feat: support max kB in upload batching

packages/core/src/__tests__/util.test.ts

@@ -19,4 +19,20 @@ describe('#chunk', () => {
   it('handles remainders', () => {
     expect(chunk([1, 2, 3, 4, 5], 2)).toEqual([[1, 2], [3, 4], [5]]);
   });
+
+  it('handles max kb', () => {
+    const almost1KBString =
+      'heisxxohwswnetphtpnwyhluoeariblrgbksynoricexcoylnpmkymrgxmtuslveckeoulxqfwmemyxqvgyjnnclrmzcsacxkpdwzdthtudexkphsshjhlplvgryuqbqecgpbyussbgwuearvcmmrnkghcmcmynzxcswkghgrkhumeswvmhnhvymohluqooxmlbqdtxmeyikacyxvdkntrlwggoyrldmeaezowghoyukeeuaulxohknbbpyxpotonrtwfhoevysrfonevjtpjfywpptklmshzobtyklrahpkqnjifqtnxbeleeqxjacnphcomsdplxqmkcvnpxwbbmnmmzqvxmvpdrqaytlybmyghuckxnqlkgzptowgycanrenlgsgtwwbyudctiqmlszpzczfoglfoeuvdkublajngfmgstufgaroktarkeydqpmrgspuhbsqxqebzhycttsxkfcbgdqgctjgkrahoubqrvscdowafwfbysqerxqmzmakqibepyrjxjgcsnabjmovsgothxcawjsqzprxdnolyyhjfujgcxvniupmffqkhzknkzglazjflnnloazwslomxpegluqywyuvalysliziukrunfprejkojyihjjfcqdgsundoinipuzoxkqthslqgminkwcqglrjuhbqcirmuowdvizfsyubnllxxtjxqrctlzqmbfunxwzzbovogpvgrzeunvucaniqmqhwzcwqwzqvjaaeuajxgdhgitsekkowqejuouhaguudfkbyyivkayzsyoiymvavbabvtrgwmoadkvxxvgenttbohzngbzqiguficxkqvbxlrvkpycotawgalfdpmnmcfligffhivuzjtuxrwjjmzqcvbaocxfdmhnrcwhdmnpccikjvrskojxwnoiskacbivzfmbwyqfrhzjtdsjomzifswvaqdlorszbasmskosoullyyzpzjjzzkuvgoqwrzasqiroyaomrrxyqgzdigcwde';
+    expect(
+      chunk([almost1KBString, almost1KBString, almost1KBString], 5, 1)
+    ).toEqual([[almost1KBString], [almost1KBString], [almost1KBString]]);
+  });
+
+  it('handles max kb and count at the same time', () => {
+    const about500bString =
+      'jfywukefmafvjroiisrqxudmmkqibiwaknaywfberhhtiymwrzcyoitzemqhwqamkccbgccsinagjddhhnijndpqyejfaztdbmcunmdaewraamfzssfwyzddttjkdpwehphowxrbpohntfohxvmufgoyovfguxlghvoisbmtufpyxqgqylufzhvavjylkhavobvmzyqbojniyjhkgssmnujyicucskiaenpeutaqbxsnzuinhnvfqbkmmoxzxhangjxhnuhskmldksucmridbyacvhycxpdgkkibypdphhwrkpmwivtgbvnfkilxmwvxbxuxklqstjltilksgaaxqmhdtmublkwzobrcoofiyygahosrmbmgftybdmdpyptsrntukfyczdusqlfgyexyojcfuloyyuepgkyhivmiqruqxbvixfotvwzjbamrqopsjiyftiuwmhbcfsvkrmjpmwthpwwszgjarargfpmxxkuwaofahreb';
+    expect(
+      chunk([about500bString, about500bString, about500bString], 2, 1)
+    ).toEqual([[about500bString, about500bString], [about500bString]]);
+  });
 });

packages/core/src/plugins/SegmentDestination.ts

@@ -7,6 +7,7 @@ import { DestinationMetadataEnrichment } from './DestinationMetadataEnrichment';
 import { QueueFlushingPlugin } from './QueueFlushingPlugin';
 
 const MAX_EVENTS_PER_BATCH = 100;
+const MAX_PAYLOAD_SIZE_IN_KB = 500;
 export const SEGMENT_DESTINATION_KEY = 'Segment.io';
 
 export class SegmentDestination extends DestinationPlugin {
@@ -21,7 +22,8 @@ export class SegmentDestination extends DestinationPlugin {
 
     const chunkedEvents: SegmentEvent[][] = chunk(
       events,
-      this.analytics?.getConfig().maxBatchSize ?? MAX_EVENTS_PER_BATCH
+      this.analytics?.getConfig().maxBatchSize ?? MAX_EVENTS_PER_BATCH,
+      MAX_PAYLOAD_SIZE_IN_KB
     );
 
     let sentEvents: SegmentEvent[] = [];

packages/core/src/util.ts

@@ -1,18 +1,43 @@
 import type { EventPlugin } from './plugin';
 import type { Timeline } from './timeline';
 
-export const chunk = (array: any[], size: number) => {
-  if (!array.length || !size) {
+const sizeOf = (obj: unknown): number => {
+  const size = encodeURI(JSON.stringify(obj)).split(/%..|./).length - 1;
+  return size / 1024;
+};
+
+export const chunk = (array: any[], count: number, maxKB?: number) => {
+  if (!array.length || !count) {
     return [];
   }
 
-  let index = 0;
-  let resIndex = 0;
-  const result = new Array(Math.ceil(array.length / size));
+  let currentChunk = 0;
+  let rollingKBSize = 0;
+  const result: any[] = array.reduce(
+    (chunks: any[][], item: any, index: number) => {
+      if (maxKB !== undefined) {
+        rollingKBSize += sizeOf(item);
+        // If we overflow chunk until the previous index, else keep going
+        if (rollingKBSize >= maxKB) {
+          chunks[++currentChunk] = [item];
+          return chunks;
+        }
+      }
+
+      if (index !== 0 && index % count === 0) {
+        chunks[++currentChunk] = [item];
+      } else {
+        if (chunks[currentChunk] === undefined) {
+          chunks[currentChunk] = [];
+        }
+        chunks[currentChunk].push(item);
+      }
+
+      return chunks;
+    },
+    []
+  );
 
-  while (index < array.length) {
-    result[resIndex++] = array.slice(index, (index += size));
-  }
   return result;
 };