Export axiosClient, fix circular structure logging. Add normalization for attachments.

.gitignore

@@ -130,4 +130,4 @@ dist
 .pnp.*
 
 .npmrc
-.idea
+.idea

README.md

@@ -2,6 +2,13 @@
 
 ## Release Notes
 
+### v1.1.3
+
+- Exported `axios` and `axiosClient` with exponential backoff retry mechanism for HTTP requests and omitting Authorization headers from Axios errors.
+- Resolved issues with circular structure logging.
+- Fixed attachments metadata normalization bug.
+- Improved repository logging.
+
 #### v1.1.2
 
 - Unified incoming and outgoing event context.

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@devrev/ts-adaas",
-  "version": "1.1.2",
+  "version": "1.1.3",
   "description": "Typescript library containing the ADaaS(AirDrop as a Service) control protocol.",
   "type": "commonjs",
   "main": "./dist/index.js",
@@ -37,7 +37,8 @@
   },
   "dependencies": {
     "@devrev/typescript-sdk": "^1.1.27",
-    "axios": "^1.5.1",
+    "axios": "^1.7.9",
+    "axios-retry": "^4.5.0",
     "form-data": "^4.0.1",
     "js-jsonl": "^1.1.1",
     "lambda-log": "^3.1.0",

src/common/control-protocol.ts

@@ -1,5 +1,4 @@
-import axios from 'axios';
-
+import { axios, axiosClient } from '../http/axios-client';
 import {
   AirdropEvent,
   EventData,
@@ -33,7 +32,7 @@ export const emit = async ({
     console.info('Emitting event', JSON.stringify(newEvent));
 
     try {
-      await axios.post(
+      await axiosClient.post(
         event.payload.event_context.callback_url,
         { ...newEvent },
         {

src/common/helpers.ts

@@ -143,3 +143,18 @@ export function addReportToLoaderReport({
 
   return loaderReports;
 }
+
+// https://stackoverflow.com/a/53731154
+export function getCircularReplacer() {
+  const seen = new WeakSet();
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  return (key: any, value: any) => {
+    if (typeof value === 'object' && value !== null) {
+      if (seen.has(value)) {
+        return;
+      }
+      seen.add(value);
+    }
+    return value;
+  };
+}

src/common/install-initial-domain-mapping.ts

@@ -1,4 +1,4 @@
-import axios from 'axios';
+import { axios, axiosClient } from '../http/axios-client';
 import { FunctionInput } from '@devrev/typescript-sdk/dist/snap-ins';
 
 import { InitialDomainMapping } from '../types/common';
@@ -18,7 +18,7 @@ export async function installInitialDomainMapping(
   }
 
   try {
-    const snapInResponse = await axios.get(
+    const snapInResponse = await axiosClient.get(
       devrevEndpoint + '/internal/snap-ins.get',
       {
         headers: {
@@ -50,7 +50,7 @@ export async function installInitialDomainMapping(
       Object.keys(startingRecipeBlueprint).length !== 0
     ) {
       try {
-        const recipeBlueprintResponse = await axios.post(
+        const recipeBlueprintResponse = await axiosClient.post(
           `${devrevEndpoint}/internal/airdrop.recipe.blueprints.create`,
           {
             ...startingRecipeBlueprint,
@@ -83,7 +83,7 @@ export async function installInitialDomainMapping(
       // 2. Install the initial domain mappings
       const additionalMappings =
         initialDomainMappingJson.additional_mappings || {};
-      const initialDomainMappingInstallResponse = await axios.post(
+      const initialDomainMappingInstallResponse = await axiosClient.post(
         `${devrevEndpoint}/internal/airdrop.recipe.initial-domain-mappings.install`,
         {
           external_system_type: 'ADaaS',

src/http/axios-client.ts

@@ -0,0 +1,36 @@
+import axios, { AxiosError } from 'axios';
+import axiosRetry from 'axios-retry';
+
+const axiosClient = axios.create();
+
+// Exponential backoff algorithm: Retry 3 times and there will be a delay of more than 1 * no. of retries second + random number of milliseconds between each retry.
+axiosRetry(axiosClient, {
+  retries: 3,
+  retryDelay: (retryCount, error) => {
+    console.log(`Retry attempt: ${retryCount} of ${error.config?.url}.`);
+    return axiosRetry.exponentialDelay(retryCount, error, 1000);
+  },
+  retryCondition: (error: AxiosError) => {
+    if (
+      error.response?.status &&
+      error.response?.status >= 500 &&
+      error.response?.status <= 599
+    ) {
+      return true;
+    } else if (error.response?.status === 429) {
+      console.log(
+        'Rate limit exceeded. Delay: ' + error.response.headers['retry-after']
+      );
+      return false;
+    } else {
+      return false;
+    }
+  },
+  onMaxRetryTimesExceeded(error: AxiosError, retryCount) {
+    console.log(`Max retries attempted: ${retryCount}`);
+    delete error.config?.headers.Authorization;
+    delete error.request._header;
+  },
+});
+
+export { axios, axiosClient };

src/http/client.ts

@@ -19,6 +19,10 @@ export const defaultResponse: HTTPResponse = {
   success: false,
 };
 
+/**
+ * HTTPClient class to make HTTP requests
+ * @deprecated
+ */
 export class HTTPClient {
   private retryAfter = 0;
   private retryAt = 0;

src/http/index.ts

@@ -1,2 +1,3 @@
 export * from './client';
 export * from './types';
+export * from './axios-client';

src/http/types.ts

@@ -1,3 +1,7 @@
+/**
+ * HTTP Response type
+ * @deprecated
+ */
 export type HTTPResponse = {
   success: boolean;
   message: string;

src/logger/logger.ts

@@ -10,6 +10,7 @@ import {
 import { isMainThread, parentPort } from 'node:worker_threads';
 import { WorkerAdapterOptions, WorkerMessageSubject } from '../types/workers';
 import { AxiosError } from 'axios';
+import { getCircularReplacer } from '../common/helpers';
 
 export class Logger extends Console {
   private options?: WorkerAdapterOptions;
@@ -38,7 +39,7 @@ export class Logger extends Console {
       parentPort?.postMessage({
         subject: WorkerMessageSubject.WorkerMessageLog,
         payload: {
-          args: JSON.parse(JSON.stringify(args)),
+          args: JSON.parse(JSON.stringify(args, getCircularReplacer())),
           level,
         },
       });

src/mappers/mappers.ts

@@ -1,4 +1,5 @@
-import axios, { AxiosResponse } from 'axios';
+import { axiosClient } from '../http/axios-client';
+import { AxiosResponse } from 'axios';
 
 import {
   MappersFactoryInterface,
@@ -23,7 +24,7 @@ export class Mappers {
     params: MappersGetByTargetIdParams
   ): Promise<AxiosResponse<MappersGetByTargetIdResponse>> {
     const { sync_unit, target } = params;
-    return axios.get<MappersGetByTargetIdResponse>(
+    return axiosClient.get<MappersGetByTargetIdResponse>(
       `${this.endpoint}/internal/airdrop.sync-mapper-record.get-by-target`,
       {
         headers: {
@@ -37,7 +38,7 @@ export class Mappers {
   async create(
     params: MappersCreateParams
   ): Promise<AxiosResponse<MappersCreateResponse>> {
-    return axios.post<MappersCreateResponse>(
+    return axiosClient.post<MappersCreateResponse>(
       `${this.endpoint}/internal/airdrop.sync-mapper-record.create`,
       params,
       {
@@ -51,7 +52,7 @@ export class Mappers {
   async update(
     params: MappersUpdateParams
   ): Promise<AxiosResponse<MappersUpdateResponse>> {
-    return axios.post<MappersUpdateResponse>(
+    return axiosClient.post<MappersUpdateResponse>(
       `${this.endpoint}/internal/airdrop.sync-mapper-record.update`,
       params,
       {

src/repo/repo.test.ts

@@ -54,24 +54,36 @@ describe('Repo class push method', () => {
     expect(normalize).not.toHaveBeenCalled();
   });
 
-  describe('should not normalize items if they are airdrop default item types', () => {
-    it.each(Object.values(AIRDROP_DEFAULT_ITEM_TYPES))(
-      'item type: %s',
-      async (itemType) => {
-        repo = new Repo({
-          event: createEvent({ eventType: EventType.ExtractionDataStart }),
-          itemType,
-          normalize,
-          onUpload: jest.fn(),
-          options: {},
-        });
-
-        const items = createItems(10);
-        await repo.push(items);
-
-        expect(normalize).not.toHaveBeenCalled();
-      }
-    );
+  describe('should not normalize items if type is "external_domain_metadata" or "ssor_attachment"', () => {
+    it('item type: external_domain_metadata', async () => {
+      repo = new Repo({
+        event: createEvent({ eventType: EventType.ExtractionDataStart }),
+        itemType: AIRDROP_DEFAULT_ITEM_TYPES.EXTERNAL_DOMAIN_METADATA,
+        normalize,
+        onUpload: jest.fn(),
+        options: {},
+      });
+
+      const items = createItems(10);
+      await repo.push(items);
+
+      expect(normalize).not.toHaveBeenCalled();
+    });
+
+    it('item type: ssor_attachment', async () => {
+      repo = new Repo({
+        event: createEvent({ eventType: EventType.ExtractionDataStart }),
+        itemType: AIRDROP_DEFAULT_ITEM_TYPES.SSOR_ATTACHMENT,
+        normalize,
+        onUpload: jest.fn(),
+        options: {},
+      });
+
+      const items = createItems(10);
+      await repo.push(items);
+
+      expect(normalize).not.toHaveBeenCalled();
+    });
   });
 
   it('should leave 5 items in the items array after pushing 2005 items with batch size of 2000', async () => {