New IDM installation flow, attachments streaming bugfix, retry all 5xx requests (#32)

Author: radovanjorgic

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "@devrev/ts-adaas",
-  "version": "1.2.6",
+  "version": "1.3.0",
   "description": "DevRev ADaaS (AirDrop-as-a-Service) Typescript SDK.",
   "type": "commonjs",
   "main": "./dist/index.js",
@@ -37,7 +37,7 @@
     "typescript": "^5.3.3"
   },
   "dependencies": {
-    "@devrev/typescript-sdk": "^1.1.27",
+    "@devrev/typescript-sdk": "^1.1.54",
     "axios": "^1.7.9",
     "axios-retry": "^4.5.0",
     "form-data": "^4.0.1",

package.json

@@ -1,11 +1,11 @@
 import { axios, axiosClient } from '../http/axios-client';
-import { FunctionInput } from '@devrev/typescript-sdk/dist/snap-ins';
+import { AirdropEvent } from '../types/extraction';
 
 import { InitialDomainMapping } from '../types/common';
 import { serializeAxiosError } from '../logger/logger';
 
 export async function installInitialDomainMapping(
-  event: FunctionInput,
+  event: AirdropEvent,
   initialDomainMappingJson: InitialDomainMapping
 ) {
   const devrevEndpoint = event.execution_metadata.devrev_endpoint;

src/common/install-initial-domain-mapping.ts

@@ -1,3 +1,27 @@
+/**
+ * Axios client setup with retry capabilities using axios-retry.
+ *
+ * This module exports an Axios client instance (`axiosClient`) that is configured to automatically retry
+ * failed requests under certain conditions.
+ *
+ * Retry Conditions:
+ * 1. Network errors (where no response is received).
+ * 2. Idempotent requests (defaults include GET, HEAD, OPTIONS, PUT).
+ * 3. All 5xx server errors.
+ *
+ * Retry Strategy:
+ * - A maximum of 5 retries are attempted.
+ * - Exponential backoff delay is applied between retries, increasing with each retry attempt.
+ *
+ * Additional Features:
+ * - When the maximum number of retry attempts is reached, sensitive headers (like authorization)
+ *   are removed from error logs for security reasons.
+ *
+ * Exported:
+ * - `axios`: Original axios instance for additional customizations or direct use.
+ * - `axiosClient`: Configured axios instance with retry logic.
+ */
+
 import axios, { AxiosError } from 'axios';
 import axiosRetry from 'axios-retry';
 
@@ -17,8 +41,9 @@ axiosRetry(axiosClient, {
   },
   retryCondition: (error: AxiosError) => {
     return (
-      axiosRetry.isNetworkOrIdempotentRequestError(error) &&
-      error.response?.status !== 429
+      (axiosRetry.isNetworkOrIdempotentRequestError(error) &&
+        error.response?.status !== 429) ||
+      (error.response?.status ?? 0) >= 500
     );
   },
   onMaxRetryTimesExceeded(error: AxiosError) {

src/http/axios-client.ts

@@ -1,16 +1,19 @@
 import { AirdropEvent } from '../types/extraction';
 import { FileToLoad } from '../types/loading';
 import { WorkerAdapterOptions } from '../types/workers';
+import { InitialDomainMapping } from '../types/common';
 
 export interface SdkState {
   lastSyncStarted?: string;
   lastSuccessfulSyncStarted?: string;
   toDevRev?: ToDevRev;
   fromDevRev?: FromDevRev;
+  snapInVersionId?: string;
 }
 
 /**
- * AdapterState is an interface that defines the structure of the adapter state that is used by the external extractor. It extends the connector state with additional fields: lastSyncStarted, lastSuccessfulSyncStarted, and attachmentsMetadata.
+ * AdapterState is an interface that defines the structure of the adapter state that is used by the external extractor.
+ * It extends the connector state with additional fields: lastSyncStarted, lastSuccessfulSyncStarted, snapInVersionId and attachmentsMetadata.
  */
 export type AdapterState<ConnectorState> = ConnectorState & SdkState;
 
@@ -28,5 +31,6 @@ export interface FromDevRev {
 export interface StateInterface<ConnectorState> {
   event: AirdropEvent;
   initialState: ConnectorState;
+  initialDomainMapping?: InitialDomainMapping;
   options?: WorkerAdapterOptions;
 }

src/state/state.interfaces.ts

@@ -4,25 +4,54 @@ import { AirdropEvent, EventType, SyncMode } from '../types/extraction';
 import { STATELESS_EVENT_TYPES } from '../common/constants';
 import { serializeAxiosError, getPrintableState } from '../logger/logger';
 import { ErrorRecord } from '../types/common';
+import { installInitialDomainMapping } from '../common/install-initial-domain-mapping';
 
 import { AdapterState, SdkState, StateInterface } from './state.interfaces';
 import { getSyncDirection } from '../common/helpers';
 
 export async function createAdapterState<ConnectorState>({
   event,
   initialState,
+  initialDomainMapping,
   options,
 }: StateInterface<ConnectorState>): Promise<State<ConnectorState>> {
   const newInitialState = structuredClone(initialState);
   const as = new State<ConnectorState>({
     event,
     initialState: newInitialState,
+    initialDomainMapping,
     options,
   });
 
   if (!STATELESS_EVENT_TYPES.includes(event.payload.event_type)) {
     await as.fetchState(newInitialState);
 
+    const snapInVersionId = event.context.snap_in_version_id;
+
+    const hasSnapInVersionInState = 'snapInVersionId' in as.state;
+
+    const shouldUpdateIDM =
+      !hasSnapInVersionInState || as.state.snapInVersionId !== snapInVersionId;
+
+    if (shouldUpdateIDM) {
+      console.log(
+        `Snap-in version in state (${as.state?.snapInVersionId}) differs from the version in event context (${snapInVersionId}) - initial domain mapping needs to be updated.`
+      );
+      if (initialDomainMapping) {
+        await installInitialDomainMapping(event, initialDomainMapping);
+        as.state.snapInVersionId = snapInVersionId;
+        console.log('Successfully installed new initial domain mapping.');
+      } else {
+        console.warn(
+          'No initial domain mapping was passed to spawn function. Skipping initial domain mapping installation.'
+        );
+      }
+    } else {
+      console.log(
+        `Snap-in version in state matches the version in event context (${snapInVersionId}). Skipping initial domain mapping installation.`
+      );
+    }
+
     if (
       event.payload.event_type === EventType.ExtractionDataStart &&
       !as.state.lastSyncStarted
@@ -47,13 +76,15 @@ export class State<ConnectorState> {
     this.initialSdkState =
       getSyncDirection({ event }) === SyncMode.LOADING
         ? {
+            snapInVersionId: '',
             fromDevRev: {
               filesToLoad: [],
             },
           }
         : {
             lastSyncStarted: '',
             lastSuccessfulSyncStarted: '',
+            snapInVersionId: '',
             toDevRev: {
               attachmentsMetadata: {
                 artifactIds: [],
@@ -132,19 +163,15 @@ export class State<ConnectorState> {
     );
 
     try {
-      const response = await axiosClient.post(
-        this.workerUrl + '.get',
-        {},
-        {
-          headers: {
-            Authorization: this.devrevToken,
-          },
-          params: {
-            sync_unit: this.event.payload.event_context.sync_unit_id,
-            request_id: this.event.payload.event_context.uuid,
-          },
-        }
-      );
+      const response = await axiosClient.get(this.workerUrl + '.get', {
+        headers: {
+          Authorization: this.devrevToken,
+        },
+        params: {
+          sync_unit: this.event.payload.event_context.sync_unit_id,
+          request_id: this.event.payload.event_context.uuid,
+        },
+      });
 
       this.state = JSON.parse(response.data.state);
 

src/state/state.ts

@@ -21,6 +21,7 @@ export function createEvent({
         service_account_token: 'test_token',
       },
       snap_in_version_id: 'test_snap_in_version_id',
+      snap_in_id: 'test_snap_in_id',
       ...contextOverrides,
     },
     payload: {

src/tests/test-helpers.ts

@@ -221,6 +221,7 @@ export interface AirdropEvent {
       service_account_token: string;
     };
     snap_in_version_id: string;
+    snap_in_id: string;
   };
   payload: AirdropMessage;
   execution_metadata: {

src/types/extraction.ts

@@ -7,6 +7,8 @@ import { ExtractorEventType, AirdropEvent } from './extraction';
 
 import { LoaderEventType } from './loading';
 
+import { InitialDomainMapping } from './common';
+
 /**
  * WorkerAdapterInterface is an interface for WorkerAdapter class.
  * @interface WorkerAdapterInterface
@@ -59,13 +61,15 @@ export interface SpawnInterface {
  * @param {AirdropEvent} event - The event object received from the platform
  * @param {object=} initialState - The initial state of the adapter
  * @param {string} workerPath - The path to the worker file
+ * @param {string} initialDomainMapping - The initial domain mapping
  * @param {WorkerAdapterOptions} options - The options to create a new instance of Spawn class
  */
 export interface SpawnFactoryInterface<ConnectorState> {
   event: AirdropEvent;
   initialState: ConnectorState;
   workerPath?: string;
   options?: WorkerAdapterOptions;
+  initialDomainMapping?: InitialDomainMapping;
 }
 
 /**
@@ -149,6 +153,7 @@ export interface WorkerData<ConnectorState> {
   event: AirdropEvent;
   initialState: ConnectorState;
   workerPath: string;
+  initialDomainMapping?: InitialDomainMapping;
   options?: WorkerAdapterOptions;
 }
 

src/types/workers.ts

@@ -17,12 +17,14 @@ export function processTask<ConnectorState>({
     void (async () => {
       const event = workerData.event;
       const initialState = workerData.initialState as ConnectorState;
+      const initialDomainMapping = workerData.initialDomainMapping;
       const options = workerData.options;
       console = new Logger({ event, options });
 
       const adapterState = await createAdapterState<ConnectorState>({
         event,
         initialState,
+        initialDomainMapping,
         options,
       });
 

src/workers/process-task.ts

@@ -86,6 +86,7 @@ export async function spawn<ConnectorState>({
   event,
   initialState,
   workerPath,
+  initialDomainMapping,
   options,
 }: SpawnFactoryInterface<ConnectorState>): Promise<void> {
   const logger = new Logger({ event, options });
@@ -115,6 +116,7 @@ export async function spawn<ConnectorState>({
         event,
         initialState,
         workerPath: script,
+        initialDomainMapping,
         options,
       });