Add local flag, headers when emitting, bugfixes (#27)

Author: radovan-jorgic

README.md

@@ -2,6 +2,13 @@
 
 ## Release Notes
 
+### v1.2.3
+
+- Add `local` flag to use for local development of the ADaaS snap-ins.
+- Send library version, snap-in version and snap-in slug in headers while emitting.
+- Make `actor_id` field optional for `SsorAttachment` interface.
+- Fix bugs related to event handling, error logging.
+
 ### v1.2.2
 
 - Add library version as a part of control protocol.

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "@devrev/ts-adaas",
-  "version": "1.2.2",
+  "version": "1.2.3",
   "description": "DevRev ADaaS (AirDrop-as-a-Service) Typescript SDK.",
   "type": "commonjs",
   "main": "./dist/index.js",
@@ -26,6 +26,7 @@
     "@types/jest": "^29.5.12",
     "@types/lambda-log": "^3.0.3",
     "@types/node": "20.16.11",
+    "@types/yargs": "^17.0.33",
     "@typescript-eslint/eslint-plugin": "^7.12.0",
     "@typescript-eslint/parser": "^7.12.0",
     "eslint": "^8.57.0",
@@ -42,7 +43,8 @@
     "form-data": "^4.0.1",
     "js-jsonl": "^1.1.1",
     "lambda-log": "^3.1.0",
-    "ts-node": "^10.9.2"
+    "ts-node": "^10.9.2",
+    "yargs": "^17.7.2"
   },
   "files": [
     "dist"

package.json

@@ -1,4 +1,5 @@
 import { EventType } from '../types/extraction';
+import { getLibraryVersion } from './helpers';
 
 export const STATELESS_EVENT_TYPES = [
   EventType.ExtractionExternalSyncUnitsStart,
@@ -40,3 +41,5 @@ export const AIRDROP_DEFAULT_ITEM_TYPES = {
   ATTACHMENTS: 'attachments',
   SSOR_ATTACHMENT: 'ssor_attachment',
 };
+
+export const LIBRARY_VERSION = getLibraryVersion();

src/common/constants.ts

@@ -8,7 +8,7 @@ import {
   LoaderEvent,
 } from '../types/extraction';
 import { LoaderEventType } from '../types/loading';
-import { getLibraryVersion } from './helpers';
+import { LIBRARY_VERSION } from './constants';
 
 export interface EmitInterface {
   event: AirdropEvent;
@@ -28,7 +28,7 @@ export const emit = async ({
       ...data,
     },
     worker_metadata: {
-      adaas_library_version: getLibraryVersion(),
+      adaas_library_version: LIBRARY_VERSION,
     },
   };
 
@@ -42,6 +42,9 @@ export const emit = async ({
         Accept: 'application/json, text/plain, */*',
         Authorization: event.context.secrets.service_account_token,
         'Content-Type': 'application/json',
+        'X-DevRev-Client-Platform': event.payload.event_context.snap_in_slug,
+        'X-DevRev-Client-Id': event.payload.event_context.snap_in_version_id,
+        'X-DevRev-Client-Version': LIBRARY_VERSION,
       },
     }
   );

src/common/control-protocol.ts

@@ -60,6 +60,12 @@ export function getTimeoutErrorEventType(eventType: EventType): {
         eventType: LoaderEventType.LoaderStateDeletionError,
       };
 
+    case EventType.StartLoadingAttachments:
+    case EventType.ContinueLoadingAttachments:
+      return {
+        eventType: LoaderEventType.AttachmentLoadingError,
+      };
+
     case EventType.StartDeletingLoaderAttachmentState:
       return {
         eventType: LoaderEventType.LoaderAttachmentStateDeletionError,
@@ -172,9 +178,14 @@ export function getCircularReplacer() {
 // read adaas library version from package.json
 export function getLibraryVersion() {
   try {
-    return JSON.parse(
+    const version = JSON.parse(
       readFileSync(path.resolve(__dirname, '../../package.json'), 'utf8')
     )?.version;
+
+    if (version) {
+      return version;
+    }
+    return '';
   } catch (error) {
     console.error(
       'Error reading adaas library version from package.json',

src/common/helpers.ts

@@ -39,8 +39,8 @@ export interface NormalizedAttachment {
   url: string;
   id: string;
   file_name: string;
-  author_id: string;
   parent_id: string;
+  author_id?: string;
   grand_parent_id?: number;
 }
 

src/repo/repo.interfaces.ts

@@ -45,6 +45,7 @@ export function createEvent({
         mode: 'test_mode',
         request_id: 'test_request_id',
         snap_in_slug: 'test_snap_in_slug',
+        snap_in_version_id: 'test_snap_in_version_id',
         sync_run: 'test_sync_run',
         sync_run_id: 'test_sync_run_id',
         sync_tier: 'test_sync_tier',

src/tests/test-helpers.ts

@@ -148,6 +148,7 @@ export interface EventContext {
   mode: string;
   request_id: string;
   snap_in_slug: string;
+  snap_in_version_id: string;
   sync_run: string;
   sync_run_id: string;
   sync_tier: string;

src/types/extraction.ts

@@ -1,5 +1,4 @@
 import { Worker } from 'worker_threads';
-import { MessagePort } from 'node:worker_threads';
 
 import { State } from '../state/state';
 import { WorkerAdapter } from '../workers/worker-adapter';

src/types/workers.ts

@@ -65,7 +65,7 @@ export interface SsorAttachment {
   parent_id: {
     external: string;
   };
-  actor_id: {
+  actor_id?: {
     external: string;
   };
 }

src/uploader/uploader.interfaces.ts

@@ -1,7 +1,7 @@
 import { isMainThread, Worker } from 'node:worker_threads';
 
 import { WorkerData, WorkerEvent } from '../types/workers';
-import { Logger, serializeError } from '../logger/logger';
+import { Logger } from '../logger/logger';
 
 async function createWorker<ConnectorState>(
   workerData: WorkerData<ConnectorState>
@@ -19,7 +19,7 @@ async function createWorker<ConnectorState>(
       } as WorkerOptions);
 
       worker.on(WorkerEvent.WorkerError, (error) => {
-        logger.error('Worker error', serializeError(error));
+        logger.error('Worker error', error);
         reject();
       });
       worker.on(WorkerEvent.WorkerOnline, () => {

src/workers/create-worker.ts

@@ -1,4 +1,7 @@
 import axios from 'axios';
+import { hideBin } from 'yargs/helpers';
+import yargs from 'yargs';
+
 import {
   AirdropEvent,
   EventType,
@@ -91,6 +94,21 @@ export async function spawn<ConnectorState>({
     connectorWorkerPath: workerPath,
   });
 
+  if (options?.isLocalDevelopment) {
+    logger.warn(
+      'WARN: isLocalDevelopment is deprecated. Please use the -- local flag instead.'
+    );
+  }
+
+  // read the command line arguments to check if the local flag is passed
+  const argv = await yargs(hideBin(process.argv)).argv;
+  if (argv._.includes('local')) {
+    options = {
+      ...(options || {}),
+      isLocalDevelopment: true,
+    };
+  }
+
   if (script) {
     try {
       const worker = await createWorker<ConnectorState>({

src/workers/spawn.ts

@@ -727,11 +727,14 @@ export class WorkerAdapter<ConnectorState> {
         parent_id: {
           external: attachment.parent_id,
         },
-        actor_id: {
-          external: attachment.author_id,
-        },
       };
 
+      if (attachment.author_id) {
+        ssorAttachment.actor_id = {
+          external: attachment.author_id,
+        };
+      }
+
       await this.getRepo('ssor_attachment')?.push([ssorAttachment]);
     }
     return;