Attachments extraction fixes, bump to the latest SDK version (#24)

https://app.devrev.ai/devrev/works/ISS-151039

Author: radovan-jorgic

.github/templates/manifest.yaml

@@ -29,7 +29,7 @@ keyring_types:
         method: GET
 
 imports:
-  - slug: %PROJECT_NAME%-extractor
+  - slug: %PROJECT_NAME%
     display_name: %PRETTY_NAME%
     description: %PRETTY_NAME%
     extractor_function: extraction

README.md

@@ -1,6 +1,6 @@
 # ADaaS Template
 
-This GitHub repository provides a template with example code to implement an Airdrop as a Service (ADaaS) Snap-in .
+This GitHub repository provides a template with example code to implement an Airdrop as a Service (ADaaS) Snap-in.
 
 ## Prerequisites
 
@@ -13,7 +13,7 @@ This GitHub repository provides a template with example code to implement an Air
 1\. Clone Repository:
 
 - Either clone this repository or create a new repository from it by clicking the "Use this template" button above.
-- Set the desired repository name for your own copy (e.g., `airdrop-<external system>-snap-in`).
+- The repository name must start with `airdrop-` (e.g., `airdrop-<external system>-snap-in`).
 
 2\. Open the project in your IDE and set up project environment variables, by following these steps:
 

code/package-lock.json

@@ -55,7 +55,7 @@
     "yargs": "^17.6.2"
   },
   "dependencies": {
-    "@devrev/ts-adaas": "^1.1.6",
+    "@devrev/ts-adaas": "1.2.1",
     "axios": "^1.7.7",
     "dotenv": "^16.0.3",
     "js-jsonl": "^1.1.1",

code/package.json

@@ -1,6 +1,11 @@
 import axios, { AxiosRequestConfig, AxiosResponse } from 'axios';
 
-import { AirdropEvent, ExternalSystemItemLoadingParams, ExternalSystemItemLoadingResponse } from '@devrev/ts-adaas';
+import {
+  AirdropEvent,
+  ExternalSystemItem,
+  ExternalSystemItemLoadingParams,
+  ExternalSystemItemLoadingResponse,
+} from '@devrev/ts-adaas';
 
 export class HttpClient {
   private apiEndpoint: string;
@@ -19,15 +24,15 @@ export class HttpClient {
     item,
     mappers,
     event,
-  }: ExternalSystemItemLoadingParams): Promise<ExternalSystemItemLoadingResponse> {
+  }: ExternalSystemItemLoadingParams<ExternalSystemItem>): Promise<ExternalSystemItemLoadingResponse> {
     return { error: 'Could not create an issue in external system.' };
   }
 
   async updateIssue({
     item,
     mappers,
     event,
-  }: ExternalSystemItemLoadingParams): Promise<ExternalSystemItemLoadingResponse> {
+  }: ExternalSystemItemLoadingParams<ExternalSystemItem>): Promise<ExternalSystemItemLoadingResponse> {
     return { error: 'Could not update an issue in external system.' };
   }
 }

code/src/functions/external-system/http-client.ts

@@ -1,5 +1,6 @@
 import {
   axios,
+  axiosClient,
   ExternalSystemAttachmentStreamingParams,
   ExternalSystemAttachmentStreamingResponse,
   ExtractorEventType,
@@ -9,22 +10,24 @@ import {
 
 const getAttachmentStream = async ({
   item,
-  event,
 }: ExternalSystemAttachmentStreamingParams): Promise<ExternalSystemAttachmentStreamingResponse> => {
-  const { id, url, inline } = item;
+  const { id, url } = item;
 
   try {
-    const fileStreamResponse = await axios.get(url, {
+    const fileStreamResponse = await axiosClient.get(url, {
       responseType: 'stream',
     });
 
     return { httpStream: fileStreamResponse };
   } catch (error) {
     if (axios.isAxiosError(error)) {
-      console.error('Error while fetching attachment from URL.', serializeAxiosError(error));
+      console.warn(`Error while fetching attachment ${id} from URL.`, serializeAxiosError(error));
+      console.warn('Failed attachment metadata', item);
     } else {
-      console.error('Error while fetching attachment from URL.', error);
+      console.warn(`Error while fetching attachment ${id} from URL.`, error);
+      console.warn('Failed attachment metadata', item);
     }
+
     return {
       error: {
         message: 'Error while fetching attachment ' + id + ' from URL.',
@@ -35,20 +38,24 @@ const getAttachmentStream = async ({
 
 processTask({
   task: async ({ adapter }) => {
-    const { error, delay } = await adapter.streamAttachments({
-      stream: getAttachmentStream,
-    });
-
-    if (delay) {
-      await adapter.emit(ExtractorEventType.ExtractionAttachmentsDelay, {
-        delay,
+    try {
+      const response = await adapter.streamAttachments({
+        stream: getAttachmentStream,
       });
-    } else if (error) {
-      await adapter.emit(ExtractorEventType.ExtractionAttachmentsError, {
-        error,
-      });
-    } else {
-      await adapter.emit(ExtractorEventType.ExtractionAttachmentsDone);
+
+      if (response?.delay) {
+        await adapter.emit(ExtractorEventType.ExtractionAttachmentsDelay, {
+          delay: response.delay,
+        });
+      } else if (response?.error) {
+        await adapter.emit(ExtractorEventType.ExtractionAttachmentsError, {
+          error: response.error,
+        });
+      } else {
+        await adapter.emit(ExtractorEventType.ExtractionAttachmentsDone);
+      }
+    } catch (error) {
+      console.error('An error occured while processing a task.', error);
     }
   },
   onTimeout: async ({ adapter }) => {

code/src/functions/extraction/workers/attachments-extraction.ts

@@ -7,7 +7,7 @@ function getWorkerPerLoadingPhase(event: AirdropEvent) {
   switch (event.payload.event_type) {
     case EventType.StartLoadingData:
     case EventType.ContinueLoadingData:
-      path = __dirname + '/workers/data-loading';
+      path = __dirname + '/workers/load-data';
       break;
   }
   return path;