Fix for attachments streaming. Expose getter to end developer. Increase delayFactor of exponential backoff mechanism. (#18)

Author: radovan-jorgic

README.md

@@ -2,71 +2,77 @@
 
 ## Release Notes
 
+### v1.1.5
+
+- Increase `delayFactor` and number of retries for the exponential backoff retry mechanism for HTTP requests.
+- Provide an inject function for streaming attachments.
+- Fix the attachments streaming bug.
+
 ### v1.1.4
 
-- Provide log line and stack trace on runtime worker errors.
+- Provide log lines and stack traces for runtime worker errors.
 
 ### v1.1.3
 
-- Exported `axios` and `axiosClient` with exponential backoff retry mechanism for HTTP requests and omitting Authorization headers from Axios errors.
-- Resolved issues with circular structure logging.
-- Fixed attachments metadata normalization bug.
-- Improved repository logging.
+- Export `axios` and `axiosClient` with the exponential backoff retry mechanism for HTTP requests and omit Authorization headers from Axios errors.
+- Resolve circular structure logging issues.
+- Fix the attachments metadata normalization bug.
+- Improve repository logging.
 
-#### v1.1.2
+### v1.1.2
 
-- Unified incoming and outgoing event context.
-- Added `dev_oid` to logger tags.
+- Unify incoming and outgoing event context.
+- Add `dev_oid` to logger tags.
 
-#### v1.1.1
+### v1.1.1
 
-- Added default workers for loading deletion events.
+- Add default workers for loading deletion events.
 
-#### v1.1.0
+### v1.1.0
 
-- Support for sync from DevRev to external system. Known limitations: no support for loading attachments.
+- Support sync from DevRev to the external system. (Known limitations: no support for loading attachments.)
 
-#### v1.0.4
+### v1.0.4
 
 - Fix logging from worker threads.
 
-#### v1.0.3
+### v1.0.3
 
-- Added Release notes.
+- Add release notes.
 
-#### v1.0.2
+### v1.0.2
 
-- Bug fixes and improvements in local development.
+- Fix bugs and improve local development.
 - Expose `formatAxiosError` function for error handling.
 
-#### v1.0.1
+### v1.0.1
 
-- Bug fixes and improvements in logging.
+- Fix bugs and improve logging.
 
-#### v1.0.0
+### v1.0.0
 
-- Allow extractions to use full lambda runtime and gracefully handle execution context timeout.
-- Simplified metadata and data normalization and uploading with repo implementation.
-- Default handling of attachment extraction phase in ADaaS SDK library.
-- Reduced file size, streamlined process by gzip compression.
-- Bug fixes and improvements in error handling.
+- Enable extractions to use the full lambda runtime and gracefully handle execution context timeout.
+- Simplify metadata and data normalization and uploading with the repo implementation.
+- Provide default handling of the attachment extraction phase in the ADaaS SDK library.
+- Reduce file size and streamline processes with gzip compression.
+- Fix bugs and improve error handling.
 
-#### v0.0.3
+### v0.0.3
 
-- Support for new recipe management
+- Support new recipe management.
 
-#### v0.0.2
+### v0.0.2
 
-- Support for the State API
-- HTTP client for API requests
-- Local development environment creates local artifact files
-- Improvements in logging
+- Support the State API.
+- Provide an HTTP client for API requests.
+- Create local artifact files in the local development environment.
+- Improve logging.
 
-#### v0.0.1
+### v0.0.1
 
-- Demo implementation of ADaaS snap-in
-- Adapter for ADaaS control protocol with helper functions
-- Uploader for uploading artifacts
+- Implement a demo of the ADaaS snap-in.
+- Add an adapter for the ADaaS control protocol with helper functions.
+- Provide an uploader for uploading artifacts.
 
 # Overview
 

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "@devrev/ts-adaas",
-  "version": "1.1.4",
+  "version": "1.1.5",
   "description": "Typescript library containing the ADaaS(AirDrop as a Service) control protocol.",
   "type": "commonjs",
   "main": "./dist/index.js",

package.json

@@ -3,12 +3,15 @@ import axiosRetry from 'axios-retry';
 
 const axiosClient = axios.create();
 
-// Exponential backoff algorithm: Retry 3 times and there will be a delay of more than 1 * no. of retries second + random number of milliseconds between each retry.
 axiosRetry(axiosClient, {
-  retries: 3,
+  retries: 5,
   retryDelay: (retryCount, error) => {
-    console.log(`Retry attempt: ${retryCount} of ${error.config?.url}.`);
-    return axiosRetry.exponentialDelay(retryCount, error, 1000);
+    console.warn(
+      'Retry attempt: ' + retryCount + 'to url: ' + error.config?.url + '.'
+    );
+
+    // Exponential backoff algorithm: 1 * 2 ^ retryCount * 5000ms
+    return axiosRetry.exponentialDelay(retryCount, error, 5000);
   },
   retryCondition: (error: AxiosError) => {
     if (

src/http/axios-client.ts

@@ -41,6 +41,7 @@ export interface NormalizedAttachment {
   file_name: string;
   author_id: string;
   parent_id: string;
+  inline?: boolean;
 }
 
 /**

src/repo/repo.interfaces.ts

@@ -7,8 +7,7 @@ export interface SdkState {
   lastSuccessfulSyncStarted?: string;
   toDevRev?: ToDevRev;
   fromDevRev?: FromDevRev;
-};
-
+}
 
 /**
  * AdapterState is an interface that defines the structure of the adapter state that is used by the external extractor. It extends the connector state with additional fields: lastSyncStarted, lastSuccessfulSyncStarted, and attachmentsMetadata.

src/state/state.interfaces.ts

@@ -4,7 +4,9 @@ import { Artifact } from '../uploader/uploader.interfaces';
 
 import { ErrorRecord } from './common';
 
-import { DonV2, LoaderReport } from './loading';
+import { DonV2, LoaderReport, RateLimited } from './loading';
+import { NormalizedAttachment } from 'repo/repo.interfaces';
+import { AxiosResponse } from 'axios';
 
 /**
  * EventType is an enum that defines the different types of events that can be sent to the external extractor from ADaaS.
@@ -244,3 +246,25 @@ export interface LoaderEvent {
   event_context: EventContext;
   event_data?: EventData;
 }
+
+export type ExternalSystemAttachmentStreamingFunction = ({
+  item,
+  event,
+}: ExternalSystemAttachmentStreamingParams) => Promise<ExternalSystemAttachmentStreamingResponse>;
+
+export interface ExternalSystemAttachmentStreamingParams {
+  item: NormalizedAttachment;
+  event: AirdropEvent;
+}
+
+export interface ExternalSystemAttachmentStreamingResponse {
+  httpStream?: AxiosResponse;
+  error?: ErrorRecord;
+  delay?: number;
+}
+
+export interface StreamAttachmentsResponse {
+  error?: ErrorRecord;
+  report?: LoaderReport;
+  rateLimit?: RateLimited;
+}

src/types/extraction.ts

@@ -22,6 +22,9 @@ export {
   AirdropMessage,
   ExtractorEvent,
   SyncMode,
+  ExternalSystemAttachmentStreamingParams,
+  ExternalSystemAttachmentStreamingResponse,
+  ExternalSystemAttachmentStreamingFunction,
 } from './extraction';
 
 // Loading