Skip to content

Commit 2d1710c

Browse files
matt-aitkenmatt-aitken
committed
v2: Better handle recovering from platform communication errors by auto-yielding back to the platform in case of temporary API failures
1 parent 70f9bd0 commit 2d1710c

File tree

4 files changed

+251
-102
lines changed

4 files changed

+251
-102
lines changed

‎.changeset/strange-sheep-pull.md

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
---
2+
"@trigger.dev/sdk": patch
3+
"@trigger.dev/core": patch
4+
---
5+
6+
v2: Better handle recovering from platform communication errors by auto-yielding back to the platform in case of temporary API failures

‎packages/trigger-sdk/src/apiClient.ts

Lines changed: 187 additions & 85 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,18 @@ export type RunRecord = {
7474
event: ApiEventLog;
7575
};
7676

77+
export class UnknownVersionError extends Error {
78+
constructor(version: string) {
79+
super(`Unknown version ${version}`);
80+
}
81+
}
82+
83+
const MAX_RETRIES = 8;
84+
const EXPONENT_FACTOR = 2;
85+
const MIN_DELAY_IN_MS = 80;
86+
const MAX_DELAY_IN_MS = 2000;
87+
const JITTER_IN_MS = 50;
88+
7789
export class ApiClient {
7890
#apiUrl: string;
7991
#options: ApiClientOptions;
@@ -129,11 +141,10 @@ export class ApiClient {
129141
) {
130142
const apiKey = await this.#apiKey();
131143

132-
this.#logger.debug("Running Task", {
133-
task,
134-
});
144+
this.#logger.debug(`[ApiClient] runTask ${task.displayKey}`);
135145

136146
return await zodfetchWithVersions(
147+
this.#logger,
137148
{
138149
[API_VERSIONS.LAZY_LOADED_CACHED_TASKS]: RunTaskResponseWithCachedTasksBodySchema,
139150
},
@@ -771,6 +782,7 @@ async function zodfetchWithVersions<
771782
TUnversionedResponseBodySchema extends z.ZodTypeAny,
772783
TOptional extends boolean = false,
773784
>(
785+
logger: Logger,
774786
versionedSchemaMap: TVersionedResponseBodyMap,
775787
unversionedSchema: TUnversionedResponseBodySchema,
776788
url: string,
@@ -785,66 +797,132 @@ async function zodfetchWithVersions<
785797
? VersionedResponseBody<TVersionedResponseBodyMap, TUnversionedResponseBodySchema> | undefined
786798
: VersionedResponseBody<TVersionedResponseBodyMap, TUnversionedResponseBodySchema>
787799
> {
788-
const response = await fetch(url, requestInitWithCache(requestInit));
800+
try {
801+
const fullRequestInit = requestInitWithCache(requestInit);
789802

790-
if (
791-
(!requestInit || requestInit.method === "GET") &&
792-
response.status === 404 &&
793-
options?.optional
794-
) {
795-
// @ts-ignore
796-
return;
797-
}
803+
const response = await fetch(url, fullRequestInit);
798804

799-
if (response.status >= 400 && response.status < 500) {
800-
const body = await response.json();
805+
logger.debug(`[ApiClient] zodfetchWithVersions ${url} (attempt ${retryCount + 1})`, {
806+
url,
807+
retryCount,
808+
requestHeaders: fullRequestInit?.headers,
809+
responseHeaders: Object.fromEntries(response.headers.entries()),
810+
});
801811

802-
throw new Error(body.error);
803-
}
812+
if (
813+
(!requestInit || requestInit.method === "GET") &&
814+
response.status === 404 &&
815+
options?.optional
816+
) {
817+
// @ts-ignore
818+
return;
819+
}
804820

805-
if (response.status >= 500 && retryCount < 6) {
806-
// retry with exponential backoff and jitter
807-
const delay = exponentialBackoff(retryCount + 1, 2, 50, 1150, 50);
821+
if (response.status >= 400 && response.status < 500) {
822+
const rawBody = await safeResponseText(response);
823+
const body = safeJsonParse(rawBody);
824+
825+
logger.error(`[ApiClient] zodfetchWithVersions failed with ${response.status}`, {
826+
url,
827+
retryCount,
828+
requestHeaders: fullRequestInit?.headers,
829+
responseHeaders: Object.fromEntries(response.headers.entries()),
830+
status: response.status,
831+
rawBody,
832+
});
833+
834+
if (body && body.error) {
835+
throw new Error(body.error);
836+
} else {
837+
throw new Error(rawBody);
838+
}
839+
}
808840

809-
await new Promise((resolve) => setTimeout(resolve, delay));
841+
if (response.status >= 500 && retryCount < MAX_RETRIES) {
842+
// retry with exponential backoff and jitter
843+
const delay = exponentialBackoff(retryCount + 1);
844+
845+
await new Promise((resolve) => setTimeout(resolve, delay));
846+
847+
return zodfetchWithVersions(
848+
logger,
849+
versionedSchemaMap,
850+
unversionedSchema,
851+
url,
852+
requestInit,
853+
options,
854+
retryCount + 1
855+
);
856+
}
810857

811-
return zodfetchWithVersions(
812-
versionedSchemaMap,
813-
unversionedSchema,
814-
url,
815-
requestInit,
816-
options,
817-
retryCount + 1
818-
);
819-
}
858+
if (response.status !== 200) {
859+
const rawBody = await safeResponseText(response);
860+
861+
logger.error(`[ApiClient] zodfetchWithVersions failed with ${response.status}`, {
862+
url,
863+
retryCount,
864+
requestHeaders: fullRequestInit?.headers,
865+
responseHeaders: Object.fromEntries(response.headers.entries()),
866+
status: response.status,
867+
rawBody,
868+
});
869+
870+
throw new Error(
871+
options?.errorMessage ?? `Failed to fetch ${url}, got status code ${response.status}`
872+
);
873+
}
820874

821-
if (response.status !== 200) {
822-
throw new Error(
823-
options?.errorMessage ?? `Failed to fetch ${url}, got status code ${response.status}`
824-
);
825-
}
875+
const jsonBody = await response.json();
826876

827-
const jsonBody = await response.json();
877+
const version = response.headers.get("trigger-version");
828878

829-
const version = response.headers.get("trigger-version");
879+
if (!version) {
880+
return {
881+
version: "unversioned",
882+
body: unversionedSchema.parse(jsonBody),
883+
};
884+
}
885+
886+
const versionedSchema = versionedSchemaMap[version];
887+
888+
if (!versionedSchema) {
889+
throw new UnknownVersionError(version);
890+
}
830891

831-
if (!version) {
832892
return {
833-
version: "unversioned",
834-
body: unversionedSchema.parse(jsonBody),
893+
version,
894+
body: versionedSchema.parse(jsonBody),
835895
};
836-
}
896+
} catch (error) {
897+
if (error instanceof UnknownVersionError) {
898+
throw error;
899+
}
837900

838-
const versionedSchema = versionedSchemaMap[version];
901+
logger.error(`[ApiClient] zodfetchWithVersions failed with a connection error`, {
902+
url,
903+
retryCount,
904+
error,
905+
});
839906

840-
if (!versionedSchema) {
841-
throw new Error(`Unknown version ${version}`);
842-
}
907+
if (retryCount < MAX_RETRIES) {
908+
// retry with exponential backoff and jitter
909+
const delay = exponentialBackoff(retryCount + 1);
910+
911+
await new Promise((resolve) => setTimeout(resolve, delay));
912+
913+
return zodfetchWithVersions(
914+
logger,
915+
versionedSchemaMap,
916+
unversionedSchema,
917+
url,
918+
requestInit,
919+
options,
920+
retryCount + 1
921+
);
922+
}
843923

844-
return {
845-
version,
846-
body: versionedSchema.parse(jsonBody),
847-
};
924+
throw error;
925+
}
848926
}
849927

850928
function requestInitWithCache(requestInit?: RequestInit): RequestInit {
@@ -873,9 +951,9 @@ async function fetchHead(
873951
};
874952
const response = await fetch(url, requestInitWithCache(requestInit));
875953

876-
if (response.status >= 500 && retryCount < 6) {
954+
if (response.status >= 500 && retryCount < MAX_RETRIES) {
877955
// retry with exponential backoff and jitter
878-
const delay = exponentialBackoff(retryCount + 1, 2, 50, 1150, 50);
956+
const delay = exponentialBackoff(retryCount + 1);
879957

880958
await new Promise((resolve) => setTimeout(resolve, delay));
881959

@@ -897,56 +975,80 @@ async function zodfetch<TResponseSchema extends z.ZodTypeAny, TOptional extends
897975
): Promise<
898976
TOptional extends true ? z.infer<TResponseSchema> | undefined : z.infer<TResponseSchema>
899977
> {
900-
const response = await fetch(url, requestInitWithCache(requestInit));
978+
try {
979+
const response = await fetch(url, requestInitWithCache(requestInit));
980+
981+
if (
982+
(!requestInit || requestInit.method === "GET") &&
983+
response.status === 404 &&
984+
options?.optional
985+
) {
986+
// @ts-ignore
987+
return;
988+
}
901989

902-
if (
903-
(!requestInit || requestInit.method === "GET") &&
904-
response.status === 404 &&
905-
options?.optional
906-
) {
907-
// @ts-ignore
908-
return;
909-
}
990+
if (response.status >= 400 && response.status < 500) {
991+
const body = await response.json();
910992

911-
if (response.status >= 400 && response.status < 500) {
912-
const body = await response.json();
993+
throw new Error(body.error);
994+
}
913995

914-
throw new Error(body.error);
915-
}
996+
if (response.status >= 500 && retryCount < MAX_RETRIES) {
997+
// retry with exponential backoff and jitter
998+
const delay = exponentialBackoff(retryCount + 1);
916999

917-
if (response.status >= 500 && retryCount < 6) {
918-
// retry with exponential backoff and jitter
919-
const delay = exponentialBackoff(retryCount + 1, 2, 50, 1150, 50);
1000+
await new Promise((resolve) => setTimeout(resolve, delay));
9201001

921-
await new Promise((resolve) => setTimeout(resolve, delay));
1002+
return zodfetch(schema, url, requestInit, options, retryCount + 1);
1003+
}
9221004

923-
return zodfetch(schema, url, requestInit, options, retryCount + 1);
924-
}
1005+
if (response.status !== 200) {
1006+
throw new Error(
1007+
options?.errorMessage ?? `Failed to fetch ${url}, got status code ${response.status}`
1008+
);
1009+
}
9251010

926-
if (response.status !== 200) {
927-
throw new Error(
928-
options?.errorMessage ?? `Failed to fetch ${url}, got status code ${response.status}`
929-
);
930-
}
1011+
const jsonBody = await response.json();
1012+
1013+
return schema.parse(jsonBody);
1014+
} catch (error) {
1015+
if (retryCount < MAX_RETRIES) {
1016+
// retry with exponential backoff and jitter
1017+
const delay = exponentialBackoff(retryCount + 1);
9311018

932-
const jsonBody = await response.json();
1019+
await new Promise((resolve) => setTimeout(resolve, delay));
9331020

934-
return schema.parse(jsonBody);
1021+
return zodfetch(schema, url, requestInit, options, retryCount + 1);
1022+
}
1023+
1024+
throw error;
1025+
}
9351026
}
9361027

937-
function exponentialBackoff(
938-
retryCount: number,
939-
exponential: number,
940-
minDelay: number,
941-
maxDelay: number,
942-
jitter: number
943-
): number {
1028+
// First retry will have a delay of 80ms, second 160ms, third 320ms, etc.
1029+
function exponentialBackoff(retryCount: number): number {
9441030
// Calculate the delay using the exponential backoff formula
945-
const delay = Math.min(Math.pow(exponential, retryCount) * minDelay, maxDelay);
1031+
const delay = Math.min(Math.pow(EXPONENT_FACTOR, retryCount) * MIN_DELAY_IN_MS, MAX_DELAY_IN_MS);
9461032

9471033
// Calculate the jitter
948-
const jitterValue = Math.random() * jitter;
1034+
const jitterValue = Math.random() * JITTER_IN_MS;
9491035

9501036
// Return the calculated delay with jitter
9511037
return delay + jitterValue;
9521038
}
1039+
1040+
function safeJsonParse(rawBody: string) {
1041+
try {
1042+
return JSON.parse(rawBody);
1043+
} catch (error) {
1044+
return;
1045+
}
1046+
}
1047+
1048+
async function safeResponseText(response: Response) {
1049+
try {
1050+
return await response.text();
1051+
} catch (error) {
1052+
return "";
1053+
}
1054+
}

0 commit comments

Comments
 (0)