v3: recover from server rate limiting errors in a more reliable way

- Changing from sliding window to token bucket in the API rate limiter, to help smooth out traffic
- Adding spans to the API Client core & SDK functions
- Added waiting spans when retrying in the API Client
- Retrying in the API Client now respects the x-ratelimit-reset
- Retrying ApiError’s in tasks now respects the x-ratelimit-reset
- Added AbortTaskRunError that when thrown will stop retries
- Added idempotency keys SDK functions and automatically injecting the run ID when inside a task
- Added the ability to configure ApiRequestOptions (retries only for now) globally and on specific calls
- Implement the maxAttempts TaskRunOption (it wasn’t doing anything before)

Author: ericallam

apps/webapp/app/env.server.ts

@@ -97,8 +97,9 @@ const EnvironmentSchema = z.object({
    * @example "1000ms"
    * @example "1000s"
    */
-  API_RATE_LIMIT_WINDOW: z.string().default("60s"),
-  API_RATE_LIMIT_MAX: z.coerce.number().int().default(600),
+  API_RATE_LIMIT_REFILL_INTERVAL: z.string().default("10s"), // refill 250 tokens every 10 seconds
+  API_RATE_LIMIT_MAX: z.coerce.number().int().default(750), // allow bursts of 750 requests
+  API_RATE_LIMIT_REFILL_RATE: z.coerce.number().int().default(250), // refix 250 tokens every 10 seconds
   API_RATE_LIMIT_REQUEST_LOGS_ENABLED: z.string().default("0"),
   API_RATE_LIMIT_REJECTION_LOGS_ENABLED: z.string().default("1"),
 

apps/webapp/app/services/apiRateLimit.server.ts

@@ -106,8 +106,10 @@ export function authorizationRateLimitMiddleware({
       hashedAuthorizationValue
     );
 
+    const $remaining = Math.max(0, remaining); // remaining can be negative if the user has exceeded the limit, so clamp it to 0
+
     res.set("x-ratelimit-limit", limit.toString());
-    res.set("x-ratelimit-remaining", remaining.toString());
+    res.set("x-ratelimit-remaining", $remaining.toString());
     res.set("x-ratelimit-reset", reset.toString());
 
     if (success) {
@@ -122,12 +124,12 @@ export function authorizationRateLimitMiddleware({
           title: "Rate Limit Exceeded",
           status: 429,
           type: "https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/429",
-          detail: `Rate limit exceeded ${remaining}/${limit} requests remaining. Retry in ${secondsUntilReset} seconds.`,
+          detail: `Rate limit exceeded ${$remaining}/${limit} requests remaining. Retry in ${secondsUntilReset} seconds.`,
           reset,
           limit,
           remaining,
           secondsUntilReset,
-          error: `Rate limit exceeded ${remaining}/${limit} requests remaining. Retry in ${secondsUntilReset} seconds.`,
+          error: `Rate limit exceeded ${$remaining}/${limit} requests remaining. Retry in ${secondsUntilReset} seconds.`,
         },
         null,
         2
@@ -138,7 +140,11 @@ export function authorizationRateLimitMiddleware({
 
 export const apiRateLimiter = authorizationRateLimitMiddleware({
   keyPrefix: "api",
-  limiter: Ratelimit.slidingWindow(env.API_RATE_LIMIT_MAX, env.API_RATE_LIMIT_WINDOW as Duration),
+  limiter: Ratelimit.tokenBucket(
+    env.API_RATE_LIMIT_REFILL_RATE,
+    env.API_RATE_LIMIT_REFILL_INTERVAL as Duration,
+    env.API_RATE_LIMIT_MAX
+  ),
   pathMatchers: [/^\/api/],
   // Allow /api/v1/tasks/:id/callback/:secret
   pathWhiteList: [
@@ -152,6 +158,8 @@ export const apiRateLimiter = authorizationRateLimitMiddleware({
     /^\/api\/v1\/sources\/http\/[^\/]+$/, // /api/v1/sources/http/$id
     /^\/api\/v1\/endpoints\/[^\/]+\/[^\/]+\/index\/[^\/]+$/, // /api/v1/endpoints/$environmentId/$endpointSlug/index/$indexHookIdentifier
     "/api/v1/timezones",
+    "/api/v1/usage/ingest",
+    /^\/api\/v1\/runs\/[^\/]+\/attempts$/, // /api/v1/runs/$runFriendlyId/attempts
   ],
   log: {
     rejections: env.API_RATE_LIMIT_REJECTION_LOGS_ENABLED === "1",

apps/webapp/app/v3/services/createTaskRunAttempt.server.ts

@@ -52,7 +52,14 @@ export class CreateTaskRunAttemptService extends BaseService {
           },
           lockedBy: {
             include: {
-              worker: true,
+              worker: {
+                select: {
+                  id: true,
+                  version: true,
+                  sdkVersion: true,
+                  cliVersion: true,
+                },
+              },
             },
           },
           batchItems: {
@@ -185,6 +192,7 @@ export class CreateTaskRunAttemptService extends BaseService {
           durationMs: taskRun.usageDurationMs,
           costInCents: taskRun.costInCents,
           baseCostInCents: taskRun.baseCostInCents,
+          maxAttempts: taskRun.maxAttempts ?? undefined,
         },
         queue: {
           id: queue.friendlyId,

apps/webapp/app/v3/services/triggerTask.server.ts

@@ -206,6 +206,7 @@ export class TriggerTaskService extends BaseService {
                   isTest: body.options?.test ?? false,
                   delayUntil,
                   queuedAt: delayUntil ? undefined : new Date(),
+                  maxAttempts: body.options?.maxAttempts,
                   ttl,
                 },
               });

docs/v3/errors-retrying.mdx

@@ -255,11 +255,39 @@ export const openaiTask = task({
 });
 ```
 
-## Using try/catch to prevent retries
+## Preventing retries
+
+### Using `AbortTaskRunError`
+
+You can prevent retries by throwing an `AbortTaskRunError`. This will fail the task attempt and disable retrying.
+
+```ts /trigger/myTasks.ts
+import { task, AbortTaskRunError } from "@trigger.dev/sdk/v3";
+
+export const openaiTask = task({
+  id: "openai-task",
+  run: async (payload: { prompt: string }) => {
+    //if this fails, it will throw an error and stop retrying
+    const chatCompletion = await openai.chat.completions.create({
+      messages: [{ role: "user", content: payload.prompt }],
+      model: "gpt-3.5-turbo",
+    });
+
+    if (chatCompletion.choices[0]?.message.content === undefined) {
+      // If OpenAI returns an empty response, abort retrying
+      throw new AbortTaskRunError("OpenAI call failed");
+    }
+
+    return chatCompletion.choices[0].message.content;
+  },
+});
+```
+
+### Using try/catch
 
 Sometimes you want to catch an error and don't want to retry the task. You can use try/catch as you normally would. In this example we fallback to using Replicate if OpenAI fails.
 
-```ts /trigger/
+```ts /trigger/myTasks.ts
 import { task } from "@trigger.dev/sdk/v3";
 
 export const openaiTask = task({