v3: add 3MB limit on batch and single payloads

Author: ericallam

apps/webapp/app/env.server.ts

@@ -208,6 +208,7 @@ const EnvironmentSchema = z.object({
   MAXIMUM_LIVE_RELOADING_EVENTS: z.coerce.number().int().default(1000),
   MAXIMUM_TRACE_SUMMARY_VIEW_COUNT: z.coerce.number().int().default(25_000),
   TASK_PAYLOAD_OFFLOAD_THRESHOLD: z.coerce.number().int().default(524_288), // 512KB
+  TASK_PAYLOAD_MAXIMUM_SIZE: z.coerce.number().int().default(3_145_728), // 3MB
 });
 
 export type Environment = z.infer<typeof EnvironmentSchema>;

apps/webapp/app/routes/api.v1.tasks.$taskId.batch.ts

@@ -7,6 +7,7 @@ import { authenticateApiRequest } from "~/services/apiAuth.server";
 import { logger } from "~/services/logger.server";
 import { BatchTriggerTaskService } from "~/v3/services/batchTriggerTask.server";
 import { HeadersSchema } from "./api.v1.tasks.$taskId.trigger";
+import { env } from "~/env.server";
 
 const ParamsSchema = z.object({
   taskId: z.string(),
@@ -43,6 +44,12 @@ export async function action({ request, params }: ActionFunctionArgs) {
 
   const { taskId } = ParamsSchema.parse(params);
 
+  const contentLength = request.headers.get("content-length");
+
+  if (!contentLength || parseInt(contentLength) > env.TASK_PAYLOAD_MAXIMUM_SIZE) {
+    return json({ error: "Request body too large" }, { status: 413 });
+  }
+
   // Now parse the request body
   const anyBody = await request.json();
 

apps/webapp/app/routes/api.v1.tasks.$taskId.trigger.ts

@@ -2,6 +2,7 @@ import type { ActionFunctionArgs } from "@remix-run/server-runtime";
 import { json } from "@remix-run/server-runtime";
 import { TriggerTaskRequestBody } from "@trigger.dev/core/v3";
 import { z } from "zod";
+import { env } from "~/env.server";
 import { authenticateApiRequest } from "~/services/apiAuth.server";
 import { logger } from "~/services/logger.server";
 import { parseRequestJsonAsync } from "~/utils/parseRequestJson.server";
@@ -36,7 +37,7 @@ export async function action({ request, params }: ActionFunctionArgs) {
 
   const contentLength = request.headers.get("content-length");
 
-  if (!contentLength || parseInt(contentLength) > 10 * 1000 * 1000) {
+  if (!contentLength || parseInt(contentLength) > env.TASK_PAYLOAD_MAXIMUM_SIZE) {
     return json({ error: "Request body too large" }, { status: 413 });
   }
 

docs/v3/limits.mdx

@@ -37,3 +37,13 @@ If you add them dynamically using code make sure you add a `deduplicationKey` so
 If you're creating schedules for your user you will definitely need to request more schedules from us.
 
 <Snippet file="v3/soft-limit.mdx" />
+
+## Task payloads and outputs
+
+| Limit                  | Details                                       |
+| ---------------------- | --------------------------------------------- |
+| Single trigger payload | Must not exceed 3MB                           |
+| Batch trigger payload  | The total of all payloads must not exceed 3MB |
+| Task outputs           | Must not exceed 10MB                          |
+
+Payloads and outputs that exceed 512KB will be offloaded to object storage and a presigned URL will be provided to download the data when calling `runs.retrieve`. You don't need to do anything to handle this in your tasks however, as we will transparently upload/download these during operation.

docs/v3/triggering.mdx

@@ -539,7 +539,7 @@ export async function create() {
 
 ## Large Payloads
 
-We recommend keeping your task payloads as small as possible. We currently have a hard limit on task payloads above 10MB.
+We recommend keeping your task payloads as small as possible. We currently have a hard limit on task payloads above 3MB.
 
 If your payload size is larger than 512KB, instead of saving the payload to the database, we will upload it to an S3-compatible object store and store the URL in the database.
 
@@ -560,5 +560,62 @@ if (run.payloadPresignedUrl) {
 
 <Note>
   We also use this same system for dealing with large task outputs, and subsequently will return a
-  corresponding `outputPresignedUrl`
+  corresponding `outputPresignedUrl`. Task outputs are limited to 10MB.
 </Note>
+
+If you need to pass larger payloads, you'll need to upload the payload to your own storage and pass a URL to the file in the payload instead. For example, uploading to S3 and then sending a presigned URL that expires in URL:
+
+<CodeGroup>
+
+```ts /yourServer.ts
+import { myTask } from "./trigger/myTasks";
+import { s3Client, getSignedUrl, PutObjectCommand, GetObjectCommand } from "./s3";
+import { createReadStream } from "node:fs";
+
+// Upload file to S3
+await s3Client.send(
+  new PutObjectCommand({
+    Bucket: "my-bucket",
+    Key: "myfile.json",
+    Body: createReadStream("large-payload.json"),
+  })
+);
+
+// Create presigned URL
+const presignedUrl = await getSignedUrl(
+  s3Client,
+  new GetObjectCommand({
+    Bucket: "my-bucket",
+    Key: "my-file.json",
+  }),
+  {
+    expiresIn: 3600, // expires in 1 hour
+  }
+);
+
+// Now send the URL to the task
+const handle = await myTask.trigger({
+  url: presignedUrl,
+});
+```
+
+```ts /trigger/myTasks.ts
+import { task } from "@trigger.dev/sdk/v3";
+
+export const myTask = task({
+  id: "my-task",
+  run: async (payload: { url: string }) => {
+    // Download the file from the URL
+    const response = await fetch(payload.url);
+    const data = await response.json();
+
+    // Do something with the data
+  },
+});
+```
+
+</CodeGroup>
+
+### Batch Triggering
+
+When using `batchTrigger` or `batchTriggerAndWait`, the total size of all payloads cannot exceed 3MB. This means if you are doing a batch of 100 runs, each payload should be less than 30KB.