How it works doc

Author: ericallam

docs/how-it-works.mdx

@@ -4,83 +4,79 @@ sidebarTitle: "How it works"
 description: "Understand how Trigger.dev works and how it can help you."
 ---
 
+## Introduction
+
 Trigger.dev v3 allows you to embed long-running async tasks into your application and run them in the background. This allows you to offload tasks that take a long time to complete, such as sending emails, processing videos, or running long chains of AI tasks.
 
 For example, the below task processes a video with `ffmpeg` and sends the results to an s3 bucket, then updates a database with the results and sends an email to the user.
 
 ```ts /trigger/video.ts
-import { task, logger } from "@trigger.dev/sdk/v3";
+import { logger, task } from "@trigger.dev/sdk/v3";
+import { updateVideoUrl } from "../db.js";
 import ffmpeg from "fluent-ffmpeg";
 import { Readable } from "node:stream";
+import type { ReadableStream } from "node:stream/web";
 import * as fs from "node:fs/promises";
 import * as path from "node:path";
 import { S3Client, PutObjectCommand } from "@aws-sdk/client-s3";
-import { updateDatabase } from "./database";
-import { sendEmail } from "./email";
+import { sendEmail } from "../email.js";
+import { getVideo } from "../db.js";
 
 // Initialize S3 client
-const s3Client = new S3Client({ region: process.env.AWS_REGION }); // Replace with your preferred region
+const s3Client = new S3Client({
+  region: process.env.AWS_REGION,
+});
 
 export const convertVideo = task({
   id: "convert-video",
-  run: async (payload: { videoUrl: string; videoId: string; userId: string }) => {
-    const { videoUrl, videoId, userId } = payload;
+  retry: {
+    maxAttempts: 5,
+    minTimeoutInMs: 1000,
+    maxTimeoutInMs: 10000,
+    factor: 2,
+  },
+  run: async ({ videoId }: { videoId: string }) => {
+    const { url, userId } = await getVideo(videoId);
 
-    const outputPath = path.join("/tmp", `output_${Date.now()}.mp4`);
+    const outputPath = path.join("/tmp", `output_${videoId}.mp4`);
 
-    // Fetch the video
-    const response = await fetch(videoUrl);
+    const response = await fetch(url);
 
-    // Process the video
     await new Promise((resolve, reject) => {
-      ffmpeg(Readable.fromWeb(response.body))
-        .videoFilters("scale=iw/2:ih/2") // This halves both width and height
+      ffmpeg(Readable.fromWeb(response.body as ReadableStream))
+        .videoFilters("scale=iw/2:ih/2")
         .output(outputPath)
         .on("end", resolve)
         .on("error", reject)
         .run();
     });
 
-    // Upload the video to S3
-    const s3Key = `processed-videos/${path.basename(outputPath)}`;
-
-    try {
-      const fileContent = await fs.readFile(outputPath);
-      const uploadParams = {
-        Bucket: process.env.S3_BUCKET,
-        Key: s3Key,
-        Body: fileContent,
-      };
-
-      await s3Client.send(new PutObjectCommand(uploadParams));
-
-      logger.info("Video uploaded successfully to S3");
+    const processedContent = await fs.readFile(outputPath);
 
-      // Generate the S3 URL
-      const s3Url = `https://${process.env.S3_BUCKET}.s3.amazonaws.com/${s3Key}`;
+    // Upload to S3
+    const s3Key = `processed-videos/output_${videoId}.mp4`;
 
-      // Update the database with the S3 URL
-      await updateDatabase(videoId, s3Url);
+    const uploadParams = {
+      Bucket: process.env.S3_BUCKET,
+      Key: s3Key,
+      Body: processedContent,
+    };
 
-      logger.info("Database updated with S3 URL");
+    await s3Client.send(new PutObjectCommand(uploadParams));
+    const s3Url = `https://${process.env.S3_BUCKET}.s3.amazonaws.com/${s3Key}`;
 
-      // Send an email to the user
-      await sendEmail(
-        userId,
-        "Video Processing Complete",
-        `Your video has been processed and is available at: ${s3Url}`
-      );
+    logger.info("Video converted", { videoId, s3Url });
 
-      logger.info("Email sent to user");
+    // Update database
+    await updateVideoUrl(videoId, s3Url);
 
-      // Clean up the temporary file
-      await fs.unlink(outputPath);
+    await sendEmail(
+      userId,
+      "Video Processing Complete",
+      `Your video has been processed and is available at: ${s3Url}`
+    );
 
-      return { success: true, s3Url };
-    } catch (error) {
-      logger.error("Error processing video:", error);
-      throw error;
-    }
+    return { success: true, s3Url };
   },
 });
 ```
@@ -96,6 +92,7 @@ import type { convertVideo } from "./trigger/video";
 export async function POST(request: Request) {
   const body = await request.json();
 
+  // Trigger the task, this will return before the task is completed
   const handle = await tasks.trigger<typeof convertVideo>("convert-video", body);
 
   return NextResponse.json(handle);
@@ -104,6 +101,25 @@ export async function POST(request: Request) {
 
 This will schedule the task to run in the background and return a handle that you can use to check the status of the task. This allows your backend application to respond quickly to the user and offload the long-running task to Trigger.dev.
 
+## The CLI
+
+Trigger.dev comes with a CLI that allows you to initialize Trigger.dev into your project, deploy your tasks, and run your tasks locally. You can run it via `npx` like so:
+
+```sh
+npx trigger.dev@latest login # Log in to your Trigger.dev account
+npx trigger.dev@latest init # Initialize Trigger.dev in your project
+npx trigger.dev@latest dev # Run your tasks locally
+npx trigger.dev@latest deploy # Deploy your tasks to the Trigger.dev instance
+```
+
+All these commands work with the Trigger.dev cloud and/or your self-hosted instance. It supports multiple profiles so you can easily switch between different accounts or instances.
+
+```sh
+npx trigger.dev@latest login --profile <profile> -a https://trigger.example.com # Log in to a specific profile into a self-hosted instance
+npx trigger.dev@latest dev --profile <profile> # Initialize Trigger.dev in your project
+npx trigger.dev@latest deploy --profile <profile> # Deploy your tasks to the Trigger.dev instance
+```
+
 ## Trigger.dev architecture
 
 Trigger.dev implements a serverless architecture (without timeouts!) that allows you to run your tasks in a scalable and reliable way. When you run `npx trigger.dev@latest deploy`, we build and deploy your task code to your Trigger.dev instance. Then, when you trigger a task from your application, it's run in a secure, isolated environment with the resources you need to complete the task. A simplified diagram for a task execution looks like this:
@@ -233,36 +249,37 @@ Let's rewrite the `convert-video` task above to be more durable:
 <CodeGroup>
 
 ```ts /trigger/video.ts
-import { task, logger } from "@trigger.dev/sdk/v3";
-import { processVideo, uploadToS3, sendUserEmail } from "./tasks.js";
-import { updateDatabase } from "./database.js";
+import { idempotencyKeys, logger, task } from "@trigger.dev/sdk/v3";
+import { processVideo, sendUserEmail, uploadToS3 } from "./tasks.js";
+import { updateVideoUrl } from "../db.js";
 
 export const convertVideo = task({
   id: "convert-video",
-  retries: {
-    maxAttempts: 10,
+  retry: {
+    maxAttempts: 5,
     minTimeoutInMs: 1000,
     maxTimeoutInMs: 10000,
     factor: 2,
   },
-  run: async (payload: { videoUrl: string; videoId: string; userId: string }) => {
-    const { videoUrl, userId, videoId } = payload;
+  run: async ({ videoId }: { videoId: string }) => {
+    // Automatically scope the idempotency key to this run, across retries
+    const idempotencyKey = await idempotencyKeys.create(videoId);
 
     // Process video
     const { processedContent } = await processVideo
-      .triggerAndWait({ videoUrl, videoId }, { idempotencyKey: ["process", videoId] })
+      .triggerAndWait({ videoId }, { idempotencyKey })
       .unwrap(); // Calling unwrap will return the output of the subtask, or throw an error if the subtask failed
 
     // Upload to S3
     const { s3Url } = await uploadToS3
-      .triggerAndWait({ processedContent, videoId }, { idempotencyKey: ["upload", videoId] })
+      .triggerAndWait({ processedContent, videoId }, { idempotencyKey })
       .unwrap();
 
     // Update database
-    await updateDatabase(videoId, s3Url);
+    await updateVideoUrl(videoId, s3Url);
 
     // Send email, we don't need to wait for this to finish
-    await sendUserEmail.trigger({ userId, s3Url }, { idempotencyKey: ["email", videoId] });
+    await sendUserEmail.trigger({ videoId, s3Url }, { idempotencyKey });
 
     return { success: true, s3Url };
   },
@@ -273,32 +290,40 @@ export const convertVideo = task({
 import { task, logger } from "@trigger.dev/sdk/v3";
 import ffmpeg from "fluent-ffmpeg";
 import { Readable } from "node:stream";
+import type { ReadableStream } from "node:stream/web";
 import * as fs from "node:fs/promises";
 import * as path from "node:path";
 import { S3Client, PutObjectCommand } from "@aws-sdk/client-s3";
-import { sendEmail } from "./email";
+import { sendEmail } from "../email.js";
+import { getVideo } from "../db.js";
 
 // Initialize S3 client
-const s3Client = new S3Client({ region: process.env.AWS_REGION });
+const s3Client = new S3Client({
+  region: process.env.AWS_REGION,
+});
 
 export const processVideo = task({
   id: "process-video",
-  run: async (payload: { videoUrl: string; videoId: string }) => {
-    const { videoUrl, videoId } = payload;
-    const outputPath = path.join("/tmp", `output_${videoId}.mp4`);
-    const response = await fetch(videoUrl);
+  run: async ({ videoId }: { videoId: string }) => {
+    const { url } = await getVideo(videoId);
 
-    await new Promise((resolve, reject) => {
-      ffmpeg(Readable.fromWeb(response.body))
-        .videoFilters("scale=iw/2:ih/2")
-        .output(outputPath)
-        .on("end", resolve)
-        .on("error", reject)
-        .run();
+    const outputPath = path.join("/tmp", `output_${videoId}.mp4`);
+    const response = await fetch(url);
+
+    await logger.trace("ffmpeg", async (span) => {
+      await new Promise((resolve, reject) => {
+        ffmpeg(Readable.fromWeb(response.body as ReadableStream))
+          .videoFilters("scale=iw/2:ih/2")
+          .output(outputPath)
+          .on("end", resolve)
+          .on("error", reject)
+          .run();
+      });
     });
 
     const processedContent = await fs.readFile(outputPath);
-    await fs.unlink(outputPath); // Clean up the temporary file
+
+    await fs.unlink(outputPath);
 
     return { processedContent: processedContent.toString("base64") };
   },
@@ -308,7 +333,9 @@ export const uploadToS3 = task({
   id: "upload-to-s3",
   run: async (payload: { processedContent: string; videoId: string }) => {
     const { processedContent, videoId } = payload;
+
     const s3Key = `processed-videos/output_${videoId}.mp4`;
+
     const uploadParams = {
       Bucket: process.env.S3_BUCKET,
       Key: s3Key,
@@ -324,9 +351,10 @@ export const uploadToS3 = task({
 
 export const sendUserEmail = task({
   id: "send-user-email",
-  run: async (payload: { userId: string; s3Url: string }) => {
-    const { userId, s3Url } = payload;
-    await sendEmail(
+  run: async ({ videoId, s3Url }: { videoId: string; s3Url: string }) => {
+    const { userId } = await getVideo(videoId);
+
+    return await sendEmail(
       userId,
       "Video Processing Complete",
       `Your video has been processed and is available at: ${s3Url}`
@@ -374,14 +402,50 @@ sequenceDiagram
 
 ## The build system
 
-When you run `npx trigger.dev@latest deploy` or `npx trigger.dev@latest dev`, we build your task code using our build system, which is powered by esbuild. When deploying, the code is packaged up into a Docker image and deployed to your Trigger.dev instance. When running in development mode, the code is built and run locally on your machine. Some features of our build system include:
+When you run `npx trigger.dev@latest deploy` or `npx trigger.dev@latest dev`, we build your task code using our build system, which is powered by [esbuild](https://esbuild.github.io/). When deploying, the code is packaged up into a Docker image and deployed to your Trigger.dev instance. When running in dev mode, the code is built and run locally on your machine. Some features of our build system include:
 
 - **Bundled by default**: Code + dependencies are bundled and tree-shaked by default.
 - **Build extensions**: Use and write custom build extensions to transform your code or the resulting docker image.
 - **ESM ouput**: We output to ESM, which allows tree-shaking and better performance.
 
-You can learn more about working with our build system in the [configuration docs](/config/config-file).
+You can review the build output by running deploy with the `--dry-run` flag, which will output the Containerfile and the build output.
+
+Learn more about working with our build system in the [configuration docs](/config/config-file).
+
+## Dev mode
+
+When you run `npx trigger.dev@latest dev`, we run your task code locally on your machine. All scheduling is still done in the Trigger.dev server instance, but the task code is run locally. This allows you to develop and test your tasks locally before deploying them to the cloud, and is especially useful for debugging and testing.
+
+- The same build system is used in dev mode, so you can be sure that your code will run the same locally as it does in the cloud.
+- Changes are automatically detected and a new version is spun up when you save your code.
+- Add debuggers and breakpoints to your code and debug it locally.
+- Each task is run in a separate process, so you can run multiple tasks in parallel.
+- Auto-cancels tasks when you stop the dev server.
+
+<Note>
+  Trigger.dev currently does not support "offline" dev mode, where you can run tasks without an
+  internet connection. Please let us know if this is a feature you want/need.
+</Note>
+
+## Staging and production environments
+
+Trigger.dev supports deploying to multiple "deployed" environments, such as staging and production. This allows you to test your tasks in a staging environment before deploying them to production. You can deploy to a new environment by running `npx trigger.dev@latest deploy --env <env>`, where `<env>` is the name of the environment you want to deploy to. Each environment has its own API Key, which you can use to trigger tasks in that environment.
 
 ## OpenTelemetry
 
-The Trigger.dev logging and task dashboard is powered by OpenTelemetry traces and logs, which allows you to trace your tasks and auto-instrument your code.
+The Trigger.dev logging and task dashboard is powered by OpenTelemetry traces and logs, which allows you to trace your tasks and auto-instrument your code. We also auto-correlate logs from subtasks and parent tasks, making it easy view the entire trace of a task execution. A single run of the video processing task above looks like this in the dashboard:
+
+![OpenTelemetry trace](/images/opentelemetry-trace.png)
+
+Because we use standard OpenTelemetry, you can instrument your code and OpenTelemetry compatible libraries to get detailed traces and logs of your tasks. The above trace instruments both Prisma and the AWS SDK:
+
+```ts trigger.config.ts
+import { defineConfig } from "@trigger.dev/sdk/v3";
+import { PrismaInstrumentation } from "@prisma/instrumentation";
+import { AwsInstrumentation } from "@opentelemetry/instrumentation-aws-sdk";
+
+export default defineConfig({
+  project: "<your-project-ref>",
+  instrumentations: [new PrismaInstrumentation(), new AwsInstrumentation()],
+});
+```