v3: improve non-zero exit errors (#1179)

* docker provider will optionally enforce machine presets

* update task monitor oom message

* add oom task to v3 catalog

* improve handling of non-zero exit errors

* add changeset

Author: nicktrn

.changeset/nervous-planets-sparkle.md

@@ -0,0 +1,9 @@
+---
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+- Improve non-zero exit code error messages
+- Detect OOM conditions within worker child processes
+- Internal errors can have optional stack traces
+- Docker provider can be set to enforce machine presets

apps/docker-provider/src/index.ts

@@ -109,22 +109,27 @@ class DockerTaskOperations implements TaskOperations {
 
     const containerName = this.#getRunContainerName(opts.runId);
 
+    const runArgs = [
+      "run",
+      "--network=host",
+      "--detach",
+      `--env=TRIGGER_ENV_ID=${opts.envId}`,
+      `--env=TRIGGER_RUN_ID=${opts.runId}`,
+      `--env=OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT}`,
+      `--env=POD_NAME=${containerName}`,
+      `--env=COORDINATOR_HOST=${COORDINATOR_HOST}`,
+      `--env=COORDINATOR_PORT=${COORDINATOR_PORT}`,
+      `--name=${containerName}`,
+    ];
+
+    if (process.env.ENFORCE_MACHINE_PRESETS) {
+      runArgs.push(`--cpus=${opts.machine.cpu}`, `--memory=${opts.machine.memory}G`);
+    }
+
+    runArgs.push(`${opts.image}`);
+
     try {
-      logger.debug(
-        await execa("docker", [
-          "run",
-          "--network=host",
-          "--detach",
-          `--env=TRIGGER_ENV_ID=${opts.envId}`,
-          `--env=TRIGGER_RUN_ID=${opts.runId}`,
-          `--env=OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT}`,
-          `--env=POD_NAME=${containerName}`,
-          `--env=COORDINATOR_HOST=${COORDINATOR_HOST}`,
-          `--env=COORDINATOR_PORT=${COORDINATOR_PORT}`,
-          `--name=${containerName}`,
-          `${opts.image}`,
-        ])
-      );
+      logger.debug(await execa("docker", runArgs));
     } catch (error) {
       if (!isExecaChildProcess(error)) {
         throw error;

apps/kubernetes-provider/src/taskMonitor.ts

@@ -181,7 +181,8 @@ export class TaskMonitor {
         }
         break;
       case "OOMKilled":
-        reason = "Out of memory! Try increasing the memory on this task.";
+        reason =
+          "Process ran out of memory! Try choosing a machine preset with more memory for this task.";
         break;
       default:
         break;

apps/webapp/app/v3/eventRepository.server.ts

@@ -1118,6 +1118,7 @@ export function createExceptionPropertiesFromError(error: TaskRunError): Excepti
       return {
         type: "Internal error",
         message: [error.code, error.message].filter(Boolean).join(": "),
+        stacktrace: error.stackTrace,
       };
     }
     case "STRING_ERROR": {

packages/cli-v3/src/workers/common/errors.ts

@@ -23,7 +23,11 @@ export class TaskMetadataParseError extends Error {
 }
 
 export class UnexpectedExitError extends Error {
-  constructor(public code: number) {
+  constructor(
+    public code: number,
+    public signal: NodeJS.Signals | null,
+    public stderr: string | undefined
+  ) {
     super(`Unexpected exit with code ${code}`);
 
     this.name = "UnexpectedExitError";
@@ -61,3 +65,25 @@ export class GracefulExitTimeoutError extends Error {
     this.name = "GracefulExitTimeoutError";
   }
 }
+
+export function getFriendlyErrorMessage(
+  code: number,
+  signal: NodeJS.Signals | null,
+  stderr: string | undefined
+) {
+  const message = (text: string) => {
+    if (signal) {
+      return `[${signal}] ${text}`;
+    } else {
+      return text;
+    }
+  };
+
+  if (code === 137 || stderr?.includes("OOMErrorHandler")) {
+    return message(
+      "Process ran out of memory! Try choosing a machine preset with more memory for this task."
+    );
+  }
+
+  return message(`Process exited with code ${code}.`);
+}

packages/cli-v3/src/workers/dev/backgroundWorker.ts

@@ -45,6 +45,7 @@ import {
   TaskMetadataParseError,
   UncaughtExceptionError,
   UnexpectedExitError,
+  getFriendlyErrorMessage,
 } from "../common/errors.js";
 import { CliApiClient } from "../../apiClient.js";
 
@@ -723,6 +724,8 @@ export class BackgroundWorker {
           error: {
             type: "INTERNAL_ERROR",
             code: TaskRunErrorCodes.TASK_PROCESS_EXITED_WITH_NON_ZERO_CODE,
+            message: getFriendlyErrorMessage(e.code, e.signal, e.stderr),
+            stackTrace: e.stderr,
           },
         };
       }
@@ -782,6 +785,7 @@ class TaskRunProcess {
   private _currentExecution: TaskRunExecution | undefined;
   private _isBeingKilled: boolean = false;
   private _isBeingCancelled: boolean = false;
+  private _stderr: Array<string> = [];
   /**
    * @deprecated use onTaskRunHeartbeat instead
    */
@@ -1009,7 +1013,13 @@ class TaskRunProcess {
         } else if (this._isBeingKilled) {
           rejecter(new CleanupProcessError());
         } else {
-          rejecter(new UnexpectedExitError(code ?? -1));
+          rejecter(
+            new UnexpectedExitError(
+              code ?? -1,
+              signal,
+              this._stderr.length ? this._stderr.join("\n") : undefined
+            )
+          );
         }
       }
     }
@@ -1048,9 +1058,16 @@ class TaskRunProcess {
       `${this._currentExecution.run.id}.${this._currentExecution.attempt.number}`
     );
 
+    const errorLine = data.toString();
+
     logger.log(
-      `${chalkError("○")} ${chalkGrey(prettyPrintDate(new Date()))} ${runId} ${data.toString()}`
+      `${chalkError("○")} ${chalkGrey(prettyPrintDate(new Date()))} ${runId} ${errorLine}`
     );
+
+    if (this._stderr.length > 100) {
+      this._stderr.shift();
+    }
+    this._stderr.push(errorLine);
   }
 
   #kill() {

packages/cli-v3/src/workers/prod/backgroundWorker.ts

@@ -29,6 +29,7 @@ import {
   TaskMetadataParseError,
   UncaughtExceptionError,
   UnexpectedExitError,
+  getFriendlyErrorMessage,
 } from "../common/errors";
 
 type BackgroundWorkerParams = {
@@ -462,6 +463,8 @@ export class ProdBackgroundWorker {
           error: {
             type: "INTERNAL_ERROR",
             code: TaskRunErrorCodes.TASK_PROCESS_EXITED_WITH_NON_ZERO_CODE,
+            message: getFriendlyErrorMessage(e.code, e.signal, e.stderr),
+            stackTrace: e.stderr,
           },
         };
       }
@@ -576,6 +579,7 @@ class TaskRunProcess {
   private _isBeingKilled: boolean = false;
   private _isBeingCancelled: boolean = false;
   private _gracefulExitTimeoutElapsed: boolean = false;
+  private _stderr: Array<string> = [];
 
   /**
    * @deprecated use onTaskRunHeartbeat instead
@@ -854,7 +858,13 @@ class TaskRunProcess {
         } else if (this._isBeingKilled) {
           rejecter(new CleanupProcessError());
         } else {
-          rejecter(new UnexpectedExitError(code ?? -1));
+          rejecter(
+            new UnexpectedExitError(
+              code ?? -1,
+              signal,
+              this._stderr.length ? this._stderr.join("\n") : undefined
+            )
+          );
         }
       }
     }
@@ -867,7 +877,13 @@ class TaskRunProcess {
   }
 
   #handleStdErr(data: Buffer) {
-    console.error(data.toString());
+    const text = data.toString();
+    console.error(text);
+
+    if (this._stderr.length > 100) {
+      this._stderr.shift();
+    }
+    this._stderr.push(text);
   }
 
   async kill(signal?: number | NodeJS.Signals, timeoutInMs?: number) {

packages/core/src/v3/schemas/common.ts

@@ -109,6 +109,7 @@ export const TaskRunInternalError = z.object({
     "TASK_RUN_HEARTBEAT_TIMEOUT",
   ]),
   message: z.string().optional(),
+  stackTrace: z.string().optional(),
 });
 
 export type TaskRunInternalError = z.infer<typeof TaskRunInternalError>;

references/v3-catalog/src/trigger/other.ts

@@ -82,3 +82,28 @@ export const unfriendlyIdTask = task({
     console.log("Hello world");
   },
 });
+
+export const oomTask = task({
+  id: "oom-task",
+  machine: {
+    preset: "micro",
+  },
+  run: async () => {
+    logger.info("running out of memory below this line");
+
+    let a = "a";
+
+    try {
+      while (true) {
+        a += a;
+      }
+    } catch (error) {
+      logger.error(error instanceof Error ? error.message : "Unknown error", { error });
+
+      let b = [];
+      while (true) {
+        b.push(a.replace(/a/g, "b"));
+      }
+    }
+  },
+});