better errors for crashes

Author: nicktrn

apps/kubernetes-provider/src/taskMonitor.ts

@@ -3,13 +3,14 @@ import { SimpleLogger } from "@trigger.dev/core/v3/apps";
 import { EXIT_CODE_ALREADY_HANDLED, EXIT_CODE_CHILD_NONZERO } from "@trigger.dev/core/v3/apps";
 import { setTimeout } from "timers/promises";
 import PQueue from "p-queue";
-import type { Prettify } from "@trigger.dev/core/v3";
+import { TaskRunErrorCodes, type Prettify, type TaskRunInternalError } from "@trigger.dev/core/v3";
 
 type FailureDetails = Prettify<{
   exitCode: number;
   reason: string;
   logs: string;
   overrideCompletion: boolean;
+  errorCode: TaskRunInternalError["code"];
 }>;
 
 type IndexFailureHandler = (deploymentId: string, details: FailureDetails) => Promise<any>;
@@ -160,18 +161,23 @@ export class TaskMonitor {
     let reason = rawReason || "Unknown error";
     let logs = rawLogs || "";
     let overrideCompletion = false;
+    let errorCode: TaskRunInternalError["code"] = TaskRunErrorCodes.POD_UNKNOWN_ERROR;
 
     switch (rawReason) {
       case "Error":
         reason = "Unknown error.";
+        errorCode = TaskRunErrorCodes.POD_UNKNOWN_ERROR;
         break;
       case "Evicted":
         if (message.startsWith("Pod ephemeral local storage usage")) {
           reason = "Storage limit exceeded.";
+          errorCode = TaskRunErrorCodes.DISK_SPACE_EXCEEDED;
         } else if (message) {
           reason = `Evicted: ${message}`;
+          errorCode = TaskRunErrorCodes.POD_EVICTED;
         } else {
           reason = "Evicted for unknown reason.";
+          errorCode = TaskRunErrorCodes.POD_EVICTED;
         }
 
         if (logs.startsWith("failed to try resolving symlinks")) {
@@ -183,6 +189,7 @@ export class TaskMonitor {
         reason = `${
           exitCode === EXIT_CODE_CHILD_NONZERO ? "Child process" : "Parent process"
         } ran out of memory! Try choosing a machine preset with more memory for this task.`;
+        errorCode = TaskRunErrorCodes.TASK_PROCESS_OOM_KILLED;
         break;
       default:
         break;
@@ -193,6 +200,7 @@ export class TaskMonitor {
       reason,
       logs,
       overrideCompletion,
+      errorCode,
     } satisfies FailureDetails;
 
     const app = pod.metadata?.labels?.app;

apps/webapp/app/v3/services/crashTaskRun.server.ts

@@ -5,7 +5,7 @@ import { BaseService } from "./baseService.server";
 import { logger } from "~/services/logger.server";
 import { AuthenticatedEnvironment } from "~/services/apiAuth.server";
 import { CRASHABLE_ATTEMPT_STATUSES, isCrashableRunStatus } from "../taskStatus";
-import { sanitizeError } from "@trigger.dev/core/v3";
+import { sanitizeError, TaskRunInternalError } from "@trigger.dev/core/v3";
 import { FinalizeTaskRunService } from "./finalizeTaskRun.server";
 
 export type CrashTaskRunServiceOptions = {
@@ -15,6 +15,7 @@ export type CrashTaskRunServiceOptions = {
   crashAttempts?: boolean;
   crashedAt?: Date;
   overrideCompletion?: boolean;
+  errorCode?: TaskRunInternalError["code"];
 };
 
 export class CrashTaskRunService extends BaseService {
@@ -26,6 +27,8 @@ export class CrashTaskRunService extends BaseService {
       ...options,
     };
 
+    logger.debug("CrashTaskRunService.call", { runId, opts });
+
     const taskRun = await this._prisma.taskRun.findFirst({
       where: {
         id: runId,
@@ -71,7 +74,7 @@ export class CrashTaskRunService extends BaseService {
       attemptStatus: "FAILED",
       error: {
         type: "INTERNAL_ERROR",
-        code: "TASK_RUN_CRASHED",
+        code: opts.errorCode ?? "TASK_RUN_CRASHED",
         message: opts.reason,
         stackTrace: opts.logs,
       },
@@ -129,6 +132,7 @@ export class CrashTaskRunService extends BaseService {
     error: {
       reason: string;
       logs?: string;
+      code?: TaskRunInternalError["code"];
     }
   ) {
     return await this.traceWithEnv("failAttempt()", environment, async (span) => {
@@ -146,7 +150,7 @@ export class CrashTaskRunService extends BaseService {
           completedAt: failedAt,
           error: sanitizeError({
             type: "INTERNAL_ERROR",
-            code: "TASK_RUN_CRASHED",
+            code: error.code ?? "TASK_RUN_CRASHED",
             message: error.reason,
             stackTrace: error.logs,
           }),

packages/core/src/v3/schemas/common.ts

@@ -95,6 +95,9 @@ export const TaskRunErrorCodes = {
   GRACEFUL_EXIT_TIMEOUT: "GRACEFUL_EXIT_TIMEOUT",
   TASK_RUN_CRASHED: "TASK_RUN_CRASHED",
   MAX_DURATION_EXCEEDED: "MAX_DURATION_EXCEEDED",
+  DISK_SPACE_EXCEEDED: "DISK_SPACE_EXCEEDED",
+  POD_EVICTED: "POD_EVICTED",
+  POD_UNKNOWN_ERROR: "POD_UNKNOWN_ERROR",
 } as const;
 
 export const TaskRunInternalError = z.object({
@@ -118,6 +121,9 @@ export const TaskRunInternalError = z.object({
     "TASK_RUN_HEARTBEAT_TIMEOUT",
     "TASK_RUN_CRASHED",
     "MAX_DURATION_EXCEEDED",
+    "DISK_SPACE_EXCEEDED",
+    "POD_EVICTED",
+    "POD_UNKNOWN_ERROR",
   ]),
   message: z.string().optional(),
   stackTrace: z.string().optional(),

packages/core/src/v3/schemas/messages.ts

@@ -5,6 +5,7 @@ import {
   TaskRunExecution,
   TaskRunExecutionResult,
   TaskRunFailedExecutionResult,
+  TaskRunInternalError,
 } from "./common.js";
 import { TaskResource } from "./resources.js";
 import {
@@ -252,6 +253,7 @@ export const ProviderToPlatformMessages = {
       message: z.string().optional(),
       logs: z.string().optional(),
       overrideCompletion: z.boolean().optional(),
+      errorCode: TaskRunInternalError.shape.code.optional(),
     }),
   },
   INDEXING_FAILED: {