v3: stop swallowing deployment errors and display them better (#1020)

* exit worker with code 111 for handled errors

* task monitor will ignore previously handled errors

* fix logging of index errors

* deployment error component

* changeset

Author: nicktrn

.changeset/tender-oranges-rhyme.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Stop swallowing deployment errors and display them better

apps/kubernetes-provider/src/taskMonitor.ts

@@ -136,6 +136,13 @@ export class TaskMonitor {
 
     const podStatus = this.#getPodStatusSummary(pod.status);
     const containerState = this.#getContainerStateSummary(containerStatus.state);
+    const exitCode = containerState.exitCode ?? -1;
+
+    // We use this special exit code to signal any errors were already handled elsewhere
+    if (exitCode === 111) {
+      return;
+    }
+
     const rawLogs = await this.#getLogTail(podName);
 
     this.#logger.log(`${podName} failed with:`, {
@@ -144,7 +151,6 @@ export class TaskMonitor {
       rawLogs,
     });
 
-    const exitCode = containerState.exitCode ?? -1;
     const rawReason = podStatus.reason ?? containerState.reason ?? "";
     const message = podStatus.message ?? containerState.message ?? "";
 

apps/webapp/app/components/runs/v3/DeploymentError.tsx

@@ -0,0 +1,39 @@
+import { CodeBlock } from "~/components/code/CodeBlock";
+import { Callout } from "~/components/primitives/Callout";
+import { Header2 } from "~/components/primitives/Headers";
+import type { ErrorData } from "~/presenters/v3/DeploymentPresenter.server";
+
+type DeploymentErrorProps = {
+  errorData: ErrorData;
+};
+
+export function DeploymentError({ errorData }: DeploymentErrorProps) {
+  return (
+    <div className="flex flex-col gap-2 rounded-sm border border-rose-500/50 p-3">
+      <DeploymentErrorHeader title={errorData.name ?? "Error"} titleClassName="text-rose-500" />
+      {errorData.message && <Callout variant="error">{errorData.message}</Callout>}
+      {errorData.stack && (
+        <CodeBlock
+          showCopyButton={false}
+          showLineNumbers={false}
+          code={errorData.stack}
+          maxLines={20}
+        />
+      )}
+    </div>
+  );
+}
+
+function DeploymentErrorHeader({
+  title,
+  titleClassName,
+}: {
+  title: string;
+  titleClassName?: string;
+}) {
+  return (
+    <div className="flex items-center justify-between">
+      <Header2 className={titleClassName}>{title}</Header2>
+    </div>
+  );
+}

apps/webapp/app/presenters/v3/DeploymentPresenter.server.ts

@@ -12,6 +12,12 @@ import { User } from "~/models/user.server";
 import { safeJsonParse } from "~/utils/json";
 import { getUsername } from "~/utils/username";
 
+export type ErrorData = {
+  name: string;
+  message: string;
+  stack?: string;
+};
+
 export class DeploymentPresenter {
   #prismaClient: PrismaClient;
 
@@ -133,7 +139,7 @@ export class DeploymentPresenter {
     };
   }
 
-  #prepareErrorData(errorData: WorkerDeployment["errorData"]) {
+  #prepareErrorData(errorData: WorkerDeployment["errorData"]): ErrorData | undefined {
     if (!errorData) {
       return;
     }

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.deployments.$deploymentParam/route.tsx

@@ -19,6 +19,7 @@ import {
   TableHeaderCell,
   TableRow,
 } from "~/components/primitives/Table";
+import { DeploymentError } from "~/components/runs/v3/DeploymentError";
 import { DeploymentStatus } from "~/components/runs/v3/DeploymentStatus";
 import { TaskFunctionName } from "~/components/runs/v3/TaskPath";
 import { useOrganization } from "~/hooks/useOrganizations";
@@ -158,25 +159,7 @@ export default function Page() {
               </Table>
             </div>
           ) : deployment.errorData ? (
-            <div className="flex flex-col">
-              {deployment.errorData.stack ? (
-                <CodeBlock
-                  language="markdown"
-                  rowTitle={deployment.errorData.message}
-                  code={deployment.errorData.stack}
-                  maxLines={20}
-                />
-              ) : (
-                <div className="flex flex-col">
-                  <Paragraph
-                    variant="base/bright"
-                    className="w-full border-b border-grid-dimmed py-2.5"
-                  >
-                    {deployment.errorData.message}
-                  </Paragraph>
-                </div>
-              )}
-            </div>
+            <DeploymentError errorData={deployment.errorData} />
           ) : null}
         </div>
       </div>

packages/cli-v3/src/workers/prod/entry-point.ts

@@ -417,7 +417,10 @@ class ProdWorker {
             }
           } catch (e) {
             if (e instanceof TaskMetadataParseError) {
-              logger.error("tasks metadata parse error", { message: e.zodIssues, tasks: e.tasks });
+              logger.error("tasks metadata parse error", {
+                zodIssues: e.zodIssues,
+                tasks: e.tasks,
+              });
 
               socket.emit("INDEXING_FAILED", {
                 version: "v1",
@@ -429,31 +432,35 @@ class ProdWorker {
                 },
               });
             } else if (e instanceof UncaughtExceptionError) {
-              logger.error("uncaught exception", { message: e.originalError.message });
+              const error = {
+                name: e.originalError.name,
+                message: e.originalError.message,
+                stack: e.originalError.stack,
+              };
+
+              logger.error("uncaught exception", { originalError: error });
 
               socket.emit("INDEXING_FAILED", {
                 version: "v1",
                 deploymentId: this.deploymentId,
-                error: {
-                  name: e.originalError.name,
-                  message: e.originalError.message,
-                  stack: e.originalError.stack,
-                },
+                error,
               });
             } else if (e instanceof Error) {
-              logger.error("error", { message: e.message });
+              const error = {
+                name: e.name,
+                message: e.message,
+                stack: e.stack,
+              };
+
+              logger.error("error", { error });
 
               socket.emit("INDEXING_FAILED", {
                 version: "v1",
                 deploymentId: this.deploymentId,
-                error: {
-                  name: e.name,
-                  message: e.message,
-                  stack: e.stack,
-                },
+                error,
               });
             } else if (typeof e === "string") {
-              logger.error("string error", { message: e });
+              logger.error("string error", { error: { message: e } });
 
               socket.emit("INDEXING_FAILED", {
                 version: "v1",
@@ -477,7 +484,8 @@ class ProdWorker {
             }
 
             await setTimeout(200);
-            process.exit(1);
+            // Use exit code 111 so we can ignore those failures in the task monitor
+            process.exit(111);
           }
         }