v3: prod image upgrade and fixes (#1003)

* fix shutdown after final attempt

* fix workdir permissions

* correctly handle self-hosted deploy errors

* switch to node:20-bookworm-slim

* add binaries catalog file

* changeset

* skip recommended packages

* post start hook retries

* comment and improve lifecycle retry

Author: nicktrn

.changeset/light-dragons-complain.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Correctly handle self-hosted deploy command errors

apps/kubernetes-provider/src/index.ts

@@ -404,7 +404,11 @@ class KubernetesTaskOperations implements TaskOperations {
     type: THookType,
     cause: THookType extends "postStart" ? PostStartCauses : PreStopCauses
   ) {
-    return ["/bin/sh", "-c", `sleep 1; wget -q -O- 127.0.0.1:8000/${type}?cause=${cause}`];
+    const retries = 5
+
+    // This will retry sending the lifecycle hook up to `retries` times
+    // The sleep is required as this may start running before the HTTP server is up
+    return ["/bin/sh", "-c", `for i in $(seq ${retries}); do sleep 1; wget -q -O- 127.0.0.1:8000/${type}?cause=${cause} && break; done`];
   }
 
   #getIndexContainerName(suffix: string) {

packages/cli-v3/src/Containerfile.prod

@@ -1,7 +1,10 @@
-FROM node:20-alpine@sha256:bf77dc26e48ea95fca9d1aceb5acfa69d2e546b765ec2abfb502975f1a2d4def AS base
+FROM node:20-bookworm-slim@sha256:d4cdfc305abe5ea78da7167bf78263c22596dc332f2654b662890777ea166224 AS base
 
-RUN apk add --no-cache dumb-init
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && apt-get install -y --no-install-recommends dumb-init && rm -rf /var/lib/apt/lists/*
 
+# Create and set workdir with appropriate permissions
+RUN mkdir /app && chown node:node /app
 WORKDIR /app
 
 # copy all the files just in case anything is needed in postinstall

packages/cli-v3/src/commands/deploy.ts

@@ -792,7 +792,7 @@ async function buildAndPushSelfHostedImage(
     let digest: string | undefined;
 
     try {
-      await new Promise<void>((res, rej) => {
+      const processCode = await new Promise<number | null>((res, rej) => {
         // For some reason everything is output on stderr, not stdout
         buildProcess.stderr?.on("data", (data: Buffer) => {
           const text = data.toString();
@@ -802,9 +802,17 @@ async function buildAndPushSelfHostedImage(
         });
 
         buildProcess.on("error", (e) => rej(e));
-        buildProcess.on("close", () => res());
+        buildProcess.on("close", (code) => res(code));
       });
 
+      if (processCode !== 0) {
+        return {
+          ok: false as const,
+          error: "Error building image",
+          logs: extractLogs(errors),
+        };
+      }
+
       digest = extractImageDigest(errors);
 
       span.setAttributes({
@@ -835,7 +843,7 @@ async function buildAndPushSelfHostedImage(
       });
 
       try {
-        await new Promise<void>((res, rej) => {
+        const processCode = await new Promise<number | null>((res, rej) => {
           pushProcess.stdout?.on("data", (data: Buffer) => {
             const text = data.toString();
 
@@ -849,9 +857,17 @@ async function buildAndPushSelfHostedImage(
           });
 
           pushProcess.on("error", (e) => rej(e));
-          pushProcess.on("close", () => res());
+          pushProcess.on("close", (code) => res(code));
         });
 
+        if (processCode !== 0) {
+          return {
+            ok: false as const,
+            error: "Error pushing image",
+            logs: extractLogs(errors),
+          };
+        }
+
         span.end();
       } catch (e) {
         recordSpanException(span, e);

packages/cli-v3/src/workers/prod/backgroundWorker.ts

@@ -267,9 +267,6 @@ export class ProdBackgroundWorker {
 
       const result = await taskRunProcess.executeTaskRun(payload);
 
-      // Kill the worker if the task was successful or if it's not going to be retried);
-      await taskRunProcess.cleanup(result.ok || result.retry === undefined);
-
       if (result.ok) {
         return result;
       }

packages/cli-v3/src/workers/prod/entry-point.ts

@@ -358,8 +358,6 @@ class ProdWorker {
 
           this.completed.add(executionPayload.execution.attempt.id);
 
-          await this.#backgroundWorker.flushTelemetry();
-
           const { willCheckpointAndRestore, shouldExit } =
             await this.#coordinatorSocket.socket.emitWithAck("TASK_RUN_COMPLETED", {
               version: "v1",

pnpm-lock.yaml

@@ -9,11 +9,12 @@
     "@opentelemetry/api": "^1.8.0",
     "@sindresorhus/slugify": "^2.2.1",
     "@traceloop/instrumentation-openai": "^0.3.9",
-    "@trigger.dev/sdk": "workspace:^3.0.0-beta.0",
     "@trigger.dev/core": "workspace:^3.0.0-beta.0",
+    "@trigger.dev/sdk": "workspace:^3.0.0-beta.0",
     "msw": "^2.2.1",
     "openai": "^4.28.0",
-    "stripe": "^12.14.0"
+    "stripe": "^12.14.0",
+    "yt-dlp-wrap": "^2.3.12"
   },
   "devDependencies": {
     "@trigger.dev/tsconfig": "workspace:*",

references/v3-catalog/package.json

@@ -0,0 +1,22 @@
+import { logger, task } from "@trigger.dev/sdk/v3";
+import { chmod } from "node:fs/promises";
+import YTDlpWrap from "yt-dlp-wrap";
+
+export const ytDlp = task({
+  id: "yt-dlp",
+  run: async () => {
+    const releaseArtifact = "yt-dlp_linux";
+    const filePath = `./${releaseArtifact}`;
+    const fileURL = `https://github.com/yt-dlp/yt-dlp/releases/latest/download/${releaseArtifact}`;
+
+    await YTDlpWrap.downloadFile(fileURL, filePath);
+    await chmod(filePath, "777");
+
+    logger.log("downloaded", { filePath, fileURL });
+
+    const ytDlpWrap = new YTDlpWrap(filePath);
+    const version = await ytDlpWrap.getVersion();
+
+    logger.log("version", { version });
+  },
+});