Merge remote-tracking branch 'origin/main' into run-engine-2-retry-oom

# Conflicts:
#	packages/core/package.json

Author: matt-aitken

.dockerignore

@@ -9,7 +9,7 @@
 
 # dependencies
 
-node_modules
+**/node_modules
 .pnp
 .pnp.js
 

.github/actions/get-image-tag/action.yml

@@ -31,6 +31,11 @@ runs:
             sha=$(echo ${{ github.sha }} | head -c7)
             ts=$(date +%s)
             tag=${env}-${sha}-${ts}
+          elif [[ "${{ github.ref_name }}" == re2-*-* ]]; then
+            env=$(echo ${{ github.ref_name }} | cut -d- -f2)
+            sha=$(echo ${{ github.sha }} | head -c7)
+            ts=$(date +%s)
+            tag=${env}-${sha}-${ts}
           elif [[ "${{ github.ref_name }}" == v.docker.* ]]; then
             version="${GITHUB_REF_NAME#v.docker.}"
             tag="v${version}"

.github/workflows/publish-worker-re2.yml

@@ -0,0 +1,95 @@
+name: "⚒️ Publish Worker RE2"
+
+on:
+  workflow_call:
+    inputs:
+      image_tag:
+        description: The image tag to publish
+        type: string
+        required: false
+        default: ""
+  push:
+    tags:
+      - "re2-test-*"
+      - "re2-prod-*"
+
+permissions:
+  packages: write
+  contents: read
+
+jobs:
+  check-branch:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Fail if re2-prod-* is pushed from a non-main branch
+        if: startsWith(github.ref_name, 're2-prod-') && github.base_ref != 'main'
+        run: |
+          echo "🚫 re2-prod-* tags can only be pushed from the main branch."
+          exit 1
+  build:
+    needs: check-branch
+    strategy:
+      matrix:
+        package: [supervisor]
+    runs-on: ubuntu-latest
+    env:
+      DOCKER_BUILDKIT: "1"
+    steps:
+      - name: ⬇️ Checkout git repo
+        uses: actions/checkout@v4
+
+      - name: 📦 Get image repo
+        id: get_repository
+        run: |
+          if [[ "${{ matrix.package }}" == *-provider ]]; then
+            provider_type=$(echo "${{ matrix.package }}" | cut -d- -f1)
+            repo=provider/${provider_type}
+          else
+            repo="${{ matrix.package }}"
+          fi
+          echo "repo=${repo}" >> "$GITHUB_OUTPUT"
+
+      - id: get_tag
+        uses: ./.github/actions/get-image-tag
+        with:
+          tag: ${{ inputs.image_tag }}
+
+      - name: 🐋 Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      # ..to avoid rate limits when pulling images
+      - name: 🐳 Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: 🚢 Build Container Image
+        run: |
+          docker build -t infra_image -f ./apps/${{ matrix.package }}/Containerfile .
+
+      # ..to push image
+      - name: 🐙 Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: 🐙 Push to GitHub Container Registry
+        run: |
+          docker tag infra_image "$REGISTRY/$REPOSITORY:$IMAGE_TAG"
+          docker push "$REGISTRY/$REPOSITORY:$IMAGE_TAG"
+        env:
+          REGISTRY: ghcr.io/triggerdotdev
+          REPOSITORY: ${{ steps.get_repository.outputs.repo }}
+          IMAGE_TAG: ${{ steps.get_tag.outputs.tag }}
+
+      - name: 🐙 Push 'v3' tag to GitHub Container Registry
+        if: steps.get_tag.outputs.is_semver == 'true'
+        run: |
+          docker tag infra_image "$REGISTRY/$REPOSITORY:v3"
+          docker push "$REGISTRY/$REPOSITORY:v3"
+        env:
+          REGISTRY: ghcr.io/triggerdotdev
+          REPOSITORY: ${{ steps.get_repository.outputs.repo }}

apps/coordinator/src/checkpointer.ts

@@ -1,5 +1,5 @@
 import { ExponentialBackoff } from "@trigger.dev/core/v3/apps";
-import { testDockerCheckpoint } from "@trigger.dev/core/v3/checkpoints";
+import { testDockerCheckpoint } from "@trigger.dev/core/v3/serverOnly";
 import { nanoid } from "nanoid";
 import fs from "node:fs/promises";
 import { ChaosMonkey } from "./chaosMonkey";

apps/supervisor/.env.example

@@ -0,0 +1,18 @@
+# This needs to match the token of the worker group you want to connect to
+TRIGGER_WORKER_TOKEN=
+
+# This needs to match the MANAGED_WORKER_SECRET env var on the webapp
+MANAGED_WORKER_SECRET=managed-secret
+
+# Point this at the webapp in prod
+TRIGGER_API_URL=http://localhost:3030
+
+# Point this at the OTel collector in prod
+OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:3030/otel
+# Use this on macOS
+# OTEL_EXPORTER_OTLP_ENDPOINT=http://host.docker.internal:3030/otel
+
+# Optional settings
+DEBUG=1
+ENFORCE_MACHINE_PRESETS=1
+TRIGGER_DEQUEUE_INTERVAL_MS=1000

apps/supervisor/.nvmrc

@@ -0,0 +1 @@
+v22.12.0

apps/supervisor/Containerfile

@@ -0,0 +1,63 @@
+FROM node:22-alpine@sha256:9bef0ef1e268f60627da9ba7d7605e8831d5b56ad07487d24d1aa386336d1944 AS node-22-alpine
+
+WORKDIR /app
+
+FROM node-22-alpine AS pruner
+
+COPY --chown=node:node . .
+RUN npx -q [email protected] prune --scope=supervisor --docker
+RUN find . -name "node_modules" -type d -prune -exec rm -rf '{}' +
+
+FROM node-22-alpine AS base
+
+RUN apk add --no-cache dumb-init
+
+COPY --chown=node:node .gitignore .gitignore
+COPY --from=pruner --chown=node:node /app/out/json/ .
+COPY --from=pruner --chown=node:node /app/out/pnpm-lock.yaml ./pnpm-lock.yaml
+COPY --from=pruner --chown=node:node /app/out/pnpm-workspace.yaml ./pnpm-workspace.yaml
+
+FROM base AS dev-deps
+RUN corepack enable
+ENV NODE_ENV development
+
+RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm fetch --no-frozen-lockfile
+RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm install --ignore-scripts --no-frozen-lockfile
+
+FROM base AS prod-deps
+RUN corepack enable
+ENV NODE_ENV production
+
+RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm install --prod --no-frozen-lockfile
+
+COPY --from=pruner --chown=node:node /app/internal-packages/database/prisma/schema.prisma /app/internal-packages/database/prisma/schema.prisma
+
+ENV NPM_CONFIG_IGNORE_WORKSPACE_ROOT_CHECK true
+RUN pnpx [email protected] generate --schema /app/internal-packages/database/prisma/schema.prisma
+
+FROM base AS builder
+RUN corepack enable
+
+COPY --from=pruner --chown=node:node /app/out/full/ .
+COPY --from=dev-deps --chown=node:node /app/ .
+COPY --chown=node:node turbo.json turbo.json
+COPY --chown=node:node .configs/tsconfig.base.json .configs/tsconfig.base.json
+COPY --chown=node:node scripts/updateVersion.ts scripts/updateVersion.ts
+
+RUN pnpm run generate && \
+  pnpm run -r --filter supervisor... build
+
+FROM base AS runner
+
+RUN corepack enable
+ENV NODE_ENV production
+
+COPY --from=pruner --chown=node:node /app/out/full/ .
+COPY --from=prod-deps --chown=node:node /app .
+COPY --from=builder --chown=node:node /app/apps/supervisor ./apps/supervisor
+
+EXPOSE 8000
+
+USER node
+
+CMD [ "/usr/bin/dumb-init", "--", "pnpm", "run", "--filter", "supervisor", "start"]

apps/supervisor/README.md

@@ -0,0 +1,49 @@
+# Supervisor
+
+## Dev setup
+
+1. Create a worker group
+
+```sh
+api_url=http://localhost:3030
+wg_name=my-worker
+
+# edit these
+admin_pat=tr_pat_...
+project_id=clsw6q8wz...
+
+curl -sS \
+    -X POST \
+    "$api_url/admin/api/v1/workers" \
+    -H "Authorization: Bearer $admin_pat" \
+    -H "Content-Type: application/json" \
+    -d "{
+        \"name\": \"$wg_name\",
+        \"makeDefault\": true,
+        \"projectId\": \"$project_id\"
+    }"
+```
+
+2. Create `.env` and set the worker token
+
+```sh
+cp .env.example .env
+
+# Then edit your .env and set this to the token.plaintext value
+TRIGGER_WORKER_TOKEN=tr_wgt_...
+```
+
+3. Start the supervisor
+
+```sh
+pnpm dev
+```
+
+4. Build CLI, then deploy a reference project
+
+```sh
+pnpm exec trigger deploy --self-hosted
+
+# The additional network flag is required on linux
+pnpm exec trigger deploy --self-hosted --network host
+```

apps/supervisor/package.json

@@ -0,0 +1,27 @@
+{
+  "name": "supervisor",
+  "private": true,
+  "version": "0.0.1",
+  "main": "dist/index.js",
+  "type": "module",
+  "scripts": {
+    "build": "tsc",
+    "dev": "tsx --experimental-sqlite --require dotenv/config --watch src/index.ts",
+    "start": "node --experimental-sqlite dist/index.js",
+    "typecheck": "tsc --noEmit"
+  },
+  "dependencies": {
+    "@kubernetes/client-node": "^1.0.0",
+    "@trigger.dev/core": "workspace:*",
+    "dockerode": "^4.0.3",
+    "nanoid": "^5.0.9",
+    "socket.io": "4.7.4",
+    "std-env": "^3.8.0",
+    "tinyexec": "^0.3.1",
+    "zod": "3.23.8"
+  },
+  "devDependencies": {
+    "@types/dockerode": "^3.3.33",
+    "docker-api-ts": "^0.2.2"
+  }
+}

apps/supervisor/src/clients/kubernetes.ts

@@ -0,0 +1,39 @@
+import * as k8s from "@kubernetes/client-node";
+import { assertExhaustive } from "@trigger.dev/core/utils";
+
+export const RUNTIME_ENV = process.env.KUBERNETES_PORT ? "kubernetes" : "local";
+
+export function createK8sApi() {
+  const kubeConfig = getKubeConfig();
+
+  const api = {
+    core: kubeConfig.makeApiClient(k8s.CoreV1Api),
+    batch: kubeConfig.makeApiClient(k8s.BatchV1Api),
+    apps: kubeConfig.makeApiClient(k8s.AppsV1Api),
+  };
+
+  return api;
+}
+
+export type K8sApi = ReturnType<typeof createK8sApi>;
+
+function getKubeConfig() {
+  console.log("getKubeConfig()", { RUNTIME_ENV });
+
+  const kubeConfig = new k8s.KubeConfig();
+
+  switch (RUNTIME_ENV) {
+    case "local":
+      kubeConfig.loadFromDefault();
+      break;
+    case "kubernetes":
+      kubeConfig.loadFromCluster();
+      break;
+    default:
+      assertExhaustive(RUNTIME_ENV);
+  }
+
+  return kubeConfig;
+}
+
+export { k8s };

apps/supervisor/src/env.ts

@@ -0,0 +1,30 @@
+import { randomUUID } from "crypto";
+import { env as stdEnv } from "std-env";
+import { z } from "zod";
+import { getDockerHostDomain } from "./util.js";
+
+const Env = z.object({
+  // This will come from `status.hostIP` in k8s
+  WORKER_HOST_IP: z.string().default(getDockerHostDomain()),
+  TRIGGER_API_URL: z.string().url(),
+  TRIGGER_WORKER_TOKEN: z.string(),
+  // This will come from `spec.nodeName` in k8s
+  TRIGGER_WORKER_INSTANCE_NAME: z.string().default(randomUUID()),
+  MANAGED_WORKER_SECRET: z.string(),
+  TRIGGER_WORKLOAD_API_PORT: z.coerce.number().default(8020),
+  TRIGGER_WORKLOAD_API_PORT_EXTERNAL: z.coerce.number().default(8020),
+  TRIGGER_WARM_START_URL: z.string().optional(),
+  TRIGGER_CHECKPOINT_URL: z.string().optional(),
+  TRIGGER_DEQUEUE_INTERVAL_MS: z.coerce.number().int().default(1000),
+
+  // Used by the workload manager, e.g docker/k8s
+  DOCKER_NETWORK: z.string().default("host"),
+  OTEL_EXPORTER_OTLP_ENDPOINT: z.string().url(),
+  ENFORCE_MACHINE_PRESETS: z.coerce.boolean().default(false),
+
+  // Used by the resource monitor
+  OVERRIDE_CPU_TOTAL: z.coerce.number().optional(),
+  OVERRIDE_MEMORY_TOTAL_GB: z.coerce.number().optional(),
+});
+
+export const env = Env.parse(stdEnv);