v3: checkpoint and reliability improvements (#1198)

* only checkpoint retries with delays greater than threshold

* rename checkpoint threshold env var

* log task monitor ignores

* crash runs with unbounded attempts

* fix retry check in shared queue consumer

* add missing stop for env var sync spinner

* prod entry point refactor

* missing awaits

* more verbose prod flush and exit logs

* reduce checkpoint support logs

* heartbeat while checkpointing between retries

* dynamic coordinator config

* measure lazy attempt creation time in prod

* simplify delay threshold

* heartbeat clarifications

* crash run if it doesn't reach checkpointable state

* require dynamic config threshold

* fix retry prep, await previous worker kill

* unify wait mechanics

* fix prod worker without tasks error

* ensure worker is ready to be checkpointed for dependency waits

* improve worker attempt creation logging

* prevent crashes caused by failed socket schema parsing

* fix dynamic imports in v3 catalog

* clarify attempt retry mechanics

* move backoff helper to core-apps

* remove core-apps barrel file

* add backoff execute with callback

* deprecate non-lazy attempt messages

* update socket.io-client to v4.7.5

* fix socket.io types for emits with timeout

* retry all the things

* remove todo

* fix retry restores

* improve index failure logs

* retry incomplete dependency waits

* fix checkpoint in-progress detection

* prevent losing messages during reconnect

* checkpoint when greater or equal to threshold

* improve handling of duration wait edge cases

* add ready for lazy attempt replay

* retry attempt completion

* allow failing runs with unfriendly run id

* fix min max jitter

* cancel checkpoints on run failure

* improve attempt creation errors

* prevent crashing run on failed cleanup

* handle at-least-once execute lazy attempt delivery

* log exit code on prepare for retry

* fix timeout promise

* mark some things

* chaos monkey superpowers

* refactor checkpointer

* set chaos monkey defaults

* less chaos

* fix backoff

* handle uncaught entry point exceptions

* only replay rpcs on true reconnects

* allow resume unless final run status

* add changeset

* small fixes

Author: nicktrn

.changeset/mighty-eggs-grab.md

@@ -0,0 +1,30 @@
+---
+"@trigger.dev/core-apps": patch
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+Tasks should now be much more robust and resilient to reconnects during crucial operations and other failure scenarios.
+
+Task runs now have to signal checkpointable state prior to ALL checkpoints. This ensures flushing always happens.
+
+All important socket.io RPCs will now be retried with backoff. Actions relying on checkpoints will be replayed if we haven't been checkpointed and restored as expected, e.g. after reconnect.
+
+Other changes:
+
+- Fix retry check in shared queue
+- Fix env var sync spinner
+- Heartbeat between retries
+- Fix retry prep
+- Fix prod worker no tasks detection
+- Fail runs above `MAX_TASK_RUN_ATTEMPTS`
+- Additional debug logs in all places
+- Prevent crashes due to failed socket schema parsing
+- Remove core-apps barrel
+- Upgrade socket.io-client to fix an ACK memleak
+- Additional index failure logs
+- Prevent message loss during reconnect
+- Prevent burst of heartbeats on reconnect
+- Prevent crash on failed cleanup
+- Handle at-least-once lazy execute message delivery
+- Handle uncaught entry point exceptions

.env.example

@@ -71,7 +71,7 @@ COORDINATOR_SECRET=coordinator-secret # generate the actual secret with `openssl
 # OBJECT_STORE_BASE_URL="https://{bucket}.{accountId}.r2.cloudflarestorage.com"
 # OBJECT_STORE_ACCESS_KEY_ID=
 # OBJECT_STORE_SECRET_ACCESS_KEY=
-# RUNTIME_WAIT_THRESHOLD_IN_MS=10000
+# CHECKPOINT_THRESHOLD_IN_MS=10000
 
 # These control the server-side internal telemetry
 # INTERNAL_OTEL_TRACE_EXPORTER_URL=<URL to send traces to>

apps/coordinator/package.json

@@ -21,8 +21,7 @@
     "execa": "^8.0.1",
     "nanoid": "^5.0.6",
     "prom-client": "^15.1.0",
-    "socket.io": "4.7.4",
-    "socket.io-client": "4.7.4"
+    "socket.io": "4.7.4"
   },
   "devDependencies": {
     "@types/node": "^18",

apps/coordinator/src/chaosMonkey.ts

@@ -0,0 +1,95 @@
+import type { Execa$ } from "execa";
+import { setTimeout as timeout } from "node:timers/promises";
+
+class ChaosMonkeyError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "ChaosMonkeyError";
+  }
+}
+
+export class ChaosMonkey {
+  private chaosEventRate = 0.2;
+  private delayInSeconds = 45;
+
+  constructor(private enabled = false) {
+    if (this.enabled) {
+      console.log("🍌 Chaos monkey enabled");
+    }
+  }
+
+  static Error = ChaosMonkeyError;
+
+  enable() {
+    this.enabled = true;
+    console.log("🍌 Chaos monkey enabled");
+  }
+
+  disable() {
+    this.enabled = false;
+    console.log("🍌 Chaos monkey disabled");
+  }
+
+  async call({
+    $,
+    throwErrors = true,
+    addDelays = true,
+  }: {
+    $?: Execa$<string>;
+    throwErrors?: boolean;
+    addDelays?: boolean;
+  } = {}) {
+    if (!this.enabled) {
+      return;
+    }
+
+    const random = Math.random();
+
+    if (random > this.chaosEventRate) {
+      // Don't interfere with normal operation
+      return;
+    }
+
+    const chaosEvents: Array<() => Promise<any>> = [];
+
+    if (addDelays) {
+      chaosEvents.push(async () => {
+        console.log("🍌 Chaos monkey: Add delay");
+
+        if ($) {
+          await $`sleep ${this.delayInSeconds}`;
+        } else {
+          await timeout(this.delayInSeconds * 1000);
+        }
+      });
+    }
+
+    if (throwErrors) {
+      chaosEvents.push(async () => {
+        console.log("🍌 Chaos monkey: Throw error");
+
+        if ($) {
+          await $`false`;
+        } else {
+          throw new ChaosMonkey.Error("🍌 Chaos monkey: Throw error");
+        }
+      });
+    }
+
+    if (chaosEvents.length === 0) {
+      console.error("🍌 Chaos monkey: No events selected");
+      return;
+    }
+
+    const randomIndex = Math.floor(Math.random() * chaosEvents.length);
+
+    const chaosEvent = chaosEvents[randomIndex];
+
+    if (!chaosEvent) {
+      console.error("🍌 Chaos monkey: No event found");
+      return;
+    }
+
+    await chaosEvent();
+  }
+}