Added DLQ tests

Author: matt-aitken

internal-packages/redis-worker/src/worker.test.ts

@@ -1,10 +1,9 @@
 import { redisTest } from "@internal/testcontainers";
-import { describe, it } from "node:test";
+import { Logger } from "@trigger.dev/core/logger";
+import { describe } from "node:test";
 import { expect } from "vitest";
 import { z } from "zod";
 import { Worker } from "./worker.js";
-import { Logger } from "@trigger.dev/core/logger";
-import { SimpleQueue } from "./queue.js";
 
 describe("Worker", () => {
   redisTest("Process items that don't throw", { timeout: 30_000 }, async ({ redisContainer }) => {
@@ -35,26 +34,27 @@ describe("Worker", () => {
       },
       logger: new Logger("test", "log"),
     });
+    try {
+      // Enqueue 10 items
+      for (let i = 0; i < 10; i++) {
+        await worker.enqueue({
+          id: `item-${i}`,
+          job: "testJob",
+          payload: { value: i },
+          visibilityTimeoutMs: 5000,
+        });
+      }
 
-    // Enqueue 10 items
-    for (let i = 0; i < 10; i++) {
-      await worker.enqueue({
-        id: `item-${i}`,
-        job: "testJob",
-        payload: { value: i },
-        visibilityTimeoutMs: 5000,
-      });
-    }
-
-    worker.start();
-
-    // Wait for items to be processed
-    await new Promise((resolve) => setTimeout(resolve, 600));
+      worker.start();
 
-    worker.stop();
+      // Wait for items to be processed
+      await new Promise((resolve) => setTimeout(resolve, 600));
 
-    expect(processedItems.length).toBe(10);
-    expect(new Set(processedItems).size).toBe(10); // Ensure all items were processed uniquely
+      expect(processedItems.length).toBe(10);
+      expect(new Set(processedItems).size).toBe(10); // Ensure all items were processed uniquely
+    } finally {
+      worker.stop();
+    }
   });
 
   redisTest(
@@ -97,28 +97,103 @@ describe("Worker", () => {
         logger: new Logger("test", "error"),
       });
 
-      // Enqueue 10 items
-      for (let i = 0; i < 10; i++) {
+      try {
+        // Enqueue 10 items
+        for (let i = 0; i < 10; i++) {
+          await worker.enqueue({
+            id: `item-${i}`,
+            job: "testJob",
+            payload: { value: i },
+            visibilityTimeoutMs: 5000,
+          });
+        }
+
+        worker.start();
+
+        // Wait for items to be processed
+        await new Promise((resolve) => setTimeout(resolve, 500));
+
+        expect(processedItems.length).toBe(10);
+        expect(new Set(processedItems).size).toBe(10); // Ensure all items were processed uniquely
+      } finally {
+        worker.stop();
+      }
+    }
+  );
+
+  redisTest(
+    "Process an item that permanently fails and ends up in DLQ",
+    { timeout: 30_000 },
+    async ({ redisContainer }) => {
+      const processedItems: number[] = [];
+      const failedItemId = "permanent-fail-item";
+
+      const worker = new Worker({
+        name: "test-worker",
+        redisOptions: {
+          host: redisContainer.getHost(),
+          port: redisContainer.getPort(),
+          password: redisContainer.getPassword(),
+        },
+        catalog: {
+          testJob: {
+            schema: z.object({ value: z.number() }),
+            visibilityTimeoutMs: 1000,
+            retry: { maxAttempts: 3, minTimeoutInMs: 10, maxTimeoutInMs: 50 },
+          },
+        },
+        jobs: {
+          testJob: async ({ id, payload }) => {
+            if (id === failedItemId) {
+              throw new Error("Permanent failure");
+            }
+            processedItems.push(payload.value);
+          },
+        },
+        concurrency: {
+          workers: 1,
+          tasksPerWorker: 1,
+        },
+        pollIntervalMs: 50,
+        logger: new Logger("test", "error"),
+      });
+
+      try {
+        // Enqueue the item that will permanently fail
         await worker.enqueue({
-          id: `item-${i}`,
+          id: failedItemId,
           job: "testJob",
-          payload: { value: i },
-          visibilityTimeoutMs: 5000,
+          payload: { value: 999 },
         });
-      }
 
-      worker.start();
+        // Enqueue a normal item
+        await worker.enqueue({
+          id: "normal-item",
+          job: "testJob",
+          payload: { value: 1 },
+        });
 
-      // Wait for items to be processed
-      await new Promise((resolve) => setTimeout(resolve, 500));
+        worker.start();
 
-      worker.stop();
+        // Wait for items to be processed and retried
+        await new Promise((resolve) => setTimeout(resolve, 1000));
 
-      expect(processedItems.length).toBe(10);
-      expect(new Set(processedItems).size).toBe(10); // Ensure all items were processed uniquely
+        // Check that the normal item was processed
+        expect(processedItems).toEqual([1]);
+
+        // Check that the failed item is in the DLQ
+        const dlqSize = await worker.queue.sizeOfDeadLetterQueue();
+        expect(dlqSize).toBe(1);
+      } finally {
+        worker.stop();
+      }
     }
   );
 
   //todo test that throwing an error doesn't screw up the other items
   //todo process more items when finished
+
+  //todo add a Dead Letter Queue when items are failed, with the error
+  //todo add a function on the worker to redrive them
+  //todo add an API endpoint to redrive with an ID
 });

internal-packages/redis-worker/src/worker.ts

@@ -42,7 +42,7 @@ type WorkerOptions<TCatalog extends WorkerCatalog> = {
 };
 
 class Worker<TCatalog extends WorkerCatalog> {
-  private queue: SimpleQueue<QueueCatalogFromWorkerCatalog<TCatalog>>;
+  queue: SimpleQueue<QueueCatalogFromWorkerCatalog<TCatalog>>;
   private jobs: WorkerOptions<TCatalog>["jobs"];
   private logger: Logger;
   private workers: NodeWorker[] = [];
@@ -157,15 +157,19 @@ class Worker<TCatalog extends WorkerCatalog> {
 
           try {
             await handler({ id, payload: item, visibilityTimeoutMs, attempt });
+
+            //succeeded, acking the item
             await this.queue.ack(id);
           } catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
             this.logger.error(`Error processing item, it threw an error:`, {
               name: this.options.name,
               id,
               job,
               item,
               visibilityTimeoutMs,
               error,
+              errorMessage,
             });
             // Requeue the failed item with a delay
             try {
@@ -174,15 +178,20 @@ class Worker<TCatalog extends WorkerCatalog> {
               const retryDelay = calculateNextRetryDelay(catalogItem.retry, attempt);
 
               if (!retryDelay) {
-                this.logger.error(`Failed item ${id} has reached max attempts, acking.`, {
-                  name: this.options.name,
-                  id,
-                  job,
-                  item,
-                  visibilityTimeoutMs,
-                  attempt,
-                });
-                await this.queue.ack(id);
+                this.logger.error(
+                  `Failed item ${id} has reached max attempts, moving to the DLQ.`,
+                  {
+                    name: this.options.name,
+                    id,
+                    job,
+                    item,
+                    visibilityTimeoutMs,
+                    attempt,
+                    errorMessage,
+                  }
+                );
+
+                await this.queue.moveToDeadLetterQueue(id, errorMessage);
                 return;
               }