Fix slow job retries by optimizing how errored tasks are deleted

Author: ericallam

apps/webapp/app/services/runs/continueRun.server.ts

@@ -32,6 +32,14 @@ export class ContinueRunService {
           throw new Error("Run is not resumable");
         }
 
+        // Delete any tasks that are errored
+        const erroredTasks = await tx.task.findMany({
+          where: {
+            runId: runId,
+            status: "ERRORED",
+          },
+        });
+
         await tx.jobRun.update({
           where: { id: runId },
           data: {
@@ -45,17 +53,15 @@ export class ContinueRunService {
           },
         });
 
-        // Delete any tasks that are errored
-        await tx.task.deleteMany({
-          where: {
-            runId: runId,
-            status: "ERRORED",
-          },
-        });
+        for (const task of erroredTasks) {
+          await tx.task.delete({
+            where: { id: task.id },
+          });
+        }
 
         await ResumeRunService.enqueue(run, tx);
       },
-      { timeout: 10000 }
+      { timeout: 30_000 }
     );
   }
 }

packages/database/prisma/migrations/20240424210540_change_on_delete_ref_action_on_run_execution/migration.sql

@@ -0,0 +1,5 @@
+-- DropForeignKey
+ALTER TABLE "JobRunExecution" DROP CONSTRAINT "JobRunExecution_resumeTaskId_fkey";
+
+-- AddForeignKey
+ALTER TABLE "JobRunExecution" ADD CONSTRAINT "JobRunExecution_resumeTaskId_fkey" FOREIGN KEY ("resumeTaskId") REFERENCES "Task"("id") ON DELETE SET NULL ON UPDATE CASCADE;

packages/database/prisma/schema.prisma

@@ -991,7 +991,7 @@ model JobRunExecution {
   reason JobRunExecutionReason @default(EXECUTE_JOB)
   status JobRunExecutionStatus @default(PENDING)
 
-  resumeTask   Task?   @relation(fields: [resumeTaskId], references: [id], onDelete: Cascade, onUpdate: Cascade)
+  resumeTask   Task?   @relation(fields: [resumeTaskId], references: [id], onDelete: SetNull, onUpdate: Cascade)
   resumeTaskId String?
 
   graphileJobId String?