Set endpoint URLs to null, instead of deleting them (#878)

* Added Endpoint deletedAt column

* Only show endpoints where they’re not deleted

* Don’t delete Endpoints, set the deletedAt and change their slug name

* Only perform indexing if the endpoint isn’t deleted

* Have a nullable URL for endpoints

* Deal with null URLs throughout the app

* Re-running and retrying behaves properly when there’s no endpoint URL

* Remove console.log

* Better error message when doing a run

Author: matt-aitken

apps/webapp/app/components/run/RunOverview.tsx

@@ -117,6 +117,7 @@ export function RunOverview({ run, trigger, showRerun, paths, currentUser }: Run
             {showRerun && run.isFinished && (
               <RerunPopover
                 runId={run.id}
+                runPath={paths.run}
                 runsPath={paths.runsPath}
                 environmentType={run.environment.type}
                 status={run.basicStatus}
@@ -317,18 +318,20 @@ function BlankTasks({ status }: { status: RunBasicStatus }) {
 
 function RerunPopover({
   runId,
+  runPath,
   runsPath,
   environmentType,
   status,
 }: {
   runId: string;
+  runPath: string;
   runsPath: string;
   environmentType: RuntimeEnvironmentType;
   status: RunBasicStatus;
 }) {
   const lastSubmission = useActionData();
 
-  const [form, { successRedirect }] = useForm({
+  const [form, { successRedirect, failureRedirect }] = useForm({
     id: "rerun",
     // TODO: type this
     lastSubmission: lastSubmission as any,
@@ -347,6 +350,7 @@ function RerunPopover({
       <PopoverContent className="flex min-w-[20rem] max-w-[20rem] flex-col gap-2 p-0" align="end">
         <Form method="post" action={`/resources/runs/${runId}/rerun`} {...form.props}>
           <input {...conform.input(successRedirect, { type: "hidden" })} defaultValue={runsPath} />
+          <input {...conform.input(failureRedirect, { type: "hidden" })} defaultValue={runPath} />
           {environmentType === "PRODUCTION" && (
             <div className="px-4 pt-4">
               <Callout variant="warning">

apps/webapp/app/presenters/EnvironmentsPresenter.server.ts

@@ -38,7 +38,7 @@ export type ClientEndpoint =
       state: "configured";
       id: string;
       slug: string;
-      url: string;
+      url: string | null;
       indexWebhookPath: string;
       latestIndex?: {
         status: EndpointIndexStatus;
@@ -102,6 +102,11 @@ export class EnvironmentsPresenter {
               },
             },
           },
+          where: {
+            url: {
+              not: null,
+            },
+          },
         },
       },
       where: {

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.environments/ConfigureEndpointSheet.tsx

@@ -111,7 +111,7 @@ export function ConfigureEndpointSheet({ slug, endpoint, onClose }: ConfigureEnd
                 <Input
                   className="rounded-r-none"
                   {...conform.input(url, { type: "url" })}
-                  defaultValue={"url" in endpoint ? endpoint.url : ""}
+                  defaultValue={"url" in endpoint ? endpoint.url ?? "" : ""}
                   placeholder="URL for your Trigger API route"
                 />
                 <Button

apps/webapp/app/routes/resources.runs.$runId.rerun.ts

@@ -1,12 +1,18 @@
 import { parse } from "@conform-to/zod";
 import { ActionFunction, json } from "@remix-run/node";
 import { z } from "zod";
-import { redirectBackWithErrorMessage, redirectWithSuccessMessage } from "~/models/message.server";
+import {
+  redirectBackWithErrorMessage,
+  redirectWithErrorMessage,
+  redirectWithSuccessMessage,
+} from "~/models/message.server";
 import { ContinueRunService } from "~/services/runs/continueRun.server";
 import { ReRunService } from "~/services/runs/reRun.server";
+import { rootPath, runPath } from "~/utils/pathBuilder";
 
 export const schema = z.object({
   successRedirect: z.string(),
+  failureRedirect: z.string(),
 });
 
 const ParamSchema = z.object({
@@ -20,7 +26,11 @@ export const action: ActionFunction = async ({ request, params }) => {
   const submission = parse(formData, { schema });
 
   if (!submission.value) {
-    return json(submission);
+    return redirectWithErrorMessage(
+      rootPath(),
+      request,
+      submission.error ? JSON.stringify(submission.error) : "Invalid form"
+    );
   }
 
   try {
@@ -29,7 +39,11 @@ export const action: ActionFunction = async ({ request, params }) => {
       const run = await rerunService.call({ runId });
 
       if (!run) {
-        return redirectBackWithErrorMessage(request, "Unable to retry run");
+        return redirectWithErrorMessage(
+          submission.value.failureRedirect,
+          request,
+          "Unable to retry run"
+        );
       }
 
       return redirectWithSuccessMessage(
@@ -48,6 +62,10 @@ export const action: ActionFunction = async ({ request, params }) => {
       );
     }
   } catch (error: any) {
-    return json({ errors: { body: error.message } }, { status: 400 });
+    return redirectWithErrorMessage(
+      submission.value.failureRedirect,
+      request,
+      error instanceof Error ? error.message : JSON.stringify(error)
+    );
   }
 };

apps/webapp/app/services/endpoints/deleteEndpointService.ts

@@ -9,7 +9,10 @@ export class DeleteEndpointIndexService {
   }
 
   public async call(id: string, userId: string): Promise<void> {
-    await this.#prismaClient.endpoint.delete({
+    await this.#prismaClient.endpoint.update({
+      data: {
+        url: null,
+      },
       where: {
         id,
         organization: {

apps/webapp/app/services/endpoints/performEndpointIndexService.ts

@@ -54,6 +54,13 @@ export class PerformEndpointIndexService {
 
     logger.debug("Performing endpoint index", endpointIndex);
 
+    if (!endpointIndex.endpoint.url) {
+      logger.debug("Endpoint URL is not set", endpointIndex);
+      return updateEndpointIndexWithError(this.#prismaClient, id, {
+        message: "Endpoint URL is not set",
+      });
+    }
+
     // Make a request to the endpoint to fetch a list of jobs
     const client = new EndpointApi(
       endpointIndex.endpoint.environment.apiKey,

apps/webapp/app/services/endpoints/probeEndpoint.server.ts

@@ -29,6 +29,13 @@ export class ProbeEndpointService {
       id,
     });
 
+    if (!endpoint.url) {
+      logger.debug(`Endpoint has no url`, {
+        id,
+      });
+      return;
+    }
+
     const client = new EndpointApi(endpoint.environment.apiKey, endpoint.url);
 
     const { response, durationInMs } = await client.probe(MAX_RUN_CHUNK_EXECUTION_LIMIT);

apps/webapp/app/services/endpoints/recurringEndpointIndex.server.ts

@@ -16,6 +16,9 @@ export class RecurringEndpointIndexService {
 
     const endpoints = await this.#prismaClient.endpoint.findMany({
       where: {
+        url: {
+          not: null,
+        },
         environment: {
           type: {
             in: [RuntimeEnvironmentType.PRODUCTION, RuntimeEnvironmentType.STAGING],

apps/webapp/app/services/httpendpoint/HandleHttpEndpointService.ts

@@ -96,6 +96,14 @@ export class HandleHttpEndpointService {
       );
     }
 
+    if (!httpEndpointEnvironment.endpoint.url) {
+      logger.debug("Endpoint has no url", {
+        httpEndpointId: httpEndpoint.id,
+        environmentId: environment.id,
+      });
+      return json({ error: true, message: "Endpoint has no url" }, { status: 404 });
+    }
+
     const immediateResponseFilter = RequestFilterSchema.nullable().safeParse(
       httpEndpointEnvironment.immediateResponseFilter
     );

apps/webapp/app/services/runs/createRun.server.ts

@@ -37,6 +37,11 @@ export class CreateRunService {
       },
     });
 
+    if (!endpoint.url) {
+      logger.debug("Endpoint has no url", endpoint);
+      return;
+    }
+
     const eventRecord = await this.#prismaClient.eventRecord.findUniqueOrThrow({
       where: {
         id: eventId,

apps/webapp/app/services/runs/deliverRunSubscription.server.ts

@@ -97,6 +97,10 @@ export class DeliverRunSubscriptionService {
           return true;
         }
 
+        if (subscription.run.endpoint.url === null) {
+          return true;
+        }
+
         const client = new EndpointApi(
           subscription.run.environment.apiKey,
           subscription.run.endpoint.url

apps/webapp/app/services/runs/performRunExecutionV3.server.ts

@@ -135,6 +135,12 @@ export class PerformRunExecutionV3Service {
         return;
       }
 
+      if (!run.endpoint.url) {
+        return await this.#failRunExecution(this.#prismaClient, run, {
+          message: `Endpoint has no URL set`,
+        });
+      }
+
       const client = new EndpointApi(run.environment.apiKey, run.endpoint.url);
       const event = eventRecordToApiJson(run.event);
 

apps/webapp/app/services/sources/deliverHttpSourceRequest.server.ts

@@ -43,6 +43,10 @@ export class DeliverHttpSourceRequestService {
       return;
     }
 
+    if (!httpSourceRequest.endpoint.url) {
+      return;
+    }
+
     const secretStore = getSecretStore(httpSourceRequest.source.secretReference.provider);
 
     const secret = await secretStore.getSecret(

apps/webapp/app/services/sources/deliverWebhookRequest.server.ts

@@ -49,6 +49,10 @@ export class DeliverWebhookRequestService {
       return;
     }
 
+    if (!requestDelivery.endpoint.url) {
+      return;
+    }
+
     const { secretReference } = requestDelivery.webhook.httpEndpoint;
 
     const secretStore = getSecretStore(secretReference.provider);

apps/webapp/app/services/triggers/initializeTrigger.server.ts

@@ -35,6 +35,10 @@ export class InitializeTriggerService {
       },
     });
 
+    if (!endpoint.url) {
+      throw new Error("This environment's endpoint doesn't have a URL set");
+    }
+
     const dynamicTrigger = await this.#prismaClient.dynamicTrigger.findUniqueOrThrow({
       where: {
         endpointId_slug_type: {

packages/database/prisma/migrations/20240129140944_endpoint_deleted_at_column/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "Endpoint" ADD COLUMN     "deletedAt" TIMESTAMP(3);

packages/database/prisma/migrations/20240129161848_endpoint_nullable_slug_instead_of_deletedat_column/migration.sql

@@ -0,0 +1,9 @@
+/*
+  Warnings:
+
+  - You are about to drop the column `deletedAt` on the `Endpoint` table. All the data in the column will be lost.
+
+*/
+-- AlterTable
+ALTER TABLE "Endpoint" DROP COLUMN "deletedAt",
+ALTER COLUMN "url" DROP NOT NULL;

packages/database/prisma/schema.prisma

@@ -413,9 +413,9 @@ model Project {
 }
 
 model Endpoint {
-  id   String @id @default(cuid())
+  id   String  @id @default(cuid())
   slug String
-  url  String
+  url  String?
 
   environment   RuntimeEnvironment @relation(fields: [environmentId], references: [id], onDelete: Cascade, onUpdate: Cascade)
   environmentId String