Skip to content

Commit 2496917

Browse files
ericallamericallam
authored
v2: Graphile auto-cleanup and auto-endpoint disabling (#1103)
* Auto-cleanup failed graphile jobs instead of keeping them around * Disable endpoint after a period of sequential indexing failures * Remove log
1 parent dc53f0f commit 2496917

File tree

8 files changed

+144
-40
lines changed

8 files changed

+144
-40
lines changed

‎apps/webapp/app/env.server.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -158,6 +158,8 @@ const EnvironmentSchema = z.object({
158158

159159
ORG_SLACK_INTEGRATION_CLIENT_ID: z.string().optional(),
160160
ORG_SLACK_INTEGRATION_CLIENT_SECRET: z.string().optional(),
161+
162+
MAX_SEQUENTIAL_INDEX_FAILURE_COUNT: z.coerce.number().default(96),
161163
});
162164

163165
export type Environment = z.infer<typeof EnvironmentSchema>;

‎apps/webapp/app/platform/zodWorker.server.ts

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -577,6 +577,17 @@ export class ZodWorker<TMessageCatalog extends MessageCatalogSchema> {
577577
span.recordException(new Error(String(error)));
578578
}
579579

580+
if (job.attempts >= job.max_attempts) {
581+
logger.error("Job failed after max attempts", {
582+
job,
583+
attempts: job.attempts,
584+
max_attempts: job.max_attempts,
585+
error: error instanceof Error ? error.message : error,
586+
});
587+
588+
return;
589+
}
590+
580591
throw error;
581592
} finally {
582593
span.end();

‎apps/webapp/app/routes/api.v1.endpoints.$environmentId.$endpointSlug.index.$indexHookIdentifier.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ export class TriggerEndpointIndexHookService {
106106
},
107107
});
108108

109-
if (!endpoint) {
109+
if (!endpoint || !endpoint.url) {
110110
throw new Error("Endpoint not found");
111111
}
112112

‎apps/webapp/app/services/apiRateLimit.server.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -191,6 +191,7 @@ export const apiRateLimiter = authorizationRateLimitMiddleware({
191191
pathMatchers: [/^\/api/],
192192
// Allow /api/v1/tasks/:id/callback/:secret
193193
pathWhiteList: [
194+
"/api/internal/stripe_webhooks",
194195
"/api/v1/authorization-code",
195196
"/api/v1/token",
196197
/^\/api\/v1\/tasks\/[^\/]+\/callback\/[^\/]+$/, // /api/v1/tasks/$id/callback/$secret

‎apps/webapp/app/services/endpoints/performEndpointIndexService.ts

Lines changed: 124 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,9 @@ import { IndexEndpointStats } from "@trigger.dev/core";
1414
import { RegisterHttpEndpointService } from "../triggers/registerHttpEndpoint.server";
1515
import { RegisterWebhookService } from "../triggers/registerWebhook.server";
1616
import { EndpointIndex } from "@trigger.dev/database";
17+
import { env } from "~/env.server";
18+
19+
const MAX_SEQUENTIAL_FAILURE_COUNT = env.MAX_SEQUENTIAL_INDEX_FAILURE_COUNT;
1720

1821
export class PerformEndpointIndexService {
1922
#prismaClient: PrismaClient;
@@ -56,9 +59,16 @@ export class PerformEndpointIndexService {
5659

5760
if (!endpointIndex.endpoint.url) {
5861
logger.debug("Endpoint URL is not set", endpointIndex);
59-
return updateEndpointIndexWithError(this.#prismaClient, id, {
60-
message: "Endpoint URL is not set",
61-
});
62+
63+
return updateEndpointIndexWithError(
64+
this.#prismaClient,
65+
id,
66+
endpointIndex.endpoint.id,
67+
{
68+
message: "Endpoint URL is not set",
69+
},
70+
false
71+
);
6272
}
6373

6474
// Make a request to the endpoint to fetch a list of jobs
@@ -69,9 +79,15 @@ export class PerformEndpointIndexService {
6979
const { response, parser, headerParser, errorParser } = await client.indexEndpoint();
7080

7181
if (!response) {
72-
return updateEndpointIndexWithError(this.#prismaClient, id, {
73-
message: `Could not connect to endpoint ${endpointIndex.endpoint.url}`,
74-
});
82+
return updateEndpointIndexWithError(
83+
this.#prismaClient,
84+
id,
85+
endpointIndex.endpoint.id,
86+
{
87+
message: `Could not connect to endpoint ${endpointIndex.endpoint.url}`,
88+
},
89+
endpointIndex.endpoint.environment.type !== "DEVELOPMENT"
90+
);
7591
}
7692

7793
if (isRedirect(response.status)) {
@@ -83,15 +99,27 @@ export class PerformEndpointIndexService {
8399
const location = response.headers.get("location");
84100

85101
if (!location) {
86-
return updateEndpointIndexWithError(this.#prismaClient, id, {
87-
message: `Endpoint ${endpointIndex.endpoint.url} is redirecting but no location header is present`,
88-
});
102+
return updateEndpointIndexWithError(
103+
this.#prismaClient,
104+
id,
105+
endpointIndex.endpoint.id,
106+
{
107+
message: `Endpoint ${endpointIndex.endpoint.url} is redirecting but no location header is present`,
108+
},
109+
endpointIndex.endpoint.environment.type !== "DEVELOPMENT"
110+
);
89111
}
90112

91113
if (redirectCount > 5) {
92-
return updateEndpointIndexWithError(this.#prismaClient, id, {
93-
message: `Endpoint ${endpointIndex.endpoint.url} is redirecting too many times`,
94-
});
114+
return updateEndpointIndexWithError(
115+
this.#prismaClient,
116+
id,
117+
endpointIndex.endpoint.id,
118+
{
119+
message: `Endpoint ${endpointIndex.endpoint.url} is redirecting too many times`,
120+
},
121+
endpointIndex.endpoint.environment.type !== "DEVELOPMENT"
122+
);
95123
}
96124

97125
await this.#prismaClient.endpoint.update({
@@ -111,20 +139,38 @@ export class PerformEndpointIndexService {
111139
const body = await safeBodyFromResponse(response, errorParser);
112140

113141
if (body) {
114-
return updateEndpointIndexWithError(this.#prismaClient, id, {
115-
message: body.message,
116-
});
142+
return updateEndpointIndexWithError(
143+
this.#prismaClient,
144+
id,
145+
endpointIndex.endpoint.id,
146+
{
147+
message: body.message,
148+
},
149+
endpointIndex.endpoint.environment.type !== "DEVELOPMENT"
150+
);
117151
}
118152

119-
return updateEndpointIndexWithError(this.#prismaClient, id, {
120-
message: "Trigger API key is invalid",
121-
});
153+
return updateEndpointIndexWithError(
154+
this.#prismaClient,
155+
id,
156+
endpointIndex.endpoint.id,
157+
{
158+
message: "Trigger API key is invalid",
159+
},
160+
endpointIndex.endpoint.environment.type !== "DEVELOPMENT"
161+
);
122162
}
123163

124164
if (!response.ok) {
125-
return updateEndpointIndexWithError(this.#prismaClient, id, {
126-
message: `Could not connect to endpoint ${endpointIndex.endpoint.url}. Status code: ${response.status}`,
127-
});
165+
return updateEndpointIndexWithError(
166+
this.#prismaClient,
167+
id,
168+
endpointIndex.endpoint.id,
169+
{
170+
message: `Could not connect to endpoint ${endpointIndex.endpoint.url}. Status code: ${response.status}`,
171+
},
172+
endpointIndex.endpoint.environment.type !== "DEVELOPMENT"
173+
);
128174
}
129175

130176
const anyBody = await response.json();
@@ -152,21 +198,33 @@ export class PerformEndpointIndexService {
152198
}).message;
153199
}
154200

155-
return updateEndpointIndexWithError(this.#prismaClient, id, {
156-
message: friendlyError,
157-
raw: fromZodError(bodyResult.error).message,
158-
});
201+
return updateEndpointIndexWithError(
202+
this.#prismaClient,
203+
id,
204+
endpointIndex.endpoint.id,
205+
{
206+
message: friendlyError,
207+
raw: fromZodError(bodyResult.error).message,
208+
},
209+
endpointIndex.endpoint.environment.type !== "DEVELOPMENT"
210+
);
159211
}
160212

161213
const headerResult = headerParser.safeParse(Object.fromEntries(response.headers.entries()));
162214
if (!headerResult.success) {
163215
const friendlyError = fromZodError(headerResult.error, {
164216
prefix: "Your headers are invalid",
165217
});
166-
return updateEndpointIndexWithError(this.#prismaClient, id, {
167-
message: friendlyError.message,
168-
raw: headerResult.error.issues,
169-
});
218+
return updateEndpointIndexWithError(
219+
this.#prismaClient,
220+
id,
221+
endpointIndex.endpoint.id,
222+
{
223+
message: friendlyError.message,
224+
raw: headerResult.error.issues,
225+
},
226+
endpointIndex.endpoint.environment.type !== "DEVELOPMENT"
227+
);
170228
}
171229

172230
const { jobs, sources, dynamicTriggers, dynamicSchedules, httpEndpoints, webhooks } =
@@ -407,8 +465,44 @@ export class PerformEndpointIndexService {
407465
async function updateEndpointIndexWithError(
408466
prismaClient: PrismaClient,
409467
id: string,
410-
error: EndpointIndexError
468+
endpointId: string,
469+
error: EndpointIndexError,
470+
checkDisabling = true
411471
) {
472+
// Check here to see if this endpoint has only failed for the last 50 times
473+
// And if so, we disable the endpoint by setting the url to null
474+
if (checkDisabling) {
475+
const recentIndexes = await prismaClient.endpointIndex.findMany({
476+
where: {
477+
endpointId,
478+
id: {
479+
not: id,
480+
},
481+
},
482+
orderBy: {
483+
createdAt: "desc",
484+
},
485+
take: MAX_SEQUENTIAL_FAILURE_COUNT - 1,
486+
select: {
487+
status: true,
488+
},
489+
});
490+
491+
if (
492+
recentIndexes.length === MAX_SEQUENTIAL_FAILURE_COUNT - 1 &&
493+
recentIndexes.every((index) => index.status === "FAILURE")
494+
) {
495+
await prismaClient.endpoint.update({
496+
where: {
497+
id: endpointId,
498+
},
499+
data: {
500+
url: null,
501+
},
502+
});
503+
}
504+
}
505+
412506
return await prismaClient.endpointIndex.update({
413507
where: {
414508
id,

‎apps/webapp/app/services/sources/handleHttpSource.server.ts

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -32,10 +32,11 @@ export class HandleHttpSourceService {
3232
return { status: 200 };
3333
}
3434

35+
if (!triggerSource.endpoint.url) {
36+
return { status: 404 };
37+
}
38+
3539
if (!triggerSource.organization.runsEnabled) {
36-
logger.debug("HandleHttpSourceService: Runs are disabled for this organization", {
37-
organizationId: triggerSource.organization.id,
38-
});
3940
return { status: 404 };
4041
}
4142

‎apps/webapp/app/services/worker.server.ts

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -232,11 +232,6 @@ function getWorkerQueue() {
232232
return new ZodWorker({
233233
name: "workerQueue",
234234
prisma,
235-
cleanup: {
236-
frequencyExpression: "13,27,43 * * * *",
237-
ttl: env.WORKER_CLEANUP_TTL_DAYS * 24 * 60 * 60 * 1000, // X days
238-
maxCount: 1000,
239-
},
240235
runnerOptions: {
241236
connectionString: env.DATABASE_URL,
242237
concurrency: env.WORKER_CONCURRENCY,

‎apps/webapp/app/v3/tracer.server.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@ function getTracer() {
100100
if (env.INTERNAL_OTEL_TRACE_EXPORTER_URL) {
101101
const exporter = new OTLPTraceExporter({
102102
url: env.INTERNAL_OTEL_TRACE_EXPORTER_URL,
103-
timeoutMillis: 1000,
103+
timeoutMillis: 10_000,
104104
headers:
105105
env.INTERNAL_OTEL_TRACE_EXPORTER_AUTH_HEADER_NAME &&
106106
env.INTERNAL_OTEL_TRACE_EXPORTER_AUTH_HEADER_VALUE

0 commit comments

Comments
 (0)