Skip to content

Commit 3a953c8

Browse files
ericallamericallam
committed
Disable endpoint after a period of sequential indexing failures
1 parent 078b916 commit 3a953c8

File tree

5 files changed

+137
-35
lines changed

5 files changed

+137
-35
lines changed

‎apps/webapp/app/env.server.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -158,6 +158,8 @@ const EnvironmentSchema = z.object({
158158

159159
ORG_SLACK_INTEGRATION_CLIENT_ID: z.string().optional(),
160160
ORG_SLACK_INTEGRATION_CLIENT_SECRET: z.string().optional(),
161+
162+
MAX_SEQUENTIAL_INDEX_FAILURE_COUNT: z.coerce.number().default(96),
161163
});
162164

163165
export type Environment = z.infer<typeof EnvironmentSchema>;

‎apps/webapp/app/routes/api.v1.endpoints.$environmentId.$endpointSlug.index.$indexHookIdentifier.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ export class TriggerEndpointIndexHookService {
106106
},
107107
});
108108

109-
if (!endpoint) {
109+
if (!endpoint || !endpoint.url) {
110110
throw new Error("Endpoint not found");
111111
}
112112

‎apps/webapp/app/services/endpoints/performEndpointIndexService.ts

Lines changed: 129 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,9 @@ import { IndexEndpointStats } from "@trigger.dev/core";
1414
import { RegisterHttpEndpointService } from "../triggers/registerHttpEndpoint.server";
1515
import { RegisterWebhookService } from "../triggers/registerWebhook.server";
1616
import { EndpointIndex } from "@trigger.dev/database";
17+
import { env } from "~/env.server";
18+
19+
const MAX_SEQUENTIAL_FAILURE_COUNT = env.MAX_SEQUENTIAL_INDEX_FAILURE_COUNT;
1720

1821
export class PerformEndpointIndexService {
1922
#prismaClient: PrismaClient;
@@ -56,9 +59,16 @@ export class PerformEndpointIndexService {
5659

5760
if (!endpointIndex.endpoint.url) {
5861
logger.debug("Endpoint URL is not set", endpointIndex);
59-
return updateEndpointIndexWithError(this.#prismaClient, id, {
60-
message: "Endpoint URL is not set",
61-
});
62+
63+
return updateEndpointIndexWithError(
64+
this.#prismaClient,
65+
id,
66+
endpointIndex.endpoint.id,
67+
{
68+
message: "Endpoint URL is not set",
69+
},
70+
false
71+
);
6272
}
6373

6474
// Make a request to the endpoint to fetch a list of jobs
@@ -69,9 +79,15 @@ export class PerformEndpointIndexService {
6979
const { response, parser, headerParser, errorParser } = await client.indexEndpoint();
7080

7181
if (!response) {
72-
return updateEndpointIndexWithError(this.#prismaClient, id, {
73-
message: `Could not connect to endpoint ${endpointIndex.endpoint.url}`,
74-
});
82+
return updateEndpointIndexWithError(
83+
this.#prismaClient,
84+
id,
85+
endpointIndex.endpoint.id,
86+
{
87+
message: `Could not connect to endpoint ${endpointIndex.endpoint.url}`,
88+
},
89+
endpointIndex.endpoint.environment.type !== "DEVELOPMENT"
90+
);
7591
}
7692

7793
if (isRedirect(response.status)) {
@@ -83,15 +99,27 @@ export class PerformEndpointIndexService {
8399
const location = response.headers.get("location");
84100

85101
if (!location) {
86-
return updateEndpointIndexWithError(this.#prismaClient, id, {
87-
message: `Endpoint ${endpointIndex.endpoint.url} is redirecting but no location header is present`,
88-
});
102+
return updateEndpointIndexWithError(
103+
this.#prismaClient,
104+
id,
105+
endpointIndex.endpoint.id,
106+
{
107+
message: `Endpoint ${endpointIndex.endpoint.url} is redirecting but no location header is present`,
108+
},
109+
endpointIndex.endpoint.environment.type !== "DEVELOPMENT"
110+
);
89111
}
90112

91113
if (redirectCount > 5) {
92-
return updateEndpointIndexWithError(this.#prismaClient, id, {
93-
message: `Endpoint ${endpointIndex.endpoint.url} is redirecting too many times`,
94-
});
114+
return updateEndpointIndexWithError(
115+
this.#prismaClient,
116+
id,
117+
endpointIndex.endpoint.id,
118+
{
119+
message: `Endpoint ${endpointIndex.endpoint.url} is redirecting too many times`,
120+
},
121+
endpointIndex.endpoint.environment.type !== "DEVELOPMENT"
122+
);
95123
}
96124

97125
await this.#prismaClient.endpoint.update({
@@ -111,20 +139,38 @@ export class PerformEndpointIndexService {
111139
const body = await safeBodyFromResponse(response, errorParser);
112140

113141
if (body) {
114-
return updateEndpointIndexWithError(this.#prismaClient, id, {
115-
message: body.message,
116-
});
142+
return updateEndpointIndexWithError(
143+
this.#prismaClient,
144+
id,
145+
endpointIndex.endpoint.id,
146+
{
147+
message: body.message,
148+
},
149+
endpointIndex.endpoint.environment.type !== "DEVELOPMENT"
150+
);
117151
}
118152

119-
return updateEndpointIndexWithError(this.#prismaClient, id, {
120-
message: "Trigger API key is invalid",
121-
});
153+
return updateEndpointIndexWithError(
154+
this.#prismaClient,
155+
id,
156+
endpointIndex.endpoint.id,
157+
{
158+
message: "Trigger API key is invalid",
159+
},
160+
endpointIndex.endpoint.environment.type !== "DEVELOPMENT"
161+
);
122162
}
123163

124164
if (!response.ok) {
125-
return updateEndpointIndexWithError(this.#prismaClient, id, {
126-
message: `Could not connect to endpoint ${endpointIndex.endpoint.url}. Status code: ${response.status}`,
127-
});
165+
return updateEndpointIndexWithError(
166+
this.#prismaClient,
167+
id,
168+
endpointIndex.endpoint.id,
169+
{
170+
message: `Could not connect to endpoint ${endpointIndex.endpoint.url}. Status code: ${response.status}`,
171+
},
172+
endpointIndex.endpoint.environment.type !== "DEVELOPMENT"
173+
);
128174
}
129175

130176
const anyBody = await response.json();
@@ -152,21 +198,33 @@ export class PerformEndpointIndexService {
152198
}).message;
153199
}
154200

155-
return updateEndpointIndexWithError(this.#prismaClient, id, {
156-
message: friendlyError,
157-
raw: fromZodError(bodyResult.error).message,
158-
});
201+
return updateEndpointIndexWithError(
202+
this.#prismaClient,
203+
id,
204+
endpointIndex.endpoint.id,
205+
{
206+
message: friendlyError,
207+
raw: fromZodError(bodyResult.error).message,
208+
},
209+
endpointIndex.endpoint.environment.type !== "DEVELOPMENT"
210+
);
159211
}
160212

161213
const headerResult = headerParser.safeParse(Object.fromEntries(response.headers.entries()));
162214
if (!headerResult.success) {
163215
const friendlyError = fromZodError(headerResult.error, {
164216
prefix: "Your headers are invalid",
165217
});
166-
return updateEndpointIndexWithError(this.#prismaClient, id, {
167-
message: friendlyError.message,
168-
raw: headerResult.error.issues,
169-
});
218+
return updateEndpointIndexWithError(
219+
this.#prismaClient,
220+
id,
221+
endpointIndex.endpoint.id,
222+
{
223+
message: friendlyError.message,
224+
raw: headerResult.error.issues,
225+
},
226+
endpointIndex.endpoint.environment.type !== "DEVELOPMENT"
227+
);
170228
}
171229

172230
const { jobs, sources, dynamicTriggers, dynamicSchedules, httpEndpoints, webhooks } =
@@ -407,8 +465,49 @@ export class PerformEndpointIndexService {
407465
async function updateEndpointIndexWithError(
408466
prismaClient: PrismaClient,
409467
id: string,
410-
error: EndpointIndexError
468+
endpointId: string,
469+
error: EndpointIndexError,
470+
checkDisabling = true
411471
) {
472+
// Check here to see if this endpoint has only failed for the last 50 times
473+
// And if so, we disable the endpoint by setting the url to null
474+
if (checkDisabling) {
475+
const recentIndexes = await prismaClient.endpointIndex.findMany({
476+
where: {
477+
endpointId,
478+
id: {
479+
not: id,
480+
},
481+
},
482+
orderBy: {
483+
createdAt: "desc",
484+
},
485+
take: MAX_SEQUENTIAL_FAILURE_COUNT - 1,
486+
select: {
487+
status: true,
488+
},
489+
});
490+
491+
logger.debug("Recent indexes", {
492+
recentIndexes,
493+
endpointId,
494+
});
495+
496+
if (
497+
recentIndexes.length === MAX_SEQUENTIAL_FAILURE_COUNT - 1 &&
498+
recentIndexes.every((index) => index.status === "FAILURE")
499+
) {
500+
await prismaClient.endpoint.update({
501+
where: {
502+
id: endpointId,
503+
},
504+
data: {
505+
url: null,
506+
},
507+
});
508+
}
509+
}
510+
412511
return await prismaClient.endpointIndex.update({
413512
where: {
414513
id,

‎apps/webapp/app/services/sources/handleHttpSource.server.ts

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -32,10 +32,11 @@ export class HandleHttpSourceService {
3232
return { status: 200 };
3333
}
3434

35+
if (!triggerSource.endpoint.url) {
36+
return { status: 404 };
37+
}
38+
3539
if (!triggerSource.organization.runsEnabled) {
36-
logger.debug("HandleHttpSourceService: Runs are disabled for this organization", {
37-
organizationId: triggerSource.organization.id,
38-
});
3940
return { status: 404 };
4041
}
4142

‎apps/webapp/app/v3/tracer.server.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@ function getTracer() {
100100
if (env.INTERNAL_OTEL_TRACE_EXPORTER_URL) {
101101
const exporter = new OTLPTraceExporter({
102102
url: env.INTERNAL_OTEL_TRACE_EXPORTER_URL,
103-
timeoutMillis: 1000,
103+
timeoutMillis: 10_000,
104104
headers:
105105
env.INTERNAL_OTEL_TRACE_EXPORTER_AUTH_HEADER_NAME &&
106106
env.INTERNAL_OTEL_TRACE_EXPORTER_AUTH_HEADER_VALUE

0 commit comments

Comments
 (0)