Skip to content

Commit 252db30

Browse files
svenefftingesvenefftinge
authored
[server] sequentially gc workspaces (#18456)
1 parent e99800d commit 252db30

File tree

1 file changed

+78
-37
lines changed

1 file changed

+78
-37
lines changed

‎components/server/src/jobs/workspace-gc.ts

Lines changed: 78 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -49,16 +49,26 @@ export class WorkspaceGarbageCollector implements Job {
4949
return;
5050
}
5151

52-
await Promise.all([
53-
this.softDeleteOldWorkspaces().catch((err) => log.error("workspace-gc: error during soft-deletion", err)),
54-
this.deleteWorkspaceContentAfterRetentionPeriod().catch((err) =>
55-
log.error("workspace-gc: error during content deletion", err),
56-
),
57-
this.purgeWorkspacesAfterPurgeRetentionPeriod().catch((err) =>
58-
log.error("workspace-gc: error during hard deletion of workspaces", err),
59-
),
60-
this.deleteOldPrebuilds().catch((err) => log.error("workspace-gc: error during prebuild deletion", err)),
61-
]);
52+
try {
53+
await this.softDeleteOldWorkspaces();
54+
} catch (error) {
55+
log.error("workspace-gc: error during garbage collection", error);
56+
}
57+
try {
58+
await this.deleteWorkspaceContentAfterRetentionPeriod();
59+
} catch (error) {
60+
log.error("workspace-gc: error during content deletion", error);
61+
}
62+
try {
63+
await this.purgeWorkspacesAfterPurgeRetentionPeriod();
64+
} catch (err) {
65+
log.error("workspace-gc: error during hard deletion of workspaces", err);
66+
}
67+
try {
68+
await this.deleteOldPrebuilds();
69+
} catch (err) {
70+
log.error("workspace-gc: error during prebuild deletion", err);
71+
}
6272
}
6373

6474
/**
@@ -80,16 +90,21 @@ export class WorkspaceGarbageCollector implements Job {
8090
this.config.workspaceGarbageCollection.chunkLimit,
8191
);
8292
const afterSelect = new Date();
83-
const deletes = await Promise.all(
84-
workspaces.map((ws) => this.workspaceService.deleteWorkspace(SYSTEM_USER, ws.id, "gc")),
85-
);
93+
log.info(`workspace-gc: about to soft-delete ${workspaces.length} workspaces`);
94+
for (const ws of workspaces) {
95+
try {
96+
await this.workspaceService.deleteWorkspace(SYSTEM_USER, ws.id, "gc");
97+
} catch (err) {
98+
log.error({ workspaceId: ws.id }, "workspace-gc: error during workspace soft-deletion", err);
99+
}
100+
}
86101
const afterDelete = new Date();
87102

88-
log.info(`workspace-gc: successfully soft-deleted ${deletes.length} workspaces`, {
103+
log.info(`workspace-gc: successfully soft-deleted ${workspaces.length} workspaces`, {
89104
selectionTimeMs: afterSelect.getTime() - now.getTime(),
90105
deletionTimeMs: afterDelete.getTime() - afterSelect.getTime(),
91106
});
92-
span.addTags({ nrOfCollectedWorkspaces: deletes.length });
107+
span.addTags({ nrOfCollectedWorkspaces: workspaces.length });
93108
} catch (err) {
94109
TraceContext.setError({ span }, err);
95110
throw err;
@@ -101,16 +116,29 @@ export class WorkspaceGarbageCollector implements Job {
101116
private async deleteWorkspaceContentAfterRetentionPeriod() {
102117
const span = opentracing.globalTracer().startSpan("deleteWorkspaceContentAfterRetentionPeriod");
103118
try {
119+
const now = new Date();
104120
const workspaces = await this.workspaceDB
105121
.trace({ span })
106122
.findWorkspacesForContentDeletion(
107123
this.config.workspaceGarbageCollection.contentRetentionPeriodDays,
108124
this.config.workspaceGarbageCollection.contentChunkLimit,
109125
);
110-
const deletes = await Promise.all(workspaces.map((ws) => this.garbageCollectWorkspace({ span }, ws)));
126+
const afterSelect = new Date();
127+
log.info(`workspace-gc: about to delete the content of ${workspaces.length} workspaces`);
128+
for (const ws of workspaces) {
129+
try {
130+
await this.garbageCollectWorkspace({ span }, ws);
131+
} catch (err) {
132+
log.error({ workspaceId: ws.id }, "workspace-gc: error during workspace content deletion", err);
133+
}
134+
}
135+
const afterDelete = new Date();
111136

112-
log.info(`workspace-gc: successfully deleted the content of ${deletes.length} workspaces`);
113-
span.addTags({ nrOfCollectedWorkspaces: deletes.length });
137+
log.info(`workspace-gc: successfully deleted the content of ${workspaces.length} workspaces`, {
138+
selectionTimeMs: afterSelect.getTime() - now.getTime(),
139+
deletionTimeMs: afterDelete.getTime() - afterSelect.getTime(),
140+
});
141+
span.addTags({ nrOfCollectedWorkspaces: workspaces.length });
114142
} catch (err) {
115143
TraceContext.setError({ span }, err);
116144
throw err;
@@ -133,22 +161,22 @@ export class WorkspaceGarbageCollector implements Job {
133161
this.config.workspaceGarbageCollection.purgeChunkLimit,
134162
now,
135163
);
136-
const deletes = await Promise.all(
137-
workspaces.map((ws) =>
138-
this.workspaceService
139-
.hardDeleteWorkspace(SYSTEM_USER, ws.id)
140-
.catch((err) =>
141-
log.error(
142-
{ userId: ws.ownerId, workspaceId: ws.id },
143-
"failed to hard-delete workspace",
144-
err,
145-
),
146-
),
147-
),
148-
);
149-
150-
log.info(`workspace-gc: successfully purged ${deletes.length} workspaces`);
151-
span.addTags({ nrOfCollectedWorkspaces: deletes.length });
164+
const afterSelect = new Date();
165+
log.info(`workspace-gc: about to purge ${workspaces.length} workspaces`);
166+
for (const ws of workspaces) {
167+
try {
168+
await this.workspaceService.hardDeleteWorkspace(SYSTEM_USER, ws.id);
169+
} catch (err) {
170+
log.error({ workspaceId: ws.id }, "workspace-gc: failed to purge workspace", err);
171+
}
172+
}
173+
const afterDelete = new Date();
174+
175+
log.info(`workspace-gc: successfully purged ${workspaces.length} workspaces`, {
176+
selectionTimeMs: afterSelect.getTime() - now.getTime(),
177+
deletionTimeMs: afterDelete.getTime() - afterSelect.getTime(),
178+
});
179+
span.addTags({ nrOfCollectedWorkspaces: workspaces.length });
152180
} catch (err) {
153181
TraceContext.setError({ span }, err);
154182
throw err;
@@ -160,16 +188,29 @@ export class WorkspaceGarbageCollector implements Job {
160188
private async deleteOldPrebuilds() {
161189
const span = opentracing.globalTracer().startSpan("deleteOldPrebuilds");
162190
try {
191+
const now = new Date();
163192
const workspaces = await this.workspaceDB
164193
.trace({ span })
165194
.findPrebuiltWorkspacesForGC(
166195
this.config.workspaceGarbageCollection.minAgePrebuildDays,
167196
this.config.workspaceGarbageCollection.chunkLimit,
168197
);
169-
const deletes = await Promise.all(workspaces.map((ws) => this.garbageCollectPrebuild({ span }, ws)));
198+
const afterSelect = new Date();
199+
log.info(`workspace-gc: about to delete ${workspaces.length} prebuilds`);
200+
for (const ws of workspaces) {
201+
try {
202+
await this.garbageCollectPrebuild({ span }, ws);
203+
} catch (err) {
204+
log.error({ workspaceId: ws.id }, "workspace-gc: failed to delete prebuild", err);
205+
}
206+
}
207+
const afterDelete = new Date();
170208

171-
log.info(`workspace-gc: successfully deleted ${deletes.length} prebuilds`);
172-
span.addTags({ nrOfCollectedPrebuilds: deletes.length });
209+
log.info(`workspace-gc: successfully deleted ${workspaces.length} prebuilds`, {
210+
selectionTimeMs: afterSelect.getTime() - now.getTime(),
211+
deletionTimeMs: afterDelete.getTime() - afterSelect.getTime(),
212+
});
213+
span.addTags({ nrOfCollectedPrebuilds: workspaces.length });
173214
} catch (err) {
174215
TraceContext.setError({ span }, err);
175216
throw err;

0 commit comments

Comments
 (0)