v3: more visibility into flushing worker otel data in prod

Author: ericallam

packages/cli-v3/src/workers/prod/worker-facade.ts

@@ -202,18 +202,54 @@ const zodIpc = new ZodIpcConnection({
     },
     CLEANUP: async ({ flush, kill }, sender) => {
       if (kill) {
-        await Promise.all([prodUsageManager.flush(), tracingSDK.flush()]);
+        await flushAll();
         // Now we need to exit the process
         await sender.send("READY_TO_DISPOSE", undefined);
       } else {
         if (flush) {
-          await Promise.all([prodUsageManager.flush(), tracingSDK.flush()]);
+          await flushAll();
         }
       }
     },
   },
 });
 
+async function flushAll(timeoutInMs: number = 10_000) {
+  const now = performance.now();
+
+  console.log(`Flushing at ${now}`);
+
+  await Promise.all([flushUsage(), flushTracingSDK()]);
+
+  const duration = performance.now() - now;
+
+  console.log(`Flushed in ${duration}ms`);
+}
+
+async function flushUsage() {
+  const now = performance.now();
+
+  console.log(`Flushing usage at ${now}`);
+
+  await prodUsageManager.flush();
+
+  const duration = performance.now() - now;
+
+  console.log(`Flushed usage in ${duration}ms`);
+}
+
+async function flushTracingSDK() {
+  const now = performance.now();
+
+  console.log(`Flushing tracingSDK at ${now}`);
+
+  await tracingSDK.flush();
+
+  const duration = performance.now() - now;
+
+  console.log(`Flushed tracingSDK in ${duration}ms`);
+}
+
 // Ignore SIGTERM, handled by entry point
 process.on("SIGTERM", async () => {});
 

packages/cli-v3/src/workers/prod/worker-setup.ts

@@ -16,7 +16,9 @@ export const tracingSDK = new TracingSDK({
   url: process.env.OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://0.0.0.0:4318",
   instrumentations: setupImportedConfig?.instrumentations ?? [],
   diagLogLevel: (process.env.OTEL_LOG_LEVEL as TracingDiagnosticLogLevel) ?? "none",
-  forceFlushTimeoutMillis: 1_000,
+  forceFlushTimeoutMillis: process.env.OTEL_FORCE_FLUSH_TIMEOUT
+    ? parseInt(process.env.OTEL_FORCE_FLUSH_TIMEOUT, 10)
+    : 1_000,
 });
 
 export const otelTracer: Tracer = tracingSDK.getTracer("trigger-prod-worker", packageJson.version);