Get the internal telemetry ready for test/prod

Author: ericallam

.env.example

@@ -64,4 +64,12 @@ COORDINATOR_SECRET=coordinator-secret # generate the actual secret with `openssl
 # OBJECT_STORE_BASE_URL="https://{bucket}.{accountId}.r2.cloudflarestorage.com"
 # OBJECT_STORE_ACCESS_KEY_ID=
 # OBJECT_STORE_SECRET_ACCESS_KEY=
-# RUNTIME_WAIT_THRESHOLD_IN_MS=10000
+# RUNTIME_WAIT_THRESHOLD_IN_MS=10000
+
+# These control the server-side internal telemetry
+# INTERNAL_OTEL_TRACE_EXPORTER_URL=<URL to send traces to>
+# INTERNAL_OTEL_TRACE_EXPORTER_AUTH_HEADER_NAME=<Header name for the auth token>
+# INTERNAL_OTEL_TRACE_EXPORTER_AUTH_HEADER_VALUE=<Auth token value>
+# INTERNAL_OTEL_TRACE_LOGGING_ENABLED=1
+# INTERNAL_OTEL_TRACE_SAMPING_RATE=20 # this means 1/20 traces or 5% of traces will be sampled (sampled = recorded)
+# INTERNAL_OTEL_TRACE_INSTRUMENT_PRISMA_ENABLED=0,

apps/webapp/app/env.server.ts

@@ -85,8 +85,6 @@ const EnvironmentSchema = z.object({
 
   //v3
   V3_ENABLED: z.string().default("false"),
-  OTLP_EXPORTER_TRACES_URL: z.string().optional(),
-  LOG_TELEMETRY: z.string().default("true"),
   IMAGE_REGISTRY: z.string().default("docker.io"),
   IMAGE_REPO: z.string().default("task"),
   PROVIDER_SECRET: z.string().default("provider-secret"),
@@ -117,6 +115,15 @@ const EnvironmentSchema = z.object({
   DEV_OTEL_LOG_EXPORT_TIMEOUT_MILLIS: z.string().default("30000"),
   DEV_OTEL_LOG_MAX_QUEUE_SIZE: z.string().default("512"),
   RUNTIME_WAIT_THRESHOLD_IN_MS: z.coerce.number().int().default(30000),
+
+  // Internal OTEL environment variables
+  INTERNAL_OTEL_TRACE_EXPORTER_URL: z.string().optional(),
+  INTERNAL_OTEL_TRACE_EXPORTER_AUTH_HEADER_NAME: z.string().optional(),
+  INTERNAL_OTEL_TRACE_EXPORTER_AUTH_HEADER_VALUE: z.string().optional(),
+  INTERNAL_OTEL_TRACE_LOGGING_ENABLED: z.string().default("1"),
+  // this means 1/20 traces or 5% of traces will be sampled (sampled = recorded)
+  INTERNAL_OTEL_TRACE_SAMPING_RATE: z.string().default("20"),
+  INTERNAL_OTEL_TRACE_INSTRUMENT_PRISMA_ENABLED: z.string().default("0"),
 });
 
 export type Environment = z.infer<typeof EnvironmentSchema>;

apps/webapp/app/v3/tracer.server.ts

@@ -9,7 +9,7 @@ import {
   trace,
 } from "@opentelemetry/api";
 import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
-import { registerInstrumentations } from "@opentelemetry/instrumentation";
+import { InstrumentationOption, registerInstrumentations } from "@opentelemetry/instrumentation";
 import { ExpressInstrumentation } from "@opentelemetry/instrumentation-express";
 import { HttpInstrumentation } from "@opentelemetry/instrumentation-http";
 import { Resource } from "@opentelemetry/resources";
@@ -23,12 +23,13 @@ import {
   TraceIdRatioBasedSampler,
 } from "@opentelemetry/sdk-trace-base";
 import { NodeTracerProvider } from "@opentelemetry/sdk-trace-node";
-import { SemanticResourceAttributes } from "@opentelemetry/semantic-conventions";
+import { SEMRESATTRS_SERVICE_NAME } from "@opentelemetry/semantic-conventions";
 import { PrismaInstrumentation } from "@prisma/instrumentation";
 import { env } from "~/env.server";
 import { AuthenticatedEnvironment } from "~/services/apiAuth.server";
 import { singleton } from "~/utils/singleton";
 import { LoggerSpanExporter } from "./telemetry/loggerExporter.server";
+
 class CustomWebappSampler implements Sampler {
   constructor(private readonly _baseSampler: Sampler) {}
 
@@ -44,10 +45,7 @@ class CustomWebappSampler implements Sampler {
     const parentContext = trace.getSpanContext(context);
 
     // Exclude Prisma spans (adjust this logic as needed for your use case)
-    if (
-      !parentContext &&
-      ((attributes && attributes["model"] && attributes["method"]) || name.includes("prisma"))
-    ) {
+    if (!parentContext && name.includes("prisma")) {
       return { decision: SamplingDecision.NOT_RECORD };
     }
 
@@ -65,29 +63,46 @@ export const tracer = singleton("tracer", getTracer);
 function getTracer() {
   diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.ERROR);
 
+  const samplingRate = 1.0 / Math.max(parseInt(env.INTERNAL_OTEL_TRACE_SAMPING_RATE, 10), 1);
+
   const provider = new NodeTracerProvider({
     forceFlushTimeoutMillis: 500,
     resource: new Resource({
-      [SemanticResourceAttributes.SERVICE_NAME]: "trigger.dev",
+      [SEMRESATTRS_SERVICE_NAME]: "trigger.dev",
     }),
     sampler: new ParentBasedSampler({
-      root: new CustomWebappSampler(
-        new TraceIdRatioBasedSampler(env.APP_ENV === "development" ? 1.0 : 0.05)
-      ), // 5% sampling
+      root: new CustomWebappSampler(new TraceIdRatioBasedSampler(samplingRate)), // 5% sampling
     }), // 5% sampling
   });
 
-  if (env.OTLP_EXPORTER_TRACES_URL) {
+  if (env.INTERNAL_OTEL_TRACE_EXPORTER_URL) {
     const exporter = new OTLPTraceExporter({
-      url: env.OTLP_EXPORTER_TRACES_URL,
+      url: env.INTERNAL_OTEL_TRACE_EXPORTER_URL,
       timeoutMillis: 1000,
+      headers:
+        env.INTERNAL_OTEL_TRACE_EXPORTER_AUTH_HEADER_NAME &&
+        env.INTERNAL_OTEL_TRACE_EXPORTER_AUTH_HEADER_VALUE
+          ? {
+              [env.INTERNAL_OTEL_TRACE_EXPORTER_AUTH_HEADER_NAME]:
+                env.INTERNAL_OTEL_TRACE_EXPORTER_AUTH_HEADER_VALUE,
+            }
+          : undefined,
     });
 
-    provider.addSpanProcessor(new BatchSpanProcessor(exporter));
+    provider.addSpanProcessor(
+      new BatchSpanProcessor(exporter, {
+        maxExportBatchSize: 512,
+        scheduledDelayMillis: 200,
+        exportTimeoutMillis: 30000,
+        maxQueueSize: 2048,
+      })
+    );
 
-    console.log(`⚡ Tracer: OTLP exporter enabled to ${env.OTLP_EXPORTER_TRACES_URL}`);
+    console.log(`🔦 Tracer: OTLP exporter enabled to ${env.INTERNAL_OTEL_TRACE_EXPORTER_URL}`);
   } else {
-    if (env.LOG_TELEMETRY === "true") {
+    if (env.INTERNAL_OTEL_TRACE_LOGGING_ENABLED === "1") {
+      console.log(`🔦 Tracer: Logger exporter enabled`);
+
       const loggerExporter = new LoggerSpanExporter();
 
       provider.addSpanProcessor(new SimpleSpanProcessor(loggerExporter));
@@ -96,13 +111,18 @@ function getTracer() {
 
   provider.register();
 
+  let instrumentations: InstrumentationOption[] = [
+    new HttpInstrumentation(),
+    new ExpressInstrumentation(),
+  ];
+
+  if (env.INTERNAL_OTEL_TRACE_INSTRUMENT_PRISMA_ENABLED === "1") {
+    instrumentations.push(new PrismaInstrumentation());
+  }
+
   registerInstrumentations({
     tracerProvider: provider,
-    instrumentations: [
-      new HttpInstrumentation(),
-      new ExpressInstrumentation(),
-      new PrismaInstrumentation(),
-    ],
+    instrumentations,
   });
 
   return provider.getTracer("trigger.dev", "3.0.0.dp.1");

packages/core-backend/src/logger.ts

@@ -94,20 +94,22 @@ export class Logger {
     ...args: Array<Record<string, unknown> | undefined>
   ) {
     // Get the current context from trace if it exists
-    const currentContext = trace.getSpan(context.active());
-
-    const structuredLog = {
-      ...structureArgs(safeJsonClone(args) as Record<string, unknown>[], this.#filteredKeys),
-      ...this.#additionalFields(),
-      timestamp: new Date(),
-      name: this.#name,
-      message,
-      level,
-      traceId: currentContext?.spanContext().traceId,
-      parentSpanId: currentContext?.spanContext().spanId,
-    };
-
-    loggerFunction(JSON.stringify(structuredLog, this.#jsonReplacer));
+    const currentSpan = trace.getSpan(context.active());
+
+    if (!currentSpan || currentSpan.isRecording()) {
+      const structuredLog = {
+        ...structureArgs(safeJsonClone(args) as Record<string, unknown>[], this.#filteredKeys),
+        ...this.#additionalFields(),
+        timestamp: new Date(),
+        name: this.#name,
+        message,
+        level,
+        traceId: currentSpan?.spanContext().traceId,
+        parentSpanId: currentSpan?.spanContext().spanId,
+      };
+
+      loggerFunction(JSON.stringify(structuredLog, this.#jsonReplacer));
+    }
   }
 }