Add Vercel AI integration with telemetry for Next.js 15 test app

Author: cursoragent

dev-packages/e2e-tests/test-applications/nextjs-15/AI_INTEGRATION_SUMMARY.md

@@ -0,0 +1,59 @@
+# Vercel AI Integration - Next.js 15 E2E Test Implementation
+
+## Overview
+This document summarizes the implementation of the Vercel AI integration for the Next.js 15 E2E test application.
+
+## Changes Made
+
+### 1. Updated Dependencies (package.json)
+Added the following dependencies:
+- `ai`: ^3.0.0 - Vercel AI SDK
+- `zod`: ^3.22.4 - For tool parameter schemas
+
+### 2. Server Configuration (sentry.server.config.ts)
+Added the Vercel AI integration to the Sentry initialization:
+```typescript
+integrations: [
+  Sentry.vercelAIIntegration(),
+],
+```
+
+### 3. Test Page (app/ai-test/page.tsx)
+Created a new test page that demonstrates various AI SDK features:
+- Basic text generation with automatic telemetry
+- Explicit telemetry configuration
+- Tool calls and execution
+- Disabled telemetry
+
+The page wraps AI operations in a Sentry span for proper tracing.
+
+### 4. Test Suite (tests/ai-test.test.ts)
+Created a Playwright test that verifies:
+- AI spans are created with correct operations (`ai.pipeline.generate_text`, `gen_ai.generate_text`, `gen_ai.execute_tool`)
+- Span attributes match expected values (model info, tokens, prompts, etc.)
+- Input/output recording respects `sendDefaultPii: true` setting
+- Tool calls are properly traced
+- Disabled telemetry prevents span creation
+
+## Expected Behavior
+
+When `sendDefaultPii: true` (as configured in this test app):
+1. AI operations automatically enable telemetry
+2. Input prompts and output responses are recorded in spans
+3. Tool calls include arguments and results
+4. Token usage is tracked
+
+## Running the Tests
+
+Prerequisites:
+1. Build packages: `yarn build:tarball` (from repository root)
+2. Start the test registry (Verdaccio)
+3. Run the test: `yarn test:e2e nextjs-15` or `yarn test:run nextjs-15`
+
+## Instrumentation Notes
+
+The Vercel AI integration uses OpenTelemetry instrumentation to automatically patch the `ai` module methods. The instrumentation:
+- Enables telemetry by default for all AI operations
+- Respects the `sendDefaultPii` client option for recording inputs/outputs
+- Allows per-call telemetry configuration via `experimental_telemetry`
+- Follows a precedence hierarchy: integration options > method options > defaults

dev-packages/e2e-tests/test-applications/nextjs-15/app/ai-test/page.tsx

@@ -0,0 +1,101 @@
+import { generateText } from 'ai';
+import { MockLanguageModelV1 } from 'ai/test';
+import { z } from 'zod';
+import * as Sentry from '@sentry/nextjs';
+
+export const dynamic = 'force-dynamic';
+
+async function runAITest() {
+  // First span - telemetry should be enabled automatically but no input/output recorded when sendDefaultPii: true
+  const result1 = await generateText({
+    model: new MockLanguageModelV1({
+      doGenerate: async () => ({
+        rawCall: { rawPrompt: null, rawSettings: {} },
+        finishReason: 'stop',
+        usage: { promptTokens: 10, completionTokens: 20 },
+        text: 'First span here!',
+      }),
+    }),
+    prompt: 'Where is the first span?',
+  });
+
+  // Second span - explicitly enabled telemetry, should record inputs/outputs
+  const result2 = await generateText({
+    experimental_telemetry: { isEnabled: true },
+    model: new MockLanguageModelV1({
+      doGenerate: async () => ({
+        rawCall: { rawPrompt: null, rawSettings: {} },
+        finishReason: 'stop',
+        usage: { promptTokens: 10, completionTokens: 20 },
+        text: 'Second span here!',
+      }),
+    }),
+    prompt: 'Where is the second span?',
+  });
+
+  // Third span - with tool calls and tool results
+  const result3 = await generateText({
+    model: new MockLanguageModelV1({
+      doGenerate: async () => ({
+        rawCall: { rawPrompt: null, rawSettings: {} },
+        finishReason: 'tool-calls',
+        usage: { promptTokens: 15, completionTokens: 25 },
+        text: 'Tool call completed!',
+        toolCalls: [
+          {
+            toolCallType: 'function',
+            toolCallId: 'call-1',
+            toolName: 'getWeather',
+            args: '{ "location": "San Francisco" }',
+          },
+        ],
+      }),
+    }),
+    tools: {
+      getWeather: {
+        parameters: z.object({ location: z.string() }),
+        execute: async (args) => {
+          return `Weather in ${args.location}: Sunny, 72°F`;
+        },
+      },
+    },
+    prompt: 'What is the weather in San Francisco?',
+  });
+
+  // Fourth span - explicitly disabled telemetry, should not be captured
+  const result4 = await generateText({
+    experimental_telemetry: { isEnabled: false },
+    model: new MockLanguageModelV1({
+      doGenerate: async () => ({
+        rawCall: { rawPrompt: null, rawSettings: {} },
+        finishReason: 'stop',
+        usage: { promptTokens: 10, completionTokens: 20 },
+        text: 'Third span here!',
+      }),
+    }),
+    prompt: 'Where is the third span?',
+  });
+
+  return {
+    result1: result1.text,
+    result2: result2.text,
+    result3: result3.text,
+    result4: result4.text,
+  };
+}
+
+export default async function Page() {
+  const results = await Sentry.startSpan(
+    { op: 'function', name: 'ai-test' },
+    async () => {
+      return await runAITest();
+    }
+  );
+
+  return (
+    <div>
+      <h1>AI Test Results</h1>
+      <pre id="ai-results">{JSON.stringify(results, null, 2)}</pre>
+    </div>
+  );
+}

dev-packages/e2e-tests/test-applications/nextjs-15/package.json

@@ -18,10 +18,12 @@
     "@types/node": "^18.19.1",
     "@types/react": "18.0.26",
     "@types/react-dom": "18.0.9",
+    "ai": "^3.0.0",
     "next": "15.3.0-canary.33",
     "react": "beta",
     "react-dom": "beta",
-    "typescript": "~5.0.0"
+    "typescript": "~5.0.0",
+    "zod": "^3.22.4"
   },
   "devDependencies": {
     "@playwright/test": "~1.50.0",

dev-packages/e2e-tests/test-applications/nextjs-15/sentry.server.config.ts

@@ -10,4 +10,7 @@ Sentry.init({
     // We are doing a lot of events at once in this test
     bufferSize: 1000,
   },
+  integrations: [
+    Sentry.vercelAIIntegration(),
+  ],
 });

dev-packages/e2e-tests/test-applications/nextjs-15/tests/ai-test.test.ts

@@ -0,0 +1,70 @@
+import { expect, test } from '@playwright/test';
+import { waitForTransaction } from '@sentry-internal/test-utils';
+
+test('should create AI spans with correct attributes', async ({ page }) => {
+  const aiTransactionPromise = waitForTransaction('nextjs-15', async transactionEvent => {
+    return transactionEvent?.transaction === 'ai-test';
+  });
+
+  await page.goto('/ai-test');
+
+  const aiTransaction = await aiTransactionPromise;
+
+  expect(aiTransaction).toBeDefined();
+  expect(aiTransaction.contexts?.trace?.op).toBe('function');
+  expect(aiTransaction.transaction).toBe('ai-test');
+
+  const spans = aiTransaction.spans || [];
+
+  // We expect spans for the first 3 AI calls (4th is disabled)
+  // Each generateText call should create 2 spans: one for the pipeline and one for doGenerate
+  // Plus a span for the tool call
+  const aiPipelineSpans = spans.filter(span => span.op === 'ai.pipeline.generate_text');
+  const aiGenerateSpans = spans.filter(span => span.op === 'gen_ai.generate_text');
+  const toolCallSpans = spans.filter(span => span.op === 'gen_ai.execute_tool');
+
+  expect(aiPipelineSpans.length).toBeGreaterThanOrEqual(3);
+  expect(aiGenerateSpans.length).toBeGreaterThanOrEqual(3);
+  expect(toolCallSpans.length).toBeGreaterThanOrEqual(1);
+
+  // First AI call - should have telemetry enabled and record inputs/outputs (sendDefaultPii: true)
+  const firstPipelineSpan = aiPipelineSpans[0];
+  expect(firstPipelineSpan?.data?.['ai.model.id']).toBe('mock-model-id');
+  expect(firstPipelineSpan?.data?.['ai.model.provider']).toBe('mock-provider');
+  expect(firstPipelineSpan?.data?.['ai.prompt']).toContain('Where is the first span?');
+  expect(firstPipelineSpan?.data?.['ai.response.text']).toBe('First span here!');
+  expect(firstPipelineSpan?.data?.['gen_ai.usage.input_tokens']).toBe(10);
+  expect(firstPipelineSpan?.data?.['gen_ai.usage.output_tokens']).toBe(20);
+
+  // Second AI call - explicitly enabled telemetry
+  const secondPipelineSpan = aiPipelineSpans[1];
+  expect(secondPipelineSpan?.data?.['ai.prompt']).toContain('Where is the second span?');
+  expect(secondPipelineSpan?.data?.['ai.response.text']).toContain('Second span here!');
+
+  // Third AI call - with tool calls
+  const thirdPipelineSpan = aiPipelineSpans[2];
+  expect(thirdPipelineSpan?.data?.['ai.response.finishReason']).toBe('tool-calls');
+  expect(thirdPipelineSpan?.data?.['gen_ai.usage.input_tokens']).toBe(15);
+  expect(thirdPipelineSpan?.data?.['gen_ai.usage.output_tokens']).toBe(25);
+
+  // Tool call span
+  const toolSpan = toolCallSpans[0];
+  expect(toolSpan?.data?.['ai.toolCall.name']).toBe('getWeather');
+  expect(toolSpan?.data?.['ai.toolCall.id']).toBe('call-1');
+  expect(toolSpan?.data?.['ai.toolCall.args']).toContain('San Francisco');
+  expect(toolSpan?.data?.['ai.toolCall.result']).toContain('Sunny, 72°F');
+
+  // Verify the fourth call was not captured (telemetry disabled)
+  const promptsInSpans = spans
+    .map(span => span.data?.['ai.prompt'])
+    .filter(Boolean);
+  const hasDisabledPrompt = promptsInSpans.some(prompt => prompt.includes('Where is the third span?'));
+  expect(hasDisabledPrompt).toBe(false);
+
+  // Verify results are displayed on the page
+  const resultsText = await page.locator('#ai-results').textContent();
+  expect(resultsText).toContain('First span here!');
+  expect(resultsText).toContain('Second span here!');
+  expect(resultsText).toContain('Tool call completed!');
+  expect(resultsText).toContain('Third span here!');
+});

packages/node/src/integrations/tracing/vercelai/instrumentation.ts

@@ -109,7 +109,7 @@ export class SentryVercelAiInstrumentation extends InstrumentationBase {
     this._callbacks = [];
 
     function generatePatch(originalMethod: (...args: MethodArgs) => unknown) {
-      return (...args: MethodArgs) => {
+      return function (this: unknown, ...args: MethodArgs) {
         const existingExperimentalTelemetry = args[0].experimental_telemetry || {};
         const isEnabled = existingExperimentalTelemetry.isEnabled;
 
@@ -132,7 +132,6 @@ export class SentryVercelAiInstrumentation extends InstrumentationBase {
           recordOutputs,
         };
 
-        // @ts-expect-error we know that the method exists
         return originalMethod.apply(this, args);
       };
     }