fix(ci): Reduce flakiness of Replay integration tests

packages/integration-tests/suites/replay/captureReplay/test.ts

@@ -1,16 +1,20 @@
 import { expect } from '@playwright/test';
 import { SDK_VERSION } from '@sentry/browser';
-import type { Event } from '@sentry/types';
+import type { ReplayEvent } from '@sentry/types';
 
 import { sentryTest } from '../../../utils/fixtures';
-import { getFirstSentryEnvelopeRequest } from '../../../utils/helpers';
+import { envelopeRequestParser } from '../../../utils/helpers';
+import { waitForReplayRequest } from '../../../utils/replayHelpers';
 
-sentryTest('captureReplay', async ({ getLocalTestPath, page }) => {
+sentryTest('should capture replays', async ({ getLocalTestPath, page }) => {
   // Replay bundles are es6 only
   if (process.env.PW_BUNDLE && process.env.PW_BUNDLE.startsWith('bundle_es5')) {
     sentryTest.skip();
   }
 
+  const reqPromise0 = waitForReplayRequest(page, 0);
+  const reqPromise1 = waitForReplayRequest(page, 1);
+
   await page.route('https://dsn.ingest.sentry.io/**/*', route => {
     return route.fulfill({
       status: 200,
@@ -20,22 +24,61 @@ sentryTest('captureReplay', async ({ getLocalTestPath, page }) => {
   });
 
   const url = await getLocalTestPath({ testDir: __dirname });
+
   await page.goto(url);
+  const replayEvent0 = envelopeRequestParser(await reqPromise0) as ReplayEvent;
 
   await page.click('button');
-  await page.waitForTimeout(300);
-
-  const replayEvent = await getFirstSentryEnvelopeRequest<Event>(page, url);
+  const replayEvent1 = envelopeRequestParser(await reqPromise1) as ReplayEvent;
 
-  expect(replayEvent).toBeDefined();
-  expect(replayEvent).toEqual({
+  expect(replayEvent0).toBeDefined();
+  expect(replayEvent0).toEqual({
     type: 'replay_event',
     timestamp: expect.any(Number),
     error_ids: [],
     trace_ids: [],
     urls: [expect.stringContaining('/dist/index.html')],
     replay_id: expect.stringMatching(/\w{32}/),
-    segment_id: 2,
+    replay_start_timestamp: expect.any(Number),
+    segment_id: 0,
+    replay_type: 'session',
+    event_id: expect.stringMatching(/\w{32}/),
+    environment: 'production',
+    sdk: {
+      integrations: [
+        'InboundFilters',
+        'FunctionToString',
+        'TryCatch',
+        'Breadcrumbs',
+        'GlobalHandlers',
+        'LinkedErrors',
+        'Dedupe',
+        'HttpContext',
+        'Replay',
+      ],
+      version: SDK_VERSION,
+      name: 'sentry.javascript.browser',
+    },
+    sdkProcessingMetadata: {},
+    request: {
+      url: expect.stringContaining('/dist/index.html'),
+      headers: {
+        'User-Agent': expect.stringContaining(''),
+      },
+    },
+    platform: 'javascript',
+    tags: { sessionSampleRate: 1, errorSampleRate: 0 },
+  });
+
+  expect(replayEvent1).toBeDefined();
+  expect(replayEvent1).toEqual({
+    type: 'replay_event',
+    timestamp: expect.any(Number),
+    error_ids: [],
+    trace_ids: [],
+    urls: [],
+    replay_id: expect.stringMatching(/\w{32}/),
+    segment_id: 1,
     replay_type: 'session',
     event_id: expect.stringMatching(/\w{32}/),
     environment: 'production',

packages/integration-tests/suites/replay/captureReplayViaBrowser/test.ts

@@ -1,17 +1,21 @@
 import { expect } from '@playwright/test';
 import { SDK_VERSION } from '@sentry/browser';
-import type { Event } from '@sentry/types';
+import type { ReplayEvent } from '@sentry/types';
 
 import { sentryTest } from '../../../utils/fixtures';
-import { getFirstSentryEnvelopeRequest } from '../../../utils/helpers';
+import { envelopeRequestParser } from '../../../utils/helpers';
+import { waitForReplayRequest } from '../../../utils/replayHelpers';
 
-sentryTest('captureReplay', async ({ getLocalTestPath, page }) => {
+sentryTest('should capture replays (@sentry/browser export)', async ({ getLocalTestPath, page }) => {
   // For this test, we skip all bundle tests, as we're only interested in Replay being correctly
   // exported from the `@sentry/browser` npm package.
   if (process.env.PW_BUNDLE && process.env.PW_BUNDLE.startsWith('bundle_')) {
     sentryTest.skip();
   }
 
+  const reqPromise0 = waitForReplayRequest(page, 0);
+  const reqPromise1 = waitForReplayRequest(page, 1);
+
   await page.route('https://dsn.ingest.sentry.io/**/*', route => {
     return route.fulfill({
       status: 200,
@@ -21,22 +25,61 @@ sentryTest('captureReplay', async ({ getLocalTestPath, page }) => {
   });
 
   const url = await getLocalTestPath({ testDir: __dirname });
+
   await page.goto(url);
+  const replayEvent0 = envelopeRequestParser(await reqPromise0) as ReplayEvent;
 
   await page.click('button');
-  await page.waitForTimeout(300);
-
-  const replayEvent = await getFirstSentryEnvelopeRequest<Event>(page, url);
+  const replayEvent1 = envelopeRequestParser(await reqPromise1) as ReplayEvent;
 
-  expect(replayEvent).toBeDefined();
-  expect(replayEvent).toEqual({
+  expect(replayEvent0).toBeDefined();
+  expect(replayEvent0).toEqual({
     type: 'replay_event',
     timestamp: expect.any(Number),
     error_ids: [],
     trace_ids: [],
     urls: [expect.stringContaining('/dist/index.html')],
     replay_id: expect.stringMatching(/\w{32}/),
-    segment_id: 2,
+    replay_start_timestamp: expect.any(Number),
+    segment_id: 0,
+    replay_type: 'session',
+    event_id: expect.stringMatching(/\w{32}/),
+    environment: 'production',
+    sdk: {
+      integrations: [
+        'InboundFilters',
+        'FunctionToString',
+        'TryCatch',
+        'Breadcrumbs',
+        'GlobalHandlers',
+        'LinkedErrors',
+        'Dedupe',
+        'HttpContext',
+        'Replay',
+      ],
+      version: SDK_VERSION,
+      name: 'sentry.javascript.browser',
+    },
+    sdkProcessingMetadata: {},
+    request: {
+      url: expect.stringContaining('/dist/index.html'),
+      headers: {
+        'User-Agent': expect.stringContaining(''),
+      },
+    },
+    platform: 'javascript',
+    tags: { sessionSampleRate: 1, errorSampleRate: 0 },
+  });
+
+  expect(replayEvent1).toBeDefined();
+  expect(replayEvent1).toEqual({
+    type: 'replay_event',
+    timestamp: expect.any(Number),
+    error_ids: [],
+    trace_ids: [],
+    urls: [],
+    replay_id: expect.stringMatching(/\w{32}/),
+    segment_id: 1,
     replay_type: 'session',
     event_id: expect.stringMatching(/\w{32}/),
     environment: 'production',

packages/integration-tests/suites/replay/errorResponse/test.ts

@@ -1,9 +1,9 @@
 import { expect } from '@playwright/test';
 
 import { sentryTest } from '../../../utils/fixtures';
-import { getReplaySnapshot } from '../../../utils/helpers';
+import { getReplaySnapshot, REPLAY_DEFAULT_FLUSH_MAX_DELAY, waitForReplayRequest } from '../../../utils/replayHelpers';
 
-sentryTest('errorResponse', async ({ getLocalTestPath, page }) => {
+sentryTest('should stop recording after receiving an error response', async ({ getLocalTestPath, page }) => {
   // Currently bundle tests are not supported for replay
   if (process.env.PW_BUNDLE && process.env.PW_BUNDLE.startsWith('bundle_es5')) {
     sentryTest.skip();
@@ -22,13 +22,15 @@ sentryTest('errorResponse', async ({ getLocalTestPath, page }) => {
   const url = await getLocalTestPath({ testDir: __dirname });
   await page.goto(url);
 
+  await waitForReplayRequest(page);
   await page.click('button');
-  await page.waitForTimeout(300);
 
   expect(called).toBe(1);
 
   // Should immediately skip retrying and just cancel, no backoff
-  await page.waitForTimeout(5001);
+  // This waitForTimeout call should be okay, as we're not checking for any
+  // further network requests afterwards.
+  await page.waitForTimeout(REPLAY_DEFAULT_FLUSH_MAX_DELAY + 1);
 
   expect(called).toBe(1);
 

packages/integration-tests/suites/replay/sampling/test.ts

@@ -1,9 +1,9 @@
 import { expect } from '@playwright/test';
 
 import { sentryTest } from '../../../utils/fixtures';
-import { getReplaySnapshot } from '../../../utils/helpers';
+import { getReplaySnapshot } from '../../../utils/replayHelpers';
 
-sentryTest('sampling', async ({ getLocalTestPath, page }) => {
+sentryTest('should not send replays if both sample rates are 0', async ({ getLocalTestPath, page }) => {
   // Replay bundles are es6 only
   if (process.env.PW_BUNDLE && process.env.PW_BUNDLE.startsWith('bundle_es5')) {
     sentryTest.skip();
@@ -24,7 +24,6 @@ sentryTest('sampling', async ({ getLocalTestPath, page }) => {
   await page.goto(url);
 
   await page.click('button');
-  await page.waitForTimeout(200);
 
   const replay = await getReplaySnapshot(page);
 

packages/integration-tests/utils/helpers.ts

@@ -1,10 +1,9 @@
 import type { Page, Request } from '@playwright/test';
-import type { ReplayContainer } from '@sentry/replay/build/npm/types/types';
 import type { EnvelopeItemType, Event, EventEnvelopeHeaders } from '@sentry/types';
 
 const envelopeUrlRegex = /\.sentry\.io\/api\/\d+\/envelope\//;
 
-const envelopeRequestParser = (request: Request | null): Event => {
+export const envelopeRequestParser = (request: Request | null): Event => {
   // https://develop.sentry.dev/sdk/envelopes/
   const envelope = request?.postData() || '';
 
@@ -105,16 +104,6 @@ async function getSentryEvents(page: Page, url?: string): Promise<Array<Event>>
   return eventsHandle.jsonValue();
 }
 
-/**
- * This returns the replay container (assuming it exists).
- * Note that due to how this works with playwright, this is a POJO copy of replay.
- * This means that we cannot access any methods on it, and also not mutate it in any way.
- */
-export async function getReplaySnapshot(page: Page): Promise<ReplayContainer> {
-  const replayIntegration = await page.evaluate<{ _replay: ReplayContainer }>('window.Replay');
-  return replayIntegration._replay;
-}
-
 /**
  * Waits until a number of requests matching urlRgx at the given URL arrive.
  * If the timout option is configured, this function will abort waiting, even if it hasn't reveived the configured

packages/integration-tests/utils/replayHelpers.ts

@@ -0,0 +1,58 @@
+import type { ReplayContainer } from '@sentry/replay/build/npm/types/types';
+import type { Event, ReplayEvent } from '@sentry/types';
+import type { Page, Request } from 'playwright';
+
+import { envelopeRequestParser } from './helpers';
+
+/**
+ * Waits for a replay request to be sent by the page and returns it.
+ *
+ * Optionally, you can specify a segmentId to wait for a specific replay request, containing
+ * the segment_id in the replay envelope.
+ * This is useful for tests where you want to wait on multiple replay requests or check
+ * segment order.
+ *
+ * @param page the playwright page object
+ * @param segmentId the segment_id of the replay event
+ * @returns
+ */
+export function waitForReplayRequest(page: Page, segmentId?: number): Promise<Request> {
+  return page.waitForRequest(req => {
+    const postData = req.postData();
+    if (!postData) {
+      return false;
+    }
+
+    try {
+      const event = envelopeRequestParser(req);
+
+      if (!isReplayEvent(event)) {
+        return false;
+      }
+
+      if (segmentId !== undefined) {
+        return event.segment_id === segmentId;
+      }
+
+      return true;
+    } catch {
+      return false;
+    }
+  });
+}
+
+function isReplayEvent(event: Event): event is ReplayEvent {
+  return event.type === 'replay_event';
+}
+
+/**
+ * This returns the replay container (assuming it exists).
+ * Note that due to how this works with playwright, this is a POJO copy of replay.
+ * This means that we cannot access any methods on it, and also not mutate it in any way.
+ */
+export async function getReplaySnapshot(page: Page): Promise<ReplayContainer> {
+  const replayIntegration = await page.evaluate<{ _replay: ReplayContainer }>('window.Replay');
+  return replayIntegration._replay;
+}
+
+export const REPLAY_DEFAULT_FLUSH_MAX_DELAY = 5_000;