Skip to content

Commit 4b1bc47

Browse files
Lms24Lms24
authored
fix(ci): Reduce flakiness of Replay integration tests (#6823)
Replace a bunch of `page.waitForTimeout` calls with `page.waitForRequest` waits which should make Replay integration tests more stable. Added a `waitForReplayRequest`, which ensures we're actually waiting for a replay request. Optionally, this helper takes a `segmentId` to wait for a replay request of a specific replay segment. This might be useful for more involved testing scenarios.
1 parent c06756c commit 4b1bc47

File tree

6 files changed

+171
-37
lines changed

6 files changed

+171
-37
lines changed

‎packages/integration-tests/suites/replay/captureReplay/test.ts

Lines changed: 52 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,20 @@
11
import { expect } from '@playwright/test';
22
import { SDK_VERSION } from '@sentry/browser';
3-
import type { Event } from '@sentry/types';
3+
import type { ReplayEvent } from '@sentry/types';
44

55
import { sentryTest } from '../../../utils/fixtures';
6-
import { getFirstSentryEnvelopeRequest } from '../../../utils/helpers';
6+
import { envelopeRequestParser } from '../../../utils/helpers';
7+
import { waitForReplayRequest } from '../../../utils/replayHelpers';
78

8-
sentryTest('captureReplay', async ({ getLocalTestPath, page }) => {
9+
sentryTest('should capture replays', async ({ getLocalTestPath, page }) => {
910
// Replay bundles are es6 only
1011
if (process.env.PW_BUNDLE && process.env.PW_BUNDLE.startsWith('bundle_es5')) {
1112
sentryTest.skip();
1213
}
1314

15+
const reqPromise0 = waitForReplayRequest(page, 0);
16+
const reqPromise1 = waitForReplayRequest(page, 1);
17+
1418
await page.route('https://dsn.ingest.sentry.io/**/*', route => {
1519
return route.fulfill({
1620
status: 200,
@@ -20,22 +24,61 @@ sentryTest('captureReplay', async ({ getLocalTestPath, page }) => {
2024
});
2125

2226
const url = await getLocalTestPath({ testDir: __dirname });
27+
2328
await page.goto(url);
29+
const replayEvent0 = envelopeRequestParser(await reqPromise0) as ReplayEvent;
2430

2531
await page.click('button');
26-
await page.waitForTimeout(300);
27-
28-
const replayEvent = await getFirstSentryEnvelopeRequest<Event>(page, url);
32+
const replayEvent1 = envelopeRequestParser(await reqPromise1) as ReplayEvent;
2933

30-
expect(replayEvent).toBeDefined();
31-
expect(replayEvent).toEqual({
34+
expect(replayEvent0).toBeDefined();
35+
expect(replayEvent0).toEqual({
3236
type: 'replay_event',
3337
timestamp: expect.any(Number),
3438
error_ids: [],
3539
trace_ids: [],
3640
urls: [expect.stringContaining('/dist/index.html')],
3741
replay_id: expect.stringMatching(/\w{32}/),
38-
segment_id: 2,
42+
replay_start_timestamp: expect.any(Number),
43+
segment_id: 0,
44+
replay_type: 'session',
45+
event_id: expect.stringMatching(/\w{32}/),
46+
environment: 'production',
47+
sdk: {
48+
integrations: [
49+
'InboundFilters',
50+
'FunctionToString',
51+
'TryCatch',
52+
'Breadcrumbs',
53+
'GlobalHandlers',
54+
'LinkedErrors',
55+
'Dedupe',
56+
'HttpContext',
57+
'Replay',
58+
],
59+
version: SDK_VERSION,
60+
name: 'sentry.javascript.browser',
61+
},
62+
sdkProcessingMetadata: {},
63+
request: {
64+
url: expect.stringContaining('/dist/index.html'),
65+
headers: {
66+
'User-Agent': expect.stringContaining(''),
67+
},
68+
},
69+
platform: 'javascript',
70+
tags: { sessionSampleRate: 1, errorSampleRate: 0 },
71+
});
72+
73+
expect(replayEvent1).toBeDefined();
74+
expect(replayEvent1).toEqual({
75+
type: 'replay_event',
76+
timestamp: expect.any(Number),
77+
error_ids: [],
78+
trace_ids: [],
79+
urls: [],
80+
replay_id: expect.stringMatching(/\w{32}/),
81+
segment_id: 1,
3982
replay_type: 'session',
4083
event_id: expect.stringMatching(/\w{32}/),
4184
environment: 'production',

‎packages/integration-tests/suites/replay/captureReplayViaBrowser/test.ts

Lines changed: 52 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,21 @@
11
import { expect } from '@playwright/test';
22
import { SDK_VERSION } from '@sentry/browser';
3-
import type { Event } from '@sentry/types';
3+
import type { ReplayEvent } from '@sentry/types';
44

55
import { sentryTest } from '../../../utils/fixtures';
6-
import { getFirstSentryEnvelopeRequest } from '../../../utils/helpers';
6+
import { envelopeRequestParser } from '../../../utils/helpers';
7+
import { waitForReplayRequest } from '../../../utils/replayHelpers';
78

8-
sentryTest('captureReplay', async ({ getLocalTestPath, page }) => {
9+
sentryTest('should capture replays (@sentry/browser export)', async ({ getLocalTestPath, page }) => {
910
// For this test, we skip all bundle tests, as we're only interested in Replay being correctly
1011
// exported from the `@sentry/browser` npm package.
1112
if (process.env.PW_BUNDLE && process.env.PW_BUNDLE.startsWith('bundle_')) {
1213
sentryTest.skip();
1314
}
1415

16+
const reqPromise0 = waitForReplayRequest(page, 0);
17+
const reqPromise1 = waitForReplayRequest(page, 1);
18+
1519
await page.route('https://dsn.ingest.sentry.io/**/*', route => {
1620
return route.fulfill({
1721
status: 200,
@@ -21,22 +25,61 @@ sentryTest('captureReplay', async ({ getLocalTestPath, page }) => {
2125
});
2226

2327
const url = await getLocalTestPath({ testDir: __dirname });
28+
2429
await page.goto(url);
30+
const replayEvent0 = envelopeRequestParser(await reqPromise0) as ReplayEvent;
2531

2632
await page.click('button');
27-
await page.waitForTimeout(300);
28-
29-
const replayEvent = await getFirstSentryEnvelopeRequest<Event>(page, url);
33+
const replayEvent1 = envelopeRequestParser(await reqPromise1) as ReplayEvent;
3034

31-
expect(replayEvent).toBeDefined();
32-
expect(replayEvent).toEqual({
35+
expect(replayEvent0).toBeDefined();
36+
expect(replayEvent0).toEqual({
3337
type: 'replay_event',
3438
timestamp: expect.any(Number),
3539
error_ids: [],
3640
trace_ids: [],
3741
urls: [expect.stringContaining('/dist/index.html')],
3842
replay_id: expect.stringMatching(/\w{32}/),
39-
segment_id: 2,
43+
replay_start_timestamp: expect.any(Number),
44+
segment_id: 0,
45+
replay_type: 'session',
46+
event_id: expect.stringMatching(/\w{32}/),
47+
environment: 'production',
48+
sdk: {
49+
integrations: [
50+
'InboundFilters',
51+
'FunctionToString',
52+
'TryCatch',
53+
'Breadcrumbs',
54+
'GlobalHandlers',
55+
'LinkedErrors',
56+
'Dedupe',
57+
'HttpContext',
58+
'Replay',
59+
],
60+
version: SDK_VERSION,
61+
name: 'sentry.javascript.browser',
62+
},
63+
sdkProcessingMetadata: {},
64+
request: {
65+
url: expect.stringContaining('/dist/index.html'),
66+
headers: {
67+
'User-Agent': expect.stringContaining(''),
68+
},
69+
},
70+
platform: 'javascript',
71+
tags: { sessionSampleRate: 1, errorSampleRate: 0 },
72+
});
73+
74+
expect(replayEvent1).toBeDefined();
75+
expect(replayEvent1).toEqual({
76+
type: 'replay_event',
77+
timestamp: expect.any(Number),
78+
error_ids: [],
79+
trace_ids: [],
80+
urls: [],
81+
replay_id: expect.stringMatching(/\w{32}/),
82+
segment_id: 1,
4083
replay_type: 'session',
4184
event_id: expect.stringMatching(/\w{32}/),
4285
environment: 'production',

‎packages/integration-tests/suites/replay/errorResponse/test.ts

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
import { expect } from '@playwright/test';
22

33
import { sentryTest } from '../../../utils/fixtures';
4-
import { getReplaySnapshot } from '../../../utils/helpers';
4+
import { getReplaySnapshot, REPLAY_DEFAULT_FLUSH_MAX_DELAY, waitForReplayRequest } from '../../../utils/replayHelpers';
55

6-
sentryTest('errorResponse', async ({ getLocalTestPath, page }) => {
6+
sentryTest('should stop recording after receiving an error response', async ({ getLocalTestPath, page }) => {
77
// Currently bundle tests are not supported for replay
88
if (process.env.PW_BUNDLE && process.env.PW_BUNDLE.startsWith('bundle_es5')) {
99
sentryTest.skip();
@@ -22,13 +22,15 @@ sentryTest('errorResponse', async ({ getLocalTestPath, page }) => {
2222
const url = await getLocalTestPath({ testDir: __dirname });
2323
await page.goto(url);
2424

25+
await waitForReplayRequest(page);
2526
await page.click('button');
26-
await page.waitForTimeout(300);
2727

2828
expect(called).toBe(1);
2929

3030
// Should immediately skip retrying and just cancel, no backoff
31-
await page.waitForTimeout(5001);
31+
// This waitForTimeout call should be okay, as we're not checking for any
32+
// further network requests afterwards.
33+
await page.waitForTimeout(REPLAY_DEFAULT_FLUSH_MAX_DELAY + 1);
3234

3335
expect(called).toBe(1);
3436

‎packages/integration-tests/suites/replay/sampling/test.ts

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
import { expect } from '@playwright/test';
22

33
import { sentryTest } from '../../../utils/fixtures';
4-
import { getReplaySnapshot } from '../../../utils/helpers';
4+
import { getReplaySnapshot } from '../../../utils/replayHelpers';
55

6-
sentryTest('sampling', async ({ getLocalTestPath, page }) => {
6+
sentryTest('should not send replays if both sample rates are 0', async ({ getLocalTestPath, page }) => {
77
// Replay bundles are es6 only
88
if (process.env.PW_BUNDLE && process.env.PW_BUNDLE.startsWith('bundle_es5')) {
99
sentryTest.skip();
@@ -24,7 +24,6 @@ sentryTest('sampling', async ({ getLocalTestPath, page }) => {
2424
await page.goto(url);
2525

2626
await page.click('button');
27-
await page.waitForTimeout(200);
2827

2928
const replay = await getReplaySnapshot(page);
3029

‎packages/integration-tests/utils/helpers.ts

Lines changed: 1 addition & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,9 @@
11
import type { Page, Request } from '@playwright/test';
2-
import type { ReplayContainer } from '@sentry/replay/build/npm/types/types';
32
import type { EnvelopeItemType, Event, EventEnvelopeHeaders } from '@sentry/types';
43

54
const envelopeUrlRegex = /\.sentry\.io\/api\/\d+\/envelope\//;
65

7-
const envelopeRequestParser = (request: Request | null): Event => {
6+
export const envelopeRequestParser = (request: Request | null): Event => {
87
// https://develop.sentry.dev/sdk/envelopes/
98
const envelope = request?.postData() || '';
109

@@ -105,16 +104,6 @@ async function getSentryEvents(page: Page, url?: string): Promise<Array<Event>>
105104
return eventsHandle.jsonValue();
106105
}
107106

108-
/**
109-
* This returns the replay container (assuming it exists).
110-
* Note that due to how this works with playwright, this is a POJO copy of replay.
111-
* This means that we cannot access any methods on it, and also not mutate it in any way.
112-
*/
113-
export async function getReplaySnapshot(page: Page): Promise<ReplayContainer> {
114-
const replayIntegration = await page.evaluate<{ _replay: ReplayContainer }>('window.Replay');
115-
return replayIntegration._replay;
116-
}
117-
118107
/**
119108
* Waits until a number of requests matching urlRgx at the given URL arrive.
120109
* If the timout option is configured, this function will abort waiting, even if it hasn't reveived the configured

‎packages/integration-tests/utils/replayHelpers.ts

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
import type { ReplayContainer } from '@sentry/replay/build/npm/types/types';
2+
import type { Event, ReplayEvent } from '@sentry/types';
3+
import type { Page, Request } from 'playwright';
4+
5+
import { envelopeRequestParser } from './helpers';
6+
7+
/**
8+
* Waits for a replay request to be sent by the page and returns it.
9+
*
10+
* Optionally, you can specify a segmentId to wait for a specific replay request, containing
11+
* the segment_id in the replay envelope.
12+
* This is useful for tests where you want to wait on multiple replay requests or check
13+
* segment order.
14+
*
15+
* @param page the playwright page object
16+
* @param segmentId the segment_id of the replay event
17+
* @returns
18+
*/
19+
export function waitForReplayRequest(page: Page, segmentId?: number): Promise<Request> {
20+
return page.waitForRequest(req => {
21+
const postData = req.postData();
22+
if (!postData) {
23+
return false;
24+
}
25+
26+
try {
27+
const event = envelopeRequestParser(req);
28+
29+
if (!isReplayEvent(event)) {
30+
return false;
31+
}
32+
33+
if (segmentId !== undefined) {
34+
return event.segment_id === segmentId;
35+
}
36+
37+
return true;
38+
} catch {
39+
return false;
40+
}
41+
});
42+
}
43+
44+
function isReplayEvent(event: Event): event is ReplayEvent {
45+
return event.type === 'replay_event';
46+
}
47+
48+
/**
49+
* This returns the replay container (assuming it exists).
50+
* Note that due to how this works with playwright, this is a POJO copy of replay.
51+
* This means that we cannot access any methods on it, and also not mutate it in any way.
52+
*/
53+
export async function getReplaySnapshot(page: Page): Promise<ReplayContainer> {
54+
const replayIntegration = await page.evaluate<{ _replay: ReplayContainer }>('window.Replay');
55+
return replayIntegration._replay;
56+
}
57+
58+
export const REPLAY_DEFAULT_FLUSH_MAX_DELAY = 5_000;

0 commit comments

Comments
 (0)