Skip to content

Commit e9d5d1c

Browse files
mydeamydea
committed
test: Fix flaky replay test
1 parent eedd811 commit e9d5d1c

File tree

3 files changed

+142
-116
lines changed

3 files changed

+142
-116
lines changed

‎packages/integration-tests/suites/replay/errors/errorMode/test.ts

Lines changed: 124 additions & 115 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import { expect } from '@playwright/test';
22

33
import { sentryTest } from '../../../../utils/fixtures';
4-
import { envelopeRequestParser } from '../../../../utils/helpers';
4+
import { envelopeRequestParser, waitForErrorRequest } from '../../../../utils/helpers';
55
import {
66
expectedClickBreadcrumb,
77
expectedConsoleBreadcrumb,
@@ -10,124 +10,133 @@ import {
1010
import {
1111
getReplayEvent,
1212
getReplayRecordingContent,
13+
isReplayEvent,
1314
shouldSkipReplayTest,
1415
waitForReplayRequest,
1516
} from '../../../../utils/replayHelpers';
1617

17-
sentryTest(
18-
'[error-mode] should start recording and switch to session mode once an error is thrown',
19-
async ({ getLocalTestPath, page }) => {
20-
if (shouldSkipReplayTest()) {
21-
sentryTest.skip();
22-
}
23-
24-
let callsToSentry = 0;
25-
let errorEventId: string | undefined;
26-
const reqPromise0 = waitForReplayRequest(page, 0);
27-
const reqPromise1 = waitForReplayRequest(page, 1);
28-
const reqPromise2 = waitForReplayRequest(page, 2);
29-
30-
await page.route('https://dsn.ingest.sentry.io/**/*', route => {
31-
const event = envelopeRequestParser(route.request());
32-
// error events have no type field
33-
if (event && !event.type && event.event_id) {
34-
errorEventId = event.event_id;
18+
// TODO: Try to run this many times to identify flakes
19+
for (let i = 0; i < 100; i++) {
20+
sentryTest(
21+
`[error-mode] should start recording and switch to session mode once an error is thrown RUN ${i}`,
22+
async ({ getLocalTestPath, page }) => {
23+
if (shouldSkipReplayTest()) {
24+
sentryTest.skip();
3525
}
36-
callsToSentry++;
3726

38-
return route.fulfill({
39-
status: 200,
40-
contentType: 'application/json',
41-
body: JSON.stringify({ id: 'test-id' }),
27+
let callsToSentry = 0;
28+
let errorEventId: string | undefined;
29+
const reqPromise0 = waitForReplayRequest(page, 0);
30+
const reqPromise1 = waitForReplayRequest(page, 1);
31+
const reqPromise2 = waitForReplayRequest(page, 2);
32+
const reqErrorPromise = waitForErrorRequest(page);
33+
34+
await page.route('https://dsn.ingest.sentry.io/**/*', route => {
35+
const event = envelopeRequestParser(route.request());
36+
// error events have no type field
37+
if (event && !event.type && event.event_id) {
38+
errorEventId = event.event_id;
39+
}
40+
// We only want to count errors & replays here
41+
if (event && (!event.type || isReplayEvent(event))) {
42+
callsToSentry++;
43+
}
44+
45+
return route.fulfill({
46+
status: 200,
47+
contentType: 'application/json',
48+
body: JSON.stringify({ id: 'test-id' }),
49+
});
4250
});
43-
});
44-
45-
const url = await getLocalTestPath({ testDir: __dirname });
46-
47-
await page.goto(url);
48-
await page.click('#go-background');
49-
expect(callsToSentry).toEqual(0);
50-
51-
await page.click('#error');
52-
const req0 = await reqPromise0;
53-
54-
await page.click('#go-background');
55-
const req1 = await reqPromise1;
56-
57-
expect(callsToSentry).toEqual(3); // 1 error, 2 replay events
58-
59-
await page.click('#log');
60-
await page.click('#go-background');
61-
const req2 = await reqPromise2;
62-
63-
const event0 = getReplayEvent(req0);
64-
const content0 = getReplayRecordingContent(req0);
65-
66-
const event1 = getReplayEvent(req1);
67-
const content1 = getReplayRecordingContent(req1);
68-
69-
const event2 = getReplayEvent(req2);
70-
const content2 = getReplayRecordingContent(req2);
71-
72-
expect(event0).toEqual(
73-
getExpectedReplayEvent({
74-
contexts: { replay: { error_sample_rate: 1, session_sample_rate: 0 } },
75-
// @ts-ignore this is fine
76-
error_ids: [errorEventId],
77-
replay_type: 'error',
78-
}),
79-
);
80-
81-
// The first event should have both, full and incremental snapshots,
82-
// as we recorded and kept all events in the buffer
83-
expect(content0.fullSnapshots).toHaveLength(1);
84-
// We don't know how many incremental snapshots we'll have (also browser-dependent),
85-
// but we know that we have at least 5
86-
expect(content0.incrementalSnapshots.length).toBeGreaterThan(5);
87-
// We want to make sure that the event that triggered the error was recorded.
88-
expect(content0.breadcrumbs).toEqual(
89-
expect.arrayContaining([
90-
{
91-
...expectedClickBreadcrumb,
92-
message: 'body > button#error',
93-
},
94-
]),
95-
);
96-
97-
expect(event1).toEqual(
98-
getExpectedReplayEvent({
99-
contexts: { replay: { error_sample_rate: 1, session_sample_rate: 0 } },
100-
// @ts-ignore this is fine
101-
replay_type: 'error', // although we're in session mode, we still send 'error' as replay_type
102-
replay_start_timestamp: undefined,
103-
segment_id: 1,
104-
urls: [],
105-
}),
106-
);
107-
108-
// Also the second snapshot should have a full snapshot, as we switched from error to session
109-
// mode which triggers another checkout
110-
expect(content1.fullSnapshots).toHaveLength(1);
111-
expect(content1.incrementalSnapshots).toHaveLength(0);
112-
113-
// The next event should just be a normal replay event as we're now in session mode and
114-
// we continue recording everything
115-
expect(event2).toEqual(
116-
getExpectedReplayEvent({
117-
contexts: { replay: { error_sample_rate: 1, session_sample_rate: 0 } },
118-
// @ts-ignore this is fine
119-
replay_type: 'error',
120-
replay_start_timestamp: undefined,
121-
segment_id: 2,
122-
urls: [],
123-
}),
124-
);
125-
126-
expect(content2.breadcrumbs).toEqual(
127-
expect.arrayContaining([
128-
{ ...expectedClickBreadcrumb, message: 'body > button#log' },
129-
{ ...expectedConsoleBreadcrumb, level: 'log', message: 'Some message' },
130-
]),
131-
);
132-
},
133-
);
51+
52+
const url = await getLocalTestPath({ testDir: __dirname });
53+
54+
await page.goto(url);
55+
await page.click('#go-background');
56+
expect(callsToSentry).toEqual(0);
57+
58+
await page.click('#error');
59+
const req0 = await reqPromise0;
60+
61+
void page.click('#go-background');
62+
const req1 = await reqPromise1;
63+
await reqErrorPromise;
64+
65+
expect(callsToSentry).toEqual(3); // 1 error, 2 replay events
66+
67+
void page.click('#log');
68+
void page.click('#go-background');
69+
const req2 = await reqPromise2;
70+
71+
const event0 = getReplayEvent(req0);
72+
const content0 = getReplayRecordingContent(req0);
73+
74+
const event1 = getReplayEvent(req1);
75+
const content1 = getReplayRecordingContent(req1);
76+
77+
const event2 = getReplayEvent(req2);
78+
const content2 = getReplayRecordingContent(req2);
79+
80+
expect(event0).toEqual(
81+
getExpectedReplayEvent({
82+
contexts: { replay: { error_sample_rate: 1, session_sample_rate: 0 } },
83+
// @ts-ignore this is fine
84+
error_ids: [errorEventId],
85+
replay_type: 'error',
86+
}),
87+
);
88+
89+
// The first event should have both, full and incremental snapshots,
90+
// as we recorded and kept all events in the buffer
91+
expect(content0.fullSnapshots).toHaveLength(1);
92+
// We don't know how many incremental snapshots we'll have (also browser-dependent),
93+
// but we know that we have at least 5
94+
expect(content0.incrementalSnapshots.length).toBeGreaterThan(5);
95+
// We want to make sure that the event that triggered the error was recorded.
96+
expect(content0.breadcrumbs).toEqual(
97+
expect.arrayContaining([
98+
{
99+
...expectedClickBreadcrumb,
100+
message: 'body > button#error',
101+
},
102+
]),
103+
);
104+
105+
expect(event1).toEqual(
106+
getExpectedReplayEvent({
107+
contexts: { replay: { error_sample_rate: 1, session_sample_rate: 0 } },
108+
// @ts-ignore this is fine
109+
replay_type: 'error', // although we're in session mode, we still send 'error' as replay_type
110+
replay_start_timestamp: undefined,
111+
segment_id: 1,
112+
urls: [],
113+
}),
114+
);
115+
116+
// Also the second snapshot should have a full snapshot, as we switched from error to session
117+
// mode which triggers another checkout
118+
expect(content1.fullSnapshots).toHaveLength(1);
119+
expect(content1.incrementalSnapshots).toHaveLength(0);
120+
121+
// The next event should just be a normal replay event as we're now in session mode and
122+
// we continue recording everything
123+
expect(event2).toEqual(
124+
getExpectedReplayEvent({
125+
contexts: { replay: { error_sample_rate: 1, session_sample_rate: 0 } },
126+
// @ts-ignore this is fine
127+
replay_type: 'error',
128+
replay_start_timestamp: undefined,
129+
segment_id: 2,
130+
urls: [],
131+
}),
132+
);
133+
134+
expect(content2.breadcrumbs).toEqual(
135+
expect.arrayContaining([
136+
{ ...expectedClickBreadcrumb, message: 'body > button#log' },
137+
{ ...expectedConsoleBreadcrumb, level: 'log', message: 'Some message' },
138+
]),
139+
);
140+
},
141+
);
142+
}

‎packages/integration-tests/utils/helpers.ts

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,23 @@ async function getSentryEvents(page: Page, url?: string): Promise<Array<Event>>
108108
return eventsHandle.jsonValue();
109109
}
110110

111+
export function waitForErrorRequest(page: Page): Promise<Request> {
112+
return page.waitForRequest(req => {
113+
const postData = req.postData();
114+
if (!postData) {
115+
return false;
116+
}
117+
118+
try {
119+
const event = envelopeRequestParser(req);
120+
121+
return !event.type;
122+
} catch {
123+
return false;
124+
}
125+
});
126+
}
127+
111128
/**
112129
* Waits until a number of requests matching urlRgx at the given URL arrive.
113130
* If the timout option is configured, this function will abort waiting, even if it hasn't reveived the configured

‎packages/integration-tests/utils/replayHelpers.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ export function waitForReplayRequest(page: Page, segmentId?: number): Promise<Re
6464
});
6565
}
6666

67-
function isReplayEvent(event: Event): event is ReplayEvent {
67+
export function isReplayEvent(event: Event): event is ReplayEvent {
6868
return event.type === 'replay_event';
6969
}
7070

0 commit comments

Comments
 (0)