Skip to content

Commit 34f9cc7

Browse files
mydeamydea
authored
test: Make some replay tests less flaky (#10418)
This changes the replay fetch/xhr tests to not wait for the initial replay, but wait for any replay being captured, and collect snapshots as we go. This way, we may be a bit more resilient for small timing issues - hopefully...
1 parent e168692 commit 34f9cc7

File tree

17 files changed

+295
-205
lines changed

17 files changed

+295
-205
lines changed

‎.github/workflows/build.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -381,7 +381,7 @@ jobs:
381381
name: Browser Unit Tests
382382
needs: [job_get_metadata, job_build]
383383
timeout-minutes: 10
384-
runs-on: ubuntu-latest
384+
runs-on: ubuntu-20.04
385385
steps:
386386
- name: Check out current commit (${{ needs.job_get_metadata.outputs.commit_label }})
387387
uses: actions/checkout@v4

‎.github/workflows/flaky-test-detector.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ concurrency:
2323

2424
jobs:
2525
flaky-detector:
26-
runs-on: ubuntu-20.04
26+
runs-on: ubuntu-20.04-large-js
2727
timeout-minutes: 60
2828
name: 'Check tests for flakiness'
2929
# Also skip if PR is from master -> develop

‎dev-packages/browser-integration-tests/suites/replay/extendNetworkBreadcrumbs/fetch/captureRequestBody/test.ts

Lines changed: 27 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,9 @@ import { expect } from '@playwright/test';
33
import { sentryTest } from '../../../../../utils/fixtures';
44
import { envelopeRequestParser, waitForErrorRequest } from '../../../../../utils/helpers';
55
import {
6-
getCustomRecordingEvents,
6+
collectReplayRequests,
7+
getReplayPerformanceSpans,
78
shouldSkipReplayTest,
8-
waitForReplayRequest,
99
} from '../../../../../utils/replayHelpers';
1010

1111
sentryTest('captures text request body', async ({ getLocalTestPath, page, browserName }) => {
@@ -30,7 +30,9 @@ sentryTest('captures text request body', async ({ getLocalTestPath, page, browse
3030
});
3131

3232
const requestPromise = waitForErrorRequest(page);
33-
const replayRequestPromise1 = waitForReplayRequest(page, 0);
33+
const replayRequestPromise = collectReplayRequests(page, recordingEvents => {
34+
return getReplayPerformanceSpans(recordingEvents).some(span => span.op === 'resource.fetch');
35+
});
3436

3537
const url = await getLocalTestPath({ testDir: __dirname });
3638
await page.goto(url);
@@ -65,9 +67,8 @@ sentryTest('captures text request body', async ({ getLocalTestPath, page, browse
6567
},
6668
});
6769

68-
const replayReq1 = await replayRequestPromise1;
69-
const { performanceSpans: performanceSpans1 } = getCustomRecordingEvents(replayReq1);
70-
expect(performanceSpans1.filter(span => span.op === 'resource.fetch')).toEqual([
70+
const { replayRecordingSnapshots } = await replayRequestPromise;
71+
expect(getReplayPerformanceSpans(replayRecordingSnapshots).filter(span => span.op === 'resource.fetch')).toEqual([
7172
{
7273
data: {
7374
method: 'POST',
@@ -109,7 +110,9 @@ sentryTest('captures JSON request body', async ({ getLocalTestPath, page, browse
109110
});
110111

111112
const requestPromise = waitForErrorRequest(page);
112-
const replayRequestPromise1 = waitForReplayRequest(page, 0);
113+
const replayRequestPromise = collectReplayRequests(page, recordingEvents => {
114+
return getReplayPerformanceSpans(recordingEvents).some(span => span.op === 'resource.fetch');
115+
});
113116

114117
const url = await getLocalTestPath({ testDir: __dirname });
115118
await page.goto(url);
@@ -144,9 +147,8 @@ sentryTest('captures JSON request body', async ({ getLocalTestPath, page, browse
144147
},
145148
});
146149

147-
const replayReq1 = await replayRequestPromise1;
148-
const { performanceSpans: performanceSpans1 } = getCustomRecordingEvents(replayReq1);
149-
expect(performanceSpans1.filter(span => span.op === 'resource.fetch')).toEqual([
150+
const { replayRecordingSnapshots } = await replayRequestPromise;
151+
expect(getReplayPerformanceSpans(replayRecordingSnapshots).filter(span => span.op === 'resource.fetch')).toEqual([
150152
{
151153
data: {
152154
method: 'POST',
@@ -188,7 +190,9 @@ sentryTest('captures non-text request body', async ({ getLocalTestPath, page, br
188190
});
189191

190192
const requestPromise = waitForErrorRequest(page);
191-
const replayRequestPromise1 = waitForReplayRequest(page, 0);
193+
const replayRequestPromise = collectReplayRequests(page, recordingEvents => {
194+
return getReplayPerformanceSpans(recordingEvents).some(span => span.op === 'resource.fetch');
195+
});
192196

193197
const url = await getLocalTestPath({ testDir: __dirname });
194198
await page.goto(url);
@@ -227,9 +231,8 @@ sentryTest('captures non-text request body', async ({ getLocalTestPath, page, br
227231
},
228232
});
229233

230-
const replayReq1 = await replayRequestPromise1;
231-
const { performanceSpans: performanceSpans1 } = getCustomRecordingEvents(replayReq1);
232-
expect(performanceSpans1.filter(span => span.op === 'resource.fetch')).toEqual([
234+
const { replayRecordingSnapshots } = await replayRequestPromise;
235+
expect(getReplayPerformanceSpans(replayRecordingSnapshots).filter(span => span.op === 'resource.fetch')).toEqual([
233236
{
234237
data: {
235238
method: 'POST',
@@ -271,7 +274,9 @@ sentryTest('captures text request body when matching relative URL', async ({ get
271274
});
272275

273276
const requestPromise = waitForErrorRequest(page);
274-
const replayRequestPromise1 = waitForReplayRequest(page, 0);
277+
const replayRequestPromise = collectReplayRequests(page, recordingEvents => {
278+
return getReplayPerformanceSpans(recordingEvents).some(span => span.op === 'resource.fetch');
279+
});
275280

276281
const url = await getLocalTestUrl({ testDir: __dirname });
277282
await page.goto(url);
@@ -306,9 +311,8 @@ sentryTest('captures text request body when matching relative URL', async ({ get
306311
},
307312
});
308313

309-
const replayReq1 = await replayRequestPromise1;
310-
const { performanceSpans: performanceSpans1 } = getCustomRecordingEvents(replayReq1);
311-
expect(performanceSpans1.filter(span => span.op === 'resource.fetch')).toEqual([
314+
const { replayRecordingSnapshots } = await replayRequestPromise;
315+
expect(getReplayPerformanceSpans(replayRecordingSnapshots).filter(span => span.op === 'resource.fetch')).toEqual([
312316
{
313317
data: {
314318
method: 'POST',
@@ -348,7 +352,9 @@ sentryTest('does not capture request body when URL does not match', async ({ get
348352
});
349353

350354
const requestPromise = waitForErrorRequest(page);
351-
const replayRequestPromise1 = waitForReplayRequest(page, 0);
355+
const replayRequestPromise = collectReplayRequests(page, recordingEvents => {
356+
return getReplayPerformanceSpans(recordingEvents).some(span => span.op === 'resource.fetch');
357+
});
352358

353359
const url = await getLocalTestPath({ testDir: __dirname });
354360
await page.goto(url);
@@ -383,9 +389,8 @@ sentryTest('does not capture request body when URL does not match', async ({ get
383389
},
384390
});
385391

386-
const replayReq1 = await replayRequestPromise1;
387-
const { performanceSpans: performanceSpans1 } = getCustomRecordingEvents(replayReq1);
388-
expect(performanceSpans1.filter(span => span.op === 'resource.fetch')).toEqual([
392+
const { replayRecordingSnapshots } = await replayRequestPromise;
393+
expect(getReplayPerformanceSpans(replayRecordingSnapshots).filter(span => span.op === 'resource.fetch')).toEqual([
389394
{
390395
data: {
391396
method: 'POST',

‎dev-packages/browser-integration-tests/suites/replay/extendNetworkBreadcrumbs/fetch/captureRequestHeaders/test.ts

Lines changed: 27 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,9 @@ import { expect } from '@playwright/test';
33
import { sentryTest } from '../../../../../utils/fixtures';
44
import { envelopeRequestParser, waitForErrorRequest } from '../../../../../utils/helpers';
55
import {
6-
getCustomRecordingEvents,
6+
collectReplayRequests,
7+
getReplayPerformanceSpans,
78
shouldSkipReplayTest,
8-
waitForReplayRequest,
99
} from '../../../../../utils/replayHelpers';
1010

1111
sentryTest('handles empty/missing request headers', async ({ getLocalTestPath, page, browserName }) => {
@@ -28,7 +28,9 @@ sentryTest('handles empty/missing request headers', async ({ getLocalTestPath, p
2828
});
2929

3030
const requestPromise = waitForErrorRequest(page);
31-
const replayRequestPromise1 = waitForReplayRequest(page, 0);
31+
const replayRequestPromise = collectReplayRequests(page, recordingEvents => {
32+
return getReplayPerformanceSpans(recordingEvents).some(span => span.op === 'resource.fetch');
33+
});
3234

3335
const url = await getLocalTestPath({ testDir: __dirname });
3436
await page.goto(url);
@@ -61,9 +63,8 @@ sentryTest('handles empty/missing request headers', async ({ getLocalTestPath, p
6163
},
6264
});
6365

64-
const replayReq1 = await replayRequestPromise1;
65-
const { performanceSpans: performanceSpans1 } = getCustomRecordingEvents(replayReq1);
66-
expect(performanceSpans1.filter(span => span.op === 'resource.fetch')).toEqual([
66+
const { replayRecordingSnapshots } = await replayRequestPromise;
67+
expect(getReplayPerformanceSpans(replayRecordingSnapshots).filter(span => span.op === 'resource.fetch')).toEqual([
6768
{
6869
data: {
6970
method: 'POST',
@@ -100,7 +101,9 @@ sentryTest('captures request headers as POJO', async ({ getLocalTestPath, page,
100101
});
101102

102103
const requestPromise = waitForErrorRequest(page);
103-
const replayRequestPromise1 = waitForReplayRequest(page, 0);
104+
const replayRequestPromise = collectReplayRequests(page, recordingEvents => {
105+
return getReplayPerformanceSpans(recordingEvents).some(span => span.op === 'resource.fetch');
106+
});
104107

105108
const url = await getLocalTestPath({ testDir: __dirname });
106109
await page.goto(url);
@@ -140,9 +143,8 @@ sentryTest('captures request headers as POJO', async ({ getLocalTestPath, page,
140143
},
141144
});
142145

143-
const replayReq1 = await replayRequestPromise1;
144-
const { performanceSpans: performanceSpans1 } = getCustomRecordingEvents(replayReq1);
145-
expect(performanceSpans1.filter(span => span.op === 'resource.fetch')).toEqual([
146+
const { replayRecordingSnapshots } = await replayRequestPromise;
147+
expect(getReplayPerformanceSpans(replayRecordingSnapshots).filter(span => span.op === 'resource.fetch')).toEqual([
146148
{
147149
data: {
148150
method: 'POST',
@@ -184,7 +186,9 @@ sentryTest('captures request headers on Request', async ({ getLocalTestPath, pag
184186
});
185187

186188
const requestPromise = waitForErrorRequest(page);
187-
const replayRequestPromise1 = waitForReplayRequest(page, 0);
189+
const replayRequestPromise = collectReplayRequests(page, recordingEvents => {
190+
return getReplayPerformanceSpans(recordingEvents).some(span => span.op === 'resource.fetch');
191+
});
188192

189193
const url = await getLocalTestPath({ testDir: __dirname });
190194
await page.goto(url);
@@ -224,9 +228,8 @@ sentryTest('captures request headers on Request', async ({ getLocalTestPath, pag
224228
},
225229
});
226230

227-
const replayReq1 = await replayRequestPromise1;
228-
const { performanceSpans: performanceSpans1 } = getCustomRecordingEvents(replayReq1);
229-
expect(performanceSpans1.filter(span => span.op === 'resource.fetch')).toEqual([
231+
const { replayRecordingSnapshots } = await replayRequestPromise;
232+
expect(getReplayPerformanceSpans(replayRecordingSnapshots).filter(span => span.op === 'resource.fetch')).toEqual([
230233
{
231234
data: {
232235
method: 'POST',
@@ -267,7 +270,9 @@ sentryTest('captures request headers as Headers instance', async ({ getLocalTest
267270
});
268271

269272
const requestPromise = waitForErrorRequest(page);
270-
const replayRequestPromise1 = waitForReplayRequest(page, 0);
273+
const replayRequestPromise = collectReplayRequests(page, recordingEvents => {
274+
return getReplayPerformanceSpans(recordingEvents).some(span => span.op === 'resource.fetch');
275+
});
271276

272277
const url = await getLocalTestPath({ testDir: __dirname });
273278

@@ -308,9 +313,8 @@ sentryTest('captures request headers as Headers instance', async ({ getLocalTest
308313
},
309314
});
310315

311-
const replayReq1 = await replayRequestPromise1;
312-
const { performanceSpans: performanceSpans1 } = getCustomRecordingEvents(replayReq1);
313-
expect(performanceSpans1.filter(span => span.op === 'resource.fetch')).toEqual([
316+
const { replayRecordingSnapshots } = await replayRequestPromise;
317+
expect(getReplayPerformanceSpans(replayRecordingSnapshots).filter(span => span.op === 'resource.fetch')).toEqual([
314318
{
315319
data: {
316320
method: 'POST',
@@ -351,7 +355,9 @@ sentryTest('does not captures request headers if URL does not match', async ({ g
351355
});
352356

353357
const requestPromise = waitForErrorRequest(page);
354-
const replayRequestPromise1 = waitForReplayRequest(page, 0);
358+
const replayRequestPromise = collectReplayRequests(page, recordingEvents => {
359+
return getReplayPerformanceSpans(recordingEvents).some(span => span.op === 'resource.fetch');
360+
});
355361

356362
const url = await getLocalTestPath({ testDir: __dirname });
357363
await page.goto(url);
@@ -391,9 +397,8 @@ sentryTest('does not captures request headers if URL does not match', async ({ g
391397
},
392398
});
393399

394-
const replayReq1 = await replayRequestPromise1;
395-
const { performanceSpans: performanceSpans1 } = getCustomRecordingEvents(replayReq1);
396-
expect(performanceSpans1.filter(span => span.op === 'resource.fetch')).toEqual([
400+
const { replayRecordingSnapshots } = await replayRequestPromise;
401+
expect(getReplayPerformanceSpans(replayRecordingSnapshots).filter(span => span.op === 'resource.fetch')).toEqual([
397402
{
398403
data: {
399404
method: 'POST',

‎dev-packages/browser-integration-tests/suites/replay/extendNetworkBreadcrumbs/fetch/captureRequestSize/test.ts

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,9 @@ import { expect } from '@playwright/test';
33
import { sentryTest } from '../../../../../utils/fixtures';
44
import { envelopeRequestParser, waitForErrorRequest } from '../../../../../utils/helpers';
55
import {
6-
getCustomRecordingEvents,
6+
collectReplayRequests,
7+
getReplayPerformanceSpans,
78
shouldSkipReplayTest,
8-
waitForReplayRequest,
99
} from '../../../../../utils/replayHelpers';
1010

1111
sentryTest('captures request body size when body is sent', async ({ getLocalTestPath, page }) => {
@@ -28,7 +28,9 @@ sentryTest('captures request body size when body is sent', async ({ getLocalTest
2828
});
2929

3030
const requestPromise = waitForErrorRequest(page);
31-
const replayRequestPromise1 = waitForReplayRequest(page, 0);
31+
const replayRequestPromise = collectReplayRequests(page, recordingEvents => {
32+
return getReplayPerformanceSpans(recordingEvents).some(span => span.op === 'resource.fetch');
33+
});
3234

3335
const url = await getLocalTestPath({ testDir: __dirname });
3436
await page.goto(url);
@@ -63,9 +65,8 @@ sentryTest('captures request body size when body is sent', async ({ getLocalTest
6365
},
6466
});
6567

66-
const replayReq1 = await replayRequestPromise1;
67-
const { performanceSpans: performanceSpans1 } = getCustomRecordingEvents(replayReq1);
68-
expect(performanceSpans1.filter(span => span.op === 'resource.fetch')).toEqual([
68+
const { replayRecordingSnapshots } = await replayRequestPromise;
69+
expect(getReplayPerformanceSpans(replayRecordingSnapshots).filter(span => span.op === 'resource.fetch')).toEqual([
6970
{
7071
data: {
7172
method: 'POST',
@@ -112,7 +113,9 @@ sentryTest('captures request size from non-text request body', async ({ getLocal
112113
});
113114

114115
const requestPromise = waitForErrorRequest(page);
115-
const replayRequestPromise1 = waitForReplayRequest(page, 0);
116+
const replayRequestPromise = collectReplayRequests(page, recordingEvents => {
117+
return getReplayPerformanceSpans(recordingEvents).some(span => span.op === 'resource.fetch');
118+
});
116119

117120
const url = await getLocalTestPath({ testDir: __dirname });
118121
await page.goto(url);
@@ -149,9 +152,8 @@ sentryTest('captures request size from non-text request body', async ({ getLocal
149152
},
150153
});
151154

152-
const replayReq1 = await replayRequestPromise1;
153-
const { performanceSpans: performanceSpans1 } = getCustomRecordingEvents(replayReq1);
154-
expect(performanceSpans1.filter(span => span.op === 'resource.fetch')).toEqual([
155+
const { replayRecordingSnapshots } = await replayRequestPromise;
156+
expect(getReplayPerformanceSpans(replayRecordingSnapshots).filter(span => span.op === 'resource.fetch')).toEqual([
155157
{
156158
data: {
157159
method: 'POST',

0 commit comments

Comments
 (0)