Skip to content

Commit 881fbb8

Browse files
mydeamydea
committed
ci: Add workflow to detect flaky tests
1 parent 30bba1f commit 881fbb8

File tree

4 files changed

+168
-2
lines changed

4 files changed

+168
-2
lines changed

‎.github/workflows/flaky-test-detector.yml

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
name: 'Detect flaky tests'
2+
on:
3+
workflow_dispatch:
4+
pull_request:
5+
paths:
6+
- 'packages/browser-integration-tests/suites/**'
7+
8+
env:
9+
HEAD_COMMIT: ${{ github.event.inputs.commit || github.sha }}
10+
11+
NX_CACHE_RESTORE_KEYS: |
12+
nx-Linux-${{ github.ref }}-${{ github.event.inputs.commit || github.sha }}
13+
nx-Linux-${{ github.ref }}
14+
nx-Linux
15+
16+
# Cancel in progress workflows on pull_requests.
17+
# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value
18+
concurrency:
19+
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
20+
cancel-in-progress: true
21+
22+
jobs:
23+
flaky-detector:
24+
runs-on: ubuntu-20.04
25+
timeout-minutes: 60
26+
name: 'Check tests for flakiness'
27+
steps:
28+
- name: Check out current branch
29+
uses: actions/checkout@v3
30+
- name: Set up Node
31+
uses: volta-cli/action@v4
32+
33+
- name: Install dependencies
34+
run: yarn install --ignore-engines --frozen-lockfile
35+
36+
- name: NX cache
37+
uses: actions/cache/restore@v3
38+
with:
39+
path: .nxcache
40+
key: nx-Linux-${{ github.ref }}-${{ env.HEAD_COMMIT }}
41+
restore-keys: ${{ env.NX_CACHE_RESTORE_KEYS }}
42+
43+
- name: Build packages
44+
run: yarn build
45+
46+
- name: Get npm cache directory
47+
id: npm-cache-dir
48+
run: echo "dir=$(npm config get cache)" >> $GITHUB_OUTPUT
49+
- name: Get Playwright version
50+
id: playwright-version
51+
run: echo "version=$(node -p "require('@playwright/test/package.json').version")" >> $GITHUB_OUTPUT
52+
- uses: actions/cache@v3
53+
name: Check if Playwright browser is cached
54+
id: playwright-cache
55+
with:
56+
path: ${{ steps.npm-cache-dir.outputs.dir }}
57+
key: ${{ runner.os }}-Playwright-${{steps.playwright-version.outputs.version}}
58+
- name: Install Playwright browser if not cached
59+
if: steps.playwright-cache.outputs.cache-hit != 'true'
60+
run: npx playwright install --with-deps
61+
env:
62+
PLAYWRIGHT_BROWSERS_PATH: ${{steps.npm-cache-dir.outputs.dir}}
63+
- name: Install OS dependencies of Playwright if cache hit
64+
if: steps.playwright-cache.outputs.cache-hit == 'true'
65+
run: npx playwright install-deps
66+
67+
- name: Determine changed tests
68+
uses: getsentry/[email protected]
69+
id: changed
70+
with:
71+
list-files: json
72+
filters: |
73+
browser_integration: packages/browser-integration-tests/suites/**
74+
75+
- name: Detect flaky tests
76+
run: yarn test:detect-flaky
77+
working-directory: packages/browser-integration-tests
78+
env:
79+
CHANGED_TEST_PATHS: ${{ steps.changed.outputs.browser_integration_files }}
80+
# Run 100 times when detecting changed test(s), else run all tests 5x
81+
TEST_RUN_COUNT: ${{ steps.changed.outputs.browser_integration == 'true' && 100 || 5 }}

‎packages/browser-integration-tests/.eslintrc.js

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ module.exports = {
44
node: true,
55
},
66
extends: ['../../.eslintrc.js'],
7-
ignorePatterns: ['suites/**/subject.js', 'suites/**/dist/*'],
7+
ignorePatterns: ['suites/**/subject.js', 'suites/**/dist/*', 'scripts/**'],
88
parserOptions: {
99
sourceType: 'module',
1010
},

‎packages/browser-integration-tests/package.json

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,8 @@
2828
"test:cjs": "PW_BUNDLE=cjs yarn test",
2929
"test:esm": "PW_BUNDLE=esm yarn test",
3030
"test:ci": "playwright test ./suites --browser='all' --reporter='line'",
31-
"test:update-snapshots": "yarn test --update-snapshots --browser='all' && yarn test --update-snapshots"
31+
"test:update-snapshots": "yarn test --update-snapshots --browser='all' && yarn test --update-snapshots",
32+
"test:detect-flaky": "ts-node scripts/detectFlakyTests.ts"
3233
},
3334
"dependencies": {
3435
"@babel/preset-typescript": "^7.16.7",
@@ -40,6 +41,10 @@
4041
"typescript": "^4.5.2",
4142
"webpack": "^5.52.0"
4243
},
44+
"devDependencies": {
45+
"glob": "8.0.3",
46+
"@types/glob": "8.0.0"
47+
},
4348
"volta": {
4449
"extends": "../../package.json"
4550
}

‎packages/browser-integration-tests/scripts/detectFlakyTests.ts

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
import * as glob from 'glob';
2+
import * as path from 'path';
3+
import * as childProcess from 'child_process';
4+
import { promisify } from 'util';
5+
6+
const exec = promisify(childProcess.exec);
7+
8+
async function run(): Promise<void> {
9+
let testPaths = getTestPaths();
10+
let failed = [];
11+
12+
try {
13+
const changedPaths: string[] = process.env.CHANGED_TEST_PATHS ? JSON.parse(process.env.CHANGED_TEST_PATHS) : [];
14+
15+
if (changedPaths.length > 0) {
16+
console.log(`Detected changed test paths:
17+
${changedPaths.join('\n')}
18+
19+
`);
20+
21+
testPaths = testPaths.filter(p => changedPaths.some(changedPath => changedPath.includes(p)));
22+
}
23+
} catch {
24+
console.log('Could not detect changed test paths, running all tests.');
25+
}
26+
27+
const cwd = path.join(__dirname, '../');
28+
const runCount = parseInt(process.env.TEST_RUN_COUNT || '10');
29+
30+
for (const testPath of testPaths) {
31+
console.log(`Running test: ${testPath}`);
32+
const start = Date.now();
33+
34+
try {
35+
await exec(`yarn playwright test ${testPath} --browser='all' --repeat-each ${runCount}`, {
36+
cwd,
37+
});
38+
const end = Date.now();
39+
console.log(` ☑️ Passed ${runCount} times, avg. duration ${Math.ceil((end - start) / runCount)}ms`);
40+
} catch (error) {
41+
logError(error);
42+
failed.push(testPath);
43+
}
44+
}
45+
46+
console.log('');
47+
console.log('');
48+
49+
if (failed.length > 0) {
50+
console.error(`⚠️ ${failed.length} test(s) failed.`);
51+
process.exit(1);
52+
} else {
53+
console.log(`☑️ ${testPaths.length} test(s) passed.`);
54+
}
55+
}
56+
57+
function getTestPaths(): string[] {
58+
const paths = glob.sync('suites/**/test.{ts,js}', {
59+
cwd: path.join(__dirname, '../'),
60+
});
61+
62+
return paths.map(p => path.dirname(p));
63+
}
64+
65+
function logError(error: unknown) {
66+
if (process.env.CI) {
67+
console.log('::group::Test failed');
68+
} else {
69+
console.error(' ⚠️ Test failed:');
70+
}
71+
72+
console.log((error as any).stdout);
73+
console.log((error as any).stderr);
74+
75+
if (process.env.CI) {
76+
console.log('::endgroup::');
77+
}
78+
}
79+
80+
run();

0 commit comments

Comments
 (0)