ci: Improve flaky test detector performance (#7569)

We now run all tests into a single spawned process, which should hopefully speed this up a bit.
We also reduce the # of repetitions from 100 to 50 to ensure we do not run overtime.

Author: mydea

.github/workflows/flaky-test-detector.yml

@@ -77,5 +77,5 @@ jobs:
         working-directory: packages/browser-integration-tests
         env:
           CHANGED_TEST_PATHS: ${{ steps.changed.outputs.browser_integration_files }}
-          # Run 100 times when detecting changed test(s), else run all tests 5x
-          TEST_RUN_COUNT: ${{ steps.changed.outputs.browser_integration == 'true' && 100 || 5 }}
+          # Run 50 times when detecting changed test(s), else run all tests 5x
+          TEST_RUN_COUNT: ${{ steps.changed.outputs.browser_integration == 'true' && 50 || 5 }}

packages/browser-integration-tests/scripts/detectFlakyTests.ts

@@ -6,52 +6,72 @@ import { promisify } from 'util';
 const exec = promisify(childProcess.exec);
 
 async function run(): Promise<void> {
-  let testPaths = getTestPaths();
-  let failed = [];
+  let testPaths: string[] = [];
 
-  try {
-    const changedPaths: string[] = process.env.CHANGED_TEST_PATHS ? JSON.parse(process.env.CHANGED_TEST_PATHS) : [];
+  const changedPaths: string[] = process.env.CHANGED_TEST_PATHS ? JSON.parse(process.env.CHANGED_TEST_PATHS) : [];
 
-    if (changedPaths.length > 0) {
-      console.log(`Detected changed test paths:
+  if (changedPaths.length > 0) {
+    console.log(`Detected changed test paths:
 ${changedPaths.join('\n')}
 
 `);
 
-      testPaths = testPaths.filter(p => changedPaths.some(changedPath => changedPath.includes(p)));
+    testPaths = getTestPaths().filter(p => changedPaths.some(changedPath => changedPath.includes(p)));
+    if (testPaths.length === 0) {
+      console.log('Could not find matching tests, aborting...');
+      process.exit(1);
     }
-  } catch {
-    console.log('Could not detect changed test paths, running all tests.');
   }
 
   const cwd = path.join(__dirname, '../');
   const runCount = parseInt(process.env.TEST_RUN_COUNT || '10');
 
-  for (const testPath of testPaths) {
-    console.log(`Running test: ${testPath}`);
-    const start = Date.now();
+  try {
+    await new Promise<void>((resolve, reject) => {
+      const cp = childProcess.spawn(
+        `yarn playwright test ${
+          testPaths.length ? testPaths.join(' ') : './suites'
+        } --browser='all' --reporter='line' --repeat-each ${runCount}`,
+        { shell: true, cwd },
+      );
+
+      let error: Error | undefined;
+
+      cp.stdout.on('data', data => {
+        console.log(data ? (data as object).toString() : '');
+      });
 
-    try {
-      await exec(`yarn playwright test ${testPath} --browser='all' --repeat-each ${runCount}`, {
-        cwd,
+      cp.stderr.on('data', data => {
+        console.log(data ? (data as object).toString() : '');
       });
-      const end = Date.now();
-      console.log(`  ☑️  Passed ${runCount} times, avg. duration ${Math.ceil((end - start) / runCount)}ms`);
-    } catch (error) {
-      logError(error);
-      failed.push(testPath);
-    }
-  }
 
-  console.log('');
-  console.log('');
+      cp.on('error', e => {
+        console.error(e);
+        error = e;
+      });
 
-  if (failed.length > 0) {
-    console.error(`⚠️ ${failed.length} test(s) failed.`);
+      cp.on('close', status => {
+        const err = error || (status !== 0 ? new Error(`Process exited with status ${status}`) : undefined);
+
+        if (err) {
+          reject(err);
+        } else {
+          resolve();
+        }
+      });
+    });
+  } catch (error) {
+    console.log('');
+    console.log('');
+
+    console.error(`⚠️ Some tests failed.`);
+    console.error(error);
     process.exit(1);
-  } else {
-    console.log(`☑️ ${testPaths.length} test(s) passed.`);
   }
+
+  console.log('');
+  console.log('');
+  console.log(`☑️ All tests passed.`);
 }
 
 function getTestPaths(): string[] {