test: Improve node integration test running (#10673)

This improves the output and formatting of the node integration tests,
making it a bit easier to work with them:

1. Removed the coverage output, making the general output much clearer
2. Ensure we have color in the jest output, making it easier to scan it
3. Remove logging around started tests & workers, streamlining the
output and aligning it more with the usual jest output.
4. Move skipped output to the end of the test, clearing up the output
for actually run tests
5. Update summary at the bottom to show success/failed/skipped test
numbers
6. Add color to summary output (error/success/skipped) to make it
clearer.
7. Show relative path names instead of full path names

Skipped tests:
<img width="528" alt="Screenshot 2024-02-15 at 12 04 29"
src="https://github.com/getsentry/sentry-javascript/assets/2411343/02d849c8-b8f4-4775-bd5c-ce4e8e18cfc2">

Failed test:
<img width="1138" alt="Screenshot 2024-02-15 at 11 46 42"
src="https://github.com/getsentry/sentry-javascript/assets/2411343/72ee1d31-6c49-4108-b891-ed5492d183ac">

Successful test:

<img width="1002" alt="Screenshot 2024-02-15 at 11 48 28"
src="https://github.com/getsentry/sentry-javascript/assets/2411343/1ee3b2a2-ecd0-4eaf-b68b-2d534f1c4622">

Failed test summary:
<img width="491" alt="Screenshot 2024-02-15 at 11 52 19"
src="https://github.com/getsentry/sentry-javascript/assets/2411343/b19d051d-189b-4459-8a28-9b9c6dcc2bb5">

Successful test summary:
<img width="296" alt="Screenshot 2024-02-15 at 11 55 28"
src="https://github.com/getsentry/sentry-javascript/assets/2411343/bc369482-08ab-4b50-8cf8-3c7c2e9cf51c">

Author: mydea

dev-packages/node-integration-tests/jest.config.js

@@ -5,4 +5,5 @@ module.exports = {
   ...baseConfig,
   testMatch: ['**/test.ts'],
   setupFilesAfterEnv: ['./jest.setup.js'],
+  coverageReporters: ['json', 'lcov', 'clover'],
 };

dev-packages/node-integration-tests/suites/anr/basic-session.js

@@ -17,7 +17,6 @@ Sentry.init({
 function longWork() {
   for (let i = 0; i < 20; i++) {
     const salt = crypto.randomBytes(128).toString('base64');
-    // eslint-disable-next-line no-unused-vars
     const hash = crypto.pbkdf2Sync('myPassword', salt, 10000, 512, 'sha512');
     assert.ok(hash);
   }

dev-packages/node-integration-tests/suites/anr/basic.js

@@ -18,7 +18,6 @@ Sentry.init({
 function longWork() {
   for (let i = 0; i < 20; i++) {
     const salt = crypto.randomBytes(128).toString('base64');
-    // eslint-disable-next-line no-unused-vars
     const hash = crypto.pbkdf2Sync('myPassword', salt, 10000, 512, 'sha512');
     assert.ok(hash);
   }

dev-packages/node-integration-tests/suites/anr/basic.mjs

@@ -18,7 +18,6 @@ Sentry.init({
 function longWork() {
   for (let i = 0; i < 20; i++) {
     const salt = crypto.randomBytes(128).toString('base64');
-    // eslint-disable-next-line no-unused-vars
     const hash = crypto.pbkdf2Sync('myPassword', salt, 10000, 512, 'sha512');
     assert.ok(hash);
   }

dev-packages/node-integration-tests/suites/anr/forked.js

@@ -10,15 +10,14 @@ setTimeout(() => {
 Sentry.init({
   dsn: 'https://[email protected]/1337',
   release: '1.0',
-  debug: true,
   autoSessionTracking: false,
+  debug: true,
   integrations: [Sentry.anrIntegration({ captureStackTrace: true, anrThreshold: 100 })],
 });
 
 function longWork() {
   for (let i = 0; i < 20; i++) {
     const salt = crypto.randomBytes(128).toString('base64');
-    // eslint-disable-next-line no-unused-vars
     const hash = crypto.pbkdf2Sync('myPassword', salt, 10000, 512, 'sha512');
     assert.ok(hash);
   }

dev-packages/node-integration-tests/suites/anr/should-exit-forced.js

@@ -4,8 +4,8 @@ function configureSentry() {
   Sentry.init({
     dsn: 'https://[email protected]/1337',
     release: '1.0',
-    debug: true,
     autoSessionTracking: false,
+    debug: true,
     integrations: [Sentry.anrIntegration({ captureStackTrace: true })],
   });
 }

dev-packages/node-integration-tests/suites/anr/should-exit.js

@@ -4,8 +4,8 @@ function configureSentry() {
   Sentry.init({
     dsn: 'https://[email protected]/1337',
     release: '1.0',
-    debug: true,
     autoSessionTracking: false,
+    debug: true,
     integrations: [Sentry.anrIntegration({ captureStackTrace: true })],
   });
 }

dev-packages/node-integration-tests/suites/proxy/basic.js

@@ -8,7 +8,6 @@ proxy.listen(0, () => {
 
   Sentry.init({
     dsn: process.env.SENTRY_DSN,
-    debug: true,
     transportOptions: {
       proxy: `http://localhost:${proxyPort}`,
     },

dev-packages/node-integration-tests/suites/tracing-experimental/hapi/scenario.js

@@ -4,7 +4,6 @@ const Sentry = require('@sentry/node-experimental');
 Sentry.init({
   dsn: 'https://[email protected]/1337',
   release: '1.0',
-  debug: true,
   tracesSampleRate: 1.0,
   transport: loggingTransport,
 });

dev-packages/node-integration-tests/suites/tracing-experimental/mongoose/scenario.js

@@ -4,7 +4,6 @@ const Sentry = require('@sentry/node-experimental');
 Sentry.init({
   dsn: 'https://[email protected]/1337',
   release: '1.0',
-  debug: true,
   tracesSampleRate: 1.0,
   transport: loggingTransport,
 });

dev-packages/node-integration-tests/suites/tracing-new/httpIntegration/spans/scenario.ts

@@ -8,7 +8,6 @@ Sentry.init({
   release: '1.0',
   tracesSampleRate: 1.0,
   integrations: [Sentry.httpIntegration({})],
-  debug: true,
 });
 
 // eslint-disable-next-line @typescript-eslint/no-floating-promises

dev-packages/node-integration-tests/utils/run-tests.ts

@@ -1,6 +1,7 @@
 /* eslint-disable no-console */
 import childProcess from 'child_process';
 import os from 'os';
+import path from 'path';
 import yargs from 'yargs';
 
 const args = yargs
@@ -19,14 +20,19 @@ const testPaths = childProcess.execSync('jest --listTests', { encoding: 'utf8' }
 
 const numTests = testPaths.length;
 const fails: string[] = [];
+const skips: string[] = [];
+
+function getTestPath(testPath: string): string {
+  const cwd = process.cwd();
+  return path.relative(cwd, testPath);
+}
 
 // We're creating a worker for each CPU core.
-const workers = os.cpus().map(async (_, i) => {
+const workers = os.cpus().map(async () => {
   while (testPaths.length > 0) {
-    const testPath = testPaths.pop();
-    console.log(`(Worker ${i}) Running test "${testPath}"`);
+    const testPath = testPaths.pop() as string;
     await new Promise<void>(resolve => {
-      const jestArgs = ['--runTestsByPath', testPath as string, '--forceExit'];
+      const jestArgs = ['--runTestsByPath', testPath as string, '--forceExit', '--colors'];
 
       if (args.t) {
         jestArgs.push('-t', args.t);
@@ -51,18 +57,24 @@ const workers = os.cpus().map(async (_, i) => {
       });
 
       jestProcess.on('error', error => {
+        console.log(`"${getTestPath(testPath)}" finished with error`, error);
         console.log(output);
-        console.log(`(Worker ${i}) Error in test "${testPath}"`, error);
-        fails.push(`FAILED: "${testPath}"`);
+        fails.push(`FAILED: ${getTestPath(testPath)}`);
         resolve();
       });
 
       jestProcess.on('exit', exitcode => {
-        output = checkSkippedAllTests(output, i, testPath);
-        console.log(`(Worker ${i}) Finished test "${testPath}"`);
-        console.log(output);
-        if (exitcode !== 0) {
-          fails.push(`FAILED: "${testPath}"`);
+        const hasError = exitcode !== 0;
+        const skippedOutput = checkSkippedAllTests(output);
+
+        if (skippedOutput && !hasError) {
+          skips.push(`SKIPPED: ${getTestPath(testPath)}`);
+        } else {
+          console.log(output);
+        }
+
+        if (hasError) {
+          fails.push(`FAILED: ${getTestPath(testPath)}`);
         }
         resolve();
       });
@@ -73,15 +85,35 @@ const workers = os.cpus().map(async (_, i) => {
 // eslint-disable-next-line @typescript-eslint/no-floating-promises
 Promise.all(workers).then(() => {
   console.log('-------------------');
-  console.log(`Successfully ran ${numTests} tests.`);
-  if (fails.length > 0) {
-    console.log('Not all tests succeeded:\n');
+
+  const failCount = fails.length;
+  const skipCount = skips.length;
+  const totalCount = numTests;
+  const successCount = numTests - failCount - skipCount;
+  const nonSkippedCount = totalCount - skipCount;
+
+  if (skips.length) {
+    console.log('\x1b[2m%s\x1b[0m', '\nSkipped tests:');
+    skips.forEach(skip => {
+      console.log('\x1b[2m%s\x1b[0m', `● ${skip}`);
+    });
+  }
+
+  if (failCount > 0) {
+    console.log(
+      '\x1b[31m%s\x1b[0m',
+      `\n${failCount} of ${nonSkippedCount} tests failed${skipCount ? ` (${skipCount} skipped)` : ''}:\n`,
+    );
     fails.forEach(fail => {
-      console.log(`● ${fail}`);
+      console.log('\x1b[31m%s\x1b[0m', `● ${fail}`);
     });
     process.exit(1);
   } else {
-    console.log('All tests succeeded.');
+    console.log(
+      '\x1b[32m%s\x1b[0m',
+      `\nSuccessfully ran ${successCount} tests${skipCount ? ` (${skipCount} skipped)` : ''}.`,
+    );
+    console.log('\x1b[32m%s\x1b[0m', 'All tests succeeded.');
     process.exit(0);
   }
 });
@@ -90,15 +122,17 @@ Promise.all(workers).then(() => {
  * Suppress jest output for test suites where all tests were skipped.
  * This only clutters the logs and we can safely print a one-liner instead.
  */
-function checkSkippedAllTests(output: string, workerNumber: number, testPath: string | undefined): string {
-  const regex = /Tests:\s+(\d+) skipped, (\d+) total/gm;
+function checkSkippedAllTests(output: string): boolean {
+  const regex = /(.+)Tests:(.+)\s+(.+?)(\d+) skipped(.+), (\d+) total/gm;
   const matches = regex.exec(output);
+
   if (matches) {
-    const skipped = Number(matches[1]);
-    const total = Number(matches[2]);
+    const skipped = Number(matches[4]);
+    const total = Number(matches[6]);
     if (!isNaN(skipped) && !isNaN(total) && total === skipped) {
-      return `(Worker ${workerNumber}) > Skipped all (${total} tests) in ${testPath}`;
+      return true;
     }
   }
-  return output;
+
+  return false;
 }