test: Improve node integration test running

dev-packages/node-integration-tests/jest.config.js

@@ -5,4 +5,5 @@ module.exports = {
   ...baseConfig,
   testMatch: ['**/test.ts'],
   setupFilesAfterEnv: ['./jest.setup.js'],
+  coverageReporters: ['json', 'lcov', 'clover'],
 };

dev-packages/node-integration-tests/suites/anr/basic-session.js

@@ -17,7 +17,6 @@ Sentry.init({
 function longWork() {
   for (let i = 0; i < 20; i++) {
     const salt = crypto.randomBytes(128).toString('base64');
-    // eslint-disable-next-line no-unused-vars
     const hash = crypto.pbkdf2Sync('myPassword', salt, 10000, 512, 'sha512');
     assert.ok(hash);
   }

dev-packages/node-integration-tests/suites/anr/basic.js

@@ -18,7 +18,6 @@ Sentry.init({
 function longWork() {
   for (let i = 0; i < 20; i++) {
     const salt = crypto.randomBytes(128).toString('base64');
-    // eslint-disable-next-line no-unused-vars
     const hash = crypto.pbkdf2Sync('myPassword', salt, 10000, 512, 'sha512');
     assert.ok(hash);
   }

dev-packages/node-integration-tests/suites/anr/basic.mjs

@@ -18,7 +18,6 @@ Sentry.init({
 function longWork() {
   for (let i = 0; i < 20; i++) {
     const salt = crypto.randomBytes(128).toString('base64');
-    // eslint-disable-next-line no-unused-vars
     const hash = crypto.pbkdf2Sync('myPassword', salt, 10000, 512, 'sha512');
     assert.ok(hash);
   }

dev-packages/node-integration-tests/suites/anr/forked.js

@@ -10,15 +10,14 @@ setTimeout(() => {
 Sentry.init({
   dsn: 'https://[email protected]/1337',
   release: '1.0',
-  debug: true,
   autoSessionTracking: false,
+  debug: true,
   integrations: [Sentry.anrIntegration({ captureStackTrace: true, anrThreshold: 100 })],
 });
 
 function longWork() {
   for (let i = 0; i < 20; i++) {
     const salt = crypto.randomBytes(128).toString('base64');
-    // eslint-disable-next-line no-unused-vars
     const hash = crypto.pbkdf2Sync('myPassword', salt, 10000, 512, 'sha512');
     assert.ok(hash);
   }

dev-packages/node-integration-tests/suites/anr/should-exit-forced.js

@@ -4,8 +4,8 @@ function configureSentry() {
   Sentry.init({
     dsn: 'https://[email protected]/1337',
     release: '1.0',
-    debug: true,
     autoSessionTracking: false,
+    debug: true,
     integrations: [Sentry.anrIntegration({ captureStackTrace: true })],
   });
 }

dev-packages/node-integration-tests/suites/anr/should-exit.js

@@ -4,8 +4,8 @@ function configureSentry() {
   Sentry.init({
     dsn: 'https://[email protected]/1337',
     release: '1.0',
-    debug: true,
     autoSessionTracking: false,
+    debug: true,
     integrations: [Sentry.anrIntegration({ captureStackTrace: true })],
   });
 }

dev-packages/node-integration-tests/suites/proxy/basic.js

@@ -8,7 +8,6 @@ proxy.listen(0, () => {
 
   Sentry.init({
     dsn: process.env.SENTRY_DSN,
-    debug: true,
     transportOptions: {
       proxy: `http://localhost:${proxyPort}`,
     },

dev-packages/node-integration-tests/suites/tracing-experimental/hapi/scenario.js

@@ -4,7 +4,6 @@ const Sentry = require('@sentry/node-experimental');
 Sentry.init({
   dsn: 'https://[email protected]/1337',
   release: '1.0',
-  debug: true,
   tracesSampleRate: 1.0,
   transport: loggingTransport,
 });

dev-packages/node-integration-tests/suites/tracing-experimental/mongoose/scenario.js

@@ -4,7 +4,6 @@ const Sentry = require('@sentry/node-experimental');
 Sentry.init({
   dsn: 'https://[email protected]/1337',
   release: '1.0',
-  debug: true,
   tracesSampleRate: 1.0,
   transport: loggingTransport,
 });

dev-packages/node-integration-tests/suites/tracing-new/httpIntegration/spans/scenario.ts

@@ -8,7 +8,6 @@ Sentry.init({
   release: '1.0',
   tracesSampleRate: 1.0,
   integrations: [Sentry.httpIntegration({})],
-  debug: true,
 });
 
 // eslint-disable-next-line @typescript-eslint/no-floating-promises

dev-packages/node-integration-tests/utils/run-tests.ts

@@ -1,6 +1,7 @@
 /* eslint-disable no-console */
 import childProcess from 'child_process';
 import os from 'os';
+import path from 'path';
 import yargs from 'yargs';
 
 const args = yargs
@@ -19,14 +20,19 @@ const testPaths = childProcess.execSync('jest --listTests', { encoding: 'utf8' }
 
 const numTests = testPaths.length;
 const fails: string[] = [];
+const skips: string[] = [];
+
+function getTestPath(testPath: string): string {
+  const cwd = process.cwd();
+  return path.relative(cwd, testPath);
+}
 
 // We're creating a worker for each CPU core.
-const workers = os.cpus().map(async (_, i) => {
+const workers = os.cpus().map(async () => {
   while (testPaths.length > 0) {
-    const testPath = testPaths.pop();
-    console.log(`(Worker ${i}) Running test "${testPath}"`);
+    const testPath = testPaths.pop() as string;
     await new Promise<void>(resolve => {
-      const jestArgs = ['--runTestsByPath', testPath as string, '--forceExit'];
+      const jestArgs = ['--runTestsByPath', testPath as string, '--forceExit', '--colors'];
 
       if (args.t) {
         jestArgs.push('-t', args.t);
@@ -51,18 +57,24 @@ const workers = os.cpus().map(async (_, i) => {
       });
 
       jestProcess.on('error', error => {
+        console.log(`"${getTestPath(testPath)}" finished with error`, error);
         console.log(output);
-        console.log(`(Worker ${i}) Error in test "${testPath}"`, error);
-        fails.push(`FAILED: "${testPath}"`);
+        fails.push(`FAILED: ${getTestPath(testPath)}`);
         resolve();
       });
 
       jestProcess.on('exit', exitcode => {
-        output = checkSkippedAllTests(output, i, testPath);
-        console.log(`(Worker ${i}) Finished test "${testPath}"`);
-        console.log(output);
-        if (exitcode !== 0) {
-          fails.push(`FAILED: "${testPath}"`);
+        const hasError = exitcode !== 0;
+        const skippedOutput = checkSkippedAllTests(output);
+
+        if (skippedOutput && !hasError) {
+          skips.push(`SKIPPED: ${getTestPath(testPath)}`);
+        } else {
+          console.log(output);
+        }
+
+        if (hasError) {
+          fails.push(`FAILED: ${getTestPath(testPath)}`);
         }
         resolve();
       });
@@ -73,15 +85,35 @@ const workers = os.cpus().map(async (_, i) => {
 // eslint-disable-next-line @typescript-eslint/no-floating-promises
 Promise.all(workers).then(() => {
   console.log('-------------------');
-  console.log(`Successfully ran ${numTests} tests.`);
-  if (fails.length > 0) {
-    console.log('Not all tests succeeded:\n');
+
+  const failCount = fails.length;
+  const skipCount = skips.length;
+  const totalCount = numTests;
+  const successCount = numTests - failCount - skipCount;
+  const nonSkippedCount = totalCount - skipCount;
+
+  if (skips.length) {
+    console.log('\x1b[2m%s\x1b[0m', '\nSkipped tests:');
+    skips.forEach(skip => {
+      console.log('\x1b[2m%s\x1b[0m', `● ${skip}`);
+    });
+  }
+
+  if (failCount > 0) {
+    console.log(
+      '\x1b[31m%s\x1b[0m',
+      `\n${failCount} of ${nonSkippedCount} tests failed${skipCount ? ` (${skipCount} skipped)` : ''}:\n`,
+    );
     fails.forEach(fail => {
-      console.log(`● ${fail}`);
+      console.log('\x1b[31m%s\x1b[0m', `● ${fail}`);
     });
     process.exit(1);
   } else {
-    console.log('All tests succeeded.');
+    console.log(
+      '\x1b[32m%s\x1b[0m',
+      `\nSuccessfully ran ${successCount} tests${skipCount ? ` (${skipCount} skipped)` : ''}.`,
+    );
+    console.log('\x1b[32m%s\x1b[0m', 'All tests succeeded.');
     process.exit(0);
   }
 });
@@ -90,15 +122,17 @@ Promise.all(workers).then(() => {
  * Suppress jest output for test suites where all tests were skipped.
  * This only clutters the logs and we can safely print a one-liner instead.
  */
-function checkSkippedAllTests(output: string, workerNumber: number, testPath: string | undefined): string {
-  const regex = /Tests:\s+(\d+) skipped, (\d+) total/gm;
+function checkSkippedAllTests(output: string): boolean {
+  const regex = /(.+)Tests:(.+)\s+(.+?)(\d+) skipped(.+), (\d+) total/gm;
   const matches = regex.exec(output);
+
   if (matches) {
-    const skipped = Number(matches[1]);
-    const total = Number(matches[2]);
+    const skipped = Number(matches[4]);
+    const total = Number(matches[6]);
     if (!isNaN(skipped) && !isNaN(total) && total === skipped) {
-      return `(Worker ${workerNumber}) > Skipped all (${total} tests) in ${testPath}`;
+      return true;
     }
   }
-  return output;
+
+  return false;
 }