Generalize integration tests runner

packages/nextjs/test/integration/test/client.js

@@ -1,182 +1,41 @@
-const fs = require('fs').promises;
-const { createServer } = require('http');
-const { parse } = require('url');
 const path = require('path');
-
-const yargs = require('yargs/yargs');
-const next = require('next');
 const puppeteer = require('puppeteer');
+const { run } = require('./runner');
+const { createNextServer, startServer } = require('./utils/common');
+const { createRequestInterceptor } = require('./utils/client');
 
-const {
-  colorize,
-  extractEnvelopeFromRequest,
-  extractEventFromRequest,
-  isEventRequest,
-  isSentryRequest,
-  isSessionRequest,
-  isTransactionRequest,
-  logIf,
-} = require('./utils');
-const { log } = console;
-
-/**
- * client.js
- *
- * Start the test-runner
- *
- * Options:
- *   --filter   Filter scenarios based on filename (case-insensitive)      [string]
- *   --silent   Hide all stdout and console logs except test results      [boolean]
- *   --debug    Log intercepted requests and debug messages               [boolean]
- *   --depth    Set the logging depth for intercepted requests             [number]
- */
-
-const argv = yargs(process.argv.slice(2))
-  .command('$0', 'Start the test-runner')
-  .option('filter', {
-    type: 'string',
-    description: 'Filter scenarios based on filename (case-insensitive)',
-  })
-  .option('silent', {
-    type: 'boolean',
-    description: 'Hide all stdout and console logs except test results',
-  })
-  .option('debug', {
-    type: 'boolean',
-    description: 'Log intercepted requests and debug messages',
-  })
-  .option('depth', {
-    type: 'number',
-    description: 'Set the logging depth for intercepted requests',
-  }).argv;
-
-(async () => {
-  let scenarios = await fs.readdir(path.resolve(__dirname, './client'));
-
-  if (argv.filter) {
-    scenarios = scenarios.filter(file => file.toLowerCase().includes(argv.filter));
-  }
-
-  if (scenarios.length === 0) {
-    log('No test suites found');
-    process.exit(0);
-  } else {
-    if (!argv.silent) {
-      scenarios.forEach(s => log(`⊙ Test suites found: ${s}`));
-    }
-  }
-
-  // Silence all the unnecessary server noise. We are capturing errors manualy anyway.
-  if (argv.silent) {
-    console.log = () => {};
-    console.error = () => {};
-  }
-
-  const app = next({ dev: false, dir: path.resolve(__dirname, '..') });
-  const handle = app.getRequestHandler();
-  await app.prepare();
-  const server = createServer((req, res) => handle(req, res, parse(req.url, true)));
-
+const setup = async () => {
+  const server = await createNextServer({ dev: false, dir: path.resolve(__dirname, '..') });
   const browser = await puppeteer.launch({
     devtools: false,
   });
-
-  const success = await new Promise(resolve => {
-    server.listen(0, err => {
-      if (err) throw err;
-
-      const cases = scenarios.map(async testCase => {
-        const page = await browser.newPage();
-        page.setDefaultTimeout(2000);
-
-        page.on('console', msg => logIf(argv.debug, msg.text()));
-
-        // Capturing requests this way allows us to have a reproducible order,
-        // where using `Promise.all([page.waitForRequest(isEventRequest), page.waitForRequest(isEventRequest)])` doesn't guarantee it.
-        const testInput = {
-          page,
-          url: `http://localhost:${server.address().port}`,
-          requests: {
-            events: [],
-            sessions: [],
-            transactions: [],
-          },
-        };
-
-        await page.setRequestInterception(true);
-        page.on('request', request => {
-          if (
-            isSentryRequest(request) ||
-            // Used for testing http tracing
-            request.url().includes('http://example.com')
-          ) {
-            request.respond({
-              status: 200,
-              contentType: 'application/json',
-              headers: {
-                'Access-Control-Allow-Origin': '*',
-              },
-            });
-          } else {
-            request.continue();
-          }
-
-          if (isEventRequest(request)) {
-            logIf(argv.debug, 'Intercepted Event', extractEventFromRequest(request), argv.depth);
-            testInput.requests.events.push(request);
-          }
-
-          if (isSessionRequest(request)) {
-            logIf(argv.debug, 'Intercepted Session', extractEnvelopeFromRequest(request), argv.depth);
-            testInput.requests.sessions.push(request);
-          }
-
-          if (isTransactionRequest(request)) {
-            logIf(argv.debug, 'Intercepted Transaction', extractEnvelopeFromRequest(request), argv.depth);
-            testInput.requests.transactions.push(request);
-          }
-        });
-
-        try {
-          await require(`./client/${testCase}`)(testInput);
-          log(colorize(`✓ Scenario succeded: ${testCase}`, 'green'));
-          return true;
-        } catch (error) {
-          const testCaseFrames = error.stack.split('\n').filter(l => l.includes(testCase));
-          if (testCaseFrames.length === 0) {
-            log(error);
-            return false;
-          }
-          /**
-           * Find first frame that matches our scenario filename and extract line number from it, eg.:
-           *
-           * at assertObjectMatches (/test/integration/test/utils.js:184:7)
-           * at module.exports.expectEvent (/test/integration/test/utils.js:122:10)
-           * at module.exports (/test/integration/test/client/errorGlobal.js:6:3)
-           */
-          const line = testCaseFrames[0].match(/.+:(\d+):/)[1];
-          log(colorize(`X Scenario failed: ${testCase} (line: ${line})`, 'red'));
-          log(error.message);
-          return false;
-        }
-      });
-
-      Promise.all(cases).then(result => {
-        // Awaiting server being correctly closed and resolving promise in it's callback
-        // adds ~4-5sec overhead for some reason. It should be safe to skip it though.
-        server.close();
-        resolve(result.every(Boolean));
-      });
-    });
-  });
-
-  await browser.close();
-
-  if (success) {
-    log(colorize(`✓ All scenarios succeded`, 'green'));
-    process.exit(0);
-  } else {
-    log(colorize(`X Some scenarios failed`, 'red'));
-    process.exit(1);
-  }
-})();
+  return startServer(server, { browser });
+};
+
+const teardown = async ({ browser, server }) => {
+  return Promise.all([browser.close(), new Promise(resolve => server.close(resolve))]);
+};
+
+const execute = async (scenario, env) => {
+  // Capturing requests this way allows us to have a reproducible, guaranteed order, as `Promise.all` does not do that.
+  // Eg. this won't be enough: `const [resp1, resp2] = Promise.all([page.waitForRequest(isEventRequest), page.waitForRequest(isEventRequest)])`
+  env.requests = {
+    events: [],
+    sessions: [],
+    transactions: [],
+  };
+
+  const page = (env.page = await env.browser.newPage());
+  await page.setRequestInterception(true);
+  page.setDefaultTimeout(2000);
+  page.on('request', createRequestInterceptor(env));
+
+  return scenario(env);
+};
+
+run({
+  setup,
+  teardown,
+  execute,
+  scenariosDir: path.resolve(__dirname, './client'),
+});

packages/nextjs/test/integration/test/client/errorClick.js

@@ -1,6 +1,8 @@
-const { expectRequestCount, waitForAll, isEventRequest, expectEvent } = require('../utils');
+const { waitForAll } = require('../utils/common');
+const { expectRequestCount, isEventRequest, expectEvent } = require('../utils/client');
 
 module.exports = async ({ page, url, requests }) => {
+  console.log(page, url, requests);
   await page.goto(`${url}/errorClick`);
 
   await waitForAll([page.click('button'), page.waitForRequest(isEventRequest)]);

packages/nextjs/test/integration/test/client/errorGlobal.js

@@ -1,4 +1,5 @@
-const { expectRequestCount, waitForAll, isEventRequest, expectEvent } = require('../utils');
+const { waitForAll } = require('../utils/common');
+const { expectRequestCount, isEventRequest, expectEvent } = require('../utils/client');
 
 module.exports = async ({ page, url, requests }) => {
   await waitForAll([page.goto(`${url}/crashed`), page.waitForRequest(isEventRequest)]);

packages/nextjs/test/integration/test/client/sessionCrashed.js

@@ -1,4 +1,5 @@
-const { expectRequestCount, waitForAll, isSessionRequest, expectSession } = require('../utils');
+const { waitForAll } = require('../utils/common');
+const { expectRequestCount, isSessionRequest, expectSession } = require('../utils/client');
 
 module.exports = async ({ page, url, requests }) => {
   await waitForAll([

packages/nextjs/test/integration/test/client/sessionHealthy.js

@@ -1,4 +1,5 @@
-const { expectRequestCount, waitForAll, expectSession, isSessionRequest } = require('../utils');
+const { waitForAll } = require('../utils/common');
+const { expectRequestCount, isSessionRequest, expectSession } = require('../utils/client');
 
 module.exports = async ({ page, url, requests }) => {
   await waitForAll([page.goto(`${url}/healthy`), page.waitForRequest(isSessionRequest)]);

packages/nextjs/test/integration/test/client/sessionNavigate.js

@@ -1,4 +1,5 @@
-const { expectRequestCount, waitForAll, isSessionRequest, expectSession, sleep } = require('../utils');
+const { sleep, waitForAll } = require('../utils/common');
+const { expectRequestCount, isSessionRequest, expectSession } = require('../utils/client');
 
 module.exports = async ({ page, url, requests }) => {
   await waitForAll([page.goto(`${url}/healthy`), page.waitForRequest(isSessionRequest)]);

packages/nextjs/test/integration/test/client/tracingDynamicRoute.js

@@ -1,4 +1,4 @@
-const { expectRequestCount, isTransactionRequest, expectTransaction } = require('../utils');
+const { expectRequestCount, isTransactionRequest, expectTransaction } = require('../utils/client');
 
 module.exports = async ({ page, url, requests }) => {
   await page.goto(`${url}/users/102`);

packages/nextjs/test/integration/test/client/tracingFetch.js

@@ -1,4 +1,4 @@
-const { expectRequestCount, expectTransaction, isTransactionRequest } = require('../utils');
+const { expectRequestCount, isTransactionRequest, expectTransaction } = require('../utils/client');
 
 module.exports = async ({ page, url, requests }) => {
   await page.goto(`${url}/fetch`);

packages/nextjs/test/integration/test/client/tracingNavigate.js

@@ -1,4 +1,5 @@
-const { expectRequestCount, expectTransaction, isTransactionRequest, sleep } = require('../utils');
+const { sleep } = require('../utils/common');
+const { expectRequestCount, isTransactionRequest, expectTransaction } = require('../utils/client');
 
 module.exports = async ({ page, url, requests }) => {
   await page.goto(`${url}/healthy`);

packages/nextjs/test/integration/test/client/tracingPageLoad.js

@@ -1,4 +1,4 @@
-const { expectRequestCount, expectTransaction, isTransactionRequest } = require('../utils');
+const { expectRequestCount, isTransactionRequest, expectTransaction } = require('../utils/client');
 
 module.exports = async ({ page, url, requests }) => {
   await page.goto(`${url}/healthy`);

packages/nextjs/test/integration/test/runner.js

@@ -0,0 +1,107 @@
+const fs = require('fs').promises;
+const path = require('path');
+
+const yargs = require('yargs/yargs');
+
+const { colorize, verifyDir } = require('./utils/common');
+const { error, log } = console;
+
+const argv = yargs(process.argv.slice(2))
+  .option('filter', {
+    type: 'string',
+    description: 'Filter scenarios based on filename (case-insensitive)',
+  })
+  .option('silent', {
+    type: 'boolean',
+    description: 'Hide all stdout and console logs except test results',
+  })
+  .option('debug', {
+    type: 'boolean',
+    description: 'Log intercepted requests and debug messages',
+  })
+  .option('depth', {
+    type: 'number',
+    description: 'Set the logging depth for intercepted requests',
+  }).argv;
+
+const runScenario = async (scenario, execute, env) => {
+  try {
+    await execute(require(scenario), { ...env });
+    log(colorize(`✓ Scenario succeded: ${path.basename(scenario)}`, 'green'));
+    return true;
+  } catch (error) {
+    const scenarioFrames = error.stack.split('\n').filter(l => l.includes(scenario));
+
+    if (scenarioFrames.length === 0) {
+      log(error);
+      return false;
+    }
+
+    /**
+     * Find first frame that matches our scenario filename and extract line number from it, eg.:
+     *
+     * at assertObjectMatches (/test/integration/test/utils.js:184:7)
+     * at module.exports.expectEvent (/test/integration/test/utils.js:122:10)
+     * at module.exports (/test/integration/test/client/errorGlobal.js:6:3)
+     */
+    const line = scenarioFrames[0].match(/.+:(\d+):/)[1];
+    log(colorize(`X Scenario failed: ${path.basename(scenario)} (line: ${line})`, 'red'));
+    log(error.message);
+    return false;
+  }
+};
+
+const runScenarios = async (scenarios, execute, env) => {
+  return Promise.all(scenarios.map(scenario => runScenario(scenario, execute, env)));
+};
+
+module.exports.run = async ({
+  setup = async () => {},
+  teardown = async () => {},
+  execute = async (scenario, env) => scenario(env),
+  scenariosDir,
+}) => {
+  try {
+    await verifyDir(scenariosDir);
+
+    let scenarios = await fs.readdir(scenariosDir);
+    if (argv.filter) {
+      scenarios = scenarios.filter(file => file.toLowerCase().includes(argv.filter));
+    }
+    scenarios = scenarios.map(s => path.resolve(scenariosDir, s));
+
+    if (scenarios.length === 0) {
+      log('No scenarios found');
+      process.exit(0);
+    } else {
+      if (!argv.silent) {
+        scenarios.forEach(s => log(`⊙ Scenario found: ${path.basename(s)}`));
+      }
+    }
+    // Silence all the unnecessary server noise. We are capturing errors manualy anyway.
+    if (argv.silent) {
+      for (const level of ['log', 'warn', 'info', 'error']) {
+        console[level] = () => {};
+      }
+    }
+
+    const env = {
+      argv,
+      ...(await setup({ argv })),
+    };
+    const results = await runScenarios(scenarios, execute, env);
+    const success = results.every(Boolean);
+    await teardown(env);
+
+    if (success) {
+      log(colorize(`✓ All scenarios succeded`, 'green'));
+      process.exit(0);
+    } else {
+      log(colorize(`X Some scenarios failed`, 'red'));
+      process.exit(1);
+    }
+  } catch (e) {
+    error(e.message);
+    process.exit(1);
+  }
+};