test: Make E2E tests async & parallelizable

.github/workflows/build.yml

@@ -185,7 +185,7 @@ jobs:
     name: Build
     needs: [job_get_metadata, job_install_deps]
     runs-on: ubuntu-20.04
-    timeout-minutes: 20
+    timeout-minutes: 30
     steps:
       - name: Check out current commit (${{ needs.job_get_metadata.outputs.commit_label }})
         uses: actions/checkout@v3

packages/e2e-tests/.eslintrc.js

@@ -3,7 +3,7 @@ module.exports = {
     node: true,
   },
   extends: ['../../.eslintrc.js'],
-  ignorePatterns: ['test-applications/**'],
+  ignorePatterns: ['test-applications/**', 'tmp/**'],
   parserOptions: {
     sourceType: 'module',
   },

packages/e2e-tests/.gitignore

@@ -1 +1,2 @@
 .env
+tmp

packages/e2e-tests/README.md

@@ -54,7 +54,7 @@ To get you started with the recipe, you can copy the following into `test-recipe
 {
   "$schema": "../../test-recipe-schema.json",
   "testApplicationName": "My New Test Application",
-  "buildCommand": "yarn install --pure-lockfile",
+  "buildCommand": "yarn install --network-concurrency 1",
   "tests": [
     {
       "testName": "My new test",

packages/e2e-tests/lib/buildApp.ts

@@ -0,0 +1,80 @@
+/* eslint-disable no-console */
+
+import * as fs from 'fs-extra';
+import * as path from 'path';
+
+import { DEFAULT_BUILD_TIMEOUT_SECONDS } from './constants';
+import type { Env, RecipeInstance } from './types';
+import { spawnAsync } from './utils';
+
+export async function buildApp(appDir: string, recipeInstance: RecipeInstance, env: Env): Promise<void> {
+  const { recipe, label, dependencyOverrides } = recipeInstance;
+
+  const packageJsonPath = path.resolve(appDir, 'package.json');
+
+  if (dependencyOverrides) {
+    // Override dependencies
+    const packageJson: { dependencies?: Record<string, string> } = JSON.parse(
+      fs.readFileSync(packageJsonPath, { encoding: 'utf-8' }),
+    );
+    packageJson.dependencies = packageJson.dependencies
+      ? { ...packageJson.dependencies, ...dependencyOverrides }
+      : dependencyOverrides;
+    fs.writeFileSync(packageJsonPath, JSON.stringify(packageJson, null, 2), {
+      encoding: 'utf-8',
+    });
+  }
+
+  if (recipe.buildCommand) {
+    console.log(`Running build command for test application "${label}"`);
+
+    const buildResult = await spawnAsync(recipe.buildCommand, {
+      cwd: appDir,
+      timeout: (recipe.buildTimeoutSeconds ?? DEFAULT_BUILD_TIMEOUT_SECONDS) * 1000,
+      env: {
+        ...process.env,
+        ...env,
+      } as unknown as NodeJS.ProcessEnv,
+    });
+
+    if (buildResult.error) {
+      console.log(`Build failed for test application "${label}"`);
+
+      // Prepends some text to the output build command's output so we can distinguish it from logging in this script
+      console.log(buildResult.stdout.replace(/^/gm, '  [BUILD OUTPUT] '));
+      console.log(buildResult.stderr.replace(/^/gm, '  [BUILD OUTPUT] '));
+
+      console.log('[BUILD ERROR] ', buildResult.error);
+      throw buildResult.error;
+    }
+
+    if (recipe.buildAssertionCommand) {
+      console.log(`Running build assertion for test application "${label}"`);
+
+      const buildAssertionResult = await spawnAsync(
+        recipe.buildAssertionCommand,
+        {
+          cwd: appDir,
+          timeout: (recipe.buildTimeoutSeconds ?? DEFAULT_BUILD_TIMEOUT_SECONDS) * 1000,
+          env: {
+            ...process.env,
+            ...env,
+          } as unknown as NodeJS.ProcessEnv,
+        },
+        buildResult.stdout,
+      );
+
+      if (buildAssertionResult.error) {
+        console.log(`Build assertion failed for test application "${label}"`);
+
+        // Prepends some text to the output build command's output so we can distinguish it from logging in this script
+        console.log(buildAssertionResult.stdout.replace(/^/gm, '  [BUILD ASSERTION OUTPUT] '));
+        console.log(buildAssertionResult.stderr.replace(/^/gm, '  [BUILD ASSERTION OUTPUT] '));
+
+        console.log('[BUILD ASSERTION ERROR] ', buildAssertionResult.error);
+
+        throw buildAssertionResult.error;
+      }
+    }
+  }
+}

packages/e2e-tests/lib/buildRecipeInstances.ts

@@ -0,0 +1,47 @@
+import * as fs from 'fs';
+
+import type { Recipe, RecipeInput, RecipeInstance } from './types';
+
+export function buildRecipeInstances(recipePaths: string[]): RecipeInstance[] {
+  const recipes = buildRecipes(recipePaths);
+  const recipeInstances: RecipeInstance[] = [];
+
+  const basePort = 3001;
+
+  recipes.forEach((recipe, i) => {
+    recipe.versions.forEach(version => {
+      const dependencyOverrides =
+        Object.keys(version.dependencyOverrides).length > 0 ? version.dependencyOverrides : undefined;
+      const dependencyOverridesInformationString = dependencyOverrides
+        ? ` (Dependency overrides: ${JSON.stringify(dependencyOverrides)})`
+        : '';
+
+      recipeInstances.push({
+        label: `${recipe.testApplicationName}${dependencyOverridesInformationString}`,
+        recipe,
+        dependencyOverrides,
+        port: basePort + i,
+      });
+    });
+  });
+
+  return recipeInstances;
+}
+
+function buildRecipes(recipePaths: string[]): Recipe[] {
+  return recipePaths.map(recipePath => buildRecipe(recipePath));
+}
+
+function buildRecipe(recipePath: string): Recipe {
+  const recipe: RecipeInput = JSON.parse(fs.readFileSync(recipePath, 'utf-8'));
+
+  const versions = process.env.CANARY_E2E_TEST
+    ? recipe.canaryVersions ?? []
+    : recipe.versions ?? [{ dependencyOverrides: {} }];
+
+  return {
+    ...recipe,
+    path: recipePath,
+    versions,
+  };
+}

packages/e2e-tests/lib/constants.ts

@@ -0,0 +1,6 @@
+export const TEST_REGISTRY_CONTAINER_NAME = 'verdaccio-e2e-test-registry';
+export const DEFAULT_BUILD_TIMEOUT_SECONDS = 60 * 5;
+export const DEFAULT_TEST_TIMEOUT_SECONDS = 60 * 2;
+export const VERDACCIO_VERSION = '5.22.1';
+export const PUBLISH_PACKAGES_DOCKER_IMAGE_NAME = 'publish-packages';
+export const TMP_DIR = 'tmp';

packages/e2e-tests/lib/runAllTestApps.ts

@@ -0,0 +1,84 @@
+/* eslint-disable no-console */
+import { buildRecipeInstances } from './buildRecipeInstances';
+import { buildAndTestApp } from './runTestApp';
+import type { RecipeInstance, RecipeTestResult } from './types';
+
+export async function runAllTestApps(
+  recipePaths: string[],
+  envVarsToInject: Record<string, string | undefined>,
+): Promise<void> {
+  const maxParallel = process.env.CI ? 2 : 5;
+
+  const recipeInstances = buildRecipeInstances(recipePaths);
+
+  const results = await shardPromises(
+    recipeInstances,
+    recipeInstance => buildAndTestApp(recipeInstance, envVarsToInject),
+    maxParallel,
+  );
+
+  console.log('--------------------------------------');
+  console.log('Test Result Summary:');
+
+  results.forEach(result => {
+    if (result.buildFailed) {
+      console.log(`● BUILD FAILED - ${result.label} (${result.recipe.path}`);
+    } else {
+      console.log(`● BUILD SUCCEEDED - ${result.label}`);
+      result.tests.forEach(testResult => {
+        console.log(`  ● ${testResult.result.padEnd(7, ' ')} ${testResult.testName}`);
+      });
+    }
+  });
+
+  const failed = results.filter(result => result.buildFailed || result.testFailed);
+
+  if (failed.length) {
+    console.log(`${failed.length} test(s) failed.`);
+    process.exit(1);
+  }
+
+  console.log('All tests succeeded. 🎉');
+}
+
+// Always run X promises at a time
+function shardPromises(
+  recipes: RecipeInstance[],
+  callback: (recipe: RecipeInstance) => Promise<RecipeTestResult>,
+  maxParallel: number,
+): Promise<RecipeTestResult[]> {
+  return new Promise(resolve => {
+    console.log(`Running a total of ${recipes.length} jobs, with up to ${maxParallel} jobs in parallel...`);
+    const results: RecipeTestResult[] = [];
+    const remaining = recipes.slice();
+    const running: Promise<unknown>[] = [];
+
+    function runNext(): void {
+      if (running.length < maxParallel && remaining.length > 0) {
+        // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+        const next = remaining.shift()!;
+        const promise = callback(next);
+
+        console.log(`Running job ${next.label}, ${remaining.length} remaining...`);
+
+        running.push(promise);
+
+        promise
+          .then(result => results.push(result))
+          .finally(() => {
+            const pos = running.indexOf(promise);
+            running.splice(pos, 1);
+
+            runNext();
+          });
+      } else if (remaining.length === 0 && running.length === 0) {
+        resolve(results);
+      }
+    }
+
+    // Initial runs
+    for (let i = 0; i < maxParallel; i++) {
+      runNext();
+    }
+  });
+}

packages/e2e-tests/lib/runTestApp.ts

@@ -0,0 +1,55 @@
+/* eslint-disable no-console */
+
+import * as fs from 'fs-extra';
+import * as path from 'path';
+
+import { buildApp } from './buildApp';
+import { TMP_DIR } from './constants';
+import { testApp } from './testApp';
+import type { Env, RecipeInstance, RecipeTestResult } from './types';
+
+let tmpDirCount = 0;
+
+// This should never throw, we always return a result here
+export async function buildAndTestApp(
+  recipeInstance: RecipeInstance,
+  envVarsToInject: Record<string, string | undefined>,
+): Promise<RecipeTestResult> {
+  const { recipe, port } = recipeInstance;
+  const recipeDirname = path.dirname(recipe.path);
+
+  const targetDir = path.join(TMP_DIR, `${recipe.testApplicationName}-${tmpDirCount++}`);
+
+  await fs.copy(recipeDirname, targetDir);
+
+  const env: Env = {
+    ...envVarsToInject,
+    PORT: port.toString(),
+  };
+
+  try {
+    await buildApp(targetDir, recipeInstance, env);
+  } catch (error) {
+    await fs.remove(targetDir);
+
+    return {
+      ...recipeInstance,
+      buildFailed: true,
+      testFailed: false,
+      tests: [],
+    };
+  }
+
+  // This cannot throw, we always return a result here
+  const results = await testApp(targetDir, recipeInstance, env);
+
+  // Cleanup
+  await fs.remove(targetDir);
+
+  return {
+    ...recipeInstance,
+    buildFailed: false,
+    testFailed: results.some(result => result.result !== 'PASS'),
+    tests: results,
+  };
+}

packages/e2e-tests/lib/testApp.ts

@@ -0,0 +1,50 @@
+/* eslint-disable no-console */
+
+import { DEFAULT_TEST_TIMEOUT_SECONDS } from './constants';
+import type { Env, RecipeInstance, TestDef, TestResult } from './types';
+import { spawnAsync } from './utils';
+
+export async function testApp(appDir: string, recipeInstance: RecipeInstance, env: Env): Promise<TestResult[]> {
+  const { recipe } = recipeInstance;
+
+  const results: TestResult[] = [];
+  for (const test of recipe.tests) {
+    results.push(await runTest(appDir, recipeInstance, test, env));
+  }
+
+  return results;
+}
+
+async function runTest(appDir: string, recipeInstance: RecipeInstance, test: TestDef, env: Env): Promise<TestResult> {
+  const { recipe, label } = recipeInstance;
+  console.log(`Running test command for test application "${label}", test "${test.testName}"`);
+
+  const testResult = await spawnAsync(test.testCommand, {
+    cwd: appDir,
+    timeout: (recipe.testTimeoutSeconds ?? DEFAULT_TEST_TIMEOUT_SECONDS) * 1000,
+    env: {
+      ...process.env,
+      ...env,
+    } as unknown as NodeJS.ProcessEnv,
+  });
+
+  if (testResult.error) {
+    console.log(`Test failed for test application "${label}", test "${test.testName}"`);
+
+    // Prepends some text to the output test command's output so we can distinguish it from logging in this script
+    console.log(testResult.stdout.replace(/^/gm, '  [TEST OUTPUT] '));
+    console.log(testResult.stderr.replace(/^/gm, '  [TEST OUTPUT] '));
+
+    console.log('[TEST ERROR] ', testResult.error);
+
+    return {
+      testName: test.testName,
+      result: testResult.error?.message.includes('ETDIMEDOUT') ? 'TIMEOUT' : 'FAIL',
+    };
+  }
+
+  return {
+    testName: test.testName,
+    result: 'PASS',
+  };
+}