test(e2e): Add framework for canary testing

Luca Forstner · Luca Forstner · commit 185790caf0b1 · 2022-10-12T10:52:00.000Z
diff --git a/packages/e2e-tests/run.ts b/packages/e2e-tests/run.ts
@@ -135,156 +135,218 @@ type TestResult = {
   result: 'PASS' | 'FAIL' | 'TIMEOUT';
 };
 
+type VersionResult = {
+  dependencyOverrides?: Record<string, string>;
+  buildFailed: boolean;
+  testResults: TestResult[];
+};
+
 type RecipeResult = {
   testApplicationName: string;
   testApplicationPath: string;
-  buildFailed: boolean;
-  testResults: TestResult[];
+  versionResults: VersionResult[];
 };
 
-const recipeResults: RecipeResult[] = recipePaths.map(recipePath => {
-  type Recipe = {
-    testApplicationName: string;
-    buildCommand?: string;
-    buildTimeoutSeconds?: number;
-    tests: {
-      testName: string;
-      testCommand: string;
-      timeoutSeconds?: number;
-    }[];
-  };
+type Recipe = {
+  testApplicationName: string;
+  buildCommand?: string;
+  buildTimeoutSeconds?: number;
+  tests: {
+    testName: string;
+    testCommand: string;
+    timeoutSeconds?: number;
+  }[];
+  versions?: { dependencyOverrides: Record<string, string> }[];
+  canaryVersions?: { dependencyOverrides: Record<string, string> }[];
+};
 
+const recipeResults: RecipeResult[] = recipePaths.map(recipePath => {
   const recipe: Recipe = JSON.parse(fs.readFileSync(recipePath, 'utf-8'));
+  const recipeDirname = path.dirname(recipePath);
 
-  if (recipe.buildCommand) {
-    console.log(`Running E2E test build command for test application "${recipe.testApplicationName}"`);
-    const buildCommandProcess = childProcess.spawnSync(recipe.buildCommand, {
-      cwd: path.dirname(recipePath),
-      encoding: 'utf8',
-      shell: true, // needed so we can pass the build command in as whole without splitting it up into args
-      timeout: (recipe.buildTimeoutSeconds ?? DEFAULT_BUILD_TIMEOUT_SECONDS) * 1000,
-      env: {
-        ...process.env,
-        ...envVarsToInject,
-      },
-    });
-
-    // Prepends some text to the output build command's output so we can distinguish it from logging in this script
-    console.log(buildCommandProcess.stdout.replace(/^/gm, '[BUILD OUTPUT] '));
-    console.log(buildCommandProcess.stderr.replace(/^/gm, '[BUILD OUTPUT] '));
+  function runRecipe(dependencyOverrides: Record<string, string> | undefined): VersionResult {
+    const dependencyOverridesInformationString = dependencyOverrides
+      ? ` (Dependency overrides: ${JSON.stringify(dependencyOverrides)})`
+      : '';
 
-    const error: undefined | (Error & { code?: string }) = buildCommandProcess.error;
-
-    if (error?.code === 'ETIMEDOUT') {
-      processShouldExitWithError = true;
-
-      printCIErrorMessage(
-        `Build command in test application "${recipe.testApplicationName}" (${path.dirname(recipePath)}) timed out!`,
-      );
-
-      return {
-        testApplicationName: recipe.testApplicationName,
-        testApplicationPath: recipePath,
-        buildFailed: true,
-        testResults: [],
-      };
-    } else if (buildCommandProcess.status !== 0) {
-      processShouldExitWithError = true;
-
-      printCIErrorMessage(
-        `Build command in test application "${recipe.testApplicationName}" (${path.dirname(recipePath)}) failed!`,
+    if (recipe.buildCommand) {
+      console.log(
+        `Running E2E test build command for test application "${recipe.testApplicationName}"${dependencyOverridesInformationString}`,
       );
-
-      return {
-        testApplicationName: recipe.testApplicationName,
-        testApplicationPath: recipePath,
-        buildFailed: true,
-        testResults: [],
-      };
+      const buildCommandProcess = childProcess.spawnSync(recipe.buildCommand, {
+        cwd: path.dirname(recipePath),
+        encoding: 'utf8',
+        shell: true, // needed so we can pass the build command in as whole without splitting it up into args
+        timeout: (recipe.buildTimeoutSeconds ?? DEFAULT_BUILD_TIMEOUT_SECONDS) * 1000,
+        env: {
+          ...process.env,
+          ...envVarsToInject,
+        },
+      });
+
+      // Prepends some text to the output build command's output so we can distinguish it from logging in this script
+      console.log(buildCommandProcess.stdout.replace(/^/gm, '[BUILD OUTPUT] '));
+      console.log(buildCommandProcess.stderr.replace(/^/gm, '[BUILD OUTPUT] '));
+
+      const error: undefined | (Error & { code?: string }) = buildCommandProcess.error;
+
+      if (error?.code === 'ETIMEDOUT') {
+        processShouldExitWithError = true;
+
+        printCIErrorMessage(
+          `Build command in test application "${recipe.testApplicationName}" (${path.dirname(recipePath)}) timed out!`,
+        );
+
+        return {
+          dependencyOverrides,
+          buildFailed: true,
+          testResults: [],
+        };
+      } else if (buildCommandProcess.status !== 0) {
+        processShouldExitWithError = true;
+
+        printCIErrorMessage(
+          `Build command in test application "${recipe.testApplicationName}" (${path.dirname(recipePath)}) failed!`,
+        );
+
+        return {
+          dependencyOverrides,
+          buildFailed: true,
+          testResults: [],
+        };
+      }
     }
-  }
 
-  const testResults: TestResult[] = recipe.tests.map(test => {
-    console.log(
-      `Running E2E test command for test application "${recipe.testApplicationName}", test "${test.testName}"`,
-    );
+    const testResults: TestResult[] = recipe.tests.map(test => {
+      console.log(
+        `Running E2E test command for test application "${recipe.testApplicationName}", test "${test.testName}"${dependencyOverridesInformationString}`,
+      );
 
-    const testProcessResult = childProcess.spawnSync(test.testCommand, {
-      cwd: path.dirname(recipePath),
-      timeout: (test.timeoutSeconds ?? DEFAULT_TEST_TIMEOUT_SECONDS) * 1000,
-      encoding: 'utf8',
-      shell: true, // needed so we can pass the test command in as whole without splitting it up into args
-      env: {
-        ...process.env,
-        ...envVarsToInject,
-      },
+      const testProcessResult = childProcess.spawnSync(test.testCommand, {
+        cwd: path.dirname(recipePath),
+        timeout: (test.timeoutSeconds ?? DEFAULT_TEST_TIMEOUT_SECONDS) * 1000,
+        encoding: 'utf8',
+        shell: true, // needed so we can pass the test command in as whole without splitting it up into args
+        env: {
+          ...process.env,
+          ...envVarsToInject,
+        },
+      });
+
+      // Prepends some text to the output test command's output so we can distinguish it from logging in this script
+      console.log(testProcessResult.stdout.replace(/^/gm, '[TEST OUTPUT] '));
+      console.log(testProcessResult.stderr.replace(/^/gm, '[TEST OUTPUT] '));
+
+      const error: undefined | (Error & { code?: string }) = testProcessResult.error;
+
+      if (error?.code === 'ETIMEDOUT') {
+        processShouldExitWithError = true;
+        printCIErrorMessage(
+          `Test "${test.testName}" in test application "${recipe.testApplicationName}" (${path.dirname(
+            recipePath,
+          )}) timed out.`,
+        );
+        return {
+          testName: test.testName,
+          result: 'TIMEOUT',
+        };
+      } else if (testProcessResult.status !== 0) {
+        processShouldExitWithError = true;
+        printCIErrorMessage(
+          `Test "${test.testName}" in test application "${recipe.testApplicationName}" (${path.dirname(
+            recipePath,
+          )}) failed.`,
+        );
+        return {
+          testName: test.testName,
+          result: 'FAIL',
+        };
+      } else {
+        console.log(
+          `Test "${test.testName}" in test application "${recipe.testApplicationName}" (${path.dirname(
+            recipePath,
+          )}) succeeded.`,
+        );
+        return {
+          testName: test.testName,
+          result: 'PASS',
+        };
+      }
     });
 
-    // Prepends some text to the output test command's output so we can distinguish it from logging in this script
-    console.log(testProcessResult.stdout.replace(/^/gm, '[TEST OUTPUT] '));
-    console.log(testProcessResult.stderr.replace(/^/gm, '[TEST OUTPUT] '));
+    return {
+      dependencyOverrides,
+      buildFailed: false,
+      testResults,
+    };
+  }
 
-    const error: undefined | (Error & { code?: string }) = testProcessResult.error;
+  const versionsToRun: {
+    dependencyOverrides?: Record<string, string>;
+  }[] = (process.env.CANARY_E2E_TEST ? recipe.canaryVersions : recipe.versions) ?? [{}];
 
-    if (error?.code === 'ETIMEDOUT') {
-      processShouldExitWithError = true;
-      printCIErrorMessage(
-        `Test "${test.testName}" in test application "${recipe.testApplicationName}" (${path.dirname(
-          recipePath,
-        )}) timed out.`,
-      );
-      return {
-        testName: test.testName,
-        result: 'TIMEOUT',
-      };
-    } else if (testProcessResult.status !== 0) {
-      processShouldExitWithError = true;
-      printCIErrorMessage(
-        `Test "${test.testName}" in test application "${recipe.testApplicationName}" (${path.dirname(
-          recipePath,
-        )}) failed.`,
-      );
-      return {
-        testName: test.testName,
-        result: 'FAIL',
-      };
-    } else {
-      console.log(
-        `Test "${test.testName}" in test application "${recipe.testApplicationName}" (${path.dirname(
-          recipePath,
-        )}) succeeded.`,
+  const versionResults = versionsToRun.map(({ dependencyOverrides }) => {
+    if (dependencyOverrides) {
+      // Back up original package.json
+      fs.copyFileSync(path.resolve(recipeDirname, 'package.json'), path.resolve(recipeDirname, 'package.json.bak'));
+
+      // Override dependencies
+      const packageJson: { dependencies?: Record<string, string> } = JSON.parse(
+        fs.readFileSync(path.resolve(recipeDirname, 'package.json'), { encoding: 'utf-8' }),
       );
-      return {
-        testName: test.testName,
-        result: 'PASS',
-      };
+      packageJson.dependencies = packageJson.dependencies
+        ? { ...packageJson.dependencies, ...dependencyOverrides }
+        : dependencyOverrides;
+      fs.writeFileSync(path.resolve(recipeDirname, 'package.json'), JSON.stringify(packageJson, null, 2), {
+        encoding: 'utf-8',
+      });
+    }
+
+    try {
+      return runRecipe(dependencyOverrides);
+    } finally {
+      if (dependencyOverrides) {
+        // Restore original package.json
+        fs.rmSync(path.resolve(recipeDirname, 'package.json'), { force: true });
+        fs.copyFileSync(path.resolve(recipeDirname, 'package.json.bak'), path.resolve(recipeDirname, 'package.json'));
+        fs.rmSync(path.resolve(recipeDirname, 'package.json.bak'), { force: true });
+      }
     }
   });
 
   return {
     testApplicationName: recipe.testApplicationName,
     testApplicationPath: recipePath,
-    buildFailed: false,
-    testResults,
+    versionResults,
   };
 });
 
 console.log('--------------------------------------');
 console.log('Test Result Summary:');
 
 recipeResults.forEach(recipeResult => {
-  if (recipeResult.buildFailed) {
-    console.log(
-      `● BUILD FAILED - ${recipeResult.testApplicationName} (${path.dirname(recipeResult.testApplicationPath)})`,
-    );
-  } else {
-    console.log(
-      `● BUILD SUCCEEDED - ${recipeResult.testApplicationName} (${path.dirname(recipeResult.testApplicationPath)})`,
-    );
-    recipeResult.testResults.forEach(testResult => {
-      console.log(`  ● ${testResult.result.padEnd(7, ' ')} ${testResult.testName}`);
-    });
-  }
+  recipeResult.versionResults.forEach(versionResult => {
+    const dependencyOverridesInformationString = versionResult.dependencyOverrides
+      ? ` (Dependency overrides: ${JSON.stringify(versionResult.dependencyOverrides)})`
+      : '';
+
+    if (versionResult.buildFailed) {
+      console.log(
+        `● BUILD FAILED - ${recipeResult.testApplicationName} (${path.dirname(
+          recipeResult.testApplicationPath,
+        )})${dependencyOverridesInformationString}`,
+      );
+    } else {
+      console.log(
+        `● BUILD SUCCEEDED - ${recipeResult.testApplicationName} (${path.dirname(
+          recipeResult.testApplicationPath,
+        )})${dependencyOverridesInformationString}`,
+      );
+      versionResult.testResults.forEach(testResult => {
+        console.log(`  ● ${testResult.result.padEnd(7, ' ')} ${testResult.testName}`);
+      });
+    }
+  });
 });
 
 groupCIOutput('Cleanup', () => {
diff --git a/packages/e2e-tests/test-applications/standard-frontend-react/test-recipe.json b/packages/e2e-tests/test-applications/standard-frontend-react/test-recipe.json
@@ -7,5 +7,33 @@
       "testName": "Playwright tests",
       "testCommand": "yarn test"
     }
+  ],
+  "versions": [
+    {
+      "dependencyOverrides": {
+        "react": "^18",
+        "react-dom": "^18"
+      }
+    },
+    {
+      "dependencyOverrides": {
+        "react": "^17",
+        "react-dom": "^17"
+      }
+    },
+    {
+      "dependencyOverrides": {
+        "react": "^16",
+        "react-dom": "^16"
+      }
+    }
+  ],
+  "canaryVersions": [
+    {
+      "dependencyOverrides": {
+        "react": "latest",
+        "react-dom": "latest"
+      }
+    }
   ]
 }
