assert on build

Author: Luca Forstner

packages/e2e-tests/run.ts

@@ -174,6 +174,7 @@ type RecipeResult = {
 type Recipe = {
   testApplicationName: string;
   buildCommand?: string;
+  buildAssertionCommand?: string;
   buildTimeoutSeconds?: number;
   tests: {
     testName: string;
@@ -212,9 +213,9 @@ const recipeResults: RecipeResult[] = recipePaths.map(recipePath => {
       console.log(buildCommandProcess.stdout.replace(/^/gm, '[BUILD OUTPUT] '));
       console.log(buildCommandProcess.stderr.replace(/^/gm, '[BUILD OUTPUT] '));
 
-      const error: undefined | (Error & { code?: string }) = buildCommandProcess.error;
+      const buildCommandProcessError: undefined | (Error & { code?: string }) = buildCommandProcess.error;
 
-      if (error?.code === 'ETIMEDOUT') {
+      if (buildCommandProcessError?.code === 'ETIMEDOUT') {
         processShouldExitWithError = true;
 
         printCIErrorMessage(
@@ -239,6 +240,58 @@ const recipeResults: RecipeResult[] = recipePaths.map(recipePath => {
           testResults: [],
         };
       }
+
+      if (recipe.buildAssertionCommand) {
+        console.log(
+          `Running E2E test build assertion for test application "${recipe.testApplicationName}"${dependencyOverridesInformationString}`,
+        );
+        const buildAssertionCommandProcess = childProcess.spawnSync(recipe.buildAssertionCommand, {
+          cwd: path.dirname(recipePath),
+          input: buildCommandProcess.stdout,
+          encoding: 'utf8',
+          shell: true, // needed so we can pass the build command in as whole without splitting it up into args
+          timeout: (recipe.buildTimeoutSeconds ?? DEFAULT_BUILD_TIMEOUT_SECONDS) * 1000,
+          env: {
+            ...process.env,
+            ...envVarsToInject,
+          },
+        });
+
+        // Prepends some text to the output build command's output so we can distinguish it from logging in this script
+        console.log(buildAssertionCommandProcess.stdout.replace(/^/gm, '[BUILD ASSERTION OUTPUT] '));
+        console.log(buildAssertionCommandProcess.stderr.replace(/^/gm, '[BUILD ASSERTION OUTPUT] '));
+
+        const buildAssertionCommandProcessError: undefined | (Error & { code?: string }) =
+          buildAssertionCommandProcess.error;
+
+        if (buildAssertionCommandProcessError?.code === 'ETIMEDOUT') {
+          processShouldExitWithError = true;
+
+          printCIErrorMessage(
+            `Build assertion in test application "${recipe.testApplicationName}" (${path.dirname(
+              recipePath,
+            )}) timed out!`,
+          );
+
+          return {
+            dependencyOverrides,
+            buildFailed: true,
+            testResults: [],
+          };
+        } else if (buildAssertionCommandProcess.status !== 0) {
+          processShouldExitWithError = true;
+
+          printCIErrorMessage(
+            `Build assertion in test application "${recipe.testApplicationName}" (${path.dirname(recipePath)}) failed!`,
+          );
+
+          return {
+            dependencyOverrides,
+            buildFailed: true,
+            testResults: [],
+          };
+        }
+      }
     }
 
     const testResults: TestResult[] = recipe.tests.map(test => {

packages/e2e-tests/test-applications/nextjs-app-dir/assert-build.ts

@@ -0,0 +1,16 @@
+import * as fs from 'fs';
+import * as assert from 'assert/strict';
+
+const stdin = fs.readFileSync(0).toString();
+
+// Assert that all static components stay static and ally dynamic components stay dynamic
+
+assert.match(stdin, /○ \/client-component/);
+assert.match(stdin, /● \/client-component\/parameter\/\[\.\.\.parameters\]/);
+assert.match(stdin, /● \/client-component\/parameter\/\[parameter\]/);
+
+assert.match(stdin, /λ \/server-component/);
+assert.match(stdin, /λ \/server-component\/parameter\/\[\.\.\.parameters\]/);
+assert.match(stdin, /λ \/server-component\/parameter\/\[parameter\]/);
+
+export {};

packages/e2e-tests/test-applications/nextjs-app-dir/test-recipe.json

@@ -2,6 +2,7 @@
   "$schema": "../../test-recipe-schema.json",
   "testApplicationName": "nextjs-13-app-dir",
   "buildCommand": "yarn install --pure-lockfile && npx playwright install && yarn build",
+  "buildAssertionCommand": "yarn ts-node --script-mode assert-build.ts",
   "tests": [
     {
       "testName": "Prod Mode",

packages/e2e-tests/test-applications/nextjs-app-dir/tests/devErrorSymbolification.test.ts

@@ -21,15 +21,15 @@ test.describe('dev mode error symbolification', () => {
 
     expect(errorEventFrames?.[errorEventFrames?.length - 1]).toEqual(
       expect.objectContaining({
+        function: 'onClick',
         filename: 'components/client-error-debug-tools.tsx',
         abs_path: 'webpack-internal:///(app-client)/./components/client-error-debug-tools.tsx',
-        function: 'onClick',
-        in_app: true,
-        lineno: 32,
+        lineno: 54,
         colno: 16,
-        post_context: ['         }}', '       >', '         Throw error'],
-        context_line: "           throw new Error('Click Error');",
+        in_app: true,
         pre_context: ['       <button', '         onClick={() => {'],
+        context_line: "           throw new Error('Click Error');",
+        post_context: ['         }}', '       >', '         Throw error'],
       }),
     );
   });

packages/e2e-tests/test-recipe-schema.json

@@ -11,6 +11,10 @@
       "type": "string",
       "description": "Command that is run to install dependencies and build the test application. This command is only run once before all tests. Working directory of the command is the root of the test application."
     },
+    "buildAssertionCommand": {
+      "type": "string",
+      "description": "Command to verify build output. This command will be run after the build is complete. The command will receive the STDOUT of the `buildCommand` as STDIN."
+    },
     "buildTimeoutSeconds": {
       "type": "number",
       "description": "Timeout for the build command in seconds. Default: 60"