test(e2e): Add behaviour test for Errors in standard React E2E tests application (#5909)

Author: Luca Forstner

.github/workflows/build.yml

@@ -648,6 +648,8 @@ jobs:
       - name: Run E2E tests
         env:
           E2E_TEST_PUBLISH_SCRIPT_NODE_VERSION: ${{ env.DEFAULT_NODE_VERSION }}
+          E2E_TEST_AUTH_TOKEN: ${{ secrets.E2E_TEST_AUTH_TOKEN }}
+          E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }}
         run: |
           cd packages/e2e-tests
           yarn test:e2e

packages/e2e-tests/.env.example

@@ -0,0 +1,2 @@
+E2E_TEST_AUTH_TOKEN=
+E2E_TEST_DSN=

packages/e2e-tests/.gitignore

@@ -0,0 +1 @@
+.env

packages/e2e-tests/package.json

@@ -13,10 +13,10 @@
     "lint": "run-s lint:prettier lint:eslint",
     "lint:eslint": "eslint . --cache --cache-location '../../eslintcache/' --format stylish",
     "lint:prettier": "prettier --config ../../.prettierrc.json --check .",
-    "test:e2e": "run-s test:validate-configuration test:test-app-setups test:run",
+    "test:e2e": "run-s test:validate-configuration test:validate-test-app-setups test:run",
     "test:run": "ts-node run.ts",
     "test:validate-configuration": "ts-node validate-verdaccio-configuration.ts",
-    "test:test-app-setups": "ts-node validate-test-app-setups.ts"
+    "test:validate-test-app-setups": "ts-node validate-test-app-setups.ts"
   },
   "devDependencies": {
     "@types/glob": "8.0.0",

packages/e2e-tests/run.ts

@@ -13,9 +13,27 @@ const PUBLISH_PACKAGES_DOCKER_IMAGE_NAME = 'publish-packages';
 
 const publishScriptNodeVersion = process.env.E2E_TEST_PUBLISH_SCRIPT_NODE_VERSION;
 
-const DEFAULT_BUILD_TIMEOUT_SECONDS = 60;
+const DEFAULT_BUILD_TIMEOUT_SECONDS = 60 * 5;
 const DEFAULT_TEST_TIMEOUT_SECONDS = 60;
 
+if (!process.env.E2E_TEST_AUTH_TOKEN) {
+  console.log(
+    "No auth token configured! Please configure the E2E_TEST_AUTH_TOKEN environment variable with an auth token that has the scope 'project:read'!",
+  );
+}
+
+if (!process.env.E2E_TEST_DSN) {
+  console.log('No DSN configured! Please configure the E2E_TEST_DSN environment variable with a DSN!');
+}
+
+if (!process.env.E2E_TEST_AUTH_TOKEN || !process.env.E2E_TEST_DSN) {
+  process.exit(1);
+}
+
+const envVarsToInject = {
+  REACT_APP_E2E_TEST_DSN: process.env.E2E_TEST_DSN,
+};
+
 // https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#grouping-log-lines
 function groupCIOutput(groupTitle: string, fn: () => void): void {
   if (process.env.CI) {
@@ -145,13 +163,32 @@ const recipeResults: RecipeResult[] = recipePaths.map(recipePath => {
       encoding: 'utf8',
       shell: true, // needed so we can pass the build command in as whole without splitting it up into args
       timeout: (recipe.buildTimeoutSeconds ?? DEFAULT_BUILD_TIMEOUT_SECONDS) * 1000,
+      env: {
+        ...process.env,
+        ...envVarsToInject,
+      },
     });
 
     // Prepends some text to the output build command's output so we can distinguish it from logging in this script
     console.log(buildCommandProcess.stdout.replace(/^/gm, '[BUILD OUTPUT] '));
     console.log(buildCommandProcess.stderr.replace(/^/gm, '[BUILD OUTPUT] '));
 
-    if (buildCommandProcess.status !== 0) {
+    const error: undefined | (Error & { code?: string }) = buildCommandProcess.error;
+
+    if (error?.code === 'ETIMEDOUT') {
+      processShouldExitWithError = true;
+
+      printCIErrorMessage(
+        `Build command in test application "${recipe.testApplicationName}" (${path.dirname(recipePath)}) timed out!`,
+      );
+
+      return {
+        testApplicationName: recipe.testApplicationName,
+        testApplicationPath: recipePath,
+        buildFailed: true,
+        testResults: [],
+      };
+    } else if (buildCommandProcess.status !== 0) {
       processShouldExitWithError = true;
 
       printCIErrorMessage(
@@ -177,6 +214,10 @@ const recipeResults: RecipeResult[] = recipePaths.map(recipePath => {
       timeout: (test.timeoutSeconds ?? DEFAULT_TEST_TIMEOUT_SECONDS) * 1000,
       encoding: 'utf8',
       shell: true, // needed so we can pass the test command in as whole without splitting it up into args
+      env: {
+        ...process.env,
+        ...envVarsToInject,
+      },
     });
 
     // Prepends some text to the output test command's output so we can distinguish it from logging in this script

packages/e2e-tests/test-applications/standard-frontend-react/.gitignore

@@ -22,4 +22,8 @@ npm-debug.log*
 yarn-debug.log*
 yarn-error.log*
 
+/test-results/
+/playwright-report/
+/playwright/.cache/
+
 !*.d.ts

packages/e2e-tests/test-applications/standard-frontend-react/package.json

@@ -20,8 +20,9 @@
     "web-vitals": "2.1.0"
   },
   "scripts": {
-    "start": "react-scripts start",
-    "build": "react-scripts build"
+    "build": "react-scripts build",
+    "start": "serve -s build",
+    "test": "playwright test"
   },
   "eslintConfig": {
     "extends": [
@@ -40,5 +41,10 @@
       "last 1 firefox version",
       "last 1 safari version"
     ]
+  },
+  "devDependencies": {
+    "@playwright/test": "1.26.1",
+    "axios": "1.1.2",
+    "serve": "14.0.1"
   }
 }

packages/e2e-tests/test-applications/standard-frontend-react/playwright.config.ts

@@ -0,0 +1,69 @@
+import type { PlaywrightTestConfig } from '@playwright/test';
+import { devices } from '@playwright/test';
+
+/**
+ * See https://playwright.dev/docs/test-configuration.
+ */
+const config: PlaywrightTestConfig = {
+  testDir: './tests',
+  /* Maximum time one test can run for. */
+  timeout: 60 * 1000,
+  expect: {
+    /**
+     * Maximum time expect() should wait for the condition to be met.
+     * For example in `await expect(locator).toHaveText();`
+     */
+    timeout: 5000,
+  },
+  /* Run tests in files in parallel */
+  fullyParallel: true,
+  /* Fail the build on CI if you accidentally left test.only in the source code. */
+  forbidOnly: !!process.env.CI,
+  /* Retry on CI only */
+  retries: 0,
+  /* Opt out of parallel tests on CI. */
+  workers: 1,
+  /* Reporter to use. See https://playwright.dev/docs/test-reporters */
+  reporter: 'dot',
+  /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
+  use: {
+    /* Maximum time each action such as `click()` can take. Defaults to 0 (no limit). */
+    actionTimeout: 0,
+    /* Base URL to use in actions like `await page.goto('/')`. */
+    // baseURL: 'http://localhost:3000',
+
+    /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
+    trace: 'on-first-retry',
+  },
+
+  /* Configure projects for major browsers */
+  projects: [
+    {
+      name: 'chromium',
+      use: {
+        ...devices['Desktop Chrome'],
+      },
+    },
+    // For now we only test Chrome!
+    // {
+    //   name: 'firefox',
+    //   use: {
+    //     ...devices['Desktop Firefox'],
+    //   },
+    // },
+    // {
+    //   name: 'webkit',
+    //   use: {
+    //     ...devices['Desktop Safari'],
+    //   },
+    // },
+  ],
+
+  /* Run your local dev server before starting the tests */
+  webServer: {
+    command: 'yarn start',
+    port: 3000,
+  },
+};
+
+export default config;

packages/e2e-tests/test-applications/standard-frontend-react/src/globals.d.ts

@@ -0,0 +1,4 @@
+interface Window {
+  recordedTransactions?: string[];
+  capturedExceptionId?: string;
+}

packages/e2e-tests/test-applications/standard-frontend-react/src/index.tsx

@@ -15,7 +15,7 @@ import Index from './pages/Index';
 import User from './pages/User';
 
 Sentry.init({
-  dsn: 'https://[email protected]/1337',
+  dsn: process.env.REACT_APP_E2E_TEST_DSN,
   integrations: [
     new BrowserTracing({
       routingInstrumentation: Sentry.reactRouterV6Instrumentation(
@@ -39,10 +39,10 @@ Sentry.addGlobalEventProcessor(event => {
     (event.contexts?.trace?.op === 'pageload' || event.contexts?.trace?.op === 'navigation')
   ) {
     const eventId = event.event_id;
-    // @ts-ignore
-    window.recordedTransactions = window.recordedTransactions || [];
-    // @ts-ignore
-    window.recordedTransactions.push(eventId);
+    if (eventId) {
+      window.recordedTransactions = window.recordedTransactions || [];
+      window.recordedTransactions.push(eventId);
+    }
   }
 
   return event;

packages/e2e-tests/test-applications/standard-frontend-react/src/pages/Index.tsx

@@ -11,7 +11,6 @@ const Index = () => {
         id="exception-button"
         onClick={() => {
           const eventId = Sentry.captureException(new Error('I am an error!'));
-          // @ts-ignore
           window.capturedExceptionId = eventId;
         }}
       />

packages/e2e-tests/test-applications/standard-frontend-react/test-recipe.json

@@ -1,6 +1,11 @@
 {
   "$schema": "../../test-recipe-schema.json",
   "testApplicationName": "standard-frontend-react",
-  "buildCommand": "yarn install && yarn build",
-  "tests": []
+  "buildCommand": "yarn install --pure-lockfile && npx playwright install && yarn build",
+  "tests": [
+    {
+      "testName": "Playwright tests",
+      "testCommand": "yarn test"
+    }
+  ]
 }

packages/e2e-tests/test-applications/standard-frontend-react/tests/behaviour-test.spec.ts

@@ -0,0 +1,47 @@
+import { test, expect } from '@playwright/test';
+import axios, { AxiosError } from 'axios';
+
+const SENTRY_TEST_ORG_SLUG = 'sentry-sdks';
+const SENTRY_TEST_PROJECT = 'sentry-javascript-e2e-tests';
+
+const EVENT_POLLING_TIMEOUT = 45000;
+const EVENT_POLLING_RETRY_INTERVAL = 1000;
+
+const authToken = process.env.E2E_TEST_AUTH_TOKEN;
+
+test('Sends an exception to Sentry', async ({ page }) => {
+  await page.goto('/');
+
+  const exceptionButton = page.locator('id=exception-button');
+  await exceptionButton.click();
+
+  const exceptionIdHandle = await page.waitForFunction(() => window.capturedExceptionId);
+  const exceptionEventId = await exceptionIdHandle.jsonValue();
+
+  let lastErrorResponse: AxiosError | undefined;
+
+  const timeout = setTimeout(() => {
+    if (lastErrorResponse?.response?.status) {
+      throw new Error(
+        `Timeout reached while polling event. Last received status code: ${lastErrorResponse.response.status}`,
+      );
+    } else {
+      throw new Error('Timeout reached while polling event.');
+    }
+  }, EVENT_POLLING_TIMEOUT);
+
+  while (true) {
+    try {
+      const response = await axios.get(
+        `https://sentry.io/api/0/projects/${SENTRY_TEST_ORG_SLUG}/${SENTRY_TEST_PROJECT}/events/${exceptionEventId}/`,
+        { headers: { Authorization: `Bearer ${authToken}` } },
+      );
+      clearTimeout(timeout);
+      expect(response?.status).toBe(200);
+      break;
+    } catch (e) {
+      lastErrorResponse = e;
+      await new Promise(resolve => setTimeout(resolve, EVENT_POLLING_RETRY_INTERVAL));
+    }
+  }
+});