Browser SDK Integration Tests

.github/workflows/build.yml

@@ -271,6 +271,33 @@ jobs:
             ${{ github.workspace }}/packages/**/*.tgz
             ${{ github.workspace }}/packages/serverless/dist-awslambda-layer/*.zip
 
+  job_browser_playwright_tests:
+    name: Browser Playwright Tests
+    needs: job_build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out current commit (${{ github.sha }})
+        uses: actions/checkout@v2
+      - name: Set up Node
+        uses: actions/setup-node@v1
+        with:
+          node-version: '16'
+      - name: Check dependency cache
+        uses: actions/cache@v2
+        with:
+          path: ${{ env.CACHED_DEPENDENCY_PATHS }}
+          key: ${{ needs.job_build.outputs.dependency_cache_key }}
+      - name: Check build cache
+        uses: actions/cache@v2
+        with:
+          path: ${{ env.CACHED_BUILD_PATHS }}
+          key: ${{ env.BUILD_CACHE_KEY }}
+      - name: Run Playwright tests
+        run: |
+          cd packages/integration-tests
+          yarn run playwright install-deps webkit
+          yarn test:ci
+
   job_browser_integration_tests:
     name: Browser Integration Tests (${{ matrix.browser }})
     needs: job_build

package.json

@@ -19,7 +19,7 @@
     "pack:changed": "lerna run pack --since",
     "prepublishOnly": "lerna run --stream --concurrency 1 prepublishOnly",
     "postpublish": "make publish-docs && lerna run --stream --concurrency 1 postpublish",
-    "test": "lerna run --stream --concurrency 1 --sort test"
+    "test": "lerna run --ignore @sentry-internal/browser-integration-tests --stream --concurrency 1 --sort test"
   },
   "volta": {
     "node": "14.17.0",
@@ -34,6 +34,7 @@
     "packages/eslint-plugin-sdk",
     "packages/gatsby",
     "packages/hub",
+    "packages/integration-tests",
     "packages/integrations",
     "packages/minimal",
     "packages/nextjs",

packages/eslint-config-sdk/src/index.js

@@ -164,7 +164,7 @@ module.exports = {
       env: {
         jest: true,
       },
-      files: ['*.test.ts', '*.test.tsx', '*.test.js', '*.test.jsx', 'test/**/*.ts', 'test/**/*.js'],
+      files: ['test.ts', '*.test.ts', '*.test.tsx', '*.test.js', '*.test.jsx', 'test/**/*.ts', 'test/**/*.js'],
       rules: {
         'max-lines': 'off',
         '@typescript-eslint/explicit-function-return-type': 'off',

packages/integration-tests/.eslintrc.js

@@ -0,0 +1,11 @@
+module.exports = {
+  env: {
+    browser: true,
+    node: true,
+  },
+  extends: ['../../.eslintrc.js'],
+  ignorePatterns: ['suites/**/subject.js', 'suites/**/dist/*'],
+  parserOptions: {
+    sourceType: 'module',
+  },
+};

packages/integration-tests/.gitignore

@@ -0,0 +1 @@
+dist

packages/integration-tests/README.md

@@ -0,0 +1,72 @@
+# Integration Tests for Sentry Browser SDK
+
+Integration tests for Sentry's Browser SDK use [Playwright](https://playwright.dev/) internally. These tests are run on latest stable versions of Chromium, Firefox and Webkit.
+
+## Structure
+
+The tests are grouped by their scope such as `breadcrumbs` or `onunhandledrejection`. In every group of tests, there are multiple folders containing test cases with their optional supporting assets.
+
+Each case group has a default HTML skeleton named `template.hbs`, and also a default initialization script named `init.js `, which contains the `Sentry.init()` call. These defaults are used as fallbacks when a specific `template.hbs` or `init.js` is not defined in a case folder.
+
+`subject.js` contains the logic that sets up the environment to be tested. It also can be defined locally and as a group fallback. Unlike `template.hbs` and `init.js`, it's not required to be defined for a group, as there may be cases that does not require a subject, instead the logic is injected using `injectScriptAndGetEvents` from `utils/helpers.ts`.
+
+`test.ts` is required for each test case, which contains the assertions (and if required the script injection logic). For every case, any set of `init.js`, `template.hbs` and `subject.js` can be defined locally, and each one of them  will have precedence over the default definitions of the test group.
+
+```
+suites/
+|---- breadcrumbs/
+      |---- template.hbs [fallback template for breadcrumb tests]
+      |---- init.js [fallback init for breadcrumb tests]
+      |---- subject.js [optional fallback subject for breadcrumb tests]
+      |---- click_event_tree/
+            |---- template.hbs [optional case specific template]
+            |---- init.js [optional case specific init]
+            |---- subject.js [optional case specific subject]
+            |---- test.ts [assertions]
+```
+
+## Writing Tests
+
+### Helpers
+
+`utils/helpers.ts` contains helpers that could be used in assertions (`test.ts`). These helpers define a convenient and reliable API to interact with Playwright's native API. It's highly recommended to define all common patterns of Playwright usage in helpers.
+
+### Fixtures
+
+[Fixtures](https://playwright.dev/docs/api/class-fixtures) allows us to define the globals and test-specific information in assertion groups (`test.ts` files). In it's current state, `fixtures.ts` contains an extension over the pure version of `test()` function of Playwright. All the tests should import `sentryTest` function from `utils/fixtures.ts` instead of `@playwright/test` to be able to access the extra fixtures.
+
+## Running Tests Locally
+
+Tests can be run locally using the latest version of Chromium with:
+
+`yarn test`
+
+To run tests with a different browser such as `firefox` or `webkit`:
+
+`yarn test --browser='firefox'`
+`yarn test --browser='webkit'`
+
+Or to run on all three browsers:
+
+`yarn test --browser='all'`
+
+To filter tests by their title:
+
+`yarn test -g "XMLHttpRequest without any handlers set"`
+
+You can refer to [Playwright documentation](https://playwright.dev/docs/test-cli) for other CLI options.
+
+### Troubleshooting
+
+Apart from [Playwright-specific issues](https://playwright.dev/docs/troubleshooting), below are common issues that might occur while writing tests for Sentry Browser SDK.
+
+- #### Flaky Tests
+  If a test fails randomly, giving a `Page Closed`, `Target Closed` or a similar error, most of the times, the reason is a race condition between the page action defined in the `subject` and the listeners of the Sentry event / request. It's recommended to firstly check `utils/helpers.ts` whether if that async logic can be replaced by one of the helpers. If not, whether the awaited (or non-awaited on purpose in some cases) Playwright methods can be orchestrated by [`Promise.all`](http://mdn.io/promise.all). Manually-defined waiting logic such as timeouts are not recommended, and should not be required in most of the cases.
+
+- #### Build Errors
+  Before running, a page for each test case is built under the case folder inside `dist`. If a page build is failed, it's recommended to check:
+
+  - If both default `template.hbs` and `init.js` are defined for the test group.
+  - If a `subject.js` is defined for the test case.
+  - If either of `init.js` or `subject.js` contain non-browser code.
+  - If the webpack configuration is valid.

packages/integration-tests/package.json

@@ -0,0 +1,30 @@
+{
+  "name": "@sentry-internal/browser-integration-tests",
+  "version": "1.0.0",
+  "main": "index.js",
+  "license": "MIT",
+  "engines": {
+    "node": ">=10"
+  },
+  "private": true,
+  "scripts": {
+    "clean": "rimraf -g suites/**/dist",
+    "install-browsers": "playwright install --with-deps",
+    "lint": "run-s lint:prettier lint:eslint",
+    "lint:eslint": "eslint . --cache --cache-location '../../eslintcache/' --format stylish",
+    "lint:prettier": "prettier --check \"{suites,utils}/**/*.ts\"",
+    "test:ci": "playwright test ./suites --browser='all' --reporter='line'",
+    "type-check": "tsc",
+    "pretest": "yarn clean && yarn type-check",
+    "test": "playwright test ./suites"
+  },
+  "dependencies": {
+    "@playwright/test": "^1.17.0",
+    "babel-loader": "^8.2.2",
+    "handlebars-loader": "^1.7.1",
+    "html-webpack-plugin": "^5.5.0",
+    "playwright": "^1.17.1",
+    "typescript": "^4.5.2",
+    "webpack": "^5.52.0"
+  }
+}

packages/integration-tests/playwright.config.ts

@@ -0,0 +1,8 @@
+import { PlaywrightTestConfig } from '@playwright/test';
+
+const config: PlaywrightTestConfig = {
+  retries: 2,
+  timeout: 12000,
+  workers: 3,
+};
+export default config;

packages/integration-tests/suites/demo/init.js

@@ -0,0 +1,7 @@
+import * as Sentry from '@sentry/browser';
+
+window.Sentry = Sentry;
+
+Sentry.init({
+  dsn: 'https://[email protected]/1337',
+});

packages/integration-tests/suites/demo/template.hbs

@@ -0,0 +1,11 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta charset="utf-8" />
+    <title></title>
+  <script src="{{htmlWebpackPlugin.options.initialization}}"></script>
+  </head>
+  <body>
+    <script src="{{htmlWebpackPlugin.options.subject}}"></script>
+  </body>
+</html>

packages/integration-tests/suites/demo/tmp/subject.js

@@ -0,0 +1 @@
+Sentry.captureMessage(1);

packages/integration-tests/suites/demo/tmp/test.ts

@@ -0,0 +1,12 @@
+import { expect } from '@playwright/test';
+
+import { sentryTest } from '../../../utils/fixtures';
+import { getSentryRequest } from '../../../utils/helpers';
+
+sentryTest('should fail', async ({ getLocalTestPath, page }) => {
+  const url = await getLocalTestPath({ testDir: __dirname });
+
+  const eventData = await getSentryRequest(page, url);
+
+  expect(eventData.message).toBe('1');
+});

packages/integration-tests/tsconfig.json

@@ -0,0 +1,12 @@
+{
+  "extends": "../../tsconfig.json",
+
+  "compilerOptions": {
+    "lib": ["dom", "es2019"],
+    "moduleResolution": "node",
+    "noEmit": true,
+    "strict": true
+  },
+  "include": ["**/*.ts"],
+  "exclude": ["node_modules"]
+}

packages/integration-tests/utils/fixtures.ts

@@ -0,0 +1,46 @@
+import { test as base } from '@playwright/test';
+import fs from 'fs';
+import path from 'path';
+
+import { generatePage } from './generatePage';
+
+const getAsset = (assetDir: string, asset: string): string => {
+  const assetPath = `${assetDir}/${asset}`;
+
+  if (fs.existsSync(assetPath)) {
+    return assetPath;
+  }
+
+  return `${path.dirname(assetDir)}/${asset}`;
+};
+
+export type TestOptions = {
+  testDir: string;
+};
+
+export type TestFixtures = {
+  testDir: string;
+  getLocalTestPath: (options: TestOptions) => Promise<string>;
+};
+
+const sentryTest = base.extend<TestFixtures>({
+  // eslint-disable-next-line no-empty-pattern
+  getLocalTestPath: ({}, use, testInfo) => {
+    return use(async ({ testDir }) => {
+      const pagePath = `file:///${path.resolve(testDir, './dist/index.html')}`;
+
+      // Build test page if it doesn't exist
+      if (!fs.existsSync(pagePath)) {
+        const testDir = path.dirname(testInfo.file);
+        const subject = getAsset(testDir, 'subject.js');
+        const template = getAsset(testDir, 'template.hbs');
+        const init = getAsset(testDir, 'init.js');
+
+        await generatePage(init, subject, template, testDir);
+      }
+      return pagePath;
+    });
+  },
+});
+
+export { sentryTest };

packages/integration-tests/utils/generatePage.ts

@@ -0,0 +1,89 @@
+import { Package } from '@sentry/types';
+import { existsSync, mkdirSync, promises } from 'fs';
+import HtmlWebpackPlugin from 'html-webpack-plugin';
+import path from 'path';
+import webpack from 'webpack';
+
+import webpackConfig from '../webpack.config';
+
+const PACKAGE_PATH = '../../packages';
+
+/**
+ * Generate webpack aliases based on packages in monorepo
+ * Example of an alias: '@sentry/serverless': 'path/to/sentry-javascript/packages/serverless',
+ */
+async function generateSentryAlias(): Promise<Record<string, string>> {
+  const dirents = (await promises.readdir(PACKAGE_PATH, { withFileTypes: true }))
+    .filter(dirent => dirent.isDirectory())
+    .map(dir => dir.name);
+
+  return Object.fromEntries(
+    await Promise.all(
+      dirents.map(async d => {
+        const packageJSON: Package = JSON.parse(
+          (await promises.readFile(path.resolve(PACKAGE_PATH, d, 'package.json'), { encoding: 'utf-8' })).toString(),
+        );
+        return [packageJSON['name'], path.resolve(PACKAGE_PATH, d)];
+      }),
+    ),
+  );
+}
+
+export async function generatePage(
+  initializationPath: string,
+  subjectPath: string,
+  templatePath: string,
+  outPath: string,
+): Promise<void> {
+  const localPath = `${outPath}/dist`;
+  const bundlePath = `${localPath}/index.html`;
+
+  const alias = await generateSentryAlias();
+
+  if (!existsSync(localPath)) {
+    mkdirSync(localPath, { recursive: true });
+  }
+
+  if (!existsSync(bundlePath)) {
+    await new Promise<void>((resolve, reject) => {
+      const compiler = webpack(
+        webpackConfig({
+          resolve: {
+            alias,
+          },
+          entry: {
+            initialization: initializationPath,
+            subject: subjectPath,
+          },
+          output: {
+            path: localPath,
+            filename: '[name].bundle.js',
+          },
+          plugins: [
+            new HtmlWebpackPlugin({
+              filename: 'index.html',
+              template: templatePath,
+              initialization: 'initialization.bundle.js',
+              subject: `subject.bundle.js`,
+              inject: false,
+            }),
+          ],
+        }),
+      );
+
+      compiler.run(err => {
+        if (err) {
+          reject(err);
+        }
+
+        compiler.close(err => {
+          if (err) {
+            reject(err);
+          }
+
+          resolve();
+        });
+      });
+    });
+  }
+}