Merge branch 'develop' into fix/fetch-not-release

Author: Lei-k

.github/workflows/build.yml

@@ -801,7 +801,15 @@ jobs:
     needs: [job_get_metadata, job_build, job_compile_bindings_profiling_node]
     runs-on: ubuntu-20.04-large-js
     timeout-minutes: 15
+    outputs:
+      matrix: ${{ steps.matrix.outputs.matrix }}
+      matrix-optional: ${{ steps.matrix-optional.outputs.matrix }}
     steps:
+      - name: Check out base commit (${{ github.event.pull_request.base.sha }})
+        uses: actions/checkout@v4
+        if: github.event_name == 'pull_request'
+        with:
+          ref: ${{ github.event.pull_request.base.sha }}
       - name: Check out current commit (${{ needs.job_get_metadata.outputs.commit_label }})
         uses: actions/checkout@v4
         with:
@@ -851,11 +859,21 @@ jobs:
           path: ${{ github.workspace }}/packages/*/*.tgz
           key: ${{ env.BUILD_CACHE_TARBALL_KEY }}
 
+      - name: Determine which E2E test applications should be run
+        id: matrix
+        run: yarn --silent ci:build-matrix --base=${{ (github.event_name == 'pull_request' && github.event.pull_request.base.sha) || '' }} >> $GITHUB_OUTPUT
+        working-directory: dev-packages/e2e-tests
+
+      - name: Determine which optional E2E test applications should be run
+        id: matrix-optional
+        run: yarn --silent ci:build-matrix-optional --base=${{ (github.event_name == 'pull_request' && github.event.pull_request.base.sha) || '' }} >> $GITHUB_OUTPUT
+        working-directory: dev-packages/e2e-tests
+
   job_e2e_tests:
     name: E2E ${{ matrix.label || matrix.test-application }} Test
     # We need to add the `always()` check here because the previous step has this as well :(
     # See: https://github.com/actions/runner/issues/2205
-    if: always() && needs.job_e2e_prepare.result == 'success'
+    if: always() && needs.job_e2e_prepare.result == 'success' && needs.job_e2e_prepare.outputs.matrix != '{"include":[]}'
     needs: [job_get_metadata, job_build, job_e2e_prepare]
     runs-on: ubuntu-20.04
     timeout-minutes: 15
@@ -870,105 +888,7 @@ jobs:
       E2E_TEST_SENTRY_PROJECT: 'sentry-javascript-e2e-tests'
     strategy:
       fail-fast: false
-      matrix:
-        is_dependabot:
-          - ${{ github.actor == 'dependabot[bot]' }}
-        test-application:
-          [
-            'angular-17',
-            'angular-18',
-            'astro-4',
-            'aws-lambda-layer-cjs',
-            'aws-serverless-esm',
-            'node-express',
-            'create-react-app',
-            'create-next-app',
-            'create-remix-app',
-            'create-remix-app-legacy',
-            'create-remix-app-v2',
-            'create-remix-app-v2-legacy',
-            'create-remix-app-express',
-            'create-remix-app-express-legacy',
-            'create-remix-app-express-vite-dev',
-            'default-browser',
-            'node-express-esm-loader',
-            'node-express-esm-preload',
-            'node-express-esm-without-loader',
-            'node-express-cjs-preload',
-            'node-otel-sdk-node',
-            'node-otel-custom-sampler',
-            'node-otel-without-tracing',
-            'ember-classic',
-            'ember-embroider',
-            'nextjs-app-dir',
-            'nextjs-13',
-            'nextjs-14',
-            'nextjs-15',
-            'nextjs-turbo',
-            'nextjs-t3',
-            'react-17',
-            'react-19',
-            'react-create-hash-router',
-            'react-router-6-use-routes',
-            'react-router-5',
-            'react-router-6',
-            'solid',
-            'solidstart',
-            'solidstart-spa',
-            'svelte-5',
-            'sveltekit',
-            'sveltekit-2',
-            'sveltekit-2-svelte-5',
-            'sveltekit-2-twp',
-            'tanstack-router',
-            'generic-ts3.8',
-            'node-fastify',
-            'node-fastify-5',
-            'node-hapi',
-            'node-nestjs-basic',
-            'node-nestjs-distributed-tracing',
-            'nestjs-basic',
-            'nestjs-8',
-            'nestjs-distributed-tracing',
-            'nestjs-with-submodules',
-            'nestjs-with-submodules-decorator',
-            'nestjs-basic-with-graphql',
-            'nestjs-graphql',
-            'node-exports-test-app',
-            'node-koa',
-            'node-connect',
-            'nuxt-3',
-            'nuxt-3-min',
-            'nuxt-4',
-            'vue-3',
-            'webpack-4',
-            'webpack-5'
-          ]
-        build-command:
-          - false
-        label:
-          - false
-        # Add any variations of a test app here
-        # You should provide an alternate build-command as well as a matching label
-        include:
-          - test-application: 'create-react-app'
-            build-command: 'test:build-ts3.8'
-            label: 'create-react-app (TS 3.8)'
-          - test-application: 'react-router-6'
-            build-command: 'test:build-ts3.8'
-            label: 'react-router-6 (TS 3.8)'
-          - test-application: 'create-next-app'
-            build-command: 'test:build-13'
-            label: 'create-next-app (next@13)'
-          - test-application: 'nextjs-app-dir'
-            build-command: 'test:build-13'
-            label: 'nextjs-app-dir (next@13)'
-        exclude:
-          - is_dependabot: true
-            test-application: 'cloudflare-astro'
-          - is_dependabot: true
-            test-application: 'cloudflare-workers'
-
+      matrix: ${{ fromJson(needs.job_e2e_prepare.outputs.matrix) }}
     steps:
       - name: Check out current commit (${{ needs.job_get_metadata.outputs.commit_label }})
         uses: actions/checkout@v4
@@ -1069,6 +989,7 @@ jobs:
     # See: https://github.com/actions/runner/issues/2205
     if:
       always() && needs.job_e2e_prepare.result == 'success' &&
+      needs.job_e2e_prepare.outputs.matrix-optional != '{"include":[]}' &&
       (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) &&
       github.actor != 'dependabot[bot]'
     needs: [job_get_metadata, job_build, job_e2e_prepare]
@@ -1085,58 +1006,7 @@ jobs:
       E2E_TEST_SENTRY_PROJECT: 'sentry-javascript-e2e-tests'
     strategy:
       fail-fast: false
-      matrix:
-        test-application:
-          [
-            'cloudflare-astro',
-            'cloudflare-workers',
-            'react-send-to-sentry',
-            'node-express-send-to-sentry',
-            'debug-id-sourcemaps',
-          ]
-        build-command:
-          - false
-        assert-command:
-          - false
-        label:
-          - false
-        include:
-          - test-application: 'create-remix-app'
-            assert-command: 'test:assert-sourcemaps'
-            label: 'create-remix-app (sourcemaps)'
-          - test-application: 'create-remix-app-legacy'
-            assert-command: 'test:assert-sourcemaps'
-            label: 'create-remix-app-legacy (sourcemaps)'
-          - test-application: 'nextjs-app-dir'
-            build-command: 'test:build-canary'
-            label: 'nextjs-app-dir (canary)'
-          - test-application: 'nextjs-app-dir'
-            build-command: 'test:build-latest'
-            label: 'nextjs-app-dir (latest)'
-          - test-application: 'nextjs-13'
-            build-command: 'test:build-canary'
-            label: 'nextjs-13 (canary)'
-          - test-application: 'nextjs-13'
-            build-command: 'test:build-latest'
-            label: 'nextjs-13 (latest)'
-          - test-application: 'nextjs-14'
-            build-command: 'test:build-canary'
-            label: 'nextjs-14 (canary)'
-          - test-application: 'nextjs-14'
-            build-command: 'test:build-latest'
-            label: 'nextjs-14 (latest)'
-          - test-application: 'nextjs-15'
-            build-command: 'test:build-canary'
-            label: 'nextjs-15 (canary)'
-          - test-application: 'nextjs-15'
-            build-command: 'test:build-latest'
-            label: 'nextjs-15 (latest)'
-          - test-application: 'nextjs-turbo'
-            build-command: 'test:build-canary'
-            label: 'nextjs-turbo (canary)'
-          - test-application: 'nextjs-turbo'
-            build-command: 'test:build-latest'
-            label: 'nextjs-turbo (latest)'
+      matrix: ${{ fromJson(needs.job_e2e_prepare.outputs.matrix-optional) }}
 
     steps:
       - name: Check out current commit (${{ needs.job_get_metadata.outputs.commit_label }})

dev-packages/e2e-tests/lib/getTestMatrix.ts

@@ -0,0 +1,155 @@
+import { execSync } from 'child_process';
+import * as fs from 'fs';
+import * as path from 'path';
+import { dirname } from 'path';
+import { parseArgs } from 'util';
+import { sync as globSync } from 'glob';
+
+interface MatrixInclude {
+  /** The test application (directory) name. */
+  'test-application': string;
+  /** Optional override for the build command to run. */
+  'build-command'?: string;
+  /** Optional override for the assert command to run. */
+  'assert-command'?: string;
+  /** Optional label for the test run. If not set, defaults to value of `test-application`. */
+  label?: string;
+}
+
+interface PackageJsonSentryTestConfig {
+  /** If this is true, the test app is optional. */
+  optional?: boolean;
+  /** Variant configs that should be run in non-optional test runs. */
+  variants?: Partial<MatrixInclude>[];
+  /** Variant configs that should be run in optional test runs. */
+  optionalVariants?: Partial<MatrixInclude>[];
+  /** Skip this test app for matrix generation. */
+  skip?: boolean;
+}
+
+/**
+ * This methods generates a matrix for the GitHub Actions workflow to run the E2E tests.
+ * It checks which test applications are affected by the current changes in the PR and then generates a matrix
+ * including all test apps that have at least one dependency that was changed in the PR.
+ * If no `--base=xxx` is provided, it will output all test applications.
+ *
+ * If `--optional=true` is set, it will generate a matrix of optional test applications only.
+ * Otherwise, these will be skipped.
+ */
+function run(): void {
+  const { values } = parseArgs({
+    args: process.argv.slice(2),
+    options: {
+      base: { type: 'string' },
+      head: { type: 'string' },
+      optional: { type: 'string', default: 'false' },
+    },
+  });
+
+  const { base, head, optional } = values;
+
+  const testApplications = globSync('*/package.json', {
+    cwd: `${__dirname}/../test-applications`,
+  }).map(filePath => dirname(filePath));
+
+  // If `--base=xxx` is defined, we only want to get test applications changed since that base
+  // Else, we take all test applications (e.g. on push)
+  const includedTestApplications = base
+    ? getAffectedTestApplications(testApplications, { base, head })
+    : testApplications;
+
+  const optionalMode = optional === 'true';
+  const includes: MatrixInclude[] = [];
+
+  includedTestApplications.forEach(testApp => {
+    addIncludesForTestApp(testApp, includes, { optionalMode });
+  });
+
+  // We print this to the output, so the GHA can use it for the matrix
+  // eslint-disable-next-line no-console
+  console.log(`matrix=${JSON.stringify({ include: includes })}`);
+}
+
+function addIncludesForTestApp(
+  testApp: string,
+  includes: MatrixInclude[],
+  { optionalMode }: { optionalMode: boolean },
+): void {
+  const packageJson = getPackageJson(testApp);
+
+  const shouldSkip = packageJson.sentryTest?.skip || false;
+  const isOptional = packageJson.sentryTest?.optional || false;
+  const variants = (optionalMode ? packageJson.sentryTest?.optionalVariants : packageJson.sentryTest?.variants) || [];
+
+  if (shouldSkip) {
+    return;
+  }
+
+  // Add the basic test-application itself, if it is in the current mode
+  if (optionalMode === isOptional) {
+    includes.push({
+      'test-application': testApp,
+    });
+  }
+
+  variants.forEach(variant => {
+    includes.push({
+      'test-application': testApp,
+      ...variant,
+    });
+  });
+}
+
+function getSentryDependencies(appName: string): string[] {
+  const packageJson = getPackageJson(appName) || {};
+
+  const dependencies = {
+    ...packageJson.devDependencies,
+    ...packageJson.dependencies,
+  };
+
+  return Object.keys(dependencies).filter(key => key.startsWith('@sentry'));
+}
+
+function getPackageJson(appName: string): {
+  dependencies?: { [key: string]: string };
+  devDependencies?: { [key: string]: string };
+  sentryTest?: PackageJsonSentryTestConfig;
+} {
+  const fullPath = path.resolve(__dirname, '..', 'test-applications', appName, 'package.json');
+
+  if (!fs.existsSync(fullPath)) {
+    throw new Error(`Could not find package.json for ${appName}`);
+  }
+
+  return JSON.parse(fs.readFileSync(fullPath, 'utf8'));
+}
+
+run();
+
+function getAffectedTestApplications(
+  testApplications: string[],
+  { base = 'develop', head }: { base?: string; head?: string },
+): string[] {
+  const additionalArgs = [`--base=${base}`];
+
+  if (head) {
+    additionalArgs.push(`--head=${head}`);
+  }
+
+  const affectedProjects = execSync(`yarn --silent nx show projects --affected ${additionalArgs.join(' ')}`)
+    .toString()
+    .split('\n')
+    .map(line => line.trim())
+    .filter(Boolean);
+
+  // If something in e2e tests themselves are changed, just run everything
+  if (affectedProjects.includes('@sentry-internal/e2e-tests')) {
+    return testApplications;
+  }
+
+  return testApplications.filter(testApp => {
+    const sentryDependencies = getSentryDependencies(testApp);
+    return sentryDependencies.some(dep => affectedProjects.includes(dep));
+  });
+}

dev-packages/e2e-tests/package.json

@@ -14,11 +14,13 @@
     "test:prepare": "ts-node prepare.ts",
     "test:validate": "run-s test:validate-configuration test:validate-test-app-setups",
     "clean": "rimraf tmp node_modules pnpm-lock.yaml && yarn clean:test-applications",
+    "ci:build-matrix": "ts-node ./lib/getTestMatrix.ts",
+    "ci:build-matrix-optional": "ts-node ./lib/getTestMatrix.ts --optional=true",
     "clean:test-applications": "rimraf --glob test-applications/**/{node_modules,dist,build,.next,.sveltekit,pnpm-lock.yaml} .last-run.json && pnpm store prune"
   },
   "devDependencies": {
     "@types/glob": "8.0.0",
-    "@types/node": "^14.18.0",
+    "@types/node": "^18.0.0",
     "dotenv": "16.0.3",
     "esbuild": "0.20.0",
     "glob": "8.0.3",

dev-packages/e2e-tests/test-applications/cloudflare-astro/package.json

@@ -26,5 +26,8 @@
   },
   "volta": {
     "extends": "../../package.json"
+  },
+  "sentryTest": {
+    "optional": true
   }
 }