Skip to content

Commit d03f6e7

Browse files
mydeamydea
committed
WIP: Manually run E2E tests on CI?
1 parent 0f2d390 commit d03f6e7

File tree

32 files changed

+268
-71
lines changed

32 files changed

+268
-71
lines changed

‎.github/workflows/build.yml

Lines changed: 91 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ env:
4343
${{ github.workspace }}/packages/utils/esm
4444
4545
BUILD_CACHE_KEY: ${{ github.event.inputs.commit || github.sha }}
46+
BUILD_CACHE_TARBALL_KEY: tarball-${{ github.event.inputs.commit || github.sha }}
4647

4748
# GH will use the first restore-key it finds that matches
4849
# So it will start by looking for one from the same branch, else take the newest one it can find elsewhere
@@ -729,20 +730,82 @@ jobs:
729730
cd packages/remix
730731
yarn test:integration:ci
731732
733+
job_e2e_prepare:
734+
name: Prepare E2E tests
735+
if:
736+
(github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) &&
737+
github.actor != 'dependabot[bot]'
738+
needs: [job_get_metadata, job_build]
739+
runs-on: ubuntu-20.04
740+
timeout-minutes: 15
741+
steps:
742+
- name: Check out current commit (${{ needs.job_get_metadata.outputs.commit_label }})
743+
uses: actions/checkout@v3
744+
with:
745+
ref: ${{ env.HEAD_COMMIT }}
746+
- name: Set up Node
747+
uses: actions/setup-node@v3
748+
with:
749+
node-version-file: 'package.json'
750+
- name: Restore caches
751+
uses: ./.github/actions/restore-cache
752+
env:
753+
DEPENDENCY_CACHE_KEY: ${{ needs.job_build.outputs.dependency_cache_key }}
754+
- name: Check tarball cache
755+
uses: actions/cache@v3
756+
with:
757+
path: ${{ github.workspace }}/packages/*/*.tgz
758+
key: ${{ env.BUILD_CACHE_TARBALL_KEY }}
759+
- name: Build tarballs
760+
run: yarn build:tarball
761+
732762
job_e2e_tests:
733-
name: E2E (Shard ${{ matrix.shard }}) Tests
763+
name: E2E ${{ matrix.label || matrix.test-application }} Test
734764
# We only run E2E tests for non-fork PRs because the E2E tests require secrets to work and they can't be accessed from forks
735765
# Dependabot PRs sadly also don't have access to secrets, so we skip them as well
736766
if:
737767
(github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) &&
738768
github.actor != 'dependabot[bot]'
739-
needs: [job_get_metadata, job_build]
769+
needs: [job_get_metadata, job_build, job_e2e_prepare]
740770
runs-on: ubuntu-20.04
741-
timeout-minutes: 30
771+
timeout-minutes: 10
772+
env:
773+
E2E_TEST_AUTH_TOKEN: ${{ secrets.E2E_TEST_AUTH_TOKEN }}
774+
E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }}
775+
# Needed because some apps expect a certain prefix
776+
NEXT_PUBLIC_E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }}
777+
PUBLIC_E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }}
778+
REACT_APP_E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }}
779+
E2E_TEST_SENTRY_ORG_SLUG: 'sentry-javascript-sdks'
780+
E2E_TEST_SENTRY_TEST_PROJECT: 'sentry-javascript-e2e-tests'
742781
strategy:
743782
fail-fast: false
744783
matrix:
745-
shard: [1, 2, 3, 4]
784+
test-application:
785+
[
786+
'node-express-app',
787+
'create-react-app',
788+
'create-next-app',
789+
'create-remix-app',
790+
'nextjs-app-dir',
791+
'react-create-hash-router',
792+
'standard-frontend-react',
793+
'standard-frontend-react-tracing-import',
794+
'sveltekit',
795+
]
796+
build-command:
797+
- false
798+
label:
799+
- false
800+
# Add any variations of a test app here
801+
# You should provide an alternate build-command as well as a matching label
802+
include:
803+
- test-application: 'create-react-app'
804+
build-command: 'test:build-ts3.8'
805+
label: 'create-react-app (TS 3.8)'
806+
- test-application: 'standard-frontend-react'
807+
build-command: 'test:build-ts3.8'
808+
label: 'standard-frontend-react (TS 3.8)'
746809

747810
steps:
748811
- name: Check out current commit (${{ needs.job_get_metadata.outputs.commit_label }})
@@ -761,22 +824,36 @@ jobs:
761824
env:
762825
DEPENDENCY_CACHE_KEY: ${{ needs.job_build.outputs.dependency_cache_key }}
763826

827+
- name: Restore tarball cache
828+
uses: actions/cache/restore@v3
829+
with:
830+
path: ${{ github.workspace }}/packages/*/*.tgz
831+
key: ${{ env.BUILD_CACHE_TARBALL_KEY }}
832+
764833
- name: Get node version
765834
id: versions
766835
run: |
767836
echo "echo node=$(jq -r '.volta.node' package.json)" >> $GITHUB_OUTPUT
768-
- name: Run E2E tests
837+
838+
- name: Validate Verdaccio
839+
run: yarn test:validate
840+
working-directory: packages/e2e-tests
841+
842+
- name: Prepare Verdaccio
843+
run: yarn test:prepare
844+
working-directory: packages/e2e-tests
769845
env:
770846
E2E_TEST_PUBLISH_SCRIPT_NODE_VERSION: ${{ steps.versions.outputs.node }}
771-
E2E_TEST_AUTH_TOKEN: ${{ secrets.E2E_TEST_AUTH_TOKEN }}
772-
E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }}
773-
E2E_TEST_SENTRY_ORG_SLUG: 'sentry-javascript-sdks'
774-
E2E_TEST_SENTRY_TEST_PROJECT: 'sentry-javascript-e2e-tests'
775-
E2E_TEST_SHARD: ${{ matrix.shard }}
776-
E2E_TEST_SHARD_AMOUNT: 4
777-
run: |
778-
cd packages/e2e-tests
779-
yarn test:e2e
847+
848+
- name: Build E2E app
849+
working-directory: packages/e2e-tests/test-applications/${{ matrix.test-application }}
850+
timeout-minutes: 5
851+
run: yarn ${{ matrix.build-command || 'test:build' }}
852+
853+
- name: Run E2E test
854+
working-directory: packages/e2e-tests/test-applications/${{ matrix.test-application }}
855+
timeout-minutes: 5
856+
run: yarn test:assert
780857

781858
job_required_tests:
782859
name: All required tests passed or skipped

‎.github/workflows/canary.yml

Lines changed: 92 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -17,41 +17,116 @@ permissions:
1717
issues: write
1818

1919
jobs:
20-
job_canary_test:
21-
name: Canary Tests
20+
job_e2e_prepare:
21+
name: Prepare E2E Canary tests
2222
runs-on: ubuntu-20.04
23-
timeout-minutes: 60
23+
timeout-minutes: 30
2424
steps:
25-
- name: 'Check out current commit'
25+
- name: Check out current commit
2626
uses: actions/checkout@v3
2727
with:
2828
ref: ${{ env.HEAD_COMMIT }}
29-
- uses: pnpm/action-setup@v2
30-
with:
31-
version: 8.3.1
3229
- name: Set up Node
3330
uses: actions/setup-node@v3
3431
with:
3532
node-version-file: 'package.json'
33+
- name: Check canary cache
34+
uses: actions/cache@v3
35+
with:
36+
path: |
37+
${{ github.workspace }}/packages/*/*.tgz
38+
${{ github.workspace }}/node_modules
39+
${{ github.workspace }}/packages/*/node_modules
40+
key: canary-${{ env.HEAD_COMMIT }}
3641
- name: Install dependencies
37-
run: yarn install --ignore-engines --frozen-lockfile
42+
run: yarn install
3843
- name: Build packages
3944
run: yarn build
45+
46+
- name: Build tarballs
47+
run: yarn build:tarball
48+
49+
job_e2e_tests:
50+
name: E2E ${{ matrix.label }} Test
51+
needs: [job_e2e_prepare]
52+
runs-on: ubuntu-20.04
53+
timeout-minutes: 15
54+
env:
55+
E2E_TEST_AUTH_TOKEN: ${{ secrets.E2E_TEST_AUTH_TOKEN }}
56+
E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }}
57+
# Needed because certain apps expect a certain prefix
58+
NEXT_PUBLIC_E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }}
59+
PUBLIC_E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }}
60+
REACT_APP_E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }}
61+
E2E_TEST_SENTRY_ORG_SLUG: 'sentry-javascript-sdks'
62+
E2E_TEST_SENTRY_TEST_PROJECT: 'sentry-javascript-e2e-tests'
63+
strategy:
64+
fail-fast: false
65+
matrix:
66+
include:
67+
- test-application: 'create-react-app'
68+
build-command: 'test:build-canary'
69+
label: 'create-react-app (canary)'
70+
- test-application: 'nextjs-app-dir'
71+
build-command: 'test:build-canary'
72+
label: 'nextjs-app-dir (canary)'
73+
- test-application: 'nextjs-app-dir'
74+
build-command: 'test:build-latest'
75+
label: 'nextjs-app-dir (latest)'
76+
- test-application: 'react-create-hash-router'
77+
build-command: 'test:build-canary'
78+
label: 'react-create-hash-router (canary)'
79+
- test-application: 'standard-frontend-react'
80+
build-command: 'test:build-canary'
81+
label: 'standard-frontend-react (canary)'
82+
83+
steps:
84+
- name: Check out current commit
85+
uses: actions/checkout@v3
86+
with:
87+
ref: ${{ env.HEAD_COMMIT }}
88+
- uses: pnpm/action-setup@v2
89+
with:
90+
version: 8.3.1
91+
- name: Set up Node
92+
uses: actions/setup-node@v3
93+
with:
94+
node-version-file: 'package.json'
95+
96+
- name: Restore canary cache
97+
uses: actions/cache/restore@v3
98+
with:
99+
path: |
100+
${{ github.workspace }}/packages/*/*.tgz
101+
${{ github.workspace }}/node_modules
102+
${{ github.workspace }}/packages/*/node_modules
103+
key: canary-${{ env.HEAD_COMMIT }}
104+
40105
- name: Get node version
41106
id: versions
42107
run: |
43108
echo "echo node=$(jq -r '.volta.node' package.json)" >> $GITHUB_OUTPUT
44-
- name: Run Canary Tests
109+
110+
- name: Validate Verdaccio
111+
run: yarn test:validate
112+
working-directory: packages/e2e-tests
113+
114+
- name: Prepare Verdaccio
115+
run: yarn test:prepare
116+
working-directory: packages/e2e-tests
45117
env:
46118
E2E_TEST_PUBLISH_SCRIPT_NODE_VERSION: ${{ steps.versions.outputs.node }}
47-
E2E_TEST_AUTH_TOKEN: ${{ secrets.E2E_TEST_AUTH_TOKEN }}
48-
E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }}
49-
E2E_TEST_SENTRY_ORG_SLUG: 'sentry-javascript-sdks'
50-
E2E_TEST_SENTRY_TEST_PROJECT: 'sentry-javascript-e2e-tests'
51-
CANARY_E2E_TEST: 'yes'
52-
run: |
53-
cd packages/e2e-tests
54-
yarn test:e2e
119+
120+
- name: Build E2E app
121+
working-directory: packages/e2e-tests/test-applications/${{ matrix.test-application }}
122+
timeout-minutes: 5
123+
run: yarn ${{ matrix.build-command }}
124+
125+
- name: Run E2E test
126+
working-directory: packages/e2e-tests/test-applications/${{ matrix.test-application }}
127+
timeout-minutes: 5
128+
run: yarn test:assert
129+
55130
- name: Create Issue
56131
if: failure() && github.event_name == 'schedule'
57132
uses: JasonEtco/create-an-issue@e27dddc79c92bc6e4562f268fffa5ed752639abd
@@ -101,4 +176,3 @@ jobs:
101176
with:
102177
filename: .github/CANARY_FAILURE_TEMPLATE.md
103178
update_existing: true
104-
title: 'Ember Canary tests failed'

‎packages/e2e-tests/package.json

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@
1818
"test:run": "ts-node run.ts",
1919
"test:validate-configuration": "ts-node validate-verdaccio-configuration.ts",
2020
"test:validate-test-app-setups": "ts-node validate-test-app-setups.ts",
21+
"test:prepare": "ts-node prepare.ts",
22+
"test:validate": "run-s test:validate-configuration test:validate-test-app-setups",
2123
"clean": "rimraf tmp test-applications/**/node_modules test-applications/**/dist"
2224
},
2325
"devDependencies": {

‎packages/e2e-tests/prepare.ts

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
/* eslint-disable max-lines */
2+
/* eslint-disable no-console */
3+
import * as dotenv from 'dotenv';
4+
5+
import { validate } from './lib/validate';
6+
import { registrySetup } from './registrySetup';
7+
8+
async function run(): Promise<void> {
9+
// Load environment variables from .env file locally
10+
dotenv.config();
11+
12+
if (!validate()) {
13+
process.exit(1);
14+
}
15+
16+
try {
17+
registrySetup();
18+
} catch (error) {
19+
console.error(error);
20+
process.exit(1);
21+
}
22+
}
23+
24+
void run();

‎packages/e2e-tests/publish-packages.ts

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,6 @@ import * as path from 'path';
55

66
const repositoryRoot = path.resolve(__dirname, '../..');
77

8-
// Create tarballs
9-
childProcess.execSync('yarn build:tarball', { encoding: 'utf8', cwd: repositoryRoot, stdio: 'inherit' });
10-
118
// Get absolute paths of all the packages we want to publish to the fake registry
129
const packageTarballPaths = glob.sync('packages/*/sentry-*.tgz', {
1310
cwd: repositoryRoot,

‎packages/e2e-tests/run.ts

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,8 @@ async function run(): Promise<void> {
1616
}
1717

1818
const envVarsToInject = {
19-
REACT_APP_E2E_TEST_DSN: process.env.E2E_TEST_DSN,
20-
REMIX_APP_E2E_TEST_DSN: process.env.E2E_TEST_DSN,
2119
NEXT_PUBLIC_E2E_TEST_DSN: process.env.E2E_TEST_DSN,
2220
PUBLIC_E2E_TEST_DSN: process.env.E2E_TEST_DSN,
23-
BASE_PORT: '27496', // just some random port
2421
};
2522

2623
try {

‎packages/e2e-tests/test-applications/create-next-app/package.json

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,9 @@
55
"scripts": {
66
"build": "next build",
77
"test:prod": "TEST_ENV=prod playwright test",
8-
"test:dev": "TEST_ENV=dev playwright test"
8+
"test:dev": "TEST_ENV=dev playwright test",
9+
"test:build": "pnpm install && npx playwright install && pnpm build",
10+
"test:assert": "pnpm test:prod && pnpm test:dev"
911
},
1012
"dependencies": {
1113
"@next/font": "13.0.7",

‎packages/e2e-tests/test-applications/create-next-app/playwright.config.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ if (!testEnv) {
77
throw new Error('No test env defined');
88
}
99

10-
const port = Number(process.env.BASE_PORT) + Number(process.env.PORT_MODULO);
10+
const port = 9000;
1111

1212
/**
1313
* See https://playwright.dev/docs/test-configuration.

‎packages/e2e-tests/test-applications/create-react-app/package.json

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,11 @@
2222
"start": "react-scripts start",
2323
"build": "react-scripts build",
2424
"test": "react-scripts test",
25-
"eject": "react-scripts eject"
25+
"eject": "react-scripts eject",
26+
"test:build": "pnpm install && pnpm build",
27+
"test:build-ts3.8": "pnpm install && pnpm add [email protected] && pnpm build",
28+
"test:build-canary": "pnpm install && pnpm add react@canary react-dom@canary && pnpm build",
29+
"test:assert": "pnpm -v"
2630
},
2731
"eslintConfig": {
2832
"extends": [

‎packages/e2e-tests/test-applications/create-remix-app/app/entry.client.tsx

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ import { hydrateRoot } from 'react-dom/client';
1010
import * as Sentry from '@sentry/remix';
1111

1212
Sentry.init({
13-
dsn: process.env.REMIX_APP_E2E_TEST_DSN,
13+
dsn: process.env.E2E_TEST_DSN,
1414
integrations: [
1515
new Sentry.BrowserTracing({
1616
routingInstrumentation: Sentry.remixRouterInstrumentation(useEffect, useLocation, useMatches),

‎packages/e2e-tests/test-applications/create-remix-app/app/entry.server.tsx

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ import * as Sentry from '@sentry/remix';
1616
const ABORT_DELAY = 5_000;
1717

1818
Sentry.init({
19-
dsn: process.env.REMIX_APP_E2E_TEST_DSN,
19+
dsn: process.env.E2E_TEST_DSN,
2020
// Performance Monitoring
2121
tracesSampleRate: 1.0, // Capture 100% of the transactions, reduce in production!
2222
});

‎packages/e2e-tests/test-applications/create-remix-app/package.json

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,9 @@
55
"build": "remix build",
66
"dev": "remix dev",
77
"start": "remix-serve build",
8-
"typecheck": "tsc"
8+
"typecheck": "tsc",
9+
"test:build": "pnpm install && pnpm build",
10+
"test:assert": "pnpm -v"
911
},
1012
"dependencies": {
1113
"@sentry/remix": "*",

0 commit comments

Comments
 (0)