Skip to content

Commit 908b2b9

Browse files
author
Luca Forstner
committed
Add logic
1 parent 30e978b commit 908b2b9

File tree

17 files changed

+274
-12
lines changed

17 files changed

+274
-12
lines changed

‎packages/e2e-tests/.eslintrc.js

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ module.exports = {
33
node: true,
44
},
55
extends: ['../../.eslintrc.js'],
6-
ignorePatterns: [],
6+
ignorePatterns: ['test-applications/**'],
77
parserOptions: {
88
sourceType: 'module',
99
},

‎packages/e2e-tests/run.ts

Lines changed: 149 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
/* eslint-disable no-console */
22
import * as childProcess from 'child_process';
3+
import * as fs from 'fs';
4+
import * as glob from 'glob';
35
import * as path from 'path';
46

57
const repositoryRoot = path.resolve(__dirname, '../..');
@@ -11,46 +13,85 @@ const PUBLISH_PACKAGES_DOCKER_IMAGE_NAME = 'publish-packages';
1113

1214
const publishScriptNodeVersion = process.env.E2E_TEST_PUBLISH_SCRIPT_NODE_VERSION;
1315

16+
const DEFAULT_TEST_TIMEOUT_SECONDS = 60;
17+
1418
// https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#grouping-log-lines
1519
function groupCIOutput(groupTitle: string, fn: () => void): void {
1620
if (process.env.CI) {
17-
console.log(`::group::{${groupTitle}}`);
21+
console.log(`::group::${groupTitle}`);
1822
fn();
1923
console.log('::endgroup::');
2024
} else {
2125
fn();
2226
}
2327
}
2428

29+
// https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-error-message
30+
function printCIErrorMessage(message: string): void {
31+
if (process.env.CI) {
32+
console.log(`::error::${message}`);
33+
} else {
34+
console.log(message);
35+
}
36+
}
37+
2538
groupCIOutput('Test Registry Setup', () => {
2639
try {
2740
// Stop test registry container (Verdaccio) if it was already running
28-
childProcess.execSync(`docker stop ${TEST_REGISTRY_CONTAINER_NAME}`, { stdio: 'ignore' });
41+
childProcess.spawnSync('docker', ['stop', TEST_REGISTRY_CONTAINER_NAME], { stdio: 'ignore' });
2942
console.log('Stopped previously running test registry');
3043
} catch (e) {
3144
// Don't throw if container wasn't running
3245
}
3346

3447
// Start test registry (Verdaccio)
35-
childProcess.execSync(
36-
`docker run --detach --rm --name ${TEST_REGISTRY_CONTAINER_NAME} -p 4873:4873 -v ${__dirname}/verdaccio-config:/verdaccio/conf verdaccio/verdaccio:${VERDACCIO_VERSION}`,
48+
childProcess.spawnSync(
49+
'docker',
50+
[
51+
'run',
52+
'--detach',
53+
'--rm',
54+
'--name',
55+
TEST_REGISTRY_CONTAINER_NAME,
56+
'-p',
57+
'4873:4873',
58+
'-v',
59+
`${__dirname}/verdaccio-config:/verdaccio/conf`,
60+
`verdaccio/verdaccio:${VERDACCIO_VERSION}`,
61+
],
3762
{ encoding: 'utf8', stdio: 'inherit' },
3863
);
3964

4065
// Build container image that is uploading our packages to fake registry with specific Node.js/npm version
41-
childProcess.execSync(
42-
`docker build --tag ${PUBLISH_PACKAGES_DOCKER_IMAGE_NAME} --file ./Dockerfile.publish-packages ${
43-
publishScriptNodeVersion ? `--build-arg NODE_VERSION=${publishScriptNodeVersion}` : ''
44-
} .`,
66+
childProcess.spawnSync(
67+
'docker',
68+
[
69+
'build',
70+
'--tag',
71+
PUBLISH_PACKAGES_DOCKER_IMAGE_NAME,
72+
'--file',
73+
'./Dockerfile.publish-packages',
74+
publishScriptNodeVersion ? `--build-arg NODE_VERSION=${publishScriptNodeVersion}` : undefined,
75+
'.',
76+
].filter((arg): arg is string => arg !== undefined),
4577
{
4678
encoding: 'utf8',
4779
stdio: 'inherit',
4880
},
4981
);
5082

5183
// Run container that uploads our packages to fake registry
52-
childProcess.execSync(
53-
`docker run --rm -v ${repositoryRoot}:/sentry-javascript --network host ${PUBLISH_PACKAGES_DOCKER_IMAGE_NAME}`,
84+
childProcess.spawnSync(
85+
'docker',
86+
[
87+
'run',
88+
'--rm',
89+
'-v',
90+
`${repositoryRoot}:/sentry-javascript`,
91+
'--network',
92+
'host',
93+
PUBLISH_PACKAGES_DOCKER_IMAGE_NAME,
94+
],
5495
{
5596
encoding: 'utf8',
5697
stdio: 'inherit',
@@ -60,10 +101,107 @@ groupCIOutput('Test Registry Setup', () => {
60101

61102
groupCIOutput('Run E2E Test Suites', () => {
62103
// TODO: Run e2e tests here
104+
const recipePaths = glob.sync(`${__dirname}/test-applications/*/test-recipe.json`, { absolute: true });
105+
106+
const recipeResults = recipePaths.map(recipePath => {
107+
type Recipe = {
108+
testApplicationName: string;
109+
buildCommand?: string;
110+
tests: {
111+
testName: string;
112+
testCommand: string;
113+
timeoutSeconds?: number;
114+
}[];
115+
};
116+
117+
const recipe: Recipe = JSON.parse(fs.readFileSync(recipePath, 'utf-8'));
118+
119+
if (recipe.buildCommand) {
120+
console.log(`Running E2E test build command for test application "${recipe.testApplicationName}"`);
121+
const [buildCommand, ...buildCommandArgs] = recipe.buildCommand.split(' ');
122+
childProcess.spawnSync(buildCommand, buildCommandArgs, {
123+
cwd: path.dirname(recipePath),
124+
encoding: 'utf8',
125+
stdio: 'inherit',
126+
});
127+
}
128+
129+
type TestResult = {
130+
testName: string;
131+
result: 'PASS' | 'FAIL' | 'TIMEOUT';
132+
};
133+
134+
const testResults: TestResult[] = recipe.tests.map(test => {
135+
console.log(
136+
`Running E2E test command for test application "${recipe.testApplicationName}", test "${test.testName}"`,
137+
);
138+
139+
const [testCommand, ...testCommandArgs] = test.testCommand.split(' ');
140+
const testProcessResult = childProcess.spawnSync(testCommand, testCommandArgs, {
141+
cwd: path.dirname(recipePath),
142+
timeout: (test.timeoutSeconds ?? DEFAULT_TEST_TIMEOUT_SECONDS) * 1000,
143+
encoding: 'utf8',
144+
stdio: 'pipe',
145+
});
146+
147+
console.log(testProcessResult.stdout.replace(/^/gm, '[TEST OUTPUT] '));
148+
console.log(testProcessResult.stderr.replace(/^/gm, '[TEST OUTPUT] '));
149+
150+
const error: undefined | (Error & { code?: string }) = testProcessResult.error;
151+
152+
if (error?.code === 'ETIMEDOUT') {
153+
printCIErrorMessage(
154+
`Test "${test.testName}" in test application "${recipe.testApplicationName}" (${path.dirname(
155+
recipePath,
156+
)}) timed out.`,
157+
);
158+
return {
159+
testName: test.testName,
160+
result: 'TIMEOUT',
161+
};
162+
} else if (testProcessResult.status !== 0) {
163+
printCIErrorMessage(
164+
`Test "${test.testName}" in test application "${recipe.testApplicationName}" (${path.dirname(
165+
recipePath,
166+
)}) failed.`,
167+
);
168+
return {
169+
testName: test.testName,
170+
result: 'FAIL',
171+
};
172+
} else {
173+
console.log(
174+
`Test "${test.testName}" in test application "${recipe.testApplicationName}" (${path.dirname(
175+
recipePath,
176+
)}) succeeded.`,
177+
);
178+
return {
179+
testName: test.testName,
180+
result: 'PASS',
181+
};
182+
}
183+
});
184+
185+
return {
186+
testApplicationName: recipe.testApplicationName,
187+
testApplicationPath: recipePath,
188+
testResults,
189+
};
190+
});
191+
192+
console.log('--------------------------------------');
193+
console.log('Test Result Summary:');
194+
195+
recipeResults.forEach(recipeResult => {
196+
console.log(`● ${recipeResult.testApplicationName} (${path.dirname(recipeResult.testApplicationPath)})`);
197+
recipeResult.testResults.forEach(testResult => {
198+
console.log(` ● ${testResult.result.padEnd(7, ' ')} ${testResult.testName}`);
199+
});
200+
});
63201
});
64202

65203
groupCIOutput('Cleanup', () => {
66204
// Stop test registry
67-
childProcess.execSync(`docker stop ${TEST_REGISTRY_CONTAINER_NAME}`, { encoding: 'utf8', stdio: 'ignore' });
205+
childProcess.spawnSync(`docker stop ${TEST_REGISTRY_CONTAINER_NAME}`, { encoding: 'utf8', stdio: 'ignore' });
68206
console.log('Successfully stopped test registry container'); // Output from command above is not good so we `ignore` it and emit our own
69207
});

‎packages/e2e-tests/test-applications/temporary-app-1/.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
yarn.lock

‎packages/e2e-tests/test-applications/temporary-app-1/.npmrc

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
@sentry:registry=http://localhost:4873
2+
@sentry-internal:registry=http://localhost:4873
3+
//localhost:4873/:_authToken=some-token

‎packages/e2e-tests/test-applications/temporary-app-1/bad.js

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
throw new Error('Sad :(');

‎packages/e2e-tests/test-applications/temporary-app-1/good.js

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
console.log('Happy :)');

‎packages/e2e-tests/test-applications/temporary-app-1/package.json

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
{
2+
"name": "temporary-app-1",
3+
"version": "1.0.0",
4+
"private": true,
5+
"scripts": {
6+
"start:good": "node good.js",
7+
"start:bad": "node bad.js",
8+
"start:timeout": "node timeout.js"
9+
},
10+
"dependencies": {
11+
"@sentry/node": "*"
12+
}
13+
}

‎packages/e2e-tests/test-applications/temporary-app-1/test-recipe.json

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
{
2+
"$schema": "../../test-recipe-schema.json",
3+
"testApplicationName": "Temporary Application 1",
4+
"buildCommand": "yarn install",
5+
"tests": [
6+
{
7+
"testName": "Example Test (Should Succeed)",
8+
"testCommand": "yarn start:good",
9+
"timeoutSeconds": 30
10+
},
11+
{
12+
"testName": "Example Test (Should Fail)",
13+
"testCommand": "yarn start:bad"
14+
},
15+
{
16+
"testName": "Example Test (Should time out)",
17+
"testCommand": "yarn start:timeout",
18+
"timeoutSeconds": 5
19+
}
20+
]
21+
}

‎packages/e2e-tests/test-applications/temporary-app-1/timeout.js

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
setTimeout(() => {
2+
console.log('Bored :/');
3+
}, 6000);

‎packages/e2e-tests/test-applications/temporary-app-2/.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
yarn.lock

‎packages/e2e-tests/test-applications/temporary-app-2/.npmrc

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
@sentry:registry=http://localhost:4873
2+
@sentry-internal:registry=http://localhost:4873
3+
//localhost:4873/:_authToken=some-token

‎packages/e2e-tests/test-applications/temporary-app-2/bad.js

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
throw new Error('Sad :(');

‎packages/e2e-tests/test-applications/temporary-app-2/good.js

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
console.log('Happy :)');

‎packages/e2e-tests/test-applications/temporary-app-2/package.json

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
{
2+
"name": "temporary-app-2",
3+
"version": "1.0.0",
4+
"private": true,
5+
"scripts": {
6+
"start:good": "node good.js",
7+
"start:bad": "node bad.js",
8+
"start:timeout": "node timeout.js"
9+
},
10+
"dependencies": {
11+
"@sentry/node": "*"
12+
}
13+
}

‎packages/e2e-tests/test-applications/temporary-app-2/test-recipe.json

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
{
2+
"$schema": "../../test-recipe-schema.json",
3+
"testApplicationName": "Temporary Application 2",
4+
"buildCommand": "yarn install",
5+
"tests": [
6+
{
7+
"testName": "Example Test (Should Succeed)",
8+
"testCommand": "yarn start:good",
9+
"timeoutSeconds": 60
10+
},
11+
{
12+
"testName": "Example Test (Should Fail)",
13+
"testCommand": "yarn start:bad"
14+
},
15+
{
16+
"testName": "Example Test (Should time out)",
17+
"testCommand": "yarn start:timeout",
18+
"timeoutSeconds": 5
19+
}
20+
]
21+
}

‎packages/e2e-tests/test-applications/temporary-app-2/timeout.js

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
setTimeout(() => {
2+
console.log('Bored :/');
3+
}, 6000);

‎packages/e2e-tests/test-recipe-schema.json

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
{
2+
"$schema": "https://json-schema.org/draft/2020-12/schema",
3+
"title": "Sentry JavaScript E2E Test Recipe",
4+
"type": "object",
5+
"properties": {
6+
"testApplicationName": {
7+
"type": "string",
8+
"description": "Name displayed in test output"
9+
},
10+
"buildCommand": {
11+
"type": "string",
12+
"description": "Command that is run to install dependencies and build the test application. This command is only run once before all tests. Working directory of the command is the root of the test application."
13+
},
14+
"tests": {
15+
"type": "array",
16+
"description": "Tests to run in this test application",
17+
"items": {
18+
"type": "object",
19+
"properties": {
20+
"testName": {
21+
"type": "string",
22+
"description": "Name displayed in test output"
23+
},
24+
"testCommand": {
25+
"type": "string",
26+
"description": "Command that is run to start the test. Working directory of the command is the root of the test application. If this command returns a non-zero exit code the test counts as failed."
27+
},
28+
"timeoutSeconds": {
29+
"type": "number",
30+
"description": "Test timeout in seconds. Default: 60"
31+
}
32+
},
33+
"required": ["testName", "testCommand"]
34+
}
35+
}
36+
},
37+
"required": ["testApplicationName", "tests"]
38+
}

0 commit comments

Comments
 (0)