Skip to content

Commit b07f9a5

Browse files
rchiodorchiodo
authored
Split functional tests into per file so they don't run out of memory during failures (#13534)
* Split functional tests on each file * Try js instead * Actually use node instead of python * More logging * More logging * More logging * Try a different way to run to see if python works * Remove grep * Add comment about environment * Try fixing the infinite loop on session shutdown * Fix file system failures * Increase timeout and fix flakey test * Fix path comparisons * Add more indicators on the output name * Remove unused file * Fix unit tests * Remove splitting on PRs and fix the current directory test
1 parent b57bc1f commit b07f9a5

16 files changed

+245
-148
lines changed

‎.gitignore

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ obj/**
3232
tmp/**
3333
.python-version
3434
.vs/
35-
test-results.xml
35+
test-results*.xml
3636
xunit-test-results.xml
3737
build/ci/performance/performance-results.json
3838
!build/

‎.vscode/launch.json

Lines changed: 47 additions & 100 deletions
Original file line numberDiff line numberDiff line change
@@ -2,25 +2,36 @@
22
{
33
"version": "0.1.0",
44
"configurations": [
5+
{
6+
"name": "Node: Current File",
7+
"program": "${file}",
8+
"request": "launch",
9+
"skipFiles": [
10+
"<node_internals>/**"
11+
],
12+
"type": "pwa-node"
13+
},
14+
{
15+
"name": "Python: Current File",
16+
"type": "python",
17+
"justMyCode": true,
18+
"request": "launch",
19+
"program": "${file}",
20+
"console": "integratedTerminal",
21+
"cwd": "${workspaceFolder}"
22+
},
523
{
624
"name": "Extension",
725
"type": "extensionHost",
826
"request": "launch",
927
"runtimeExecutable": "${execPath}",
10-
"args": [
11-
"--extensionDevelopmentPath=${workspaceFolder}"
12-
],
28+
"args": ["--extensionDevelopmentPath=${workspaceFolder}"],
1329
"stopOnEntry": false,
1430
"smartStep": true,
1531
"sourceMaps": true,
16-
"outFiles": [
17-
"${workspaceFolder}/out/**/*",
18-
"!${workspaceFolder}/**/node_modules**/*"
19-
],
32+
"outFiles": ["${workspaceFolder}/out/**/*", "!${workspaceFolder}/**/node_modules**/*"],
2033
"preLaunchTask": "Compile",
21-
"skipFiles": [
22-
"<node_internals>/**"
23-
],
34+
"skipFiles": ["<node_internals>/**"],
2435
"env": {
2536
// Enable this to turn on redux logging during debugging
2637
"XVSC_PYTHON_FORCE_LOGGING": "1",
@@ -37,40 +48,27 @@
3748
"type": "extensionHost",
3849
"request": "launch",
3950
"runtimeExecutable": "${execPath}",
40-
"args": [
41-
"--extensionDevelopmentPath=${workspaceFolder}"
42-
],
51+
"args": ["--extensionDevelopmentPath=${workspaceFolder}"],
4352
"stopOnEntry": false,
4453
"smartStep": true,
4554
"sourceMaps": true,
46-
"outFiles": [
47-
"${workspaceFolder}/out/**/*",
48-
"!${workspaceFolder}/**/node_modules**/*"
49-
],
55+
"outFiles": ["${workspaceFolder}/out/**/*", "!${workspaceFolder}/**/node_modules**/*"],
5056
"preLaunchTask": "Inject DS WebBrowser UI",
5157
"env": {
5258
"VSC_PYTHON_DS_UI_PROMPT": "1"
5359
},
54-
"skipFiles": [
55-
"<node_internals>/**"
56-
]
60+
"skipFiles": ["<node_internals>/**"]
5761
},
5862
{
5963
"name": "Extension inside container",
6064
"type": "extensionHost",
6165
"request": "launch",
6266
"runtimeExecutable": "${execPath}",
63-
"args": [
64-
"--extensionDevelopmentPath=${workspaceFolder}",
65-
"${workspaceFolder}/data"
66-
],
67+
"args": ["--extensionDevelopmentPath=${workspaceFolder}", "${workspaceFolder}/data"],
6768
"stopOnEntry": false,
6869
"smartStep": true,
6970
"sourceMaps": true,
70-
"outFiles": [
71-
"${workspaceFolder}/out/**/*",
72-
"!${workspaceFolder}/**/node_modules**/*"
73-
],
71+
"outFiles": ["${workspaceFolder}/out/**/*", "!${workspaceFolder}/**/node_modules**/*"],
7472
"preLaunchTask": "Compile"
7573
},
7674
{
@@ -79,9 +77,7 @@
7977
"request": "launch",
8078
"module": "IPython",
8179
"console": "integratedTerminal",
82-
"args": [
83-
"${file}"
84-
] // Additional args should be prefixed with a '--' first.
80+
"args": ["${file}"] // Additional args should be prefixed with a '--' first.
8581
},
8682
{
8783
"name": "Python: Current File",
@@ -104,17 +100,12 @@
104100
"stopOnEntry": false,
105101
"sourceMaps": true,
106102
"smartStep": true,
107-
"outFiles": [
108-
"${workspaceFolder}/out/**/*",
109-
"!${workspaceFolder}/**/node_modules**/*"
110-
],
103+
"outFiles": ["${workspaceFolder}/out/**/*", "!${workspaceFolder}/**/node_modules**/*"],
111104
"preLaunchTask": "Compile",
112105
"env": {
113106
"IS_CI_SERVER_TEST_DEBUGGER": "1"
114107
},
115-
"skipFiles": [
116-
"<node_internals>/**"
117-
]
108+
"skipFiles": ["<node_internals>/**"]
118109
},
119110
{
120111
// Note, for the smoke test you want to debug, you may need to copy the file,
@@ -134,14 +125,9 @@
134125
},
135126
"stopOnEntry": false,
136127
"sourceMaps": true,
137-
"outFiles": [
138-
"${workspaceFolder}/out/**/*.js",
139-
"!${workspaceFolder}/**/node_modules**/*"
140-
],
128+
"outFiles": ["${workspaceFolder}/out/**/*.js", "!${workspaceFolder}/**/node_modules**/*"],
141129
"preLaunchTask": "Compile",
142-
"skipFiles": [
143-
"<node_internals>/**"
144-
]
130+
"skipFiles": ["<node_internals>/**"]
145131
},
146132
{
147133
"name": "Tests (Single Workspace, VS Code, *.test.ts)",
@@ -159,14 +145,9 @@
159145
},
160146
"stopOnEntry": false,
161147
"sourceMaps": true,
162-
"outFiles": [
163-
"${workspaceFolder}/out/**/*.js",
164-
"!${workspaceFolder}/**/node_modules**/*"
165-
],
148+
"outFiles": ["${workspaceFolder}/out/**/*.js", "!${workspaceFolder}/**/node_modules**/*"],
166149
"preLaunchTask": "Compile",
167-
"skipFiles": [
168-
"<node_internals>/**"
169-
]
150+
"skipFiles": ["<node_internals>/**"]
170151
},
171152
{
172153
"name": "Tests (DataScience, *.ds.test.ts)",
@@ -189,14 +170,9 @@
189170
},
190171
"stopOnEntry": false,
191172
"sourceMaps": true,
192-
"outFiles": [
193-
"${workspaceFolder}/out/**/*.js",
194-
"!${workspaceFolder}/**/node_modules**/*"
195-
],
173+
"outFiles": ["${workspaceFolder}/out/**/*.js", "!${workspaceFolder}/**/node_modules**/*"],
196174
"preLaunchTask": "Compile",
197-
"skipFiles": [
198-
"<node_internals>/**"
199-
]
175+
"skipFiles": ["<node_internals>/**"]
200176
},
201177
{
202178
"name": "Tests (Multiroot, VS Code, *.test.ts)",
@@ -212,14 +188,9 @@
212188
"stopOnEntry": false,
213189
"sourceMaps": true,
214190
"smartStep": true,
215-
"outFiles": [
216-
"${workspaceFolder}/out/**/*",
217-
"!${workspaceFolder}/**/node_modules**/*"
218-
],
191+
"outFiles": ["${workspaceFolder}/out/**/*", "!${workspaceFolder}/**/node_modules**/*"],
219192
"preLaunchTask": "Compile",
220-
"skipFiles": [
221-
"<node_internals>/**"
222-
]
193+
"skipFiles": ["<node_internals>/**"]
223194
},
224195
{
225196
"name": "Unit Tests (without VS Code, *.unit.test.ts)",
@@ -237,14 +208,9 @@
237208
//"--grep", "<suite name>",
238209
"--timeout=300000"
239210
],
240-
"outFiles": [
241-
"${workspaceFolder}/out/**/*.js",
242-
"!${workspaceFolder}/**/node_modules**/*"
243-
],
211+
"outFiles": ["${workspaceFolder}/out/**/*.js", "!${workspaceFolder}/**/node_modules**/*"],
244212
"preLaunchTask": "Compile",
245-
"skipFiles": [
246-
"<node_internals>/**"
247-
]
213+
"skipFiles": ["<node_internals>/**"]
248214
},
249215
{
250216
"name": "Unit Tests (fast, without VS Code and without react/monaco, *.unit.test.ts)",
@@ -263,14 +229,9 @@
263229
"--timeout=300000",
264230
"--fast"
265231
],
266-
"outFiles": [
267-
"${workspaceFolder}/out/**/*.js",
268-
"!${workspaceFolder}/**/node_modules**/*"
269-
],
232+
"outFiles": ["${workspaceFolder}/out/**/*.js", "!${workspaceFolder}/**/node_modules**/*"],
270233
"preLaunchTask": "Compile",
271-
"skipFiles": [
272-
"<node_internals>/**"
273-
]
234+
"skipFiles": ["<node_internals>/**"]
274235
},
275236
{
276237
"name": "Functional Tests (without VS Code, *.functional.test.ts)",
@@ -301,14 +262,9 @@
301262
// Remove 'X' prefix to dump output for debugger. Directory has to exist prior to launch
302263
"XDEBUGPY_LOG_DIR": "${workspaceRoot}/tmp/Debug_Output"
303264
},
304-
"outFiles": [
305-
"${workspaceFolder}/out/**/*.js",
306-
"!${workspaceFolder}/**/node_modules**/*"
307-
],
265+
"outFiles": ["${workspaceFolder}/out/**/*.js", "!${workspaceFolder}/**/node_modules**/*"],
308266
"preLaunchTask": "Compile",
309-
"skipFiles": [
310-
"<node_internals>/**"
311-
]
267+
"skipFiles": ["<node_internals>/**"]
312268
},
313269
{
314270
"name": "Functional DS UI Tests (without VS Code, *.ui.functional.test.ts)",
@@ -337,26 +293,17 @@
337293
// Remove `X` prefix and update path to test with real python interpreter (for DS functional tests).
338294
"XCI_PYTHON_PATH": "<Python Path>"
339295
},
340-
"outFiles": [
341-
"${workspaceFolder}/out/**/*.js",
342-
"!${workspaceFolder}/**/node_modules**/*"
343-
],
296+
"outFiles": ["${workspaceFolder}/out/**/*.js", "!${workspaceFolder}/**/node_modules**/*"],
344297
"preLaunchTask": "Compile",
345-
"skipFiles": [
346-
"<node_internals>/**"
347-
]
298+
"skipFiles": ["<node_internals>/**"]
348299
},
349300
{
350301
"type": "node",
351302
"request": "launch",
352303
"name": "Gulp tasks (helpful for debugging gulpfile.js)",
353304
"program": "${workspaceFolder}/node_modules/gulp/bin/gulp.js",
354-
"args": [
355-
"watch"
356-
],
357-
"skipFiles": [
358-
"<node_internals>/**"
359-
]
305+
"args": ["watch"],
306+
"skipFiles": ["<node_internals>/**"]
360307
}
361308
]
362309
}

‎build/ci/scripts/runFunctionalTests.js

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
// This script will run all of the functional tests for each functional test file in sequence
2+
// This prevents mocha from running out of memory when running all of the tests
3+
//
4+
// This could potentially be improved to run tests in parallel (try that later)
5+
//
6+
// Additionally this was written in python at first but running python on an azure
7+
// machine may pick up an invalid environment for the subprocess. Node doesn't have this problem
8+
var path = require('path');
9+
var glob = require('glob');
10+
var child_process = require('child_process');
11+
12+
// Create a base for the output file
13+
var originalMochaFile = process.env['MOCHA_FILE'];
14+
var mochaFile = originalMochaFile || './test-results.xml';
15+
var mochaBaseFile = path.join(path.dirname(mochaFile), path.basename(mochaFile, '.xml'));
16+
var mochaFileExt = '.xml';
17+
18+
// Wrap async code in a function so can wait till done
19+
async function main() {
20+
console.log('Globbing files for functional tests');
21+
22+
// Glob all of the files that we usually send to mocha as a group (see mocha.functional.opts.xml)
23+
var files = await new Promise((resolve, reject) => {
24+
glob('./out/test/**/*.functional.test.js', (ex, res) => {
25+
if (ex) {
26+
reject(ex);
27+
} else {
28+
resolve(res);
29+
}
30+
});
31+
});
32+
33+
// Iterate over them, running mocha on each
34+
var returnCode = 0;
35+
36+
// Go through each one at a time
37+
try {
38+
for (var index = 0; index < files.length; index += 1) {
39+
// Each run with a file will expect a $MOCHA_FILE$ variable. Generate one for each
40+
// Note: this index is used as a pattern when setting mocha file in the test_phases.yml
41+
var subMochaFile = `${mochaBaseFile}_${index}_${path.basename(files[index])}${mochaFileExt}`;
42+
process.env['MOCHA_FILE'] = subMochaFile;
43+
var exitCode = await new Promise((resolve) => {
44+
// Spawn the sub node process
45+
var proc = child_process.fork('./node_modules/mocha/bin/_mocha', [
46+
files[index],
47+
'--require=out/test/unittests.js',
48+
'--exclude=out/**/*.jsx',
49+
'--reporter=mocha-multi-reporters',
50+
'--reporter-option=configFile=build/.mocha-multi-reporters.config',
51+
'--ui=tdd',
52+
'--recursive',
53+
'--colors',
54+
'--exit',
55+
'--timeout=180000'
56+
]);
57+
proc.on('exit', resolve);
58+
});
59+
60+
// If failed keep track
61+
if (exitCode !== 0) {
62+
console.log(`Functional tests for ${files[index]} failed.`);
63+
returnCode = exitCode;
64+
}
65+
}
66+
} catch (ex) {
67+
console.log(`Functional tests run failure: ${ex}.`);
68+
returnCode = -1;
69+
}
70+
71+
// Reset the mocha file variable
72+
if (originalMochaFile) {
73+
process.env['MOCHA_FILE'] = originalMochaFile;
74+
}
75+
76+
// Indicate error code
77+
console.log(`Functional test run result: ${returnCode}`);
78+
process.exit(returnCode);
79+
}
80+
81+
// Call the main function. It will exit when promise is finished.
82+
main();

0 commit comments

Comments
 (0)