Skip to content

Commit b78d7a2

Browse files
plemarquandplemarquand
committed
Pull issues from test output during parallel run
Use a combination of the xunit xml output and the test output to capture test pass/fail state and issues.
1 parent 97972d1 commit b78d7a2

File tree

6 files changed

+223
-91
lines changed

6 files changed

+223
-91
lines changed

‎src/TestExplorer/TestParsers/XCTestOutputParser.ts

Lines changed: 65 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414

1515
import { ITestRunState } from "./TestRunState";
1616
import { sourceLocationToVSCodeLocation } from "../../utilities/utilities";
17+
import { MarkdownString, Location } from "vscode";
1718

1819
/** Regex for parsing XCTest output */
1920
interface TestRegex {
@@ -67,7 +68,69 @@ export const nonDarwinTestRegex = {
6768
failedSuite: /^Test Suite '(.*)' failed/,
6869
};
6970

70-
export class XCTestOutputParser {
71+
export interface IXCTestOutputParser {
72+
parseResult(output: string, runState: ITestRunState): void;
73+
}
74+
75+
export class ParallelXCTestOutputParser implements IXCTestOutputParser {
76+
private outputParser: XCTestOutputParser;
77+
78+
/**
79+
* Create an ParallelXCTestOutputParser.
80+
* Optional regex can be supplied for tests.
81+
*/
82+
constructor(
83+
private hasMultiLineParallelTestOutput: boolean,
84+
regex?: TestRegex
85+
) {
86+
this.outputParser = new XCTestOutputParser(regex);
87+
}
88+
89+
public parseResult(output: string, runState: ITestRunState) {
90+
// From 5.7 to 5.10 running with the --parallel option dumps the test results out
91+
// to the console with no newlines, so it isn't possible to distinguish where errors
92+
// begin and end. Consequently we can't record them, and so we manually mark them
93+
// as passed or failed here with a manufactured issue.
94+
// Don't attempt to parse the console output of parallel tests between 5.7 and 5.10
95+
// as it doesn't have newlines. You might get lucky and find the output is split
96+
// in the right spot, but more often than not we wont be able to parse it.
97+
if (!this.hasMultiLineParallelTestOutput) {
98+
return;
99+
}
100+
101+
// For parallel XCTest runs we get pass/fail results from the xunit XML
102+
// produced at the end of the run, but we still want to monitor the output
103+
// for the individual assertion failures. Wrap the run state and only forward
104+
// along the issues captured during a test run, and let the `TestXUnitParser`
105+
// handle marking tests as completed.
106+
this.outputParser.parseResult(output, new ParallelXCTestRunStateProxy(runState));
107+
}
108+
}
109+
110+
/* eslint-disable @typescript-eslint/no-unused-vars */
111+
class ParallelXCTestRunStateProxy implements ITestRunState {
112+
constructor(private runState: ITestRunState) {}
113+
114+
getTestItemIndex(id: string, filename: string | undefined): number {
115+
return this.runState.getTestItemIndex(id, filename);
116+
}
117+
recordIssue(
118+
index: number,
119+
message: string | MarkdownString,
120+
location?: Location | undefined
121+
): void {
122+
this.runState.recordIssue(index, message, location);
123+
}
124+
started(index: number, startTime?: number | undefined): void {}
125+
completed(index: number, timing: { duration: number } | { timestamp: number }): void {}
126+
skipped(index: number): void {}
127+
startedSuite(name: string): void {}
128+
passedSuite(name: string): void {}
129+
failedSuite(name: string): void {}
130+
}
131+
/* eslint-enable @typescript-eslint/no-unused-vars */
132+
133+
export class XCTestOutputParser implements IXCTestOutputParser {
71134
private regex: TestRegex;
72135

73136
/**
@@ -93,7 +156,7 @@ export class XCTestOutputParser {
93156
lines.pop();
94157
}
95158
// if submitted text does not end with a newline then pop that off and store in excess
96-
// for next call of parseResultDarwin
159+
// for next call of parseResult
97160
if (output2[output2.length - 1] !== "\n") {
98161
runState.excess = lines.pop();
99162
} else {

‎src/TestExplorer/TestRunner.ts

Lines changed: 67 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -22,11 +22,15 @@ import { execFile, getErrorDescription } from "../utilities/utilities";
2222
import { createSwiftTask } from "../tasks/SwiftTaskProvider";
2323
import configuration from "../configuration";
2424
import { WorkspaceContext } from "../WorkspaceContext";
25-
import { XCTestOutputParser } from "./TestParsers/XCTestOutputParser";
25+
import {
26+
IXCTestOutputParser,
27+
ParallelXCTestOutputParser,
28+
XCTestOutputParser,
29+
} from "./TestParsers/XCTestOutputParser";
2630
import { SwiftTestingOutputParser } from "./TestParsers/SwiftTestingOutputParser";
2731
import { LoggingDebugAdapterTracker } from "../debugger/logTracker";
2832
import { TaskOperation } from "../tasks/TaskQueue";
29-
import { TestXUnitParser, iXUnitTestState } from "./TestXUnitParser";
33+
import { TestXUnitParser } from "./TestXUnitParser";
3034
import { ITestRunState } from "./TestParsers/TestRunState";
3135
import { TestRunArguments } from "./TestRunArguments";
3236
import { TemporaryFolder } from "../utilities/tempFolder";
@@ -68,7 +72,10 @@ export class TestRunProxy {
6872

6973
// Allows for introspection on the state of TestItems after a test run.
7074
public runState = {
71-
failed: [] as vscode.TestItem[],
75+
failed: [] as {
76+
test: vscode.TestItem;
77+
message: vscode.TestMessage | readonly vscode.TestMessage[];
78+
}[],
7279
passed: [] as vscode.TestItem[],
7380
skipped: [] as vscode.TestItem[],
7481
errored: [] as vscode.TestItem[],
@@ -179,7 +186,7 @@ export class TestRunProxy {
179186
message: vscode.TestMessage | readonly vscode.TestMessage[],
180187
duration?: number
181188
) {
182-
this.runState.failed.push(test);
189+
this.runState.failed.push({ test, message });
183190
this.testRun?.failed(test, message, duration);
184191
}
185192

@@ -218,7 +225,7 @@ export class TestRunProxy {
218225
export class TestRunner {
219226
private testRun: TestRunProxy;
220227
private testArgs: TestRunArguments;
221-
private xcTestOutputParser: XCTestOutputParser;
228+
private xcTestOutputParser: IXCTestOutputParser;
222229
private swiftTestOutputParser: SwiftTestingOutputParser;
223230

224231
/**
@@ -228,13 +235,19 @@ export class TestRunner {
228235
* @param controller Test controller
229236
*/
230237
constructor(
238+
private testKind: TestKind,
231239
private request: vscode.TestRunRequest,
232240
private folderContext: FolderContext,
233241
private controller: vscode.TestController
234242
) {
235243
this.testArgs = new TestRunArguments(this.ensureRequestIncludesTests(this.request));
236244
this.testRun = new TestRunProxy(request, controller, this.testArgs, folderContext);
237-
this.xcTestOutputParser = new XCTestOutputParser();
245+
this.xcTestOutputParser =
246+
testKind === TestKind.parallel
247+
? new ParallelXCTestOutputParser(
248+
this.folderContext.workspaceContext.toolchain.hasMultiLineParallelTestOutput
249+
)
250+
: new XCTestOutputParser();
238251
this.swiftTestOutputParser = new SwiftTestingOutputParser(
239252
this.testRun.testRunStarted,
240253
this.testRun.addParameterizedTestCase
@@ -274,9 +287,14 @@ export class TestRunner {
274287
RunProfileName.run,
275288
vscode.TestRunProfileKind.Run,
276289
async (request, token) => {
277-
const runner = new TestRunner(request, folderContext, controller);
290+
const runner = new TestRunner(
291+
TestKind.standard,
292+
request,
293+
folderContext,
294+
controller
295+
);
278296
onCreateTestRun.fire(runner.testRun);
279-
await runner.runHandler(false, TestKind.standard, token);
297+
await runner.runHandler(token);
280298
},
281299
true,
282300
runnableTag
@@ -286,9 +304,14 @@ export class TestRunner {
286304
RunProfileName.runParallel,
287305
vscode.TestRunProfileKind.Run,
288306
async (request, token) => {
289-
const runner = new TestRunner(request, folderContext, controller);
307+
const runner = new TestRunner(
308+
TestKind.parallel,
309+
request,
310+
folderContext,
311+
controller
312+
);
290313
onCreateTestRun.fire(runner.testRun);
291-
await runner.runHandler(false, TestKind.parallel, token);
314+
await runner.runHandler(token);
292315
},
293316
false,
294317
runnableTag
@@ -298,14 +321,19 @@ export class TestRunner {
298321
RunProfileName.coverage,
299322
vscode.TestRunProfileKind.Coverage,
300323
async (request, token) => {
301-
const runner = new TestRunner(request, folderContext, controller);
324+
const runner = new TestRunner(
325+
TestKind.coverage,
326+
request,
327+
folderContext,
328+
controller
329+
);
302330
onCreateTestRun.fire(runner.testRun);
303331
if (request.profile) {
304332
request.profile.loadDetailedCoverage = async (testRun, fileCoverage) => {
305333
return runner.testRun.coverage.loadDetailedCoverage(fileCoverage.uri);
306334
};
307335
}
308-
await runner.runHandler(false, TestKind.coverage, token);
336+
await runner.runHandler(token);
309337
await runner.testRun.computeCoverage();
310338
},
311339
false,
@@ -316,9 +344,14 @@ export class TestRunner {
316344
RunProfileName.debug,
317345
vscode.TestRunProfileKind.Debug,
318346
async (request, token) => {
319-
const runner = new TestRunner(request, folderContext, controller);
347+
const runner = new TestRunner(
348+
TestKind.debug,
349+
request,
350+
folderContext,
351+
controller
352+
);
320353
onCreateTestRun.fire(runner.testRun);
321-
await runner.runHandler(true, TestKind.debug, token);
354+
await runner.runHandler(token);
322355
await vscode.commands.executeCommand("testing.openCoverage");
323356
},
324357
false,
@@ -333,13 +366,13 @@ export class TestRunner {
333366
* @param token Cancellation token
334367
* @returns When complete
335368
*/
336-
async runHandler(shouldDebug: boolean, testKind: TestKind, token: vscode.CancellationToken) {
369+
async runHandler(token: vscode.CancellationToken) {
337370
const runState = new TestRunnerTestRunState(this.testRun);
338371
try {
339-
if (shouldDebug) {
372+
if (this.testKind === TestKind.debug) {
340373
await this.debugSession(token, runState);
341374
} else {
342-
await this.runSession(token, testKind, runState);
375+
await this.runSession(token, runState);
343376
}
344377
} catch (error) {
345378
this.workspaceContext.outputChannel.log(`Error: ${getErrorDescription(error)}`);
@@ -350,11 +383,7 @@ export class TestRunner {
350383
}
351384

352385
/** Run test session without attaching to a debugger */
353-
async runSession(
354-
token: vscode.CancellationToken,
355-
testKind: TestKind,
356-
runState: TestRunnerTestRunState
357-
) {
386+
async runSession(token: vscode.CancellationToken, runState: TestRunnerTestRunState) {
358387
// Run swift-testing first, then XCTest.
359388
// swift-testing being parallel by default should help these run faster.
360389
if (this.testArgs.hasSwiftTestingTests) {
@@ -373,7 +402,7 @@ export class TestRunner {
373402
const testBuildConfig = TestingDebugConfigurationFactory.swiftTestingConfig(
374403
this.folderContext,
375404
fifoPipePath,
376-
testKind,
405+
this.testKind,
377406
this.testArgs.swiftTestArgs,
378407
true
379408
);
@@ -401,7 +430,8 @@ export class TestRunner {
401430
await this.swiftTestOutputParser.watch(fifoPipePath, runState);
402431

403432
await this.launchTests(
404-
testKind === TestKind.parallel ? TestKind.standard : testKind,
433+
runState,
434+
this.testKind === TestKind.parallel ? TestKind.standard : this.testKind,
405435
token,
406436
outputStream,
407437
testBuildConfig,
@@ -413,7 +443,7 @@ export class TestRunner {
413443
if (this.testArgs.hasXCTests) {
414444
const testBuildConfig = TestingDebugConfigurationFactory.xcTestConfig(
415445
this.folderContext,
416-
testKind,
446+
this.testKind,
417447
this.testArgs.xcTestArgs,
418448
true
419449
);
@@ -425,13 +455,7 @@ export class TestRunner {
425455
write: (chunk, encoding, next) => {
426456
const text = chunk.toString();
427457
this.testRun.appendOutput(text.replace(/\n/g, "\r\n"));
428-
429-
// If a test fails in a parallel run the XCTest output is printed.
430-
// Since we get all results from the xunit xml we don't parse here
431-
// to prevent parsing twice.
432-
if (testKind !== TestKind.parallel) {
433-
this.xcTestOutputParser.parseResult(text, runState);
434-
}
458+
this.xcTestOutputParser.parseResult(text, runState);
435459
next();
436460
},
437461
});
@@ -445,7 +469,8 @@ export class TestRunner {
445469
this.testRun.testRunStarted();
446470

447471
await this.launchTests(
448-
testKind,
472+
runState,
473+
this.testKind,
449474
token,
450475
parsedOutputStream,
451476
testBuildConfig,
@@ -455,6 +480,7 @@ export class TestRunner {
455480
}
456481

457482
private async launchTests(
483+
runState: TestRunnerTestRunState,
458484
testKind: TestKind,
459485
token: vscode.CancellationToken,
460486
outputStream: stream.Writable,
@@ -472,7 +498,7 @@ export class TestRunner {
472498
);
473499
break;
474500
case TestKind.parallel:
475-
await this.runParallelSession(token, outputStream, testBuildConfig);
501+
await this.runParallelSession(token, outputStream, testBuildConfig, runState);
476502
break;
477503
default:
478504
await this.runStandardSession(token, outputStream, testBuildConfig, testKind);
@@ -581,7 +607,8 @@ export class TestRunner {
581607
async runParallelSession(
582608
token: vscode.CancellationToken,
583609
outputStream: stream.Writable,
584-
testBuildConfig: vscode.DebugConfiguration
610+
testBuildConfig: vscode.DebugConfiguration,
611+
runState: TestRunnerTestRunState
585612
) {
586613
await this.workspaceContext.tempFolder.withTemporaryFile("xml", async filename => {
587614
const args = [...(testBuildConfig.args ?? []), "--xunit-output", filename];
@@ -602,12 +629,12 @@ export class TestRunner {
602629
throw error;
603630
}
604631
}
632+
605633
const buffer = await asyncfs.readFile(filename, "utf8");
606-
const xUnitParser = new TestXUnitParser();
607-
const results = await xUnitParser.parse(
608-
buffer,
609-
new TestRunnerXUnitTestState(this.testItemFinder, this.testRun)
634+
const xUnitParser = new TestXUnitParser(
635+
this.folderContext.workspaceContext.toolchain.hasMultiLineParallelTestOutput
610636
);
637+
const results = await xUnitParser.parse(buffer, runState);
611638
if (results) {
612639
this.testRun.appendOutput(
613640
`\r\nExecuted ${results.tests} tests, with ${results.failures} failures and ${results.errors} errors.\r\n`
@@ -863,7 +890,7 @@ class NonDarwinTestItemFinder implements TestItemFinder {
863890
/**
864891
* Store state of current test run output parse
865892
*/
866-
class TestRunnerTestRunState implements ITestRunState {
893+
export class TestRunnerTestRunState implements ITestRunState {
867894
constructor(private testRun: TestRunProxy) {}
868895

869896
public currentTestItem?: vscode.TestItem;
@@ -953,30 +980,3 @@ class TestRunnerTestRunState implements ITestRunState {
953980
// Nothing to do here
954981
}
955982
}
956-
957-
class TestRunnerXUnitTestState implements iXUnitTestState {
958-
constructor(
959-
private testItemFinder: TestItemFinder,
960-
private testRun: TestRunProxy
961-
) {}
962-
963-
passTest(id: string, duration: number): void {
964-
const index = this.testItemFinder.getIndex(id);
965-
if (index !== -1) {
966-
this.testRun.passed(this.testItemFinder.testItems[index], duration);
967-
}
968-
}
969-
failTest(id: string, duration: number, message?: string): void {
970-
const index = this.testItemFinder.getIndex(id);
971-
if (index !== -1) {
972-
const testMessage = new vscode.TestMessage(message ?? "Failed");
973-
this.testRun.failed(this.testItemFinder.testItems[index], testMessage, duration);
974-
}
975-
}
976-
skipTest(id: string): void {
977-
const index = this.testItemFinder.getIndex(id);
978-
if (index !== -1) {
979-
this.testRun.skipped(this.testItemFinder.testItems[index]);
980-
}
981-
}
982-
}

0 commit comments

Comments
 (0)