Skip to content

Commit 1c3a35d

Browse files
plemarquandplemarquand
committed
Pull issues from test output during parallel run
Use a combination of the xunit xml output and the test output to capture test pass/fail state and issues.
1 parent 97972d1 commit 1c3a35d

File tree

5 files changed

+200
-91
lines changed

5 files changed

+200
-91
lines changed

‎src/TestExplorer/TestParsers/XCTestOutputParser.ts

Lines changed: 51 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414

1515
import { ITestRunState } from "./TestRunState";
1616
import { sourceLocationToVSCodeLocation } from "../../utilities/utilities";
17+
import { MarkdownString, Location } from "vscode";
1718

1819
/** Regex for parsing XCTest output */
1920
interface TestRegex {
@@ -67,7 +68,55 @@ export const nonDarwinTestRegex = {
6768
failedSuite: /^Test Suite '(.*)' failed/,
6869
};
6970

70-
export class XCTestOutputParser {
71+
export interface IXCTestOutputParser {
72+
parseResult(output: string, runState: ITestRunState): void;
73+
}
74+
75+
export class ParallelXCTestOutputParser implements IXCTestOutputParser {
76+
private outputParser: XCTestOutputParser;
77+
78+
/**
79+
* Create an ParallelXCTestOutputParser.
80+
* Optional regex can be supplied for tests.
81+
*/
82+
constructor(regex?: TestRegex) {
83+
this.outputParser = new XCTestOutputParser(regex);
84+
}
85+
86+
public parseResult(output: string, runState: ITestRunState) {
87+
// For parallel XCTest runs we get pass/fail results from the xunit XML
88+
// produced at the end of the run, but we still want to monitor the output
89+
// for the individual assertion failures. Wrap the run state and only forward
90+
// along the issues captured during a test run, and let the `TestXUnitParser`
91+
// handle marking tests as completed.
92+
this.outputParser.parseResult(output, new ParallelXCTestRunStateProxy(runState));
93+
}
94+
}
95+
96+
/* eslint-disable @typescript-eslint/no-unused-vars */
97+
class ParallelXCTestRunStateProxy implements ITestRunState {
98+
constructor(private runState: ITestRunState) {}
99+
100+
getTestItemIndex(id: string, filename: string | undefined): number {
101+
return this.runState.getTestItemIndex(id, filename);
102+
}
103+
recordIssue(
104+
index: number,
105+
message: string | MarkdownString,
106+
location?: Location | undefined
107+
): void {
108+
this.runState.recordIssue(index, message, location);
109+
}
110+
started(index: number, startTime?: number | undefined): void {}
111+
completed(index: number, timing: { duration: number } | { timestamp: number }): void {}
112+
skipped(index: number): void {}
113+
startedSuite(name: string): void {}
114+
passedSuite(name: string): void {}
115+
failedSuite(name: string): void {}
116+
}
117+
/* eslint-enable @typescript-eslint/no-unused-vars */
118+
119+
export class XCTestOutputParser implements IXCTestOutputParser {
71120
private regex: TestRegex;
72121

73122
/**
@@ -93,7 +142,7 @@ export class XCTestOutputParser {
93142
lines.pop();
94143
}
95144
// if submitted text does not end with a newline then pop that off and store in excess
96-
// for next call of parseResultDarwin
145+
// for next call of parseResult
97146
if (output2[output2.length - 1] !== "\n") {
98147
runState.excess = lines.pop();
99148
} else {

‎src/TestExplorer/TestRunner.ts

Lines changed: 65 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -22,11 +22,15 @@ import { execFile, getErrorDescription } from "../utilities/utilities";
2222
import { createSwiftTask } from "../tasks/SwiftTaskProvider";
2323
import configuration from "../configuration";
2424
import { WorkspaceContext } from "../WorkspaceContext";
25-
import { XCTestOutputParser } from "./TestParsers/XCTestOutputParser";
25+
import {
26+
IXCTestOutputParser,
27+
ParallelXCTestOutputParser,
28+
XCTestOutputParser,
29+
} from "./TestParsers/XCTestOutputParser";
2630
import { SwiftTestingOutputParser } from "./TestParsers/SwiftTestingOutputParser";
2731
import { LoggingDebugAdapterTracker } from "../debugger/logTracker";
2832
import { TaskOperation } from "../tasks/TaskQueue";
29-
import { TestXUnitParser, iXUnitTestState } from "./TestXUnitParser";
33+
import { TestXUnitParser } from "./TestXUnitParser";
3034
import { ITestRunState } from "./TestParsers/TestRunState";
3135
import { TestRunArguments } from "./TestRunArguments";
3236
import { TemporaryFolder } from "../utilities/tempFolder";
@@ -68,7 +72,10 @@ export class TestRunProxy {
6872

6973
// Allows for introspection on the state of TestItems after a test run.
7074
public runState = {
71-
failed: [] as vscode.TestItem[],
75+
failed: [] as {
76+
test: vscode.TestItem;
77+
message: vscode.TestMessage | readonly vscode.TestMessage[];
78+
}[],
7279
passed: [] as vscode.TestItem[],
7380
skipped: [] as vscode.TestItem[],
7481
errored: [] as vscode.TestItem[],
@@ -179,7 +186,7 @@ export class TestRunProxy {
179186
message: vscode.TestMessage | readonly vscode.TestMessage[],
180187
duration?: number
181188
) {
182-
this.runState.failed.push(test);
189+
this.runState.failed.push({ test, message });
183190
this.testRun?.failed(test, message, duration);
184191
}
185192

@@ -218,7 +225,7 @@ export class TestRunProxy {
218225
export class TestRunner {
219226
private testRun: TestRunProxy;
220227
private testArgs: TestRunArguments;
221-
private xcTestOutputParser: XCTestOutputParser;
228+
private xcTestOutputParser: IXCTestOutputParser;
222229
private swiftTestOutputParser: SwiftTestingOutputParser;
223230

224231
/**
@@ -228,13 +235,17 @@ export class TestRunner {
228235
* @param controller Test controller
229236
*/
230237
constructor(
238+
private testKind: TestKind,
231239
private request: vscode.TestRunRequest,
232240
private folderContext: FolderContext,
233241
private controller: vscode.TestController
234242
) {
235243
this.testArgs = new TestRunArguments(this.ensureRequestIncludesTests(this.request));
236244
this.testRun = new TestRunProxy(request, controller, this.testArgs, folderContext);
237-
this.xcTestOutputParser = new XCTestOutputParser();
245+
this.xcTestOutputParser =
246+
testKind === TestKind.parallel
247+
? new ParallelXCTestOutputParser()
248+
: new XCTestOutputParser();
238249
this.swiftTestOutputParser = new SwiftTestingOutputParser(
239250
this.testRun.testRunStarted,
240251
this.testRun.addParameterizedTestCase
@@ -274,9 +285,14 @@ export class TestRunner {
274285
RunProfileName.run,
275286
vscode.TestRunProfileKind.Run,
276287
async (request, token) => {
277-
const runner = new TestRunner(request, folderContext, controller);
288+
const runner = new TestRunner(
289+
TestKind.standard,
290+
request,
291+
folderContext,
292+
controller
293+
);
278294
onCreateTestRun.fire(runner.testRun);
279-
await runner.runHandler(false, TestKind.standard, token);
295+
await runner.runHandler(token);
280296
},
281297
true,
282298
runnableTag
@@ -286,9 +302,14 @@ export class TestRunner {
286302
RunProfileName.runParallel,
287303
vscode.TestRunProfileKind.Run,
288304
async (request, token) => {
289-
const runner = new TestRunner(request, folderContext, controller);
305+
const runner = new TestRunner(
306+
TestKind.parallel,
307+
request,
308+
folderContext,
309+
controller
310+
);
290311
onCreateTestRun.fire(runner.testRun);
291-
await runner.runHandler(false, TestKind.parallel, token);
312+
await runner.runHandler(token);
292313
},
293314
false,
294315
runnableTag
@@ -298,14 +319,19 @@ export class TestRunner {
298319
RunProfileName.coverage,
299320
vscode.TestRunProfileKind.Coverage,
300321
async (request, token) => {
301-
const runner = new TestRunner(request, folderContext, controller);
322+
const runner = new TestRunner(
323+
TestKind.coverage,
324+
request,
325+
folderContext,
326+
controller
327+
);
302328
onCreateTestRun.fire(runner.testRun);
303329
if (request.profile) {
304330
request.profile.loadDetailedCoverage = async (testRun, fileCoverage) => {
305331
return runner.testRun.coverage.loadDetailedCoverage(fileCoverage.uri);
306332
};
307333
}
308-
await runner.runHandler(false, TestKind.coverage, token);
334+
await runner.runHandler(token);
309335
await runner.testRun.computeCoverage();
310336
},
311337
false,
@@ -316,9 +342,14 @@ export class TestRunner {
316342
RunProfileName.debug,
317343
vscode.TestRunProfileKind.Debug,
318344
async (request, token) => {
319-
const runner = new TestRunner(request, folderContext, controller);
345+
const runner = new TestRunner(
346+
TestKind.debug,
347+
request,
348+
folderContext,
349+
controller
350+
);
320351
onCreateTestRun.fire(runner.testRun);
321-
await runner.runHandler(true, TestKind.debug, token);
352+
await runner.runHandler(token);
322353
await vscode.commands.executeCommand("testing.openCoverage");
323354
},
324355
false,
@@ -333,13 +364,13 @@ export class TestRunner {
333364
* @param token Cancellation token
334365
* @returns When complete
335366
*/
336-
async runHandler(shouldDebug: boolean, testKind: TestKind, token: vscode.CancellationToken) {
367+
async runHandler(token: vscode.CancellationToken) {
337368
const runState = new TestRunnerTestRunState(this.testRun);
338369
try {
339-
if (shouldDebug) {
370+
if (this.testKind === TestKind.debug) {
340371
await this.debugSession(token, runState);
341372
} else {
342-
await this.runSession(token, testKind, runState);
373+
await this.runSession(token, runState);
343374
}
344375
} catch (error) {
345376
this.workspaceContext.outputChannel.log(`Error: ${getErrorDescription(error)}`);
@@ -350,11 +381,7 @@ export class TestRunner {
350381
}
351382

352383
/** Run test session without attaching to a debugger */
353-
async runSession(
354-
token: vscode.CancellationToken,
355-
testKind: TestKind,
356-
runState: TestRunnerTestRunState
357-
) {
384+
async runSession(token: vscode.CancellationToken, runState: TestRunnerTestRunState) {
358385
// Run swift-testing first, then XCTest.
359386
// swift-testing being parallel by default should help these run faster.
360387
if (this.testArgs.hasSwiftTestingTests) {
@@ -373,7 +400,7 @@ export class TestRunner {
373400
const testBuildConfig = TestingDebugConfigurationFactory.swiftTestingConfig(
374401
this.folderContext,
375402
fifoPipePath,
376-
testKind,
403+
this.testKind,
377404
this.testArgs.swiftTestArgs,
378405
true
379406
);
@@ -401,7 +428,8 @@ export class TestRunner {
401428
await this.swiftTestOutputParser.watch(fifoPipePath, runState);
402429

403430
await this.launchTests(
404-
testKind === TestKind.parallel ? TestKind.standard : testKind,
431+
runState,
432+
this.testKind === TestKind.parallel ? TestKind.standard : this.testKind,
405433
token,
406434
outputStream,
407435
testBuildConfig,
@@ -413,7 +441,7 @@ export class TestRunner {
413441
if (this.testArgs.hasXCTests) {
414442
const testBuildConfig = TestingDebugConfigurationFactory.xcTestConfig(
415443
this.folderContext,
416-
testKind,
444+
this.testKind,
417445
this.testArgs.xcTestArgs,
418446
true
419447
);
@@ -425,13 +453,7 @@ export class TestRunner {
425453
write: (chunk, encoding, next) => {
426454
const text = chunk.toString();
427455
this.testRun.appendOutput(text.replace(/\n/g, "\r\n"));
428-
429-
// If a test fails in a parallel run the XCTest output is printed.
430-
// Since we get all results from the xunit xml we don't parse here
431-
// to prevent parsing twice.
432-
if (testKind !== TestKind.parallel) {
433-
this.xcTestOutputParser.parseResult(text, runState);
434-
}
456+
this.xcTestOutputParser.parseResult(text, runState);
435457
next();
436458
},
437459
});
@@ -445,7 +467,8 @@ export class TestRunner {
445467
this.testRun.testRunStarted();
446468

447469
await this.launchTests(
448-
testKind,
470+
runState,
471+
this.testKind,
449472
token,
450473
parsedOutputStream,
451474
testBuildConfig,
@@ -455,6 +478,7 @@ export class TestRunner {
455478
}
456479

457480
private async launchTests(
481+
runState: TestRunnerTestRunState,
458482
testKind: TestKind,
459483
token: vscode.CancellationToken,
460484
outputStream: stream.Writable,
@@ -472,7 +496,7 @@ export class TestRunner {
472496
);
473497
break;
474498
case TestKind.parallel:
475-
await this.runParallelSession(token, outputStream, testBuildConfig);
499+
await this.runParallelSession(token, outputStream, testBuildConfig, runState);
476500
break;
477501
default:
478502
await this.runStandardSession(token, outputStream, testBuildConfig, testKind);
@@ -581,7 +605,8 @@ export class TestRunner {
581605
async runParallelSession(
582606
token: vscode.CancellationToken,
583607
outputStream: stream.Writable,
584-
testBuildConfig: vscode.DebugConfiguration
608+
testBuildConfig: vscode.DebugConfiguration,
609+
runState: TestRunnerTestRunState
585610
) {
586611
await this.workspaceContext.tempFolder.withTemporaryFile("xml", async filename => {
587612
const args = [...(testBuildConfig.args ?? []), "--xunit-output", filename];
@@ -602,12 +627,12 @@ export class TestRunner {
602627
throw error;
603628
}
604629
}
630+
605631
const buffer = await asyncfs.readFile(filename, "utf8");
606-
const xUnitParser = new TestXUnitParser();
607-
const results = await xUnitParser.parse(
608-
buffer,
609-
new TestRunnerXUnitTestState(this.testItemFinder, this.testRun)
632+
const xUnitParser = new TestXUnitParser(
633+
this.folderContext.workspaceContext.toolchain.swiftVersion
610634
);
635+
const results = await xUnitParser.parse(buffer, runState);
611636
if (results) {
612637
this.testRun.appendOutput(
613638
`\r\nExecuted ${results.tests} tests, with ${results.failures} failures and ${results.errors} errors.\r\n`
@@ -863,7 +888,7 @@ class NonDarwinTestItemFinder implements TestItemFinder {
863888
/**
864889
* Store state of current test run output parse
865890
*/
866-
class TestRunnerTestRunState implements ITestRunState {
891+
export class TestRunnerTestRunState implements ITestRunState {
867892
constructor(private testRun: TestRunProxy) {}
868893

869894
public currentTestItem?: vscode.TestItem;
@@ -953,30 +978,3 @@ class TestRunnerTestRunState implements ITestRunState {
953978
// Nothing to do here
954979
}
955980
}
956-
957-
class TestRunnerXUnitTestState implements iXUnitTestState {
958-
constructor(
959-
private testItemFinder: TestItemFinder,
960-
private testRun: TestRunProxy
961-
) {}
962-
963-
passTest(id: string, duration: number): void {
964-
const index = this.testItemFinder.getIndex(id);
965-
if (index !== -1) {
966-
this.testRun.passed(this.testItemFinder.testItems[index], duration);
967-
}
968-
}
969-
failTest(id: string, duration: number, message?: string): void {
970-
const index = this.testItemFinder.getIndex(id);
971-
if (index !== -1) {
972-
const testMessage = new vscode.TestMessage(message ?? "Failed");
973-
this.testRun.failed(this.testItemFinder.testItems[index], testMessage, duration);
974-
}
975-
}
976-
skipTest(id: string): void {
977-
const index = this.testItemFinder.getIndex(id);
978-
if (index !== -1) {
979-
this.testRun.skipped(this.testItemFinder.testItems[index]);
980-
}
981-
}
982-
}

0 commit comments

Comments
 (0)