Run tests in one step

Instead of a build followed by a test run with --skip-build, use a
SwiftExecution task to do swift test with the appropriate arguments.
This shows the build progress in the status bar.

Author: plemarquand

src/TestExplorer/TestRunner.ts

@@ -19,12 +19,11 @@ import * as os from "os";
 import * as asyncfs from "fs/promises";
 import { FolderContext } from "../FolderContext";
 import { execFile, getErrorDescription } from "../utilities/utilities";
-import { getBuildAllTask } from "../tasks/SwiftTaskProvider";
+import { createSwiftTask } from "../tasks/SwiftTaskProvider";
 import configuration from "../configuration";
 import { WorkspaceContext } from "../WorkspaceContext";
 import { XCTestOutputParser } from "./TestParsers/XCTestOutputParser";
 import { SwiftTestingOutputParser } from "./TestParsers/SwiftTestingOutputParser";
-import { Version } from "../utilities/version";
 import { LoggingDebugAdapterTracker } from "../debugger/logTracker";
 import { TaskOperation } from "../tasks/TaskQueue";
 import { TestXUnitParser, iXUnitTestState } from "./TestXUnitParser";
@@ -34,16 +33,15 @@ import { TemporaryFolder } from "../utilities/tempFolder";
 import { TestClass, runnableTag, upsertTestItem } from "./TestDiscovery";
 import { TestCoverage } from "../coverage/LcovResults";
 import { DebugConfigurationFactory } from "../debugger/buildConfig";
-import { SwiftPtyProcess } from "../tasks/SwiftProcess";
 
 /** Workspace Folder events */
 export enum TestKind {
     // run tests serially
-    standard = "standard",
+    standard = "Standard",
     // run tests in parallel
-    parallel = "parallel",
+    parallel = "Parallel",
     // run tests and extract test coverage
-    coverage = "coverage",
+    coverage = "Coverage",
 }
 
 export enum TestLibrary {
@@ -309,28 +307,6 @@ export class TestRunner {
     async runHandler(shouldDebug: boolean, testKind: TestKind, token: vscode.CancellationToken) {
         const runState = new TestRunnerTestRunState(this.testRun);
         try {
-            // run associated build task
-            // don't do this if generating code test coverage data as the
-            // `swift test --enable-code-coverage` command will rebuild everything again.
-            if (testKind !== TestKind.coverage) {
-                const task = await getBuildAllTask(this.folderContext);
-                task.definition.dontTriggerTestDiscovery =
-                    this.folderContext.workspaceContext.swiftVersion.isGreaterThanOrEqual(
-                        new Version(6, 0, 0)
-                    );
-
-                const exitCode = await this.folderContext.taskQueue.queueOperation(
-                    new TaskOperation(task),
-                    token
-                );
-
-                // if build failed then exit
-                if (exitCode === undefined || exitCode !== 0) {
-                    await this.testRun.end();
-                    return;
-                }
-            }
-
             if (shouldDebug) {
                 await this.debugSession(token, runState);
             } else {
@@ -449,7 +425,6 @@ export class TestRunner {
         testBuildConfig: vscode.DebugConfiguration,
         testLibrary: TestLibrary
     ) {
-        this.testRun.appendOutput(`> Test run started at ${new Date().toLocaleString()} <\r\n\r\n`);
         try {
             switch (testKind) {
                 case TestKind.coverage:
@@ -464,7 +439,7 @@ export class TestRunner {
                     await this.runParallelSession(token, outputStream, testBuildConfig);
                     break;
                 default:
-                    await this.runStandardSession(token, outputStream, testBuildConfig);
+                    await this.runStandardSession(token, outputStream, testBuildConfig, testKind);
                     break;
             }
         } catch (error) {
@@ -481,35 +456,53 @@ export class TestRunner {
     async runStandardSession(
         token: vscode.CancellationToken,
         outputStream: stream.Writable,
-        testBuildConfig: vscode.DebugConfiguration
+        testBuildConfig: vscode.DebugConfiguration,
+        testKind: TestKind
     ) {
         return new Promise<void>((resolve, reject) => {
             const args = testBuildConfig.args ?? [];
-            let didError = false;
-            let cancellation: vscode.Disposable;
+            this.folderContext?.workspaceContext.outputChannel.logDiagnostic(
+                `Exec: ${testBuildConfig.program} ${args.join(" ")}`,
+                this.folderContext.name
+            );
 
-            const exec = new SwiftPtyProcess(testBuildConfig.program, args, {
-                cwd: testBuildConfig.cwd,
-                env: { ...process.env, ...testBuildConfig.env },
-            });
+            let kindLabel: string;
+            switch (testKind) {
+                case TestKind.coverage:
+                    kindLabel = " With Code Coverage";
+                    break;
+                case TestKind.parallel:
+                    kindLabel = " In Parallel";
+                    break;
+                case TestKind.standard:
+                    kindLabel = "";
+            }
 
-            exec.onDidWrite(str => {
-                // Work around SPM still emitting progress when doing --no-build.
-                const replaced = str.replace("[1/1] Planning build", "");
-                outputStream.write(replaced);
-            });
+            const task = createSwiftTask(
+                args,
+                `Building and Running Tests${kindLabel}`,
+                {
+                    cwd: this.folderContext.folder,
+                    scope: this.folderContext.workspaceFolder,
+                    prefix: this.folderContext.name,
+                    presentationOptions: { reveal: vscode.TaskRevealKind.Silent },
+                },
+                this.folderContext.workspaceContext.toolchain,
+                { ...process.env, ...testBuildConfig.env }
+            );
 
-            exec.onDidThrowError(err => {
-                didError = true;
-                reject(err);
+            task.execution.onDidWrite(str => {
+                const replaced = str
+                    .replace("[1/1] Planning build", "") // Work around SPM still emitting progress when doing --no-build.
+                    .replace(
+                        /LLVM Profile Error: Failed to write file "default.profraw": Operation not permitted\r\n/gm,
+                        ""
+                    ); // Work around benign LLVM coverage warnings
+                outputStream.write(replaced);
             });
 
-            exec.onDidClose(code => {
-                // onDidClose is still called after an error
-                if (didError) {
-                    return;
-                }
-
+            let cancellation: vscode.Disposable;
+            task.execution.onDidClose(code => {
                 if (cancellation) {
                     cancellation.dispose();
                 }
@@ -522,17 +515,7 @@ export class TestRunner {
                 }
             });
 
-            if (token) {
-                cancellation = token.onCancellationRequested(() => {
-                    exec.kill();
-                });
-            }
-
-            this.folderContext?.workspaceContext.outputChannel.logDiagnostic(
-                `Exec: ${testBuildConfig.program} ${args.join(" ")}`,
-                this.folderContext.name
-            );
-            exec.spawn();
+            this.folderContext.taskQueue.queueOperation(new TaskOperation(task), token);
         });
     }
 
@@ -544,7 +527,7 @@ export class TestRunner {
         testLibrary: TestLibrary
     ) {
         try {
-            await this.runStandardSession(token, outputStream, testBuildConfig);
+            await this.runStandardSession(token, outputStream, testBuildConfig, TestKind.coverage);
         } catch (error) {
             // If this isn't a standard test failure, forward the error and skip generating coverage.
             if (error !== 1) {
@@ -578,10 +561,15 @@ export class TestRunner {
             this.testRun.testRunStarted();
 
             try {
-                testBuildConfig.args = await this.runStandardSession(token, outputStream, {
-                    ...testBuildConfig,
-                    args: [...args, filterArgs],
-                });
+                testBuildConfig.args = await this.runStandardSession(
+                    token,
+                    outputStream,
+                    {
+                        ...testBuildConfig,
+                        args: [...args, filterArgs],
+                    },
+                    TestKind.parallel
+                );
             } catch (error) {
                 // If this isn't a standard test failure, forward the error and skip generating coverage.
                 if (error !== 1) {
@@ -727,11 +715,6 @@ export class TestRunner {
                                 .then(
                                     started => {
                                         if (started) {
-                                            if (config === validBuildConfigs[0]) {
-                                                this.testRun.appendOutput(
-                                                    `> Test run started at ${new Date().toLocaleString()} <\r\n\r\n`
-                                                );
-                                            }
                                             // show test results pane
                                             vscode.commands.executeCommand(
                                                 "testing.showMostRecentOutput"

src/coverage/LcovResults.ts

@@ -112,10 +112,6 @@ export class TestCoverage {
 
     private async computeLCOVCoverage(): Promise<lcov.LcovFile[]> {
         if (this.lcovFiles.length === 0) {
-            this.folderContext.workspaceContext.outputChannel.logDiagnostic(
-                `Attempted to compute code coverage with no coverage files captured.`,
-                this.folderContext.name
-            );
             return [];
         }
 

src/debugger/buildConfig.ts

@@ -86,12 +86,10 @@ export class DebugConfigurationFactory {
         }
 
         switch (process.platform) {
-            case "darwin":
-                return this.buidDarwinConfig();
             case "win32":
                 return this.buildWindowsConfig();
             default:
-                return this.buildLinuxConfig();
+                return this.buidDarwinConfig();
         }
     }
 
@@ -130,54 +128,44 @@ export class DebugConfigurationFactory {
                             : this.baseConfig.preLaunchTask,
                 };
             case "XCTest":
-                switch (this.testKind) {
-                    // If we want code coverage on XCTests we have to run with `swift test --enable-code-coverage`.
-                    case TestKind.coverage:
-                        let args = ["test", "--enable-code-coverage"];
-                        if (this.swiftVersionGreaterOrEqual(6, 0, 0)) {
-                            args = [
-                                ...args,
-                                "--enable-xctest",
-                                "--disable-experimental-swift-testing",
-                            ];
-                        }
-
-                        return {
-                            ...this.baseConfig,
-                            program: this.swiftProgramPath,
-                            args: this.addTestsToArgs(args),
-                            env: {
-                                ...this.testEnv,
-                                ...this.sanitizerRuntimeEnvironment,
-                            },
-                        };
-                    default:
-                        const swiftVersion = this.ctx.workspaceContext.toolchain.swiftVersion;
-                        if (
-                            swiftVersion.isLessThan(new Version(5, 7, 0)) &&
-                            swiftVersion.isGreaterThanOrEqual(new Version(5, 6, 0))
-                        ) {
-                            // if debugging on macOS with Swift 5.6 we need to create a custom launch
-                            // configuration so we can set the system architecture
-                            return this.createDarwin56TestConfiguration();
-                        }
-
-                        // If we don't need code coverage we can just run the built <Package>PackageTests.xctest binary with xctest.
-                        const xcTestBinary = this.xcTestBinaryPath;
-                        if (!xcTestBinary) {
-                            return null;
-                        }
+                const swiftVersion = this.ctx.workspaceContext.toolchain.swiftVersion;
+                if (
+                    swiftVersion.isLessThan(new Version(5, 7, 0)) &&
+                    swiftVersion.isGreaterThanOrEqual(new Version(5, 6, 0))
+                ) {
+                    // if debugging on macOS with Swift 5.6 we need to create a custom launch
+                    // configuration so we can set the system architecture
+                    return this.createDarwin56TestConfiguration();
+                }
 
-                        return {
-                            ...this.baseConfig,
-                            program: this.xcTestBinaryPath,
-                            args:
-                                this.testList.length > 0
-                                    ? ["-XCTest", this.testList.join(","), this.xcTestOutputPath]
-                                    : [this.xcTestOutputPath],
-                            env: this.testEnv,
-                        };
+                let xcTestArgs = [
+                    "test",
+                    ...(this.testKind === TestKind.coverage ? ["--enable-code-coverage"] : []),
+                ];
+                if (this.swiftVersionGreaterOrEqual(6, 0, 0)) {
+                    xcTestArgs = [
+                        ...xcTestArgs,
+                        "--enable-xctest",
+                        "--disable-experimental-swift-testing",
+                    ];
                 }
+
+                return {
+                    ...this.baseConfig,
+                    program: this.swiftProgramPath,
+                    args: this.addTestsToArgs(xcTestArgs),
+                    env: {
+                        ...this.testEnv,
+                        ...this.sanitizerRuntimeEnvironment,
+                        SWT_SF_SYMBOLS_ENABLED: "0",
+                    },
+                    // For coverage we need to rebuild so do the build/test all in one step,
+                    // otherwise we do a build, then test, to give better progress.
+                    preLaunchTask:
+                        this.testKind === TestKind.coverage
+                            ? undefined
+                            : this.baseConfig.preLaunchTask,
+                };
             default:
                 return null;
         }
@@ -191,7 +179,7 @@ export class DebugConfigurationFactory {
             case TestLibrary.xctest:
                 switch (this.testKind) {
                     case TestKind.coverage:
-                        return this.buildLinuxConfig();
+                        return this.buidDarwinConfig();
                     default:
                         const xcTestPath = this.ctx.workspaceContext.toolchain.xcTestPath;
                         const runtimePath = this.ctx.workspaceContext.toolchain.runtimePath;
@@ -229,43 +217,6 @@ export class DebugConfigurationFactory {
                 }
         }
     }
-
-    private buildLinuxConfig(): vscode.DebugConfiguration | null {
-        switch (this.testLibrary) {
-            case TestLibrary.swiftTesting:
-                return this.buidDarwinConfig();
-            case TestLibrary.xctest:
-                switch (this.testKind) {
-                    case TestKind.coverage:
-                        let args = ["test", "--enable-code-coverage"];
-                        if (this.swiftVersionGreaterOrEqual(6, 0, 0)) {
-                            args = [
-                                ...args,
-                                "--enable-xctest",
-                                "--disable-experimental-swift-testing",
-                            ];
-                        }
-
-                        return {
-                            ...this.baseConfig,
-                            program: this.swiftProgramPath,
-                            args: this.addTestsToArgs(args),
-                            env: this.testEnv,
-                            preLaunchTask:
-                                this.testKind === TestKind.coverage
-                                    ? undefined
-                                    : this.baseConfig.preLaunchTask,
-                        };
-                    default:
-                        return {
-                            ...this.baseConfig,
-                            program: this.xcTestOutputPath,
-                            args: [this.testList.join(",")],
-                            env: this.testEnv,
-                        };
-                }
-        }
-    }
     /* eslint-enable no-case-declarations */
 
     /**
@@ -339,12 +290,6 @@ export class DebugConfigurationFactory {
 
     private get swiftProgramPath(): string {
         return this.ctx.workspaceContext.toolchain.getToolchainExecutable("swift");
-        // return path.join(this.ctx.workspaceContext.toolchain.swiftFolderPath, "swift");
-    }
-
-    private get xcTestBinaryPath(): string | undefined {
-        const xcTestPath = this.ctx.workspaceContext.toolchain.xcTestPath;
-        return xcTestPath ? path.join(xcTestPath, "xctest") : undefined;
     }
 
     private get buildDirectory(): string {