Support a Unified Test Binary (#986)

With the merge of swiftlang/swift-package-manager#7766
a `swift build --build-tests` produces a single binary that contains
both XCTests and swift-testing tests.

Update the build configs when debugging tests to use this new unified
test binary if it exists.

To determine if we're running a unified binary on macOS we can check for
the presence of the swiftpm-testing-helper, which is used to run the
swift-testing tests in the unified binary.

This utility isn't required on Windows and Linux as the test binary can
be invoked directly. To determine if we're running a unified binary we
check if the latest build produced a .swift-testing binary in the build
folder. If it doesn't exist, or if it does exist but its `mtime` is
older than the last build start time, we know its a unified binary. The
`mtime` check is required in case the user updates their toolchain to a
version that produces a unified binary but still has a .swift-testing
binary in their .build folder left over from an old build.

In the future we can remove these checks once we're confident users on
6.0 and main toolchains have moved on to versions that generate a
unified test binary. Tentatively I propose this to be around when
Swift 6.0 is finalized and released.

This change only affects debugging tests. All other testing code paths
use `swift test` which lets SwiftPM automatically handle the switch to
the new unified binary.

Author: plemarquand

src/TestExplorer/TestParsers/TestEventStreamReader.ts

@@ -74,8 +74,7 @@ export class UnixNamedPipeReader implements INamedPipeReader {
                     });
 
                     readable.on("drain", () => pipe.resume());
-
-                    pipe.on("error", () => fs.close(fd));
+                    pipe.on("error", () => pipe.close());
                     pipe.on("end", () => {
                         readable.push(null);
                         fs.close(fd);

src/TestExplorer/TestRunner.ts

@@ -425,11 +425,12 @@ export class TestRunner {
                     await execFile("mkfifo", [fifoPipePath], undefined, this.folderContext);
                 }
 
-                const testBuildConfig = TestingDebugConfigurationFactory.swiftTestingConfig(
+                const testBuildConfig = await TestingDebugConfigurationFactory.swiftTestingConfig(
                     this.folderContext,
                     fifoPipePath,
                     this.testKind,
                     this.testArgs.swiftTestArgs,
+                    new Date(),
                     true
                 );
 
@@ -467,7 +468,7 @@ export class TestRunner {
         }
 
         if (this.testArgs.hasXCTests) {
-            const testBuildConfig = TestingDebugConfigurationFactory.xcTestConfig(
+            const testBuildConfig = await TestingDebugConfigurationFactory.xcTestConfig(
                 this.folderContext,
                 this.testKind,
                 this.testArgs.xcTestArgs,
@@ -710,6 +711,8 @@ export class TestRunner {
         });
         subscriptions.push(buildTask);
 
+        const buildStartTime = new Date();
+
         // Perform a build all before the tests are run. We want to avoid the "Debug Anyway" dialog
         // since choosing that with no prior build, when debugging swift-testing tests, will cause
         // the extension to start listening to the fifo pipe when no results will be produced,
@@ -731,13 +734,15 @@ export class TestRunner {
                     await execFile("mkfifo", [fifoPipePath], undefined, this.folderContext);
                 }
 
-                const swiftTestBuildConfig = TestingDebugConfigurationFactory.swiftTestingConfig(
-                    this.folderContext,
-                    fifoPipePath,
-                    this.testKind,
-                    this.testArgs.swiftTestArgs,
-                    true
-                );
+                const swiftTestBuildConfig =
+                    await TestingDebugConfigurationFactory.swiftTestingConfig(
+                        this.folderContext,
+                        fifoPipePath,
+                        this.testKind,
+                        this.testArgs.swiftTestArgs,
+                        buildStartTime,
+                        true
+                    );
 
                 if (swiftTestBuildConfig !== null) {
                     swiftTestBuildConfig.testType = TestLibrary.swiftTesting;
@@ -765,7 +770,7 @@ export class TestRunner {
 
             // create launch config for testing
             if (this.testArgs.hasXCTests) {
-                const xcTestBuildConfig = TestingDebugConfigurationFactory.xcTestConfig(
+                const xcTestBuildConfig = await TestingDebugConfigurationFactory.xcTestConfig(
                     this.folderContext,
                     this.testKind,
                     this.testArgs.xcTestArgs,

src/debugger/buildConfig.ts

@@ -15,6 +15,7 @@
 import * as os from "os";
 import * as path from "path";
 import * as vscode from "vscode";
+import * as fs from "fs/promises";
 import configuration from "../configuration";
 import { FolderContext } from "../FolderContext";
 import { BuildFlags } from "../toolchain/BuildFlags";
@@ -32,35 +33,38 @@ import { TestKind, isDebugging, isRelease } from "../TestExplorer/TestKind";
  * and `xcTestConfig` functions to create
  */
 export class TestingDebugConfigurationFactory {
-    public static swiftTestingConfig(
+    public static async swiftTestingConfig(
         ctx: FolderContext,
         fifoPipePath: string,
         testKind: TestKind,
         testList: string[],
+        buildStartTime: Date,
         expandEnvVariables = false
-    ): vscode.DebugConfiguration | null {
+    ): Promise<vscode.DebugConfiguration | null> {
         return new TestingDebugConfigurationFactory(
             ctx,
             fifoPipePath,
             testKind,
             TestLibrary.swiftTesting,
             testList,
+            buildStartTime,
             expandEnvVariables
         ).build();
     }
 
-    public static xcTestConfig(
+    public static async xcTestConfig(
         ctx: FolderContext,
         testKind: TestKind,
         testList: string[],
         expandEnvVariables = false
-    ): vscode.DebugConfiguration | null {
+    ): Promise<vscode.DebugConfiguration | null> {
         return new TestingDebugConfigurationFactory(
             ctx,
             "",
             testKind,
             TestLibrary.xctest,
             testList,
+            null,
             expandEnvVariables
         ).build();
     }
@@ -71,6 +75,7 @@ export class TestingDebugConfigurationFactory {
         private testKind: TestKind,
         private testLibrary: TestLibrary,
         private testList: string[],
+        private buildStartTime: Date | null,
         private expandEnvVariables = false
     ) {}
 
@@ -82,7 +87,7 @@ export class TestingDebugConfigurationFactory {
      * - Test Kind (coverage, debugging)
      * - Test Library (XCTest, swift-testing)
      */
-    private build(): vscode.DebugConfiguration | null {
+    private async build(): Promise<vscode.DebugConfiguration | null> {
         if (!this.hasTestTarget) {
             return null;
         }
@@ -98,7 +103,7 @@ export class TestingDebugConfigurationFactory {
     }
 
     /* eslint-disable no-case-declarations */
-    private buildWindowsConfig(): vscode.DebugConfiguration | null {
+    private async buildWindowsConfig(): Promise<vscode.DebugConfiguration | null> {
         if (isDebugging(this.testKind)) {
             const testEnv = {
                 ...swiftRuntimeEnv(),
@@ -114,8 +119,8 @@ export class TestingDebugConfigurationFactory {
 
             return {
                 ...this.baseConfig,
-                program: this.testExecutableOutputPath,
-                args: this.debuggingTestExecutableArgs,
+                program: await this.testExecutableOutputPath(),
+                args: await this.debuggingTestExecutableArgs(),
                 env: testEnv,
             };
         } else {
@@ -124,39 +129,69 @@ export class TestingDebugConfigurationFactory {
     }
 
     /* eslint-disable no-case-declarations */
-    private buildLinuxConfig(): vscode.DebugConfiguration | null {
+    private async buildLinuxConfig(): Promise<vscode.DebugConfiguration | null> {
         if (isDebugging(this.testKind) && this.testLibrary === TestLibrary.xctest) {
             return {
                 ...this.baseConfig,
-                program: this.testExecutableOutputPath,
-                args: this.debuggingTestExecutableArgs,
+                program: await this.testExecutableOutputPath(),
+                args: await this.debuggingTestExecutableArgs(),
                 env: {
                     ...swiftRuntimeEnv(),
                     ...configuration.folder(this.ctx.workspaceFolder).testEnvironmentVariables,
                 },
             };
         } else {
-            return this.buildDarwinConfig();
+            return await this.buildDarwinConfig();
         }
     }
 
-    private buildDarwinConfig(): vscode.DebugConfiguration | null {
+    private async buildDarwinConfig(): Promise<vscode.DebugConfiguration | null> {
         switch (this.testLibrary) {
             case TestLibrary.swiftTesting:
                 switch (this.testKind) {
                     case TestKind.debugRelease:
                     case TestKind.debug:
-                        // In the debug case we need to build the .swift-testing executable and then
+                        // In the debug case we need to build the testing executable and then
                         // launch it with LLDB instead of going through `swift test`.
                         const toolchain = this.ctx.workspaceContext.toolchain;
                         const libraryPath = toolchain.swiftTestingLibraryPath();
                         const frameworkPath = toolchain.swiftTestingFrameworkPath();
+                        const swiftPMTestingHelperPath = toolchain.swiftPMTestingHelperPath;
+
+                        // Toolchains that contain https://github.com/swiftlang/swift-package-manager/commit/844bd137070dcd18d0f46dd95885ef7907ea0697
+                        // produce a single testing binary for both xctest and swift-testing (called <ProductName>.xctest).
+                        // We can continue to invoke it with the xctest utility, but to run swift-testing tests
+                        // we need to invoke then using the swiftpm-testing-helper utility. If this helper utility exists
+                        // then we know we're working with a unified binary.
+                        if (swiftPMTestingHelperPath) {
+                            const result = {
+                                ...this.baseConfig,
+                                program: swiftPMTestingHelperPath,
+                                args: this.addBuildOptionsToArgs(
+                                    this.addTestsToArgs(
+                                        this.addSwiftTestingFlagsArgs([
+                                            "--test-bundle-path",
+                                            this.unifiedTestingOutputPath(),
+                                            "--testing-library",
+                                            "swift-testing",
+                                        ])
+                                    )
+                                ),
+                                env: {
+                                    ...this.testEnv,
+                                    ...this.sanitizerRuntimeEnvironment,
+                                    DYLD_FRAMEWORK_PATH: frameworkPath,
+                                    DYLD_LIBRARY_PATH: libraryPath,
+                                    SWT_SF_SYMBOLS_ENABLED: "0",
+                                },
+                            };
+                            return result;
+                        }
+
                         const result = {
                             ...this.baseConfig,
-                            program: this.swiftTestingOutputPath,
-                            args: this.addBuildOptionsToArgs(
-                                this.addTestsToArgs(this.addSwiftTestingFlagsArgs([]))
-                            ),
+                            program: await this.testExecutableOutputPath(),
+                            args: await this.debuggingTestExecutableArgs(),
                             env: {
                                 ...this.testEnv,
                                 ...this.sanitizerRuntimeEnvironment,
@@ -208,7 +243,7 @@ export class TestingDebugConfigurationFactory {
                         return {
                             ...this.baseConfig,
                             program: path.join(xcTestPath, "xctest"),
-                            args: this.addXCTestExecutableTestsToArgs([this.xcTestOutputPath]),
+                            args: this.addXCTestExecutableTestsToArgs([this.xcTestOutputPath()]),
                             env: {
                                 ...this.testEnv,
                                 ...this.sanitizerRuntimeEnvironment,
@@ -315,7 +350,7 @@ export class TestingDebugConfigurationFactory {
             targetCreateCommands: [`file -a ${arch} ${xctestPath}/xctest`],
             processCreateCommands: [
                 ...envCommands,
-                `process launch -w ${folder} -- ${testFilterArg} ${this.xcTestOutputPath}`,
+                `process launch -w ${folder} -- ${testFilterArg} ${this.xcTestOutputPath()}`,
             ],
             preLaunchTask: `swift: Build All${nameSuffix}`,
         };
@@ -387,37 +422,82 @@ export class TestingDebugConfigurationFactory {
         return isRelease(this.testKind) ? "release" : "debug";
     }
 
-    private get xcTestOutputPath(): string {
+    private xcTestOutputPath(): string {
         return path.join(
             this.buildDirectory,
             this.artifactFolderForTestKind,
-            this.ctx.swiftPackage.name + "PackageTests.xctest"
+            `${this.ctx.swiftPackage.name}PackageTests.xctest`
         );
     }
 
-    private get swiftTestingOutputPath(): string {
+    private swiftTestingOutputPath(): string {
         return path.join(
             this.buildDirectory,
             this.artifactFolderForTestKind,
             `${this.ctx.swiftPackage.name}PackageTests.swift-testing`
         );
     }
 
-    private get testExecutableOutputPath(): string {
+    private async isUnifiedTestingBinary(): Promise<boolean> {
+        // Toolchains that contain https://github.com/swiftlang/swift-package-manager/commit/844bd137070dcd18d0f46dd95885ef7907ea0697
+        // no longer produce a .swift-testing binary, instead we want to use `unifiedTestingOutputPath`.
+        // In order to determine if we're working with a unified binary we need to check if the .swift-testing
+        // binary _doesn't_ exist, and if it does exist we need to check that it wasn't built by an old toolchain
+        // and is still in the .build directory. We do this by checking its mtime and seeing if it is after
+        // the `buildStartTime`.
+
+        // TODO: When Swift 6 is released and enough time has passed that we're sure no one is building the .swift-testing
+        //       binary anymore this fs.stat workaround can be removed and `swiftTestingPath` can be returned. The
+        //       `buildStartTime` argument can be removed and the build config generation can be made synchronous again.
+
+        try {
+            const stat = await fs.stat(this.swiftTestingOutputPath());
+            return !this.buildStartTime || stat.mtime.getTime() < this.buildStartTime.getTime();
+        } catch {
+            // If the .swift-testing binary doesn't exist then swift-testing tests are in the unified binary.
+            return true;
+        }
+    }
+
+    private unifiedTestingOutputPath(): string {
+        // The unified binary that contains both swift-testing and XCTests
+        // is named the same as the old style .xctest binary. The swiftpm-testing-helper
+        // requires the full path to the binary.
+        if (process.platform === "darwin") {
+            return path.join(
+                this.xcTestOutputPath(),
+                "Contents",
+                "MacOS",
+                `${this.ctx.swiftPackage.name}PackageTests`
+            );
+        } else {
+            return this.xcTestOutputPath();
+        }
+    }
+
+    private async testExecutableOutputPath(): Promise<string> {
         switch (this.testLibrary) {
             case TestLibrary.swiftTesting:
-                return this.swiftTestingOutputPath;
+                return (await this.isUnifiedTestingBinary())
+                    ? this.unifiedTestingOutputPath()
+                    : this.swiftTestingOutputPath();
             case TestLibrary.xctest:
-                return this.xcTestOutputPath;
+                return this.xcTestOutputPath();
         }
     }
 
-    private get debuggingTestExecutableArgs(): string[] {
+    private async debuggingTestExecutableArgs(): Promise<string[]> {
         switch (this.testLibrary) {
-            case TestLibrary.swiftTesting:
+            case TestLibrary.swiftTesting: {
+                const isUnifiedBinary = await this.isUnifiedTestingBinary();
+                const swiftTestingArgs = isUnifiedBinary
+                    ? ["--testing-library", "swift-testing"]
+                    : [];
+
                 return this.addBuildOptionsToArgs(
-                    this.addTestsToArgs(this.addSwiftTestingFlagsArgs([]))
+                    this.addTestsToArgs(this.addSwiftTestingFlagsArgs(swiftTestingArgs))
                 );
+            }
             case TestLibrary.xctest:
                 return [this.testList.join(",")];
         }