Fill in the implementation of running tests from a plugin

This extends SwiftTestTool, where most of the implementation already exists. A future direction would be to factor out this functionality so that it can be used from both plugins and the command line tools (both really have the same purpose).

Author: abertelrud

Sources/Commands/SwiftPackageTool.swift

@@ -1207,8 +1207,7 @@ final class PluginDelegate: PluginInvocationDelegate {
     }
 
     private func doTest(subset: PluginInvocationTestSubset, parameters: PluginInvocationTestParameters) throws -> PluginInvocationTestResult {
-        // FIXME: To implement this we should factor out a lot of the code in SwiftTestTool.
-        throw StringError("Running tests from a plugin is not yet implemented")
+        return try SwiftTestTool.doTest(subset: subset, parameters: parameters, swiftTool: swiftTool)
     }
 
     func pluginRequestedSymbolGraph(forTarget targetName: String, options: PluginInvocationSymbolGraphOptions, completion: @escaping (Result<PluginInvocationSymbolGraphResult, Error>) -> Void) {

Sources/Commands/SwiftTestTool.swift

@@ -1,7 +1,7 @@
 /*
 This source file is part of the Swift.org open source project
 
-Copyright 2015 - 2016 Apple Inc. and the Swift project authors
+Copyright 2015 - 2022 Apple Inc. and the Swift project authors
 Licensed under Apache License v2.0 with Runtime Library Exception
 
 See http://swift.org/LICENSE.txt for license information
@@ -11,6 +11,7 @@ See http://swift.org/CONTRIBUTORS.txt for Swift project authors
 import ArgumentParser
 import Basics
 import Build
+import Dispatch
 import class Foundation.ProcessInfo
 import PackageGraph
 import SPMBuildCore
@@ -209,7 +210,7 @@ public struct SwiftTestTool: SwiftCommand {
         switch options.mode {
         case .listTests:
             let testProducts = try buildTestsIfNeeded(swiftTool: swiftTool)
-            let testSuites = try getTestSuites(in: testProducts, swiftTool: swiftTool)
+            let testSuites = try Self.getTestSuites(in: testProducts, swiftTool: swiftTool, swiftOptions: swiftOptions)
             let tests = try testSuites
                 .filteredTests(specifier: options.testCaseSpecifier)
                 .skippedTests(specifier: options.testCaseSkip)
@@ -245,7 +246,7 @@ public struct SwiftTestTool: SwiftCommand {
             #endif
             let graph = try swiftTool.loadPackageGraph()
             let testProducts = try buildTestsIfNeeded(swiftTool: swiftTool)
-            let testSuites = try getTestSuites(in: testProducts, swiftTool: swiftTool)
+            let testSuites = try Self.getTestSuites(in: testProducts, swiftTool: swiftTool, swiftOptions: swiftOptions)
             let allTestSuites = testSuites.values.flatMap { $0 }
             let generator = LinuxMainGenerator(graph: graph, testSuites: allTestSuites)
             try generator.generate()
@@ -278,7 +279,7 @@ public struct SwiftTestTool: SwiftCommand {
                 }
 
                 // Find the tests we need to run.
-                let testSuites = try getTestSuites(in: testProducts, swiftTool: swiftTool)
+                let testSuites = try Self.getTestSuites(in: testProducts, swiftTool: swiftTool, swiftOptions: swiftOptions)
                 let tests = try testSuites
                     .filteredTests(specifier: options.testCaseSpecifier)
                     .skippedTests(specifier: options.testCaseSkip)
@@ -317,7 +318,7 @@ public struct SwiftTestTool: SwiftCommand {
         case .runParallel:
             let toolchain = try swiftTool.getToolchain()
             let testProducts = try buildTestsIfNeeded(swiftTool: swiftTool)
-            let testSuites = try getTestSuites(in: testProducts, swiftTool: swiftTool)
+            let testSuites = try Self.getTestSuites(in: testProducts, swiftTool: swiftTool, swiftOptions: swiftOptions)
             let tests = try testSuites
                 .filteredTests(specifier: options.testCaseSpecifier)
                 .skippedTests(specifier: options.testCaseSkip)
@@ -465,7 +466,7 @@ public struct SwiftTestTool: SwiftCommand {
     /// Note: It is a fatalError if we are not able to locate the tool.
     ///
     /// - Returns: Path to XCTestHelper tool.
-    private func xctestHelperPath(swiftTool: SwiftTool) throws -> AbsolutePath {
+    private static func xctestHelperPath(swiftTool: SwiftTool) throws -> AbsolutePath {
         let xctestHelperBin = "swiftpm-xctest-helper"
         let binDirectory = AbsolutePath(CommandLine.arguments.first!,
             relativeTo: swiftTool.originalWorkingDirectory).parentDirectory
@@ -483,9 +484,9 @@ public struct SwiftTestTool: SwiftCommand {
         throw InternalError("XCTestHelper binary not found.")
     }
 
-    fileprivate func getTestSuites(in testProducts: [BuiltTestProduct], swiftTool: SwiftTool) throws -> [AbsolutePath: [TestSuite]] {
+    fileprivate static func getTestSuites(in testProducts: [BuiltTestProduct], swiftTool: SwiftTool, swiftOptions: SwiftToolOptions) throws -> [AbsolutePath: [TestSuite]] {
         let testSuitesByProduct = try testProducts
-            .map { try ($0.bundlePath, self.getTestSuites(fromTestAt: $0.bundlePath, swiftTool: swiftTool)) }
+            .map { try ($0.bundlePath, Self.getTestSuites(fromTestAt: $0.bundlePath, swiftTool: swiftTool, swiftOptions: swiftOptions)) }
         return Dictionary(uniqueKeysWithValues: testSuitesByProduct)
     }
 
@@ -499,7 +500,7 @@ public struct SwiftTestTool: SwiftCommand {
     /// - Throws: TestError, SystemError, TSCUtility.Error
     ///
     /// - Returns: Array of TestSuite
-    fileprivate func getTestSuites(fromTestAt path: AbsolutePath, swiftTool: SwiftTool) throws -> [TestSuite] {
+    fileprivate static func getTestSuites(fromTestAt path: AbsolutePath, swiftTool: SwiftTool, swiftOptions: SwiftToolOptions) throws -> [TestSuite] {
         // Run the correct tool.
         #if os(macOS)
         let data: String = try withTemporaryFile { tempFile in
@@ -1139,3 +1140,141 @@ private extension SwiftTool {
         return parameters
     }
 }
+
+extension SwiftTestTool {
+
+    public static func doTest(subset: PluginInvocationTestSubset, parameters: PluginInvocationTestParameters, swiftTool: SwiftTool) throws -> PluginInvocationTestResult {
+        // Build the tests. Ideally we should only build those that match the subset, but we don't have a way to know which ones they are until we've built them and can examine the binaries.
+        let toolchain = try swiftTool.getToolchain()
+        var buildParameters = try swiftTool.buildParameters()
+        buildParameters.enableTestability = true
+        buildParameters.enableCodeCoverage = parameters.enableCodeCoverage
+        let buildSystem = try swiftTool.createBuildSystem(buildParameters: buildParameters)
+        try buildSystem.build(subset: .allIncludingTests)
+
+        // Clean out the code coverage directory that may contain stale `profraw` files from a previous run of the code coverage tool.
+        if parameters.enableCodeCoverage {
+            try localFileSystem.removeFileTree(buildParameters.codeCovPath)
+        }
+
+        // Construct the environment we'll pass down to the tests.
+        var environmentOptions = swiftTool.options
+        environmentOptions.shouldEnableCodeCoverage = parameters.enableCodeCoverage
+        let testEnvironment = try constructTestEnvironment(
+            toolchain: toolchain,
+            options: environmentOptions,
+            buildParameters: buildParameters)
+
+        // Iterate over the tests and run those that match the filter.
+        var testTargetResults: [PluginInvocationTestResult.TestTarget] = []
+        var numFailedTests = 0
+        for testProduct in buildSystem.builtTestProducts {
+            // Get the test suites in the bundle. Each is just a container for test cases.
+            let testSuites = try Self.getTestSuites(fromTestAt: testProduct.bundlePath, swiftTool: swiftTool, swiftOptions: swiftTool.options)
+            for testSuite in testSuites {
+                // Each test suite is just a container for test cases (confusingly called "tests", though they are test cases).
+                for testCase in testSuite.tests {
+                    // Each test case corresponds to a combination of target and a XCTestCase, and is a collection of tests that can actually be run.
+                    var testResults: [PluginInvocationTestResult.TestTarget.TestCase.Test] = []
+                    for testName in testCase.tests {
+                        // Check if we should filter out this test.
+                        let testSpecifier = testCase.name + "/" + testName
+                        if case .filtered(let regexes) = subset {
+                            guard regexes.contains(where: { testSpecifier.range(of: $0, options: .regularExpression) != nil }) else {
+                                continue
+                            }
+                        }
+
+                        // Configure a test runner.
+                        let testRunner = TestRunner(
+                            bundlePaths: [testProduct.bundlePath],
+                            xctestArg: testSpecifier,
+                            processSet: swiftTool.processSet,
+                            toolchain: toolchain,
+                            testEnv: testEnvironment,
+                            outputStream: swiftTool.outputStream,
+                            observabilityScope: swiftTool.observabilityScope)
+
+                        // Run the test — for now we run the sequentially so we can capture accurate timing results.
+                        let startTime = DispatchTime.now()
+                        let (success, _) = testRunner.test(writeToOutputStream: false)
+                        let duration = startTime.distance(to: .now()).asSeconds ?? 0.0
+                        numFailedTests += success ? 0 : 1
+                        testResults.append(.init(name: testName, result: success ? .succeeded : .failed, duration: duration))
+                    }
+
+                    // Don't add any results if we didn't run any tests.
+                    if testResults.isEmpty { continue }
+
+                    // Otherwise we either create a new create a new target result or add to the previous one, depending on whether the target name is the same.
+                    let testTargetName = testCase.name.prefix(while: { $0 != "." })
+                    if let lastTestTargetName = testTargetResults.last?.name, testTargetName == lastTestTargetName {
+                        // Same as last one, just extend its list of cases. We know we have a last one at this point.
+                        testTargetResults[testTargetResults.count-1].testCases.append(.init(name: testCase.name, tests: testResults))
+                    }
+                    else {
+                        // Not the same, so start a new target result.
+                        testTargetResults.append(.init(name: String(testTargetName), testCases: [.init(name: testCase.name, tests: testResults)]))
+                    }
+                }
+            }
+        }
+
+        // Deal with code coverage, if enabled.
+        let codeCoverageDataFile: AbsolutePath?
+        if parameters.enableCodeCoverage {
+            // Use `llvm-prof` to merge all the `.profraw` files into a single `.profdata` file.
+            let mergedCovFile = buildParameters.codeCovDataFile
+            let codeCovFileNames = try localFileSystem.getDirectoryContents(buildParameters.codeCovPath)
+            var llvmProfCommand = [try toolchain.getLLVMProf().pathString]
+            llvmProfCommand += ["merge", "-sparse"]
+            for fileName in codeCovFileNames where fileName.hasSuffix(".profraw") {
+                let filePath = buildParameters.codeCovPath.appending(component: fileName)
+                llvmProfCommand.append(filePath.pathString)
+            }
+            llvmProfCommand += ["-o", mergedCovFile.pathString]
+            try Process.checkNonZeroExit(arguments: llvmProfCommand)
+
+            // Use `llvm-cov` to export the merged `.profdata` file contents in JSON form.
+            var llvmCovCommand = [try toolchain.getLLVMCov().pathString]
+            llvmCovCommand += ["export", "-instr-profile=\(mergedCovFile.pathString)"]
+            for product in buildSystem.builtTestProducts {
+                llvmCovCommand.append("-object")
+                llvmCovCommand.append(product.binaryPath.pathString)
+            }
+            // We get the output on stdout, and have to write it to a JSON ourselves.
+            let jsonOutput = try Process.checkNonZeroExit(arguments: llvmCovCommand)
+            let jsonCovFile = buildParameters.codeCovDataFile.parentDirectory.appending(component: buildParameters.codeCovDataFile.basenameWithoutExt + ".json")
+            try localFileSystem.writeFileContents(jsonCovFile, string: jsonOutput)
+
+            // Return the path of the exported code coverage data file.
+            codeCoverageDataFile = jsonCovFile
+        }
+        else {
+            codeCoverageDataFile = nil
+        }
+
+        // Return the results to the plugin. We only consider the test run a success if no test failed.
+        return PluginInvocationTestResult(
+            succeeded: (numFailedTests == 0),
+            testTargets: testTargetResults,
+            codeCoverageDataFile: codeCoverageDataFile?.pathString)
+    }
+}
+
+fileprivate extension DispatchTimeInterval {
+    var asSeconds: Double? {
+        switch self {
+        case .seconds(let value):
+            return Double(value)
+        case .milliseconds(let value):
+            return Double(value) / 1000
+        case .microseconds(let value):
+            return Double(value) / 1_000_000
+        case .nanoseconds(let value):
+            return Double(value) / 1_000_000_000
+        default:
+            return .none
+        }
+    }
+}

Sources/Workspace/UserToolchain.swift

@@ -223,12 +223,12 @@ public final class UserToolchain: Toolchain {
 
     /// Returns the path to llvm-cov tool.
     public func getLLVMCov() throws -> AbsolutePath {
-        return try UserToolchain.getTool("llvm-cov", binDir: self.destination.binDir)
+        return try UserToolchain.getTool("llvm-cov", binDir: self.swiftCompilerPath.parentDirectory)
     }
 
     /// Returns the path to llvm-prof tool.
     public func getLLVMProf() throws -> AbsolutePath {
-        return try UserToolchain.getTool("llvm-profdata", binDir: self.destination.binDir)
+        return try UserToolchain.getTool("llvm-profdata", binDir: self.swiftCompilerPath.parentDirectory)
     }
 
     public func getSwiftAPIDigester() throws -> AbsolutePath {