Fill in the implementation of running tests from a plugin

This uses most of the same implementation that existed in a very specify way in SwiftTestTool.swift, factoring it out into a new shared TestHelpers.swift file and making it more generic.  Future refactoring should move it down to a new module that provides the shared test support for use from everywhere.

Author: abertelrud

Sources/Basics/DispatchTimeInterval+Extensions.swift

@@ -1,7 +1,7 @@
 /*
  This source file is part of the Swift.org open source project
 
- Copyright (c) 2020 Apple Inc. and the Swift project authors
+ Copyright (c) 2020-2022 Apple Inc. and the Swift project authors
  Licensed under Apache License v2.0 with Runtime Library Exception
 
  See http://swift.org/LICENSE.txt for license information
@@ -12,7 +12,7 @@ import Dispatch
 import struct Foundation.TimeInterval
 
 extension DispatchTimeInterval {
-    func timeInterval() -> TimeInterval? {
+    public func timeInterval() -> TimeInterval? {
         switch self {
         case .seconds(let value):
             return Double(value)
@@ -27,7 +27,7 @@ extension DispatchTimeInterval {
         }
     }
 
-    func milliseconds() -> Int? {
+    public func milliseconds() -> Int? {
         switch self {
         case .seconds(let value):
             return value.multipliedReportingOverflow(by: 1000).partialValue
@@ -42,7 +42,7 @@ extension DispatchTimeInterval {
         }
     }
 
-    func seconds() -> Int? {
+    public func seconds() -> Int? {
         switch self {
         case .seconds(let value):
             return value

Sources/Commands/CMakeLists.txt

@@ -1,6 +1,6 @@
 # This source file is part of the Swift.org open source project
 #
-# Copyright (c) 2014 - 2019 Apple Inc. and the Swift project authors
+# Copyright (c) 2014 - 2022 Apple Inc. and the Swift project authors
 # Licensed under Apache License v2.0 with Runtime Library Exception
 #
 # See http://swift.org/LICENSE.txt for license information
@@ -28,6 +28,7 @@ add_library(Commands
   SwiftTestTool.swift
   SwiftTool.swift
   SymbolGraphExtract.swift
+  TestHelpers.swift
   WatchmanHelper.swift)
 target_link_libraries(Commands PUBLIC
   ArgumentParser

Sources/Commands/SwiftPackageTool.swift

@@ -1206,9 +1206,122 @@ final class PluginDelegate: PluginInvocationDelegate {
         }
     }
 
-    private func doTest(subset: PluginInvocationTestSubset, parameters: PluginInvocationTestParameters) throws -> PluginInvocationTestResult {
-        // FIXME: To implement this we should factor out a lot of the code in SwiftTestTool.
-        throw StringError("Running tests from a plugin is not yet implemented")
+    func doTest(subset: PluginInvocationTestSubset, parameters: PluginInvocationTestParameters) throws -> PluginInvocationTestResult {
+        // Build the tests. Ideally we should only build those that match the subset, but we don't have a way to know which ones they are until we've built them and can examine the binaries.
+        let toolchain = try swiftTool.getToolchain()
+        var buildParameters = try swiftTool.buildParameters()
+        buildParameters.enableTestability = true
+        buildParameters.enableCodeCoverage = parameters.enableCodeCoverage
+        let buildSystem = try swiftTool.createBuildSystem(buildParameters: buildParameters)
+        try buildSystem.build(subset: .allIncludingTests)
+
+        // Clean out the code coverage directory that may contain stale `profraw` files from a previous run of the code coverage tool.
+        if parameters.enableCodeCoverage {
+            try localFileSystem.removeFileTree(buildParameters.codeCovPath)
+        }
+
+        // Construct the environment we'll pass down to the tests.
+        var environmentOptions = swiftTool.options
+        environmentOptions.shouldEnableCodeCoverage = parameters.enableCodeCoverage
+        let testEnvironment = try constructTestEnvironment(
+            toolchain: toolchain,
+            options: environmentOptions,
+            buildParameters: buildParameters)
+
+        // Iterate over the tests and run those that match the filter.
+        var testTargetResults: [PluginInvocationTestResult.TestTarget] = []
+        var numFailedTests = 0
+        for testProduct in buildSystem.builtTestProducts {
+            // Get the test suites in the bundle. Each is just a container for test cases.
+            let testSuites = try getTestSuites(fromTestAt: testProduct.bundlePath, swiftTool: swiftTool, swiftOptions: swiftTool.options)
+            for testSuite in testSuites {
+                // Each test suite is just a container for test cases (confusingly called "tests", though they are test cases).
+                for testCase in testSuite.tests {
+                    // Each test case corresponds to a combination of target and a XCTestCase, and is a collection of tests that can actually be run.
+                    var testResults: [PluginInvocationTestResult.TestTarget.TestCase.Test] = []
+                    for testName in testCase.tests {
+                        // Check if we should filter out this test.
+                        let testSpecifier = testCase.name + "/" + testName
+                        if case .filtered(let regexes) = subset {
+                            guard regexes.contains(where: { testSpecifier.range(of: $0, options: .regularExpression) != nil }) else {
+                                continue
+                            }
+                        }
+
+                        // Configure a test runner.
+                        let testRunner = TestRunner(
+                            bundlePaths: [testProduct.bundlePath],
+                            xctestArg: testSpecifier,
+                            processSet: swiftTool.processSet,
+                            toolchain: toolchain,
+                            testEnv: testEnvironment,
+                            outputStream: swiftTool.outputStream,
+                            observabilityScope: swiftTool.observabilityScope)
+
+                        // Run the test — for now we run the sequentially so we can capture accurate timing results.
+                        let startTime = DispatchTime.now()
+                        let (success, _) = testRunner.test(writeToOutputStream: false)
+                        let duration = Double(startTime.distance(to: .now()).milliseconds() ?? 0) / 1000.0
+                        numFailedTests += success ? 0 : 1
+                        testResults.append(.init(name: testName, result: success ? .succeeded : .failed, duration: duration))
+                    }
+
+                    // Don't add any results if we didn't run any tests.
+                    if testResults.isEmpty { continue }
+
+                    // Otherwise we either create a new create a new target result or add to the previous one, depending on whether the target name is the same.
+                    let testTargetName = testCase.name.prefix(while: { $0 != "." })
+                    if let lastTestTargetName = testTargetResults.last?.name, testTargetName == lastTestTargetName {
+                        // Same as last one, just extend its list of cases. We know we have a last one at this point.
+                        testTargetResults[testTargetResults.count-1].testCases.append(.init(name: testCase.name, tests: testResults))
+                    }
+                    else {
+                        // Not the same, so start a new target result.
+                        testTargetResults.append(.init(name: String(testTargetName), testCases: [.init(name: testCase.name, tests: testResults)]))
+                    }
+                }
+            }
+        }
+
+        // Deal with code coverage, if enabled.
+        let codeCoverageDataFile: AbsolutePath?
+        if parameters.enableCodeCoverage {
+            // Use `llvm-prof` to merge all the `.profraw` files into a single `.profdata` file.
+            let mergedCovFile = buildParameters.codeCovDataFile
+            let codeCovFileNames = try localFileSystem.getDirectoryContents(buildParameters.codeCovPath)
+            var llvmProfCommand = [try toolchain.getLLVMProf().pathString]
+            llvmProfCommand += ["merge", "-sparse"]
+            for fileName in codeCovFileNames where fileName.hasSuffix(".profraw") {
+                let filePath = buildParameters.codeCovPath.appending(component: fileName)
+                llvmProfCommand.append(filePath.pathString)
+            }
+            llvmProfCommand += ["-o", mergedCovFile.pathString]
+            try Process.checkNonZeroExit(arguments: llvmProfCommand)
+
+            // Use `llvm-cov` to export the merged `.profdata` file contents in JSON form.
+            var llvmCovCommand = [try toolchain.getLLVMCov().pathString]
+            llvmCovCommand += ["export", "-instr-profile=\(mergedCovFile.pathString)"]
+            for product in buildSystem.builtTestProducts {
+                llvmCovCommand.append("-object")
+                llvmCovCommand.append(product.binaryPath.pathString)
+            }
+            // We get the output on stdout, and have to write it to a JSON ourselves.
+            let jsonOutput = try Process.checkNonZeroExit(arguments: llvmCovCommand)
+            let jsonCovFile = buildParameters.codeCovDataFile.parentDirectory.appending(component: buildParameters.codeCovDataFile.basenameWithoutExt + ".json")
+            try localFileSystem.writeFileContents(jsonCovFile, string: jsonOutput)
+
+            // Return the path of the exported code coverage data file.
+            codeCoverageDataFile = jsonCovFile
+        }
+        else {
+            codeCoverageDataFile = nil
+        }
+
+        // Return the results to the plugin. We only consider the test run a success if no test failed.
+        return PluginInvocationTestResult(
+            succeeded: (numFailedTests == 0),
+            testTargets: testTargetResults,
+            codeCoverageDataFile: codeCoverageDataFile?.pathString)
     }
 
     func pluginRequestedSymbolGraph(forTarget targetName: String, options: PluginInvocationSymbolGraphOptions, completion: @escaping (Result<PluginInvocationSymbolGraphResult, Error>) -> Void) {

Sources/Commands/SwiftTestTool.swift

@@ -1,7 +1,7 @@
 /*
 This source file is part of the Swift.org open source project
 
-Copyright 2015 - 2016 Apple Inc. and the Swift project authors
+Copyright 2015 - 2022 Apple Inc. and the Swift project authors
 Licensed under Apache License v2.0 with Runtime Library Exception
 
 See http://swift.org/LICENSE.txt for license information
@@ -209,7 +209,7 @@ public struct SwiftTestTool: SwiftCommand {
         switch options.mode {
         case .listTests:
             let testProducts = try buildTestsIfNeeded(swiftTool: swiftTool)
-            let testSuites = try getTestSuites(in: testProducts, swiftTool: swiftTool)
+            let testSuites = try getTestSuites(in: testProducts, swiftTool: swiftTool, swiftOptions: swiftOptions)
             let tests = try testSuites
                 .filteredTests(specifier: options.testCaseSpecifier)
                 .skippedTests(specifier: options.testCaseSkip)
@@ -245,7 +245,7 @@ public struct SwiftTestTool: SwiftCommand {
             #endif
             let graph = try swiftTool.loadPackageGraph()
             let testProducts = try buildTestsIfNeeded(swiftTool: swiftTool)
-            let testSuites = try getTestSuites(in: testProducts, swiftTool: swiftTool)
+            let testSuites = try getTestSuites(in: testProducts, swiftTool: swiftTool, swiftOptions: swiftOptions)
             let allTestSuites = testSuites.values.flatMap { $0 }
             let generator = LinuxMainGenerator(graph: graph, testSuites: allTestSuites)
             try generator.generate()
@@ -278,7 +278,7 @@ public struct SwiftTestTool: SwiftCommand {
                 }
 
                 // Find the tests we need to run.
-                let testSuites = try getTestSuites(in: testProducts, swiftTool: swiftTool)
+                let testSuites = try getTestSuites(in: testProducts, swiftTool: swiftTool, swiftOptions: swiftOptions)
                 let tests = try testSuites
                     .filteredTests(specifier: options.testCaseSpecifier)
                     .skippedTests(specifier: options.testCaseSkip)
@@ -317,7 +317,7 @@ public struct SwiftTestTool: SwiftCommand {
         case .runParallel:
             let toolchain = try swiftTool.getToolchain()
             let testProducts = try buildTestsIfNeeded(swiftTool: swiftTool)
-            let testSuites = try getTestSuites(in: testProducts, swiftTool: swiftTool)
+            let testSuites = try getTestSuites(in: testProducts, swiftTool: swiftTool, swiftOptions: swiftOptions)
             let tests = try testSuites
                 .filteredTests(specifier: options.testCaseSpecifier)
                 .skippedTests(specifier: options.testCaseSkip)
@@ -461,70 +461,6 @@ public struct SwiftTestTool: SwiftCommand {
         }
     }
 
-    /// Locates XCTestHelper tool inside the libexec directory and bin directory.
-    /// Note: It is a fatalError if we are not able to locate the tool.
-    ///
-    /// - Returns: Path to XCTestHelper tool.
-    private func xctestHelperPath(swiftTool: SwiftTool) throws -> AbsolutePath {
-        let xctestHelperBin = "swiftpm-xctest-helper"
-        let binDirectory = AbsolutePath(CommandLine.arguments.first!,
-            relativeTo: swiftTool.originalWorkingDirectory).parentDirectory
-        // XCTestHelper tool is installed in libexec.
-        let maybePath = binDirectory.parentDirectory.appending(components: "libexec", "swift", "pm", xctestHelperBin)
-        if localFileSystem.isFile(maybePath) {
-            return maybePath
-        }
-        // This will be true during swiftpm development.
-        // FIXME: Factor all of the development-time resource location stuff into a common place.
-        let path = binDirectory.appending(component: xctestHelperBin)
-        if localFileSystem.isFile(path) {
-            return path
-        }
-        throw InternalError("XCTestHelper binary not found.")
-    }
-
-    fileprivate func getTestSuites(in testProducts: [BuiltTestProduct], swiftTool: SwiftTool) throws -> [AbsolutePath: [TestSuite]] {
-        let testSuitesByProduct = try testProducts
-            .map { try ($0.bundlePath, self.getTestSuites(fromTestAt: $0.bundlePath, swiftTool: swiftTool)) }
-        return Dictionary(uniqueKeysWithValues: testSuitesByProduct)
-    }
-
-    /// Runs the corresponding tool to get tests JSON and create TestSuite array.
-    /// On macOS, we use the swiftpm-xctest-helper tool bundled with swiftpm.
-    /// On Linux, XCTest can dump the json using `--dump-tests-json` mode.
-    ///
-    /// - Parameters:
-    ///     - path: Path to the XCTest bundle(macOS) or executable(Linux).
-    ///
-    /// - Throws: TestError, SystemError, TSCUtility.Error
-    ///
-    /// - Returns: Array of TestSuite
-    fileprivate func getTestSuites(fromTestAt path: AbsolutePath, swiftTool: SwiftTool) throws -> [TestSuite] {
-        // Run the correct tool.
-        #if os(macOS)
-        let data: String = try withTemporaryFile { tempFile in
-            let args = [try xctestHelperPath(swiftTool: swiftTool).pathString, path.pathString, tempFile.path.pathString]
-            var env = try constructTestEnvironment(toolchain: try swiftTool.getToolchain(), options: swiftOptions, buildParameters: swiftTool.buildParametersForTest())
-            // Add the sdk platform path if we have it. If this is not present, we
-            // might always end up failing.
-            if let sdkPlatformFrameworksPath = Destination.sdkPlatformFrameworkPaths() {
-                // appending since we prefer the user setting (if set) to the one we inject
-                env.appendPath("DYLD_FRAMEWORK_PATH", value: sdkPlatformFrameworksPath.fwk.pathString)
-                env.appendPath("DYLD_LIBRARY_PATH", value: sdkPlatformFrameworksPath.lib.pathString)
-            }
-            try Process.checkNonZeroExit(arguments: args, environment: env)
-            // Read the temporary file's content.
-            return try localFileSystem.readFileContents(tempFile.path).validDescription ?? ""
-        }
-        #else
-        let env = try constructTestEnvironment(toolchain: try swiftTool.getToolchain(), options: swiftOptions, buildParameters: swiftTool.buildParametersForTest())
-        let args = [path.description, "--dump-tests-json"]
-        let data = try Process.checkNonZeroExit(arguments: args, environment: env)
-        #endif
-        // Parse json and return TestSuites.
-        return try TestSuite.parse(jsonString: data)
-    }
-
     /// Private function that validates the commands arguments
     ///
     /// - Throws: if a command argument is invalid
@@ -1022,54 +958,6 @@ fileprivate extension Array where Element == UnitTest {
     }
 }
 
-/// Creates the environment needed to test related tools.
-fileprivate func constructTestEnvironment(
-    toolchain: UserToolchain,
-    options: SwiftToolOptions,
-    buildParameters: BuildParameters
-) throws -> EnvironmentVariables {
-    var env = EnvironmentVariables.process()
-
-    // Add the code coverage related variables.
-    if options.shouldEnableCodeCoverage {
-        // Defines the path at which the profraw files will be written on test execution.
-        //
-        // `%m` will create a pool of profraw files and append the data from
-        // each execution in one of the files. This doesn't matter for serial
-        // execution but is required when the tests are running in parallel as
-        // SwiftPM repeatedly invokes the test binary with the test case name as
-        // the filter.
-        let codecovProfile = buildParameters.buildPath.appending(components: "codecov", "default%m.profraw")
-        env["LLVM_PROFILE_FILE"] = codecovProfile.pathString
-    }
-    #if !os(macOS)
-    #if os(Windows)
-    if let location = toolchain.configuration.xctestPath {
-        env.prependPath("Path", value: location.pathString)
-    }
-    #endif
-    return env
-    #else
-    // Fast path when no sanitizers are enabled.
-    if options.sanitizers.isEmpty {
-        return env
-    }
-
-    // Get the runtime libraries.
-    var runtimes = try options.sanitizers.map({ sanitizer in
-        return try toolchain.runtimeLibrary(for: sanitizer).pathString
-    })
-
-    // Append any existing value to the front.
-    if let existingValue = env["DYLD_INSERT_LIBRARIES"], !existingValue.isEmpty {
-        runtimes.insert(existingValue, at: 0)
-    }
-
-    env["DYLD_INSERT_LIBRARIES"] = runtimes.joined(separator: ":")
-    return env
-    #endif
-}
-
 /// xUnit XML file generator for a swift-test run.
 final class XUnitGenerator {
     typealias TestResult = ParallelTestRunner.TestResult
@@ -1130,12 +1018,3 @@ private extension Basics.Diagnostic {
         .warning("No matching test cases were run")
     }
 }
-
-private extension SwiftTool {
-    func buildParametersForTest() throws -> BuildParameters {
-        var parameters = try self.buildParameters()
-        // for test commands, alway enable building with testability enabled
-        parameters.enableTestability = true
-        return parameters
-    }
-}