Cross compilation improvements

A collection of fixes and improvements related to cross-compilation.

An attempt to make cross-compilation support more robust, consistent, error-prone and deal with performance regressions related to plugins.

- `BuildParameters` are now associated with a destination (host or target) to make sure that they are always used in accordance with their original purpose.
- `Buildset` gains a default `for:` parameter to indicate the intended destination of a product/target
- LLBuild names for products and modules are computed based on the build parameters instead of relying on `buildTriple` of an product/target
- Adds a way to get a command and llbuild name from a build description instead of having to reach for an underlying resolved module/product and pass parameters to it
- Refactors most of the uses of `allTargets` and `allProducts` to use the graph and supply an intended destination
- `SwiftTargetBuildDescription` no longer needs to reference `toolsBuildParameters` by reworking the way requires macros
are referenced
- Removes `ModulesGraph.updateBuildTripleRecursively`
- Plugins:
  - Fixes a bug where graph doesn't get a "tools" product synthesized for an executable target referenced by a plugin
  - Fixes a bug where plugin outputs weren't added to test targets that a rebuilt for "tools"
- Tests:
  - Introduces a new convenient way to mock build plans that respects tools/destination separation of parameters
  - Adds a "destination" parameter to `check{Target, Product}` default to `.destination`

- `swift build` with `--target` and `--product` now behave consistently
- performance regression related to plugins has been fixed
- It should be harder to request product/target without a destination

Author: xedin

Sources/Build/BuildDescription/PluginDescription.swift

@@ -29,6 +29,9 @@ public final class PluginDescription: Codable {
     /// the plugin).
     public let targetName: String
 
+    /// The language-level target name.
+    public let targetC99Name: String
+
     /// The names of any plugin products in that package that vend the plugin
     /// to other packages.
     public let productNames: [String]
@@ -56,6 +59,7 @@ public final class PluginDescription: Codable {
 
         self.package = package.identity
         self.targetName = target.name
+        self.targetC99Name = target.c99name
         self.productNames = products.map(\.name)
         self.toolsVersion = toolsVersion
         self.sources = target.sources

Sources/Build/BuildDescription/ResolvedModule+BuildDescription.swift

@@ -17,7 +17,7 @@ import SPMBuildCore
 
 extension ResolvedModule {
     func tempsPath(_ buildParameters: BuildParameters) -> AbsolutePath {
-        let suffix = buildParameters.suffix(triple: self.buildTriple)
+        let suffix = buildParameters.suffix
         return buildParameters.buildPath.appending(component: "\(self.c99name)\(suffix).build")
     }
 }

Sources/Build/BuildDescription/SwiftTargetBuildDescription.swift

@@ -101,7 +101,7 @@ public enum TargetBuildDescription {
     var buildParameters: BuildParameters {
         switch self {
         case .swift(let swiftTargetBuildDescription):
-            return swiftTargetBuildDescription.defaultBuildParameters
+            return swiftTargetBuildDescription.buildParameters
         case .clang(let clangTargetBuildDescription):
             return clangTargetBuildDescription.buildParameters
         }

Sources/Build/BuildDescription/TargetBuildDescription.swift

@@ -93,7 +93,7 @@ extension LLBuildManifestBuilder {
         let additionalInputs = try addBuildToolPlugins(.clang(target))
 
         // Create a phony node to represent the entire target.
-        let targetName = target.target.getLLBuildTargetName(buildParameters: target.buildParameters)
+        let targetName = target.llbuildTargetName
         let output: Node = .virtual(targetName)
 
         self.manifest.addNode(output, toTarget: targetName)

Sources/Build/BuildManifest/LLBuildManifestBuilder+Clang.swift

@@ -11,11 +11,14 @@
 //===----------------------------------------------------------------------===//
 
 import struct Basics.AbsolutePath
+import struct Basics.InternalError
 import struct LLBuildManifest.Node
+import struct SPMBuildCore.BuildParameters
+import struct PackageGraph.ResolvedProduct
 
 extension LLBuildManifestBuilder {
     func createProductCommand(_ buildProduct: ProductBuildDescription) throws {
-        let cmdName = try buildProduct.product.getCommandName(buildParameters: buildProduct.buildParameters)
+        let cmdName = try buildProduct.commandName
 
         // Add dependency on Info.plist generation on Darwin platforms.
         let testInputs: [AbsolutePath]
@@ -34,7 +37,7 @@ extension LLBuildManifestBuilder {
         }
 
         // Create a phony node to represent the entire target.
-        let targetName = try buildProduct.product.getLLBuildTargetName(buildParameters: buildProduct.buildParameters)
+        let targetName = try buildProduct.llbuildTargetName
         let output: Node = .virtual(targetName)
 
         let finalProductNode: Node
@@ -85,7 +88,7 @@ extension LLBuildManifestBuilder {
                     outputPath: plistPath
                 )
 
-                let cmdName = try buildProduct.product.getCommandName(buildParameters: buildProduct.buildParameters)
+                let cmdName = try buildProduct.commandName
                 let codeSigningOutput = Node.virtual(targetName + "-CodeSigning")
                 try self.manifest.addShellCmd(
                     name: "\(cmdName)-entitlements",
@@ -120,3 +123,48 @@ extension LLBuildManifestBuilder {
         )
     }
 }
+
+extension ProductBuildDescription {
+    package var llbuildTargetName: String {
+        get throws {
+            try self.product.getLLBuildTargetName(buildParameters: self.buildParameters)
+        }
+    }
+
+    package var commandName: String {
+        get throws {
+            try "C.\(self.llbuildTargetName)\(self.buildParameters.suffix)"
+        }
+    }
+}
+
+extension ResolvedProduct {
+    public func getLLBuildTargetName(buildParameters: BuildParameters) throws -> String {
+        let triple = buildParameters.triple.tripleString
+        let config = buildParameters.buildConfig
+        let suffix = buildParameters.suffix
+        let potentialExecutableTargetName = "\(name)-\(triple)-\(config)\(suffix).exe"
+        let potentialLibraryTargetName = "\(name)-\(triple)-\(config)\(suffix).dylib"
+
+        switch type {
+        case .library(.dynamic):
+            return potentialLibraryTargetName
+        case .test:
+            return "\(name)-\(triple)-\(config)\(suffix).test"
+        case .library(.static):
+            return "\(name)-\(triple)-\(config)\(suffix).a"
+        case .library(.automatic):
+            throw InternalError("automatic library not supported")
+        case .executable, .snippet:
+            return potentialExecutableTargetName
+        case .macro:
+            #if BUILD_MACROS_AS_DYLIBS
+            return potentialLibraryTargetName
+            #else
+            return potentialExecutableTargetName
+            #endif
+        case .plugin:
+            throw InternalError("unexpectedly asked for the llbuild target name of a plugin product")
+        }
+    }
+}

Sources/Build/BuildManifest/LLBuildManifestBuilder+Product.swift

@@ -45,7 +45,7 @@ extension LLBuildManifestBuilder {
             outputs.append(output)
         }
 
-        let cmdName = target.target.getLLBuildResourcesCmdName(buildParameters: target.buildParameters)
+        let cmdName = target.llbuildResourcesCmdName
         self.manifest.addPhonyCmd(name: cmdName, inputs: outputs, outputs: [.virtual(cmdName)])
 
         return .virtual(cmdName)

Sources/Build/BuildManifest/LLBuildManifestBuilder+Resources.swift

@@ -45,7 +45,7 @@ extension LLBuildManifestBuilder {
         let moduleNode = Node.file(target.moduleOutputPath)
         let cmdOutputs = objectNodes + [moduleNode]
 
-        if target.defaultBuildParameters.driverParameters.useIntegratedSwiftDriver {
+        if target.buildParameters.driverParameters.useIntegratedSwiftDriver {
             try self.addSwiftCmdsViaIntegratedDriver(
                 target,
                 inputs: inputs,
@@ -68,7 +68,7 @@ extension LLBuildManifestBuilder {
         // jobs needed to build this Swift target.
         var commandLine = try target.emitCommandLine()
         commandLine.append("-driver-use-frontend-path")
-        commandLine.append(target.defaultBuildParameters.toolchain.swiftCompilerPath.pathString)
+        commandLine.append(target.buildParameters.toolchain.swiftCompilerPath.pathString)
         // FIXME: At some point SwiftPM should provide its own executor for
         // running jobs/launching processes during planning
         let resolver = try ArgsResolver(fileSystem: target.fileSystem)
@@ -132,7 +132,7 @@ extension LLBuildManifestBuilder {
             // common intermediate dependency modules, such dependencies can lead
             // to cycles in the resulting manifest.
             var manifestNodeInputs: [Node] = []
-            if targetDescription.defaultBuildParameters.driverParameters.useExplicitModuleBuild && !isMainModule(job) {
+            if targetDescription.buildParameters.driverParameters.useExplicitModuleBuild && !isMainModule(job) {
                 manifestNodeInputs = jobInputs
             } else {
                 manifestNodeInputs = (inputs + jobInputs).uniqued()
@@ -191,11 +191,8 @@ extension LLBuildManifestBuilder {
     public func addTargetsToExplicitBuildManifest() throws {
         // Sort the product targets in topological order in order to collect and "bubble up"
         // their respective dependency graphs to the depending targets.
-        let nodes: [ResolvedModule.Dependency] = try self.plan.targetMap.keys.compactMap {
-            guard let target = self.plan.graph.allTargets[$0] else {
-                throw InternalError("unknown target \($0)")
-            }
-            return ResolvedModule.Dependency.target(target, conditions: [])
+        let nodes = self.plan.targets.compactMap {
+            ResolvedModule.Dependency.target($0.target, conditions: [])
         }
         let allPackageDependencies = try topologicalSort(nodes, successors: { $0.dependencies })
         // Instantiate the inter-module dependency oracle which will cache commonly-scanned
@@ -287,7 +284,7 @@ extension LLBuildManifestBuilder {
         // jobs needed to build this Swift target.
         var commandLine = try targetDescription.emitCommandLine()
         commandLine.append("-driver-use-frontend-path")
-        commandLine.append(targetDescription.defaultBuildParameters.toolchain.swiftCompilerPath.pathString)
+        commandLine.append(targetDescription.buildParameters.toolchain.swiftCompilerPath.pathString)
         commandLine.append("-experimental-explicit-module-build")
         let resolver = try ArgsResolver(fileSystem: self.fileSystem)
         let executor = SPMSwiftDriverExecutor(
@@ -378,14 +375,14 @@ extension LLBuildManifestBuilder {
         cmdOutputs: [Node]
     ) throws {
         let isLibrary = target.target.type == .library || target.target.type == .test
-        let cmdName = target.target.getCommandName(buildParameters: target.defaultBuildParameters)
+        let cmdName = target.getCommandName()
 
         self.manifest.addWriteSourcesFileListCommand(sources: target.sources, sourcesFileListPath: target.sourcesFileListPath)
         self.manifest.addSwiftCmd(
             name: cmdName,
             inputs: inputs + [Node.file(target.sourcesFileListPath)],
             outputs: cmdOutputs,
-            executable: target.defaultBuildParameters.toolchain.swiftCompilerPath,
+            executable: target.buildParameters.toolchain.swiftCompilerPath,
             moduleName: target.target.c99name,
             moduleAliases: target.target.moduleAliases,
             moduleOutputPath: target.moduleOutputPath,
@@ -396,7 +393,7 @@ extension LLBuildManifestBuilder {
             sources: target.sources,
             fileList: target.sourcesFileListPath,
             isLibrary: isLibrary,
-            wholeModuleOptimization: target.defaultBuildParameters.configuration == .release,
+            wholeModuleOptimization: target.buildParameters.configuration == .release,
             outputFileMapPath: try target.writeOutputFileMap() // FIXME: Eliminate side effect.
         )
     }
@@ -406,7 +403,7 @@ extension LLBuildManifestBuilder {
     ) throws -> [Node] {
         var inputs = target.sources.map(Node.file)
 
-        let swiftVersionFilePath = addSwiftGetVersionCommand(buildParameters: target.defaultBuildParameters)
+        let swiftVersionFilePath = addSwiftGetVersionCommand(buildParameters: target.buildParameters)
         inputs.append(.file(swiftVersionFilePath))
 
         // Add resources node as the input to the target. This isn't great because we
@@ -433,14 +430,10 @@ extension LLBuildManifestBuilder {
             // Depend on the binary for executable targets.
             if target.type == .executable {
                 // FIXME: Optimize.
-                let product = try plan.graph.allProducts.first {
-                    try $0.type == .executable && $0.executableTarget.id == target.id
-                }
-                if let product {
-                    guard let planProduct = plan.productMap[product.id] else {
-                        throw InternalError("unknown product \(product)")
-                    }
-                    try inputs.append(file: planProduct.binaryPath)
+                if let productDescription = try plan.productMap.values.first(where: {
+                    try $0.product.type == .executable && $0.product.executableTarget.id == target.id
+                }) {
+                    try inputs.append(file: productDescription.binaryPath)
                 }
                 return
             }
@@ -457,7 +450,7 @@ extension LLBuildManifestBuilder {
             }
         }
 
-        for dependency in target.target.dependencies(satisfying: target.defaultBuildParameters.buildEnvironment) {
+        for dependency in target.target.dependencies(satisfying: target.buildParameters.buildEnvironment) {
             switch dependency {
             case .target(let target, _):
                 try addStaticTargetInputs(target)
@@ -484,7 +477,7 @@ extension LLBuildManifestBuilder {
         }
 
         for binaryPath in target.libraryBinaryPaths {
-            let path = target.defaultBuildParameters.destinationPath(forBinaryAt: binaryPath)
+            let path = target.buildParameters.destinationPath(forBinaryAt: binaryPath)
             if self.fileSystem.isDirectory(binaryPath) {
                 inputs.append(directory: path)
             } else {
@@ -496,7 +489,7 @@ extension LLBuildManifestBuilder {
 
         // Depend on any required macro product's output.
         try target.requiredMacroProducts.forEach { macro in
-            try inputs.append(.virtual(macro.getLLBuildTargetName(buildParameters: target.defaultBuildParameters)))
+            try inputs.append(.virtual(macro.llbuildTargetName))
         }
 
         return inputs + additionalInputs
@@ -505,7 +498,7 @@ extension LLBuildManifestBuilder {
     /// Adds a top-level phony command that builds the entire target.
     private func addTargetCmd(_ target: SwiftTargetBuildDescription, cmdOutputs: [Node]) {
         // Create a phony node to represent the entire target.
-        let targetName = target.target.getLLBuildTargetName(buildParameters: target.defaultBuildParameters)
+        let targetName = target.getLLBuildTargetName()
         let targetOutput: Node = .virtual(targetName)
 
         self.manifest.addNode(targetOutput, toTarget: targetName)
@@ -514,7 +507,7 @@ extension LLBuildManifestBuilder {
             inputs: cmdOutputs,
             outputs: [targetOutput]
         )
-        if self.plan.graph.isInRootPackages(target.target, satisfying: target.defaultBuildParameters.buildEnvironment) {
+        if self.plan.graph.isInRootPackages(target.target, satisfying: target.buildParameters.buildEnvironment) {
             if !target.isTestTarget {
                 self.addNode(targetOutput, toTarget: .main)
             }
@@ -524,13 +517,13 @@ extension LLBuildManifestBuilder {
 
     private func addModuleWrapCmd(_ target: SwiftTargetBuildDescription) throws {
         // Add commands to perform the module wrapping Swift modules when debugging strategy is `modulewrap`.
-        guard target.defaultBuildParameters.debuggingStrategy == .modulewrap else { return }
+        guard target.buildParameters.debuggingStrategy == .modulewrap else { return }
         var moduleWrapArgs = [
-            target.defaultBuildParameters.toolchain.swiftCompilerPath.pathString,
+            target.buildParameters.toolchain.swiftCompilerPath.pathString,
             "-modulewrap", target.moduleOutputPath.pathString,
             "-o", target.wrappedModuleOutputPath.pathString,
         ]
-        moduleWrapArgs += try target.defaultBuildParameters.tripleArgs(for: target.target)
+        moduleWrapArgs += try target.buildParameters.tripleArgs(for: target.target)
         self.manifest.addShellCmd(
             name: target.wrappedModuleOutputPath.pathString,
             description: "Wrapping AST for \(target.target.name) for debugging",
@@ -611,3 +604,13 @@ extension Driver {
         }
     }
 }
+
+extension SwiftTargetBuildDescription {
+    public func getCommandName() -> String {
+        "C." + self.getLLBuildTargetName()
+    }
+
+    public func getLLBuildTargetName() -> String {
+        self.target.getLLBuildTargetName(buildParameters: self.buildParameters)
+    }
+}

Sources/Build/BuildManifest/LLBuildManifestBuilder+Swift.swift

@@ -316,52 +316,21 @@ extension TargetBuildDescription {
     }
 }
 
-extension ResolvedModule {
-    public func getCommandName(buildParameters: BuildParameters) -> String {
-        "C." + self.getLLBuildTargetName(buildParameters: buildParameters)
-    }
-
-    public func getLLBuildTargetName(buildParameters: BuildParameters) -> String {
-        "\(self.name)-\(buildParameters.triple.tripleString)-\(buildParameters.buildConfig)\(buildParameters.suffix(triple: self.buildTriple)).module"
-    }
-
-    public func getLLBuildResourcesCmdName(buildParameters: BuildParameters) -> String {
-        "\(self.name)-\(buildParameters.triple.tripleString)-\(buildParameters.buildConfig)\(buildParameters.suffix(triple: self.buildTriple)).module-resources"
+extension TargetBuildDescription {
+    package var llbuildResourcesCmdName: String {
+        "\(self.target.name)-\(self.buildParameters.triple.tripleString)-\(self.buildParameters.buildConfig)\(self.buildParameters.suffix).module-resources"
     }
 }
 
-extension ResolvedProduct {
-    public func getLLBuildTargetName(buildParameters: BuildParameters) throws -> String {
-        let triple = buildParameters.triple.tripleString
-        let config = buildParameters.buildConfig
-        let suffix = buildParameters.suffix(triple: self.buildTriple)
-        let potentialExecutableTargetName = "\(name)-\(triple)-\(config)\(suffix).exe"
-        let potentialLibraryTargetName = "\(name)-\(triple)-\(config)\(suffix).dylib"
-
-        switch type {
-        case .library(.dynamic):
-            return potentialLibraryTargetName
-        case .test:
-            return "\(name)-\(triple)-\(config)\(suffix).test"
-        case .library(.static):
-            return "\(name)-\(triple)-\(config)\(suffix).a"
-        case .library(.automatic):
-            throw InternalError("automatic library not supported")
-        case .executable, .snippet:
-            return potentialExecutableTargetName
-        case .macro:
-            #if BUILD_MACROS_AS_DYLIBS
-            return potentialLibraryTargetName
-            #else
-            return potentialExecutableTargetName
-            #endif
-        case .plugin:
-            throw InternalError("unexpectedly asked for the llbuild target name of a plugin product")
-        }
+extension ClangTargetBuildDescription {
+    package var llbuildTargetName: String {
+        self.target.getLLBuildTargetName(buildParameters: self.buildParameters)
     }
+}
 
-    public func getCommandName(buildParameters: BuildParameters) throws -> String {
-        try "C.\(self.getLLBuildTargetName(buildParameters: buildParameters))\(buildParameters.suffix(triple: self.buildTriple))"
+extension ResolvedModule {
+    public func getLLBuildTargetName(buildParameters: BuildParameters) -> String {
+        "\(self.name)-\(buildParameters.triple.tripleString)-\(buildParameters.buildConfig)\(buildParameters.suffix).module"
     }
 }