Refactor build system support

This is an initial step of unwinding the direct dependency of individual commands to a concrete build system implementation, through two major changes:
- Adding a new target `DriverSupport` which contains integration with swift-driver that is independent of a concrete build system being used (today mostly `checkSupportedFrontendFlags(flags:fileSystem)` but there could be more in the future)
- Adding a new `BuildSystemProvider` API which handles instantiation of a concrete build system. This will e.g. allow creating a `SwiftCommand` which doesn't depend on the `XCBuildSupport` module (which is going to help with #5807) and also provides a way to generally clean up (almost) all callsites from directly interacting with a concrete build system.

Author: neonichu

Package.swift

@@ -308,6 +308,14 @@ let package = Package(
                 "SPMBuildCore",
                 "SPMLLBuild",
                 .product(name: "SwiftDriver", package: "swift-driver"),
+                "DriverSupport",
+            ],
+            exclude: ["CMakeLists.txt"]
+        ),
+        .target(
+            name: "DriverSupport",
+            dependencies: [
+                .product(name: "SwiftDriver", package: "swift-driver"),
             ],
             exclude: ["CMakeLists.txt"]
         ),

Sources/Build/BuildOperation.swift

@@ -11,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 import Basics
+@_implementationOnly import DriverSupport
 import LLBuildManifest
 import PackageGraph
 import PackageLoading
@@ -57,7 +58,17 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
     public let cacheBuildManifest: Bool
 
     /// The build plan that was computed, if any.
-    public private(set) var buildPlan: BuildPlan?
+    public private(set) var _buildPlan: BuildPlan?
+
+    public var buildPlan: SPMBuildCore.BuildPlan {
+        get throws {
+            if let buildPlan = _buildPlan {
+                return buildPlan
+            } else {
+                throw StringError("did not compute a build plan yet")
+            }
+        }
+    }
 
     /// The build description resulting from planing.
     private let buildDescription = ThreadSafeBox<BuildDescription>()
@@ -166,7 +177,7 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
             return
         }
         // Ensure the compiler supports the import-scan operation
-        guard SwiftTargetBuildDescription.checkSupportedFrontendFlags(flags: ["import-prescan"], fileSystem: localFileSystem) else {
+        guard DriverSupport.checkSupportedFrontendFlags(flags: ["import-prescan"], fileSystem: localFileSystem) else {
             return
         }
 
@@ -457,7 +468,7 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
             fileSystem: self.fileSystem,
             observabilityScope: self.observabilityScope
         )
-        self.buildPlan = plan
+        self._buildPlan = plan
 
         let (buildDescription, buildManifest) = try BuildDescription.create(
             with: plan,
@@ -568,14 +579,14 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
 
     public func provideBuildErrorAdvice(for target: String, command: String, message: String) -> String? {
         // Find the target for which the error was emitted.  If we don't find it, we can't give any advice.
-        guard let _ = self.buildPlan?.targets.first(where: { $0.target.name == target }) else { return nil }
+        guard let _ = self._buildPlan?.targets.first(where: { $0.target.name == target }) else { return nil }
 
         // Check for cases involving modules that cannot be found.
         if let importedModule = try? RegEx(pattern: "no such module '(.+)'").matchGroups(in: message).first?.first {
             // A target is importing a module that can't be found.  We take a look at the build plan and see if can offer any advice.
 
             // Look for a target with the same module name as the one that's being imported.
-            if let importedTarget = self.buildPlan?.targets.first(where: { $0.target.c99name == importedModule }) {
+            if let importedTarget = self._buildPlan?.targets.first(where: { $0.target.c99name == importedModule }) {
                 // For the moment we just check for executables that other targets try to import.
                 if importedTarget.target.type == .executable {
                     return "module '\(importedModule)' is the main module of an executable, and cannot be imported by tests and other targets"

Sources/Build/BuildOperationBuildSystemDelegateHandler.swift

@@ -354,7 +354,7 @@ public struct BuildDescription: Codable {
         self.builtTestProducts = plan.buildProducts.filter{ $0.product.type == .test }.map { desc in
             return BuiltTestProduct(
                 productName: desc.product.name,
-                binaryPath: desc.binary
+                binaryPath: desc.binaryPath
             )
         }
         self.pluginDescriptions = pluginDescriptions

Sources/Build/BuildPlan.swift

@@ -851,16 +851,6 @@ public final class SwiftTargetBuildDescription {
         try self.fileSystem.writeIfChanged(path: path, bytes: stream.bytes)
     }
 
-    public static func checkSupportedFrontendFlags(flags: Set<String>, fileSystem: FileSystem) -> Bool {
-        do {
-            let executor = try SPMSwiftDriverExecutor(resolver: ArgsResolver(fileSystem: fileSystem), fileSystem: fileSystem, env: [:])
-            let driver = try Driver(args: ["swiftc"], executor: executor)
-            return driver.supportedFrontendFlags.intersection(flags) == flags
-        } catch {
-            return false
-        }
-    }
-
     /// The arguments needed to compile this target.
     public func compileArguments() throws -> [String] {
         var args = [String]()
@@ -1262,7 +1252,7 @@ public final class SwiftTargetBuildDescription {
 }
 
 /// The build description for a product.
-public final class ProductBuildDescription {
+public final class ProductBuildDescription: SPMBuildCore.ProductBuildDescription {
 
     /// The reference to the product.
     public let package: ResolvedPackage
@@ -1276,12 +1266,7 @@ public final class ProductBuildDescription {
     public let toolsVersion: ToolsVersion
 
     /// The build parameters.
-    let buildParameters: BuildParameters
-
-    /// The path to the product binary produced.
-    public var binary: AbsolutePath {
-        return buildParameters.binaryPath(for: product)
-    }
+    public let buildParameters: BuildParameters
 
     /// All object files to link into this product.
     ///
@@ -1387,12 +1372,12 @@ public final class ProductBuildDescription {
         let librarian = buildParameters.toolchain.librarianPath.pathString
         let triple = buildParameters.triple
         if triple.isWindows(), librarian.hasSuffix("link") || librarian.hasSuffix("link.exe") {
-            return [librarian, "/LIB", "/OUT:\(binary.pathString)", "@\(linkFileListPath.pathString)"]
+            return [librarian, "/LIB", "/OUT:\(binaryPath.pathString)", "@\(linkFileListPath.pathString)"]
         }
         if triple.isDarwin(), librarian.hasSuffix("libtool") {
-            return [librarian, "-o", binary.pathString, "@\(linkFileListPath.pathString)"]
+            return [librarian, "-o", binaryPath.pathString, "@\(linkFileListPath.pathString)"]
         }
-        return [librarian, "crs", binary.pathString, "@\(linkFileListPath.pathString)"]
+        return [librarian, "crs", binaryPath.pathString, "@\(linkFileListPath.pathString)"]
     }
 
     /// The arguments to link and create this product.
@@ -1416,7 +1401,7 @@ public final class ProductBuildDescription {
         }
 
         args += ["-L", buildParameters.buildPath.pathString]
-        args += ["-o", binary.pathString]
+        args += ["-o", binaryPath.pathString]
         args += ["-module-name", product.name.spm_mangledToC99ExtendedIdentifier()]
         args += dylibs.map({ "-l" + $0.product.name })
 
@@ -1639,7 +1624,7 @@ public final class PluginDescription: Codable {
 }
 
 /// A build plan for a package graph.
-public class BuildPlan {
+public class BuildPlan: SPMBuildCore.BuildPlan {
 
     public enum Error: Swift.Error, CustomStringConvertible, Equatable {
         /// There is no buildable target in the graph.
@@ -1680,8 +1665,8 @@ public class BuildPlan {
     }
 
     /// The products in this plan.
-    public var buildProducts: AnySequence<ProductBuildDescription> {
-        return AnySequence(productMap.values)
+    public var buildProducts: AnySequence<SPMBuildCore.ProductBuildDescription> {
+        return AnySequence(productMap.values.map { $0 as SPMBuildCore.ProductBuildDescription })
     }
 
     /// The results of invoking any build tool plugins used by targets in this build.
@@ -2056,7 +2041,7 @@ public class BuildPlan {
 
         // Plan products.
         for buildProduct in buildProducts {
-            try plan(buildProduct)
+            try plan(buildProduct as! ProductBuildDescription)
         }
         // FIXME: We need to find out if any product has a target on which it depends
         // both static and dynamically and then issue a suitable diagnostic or auto

Sources/Build/CMakeLists.txt

@@ -11,7 +11,6 @@ add_library(Build
   BuildOperation.swift
   BuildPlan.swift
   LLBuildManifestBuilder.swift
-  SPMSwiftDriverExecutor.swift
   SwiftCompilerOutputParser.swift)
 target_link_libraries(Build PUBLIC
   TSCBasic
@@ -21,6 +20,7 @@ target_link_libraries(Build PUBLIC
   SPMBuildCore
   SPMLLBuild)
 target_link_libraries(Build PRIVATE
+  DriverSupport
   SwiftDriver)
 
 # NOTE(compnerd) workaround for CMake not setting up include flags yet

Sources/Build/LLBuildManifestBuilder.swift

@@ -15,6 +15,7 @@ import PackageGraph
 import PackageModel
 import LLBuildManifest
 import SPMBuildCore
+@_implementationOnly import DriverSupport
 @_implementationOnly import SwiftDriver
 import TSCBasic
 
@@ -559,7 +560,7 @@ extension LLBuildManifestBuilder {
                     guard let planProduct = plan.productMap[product] else {
                         throw InternalError("unknown product \(product)")
                     }
-                    inputs.append(file: planProduct.binary)
+                    inputs.append(file: planProduct.binaryPath)
                 }
                 return
             }
@@ -588,7 +589,7 @@ extension LLBuildManifestBuilder {
                         throw InternalError("unknown product \(product)")
                     }
                     // Establish a dependency on binary of the product.
-                    inputs.append(file: planProduct.binary)
+                    inputs.append(file: planProduct.binaryPath)
 
                 // For automatic and static libraries, and plugins, add their targets as static input.
                 case .library(.automatic), .library(.static), .plugin:
@@ -742,7 +743,7 @@ extension LLBuildManifestBuilder {
                         throw InternalError("unknown product \(product)")
                     }
                     // Establish a dependency on binary of the product.
-                    let binary = planProduct.binary
+                    let binary = planProduct.binaryPath
                     inputs.append(file: binary)
 
                 case .library(.automatic), .library(.static), .plugin:
@@ -894,20 +895,20 @@ extension LLBuildManifestBuilder {
         case .library(.static):
             manifest.addShellCmd(
                 name: cmdName,
-                description: "Archiving \(buildProduct.binary.prettyPath())",
+                description: "Archiving \(buildProduct.binaryPath.prettyPath())",
                 inputs: buildProduct.objects.map(Node.file),
-                outputs: [.file(buildProduct.binary)],
+                outputs: [.file(buildProduct.binaryPath)],
                 arguments: try buildProduct.archiveArguments()
             )
 
         default:
-            let inputs = buildProduct.objects + buildProduct.dylibs.map({ $0.binary })
+            let inputs = buildProduct.objects + buildProduct.dylibs.map({ $0.binaryPath })
 
             manifest.addShellCmd(
                 name: cmdName,
-                description: "Linking \(buildProduct.binary.prettyPath())",
+                description: "Linking \(buildProduct.binaryPath.prettyPath())",
                 inputs: inputs.map(Node.file),
-                outputs: [.file(buildProduct.binary)],
+                outputs: [.file(buildProduct.binaryPath)],
                 arguments: try buildProduct.linkArguments()
             )
         }
@@ -919,7 +920,7 @@ extension LLBuildManifestBuilder {
         manifest.addNode(output, toTarget: targetName)
         manifest.addPhonyCmd(
             name: output.name,
-            inputs: [.file(buildProduct.binary)],
+            inputs: [.file(buildProduct.binaryPath)],
             outputs: [output]
         )
 

Sources/CMakeLists.txt

@@ -10,6 +10,7 @@ add_subdirectory(SPMSQLite3)
 add_subdirectory(Basics)
 add_subdirectory(Build)
 add_subdirectory(Commands)
+add_subdirectory(DriverSupport)
 add_subdirectory(LLBuildManifest)
 add_subdirectory(PackageCollections)
 add_subdirectory(PackageCollectionsModel)

Sources/Commands/CMakeLists.txt

@@ -7,7 +7,6 @@
 # See http://swift.org/CONTRIBUTORS.txt for Swift project authors
 
 add_library(Commands
-
   Options.swift
   Snippets/CardEvent.swift
   Snippets/Cards/SnippetCard.swift
@@ -44,6 +43,7 @@ target_link_libraries(Commands PUBLIC
   Workspace
   XCBuildSupport)
 target_link_libraries(Commands PRIVATE
+  DriverSupport
   $<$<NOT:$<PLATFORM_ID:Darwin>>:FoundationXML>)
 # NOTE(compnerd) workaround for CMake not setting up include flags yet
 set_target_properties(Commands PROPERTIES

Sources/Commands/Options.swift

@@ -15,7 +15,6 @@ import TSCBasic
 import PackageFingerprint
 import PackageModel
 import SPMBuildCore
-import Build
 
 import struct TSCUtility.Triple
 
@@ -346,9 +345,9 @@ struct BuildOptions: ParsableArguments {
 
     /// The build system to use.
     @Option(name: .customLong("build-system"))
-    var _buildSystem: BuildSystemKind = .native
+    var _buildSystem: BuildSystemProvider.Kind = .native
 
-    var buildSystem: BuildSystemKind {
+    var buildSystem: BuildSystemProvider.Kind {
         #if os(macOS)
         // Force the Xcode build system if we want to build more than one arch.
         return archs.count > 1 ? .xcode : self._buildSystem
@@ -377,11 +376,6 @@ struct BuildOptions: ParsableArguments {
         case disableIndexStore
     }
 
-    enum BuildSystemKind: String, ExpressibleByArgument, CaseIterable {
-        case native
-        case xcode
-    }
-
     enum TargetDependencyImportCheckingMode : String, Codable, ExpressibleByArgument {
         case none
         case warn
@@ -462,3 +456,6 @@ extension Sanitizer {
         return Sanitizer.allCases.map(\.rawValue).joined(separator: ", ")
     }
 }
+
+extension BuildSystemProvider.Kind: ExpressibleByArgument {
+}

Sources/Commands/SwiftBuildTool.swift

@@ -105,7 +105,10 @@ public struct SwiftBuildTool: SwiftCommand {
         }
 
         if options.printManifestGraphviz {
-            let buildOperation = try swiftTool.createBuildOperation()
+            // FIXME: Doesn't seem ideal that we need an explicit build operation, but this concretely uses the `LLBuildManifest`.
+            guard let buildOperation = try swiftTool.createBuildSystem(explicitBuildSystem: .native) as? BuildOperation else {
+                throw StringError("asked for native build system but did not get it")
+            }
             let buildManifest = try buildOperation.getBuildManifest()
             var serializer = DOTManifestSerializer(manifest: buildManifest)
             // print to stdout

Sources/Commands/SwiftPackageRegistryTool.swift

@@ -14,12 +14,10 @@ import ArgumentParser
 import Basics
 import TSCBasic
 import SPMBuildCore
-import Build
 import PackageModel
 import PackageLoading
 import PackageGraph
 import SourceControl
-import XCBuildSupport
 import Workspace
 import Foundation
 import PackageRegistry