simplify bootstrap build (#5939)

motivation: remove depednecy on workspace that carries over dependencies on source control and registry

changes:
* move minimal-built-tool code into swift-bootstrap
* add dependency resolution code swift-bootstrap so it does not require dependency on workspace
* cleanup bootstrap arguments for those that can be hard coded
* adjust target dependencies and cmake setup accordingly

Author: tomerd

Package.swift

@@ -345,7 +345,9 @@ let package = Package(
                 "Basics",
                 "Build",
                 "PackageFingerprint",
+                "PackageLoading",
                 "PackageModel",
+                "PackageGraph",
                 "Workspace",
                 "XCBuildSupport",
             ],
@@ -414,7 +416,15 @@ let package = Package(
         .executableTarget(
             /** Builds SwiftPM itself for bootstrapping (minimal version of `swift-build`) */
             name: "swift-bootstrap",
-            dependencies: ["CoreCommands"],
+            dependencies: [
+                .product(name: "ArgumentParser", package: "swift-argument-parser"),
+                "Basics",
+                "Build",
+                "PackageGraph",
+                "PackageLoading",
+                "PackageModel",
+                "XCBuildSupport",
+            ],
             exclude: ["CMakeLists.txt"]
         ),
         .executableTarget(

Sources/Basics/CMakeLists.txt

@@ -29,7 +29,6 @@ add_library(Basics
   SQLite.swift
   Sandbox.swift
   String+Extensions.swift
-  Triple+Extensions.swift
   SwiftVersion.swift
   SQLiteBackedCache.swift
   Version+Extensions.swift

Sources/Build/BuildOperation.swift

@@ -29,7 +29,6 @@ import protocol TSCUtility.ProgressAnimationProtocol
 @_implementationOnly import SwiftDriver
 
 public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildSystem, BuildErrorAdviceProvider {
-
     /// The delegate used by the build system.
     public weak var delegate: SPMBuildCore.BuildSystemDelegate?
 
@@ -39,14 +38,8 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
     /// The closure for loading the package graph.
     let packageGraphLoader: () throws -> PackageGraph
 
-    /// Entity responsible for compiling and running plugin scripts.
-    let pluginScriptRunner: PluginScriptRunner
-
-    /// Directory where plugin intermediate files are stored.
-    let pluginWorkDirectory: AbsolutePath
-
-    /// Whether to sandbox commands from build tool plugins.
-    public let disableSandboxForPluginCommands: Bool
+    /// the plugin configuration for build plugins
+    let pluginConfiguration: PluginConfiguration?
 
     /// The llbuild build delegate reference.
     private var buildSystemDelegate: BuildOperationBuildSystemDelegateHandler?
@@ -99,10 +92,8 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
         buildParameters: BuildParameters,
         cacheBuildManifest: Bool,
         packageGraphLoader: @escaping () throws -> PackageGraph,
+        pluginConfiguration: PluginConfiguration? = .none,
         additionalFileRules: [FileRuleDescription],
-        pluginScriptRunner: PluginScriptRunner,
-        pluginWorkDirectory: AbsolutePath,
-        disableSandboxForPluginCommands: Bool,
         outputStream: OutputByteStream,
         logLevel: Basics.Diagnostic.Severity,
         fileSystem: TSCBasic.FileSystem,
@@ -116,9 +107,7 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
         self.cacheBuildManifest = cacheBuildManifest
         self.packageGraphLoader = packageGraphLoader
         self.additionalFileRules = additionalFileRules
-        self.pluginScriptRunner = pluginScriptRunner
-        self.pluginWorkDirectory = pluginWorkDirectory
-        self.disableSandboxForPluginCommands = disableSandboxForPluginCommands
+        self.pluginConfiguration = pluginConfiguration
         self.outputStream = outputStream
         self.logLevel = logLevel
         self.fileSystem = fileSystem
@@ -323,6 +312,10 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
     // Compiles a single plugin, emitting its output and throwing an error if it
     // fails.
     func compilePlugin(_ plugin: PluginDescription) throws {
+        guard let pluginConfiguration = self.pluginConfiguration else {
+            throw InternalError("unknown plugin script runner")
+
+        }
         // Compile the plugin, getting back a PluginCompilationResult.
         class Delegate: PluginScriptCompilerDelegate {
             let preparationStepName: String
@@ -351,7 +344,7 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
         }
         let delegate = Delegate(preparationStepName: "Compiling plugin \(plugin.targetName)", buildSystemDelegate: self.buildSystemDelegate)
         let result = try tsc_await {
-            self.pluginScriptRunner.compilePluginScript(
+            pluginConfiguration.scriptRunner.compilePluginScript(
                 sourceFiles: plugin.sources.paths,
                 pluginName: plugin.targetName,
                 toolsVersion: plugin.toolsVersion,
@@ -393,40 +386,47 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
         // Load the package graph.
         let graph = try getPackageGraph()
 
+        let buildToolPluginInvocationResults: [ResolvedTarget: [BuildToolPluginInvocationResult]]
+        let prebuildCommandResults: [ResolvedTarget: [PrebuildCommandResult]]
         // Invoke any build tool plugins in the graph to generate prebuild commands and build commands.
-        let buildToolPluginInvocationResults = try graph.invokeBuildToolPlugins(
-            outputDir: self.pluginWorkDirectory.appending(component: "outputs"),
-            builtToolsDir: self.buildParameters.buildPath,
-            buildEnvironment: self.buildParameters.buildEnvironment,
-            toolSearchDirectories: [self.buildParameters.toolchain.swiftCompilerPath.parentDirectory],
-            pluginScriptRunner: self.pluginScriptRunner,
-            observabilityScope: self.observabilityScope,
-            fileSystem: self.fileSystem
-        )
-
-        // Surface any diagnostics from build tool plugins.
-        for (target, results) in buildToolPluginInvocationResults {
-            // There is one result for each plugin that gets applied to a target.
-            for result in results {
-                let diagnosticsEmitter = self.observabilityScope.makeDiagnosticsEmitter {
-                    var metadata = ObservabilityMetadata()
-                    metadata.targetName = target.name
-                    metadata.pluginName = result.plugin.name
-                    return metadata
-                }
-                for line in result.textOutput.split(separator: "\n") {
-                    diagnosticsEmitter.emit(info: line)
-                }
-                for diag in result.diagnostics {
-                    diagnosticsEmitter.emit(diag)
+        if let pluginConfiguration = self.pluginConfiguration {
+            buildToolPluginInvocationResults = try graph.invokeBuildToolPlugins(
+                outputDir: pluginConfiguration.workDirectory.appending(component: "outputs"),
+                builtToolsDir: self.buildParameters.buildPath,
+                buildEnvironment: self.buildParameters.buildEnvironment,
+                toolSearchDirectories: [self.buildParameters.toolchain.swiftCompilerPath.parentDirectory],
+                pluginScriptRunner: pluginConfiguration.scriptRunner,
+                observabilityScope: self.observabilityScope,
+                fileSystem: self.fileSystem
+            )
+
+            // Surface any diagnostics from build tool plugins.
+            for (target, results) in buildToolPluginInvocationResults {
+                // There is one result for each plugin that gets applied to a target.
+                for result in results {
+                    let diagnosticsEmitter = self.observabilityScope.makeDiagnosticsEmitter {
+                        var metadata = ObservabilityMetadata()
+                        metadata.targetName = target.name
+                        metadata.pluginName = result.plugin.name
+                        return metadata
+                    }
+                    for line in result.textOutput.split(separator: "\n") {
+                        diagnosticsEmitter.emit(info: line)
+                    }
+                    for diag in result.diagnostics {
+                        diagnosticsEmitter.emit(diag)
+                    }
                 }
             }
-        }
 
-        // Run any prebuild commands provided by build tool plugins. Any failure stops the build.
-        let prebuildCommandResults = try graph.reachableTargets.reduce(into: [:], { partial, target in
-            partial[target] = try buildToolPluginInvocationResults[target].map { try self.runPrebuildCommands(for: $0) }
-        })
+            // Run any prebuild commands provided by build tool plugins. Any failure stops the build.
+            prebuildCommandResults = try graph.reachableTargets.reduce(into: [:], { partial, target in
+                partial[target] = try buildToolPluginInvocationResults[target].map { try self.runPrebuildCommands(for: $0) }
+            })
+        } else {
+            buildToolPluginInvocationResults = [:]
+            prebuildCommandResults = [:]
+        }
 
         // Emit warnings about any unhandled files in authored packages. We do this after applying build tool plugins, once we know what files they handled.
         for package in graph.rootPackages where package.manifest.toolsVersion >= .v5_3 {
@@ -472,7 +472,7 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
 
         let (buildDescription, buildManifest) = try BuildDescription.create(
             with: plan,
-            disableSandboxForPluginCommands: self.disableSandboxForPluginCommands,
+            disableSandboxForPluginCommands: self.pluginConfiguration?.disableSandbox ?? false,
             fileSystem: self.fileSystem,
             observabilityScope: self.observabilityScope
         )
@@ -526,7 +526,7 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
             buildFile: buildParameters.llbuildManifest.pathString,
             databaseFile: databasePath,
             delegate: buildSystemDelegate,
-            schedulerLanes: buildParameters.jobs
+            schedulerLanes: buildParameters.workers
         )
 
         // TODO: this seems fragile, perhaps we replace commandFailureHandler by adding relevant calls in the delegates chain
@@ -541,6 +541,10 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
     /// Runs any prebuild commands associated with the given list of plugin invocation results, in order, and returns the
     /// results of running those prebuild commands.
     private func runPrebuildCommands(for pluginResults: [BuildToolPluginInvocationResult]) throws -> [PrebuildCommandResult] {
+        guard let pluginConfiguration = self.pluginConfiguration else {
+            throw InternalError("unknown plugin script runner")
+
+        }
         // Run through all the commands from all the plugin usages in the target.
         return try pluginResults.map { pluginResult in
             // As we go we will collect a list of prebuild output directories whose contents should be input to the build,
@@ -553,7 +557,7 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
                 // Run the command configuration as a subshell. This doesn't return until it is done.
                 // TODO: We need to also use any working directory, but that support isn't yet available on all platforms at a lower level.
                 var commandLine = [command.configuration.executable.pathString] + command.configuration.arguments
-                if !self.disableSandboxForPluginCommands {
+                if !pluginConfiguration.disableSandbox {
                     commandLine = try Sandbox.apply(command: commandLine, strictness: .writableTemporaryDirectory, writableDirectories: [pluginResult.pluginOutputDirectory])
                 }
                 let processResult = try TSCBasic.Process.popen(arguments: commandLine, environment: command.configuration.environment)
@@ -613,6 +617,25 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
     }
 }
 
+extension BuildOperation {
+    public struct PluginConfiguration {
+        /// Entity responsible for compiling and running plugin scripts.
+        let scriptRunner: PluginScriptRunner
+
+        /// Directory where plugin intermediate files are stored.
+        let workDirectory: AbsolutePath
+
+        /// Whether to sandbox commands from build tool plugins.
+        let disableSandbox: Bool
+
+        public init(scriptRunner: PluginScriptRunner, workDirectory: AbsolutePath, disableSandbox: Bool) {
+            self.scriptRunner = scriptRunner
+            self.workDirectory = workDirectory
+            self.disableSandbox = disableSandbox
+        }
+    }
+}
+
 extension BuildDescription {
     static func create(with plan: BuildPlan, disableSandboxForPluginCommands: Bool, fileSystem: TSCBasic.FileSystem, observabilityScope: ObservabilityScope) throws -> (BuildDescription, LLBuildManifest.BuildManifest) {
         // Generate the llbuild manifest.

Sources/Build/BuildPlan.swift

@@ -883,7 +883,7 @@ public final class SwiftTargetBuildDescription {
         args += optimizationArguments
         args += testingArguments
         args += ["-g"]
-        args += ["-j\(buildParameters.jobs)"]
+        args += ["-j\(buildParameters.workers)"]
         args += activeCompilationConditions
         args += additionalFlags
         args += try moduleCacheArgs
@@ -1037,7 +1037,7 @@ public final class SwiftTargetBuildDescription {
         result += optimizationArguments
         result += testingArguments
         result += ["-g"]
-        result += ["-j\(buildParameters.jobs)"]
+        result += ["-j\(buildParameters.workers)"]
         result += activeCompilationConditions
         result += additionalFlags
         result += try moduleCacheArgs
@@ -1084,7 +1084,7 @@ public final class SwiftTargetBuildDescription {
         result += optimizationArguments
         result += testingArguments
         result += ["-g"]
-        result += ["-j\(buildParameters.jobs)"]
+        result += ["-j\(buildParameters.workers)"]
         result += activeCompilationConditions
         result += additionalFlags
         result += try moduleCacheArgs

Sources/Commands/PackageTools/APIDiff.swift

@@ -114,7 +114,7 @@ struct APIDiff: SwiftCommand {
 
         let results = ThreadSafeArrayStore<SwiftAPIDigester.ComparisonResult>()
         let group = DispatchGroup()
-        let semaphore = DispatchSemaphore(value: Int(try buildSystem.buildPlan.buildParameters.jobs))
+        let semaphore = DispatchSemaphore(value: Int(try buildSystem.buildPlan.buildParameters.workers))
         var skippedModules: Set<String> = []
 
         for module in modulesToDiff {

Sources/Commands/Utilities/APIDigester.swift

@@ -140,7 +140,7 @@ struct APIDigesterBaselineDumper {
         // Dump the SDK JSON.
         try swiftTool.fileSystem.createDirectory(baselineDir, recursive: true)
         let group = DispatchGroup()
-        let semaphore = DispatchSemaphore(value: Int(buildParameters.jobs))
+        let semaphore = DispatchSemaphore(value: Int(buildParameters.workers))
         let errors = ThreadSafeArrayStore<Swift.Error>()
         for module in modulesToDiff {
             semaphore.wait()

Sources/CoreCommands/BuildSystemSupport.swift

@@ -30,10 +30,12 @@ extension SwiftTool {
                         buildParameters: customBuildParameters ?? self.buildParameters(),
                         cacheBuildManifest: cacheBuildManifest && self.canUseCachedBuildManifest(),
                         packageGraphLoader: customPackageGraphLoader ?? graphLoader,
+                        pluginConfiguration: .init(
+                            scriptRunner: self.getPluginScriptRunner(),
+                            workDirectory: try self.getActiveWorkspace().location.pluginWorkingDirectory,
+                            disableSandbox: self.options.security.shouldDisableSandbox
+                        ),
                         additionalFileRules: FileRuleDescription.swiftpmFileTypes,
-                        pluginScriptRunner: self.getPluginScriptRunner(),
-                        pluginWorkDirectory: try self.getActiveWorkspace().location.pluginWorkingDirectory,
-                        disableSandboxForPluginCommands: self.options.security.shouldDisableSandbox,
                         outputStream: customOutputStream ?? self.outputStream,
                         logLevel: customLogLevel ?? self.logLevel,
                         fileSystem: self.fileSystem,

Sources/CoreCommands/CMakeLists.txt

@@ -8,7 +8,6 @@
 
 add_library(CoreCommands
   BuildSystemSupport.swift
-  MinimalBuildTool.swift
   SwiftTool.swift
   SwiftToolObservabilityHandler.swift
   Options.swift)