SE-0387: add --toolset option to build-related subcommands

Sources/CoreCommands/Options.swift

@@ -2,7 +2,7 @@
 //
 // This source file is part of the Swift open source project
 //
-// Copyright (c) 2014-2023 Apple Inc. and the Swift project authors
+// Copyright (c) 2014-2024 Apple Inc. and the Swift project authors
 // Licensed under Apache License v2.0 with Runtime Library Exception
 //
 // See http://swift.org/LICENSE.txt for license information
@@ -124,6 +124,17 @@ public struct LocationOptions: ParsableArguments {
         completion: .directory
     )
     public var swiftSDKsDirectory: AbsolutePath?
+
+    @Option(
+        name: .customLong("toolset"),
+        help: """
+            Specify a toolset JSON file to use when building for the target platform. \
+            Use the option multiple times to specify more than one toolset. Toolsets will be merged in the order \
+            they're specified into a single final toolset for the current build.
+            """,
+        completion: .file(extensions: [".json"])
+    )
+    public var toolsetPaths: [AbsolutePath] = []
 
     @Option(
         name: .customLong("pkg-config-path"),

Sources/CoreCommands/SwiftCommandState.swift

@@ -880,6 +880,7 @@ public final class SwiftCommandState {
             swiftSDK = try SwiftSDK.deriveTargetSwiftSDK(
                 hostSwiftSDK: hostSwiftSDK,
                 hostTriple: hostToolchain.targetTriple,
+                customToolsets: options.locations.toolsetPaths,
                 customCompileDestination: options.locations.customCompileDestination,
                 customCompileTriple: options.build.customCompileTriple,
                 customCompileToolchain: options.build.customCompileToolchain,

Sources/PackageModel/SwiftSDKs/SwiftSDK.swift

@@ -659,6 +659,7 @@ public struct SwiftSDK: Equatable {
     public static func deriveTargetSwiftSDK(
       hostSwiftSDK: SwiftSDK,
       hostTriple: Triple,
+      customToolsets: [AbsolutePath] = [],
       customCompileDestination: AbsolutePath? = nil,
       customCompileTriple: Triple? = nil,
       customCompileToolchain: AbsolutePath? = nil,
@@ -671,6 +672,7 @@ public struct SwiftSDK: Equatable {
     ) throws -> SwiftSDK {
         var swiftSDK: SwiftSDK
         var isBasedOnHostSDK: Bool = false
+
         // Create custom toolchain if present.
         if let customDestination = customCompileDestination {
             let swiftSDKs = try SwiftSDK.decode(
@@ -699,11 +701,19 @@ public struct SwiftSDK: Equatable {
             swiftSDK = hostSwiftSDK
             isBasedOnHostSDK = true
         }
+
+        if !customToolsets.isEmpty {
+            for toolsetPath in customToolsets {
+                let toolset = try Toolset(from: toolsetPath, at: fileSystem, observabilityScope)
+                swiftSDK.toolset.merge(with: toolset)
+            }
+        }
+
         // Apply any manual overrides.
         if let triple = customCompileTriple {
             swiftSDK.targetTriple = triple
 
-            if isBasedOnHostSDK {
+            if isBasedOnHostSDK && customToolsets.isEmpty {
                 // Don't pick up extraCLIOptions for a custom triple, since those are only valid for the host triple.
                 for tool in swiftSDK.toolset.knownTools.keys {
                     swiftSDK.toolset.knownTools[tool]?.extraCLIOptions = []

Sources/PackageModel/Toolset.swift

@@ -137,7 +137,7 @@ extension Toolset {
     /// of replacing them.
     /// - Parameter newToolset: new toolset to merge into the existing `self` toolset.
     public mutating func merge(with newToolset: Toolset) {
-        self.rootPaths.append(contentsOf: newToolset.rootPaths)
+        self.rootPaths.insert(contentsOf: newToolset.rootPaths, at: 0)
 
         for (newTool, newProperties) in newToolset.knownTools {
             if newProperties.path != nil {

Sources/PackageModel/UserToolchain.swift

@@ -2,7 +2,7 @@
 //
 // This source file is part of the Swift open source project
 //
-// Copyright (c) 2014-2017 Apple Inc. and the Swift project authors
+// Copyright (c) 2014-2024 Apple Inc. and the Swift project authors
 // Licensed under Apache License v2.0 with Runtime Library Exception
 //
 // See http://swift.org/LICENSE.txt for license information
@@ -135,6 +135,7 @@ public final class UserToolchain: Toolchain {
             // Take the first match.
             break
         }
+
         guard let toolPath else {
             throw InvalidToolchainDiagnostic("could not find CLI tool `\(name)` at any of these directories: \(binDirectories)")
         }
@@ -629,7 +630,7 @@ public final class UserToolchain: Toolchain {
                 pathString: environment[.path],
                 currentWorkingDirectory: fileSystem.currentWorkingDirectory
             )
-            self.useXcrun = true
+            self.useXcrun = !(fileSystem is InMemoryFileSystem)
         case .custom(let searchPaths, let useXcrun):
             self.envSearchPaths = searchPaths
             self.useXcrun = useXcrun

Sources/_InternalTestSupport/MockWorkspace.swift

@@ -26,9 +26,19 @@ extension UserToolchain {
     package static func mockHostToolchain(_ fileSystem: InMemoryFileSystem) throws -> UserToolchain {
         var hostSwiftSDK = try SwiftSDK.hostSwiftSDK(environment: .mockEnvironment, fileSystem: fileSystem)
         hostSwiftSDK.targetTriple = hostTriple
+
+        let env = Environment.mockEnvironment
+
         return try UserToolchain(
             swiftSDK: hostSwiftSDK,
-            environment: .mockEnvironment,
+            environment: env,
+            searchStrategy: .custom(
+                searchPaths: getEnvSearchPaths(
+                    pathString: env[.path],
+                    currentWorkingDirectory: fileSystem.currentWorkingDirectory
+                ),
+                useXcrun: true
+            ),
             fileSystem: fileSystem
         )
     }

Tests/BuildTests/BuildPlanTests.swift

@@ -4661,7 +4661,20 @@ final class BuildPlanTests: XCTestCase {
                 swiftStaticResourcesPath: "/fake/lib/swift_static"
             )
         )
-        let mockToolchain = try UserToolchain(swiftSDK: userSwiftSDK, environment: .mockEnvironment, fileSystem: fs)
+
+        let env = Environment.mockEnvironment
+        let mockToolchain = try UserToolchain(
+            swiftSDK: userSwiftSDK,
+            environment: env,
+            searchStrategy: .custom(
+                searchPaths: getEnvSearchPaths(
+                    pathString: env[.path],
+                    currentWorkingDirectory: fs.currentWorkingDirectory
+                ),
+                useXcrun: true
+            ),
+            fileSystem: fs
+        )
         let commonFlags = BuildFlags(
             cCompilerFlags: ["-clang-command-line-flag"],
             swiftCompilerFlags: ["-swift-command-line-flag"]
@@ -4775,9 +4788,18 @@ final class BuildPlanTests: XCTestCase {
                 swiftStaticResourcesPath: "/fake/lib/swift_static"
             )
         )
+
+        let env = Environment.mockEnvironment
         let mockToolchain = try UserToolchain(
             swiftSDK: userSwiftSDK,
-            environment: .mockEnvironment,
+            environment: env,
+            searchStrategy: .custom(
+                searchPaths: getEnvSearchPaths(
+                    pathString: env[.path],
+                    currentWorkingDirectory: fs.currentWorkingDirectory
+                ),
+                useXcrun: true
+            ),
             fileSystem: fs
         )
 
@@ -5065,7 +5087,20 @@ final class BuildPlanTests: XCTestCase {
                 .swiftCompiler: .init(extraCLIOptions: ["-use-ld=lld"]),
             ])
         )
-        let toolchain = try UserToolchain(swiftSDK: swiftSDK, environment: .mockEnvironment, fileSystem: fileSystem)
+
+        let env = Environment.mockEnvironment
+        let toolchain = try UserToolchain(
+            swiftSDK: swiftSDK,
+            environment: env,
+            searchStrategy: .custom(
+                searchPaths: getEnvSearchPaths(
+                    pathString: env[.path],
+                    currentWorkingDirectory: fileSystem.currentWorkingDirectory
+                ),
+                useXcrun: true
+            ),
+            fileSystem: fileSystem
+        )
         let result = try await BuildPlanResult(plan: mockBuildPlan(
             toolchain: toolchain,
             graph: graph,

Tests/CommandsTests/SwiftCommandStateTests.swift

@@ -2,7 +2,7 @@
 //
 // This source file is part of the Swift open source project
 //
-// Copyright (c) 2021-2022 Apple Inc. and the Swift project authors
+// Copyright (c) 2021-2024 Apple Inc. and the Swift project authors
 // Licensed under Apache License v2.0 with Runtime Library Exception
 //
 // See http://swift.org/LICENSE.txt for license information
@@ -215,10 +215,10 @@ final class SwiftCommandStateTests: CommandsTestCase {
                 let tool = try SwiftCommandState.makeMockState(options: options)
 
                 // There is only one AuthorizationProvider depending on platform
-                #if canImport(Security)
+#if canImport(Security)
                 let keychainProvider = try tool.getRegistryAuthorizationProvider() as? KeychainAuthorizationProvider
                 XCTAssertNotNil(keychainProvider)
-                #else
+#else
                 let netrcProvider = try tool.getRegistryAuthorizationProvider() as? NetrcAuthorizationProvider
                 XCTAssertNotNil(netrcProvider)
                 XCTAssertEqual(try netrcProvider.map { try resolveSymlinks($0.path) }, try resolveSymlinks(customPath))
@@ -232,7 +232,7 @@ final class SwiftCommandStateTests: CommandsTestCase {
                 XCTAssertThrowsError(try tool.getRegistryAuthorizationProvider(), "error expected") { error in
                     XCTAssertEqual(error as? StringError, StringError("did not find netrc file at \(customPath)"))
                 }
-                #endif
+#endif
             }
 
             // Tests should not modify user's home dir .netrc so leaving that out intentionally
@@ -246,9 +246,9 @@ final class SwiftCommandStateTests: CommandsTestCase {
 
         let observer = ObservabilitySystem.makeForTesting()
         let graph = try loadModulesGraph(fileSystem: fs, manifests: [
-                Manifest.createRootManifest(displayName: "Pkg",
-                                            path: "/Pkg",
-                                            targets: [TargetDescription(name: "exe")])
+            Manifest.createRootManifest(displayName: "Pkg",
+                                        path: "/Pkg",
+                                        targets: [TargetDescription(name: "exe")])
         ], observabilityScope: observer.topScope)
 
         var plan: BuildPlan
@@ -319,7 +319,7 @@ final class SwiftCommandStateTests: CommandsTestCase {
                            [.anySequence, "-gnone", .anySequence])
     }
 
-    func testToolchainArgument() async throws {
+    func testToolchainOption() async throws {
         let customTargetToolchain = AbsolutePath("/path/to/toolchain")
         let hostSwiftcPath = AbsolutePath("/usr/bin/swiftc")
         let hostArPath = AbsolutePath("/usr/bin/ar")
@@ -351,17 +351,16 @@ final class SwiftCommandStateTests: CommandsTestCase {
             observabilityScope: observer.topScope
         )
 
-        let options = try GlobalOptions.parse(
-            [
-                "--toolchain", customTargetToolchain.pathString,
-                "--triple", "x86_64-unknown-linux-gnu",
-            ]
-        )
+        let options = try GlobalOptions.parse([
+            "--toolchain", customTargetToolchain.pathString,
+            "--triple", "x86_64-unknown-linux-gnu",
+        ])
         let swiftCommandState = try SwiftCommandState.makeMockState(
             options: options,
             fileSystem: fs,
             environment: ["PATH": "/usr/bin"]
         )
+
         XCTAssertEqual(swiftCommandState.originalWorkingDirectory, fs.currentWorkingDirectory)
         XCTAssertEqual(
             try swiftCommandState.getTargetToolchain().swiftCompilerPath,
@@ -371,6 +370,7 @@ final class SwiftCommandStateTests: CommandsTestCase {
             try swiftCommandState.getTargetToolchain().swiftSDK.toolset.knownTools[.swiftCompiler]?.path,
             nil
         )
+
         let plan = try await BuildPlan(
             destinationBuildParameters: swiftCommandState.productsBuildParameters,
             toolsBuildParameters: swiftCommandState.toolsBuildParameters,
@@ -383,6 +383,108 @@ final class SwiftCommandStateTests: CommandsTestCase {
 
         XCTAssertMatch(arguments, [.contains("/path/to/toolchain")])
     }
+
+    func testToolsetOption() throws {
+        let targetToolchainPath = "/path/to/toolchain"
+        let customTargetToolchain = AbsolutePath(targetToolchainPath)
+        let hostSwiftcPath = AbsolutePath("/usr/bin/swiftc")
+        let hostArPath = AbsolutePath("/usr/bin/ar")
+        let targetSwiftcPath = customTargetToolchain.appending(components: ["swiftc"])
+        let targetArPath = customTargetToolchain.appending(components: ["llvm-ar"])
+
+        let fs = InMemoryFileSystem(emptyFiles: [
+            hostSwiftcPath.pathString,
+            hostArPath.pathString,
+            targetSwiftcPath.pathString,
+            targetArPath.pathString
+        ])
+
+        for path in [hostSwiftcPath, hostArPath, targetSwiftcPath, targetArPath,] {
+            try fs.updatePermissions(path, isExecutable: true)
+        }
+
+        try fs.writeFileContents("/toolset.json", string: """
+        {
+            "schemaVersion": "1.0",
+            "rootPath": "\(targetToolchainPath)"
+        }
+        """)
+
+        let options = try GlobalOptions.parse(["--toolset", "/toolset.json"])
+        let swiftCommandState = try SwiftCommandState.makeMockState(
+            options: options,
+            fileSystem: fs,
+            environment: ["PATH": "/usr/bin"]
+        )
+
+        let hostToolchain = try swiftCommandState.getHostToolchain()
+        let targetToolchain = try swiftCommandState.getTargetToolchain()
+
+        XCTAssertEqual(
+            targetToolchain.swiftSDK.toolset.rootPaths,
+            [customTargetToolchain] + hostToolchain.swiftSDK.toolset.rootPaths
+        )
+        XCTAssertEqual(targetToolchain.swiftCompilerPath, targetSwiftcPath)
+        XCTAssertEqual(targetToolchain.librarianPath, targetArPath)
+    }
+
+    func testMultipleToolsets() throws {
+        let targetToolchainPath1 = "/path/to/toolchain1"
+        let customTargetToolchain1 = AbsolutePath(targetToolchainPath1)
+        let targetToolchainPath2 = "/path/to/toolchain2"
+        let customTargetToolchain2 = AbsolutePath(targetToolchainPath2)
+        let hostSwiftcPath = AbsolutePath("/usr/bin/swiftc")
+        let hostArPath = AbsolutePath("/usr/bin/ar")
+        let targetSwiftcPath = customTargetToolchain1.appending(components: ["swiftc"])
+        let targetArPath = customTargetToolchain1.appending(components: ["llvm-ar"])
+        let targetClangPath = customTargetToolchain2.appending(components: ["clang"])
+
+        let fs = InMemoryFileSystem(emptyFiles: [
+            hostSwiftcPath.pathString,
+            hostArPath.pathString,
+            targetSwiftcPath.pathString,
+            targetArPath.pathString,
+            targetClangPath.pathString
+        ])
+
+        for path in [hostSwiftcPath, hostArPath, targetSwiftcPath, targetArPath, targetClangPath,] {
+            try fs.updatePermissions(path, isExecutable: true)
+        }
+
+        try fs.writeFileContents("/toolset1.json", string: """
+        {
+            "schemaVersion": "1.0",
+            "rootPath": "\(targetToolchainPath1)"
+        }
+        """)
+
+        try fs.writeFileContents("/toolset2.json", string: """
+        {
+            "schemaVersion": "1.0",
+            "rootPath": "\(targetToolchainPath2)"
+        }
+        """)
+
+        let options = try GlobalOptions.parse([
+            "--toolset", "/toolset1.json", "--toolset", "/toolset2.json"
+        ])
+        let swiftCommandState = try SwiftCommandState.makeMockState(
+            options: options,
+            fileSystem: fs,
+            environment: ["PATH": "/usr/bin"]
+        )
+
+        let hostToolchain = try swiftCommandState.getHostToolchain()
+        let targetToolchain = try swiftCommandState.getTargetToolchain()
+
+        XCTAssertEqual(
+            targetToolchain.swiftSDK.toolset.rootPaths,
+            [customTargetToolchain2, customTargetToolchain1] + hostToolchain.swiftSDK.toolset.rootPaths
+        )
+        XCTAssertEqual(targetToolchain.swiftCompilerPath, targetSwiftcPath)
+        XCTAssertEqual(try targetToolchain.getClangCompiler(), targetClangPath)
+        XCTAssertEqual(targetToolchain.librarianPath, targetArPath)
+    }
 }
 
 extension SwiftCommandState {

Tests/PackageModelTests/SwiftSDKBundleTests.swift

@@ -388,6 +388,7 @@ final class SwiftSDKBundleTests: XCTestCase {
             observabilityScope: system.topScope,
             outputHandler: { _ in }
         )
+
         for bundle in bundles {
             try await store.install(bundlePathOrURL: bundle.path, archiver)
         }