Inherit host Swift SDK's toolset paths to fallback to host tools (#7297)

kateinoigakukun · web-flow · commit bd821ea3f818 · 2024-01-25T12:14:03.000-08:00
Fallback to host SDK tools if target SDK doesn't have tools.

### Motivation:

Currently, Swift SDK fallbacks to system `PATH` search path for tools
that are not found in the SDK. Usually, host compiler tool is capable to
cross compile but just lacks some of artifacts like platform libraries.
In that case, we don't need to have a separate compiler tools in
target-specific Swift SDK, so we can share the host compiler tools as
long as the host Swift and target Swift are the same version.

### Modifications:

This patch makes Swift SDK fallback to host Swift SDK tools if the
target Swift SDK doesn't have such tools.

### Result:

This unlocks eliminating compiler tools from target-specific Swift SDKs,
and will result in a smaller SDK size. Also makes it easier to build
host platform independent Swift SDK.
diff --git a/Sources/CoreCommands/SwiftTool.swift b/Sources/CoreCommands/SwiftTool.swift
@@ -17,6 +17,7 @@ import class Foundation.NSLock
 import class Foundation.ProcessInfo
 import PackageGraph
 import PackageLoading
+@_spi(SwiftPMInternal)
 import PackageModel
 import SPMBuildCore
 import Workspace
@@ -767,71 +768,33 @@ public final class SwiftTool {
 
     /// Lazily compute the target toolchain.z
     private lazy var _targetToolchain: Result<UserToolchain, Swift.Error> = {
-        var swiftSDK: SwiftSDK
+        let swiftSDK: SwiftSDK
         let hostSwiftSDK: SwiftSDK
+        let store = SwiftSDKBundleStore(
+            swiftSDKsDirectory: self.sharedSwiftSDKsDirectory,
+            fileSystem: fileSystem,
+            observabilityScope: observabilityScope,
+            outputHandler: { print($0.description) }
+        )
         do {
             let hostToolchain = try _hostToolchain.get()
             hostSwiftSDK = hostToolchain.swiftSDK
-            let hostTriple = hostToolchain.targetTriple
-
-            // Create custom toolchain if present.
-            if let customDestination = self.options.locations.customCompileDestination {
-                let swiftSDKs = try SwiftSDK.decode(
-                    fromFile: customDestination,
-                    fileSystem: self.fileSystem,
-                    observabilityScope: self.observabilityScope
-                )
-                if swiftSDKs.count == 1 {
-                    swiftSDK = swiftSDKs[0]
-                } else if swiftSDKs.count > 1,
-                          let triple = options.build.customCompileTriple,
-                          let matchingSDK = swiftSDKs.first(where: { $0.targetTriple == triple })
-                {
-                    swiftSDK = matchingSDK
-                } else {
-                    return .failure(SwiftSDKError.noSwiftSDKDecoded(customDestination))
-                }
-            } else if let triple = options.build.customCompileTriple,
-                      let targetSwiftSDK = SwiftSDK.defaultSwiftSDK(for: triple, hostSDK: hostSwiftSDK)
-            {
-                swiftSDK = targetSwiftSDK
-            } else if let swiftSDKSelector = options.build.swiftSDKSelector {
-                let store = SwiftSDKBundleStore(
-                    swiftSDKsDirectory: self.sharedSwiftSDKsDirectory,
-                    fileSystem: self.fileSystem,
-                    observabilityScope: self.observabilityScope,
-                    outputHandler: { print($0.description) }
-                )
-                swiftSDK = try store.selectBundle(matching: swiftSDKSelector, hostTriple: hostTriple)
-            } else {
-                // Otherwise use the host toolchain.
-                swiftSDK = hostSwiftSDK
-            }
+            swiftSDK = try SwiftSDK.deriveTargetSwiftSDK(
+                hostSwiftSDK: hostSwiftSDK,
+                hostTriple: hostToolchain.targetTriple,
+                customCompileDestination: options.locations.customCompileDestination,
+                customCompileTriple: options.build.customCompileTriple,
+                customCompileToolchain: options.build.customCompileToolchain,
+                customCompileSDK: options.build.customCompileSDK,
+                swiftSDKSelector: options.build.swiftSDKSelector,
+                architectures: options.build.architectures,
+                store: store,
+                observabilityScope: self.observabilityScope,
+                fileSystem: self.fileSystem
+            )
         } catch {
             return .failure(error)
         }
-        // Apply any manual overrides.
-        if let triple = options.build.customCompileTriple {
-            swiftSDK.targetTriple = triple
-        }
-        if let binDir = options.build.customCompileToolchain {
-            if !self.fileSystem.exists(binDir) {
-                self.observabilityScope.emit(
-                    warning: """
-                        Toolchain directory specified through a command-line option doesn't exist and is ignored: `\(
-                            binDir
-                        )`
-                        """
-                )
-            }
-
-            swiftSDK.add(toolsetRootPath: binDir.appending(components: "usr", "bin"))
-        }
-        if let sdk = options.build.customCompileSDK {
-            swiftSDK.pathsConfiguration.sdkRootPath = sdk
-        }
-        swiftSDK.architectures = options.build.architectures.isEmpty ? nil : options.build.architectures
-
         // Check if we ended up with the host toolchain.
         if hostSwiftSDK == swiftSDK {
             return self._hostToolchain
diff --git a/Sources/PackageModel/SwiftSDKs/SwiftSDK.swift b/Sources/PackageModel/SwiftSDKs/SwiftSDK.swift
@@ -598,6 +598,83 @@ public struct SwiftSDK: Equatable {
         return nil
     }
 
+    /// Computes the target Swift SDK for the given options.
+    @_spi(SwiftPMInternal)
+    public static func deriveTargetSwiftSDK(
+      hostSwiftSDK: SwiftSDK,
+      hostTriple: Triple,
+      customCompileDestination: AbsolutePath? = nil,
+      customCompileTriple: Triple? = nil,
+      customCompileToolchain: AbsolutePath? = nil,
+      customCompileSDK: AbsolutePath? = nil,
+      swiftSDKSelector: String? = nil,
+      architectures: [String] = [],
+      store: SwiftSDKBundleStore,
+      observabilityScope: ObservabilityScope,
+      fileSystem: FileSystem
+    ) throws -> SwiftSDK {
+        var swiftSDK: SwiftSDK
+        var isBasedOnHostSDK: Bool = false
+        // Create custom toolchain if present.
+        if let customDestination = customCompileDestination {
+            let swiftSDKs = try SwiftSDK.decode(
+                fromFile: customDestination,
+                fileSystem: fileSystem,
+                observabilityScope: observabilityScope
+            )
+            if swiftSDKs.count == 1 {
+                swiftSDK = swiftSDKs[0]
+            } else if swiftSDKs.count > 1,
+                      let triple = customCompileTriple,
+                      let matchingSDK = swiftSDKs.first(where: { $0.targetTriple == triple })
+            {
+                swiftSDK = matchingSDK
+            } else {
+                throw SwiftSDKError.noSwiftSDKDecoded(customDestination)
+            }
+        } else if let triple = customCompileTriple,
+                  let targetSwiftSDK = SwiftSDK.defaultSwiftSDK(for: triple, hostSDK: hostSwiftSDK)
+        {
+            swiftSDK = targetSwiftSDK
+        } else if let swiftSDKSelector {
+            swiftSDK = try store.selectBundle(matching: swiftSDKSelector, hostTriple: hostTriple)
+        } else {
+            // Otherwise use the host toolchain.
+            swiftSDK = hostSwiftSDK
+            isBasedOnHostSDK = true
+        }
+        // Apply any manual overrides.
+        if let triple = customCompileTriple {
+            swiftSDK.targetTriple = triple
+        }
+        if let binDir = customCompileToolchain {
+            if !fileSystem.exists(binDir) {
+                observabilityScope.emit(
+                    warning: """
+                        Toolchain directory specified through a command-line option doesn't exist and is ignored: `\(
+                            binDir
+                        )`
+                        """
+                )
+            }
+
+            swiftSDK.add(toolsetRootPath: binDir.appending(components: "usr", "bin"))
+        }
+        if let sdk = customCompileSDK {
+            swiftSDK.pathsConfiguration.sdkRootPath = sdk
+        }
+        swiftSDK.architectures = architectures.isEmpty ? nil : architectures
+
+        if !isBasedOnHostSDK {
+            // Append the host toolchain's toolset paths at the end for the case the target Swift SDK
+            // doesn't have some of the tools (e.g. swift-frontend might be shared between the host and
+            // target Swift SDKs).
+            hostSwiftSDK.toolset.rootPaths.forEach { swiftSDK.add(toolsetRootPath: $0) }
+        }
+
+        return swiftSDK
+    }
+
     /// Propagates toolchain and SDK paths known to the Swift SDK to `swiftc` CLI options.
     public mutating func applyPathCLIOptions() {
         var properties = self.toolset.knownTools[.swiftCompiler] ?? .init(extraCLIOptions: [])
diff --git a/Tests/PackageModelTests/SwiftSDKBundleTests.swift b/Tests/PackageModelTests/SwiftSDKBundleTests.swift
@@ -11,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 import Basics
+@_spi(SwiftPMInternal)
 @testable import PackageModel
 import SPMTestSupport
 import XCTest
@@ -56,19 +57,33 @@ private func generateBundleFiles(bundle: MockBundle) throws -> [(String, ByteStr
     ] + bundle.artifacts.map {
         (
             "\(bundle.path)/\($0.id)/\(targetTriple.tripleString)/swift-sdk.json",
-            ByteString(json: try generateSwiftSDKMetadata(jsonEncoder))
+            ByteString(json: try generateSwiftSDKMetadata(jsonEncoder, createToolset: $0.toolsetRootPath != nil))
         )
+    } + bundle.artifacts.compactMap { artifact in
+        artifact.toolsetRootPath.map { path in
+            (
+                "\(bundle.path)/\(artifact.id)/\(targetTriple.tripleString)/toolset.json",
+                ByteString(json: """
+                {
+                    "schemaVersion": "1.0",
+                    "rootPath": "\(path)"
+                }
+                """)
+            )
+        }
     }
 }
 
-private func generateSwiftSDKMetadata(_ encoder: JSONEncoder) throws -> SerializedJSON {
+private func generateSwiftSDKMetadata(_ encoder: JSONEncoder, createToolset: Bool) throws -> SerializedJSON {
     try """
     {
         "schemaVersion": "4.0",
         "targetTriples": \(
             String(
                 bytes: encoder.encode([
-                    targetTriple.tripleString: SwiftSDKMetadataV4.TripleProperties(sdkRootPath: "sdk")
+                    targetTriple.tripleString: SwiftSDKMetadataV4.TripleProperties(sdkRootPath: "sdk", toolsetPaths: createToolset ? [
+                        "toolset.json"
+                    ] : nil)
                 ]),
                 encoding: .utf8
             )!
@@ -86,6 +101,7 @@ private struct MockBundle {
 private struct MockArtifact {
     let id: String
     let supportedTriples: [Triple]
+    var toolsetRootPath: AbsolutePath?
 }
 
 private func generateTestFileSystem(bundleArtifacts: [MockArtifact]) throws -> (some FileSystem, [MockBundle], AbsolutePath) {
@@ -341,4 +357,88 @@ final class SwiftSDKBundleTests: XCTestCase {
             ),
         ])
     }
+
+    func testTargetSDKDeriviation() async throws {
+        let toolsetRootPath = AbsolutePath("/path/to/toolpath")
+        let (fileSystem, bundles, swiftSDKsDirectory) = try generateTestFileSystem(
+            bundleArtifacts: [
+                .init(id: "\(testArtifactID)1", supportedTriples: [arm64Triple]),
+                .init(id: "\(testArtifactID)2", supportedTriples: [arm64Triple], toolsetRootPath: toolsetRootPath),
+            ]
+        )
+        let system = ObservabilitySystem.makeForTesting()
+        let hostSwiftSDK = try SwiftSDK.hostSwiftSDK()
+        let hostTriple = try! Triple("arm64-apple-macosx14.0")
+        let archiver = MockArchiver()
+        let store = SwiftSDKBundleStore(
+            swiftSDKsDirectory: swiftSDKsDirectory,
+            fileSystem: fileSystem,
+            observabilityScope: system.topScope,
+            outputHandler: { _ in }
+        )
+        for bundle in bundles {
+            try await store.install(bundlePathOrURL: bundle.path, archiver)
+        }
+
+        do {
+            let targetSwiftSDK = try SwiftSDK.deriveTargetSwiftSDK(
+                hostSwiftSDK: hostSwiftSDK,
+                hostTriple: hostTriple,
+                store: store,
+                observabilityScope: system.topScope,
+                fileSystem: fileSystem
+            )
+            // By default, the target SDK is the same as the host SDK.
+            XCTAssertEqual(targetSwiftSDK, hostSwiftSDK)
+        }
+
+        do {
+            let targetSwiftSDK = try SwiftSDK.deriveTargetSwiftSDK(
+                hostSwiftSDK: hostSwiftSDK,
+                hostTriple: hostTriple,
+                swiftSDKSelector: "\(testArtifactID)1",
+                store: store,
+                observabilityScope: system.topScope,
+                fileSystem: fileSystem
+            )
+            // With a target SDK selector, SDK should be chosen from the store.
+            XCTAssertEqual(targetSwiftSDK.targetTriple, targetTriple)
+            // No toolset in the SDK, so it should be the same as the host SDK.
+            XCTAssertEqual(targetSwiftSDK.toolset.rootPaths, hostSwiftSDK.toolset.rootPaths)
+        }
+
+        do {
+            let targetSwiftSDK = try SwiftSDK.deriveTargetSwiftSDK(
+                hostSwiftSDK: hostSwiftSDK,
+                hostTriple: hostTriple,
+                swiftSDKSelector: "\(testArtifactID)2",
+                store: store,
+                observabilityScope: system.topScope,
+                fileSystem: fileSystem
+            )
+            // With toolset in the target SDK, it should contain the host toolset roots at the end.
+            XCTAssertEqual(targetSwiftSDK.toolset.rootPaths, [toolsetRootPath] + hostSwiftSDK.toolset.rootPaths)
+        }
+
+        do {
+            // Check explicit overriding options.
+            let customCompileSDK = AbsolutePath("/path/to/sdk")
+            let archs = ["x86_64-apple-macosx10.15"]
+            let customCompileToolchain = AbsolutePath("/path/to/toolchain")
+            try fileSystem.createDirectory(customCompileToolchain, recursive: true)
+
+            let targetSwiftSDK = try SwiftSDK.deriveTargetSwiftSDK(
+                hostSwiftSDK: hostSwiftSDK,
+                hostTriple: hostTriple,
+                customCompileToolchain: customCompileToolchain,
+                customCompileSDK: customCompileSDK,
+                architectures: archs,
+                store: store,
+                observabilityScope: system.topScope,
+                fileSystem: fileSystem
+            )
+            XCTAssertEqual(targetSwiftSDK.architectures, archs)
+            XCTAssertEqual(targetSwiftSDK.pathsConfiguration.sdkRootPath, customCompileSDK)
+        }
+    }
 }
