Patch *.sdk paths from pkgConfig files to the current SDK (#6772)

This fixes #6439 by replacing all `*.sdk` paths obtained from pkgConfigs
with the current SDK.

### Motivation:

Using Homebrew-packaged libraries such as Cairo in a system library
target on macOS currently causes build failures if the user compiles the
Swift package with an Xcode toolchain (see
[`swift-systemlib-demo`](https://github.com/fwcd/swift-systemlib-demo/tree/bb30a9ce6205a503ad85fc112b0e60a4c3a450c4)
and its CI builds for a minimal example), the build output often looks
similar to this:

```
/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX13.0.sdk/usr/include/ffi/module.modulemap:1:8: error: redefinition of module 'FFI'
module FFI [system] [extern_c] {
       ^
/Library/Developer/CommandLineTools/SDKs/MacOSX13.sdk/usr/include/ffi/module.modulemap:1:8: note: previously defined here
module FFI [system] [extern_c] {
```

The problem is that Homebrew's pre-compiled libraries sometimes bake the
Command Line Tools-paths
(`/Library/Developer/CommandLineTools/SDKs/...`) into their `pkgConfig`
files, which can clash with the current SDK's headers.

See #6439 for details.

### Modifications:

As per @neonichu's suggestion in
#6439 (comment),
this patches the include (`-I`) and library (`-L`) paths parsed from
pkgConfig files by replacing any `*.sdk` path with the current SDK.

Thus a flag such as

```
-I/Library/Developer/CommandLineTools/SDKs/MacOSX13.sdk/usr/include/ffi
```

would be mapped to

```
-I/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX13.3.sdk
```

when compiling with the macOS 13.3 SDK.

### Result:

The
[`swift-systemlib-demo`](https://github.com/fwcd/swift-systemlib-demo/tree/bb30a9ce6205a503ad85fc112b0e60a4c3a450c4)
package now compiles on macOS.

Author: fwcd

Sources/Build/BuildOperation.swift

@@ -443,6 +443,7 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
                 buildEnvironment: self.buildParameters.buildEnvironment,
                 toolSearchDirectories: [self.buildParameters.toolchain.swiftCompilerPath.parentDirectory],
                 pkgConfigDirectories: self.pkgConfigDirectories,
+                sdkRootPath: self.buildParameters.toolchain.sdkRootPath,
                 pluginScriptRunner: pluginConfiguration.scriptRunner,
                 observabilityScope: self.observabilityScope,
                 fileSystem: self.fileSystem

Sources/Build/BuildPlan.swift

@@ -1003,6 +1003,7 @@ public class BuildPlan: SPMBuildCore.BuildPlan {
         let results = try pkgConfigArgs(
             for: target,
             pkgConfigDirectories: buildParameters.pkgConfigDirectories,
+            sdkRootPath: buildParameters.toolchain.sdkRootPath,
             fileSystem: fileSystem,
             observabilityScope: observabilityScope
         )

Sources/Commands/PackageTools/PluginCommand.swift

@@ -281,7 +281,8 @@ struct PluginCommand: SwiftCommand {
         let delegateQueue = DispatchQueue(label: "plugin-invocation")
 
         // Run the command plugin.
-        let buildEnvironment = try swiftTool.buildParameters().buildEnvironment
+        let buildParameters = try swiftTool.buildParameters()
+        let buildEnvironment = buildParameters.buildEnvironment
         let _ = try temp_await { plugin.invoke(
             action: .performCommand(package: package, arguments: arguments),
             buildEnvironment: buildEnvironment,
@@ -294,6 +295,7 @@ struct PluginCommand: SwiftCommand {
             readOnlyDirectories: readOnlyDirectories,
             allowNetworkConnections: allowNetworkConnections,
             pkgConfigDirectories: swiftTool.options.locations.pkgConfigDirectories,
+            sdkRootPath: buildParameters.toolchain.sdkRootPath,
             fileSystem: swiftTool.fileSystem,
             observabilityScope: swiftTool.observabilityScope,
             callbackQueue: delegateQueue,

Sources/PackageLoading/Target+PkgConfig.swift

@@ -14,6 +14,7 @@ import Basics
 import PackageModel
 
 import class TSCBasic.Process
+import struct TSCBasic.RegEx
 
 import enum TSCUtility.Platform
 
@@ -63,6 +64,7 @@ public struct PkgConfigResult {
 public func pkgConfigArgs(
     for target: SystemLibraryTarget,
     pkgConfigDirectories: [AbsolutePath],
+    sdkRootPath: AbsolutePath? = nil,
     brewPrefix: AbsolutePath? = .none,
     fileSystem: FileSystem,
     observabilityScope: ObservabilityScope
@@ -98,7 +100,14 @@ public func pkgConfigArgs(
             let filtered = try allowlist(pcFile: pkgConfigName, flags: (pkgConfig.cFlags, pkgConfig.libs))
 
             // Remove any default flags which compiler adds automatically.
-            let (cFlags, libs) = try removeDefaultFlags(cFlags: filtered.cFlags, libs: filtered.libs)
+            var (cFlags, libs) = try removeDefaultFlags(cFlags: filtered.cFlags, libs: filtered.libs)
+
+            // Patch any paths containing an SDK to the current SDK
+            // See https://github.com/apple/swift-package-manager/issues/6439
+            if let sdkRootPath = sdkRootPath {
+                cFlags = try patchSDKPaths(in: cFlags, to: sdkRootPath)
+                libs = try patchSDKPaths(in: libs, to: sdkRootPath)
+            }
 
             // Set the error if there are any disallowed flags.
             var error: Swift.Error?
@@ -261,43 +270,64 @@ public func allowlist(
     return (filteredCFlags.allowed, filteredLibs.allowed, filteredCFlags.disallowed + filteredLibs.disallowed)
 }
 
+/// Maps values of the given flag with the given transform, removing those where the transform returns `nil`.
+private func patch(flag: String, in flags: [String], transform: (String) -> String?) throws -> [String] {
+    var result = [String]()
+    var it = flags.makeIterator()
+    while let current = it.next() {
+        if current == flag {
+            // Handle <flag><space><value> style.
+            guard let value = it.next() else {
+                throw InternalError("Expected associated value")
+            }
+            if let newValue = transform(value) {
+                result.append(flag)
+                result.append(newValue)
+            }
+        } else if current.starts(with: flag) {
+            // Handle <flag><value> style
+            let value = String(current.dropFirst(flag.count))
+            if let newValue = transform(value) {
+                result.append(flag + newValue)
+            }
+        } else {
+            // Leave other flags as-is
+            result.append(current)
+        }
+    }
+    return result
+}
+
+/// Removes the given flag from the given flags.
+private func remove(flag: String, with expectedValue: String, from flags: [String]) throws -> [String] {
+    try patch(flag: flag, in: flags) { value in value == expectedValue ? nil : value }
+}
+
 /// Remove the default flags which are already added by the compiler.
 ///
 /// This behavior is similar to pkg-config cli tool and helps avoid conflicts between
 /// sdk and default search paths in macOS.
 public func removeDefaultFlags(cFlags: [String], libs: [String]) throws -> ([String], [String]) {
-    /// removes a flag from given array of flags.
-    func remove(flag: (String, String), from flags: [String]) throws -> [String] {
-        var result = [String]()
-        var it = flags.makeIterator()
-        while let curr = it.next() {
-            switch curr {
-            case flag.0:
-                // Check for <flag><space><value> style.
-                guard let val = it.next() else {
-                    throw InternalError("Expected associated value")
-                }
-                // If we found a match, don't add these flags and just skip.
-                if val == flag.1 { continue }
-                // Otherwise add both the flags.
-                result.append(curr)
-                result.append(val)
-
-            case flag.0 + flag.1:
-                // Check for <flag><value> style.
-                continue
+    return (
+        try remove(flag: "-I", with: "/usr/include", from: cFlags),
+        try remove(flag: "-L", with: "/usr/lib", from: libs)
+    )
+}
 
-            default:
-                // Otherwise just append this flag.
-                result.append(curr)
+/// Replaces any path containing *.sdk with the current SDK to avoid conflicts.
+///
+/// See https://github.com/apple/swift-package-manager/issues/6439 for details.
+public func patchSDKPaths(in flags: [String], to sdkRootPath: AbsolutePath) throws -> [String] {
+    let sdkRegex = try! RegEx(pattern: #"^.*\.sdk(\/.*|$)"#)
+
+    return try ["-I", "-L"].reduce(flags) { (flags, flag) in
+        try patch(flag: flag, in: flags) { value in
+            guard let groups = sdkRegex.matchGroups(in: value).first else {
+                return value
             }
+            return sdkRootPath.pathString + groups[0]
         }
-        return result
     }
-    return (
-        try remove(flag: ("-I", "/usr/include"), from: cFlags),
-        try remove(flag: ("-L", "/usr/lib"), from: libs)
-    )
 }
 
 extension ObservabilityMetadata {

Sources/PackageModel/Toolchain.swift

@@ -43,6 +43,9 @@ public protocol Toolchain {
     /// Configuration from the used toolchain.
     var installedSwiftPMConfiguration: InstalledSwiftPMConfiguration { get }
 
+    /// The root path to the Swift SDK used by this toolchain.
+    var sdkRootPath: AbsolutePath? { get }
+
     /// Path of the `clang` compiler.
     func getClangCompiler() throws -> AbsolutePath
 

Sources/SPMBuildCore/PluginContextSerializer.swift

@@ -24,6 +24,7 @@ internal struct PluginContextSerializer {
     let fileSystem: FileSystem
     let buildEnvironment: BuildEnvironment
     let pkgConfigDirectories: [AbsolutePath]
+    let sdkRootPath: AbsolutePath?
     var paths: [WireInput.Path] = []
     var pathsToIds: [AbsolutePath: WireInput.Path.Id] = [:]
     var targets: [WireInput.Target] = []
@@ -108,6 +109,7 @@ internal struct PluginContextSerializer {
             for result in try pkgConfigArgs(
                 for: target,
                 pkgConfigDirectories: pkgConfigDirectories,
+                sdkRootPath: sdkRootPath,
                 fileSystem: fileSystem,
                 observabilityScope: observabilityScope
             ) {

Sources/SPMBuildCore/PluginInvocation.swift

@@ -59,6 +59,7 @@ extension PluginTarget {
         readOnlyDirectories: [AbsolutePath],
         allowNetworkConnections: [SandboxNetworkPermission],
         pkgConfigDirectories: [AbsolutePath],
+        sdkRootPath: AbsolutePath?,
         fileSystem: FileSystem,
         observabilityScope: ObservabilityScope,
         callbackQueue: DispatchQueue,
@@ -79,7 +80,8 @@ extension PluginTarget {
             var serializer = PluginContextSerializer(
                 fileSystem: fileSystem,
                 buildEnvironment: buildEnvironment,
-                pkgConfigDirectories: pkgConfigDirectories
+                pkgConfigDirectories: pkgConfigDirectories,
+                sdkRootPath: sdkRootPath
             )
             let pluginWorkDirId = try serializer.serialize(path: outputDirectory)
             let toolSearchDirIds = try toolSearchDirectories.map{ try serializer.serialize(path: $0) }
@@ -332,6 +334,7 @@ extension PackageGraph {
         buildEnvironment: BuildEnvironment,
         toolSearchDirectories: [AbsolutePath],
         pkgConfigDirectories: [AbsolutePath],
+        sdkRootPath: AbsolutePath?,
         pluginScriptRunner: PluginScriptRunner,
         observabilityScope: ObservabilityScope,
         fileSystem: FileSystem,
@@ -479,6 +482,7 @@ extension PackageGraph {
                     readOnlyDirectories: readOnlyDirectories,
                     allowNetworkConnections: [],
                     pkgConfigDirectories: pkgConfigDirectories,
+                    sdkRootPath: sdkRootPath,
                     fileSystem: fileSystem,
                     observabilityScope: observabilityScope,
                     callbackQueue: delegateQueue,

Sources/XCBuildSupport/PIFBuilder.swift

@@ -34,6 +34,9 @@ struct PIFBuilderParameters {
 
     /// An array of paths to search for pkg-config `.pc` files.
     let pkgConfigDirectories: [AbsolutePath]
+
+    /// The toolchain's SDK root path.
+    let sdkRootPath: AbsolutePath?
 }
 
 /// PIF object builder for a package graph.
@@ -711,6 +714,7 @@ final class PackagePIFProjectBuilder: PIFProjectBuilder {
         for result in try pkgConfigArgs(
             for: systemTarget,
             pkgConfigDirectories: parameters.pkgConfigDirectories,
+            sdkRootPath: parameters.sdkRootPath,
             fileSystem: fileSystem,
             observabilityScope: observabilityScope
         ) {

Sources/XCBuildSupport/XcodeBuildSystem.swift

@@ -321,7 +321,8 @@ extension PIFBuilderParameters {
             enableTestability: buildParameters.enableTestability,
             shouldCreateDylibForDynamicProducts: buildParameters.shouldCreateDylibForDynamicProducts,
             toolchainLibDir: (try? buildParameters.toolchain.toolchainLibDir) ?? .root,
-            pkgConfigDirectories: buildParameters.pkgConfigDirectories
+            pkgConfigDirectories: buildParameters.pkgConfigDirectories,
+            sdkRootPath: buildParameters.toolchain.sdkRootPath
         )
     }
 }

Tests/BuildTests/MockBuildTestHelper.swift

@@ -31,6 +31,7 @@ struct MockToolchain: PackageModel.Toolchain {
     let swiftResourcesPath: AbsolutePath? = nil
     let swiftStaticResourcesPath: AbsolutePath? = nil
     let isSwiftDevelopmentToolchain = false
+    let sdkRootPath: AbsolutePath? = nil
     let swiftPluginServerPath: AbsolutePath? = nil
     let extraFlags = PackageModel.BuildFlags()
     let installedSwiftPMConfiguration = InstalledSwiftPMConfiguration.default

Tests/FunctionalTests/PluginTests.swift

@@ -472,6 +472,7 @@ class PluginTests: XCTestCase {
                         readOnlyDirectories: [package.path],
                         allowNetworkConnections: [],
                         pkgConfigDirectories: [],
+                        sdkRootPath: nil,
                         fileSystem: localFileSystem,
                         observabilityScope: observability.topScope,
                         callbackQueue: delegateQueue,
@@ -742,6 +743,7 @@ class PluginTests: XCTestCase {
                 readOnlyDirectories: [package.path],
                 allowNetworkConnections: [],
                 pkgConfigDirectories: [],
+                sdkRootPath: try UserToolchain.default.sdkRootPath,
                 fileSystem: localFileSystem,
                 observabilityScope: observability.topScope,
                 callbackQueue: delegateQueue,

Tests/PackageLoadingTests/PkgConfigAllowlistTests.swift

@@ -10,6 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+import Basics
 import PackageLoading
 import XCTest
 
@@ -42,4 +43,12 @@ final class PkgConfigAllowlistTests: XCTestCase {
         XCTAssertEqual(result.0, ["-I", "/usr/include/Cellar/gtk+3/3.18.9/include/gtk-3.0", "-L/hello"])
         XCTAssertEqual(result.1, ["-L/usr/lib/Cellar/gtk+3/3.18.9/lib", "-lgtk-3", "-module-name", "-lcool", "ok", "name"])
     }
+
+    func testPathSDKPaths() throws {
+        let flags = ["-I/opt/homebrew/Cellar/cairo/1.16.0_5/include/cairo", "-I/Library/Developer/CommandLineTools/SDKs/MacOSX13.sdk/usr/include/ffi"]
+        let sdk = "/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX13.3.sdk"
+        let result = try patchSDKPaths(in: flags, to: AbsolutePath(sdk))
+
+        XCTAssertEqual(result, ["-I/opt/homebrew/Cellar/cairo/1.16.0_5/include/cairo", "-I\(sdk)/usr/include/ffi"])
+    }
 }

Tests/SPMBuildCoreTests/PluginInvocationTests.swift

@@ -195,6 +195,7 @@ class PluginInvocationTests: XCTestCase {
             buildEnvironment: BuildEnvironment(platform: .macOS, configuration: .debug),
             toolSearchDirectories: [UserToolchain.default.swiftCompilerPath.parentDirectory],
             pkgConfigDirectories: [],
+            sdkRootPath: UserToolchain.default.sdkRootPath,
             pluginScriptRunner: pluginRunner,
             observabilityScope: observability.topScope,
             fileSystem: fileSystem
@@ -900,6 +901,7 @@ class PluginInvocationTests: XCTestCase {
                     buildEnvironment: BuildEnvironment(platform: .macOS, configuration: .debug),
                     toolSearchDirectories: [UserToolchain.default.swiftCompilerPath.parentDirectory],
                     pkgConfigDirectories: [],
+                    sdkRootPath: UserToolchain.default.sdkRootPath,
                     pluginScriptRunner: pluginScriptRunner,
                     observabilityScope: observability.topScope,
                     fileSystem: localFileSystem
@@ -1243,6 +1245,7 @@ class PluginInvocationTests: XCTestCase {
                 buildEnvironment: BuildEnvironment(platform: .macOS, configuration: .debug),
                 toolSearchDirectories: [UserToolchain.default.swiftCompilerPath.parentDirectory],
                 pkgConfigDirectories: [],
+                sdkRootPath: UserToolchain.default.sdkRootPath,
                 pluginScriptRunner: pluginScriptRunner,
                 observabilityScope: observability.topScope,
                 fileSystem: localFileSystem

Tests/XCBuildSupportTests/PIFBuilderTests.swift

@@ -2626,7 +2626,8 @@ extension PIFBuilderParameters {
             enableTestability: false,
             shouldCreateDylibForDynamicProducts: shouldCreateDylibForDynamicProducts,
             toolchainLibDir: "/toolchain/lib",
-            pkgConfigDirectories: ["/pkg-config"]
+            pkgConfigDirectories: ["/pkg-config"],
+            sdkRootPath: "/some.sdk"
         )
     }
 }