PackageModel: relative paths for cross-compilation destinations (#5812)

Currently, `destination.json` files only support absolute paths, which makes it hard to move cross-compilation sysroots between directories and copying them to different machines.

This change support for relative paths in v2 scheme of `destination.json` files. The path is then normalized relative to the location of `destination.json` file. 

`struct DestinationInfo` is renamed to `DestinationInfoV1`. New `struct DestinationInfoV2` is added to handle the new version that supports relative paths. This is also used as an opportunity to make the schema more consistent with manifests for artifact bundles, in which properties use `camelCase` instead of `dash-case` naming. Also new in schema version 2 are `linkerFlags` and renaming of `extraCPPFlags` to `extraCXXFlags`, to follow up on #5827 and #5837.

Author: MaxDesiatov

Sources/PackageModel/Destination.swift

@@ -238,6 +238,7 @@ public struct Destination: Encodable, Equatable {
         }
         return _sdkPlatformFrameworkPath
     }
+
     /// Cache storage for sdk platform path.
     private static var _sdkPlatformFrameworkPath: (fwk: AbsolutePath, lib: AbsolutePath)? = nil
 
@@ -258,34 +259,55 @@ public struct Destination: Encodable, Equatable {
 }
 
 extension Destination {
-    /// Load a Destination description from a JSON representation from disk.
+    /// Load a ``Destination`` description from a JSON representation from disk.
     public init(fromFile path: AbsolutePath, fileSystem: FileSystem) throws {
         let decoder = JSONDecoder.makeWithDefaults()
         let version = try decoder.decode(path: path, fileSystem: fileSystem, as: VersionInfo.self)
+        
         // Check schema version.
-        guard version.version == 1 else {
+        switch version.version {
+        case 1:
+            let destination = try decoder.decode(path: path, fileSystem: fileSystem, as: DestinationInfoV1.self)
+            try self.init(
+                destinationTriple: destination.target.map{ try Triple($0) },
+                sdkRootDir: destination.sdk,
+                toolchainBinDir: destination.binDir,
+                extraFlags: .init(
+                    cCompilerFlags: destination.extraCCFlags,
+                    cxxCompilerFlags: destination.extraCPPFlags,
+                    swiftCompilerFlags: destination.extraSwiftCFlags
+                )
+            )
+        case 2:
+            let destination = try decoder.decode(path: path, fileSystem: fileSystem, as: DestinationInfoV2.self)
+            let destinationDirectory = path.parentDirectory
+
+            // TODO support multiple host and destination triple.
+            try self.init(
+                hostTriple: destination.hostTriples.map(Triple.init).first,
+                destinationTriple: destination.destinationTriples.map(Triple.init).first,
+                sdkRootDir: AbsolutePath(validating: destination.sdkRootDir, relativeTo: destinationDirectory),
+                toolchainBinDir: AbsolutePath(validating: destination.toolchainBinDir, relativeTo: destinationDirectory),
+                extraFlags: .init(
+                    cCompilerFlags: destination.extraCCFlags,
+                    cxxCompilerFlags: destination.extraCXXFlags,
+                    swiftCompilerFlags: destination.extraSwiftCFlags,
+                    linkerFlags: destination.extraLinkerFlags
+                )
+            )
+        default:
             throw DestinationError.invalidSchemaVersion
         }
-        let destination = try decoder.decode(path: path, fileSystem: fileSystem, as: DestinationInfo.self)
-        try self.init(
-            destinationTriple: destination.target.map{ try Triple($0) },
-            sdkRootDir: destination.sdk,
-            toolchainBinDir: destination.binDir,
-            extraFlags: BuildFlags(
-                cCompilerFlags: destination.extraCCFlags,
-                // maintaining `destination.extraCPPFlags` naming inconsistency for compatibility.
-                cxxCompilerFlags: destination.extraCPPFlags,
-                swiftCompilerFlags: destination.extraSwiftCFlags
-            )
-        )
     }
 }
 
+/// Version of the schema of `destination.json` files used for cross-compilation.
 fileprivate struct VersionInfo: Codable {
     let version: Int
 }
 
-fileprivate struct DestinationInfo: Codable {
+/// Represents v1 schema of `destination.json` files used for cross-compilation.
+fileprivate struct DestinationInfoV1: Codable {
     let target: String?
     let sdk: AbsolutePath?
     let binDir: AbsolutePath
@@ -302,3 +324,15 @@ fileprivate struct DestinationInfo: Codable {
         case extraCPPFlags = "extra-cpp-flags"
     }
 }
+
+/// Represents v2 schema of `destination.json` files used for cross-compilation.
+fileprivate struct DestinationInfoV2: Codable {
+    let sdkRootDir: String
+    let toolchainBinDir: String
+    let hostTriples: [String]
+    let destinationTriples: [String]
+    let extraCCFlags: [String]
+    let extraSwiftCFlags: [String]
+    let extraCXXFlags: [String]
+    let extraLinkerFlags: [String]
+}

Sources/PackageModel/Manifest.swift

@@ -180,14 +180,14 @@ public final class Manifest {
         }
 
         var requiredDependencies: Set<PackageIdentity> = []
-        for target in self.targetsRequired(for: products) {
-            for targetDependency in target.dependencies {
+        for destinationTriple in self.targetsRequired(for: products) {
+            for targetDependency in destinationTriple.dependencies {
                 if let dependency = self.packageDependency(referencedBy: targetDependency) {
                     requiredDependencies.insert(dependency.identity)
                 }
             }
 
-            target.pluginUsages?.forEach {
+            destinationTriple.pluginUsages?.forEach {
                 if let dependency = self.packageDependency(referencedBy: $0) {
                     requiredDependencies.insert(dependency.identity)
                 }

Sources/PackageModel/UserToolchain.swift

@@ -258,7 +258,7 @@ public final class UserToolchain: Toolchain {
     }
 
     internal static func deriveSwiftCFlags(triple: Triple, destination: Destination, environment: EnvironmentVariables) throws -> [String] {
-        guard let sdk = destination.sdkRootDir else {
+        guard let sdkDir = destination.sdkRootDir else {
             if triple.isWindows() {
                 // Windows uses a variable named SDKROOT to determine the root of
                 // the SDK.  This is not the same value as the SDKROOT parameter
@@ -336,7 +336,7 @@ public final class UserToolchain: Toolchain {
         }
 
         return (triple.isDarwin() || triple.isAndroid() || triple.isWASI() || triple.isWindows()
-                ? ["-sdk", sdk.pathString]
+                ? ["-sdk", sdkDir.pathString]
                 : [])
         + destination.extraFlags.swiftCompilerFlags
     }
@@ -389,9 +389,9 @@ public final class UserToolchain: Toolchain {
 
         self.extraFlags.swiftCompilerFlags = try Self.deriveSwiftCFlags(triple: triple, destination: destination, environment: environment)
 
-        if let sdk = destination.sdkRootDir {
+        if let sdkDir = destination.sdkRootDir {
             self.extraFlags.cCompilerFlags = [
-                triple.isDarwin() ? "-isysroot" : "--sysroot", sdk.pathString
+                triple.isDarwin() ? "-isysroot" : "--sysroot", sdkDir.pathString
             ] + destination.extraFlags.cCompilerFlags
 
             self.extraFlags.cxxCompilerFlags = destination.extraFlags.cxxCompilerFlags
@@ -521,12 +521,12 @@ public final class UserToolchain: Toolchain {
                 return try AbsolutePath(validating: path)
             }
         } else if triple.isWindows() {
-            let sdkroot: AbsolutePath
+            let sdkRoot: AbsolutePath
 
-            if let sdk = destination.sdkRootDir {
-                sdkroot = sdk
-            } else if let SDKROOT = environment["SDKROOT"], let sdk = try? AbsolutePath(validating: SDKROOT) {
-                sdkroot = sdk
+            if let sdkDir = destination.sdkRootDir {
+                sdkRoot = sdkDir
+            } else if let SDKROOT = environment["SDKROOT"], let sdkDir = try? AbsolutePath(validating: SDKROOT) {
+                sdkRoot = sdkDir
             } else {
                 return .none
             }
@@ -537,7 +537,7 @@ public final class UserToolchain: Toolchain {
             // Library/Developer/Platforms/[PLATFORM].platform/Developer/SDKs/[PLATFORM].sdk/...
             //
             // SDKROOT points to [PLATFORM].sdk
-            let platform = sdkroot.parentDirectory.parentDirectory.parentDirectory
+            let platform = sdkRoot.parentDirectory.parentDirectory.parentDirectory
 
             if let info = WindowsPlatformInfo(reading: platform.appending(component: "Info.plist"),
                                               diagnostics: nil, filesystem: localFileSystem) {

Tests/PackageModelTests/DestinationTests.swift

@@ -0,0 +1,97 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift open source project
+//
+// Copyright (c) 2014-2022 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See http://swift.org/LICENSE.txt for license information
+// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import Basics
+@testable import PackageModel
+import TSCBasic
+import TSCUtility
+import XCTest
+
+private let bundleRootPath = try! AbsolutePath(validating: "/tmp/cross-toolchain")
+private let toolchainBinDir = RelativePath("swift.xctoolchain/usr/bin")
+private let sdkRootDir = RelativePath("ubuntu-jammy.sdk")
+private let hostTriple = "arm64-apple-darwin22.1.0"
+private let destinationTriple = "x86_64-unknown-linux-gnu"
+private let extraFlags = BuildFlags(
+    cCompilerFlags: ["-fintegrated-as"],
+    cxxCompilerFlags: ["-fno-exceptions"],
+    swiftCompilerFlags: ["-enable-experimental-cxx-interop", "-use-ld=lld"],
+    linkerFlags: ["-R/usr/lib/swift/linux/"]
+)
+
+private let destinationV1JSON =
+    #"""
+    {
+        "version": 1,
+        "sdk": "\#(bundleRootPath.appending(sdkRootDir))",
+        "toolchain-bin-dir": "\#(bundleRootPath.appending(toolchainBinDir))",
+        "target": "\#(destinationTriple)",
+        "extra-cc-flags": \#(extraFlags.cCompilerFlags),
+        "extra-swiftc-flags": \#(extraFlags.swiftCompilerFlags),
+        "extra-cpp-flags": \#(extraFlags.cxxCompilerFlags)
+    }
+    """#
+
+private let destinationV2JSON =
+    #"""
+    {
+        "version": 2,
+        "sdkRootDir": "\#(sdkRootDir)",
+        "toolchainBinDir": "\#(toolchainBinDir)",
+        "hostTriples": ["\#(hostTriple)"],
+        "destinationTriples": ["\#(destinationTriple)"],
+        "extraCCFlags": \#(extraFlags.cCompilerFlags),
+        "extraSwiftCFlags": \#(extraFlags.swiftCompilerFlags),
+        "extraCXXFlags": \#(extraFlags.cxxCompilerFlags),
+        "extraLinkerFlags": \#(extraFlags.linkerFlags)
+    }
+    """#
+
+final class DestinationTests: XCTestCase {
+    func testDestinationCodable() throws {
+        let fs = InMemoryFileSystem(files: [
+            "\(bundleRootPath)/destinationV1.json": ByteString(encodingAsUTF8: destinationV1JSON),
+            "\(bundleRootPath)/destinationV2.json": ByteString(encodingAsUTF8: destinationV2JSON),
+        ])
+
+        let destinationV1 = try Destination(fromFile: bundleRootPath.appending(.init("destinationV1.json")), fileSystem: fs)
+
+        var flagsWithoutLinkerFlags = extraFlags
+        flagsWithoutLinkerFlags.linkerFlags = []
+
+        let sdkRootAbsolutePath = bundleRootPath.appending(sdkRootDir)
+        let toolchainBinAbsolutePath = bundleRootPath.appending(toolchainBinDir)
+
+        XCTAssertEqual(
+            destinationV1,
+            Destination(
+                destinationTriple: try Triple(destinationTriple),
+                sdkRootDir: sdkRootAbsolutePath,
+                toolchainBinDir: toolchainBinAbsolutePath,
+                extraFlags: flagsWithoutLinkerFlags
+            )
+        )
+
+        let destinationV2 = try Destination(fromFile: bundleRootPath.appending(.init("destinationV2.json")), fileSystem: fs)
+
+        XCTAssertEqual(
+            destinationV2,
+            Destination(
+                hostTriple: try Triple(hostTriple),
+                destinationTriple: try Triple(destinationTriple),
+                sdkRootDir: sdkRootAbsolutePath,
+                toolchainBinDir: toolchainBinAbsolutePath,
+                extraFlags: extraFlags
+            )
+        )
+    }
+}

Tests/PackageModelTests/PackageModelTests.swift

@@ -57,18 +57,18 @@ class PackageModelTests: XCTestCase {
 
     func testAndroidCompilerFlags() throws {
         let triple = try Triple("x86_64-unknown-linux-android")
-        let sdk = AbsolutePath(path: "/some/path/to/an/SDK.sdk")
+        let sdkDir = AbsolutePath(path: "/some/path/to/an/SDK.sdk")
         let toolchainPath = AbsolutePath(path: "/some/path/to/a/toolchain.xctoolchain")
 
         let destination = Destination(
             destinationTriple: triple,
-            sdkRootDir: sdk,
+            sdkRootDir: sdkDir,
             toolchainBinDir: toolchainPath.appending(components: "usr", "bin")
         )
 
         XCTAssertEqual(try UserToolchain.deriveSwiftCFlags(triple: triple, destination: destination, environment: .process()), [
             // Needed when cross‐compiling for Android. 2020‐03‐01
-            "-sdk", sdk.pathString,
+            "-sdk", sdkDir.pathString,
         ])
     }
 }