add artifacts archive file format (#3295)

motivation: to support the package extensions feature, we need to support binary dependencies with tools that can be distributed across different platforms

changes:
* introduce new binary dependency type "artifactsArchive" with the extension "arar"
* the structure of the archive is inspired by XCFramework, and it can contain multiple tools/executables targeting multiple platforms and architectures using target triples to indicate such support
* update BuildPlan code to parse "arar" files and extract the tools available to the target in a [String: AbsolutePath] which * can be later hooks to be used by extensions
* refactor XCFramework code
* update call-sites and tests and add new tests

Author: tomerd

Sources/Build/ArtifactsArchiveMetadata.swift

@@ -0,0 +1,108 @@
+/*
+ This source file is part of the Swift.org open source project
+
+ Copyright (c) 2021 Apple Inc. and the Swift project authors
+ Licensed under Apache License v2.0 with Runtime Library Exception
+
+ See http://swift.org/LICENSE.txt for license information
+ See http://swift.org/CONTRIBUTORS.txt for Swift project authors
+ */
+
+import Foundation
+import PackageModel
+import SPMBuildCore
+import TSCBasic
+import TSCUtility
+
+public struct ArtifactsArchiveMetadata: Equatable {
+    public let schemaVersion: String
+    public let artifacts: [String: Artifact]
+
+    public init(schemaVersion: String, artifacts: [String: Artifact]) {
+        self.schemaVersion = schemaVersion
+        self.artifacts = artifacts
+    }
+
+    public struct Artifact: Equatable {
+        let type: ArtifactType
+        let version: String
+        let variants: [Variant]
+
+        public init(type: ArtifactsArchiveMetadata.ArtifactType, version: String, variants: [Variant]) {
+            self.type = type
+            self.version = version
+            self.variants = variants
+        }
+    }
+
+    // In the future we are likely to extend the ArtifactsArchive file format to carry other types of artifacts beyond executables.
+    // Additional fields may be required to support these new artifact types e.g. headers path for libraries.
+    // This can also support resource-only artifacts as well. For example, 3d models along with associated textures, or fonts, etc.
+    public enum ArtifactType: String, RawRepresentable, Decodable {
+        case executable
+    }
+
+    public struct Variant: Equatable {
+        let path: String
+        let supportedTriples: [Triple]
+
+        public init(path: String, supportedTriples: [Triple]) {
+            self.path = path
+            self.supportedTriples = supportedTriples
+        }
+    }
+}
+
+extension ArtifactsArchiveMetadata {
+    public static func parse(fileSystem: FileSystem, rootPath: AbsolutePath) throws -> ArtifactsArchiveMetadata {
+        let path = rootPath.appending(component: "info.json")
+        guard fileSystem.exists(path) else {
+            throw StringError("ArtifactsArchive info.json not found at '\(rootPath)'")
+        }
+
+        do {
+            let bytes = try fileSystem.readFileContents(path)
+            return try bytes.withData { data in
+                let decoder = JSONDecoder.makeWithDefaults()
+                return try decoder.decode(ArtifactsArchiveMetadata.self, from: data)
+            }
+        } catch {
+            throw StringError("failed parsing ArtifactsArchive info.json at '\(path)': \(error)")
+        }
+    }
+}
+
+extension ArtifactsArchiveMetadata: Decodable {
+    enum CodingKeys: String, CodingKey {
+        case schemaVersion
+        case artifacts
+    }
+}
+
+extension ArtifactsArchiveMetadata.Artifact: Decodable {
+    enum CodingKeys: String, CodingKey {
+        case type
+        case version
+        case variants
+    }
+
+    public init(from decoder: Decoder) throws {
+        let container = try decoder.container(keyedBy: CodingKeys.self)
+        self.type = try container.decode(ArtifactsArchiveMetadata.ArtifactType.self, forKey: .type)
+        self.version = try container.decode(String.self, forKey: .version)
+        self.variants = try container.decode([ArtifactsArchiveMetadata.Variant].self, forKey: .variants)
+    }
+}
+
+extension ArtifactsArchiveMetadata.Variant: Decodable {
+    enum CodingKeys: String, CodingKey {
+        case path
+        case supportedTriples
+    }
+
+    public init(from decoder: Decoder) throws {
+        let container = try decoder.container(keyedBy: CodingKeys.self)
+        self.supportedTriples = try container.decode([String].self, forKey: .supportedTriples).map { try Triple($0) }
+        self.path = try container.decode(String.self, forKey: .path)
+    }
+}

Sources/Build/BuildPlan.swift

@@ -1043,6 +1043,9 @@ public final class ProductBuildDescription {
     /// Paths to the binary libraries the product depends on.
     fileprivate var libraryBinaryPaths: Set<AbsolutePath> = []
 
+    /// Paths to tools shipped in binary dependencies
+    var availableTools: [String: AbsolutePath] = [:]
+
     /// Path to the temporary directory for this product.
     var tempsPath: AbsolutePath {
         return buildParameters.buildPath.appending(component: product.name + ".product")
@@ -1294,8 +1297,11 @@ public class BuildPlan {
     /// Cache for pkgConfig flags.
     private var pkgConfigCache = [SystemLibraryTarget: (cFlags: [String], libs: [String])]()
 
-    /// Cache for xcframework library information.
-    private var xcFrameworkCache = [BinaryTarget: LibraryInfo?]()
+    /// Cache for  library information.
+    private var externalLibrariesCache = [BinaryTarget: [LibraryInfo]]()
+
+    /// Cache for  tools information.
+    private var externalExecutablesCache = [BinaryTarget: [ExecutableInfo]]()
 
     private static func makeTestManifestTargets(
         _ buildParameters: BuildParameters,
@@ -1499,7 +1505,7 @@ public class BuildPlan {
         // Add flags for system targets.
         for systemModule in dependencies.systemModules {
             guard case let target as SystemLibraryTarget = systemModule.underlyingTarget else {
-                fatalError("This should not be possible.")
+                throw InternalError("This should not be possible.")
             }
             // Add pkgConfig libs arguments.
             buildProduct.additionalFlags += pkgConfig(for: target).libs
@@ -1569,6 +1575,8 @@ public class BuildPlan {
         // FIXME: We should write this as a custom llbuild task once we adopt it
         // as a library.
         try buildProduct.writeLinkFilelist(fileSystem)
+
+        buildProduct.availableTools = dependencies.availableTools
     }
 
     /// Computes the dependencies of a product.
@@ -1578,7 +1586,8 @@ public class BuildPlan {
         dylibs: [ResolvedProduct],
         staticTargets: [ResolvedTarget],
         systemModules: [ResolvedTarget],
-        libraryBinaryPaths: Set<AbsolutePath>
+        libraryBinaryPaths: Set<AbsolutePath>,
+        availableTools: [String: AbsolutePath]
     ) {
 
         // Sort the product targets in topological order.
@@ -1606,6 +1615,7 @@ public class BuildPlan {
         var staticTargets = [ResolvedTarget]()
         var systemModules = [ResolvedTarget]()
         var libraryBinaryPaths: Set<AbsolutePath> = []
+        var availableTools = [String: AbsolutePath]()
 
         for dependency in allTargets {
             switch dependency {
@@ -1628,8 +1638,15 @@ public class BuildPlan {
                     guard let binaryTarget = target.underlyingTarget as? BinaryTarget else {
                         throw InternalError("invalid binary target '\(target.name)'")
                     }
-                    if case .xcframework = binaryTarget.kind, let library = self.xcFrameworkLibrary(for: binaryTarget) {
-                        libraryBinaryPaths.insert(library.binaryPath)
+                    switch binaryTarget.kind {
+                    case .xcframework:
+                        let libraries = try self.parseXCFramework(for: binaryTarget)
+                        for library in libraries {
+                            libraryBinaryPaths.insert(library.libraryPath)
+                        }
+                    case .artifactsArchive:
+                        let tools = try self.parseArtifactsArchive(for: binaryTarget)
+                        tools.forEach { availableTools[$0.name] = $0.executablePath  }
                     }
                 case .extension:
                     continue
@@ -1650,7 +1667,7 @@ public class BuildPlan {
             }
         }
 
-        return (linkLibraries, staticTargets, systemModules, libraryBinaryPaths)
+        return (linkLibraries, staticTargets, systemModules, libraryBinaryPaths, availableTools)
     }
 
     /// Plan a Clang target.
@@ -1678,11 +1695,14 @@ public class BuildPlan {
                 clangTarget.additionalFlags += ["-fmodule-map-file=\(target.moduleMapPath.pathString)"]
                 clangTarget.additionalFlags += pkgConfig(for: target).cFlags
             case let target as BinaryTarget:
-                if let library = xcFrameworkLibrary(for: target) {
-                    if let headersPath = library.headersPath {
-                        clangTarget.additionalFlags += ["-I", headersPath.pathString]
+                if case .xcframework = target.kind {
+                    let libraries = try self.parseXCFramework(for: target)
+                    for library in libraries {
+                        if let headersPath = library.headersPath {
+                            clangTarget.additionalFlags += ["-I", headersPath.pathString]
+                        }
+                        clangTarget.libraryBinaryPaths.insert(library.libraryPath)
                     }
-                    clangTarget.libraryBinaryPaths.insert(library.binaryPath)
                 }
             default: continue
             }
@@ -1712,11 +1732,14 @@ public class BuildPlan {
                 swiftTarget.additionalFlags += ["-Xcc", "-fmodule-map-file=\(target.moduleMapPath.pathString)"]
                 swiftTarget.additionalFlags += pkgConfig(for: target).cFlags
             case let target as BinaryTarget:
-                if let library = xcFrameworkLibrary(for: target) {
-                    if let headersPath = library.headersPath {
-                        swiftTarget.additionalFlags += ["-Xcc", "-I", "-Xcc", headersPath.pathString]
+                if case .xcframework = target.kind {
+                    let libraries = try self.parseXCFramework(for: target)
+                    for library in libraries {
+                        if let headersPath = library.headersPath {
+                            swiftTarget.additionalFlags += ["-Xcc", "-I", "-Xcc", headersPath.pathString]
+                        }
+                        swiftTarget.libraryBinaryPaths.insert(library.libraryPath)
                     }
-                    swiftTarget.libraryBinaryPaths.insert(library.binaryPath)
                 }
             default:
                 break
@@ -1835,53 +1858,69 @@ public class BuildPlan {
         return result
     }
 
-    /// Extracts the library to building against from a XCFramework.
-    private func xcFrameworkLibrary(for target: BinaryTarget) -> LibraryInfo? {
-        func calculateLibraryInfo() -> LibraryInfo? {
-            // Parse the XCFramework's Info.plist.
-            let infoPath = target.artifactPath.appending(component: "Info.plist")
-            guard let info = XCFrameworkInfo(path: infoPath, diagnostics: diagnostics, fileSystem: fileSystem) else {
-                return nil
-            }
+    /// Extracts the library information from an XCFramework.
+    private func parseXCFramework(for target: BinaryTarget) throws -> [LibraryInfo] {
+        try self.externalLibrariesCache.memoize(key: target) {
+            let metadata = try XCFrameworkMetadata.parse(fileSystem: self.fileSystem, rootPath: target.artifactPath)
 
             // Check that it supports the target platform and architecture.
-            guard let library = info.libraries.first(where: {
-                return $0.platform == buildParameters.triple.os.asXCFrameworkPlatformString && $0.architectures.contains(buildParameters.triple.arch.rawValue)
+            guard let library = metadata.libraries.first(where: {
+                $0.platform == buildParameters.triple.os.asXCFrameworkPlatformString && $0.architectures.contains(buildParameters.triple.arch.rawValue)
             }) else {
-                diagnostics.emit(error: """
+                throw StringError("""
                     artifact '\(target.name)' does not support the target platform and architecture \
                     ('\(buildParameters.triple)')
                     """)
-                return nil
             }
 
             let libraryDirectory = target.artifactPath.appending(component: library.libraryIdentifier)
-            let binaryPath = libraryDirectory.appending(component: library.libraryPath)
-            let headersPath = library.headersPath.map({ libraryDirectory.appending(component: $0) })
-            return LibraryInfo(binaryPath: binaryPath, headersPath: headersPath)
-        }
+            let libraryPath = libraryDirectory.appending(RelativePath(library.libraryPath))
+            let headersPath = library.headersPath.map({ libraryDirectory.appending(RelativePath($0)) })
 
-        // If we don't have the library information yet, calculate it.
-        if let xcFramework = xcFrameworkCache[target] {
-            return xcFramework
+            return [LibraryInfo(libraryPath: libraryPath, headersPath: headersPath)]
         }
+    }
+
+    /// Extracts the artifacts  from an artifactsArchive
+    private func parseArtifactsArchive(for target: BinaryTarget) throws -> [ExecutableInfo] {
+        try self.externalExecutablesCache.memoize(key: target) {
+            let metadata = try ArtifactsArchiveMetadata.parse(fileSystem: self.fileSystem, rootPath: target.artifactPath)
 
-        let xcFramework = calculateLibraryInfo()
-        xcFrameworkCache[target] = xcFramework
-        return xcFramework
+            // filter the artifacts that are relevant to the triple
+            // FIXME: this filter needs to become more sophisticated
+            let supportedArtifacts = metadata.artifacts.filter { $0.value.variants.contains(where: { $0.supportedTriples.contains(buildParameters.triple) }) }
+            // TODO: add support for libraries
+            let executables = supportedArtifacts.filter { $0.value.type == .executable }
+
+            // flatten the results for easy access
+            return executables.reduce(into: [ExecutableInfo](), { partial, entry in
+                let executables = entry.value.variants.map {
+                    ExecutableInfo(name: entry.key, executablePath: target.artifactPath.appending(RelativePath($0.path)))
+                }
+                partial.append(contentsOf: executables)
+            })
+        }
     }
 }
 
-/// Information about a library.
+/// Information about a library from a binary dependency.
 private struct LibraryInfo: Equatable {
-
     /// The path to the binary.
-    let binaryPath: AbsolutePath
+    let libraryPath: AbsolutePath
 
     /// The path to the headers directory, if one exists.
     let headersPath: AbsolutePath?
 }
 
+/// Information about an executable from a binary dependency.
+private struct ExecutableInfo: Equatable {
+    /// The tool name
+    let name: String
+
+    /// The path to the executable.
+    let executablePath: AbsolutePath
+}
+
 private extension Diagnostic.Message {
     static var swiftBackDeployError: Diagnostic.Message {
         .warning("Swift compiler no longer supports statically linking the Swift libraries. They're included in the OS by default starting with macOS Mojave 10.14.4 beta 3. For macOS Mojave 10.14.3 and earlier, there's an optional Swift library package that can be downloaded from \"More Downloads\" for Apple Developers at https://developer.apple.com/download/more/")

Sources/Build/CMakeLists.txt

@@ -7,13 +7,14 @@
 # See http://swift.org/CONTRIBUTORS.txt for Swift project authors
 
 add_library(Build
+  ArtifactsArchiveMetadata.swift
   BuildOperationBuildSystemDelegateHandler.swift
   BuildOperation.swift
   BuildPlan.swift
   ManifestBuilder.swift
   SPMSwiftDriverExecutor.swift
   SwiftCompilerOutputParser.swift
-  XCFrameworkInfo.swift)
+  XCFrameworkMetadata.swift)
 target_link_libraries(Build PUBLIC
   TSCBasic
   Basics