add support for tools archive

motivation: to support the package extensions feature, we need to support binary dependencies with tools that can be distributed across different platforms

changes:
* introuce new binary dependency type "toolsArchive" with the extension "toar"
* the structure of the archive is inspired by XCFramework, and it can contain multiple tools/executables targeting multiple platforms and architectures using target triplets to indicate such support
* update BuildPlan code to parse "toar" files and extract the tools available to the target in a [String: AbsolutePath] which can be later hooks to be used by extensions
* refactory XCFramework code
* update call-sites and tests and add new tests

Author: tomerd

Sources/Build/BuildPlan.swift

@@ -1043,6 +1043,9 @@ public final class ProductBuildDescription {
     /// Paths to the binary libraries the product depends on.
     fileprivate var libraryBinaryPaths: Set<AbsolutePath> = []
 
+    /// Paths to tools shipped in binary dependencies
+    var availableTools: [String: AbsolutePath] = [:]
+
     /// Path to the temporary directory for this product.
     var tempsPath: AbsolutePath {
         return buildParameters.buildPath.appending(component: product.name + ".product")
@@ -1294,8 +1297,11 @@ public class BuildPlan {
     /// Cache for pkgConfig flags.
     private var pkgConfigCache = [SystemLibraryTarget: (cFlags: [String], libs: [String])]()
 
-    /// Cache for xcframework library information.
-    private var xcFrameworkCache = [BinaryTarget: LibraryInfo?]()
+    /// Cache for  library information.
+    private var externalLibrariesCache = [BinaryTarget: LibraryInfo]()
+
+    /// Cache for  tools information.
+    private var externalToolsCache = [BinaryTarget: [ToolInfo]]()
 
     private static func makeTestManifestTargets(
         _ buildParameters: BuildParameters,
@@ -1499,7 +1505,7 @@ public class BuildPlan {
         // Add flags for system targets.
         for systemModule in dependencies.systemModules {
             guard case let target as SystemLibraryTarget = systemModule.underlyingTarget else {
-                fatalError("This should not be possible.")
+                throw InternalError("This should not be possible.")
             }
             // Add pkgConfig libs arguments.
             buildProduct.additionalFlags += pkgConfig(for: target).libs
@@ -1569,6 +1575,8 @@ public class BuildPlan {
         // FIXME: We should write this as a custom llbuild task once we adopt it
         // as a library.
         try buildProduct.writeLinkFilelist(fileSystem)
+
+        buildProduct.availableTools = dependencies.availableTools
     }
 
     /// Computes the dependencies of a product.
@@ -1578,7 +1586,8 @@ public class BuildPlan {
         dylibs: [ResolvedProduct],
         staticTargets: [ResolvedTarget],
         systemModules: [ResolvedTarget],
-        libraryBinaryPaths: Set<AbsolutePath>
+        libraryBinaryPaths: Set<AbsolutePath>,
+        availableTools: [String: AbsolutePath]
     ) {
 
         // Sort the product targets in topological order.
@@ -1606,6 +1615,7 @@ public class BuildPlan {
         var staticTargets = [ResolvedTarget]()
         var systemModules = [ResolvedTarget]()
         var libraryBinaryPaths: Set<AbsolutePath> = []
+        var availableTools = [String: AbsolutePath]()
 
         for dependency in allTargets {
             switch dependency {
@@ -1628,8 +1638,13 @@ public class BuildPlan {
                     guard let binaryTarget = target.underlyingTarget as? BinaryTarget else {
                         throw InternalError("invalid binary target '\(target.name)'")
                     }
-                    if case .xcframework = binaryTarget.kind, let library = self.xcFrameworkLibrary(for: binaryTarget) {
+                    switch binaryTarget.kind {
+                    case .xcframework:
+                        let library = try self.parseXCFramework(for: binaryTarget)
                         libraryBinaryPaths.insert(library.binaryPath)
+                    case .toolsArchive:
+                        let tools = try self.parseToolsArchive(for: binaryTarget)
+                        tools.forEach { availableTools[$0.name] = $0.binaryPath  }
                     }
                 case .extension:
                     continue
@@ -1650,7 +1665,7 @@ public class BuildPlan {
             }
         }
 
-        return (linkLibraries, staticTargets, systemModules, libraryBinaryPaths)
+        return (linkLibraries, staticTargets, systemModules, libraryBinaryPaths, availableTools)
     }
 
     /// Plan a Clang target.
@@ -1678,7 +1693,8 @@ public class BuildPlan {
                 clangTarget.additionalFlags += ["-fmodule-map-file=\(target.moduleMapPath.pathString)"]
                 clangTarget.additionalFlags += pkgConfig(for: target).cFlags
             case let target as BinaryTarget:
-                if let library = xcFrameworkLibrary(for: target) {
+                if case .xcframework = target.kind {
+                    let library = try self.parseXCFramework(for: target)
                     if let headersPath = library.headersPath {
                         clangTarget.additionalFlags += ["-I", headersPath.pathString]
                     }
@@ -1712,7 +1728,8 @@ public class BuildPlan {
                 swiftTarget.additionalFlags += ["-Xcc", "-fmodule-map-file=\(target.moduleMapPath.pathString)"]
                 swiftTarget.additionalFlags += pkgConfig(for: target).cFlags
             case let target as BinaryTarget:
-                if let library = xcFrameworkLibrary(for: target) {
+                if case .xcframework = target.kind {
+                    let library = try self.parseXCFramework(for: target)
                     if let headersPath = library.headersPath {
                         swiftTarget.additionalFlags += ["-Xcc", "-I", "-Xcc", headersPath.pathString]
                     }
@@ -1835,53 +1852,64 @@ public class BuildPlan {
         return result
     }
 
-    /// Extracts the library to building against from a XCFramework.
-    private func xcFrameworkLibrary(for target: BinaryTarget) -> LibraryInfo? {
-        func calculateLibraryInfo() -> LibraryInfo? {
-            // Parse the XCFramework's Info.plist.
-            let infoPath = target.artifactPath.appending(component: "Info.plist")
-            guard let info = XCFrameworkInfo(path: infoPath, diagnostics: diagnostics, fileSystem: fileSystem) else {
-                return nil
-            }
+    /// Extracts the library information from an XCFramework.
+    private func parseXCFramework(for target: BinaryTarget) throws -> LibraryInfo {
+        try self.externalLibrariesCache.memoize(key: target) {
+            let metadata = try XCFrameworkMetadata.parse(fileSystem: self.fileSystem, rootPath: target.artifactPath)
 
             // Check that it supports the target platform and architecture.
-            guard let library = info.libraries.first(where: {
-                return $0.platform == buildParameters.triple.os.asXCFrameworkPlatformString && $0.architectures.contains(buildParameters.triple.arch.rawValue)
+            guard let library = metadata.libraries.first(where: {
+                $0.platform == buildParameters.triple.os.asXCFrameworkPlatformString && $0.architectures.contains(buildParameters.triple.arch.rawValue)
             }) else {
-                diagnostics.emit(error: """
+                throw StringError("""
                     artifact '\(target.name)' does not support the target platform and architecture \
                     ('\(buildParameters.triple)')
                     """)
-                return nil
             }
 
             let libraryDirectory = target.artifactPath.appending(component: library.libraryIdentifier)
             let binaryPath = libraryDirectory.appending(component: library.libraryPath)
             let headersPath = library.headersPath.map({ libraryDirectory.appending(component: $0) })
+
             return LibraryInfo(binaryPath: binaryPath, headersPath: headersPath)
         }
+    }
 
-        // If we don't have the library information yet, calculate it.
-        if let xcFramework = xcFrameworkCache[target] {
-            return xcFramework
-        }
+    /// Extracts the executables info from an executablesArchive
+    private func parseToolsArchive(for target: BinaryTarget) throws -> [ToolInfo] {
+        try self.externalToolsCache.memoize(key: target) {
+            let metadata = try ToolsArchiveMetadata.parse(fileSystem: self.fileSystem, rootPath: target.artifactPath)
 
-        let xcFramework = calculateLibraryInfo()
-        xcFrameworkCache[target] = xcFramework
-        return xcFramework
+            // filter the tools that are relevant to the triple
+            let supportedTools = metadata.tools.filter { $0.value.contains(where: { $0.supportedTriplets.contains(buildParameters.triple) }) }
+            // flatten the tools for each access
+            return supportedTools.reduce(into: [ToolInfo](), { partial, entry in
+                let tools = entry.value.map {
+                    ToolInfo(name: entry.key, binaryPath: target.artifactPath.appending(RelativePath($0.path)))
+                }
+                partial.append(contentsOf: tools)
+            })
+        }
     }
 }
 
-/// Information about a library.
+/// Information about a library from a binary dependency.
 private struct LibraryInfo: Equatable {
-
     /// The path to the binary.
     let binaryPath: AbsolutePath
 
     /// The path to the headers directory, if one exists.
     let headersPath: AbsolutePath?
 }
 
+/// Information about an executable from a binary dependency.
+private struct ToolInfo: Equatable {
+    /// The tool name
+    let name: String
+    /// The path to the binary.
+    let binaryPath: AbsolutePath
+}
+
 private extension Diagnostic.Message {
     static var swiftBackDeployError: Diagnostic.Message {
         .warning("Swift compiler no longer supports statically linking the Swift libraries. They're included in the OS by default starting with macOS Mojave 10.14.4 beta 3. For macOS Mojave 10.14.3 and earlier, there's an optional Swift library package that can be downloaded from \"More Downloads\" for Apple Developers at https://developer.apple.com/download/more/")
@@ -1980,3 +2008,14 @@ fileprivate extension Triple {
         isLinux() || arch == .wasm32
     }
 }
+
+// FIXME: move this to TSC
+extension Triple: Hashable {
+    public func hash(into hasher: inout Hasher) {
+        hasher.combine(self.tripleString)
+        hasher.combine(self.arch)
+        hasher.combine(self.vendor)
+        hasher.combine(self.os)
+        hasher.combine(self.abi)
+    }
+}

Sources/Build/CMakeLists.txt

@@ -13,7 +13,8 @@ add_library(Build
   ManifestBuilder.swift
   SPMSwiftDriverExecutor.swift
   SwiftCompilerOutputParser.swift
-  XCFrameworkInfo.swift)
+  ToolsArchiveMetadata.swift
+  XCFrameworkMetadata.swift)
 target_link_libraries(Build PUBLIC
   TSCBasic
   Basics

Sources/Build/ToolsArchiveMetadata.swift

@@ -0,0 +1,74 @@
+/*
+ This source file is part of the Swift.org open source project
+
+ Copyright (c) 2021 Apple Inc. and the Swift project authors
+ Licensed under Apache License v2.0 with Runtime Library Exception
+
+ See http://swift.org/LICENSE.txt for license information
+ See http://swift.org/CONTRIBUTORS.txt for Swift project authors
+ */
+
+import Foundation
+import PackageModel
+import SPMBuildCore
+import TSCBasic
+import TSCUtility
+
+public struct ToolsArchiveMetadata: Equatable {
+    public let schemaVersion: String
+    public let tools: [String: [Support]]
+
+    public init(schemaVersion: String, tools: [String: [ToolsArchiveMetadata.Support]]) {
+        self.schemaVersion = schemaVersion
+        self.tools = tools
+    }
+
+    public struct Support: Equatable {
+        let path: String
+        let supportedTriplets: [Triple]
+
+        public init(path: String, supportedTriplets: [Triple]) {
+            self.path = path
+            self.supportedTriplets = supportedTriplets
+        }
+    }
+}
+
+extension ToolsArchiveMetadata {
+    public static func parse(fileSystem: FileSystem, rootPath: AbsolutePath) throws -> ToolsArchiveMetadata {
+        let path = rootPath.appending(component: "info.json")
+        guard fileSystem.exists(path) else {
+            throw StringError("ExecutablesArchive info.json not found at '\(rootPath)'")
+        }
+
+        do {
+            let bytes = try fileSystem.readFileContents(path)
+            return try bytes.withData { data in
+                let decoder = JSONDecoder.makeWithDefaults()
+                return try decoder.decode(ToolsArchiveMetadata.self, from: data)
+            }
+        } catch {
+            throw StringError("failed parsing ExecutablesArchive info.json at '\(path)': \(error)")
+        }
+    }
+}
+
+extension ToolsArchiveMetadata: Decodable {
+    enum CodingKeys: String, CodingKey {
+        case schemaVersion
+        case tools = "availableTools"
+    }
+}
+
+extension ToolsArchiveMetadata.Support: Decodable {
+    enum CodingKeys: String, CodingKey {
+        case path
+        case supportedTriplets
+    }
+
+    public init(from decoder: Decoder) throws {
+        let container = try decoder.container(keyedBy: CodingKeys.self)
+        self.path = try container.decode(String.self, forKey: .path)
+        self.supportedTriplets = try container.decode([String].self, forKey: .supportedTriplets).map { try Triple($0) }
+    }
+}

Sources/Build/XCFrameworkInfo.swift renamed to Sources/Build/XCFrameworkMetadata.swift

@@ -1,20 +1,20 @@
 /*
-This source file is part of the Swift.org open source project
+ This source file is part of the Swift.org open source project
 
-Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
-Licensed under Apache License v2.0 with Runtime Library Exception
+ Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
+ Licensed under Apache License v2.0 with Runtime Library Exception
 
-See http://swift.org/LICENSE.txt for license information
-See http://swift.org/CONTRIBUTORS.txt for Swift project authors
-*/
+ See http://swift.org/LICENSE.txt for license information
+ See http://swift.org/CONTRIBUTORS.txt for Swift project authors
+ */
 
-import TSCBasic
-import TSCUtility
+import Foundation
 import PackageModel
 import SPMBuildCore
-import Foundation
+import TSCBasic
+import TSCUtility
 
-public struct XCFrameworkInfo: Equatable {
+public struct XCFrameworkMetadata: Equatable {
     public struct Library: Equatable {
         public let libraryIdentifier: String
         public let libraryPath: String
@@ -44,35 +44,32 @@ public struct XCFrameworkInfo: Equatable {
     }
 }
 
-extension XCFrameworkInfo {
-    public init?(path: AbsolutePath, diagnostics: DiagnosticsEngine, fileSystem: FileSystem) {
+extension XCFrameworkMetadata {
+    public static func parse(fileSystem: FileSystem, rootPath: AbsolutePath) throws -> XCFrameworkMetadata {
+        let path = rootPath.appending(component: "Info.plist")
         guard fileSystem.exists(path) else {
-            diagnostics.emit(error: "missing XCFramework Info.plist at '\(path)'")
-            return nil
+            throw StringError("XCFramework Info.plist not found at '\(rootPath)'")
         }
 
         do {
-            let plistBytes = try fileSystem.readFileContents(path)
-
-            let decoder = PropertyListDecoder()
-            self = try plistBytes.withData({ data in
-                try decoder.decode(XCFrameworkInfo.self, from: data)
-            })
+            let bytes = try fileSystem.readFileContents(path)
+            return try bytes.withData { data in
+                let decoder = PropertyListDecoder()
+                return try decoder.decode(XCFrameworkMetadata.self, from: data)
+            }
         } catch {
-            diagnostics.emit(error: "failed parsing XCFramework Info.plist at '\(path)': \(error)")
-            return nil
+            throw StringError("failed parsing XCFramework Info.plist at '\(path)': \(error)")
         }
     }
 }
 
-extension XCFrameworkInfo: Decodable {
+extension XCFrameworkMetadata: Decodable {
     enum CodingKeys: String, CodingKey {
         case libraries = "AvailableLibraries"
     }
 }
 
-extension Triple.Arch: Decodable { }
-extension XCFrameworkInfo.Library: Decodable {
+extension XCFrameworkMetadata.Library: Decodable {
     enum CodingKeys: String, CodingKey {
         case libraryIdentifier = "LibraryIdentifier"
         case libraryPath = "LibraryPath"
@@ -81,3 +78,5 @@ extension XCFrameworkInfo.Library: Decodable {
         case architectures = "SupportedArchitectures"
     }
 }
+
+extension Triple.Arch: Decodable {}

Sources/PackageModel/Target.swift

@@ -517,14 +517,14 @@ public final class BinaryTarget: Target {
 
     public enum Kind: String, RawRepresentable, Codable, CaseIterable {
         case xcframework
-        // TODO: add new types
-        //case swiftLibraryArchive
-        //case swiftExecutableArchive
+        case toolsArchive
 
         public var fileExtension: String {
             switch self {
             case .xcframework:
                 return "xcframework"
+            case .toolsArchive:
+                return "toar"
             }
         }