introduce archive index files

motivation: to support the package extensions feature, we need to support binary dependencies with tools that can be distributed across different platforms. in previous PR we intriduces the archive file, this PR adds the archive index files which helps control the size of the archive files by splitting them across key platforms

changes:
* introduce new binary dependency file type named "Archive Index" with the extension "ari"
* the structure of the file is a flat list of archive files, their checksums and the target-triples they support
* add code to Workspace pre-downloading the binaryTarget "zip" files to first fetch "ari" files, parse them and add any relevant "zip" files to the *host* tripple to the list of candidate downloads
* add and adjust tests

Author: tomerd

Sources/Basics/ConcurrencyHelpers.swift

@@ -109,6 +109,14 @@ public final class ThreadSafeArrayStore<Value> {
         }
     }
 
+    @discardableResult
+    public func append(contentsOf items: [Value]) -> Int {
+        self.lock.withLock {
+            self.underlying.append(contentsOf: items)
+            return self.underlying.count
+        }
+    }
+
     public var count: Int {
         self.lock.withLock {
             self.underlying.count

Sources/SPMTestSupport/MockHTTPClient.swift

@@ -0,0 +1,23 @@
+/*
+ This source file is part of the Swift.org open source project
+
+ Copyright (c) 2021 Apple Inc. and the Swift project authors
+ Licensed under Apache License v2.0 with Runtime Library Exception
+
+ See http://swift.org/LICENSE.txt for license information
+ See http://swift.org/CONTRIBUTORS.txt for Swift project authors
+*/
+
+import Basics
+import TSCBasic
+import TSCUtility
+import Foundation
+
+extension HTTPClient {
+    public static func mock() -> HTTPClient {
+        let handler: HTTPClient.Handler = { request, _, completion in
+            completion(.success(.init(statusCode: 200, body: Data(request.url.absoluteString.utf8))))
+        }
+        return HTTPClient(handler: handler)
+    }
+}

Sources/SPMTestSupport/MockWorkspace.swift

@@ -20,6 +20,7 @@ import Basics
 public final class MockWorkspace {
     let sandbox: AbsolutePath
     let fs: FileSystem
+    public let httpClient: HTTPClient
     public let downloader: MockDownloader
     public let archiver: MockArchiver
     public let checksumAlgorithm: MockHashAlgorithm
@@ -34,10 +35,10 @@ public final class MockWorkspace {
     let skipUpdate: Bool
     let enablePubGrub: Bool
 
-
     public init(
         sandbox: AbsolutePath,
         fs: FileSystem,
+        httpClient: HTTPClient = HTTPClient.mock(),
         downloader: MockDownloader? = nil,
         archiver: MockArchiver = MockArchiver(),
         checksumAlgorithm: MockHashAlgorithm = MockHashAlgorithm(),
@@ -50,6 +51,7 @@ public final class MockWorkspace {
     ) throws {
         self.sandbox = sandbox
         self.fs = fs
+        self.httpClient = httpClient
         self.downloader = downloader ?? MockDownloader(fileSystem: fs)
         self.archiver = archiver
         self.checksumAlgorithm = checksumAlgorithm
@@ -172,6 +174,7 @@ public final class MockWorkspace {
             fileSystem: self.fs,
             repositoryProvider: self.repoProvider,
             identityResolver: self.identityResolver,
+            httpClient: self.httpClient,
             downloader: self.downloader,
             archiver: self.archiver,
             checksumAlgorithm: self.checksumAlgorithm,

Sources/Workspace/Diagnostics.swift

@@ -141,12 +141,12 @@ extension Diagnostic.Message {
         .error("checksum of downloaded artifact of binary target '\(targetName)' (\(actualChecksum)) does not match checksum specified by the manifest (\(expectedChecksum))")
     }
 
-    static func artifactFailedDownload(targetName: String, reason: String) -> Diagnostic.Message {
-        .error("artifact of binary target '\(targetName)' failed download: \(reason)")
+    static func artifactFailedDownload(artifactURL: Foundation.URL, targetName: String, reason: String) -> Diagnostic.Message {
+        .error("failed downloading '\(artifactURL.absoluteString)' which is required by binary target '\(targetName)': \(reason)")
     }
 
-    static func artifactFailedExtraction(targetName: String, reason: String) -> Diagnostic.Message {
-        .error("artifact of binary target '\(targetName)' failed extraction: \(reason)")
+    static func artifactFailedExtraction(artifactURL: Foundation.URL, targetName: String, reason: String) -> Diagnostic.Message {
+        .error("failed extracting '\(artifactURL.absoluteString)' which is required by binary target '\(targetName)': \(reason)")
     }
 
     static func artifactNotFound(targetName: String, artifactName: String) -> Diagnostic.Message {

Sources/Workspace/Workspace.swift

@@ -211,7 +211,11 @@ public class Workspace {
     /// The package container provider.
     fileprivate let containerProvider: RepositoryPackageContainerProvider
 
+    /// The http client used for downloading binary artifacts.
+    fileprivate let httpClient: HTTPClient
+
     /// The downloader used for downloading binary artifacts.
+    // FIXME: consolidate HttpClient and Downloader as they provide same functionality
     fileprivate let downloader: Downloader
 
     fileprivate let netrcFilePath: AbsolutePath?
@@ -267,6 +271,7 @@ public class Workspace {
         fileSystem: FileSystem = localFileSystem,
         repositoryProvider: RepositoryProvider = GitRepositoryProvider(),
         identityResolver: IdentityResolver? = nil,
+        httpClient: HTTPClient? = nil,
         downloader: Downloader = FoundationDownloader(),
         netrcFilePath: AbsolutePath? = nil,
         archiver: Archiver = ZipArchiver(),
@@ -286,6 +291,7 @@ public class Workspace {
         self.currentToolsVersion = currentToolsVersion
         self.toolsVersionLoader = toolsVersionLoader
         self.downloader = downloader
+        self.httpClient = httpClient ?? HTTPClient()
         self.netrcFilePath = netrcFilePath
         self.archiver = archiver
 
@@ -771,7 +777,7 @@ extension Workspace {
 
         return diagnostics.wrap {
             let contents = try fileSystem.readFileContents(path)
-            return checksumAlgorithm.hash(contents).hexadecimalRepresentation
+            return self.checksumAlgorithm.hash(contents).hexadecimalRepresentation
         } ?? ""
     }
 }
@@ -1546,7 +1552,7 @@ extension Workspace {
                             targetName: target.name,
                             path: absolutePath)
                     )
-                } else if let url = target.url, let checksum = target.checksum {
+                } else if let url = target.url.flatMap(URL.init(string:)), let checksum = target.checksum {
                     remoteArtifacts.append(
                         .init(
                             packageRef: packageRef,
@@ -1566,6 +1572,7 @@ extension Workspace {
     private func download(_ artifacts: [RemoteArtifact], diagnostics: DiagnosticsEngine) throws -> [ManagedArtifact] {
         let group = DispatchGroup()
         let tempDiagnostics = DiagnosticsEngine()
+        let result = ThreadSafeArrayStore<ManagedArtifact>()
 
         var authProvider: AuthorizationProviding? = nil
         var didDownloadAnyArtifact = false
@@ -1577,15 +1584,70 @@ extension Workspace {
         }
         #endif
 
-        let result = ThreadSafeArrayStore<ManagedArtifact>()
+        // zip files to download
+        // stored in a thread-safe way as we may fetch more from "ari" files
+        let zipArtifacts = ThreadSafeArrayStore<RemoteArtifact>(artifacts.filter { $0.url.pathExtension.lowercased() == "zip" })
 
-        for artifact in artifacts {
+        // fetch and parse "ari" files, if any
+        let hostToolchain = try UserToolchain(destination: .hostDestination())
+        let jsonDecoder = JSONDecoder.makeWithDefaults()
+        let indexFiles = artifacts.filter { $0.url.pathExtension.lowercased() == "ari" }
+        for indexFile in indexFiles {
             group.enter()
-            defer { group.leave() }
+            var request = HTTPClient.Request(method: .get, url: indexFile.url)
+            request.options.validResponseCodes = [200]
+            request.options.authorizationProvider = authProvider?.authorization(for:)
+            self.httpClient.execute(request) { result in
+                defer { group.leave() }
 
-            guard let parsedURL = URL(string: artifact.url) else {
-                throw StringError("invalid url \(artifact.url)")
+                do {
+                    switch result {
+                    case .failure(let error):
+                        throw error
+                    case .success(let response):
+                        guard let body = response.body else {
+                            throw StringError("Body is empty")
+                        }
+                        // FIXME: would be nice if checksumAlgorithm.hash took Data directly
+                        let bodyChecksum = self.checksumAlgorithm.hash(ByteString(body)).hexadecimalRepresentation
+                        guard bodyChecksum == indexFile.checksum else {
+                            throw StringError("checksum of downloaded artifact of binary target '\(indexFile.targetName)' (\(bodyChecksum)) does not match checksum specified by the manifest (\(indexFile.checksum ))")
+                        }
+                        let metadata = try jsonDecoder.decode(ArchiveIndexFile.self, from: body)
+                        // FIXME: this filter needs to become more sophisticated
+                        guard let supportedArchive = metadata.archives.first(where: { $0.fileName.lowercased().hasSuffix(".zip") && $0.supportedTriples.contains(hostToolchain.triple) }) else {
+                            throw StringError("No supported archive was found for '\(hostToolchain.triple)'")
+                        }
+                        // add relevant archive
+                        zipArtifacts.append(
+                            RemoteArtifact(
+                                packageRef: indexFile.packageRef,
+                                targetName: indexFile.targetName,
+                                url: indexFile.url.deletingLastPathComponent().appendingPathComponent(supportedArchive.fileName),
+                                checksum: supportedArchive.checksum)
+                        )
+                    }
+                } catch {
+                    tempDiagnostics.emit(.error("failed retrieving '\(indexFile.url)': \(error)"))
+                }
             }
+        }
+
+        // wait for all "ari" files to be processed
+        group.wait()
+
+        // no reason to continue if we already ran into issues
+        if tempDiagnostics.hasErrors {
+            // collect all diagnostics
+            diagnostics.append(contentsOf: tempDiagnostics)
+            throw Diagnostics.fatalError
+        }
+
+        // finally download zip files, if any
+        didDownloadAnyArtifact = zipArtifacts.count > 0
+        for artifact in (zipArtifacts.map{ $0 }) {
+            group.enter()
+            defer { group.leave() }
 
             let parentDirectory =  self.artifactsPath.appending(component: artifact.packageRef.name)
 
@@ -1596,18 +1658,16 @@ extension Workspace {
                 continue
             }
 
-            let archivePath = parentDirectory.appending(component: parsedURL.lastPathComponent)
-
-            didDownloadAnyArtifact = true
+            let archivePath = parentDirectory.appending(component: artifact.url.lastPathComponent)
 
             group.enter()
-            downloader.downloadFile(
-                at: parsedURL,
+            self.downloader.downloadFile(
+                at: artifact.url,
                 to: archivePath,
                 withAuthorizationProvider: authProvider,
                 progress: { bytesDownloaded, totalBytesToDownload in
                     self.delegate?.downloadingBinaryArtifact(
-                        from: artifact.url,
+                        from: artifact.url.absoluteString,
                         bytesDownloaded: bytesDownloaded,
                         totalBytesToDownload: totalBytesToDownload)
                 },
@@ -1643,21 +1703,21 @@ extension Workspace {
                                     .remote(
                                         packageRef: artifact.packageRef,
                                         targetName: artifact.targetName,
-                                        url: artifact.url,
+                                        url: artifact.url.absoluteString,
                                         checksum: artifact.checksum,
                                         path: artifactPath
                                     )
                                 )
                             case .failure(let error):
                                 let reason = (error as? LocalizedError)?.errorDescription ?? error.localizedDescription
-                                tempDiagnostics.emit(.artifactFailedExtraction(targetName: artifact.targetName, reason: reason))
+                                tempDiagnostics.emit(.artifactFailedExtraction(artifactURL: artifact.url, targetName: artifact.targetName, reason: reason))
                             }
 
                             tempDiagnostics.wrap { try self.fileSystem.removeFileTree(archivePath) }
                         })
                     case .failure(let error):
                         let reason = error.errorDescription ?? error.localizedDescription
-                        tempDiagnostics.emit(.artifactFailedDownload(targetName: artifact.targetName, reason: reason))
+                        tempDiagnostics.emit(.artifactFailedDownload(artifactURL: artifact.url, targetName: artifact.targetName, reason: reason))
                     }
                 })
         }
@@ -1668,9 +1728,8 @@ extension Workspace {
             delegate?.didDownloadBinaryArtifacts()
         }
 
-        for diagnostic in tempDiagnostics.diagnostics {
-            diagnostics.emit(diagnostic.message, location: diagnostic.location)
-        }
+        // collect all diagnostics
+        diagnostics.append(contentsOf: tempDiagnostics)
 
         return result.map{ $0 }
     }
@@ -2567,10 +2626,34 @@ public final class LoadableResult<Value> {
 private struct RemoteArtifact {
     let packageRef: PackageReference
     let targetName: String
-    let url: String
+    let url: Foundation.URL
     let checksum: String
 }
 
+private struct ArchiveIndexFile: Decodable {
+    let schemaVersion: String
+    let archives: [Archive]
+
+    struct Archive: Decodable {
+        let fileName: String
+        let checksum: String
+        let supportedTriples: [Triple]
+
+        enum CodingKeys: String, CodingKey {
+            case fileName
+            case checksum
+            case supportedTriples
+        }
+
+        public init(from decoder: Decoder) throws {
+            let container = try decoder.container(keyedBy: CodingKeys.self)
+            self.fileName = try container.decode(String.self, forKey: .fileName)
+            self.checksum = try container.decode(String.self, forKey: .checksum)
+            self.supportedTriples = try container.decode([String].self, forKey: .supportedTriples).map(Triple.init)
+        }
+    }
+}
+
 private extension ManagedArtifact {
     var originURL: String? {
         switch self.source {
@@ -2598,3 +2681,11 @@ private extension PackageDependencyDescription {
         }
     }
 }
+
+private extension DiagnosticsEngine {
+    func append(contentsOf other: DiagnosticsEngine) {
+        for diagnostic in other.diagnostics {
+            self.emit(diagnostic.message, location: diagnostic.location)
+        }
+    }
+}