introduce archive index files

Sources/Basics/ConcurrencyHelpers.swift

@@ -109,6 +109,14 @@ public final class ThreadSafeArrayStore<Value> {
         }
     }
 
+    @discardableResult
+    public func append(contentsOf items: [Value]) -> Int {
+        self.lock.withLock {
+            self.underlying.append(contentsOf: items)
+            return self.underlying.count
+        }
+    }
+
     public var count: Int {
         self.lock.withLock {
             self.underlying.count

Sources/SPMTestSupport/MockHTTPClient.swift

@@ -9,14 +9,12 @@
 */
 
 import Basics
-import Foundation
 import TSCBasic
 import TSCUtility
 
 extension HTTPClient {
     public static func mock(fileSystem: FileSystem) -> HTTPClient {
         let handler: HTTPClient.Handler = { request, _, completion in
-
             switch request.kind {
             case.generic:
                 completion(.success(.okay(body: request.url.absoluteString)))
@@ -33,6 +31,6 @@ extension HTTPClient {
                 }
             }
         }
-        return HTTPClient(handler: handler)       
+        return HTTPClient(handler: handler)
     }
 }

Sources/SPMTestSupport/MockHashAlgorithm.swift

@@ -6,24 +6,25 @@
 
  See http://swift.org/LICENSE.txt for license information
  See http://swift.org/CONTRIBUTORS.txt for Swift project authors
-*/
+ */
 
+import Basics
 import TSCBasic
 
 public class MockHashAlgorithm: HashAlgorithm {
     public typealias Hash = (ByteString) -> ByteString
 
-    public private(set) var hashes: [ByteString] = []
+    public private(set) var hashes = ThreadSafeArrayStore<ByteString>()
     private var hashFunction: Hash!
 
     public init(hash: Hash? = nil) {
-        hashFunction = hash ?? { hash in
+        self.hashFunction = hash ?? { hash in
             self.hashes.append(hash)
             return ByteString(hash.contents.reversed())
         }
     }
 
     public func hash(_ bytes: ByteString) -> ByteString {
-        return hashFunction(bytes)
+        return self.hashFunction(bytes)
     }
 }

Sources/SPMTestSupport/MockWorkspace.swift

@@ -34,7 +34,6 @@ public final class MockWorkspace {
     let skipUpdate: Bool
     let enablePubGrub: Bool
 
-
     public init(
         sandbox: AbsolutePath,
         fs: FileSystem,

Sources/Workspace/Diagnostics.swift

@@ -141,12 +141,12 @@ extension Diagnostic.Message {
         .error("checksum of downloaded artifact of binary target '\(targetName)' (\(actualChecksum)) does not match checksum specified by the manifest (\(expectedChecksum))")
     }
 
-    static func artifactFailedDownload(targetName: String, reason: String) -> Diagnostic.Message {
-        .error("artifact of binary target '\(targetName)' failed download: \(reason)")
+    static func artifactFailedDownload(artifactURL: Foundation.URL, targetName: String, reason: String) -> Diagnostic.Message {
+        .error("failed downloading '\(artifactURL.absoluteString)' which is required by binary target '\(targetName)': \(reason)")
     }
 
-    static func artifactFailedExtraction(targetName: String, reason: String) -> Diagnostic.Message {
-        .error("artifact of binary target '\(targetName)' failed extraction: \(reason)")
+    static func artifactFailedExtraction(artifactURL: Foundation.URL, targetName: String, reason: String) -> Diagnostic.Message {
+        .error("failed extracting '\(artifactURL.absoluteString)' which is required by binary target '\(targetName)': \(reason)")
     }
 
     static func artifactNotFound(targetName: String, artifactName: String) -> Diagnostic.Message {

Sources/Workspace/Workspace.swift

@@ -771,7 +771,7 @@ extension Workspace {
 
         return diagnostics.wrap {
             let contents = try fileSystem.readFileContents(path)
-            return checksumAlgorithm.hash(contents).hexadecimalRepresentation
+            return self.checksumAlgorithm.hash(contents).hexadecimalRepresentation
         } ?? ""
     }
 }
@@ -1546,7 +1546,7 @@ extension Workspace {
                             targetName: target.name,
                             path: absolutePath)
                     )
-                } else if let url = target.url, let checksum = target.checksum {
+                } else if let url = target.url.flatMap(URL.init(string:)), let checksum = target.checksum {
                     remoteArtifacts.append(
                         .init(
                             packageRef: packageRef,
@@ -1566,23 +1566,78 @@ extension Workspace {
     private func download(_ artifacts: [RemoteArtifact], diagnostics: DiagnosticsEngine) throws -> [ManagedArtifact] {
         let group = DispatchGroup()
         let tempDiagnostics = DiagnosticsEngine()
+        let result = ThreadSafeArrayStore<ManagedArtifact>()
 
         var authProvider: AuthorizationProviding? = nil
-        #if os(macOS)
-        // Netrc feature currently only supported on macOS
+        #if os(macOS) // Netrc feature currently only supported on macOS
         authProvider = try? Netrc.load(fromFileAtPath: netrcFilePath).get()
         #endif
 
-        var didDownloadAnyArtifact = false
-        let result = ThreadSafeArrayStore<ManagedArtifact>()
+        // zip files to download
+        // stored in a thread-safe way as we may fetch more from "ari" files
+        let zipArtifacts = ThreadSafeArrayStore<RemoteArtifact>(artifacts.filter { $0.url.pathExtension.lowercased() == "zip" })
+
+        // fetch and parse "ari" files, if any
+        let indexFiles = artifacts.filter { $0.url.pathExtension.lowercased() == "ari" }
+        if !indexFiles.isEmpty {
+            let hostToolchain = try UserToolchain(destination: .hostDestination())
+            let jsonDecoder = JSONDecoder.makeWithDefaults()
+            for indexFile in indexFiles {
+                group.enter()
+                var request = HTTPClient.Request(method: .get, url: indexFile.url)
+                request.options.validResponseCodes = [200]
+                request.options.authorizationProvider = authProvider?.authorization(for:)
+                self.httpClient.execute(request) { result in
+                    defer { group.leave() }
 
-        for artifact in artifacts {
-            group.enter()
-            defer { group.leave() }
+                    do {
+                        switch result {
+                        case .failure(let error):
+                            throw error
+                        case .success(let response):
+                            guard let body = response.body else {
+                                throw StringError("Body is empty")
+                            }
+                            // FIXME: would be nice if checksumAlgorithm.hash took Data directly
+                            let bodyChecksum = self.checksumAlgorithm.hash(ByteString(body)).hexadecimalRepresentation
+                            guard bodyChecksum == indexFile.checksum else {
+                                throw StringError("checksum of downloaded artifact of binary target '\(indexFile.targetName)' (\(bodyChecksum)) does not match checksum specified by the manifest (\(indexFile.checksum ))")
+                            }
+                            let metadata = try jsonDecoder.decode(ArchiveIndexFile.self, from: body)
+                            // FIXME: this filter needs to become more sophisticated
+                            guard let supportedArchive = metadata.archives.first(where: { $0.fileName.lowercased().hasSuffix(".zip") && $0.supportedTriples.contains(hostToolchain.triple) }) else {
+                                throw StringError("No supported archive was found for '\(hostToolchain.triple.tripleString)'")
+                            }
+                            // add relevant archive
+                            zipArtifacts.append(
+                                RemoteArtifact(
+                                    packageRef: indexFile.packageRef,
+                                    targetName: indexFile.targetName,
+                                    url: indexFile.url.deletingLastPathComponent().appendingPathComponent(supportedArchive.fileName),
+                                    checksum: supportedArchive.checksum)
+                            )
+                        }
+                    } catch {
+                        tempDiagnostics.emit(.error("failed retrieving '\(indexFile.url)': \(error)"))
+                    }
+                }
+            }
 
-            guard let parsedURL = URL(string: artifact.url) else {
-                throw StringError("invalid url \(artifact.url)")
+            // wait for all "ari" files to be processed
+            group.wait()
+
+            // no reason to continue if we already ran into issues
+            if tempDiagnostics.hasErrors {
+                // collect all diagnostics
+                diagnostics.append(contentsOf: tempDiagnostics)
+                throw Diagnostics.fatalError
             }
+        }
+
+        // finally download zip files, if any
+        for artifact in (zipArtifacts.map{ $0 }) {
+            group.enter()
+            defer { group.leave() }
 
             let parentDirectory =  self.artifactsPath.appending(component: artifact.packageRef.name)
 
@@ -1593,20 +1648,17 @@ extension Workspace {
                 continue
             }
 
-            let archivePath = parentDirectory.appending(component: parsedURL.lastPathComponent)
-
-            didDownloadAnyArtifact = true
+            let archivePath = parentDirectory.appending(component: artifact.url.lastPathComponent)
 
             group.enter()
-
-            var request = HTTPClient.Request.download(url: parsedURL, fileSystem: self.fileSystem, destination: archivePath)
+            var request = HTTPClient.Request.download(url: artifact.url, fileSystem: self.fileSystem, destination: archivePath)
             request.options.authorizationProvider = authProvider?.authorization(for:)
             request.options.validResponseCodes = [200]
             self.httpClient.execute(
                 request,
                 progress: { bytesDownloaded, totalBytesToDownload in
                     self.delegate?.downloadingBinaryArtifact(
-                        from: artifact.url,
+                        from: artifact.url.absoluteString,
                         bytesDownloaded: bytesDownloaded,
                         totalBytesToDownload: totalBytesToDownload)
                 },
@@ -1642,33 +1694,32 @@ extension Workspace {
                                     .remote(
                                         packageRef: artifact.packageRef,
                                         targetName: artifact.targetName,
-                                        url: artifact.url,
+                                        url: artifact.url.absoluteString,
                                         checksum: artifact.checksum,
                                         path: artifactPath
                                     )
                                 )
                             case .failure(let error):
                                 let reason = (error as? LocalizedError)?.errorDescription ?? error.localizedDescription
-                                tempDiagnostics.emit(.artifactFailedExtraction(targetName: artifact.targetName, reason: reason))
+                                tempDiagnostics.emit(.artifactFailedExtraction(artifactURL: artifact.url, targetName: artifact.targetName, reason: reason))
                             }
 
                             tempDiagnostics.wrap { try self.fileSystem.removeFileTree(archivePath) }
                         })
                     case .failure(let error):
-                        tempDiagnostics.emit(.artifactFailedDownload(targetName: artifact.targetName, reason: "\(error)"))
+                        tempDiagnostics.emit(.artifactFailedDownload(artifactURL: artifact.url, targetName: artifact.targetName, reason: "\(error)"))
                     }
                 })
         }
 
         group.wait()
 
-        if didDownloadAnyArtifact {
+        if zipArtifacts.count > 0 {
             delegate?.didDownloadBinaryArtifacts()
         }
 
-        for diagnostic in tempDiagnostics.diagnostics {
-            diagnostics.emit(diagnostic.message, location: diagnostic.location)
-        }
+        // collect all diagnostics
+        diagnostics.append(contentsOf: tempDiagnostics)
 
         return result.map{ $0 }
     }
@@ -2565,10 +2616,34 @@ public final class LoadableResult<Value> {
 private struct RemoteArtifact {
     let packageRef: PackageReference
     let targetName: String
-    let url: String
+    let url: Foundation.URL
     let checksum: String
 }
 
+private struct ArchiveIndexFile: Decodable {
+    let schemaVersion: String
+    let archives: [Archive]
+
+    struct Archive: Decodable {
+        let fileName: String
+        let checksum: String
+        let supportedTriples: [Triple]
+
+        enum CodingKeys: String, CodingKey {
+            case fileName
+            case checksum
+            case supportedTriples
+        }
+
+        public init(from decoder: Decoder) throws {
+            let container = try decoder.container(keyedBy: CodingKeys.self)
+            self.fileName = try container.decode(String.self, forKey: .fileName)
+            self.checksum = try container.decode(String.self, forKey: .checksum)
+            self.supportedTriples = try container.decode([String].self, forKey: .supportedTriples).map(Triple.init)
+        }
+    }
+}
+
 private extension ManagedArtifact {
     var originURL: String? {
         switch self.source {
@@ -2596,3 +2671,11 @@ private extension PackageDependencyDescription {
         }
     }
 }
+
+private extension DiagnosticsEngine {
+    func append(contentsOf other: DiagnosticsEngine) {
+        for diagnostic in other.diagnostics {
+            self.emit(diagnostic.message, location: diagnostic.location)
+        }
+    }
+}