Add package registry implementation

Package.swift

@@ -43,6 +43,7 @@ let package = Package(
                 "PackageModel",
                 "PackageLoading",
                 "PackageGraph",
+                "PackageRegistry",
                 "Build",
                 "Xcodeproj",
                 "Workspace"
@@ -58,6 +59,7 @@ let package = Package(
                 "PackageModel",
                 "PackageLoading",
                 "PackageGraph",
+                "PackageRegistry",
                 "Build",
                 "Xcodeproj",
                 "Workspace"
@@ -111,10 +113,16 @@ let package = Package(
             name: "LLBuildManifest",
             dependencies: ["SwiftToolsSupport-auto", "Basics"]),
 
+        .target(
+            /** Package registry support */
+            name: "PackageRegistry",
+            dependencies: ["SwiftToolsSupport-auto", "Basics", "PackageLoading", "PackageModel"]),
+
         .target(
             /** Source control operations */
             name: "SourceControl",
             dependencies: ["SwiftToolsSupport-auto", "Basics"]),
+
         .target(
             /** Shim for llbuild library */
             name: "SPMLLBuild",
@@ -136,7 +144,7 @@ let package = Package(
         .target(
             /** Data structures and support for complete package graphs */
             name: "PackageGraph",
-            dependencies: ["SwiftToolsSupport-auto", "Basics", "PackageLoading", "PackageModel", "SourceControl"]),
+            dependencies: ["SwiftToolsSupport-auto", "Basics", "PackageLoading", "PackageModel", "PackageRegistry", "SourceControl"]),
 
         // MARK: Package Collections
 
@@ -252,6 +260,9 @@ let package = Package(
         .testTarget(
             name: "PackageCollectionsTests",
             dependencies: ["SPMTestSupport", "PackageCollections"]),
+        .testTarget(
+            name: "PackageRegistryTests",
+            dependencies: ["SPMTestSupport", "PackageRegistry"]),
         .testTarget(
             name: "SourceControlTests",
             dependencies: ["SourceControl", "SPMTestSupport"]),

Sources/Basics/ByteString+Extensions.swift

@@ -0,0 +1,27 @@
+/*
+ This source file is part of the Swift.org open source project
+ Copyright (c) 2020 Apple Inc. and the Swift project authors
+ Licensed under Apache License v2.0 with Runtime Library Exception
+ See http://swift.org/LICENSE.txt for license information
+ See http://swift.org/CONTRIBUTORS.txt for Swift project authors
+ */
+
+import TSCBasic
+
+extension ByteString {
+    /// A lowercase, hexadecimal representation of the SHA256 hash
+    /// generated for the byte string's contents.
+    ///
+    /// This property uses the CryptoKit implementation of
+    /// Secure Hashing Algorithm 2 (SHA-2) hashing with a 256-bit digest, when available,
+    /// falling back on a native implementation in Swift provided by TSCBasic.
+    public var sha256Checksum: String {
+        #if canImport(CryptoKit)
+        if #available(macOS 10.15, *) {
+            return CryptoKitSHA256().hash(self).hexadecimalRepresentation
+        }
+        #endif
+
+        return SHA256().hash(self).hexadecimalRepresentation
+    }
+}

Sources/Basics/CMakeLists.txt

@@ -7,6 +7,7 @@
 # See http://swift.org/CONTRIBUTORS.txt for Swift project authors
 
 add_library(Basics
+  ByteString+Extensions.swift
   ConcurrencyHelpers.swift
   Dictionary+Extensions.swift
   DispatchTimeInterval+Extensions.swift

Sources/Basics/HTPClient+URLSession.swift

@@ -8,28 +8,112 @@ import struct TSCUtility.Versioning
 import FoundationNetworking
 #endif
 
-public struct URLSessionHTTPClient: HTTPClientProtocol {
-    private let configuration: URLSessionConfiguration
+public final class URLSessionHTTPClient: NSObject {
+    public var configuration: HTTPClientConfiguration = .init()
 
-    public init(configuration: URLSessionConfiguration = .default) {
-        self.configuration = configuration
+    private var session: URLSession!
+    private var taskDelegates: [URLSessionTask: TaskDelegate] = [:]
+
+    final class TaskDelegate: NSObject {
+        private let configuration: HTTPClientConfiguration
+        private let callback: (Result<HTTPClient.Response, Error>) -> Void
+        private var accumulatedData: Data = Data()
+
+        required init(configuration: HTTPClientConfiguration, callback: @escaping (Result<HTTPClient.Response, Error>) -> Void) {
+            self.configuration = configuration
+            self.callback = callback
+        }
+    }
+
+    public required init(configuration: URLSessionConfiguration = .default) {
+        super.init()
+        self.session = URLSession(configuration: configuration, delegate: self, delegateQueue: nil)
     }
+}
 
+extension URLSessionHTTPClient: HTTPClientProtocol {
     public func execute(_ request: HTTPClient.Request, callback: @escaping (Result<HTTPClient.Response, Error>) -> Void) {
-        let session = URLSession(configuration: self.configuration)
-        let task = session.dataTask(with: request.urlRequest()) { data, response, error in
-            if let error = error {
-                callback(.failure(error))
-            } else if let response = response as? HTTPURLResponse {
-                callback(.success(response.response(body: data)))
-            } else {
-                callback(.failure(HTTPClientError.invalidResponse))
+        let task = session.dataTask(with: request.urlRequest())
+        taskDelegates[task] = TaskDelegate(configuration: configuration, callback: callback)
+
+        task.resume()
+    }
+}
+
+extension URLSessionHTTPClient: URLSessionDataDelegate {
+    public func urlSession(_ session: URLSession,
+                           task: URLSessionTask,
+                           willPerformHTTPRedirection response: HTTPURLResponse,
+                           newRequest request: URLRequest,
+                           completionHandler: @escaping (URLRequest?) -> Void)
+    {
+        guard let delegate = taskDelegates[task] else {
+            return completionHandler(request)
+        }
+
+        delegate.urlSession(session, task: task, willPerformHTTPRedirection: response, newRequest: request, completionHandler: completionHandler)
+    }
+
+    public func urlSession(_ session: URLSession,
+                           task: URLSessionTask,
+                           didCompleteWithError error: Error?)
+    {
+        guard let delegate = taskDelegates[task] else { return }
+        defer { taskDelegates[task] = nil }
+
+        delegate.urlSession(session, task: task, didCompleteWithError: error)
+    }
+
+    public func urlSession(_ session: URLSession,
+                           dataTask: URLSessionDataTask,
+                           didReceive data: Data)
+    {
+        guard let delegate = taskDelegates[dataTask] else { return }
+        delegate.urlSession(session, dataTask: dataTask, didReceive: data)
+    }
+}
+
+
+extension URLSessionHTTPClient.TaskDelegate: URLSessionDataDelegate {
+    public func urlSession(_ session: URLSession,
+                           task: URLSessionTask,
+                           willPerformHTTPRedirection response: HTTPURLResponse,
+                           newRequest request: URLRequest,
+                           completionHandler: @escaping (URLRequest?) -> Void)
+    {
+        completionHandler(configuration.followRedirects ? request : nil)
+    }
+
+    public func urlSession(_ session: URLSession,
+                           task: URLSessionTask,
+                           didCompleteWithError error: Error?)
+    {
+        if let error = error {
+            configuration.callbackQueue.async {
+                self.callback(.failure(error))
+            }
+        } else if let response = task.response as? HTTPURLResponse {
+            let body = accumulatedData.isEmpty ? nil : accumulatedData
+            configuration.callbackQueue.async {
+                self.callback(.success(response.response(body: body)))
+            }
+        } else {
+            configuration.callbackQueue.async {
+                self.callback(.failure(HTTPClientError.invalidResponse))
             }
         }
-        task.resume()
+    }
+
+    public func urlSession(_ session: URLSession,
+                           dataTask: URLSessionDataTask,
+                           didReceive data: Data)
+    {
+        accumulatedData.append(data)
     }
 }
 
+// MARK: -
+
 extension HTTPClient.Request {
     func urlRequest() -> URLRequest {
         var request = URLRequest(url: self.url)

Sources/Basics/HTTPClient.swift

@@ -53,8 +53,14 @@ public struct HTTPClient: HTTPClientProtocol {
     public init(configuration: HTTPClientConfiguration = .init(), handler: Handler? = nil, diagnosticsEngine: DiagnosticsEngine? = nil) {
         self.configuration = configuration
         self.diagnosticsEngine = diagnosticsEngine
-        // FIXME: inject platform specific implementation here
-        self.underlying = handler ?? URLSessionHTTPClient().execute
+        if let handler = handler {
+            self.underlying = handler
+        } else {
+            // FIXME: inject platform-specific implementation here
+            let client = URLSessionHTTPClient()
+            client.configuration = configuration
+            self.underlying = client.execute
+        }
     }
 
     public func execute(_ request: Request, callback: @escaping (Result<Response, Error>) -> Void) {
@@ -207,13 +213,15 @@ public struct HTTPClientConfiguration {
     public var requestTimeout: DispatchTimeInterval?
     public var retryStrategy: HTTPClientRetryStrategy?
     public var circuitBreakerStrategy: HTTPClientCircuitBreakerStrategy?
+    public var followRedirects: Bool
     public var callbackQueue: DispatchQueue
 
     public init() {
         self.requestHeaders = .none
         self.requestTimeout = .none
         self.retryStrategy = .none
         self.circuitBreakerStrategy = .none
+        self.followRedirects = true
         self.callbackQueue = .global()
     }
 }

Sources/CMakeLists.txt

@@ -15,6 +15,7 @@ add_subdirectory(PackageDescription)
 add_subdirectory(PackageGraph)
 add_subdirectory(PackageLoading)
 add_subdirectory(PackageModel)
+add_subdirectory(PackageRegistry)
 add_subdirectory(SPMBuildCore)
 add_subdirectory(SPMLLBuild)
 add_subdirectory(SourceControl)

Sources/Commands/Options.swift

@@ -244,6 +244,10 @@ public struct SwiftToolOptions: ParsableArguments {
     @Flag(name: .customLong("trace-resolver"))
     var enableResolverTrace: Bool = false
 
+    @Flag(name: .customLong("enable-package-registry"),
+          help: "Enable dependency resolution using package registries when available")
+    var enablePackageRegistry: Bool = false
+
     /// The number of jobs for llbuild to start (aka the number of schedulerLanes)
     @Option(name: .shortAndLong, help: "The number of jobs to spawn in parallel during the build process")
     var jobs: UInt32?

Sources/Commands/SwiftTool.swift

@@ -156,6 +156,8 @@ private class ToolWorkspaceDelegate: WorkspaceDelegate {
                     case .unversioned:
                         self.stdoutStream <<< "unversioned"
                     }
+                case .downloaded(let version):
+                    self.stdoutStream <<< "resolved to '\(version)'"
                 case .edited?:
                     self.stdoutStream <<< "edited"
                 case .local?:
@@ -228,7 +230,7 @@ private final class DiagnosticsEngineHandler {
 
 protocol SwiftCommand: ParsableCommand {
     var swiftOptions: SwiftToolOptions { get }
-  
+
     func run(_ swiftTool: SwiftTool) throws
 }
 
@@ -312,7 +314,7 @@ public class SwiftTool {
         do {
             try Self.postprocessArgParserResult(options: options, diagnostics: diagnostics)
             self.options = options
-            
+
             // Honor package-path option is provided.
             if let packagePath = options.packagePath ?? options.chdir {
                 try ProcessEnv.chdir(packagePath)
@@ -370,29 +372,33 @@ public class SwiftTool {
         self.buildPath = getEnvBuildPath(workingDir: cwd) ??
             customBuildPath ??
             (packageRoot ?? cwd).appending(component: ".build")
-
+
+        if options.enablePackageRegistry {
+            PackageModel._useLegacyIdentities = false
+        }
+
         // Setup the globals.
         verbosity = Verbosity(rawValue: options.verbosity)
         Process.verbose = verbosity != .concise
     }
-    
+
     static func postprocessArgParserResult(options: SwiftToolOptions, diagnostics: DiagnosticsEngine) throws {
         if options.chdir != nil {
             diagnostics.emit(warning: "'--chdir/-C' option is deprecated; use '--package-path' instead")
         }
-        
+
         if options.multirootPackageDataFile != nil {
             diagnostics.emit(.unsupportedFlag("--multiroot-data-file"))
         }
-        
+
         if options.useExplicitModuleBuild && !options.useIntegratedSwiftDriver {
             diagnostics.emit(error: "'--experimental-explicit-module-build' option requires '--use-integrated-swift-driver'")
         }
-        
+
         if !options.archs.isEmpty && options.customCompileTriple != nil {
             diagnostics.emit(.mutuallyExclusiveArgumentsError(arguments: ["--arch", "--triple"]))
         }
-        
+
         if options.netrcFilePath != nil {
             // --netrc-file option only supported on macOS >=10.13
             #if os(macOS)
@@ -405,7 +411,7 @@ public class SwiftTool {
             diagnostics.emit(error: "'--netrc-file' option is only supported on macOS >=10.13")
             #endif
         }
-        
+
         if options.enableTestDiscovery {
             diagnostics.emit(warning: "'--enable-test-discovery' option is deprecated; tests are automatically discovered on all platforms")
         }
@@ -444,12 +450,12 @@ public class SwiftTool {
     private lazy var _swiftpmConfig: Result<Workspace.Configuration, Swift.Error> = {
         return Result(catching: { try Workspace.Configuration(path: try configFilePath()) })
     }()
-    
+
     func resolvedNetrcFilePath() throws -> AbsolutePath? {
         guard options.netrc ||
                 options.netrcFilePath != nil ||
                 options.netrcOptional else { return nil }
-        
+
         let resolvedPath: AbsolutePath = options.netrcFilePath ?? AbsolutePath("\(NSHomeDirectory())/.netrc")
         guard localFileSystem.exists(resolvedPath) else {
             if !options.netrcOptional {
@@ -493,7 +499,8 @@ public class SwiftTool {
             isResolverPrefetchingEnabled: options.shouldEnableResolverPrefetching,
             skipUpdate: options.skipDependencyUpdate,
             enableResolverTrace: options.enableResolverTrace,
-            cachePath: cachePath
+            cachePath: cachePath,
+            enablePackageRegistry: options.enablePackageRegistry
         )
         _workspace = workspace
         return workspace

Sources/PackageGraph/CMakeLists.txt

@@ -9,6 +9,7 @@
 add_library(PackageGraph
   BoundVersion.swift
   CheckoutState.swift
+  CompositePackageContainerProvider.swift
   DependencyMirrors.swift
   DependencyResolutionNode.swift
   DependencyResolver.swift
@@ -27,6 +28,7 @@ add_library(PackageGraph
   Pubgrub/PartialSolution.swift
   Pubgrub/PubgrubDependencyResolver.swift
   Pubgrub/Term.swift
+  RegistryPackageContainer.swift
   RepositoryPackageContainer.swift
   ResolvedPackage.swift
   ResolvedProduct.swift