Initial implementation of RegistryPackageContainer

Add PackageRegistry module target

Add PackageRegistryTests module target

Add --enable-package-registry command-line option

Adjust assertion for LocalPackageContainerProvider to accommodate root identifiers

Fix package registry manager implementation to match specification

Author: mattt

Package.swift

@@ -43,6 +43,7 @@ let package = Package(
                 "PackageModel",
                 "PackageLoading",
                 "PackageGraph",
+                "PackageRegistry",
                 "Build",
                 "Xcodeproj",
                 "Workspace"
@@ -58,6 +59,7 @@ let package = Package(
                 "PackageModel",
                 "PackageLoading",
                 "PackageGraph",
+                "PackageRegistry",
                 "Build",
                 "Xcodeproj",
                 "Workspace"
@@ -111,10 +113,16 @@ let package = Package(
             name: "LLBuildManifest",
             dependencies: ["SwiftToolsSupport-auto", "Basics"]),
 
+        .target(
+            /** Package registry support */
+            name: "PackageRegistry",
+            dependencies: ["SwiftToolsSupport-auto", "Basics", "PackageLoading", "PackageModel"]),
+
         .target(
             /** Source control operations */
             name: "SourceControl",
             dependencies: ["SwiftToolsSupport-auto", "Basics"]),
+
         .target(
             /** Shim for llbuild library */
             name: "SPMLLBuild",
@@ -136,7 +144,7 @@ let package = Package(
         .target(
             /** Data structures and support for complete package graphs */
             name: "PackageGraph",
-            dependencies: ["SwiftToolsSupport-auto", "Basics", "PackageLoading", "PackageModel", "SourceControl"]),
+            dependencies: ["SwiftToolsSupport-auto", "Basics", "PackageLoading", "PackageModel", "PackageRegistry", "SourceControl"]),
 
         // MARK: Package Collections
 
@@ -248,6 +256,9 @@ let package = Package(
         .testTarget(
             name: "PackageCollectionsTests",
             dependencies: ["SPMTestSupport", "PackageCollections"]),
+        .testTarget(
+            name: "PackageRegistryTests",
+            dependencies: ["SPMTestSupport", "PackageRegistry"]),
         .testTarget(
             name: "SourceControlTests",
             dependencies: ["SourceControl", "SPMTestSupport"]),

Sources/CMakeLists.txt

@@ -15,6 +15,7 @@ add_subdirectory(PackageDescription)
 add_subdirectory(PackageGraph)
 add_subdirectory(PackageLoading)
 add_subdirectory(PackageModel)
+add_subdirectory(PackageRegistry)
 add_subdirectory(SPMBuildCore)
 add_subdirectory(SPMLLBuild)
 add_subdirectory(SourceControl)

Sources/Commands/Options.swift

@@ -244,6 +244,10 @@ public struct SwiftToolOptions: ParsableArguments {
     @Flag(name: .customLong("trace-resolver"))
     var enableResolverTrace: Bool = false
 
+    @Flag(name: .customLong("enable-package-registry"),
+          help: "Enable dependency resolution using package registries when available")
+    var enablePackageRegistry: Bool = false
+
     /// The number of jobs for llbuild to start (aka the number of schedulerLanes)
     @Option(name: .shortAndLong, help: "The number of jobs to spawn in parallel during the build process")
     var jobs: UInt32?

Sources/Commands/SwiftTool.swift

@@ -228,7 +228,7 @@ private final class DiagnosticsEngineHandler {
 
 protocol SwiftCommand: ParsableCommand {
     var swiftOptions: SwiftToolOptions { get }
-  
+
     func run(_ swiftTool: SwiftTool) throws
 }
 
@@ -312,7 +312,7 @@ public class SwiftTool {
         do {
             try Self.postprocessArgParserResult(options: options, diagnostics: diagnostics)
             self.options = options
-            
+
             // Honor package-path option is provided.
             if let packagePath = options.packagePath ?? options.chdir {
                 try ProcessEnv.chdir(packagePath)
@@ -370,29 +370,29 @@ public class SwiftTool {
         self.buildPath = getEnvBuildPath(workingDir: cwd) ??
             customBuildPath ??
             (packageRoot ?? cwd).appending(component: ".build")
-        
+
         // Setup the globals.
         verbosity = Verbosity(rawValue: options.verbosity)
         Process.verbose = verbosity != .concise
     }
-    
+
     static func postprocessArgParserResult(options: SwiftToolOptions, diagnostics: DiagnosticsEngine) throws {
         if options.chdir != nil {
             diagnostics.emit(warning: "'--chdir/-C' option is deprecated; use '--package-path' instead")
         }
-        
+
         if options.multirootPackageDataFile != nil {
             diagnostics.emit(.unsupportedFlag("--multiroot-data-file"))
         }
-        
+
         if options.useExplicitModuleBuild && !options.useIntegratedSwiftDriver {
             diagnostics.emit(error: "'--experimental-explicit-module-build' option requires '--use-integrated-swift-driver'")
         }
-        
+
         if !options.archs.isEmpty && options.customCompileTriple != nil {
             diagnostics.emit(.mutuallyExclusiveArgumentsError(arguments: ["--arch", "--triple"]))
         }
-        
+
         if options.netrcFilePath != nil {
             // --netrc-file option only supported on macOS >=10.13
             #if os(macOS)
@@ -405,7 +405,7 @@ public class SwiftTool {
             diagnostics.emit(error: "'--netrc-file' option is only supported on macOS >=10.13")
             #endif
         }
-        
+
         if options.enableTestDiscovery {
             diagnostics.emit(warning: "'--enable-test-discovery' option is deprecated; tests are automatically discovered on all platforms")
         }
@@ -444,12 +444,12 @@ public class SwiftTool {
     private lazy var _swiftpmConfig: Result<Workspace.Configuration, Swift.Error> = {
         return Result(catching: { try Workspace.Configuration(path: try configFilePath()) })
     }()
-    
+
     func resolvedNetrcFilePath() throws -> AbsolutePath? {
         guard options.netrc ||
                 options.netrcFilePath != nil ||
                 options.netrcOptional else { return nil }
-        
+
         let resolvedPath: AbsolutePath = options.netrcFilePath ?? AbsolutePath("\(NSHomeDirectory())/.netrc")
         guard localFileSystem.exists(resolvedPath) else {
             if !options.netrcOptional {
@@ -493,7 +493,8 @@ public class SwiftTool {
             isResolverPrefetchingEnabled: options.shouldEnableResolverPrefetching,
             skipUpdate: options.skipDependencyUpdate,
             enableResolverTrace: options.enableResolverTrace,
-            cachePath: cachePath
+            cachePath: cachePath,
+            enablePackageRegistry: options.enablePackageRegistry
         )
         _workspace = workspace
         return workspace

Sources/PackageGraph/CMakeLists.txt

@@ -9,6 +9,7 @@
 add_library(PackageGraph
   BoundVersion.swift
   CheckoutState.swift
+  CompositePackageContainerProvider.swift
   DependencyMirrors.swift
   DependencyResolutionNode.swift
   DependencyResolver.swift
@@ -27,6 +28,7 @@ add_library(PackageGraph
   Pubgrub/PartialSolution.swift
   Pubgrub/PubgrubDependencyResolver.swift
   Pubgrub/Term.swift
+  RegistryPackageContainer.swift
   RepositoryPackageContainer.swift
   ResolvedPackage.swift
   ResolvedProduct.swift

Sources/PackageGraph/CompositePackageContainerProvider.swift

@@ -0,0 +1,85 @@
+/*
+ This source file is part of the Swift.org open source project
+
+ Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
+ Licensed under Apache License v2.0 with Runtime Library Exception
+
+ See http://swift.org/LICENSE.txt for license information
+ See http://swift.org/CONTRIBUTORS.txt for Swift project authors
+ */
+
+import Dispatch
+
+import PackageLoading
+import PackageModel
+import PackageRegistry
+import SourceControl
+
+import TSCBasic
+import TSCUtility
+
+public class CompositePackageContainerProvider: PackageContainerProvider {
+    let manifestLoader: ManifestLoaderProtocol
+    let repositoryManager: RepositoryManager
+    let mirrors: DependencyMirrors
+
+    /// The tools version currently in use. Only the container versions less than and equal to this will be provided by
+    /// the container.
+    let currentToolsVersion: ToolsVersion
+
+    /// The tools version loader.
+    let toolsVersionLoader: ToolsVersionLoaderProtocol
+
+    let fileSystem: FileSystem
+
+    /// Create a repository-based package provider.
+    ///
+    /// - Parameters:
+    ///   - repositoryManager: The repository manager responsible for providing repositories.
+    ///   - manifestLoader: The manifest loader instance.
+    ///   - currentToolsVersion: The current tools version in use.
+    ///   - toolsVersionLoader: The tools version loader.
+    public init(
+        repositoryManager: RepositoryManager,
+        mirrors: DependencyMirrors = DependencyMirrors(),
+        manifestLoader: ManifestLoaderProtocol,
+        currentToolsVersion: ToolsVersion = ToolsVersion.currentToolsVersion,
+        toolsVersionLoader: ToolsVersionLoaderProtocol = ToolsVersionLoader(),
+        fileSystem: FileSystem = localFileSystem
+    ) {
+        self.repositoryManager = repositoryManager
+        self.mirrors = mirrors
+        self.manifestLoader = manifestLoader
+        self.currentToolsVersion = currentToolsVersion
+        self.toolsVersionLoader = toolsVersionLoader
+        self.fileSystem = fileSystem
+    }
+
+    public func getContainer(
+        for reference: PackageReference,
+        skipUpdate: Bool,
+        on queue: DispatchQueue,
+        completion: @escaping (Result<PackageContainer, Swift.Error>) -> Void
+    ) {
+        let result: Result<PackageContainer, Swift.Error>
+
+        // For remote package references, attempt to load from registry before falling back to respository access.
+        if reference.kind == .remote {
+            do {
+                let provider = RegistryPackageContainerProvider(mirrors: mirrors, manifestLoader: manifestLoader, currentToolsVersion: currentToolsVersion, toolsVersionLoader: toolsVersionLoader)
+                let container = try tsc_await { provider.getContainer(for: reference, skipUpdate: skipUpdate, on: queue, completion: $0) }
+                result = .success(container)
+            } catch {
+                let provider = RepositoryPackageContainerProvider(repositoryManager: self.repositoryManager, mirrors: self.mirrors, manifestLoader: self.manifestLoader, currentToolsVersion: self.currentToolsVersion, toolsVersionLoader: self.toolsVersionLoader)
+                result = tsc_await { provider.getContainer(for: reference, skipUpdate: skipUpdate, on: queue, completion: $0) }
+            }
+        } else {
+            let provider = LocalPackageContainerProvider(mirrors: self.mirrors, manifestLoader: self.manifestLoader, currentToolsVersion: self.currentToolsVersion, toolsVersionLoader: self.toolsVersionLoader, fileSystem: self.fileSystem)
+            result = tsc_await { provider.getContainer(for: reference, skipUpdate: skipUpdate, on: queue, completion: $0) }
+        }
+
+        queue.async {
+            completion(result)
+        }
+    }
+}

Sources/PackageGraph/LocalPackageContainer.swift

@@ -6,14 +6,14 @@
 
  See http://swift.org/LICENSE.txt for license information
  See http://swift.org/CONTRIBUTORS.txt for Swift project authors
-*/
+ */
 
 import Dispatch
 
-import TSCBasic
 import PackageLoading
 import PackageModel
 import SourceControl
+import TSCBasic
 import TSCUtility
 
 /// Local package container.
@@ -111,3 +111,53 @@ extension LocalPackageContainer: CustomStringConvertible  {
         return "LocalPackageContainer(\(identifier.path))"
     }
 }
+
+public class LocalPackageContainerProvider: PackageContainerProvider {
+    let manifestLoader: ManifestLoaderProtocol
+    let mirrors: DependencyMirrors
+
+    /// The tools version currently in use. Only the container versions less than and equal to this will be provided by
+    /// the container.
+    let currentToolsVersion: ToolsVersion
+
+    /// The tools version loader.
+    let toolsVersionLoader: ToolsVersionLoaderProtocol
+
+    let fileSystem: FileSystem
+
+    public init(
+        mirrors: DependencyMirrors = DependencyMirrors(),
+        manifestLoader: ManifestLoaderProtocol,
+        currentToolsVersion: ToolsVersion = ToolsVersion.currentToolsVersion,
+        toolsVersionLoader: ToolsVersionLoaderProtocol = ToolsVersionLoader(),
+        fileSystem: FileSystem = localFileSystem
+    ) {
+        self.mirrors = mirrors
+        self.manifestLoader = manifestLoader
+        self.currentToolsVersion = currentToolsVersion
+        self.toolsVersionLoader = toolsVersionLoader
+        self.fileSystem = fileSystem
+    }
+
+    public func getContainer(
+        for identifier: PackageReference,
+        skipUpdate: Bool,
+        on queue: DispatchQueue,
+        completion: @escaping (Result<PackageContainer, Swift.Error>) -> Void)
+    {
+        assert(identifier.kind != .remote)
+        
+        let container = LocalPackageContainer(
+            identifier,
+            mirrors: self.mirrors,
+            manifestLoader: self.manifestLoader,
+            toolsVersionLoader: self.toolsVersionLoader,
+            currentToolsVersion: self.currentToolsVersion,
+            fs: self.fileSystem
+        )
+
+        queue.async {
+            completion(.success(container))
+        }
+    }
+}

Sources/PackageGraph/PackageContainer.swift

@@ -12,6 +12,7 @@ import Dispatch
 import PackageLoading
 import PackageModel
 import SourceControl
+import Dispatch
 import struct TSCUtility.Version
 
 /// A container of packages.