[SR-12851] System-wide cache of SwiftPM dependencies (#2985)

* implemented caching

* fixed packages with a path ending in “/”

* fetching a repo creates an alias for the fetched repository under a new RepositorySpecifier

* added purge-cache and --cache-path options

* temporarily fix tests until FileSystem has locks

* added documentation and some refactoring

* remove unnecessary extension

* remove unnecessary import

* use file lock provided by FileSystem

* added -skip-cache flag

* using copy instead of fetch

* using repository provider instead of repository manager to fetch the cached repo

* added fetchDetail to delegate methods

* remove func setURL(remote: String, url: String) throws

* fixed unit tests by making RepositoryProvider aware of copy

* fix newlines

* fix lock

* extract function initalizeCacheIfNeeded

* update comments

* remove duplicate code

* remove newlines

* re-add old comments and move repositoryPath back down

* move isCache to the top

* converted FetchDetails into a struct

* use FileSystem.swiftPMCacheDirectory

* add symlink to cache directory in .swiftpm

* add some comments

* create .swiftpm directory

* don't cache local packages

* added default implementations

* automatically infer update parameter for fetch

* deprecated old delegate methods

* add Basics to SourceControl

* use a more reliable way to determine if a git repository exists

* open repository before fetching

* added tests for repository manager

* rename cacheLocalPackages

Author: tgymnich

Sources/Commands/Options.swift

@@ -136,6 +136,12 @@ public struct SwiftToolOptions: ParsableArguments {
     @Option(help: "Specify build/cache directory")
     var buildPath: AbsolutePath?
 
+    @Option(help: "Specify the global repository cache directory")
+    var cachePath: AbsolutePath?
+
+    @Flag(help: "Skip the cache when fetching")
+    var skipCache: Bool = false
+
     /// The custom working directory that the tool should operate in (deprecated).
     @Option(name: [.long, .customShort("C")])
     var chdir: AbsolutePath?

Sources/Commands/SwiftPackageTool.swift

@@ -33,6 +33,7 @@ public struct SwiftPackageTool: ParsableCommand {
         version: Versioning.currentVersion.completeDisplayString,
         subcommands: [
             Clean.self,
+            PurgeCache.self,
             Reset.self,
             Update.self,
             Describe.self,
@@ -83,6 +84,18 @@ extension SwiftPackageTool {
             try swiftTool.getActiveWorkspace().clean(with: swiftTool.diagnostics)
         }
     }
+
+    struct PurgeCache: SwiftCommand {
+        static let configuration = CommandConfiguration(
+            abstract: "Purge the global repository cache.")
+
+        @OptionGroup()
+        var swiftOptions: SwiftToolOptions
+
+        func run(_ swiftTool: SwiftTool) throws {
+            try swiftTool.getActiveWorkspace().purgeCache(with: swiftTool.diagnostics)
+        }
+    }
 
     struct Reset: SwiftCommand {
         static let configuration = CommandConfiguration(

Sources/Commands/SwiftTool.swift

@@ -25,6 +25,7 @@ import SPMBuildCore
 import Build
 import XCBuildSupport
 import Workspace
+import Basics
 
 typealias Diagnostic = TSCBasic.Diagnostic
 
@@ -59,13 +60,18 @@ private class ToolWorkspaceDelegate: WorkspaceDelegate {
         self.diagnostics = diagnostics
     }
 
-    func fetchingWillBegin(repository: String) {
+    func fetchingWillBegin(repository: String, fetchDetails: RepositoryManager.FetchDetails?) {
         stdoutStream <<< "Fetching \(repository)"
+        if let fetchDetails = fetchDetails {
+            if fetchDetails.fromCache {
+                stdoutStream <<< " from cache"
+            }
+        }
         stdoutStream <<< "\n"
         stdoutStream.flush()
     }
 
-    func fetchingDidFinish(repository: String, diagnostic: Diagnostic?) {
+    func fetchingDidFinish(repository: String, fetchDetails: RepositoryManager.FetchDetails?, diagnostic: Diagnostic?) {
     }
 
     func repositoryWillUpdate(_ repository: String) {
@@ -456,6 +462,8 @@ public class SwiftTool {
         let isVerbose = options.verbosity != 0
         let delegate = ToolWorkspaceDelegate(self.stdoutStream, isVerbose: isVerbose, diagnostics: diagnostics)
         let provider = GitRepositoryProvider(processSet: processSet)
+        let defaultCachePath = localFileSystem.swiftPMCacheDirectory.appending(component: "repositories")
+        let cachePath = !options.skipCache ? options.cachePath ?? defaultCachePath : nil
         let workspace = Workspace(
             dataPath: buildPath,
             editablesPath: try editablesPath(),
@@ -468,7 +476,8 @@ public class SwiftTool {
             netrcFilePath: try resolvedNetrcFilePath(),
             isResolverPrefetchingEnabled: options.shouldEnableResolverPrefetching,
             skipUpdate: options.skipDependencyUpdate,
-            enableResolverTrace: options.enableResolverTrace
+            enableResolverTrace: options.enableResolverTrace,
+            cachePath: cachePath
         )
         _workspace = workspace
         return workspace

Sources/SPMTestSupport/InMemoryGitRepository.swift

@@ -335,7 +335,14 @@ public final class InMemoryGitRepositoryProvider: RepositoryProvider {
     // Note: These methods use force unwrap (instead of throwing) to honor their preconditions.
 
     public func fetch(repository: RepositorySpecifier, to path: AbsolutePath) throws {
-        fetchedMap[path] = specifierMap[RepositorySpecifier(url: repository.url.spm_dropGitSuffix())]!.copy()
+        let repo = specifierMap[RepositorySpecifier(url: repository.url.spm_dropGitSuffix())]!
+        fetchedMap[path] = repo.copy()
+        add(specifier: RepositorySpecifier(url: path.asURL.absoluteString), repository: repo)
+    }
+
+    public func copy(from sourcePath: AbsolutePath, to destinationPath: AbsolutePath) throws {
+        let repo = fetchedMap[sourcePath]!
+        fetchedMap[destinationPath] = repo.copy()
     }
 
     public func open(repository: RepositorySpecifier, at path: AbsolutePath) throws -> Repository {

Sources/SPMTestSupport/MockWorkspace.swift

@@ -15,6 +15,7 @@ import PackageModel
 import SourceControl
 import TSCBasic
 import Workspace
+import Basics
 
 public final class MockWorkspace {
     let sandbox: AbsolutePath
@@ -171,7 +172,8 @@ public final class MockWorkspace {
             checksumAlgorithm: self.checksumAlgorithm,
             isResolverPrefetchingEnabled: true,
             enablePubgrubResolver: self.enablePubGrub,
-            skipUpdate: self.skipUpdate
+            skipUpdate: self.skipUpdate,
+            cachePath: localFileSystem.swiftPMCacheDirectory.appending(component: "repositories")
         )
         return self._workspace!
     }
@@ -552,11 +554,11 @@ public final class MockWorkspaceDelegate: WorkspaceDelegate {
         self.events.append("Everything is already up-to-date")
     }
 
-    public func fetchingWillBegin(repository: String) {
+    public func fetchingWillBegin(repository: String, fetchDetails: RepositoryManager.FetchDetails?) {
         self.events.append("fetching repo: \(repository)")
     }
 
-    public func fetchingDidFinish(repository: String, diagnostic: Diagnostic?) {
+    public func fetchingDidFinish(repository: String, fetchDetails: RepositoryManager.FetchDetails?, diagnostic: Diagnostic?) {
         self.events.append("finished fetching repo: \(repository)")
     }
 

Sources/SourceControl/CMakeLists.txt

@@ -12,6 +12,7 @@ add_library(SourceControl
   RepositoryManager.swift)
 
 target_link_libraries(SourceControl PUBLIC
+  Basics
   TSCBasic
   TSCUtility
   Basics)

Sources/SourceControl/GitRepository.swift

@@ -52,12 +52,14 @@ public class GitRepositoryProvider: RepositoryProvider {
         // NOTE: We intentionally do not create a shallow clone here; the
         // expected cost of iterative updates on a full clone is less than on a
         // shallow clone.
-
         precondition(!localFileSystem.exists(path))
-
         // FIXME: Ideally we should pass `--progress` here and report status regularly.  We currently don't have callbacks for that.
         try callGit("clone", "--mirror", repository.url, path.pathString,
-            failureMessage: "Failed to clone repository \(repository.url)", repository: repository)
+                    failureMessage: "Failed to clone repository \(repository.url)", repository: repository)
+    }
+
+    public func copy(from sourcePath: AbsolutePath, to destinationPath: AbsolutePath) throws {
+        try localFileSystem.copy(from: sourcePath, to: destinationPath)
     }
 
     public func open(repository: RepositorySpecifier, at path: AbsolutePath) -> Repository {

Sources/SourceControl/Repository.swift

@@ -34,7 +34,7 @@ public struct RepositorySpecifier: Hashable, Codable {
 
     /// Returns the cleaned basename for the specifier.
     public var basename: String {
-        var basename = url.components(separatedBy: "/").last!
+        var basename = url.components(separatedBy: "/").last(where: { !$0.isEmpty }) ?? ""
         if basename.hasSuffix(".git") {
             basename = String(basename.dropLast(4))
         }
@@ -71,7 +71,7 @@ public protocol RepositoryProvider {
     ///
     /// - Parameters:
     ///   - repository: The specifier of the repository to fetch.
-    ///
+    ///   - path: The destiantion path for the fetch.
     /// - Throws: If there is any error fetching the repository.
     func fetch(repository: RepositorySpecifier, to path: AbsolutePath) throws
 
@@ -118,12 +118,23 @@ public protocol RepositoryProvider {
     ///   - path: The location of the repository on disk, at which the repository
     ///     has previously been created via `cloneCheckout`.
     func openCheckout(at path: AbsolutePath) throws -> WorkingCheckout
+
+    /// Copies the repository at path `from` to path `to`.
+    /// - Parameters:
+    ///   - sourcePath: the source path.
+    ///   - destinationPath: the destination  path.
+    func copy(from sourcePath: AbsolutePath, to destinationPath: AbsolutePath) throws
 }
 
 extension RepositoryProvider {
     public func checkoutExists(at path: AbsolutePath) throws -> Bool {
         fatalError("Unimplemented")
     }
+
+    public func copy(from sourcePath: AbsolutePath, to destinationPath: AbsolutePath) throws {
+        fatalError("Unimplemented")
+    }
+
 }
 
 /// Abstract repository operations.