[Collections] Use persistent cache for GitHub package metadata

Motivation:
Currently we use transient in-memory cache for storing GitHub package metadata.

Modifications:
- Add generic `SQLiteBackedCache` in Basics
- Change `ManifestLoader` to use `SQLiteBackedCache`
- Change `GitHubPackageMetadataProvider` to use `SQLiteBackedCache`
- Adjust tests

Author: yim-lee

Sources/Basics/SQLiteBackedCache.swift

@@ -0,0 +1,246 @@
+/*
+ This source file is part of the Swift.org open source project
+
+ Copyright (c) 2021 Apple Inc. and the Swift project authors
+ Licensed under Apache License v2.0 with Runtime Library Exception
+
+ See http://swift.org/LICENSE.txt for license information
+ See http://swift.org/CONTRIBUTORS.txt for Swift project authors
+ */
+
+import Foundation
+
+import TSCBasic
+import TSCUtility
+
+/// SQLite backed persistent cache.
+public final class SQLiteBackedCache<Value: Codable>: Closable {
+    public typealias Key = String
+
+    public let name: String
+    public let fileSystem: TSCBasic.FileSystem
+    public let location: SQLite.Location
+    public let configuration: SQLiteBackedCacheConfiguration
+
+    private var state = State.idle
+    private let stateLock = Lock()
+
+    private let diagnosticsEngine: DiagnosticsEngine?
+    private let jsonEncoder: JSONEncoder
+    private let jsonDecoder: JSONDecoder
+
+    public init(name: String, location: SQLite.Location, configuration: SQLiteBackedCacheConfiguration = .init(), diagnosticsEngine: DiagnosticsEngine? = nil) {
+        self.name = name
+        self.location = location
+        switch self.location {
+        case .path, .temporary:
+            self.fileSystem = localFileSystem
+        case .memory:
+            self.fileSystem = InMemoryFileSystem()
+        }
+        self.configuration = configuration
+        self.diagnosticsEngine = diagnosticsEngine
+        self.jsonEncoder = JSONEncoder.makeWithDefaults()
+        self.jsonDecoder = JSONDecoder.makeWithDefaults()
+    }
+
+    public convenience init(name: String, path: AbsolutePath, configuration: SQLiteBackedCacheConfiguration = .init(), diagnosticsEngine: DiagnosticsEngine? = nil) {
+        self.init(name: name, location: .path(path), configuration: configuration, diagnosticsEngine: diagnosticsEngine)
+    }
+
+    deinit {
+        // TODO: we could wrap the failure here with diagnostics if it wasn't optional throughout
+        try? self.withStateLock {
+            if case .connected(let db) = self.state {
+                assertionFailure("db should be closed")
+                try db.close()
+            }
+        }
+    }
+
+    public func close() throws {
+        try self.withStateLock {
+            if case .connected(let db) = self.state {
+                try db.close()
+            }
+            self.state = .disconnected
+        }
+    }
+
+    public func put(key: Key, value: Value, replace: Bool = false) throws {
+        do {
+            let query = "INSERT OR \(replace ? "REPLACE" : "IGNORE") INTO \(self.name) VALUES (?, ?);"
+            try self.executeStatement(query) { statement -> Void in
+                let data = try self.jsonEncoder.encode(value)
+                let bindings: [SQLite.SQLiteValue] = [
+                    .string(key),
+                    .blob(data),
+                ]
+                try statement.bind(bindings)
+                try statement.step()
+            }
+        } catch (let error as SQLite.Errors) where error == .databaseFull {
+            if !self.configuration.truncateWhenFull {
+                throw error
+            }
+            self.diagnosticsEngine?.emit(.warning("truncating \(self.name) cache database since it reached max size of \(self.configuration.maxSizeInBytes ?? 0) bytes"))
+            try self.executeStatement("DELETE FROM \(self.name);") { statement -> Void in
+                try statement.step()
+            }
+            try self.put(key: key, value: value, replace: replace)
+        } catch {
+            throw error
+        }
+    }
+
+    public func get(key: Key) throws -> Value? {
+        let query = "SELECT value FROM \(self.name) WHERE key = ? LIMIT 1;"
+        return try self.executeStatement(query) { statement -> Value? in
+            try statement.bind([.string(key)])
+            let data = try statement.step()?.blob(at: 0)
+            return try data.flatMap {
+                try self.jsonDecoder.decode(Value.self, from: $0)
+            }
+        }
+    }
+
+    public func remove(key: Key) throws {
+        let query = "DELETE FROM \(self.name) WHERE key = ? LIMIT 1;"
+        try self.executeStatement(query) { statement in
+            try statement.bind([.string(key)])
+            try statement.step()
+        }
+    }
+
+    private func executeStatement<T>(_ query: String, _ body: (SQLite.PreparedStatement) throws -> T) throws -> T {
+        try self.withDB { db in
+            let result: Result<T, Error>
+            let statement = try db.prepare(query: query)
+            do {
+                result = .success(try body(statement))
+            } catch {
+                result = .failure(error)
+            }
+            try statement.finalize()
+            switch result {
+            case .failure(let error):
+                throw error
+            case .success(let value):
+                return value
+            }
+        }
+    }
+
+    private func withDB<T>(_ body: (SQLite) throws -> T) throws -> T {
+        let createDB = { () throws -> SQLite in
+            let db = try SQLite(location: self.location, configuration: self.configuration.underlying)
+            try self.createSchemaIfNecessary(db: db)
+            return db
+        }
+
+        let db = try self.withStateLock { () -> SQLite in
+            let db: SQLite
+            switch (self.location, self.state) {
+            case (.path(let path), .connected(let database)):
+                if self.fileSystem.exists(path) {
+                    db = database
+                } else {
+                    try database.close()
+                    try self.fileSystem.createDirectory(path.parentDirectory, recursive: true)
+                    db = try createDB()
+                }
+            case (.path(let path), _):
+                if !self.fileSystem.exists(path) {
+                    try self.fileSystem.createDirectory(path.parentDirectory, recursive: true)
+                }
+                db = try createDB()
+            case (_, .connected(let database)):
+                db = database
+            case (_, _):
+                db = try createDB()
+            }
+            self.state = .connected(db)
+            return db
+        }
+
+        // FIXME: workaround linux sqlite concurrency issues causing CI failures
+        #if os(Linux)
+        return try self.withStateLock {
+            return try body(db)
+        }
+        #else
+        return try body(db)
+        #endif
+    }
+
+    private func createSchemaIfNecessary(db: SQLite) throws {
+        let table = """
+            CREATE TABLE IF NOT EXISTS \(self.name) (
+                key STRING PRIMARY KEY NOT NULL,
+                value BLOB NOT NULL
+            );
+        """
+
+        try db.exec(query: table)
+        try db.exec(query: "PRAGMA journal_mode=WAL;")
+    }
+
+    private func withStateLock<T>(_ body: () throws -> T) throws -> T {
+        switch self.location {
+        case .path(let path):
+            if !self.fileSystem.exists(path.parentDirectory) {
+                try self.fileSystem.createDirectory(path.parentDirectory)
+            }
+            return try self.fileSystem.withLock(on: path, type: .exclusive, body)
+        case .memory, .temporary:
+            return try self.stateLock.withLock(body)
+        }
+    }
+
+    private enum State {
+        case idle
+        case connected(SQLite)
+        case disconnected
+    }
+}
+
+public struct SQLiteBackedCacheConfiguration {
+    public var truncateWhenFull: Bool
+
+    fileprivate var underlying: SQLite.Configuration
+
+    public init() {
+        self.underlying = .init()
+        self.truncateWhenFull = true
+        self.maxSizeInMegabytes = 100
+        // see https://www.sqlite.org/c3ref/busy_timeout.html
+        self.busyTimeoutMilliseconds = 1000
+    }
+
+    public var maxSizeInMegabytes: Int? {
+        get {
+            self.underlying.maxSizeInMegabytes
+        }
+        set {
+            self.underlying.maxSizeInMegabytes = newValue
+        }
+    }
+
+    public var maxSizeInBytes: Int? {
+        get {
+            self.underlying.maxSizeInBytes
+        }
+        set {
+            self.underlying.maxSizeInBytes = newValue
+        }
+    }
+
+    public var busyTimeoutMilliseconds: Int32 {
+        get {
+            self.underlying.busyTimeoutMilliseconds
+        }
+        set {
+            self.underlying.busyTimeoutMilliseconds = newValue
+        }
+    }
+}

Sources/PackageCollections/PackageCollections.swift

@@ -65,6 +65,7 @@ public struct PackageCollections: PackageCollectionsProtocol {
         if self.storageContainer.owned {
             try self.storageContainer.storage.close()
         }
+        try self.metadataProvider.close()
     }
 
     // MARK: - Collections

Sources/PackageCollections/Providers/GitHubPackageMetadataProvider.swift

@@ -27,27 +27,37 @@ struct GitHubPackageMetadataProvider: PackageMetadataProvider {
     private let diagnosticsEngine: DiagnosticsEngine?
     private let decoder: JSONDecoder
 
-    private let cache: ThreadSafeKeyValueStore<PackageReference, (package: Model.PackageBasicMetadata, timestamp: DispatchTime)>?
+    private let cache: SQLiteBackedCache<CacheValue>?
 
     init(configuration: Configuration = .init(), httpClient: HTTPClient? = nil, diagnosticsEngine: DiagnosticsEngine? = nil) {
         self.configuration = configuration
         self.httpClient = httpClient ?? Self.makeDefaultHTTPClient(diagnosticsEngine: diagnosticsEngine)
         self.diagnosticsEngine = diagnosticsEngine
         self.decoder = JSONDecoder.makeWithDefaults()
-        self.cache = configuration.cacheTTLInSeconds > 0 ? .init() : nil
+        if configuration.cacheTTLInSeconds > 0 {
+            var cacheConfig = SQLiteBackedCacheConfiguration()
+            cacheConfig.maxSizeInMegabytes = configuration.cacheSizeInMegabytes
+            self.cache = SQLiteBackedCache<CacheValue>(name: "github_cache", path: configuration.cacheDir.appending(component: "package-metadata.db"), configuration: cacheConfig, diagnosticsEngine: diagnosticsEngine)
+        } else {
+            self.cache = nil
+        }
+    }
+
+    func close() throws {
+        try self.cache?.close()
     }
 
     func get(_ reference: PackageReference, callback: @escaping (Result<Model.PackageBasicMetadata, Error>) -> Void) {
         guard reference.kind == .remote else {
             return callback(.failure(Errors.invalidReferenceType(reference)))
         }
-        guard let baseURL = self.apiURL(reference.location) else {
+        guard let baseURL = Self.apiURL(reference.location) else {
             return callback(.failure(Errors.invalidGitURL(reference.location)))
         }
 
-        if let cachedMetadata = self.cache?[reference] {
-            if cachedMetadata.timestamp + DispatchTimeInterval.seconds(self.configuration.cacheTTLInSeconds) > DispatchTime.now() {
-                return callback(.success(cachedMetadata.package))
+        if let cached = try? self.cache?.get(key: reference.identity.description) {
+            if cached.dispatchTime + DispatchTimeInterval.seconds(self.configuration.cacheTTLInSeconds) > DispatchTime.now() {
+                return callback(.success(cached.package))
             }
         }
 
@@ -144,22 +154,12 @@ struct GitHubPackageMetadataProvider: PackageMetadataProvider {
                         processedAt: Date()
                     )
 
-                    if let cache = self.cache {
-                        cache[reference] = (model, DispatchTime.now())
-
-                        if cache.count > self.configuration.cacheSize {
-                            DispatchQueue.sharedConcurrent.async {
-                                // Delete oldest entries with some room for growth
-                                let sortedCacheEntries = cache.get().sorted { $0.value.timestamp < $1.value.timestamp }
-                                let deleteCount = sortedCacheEntries.count - (self.configuration.cacheSize / 2)
-                                self.diagnosticsEngine?.emit(note: "Cache size limit exceeded, deleting the oldest \(deleteCount) entries")
-
-                                for index in 0 ..< deleteCount {
-                                    _ = cache.removeValue(forKey: sortedCacheEntries[index].key)
-                                }
-                            }
-                        }
+                    do {
+                        try self.cache?.put(key: reference.identity.description, value: CacheValue(package: model, timestamp: DispatchTime.now()), replace: true)
+                    } catch {
+                        self.diagnosticsEngine?.emit(.warning("Failed to save GitHub metadata for package \(reference) to cache: \(error)"))
                     }
+
                     callback(.success(model))
                 }
             } catch {
@@ -168,7 +168,7 @@ struct GitHubPackageMetadataProvider: PackageMetadataProvider {
         }
     }
 
-    internal func apiURL(_ url: String) -> Foundation.URL? {
+    internal static func apiURL(_ url: String) -> Foundation.URL? {
         do {
             let regex = try NSRegularExpression(pattern: #"([^/@]+)[:/]([^:/]+)/([^/.]+)(\.git)?$"#, options: .caseInsensitive)
             if let match = regex.firstMatch(in: url, options: [], range: NSRange(location: 0, length: url.count)) {
@@ -214,17 +214,20 @@ struct GitHubPackageMetadataProvider: PackageMetadataProvider {
     public struct Configuration {
         public var apiLimitWarningThreshold: Int
         public var authTokens: [AuthTokenType: String]?
+        public var cacheDir: AbsolutePath
+        public var cacheSizeInMegabytes: Int
         public var cacheTTLInSeconds: Int
-        public var cacheSize: Int
 
         public init(authTokens: [AuthTokenType: String]? = nil,
                     apiLimitWarningThreshold: Int? = nil,
+                    cacheDir: AbsolutePath? = nil,
                     cacheTTLInSeconds: Int? = nil,
-                    cacheSize: Int? = nil) {
+                    cacheSizeInMegabytes: Int? = nil) {
             self.authTokens = authTokens
             self.apiLimitWarningThreshold = apiLimitWarningThreshold ?? 5
+            self.cacheDir = cacheDir.map(resolveSymlinks) ?? localFileSystem.swiftPMCacheDirectory.appending(components: "package-metadata")
+            self.cacheSizeInMegabytes = cacheSizeInMegabytes ?? 10
             self.cacheTTLInSeconds = cacheTTLInSeconds ?? 3600
-            self.cacheSize = cacheSize ?? 1000
         }
     }
 
@@ -236,6 +239,20 @@ struct GitHubPackageMetadataProvider: PackageMetadataProvider {
         case invalidAuthToken(URL)
         case apiLimitsExceeded(URL, Int)
     }
+
+    struct CacheValue: Codable {
+        let package: Model.PackageBasicMetadata
+        let timestamp: UInt64
+
+        var dispatchTime: DispatchTime {
+            DispatchTime(uptimeNanoseconds: self.timestamp)
+        }
+
+        init(package: Model.PackageBasicMetadata, timestamp: DispatchTime) {
+            self.package = package
+            self.timestamp = timestamp.uptimeNanoseconds
+        }
+    }
 }
 
 extension GitHubPackageMetadataProvider {