[5.4] limit size of sqlite manifest cache (#3178)

Sources/PackageLoading/ManifestLoader.swift

@@ -625,7 +625,11 @@ public final class ManifestLoader: ManifestLoaderProtocol {
     fileprivate func parseAndCacheManifest(key: ManifestCacheKey, diagnostics: DiagnosticsEngine?) -> ManifestParseResult {
         let cache = self.databaseCacheDir.map { cacheDir -> SQLiteManifestCache in
             let path = Self.manifestCacheDBPath(cacheDir)
-            return SQLiteManifestCache(location: .path(path), diagnosticsEngine: diagnostics)
+            var configuration = SQLiteManifestCache.Configuration()
+            // FIXME: expose as user-facing configuration
+            configuration.maxSizeInMegabytes = 100
+            configuration.truncateWhenFull = true
+            return SQLiteManifestCache(location: .path(path), configuration: configuration, diagnosticsEngine: diagnostics)
         }
 
         // TODO: we could wrap the failure here with diagnostics if it wasn't optional throughout
@@ -657,7 +661,7 @@ public final class ManifestLoader: ManifestLoaderProtocol {
         return result
     }
 
-    fileprivate struct ManifestCacheKey: Hashable {
+    internal struct ManifestCacheKey: Hashable {
         let packageIdentity: PackageIdentity
         let manifestPath: AbsolutePath
         let manifestContents: [UInt8]
@@ -708,7 +712,7 @@ public final class ManifestLoader: ManifestLoaderProtocol {
         }
     }
 
-    fileprivate struct ManifestParseResult: Codable {
+    internal struct ManifestParseResult: Codable {
         var hasErrors: Bool {
             return parsedManifest == nil
         }
@@ -1089,9 +1093,10 @@ extension TSCBasic.Diagnostic.Message {
 }
 
 /// SQLite backed persistent cache.
-private final class SQLiteManifestCache: Closable {
+internal final class SQLiteManifestCache: Closable {
     let fileSystem: FileSystem
     let location: SQLite.Location
+    let configuration: Configuration
 
     private var state = State.idle
     private let stateLock = Lock()
@@ -1100,21 +1105,22 @@ private final class SQLiteManifestCache: Closable {
     private let jsonEncoder: JSONEncoder
     private let jsonDecoder: JSONDecoder
 
-    init(location: SQLite.Location, diagnosticsEngine: DiagnosticsEngine? = nil) {
+    init(location: SQLite.Location, configuration: Configuration = .init(), diagnosticsEngine: DiagnosticsEngine? = nil) {
         self.location = location
         switch self.location {
         case .path, .temporary:
             self.fileSystem = localFileSystem
         case .memory:
             self.fileSystem = InMemoryFileSystem()
         }
+        self.configuration = configuration
         self.diagnosticsEngine = diagnosticsEngine
         self.jsonEncoder = JSONEncoder.makeWithDefaults()
         self.jsonDecoder = JSONDecoder.makeWithDefaults()
     }
 
-    convenience init(path: AbsolutePath, diagnosticsEngine: DiagnosticsEngine? = nil) {
-        self.init(location: .path(path), diagnosticsEngine: diagnosticsEngine)
+    convenience init(path: AbsolutePath, configuration: Configuration = .init(), diagnosticsEngine: DiagnosticsEngine? = nil) {
+        self.init(location: .path(path), configuration: configuration, diagnosticsEngine: diagnosticsEngine)
     }
 
     deinit {
@@ -1137,15 +1143,28 @@ private final class SQLiteManifestCache: Closable {
     }
 
     func put(key: ManifestLoader.ManifestCacheKey, manifest: ManifestLoader.ManifestParseResult) throws {
-        let query = "INSERT OR IGNORE INTO MANIFEST_CACHE VALUES (?, ?);"
-        try self.executeStatement(query) { statement -> Void in
-            let data = try self.jsonEncoder.encode(manifest)
-            let bindings: [SQLite.SQLiteValue] = [
-                .string(key.sha256Checksum),
-                .blob(data),
-            ]
-            try statement.bind(bindings)
-            try statement.step()
+        do {
+            let query = "INSERT OR IGNORE INTO MANIFEST_CACHE VALUES (?, ?);"
+            try self.executeStatement(query) { statement -> Void in
+                let data = try self.jsonEncoder.encode(manifest)
+                let bindings: [SQLite.SQLiteValue] = [
+                    .string(key.sha256Checksum),
+                    .blob(data),
+                ]
+                try statement.bind(bindings)
+                try statement.step()
+            }
+        } catch (let error as SQLite.Errors) where error == .databaseFull {
+            if !self.configuration.truncateWhenFull {
+                throw error
+            }
+            self.diagnosticsEngine?.emit(.warning("truncating manifest cache database since it reached max size of \(self.configuration.maxSizeInBytes ?? 0) bytes"))
+            try self.executeStatement("DELETE FROM MANIFEST_CACHE;") { statement -> Void in
+                try statement.step()
+            }
+            try self.put(key: key, manifest: manifest)
+        } catch {
+            throw error
         }
     }
 
@@ -1181,10 +1200,7 @@ private final class SQLiteManifestCache: Closable {
 
     private func withDB<T>(_ body: (SQLite) throws -> T) throws -> T {
         let createDB = { () throws -> SQLite in
-            // see https://www.sqlite.org/c3ref/busy_timeout.html
-            var configuration = SQLite.Configuration()
-            configuration.busyTimeoutMilliseconds = 1_000
-            let db = try SQLite(location: self.location, configuration: configuration)
+            let db = try SQLite(location: self.location, configuration: self.configuration.underlying)
             try self.createSchemaIfNecessary(db: db)
             return db
         }
@@ -1253,4 +1269,45 @@ private final class SQLiteManifestCache: Closable {
         case connected(SQLite)
         case disconnected
     }
+
+    struct Configuration {
+        var truncateWhenFull: Bool
+
+        fileprivate var underlying: SQLite.Configuration
+
+        init() {
+            self.underlying = .init()
+            self.truncateWhenFull = true
+            self.maxSizeInMegabytes = 100
+            // see https://www.sqlite.org/c3ref/busy_timeout.html
+            self.busyTimeoutMilliseconds = 1_000
+        }
+
+        var maxSizeInMegabytes: Int? {
+            get {
+                self.underlying.maxSizeInMegabytes
+            }
+            set {
+                self.underlying.maxSizeInMegabytes = newValue
+            }
+        }
+
+        var maxSizeInBytes: Int? {
+            get {
+                self.underlying.maxSizeInBytes
+            }
+            set {
+                self.underlying.maxSizeInBytes = newValue
+            }
+        }
+
+        var busyTimeoutMilliseconds: Int32 {
+            get {
+                self.underlying.busyTimeoutMilliseconds
+            }
+            set {
+                self.underlying.busyTimeoutMilliseconds = newValue
+            }
+        }
+    }
 }

Tests/PackageLoadingTests/ManifestLoaderSQLiteCacheTests.swift

@@ -0,0 +1,197 @@
+/*
+ This source file is part of the Swift.org open source project
+
+ Copyright (c) 2020 Apple Inc. and the Swift project authors
+ Licensed under Apache License v2.0 with Runtime Library Exception
+
+ See http://swift.org/LICENSE.txt for license information
+ See http://swift.org/CONTRIBUTORS.txt for Swift project authors
+*/
+
+@testable import PackageLoading
+import PackageModel
+import TSCBasic
+import TSCTestSupport
+import TSCUtility
+import XCTest
+
+class ManifestLoaderSQLiteCacheTests: XCTestCase {
+    func testHappyCase() throws {
+        try testWithTemporaryDirectory { tmpPath in
+            let path = tmpPath.appending(component: "test.db")
+            let storage = SQLiteManifestCache(path: path)
+            defer { XCTAssertNoThrow(try storage.close()) }
+
+
+            let mockManifests = try makeMockManifests(fileSystem: localFileSystem, rootPath: tmpPath)
+            try mockManifests.forEach { key, manifest in
+                _ = try storage.put(key: key, manifest: manifest)
+            }
+
+            try mockManifests.forEach { key, manifest in
+                let result = try storage.get(key: key)
+                XCTAssertEqual(result?.parsedManifest, manifest.parsedManifest)
+            }
+
+            guard case .path(let storagePath) = storage.location else {
+                return XCTFail("invalid location \(storage.location)")
+            }
+
+            XCTAssertTrue(storage.fileSystem.exists(storagePath), "expected file to be written")
+        }
+    }
+
+    func testFileDeleted() throws {
+        try testWithTemporaryDirectory { tmpPath in
+            let path = tmpPath.appending(component: "test.db")
+            let storage = SQLiteManifestCache(path: path)
+            defer { XCTAssertNoThrow(try storage.close()) }
+
+            let mockManifests = try makeMockManifests(fileSystem: localFileSystem, rootPath: tmpPath)
+            try mockManifests.forEach {  key, manifest in
+                _ = try storage.put(key: key, manifest: manifest)
+            }
+
+            try mockManifests.forEach { key, manifest in
+                let result = try storage.get(key: key)
+                XCTAssertEqual(result?.parsedManifest, manifest.parsedManifest)
+            }
+
+            guard case .path(let storagePath) = storage.location else {
+                return XCTFail("invalid location \(storage.location)")
+            }
+
+            XCTAssertTrue(storage.fileSystem.exists(storagePath), "expected file to exist at \(storagePath)")
+            try storage.fileSystem.removeFileTree(storagePath)
+
+            do {
+                let result = try storage.get(key: mockManifests.first!.key)
+                XCTAssertNil(result)
+            }
+
+            do {
+                XCTAssertNoThrow(try storage.put(key: mockManifests.first!.key, manifest: mockManifests.first!.value))
+                let result = try storage.get(key: mockManifests.first!.key)
+                XCTAssertEqual(result?.parsedManifest, mockManifests.first!.value.parsedManifest)
+            }
+
+            XCTAssertTrue(storage.fileSystem.exists(storagePath), "expected file to exist at \(storagePath)")
+        }
+    }
+
+    func testFileCorrupt() throws {
+        try testWithTemporaryDirectory { tmpPath in
+            let path = tmpPath.appending(component: "test.db")
+            let storage = SQLiteManifestCache(path: path)
+            defer { XCTAssertNoThrow(try storage.close()) }
+
+            let mockManifests = try makeMockManifests(fileSystem: localFileSystem, rootPath: tmpPath)
+            try mockManifests.forEach {  key, manifest in
+                _ = try storage.put(key: key, manifest: manifest)
+            }
+
+            try mockManifests.forEach { key, manifest in
+                let result = try storage.get(key: key)
+                XCTAssertEqual(result?.parsedManifest, manifest.parsedManifest)
+            }
+
+            guard case .path(let storagePath) = storage.location else {
+                return XCTFail("invalid location \(storage.location)")
+            }
+
+            try storage.close()
+
+            XCTAssertTrue(storage.fileSystem.exists(storagePath), "expected file to exist at \(path)")
+            try storage.fileSystem.writeFileContents(storagePath, bytes: ByteString("blah".utf8))
+
+            XCTAssertThrowsError(try storage.get(key: mockManifests.first!.key), "expected error", { error in
+                XCTAssert("\(error)".contains("is not a database"), "Expected file is not a database error")
+            })
+
+            XCTAssertThrowsError(try storage.put(key: mockManifests.first!.key, manifest: mockManifests.first!.value), "expected error", { error in
+                XCTAssert("\(error)".contains("is not a database"), "Expected file is not a database error")
+            })
+        }
+    }
+
+    func testMaxSizeNotHandled() throws {
+        try testWithTemporaryDirectory { tmpPath in
+            let path = tmpPath.appending(component: "test.db")
+            var configuration = SQLiteManifestCache.Configuration()
+            configuration.maxSizeInBytes = 1024 * 3
+            configuration.truncateWhenFull = false
+            let storage = SQLiteManifestCache(location: .path(path), configuration: configuration)
+            defer { XCTAssertNoThrow(try storage.close()) }
+
+            func create() throws {
+                let mockManifests = try makeMockManifests(fileSystem: localFileSystem, rootPath: tmpPath, count: 50)
+                try mockManifests.forEach {  key, manifest in
+                    _ = try storage.put(key: key, manifest: manifest)
+                }
+            }
+
+            XCTAssertThrowsError(try create(), "expected error", { error in
+                XCTAssertEqual(error as? SQLite.Errors, .databaseFull, "Expected 'databaseFull' error")
+            })
+        }
+    }
+
+    func testMaxSizeHandled() throws {
+        try testWithTemporaryDirectory { tmpPath in
+            let path = tmpPath.appending(component: "test.db")
+            var configuration = SQLiteManifestCache.Configuration()
+            configuration.maxSizeInBytes = 1024 * 3
+            configuration.truncateWhenFull = true
+            let storage = SQLiteManifestCache(location: .path(path), configuration: configuration)
+            defer { XCTAssertNoThrow(try storage.close()) }
+
+            var keys = [ManifestLoader.ManifestCacheKey]()
+            let mockManifests = try makeMockManifests(fileSystem: localFileSystem, rootPath: tmpPath, count: 50)
+            try mockManifests.forEach { key, manifest in
+                _ = try storage.put(key: key, manifest: manifest)
+                keys.append(key)
+            }
+
+            do {
+                let result = try storage.get(key: mockManifests.first!.key)
+                XCTAssertNil(result)
+            }
+
+            do {
+                let result = try storage.get(key: keys.last!)
+                XCTAssertEqual(result?.parsedManifest, mockManifests[keys.last!]?.parsedManifest)
+            }
+        }
+    }
+}
+
+fileprivate func makeMockManifests(fileSystem: FileSystem, rootPath: AbsolutePath, count: Int = Int.random(in: 50 ..< 100)) throws -> [ManifestLoader.ManifestCacheKey: ManifestLoader.ManifestParseResult] {
+    var manifests = [ManifestLoader.ManifestCacheKey: ManifestLoader.ManifestParseResult]()
+    for index in 0 ..< count {
+        let manifestPath = rootPath.appending(components: "\(index)", "Package.swift")
+        try fileSystem.writeFileContents(manifestPath) { stream in
+            stream <<< """
+            import PackageDescription
+            let package = Package(
+            name: "Trivial-\(index)",
+                targets: [
+                    .target(
+                        name: "foo-\(index)",
+                        dependencies: []),
+
+            )
+            """
+        }
+        let key = try ManifestLoader.ManifestCacheKey(packageIdentity: PackageIdentity.init(path: manifestPath),
+                                                      manifestPath: manifestPath,
+                                                      toolsVersion: ToolsVersion.currentToolsVersion,
+                                                      env: [:],
+                                                      swiftpmVersion: Versioning.currentVersion.displayString,
+                                                      fileSystem: fileSystem)
+        manifests[key] = ManifestLoader.ManifestParseResult(compilerOutput: "mock-output-\(index)",
+                                                            parsedManifest: "{ 'name': 'mock-manifest-\(index)' }")
+    }
+
+    return manifests
+}
+