Skip to content

Commit 9c96dd7

Browse files
authored
limit size of sqlite manifest cache (#3178) (#3204)
motivation: sqlite manifest cache is now shared by default, controlling its max size is important changes: * introduce SQLiteManifestCache::Configuraiton allowing to set max database size * when database is full, truncate it. this is not refined, but since this is a cache it should be fine. * add tests including a couple of other tests for SQLiteManifestCache that were missing * adjust to latest tsc
1 parent 2665379 commit 9c96dd7

File tree

2 files changed

+274
-20
lines changed

2 files changed

+274
-20
lines changed

Sources/PackageLoading/ManifestLoader.swift

Lines changed: 77 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -625,7 +625,11 @@ public final class ManifestLoader: ManifestLoaderProtocol {
625625
fileprivate func parseAndCacheManifest(key: ManifestCacheKey, diagnostics: DiagnosticsEngine?) -> ManifestParseResult {
626626
let cache = self.databaseCacheDir.map { cacheDir -> SQLiteManifestCache in
627627
let path = Self.manifestCacheDBPath(cacheDir)
628-
return SQLiteManifestCache(location: .path(path), diagnosticsEngine: diagnostics)
628+
var configuration = SQLiteManifestCache.Configuration()
629+
// FIXME: expose as user-facing configuration
630+
configuration.maxSizeInMegabytes = 100
631+
configuration.truncateWhenFull = true
632+
return SQLiteManifestCache(location: .path(path), configuration: configuration, diagnosticsEngine: diagnostics)
629633
}
630634

631635
// TODO: we could wrap the failure here with diagnostics if it wasn't optional throughout
@@ -657,7 +661,7 @@ public final class ManifestLoader: ManifestLoaderProtocol {
657661
return result
658662
}
659663

660-
fileprivate struct ManifestCacheKey: Hashable {
664+
internal struct ManifestCacheKey: Hashable {
661665
let packageIdentity: PackageIdentity
662666
let manifestPath: AbsolutePath
663667
let manifestContents: [UInt8]
@@ -708,7 +712,7 @@ public final class ManifestLoader: ManifestLoaderProtocol {
708712
}
709713
}
710714

711-
fileprivate struct ManifestParseResult: Codable {
715+
internal struct ManifestParseResult: Codable {
712716
var hasErrors: Bool {
713717
return parsedManifest == nil
714718
}
@@ -1089,9 +1093,10 @@ extension TSCBasic.Diagnostic.Message {
10891093
}
10901094

10911095
/// SQLite backed persistent cache.
1092-
private final class SQLiteManifestCache: Closable {
1096+
internal final class SQLiteManifestCache: Closable {
10931097
let fileSystem: FileSystem
10941098
let location: SQLite.Location
1099+
let configuration: Configuration
10951100

10961101
private var state = State.idle
10971102
private let stateLock = Lock()
@@ -1100,21 +1105,22 @@ private final class SQLiteManifestCache: Closable {
11001105
private let jsonEncoder: JSONEncoder
11011106
private let jsonDecoder: JSONDecoder
11021107

1103-
init(location: SQLite.Location, diagnosticsEngine: DiagnosticsEngine? = nil) {
1108+
init(location: SQLite.Location, configuration: Configuration = .init(), diagnosticsEngine: DiagnosticsEngine? = nil) {
11041109
self.location = location
11051110
switch self.location {
11061111
case .path, .temporary:
11071112
self.fileSystem = localFileSystem
11081113
case .memory:
11091114
self.fileSystem = InMemoryFileSystem()
11101115
}
1116+
self.configuration = configuration
11111117
self.diagnosticsEngine = diagnosticsEngine
11121118
self.jsonEncoder = JSONEncoder.makeWithDefaults()
11131119
self.jsonDecoder = JSONDecoder.makeWithDefaults()
11141120
}
11151121

1116-
convenience init(path: AbsolutePath, diagnosticsEngine: DiagnosticsEngine? = nil) {
1117-
self.init(location: .path(path), diagnosticsEngine: diagnosticsEngine)
1122+
convenience init(path: AbsolutePath, configuration: Configuration = .init(), diagnosticsEngine: DiagnosticsEngine? = nil) {
1123+
self.init(location: .path(path), configuration: configuration, diagnosticsEngine: diagnosticsEngine)
11181124
}
11191125

11201126
deinit {
@@ -1137,15 +1143,28 @@ private final class SQLiteManifestCache: Closable {
11371143
}
11381144

11391145
func put(key: ManifestLoader.ManifestCacheKey, manifest: ManifestLoader.ManifestParseResult) throws {
1140-
let query = "INSERT OR IGNORE INTO MANIFEST_CACHE VALUES (?, ?);"
1141-
try self.executeStatement(query) { statement -> Void in
1142-
let data = try self.jsonEncoder.encode(manifest)
1143-
let bindings: [SQLite.SQLiteValue] = [
1144-
.string(key.sha256Checksum),
1145-
.blob(data),
1146-
]
1147-
try statement.bind(bindings)
1148-
try statement.step()
1146+
do {
1147+
let query = "INSERT OR IGNORE INTO MANIFEST_CACHE VALUES (?, ?);"
1148+
try self.executeStatement(query) { statement -> Void in
1149+
let data = try self.jsonEncoder.encode(manifest)
1150+
let bindings: [SQLite.SQLiteValue] = [
1151+
.string(key.sha256Checksum),
1152+
.blob(data),
1153+
]
1154+
try statement.bind(bindings)
1155+
try statement.step()
1156+
}
1157+
} catch (let error as SQLite.Errors) where error == .databaseFull {
1158+
if !self.configuration.truncateWhenFull {
1159+
throw error
1160+
}
1161+
self.diagnosticsEngine?.emit(.warning("truncating manifest cache database since it reached max size of \(self.configuration.maxSizeInBytes ?? 0) bytes"))
1162+
try self.executeStatement("DELETE FROM MANIFEST_CACHE;") { statement -> Void in
1163+
try statement.step()
1164+
}
1165+
try self.put(key: key, manifest: manifest)
1166+
} catch {
1167+
throw error
11491168
}
11501169
}
11511170

@@ -1181,10 +1200,7 @@ private final class SQLiteManifestCache: Closable {
11811200

11821201
private func withDB<T>(_ body: (SQLite) throws -> T) throws -> T {
11831202
let createDB = { () throws -> SQLite in
1184-
// see https://www.sqlite.org/c3ref/busy_timeout.html
1185-
var configuration = SQLite.Configuration()
1186-
configuration.busyTimeoutMilliseconds = 1_000
1187-
let db = try SQLite(location: self.location, configuration: configuration)
1203+
let db = try SQLite(location: self.location, configuration: self.configuration.underlying)
11881204
try self.createSchemaIfNecessary(db: db)
11891205
return db
11901206
}
@@ -1253,4 +1269,45 @@ private final class SQLiteManifestCache: Closable {
12531269
case connected(SQLite)
12541270
case disconnected
12551271
}
1272+
1273+
struct Configuration {
1274+
var truncateWhenFull: Bool
1275+
1276+
fileprivate var underlying: SQLite.Configuration
1277+
1278+
init() {
1279+
self.underlying = .init()
1280+
self.truncateWhenFull = true
1281+
self.maxSizeInMegabytes = 100
1282+
// see https://www.sqlite.org/c3ref/busy_timeout.html
1283+
self.busyTimeoutMilliseconds = 1_000
1284+
}
1285+
1286+
var maxSizeInMegabytes: Int? {
1287+
get {
1288+
self.underlying.maxSizeInMegabytes
1289+
}
1290+
set {
1291+
self.underlying.maxSizeInMegabytes = newValue
1292+
}
1293+
}
1294+
1295+
var maxSizeInBytes: Int? {
1296+
get {
1297+
self.underlying.maxSizeInBytes
1298+
}
1299+
set {
1300+
self.underlying.maxSizeInBytes = newValue
1301+
}
1302+
}
1303+
1304+
var busyTimeoutMilliseconds: Int32 {
1305+
get {
1306+
self.underlying.busyTimeoutMilliseconds
1307+
}
1308+
set {
1309+
self.underlying.busyTimeoutMilliseconds = newValue
1310+
}
1311+
}
1312+
}
12561313
}
Lines changed: 197 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,197 @@
1+
/*
2+
This source file is part of the Swift.org open source project
3+
4+
Copyright (c) 2020 Apple Inc. and the Swift project authors
5+
Licensed under Apache License v2.0 with Runtime Library Exception
6+
7+
See http://swift.org/LICENSE.txt for license information
8+
See http://swift.org/CONTRIBUTORS.txt for Swift project authors
9+
*/
10+
11+
@testable import PackageLoading
12+
import PackageModel
13+
import TSCBasic
14+
import TSCTestSupport
15+
import TSCUtility
16+
import XCTest
17+
18+
class ManifestLoaderSQLiteCacheTests: XCTestCase {
19+
func testHappyCase() throws {
20+
try testWithTemporaryDirectory { tmpPath in
21+
let path = tmpPath.appending(component: "test.db")
22+
let storage = SQLiteManifestCache(path: path)
23+
defer { XCTAssertNoThrow(try storage.close()) }
24+
25+
26+
let mockManifests = try makeMockManifests(fileSystem: localFileSystem, rootPath: tmpPath)
27+
try mockManifests.forEach { key, manifest in
28+
_ = try storage.put(key: key, manifest: manifest)
29+
}
30+
31+
try mockManifests.forEach { key, manifest in
32+
let result = try storage.get(key: key)
33+
XCTAssertEqual(result?.parsedManifest, manifest.parsedManifest)
34+
}
35+
36+
guard case .path(let storagePath) = storage.location else {
37+
return XCTFail("invalid location \(storage.location)")
38+
}
39+
40+
XCTAssertTrue(storage.fileSystem.exists(storagePath), "expected file to be written")
41+
}
42+
}
43+
44+
func testFileDeleted() throws {
45+
try testWithTemporaryDirectory { tmpPath in
46+
let path = tmpPath.appending(component: "test.db")
47+
let storage = SQLiteManifestCache(path: path)
48+
defer { XCTAssertNoThrow(try storage.close()) }
49+
50+
let mockManifests = try makeMockManifests(fileSystem: localFileSystem, rootPath: tmpPath)
51+
try mockManifests.forEach { key, manifest in
52+
_ = try storage.put(key: key, manifest: manifest)
53+
}
54+
55+
try mockManifests.forEach { key, manifest in
56+
let result = try storage.get(key: key)
57+
XCTAssertEqual(result?.parsedManifest, manifest.parsedManifest)
58+
}
59+
60+
guard case .path(let storagePath) = storage.location else {
61+
return XCTFail("invalid location \(storage.location)")
62+
}
63+
64+
XCTAssertTrue(storage.fileSystem.exists(storagePath), "expected file to exist at \(storagePath)")
65+
try storage.fileSystem.removeFileTree(storagePath)
66+
67+
do {
68+
let result = try storage.get(key: mockManifests.first!.key)
69+
XCTAssertNil(result)
70+
}
71+
72+
do {
73+
XCTAssertNoThrow(try storage.put(key: mockManifests.first!.key, manifest: mockManifests.first!.value))
74+
let result = try storage.get(key: mockManifests.first!.key)
75+
XCTAssertEqual(result?.parsedManifest, mockManifests.first!.value.parsedManifest)
76+
}
77+
78+
XCTAssertTrue(storage.fileSystem.exists(storagePath), "expected file to exist at \(storagePath)")
79+
}
80+
}
81+
82+
func testFileCorrupt() throws {
83+
try testWithTemporaryDirectory { tmpPath in
84+
let path = tmpPath.appending(component: "test.db")
85+
let storage = SQLiteManifestCache(path: path)
86+
defer { XCTAssertNoThrow(try storage.close()) }
87+
88+
let mockManifests = try makeMockManifests(fileSystem: localFileSystem, rootPath: tmpPath)
89+
try mockManifests.forEach { key, manifest in
90+
_ = try storage.put(key: key, manifest: manifest)
91+
}
92+
93+
try mockManifests.forEach { key, manifest in
94+
let result = try storage.get(key: key)
95+
XCTAssertEqual(result?.parsedManifest, manifest.parsedManifest)
96+
}
97+
98+
guard case .path(let storagePath) = storage.location else {
99+
return XCTFail("invalid location \(storage.location)")
100+
}
101+
102+
try storage.close()
103+
104+
XCTAssertTrue(storage.fileSystem.exists(storagePath), "expected file to exist at \(path)")
105+
try storage.fileSystem.writeFileContents(storagePath, bytes: ByteString("blah".utf8))
106+
107+
XCTAssertThrowsError(try storage.get(key: mockManifests.first!.key), "expected error", { error in
108+
XCTAssert("\(error)".contains("is not a database"), "Expected file is not a database error")
109+
})
110+
111+
XCTAssertThrowsError(try storage.put(key: mockManifests.first!.key, manifest: mockManifests.first!.value), "expected error", { error in
112+
XCTAssert("\(error)".contains("is not a database"), "Expected file is not a database error")
113+
})
114+
}
115+
}
116+
117+
func testMaxSizeNotHandled() throws {
118+
try testWithTemporaryDirectory { tmpPath in
119+
let path = tmpPath.appending(component: "test.db")
120+
var configuration = SQLiteManifestCache.Configuration()
121+
configuration.maxSizeInBytes = 1024 * 3
122+
configuration.truncateWhenFull = false
123+
let storage = SQLiteManifestCache(location: .path(path), configuration: configuration)
124+
defer { XCTAssertNoThrow(try storage.close()) }
125+
126+
func create() throws {
127+
let mockManifests = try makeMockManifests(fileSystem: localFileSystem, rootPath: tmpPath, count: 50)
128+
try mockManifests.forEach { key, manifest in
129+
_ = try storage.put(key: key, manifest: manifest)
130+
}
131+
}
132+
133+
XCTAssertThrowsError(try create(), "expected error", { error in
134+
XCTAssertEqual(error as? SQLite.Errors, .databaseFull, "Expected 'databaseFull' error")
135+
})
136+
}
137+
}
138+
139+
func testMaxSizeHandled() throws {
140+
try testWithTemporaryDirectory { tmpPath in
141+
let path = tmpPath.appending(component: "test.db")
142+
var configuration = SQLiteManifestCache.Configuration()
143+
configuration.maxSizeInBytes = 1024 * 3
144+
configuration.truncateWhenFull = true
145+
let storage = SQLiteManifestCache(location: .path(path), configuration: configuration)
146+
defer { XCTAssertNoThrow(try storage.close()) }
147+
148+
var keys = [ManifestLoader.ManifestCacheKey]()
149+
let mockManifests = try makeMockManifests(fileSystem: localFileSystem, rootPath: tmpPath, count: 50)
150+
try mockManifests.forEach { key, manifest in
151+
_ = try storage.put(key: key, manifest: manifest)
152+
keys.append(key)
153+
}
154+
155+
do {
156+
let result = try storage.get(key: mockManifests.first!.key)
157+
XCTAssertNil(result)
158+
}
159+
160+
do {
161+
let result = try storage.get(key: keys.last!)
162+
XCTAssertEqual(result?.parsedManifest, mockManifests[keys.last!]?.parsedManifest)
163+
}
164+
}
165+
}
166+
}
167+
168+
fileprivate func makeMockManifests(fileSystem: FileSystem, rootPath: AbsolutePath, count: Int = Int.random(in: 50 ..< 100)) throws -> [ManifestLoader.ManifestCacheKey: ManifestLoader.ManifestParseResult] {
169+
var manifests = [ManifestLoader.ManifestCacheKey: ManifestLoader.ManifestParseResult]()
170+
for index in 0 ..< count {
171+
let manifestPath = rootPath.appending(components: "\(index)", "Package.swift")
172+
try fileSystem.writeFileContents(manifestPath) { stream in
173+
stream <<< """
174+
import PackageDescription
175+
let package = Package(
176+
name: "Trivial-\(index)",
177+
targets: [
178+
.target(
179+
name: "foo-\(index)",
180+
dependencies: []),
181+
182+
)
183+
"""
184+
}
185+
let key = try ManifestLoader.ManifestCacheKey(packageIdentity: PackageIdentity.init(path: manifestPath),
186+
manifestPath: manifestPath,
187+
toolsVersion: ToolsVersion.currentToolsVersion,
188+
env: [:],
189+
swiftpmVersion: Versioning.currentVersion.displayString,
190+
fileSystem: fileSystem)
191+
manifests[key] = ManifestLoader.ManifestParseResult(compilerOutput: "mock-output-\(index)",
192+
parsedManifest: "{ 'name': 'mock-manifest-\(index)' }")
193+
}
194+
195+
return manifests
196+
}
197+

0 commit comments

Comments
 (0)