@@ -20,7 +20,7 @@ import TSCUtility
20
20
21
21
final class SQLitePackageCollectionsStorage : PackageCollectionsStorage , Closable {
22
22
static let batchSize = 100
23
-
23
+
24
24
private static let packageCollectionsTableName = " package_collections "
25
25
private static let packagesFTSName = " fts_packages "
26
26
private static let targetsFTSName = " fts_targets "
@@ -40,7 +40,7 @@ final class SQLitePackageCollectionsStorage: PackageCollectionsStorage, Closable
40
40
41
41
private let cache = ThreadSafeKeyValueStore < Model . CollectionIdentifier , Model . Collection > ( )
42
42
private let cacheLock = Lock ( )
43
-
43
+
44
44
// Lock helps prevent concurrency errors with transaction statements during e.g. `refreshCollections`,
45
45
// since only one transaction is allowed per SQLite connection. We need transactions to speed up bulk updates.
46
46
// TODO: we could potentially optimize this with db connection pool
@@ -61,7 +61,7 @@ final class SQLitePackageCollectionsStorage: PackageCollectionsStorage, Closable
61
61
self . diagnosticsEngine = diagnosticsEngine
62
62
self . encoder = JSONEncoder . makeWithDefaults ( )
63
63
self . decoder = JSONDecoder . makeWithDefaults ( )
64
-
64
+
65
65
self . populateTargetTrie ( )
66
66
}
67
67
@@ -100,22 +100,22 @@ final class SQLitePackageCollectionsStorage: PackageCollectionsStorage, Closable
100
100
try statement. bind ( bindings)
101
101
try statement. step ( )
102
102
}
103
-
103
+
104
104
try self . ftsLock. withLock {
105
105
// Update search indices
106
106
try self . withDB { db in
107
107
try db. exec ( query: " BEGIN TRANSACTION; " )
108
-
108
+
109
109
// First delete existing data
110
110
try self . removeFromSearchIndices ( identifier: collection. identifier)
111
-
111
+
112
112
let packagesStatement = try db. prepare ( query: " INSERT INTO \( Self . packagesFTSName) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?); " )
113
113
let targetsStatement = try db. prepare ( query: " INSERT INTO \( Self . targetsFTSName) VALUES (?, ?, ?); " )
114
-
114
+
115
115
// Then insert new data
116
116
try collection. packages. forEach { package in
117
117
var targets = Set < String > ( )
118
-
118
+
119
119
try package . versions. forEach { version in
120
120
// Packages FTS
121
121
let packagesBindings : [ SQLite . SQLiteValue ] = [
@@ -134,20 +134,20 @@ final class SQLitePackageCollectionsStorage: PackageCollectionsStorage, Closable
134
134
135
135
try packagesStatement. clearBindings ( )
136
136
try packagesStatement. reset ( )
137
-
137
+
138
138
version. targets. forEach { targets. insert ( $0. name) }
139
139
}
140
-
140
+
141
141
let collectionPackage = CollectionPackage ( collection: collection. identifier, package : package . reference. identity)
142
142
try targets. forEach { target in
143
143
// Targets in-memory trie
144
144
self . targetTrie. insert ( word: target. lowercased ( ) , foundIn: collectionPackage)
145
-
145
+
146
146
// Targets FTS
147
147
let targetsBindings : [ SQLite . SQLiteValue ] = [
148
148
. string( try self . encoder. encode ( collection. identifier) . base64EncodedString ( ) ) ,
149
149
. string( package . repository. url) ,
150
- . string( target)
150
+ . string( target) ,
151
151
]
152
152
try targetsStatement. bind ( targetsBindings)
153
153
try targetsStatement. step ( )
@@ -156,9 +156,9 @@ final class SQLitePackageCollectionsStorage: PackageCollectionsStorage, Closable
156
156
try targetsStatement. reset ( )
157
157
}
158
158
}
159
-
159
+
160
160
try db. exec ( query: " COMMIT; " )
161
-
161
+
162
162
try packagesStatement. finalize ( )
163
163
try targetsStatement. finalize ( )
164
164
}
@@ -172,24 +172,24 @@ final class SQLitePackageCollectionsStorage: PackageCollectionsStorage, Closable
172
172
}
173
173
}
174
174
}
175
-
175
+
176
176
private func removeFromSearchIndices( identifier: Model . CollectionIdentifier ) throws {
177
177
let identifierBase64 = try self . encoder. encode ( identifier. databaseKey ( ) ) . base64EncodedString ( )
178
-
178
+
179
179
let packagesQuery = " DELETE FROM \( Self . packagesFTSName) WHERE collection_id_blob_base64 = ?; "
180
180
try self . executeStatement ( packagesQuery) { statement -> Void in
181
181
let bindings : [ SQLite . SQLiteValue ] = [ . string( identifierBase64) ]
182
182
try statement. bind ( bindings)
183
183
try statement. step ( )
184
184
}
185
-
185
+
186
186
let targetsQuery = " DELETE FROM \( Self . targetsFTSName) WHERE collection_id_blob_base64 = ?; "
187
187
try self . executeStatement ( targetsQuery) { statement -> Void in
188
188
let bindings : [ SQLite . SQLiteValue ] = [ . string( identifierBase64) ]
189
189
try statement. bind ( bindings)
190
190
try statement. step ( )
191
191
}
192
-
192
+
193
193
self . targetTrie. remove { $0. collection == identifier }
194
194
}
195
195
@@ -206,10 +206,10 @@ final class SQLitePackageCollectionsStorage: PackageCollectionsStorage, Closable
206
206
try statement. bind ( bindings)
207
207
try statement. step ( )
208
208
}
209
-
209
+
210
210
// remove from search indices
211
211
try self . removeFromSearchIndices ( identifier: identifier)
212
-
212
+
213
213
// write to cache
214
214
self . cache [ identifier] = nil
215
215
callback ( . success( ( ) ) )
@@ -327,10 +327,10 @@ final class SQLitePackageCollectionsStorage: PackageCollectionsStorage, Closable
327
327
let packageQuery = " SELECT collection_id_blob_base64, repository_url FROM \( Self . packagesFTSName) WHERE \( Self . packagesFTSName) MATCH ?; "
328
328
try self . executeStatement ( packageQuery) { statement in
329
329
try statement. bind ( [ . string( query) ] )
330
-
330
+
331
331
while let row = try statement. step ( ) {
332
332
if let collectionData = Data ( base64Encoded: row. string ( at: 0 ) ) ,
333
- let collection = try ? self . decoder. decode ( Model . CollectionIdentifier. self, from: collectionData) {
333
+ let collection = try ? self . decoder. decode ( Model . CollectionIdentifier. self, from: collectionData) {
334
334
matches. append ( ( collection: collection, package : PackageIdentity ( url: row. string ( at: 1 ) ) ) )
335
335
}
336
336
}
@@ -342,7 +342,7 @@ final class SQLitePackageCollectionsStorage: PackageCollectionsStorage, Closable
342
342
let collectionDict = collections. reduce ( into: [ Model . CollectionIdentifier: Model . Collection] ( ) ) { result, collection in
343
343
result [ collection. identifier] = collection
344
344
}
345
-
345
+
346
346
// For each package, find the containing collections
347
347
let packageCollections = matches. filter { collectionDict. keys. contains ( $0. collection) }
348
348
. reduce ( into: [ PackageIdentity : ( package : Model . Package , collections: Set < Model . CollectionIdentifier > ) ] ( ) ) { result, match in
@@ -383,10 +383,10 @@ final class SQLitePackageCollectionsStorage: PackageCollectionsStorage, Closable
383
383
let packageQuery = " SELECT collection_id_blob_base64, repository_url FROM \( Self . packagesFTSName) WHERE id = ?; "
384
384
try self . executeStatement ( packageQuery) { statement in
385
385
try statement. bind ( [ . string( identifier. description) ] )
386
-
386
+
387
387
while let row = try statement. step ( ) {
388
388
if let collectionData = Data ( base64Encoded: row. string ( at: 0 ) ) ,
389
- let collection = try ? self . decoder. decode ( Model . CollectionIdentifier. self, from: collectionData) {
389
+ let collection = try ? self . decoder. decode ( Model . CollectionIdentifier. self, from: collectionData) {
390
390
matches. append ( ( collection: collection, package : PackageIdentity ( url: row. string ( at: 1 ) ) ) )
391
391
}
392
392
}
@@ -398,7 +398,7 @@ final class SQLitePackageCollectionsStorage: PackageCollectionsStorage, Closable
398
398
let collectionDict = collections. reduce ( into: [ Model . CollectionIdentifier: Model . Collection] ( ) ) { result, collection in
399
399
result [ collection. identifier] = collection
400
400
}
401
-
401
+
402
402
let collections = matches. filter { collectionDict. keys. contains ( $0. collection) }
403
403
. compactMap { collectionDict [ $0. collection] }
404
404
// Sort collections by processing date so the latest metadata is first
@@ -407,7 +407,7 @@ final class SQLitePackageCollectionsStorage: PackageCollectionsStorage, Closable
407
407
guard let package = collections. compactMap ( { $0. packages. first { $0. reference. identity == identifier } } ) . first else {
408
408
return callback ( . failure( NotFoundError ( " \( identifier) " ) ) )
409
409
}
410
-
410
+
411
411
callback ( . success( . init( package : package , collections: collections. map { $0. identifier } ) ) )
412
412
}
413
413
}
@@ -458,7 +458,7 @@ final class SQLitePackageCollectionsStorage: PackageCollectionsStorage, Closable
458
458
459
459
while let row = try statement. step ( ) {
460
460
if let collectionData = Data ( base64Encoded: row. string ( at: 0 ) ) ,
461
- let collection = try ? self . decoder. decode ( Model . CollectionIdentifier. self, from: collectionData) {
461
+ let collection = try ? self . decoder. decode ( Model . CollectionIdentifier. self, from: collectionData) {
462
462
matches. append ( (
463
463
collection: collection,
464
464
package : PackageIdentity ( url: row. string ( at: 1 ) ) ,
@@ -471,7 +471,7 @@ final class SQLitePackageCollectionsStorage: PackageCollectionsStorage, Closable
471
471
return callback ( . failure( error) )
472
472
}
473
473
}
474
-
474
+
475
475
let collectionDict = collections. reduce ( into: [ Model . CollectionIdentifier: Model . Collection] ( ) ) { result, collection in
476
476
result [ collection. identifier] = collection
477
477
}
@@ -480,7 +480,7 @@ final class SQLitePackageCollectionsStorage: PackageCollectionsStorage, Closable
480
480
var packageCollections = [ PackageIdentity : ( package : Model . Package , collections: Set < Model . CollectionIdentifier > ) ] ( )
481
481
// For each matching target, find the containing package version(s)
482
482
var targetPackageVersions = [ Model . Target : [ PackageIdentity : Set < Model . TargetListResult . PackageVersion > ] ] ( )
483
-
483
+
484
484
matches. filter { collectionDict. keys. contains ( $0. collection) } . forEach { match in
485
485
var packageEntry = packageCollections. removeValue ( forKey: match. package )
486
486
if packageEntry == nil {
@@ -495,7 +495,7 @@ final class SQLitePackageCollectionsStorage: PackageCollectionsStorage, Closable
495
495
if var packageEntry = packageEntry {
496
496
packageEntry. collections. insert ( match. collection)
497
497
packageCollections [ match. package ] = packageEntry
498
-
498
+
499
499
packageEntry. package . versions. forEach { version in
500
500
let targets = version. targets. filter { $0. name. lowercased ( ) == match. targetName. lowercased ( ) }
501
501
targets. forEach { target in
@@ -505,10 +505,10 @@ final class SQLitePackageCollectionsStorage: PackageCollectionsStorage, Closable
505
505
targetEntry [ packageEntry. package . reference. identity] = targetPackageEntry
506
506
targetPackageVersions [ target] = targetEntry
507
507
}
508
- }
508
+ }
509
509
}
510
510
}
511
-
511
+
512
512
let result = Model . TargetSearchResult ( items: targetPackageVersions. map { target, packageVersions in
513
513
let targetPackages : [ Model . TargetListItem . Package ] = packageVersions. compactMap { reference, versions in
514
514
guard let packageEntry = packageCollections [ reference] else {
@@ -543,7 +543,7 @@ final class SQLitePackageCollectionsStorage: PackageCollectionsStorage, Closable
543
543
);
544
544
"""
545
545
try db. exec ( query: table)
546
-
546
+
547
547
let ftsPackages = """
548
548
CREATE VIRTUAL TABLE IF NOT EXISTS \( Self . packagesFTSName) USING fts4(
549
549
collection_id_blob_base64, id, version, name, repository_url, summary, keywords, products, targets,
@@ -552,7 +552,7 @@ final class SQLitePackageCollectionsStorage: PackageCollectionsStorage, Closable
552
552
);
553
553
"""
554
554
try db. exec ( query: ftsPackages)
555
-
555
+
556
556
let ftsTargets = """
557
557
CREATE VIRTUAL TABLE IF NOT EXISTS \( Self . targetsFTSName) USING fts4(
558
558
collection_id_blob_base64, package_repository_url, name,
@@ -561,7 +561,7 @@ final class SQLitePackageCollectionsStorage: PackageCollectionsStorage, Closable
561
561
);
562
562
"""
563
563
try db. exec ( query: ftsTargets)
564
-
564
+
565
565
try db. exec ( query: " PRAGMA journal_mode=WAL; " )
566
566
}
567
567
@@ -625,7 +625,7 @@ final class SQLitePackageCollectionsStorage: PackageCollectionsStorage, Closable
625
625
626
626
return try body ( db)
627
627
}
628
-
628
+
629
629
func populateTargetTrie( callback: @escaping ( Result < Void , Error > ) -> Void = { _ in } ) {
630
630
self . queue. async {
631
631
do {
@@ -636,7 +636,7 @@ final class SQLitePackageCollectionsStorage: PackageCollectionsStorage, Closable
636
636
let targetName = row. string ( at: 2 )
637
637
638
638
if let collectionData = Data ( base64Encoded: row. string ( at: 0 ) ) ,
639
- let collection = try ? self . decoder. decode ( Model . CollectionIdentifier. self, from: collectionData) {
639
+ let collection = try ? self . decoder. decode ( Model . CollectionIdentifier. self, from: collectionData) {
640
640
let collectionPackage = CollectionPackage ( collection: collection, package : PackageIdentity ( url: row. string ( at: 1 ) ) )
641
641
self . targetTrie. insert ( word: targetName. lowercased ( ) , foundIn: collectionPackage)
642
642
}
@@ -650,7 +650,7 @@ final class SQLitePackageCollectionsStorage: PackageCollectionsStorage, Closable
650
650
}
651
651
}
652
652
}
653
-
653
+
654
654
// For `Trie`
655
655
private struct CollectionPackage : Hashable , CustomStringConvertible {
656
656
let collection : Model . CollectionIdentifier
0 commit comments