Skip to content

Commit 1248164

Browse files
yim-leeyim-lee
authored
[Collection] Optimize collection fetch in search methods (#3491)
Motivation: All the search methods in `SQLitePackageCollectionsStorage` start by fetching a subset or all of the collections from database. This is because prior to using FTS, search was done directly on the `Collection` instances. In case of FTS, we can optimize things by: 1. Fetching collections from the database iff there are matches. i.e., return `NotFoundError` or empty result early 2. Fetching only collections where matches are found Modifications: Rearrange logic in search methods to do the optimizations described above. Related to rdar://77873780
1 parent 5f72e9f commit 1248164

File tree

1 file changed

+167
-110
lines changed

1 file changed

+167
-110
lines changed

‎Sources/PackageCollections/Storage/SQLitePackageCollectionsStorage.swift

Lines changed: 167 additions & 110 deletions
Original file line numberDiff line numberDiff line change
@@ -286,29 +286,38 @@ final class SQLitePackageCollectionsStorage: PackageCollectionsStorage, Closable
286286
func searchPackages(identifiers: [Model.CollectionIdentifier]? = nil,
287287
query: String,
288288
callback: @escaping (Result<Model.PackageSearchResult, Error>) -> Void) {
289-
self.list(identifiers: identifiers) { result in
290-
switch result {
291-
case .failure(let error):
292-
callback(.failure(error))
293-
case .success(let collections):
294-
if self.useSearchIndices.get() ?? false {
295-
var matches = [(collection: Model.CollectionIdentifier, package: PackageIdentity)]()
296-
do {
297-
let packageQuery = "SELECT collection_id_blob_base64, repository_url FROM \(Self.packagesFTSName) WHERE \(Self.packagesFTSName) MATCH ?;"
298-
try self.executeStatement(packageQuery) { statement in
299-
try statement.bind([.string(query)])
289+
if self.useSearchIndices.get() ?? false {
290+
var matches = [(collection: Model.CollectionIdentifier, package: PackageIdentity)]()
291+
var matchingCollections = Set<Model.CollectionIdentifier>()
300292

301-
while let row = try statement.step() {
302-
if let collectionData = Data(base64Encoded: row.string(at: 0)),
303-
let collection = try? self.decoder.decode(Model.CollectionIdentifier.self, from: collectionData) {
304-
matches.append((collection: collection, package: PackageIdentity(url: row.string(at: 1))))
305-
}
306-
}
293+
do {
294+
let packageQuery = "SELECT collection_id_blob_base64, repository_url FROM \(Self.packagesFTSName) WHERE \(Self.packagesFTSName) MATCH ?;"
295+
try self.executeStatement(packageQuery) { statement in
296+
try statement.bind([.string(query)])
297+
298+
while let row = try statement.step() {
299+
if let collectionData = Data(base64Encoded: row.string(at: 0)),
300+
let collection = try? self.decoder.decode(Model.CollectionIdentifier.self, from: collectionData) {
301+
matches.append((collection: collection, package: PackageIdentity(url: row.string(at: 1))))
302+
matchingCollections.insert(collection)
307303
}
308-
} catch {
309-
return callback(.failure(error))
310304
}
305+
}
306+
} catch {
307+
return callback(.failure(error))
308+
}
309+
310+
// Optimization: return early if no matches
311+
guard !matches.isEmpty else {
312+
return callback(.success(Model.PackageSearchResult(items: [])))
313+
}
311314

315+
// Optimization: fetch only those collections that contain matching packages
316+
self.list(identifiers: Array(identifiers.map { Set($0).intersection(matchingCollections) } ?? matchingCollections)) { result in
317+
switch result {
318+
case .failure(let error):
319+
callback(.failure(error))
320+
case .success(let collections):
312321
let collectionDict = collections.reduce(into: [Model.CollectionIdentifier: Model.Collection]()) { result, collection in
313322
result[collection.identifier] = collection
314323
}
@@ -338,7 +347,14 @@ final class SQLitePackageCollectionsStorage: PackageCollectionsStorage, Closable
338347
.init(package: entry.value.package, collections: Array(entry.value.collections))
339348
})
340349
callback(.success(result))
341-
} else {
350+
}
351+
}
352+
} else {
353+
self.list(identifiers: identifiers) { result in
354+
switch result {
355+
case .failure(let error):
356+
callback(.failure(error))
357+
case .success(let collections):
342358
let queryString = query.lowercased()
343359
let collectionsPackages = collections.reduce([Model.CollectionIdentifier: [Model.Package]]()) { partial, collection in
344360
var map = partial
@@ -380,29 +396,38 @@ final class SQLitePackageCollectionsStorage: PackageCollectionsStorage, Closable
380396
func findPackage(identifier: PackageIdentity,
381397
collectionIdentifiers: [Model.CollectionIdentifier]?,
382398
callback: @escaping (Result<Model.PackageSearchResult.Item, Error>) -> Void) {
383-
self.list(identifiers: collectionIdentifiers) { result in
384-
switch result {
385-
case .failure(let error):
386-
return callback(.failure(error))
387-
case .success(let collections):
388-
if self.useSearchIndices.get() ?? false {
389-
var matches = [(collection: Model.CollectionIdentifier, package: PackageIdentity)]()
390-
do {
391-
let packageQuery = "SELECT collection_id_blob_base64, repository_url FROM \(Self.packagesFTSName) WHERE id = ?;"
392-
try self.executeStatement(packageQuery) { statement in
393-
try statement.bind([.string(identifier.description)])
399+
if self.useSearchIndices.get() ?? false {
400+
var matches = [(collection: Model.CollectionIdentifier, package: PackageIdentity)]()
401+
var matchingCollections = Set<Model.CollectionIdentifier>()
394402

395-
while let row = try statement.step() {
396-
if let collectionData = Data(base64Encoded: row.string(at: 0)),
397-
let collection = try? self.decoder.decode(Model.CollectionIdentifier.self, from: collectionData) {
398-
matches.append((collection: collection, package: PackageIdentity(url: row.string(at: 1))))
399-
}
400-
}
403+
do {
404+
let packageQuery = "SELECT collection_id_blob_base64, repository_url FROM \(Self.packagesFTSName) WHERE id = ?;"
405+
try self.executeStatement(packageQuery) { statement in
406+
try statement.bind([.string(identifier.description)])
407+
408+
while let row = try statement.step() {
409+
if let collectionData = Data(base64Encoded: row.string(at: 0)),
410+
let collection = try? self.decoder.decode(Model.CollectionIdentifier.self, from: collectionData) {
411+
matches.append((collection: collection, package: PackageIdentity(url: row.string(at: 1))))
412+
matchingCollections.insert(collection)
401413
}
402-
} catch {
403-
return callback(.failure(error))
404414
}
415+
}
416+
} catch {
417+
return callback(.failure(error))
418+
}
419+
420+
// Optimization: return early if no matches
421+
guard !matches.isEmpty else {
422+
return callback(.failure(NotFoundError("\(identifier)")))
423+
}
405424

425+
// Optimization: fetch only those collections that contain matching packages
426+
self.list(identifiers: Array(collectionIdentifiers.map { Set($0).intersection(matchingCollections) } ?? matchingCollections)) { result in
427+
switch result {
428+
case .failure(let error):
429+
return callback(.failure(error))
430+
case .success(let collections):
406431
let collectionDict = collections.reduce(into: [Model.CollectionIdentifier: Model.Collection]()) { result, collection in
407432
result[collection.identifier] = collection
408433
}
@@ -417,7 +442,14 @@ final class SQLitePackageCollectionsStorage: PackageCollectionsStorage, Closable
417442
}
418443

419444
callback(.success(.init(package: package, collections: collections.map { $0.identifier })))
420-
} else {
445+
}
446+
}
447+
} else {
448+
self.list(identifiers: collectionIdentifiers) { result in
449+
switch result {
450+
case .failure(let error):
451+
return callback(.failure(error))
452+
case .success(let collections):
421453
// sorting by collection processing date so the latest metadata is first
422454
let collectionPackages = collections.sorted(by: { lhs, rhs in lhs.lastProcessedAt > rhs.lastProcessedAt }).compactMap { collection in
423455
collection.packages
@@ -441,65 +473,96 @@ final class SQLitePackageCollectionsStorage: PackageCollectionsStorage, Closable
441473
callback: @escaping (Result<Model.TargetSearchResult, Error>) -> Void) {
442474
let query = query.lowercased()
443475

444-
self.list(identifiers: identifiers) { result in
445-
switch result {
446-
case .failure(let error):
447-
callback(.failure(error))
448-
case .success(let collections):
449-
// For each package, find the containing collections
450-
var packageCollections = [PackageIdentity: (package: Model.Package, collections: Set<Model.CollectionIdentifier>)]()
451-
// For each matching target, find the containing package version(s)
452-
var targetPackageVersions = [Model.Target: [PackageIdentity: Set<Model.TargetListResult.PackageVersion>]]()
453-
454-
if self.useSearchIndices.get() ?? false {
455-
var matches = [(collection: Model.CollectionIdentifier, package: PackageIdentity, targetName: String)]()
456-
// Trie is more performant for target search; use it if available
457-
if self.targetTrieReady.get() ?? false {
458-
do {
459-
switch type {
460-
case .exactMatch:
461-
try self.targetTrie.find(word: query).forEach {
462-
matches.append((collection: $0.collection, package: $0.package, targetName: query))
463-
}
464-
case .prefix:
465-
try self.targetTrie.findWithPrefix(query).forEach { targetName, collectionPackages in
466-
collectionPackages.forEach {
467-
matches.append((collection: $0.collection, package: $0.package, targetName: targetName))
468-
}
469-
}
476+
// For each package, find the containing collections
477+
var packageCollections = [PackageIdentity: (package: Model.Package, collections: Set<Model.CollectionIdentifier>)]()
478+
// For each matching target, find the containing package version(s)
479+
var targetPackageVersions = [Model.Target: [PackageIdentity: Set<Model.TargetListResult.PackageVersion>]]()
480+
481+
func buildResult() {
482+
// Sort by target name for consistent ordering in results
483+
let result = Model.TargetSearchResult(items: targetPackageVersions.sorted { $0.key.name < $1.key.name }.map { target, packageVersions in
484+
let targetPackages: [Model.TargetListItem.Package] = packageVersions.compactMap { identity, versions in
485+
guard let packageEntry = packageCollections[identity] else {
486+
return nil
487+
}
488+
return Model.TargetListItem.Package(
489+
repository: packageEntry.package.repository,
490+
summary: packageEntry.package.summary,
491+
versions: Array(versions).sorted(by: >),
492+
collections: Array(packageEntry.collections)
493+
)
494+
}
495+
return Model.TargetListItem(target: target, packages: targetPackages)
496+
})
497+
498+
callback(.success(result))
499+
}
500+
501+
if self.useSearchIndices.get() ?? false {
502+
var matches = [(collection: Model.CollectionIdentifier, package: PackageIdentity, targetName: String)]()
503+
var matchingCollections = Set<Model.CollectionIdentifier>()
504+
505+
// Trie is more performant for target search; use it if available
506+
if self.targetTrieReady.get() ?? false {
507+
do {
508+
switch type {
509+
case .exactMatch:
510+
try self.targetTrie.find(word: query).forEach {
511+
matches.append((collection: $0.collection, package: $0.package, targetName: query))
512+
matchingCollections.insert($0.collection)
513+
}
514+
case .prefix:
515+
try self.targetTrie.findWithPrefix(query).forEach { targetName, collectionPackages in
516+
collectionPackages.forEach {
517+
matches.append((collection: $0.collection, package: $0.package, targetName: targetName))
518+
matchingCollections.insert($0.collection)
470519
}
471-
} catch is NotFoundError {
472-
// Do nothing if no matches found
473-
} catch {
474-
return callback(.failure(error))
475520
}
476-
} else {
477-
do {
478-
let targetQuery = "SELECT collection_id_blob_base64, package_repository_url, name FROM \(Self.targetsFTSName) WHERE name LIKE ?;"
479-
try self.executeStatement(targetQuery) { statement in
480-
switch type {
481-
case .exactMatch:
482-
try statement.bind([.string("\(query)")])
483-
case .prefix:
484-
try statement.bind([.string("\(query)%")])
485-
}
521+
}
522+
} catch is NotFoundError {
523+
// Do nothing if no matches found
524+
} catch {
525+
return callback(.failure(error))
526+
}
527+
} else {
528+
do {
529+
let targetQuery = "SELECT collection_id_blob_base64, package_repository_url, name FROM \(Self.targetsFTSName) WHERE name LIKE ?;"
530+
try self.executeStatement(targetQuery) { statement in
531+
switch type {
532+
case .exactMatch:
533+
try statement.bind([.string("\(query)")])
534+
case .prefix:
535+
try statement.bind([.string("\(query)%")])
536+
}
486537

487-
while let row = try statement.step() {
488-
if let collectionData = Data(base64Encoded: row.string(at: 0)),
489-
let collection = try? self.decoder.decode(Model.CollectionIdentifier.self, from: collectionData) {
490-
matches.append((
491-
collection: collection,
492-
package: PackageIdentity(url: row.string(at: 1)),
493-
targetName: row.string(at: 2)
494-
))
495-
}
496-
}
538+
while let row = try statement.step() {
539+
if let collectionData = Data(base64Encoded: row.string(at: 0)),
540+
let collection = try? self.decoder.decode(Model.CollectionIdentifier.self, from: collectionData) {
541+
matches.append((
542+
collection: collection,
543+
package: PackageIdentity(url: row.string(at: 1)),
544+
targetName: row.string(at: 2)
545+
))
546+
matchingCollections.insert(collection)
497547
}
498-
} catch {
499-
return callback(.failure(error))
500548
}
501549
}
550+
} catch {
551+
return callback(.failure(error))
552+
}
553+
}
554+
555+
// Optimization: return early if no matches
556+
guard !matches.isEmpty else {
557+
return callback(.success(Model.TargetSearchResult(items: [])))
558+
}
502559

560+
// Optimization: fetch only those collections that contain matching packages
561+
self.list(identifiers: Array(identifiers.map { Set($0).intersection(matchingCollections) } ?? matchingCollections)) { result in
562+
switch result {
563+
case .failure(let error):
564+
return callback(.failure(error))
565+
case .success(let collections):
503566
let collectionDict = collections.reduce(into: [Model.CollectionIdentifier: Model.Collection]()) { result, collection in
504567
result[collection.identifier] = collection
505568
}
@@ -533,7 +596,16 @@ final class SQLitePackageCollectionsStorage: PackageCollectionsStorage, Closable
533596
}
534597
}
535598
}
536-
} else {
599+
600+
buildResult()
601+
}
602+
}
603+
} else {
604+
self.list(identifiers: identifiers) { result in
605+
switch result {
606+
case .failure(let error):
607+
callback(.failure(error))
608+
case .success(let collections):
537609
let collectionsPackages = collections.reduce([Model.CollectionIdentifier: [(target: Model.Target, package: Model.Package)]]()) { partial, collection in
538610
var map = partial
539611
collection.packages.forEach { package in
@@ -581,24 +653,9 @@ final class SQLitePackageCollectionsStorage: PackageCollectionsStorage, Closable
581653
}
582654
}
583655
}
584-
}
585656

586-
// Sort by target name for consistent ordering in results
587-
let result = Model.TargetSearchResult(items: targetPackageVersions.sorted { $0.key.name < $1.key.name }.map { target, packageVersions in
588-
let targetPackages: [Model.TargetListItem.Package] = packageVersions.compactMap { identity, versions in
589-
guard let packageEntry = packageCollections[identity] else {
590-
return nil
591-
}
592-
return Model.TargetListItem.Package(
593-
repository: packageEntry.package.repository,
594-
summary: packageEntry.package.summary,
595-
versions: Array(versions).sorted(by: >),
596-
collections: Array(packageEntry.collections)
597-
)
598-
}
599-
return Model.TargetListItem(target: target, packages: targetPackages)
600-
})
601-
callback(.success(result))
657+
buildResult()
658+
}
602659
}
603660
}
604661
}

0 commit comments

Comments
 (0)