fetch package metadata (#3041)

tomerd · web-flow · commit 7801f56c5b40 · 2020-11-11T11:57:24.000-08:00
motivation: implement fetch package metadata API

changes:
* add PackageMetadataProvider API
* add business logic to fetch metadata from collection sotre + provider and merge it
* add tests

open issue:  implement more efficient storage for packages
diff --git a/Sources/PackageCollections/Model/CVE.swift b/Sources/PackageCollections/Model/CVE.swift
@@ -12,7 +12,7 @@ import struct Foundation.URL
 
 extension PackageCollectionsModel {
     /// A representation of Common Vulnerabilities and Exposures (CVE)
-    public struct CVE {
+    public struct CVE: Equatable {
         /// CVE identifier
         public let identifier: String
 
diff --git a/Sources/PackageCollections/Model/Package.swift b/Sources/PackageCollections/Model/Package.swift
@@ -15,7 +15,7 @@ import SourceControl
 
 extension PackageCollectionsModel {
     /// Package metadata
-    public struct Package {
+    public struct Package: Equatable {
         /// Package reference
         public let reference: PackageReference
 
@@ -87,7 +87,7 @@ extension PackageCollectionsModel {
 
 extension PackageCollectionsModel.Package {
     /// A representation of package version
-    public struct Version {
+    public struct Version: Equatable {
         public typealias Target = PackageCollectionsModel.PackageTarget
         public typealias Product = PackageCollectionsModel.PackageProduct
 
@@ -114,9 +114,6 @@ extension PackageCollectionsModel.Package {
         /// The package version's Swift versions verified to work
         public let verifiedSwiftVersions: [SwiftLanguageVersion]?
 
-        /// The package version's CVEs
-        public let cves: [PackageCollectionsModel.CVE]?
-
         /// The package version's license
         public let license: PackageCollectionsModel.License?
     }
@@ -149,7 +146,7 @@ extension PackageCollectionsModel {
 
 extension PackageCollectionsModel.Package {
     /// A representation of package author
-    public struct Author {
+    public struct Author: Equatable, Codable {
         /// Author's username
         public let username: String
 
@@ -160,7 +157,7 @@ extension PackageCollectionsModel.Package {
         public let service: Service?
 
         /// A representation of user service
-        public struct Service {
+        public struct Service: Equatable, Codable {
             /// The service name
             public let name: String
         }
diff --git a/Sources/PackageCollections/Model/Search.swift b/Sources/PackageCollections/Model/Search.swift
@@ -18,7 +18,7 @@ extension PackageCollectionsModel {
         public struct Item {
             // Merged package metadata from across collections
             /// The matching package
-            public let package: PackageCollectionsModel.Package
+            public let package: PackageCollectionsModel.Collection.Package
 
             /// Package collections that contain the package
             public let collections: [PackageCollectionsModel.CollectionIdentifier]
diff --git a/Sources/PackageCollections/PackageCollections.swift b/Sources/PackageCollections/PackageCollections.swift
@@ -15,18 +15,27 @@ import TSCBasic
 public struct PackageCollections: PackageCollectionsProtocol {
     private let configuration: Configuration
     private let storage: Storage
-    private let providers: [PackageCollectionsModel.CollectionSourceType: PackageCollectionProvider]
+    private let collectionProviders: [PackageCollectionsModel.CollectionSourceType: PackageCollectionProvider]
+    private let metadataProvider: PackageMetadataProvider
 
-    init(configuration: Configuration, storage: Storage, providers: [PackageCollectionsModel.CollectionSourceType: PackageCollectionProvider]) {
+    init(configuration: Configuration,
+         storage: Storage,
+         collectionProviders: [PackageCollectionsModel.CollectionSourceType: PackageCollectionProvider],
+         metadataProvider: PackageMetadataProvider) {
         self.configuration = configuration
         self.storage = storage
-        self.providers = providers
+        self.collectionProviders = collectionProviders
+        self.metadataProvider = metadataProvider
     }
 
+    // MARK: - Profiles
+
     public func listProfiles(callback: @escaping (Result<[PackageCollectionsModel.Profile], Error>) -> Void) {
         self.storage.collectionsProfiles.listProfiles(callback: callback)
     }
 
+    // MARK: - Collections
+
     public func listCollections(identifiers: Set<PackageCollectionsModel.CollectionIdentifier>? = nil,
                                 in profile: PackageCollectionsModel.Profile? = nil,
                                 callback: @escaping (Result<[PackageCollectionsModel.Collection], Error>) -> Void) {
@@ -60,28 +69,6 @@ public struct PackageCollections: PackageCollectionsProtocol {
         }
     }
 
-    public func findPackages(
-        _ query: String,
-        collections: Set<PackageCollectionsModel.CollectionIdentifier>? = nil,
-        profile: PackageCollectionsModel.Profile? = nil,
-        callback: @escaping (Result<PackageCollectionsModel.PackageSearchResult, Error>) -> Void
-    ) {
-        let profile = profile ?? .default
-
-        self.storage.collectionsProfiles.listSources(in: profile) { result in
-            switch result {
-            case .failure(let error):
-                callback(.failure(error))
-            case .success(let sources):
-                let identifiers = sources.map { .init(from: $0) }.filter { collections?.contains($0) ?? true }
-                if identifiers.isEmpty {
-                    return callback(.success(PackageCollectionsModel.PackageSearchResult(items: [])))
-                }
-                self.storage.collections.searchPackages(identifiers: identifiers, query: query, callback: callback)
-            }
-        }
-    }
-
     public func refreshCollections(in profile: PackageCollectionsModel.Profile? = nil,
                                    callback: @escaping (Result<[PackageCollectionsModel.CollectionSource], Error>) -> Void) {
         let profile = profile ?? .default
@@ -179,7 +166,7 @@ public struct PackageCollections: PackageCollectionsProtocol {
         self.storage.collections.get(identifier: .init(from: source)) { result in
             switch result {
             case .failure:
-                guard let provider = self.providers[source.type] else {
+                guard let provider = self.collectionProviders[source.type] else {
                     return callback(.failure(UnknownProvider(source.type)))
                 }
                 provider.get(source, callback: callback)
@@ -189,6 +176,63 @@ public struct PackageCollections: PackageCollectionsProtocol {
         }
     }
 
+    // MARK: - Packages
+
+    public func findPackages(
+        _ query: String,
+        collections: Set<PackageCollectionsModel.CollectionIdentifier>? = nil,
+        profile: PackageCollectionsModel.Profile? = nil,
+        callback: @escaping (Result<PackageCollectionsModel.PackageSearchResult, Error>) -> Void
+    ) {
+        let profile = profile ?? .default
+
+        self.storage.collectionsProfiles.listSources(in: profile) { result in
+            switch result {
+            case .failure(let error):
+                callback(.failure(error))
+            case .success(let sources):
+                let identifiers = sources.map { .init(from: $0) }.filter { collections?.contains($0) ?? true }
+                if identifiers.isEmpty {
+                    return callback(.success(PackageCollectionsModel.PackageSearchResult(items: [])))
+                }
+                self.storage.collections.searchPackages(identifiers: identifiers, query: query, callback: callback)
+            }
+        }
+    }
+
+    // MARK: - Package Metadata
+
+    public func getPackageMetadata(_ reference: PackageReference,
+                                   profile: PackageCollectionsModel.Profile? = nil,
+                                   callback: @escaping (Result<PackageCollectionsModel.PackageMetadata, Error>) -> Void) {
+        let profile = profile ?? .default
+
+        // first find in storage
+        self.findPackage(identifier: reference.identity, profile: profile) { result in
+            switch result {
+            case .failure(let error):
+                callback(.failure(error))
+            case .success(let packageSearchResult):
+                // then try to get more metadata from provider (optional)
+                self.metadataProvider.get(reference: reference) { result in
+                    switch result {
+                    case .failure(let error):
+                        callback(.failure(error))
+                    case .success(let basicMetadata):
+                        // finally merge the results
+                        let metadata = PackageCollectionsModel.PackageMetadata(
+                            package: Self.mergedPackageMetadata(package: packageSearchResult.package, basicMetadata: basicMetadata),
+                            collections: packageSearchResult.collections
+                        )
+                        callback(.success(metadata))
+                    }
+                }
+            }
+        }
+    }
+
+    // MARK: - Targets
+
     public func listTargets(
         collections: Set<PackageCollectionsModel.CollectionIdentifier>? = nil,
         in profile: PackageCollectionsModel.Profile? = nil,
@@ -229,12 +273,7 @@ public struct PackageCollections: PackageCollectionsProtocol {
         }
     }
 
-    // FIXME:
-    public func getPackageMetadata(_ reference: PackageReference,
-                                   profile: PackageCollectionsModel.Profile? = nil,
-                                   callback: @escaping (Result<PackageCollectionsModel.PackageMetadata, Error>) -> Void) {
-        fatalError("not implemented")
-    }
+    // MARK: - Private
 
     // Fetch the collection from the network and store it in local storage
     // This helps avoid network access in normal operations
@@ -245,7 +284,7 @@ public struct PackageCollections: PackageCollectionsProtocol {
         if let errors = source.validate() {
             return callback(.failure(MultipleErrors(errors)))
         }
-        guard let provider = self.providers[source.type] else {
+        guard let provider = self.collectionProviders[source.type] else {
             return callback(.failure(UnknownProvider(source.type)))
         }
         provider.get(source) { result in
@@ -258,6 +297,27 @@ public struct PackageCollections: PackageCollectionsProtocol {
         }
     }
 
+    func findPackage(
+        identifier: PackageReference.PackageIdentity,
+        profile: PackageCollectionsModel.Profile? = nil,
+        callback: @escaping (Result<PackageCollectionsModel.PackageSearchResult.Item, Error>) -> Void
+    ) {
+        let profile = profile ?? .default
+
+        self.storage.collectionsProfiles.listSources(in: profile) { result in
+            switch result {
+            case .failure(let error):
+                callback(.failure(error))
+            case .success(let sources):
+                let identifiers = sources.map { PackageCollectionsModel.CollectionIdentifier(from: $0) }
+                if identifiers.isEmpty {
+                    return callback(.failure(NotFoundError("\(identifier)")))
+                }
+                self.storage.collections.findPackage(identifier: identifier, collectionIdentifiers: identifiers, callback: callback)
+            }
+        }
+    }
+
     private func targetListResultFromCollections(_ collections: [PackageCollectionsModel.Collection]) -> PackageCollectionsModel.TargetListResult {
         var packageCollections = [PackageReference: (package: PackageCollectionsModel.Collection.Package, collections: Set<PackageCollectionsModel.CollectionIdentifier>)]()
         var targetsPackages = [String: (target: PackageCollectionsModel.PackageTarget, packages: Set<PackageReference>)]()
@@ -285,15 +345,43 @@ public struct PackageCollections: PackageCollectionsProtocol {
                 .compactMap { packageCollections[$0] }
                 .map { pair -> PackageCollectionsModel.TargetListResult.Package in
                     let versions = pair.package.versions.map { PackageCollectionsModel.TargetListResult.PackageVersion(version: $0.version, packageName: $0.packageName) }
-                    return PackageCollectionsModel.TargetListResult.Package(repository: pair.package.repository,
-                                                                            description: pair.package.summary,
-                                                                            versions: versions,
-                                                                            collections: Array(pair.collections))
+                    return .init(repository: pair.package.repository,
+                                 description: pair.package.summary,
+                                 versions: versions,
+                                 collections: Array(pair.collections))
                 }
 
             return PackageCollectionsModel.TargetListItem(target: pair.target, packages: targetPackages)
         }
     }
+
+    internal static func mergedPackageMetadata(package: PackageCollectionsModel.Collection.Package,
+                                               basicMetadata: PackageCollectionsModel.PackageBasicMetadata?) -> PackageCollectionsModel.Package {
+        var versions = package.versions.map { packageVersion -> PackageCollectionsModel.Package.Version in
+            .init(version: packageVersion.version,
+                  packageName: packageVersion.packageName,
+                  targets: packageVersion.targets,
+                  products: packageVersion.products,
+                  toolsVersion: packageVersion.toolsVersion,
+                  verifiedPlatforms: packageVersion.verifiedPlatforms,
+                  verifiedSwiftVersions: packageVersion.verifiedSwiftVersions,
+                  license: packageVersion.license)
+        }
+
+        // uses TSCUtility.Version comparator
+        versions.sort(by: { lhs, rhs in lhs.version > rhs.version })
+        let latestVersion = versions.first
+
+        return .init(
+            repository: package.repository,
+            description: basicMetadata?.description ?? package.summary,
+            versions: versions,
+            latestVersion: latestVersion,
+            watchersCount: basicMetadata?.watchersCount,
+            readmeURL: basicMetadata?.readmeURL ?? package.readmeURL,
+            authors: basicMetadata?.authors
+        )
+    }
 }
 
 private struct UnknownProvider: Error {
diff --git a/Sources/PackageCollections/PackageMetadataProvider.swift b/Sources/PackageCollections/PackageMetadataProvider.swift
@@ -0,0 +1,35 @@
+/*
+ This source file is part of the Swift.org open source project
+
+ Copyright (c) 2020 Apple Inc. and the Swift project authors
+ Licensed under Apache License v2.0 with Runtime Library Exception
+
+ See http://swift.org/LICENSE.txt for license information
+ See http://swift.org/CONTRIBUTORS.txt for Swift project authors
+ */
+
+import struct Foundation.Date
+import struct Foundation.URL
+import PackageModel
+import TSCUtility
+
+/// `PackageBasicMetadata` provider
+protocol PackageMetadataProvider {
+    /// Retrieves metadata for a package at the given repository address.
+    ///
+    /// - Parameters:
+    ///   - reference: The package's reference
+    ///   - callback: The closure to invoke when result becomes available
+    func get(reference: PackageReference, callback: @escaping (Result<PackageCollectionsModel.PackageBasicMetadata?, Error>) -> Void)
+}
+
+extension PackageCollectionsModel {
+    struct PackageBasicMetadata: Equatable {
+        let description: String?
+        let versions: [TSCUtility.Version]
+        let watchersCount: Int?
+        let readmeURL: Foundation.URL?
+        let authors: [PackageCollectionsModel.Package.Author]?
+        let processedAt: Date
+    }
+}
diff --git a/Sources/PackageCollections/Storage/PackageCollectionsStorage.swift b/Sources/PackageCollections/Storage/PackageCollectionsStorage.swift
@@ -14,6 +14,7 @@ import struct Foundation.Data
 import class Foundation.JSONDecoder
 import class Foundation.JSONEncoder
 import struct Foundation.URL
+import PackageModel
 import TSCBasic
 import TSCUtility
 
@@ -62,6 +63,16 @@ public protocol PackageCollectionsStorage {
                         query: String,
                         callback: @escaping (Result<PackageCollectionsModel.PackageSearchResult, Error>) -> Void)
 
+    /// Returns optional `PackageSearchResult.Item` for the given package identity.
+    ///
+    /// - Parameters:
+    ///   - identifier: The package identifier
+    ///   - collectionIdentifiers: Optional. The identifiers of the `PackageCollection`s
+    ///   - callback: The closure to invoke when result becomes available
+    func findPackage(identifier: PackageReference.PackageIdentity,
+                     collectionIdentifiers: [PackageCollectionsModel.CollectionIdentifier]?,
+                     callback: @escaping (Result<PackageCollectionsModel.PackageSearchResult.Item, Error>) -> Void)
+
     /// Returns `TargetSearchResult` for the given search criteria.
     ///
     /// - Parameters:
@@ -233,6 +244,31 @@ final class SQLitePackageCollectionsStorage: PackageCollectionsStorage, Closable
         fatalError("not implemented")
     }
 
+    // FIXME: this is PoC for search, need a more performant version of this
+    func findPackage(identifier: PackageReference.PackageIdentity,
+                     collectionIdentifiers: [PackageCollectionsModel.CollectionIdentifier]?,
+                     callback: @escaping (Result<PackageCollectionsModel.PackageSearchResult.Item, Error>) -> Void) {
+        self.list(identifiers: collectionIdentifiers) { result in
+            switch result {
+            case .failure(let error):
+                return callback(.failure(error))
+            case .success(let collections):
+                // sorting by collection processing date so the latest metadata is first
+                let collectionPackages = collections.sorted(by: { lhs, rhs in lhs.lastProcessedAt > rhs.lastProcessedAt }).compactMap { collection in
+                    collection.packages
+                        .first(where: { $0.reference.identity == identifier })
+                        .flatMap { (collection: collection.identifier, package: $0) }
+                }
+                // first package should have latest processing date
+                guard let package = collectionPackages.first?.package else {
+                    return callback(.failure(NotFoundError("\(identifier)")))
+                }
+                let collections = collectionPackages.map { $0.collection }
+                callback(.success(.init(package: package, collections: collections)))
+            }
+        }
+    }
+
     // FIXME: implement this
     func searchTargets(identifiers: [PackageCollectionsModel.CollectionIdentifier]? = nil,
                        query: String,
diff --git a/Tests/PackageCollectionsTests/PackageCollectionsTests.swift b/Tests/PackageCollectionsTests/PackageCollectionsTests.swift
diff --git a/Tests/PackageCollectionsTests/Utility.swift b/Tests/PackageCollectionsTests/Utility.swift
