Handle paginated registry metadata responses

In [4.1 List Package
Releases](https://github.com/swiftlang/swift-package-manager/blob/main/Documentation/PackageRegistry/Registry.md#41-list-package-releases)
it is stated that a server may respond with a `Link` header that contains
a pointer to a subsequent page of results. SPM is not checking for this
link in the `Link` header and so if a registry returns paginated results
only the first page of versions is searched when resolving.

Respect the `next` link in the `Link` header by loading the next page of
results and building up a list of versions, continuing until there is no
`next` link present in the `Link` header of the last result.

Issue: #8215

Author: plemarquand

Sources/PackageMetadata/PackageMetadata.swift

@@ -352,7 +352,7 @@ public struct PackageSearchClient {
         ) { result in
             do {
                 let metadata = try result.get()
-                let alternateLocations = metadata.alternateLocations ?? []
+                let alternateLocations = metadata.alternateLocations
                 return completion(.success(Set(alternateLocations)))
             } catch {
                 return completion(.failure(error))

Sources/PackageRegistry/RegistryClient.swift

@@ -185,14 +185,16 @@ public final class RegistryClient: Cancellable {
         observabilityScope.emit(debug: "registry for \(package): \(registry)")
 
         let underlying = {
-            self._getPackageMetadata(
-                registry: registry,
-                package: registryIdentity,
-                timeout: timeout,
-                observabilityScope: observabilityScope,
-                callbackQueue: callbackQueue,
-                completion: completion
-            )
+            _ = Task {
+                let result = await self._getPackageMetadata(
+                    registry: registry,
+                    package: registryIdentity,
+                    timeout: timeout,
+                    observabilityScope: observabilityScope,
+                    callbackQueue: callbackQueue
+                )
+                completion(result)
+            }
         }
 
         if registry.supportsAvailability {
@@ -217,64 +219,106 @@ public final class RegistryClient: Cancellable {
         package: PackageIdentity.RegistryIdentity,
         timeout: DispatchTimeInterval?,
         observabilityScope: ObservabilityScope,
-        callbackQueue: DispatchQueue,
-        completion: @escaping (Result<PackageMetadata, Error>) -> Void
-    ) {
-        let completion = self.makeAsync(completion, on: callbackQueue)
-
+        callbackQueue: DispatchQueue
+    ) async -> Result<PackageMetadata, Error> {
         guard var components = URLComponents(url: registry.url, resolvingAgainstBaseURL: true) else {
-            return completion(.failure(RegistryError.invalidURL(registry.url)))
+            return .failure(RegistryError.invalidURL(registry.url))
         }
         components.appendPathComponents("\(package.scope)", "\(package.name)")
         guard let url = components.url else {
-            return completion(.failure(RegistryError.invalidURL(registry.url)))
+            return .failure(RegistryError.invalidURL(registry.url))
         }
 
-        let request = LegacyHTTPClient.Request(
-            method: .get,
-            url: url,
-            headers: [
-                "Accept": self.acceptHeader(mediaType: .json),
-            ],
-            options: self.defaultRequestOptions(timeout: timeout, callbackQueue: callbackQueue)
-        )
+        // If the responses are paginated then iterate until we've exasuasted all the pages and have a full versions list.
+        func iterateResponses(url: URL, existingMetadata: PackageMetadata) async -> Result<PackageMetadata, Error> {
+            let response = await self._getIndividualPackageMetadata(
+                url: url,
+                registry: registry,
+                package: package,
+                timeout: timeout,
+                observabilityScope: observabilityScope,
+                callbackQueue: callbackQueue
+            )
 
-        let start = DispatchTime.now()
-        observabilityScope.emit(info: "retrieving \(package) metadata from \(request.url)")
-        self.httpClient.execute(request, observabilityScope: observabilityScope, progress: nil) { result in
-            completion(
-                result.tryMap { response in
-                    observabilityScope
-                        .emit(
-                            debug: "server response for \(request.url): \(response.statusCode) in \(start.distance(to: .now()).descriptionInSeconds)"
-                        )
-                    switch response.statusCode {
-                    case 200:
-                        let packageMetadata = try response.parseJSON(
-                            Serialization.PackageMetadata.self,
-                            decoder: self.jsonDecoder
+            if case .success(let metadata) = response {
+                let mergedMetadata = PackageMetadata(
+                    registry: registry,
+                    versions: existingMetadata.versions + metadata.versions,
+                    alternateLocations: existingMetadata.alternateLocations.count > 0
+                        ? existingMetadata.alternateLocations
+                        : metadata.alternateLocations,
+                    nextPage: metadata.nextPage
+                )
+                if let nextPage = mergedMetadata.nextPage?.url {
+                    return await iterateResponses(url: nextPage, existingMetadata: mergedMetadata)
+                } else {
+                    return .success(
+                        PackageMetadata(
+                            registry: registry,
+                            versions: mergedMetadata.versions.sorted(by: >),
+                            alternateLocations: mergedMetadata.alternateLocations,
+                            nextPage: mergedMetadata.nextPage
                         )
+                    )
+                }
+            }
+            return response
+        }
 
-                        let versions = packageMetadata.releases.filter { $0.value.problem == nil }
-                            .compactMap { Version($0.key) }
-                            .sorted(by: >)
+        return await iterateResponses(url: url, existingMetadata: PackageMetadata(registry: registry, versions: [], alternateLocations: [], nextPage: nil))
+    }
 
-                        let alternateLocations = try response.headers.parseAlternativeLocationLinks()
+    private func _getIndividualPackageMetadata(
+        url: URL,
+        registry: Registry,
+        package: PackageIdentity.RegistryIdentity,
+        timeout: DispatchTimeInterval?,
+        observabilityScope: ObservabilityScope,
+        callbackQueue: DispatchQueue
+    ) async -> Result<PackageMetadata, Error> {
+        do {
+            let start = DispatchTime.now()
+            observabilityScope.emit(info: "retrieving \(package) metadata from \(url)")
+            let response = try await self.httpClient.get(
+                url,
+                headers: [
+                    "Accept": self.acceptHeader(mediaType: .json),
+                ],
+                options: self.defaultRequestOptions(timeout: timeout, callbackQueue: callbackQueue),
+                observabilityScope: observabilityScope
+            )
 
-                        return PackageMetadata(
-                            registry: registry,
-                            versions: versions,
-                            alternateLocations: alternateLocations?.map(\.url)
-                        )
-                    case 404:
-                        throw RegistryError.packageNotFound
-                    default:
-                        throw self.unexpectedStatusError(response, expectedStatus: [200, 404])
-                    }
-                }.mapError {
-                    RegistryError.failedRetrievingReleases(registry: registry, package: package.underlying, error: $0)
+            observabilityScope
+                .emit(
+                    debug: "server response for \(url): \(response.statusCode) in \(start.distance(to: .now()).descriptionInSeconds)"
+                )
+
+            switch response.statusCode {
+                case 200:
+                    let packageMetadata = try response.parseJSON(
+                        Serialization.PackageMetadata.self,
+                        decoder: self.jsonDecoder
+                    )
+
+                    let versions = packageMetadata.releases.filter { $0.value.problem == nil }
+                        .compactMap { Version($0.key) }
+
+                    let alternateLocations = response.headers.parseAlternativeLocationLinks()
+                    let paginationLinks = response.headers.parsePagniationLinks()
+
+                    return .success(PackageMetadata(
+                        registry: registry,
+                        versions: versions,
+                        alternateLocations: alternateLocations.map(\.url),
+                        nextPage: paginationLinks.first { $0.kind == .next }?.url
+                    ))
+                case 404:
+                    return .failure(RegistryError.failedRetrievingReleases(registry: registry, package: package.underlying, error: RegistryError.packageNotFound))
+                default:
+                    return .failure(RegistryError.failedRetrievingReleases(registry: registry, package: package.underlying, error: self.unexpectedStatusError(response, expectedStatus: [200, 404])))
                 }
-            )
+        } catch {
+            return .failure(RegistryError.failedRetrievingReleases(registry: registry, package: package.underlying, error: error))
         }
     }
 
@@ -2071,7 +2115,8 @@ extension RegistryClient {
     public struct PackageMetadata {
         public let registry: Registry
         public let versions: [Version]
-        public let alternateLocations: [SourceControlURL]?
+        public let alternateLocations: [SourceControlURL]
+        public let nextPage: SourceControlURL?
     }
 
     public struct PackageVersionMetadata: Sendable {
@@ -2142,6 +2187,17 @@ extension RegistryClient {
             case alternate
         }
     }
+
+    fileprivate struct NextLocationLink {
+        let url: SourceControlURL
+        let kind: Kind
+
+        enum Kind: String {
+            // Currently we only care about `next` for pagination, but there are several other values:
+            // https://github.com/swiftlang/swift-package-manager/blob/0340bb12a56f9696b3966ad82c2aee1594135377/Documentation/PackageRegistry/Registry.md?plain=1#L403-L411
+            case next
+        }
+    }
 }
 
 extension RegistryClient {
@@ -2263,20 +2319,16 @@ extension HTTPClientResponse {
 }
 
 extension HTTPClientHeaders {
-    /*
-     <https://github.com/mona/LinkedList>; rel="canonical",
-     <ssh://[email protected]:mona/LinkedList.git>; rel="alternate",
-      */
-    fileprivate func parseAlternativeLocationLinks() throws -> [RegistryClient.AlternativeLocationLink]? {
-        try self.get("Link").map { header -> [RegistryClient.AlternativeLocationLink] in
+    fileprivate func parseLink<T>(_ factory: (String) throws -> T?) rethrows -> [T] {
+        return try self.get("Link").map { header -> [T] in
             let linkLines = header.split(separator: ",").map(String.init).map { $0.spm_chuzzle() ?? $0 }
             return try linkLines.compactMap { linkLine in
-                try parseAlternativeLocationLine(linkLine)
+                try factory(linkLine)
             }
         }.flatMap { $0 }
     }
 
-    private func parseAlternativeLocationLine(_ value: String) throws -> RegistryClient.AlternativeLocationLink? {
+    fileprivate func parseLocationLine<T>(_ value: String, _ factory: (String, String) -> T?) -> T? {
         let fields = value.split(separator: ";")
             .map(String.init)
             .map { $0.spm_chuzzle() ?? $0 }
@@ -2290,16 +2342,60 @@ extension HTTPClientHeaders {
             return nil
         }
 
-        guard let rel = fields.first(where: { $0.hasPrefix("rel=") }).flatMap({ parseLinkFieldValue($0) }),
-              let kind = RegistryClient.AlternativeLocationLink.Kind(rawValue: rel)
+        guard let rel = fields.first(where: { $0.hasPrefix("rel=") }).flatMap({ parseLinkFieldValue($0) })
         else {
             return nil
         }
 
-        return RegistryClient.AlternativeLocationLink(
-            url: SourceControlURL(link),
-            kind: kind
-        )
+        return factory(link, rel)
+    }
+}
+
+extension HTTPClientHeaders {
+    /*
+    https://github.com/swiftlang/swift-package-manager/blob/0340bb12a56f9696b3966ad82c2aee1594135377/Documentation/PackageRegistry/Registry.md?plain=1#L395C1-L401C39
+    <https://github.com/mona/LinkedList>; rel="canonical",
+    <ssh://[email protected]:mona/LinkedList.git>; rel="alternate",
+    */
+    fileprivate func parseAlternativeLocationLinks() -> [RegistryClient.AlternativeLocationLink] {
+        self.parseLink(self.parseAlternativeLocationLine(_:))
+    }
+
+    private func parseAlternativeLocationLine(_ value: String) -> RegistryClient.AlternativeLocationLink? {
+        return parseLocationLine(value) { link, rel in
+            guard let kind = RegistryClient.AlternativeLocationLink.Kind(rawValue: rel) else {
+                return nil
+            }
+
+            return RegistryClient.AlternativeLocationLink(
+                url: SourceControlURL(link),
+                kind: kind
+            )
+        }
+    }
+}
+
+extension HTTPClientHeaders {
+    /*
+    https://github.com/swiftlang/swift-package-manager/blob/0340bb12a56f9696b3966ad82c2aee1594135377/Documentation/PackageRegistry/Registry.md?plain=1#L403-L411
+    <https://github.com/mona/LinkedList?page=2>; rel="next",
+    <ssh://[email protected]:mona/LinkedList.git?page=40>; rel="last",
+    */
+    fileprivate func parsePagniationLinks() -> [RegistryClient.NextLocationLink] {
+        self.parseLink(self.parsePaginationLine(_:))
+    }
+
+    private func parsePaginationLine(_ value: String) -> RegistryClient.NextLocationLink? {
+        return parseLocationLine(value) { link, rel in
+            guard let kind = RegistryClient.NextLocationLink.Kind(rawValue: rel) else {
+                return nil
+            }
+
+            return RegistryClient.NextLocationLink(
+                url: SourceControlURL(link),
+                kind: kind
+            )
+        }
     }
 }
 
@@ -2308,12 +2404,7 @@ extension HTTPClientHeaders {
      <http://packages.example.com/mona/LinkedList/1.1.1/Package.swift?swift-version=4>; rel="alternate"; filename="[email protected]"; swift-tools-version="4.0"
      */
     fileprivate func parseManifestLinks() throws -> [RegistryClient.ManifestLink] {
-        try self.get("Link").map { header -> [RegistryClient.ManifestLink] in
-            let linkLines = header.split(separator: ",").map(String.init).map { $0.spm_chuzzle() ?? $0 }
-            return try linkLines.compactMap { linkLine in
-                try parseManifestLinkLine(linkLine)
-            }
-        }.flatMap { $0 }
+        try self.parseLink(self.parseManifestLinkLine(_:))
     }
 
     private func parseManifestLinkLine(_ value: String) throws -> RegistryClient.ManifestLink? {

Tests/PackageRegistryTests/RegistryClientTests.swift

@@ -88,7 +88,7 @@ final class RegistryClientTests: XCTestCase {
         let registryClient = makeRegistryClient(configuration: configuration, httpClient: httpClient)
         let metadata = try await registryClient.getPackageMetadata(package: identity)
         XCTAssertEqual(metadata.versions, ["1.1.1", "1.0.0"])
-        XCTAssertEqual(metadata.alternateLocations!, [
+        XCTAssertEqual(metadata.alternateLocations, [
             SourceControlURL("https://github.com/mona/LinkedList"),
             SourceControlURL("ssh://[email protected]:mona/LinkedList.git"),
             SourceControlURL("[email protected]:mona/LinkedList.git"),