package metadata: add support for auth tokens (#3077)

motivate: gfithub emposes API rate limiting, using an auth token helps mitigate low non-auth limits

changes:
* expose auth token configuration that can be set by clients of libSwiftPM
* add functionality to GitHubPackageMetadataProvider to handle auth token
* add functionality to GitHubPackageMetadataProvider to handle api limit errors
* add tests

TODO:
* allow users to configure auth tokens via config files (follow up PR)

Author: tomerd

Sources/PackageCollections/PackageCollections+Configuration.swift

@@ -13,6 +13,15 @@ extension PackageCollections {
     public struct Configuration {
         // TODO: add configuration like mx size of feed, retries, etc
 
-        public init() {}
+        /// Auth tokens for the collections or metadata provider
+        public var authTokens: [AuthTokenType: String]?
+
+        public init(authTokens: [AuthTokenType: String]? = nil) {
+            self.authTokens = authTokens
+        }
     }
 }
+
+public enum AuthTokenType: Hashable {
+    case github(_ host: String)
+}

Sources/PackageCollections/PackageCollections.swift

@@ -27,8 +27,11 @@ public struct PackageCollections: PackageCollectionsProtocol {
     public init(configuration: Configuration = .init(), diagnosticsEngine: DiagnosticsEngine? = nil) {
         let storage = Storage(sources: FilePackageCollectionsSourcesStorage(diagnosticsEngine: diagnosticsEngine),
                               collections: SQLitePackageCollectionsStorage(diagnosticsEngine: diagnosticsEngine))
+
         let collectionProviders = [Model.CollectionSourceType.json: JSONPackageCollectionProvider(diagnosticsEngine: diagnosticsEngine)]
-        let metadataProvider = GitHubPackageMetadataProvider(diagnosticsEngine: diagnosticsEngine)
+
+        let metadataProvider = GitHubPackageMetadataProvider(configuration: .init(authTokens: configuration.authTokens),
+                                                             diagnosticsEngine: diagnosticsEngine)
 
         self.configuration = configuration
         self.diagnosticsEngine = diagnosticsEngine

Sources/PackageCollections/Providers/GitHubPackageMetadataProvider.swift

@@ -21,12 +21,15 @@ import TSCBasic
 struct GitHubPackageMetadataProvider: PackageMetadataProvider {
     public var name: String = "GitHub"
 
+    var configuration: Configuration
+
     private let httpClient: HTTPClient
     private let diagnosticsEngine: DiagnosticsEngine?
     private let decoder: JSONDecoder
     private let queue: DispatchQueue
 
-    init(httpClient: HTTPClient? = nil, diagnosticsEngine: DiagnosticsEngine? = nil) {
+    init(configuration: Configuration = .init(), httpClient: HTTPClient? = nil, diagnosticsEngine: DiagnosticsEngine? = nil) {
+        self.configuration = configuration
         self.httpClient = httpClient ?? Self.makeDefaultHTTPClient(diagnosticsEngine: diagnosticsEngine)
         self.diagnosticsEngine = diagnosticsEngine
         self.decoder = JSONDecoder.makeWithDefaults()
@@ -53,28 +56,52 @@ struct GitHubPackageMetadataProvider: PackageMetadataProvider {
 
             // get the main data
             sync.enter()
-            let options = self.makeRequestOptions(validResponseCodes: [200])
-            var headers = HTTPClientHeaders()
-            headers.add(name: "Accept", value: "application/vnd.github.mercy-preview+json")
-            httpClient.get(metadataURL, headers: headers, options: options) { result in
+            var metadataHeaders = self.makeRequestHeaders(metadataURL)
+            metadataHeaders.add(name: "Accept", value: "application/vnd.github.mercy-preview+json")
+            let metadataOptions = self.makeRequestOptions(validResponseCodes: [200, 401, 403, 404])
+            httpClient.get(metadataURL, headers: metadataHeaders, options: metadataOptions) { result in
                 defer { sync.leave() }
                 resultsLock.withLock {
                     results[metadataURL] = result
                 }
-                // if successful, fan out multiple API calls
-                if case .success = result {
-                    [tagsURL, contributorsURL, readmeURL].forEach { url in
-                        sync.enter()
-                        httpClient.get(url, options: options) { result in
-                            defer { sync.leave() }
-                            resultsLock.withLock {
-                                results[url] = result
+                if case .success(let response) = result {
+                    let apiLimit = response.headers.get("X-RateLimit-Limit").first.flatMap(Int.init) ?? -1
+                    let apiRemaining = response.headers.get("X-RateLimit-Remaining").first.flatMap(Int.init) ?? -1
+
+                    switch (response.statusCode, metadataHeaders.contains("Authorization"), apiRemaining) {
+                    case (_, _, 0):
+                        self.diagnosticsEngine?.emit(warning: "Exceeded API limits on \(metadataURL.host ?? metadataURL.absoluteString) (\(apiRemaining)/\(apiLimit)), consider configuring an API token for this service.")
+                        return callback(.failure(Errors.apiLimitsExceeded(metadataURL, apiLimit)))
+                    case (401, true, _):
+                        return callback(.failure(Errors.invalidAuthToken(metadataURL)))
+                    case (401, false, _):
+                        return callback(.failure(Errors.permissionDenied(metadataURL)))
+                    case (403, _, _):
+                        return callback(.failure(Errors.permissionDenied(metadataURL)))
+                    case (404, _, _):
+                        return callback(.failure(NotFoundError("\(baseURL)")))
+                    case (200, _, _):
+                        if apiRemaining < self.configuration.apiLimitWarningThreshold {
+                            self.diagnosticsEngine?.emit(warning: "Approaching API limits on \(metadataURL.host ?? metadataURL.absoluteString) (\(apiRemaining)/\(apiLimit)), consider configuring an API token for this service.")
+                        }
+                        // if successful, fan out multiple API calls
+                        [tagsURL, contributorsURL, readmeURL].forEach { url in
+                            sync.enter()
+                            var headers = self.makeRequestHeaders(url)
+                            headers.add(name: "Accept", value: "application/vnd.github.v3+json")
+                            let options = self.makeRequestOptions(validResponseCodes: [200])
+                            httpClient.get(url, headers: headers, options: options) { result in
+                                defer { sync.leave() }
+                                resultsLock.withLock {
+                                    results[url] = result
+                                }
                             }
                         }
+                    default:
+                        return callback(.failure(Errors.invalidResponse(metadataURL, "Invalid status code: \(response.statusCode)")))
                     }
                 }
             }
-
             sync.wait()
 
             // process results
@@ -83,14 +110,12 @@ struct GitHubPackageMetadataProvider: PackageMetadataProvider {
                 // check for main request error state
                 switch results[metadataURL] {
                 case .none:
-                    throw Errors.invalidResponse(metadataURL)
-                case .some(.failure(let error)) where error as? HTTPClientError == .badResponseStatusCode(404):
-                    throw NotFoundError("\(baseURL)")
+                    throw Errors.invalidResponse(metadataURL, "Response missing")
                 case .some(.failure(let error)):
                     throw error
                 case .some(.success(let metadataResponse)):
                     guard let metadata = try metadataResponse.decodeBody(GetRepositoryResponse.self, using: self.decoder) else {
-                        throw Errors.invalidResponse(metadataURL)
+                        throw Errors.invalidResponse(metadataURL, "Empty body")
                     }
                     let tags = try results[tagsURL]?.success?.decodeBody([Tag].self, using: self.decoder) ?? []
                     let contributors = try results[contributorsURL]?.success?.decodeBody([Contributor].self, using: self.decoder)
@@ -140,6 +165,14 @@ struct GitHubPackageMetadataProvider: PackageMetadataProvider {
         return options
     }
 
+    private func makeRequestHeaders(_ url: URL) -> HTTPClientHeaders {
+        var headers = HTTPClientHeaders()
+        if let host = url.host, let token = self.configuration.authTokens?[.github(host)] {
+            headers.add(name: "Authorization", value: "token \(token)")
+        }
+        return headers
+    }
+
     private static func makeDefaultHTTPClient(diagnosticsEngine: DiagnosticsEngine?) -> HTTPClient {
         var client = HTTPClient(diagnosticsEngine: diagnosticsEngine)
         // TODO: make these defaults configurable?
@@ -149,10 +182,24 @@ struct GitHubPackageMetadataProvider: PackageMetadataProvider {
         return client
     }
 
+    public struct Configuration {
+        public var apiLimitWarningThreshold: Int
+        public var authTokens: [AuthTokenType: String]?
+
+        public init(authTokens: [AuthTokenType: String]? = nil,
+                    apiLimitWarningThreshold: Int? = nil) {
+            self.authTokens = authTokens
+            self.apiLimitWarningThreshold = apiLimitWarningThreshold ?? 5
+        }
+    }
+
     enum Errors: Error, Equatable {
         case invalidReferenceType(PackageReference)
         case invalidGitUrl(String)
-        case invalidResponse(URL)
+        case invalidResponse(URL, String)
+        case permissionDenied(URL)
+        case invalidAuthToken(URL)
+        case apiLimitsExceeded(URL, Int)
     }
 }
 

Sources/PackageCollections/Providers/JSONPackageCollectionProvider.swift

@@ -28,7 +28,7 @@ struct JSONPackageCollectionProvider: PackageCollectionProvider {
     init(configuration: Configuration = .init(), httpClient: HTTPClient? = nil, diagnosticsEngine: DiagnosticsEngine? = nil) {
         self.configuration = configuration
         self.diagnosticsEngine = diagnosticsEngine
-        self.httpClient = httpClient ?? Self.makeDefaultHTTPClient(diagnosticsEngine: diagnosticsEngine)        
+        self.httpClient = httpClient ?? Self.makeDefaultHTTPClient(diagnosticsEngine: diagnosticsEngine)
         self.decoder = JSONDecoder.makeWithDefaults()
     }
 

Sources/PackageCollections/Storage/FilePackageCollectionsSourcesStorage.swift

@@ -32,7 +32,7 @@ struct FilePackageCollectionsSourcesStorage: PackageCollectionsSourcesStorage {
         let name = "collections"
         self.path = path ?? fileSystem.dotSwiftPM.appending(components: "config", "\(name).json")
         self.diagnosticsEngine = diagnosticsEngine
-        self.encoder = JSONEncoder.makeWithDefaults()        
+        self.encoder = JSONEncoder.makeWithDefaults()
         self.decoder = JSONDecoder.makeWithDefaults()
     }
 

Tests/PackageCollectionsTests/GitHubPackageMetadataProviderTests.swift

@@ -89,19 +89,17 @@ class GitHubPackageMetadataProviderTests: XCTestCase {
     func testRepoNotFound() throws {
         let repoURL = "https://github.com/octocat/Hello-World.git"
 
-        fixture(name: "Collections") { _ in
-            let handler = { (_: HTTPClient.Request, callback: @escaping (Result<HTTPClient.Response, Error>) -> Void) in
-                callback(.success(.init(statusCode: 404)))
-            }
+        let handler = { (_: HTTPClient.Request, callback: @escaping (Result<HTTPClient.Response, Error>) -> Void) in
+            callback(.success(.init(statusCode: 404)))
+        }
 
-            var httpClient = HTTPClient(handler: handler)
-            httpClient.configuration.circuitBreakerStrategy = .none
-            httpClient.configuration.retryStrategy = .none
-            let provider = GitHubPackageMetadataProvider(httpClient: httpClient)
-            let reference = PackageReference(repository: RepositorySpecifier(url: repoURL))
-            XCTAssertThrowsError(try tsc_await { callback in provider.get(reference, callback: callback) }, "should throw error") { error in
-                XCTAssert(error is NotFoundError, "\(error)")
-            }
+        var httpClient = HTTPClient(handler: handler)
+        httpClient.configuration.circuitBreakerStrategy = .none
+        httpClient.configuration.retryStrategy = .none
+        let provider = GitHubPackageMetadataProvider(httpClient: httpClient)
+        let reference = PackageReference(repository: RepositorySpecifier(url: repoURL))
+        XCTAssertThrowsError(try tsc_await { callback in provider.get(reference, callback: callback) }, "should throw error") { error in
+            XCTAssert(error is NotFoundError, "\(error)")
         }
     }
 
@@ -110,11 +108,11 @@ class GitHubPackageMetadataProviderTests: XCTestCase {
         let apiURL = URL(string: "https://api.github.com/repos/octocat/Hello-World")!
 
         fixture(name: "Collections") { directoryPath in
+            let path = directoryPath.appending(components: "GitHub", "metadata.json")
+            let data = try Data(localFileSystem.readFileContents(path).contents)
             let handler = { (request: HTTPClient.Request, callback: @escaping (Result<HTTPClient.Response, Error>) -> Void) in
                 switch (request.method, request.url) {
                 case (.get, apiURL):
-                    let path = directoryPath.appending(components: "GitHub", "metadata.json")
-                    let data = Data(try! localFileSystem.readFileContents(path).contents)
                     callback(.success(.init(statusCode: 200,
                                             headers: .init([.init(name: "Content-Length", value: "\(data.count)")]),
                                             body: data)))
@@ -138,6 +136,93 @@ class GitHubPackageMetadataProviderTests: XCTestCase {
         }
     }
 
+    func testPermissionDenied() throws {
+        let repoURL = "https://github.com/octocat/Hello-World.git"
+        let apiURL = URL(string: "https://api.github.com/repos/octocat/Hello-World")!
+
+        let handler = { (_: HTTPClient.Request, callback: @escaping (Result<HTTPClient.Response, Error>) -> Void) in
+            callback(.success(.init(statusCode: 401)))
+        }
+
+        var httpClient = HTTPClient(handler: handler)
+        httpClient.configuration.circuitBreakerStrategy = .none
+        httpClient.configuration.retryStrategy = .none
+        let provider = GitHubPackageMetadataProvider(httpClient: httpClient)
+        let reference = PackageReference(repository: RepositorySpecifier(url: repoURL))
+        XCTAssertThrowsError(try tsc_await { callback in provider.get(reference, callback: callback) }, "should throw error") { error in
+            XCTAssertEqual(error as? GitHubPackageMetadataProvider.Errors, .permissionDenied(apiURL))
+        }
+    }
+
+    func testInvalidAuthToken() throws {
+        let repoURL = "https://github.com/octocat/Hello-World.git"
+        let apiURL = URL(string: "https://api.github.com/repos/octocat/Hello-World")!
+        let authTokens = [AuthTokenType.github("api.github.com"): "foo"]
+
+        let handler = { (request: HTTPClient.Request, callback: @escaping (Result<HTTPClient.Response, Error>) -> Void) in
+            if request.headers.get("Authorization").first == "token \(authTokens.first!.value)" {
+                callback(.success(.init(statusCode: 401)))
+            } else {
+                XCTFail("expected correct authorization header")
+                callback(.success(.init(statusCode: 500)))
+            }
+        }
+
+        var httpClient = HTTPClient(handler: handler)
+        httpClient.configuration.circuitBreakerStrategy = .none
+        httpClient.configuration.retryStrategy = .none
+        var provider = GitHubPackageMetadataProvider(httpClient: httpClient)
+        provider.configuration.authTokens = authTokens
+        let reference = PackageReference(repository: RepositorySpecifier(url: repoURL))
+        XCTAssertThrowsError(try tsc_await { callback in provider.get(reference, callback: callback) }, "should throw error") { error in
+            XCTAssertEqual(error as? GitHubPackageMetadataProvider.Errors, .invalidAuthToken(apiURL))
+        }
+    }
+
+    func testAPILimit() throws {
+        let repoURL = "https://github.com/octocat/Hello-World.git"
+        let apiURL = URL(string: "https://api.github.com/repos/octocat/Hello-World")!
+
+        let total = 5
+        var remaining = total
+
+        fixture(name: "Collections") { directoryPath in
+            let path = directoryPath.appending(components: "GitHub", "metadata.json")
+            let data = try Data(localFileSystem.readFileContents(path).contents)
+            let handler = { (request: HTTPClient.Request, callback: @escaping (Result<HTTPClient.Response, Error>) -> Void) in
+                var headers = HTTPClientHeaders()
+                headers.add(name: "X-RateLimit-Limit", value: "\(total)")
+                headers.add(name: "X-RateLimit-Remaining", value: "\(remaining)")
+                if remaining == 0 {
+                    callback(.success(.init(statusCode: 403, headers: headers)))
+                } else if request.url == apiURL {
+                    remaining = remaining - 1
+                    headers.add(name: "Content-Length", value: "\(data.count)")
+                    callback(.success(.init(statusCode: 200,
+                                            headers: headers,
+                                            body: data)))
+                } else {
+                    callback(.success(.init(statusCode: 500)))
+                }
+            }
+
+            var httpClient = HTTPClient(handler: handler)
+            httpClient.configuration.circuitBreakerStrategy = .none
+            httpClient.configuration.retryStrategy = .none
+            let provider = GitHubPackageMetadataProvider(httpClient: httpClient)
+            let reference = PackageReference(repository: RepositorySpecifier(url: repoURL))
+            for index in 0 ... total * 2 {
+                if index >= total {
+                    XCTAssertThrowsError(try tsc_await { callback in provider.get(reference, callback: callback) }, "should throw error") { error in
+                        XCTAssertEqual(error as? GitHubPackageMetadataProvider.Errors, .apiLimitsExceeded(apiURL, total))
+                    }
+                } else {
+                    XCTAssertNoThrow(try tsc_await { callback in provider.get(reference, callback: callback) })
+                }
+            }
+        }
+    }
+
     func testInvalidURL() throws {
         fixture(name: "Collections") { _ in
             let provider = GitHubPackageMetadataProvider()
@@ -169,13 +254,21 @@ class GitHubPackageMetadataProviderTests: XCTestCase {
         var httpClient = HTTPClient()
         httpClient.configuration.circuitBreakerStrategy = .none
         httpClient.configuration.retryStrategy = .none
-        let provider = GitHubPackageMetadataProvider(httpClient: httpClient)
+        httpClient.configuration.requestHeaders = .init()
+        httpClient.configuration.requestHeaders!.add(name: "Cache-Control", value: "no-cache")
+        var configuration = GitHubPackageMetadataProvider.Configuration()
+        if let token = ProcessEnv.vars["GITHUB_API_TOKEN"] {
+            configuration.authTokens = [.github("api.github.com"): token]
+        }
+        configuration.apiLimitWarningThreshold = 50
+        let provider = GitHubPackageMetadataProvider(configuration: configuration, httpClient: httpClient)
         let reference = PackageReference(repository: RepositorySpecifier(url: repoURL))
-        let metadata = try tsc_await { callback in provider.get(reference, callback: callback) }
-
-        XCTAssertNotNil(metadata)
-        XCTAssert(metadata.versions.count > 0)
-        XCTAssert(metadata.keywords!.count > 0)
-        XCTAssert(metadata.authors!.count > 0)
+        for _ in 0 ... 60 {
+            let metadata = try tsc_await { callback in provider.get(reference, callback: callback) }
+            XCTAssertNotNil(metadata)
+            XCTAssert(metadata.versions.count > 0)
+            XCTAssert(metadata.keywords!.count > 0)
+            XCTAssert(metadata.authors!.count > 0)
+        }
     }
 }