Add support for Netrc for Downloader

Sources/TSCUtility/CMakeLists.txt

@@ -20,6 +20,8 @@ add_library(TSCUtility
   IndexStore.swift
   InterruptHandler.swift
   JSONMessageStreamingParser.swift
+  misc.swift
+  Netrc.swift
   OSLog.swift
   PersistenceCache.swift
   PkgConfig.swift

Sources/TSCUtility/Downloader.swift

@@ -43,11 +43,13 @@ public protocol Downloader {
     /// - Parameters:
     ///   - url: The `URL` to the file to download.
     ///   - destination: The `AbsolutePath` to download the file to.
+    ///   - authorizationProvider: Optional provider supplying `Authorization` header to be added to `URLRequest`.
     ///   - progress: A closure to receive the download's progress as number of bytes.
     ///   - completion: A closure to be notifed of the completion of the download.
     func downloadFile(
         at url: Foundation.URL,
         to destination: AbsolutePath,
+        withAuthorizationProvider authorizationProvider: AuthorizationProviding?,
         progress: @escaping Progress,
         completion: @escaping Completion
     )
@@ -109,11 +111,18 @@ public final class FoundationDownloader: NSObject, Downloader {
     public func downloadFile(
         at url: Foundation.URL,
         to destination: AbsolutePath,
+        withAuthorizationProvider authorizationProvider: AuthorizationProviding? = nil,
         progress: @escaping Downloader.Progress,
         completion: @escaping Downloader.Completion
     ) {
         queue.addOperation {
-            let task = self.session.downloadTask(with: url)
+            var request = URLRequest(url: url)
+
+            if let authorization = authorizationProvider?.authorization(for: url) {
+                request.addValue(authorization, forHTTPHeaderField: "Authorization")
+            }
+
+            let task = self.session.downloadTask(with: request)
             let download = Download(
                 task: task,
                 destination: destination,

Sources/TSCUtility/Netrc.swift

@@ -0,0 +1,164 @@
+import Foundation
+import TSCBasic
+
+/// Supplies `Authorization` header, typically to be appended to `URLRequest`
+public protocol AuthorizationProviding {
+    /// Optional `Authorization` header, likely added to `URLRequest`
+    func authorization(for url: Foundation.URL) -> String?
+}
+
+extension AuthorizationProviding {
+    public func authorization(for url: Foundation.URL) -> String? {
+        return nil
+    }
+}
+
+#if os(Windows)
+// FIXME: - add support for Windows when regex function available
+#endif
+
+#if os(Linux)
+// FIXME: - add support for Linux when regex function available
+#endif
+
+#if os(macOS)
+/*
+ Netrc feature depends upon `NSTextCheckingResult.range(withName name: String) -> NSRange`,
+ which is only available in macOS 10.13+ at this time.
+ */
+@available (OSX 10.13, *)
+/// Container of parsed netrc connection settings
+public struct Netrc: AuthorizationProviding {
+    /// Representation of `machine` connection settings & `default` connection settings.
+    /// If `default` connection settings present, they will be last element.
+    public let machines: [Machine]
+
+    private init(machines: [Machine]) {
+        self.machines = machines
+    }
+
+    /// Basic authorization header string
+    /// - Parameter url: URI of network resource to be accessed
+    /// - Returns: (optional) Basic Authorization header string to be added to the request
+    public func authorization(for url: Foundation.URL) -> String? {
+        guard let index = machines.firstIndex(where: { $0.name == url.host }) ?? machines.firstIndex(where: { $0.isDefault }) else { return nil }
+        let machine = machines[index]
+        let authString = "\(machine.login):\(machine.password)"
+        guard let authData = authString.data(using: .utf8) else { return nil }
+        return "Basic \(authData.base64EncodedString())"
+    }
+
+    /// Reads file at path or default location, and returns parsed Netrc representation
+    /// - Parameter fileURL: Location of netrc file, defaults to `~/.netrc`
+    /// - Returns: `Netrc` container with parsed connection settings, or error
+    public static func load(fromFileAtPath filePath: AbsolutePath? = nil) -> Result<Netrc, Netrc.Error> {
+        let filePath = filePath ?? AbsolutePath("\(NSHomeDirectory())/.netrc")
+
+        guard FileManager.default.fileExists(atPath: filePath.pathString) else { return .failure(.fileNotFound(filePath)) }
+        guard FileManager.default.isReadableFile(atPath: filePath.pathString),
+              let fileContents = try? String(contentsOf: filePath.asURL, encoding: .utf8) else { return .failure(.unreadableFile(filePath)) }
+
+        return Netrc.from(fileContents)
+    }
+
+    /// Regex matching logic for deriving `Netrc` container from string content
+    /// - Parameter content: String text of netrc file
+    /// - Returns: `Netrc` container with parsed connection settings, or error
+    public static func from(_ content: String) -> Result<Netrc, Netrc.Error> {
+        let content = trimComments(from: content)
+        let regex = try! NSRegularExpression(pattern: RegexUtil.netrcPattern, options: [])
+        let matches = regex.matches(in: content, options: [], range: NSRange(content.startIndex..<content.endIndex, in: content))
+
+        let machines: [Machine] = matches.compactMap {
+            return Machine(for: $0, string: content, variant: "lp") ??
+                Machine(for: $0, string: content, variant: "pl")
+        }
+
+        if let defIndex = machines.firstIndex(where: { $0.isDefault }) {
+            guard defIndex == machines.index(before: machines.endIndex) else { return .failure(.invalidDefaultMachinePosition) }
+        }
+        guard machines.count > 0 else { return .failure(.machineNotFound) }
+        return .success(Netrc(machines: machines))
+    }
+    /// Utility method to trim comments from netrc content
+    /// - Parameter text: String text of netrc file
+    /// - Returns: String text of netrc file *sans* comments
+    private static func trimComments(from text: String) -> String {
+        let regex = try! NSRegularExpression(pattern: RegexUtil.comments, options: .anchorsMatchLines)
+        let nsString = text as NSString
+        let range = NSRange(location: 0, length: nsString.length)
+        let matches = regex.matches(in: text, range: range)
+        var trimmedCommentsText = text
+        matches.forEach {
+            trimmedCommentsText = trimmedCommentsText
+                .replacingOccurrences(of: nsString.substring(with: $0.range), with: "")
+        }
+        return trimmedCommentsText
+    }
+}
+
+@available (OSX 10.13, *)
+public extension Netrc {
+    enum Error: Swift.Error {
+        case invalidFilePath
+        case fileNotFound(AbsolutePath)
+        case unreadableFile(AbsolutePath)
+        case machineNotFound
+        case invalidDefaultMachinePosition
+    }
+
+    /// Representation of connection settings
+    /// - important: Default connection settings are stored in machine named `default`
+    struct Machine: Equatable {
+        public let name: String
+        public let login: String
+        public let password: String
+
+        public var isDefault: Bool {
+            return name == "default"
+        }
+
+        public init(name: String, login: String, password: String) {
+            self.name = name
+            self.login = login
+            self.password = password
+        }
+
+        init?(for match: NSTextCheckingResult, string: String, variant: String = "") {
+            guard let name = RegexUtil.Token.machine.capture(in: match, string: string) ?? RegexUtil.Token.default.capture(in: match, string: string),
+                let login = RegexUtil.Token.login.capture(prefix: variant, in: match, string: string),
+                let password = RegexUtil.Token.password.capture(prefix: variant, in: match, string: string) else {
+                    return nil
+            }
+            self = Machine(name: name, login: login, password: password)
+        }
+    }
+}
+
+@available (OSX 10.13, *)
+fileprivate enum RegexUtil {
+    @frozen fileprivate enum Token: String, CaseIterable {
+        case machine, login, password, account, macdef, `default`
+
+        func capture(prefix: String = "", in match: NSTextCheckingResult, string: String) -> String? {
+            guard let range = Range(match.range(withName: prefix + rawValue), in: string) else { return nil }
+            return String(string[range])
+        }
+    }
+
+    static let comments: String = "\\#[\\s\\S]*?.*$"
+    static let `default`: String = #"(?:\s*(?<default>default))"#
+    static let accountOptional: String = #"(?:\s*account\s+\S++)?"#
+    static let loginPassword: String = #"\#(namedTrailingCapture("login", prefix: "lp"))\#(accountOptional)\#(namedTrailingCapture("password", prefix: "lp"))"#
+    static let passwordLogin: String = #"\#(namedTrailingCapture("password", prefix: "pl"))\#(accountOptional)\#(namedTrailingCapture("login", prefix: "pl"))"#
+    static let netrcPattern = #"(?:(?:(\#(namedTrailingCapture("machine"))|\#(namedMatch("default"))))(?:\#(loginPassword)|\#(passwordLogin)))"#
+
+    static func namedMatch(_ string: String) -> String {
+        return #"(?:\s*(?<\#(string)>\#(string)))"#
+    }
+
+    static func namedTrailingCapture(_ string: String, prefix: String = "") -> String {
+        return #"\s*\#(string)\s+(?<\#(prefix + string)>\S++)"#
+    }
+}
+#endif

Tests/TSCUtilityTests/DownloaderTests.swift

@@ -18,6 +18,7 @@ import FoundationNetworking
 #endif
 
 class DownloaderTests: XCTestCase {
+
     func testSuccess() {
       // FIXME: Remove once https://github.com/apple/swift-corelibs-foundation/pull/2593 gets inside a toolchain.
       #if os(macOS)
@@ -72,6 +73,140 @@ class DownloaderTests: XCTestCase {
         }
       #endif
     }
+
+    #if os(macOS)
+    @available(OSX 10.13, *)
+    /// Netrc feature depends upon `NSTextCheckingResult.range(withName name: String) -> NSRange`,
+    /// which is only available in macOS 10.13+ at this time.
+    func testAuthenticatedSuccess() {
+        let netrcContent = "machine protected.downloader-tests.com login anonymous password qwerty"
+        guard case .success(let netrc) = Netrc.from(netrcContent) else {
+            return XCTFail("Cannot load netrc content")
+        }
+        let authData = "anonymous:qwerty".data(using: .utf8)!
+        let testAuthHeader = "Basic \(authData.base64EncodedString())"
+
+        let configuration = URLSessionConfiguration.default
+        configuration.protocolClasses = [MockAuthenticatingURLProtocol.self]
+        let downloader = FoundationDownloader(configuration: configuration)
+
+        mktmpdir { tmpdir in
+            let url = URL(string: "https://protected.downloader-tests.com/testBasics.zip")!
+            let destination = tmpdir.appending(component: "download")
+
+            let didStartLoadingExpectation = XCTestExpectation(description: "didStartLoading")
+            let progress50Expectation = XCTestExpectation(description: "progress50")
+            let progress100Expectation = XCTestExpectation(description: "progress100")
+            let successExpectation = XCTestExpectation(description: "success")
+            MockAuthenticatingURLProtocol.notifyDidStartLoading(for: url, completion: { didStartLoadingExpectation.fulfill() })
+
+            downloader.downloadFile(at: url, to: destination, withAuthorizationProvider: netrc, progress: { bytesDownloaded, totalBytesToDownload in
+
+                XCTAssertEqual(MockAuthenticatingURLProtocol.authenticationHeader(for: url), testAuthHeader)
+
+                switch (bytesDownloaded, totalBytesToDownload) {
+                case (512, 1024):
+                    progress50Expectation.fulfill()
+                case (1024, 1024):
+                    progress100Expectation.fulfill()
+                default:
+                    XCTFail("unexpected progress")
+                }
+            }, completion: { result in
+                switch result {
+                case .success:
+                    XCTAssert(localFileSystem.exists(destination))
+                    let bytes = ByteString(Array(repeating: 0xbe, count: 512) + Array(repeating: 0xef, count: 512))
+                    XCTAssertEqual(try! localFileSystem.readFileContents(destination), bytes)
+                    successExpectation.fulfill()
+                case .failure(let error):
+                    XCTFail("\(error)")
+                }
+            })
+
+            wait(for: [didStartLoadingExpectation], timeout: 1.0)
+
+            let response = HTTPURLResponse(url: url, statusCode: 200, httpVersion: "1.1", headerFields: [
+                "Content-Length": "1024"
+            ])!
+
+            MockAuthenticatingURLProtocol.sendResponse(response, for: url)
+            MockAuthenticatingURLProtocol.sendData(Data(repeating: 0xbe, count: 512), for: url)
+            wait(for: [progress50Expectation], timeout: 1.0)
+            MockAuthenticatingURLProtocol.sendData(Data(repeating: 0xef, count: 512), for: url)
+            wait(for: [progress100Expectation], timeout: 1.0)
+            MockAuthenticatingURLProtocol.sendCompletion(for: url)
+            wait(for: [successExpectation], timeout: 1.0)
+        }
+    }
+    #endif
+
+    #if os(macOS)
+    @available(OSX 10.13, *)
+    /// Netrc feature depends upon `NSTextCheckingResult.range(withName name: String) -> NSRange`,
+    /// which is only available in macOS 10.13+ at this time.
+    func testDefaultAuthenticationSuccess() {
+        let netrcContent = "default login default password default"
+        guard case .success(let netrc) = Netrc.from(netrcContent) else {
+            return XCTFail("Cannot load netrc content")
+        }
+        let authData = "default:default".data(using: .utf8)!
+        let testAuthHeader = "Basic \(authData.base64EncodedString())"
+
+        let configuration = URLSessionConfiguration.default
+        configuration.protocolClasses = [MockAuthenticatingURLProtocol.self]
+        let downloader = FoundationDownloader(configuration: configuration)
+
+        mktmpdir { tmpdir in
+            let url = URL(string: "https://restricted.downloader-tests.com/testBasics.zip")!
+            let destination = tmpdir.appending(component: "download")
+
+            let didStartLoadingExpectation = XCTestExpectation(description: "didStartLoading")
+            let progress50Expectation = XCTestExpectation(description: "progress50")
+            let progress100Expectation = XCTestExpectation(description: "progress100")
+            let successExpectation = XCTestExpectation(description: "success")
+            MockAuthenticatingURLProtocol.notifyDidStartLoading(for: url, completion: { didStartLoadingExpectation.fulfill() })
+
+            downloader.downloadFile(at: url, to: destination, withAuthorizationProvider: netrc, progress: { bytesDownloaded, totalBytesToDownload in
+
+                XCTAssertEqual(MockAuthenticatingURLProtocol.authenticationHeader(for: url), testAuthHeader)
+
+                switch (bytesDownloaded, totalBytesToDownload) {
+                case (512, 1024):
+                    progress50Expectation.fulfill()
+                case (1024, 1024):
+                    progress100Expectation.fulfill()
+                default:
+                    XCTFail("unexpected progress")
+                }
+            }, completion: { result in
+                switch result {
+                case .success:
+                    XCTAssert(localFileSystem.exists(destination))
+                    let bytes = ByteString(Array(repeating: 0xbe, count: 512) + Array(repeating: 0xef, count: 512))
+                    XCTAssertEqual(try! localFileSystem.readFileContents(destination), bytes)
+                    successExpectation.fulfill()
+                case .failure(let error):
+                    XCTFail("\(error)")
+                }
+            })
+
+            wait(for: [didStartLoadingExpectation], timeout: 1.0)
+
+            let response = HTTPURLResponse(url: url, statusCode: 200, httpVersion: "1.1", headerFields: [
+                "Content-Length": "1024"
+            ])!
+
+            MockAuthenticatingURLProtocol.sendResponse(response, for: url)
+            MockAuthenticatingURLProtocol.sendData(Data(repeating: 0xbe, count: 512), for: url)
+            wait(for: [progress50Expectation], timeout: 1.0)
+            MockAuthenticatingURLProtocol.sendData(Data(repeating: 0xef, count: 512), for: url)
+            wait(for: [progress100Expectation], timeout: 1.0)
+            MockAuthenticatingURLProtocol.sendCompletion(for: url)
+            wait(for: [successExpectation], timeout: 1.0)
+        }
+    }
+    #endif
 
     func testClientError() {
       // FIXME: Remove once https://github.com/apple/swift-corelibs-foundation/pull/2593 gets inside a toolchain.
@@ -208,6 +343,16 @@ private struct DummyError: Error {
 
 private typealias Action = () -> Void
 
+private class MockAuthenticatingURLProtocol: MockURLProtocol {
+
+    fileprivate static func authenticationHeader(for url: Foundation.URL) -> String? {
+        guard let instance = instance(for: url) else {
+            fatalError("url did not start loading")
+        }
+        return instance.request.allHTTPHeaderFields?["Authorization"]
+    }
+}
+
 private class MockURLProtocol: URLProtocol {
     private static var queue = DispatchQueue(label: "org.swift.swiftpm.basic-tests.mock-url-protocol")
     private static var observers: [Foundation.URL: Action] = [:]
@@ -309,6 +454,10 @@ private class MockURLProtocol: URLProtocol {
             Self.instances[url] = nil
         }
     }
+
+    fileprivate static func instance(for url: Foundation.URL) -> URLProtocol? {
+        return Self.instances[url]
+    }
 }
 
 class FailingFileSystem: FileSystem {