Add support for Netrc for Downloader (cherry-pick #2833)

Package.swift

@@ -233,7 +233,7 @@ let package = Package(
 if ProcessInfo.processInfo.environment["SWIFTPM_LLBUILD_FWK"] == nil {
     if ProcessInfo.processInfo.environment["SWIFTCI_USE_LOCAL_DEPS"] == nil {
         package.dependencies += [
-            .package(url: "https://github.com/apple/swift-llbuild.git", .branch("master")),
+            .package(url: "https://github.com/apple/swift-llbuild.git", .branch("release/5.3")),
         ]
     } else {
         // In Swift CI, use a local path to llbuild to interoperate with tools

Sources/Commands/Options.swift

@@ -35,6 +35,18 @@ public class ToolOptions {
 
     /// Enable prefetching in resolver which will kick off parallel git cloning.
     public var shouldEnableResolverPrefetching = true
+
+    /// Tells `Workspace` to attempt to locate .netrc file at HOME, or designated path.
+    public var netrc: Bool = false
+
+    /// Similar to `--netrc`, but this option makes the .netrc usage optional and not mandatory as the `--netrc` option.
+    public var netrcOptional: Bool = false
+
+    /// The path to the netrc file which should be use for authentication when downloading binary target artifacts.
+    /// Similar to `--netrc`, except that you also provide the path to the actual file to use.
+    /// This is useful when you want to provide the information in another directory or with another file name.
+    /// Respects `--netrcOptional` option.
+    public var netrcFilePath: AbsolutePath?
 
     /// If print version option was passed.
     public var shouldPrintVersion: Bool = false

Sources/Commands/SwiftTool.swift

@@ -10,6 +10,7 @@
 
 import func Foundation.NSUserName
 import class Foundation.ProcessInfo
+import func Foundation.NSHomeDirectory
 import Dispatch
 
 import TSCLibc
@@ -459,6 +460,24 @@ public class SwiftTool<Options: ToolOptions> {
         binder.bind(
             option: parser.add(option: "--build-system", kind: BuildSystemKind.self, usage: nil),
             to: { $0._buildSystem = $1 })
+
+        binder.bind(
+            option: parser.add(
+                option: "--netrc", kind: Bool.self, 
+                usage: "Makes SPM scan the .netrc file in the user's home directory for login name and password when downloading binary target artifacts."),
+            to: { $0.netrc = $1 })
+
+        binder.bind(
+            option: parser.add(
+                option: "--netrc-optional", kind: Bool.self,
+                usage: "Makes the .netrc usage optional and not mandatory as with the --netrc option.  May modify --netrc or --netrc-file."),
+            to: { $0.netrcOptional = $1 })
+
+        binder.bind(
+            option: parser.add(
+                option: "--netrc-file", kind: PathArgument.self,
+                usage: "The path to the netrc file which should be use for authentication when downloading binary target artifacts."),
+            to: { $0.netrcFilePath = $1.path })
 
         // Let subclasses bind arguments.
         type(of: self).defineArguments(parser: parser, binder: binder)
@@ -544,6 +563,21 @@ public class SwiftTool<Options: ToolOptions> {
         if result.exists(arg: "--arch") && result.exists(arg: "--triple") {
             diagnostics.emit(.mutuallyExclusiveArgumentsError(arguments: ["--arch", "--triple"]))
         }
+
+        if result.exists(arg: "--netrc") ||
+            result.exists(arg: "--netrc-file") ||
+            result.exists(arg: "--netrc-optional") {
+            // .netrc feature only supported on macOS >=10.13
+            #if os(macOS)
+            if #available(macOS 10.13, *) {
+                // ok, check succeeds
+            } else {
+                diagnostics.emit(error: ".netrc options are only supported on macOS >=10.13")
+            }
+            #else
+            diagnostics.emit(error: ".netrc options are only supported on macOS >=10.13")
+            #endif
+        }
     }
 
     class func defineArguments(parser: ArgumentParser, binder: ArgumentBinder<Options>) {
@@ -583,6 +617,24 @@ public class SwiftTool<Options: ToolOptions> {
     private lazy var _swiftpmConfig: Result<SwiftPMConfig, Swift.Error> = {
         return Result(catching: { SwiftPMConfig(path: try configFilePath()) })
     }()
+
+    func resolvedNetrcFilePath() -> AbsolutePath? {
+        guard options.netrc ||
+                options.netrcFilePath != nil ||
+                options.netrcOptional else { return nil }
+
+        let resolvedPath: AbsolutePath = options.netrcFilePath ?? AbsolutePath("\(NSHomeDirectory())/.netrc")
+        guard localFileSystem.exists(resolvedPath) else {
+            if !options.netrcOptional {
+                diagnostics.emit(error: "Cannot find mandatory .netrc file at \(resolvedPath.pathString).  To make .netrc file optional, use --netrc-optional flag.")
+                SwiftTool.exit(with: .failure)
+            } else {
+                diagnostics.emit(warning: "Did not find optional .netrc file at \(resolvedPath.pathString).")
+                return nil
+            }
+        }
+        return resolvedPath
+    }
 
     /// Holds the currently active workspace.
     ///
@@ -608,6 +660,7 @@ public class SwiftTool<Options: ToolOptions> {
             delegate: delegate,
             config: try getSwiftPMConfig(),
             repositoryProvider: provider,
+            netrcFilePath: resolvedNetrcFilePath(),
             isResolverPrefetchingEnabled: options.shouldEnableResolverPrefetching,
             skipUpdate: options.skipDependencyUpdate,
             enableResolverTrace: options.enableResolverTrace

Sources/SPMTestSupport/MockDownloader.swift

@@ -49,6 +49,7 @@ public class MockDownloader: Downloader {
     public func downloadFile(
         at url: Foundation.URL,
         to destinationPath: AbsolutePath,
+        withAuthorizationProvider authorizationProvider: AuthorizationProviding? = nil,
         progress: @escaping Downloader.Progress,
         completion: @escaping Downloader.Completion
     ) {

Sources/Workspace/Workspace.swift

@@ -326,6 +326,8 @@ public class Workspace {
 
     /// The downloader used for downloading binary artifacts.
     fileprivate let downloader: Downloader
+
+    fileprivate let netrcFilePath: AbsolutePath?
 
     /// The downloader used for unarchiving binary artifacts.
     fileprivate let archiver: Archiver
@@ -376,6 +378,7 @@ public class Workspace {
         fileSystem: FileSystem = localFileSystem,
         repositoryProvider: RepositoryProvider = GitRepositoryProvider(),
         downloader: Downloader = FoundationDownloader(),
+        netrcFilePath: AbsolutePath? = nil,
         archiver: Archiver = ZipArchiver(),
         checksumAlgorithm: HashAlgorithm = SHA256(),
         additionalFileRules: [FileRuleDescription] = [],
@@ -392,6 +395,7 @@ public class Workspace {
         self.currentToolsVersion = currentToolsVersion
         self.toolsVersionLoader = toolsVersionLoader
         self.downloader = downloader
+        self.netrcFilePath = netrcFilePath
         self.archiver = archiver
         self.checksumAlgorithm = checksumAlgorithm
         self.isResolverPrefetchingEnabled = isResolverPrefetchingEnabled
@@ -1366,7 +1370,15 @@ extension Workspace {
     private func download(_ artifacts: [ManagedArtifact], diagnostics: DiagnosticsEngine) {
         let group = DispatchGroup()
         let tempDiagnostics = DiagnosticsEngine()
-
+
+        var authProvider: AuthorizationProviding? = nil
+        #if os(macOS)
+        // Netrc feature currently only supported on macOS 10.13+ due to dependency
+        // on NSTextCheckingResult.range(with:)
+        if #available(macOS 10.13, *) {
+            authProvider = try? Netrc.load(fromFileAtPath: netrcFilePath).get()
+        }
+        #endif
         for artifact in artifacts {
             group.enter()
 
@@ -1385,9 +1397,11 @@ extension Workspace {
 
             let parsedURL = URL(string: url)!
             let archivePath = parentDirectory.appending(component: parsedURL.lastPathComponent)
+
             downloader.downloadFile(
                 at: parsedURL,
                 to: archivePath,
+                withAuthorizationProvider: authProvider,
                 progress: { bytesDownloaded, totalBytesToDownload in
                     self.delegate?.downloadingBinaryArtifact(
                         from: url,

Tests/CommandsTests/PackageToolTests.swift

@@ -41,6 +41,90 @@ final class PackageToolTests: XCTestCase {
     func testVersion() throws {
         XCTAssert(try execute(["--version"]).stdout.contains("Swift Package Manager"))
     }
+
+    func testNetrcSupportedOS() throws {
+        func verifyUnsupportedOSThrows() {
+            do {
+                // should throw and be caught
+                try execute(["update", "--netrc-file", "/Users/me/.hidden/.netrc"])
+                XCTFail()
+            } catch {
+                XCTAssert(true)
+            }
+        }
+        #if os(macOS)
+        if #available(macOS 10.13, *) {
+            // should succeed
+            XCTAssert(try execute(["--netrc"]).stdout.contains("USAGE: swift package"))
+            XCTAssert(try execute(["--netrc-file", "/Users/me/.hidden/.netrc"]).stdout.contains("USAGE: swift package"))
+            XCTAssert(try execute(["--netrc-optional"]).stdout.contains("USAGE: swift package"))
+        } else {
+            verifyUnsupportedOSThrows()
+        }
+        #else
+            verifyUnsupportedOSThrows()
+        #endif
+    }
+
+    func testNetrcFile() throws {
+        #if os(macOS)
+        if #available(macOS 10.13, *) {
+            // SUPPORTED OS
+            fixture(name: "DependencyResolution/External/Complex") { prefix in
+                let packageRoot = prefix.appending(component: "app")
+
+                let fs = localFileSystem
+                let netrcPath = prefix.appending(component: ".netrc")
+                try fs.writeFileContents(netrcPath) { stream in
+                    stream <<< "machine mymachine.labkey.org login [email protected] password mypassword"
+                }
+
+                do {
+                    // file at correct location
+                    try execute(["--netrc-file", netrcPath.pathString, "resolve"], packagePath: packageRoot)
+                    XCTAssert(true)
+                    // file does not exist, but is optional
+                    let textOutput = try execute(["--netrc-file", "/foo", "--netrc-optional", "resolve"], packagePath: packageRoot).stderr
+                    XCTAssert(textOutput.contains("warning: Did not find optional .netrc file at /foo."))
+
+                    // required file does not exist, will throw
+                    try execute(["--netrc-file", "/foo", "resolve"], packagePath: packageRoot)
+
+                } catch {
+                    XCTAssert(String(describing: error).contains("Cannot find mandatory .netrc file at /foo"))
+                }
+            }
+
+            fixture(name: "DependencyResolution/External/Complex") { prefix in
+                let packageRoot = prefix.appending(component: "app")
+                do {
+                    // Developer machine may have .netrc file at NSHomeDirectory; modify test accordingly
+                    if localFileSystem.exists(localFileSystem.homeDirectory.appending(RelativePath(".netrc"))) {
+                        try execute(["--netrc", "resolve"], packagePath: packageRoot)
+                        XCTAssert(true)
+                    } else {
+                        // file does not exist, but is optional
+                        let textOutput = try execute(["--netrc", "--netrc-optional", "resolve"], packagePath: packageRoot)
+                        XCTAssert(textOutput.stderr.contains("Did not find optional .netrc file at \(localFileSystem.homeDirectory)/.netrc."))
+
+                        // file does not exist, but is optional
+                        let textOutput2 = try execute(["--netrc-optional", "resolve"], packagePath: packageRoot)
+                        XCTAssert(textOutput2.stderr.contains("Did not find optional .netrc file at \(localFileSystem.homeDirectory)/.netrc."))
+
+                        // required file does not exist, will throw
+                        try execute(["--netrc", "resolve"], packagePath: packageRoot)
+                    }
+                } catch {
+                    XCTAssert(String(describing: error).contains("Cannot find mandatory .netrc file at \(localFileSystem.homeDirectory)/.netrc"))
+                }
+            }
+        } else {
+            // UNSUPPORTED OS, HANDLED ELSEWHERE
+        }
+        #else
+        // UNSUPPORTED OS, HANDLED ELSEWHERE
+        #endif
+    }
 
     func testResolve() throws {
         fixture(name: "DependencyResolution/External/Simple") { prefix in

swift-tools-support-core/Sources/TSCUtility/CMakeLists.txt

@@ -22,6 +22,7 @@ add_library(TSCUtility
   InterruptHandler.swift
   JSONMessageStreamingParser.swift
   misc.swift
+  Netrc.swift
   OSLog.swift
   PkgConfig.swift
   Platform.swift

swift-tools-support-core/Sources/TSCUtility/Downloader.swift

@@ -43,11 +43,13 @@ public protocol Downloader {
     /// - Parameters:
     ///   - url: The `URL` to the file to download.
     ///   - destination: The `AbsolutePath` to download the file to.
+    ///   - authorizationProvider: Optional provider supplying `Authorization` header to be added to `URLRequest`.
     ///   - progress: A closure to receive the download's progress as number of bytes.
     ///   - completion: A closure to be notifed of the completion of the download.
     func downloadFile(
         at url: Foundation.URL,
         to destination: AbsolutePath,
+        withAuthorizationProvider authorizationProvider: AuthorizationProviding?,
         progress: @escaping Progress,
         completion: @escaping Completion
     )
@@ -109,11 +111,18 @@ public final class FoundationDownloader: NSObject, Downloader {
     public func downloadFile(
         at url: Foundation.URL,
         to destination: AbsolutePath,
+        withAuthorizationProvider authorizationProvider: AuthorizationProviding? = nil,
         progress: @escaping Downloader.Progress,
         completion: @escaping Downloader.Completion
     ) {
         queue.addOperation {
-            let task = self.session.downloadTask(with: url)
+            var request = URLRequest(url: url)
+
+            if let authorization = authorizationProvider?.authorization(for: url) {
+                request.addValue(authorization, forHTTPHeaderField: "Authorization")
+            }
+
+            let task = self.session.downloadTask(with: request)
             let download = Download(
                 task: task,
                 destination: destination,