Update auth logic (#3789)

Motivation: SwiftPM should support using credentials stored in keychain for binary download and registry authentication.

Modifications:
- Wire up `KeychainAuthorizationProvider` in `SwiftTool`. Add `--no-keychain` option which allows user to opt-out.
- Update netrc logic to either:
  - Load from a user-defined `.netrc` file
  - Default to `.netrc` file found in current workspace (if any) and/or
    user's home directory.
- Deprecate `Workspace.Configuration.Netrc`

rdar://83682028

Author: yim-lee

Sources/Basics/AuthorizationProvider.swift

@@ -41,7 +41,7 @@ public extension AuthorizationProvider {
     }
 }
 
-extension Foundation.URL {
+private extension Foundation.URL {
     var authenticationID: String? {
         guard let host = host?.lowercased() else {
             return nil

Sources/Commands/Options.swift

@@ -328,6 +328,20 @@ public struct SwiftToolOptions: ParsableArguments {
         help: "Specify the .netrc file path.",
         completion: .file())
     var netrcFilePath: AbsolutePath?
+    
+    /// Whether to use keychain for authenticating with remote servers
+    /// when downloading binary artifacts or communicating with a registry.
+#if canImport(Security)
+    @Flag(inversion: .prefixedEnableDisable,
+          exclusivity: .exclusive,
+          help: "Search credentials in macOS keychain")
+    var keychain: Bool = true
+#else
+    @Flag(inversion: .prefixedEnableDisable,
+          exclusivity: .exclusive,
+          help: .hidden)
+    var keychain: Bool = false
+#endif
 
     @Flag(name: .customLong("netrc"), help: .hidden)
     var _deprecated_netrc: Bool = false

Sources/Commands/SwiftTool.swift

@@ -1,7 +1,7 @@
 /*
  This source file is part of the Swift.org open source project
 
- Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
+ Copyright (c) 2014 - 2021 Apple Inc. and the Swift project authors
  Licensed under Apache License v2.0 with Runtime Library Exception
 
  See http://swift.org/LICENSE.txt for license information
@@ -497,39 +497,60 @@ public class SwiftTool {
 
     func getAuthorizationProvider() throws -> AuthorizationProvider? {
         var providers = [AuthorizationProvider]()
+        
         // netrc file has higher specificity than keychain so use it first
-        if let netrcConfigFile = try self.getNetrcConfigFile() {
-            providers.append(try NetrcAuthorizationProvider(path: netrcConfigFile, fileSystem: localFileSystem))
+        try providers.append(contentsOf: self.getNetrcAuthorizationProviders())
+
+#if canImport(Security)
+        if self.options.keychain {
+            providers.append(KeychainAuthorizationProvider(observabilityScope: self.observabilityScope))
         }
-        
-        // TODO: add --no-keychain option to allow opt-out
-//#if canImport(Security)
-//        providers.append(KeychainAuthorizationProvider(observabilityScope: self.observabilityScope))
-//#endif
+#endif
 
         return providers.isEmpty ? .none : CompositeAuthorizationProvider(providers, observabilityScope: self.observabilityScope)
     }
 
-    func getNetrcConfigFile() throws -> AbsolutePath? {
+    func getNetrcAuthorizationProviders() throws -> [NetrcAuthorizationProvider] {
         guard options.netrc else {
-            return .none
+            return []
         }
 
+        var providers = [NetrcAuthorizationProvider]()
+        
+        // Use custom .netrc file if specified, otherwise look for it within workspace and user's home directory.
         if let configuredPath = options.netrcFilePath {
             guard localFileSystem.exists(configuredPath) else {
                 throw StringError("Did not find .netrc file at \(configuredPath).")
             }
-            return configuredPath
-        }
+            
+            providers.append(try NetrcAuthorizationProvider(path: configuredPath, fileSystem: localFileSystem))
+        } else {
+            // User didn't tell us to use these .netrc files so be more lenient with errors
+            func loadNetrcNoThrows(at path: AbsolutePath) -> NetrcAuthorizationProvider? {
+                guard localFileSystem.exists(path) else { return nil }
+                
+                do {
+                    return try NetrcAuthorizationProvider(path: path, fileSystem: localFileSystem)
+                } catch {
+                    self.observabilityScope.emit(warning: "Failed to load .netrc file at \(path). Error: \(error)")
+                    return nil
+                }
+            }
+            
+            // Workspace's .netrc file should be consulted before user-global file
+            // TODO: replace multiroot-data-file with explicit overrides
+            if let localPath = try? (options.multirootPackageDataFile ?? self.getPackageRoot()).appending(component: ".netrc"),
+               let localProvider = loadNetrcNoThrows(at: localPath) {
+                providers.append(localProvider)
+            }
 
-        // TODO: replace multiroot-data-file with explicit overrides
-        let localPath = try (options.multirootPackageDataFile ?? self.getPackageRoot()).appending(component: ".netrc")
-        if localFileSystem.exists(localPath) {
-            return localPath
+            let userHomePath = localFileSystem.homeDirectory.appending(component: ".netrc")
+            if let userHomeProvider = loadNetrcNoThrows(at: userHomePath) {
+                providers.append(userHomeProvider)
+            }
         }
-
-        let userHomePath = localFileSystem.homeDirectory.appending(component: ".netrc")
-        return localFileSystem.exists(userHomePath) ? userHomePath : .none
+        
+        return providers
     }
 
     private func getSharedCacheDirectory() throws -> AbsolutePath? {

Sources/PackageCollections/API.swift

@@ -112,7 +112,7 @@ public protocol PackageCollectionsProtocol {
     ///   - reference: The package reference
     ///   - callback: The closure to invoke when result becomes available
     // deprecated 9/21
-    @available(*, deprecated, message: "user getPackageMetadata(identity:) instead")
+    @available(*, deprecated, message: "use getPackageMetadata(identity:) instead")
     func getPackageMetadata(
         _ reference: PackageReference,
         callback: @escaping (Result<PackageCollectionsModel.PackageMetadata, Error>) -> Void
@@ -129,7 +129,7 @@ public protocol PackageCollectionsProtocol {
     ///                  processed collection will be used.
     ///   - callback: The closure to invoke when result becomes available
     // deprecated 9/21
-    @available(*, deprecated, message: "user getPackageMetadata(identity:) instead")
+    @available(*, deprecated, message: "use getPackageMetadata(identity:) instead")
     func getPackageMetadata(
         _ reference: PackageReference,
         collections: Set<PackageCollectionsModel.CollectionIdentifier>?,

Tests/CommandsTests/RunToolTests.swift

@@ -37,7 +37,7 @@ final class RunToolTests: XCTestCase {
         XCTAssert(stdout.contains("Swift Package Manager"), "got stdout:\n" + stdout)
     }
 
-    func testUnkownProductAndArgumentPassing() throws {
+    func testUnknownProductAndArgumentPassing() throws {
         fixture(name: "Miscellaneous/EchoExecutable") { path in
 
             let result = try SwiftPMProduct.SwiftRun.executeProcess(

Tests/CommandsTests/SwiftToolTests.swift

@@ -8,76 +8,108 @@
  See http://swift.org/CONTRIBUTORS.txt for Swift project authors
  */
 
-import Basics
+@testable import Basics
 @testable import Commands
 import SPMTestSupport
 import TSCBasic
 import XCTest
 
 final class SwiftToolTests: XCTestCase {
-    func testNetrcLocations() throws {
+    func testNetrcAuthorizationProviders() throws {
         fixture(name: "DependencyResolution/External/XCFramework") { packageRoot in
             let fs = localFileSystem
+            
+            let localPath = packageRoot.appending(component: ".netrc")
+            let userHomePath = fs.homeDirectory.appending(component: ".netrc")
 
             // custom .netrc file
-
             do {
                 let customPath = fs.homeDirectory.appending(component: UUID().uuidString)
                 try fs.writeFileContents(customPath) {
                     "machine mymachine.labkey.org login [email protected] password custom"
                 }
 
-
                 let options = try SwiftToolOptions.parse(["--package-path", packageRoot.pathString, "--netrc-file", customPath.pathString])
                 let tool = try SwiftTool(options: options)
-                XCTAssertEqual(try tool.getNetrcConfigFile().map(resolveSymlinks), resolveSymlinks(customPath))
+                
+                let netrcProviders = try tool.getNetrcAuthorizationProviders()
+                XCTAssertEqual(netrcProviders.count, 1)
+                XCTAssertEqual(netrcProviders.first.map { resolveSymlinks($0.path) }, resolveSymlinks(customPath))
+
                 let auth = try tool.getAuthorizationProvider()?.authentication(for: URL(string: "https://mymachine.labkey.org")!)
                 XCTAssertEqual(auth?.user, "[email protected]")
                 XCTAssertEqual(auth?.password, "custom")
 
                 // delete it
                 try localFileSystem.removeFileTree(customPath)
-                XCTAssertThrowsError(try tool.getNetrcConfigFile(), "error expected") { error in
+                XCTAssertThrowsError(try tool.getNetrcAuthorizationProviders(), "error expected") { error in
                     XCTAssertEqual(error as? StringError, StringError("Did not find .netrc file at \(customPath)."))
                 }
             }
 
             // local .netrc file
-
             do {
-                let localPath = packageRoot.appending(component: ".netrc")
+                // make sure there isn't a user home one
+                try localFileSystem.removeFileTree(userHomePath)
+                
                 try fs.writeFileContents(localPath) {
                     return "machine mymachine.labkey.org login [email protected] password local"
                 }
 
                 let options = try SwiftToolOptions.parse(["--package-path", packageRoot.pathString])
                 let tool = try SwiftTool(options: options)
+                
+                let netrcProviders = try tool.getNetrcAuthorizationProviders()
+                XCTAssertEqual(netrcProviders.count, 1)
+                XCTAssertEqual(netrcProviders.first.map { resolveSymlinks($0.path) }, resolveSymlinks(localPath))
 
-                XCTAssertEqual(try tool.getNetrcConfigFile().map(resolveSymlinks), resolveSymlinks(localPath))
                 let auth = try tool.getAuthorizationProvider()?.authentication(for: URL(string: "https://mymachine.labkey.org")!)
                 XCTAssertEqual(auth?.user, "[email protected]")
                 XCTAssertEqual(auth?.password, "local")
             }
 
             // user .netrc file
-
             do {
                 // make sure there isn't a local one
-                try localFileSystem.removeFileTree(packageRoot.appending(component: ".netrc"))
+                try localFileSystem.removeFileTree(localPath)
 
-                let userHomePath = fs.homeDirectory.appending(component: ".netrc")
                 try fs.writeFileContents(userHomePath) {
                     return "machine mymachine.labkey.org login [email protected] password user"
                 }
 
                 let options = try SwiftToolOptions.parse(["--package-path", packageRoot.pathString])
                 let tool = try SwiftTool(options: options)
 
-                XCTAssertEqual(try tool.getNetrcConfigFile().map(resolveSymlinks), resolveSymlinks(userHomePath))
+                let netrcProviders = try tool.getNetrcAuthorizationProviders()
+                XCTAssertEqual(netrcProviders.count, 1)
+                XCTAssertEqual(netrcProviders.first.map { resolveSymlinks($0.path) }, resolveSymlinks(userHomePath))
+                
                 let auth = try tool.getAuthorizationProvider()?.authentication(for: URL(string: "https://mymachine.labkey.org")!)
                 XCTAssertEqual(auth?.user, "[email protected]")
                 XCTAssertEqual(auth?.password, "user")
             }
+            
+            // both local and user .netrc file
+            do {
+                try fs.writeFileContents(localPath) {
+                    return "machine mymachine.labkey.org login [email protected] password local"
+                }
+                try fs.writeFileContents(userHomePath) {
+                    return "machine mymachine.labkey.org login [email protected] password user"
+                }
+
+                let options = try SwiftToolOptions.parse(["--package-path", packageRoot.pathString])
+                let tool = try SwiftTool(options: options)
+
+                let netrcProviders = try tool.getNetrcAuthorizationProviders()
+                XCTAssertEqual(netrcProviders.count, 2)
+                XCTAssertEqual(netrcProviders.map { resolveSymlinks($0.path) }, [localPath, userHomePath].map(resolveSymlinks))
+                
+                // local before user .netrc file
+                let auth = try tool.getAuthorizationProvider()?.authentication(for: URL(string: "https://mymachine.labkey.org")!)
+                XCTAssertEqual(auth?.user, "[email protected]")
+                XCTAssertEqual(auth?.password, "local")
+            }
         }
     }
 }