refactor toolchain SwiftPM libraries path (#3660)

motivation: improve logic in how Manifest and Plugin APIs locations are derived

changes:
* rename fields on ToolchainConfiguration to better articulate their meaning
* change logic in UserToolchain to derive the SwiftPM libraries path more clearly and reliably
* adjust call-sites and test

Author: tomerd

CONTRIBUTING.md

@@ -344,7 +344,7 @@ $> SWIFT_EXEC=/path/to/my/built/swiftc swift build
 
 SwiftPM computes the path of its runtime libraries relative to where it is
 installed. This path can be overridden by setting the environment variable
-`SWIFTPM_PD_LIBS` to a directory containing the libraries, or a colon-separated list of
+`SWIFTPM_CUSTOM_LIBS_DIR` to a directory containing the libraries, or a colon-separated list of
 absolute search paths. SwiftPM will choose the first
 path which exists on disk. If none of the paths are present on disk, it will fall
 back to built-in computation.

Sources/Commands/SwiftRunTool.swift

@@ -136,7 +136,7 @@ public struct SwiftRunTool: SwiftCommand {
             let arguments = buildOp.buildPlan!.createREPLArguments()
             print("Launching Swift REPL with arguments: \(arguments.joined(separator: " "))")
             try run(
-                swiftTool.getToolchain().swiftInterpreter,
+                swiftTool.getToolchain().swiftInterpreterPath,
                 originalWorkingDirectory: swiftTool.originalWorkingDirectory,
                 arguments: arguments)
 
@@ -171,7 +171,7 @@ public struct SwiftRunTool: SwiftCommand {
             if let executable = options.executable, isValidSwiftFilePath(executable) {
                 swiftTool.diagnostics.emit(.runFileDeprecation)
                 // Redirect execution to the toolchain's swift executable.
-                let swiftInterpreterPath = try swiftTool.getToolchain().swiftInterpreter
+                let swiftInterpreterPath = try swiftTool.getToolchain().swiftInterpreterPath
                 // Prepend the script to interpret to the arguments.
                 let arguments = [executable] + options.arguments
                 try run(

Sources/Commands/SwiftTestTool.swift

@@ -1039,10 +1039,9 @@ fileprivate func constructTestEnvironment(
         let codecovProfile = buildParameters.buildPath.appending(components: "codecov", "default%m.profraw")
         env["LLVM_PROFILE_FILE"] = codecovProfile.pathString
     }
-
   #if !os(macOS)
 #if os(Windows)
-    if let location = toolchain.manifestResources.xctestLocation {
+    if let location = toolchain.configuration.xctestPath {
       env["Path"] = "\(location.pathString);\(env["Path"] ?? "")"
     }
 #endif

Sources/PackageLoading/ManifestLoader.swift

@@ -689,21 +689,20 @@ public final class ManifestLoader: ManifestLoaderProtocol {
         let moduleCachePath = (ProcessEnv.vars["SWIFTPM_MODULECACHE_OVERRIDE"] ?? ProcessEnv.vars["SWIFTPM_TESTS_MODULECACHE"]).flatMap{ AbsolutePath.init($0) }
 
         var cmd: [String] = []
-        cmd += [self.toolchain.swiftCompiler.pathString]
+        cmd += [self.toolchain.swiftCompilerPath.pathString]
         cmd += verbosity.ccArgs
 
         let macOSPackageDescriptionPath: AbsolutePath
-        // If we got the binDir that means we could be developing SwiftPM in Xcode
+        // if runtimePath is set to "PackageFrameworks" that means we could be developing SwiftPM in Xcode
         // which produces a framework for dynamic package products.
-        let packageFrameworkPath = runtimePath.appending(component: "PackageFrameworks")
-        if self.toolchain.binDir != nil, localFileSystem.exists(packageFrameworkPath)  {
+        if runtimePath.extension == "framework" {
             cmd += [
-                "-F", packageFrameworkPath.pathString,
+                "-F", runtimePath.parentDirectory.pathString,
                 "-framework", "PackageDescription",
-                "-Xlinker", "-rpath", "-Xlinker", packageFrameworkPath.pathString,
+                "-Xlinker", "-rpath", "-Xlinker", runtimePath.parentDirectory.pathString,
             ]
 
-            macOSPackageDescriptionPath = packageFrameworkPath.appending(RelativePath("PackageDescription.framework/PackageDescription"))
+            macOSPackageDescriptionPath = runtimePath.appending(component: "PackageDescription")
         } else {
             cmd += [
                 "-L", runtimePath.pathString,
@@ -716,13 +715,13 @@ public final class ManifestLoader: ManifestLoaderProtocol {
 #endif
 
             // note: this is not correct for all platforms, but we only actually use it on macOS.
-            macOSPackageDescriptionPath = runtimePath.appending(RelativePath("libPackageDescription.dylib"))
+            macOSPackageDescriptionPath = runtimePath.appending(component: "libPackageDescription.dylib")
         }
 
         // Use the same minimum deployment target as the PackageDescription library (with a fallback of 10.15).
 #if os(macOS)
         let triple = Self._hostTriple.memoize {
-            Triple.getHostTriple(usingSwiftCompiler: self.toolchain.swiftCompiler)
+            Triple.getHostTriple(usingSwiftCompiler: self.toolchain.swiftCompilerPath)
         }
 
         let version = try Self._packageDescriptionMinimumDeploymentTarget.memoize {
@@ -853,9 +852,15 @@ public final class ManifestLoader: ManifestLoaderProtocol {
         var cmd = [String]()
         let runtimePath = self.runtimePath(for: toolsVersion)
         cmd += ["-swift-version", toolsVersion.swiftLanguageVersion.rawValue]
-        cmd += ["-I", runtimePath.pathString]
+        // if runtimePath is set to "PackageFrameworks" that means we could be developing SwiftPM in Xcode
+        // which produces a framework for dynamic package products.
+        if runtimePath.extension == "framework" {
+            cmd += ["-I", runtimePath.parentDirectory.parentDirectory.pathString]
+        } else {
+            cmd += ["-I", runtimePath.pathString]
+        }
       #if os(macOS)
-        if let sdkRoot = self.toolchain.sdkRoot ?? self.sdkRoot() {
+        if let sdkRoot = self.toolchain.sdkRootPath ?? self.sdkRoot() {
             cmd += ["-sdk", sdkRoot.pathString]
         }
       #endif
@@ -865,19 +870,14 @@ public final class ManifestLoader: ManifestLoaderProtocol {
 
     /// Returns the runtime path given the manifest version and path to libDir.
     private func runtimePath(for version: ToolsVersion) -> AbsolutePath {
-        // Bin dir will be set when developing swiftpm without building all of the runtimes.
-        if let binDir = self.toolchain.binDir {
-            return binDir
-        }
-        
-        // Otherwise we use the standard location of the manifest API in the toolchain, if it exists.
-        let manifestAPIDir = self.toolchain.libDir.appending(component: "ManifestAPI")
+        let manifestAPIDir = self.toolchain.swiftPMLibrariesLocation.manifestAPI
         if localFileSystem.exists(manifestAPIDir) {
             return manifestAPIDir
         }
-        
-        // Otherwise, fall back on the old location (this would indicate that we're using an old toolchain).
-        return self.toolchain.libDir.appending(version.runtimeSubpath)
+
+        // FIXME: how do we test this?
+        // Fall back on the old location (this would indicate that we're using an old toolchain).
+        return self.toolchain.swiftPMLibrariesLocation.manifestAPI.parentDirectory.appending(version.runtimeSubpath)
     }
 
     /// Returns path to the manifest database inside the given cache directory.

Sources/PackageModel/ToolchainConfiguration.swift

@@ -16,37 +16,56 @@ import TSCBasic
 /// using the package manager with alternate toolchains in the future.
 public struct ToolchainConfiguration {
     /// The path of the swift compiler.
-    public var swiftCompiler: AbsolutePath
+    public var swiftCompilerPath: AbsolutePath
 
     /// Extra arguments to pass the Swift compiler (defaults to the empty string).
     public var swiftCompilerFlags: [String]
 
     /// Environment to pass to the Swift compiler (defaults to the inherited environment).
     public var swiftCompilerEnvironment: [String: String]
 
-    /// The path of the library resources.
-    public var libDir: AbsolutePath
-
-    /// The bin directory.
-    public var binDir: AbsolutePath?
+    /// SwiftPM library paths.
+    public var swiftPMLibrariesLocation: SwiftPMLibrariesLocation
 
     /// The path to SDK root.
     ///
     /// If provided, it will be passed to the swift interpreter.
-    public var sdkRoot: AbsolutePath?
+    public var sdkRootPath: AbsolutePath?
 
     /// XCTest Location
-    public var xctestLocation: AbsolutePath?
+    public var xctestPath: AbsolutePath?
 
     /// Creates the set of manifest resources associated with a `swiftc` executable.
     ///
     /// - Parameters:
-    ///     - swiftCompiler: The absolute path of the associated `swiftc` executable.
-    ///     - swiftCompilerFlags: Extra flags to pass the Swift compiler.: Extra flags to pass the Swift compiler.
-    ///     - libDir: The path of the library resources.
-    ///     - binDir: The bin directory.
-    ///     - sdkRoot: The path to SDK root.
-    ///     - xctestLocation: XCTest Location
+    ///     - swiftCompilerPath: The absolute path of the associated swift compiler  executable (`swiftc`).
+    ///     - swiftCompilerFlags: Extra flags to pass to the Swift compiler.
+    ///     - swiftCompilerEnvironment: Environment variables to pass to the Swift compiler.
+    ///     - swiftPMLibrariesRootPath: Custom path for SwiftPM libraries. Computed based on the compiler path by default.
+    ///     - sdkRootPath: Optional path to SDK root.
+    ///     - xctestPath: Optional path to XCTest.
+    public init(
+        swiftCompilerPath: AbsolutePath,
+        swiftCompilerFlags: [String] = [],
+        swiftCompilerEnvironment: [String: String] = ProcessEnv.vars,
+        swiftPMLibrariesLocation: SwiftPMLibrariesLocation? = nil,
+        sdkRootPath: AbsolutePath? = nil,
+        xctestPath: AbsolutePath? = nil
+    ) {
+        let swiftPMLibrariesLocation = swiftPMLibrariesLocation ?? {
+            return .init(swiftCompilerPath: swiftCompilerPath)
+        }()
+
+        self.swiftCompilerPath = swiftCompilerPath
+        self.swiftCompilerFlags = swiftCompilerFlags
+        self.swiftCompilerEnvironment = swiftCompilerEnvironment
+        self.swiftPMLibrariesLocation = swiftPMLibrariesLocation
+        self.sdkRootPath = sdkRootPath
+        self.xctestPath = xctestPath
+    }
+
+    // deprecated 8/2021
+    @available(*, deprecated, message: "use non-deprecated initializer instead")
     public init(
         swiftCompiler: AbsolutePath,
         swiftCompilerFlags: [String] = [],
@@ -56,16 +75,37 @@ public struct ToolchainConfiguration {
         sdkRoot: AbsolutePath? = nil,
         xctestLocation: AbsolutePath? = nil
     ) {
-        self.swiftCompiler = swiftCompiler
-        self.swiftCompilerFlags = swiftCompilerFlags
-        self.swiftCompilerEnvironment = swiftCompilerEnvironment
-        self.libDir = libDir ?? Self.libDir(forBinDir: swiftCompiler.parentDirectory)
-        self.binDir = binDir
-        self.sdkRoot = sdkRoot
-        self.xctestLocation = xctestLocation
+        self.init(
+            swiftCompilerPath: swiftCompiler,
+            swiftCompilerFlags: swiftCompilerFlags,
+            swiftCompilerEnvironment: swiftCompilerEnvironment,
+            swiftPMLibrariesLocation: libDir.map { .init(root: $0) },
+            sdkRootPath: sdkRoot,
+            xctestPath: xctestLocation
+        )
     }
+}
+
+extension ToolchainConfiguration {
+    public struct SwiftPMLibrariesLocation {
+        public var manifestAPI: AbsolutePath
+        public var pluginAPI: AbsolutePath
+
+        public init(manifestAPI: AbsolutePath, pluginAPI: AbsolutePath) {
+            self.manifestAPI = manifestAPI
+            self.pluginAPI = pluginAPI
+        }
+
+        public init(root: AbsolutePath) {
+            self.init(
+                manifestAPI: root.appending(component: "ManifestAPI"),
+                pluginAPI: root.appending(component: "PluginAPI")
+            )
+        }
 
-    public static func libDir(forBinDir binDir: AbsolutePath) -> AbsolutePath {
-        return binDir.parentDirectory.appending(components: "lib", "swift", "pm")
+        public init(swiftCompilerPath: AbsolutePath) {
+            let rootPath = swiftCompilerPath.parentDirectory.parentDirectory.appending(components: "lib", "swift", "pm")
+            self.init(root: rootPath)
+        }
     }
 }

Sources/SPMPackageEditor/PackageEditor.swift

@@ -268,7 +268,7 @@ public final class PackageEditorContext {
 
         // Create toolchain.
         let hostToolchain = try UserToolchain(destination: .hostDestination())
-        self.manifestLoader = ManifestLoader(manifestResources: hostToolchain.manifestResources)
+        self.manifestLoader = ManifestLoader(manifestResources: hostToolchain.configuration)
 
         let repositoriesPath = buildDir.appending(component: "repositories")
         self.repositoryManager = RepositoryManager(

Sources/SPMTestSupport/Toolchain.swift

@@ -35,10 +35,9 @@ extension ToolchainConfiguration {
         get {
             let toolchain = UserToolchain.default
             return .init(
-                swiftCompiler: toolchain.configuration.swiftCompiler,
+                swiftCompilerPath: toolchain.configuration.swiftCompilerPath,
                 swiftCompilerFlags: [],
-                libDir: toolchain.configuration.libDir,
-                binDir: toolchain.configuration.binDir
+                swiftPMLibrariesLocation: toolchain.configuration.swiftPMLibrariesLocation
             )
         }
     }

Sources/Workspace/DefaultPluginScriptRunner.swift

@@ -40,36 +40,33 @@ public struct DefaultPluginScriptRunner: PluginScriptRunner {
 
     public var hostTriple: Triple {
         return Self._hostTriple.memoize {
-            Triple.getHostTriple(usingSwiftCompiler: self.toolchain.swiftCompiler)
+            Triple.getHostTriple(usingSwiftCompiler: self.toolchain.swiftCompilerPath)
         }
     }
 
     /// Helper function that compiles a plugin script as an executable and returns the path to it.
     fileprivate func compile(sources: Sources, toolsVersion: ToolsVersion, cacheDir: AbsolutePath) throws -> AbsolutePath {
         // FIXME: Much of this is copied from the ManifestLoader and should be consolidated.
 
-        // Bin dir will be set when developing swiftpm without building all of the runtimes.
-        let runtimePath = self.toolchain.binDir ?? self.toolchain.libDir.appending(component: "PluginAPI")
+        let runtimePath = self.toolchain.swiftPMLibrariesLocation.pluginAPI
 
         // Compile the package plugin script.
-        var command = [self.toolchain.swiftCompiler.pathString]
+        var command = [self.toolchain.swiftCompilerPath.pathString]
 
         // FIXME: Workaround for the module cache bug that's been haunting Swift CI
         // <rdar://problem/48443680>
         let moduleCachePath = ProcessEnv.vars["SWIFTPM_MODULECACHE_OVERRIDE"] ?? ProcessEnv.vars["SWIFTPM_TESTS_MODULECACHE"]
 
-        // If we got the binDir that means we could be developing SwiftPM in Xcode
-        // which produces a framework for dynamic package products.
-        let packageFrameworkPath = runtimePath.appending(component: "PackageFrameworks")
-
         let macOSPackageDescriptionPath: AbsolutePath
-        if self.toolchain.binDir != nil, localFileSystem.exists(packageFrameworkPath) {
+        // if runtimePath is set to "PackageFrameworks" that means we could be developing SwiftPM in Xcode
+        // which produces a framework for dynamic package products.
+        if runtimePath.extension == "framework" {
             command += [
-                "-F", packageFrameworkPath.pathString,
+                "-F", runtimePath.parentDirectory.pathString,
                 "-framework", "PackagePlugin",
-                "-Xlinker", "-rpath", "-Xlinker", packageFrameworkPath.pathString,
+                "-Xlinker", "-rpath", "-Xlinker", runtimePath.parentDirectory.pathString,
             ]
-            macOSPackageDescriptionPath = packageFrameworkPath.appending(RelativePath("PackagePlugin.framework/PackagePlugin"))
+            macOSPackageDescriptionPath = runtimePath.appending(component: "PackagePlugin")
         } else {
             command += [
                 "-L", runtimePath.pathString,
@@ -82,7 +79,7 @@ public struct DefaultPluginScriptRunner: PluginScriptRunner {
             #endif
 
             // note: this is not correct for all platforms, but we only actually use it on macOS.
-            macOSPackageDescriptionPath = runtimePath.appending(RelativePath("libPackagePlugin.dylib"))
+            macOSPackageDescriptionPath = runtimePath.appending(component: "libPackagePlugin.dylib")
         }
 
         // Use the same minimum deployment target as the PackageDescription library (with a fallback of 10.15).
@@ -99,9 +96,15 @@ public struct DefaultPluginScriptRunner: PluginScriptRunner {
         command += self.toolchain.swiftCompilerFlags
 
         command += ["-swift-version", toolsVersion.swiftLanguageVersion.rawValue]
-        command += ["-I", runtimePath.pathString]
+        // if runtimePath is set to "PackageFrameworks" that means we could be developing SwiftPM in Xcode
+        // which produces a framework for dynamic package products.
+        if runtimePath.extension == "framework" {
+            command += ["-I", runtimePath.parentDirectory.parentDirectory.pathString]
+        } else {
+            command += ["-I", runtimePath.pathString]
+        }
         #if os(macOS)
-        if let sdkRoot = self.toolchain.sdkRoot ?? self.sdkRoot() {
+        if let sdkRoot = self.toolchain.sdkRootPath ?? self.sdkRoot() {
             command += ["-sdk", sdkRoot.pathString]
         }
         #endif

Sources/Workspace/Destination.swift

@@ -90,8 +90,10 @@ public struct Destination: Encodable, Equatable {
         environment: [String:String] = ProcessEnv.vars
     ) throws -> Destination {
         // Select the correct binDir.
-        let customBinDir = ProcessEnv
-            .vars["SWIFTPM_CUSTOM_BINDIR"]
+        if ProcessEnv.vars["SWIFTPM_CUSTOM_BINDIR"] != nil {
+            print("SWIFTPM_CUSTOM_BINDIR was deprecated in favor of SWIFTPM_CUSTOM_BIN_DIR")
+        }
+        let customBinDir = (ProcessEnv.vars["SWIFTPM_CUSTOM_BIN_DIR"] ?? ProcessEnv.vars["SWIFTPM_CUSTOM_BINDIR"])
             .flatMap{ try? AbsolutePath(validating: $0) }
         let binDir = try customBinDir ?? binDir ?? Destination.hostBinDir(
             originalWorkingDirectory: originalWorkingDirectory)