use idiomatic location for security directory and update location of configuration directory (#3942)

motivation: use idiomatic location for security directory on macOS

changes:
* allow users to customize security directory location with new --security-path CLI option
* on macOS use <user>/Library/org.swiftpm/security for security files and symlink from ~/.swiftpm/security
* move configuration directory from <user>/Library/org.swiftpm to <user>/Library/org.swiftpm/configuration
* add migration code from old configuraiton location to new one
* add and adjust tests
* update docker setup for new locationis

Author: tomerd

Sources/Basics/FileSystem+Extensions.swift

@@ -21,10 +21,9 @@ extension FileSystem {
     public var dotSwiftPM: AbsolutePath {
         self.homeDirectory.appending(component: ".swiftpm")
     }
-    
-    /// SwiftPM security directory
-    public var swiftPMSecurityDirectory: AbsolutePath {
-        self.dotSwiftPM.appending(component: "security")
+
+    fileprivate var idiomaticSwiftPMDirectory: AbsolutePath? {
+        return FileManager.default.urls(for: .libraryDirectory, in: .userDomainMask).first.flatMap { AbsolutePath($0.path) }?.appending(component: "org.swift.swiftpm")
     }
 }
 
@@ -69,52 +68,128 @@ extension FileSystem {
     }
 }
 
-// MARK: - config
+// MARK: - configuration
 
 extension FileSystem {
-    private var idiomaticUserConfigDirectory: AbsolutePath? {
-        return FileManager.default.urls(for: .libraryDirectory, in: .userDomainMask).first.flatMap { AbsolutePath($0.path) }
-    }
-
     /// SwiftPM config directory under user's config directory (if exists)
-    public var swiftPMConfigDirectory: AbsolutePath {
-        if let path = self.idiomaticUserConfigDirectory {
-            return path.appending(component: "org.swift.swiftpm")
+    public var swiftPMConfigurationDirectory: AbsolutePath {
+        if let path = self.idiomaticSwiftPMDirectory {
+            return path.appending(component: "configuration")
         } else {
-            return self.dotSwiftPMConfigDirectory
+            return self.dotSwiftPMConfigurationDirectory
         }
     }
 
-    fileprivate var dotSwiftPMConfigDirectory: AbsolutePath {
-        return self.dotSwiftPM.appending(component: "config")
+    fileprivate var dotSwiftPMConfigurationDirectory: AbsolutePath {
+        return self.dotSwiftPM.appending(component: "configuration")
+    }
+}
+
+extension FileSystem {
+    public func getOrCreateSwiftPMConfigurationDirectory(observabilityScope: ObservabilityScope?) throws -> AbsolutePath {
+        let idiomaticConfigurationDirectory = self.swiftPMConfigurationDirectory
+
+        // temporary 5.6, remove on next version: transition from previous configuration location
+        if !self.exists(idiomaticConfigurationDirectory) {
+            try self.createDirectory(idiomaticConfigurationDirectory, recursive: true)
+        }
+
+        // in the case where ~/.swiftpm/configuration is not the idiomatic location (eg on macOS where its /Users/<user>/Library/org.swift.swiftpm/configuration)
+        if idiomaticConfigurationDirectory != self.dotSwiftPMConfigurationDirectory {
+            // copy the configuration files from old location (eg /Users/<user>/Library/org.swift.swiftpm) to new one (eg /Users/<user>/Library/org.swift.swiftpm/configuration)
+            // but leave them there for backwards compatibility (eg older xcode)
+            let oldConfigDirectory = idiomaticConfigurationDirectory.parentDirectory
+            if self.exists(oldConfigDirectory, followSymlink: false) && self.isDirectory(oldConfigDirectory) {
+                let configurationFiles = try self.getDirectoryContents(oldConfigDirectory)
+                    .map{ oldConfigDirectory.appending(component: $0) }
+                    .filter{ self.isFile($0) && !self.isSymlink($0) && $0.extension != "lock"}
+                for file in configurationFiles {
+                    let destination = idiomaticConfigurationDirectory.appending(component: file.basename)
+                    observabilityScope?.emit(warning: "Usage of \(file) has been deprecated. Please delete it and use the new \(destination) instead.")
+                    if !self.exists(destination) {
+                        try self.copy(from: file, to: destination)
+                    }
+                }
+            }
+        // in the case where ~/.swiftpm/configuration is the idiomatic location (eg on Linux)
+        } else {
+            // copy the configuration files from old location (~/.swiftpm/config) to new one (~/.swiftpm/configuration)
+            // but leave them there for backwards compatibility (eg older toolchain)
+            let oldConfigDirectory = self.dotSwiftPM.appending(component: "config")
+            if self.exists(oldConfigDirectory, followSymlink: false) && self.isDirectory(oldConfigDirectory) {
+                let configurationFiles = try self.getDirectoryContents(oldConfigDirectory)
+                    .map{ oldConfigDirectory.appending(component: $0) }
+                    .filter{ self.isFile($0) && !self.isSymlink($0) && $0.extension != "lock"}
+                for file in configurationFiles {
+                    let destination = idiomaticConfigurationDirectory.appending(component: file.basename)
+                    observabilityScope?.emit(warning: "Usage of \(file) has been deprecated. Please delete it and use the new \(destination) instead.")
+                    if !self.exists(destination) {
+                        try self.copy(from: file, to: destination)
+                    }
+                }
+            }
+        }
+        // ~temporary 5.6 migration
+
+        // Create idiomatic if necessary
+        if !self.exists(idiomaticConfigurationDirectory) {
+            try self.createDirectory(idiomaticConfigurationDirectory, recursive: true)
+        }
+        // Create ~/.swiftpm if necessary
+        if !self.exists(self.dotSwiftPM) {
+            try self.createDirectory(self.dotSwiftPM, recursive: true)
+        }
+        // Create ~/.swiftpm/configuration symlink if necessary
+        if !self.exists(self.dotSwiftPMConfigurationDirectory, followSymlink: false) {
+            try self.createSymbolicLink(dotSwiftPMConfigurationDirectory, pointingAt: idiomaticConfigurationDirectory, relative: false)
+        }
+
+        return idiomaticConfigurationDirectory
     }
 }
 
+// MARK: - security
+
 extension FileSystem {
-    public func getOrCreateSwiftPMConfigDirectory() throws -> AbsolutePath {
-        let idiomaticConfigDirectory = self.swiftPMConfigDirectory
+    /// SwiftPM security directory under user's security directory (if exists)
+    public var swiftPMSecurityDirectory: AbsolutePath {
+        if let path = self.idiomaticSwiftPMDirectory {
+            return path.appending(component: "security")
+        } else {
+            return self.dotSwiftPMSecurityDirectory
+        }
+    }
 
-        // temporary 5.5, remove on next version: transition from ~/.swiftpm/config to idiomatic location + symbolic link
-        if idiomaticConfigDirectory != self.dotSwiftPMConfigDirectory &&
-            self.exists(self.dotSwiftPMConfigDirectory) && self.isDirectory(self.dotSwiftPMConfigDirectory) &&
-            !self.exists(idiomaticConfigDirectory) {
-            print("transitioning \(self.dotSwiftPMConfigDirectory) to \(idiomaticConfigDirectory)")
-            try self.move(from: self.dotSwiftPMConfigDirectory, to: idiomaticConfigDirectory)
+    fileprivate var dotSwiftPMSecurityDirectory: AbsolutePath {
+        return self.dotSwiftPM.appending(component: "security")
+    }
+}
+
+extension FileSystem {
+    public func getOrCreateSwiftPMSecurityDirectory() throws -> AbsolutePath {
+        let idiomaticSecurityDirectory = self.swiftPMSecurityDirectory
+
+        // temporary 5.6, remove on next version: transition from ~/.swiftpm/security to idiomatic location + symbolic link
+        if idiomaticSecurityDirectory != self.dotSwiftPMSecurityDirectory &&
+            self.exists(self.dotSwiftPMSecurityDirectory) &&
+            self.isDirectory(self.dotSwiftPMSecurityDirectory) {
+            try self.removeFileTree(self.dotSwiftPMSecurityDirectory)
         }
+        // ~temporary 5.6 migration
 
         // Create idiomatic if necessary
-        if !self.exists(idiomaticConfigDirectory) {
-            try self.createDirectory(idiomaticConfigDirectory, recursive: true)
+        if !self.exists(idiomaticSecurityDirectory) {
+            try self.createDirectory(idiomaticSecurityDirectory, recursive: true)
         }
         // Create ~/.swiftpm if necessary
         if !self.exists(self.dotSwiftPM) {
             try self.createDirectory(self.dotSwiftPM, recursive: true)
         }
-        // Create ~/.swiftpm/config symlink if necessary
-        if !self.exists(self.dotSwiftPMConfigDirectory, followSymlink: false) {
-            try self.createSymbolicLink(dotSwiftPMConfigDirectory, pointingAt: idiomaticConfigDirectory, relative: false)
+        // Create ~/.swiftpm/security symlink if necessary
+        if !self.exists(self.dotSwiftPMSecurityDirectory, followSymlink: false) {
+            try self.createSymbolicLink(dotSwiftPMSecurityDirectory, pointingAt: idiomaticSecurityDirectory, relative: false)
         }
-        return idiomaticConfigDirectory
+        return idiomaticSecurityDirectory
     }
 }
 

Sources/Commands/Options.swift

@@ -149,6 +149,9 @@ public struct SwiftToolOptions: ParsableArguments {
     @Option(help: "Specify the shared configuration directory")
     var configPath: AbsolutePath?
 
+    @Option(help: "Specify the shared security directory")
+    var securityPath: AbsolutePath?
+
     /// Disables repository caching.
     @Flag(name: .customLong("repository-cache"), inversion: .prefixedEnableDisable, help: "Use a shared cache when fetching repositories")
     var useRepositoriesCache: Bool = true

Sources/Commands/SwiftTool.swift

@@ -294,6 +294,15 @@ public class SwiftTool {
     /// Path to the build directory.
     let buildPath: AbsolutePath
 
+    /// Path to the shared security directory
+    let sharedSecurityDirectory: AbsolutePath?
+
+    /// Path to the shared cache directory
+    let sharedCacheDirectory: AbsolutePath?
+
+    /// Path to the shared configuration directory
+    let sharedConfigurationDirectory: AbsolutePath?
+
     /// The process set to hold the launched processes. These will be terminated on any signal
     /// received by the swift tools.
     let processSet: ProcessSet
@@ -333,7 +342,7 @@ public class SwiftTool {
             self.observabilityScope.emit(error: "couldn't determine the current working directory")
             throw ExitCode.failure
         }
-        originalWorkingDirectory = cwd
+        self.originalWorkingDirectory = cwd
 
         do {
             try Self.postprocessArgParserResult(options: options, observabilityScope: self.observabilityScope)
@@ -416,6 +425,11 @@ public class SwiftTool {
         customBuildPath ??
         (packageRoot ?? cwd).appending(component: ".build")
 
+        // make sure common directories are created
+        self.sharedSecurityDirectory = try getSharedSecurityDirectory(options: self.options, observabilityScope: self.observabilityScope)
+        self.sharedConfigurationDirectory = try getSharedConfigurationDirectory(options: self.options, observabilityScope: self.observabilityScope)
+        self.sharedCacheDirectory = try getSharedCacheDirectory(options: self.options, observabilityScope: self.observabilityScope)
+
         // set verbosity globals.
         // TODO: get rid of this global settings in TSC
         switch self.logLevel {
@@ -492,7 +506,7 @@ public class SwiftTool {
     }
 
     func getMirrorsConfig(sharedConfigurationDirectory: AbsolutePath? = nil) throws -> Workspace.Configuration.Mirrors {
-        let sharedConfigurationDirectory = try sharedConfigurationDirectory ?? self.getSharedConfigurationDirectory()
+        let sharedConfigurationDirectory = sharedConfigurationDirectory ?? self.sharedConfigurationDirectory
         let sharedMirrorFile = sharedConfigurationDirectory.map { Workspace.DefaultLocations.mirrorsConfigurationFile(at: $0) }
         return try .init(
             localMirrorFile: self.mirrorsConfigFile(),
@@ -537,7 +551,7 @@ public class SwiftTool {
     func getRegistriesConfig(sharedConfigurationDirectory: AbsolutePath? = nil) throws -> Workspace.Configuration.Registries {
         let localRegistriesFile = try Workspace.DefaultLocations.registriesConfigurationFile(forRootPackage: self.getPackageRoot())
 
-        let sharedConfigurationDirectory = try sharedConfigurationDirectory ?? self.getSharedConfigurationDirectory()
+        let sharedConfigurationDirectory = sharedConfigurationDirectory ?? self.sharedConfigurationDirectory
         let sharedRegistriesFile = sharedConfigurationDirectory.map {
             Workspace.DefaultLocations.registriesConfigurationFile(at: $0)
         }
@@ -607,56 +621,6 @@ public class SwiftTool {
         return providers
     }
 
-    private func getSharedCacheDirectory() throws -> AbsolutePath? {
-        if let explicitCachePath = options.cachePath {
-            // Create the explicit cache path if necessary
-            if !localFileSystem.exists(explicitCachePath) {
-                try localFileSystem.createDirectory(explicitCachePath, recursive: true)
-            }
-            return explicitCachePath
-        }
-
-        do {
-            return try localFileSystem.getOrCreateSwiftPMCacheDirectory()
-        } catch {
-            self.observabilityScope.emit(warning: "Failed creating default cache location, \(error)")
-            return .none
-        }
-    }
-
-    private func getSharedConfigurationDirectory() throws -> AbsolutePath? {
-        if let explicitConfigPath = options.configPath {
-            // Create the explicit config path if necessary
-            if !localFileSystem.exists(explicitConfigPath) {
-                try localFileSystem.createDirectory(explicitConfigPath, recursive: true)
-            }
-            return explicitConfigPath
-        }
-
-        do {
-            return try localFileSystem.getOrCreateSwiftPMConfigDirectory()
-        } catch {
-            self.observabilityScope.emit(warning: "Failed creating default configuration location, \(error)")
-            return .none
-        }
-    }
-    
-    private func getSharedSecurityDirectory() throws -> AbsolutePath? {
-        do {
-            let fileSystem = localFileSystem
-            let sharedSecurityDirectory = fileSystem.swiftPMSecurityDirectory
-            if !fileSystem.exists(sharedSecurityDirectory) {
-                try fileSystem.createDirectory(sharedSecurityDirectory, recursive: true)
-            }
-            // And make sure we can write files (locking the directory writes a lock file)
-            try fileSystem.withLock(on: sharedSecurityDirectory, type: .exclusive) { }
-            return sharedSecurityDirectory
-        } catch {
-            self.observabilityScope.emit(warning: "Failed creating shared security directory: \(error)")
-            return .none
-        }
-    }
-
     /// Returns the currently active workspace.
     func getActiveWorkspace() throws -> Workspace {
         if let workspace = _workspace {
@@ -665,28 +629,25 @@ public class SwiftTool {
 
         let delegate = ToolWorkspaceDelegate(self.outputStream, logLevel: self.logLevel, observabilityScope: self.observabilityScope)
         let provider = GitRepositoryProvider(processSet: processSet)
-        let sharedSecurityDirectory = try self.getSharedSecurityDirectory()
-        let sharedCacheDirectory = try self.getSharedCacheDirectory()
-        let sharedConfigurationDirectory = try self.getSharedConfigurationDirectory()
         let isXcodeBuildSystemEnabled = self.options.buildSystem == .xcode
         let workspace = try Workspace(
             fileSystem: localFileSystem,
             location: .init(
-                workingDirectory: buildPath,
+                workingDirectory: self.buildPath,
                 editsDirectory: self.editsDirectory(),
                 resolvedVersionsFile: self.resolvedVersionsFile(),
-                sharedSecurityDirectory: sharedSecurityDirectory,
-                sharedCacheDirectory: sharedCacheDirectory,
-                sharedConfigurationDirectory: sharedConfigurationDirectory
+                sharedSecurityDirectory: self.sharedSecurityDirectory,
+                sharedCacheDirectory: self.sharedCacheDirectory,
+                sharedConfigurationDirectory: self.sharedConfigurationDirectory
             ),
-            mirrors: self.getMirrorsConfig(sharedConfigurationDirectory: sharedConfigurationDirectory).mirrors,
-            registries: try self.getRegistriesConfig(sharedConfigurationDirectory: sharedConfigurationDirectory).configuration,
+            mirrors: self.getMirrorsConfig(sharedConfigurationDirectory: self.sharedConfigurationDirectory).mirrors,
+            registries: try self.getRegistriesConfig(sharedConfigurationDirectory: self.sharedConfigurationDirectory).configuration,
             authorizationProvider: self.getAuthorizationProvider(),
             customManifestLoader: self.getManifestLoader(), // FIXME: doe we really need to customize it?
             customRepositoryProvider: provider, // FIXME: doe we really need to customize it?
             additionalFileRules: isXcodeBuildSystemEnabled ? FileRuleDescription.xcbuildFileTypes : FileRuleDescription.swiftpmFileTypes,
-            resolverUpdateEnabled: !options.skipDependencyUpdate,
-            resolverPrefetchingEnabled: options.shouldEnableResolverPrefetching,
+            resolverUpdateEnabled: !self.options.skipDependencyUpdate,
+            resolverPrefetchingEnabled: self.options.shouldEnableResolverPrefetching,
             resolverFingerprintCheckingMode: self.options.resolverFingerprintCheckingMode,
             sharedRepositoriesCacheEnabled: self.options.useRepositoriesCache,
             delegate: delegate
@@ -1011,7 +972,7 @@ public class SwiftTool {
             case (false, .local):
                 cachePath = self.buildPath
             case (false, .shared):
-                cachePath = try self.getSharedCacheDirectory().map{ Workspace.DefaultLocations.manifestsDirectory(at: $0) }
+                cachePath = self.sharedCacheDirectory.map{ Workspace.DefaultLocations.manifestsDirectory(at: $0) }
             }
 
             var extraManifestFlags = self.options.manifestFlags
@@ -1062,6 +1023,61 @@ private func getEnvBuildPath(workingDir: AbsolutePath) -> AbsolutePath? {
     return AbsolutePath(env, relativeTo: workingDir)
 }
 
+
+private func getSharedSecurityDirectory(options: SwiftToolOptions, observabilityScope: ObservabilityScope) throws -> AbsolutePath? {
+    if let explicitSecurityPath = options.securityPath {
+        // Create the explicit security path if necessary
+        if !localFileSystem.exists(explicitSecurityPath) {
+            try localFileSystem.createDirectory(explicitSecurityPath, recursive: true)
+        }
+        return explicitSecurityPath
+    }
+
+    do {
+        let sharedSecurityDirectory = try localFileSystem.getOrCreateSwiftPMSecurityDirectory()
+        // And make sure we can write files (locking the directory writes a lock file)
+        try localFileSystem.withLock(on: sharedSecurityDirectory, type: .exclusive) { }
+        return sharedSecurityDirectory
+    } catch {
+        observabilityScope.emit(warning: "Failed creating default security location, \(error)")
+        return .none
+    }
+}
+
+private func getSharedConfigurationDirectory(options: SwiftToolOptions, observabilityScope: ObservabilityScope) throws -> AbsolutePath? {
+    if let explicitConfigPath = options.configPath {
+        // Create the explicit config path if necessary
+        if !localFileSystem.exists(explicitConfigPath) {
+            try localFileSystem.createDirectory(explicitConfigPath, recursive: true)
+        }
+        return explicitConfigPath
+    }
+
+    do {
+        return try localFileSystem.getOrCreateSwiftPMConfigurationDirectory(observabilityScope: observabilityScope)
+    } catch {
+        observabilityScope.emit(warning: "Failed creating default configuration location, \(error)")
+        return .none
+    }
+}
+
+private func getSharedCacheDirectory(options: SwiftToolOptions, observabilityScope: ObservabilityScope) throws -> AbsolutePath? {
+    if let explicitCachePath = options.cachePath {
+        // Create the explicit cache path if necessary
+        if !localFileSystem.exists(explicitCachePath) {
+            try localFileSystem.createDirectory(explicitCachePath, recursive: true)
+        }
+        return explicitCachePath
+    }
+
+    do {
+        return try localFileSystem.getOrCreateSwiftPMCacheDirectory()
+    } catch {
+        observabilityScope.emit(warning: "Failed creating default cache location, \(error)")
+        return .none
+    }
+}
+
 /// A wrapper to hold the build system so we can use it inside
 /// the int. handler without requiring to initialize it.
 final class BuildSystemRef {