Unskip tests blocked by requireThreadSafeWorkingDirectory

Now that llbuild has fork/exec support, the tests can be enabled on Amazon Linux 2, OpenBSD, etc.

Author: jakepetroules

Sources/SWBApplePlatform/AssetCatalogCompiler.swift

@@ -55,7 +55,7 @@ public final class ActoolCompilerSpec : GenericCompilerSpec, SpecIdentifierType,
     }
 
     private func assetTagCombinations(catalogInputs inputs: [FileToBuild], _ cbc: CommandBuildContext, _ delegate: any TaskGenerationDelegate) async throws -> Set<Set<String>> {
-        return try await executeExternalTool(cbc, delegate, commandLine: [resolveExecutablePath(cbc, cbc.scope.actoolExecutablePath()).str, "--print-asset-tag-combinations", "--output-format", "xml1"] + inputs.map { $0.absolutePath.str }, workingDirectory: cbc.producer.defaultWorkingDirectory.str, environment: environmentFromSpec(cbc, delegate).bindingsDictionary, executionDescription: "Compute asset tag combinations") { output in
+        return try await executeExternalTool(cbc, delegate, commandLine: [resolveExecutablePath(cbc, cbc.scope.actoolExecutablePath()).str, "--print-asset-tag-combinations", "--output-format", "xml1"] + inputs.map { $0.absolutePath.str }, workingDirectory: cbc.producer.defaultWorkingDirectory, environment: environmentFromSpec(cbc, delegate).bindingsDictionary, executionDescription: "Compute asset tag combinations") { output in
             struct AssetCatalogToolOutput: Decodable {
                 struct Diagnostic: Decodable {
                     let description: String

Sources/SWBApplePlatform/CoreDataCompiler.swift

@@ -67,7 +67,7 @@ public final class CoreDataModelCompilerSpec : GenericCompilerSpec, SpecIdentifi
             // Mark the entire directory structure as being watched by the build system.
             delegate.access(path: input.absolutePath)
 
-            generatedFiles = try await generatedFilePaths(cbc, delegate, commandLine: [commandLine[0]] + ["--dry-run"] + commandLine[1...], workingDirectory: cbc.producer.defaultWorkingDirectory.str, environment: self.environmentFromSpec(cbc, delegate).bindingsDictionary, executionDescription: "Compute data model \(input.absolutePath.basename) code generation output paths") { output in
+            generatedFiles = try await generatedFilePaths(cbc, delegate, commandLine: [commandLine[0]] + ["--dry-run"] + commandLine[1...], workingDirectory: cbc.producer.defaultWorkingDirectory, environment: self.environmentFromSpec(cbc, delegate).bindingsDictionary, executionDescription: "Compute data model \(input.absolutePath.basename) code generation output paths") { output in
                 return output.unsafeStringValue.split(separator: "\n").map(Path.init).map { $0.prependingPrivatePrefixIfNeeded(otherPath: outputDir) }
             }
             guard !generatedFiles.isEmpty else {

Sources/SWBApplePlatform/CoreMLCompiler.swift

@@ -241,7 +241,7 @@ public final class CoreMLCompilerSpec : GenericCompilerSpec, SpecIdentifierType,
             // Mark the file as being watched by the build system to invalidate the build description.
             delegate.access(path: input.absolutePath)
 
-            generatedFiles = try await generatedFilePaths(cbc, delegate, commandLine: commandLine[0...3] + ["--dry-run", "yes"] + commandLine[4...], workingDirectory: cbc.producer.defaultWorkingDirectory.str, environment: self.environmentFromSpec(cbc, delegate).bindingsDictionary, executionDescription: "Compute CoreML model \(input.absolutePath.basename) code generation output paths") { output in
+            generatedFiles = try await generatedFilePaths(cbc, delegate, commandLine: commandLine[0...3] + ["--dry-run", "yes"] + commandLine[4...], workingDirectory: cbc.producer.defaultWorkingDirectory, environment: self.environmentFromSpec(cbc, delegate).bindingsDictionary, executionDescription: "Compute CoreML model \(input.absolutePath.basename) code generation output paths") { output in
                 return output.unsafeStringValue.split(separator: "\n").map(Path.init)
             }
             guard !generatedFiles.isEmpty else {

Sources/SWBApplePlatform/IntentsCompiler.swift

@@ -194,7 +194,7 @@ public final class IntentsCompilerSpec : GenericCompilerSpec, SpecIdentifierType
             // Mark the file as being watched by the build system to invalidate the build description.
             delegate.access(path: input.absolutePath)
 
-            generatedFiles = try await generatedFilePaths(cbc, delegate, commandLine: commandLine[0...1] + ["-dryRun"] + commandLine[2...], workingDirectory: cbc.producer.defaultWorkingDirectory.str, environment: self.environmentFromSpec(cbc, delegate).bindingsDictionary, executionDescription: "Compute Intent Definition \(input.absolutePath.basename) code generation output paths") { output in
+            generatedFiles = try await generatedFilePaths(cbc, delegate, commandLine: commandLine[0...1] + ["-dryRun"] + commandLine[2...], workingDirectory: cbc.producer.defaultWorkingDirectory, environment: self.environmentFromSpec(cbc, delegate).bindingsDictionary, executionDescription: "Compute Intent Definition \(input.absolutePath.basename) code generation output paths") { output in
                 return output.unsafeStringValue.split(separator: "\n").map(Path.init).map { $0.prependingPrivatePrefixIfNeeded(otherPath: outputDir) }
             }
             guard !generatedFiles.isEmpty else {

Sources/SWBApplePlatform/XCStringsCompiler.swift

@@ -62,7 +62,7 @@ public final class XCStringsCompilerSpec: GenericCompilerSpec, SpecIdentifierTyp
             // xcstringstool compile --dry-run
             dryRunCommandLine.insert("--dry-run", at: 2)
 
-            outputs = try await generatedFilePaths(cbc, delegate, commandLine: dryRunCommandLine, workingDirectory: cbc.producer.defaultWorkingDirectory.str, environment: environmentFromSpec(cbc, delegate).bindingsDictionary, executionDescription: "Compute XCStrings \(cbc.input.absolutePath.basename) output paths") { output in
+            outputs = try await generatedFilePaths(cbc, delegate, commandLine: dryRunCommandLine, workingDirectory: cbc.producer.defaultWorkingDirectory, environment: environmentFromSpec(cbc, delegate).bindingsDictionary, executionDescription: "Compute XCStrings \(cbc.input.absolutePath.basename) output paths") { output in
                 return output.unsafeStringValue.split(separator: "\n").map(Path.init)
             }
         } catch {

Sources/SWBBuildService/BuildDependencyInfo.swift

@@ -469,7 +469,7 @@ extension BuildDependencyInfo {
 
 /// Special `CoreClientDelegate`-conforming struct because our use of `GlobalProductPlan` here should never be running external tools.
 fileprivate struct UnsupportedCoreClientDelegate: CoreClientDelegate {
-    func executeExternalTool(commandLine: [String], workingDirectory: String?, environment: [String : String]) async throws -> ExternalToolResult {
+    func executeExternalTool(commandLine: [String], workingDirectory: Path?, environment: [String : String]) async throws -> ExternalToolResult {
         throw StubError.error("Running external tools is not supported when computing build dependency target info.")
     }
 }

Sources/SWBBuildService/ClientExchangeDelegate.swift

@@ -33,7 +33,7 @@ final class ClientExchangeDelegate: ClientDelegate {
         self.session = session
     }
 
-    func executeExternalTool(commandLine: [String], workingDirectory: String?, environment: [String : String]) async throws -> ExternalToolResult {
+    func executeExternalTool(commandLine: [String], workingDirectory: Path?, environment: [String : String]) async throws -> ExternalToolResult {
         // Create a synchronous client exchange which the session uses to handle the response from the client, to make the communication synchronous from the point of view of our caller.
         let exchange = SynchronousClientExchange<ExternalToolExecutionResponse>(session)
 

Sources/SWBBuildSystem/BuildOperation.swift

@@ -1122,14 +1122,14 @@ private struct OperatorSystemAdaptorDynamicContext: DynamicTaskExecutionDelegate
     }
 
     @discardableResult
-    func spawn(commandLine: [String], environment: [String: String], workingDirectory: String, processDelegate: any ProcessDelegate) async throws -> Bool {
+    func spawn(commandLine: [String], environment: [String: String], workingDirectory: Path, processDelegate: any ProcessDelegate) async throws -> Bool {
         guard let jobContext else {
             throw StubError.error("API misuse. Spawning processes is only allowed from `performTaskAction`.")
         }
 
         // This calls into llb_buildsystem_command_interface_spawn, which can block, so ensure it's shunted to a new thread so as not to block the Swift Concurrency thread pool. This shouldn't risk thread explosion because this function is only allowed to be called from performTaskAction, which in turn should be bounded to ncores based on the number of active llbuild lane threads.
         return await _Concurrency.Task.detachNewThread(name: "llb_buildsystem_command_interface_spawn") { [commandInterface, jobContext, processDelegate] in
-            commandInterface.spawn(jobContext, commandLine: commandLine, environment: environment, workingDirectory: workingDirectory, processDelegate: processDelegate)
+            commandInterface.spawn(jobContext, commandLine: commandLine, environment: environment, workingDirectory: workingDirectory.str, processDelegate: processDelegate)
         }
     }
 

Sources/SWBCore/ProcessExecutionCache.swift

@@ -14,12 +14,20 @@ public import SWBUtil
 
 public final class ProcessExecutionCache: Sendable {
     private let cache = AsyncCache<[String], Processes.ExecutionResult>()
+    private let workingDirectory: Path?
 
-    public init() { }
+    public init(workingDirectory: Path? = .root) {
+        // FIXME: Work around lack of thread-safe working directory support in Foundation (Amazon Linux 2, OpenBSD). Executing processes in the current working directory is less deterministic, but all of the clients which use this class are generally not expected to be sensitive to the working directory anyways. This workaround can be removed once we drop support for Amazon Linux 2 and/or adopt swift-subprocess and/or Foundation.Process's working directory support is made thread safe.
+        if try! Process.hasUnsafeWorkingDirectorySupport {
+            self.workingDirectory = nil
+            return
+        }
+        self.workingDirectory = workingDirectory
+    }
 
     public func run(_ delegate: any CoreClientTargetDiagnosticProducingDelegate, _ commandLine: [String], executionDescription: String?) async throws -> Processes.ExecutionResult {
         try await cache.value(forKey: commandLine) {
-            try await delegate.executeExternalTool(commandLine: commandLine, workingDirectory: "/", environment: [:], executionDescription: executionDescription)
+            try await delegate.executeExternalTool(commandLine: commandLine, workingDirectory: workingDirectory, environment: [:], executionDescription: executionDescription)
         }
     }
 }

Sources/SWBCore/SpecImplementations/CommandLineToolSpec.swift

@@ -1393,15 +1393,15 @@ open class CommandLineToolSpec : PropertyDomainSpec, SpecType, TaskTypeDescripti
         return Base.instance.shouldStart(task, buildCommand: buildCommand)
     }
 
-    public func executeExternalTool<T>(_ cbc: CommandBuildContext, _ delegate: any TaskGenerationDelegate, commandLine: [String], workingDirectory: String?, environment: [String: String], executionDescription: String?, _ parse: @escaping (ByteString) throws -> T) async throws -> T {
+    public func executeExternalTool<T>(_ cbc: CommandBuildContext, _ delegate: any TaskGenerationDelegate, commandLine: [String], workingDirectory: Path?, environment: [String: String], executionDescription: String?, _ parse: @escaping (ByteString) throws -> T) async throws -> T {
         let executionResult = try await delegate.executeExternalTool(commandLine: commandLine, workingDirectory: workingDirectory, environment: environment, executionDescription: executionDescription)
         guard executionResult.exitStatus.isSuccess else {
             throw RunProcessNonZeroExitError(args: commandLine, workingDirectory: workingDirectory, environment: .init(environment), status: executionResult.exitStatus, stdout: ByteString(executionResult.stdout), stderr: ByteString(executionResult.stderr))
         }
         return try parse(ByteString(executionResult.stdout))
     }
 
-    public func generatedFilePaths(_ cbc: CommandBuildContext, _ delegate: any TaskGenerationDelegate, commandLine: [String], workingDirectory: String?, environment: [String: String], executionDescription: String?, _ parse: @escaping (ByteString) throws -> [Path]) async throws -> [Path] {
+    public func generatedFilePaths(_ cbc: CommandBuildContext, _ delegate: any TaskGenerationDelegate, commandLine: [String], workingDirectory: Path?, environment: [String: String], executionDescription: String?, _ parse: @escaping (ByteString) throws -> [Path]) async throws -> [Path] {
         return try await executeExternalTool(cbc, delegate, commandLine: commandLine, workingDirectory: workingDirectory, environment: environment, executionDescription: executionDescription, parse)
     }
 }

Sources/SWBCore/TaskGeneration.swift

@@ -647,7 +647,7 @@ public struct PlannedTaskBuilder {
 
 /// Interface by which core classes can request information from the client.
 public protocol CoreClientDelegate {
-    func executeExternalTool(commandLine: [String], workingDirectory: String?, environment: [String: String]) async throws -> ExternalToolResult
+    func executeExternalTool(commandLine: [String], workingDirectory: Path?, environment: [String: String]) async throws -> ExternalToolResult
 }
 
 public protocol CoreClientTargetDiagnosticProducingDelegate: AnyObject, TargetDiagnosticProducingDelegate, ActivityReporter {
@@ -757,7 +757,7 @@ extension TaskGenerationDelegate {
 }
 
 extension CoreClientTargetDiagnosticProducingDelegate {
-    public func executeExternalTool(commandLine: [String], workingDirectory: String? = nil, environment: [String: String] = [:], executionDescription: String?) async throws -> Processes.ExecutionResult {
+    public func executeExternalTool(commandLine: [String], workingDirectory: Path? = nil, environment: [String: String] = [:], executionDescription: String?) async throws -> Processes.ExecutionResult {
         try await withActivity(ruleInfo: "ExecuteExternalTool " + commandLine.joined(separator: " "), executionDescription: executionDescription ?? CommandLineToolSpec.fallbackExecutionDescription, signature: ByteString(encodingAsUTF8: "\(commandLine) \(String(describing: workingDirectory)) \(environment)"), target: nil, parentActivity: ActivityID.buildDescriptionActivity) { activity in
             try await coreClientDelegate.executeExternalTool(commandLine: commandLine, workingDirectory: workingDirectory, environment: environment)
         }
@@ -768,15 +768,15 @@ extension CoreClientTargetDiagnosticProducingDelegate {
 private let externalToolExecutionQueue = AsyncOperationQueue(concurrentTasks: ProcessInfo.processInfo.activeProcessorCount)
 
 extension CoreClientDelegate {
-    package func executeExternalTool(commandLine: [String], workingDirectory: String? = nil, environment: [String: String] = [:]) async throws -> Processes.ExecutionResult {
+    package func executeExternalTool(commandLine: [String], workingDirectory: Path? = nil, environment: [String: String] = [:]) async throws -> Processes.ExecutionResult {
         switch try await executeExternalTool(commandLine: commandLine, workingDirectory: workingDirectory, environment: environment) {
         case .deferred:
             guard let url = commandLine.first.map(URL.init(fileURLWithPath:)) else {
                 throw StubError.error("Cannot execute empty command line.")
             }
 
             return try await externalToolExecutionQueue.withOperation {
-                try await Process.getOutput(url: url, arguments: Array(commandLine.dropFirst()), currentDirectoryURL: workingDirectory.map(URL.init(fileURLWithPath:)), environment: Environment.current.addingContents(of: .init(environment)))
+                try await Process.getOutput(url: url, arguments: Array(commandLine.dropFirst()), currentDirectoryURL: workingDirectory.map { URL(fileURLWithPath: $0.str) }, environment: Environment.current.addingContents(of: .init(environment)))
             }
         case let .result(status, stdout, stderr):
             return Processes.ExecutionResult(exitStatus: status, stdout: stdout, stderr: stderr)

Sources/SWBProtocol/ClientExchangeMessages.swift

@@ -21,10 +21,10 @@ public struct ExternalToolExecutionRequest: ClientExchangeMessage, Equatable {
     public let exchangeHandle: String
 
     public let commandLine: [String]
-    public let workingDirectory: String?
+    public let workingDirectory: Path?
     public let environment: [String: String]
 
-    public init(sessionHandle: String, exchangeHandle: String, commandLine: [String], workingDirectory: String?, environment: [String: String]) {
+    public init(sessionHandle: String, exchangeHandle: String, commandLine: [String], workingDirectory: Path?, environment: [String: String]) {
         self.sessionHandle = sessionHandle
         self.exchangeHandle = exchangeHandle
         self.commandLine = commandLine

Sources/SWBTaskExecution/TaskActions/ClangCompileTaskAction.swift

@@ -266,7 +266,7 @@ public final class ClangCompileTaskAction: TaskAction, BuildValueValidatingTaskA
                 let commandLine = command.arguments
                 let delegate = TaskProcessDelegate(outputDelegate: outputDelegate)
                 // The frontend invocations should be unaffected by the environment, pass an empty one.
-                try await spawn(commandLine: commandLine, environment: [:], workingDirectory: task.workingDirectory.str, dynamicExecutionDelegate: dynamicExecutionDelegate, clientDelegate: clientDelegate, processDelegate: delegate)
+                try await spawn(commandLine: commandLine, environment: [:], workingDirectory: task.workingDirectory, dynamicExecutionDelegate: dynamicExecutionDelegate, clientDelegate: clientDelegate, processDelegate: delegate)
                 lastResult = delegate.commandResult
 
                 if lastResult == .succeeded {

Sources/SWBTaskExecution/TaskActions/CodeSignTaskAction.swift

@@ -37,7 +37,7 @@ public final class CodeSignTaskAction: TaskAction {
                 commandLine.insert(preEncryptHashesFlag, at: 1)
             }
 
-            try await spawn(commandLine: commandLine, environment: task.environment.bindingsDictionary, workingDirectory: task.workingDirectory.str, dynamicExecutionDelegate: dynamicExecutionDelegate, clientDelegate: clientDelegate, processDelegate: processDelegate)
+            try await spawn(commandLine: commandLine, environment: task.environment.bindingsDictionary, workingDirectory: task.workingDirectory, dynamicExecutionDelegate: dynamicExecutionDelegate, clientDelegate: clientDelegate, processDelegate: processDelegate)
         } catch {
             outputDelegate.error(error.localizedDescription)
             return .failed

Sources/SWBTaskExecution/TaskActions/CopyTiffTaskAction.swift

@@ -145,7 +145,7 @@ public final class CopyTiffTaskAction: TaskAction {
 
                 let processDelegate = TaskProcessDelegate(outputDelegate: outputDelegate)
                 do {
-                    try await spawn(commandLine: tiffutilCommand, environment: task.environment.bindingsDictionary, workingDirectory: task.workingDirectory.str, dynamicExecutionDelegate: dynamicExecutionDelegate, clientDelegate: clientDelegate, processDelegate: processDelegate)
+                    try await spawn(commandLine: tiffutilCommand, environment: task.environment.bindingsDictionary, workingDirectory: task.workingDirectory, dynamicExecutionDelegate: dynamicExecutionDelegate, clientDelegate: clientDelegate, processDelegate: processDelegate)
                 } catch {
                     outputDelegate.error(error.localizedDescription)
                     return .failed