Merge pull request #595 from artemcm/ProcessInputSergio

Add support for compile jobs that read source input from standard input.

Author: artemcm

Package.resolved

@@ -10,6 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 import TSCBasic
+import Foundation
 
 /// Abstraction for functionality that allows working with subprocesses.
 public protocol ProcessProtocol {
@@ -19,7 +20,7 @@ public protocol ProcessProtocol {
   /// a negative number to represent a "quasi-pid".
   ///
   /// - SeeAlso: https://github.com/apple/swift/blob/main/docs/DriverParseableOutput.rst#quasi-pids
-  var processID: Process.ProcessID { get }
+  var processID: TSCBasic.Process.ProcessID { get }
 
   /// Wait for the process to finish execution.
   @discardableResult
@@ -29,15 +30,39 @@ public protocol ProcessProtocol {
     arguments: [String],
     env: [String: String]
   ) throws -> Self
+
+  static func launchProcessAndWriteInput(
+    arguments: [String],
+    env: [String: String],
+    inputFileHandle: FileHandle
+  ) throws -> Self
 }
 
-extension Process: ProcessProtocol {
+extension TSCBasic.Process: ProcessProtocol {
   public static func launchProcess(
     arguments: [String],
     env: [String: String]
-  ) throws -> Process {
+  ) throws -> TSCBasic.Process {
     let process = Process(arguments: arguments, environment: env)
     try process.launch()
     return process
   }
+
+  public static func launchProcessAndWriteInput(
+    arguments: [String],
+    env: [String: String],
+    inputFileHandle: FileHandle
+  ) throws -> TSCBasic.Process {
+    let process = Process(arguments: arguments, environment: env)
+    let processInputStream = try process.launch()
+    var input: Data
+    // Write out the contents of the input handle and close the input stream
+    repeat {
+      input = inputFileHandle.availableData
+      processInputStream.write(input)
+    } while (input.count > 0)
+    processInputStream.flush()
+    try processInputStream.close()
+    return process
+  }
 }

Sources/SwiftDriver/Execution/ProcessProtocol.swift

@@ -82,6 +82,9 @@ public final class MultiJobExecutor {
     /// The type to use when launching new processes. This mostly serves as an override for testing.
     let processType: ProcessProtocol.Type
 
+    /// The standard input `FileHandle` override for testing.
+    let testInputHandle: FileHandle?
+
     /// If a job fails, the driver needs to stop running jobs.
     private(set) var isBuildCancelled = false
 
@@ -100,7 +103,8 @@ public final class MultiJobExecutor {
       forceResponseFiles: Bool,
       recordedInputModificationDates: [TypedVirtualPath: Date],
       diagnosticsEngine: DiagnosticsEngine,
-      processType: ProcessProtocol.Type = Process.self
+      processType: ProcessProtocol.Type = Process.self,
+      inputHandleOverride: FileHandle? = nil
     ) {
       (
         jobs: self.jobs,
@@ -121,6 +125,7 @@ public final class MultiJobExecutor {
       self.recordedInputModificationDates = recordedInputModificationDates
       self.diagnosticsEngine = diagnosticsEngine
       self.processType = processType
+      self.testInputHandle = inputHandleOverride
     }
 
     private static func fillInJobsAndProducers(_ workload: DriverExecutorWorkload
@@ -248,6 +253,9 @@ public final class MultiJobExecutor {
   /// The type to use when launching new processes. This mostly serves as an override for testing.
   private let processType: ProcessProtocol.Type
 
+  /// The standard input `FileHandle`  override for testing.
+  let testInputHandle: FileHandle?
+
   public init(
     workload: DriverExecutorWorkload,
     resolver: ArgsResolver,
@@ -257,7 +265,8 @@ public final class MultiJobExecutor {
     processSet: ProcessSet? = nil,
     forceResponseFiles: Bool = false,
     recordedInputModificationDates: [TypedVirtualPath: Date] = [:],
-    processType: ProcessProtocol.Type = Process.self
+    processType: ProcessProtocol.Type = Process.self,
+    inputHandleOverride: FileHandle? = nil
   ) {
     self.workload = workload
     self.argsResolver = resolver
@@ -268,6 +277,7 @@ public final class MultiJobExecutor {
     self.forceResponseFiles = forceResponseFiles
     self.recordedInputModificationDates = recordedInputModificationDates
     self.processType = processType
+    self.testInputHandle = inputHandleOverride
   }
 
   /// Execute all jobs.
@@ -314,7 +324,8 @@ public final class MultiJobExecutor {
       forceResponseFiles: forceResponseFiles,
       recordedInputModificationDates: recordedInputModificationDates,
       diagnosticsEngine: diagnosticsEngine,
-      processType: processType
+      processType: processType,
+      inputHandleOverride: testInputHandle
     )
   }
 }
@@ -566,9 +577,20 @@ class ExecuteJobRule: LLBuildRule {
       let arguments: [String] = try resolver.resolveArgumentList(for: job,
                                                                  forceResponseFiles: context.forceResponseFiles)
 
-      let process = try context.processType.launchProcess(
-        arguments: arguments, env: env
-      )
+
+      let process : ProcessProtocol
+      // If the input comes from standard input, forward the driver's input to the compile job.
+      if job.inputs.contains(TypedVirtualPath(file: .standardInput, type: .swift)) {
+        let inputFileHandle = context.testInputHandle ?? FileHandle.standardInput
+        process = try context.processType.launchProcessAndWriteInput(
+          arguments: arguments, env: env, inputFileHandle: inputFileHandle
+        )
+      } else {
+        process = try context.processType.launchProcess(
+          arguments: arguments, env: env
+        )
+      }
+
       pid = Int(process.processID)
 
       // Add it to the process set if it's a real process.

Sources/SwiftDriverExecution/MultiJobExecutor.swift

@@ -51,8 +51,14 @@ public final class SwiftDriverExecutor: DriverExecutor {
     } else {
       var childEnv = env
       childEnv.merge(job.extraEnvironment, uniquingKeysWith: { (_, new) in new })
-
-      let process = try Process.launchProcess(arguments: arguments, env: childEnv)
+      let process : ProcessProtocol
+      if job.inputs.contains(TypedVirtualPath(file: .standardInput, type: .swift)) {
+        process = try Process.launchProcessAndWriteInput(
+          arguments: arguments, env: childEnv, inputFileHandle: FileHandle.standardInput
+        )
+      } else {
+        process = try Process.launchProcess(arguments: arguments, env: childEnv)
+      }
       return try process.waitUntilExit()
     }
   }

Sources/SwiftDriverExecution/SwiftDriverExecutor.swift

@@ -29,6 +29,11 @@ class JobCollectingDelegate: JobExecutionDelegate {
       return .init()
     }
 
+    static func launchProcessAndWriteInput(arguments: [String], env: [String : String],
+                                           inputFileHandle: FileHandle) throws -> StubProcess {
+      return .init()
+    }
+
     var processID: TSCBasic.Process.ProcessID { .init(-1) }
 
     func waitUntilExit() throws -> ProcessResult {
@@ -213,6 +218,84 @@ final class JobExecutorTests: XCTestCase {
 #endif
   }
 
+  /// Ensure the executor is capable of forwarding its standard input to the compile job that requires it.
+  func testInputForwarding() throws {
+#if os(macOS)
+    let executor = try SwiftDriverExecutor(diagnosticsEngine: DiagnosticsEngine(),
+                                           processSet: ProcessSet(),
+                                           fileSystem: localFileSystem,
+                                           env: ProcessEnv.vars)
+    let toolchain = DarwinToolchain(env: ProcessEnv.vars, executor: executor)
+    try withTemporaryDirectory { path in
+      let exec = path.appending(component: "main")
+      let compile = Job(
+        moduleName: "main",
+        kind: .compile,
+        tool: .absolute(try toolchain.getToolPath(.swiftCompiler)),
+        commandLine: [
+          "-frontend",
+          "-c",
+          "-primary-file",
+          // This compile job must read the input from STDIN
+          "-",
+          "-target", "x86_64-apple-darwin18.7.0",
+          "-enable-objc-interop",
+          "-sdk",
+          .path(.absolute(try toolchain.sdk.get())),
+          "-module-name", "main",
+          "-o", .path(.temporary(RelativePath("main.o"))),
+        ],
+        inputs: [TypedVirtualPath(file: .standardInput, type: .swift )],
+        primaryInputs: [TypedVirtualPath(file: .standardInput, type: .swift )],
+        outputs: [.init(file: VirtualPath.temporary(RelativePath("main.o")).intern(),
+                        type: .object)]
+      )
+      let link = Job(
+        moduleName: "main",
+        kind: .link,
+        tool: .absolute(try toolchain.getToolPath(.dynamicLinker)),
+        commandLine: [
+          .path(.temporary(RelativePath("main.o"))),
+          .path(.absolute(try toolchain.clangRT.get())),
+          "-syslibroot", .path(.absolute(try toolchain.sdk.get())),
+          "-lobjc", "-lSystem", "-arch", "x86_64",
+          "-force_load", .path(.absolute(try toolchain.compatibility50.get())),
+          "-force_load", .path(.absolute(try toolchain.compatibilityDynamicReplacements.get())),
+          "-L", .path(.absolute(try toolchain.resourcesDirectory.get())),
+          "-L", .path(.absolute(try toolchain.sdkStdlib(sdk: toolchain.sdk.get()))),
+          "-rpath", "/usr/lib/swift", "-macosx_version_min", "10.14.0", "-no_objc_category_merging",
+          "-o", .path(.absolute(exec)),
+        ],
+        inputs: [
+          .init(file: VirtualPath.temporary(RelativePath("main.o")).intern(), type: .object),
+        ],
+        primaryInputs: [],
+        outputs: [.init(file: VirtualPath.relative(RelativePath("main")).intern(), type: .image)]
+      )
+
+      // Create a file with inpuit
+      let inputFile = path.appending(component: "main.swift")
+      try localFileSystem.writeFileContents(inputFile) {
+        $0 <<< "print(\"Hello, World\")"
+      }
+      // We are going to override he executors standard input FileHandle to the above
+      // input file, to simulate it being piped over standard input to this compilation.
+      let testFile: FileHandle = FileHandle(forReadingAtPath: inputFile.description)!
+      let delegate = JobCollectingDelegate()
+      let resolver = try ArgsResolver(fileSystem: localFileSystem)
+      let executor = MultiJobExecutor(workload: .all([compile, link]),
+                                      resolver: resolver, executorDelegate: delegate,
+                                      diagnosticsEngine: DiagnosticsEngine(),
+                                      inputHandleOverride: testFile)
+      try executor.execute(env: toolchain.env, fileSystem: localFileSystem)
+
+      // Execute the resulting program
+      let output = try TSCBasic.Process.checkNonZeroExit(args: exec.pathString)
+      XCTAssertEqual(output, "Hello, World\n")
+    }
+#endif
+  }
+
   func testStubProcessProtocol() throws {
     // This test fails intermittently on Linux
     // rdar://70067844