Merge pull request #572 from artemcm/ParsableThroneOfLies

[Parsable Output] Break down batch compile jobs into constituent messages

Author: artemcm

Sources/SwiftDriver/Driver/Driver.swift

@@ -1018,6 +1018,7 @@ extension Driver {
       buildRecordInfo: buildRecordInfo,
       incrementalCompilationState: incrementalCompilationState,
       showJobLifecycle: showJobLifecycle,
+      argsResolver: executor.resolver,
       diagnosticEngine: diagnosticEngine)
   }
 

Sources/SwiftDriver/Driver/ToolExecutionDelegate.swift

@@ -23,7 +23,16 @@ import Glibc
 #endif
 
 /// Delegate for printing execution information on the command-line.
-final class ToolExecutionDelegate: JobExecutionDelegate {
+@_spi(Testing) public final class ToolExecutionDelegate: JobExecutionDelegate {
+  /// Quasi-PIDs are _negative_ PID-like unique keys used to
+  /// masquerade batch job constituents as (quasi)processes, when writing
+  /// parseable output to consumers that don't understand the idea of a batch
+  /// job. They are negative in order to avoid possibly colliding with real
+  /// PIDs (which are always positive). We start at -1000 here as a crude but
+  /// harmless hedge against colliding with an errno value that might slip
+  /// into the stream of real PIDs.
+  static let QUASI_PID_START = -1000
+
   public enum Mode {
     case verbose
     case parsableOutput
@@ -35,19 +44,25 @@ final class ToolExecutionDelegate: JobExecutionDelegate {
   public let incrementalCompilationState: IncrementalCompilationState?
   public let showJobLifecycle: Bool
   public let diagnosticEngine: DiagnosticsEngine
-
   public var anyJobHadAbnormalExit: Bool = false
 
-  init(mode: ToolExecutionDelegate.Mode,
-       buildRecordInfo: BuildRecordInfo?,
-       incrementalCompilationState: IncrementalCompilationState?,
-       showJobLifecycle: Bool,
-       diagnosticEngine: DiagnosticsEngine) {
+  private var nextBatchQuasiPID: Int
+  private let argsResolver: ArgsResolver
+  private var batchJobInputQuasiPIDMap = DictionaryOfDictionaries<Job, TypedVirtualPath, Int>()
+
+  @_spi(Testing) public init(mode: ToolExecutionDelegate.Mode,
+                             buildRecordInfo: BuildRecordInfo?,
+                             incrementalCompilationState: IncrementalCompilationState?,
+                             showJobLifecycle: Bool,
+                             argsResolver: ArgsResolver,
+                             diagnosticEngine: DiagnosticsEngine) {
     self.mode = mode
     self.buildRecordInfo = buildRecordInfo
     self.incrementalCompilationState = incrementalCompilationState
     self.showJobLifecycle = showJobLifecycle
     self.diagnosticEngine = diagnosticEngine
+    self.argsResolver = argsResolver
+    self.nextBatchQuasiPID = ToolExecutionDelegate.QUASI_PID_START
   }
 
   public func jobStarted(job: Job, arguments: [String], pid: Int) {
@@ -61,22 +76,10 @@ final class ToolExecutionDelegate: JobExecutionDelegate {
       stdoutStream <<< arguments.map { $0.spm_shellEscaped() }.joined(separator: " ") <<< "\n"
       stdoutStream.flush()
     case .parsableOutput:
-
-      // Compute the outputs for the message.
-      let outputs: [BeganMessage.Output] = job.outputs.map {
-        .init(path: $0.file.name, type: $0.type.description)
+      let messages = constructJobBeganMessages(job: job, arguments: arguments, pid: pid)
+      for beganMessage in messages {
+        emit(ParsableMessage(name: job.kind.rawValue, kind: .began(beganMessage)))
       }
-
-      let beganMessage = BeganMessage(
-        pid: pid,
-        inputs: job.displayInputs.map{ $0.file.name },
-        outputs: outputs,
-        commandExecutable: arguments[0],
-        commandArguments: arguments[1...].map { String($0) }
-      )
-
-      let message = ParsableMessage(name: job.kind.rawValue, kind: .began(beganMessage))
-      emit(message)
     }
   }
 
@@ -108,21 +111,26 @@ final class ToolExecutionDelegate: JobExecutionDelegate {
 
     case .parsableOutput:
       let output = (try? result.utf8Output() + result.utf8stderrOutput()).flatMap { $0.isEmpty ? nil : $0 }
-      let message: ParsableMessage
+      let messages: [ParsableMessage]
 
       switch result.exitStatus {
       case .terminated(let code):
-        let finishedMessage = FinishedMessage(exitStatus: Int(code), pid: pid, output: output)
-        message = ParsableMessage(name: job.kind.rawValue, kind: .finished(finishedMessage))
-
+        messages = constructJobFinishedMessages(job: job, exitCode: code, output: output,
+                                                pid: pid).map {
+          ParsableMessage(name: job.kind.rawValue, kind: .finished($0))
+        }
 #if !os(Windows)
       case .signalled(let signal):
         let errorMessage = strsignal(signal).map { String(cString: $0) } ?? ""
-        let signalledMessage = SignalledMessage(pid: pid, output: output, errorMessage: errorMessage, signal: Int(signal))
-        message = ParsableMessage(name: job.kind.rawValue, kind: .signalled(signalledMessage))
+        messages = constructJobSignalledMessages(job: job, error: errorMessage, output: output,
+                                                 signal: signal, pid: pid).map {
+          ParsableMessage(name: job.kind.rawValue, kind: .signalled($0))
+        }
 #endif
       }
-      emit(message)
+      for message in messages {
+        emit(message)
+      }
     }
   }
 
@@ -151,6 +159,147 @@ final class ToolExecutionDelegate: JobExecutionDelegate {
   }
 }
 
+// MARK: - Message Construction
+/// Generation of messages from jobs, including breaking down batch compile jobs into constituent messages.
+private extension ToolExecutionDelegate {
+
+  // MARK: - Job Began
+  func constructJobBeganMessages(job: Job, arguments: [String], pid: Int) -> [BeganMessage] {
+    let result : [BeganMessage]
+    if job.kind == .compile,
+       job.primaryInputs.count > 1 {
+      // Batched compile jobs need to be broken up into multiple messages, one per constituent.
+      result = constructBatchCompileBeginMessages(job: job, arguments: arguments, pid: pid,
+                                                  quasiPIDBase: nextBatchQuasiPID)
+      // Today, parseable-output messages are constructed and emitted synchronously
+      // on `MultiJobExecutor`'s `delegateQueue`. This is why the below operation is safe.
+      nextBatchQuasiPID -= result.count
+    } else {
+      result = [constructSingleBeganMessage(inputs: job.displayInputs,
+                                            outputs: job.outputs,
+                                            arguments: arguments,
+                                            pid: pid,
+                                            realPid: pid)]
+    }
+
+    return result
+  }
+
+  func constructBatchCompileBeginMessages(job: Job, arguments: [String], pid: Int,
+                                          quasiPIDBase: Int) -> [BeganMessage] {
+    precondition(job.kind == .compile && job.primaryInputs.count > 1)
+    var quasiPID = quasiPIDBase
+    var result : [BeganMessage] = []
+    for input in job.primaryInputs {
+      let outputs = job.getCompileInputOutputs(for: input) ?? []
+      let outputPaths = outputs.map {
+        TypedVirtualPath(file: try! VirtualPath.intern(path: argsResolver.resolve(.path($0.file))),
+                         type: $0.type)
+      }
+      result.append(
+        constructSingleBeganMessage(inputs: [input],
+                                    outputs: outputPaths,
+                                    arguments: arguments,
+                                    pid: quasiPID,
+                                    realPid: pid))
+      // Save the quasiPID of this job/input combination in order to generate the correct
+      // `finished` message
+      batchJobInputQuasiPIDMap[(job, input)] = quasiPID
+      quasiPID -= 1
+    }
+    return result
+  }
+
+  func constructSingleBeganMessage(inputs: [TypedVirtualPath], outputs: [TypedVirtualPath],
+                                   arguments: [String], pid: Int, realPid: Int) -> BeganMessage {
+    let outputs: [BeganMessage.Output] = outputs.map {
+      .init(path: $0.file.name, type: $0.type.description)
+    }
+
+    return BeganMessage(
+      pid: pid,
+      realPid: realPid,
+      inputs: inputs.map{ $0.file.name },
+      outputs: outputs,
+      commandExecutable: arguments[0],
+      commandArguments: arguments[1...].map { String($0) }
+    )
+  }
+
+  // MARK: - Job Finished
+  func constructJobFinishedMessages(job: Job, exitCode: Int32, output: String?, pid: Int)
+  -> [FinishedMessage] {
+    let result : [FinishedMessage]
+    if job.kind == .compile,
+       job.primaryInputs.count > 1 {
+      result = constructBatchCompileFinishedMessages(job: job, exitCode: exitCode,
+                                                     output: output, pid: pid)
+    } else {
+      result = [constructSingleFinishedMessage(exitCode: exitCode, output: output,
+                                               pid: pid, realPid: pid)]
+    }
+    return result
+  }
+
+  func constructBatchCompileFinishedMessages(job: Job, exitCode: Int32, output: String?, pid: Int)
+  -> [FinishedMessage] {
+    precondition(job.kind == .compile && job.primaryInputs.count > 1)
+    var result : [FinishedMessage] = []
+    for input in job.primaryInputs {
+      guard let quasiPid = batchJobInputQuasiPIDMap[(job, input)] else {
+        fatalError("Parsable-Output batch sub-job finished with no matching started message: \(job.description) : \(input.file.description)")
+      }
+      result.append(
+        constructSingleFinishedMessage(exitCode: exitCode, output: output,
+                                       pid: quasiPid, realPid: pid))
+    }
+    return result
+  }
+
+  func constructSingleFinishedMessage(exitCode: Int32, output: String?, pid: Int, realPid: Int)
+  -> FinishedMessage {
+    return FinishedMessage(exitStatus: Int(exitCode), output: output, pid: pid, realPid: realPid)
+  }
+
+  // MARK: - Job Signalled
+  func constructJobSignalledMessages(job: Job, error: String, output: String?,
+                                     signal: Int32, pid: Int) -> [SignalledMessage] {
+    let result : [SignalledMessage]
+    if job.kind == .compile,
+       job.primaryInputs.count > 1 {
+      result = constructBatchCompileSignalledMessages(job: job, error: error, output: output,
+                                                      signal: signal, pid: pid)
+    } else {
+      result = [constructSingleSignalledMessage(error: error, output: output, signal: signal,
+                                                pid: pid, realPid: pid)]
+    }
+    return result
+  }
+
+  func constructBatchCompileSignalledMessages(job: Job, error: String, output: String?,
+                                              signal: Int32, pid: Int)
+  -> [SignalledMessage] {
+    precondition(job.kind == .compile && job.primaryInputs.count > 1)
+    var result : [SignalledMessage] = []
+    for input in job.primaryInputs {
+      guard let quasiPid = batchJobInputQuasiPIDMap[(job, input)] else {
+        fatalError("Parsable-Output batch sub-job signalled with no matching started message: \(job.description) : \(input.file.description)")
+      }
+      result.append(
+        constructSingleSignalledMessage(error: error, output: output, signal: signal,
+                                        pid: quasiPid, realPid: pid))
+    }
+    return result
+  }
+
+  func constructSingleSignalledMessage(error: String, output: String?, signal: Int32,
+                                       pid: Int, realPid: Int)
+  -> SignalledMessage {
+    return SignalledMessage(pid: pid, realPid: realPid, output: output,
+                            errorMessage: error, signal: Int(signal))
+  }
+}
+
 fileprivate extension Diagnostic.Message {
   static func remark_job_lifecycle(_ what: String, _ job: Job
   ) -> Diagnostic.Message {

Sources/SwiftDriver/Execution/ParsableOutput.swift

@@ -39,6 +39,18 @@ import Foundation
   }
 }
 
+@_spi(Testing) public struct ActualProcess: Encodable {
+  public let realPid: Int
+
+  public init(realPid: Int) {
+    self.realPid = realPid
+  }
+
+  private enum CodingKeys: String, CodingKey {
+    case realPid = "real_pid"
+  }
+}
+
 @_spi(Testing) public struct BeganMessage: Encodable {
   public struct Output: Encodable {
     public let type: String
@@ -50,6 +62,7 @@ import Foundation
     }
   }
 
+  public let process: ActualProcess
   public let pid: Int
   public let inputs: [String]
   public let outputs: [Output]
@@ -58,12 +71,14 @@ import Foundation
 
   public init(
     pid: Int,
+    realPid: Int,
     inputs: [String],
     outputs: [Output],
     commandExecutable: String,
     commandArguments: [String]
   ) {
     self.pid = pid
+    self.process = ActualProcess(realPid: realPid)
     self.inputs = inputs
     self.outputs = outputs
     self.commandExecutable = commandExecutable
@@ -72,6 +87,7 @@ import Foundation
 
   private enum CodingKeys: String, CodingKey {
     case pid
+    case process
     case inputs
     case outputs
     case commandExecutable = "command_executable"
@@ -94,42 +110,47 @@ import Foundation
 @_spi(Testing) public struct FinishedMessage: Encodable {
   let exitStatus: Int
   let pid: Int
+  let process: ActualProcess
   let output: String?
 
-  // proc-info
-
   public init(
     exitStatus: Int,
+    output: String?,
     pid: Int,
-    output: String?
+    realPid: Int
   ) {
     self.exitStatus = exitStatus
     self.pid = pid
+    self.process = ActualProcess(realPid: realPid)
     self.output = output
   }
 
   private enum CodingKeys: String, CodingKey {
     case pid
+    case process
     case output
     case exitStatus = "exit-status"
   }
 }
 
 @_spi(Testing) public struct SignalledMessage: Encodable {
   let pid: Int
+  let process: ActualProcess
   let output: String?
   let errorMessage: String
   let signal: Int
 
-  public init(pid: Int, output: String?, errorMessage: String, signal: Int) {
+  public init(pid: Int, realPid: Int, output: String?, errorMessage: String, signal: Int) {
     self.pid = pid
+    self.process = ActualProcess(realPid: realPid)
     self.output = output
     self.errorMessage = errorMessage
     self.signal = signal
   }
 
   private enum CodingKeys: String, CodingKey {
     case pid
+    case process
     case output
     case errorMessage = "error-message"
     case signal