Merge pull request #103 from cltnschlosser/cs_largeResponseFileRuntime

Driver performance improvements (profile test/Driver/response-file.swift)

Author: DougGregor

README.md

@@ -85,7 +85,7 @@ on the command line:
 ```
 $ SWIFT_DRIVER_ENABLE_INTEGRATION_TESTS=1 \
   SWIFT_DRIVER_LIT_DIR=/path/to/build/Ninja-ReleaseAssert/swift-.../test-... \
-  swift test --parallel
+  swift test -c release --parallel
 ```
 
 #### Preparing a Linux docker for debug

Sources/SwiftDriver/Driver/Driver.swift

@@ -241,7 +241,7 @@ public struct Driver {
       throw Error.subcommandPassedToDriver
     }
 
-    var args = try Self.expandResponseFiles(args, fileSystem: fileSystem, diagnosticsEngine: self.diagnosticEngine)[...]
+    var args = try Self.expandResponseFiles(args, fileSystem: fileSystem, diagnosticsEngine: self.diagnosticEngine)
 
     self.driverKind = try Self.determineDriverKind(args: &args)
     self.optionTable = OptionTable()
@@ -508,18 +508,21 @@ extension Driver {
         continue
       } else if char.isWhitespace && !quoted {
         // This is unquoted, unescaped whitespace, start a new token.
-        tokens.append(token)
-        token = ""
+        if !token.isEmpty {
+          tokens.append(token)
+          token = ""
+        }
         continue
       }
 
       token.append(char)
     }
     // Add the final token
-    tokens.append(token)
+    if !token.isEmpty {
+      tokens.append(token)
+    }
 
-    // Filter any empty tokens that might be parsed if there's excessive whitespace.
-    return tokens.filter { !$0.isEmpty }
+    return tokens
   }
 
   /// Tokenize each line of the response file, omitting empty lines.
@@ -581,7 +584,7 @@ extension Driver {
   /// Determine the driver kind based on the command-line arguments, consuming the arguments
   /// conveying this information.
   public static func determineDriverKind(
-    args: inout ArraySlice<String>
+    args: inout [String]
   ) throws -> DriverKind {
     // Get the basename of the driver executable.
     let execRelPath = args.removeFirst()

Sources/SwiftDriver/Execution/JobExecutor.swift

@@ -118,8 +118,11 @@ public final class JobExecutor {
   /// The context required during job execution.
   struct Context {
 
-    /// This contains mapping from an output to the job that produces that output.
-    let producerMap: [VirtualPath: Job]
+    /// This contains mapping from an output to the index(in the jobs array) of the job that produces that output.
+    let producerMap: [VirtualPath: Int]
+
+    /// All the jobs being executed.
+    let jobs: [Job]
 
     /// The resolver for argument template.
     let argsResolver: ArgsResolver
@@ -155,7 +158,8 @@ public final class JobExecutor {
       argsResolver: ArgsResolver,
       env: [String: String],
       fileSystem: FileSystem,
-      producerMap: [VirtualPath: Job],
+      producerMap: [VirtualPath: Int],
+      jobs: [Job],
       executorDelegate: JobExecutorDelegate,
       jobQueue: OperationQueue,
       processSet: ProcessSet?,
@@ -164,6 +168,7 @@ public final class JobExecutor {
       diagnosticsEngine: DiagnosticsEngine
     ) {
       self.producerMap = producerMap
+      self.jobs = jobs
       self.argsResolver = argsResolver
       self.env = env
       self.fileSystem = fileSystem
@@ -227,7 +232,7 @@ public final class JobExecutor {
     let delegate = JobExecutorBuildDelegate(context)
     let engine = LLBuildEngine(delegate: delegate)
 
-    let result = try engine.build(key: ExecuteAllJobsRule.RuleKey(jobs: jobs))
+    let result = try engine.build(key: ExecuteAllJobsRule.RuleKey())
 
     // Throw the stub error the build didn't finish successfully.
     if !result.success {
@@ -237,11 +242,11 @@ public final class JobExecutor {
 
   /// Create the context required during the execution.
   func createContext(_ jobs: [Job], env: [String: String], fileSystem: FileSystem) -> Context {
-    var producerMap: [VirtualPath: Job] = [:]
-    for job in jobs {
+    var producerMap: [VirtualPath: Int] = [:]
+    for (index, job) in jobs.enumerated() {
       for output in job.outputs {
         assert(!producerMap.keys.contains(output.file), "multiple producers for output \(output): \(job) \(producerMap[output.file]!)")
-        producerMap[output.file] = job
+        producerMap[output.file] = index
       }
     }
 
@@ -254,6 +259,7 @@ public final class JobExecutor {
       env: env,
       fileSystem: fileSystem,
       producerMap: producerMap,
+      jobs: jobs,
       executorDelegate: executorDelegate,
       jobQueue: jobQueue,
       processSet: processSet,
@@ -275,7 +281,7 @@ struct JobExecutorBuildDelegate: LLBuildEngineDelegate {
   func lookupRule(rule: String, key: Key) -> Rule {
     switch rule {
     case ExecuteAllJobsRule.ruleName:
-      return ExecuteAllJobsRule(key, fileSystem: context.fileSystem)
+      return ExecuteAllJobsRule(key, jobs: context.jobs, fileSystem: context.fileSystem)
     case ExecuteJobRule.ruleName:
       return ExecuteJobRule(key, context: context)
     default:
@@ -305,26 +311,26 @@ class ExecuteAllJobsRule: LLBuildRule {
   struct RuleKey: LLBuildKey {
     typealias BuildValue = DriverBuildValue
     typealias BuildRule = ExecuteAllJobsRule
-
-    let jobs: [Job]
   }
 
   override class var ruleName: String { "\(ExecuteAllJobsRule.self)" }
 
   private let key: RuleKey
+  private let jobs: [Job]
 
   /// True if any of the inputs had any error.
   private var allInputsSucceeded: Bool = true
 
-  init(_ key: Key, fileSystem: FileSystem) {
+  init(_ key: Key, jobs: [Job], fileSystem: FileSystem) {
     self.key = RuleKey(key)
+    self.jobs = jobs
     super.init(fileSystem: fileSystem)
   }
 
   override func start(_ engine: LLTaskBuildEngine) {
-    for (idx, job) in key.jobs.enumerated() {
-        let key = ExecuteJobRule.RuleKey(job: job)
-        engine.taskNeedsInput(key, inputID: idx)
+    for index in jobs.indices {
+      let key = ExecuteJobRule.RuleKey(index: index)
+      engine.taskNeedsInput(key, inputID: index)
     }
   }
 
@@ -351,7 +357,7 @@ class ExecuteJobRule: LLBuildRule {
     typealias BuildValue = DriverBuildValue
     typealias BuildRule = ExecuteJobRule
 
-    let job: Job
+    let index: Int
   }
 
   override class var ruleName: String { "\(ExecuteJobRule.self)" }
@@ -369,9 +375,9 @@ class ExecuteJobRule: LLBuildRule {
   }
 
   override func start(_ engine: LLTaskBuildEngine) {
-    for (idx, input) in key.job.inputs.enumerated() {
-      if let producingJob = context.producerMap[input.file] {
-        let key = ExecuteJobRule.RuleKey(job: producingJob)
+    for (idx, input) in context.jobs[key.index].inputs.enumerated() {
+      if let producingJobIndex = context.producerMap[input.file] {
+        let key = ExecuteJobRule.RuleKey(index: producingJobIndex)
         engine.taskNeedsInput(key, inputID: idx)
       }
     }
@@ -404,7 +410,7 @@ class ExecuteJobRule: LLBuildRule {
   private func executeJob(_ engine: LLTaskBuildEngine) {
     let context = self.context
     let resolver = context.argsResolver
-    let job = key.job
+    let job = context.jobs[key.index]
     let env = context.env.merging(job.extraEnvironment, uniquingKeysWith: { $1 })
 
     let value: DriverBuildValue

Tests/SwiftDriverTests/SwiftDriverTests.swift

@@ -58,21 +58,21 @@ final class SwiftDriverTests: XCTestCase {
     func assertArgs(
       _ args: String...,
       parseTo driverKind: DriverKind,
-      leaving remainingArgs: ArraySlice<String>,
+      leaving remainingArgs: [String],
       file: StaticString = #file, line: UInt = #line
     ) throws {
-      var slice = args[...]
-      let result = try Driver.determineDriverKind(args: &slice)
+      var args = args
+      let result = try Driver.determineDriverKind(args: &args)
 
       XCTAssertEqual(result, driverKind, file: file, line: line)
-      XCTAssertEqual(slice, remainingArgs, file: file, line: line)
+      XCTAssertEqual(args, remainingArgs, file: file, line: line)
     }
     func assertArgsThrow(
       _ args: String...,
       file: StaticString = #file, line: UInt = #line
     ) throws {
-      var slice = args[...]
-      XCTAssertThrowsError(try Driver.determineDriverKind(args: &slice))
+      var args = args
+      XCTAssertThrowsError(try Driver.determineDriverKind(args: &args))
     }
 
     try assertArgs("swift", parseTo: .interactive, leaving: [])