Batch and test batching of 1st wave

Author: David Ungar

Sources/SwiftDriver/Incremental Compilation/IncrementalCompilationState.swift

@@ -369,28 +369,28 @@ extension IncrementalCompilationState {
   func addPreOrCompileJobGroups(_ groups: [[Job]],
                                 formBatchedJobs: ([Job]) throws -> [Job]
   ) throws {
-    for group in groups {
+    let mandatoryPreOrCompileJobs = groups.flatMap { group -> [Job] in
       if let firstJob = group.first, isSkipped(firstJob) {
         recordSkippedGroup(group)
+        return []
       }
-      else {
-        try scheduleMandatoryPreOrCompile(group: group)
-      }
+      return group
     }
+    let batchedMandatoryPreOrCompileJobs = try formBatchedJobs(mandatoryPreOrCompileJobs)
+    scheduleMandatoryPreOrCompile(jobs: batchedMandatoryPreOrCompileJobs)
   }
 
   /// Remember that `group` (a compilation and possibly bitcode generation)
   /// must definitely be executed.
-  private func scheduleMandatoryPreOrCompile(
-    group: [Job]) {
+  private func scheduleMandatoryPreOrCompile(jobs: [Job]) {
     if let report = reportIncrementalDecision {
-      for job in group {
+      for job in jobs {
         report("Queuing \(job.descriptionForLifecycle)", nil)
       }
     }
-    mandatoryPreOrCompileJobsInOrder.append(contentsOf: group)
-    unfinishedMandatoryJobs.formUnion(group)
-    let mandatoryCompilationInputs = group
+    mandatoryPreOrCompileJobsInOrder.append(contentsOf: jobs)
+    unfinishedMandatoryJobs.formUnion(jobs)
+    let mandatoryCompilationInputs = jobs
       .flatMap {$0.kind == .compile ? $0.primaryInputs : []}
     pendingInputs.formUnion(mandatoryCompilationInputs)
   }

Sources/SwiftDriver/Jobs/Planning.swift

@@ -599,14 +599,14 @@ extension Driver {
           diagnosticEngine
             .emit(
               .remark(
-                "Adding {compile: \($0.file.basename)} to batch \(idx)\n"))
+                "Adding {compile: \($0.file.basename)} to batch \(idx)"))
         }
 
         let constituents = primaryInputs.map {$0.file.basename}.joined(separator: ", ")
         diagnosticEngine
           .emit(
             .remark(
-              "Forming batch job from \(primaryInputs.count) constituents: \(constituents)\n"))
+              "Forming batch job from \(primaryInputs.count) constituents: \(constituents)"))
       }
       let constituentsEmittedModuleTrace = !inputsRequiringModuleTrace.intersection(primaryInputs).isEmpty
       // no need to add job outputs again

Tests/SwiftDriverTests/IncrementalCompilationTests.swift

@@ -368,6 +368,7 @@ final class IncrementalCompilationTests: XCTestCase {
     #if true // sometimes want to skip for debugging
     tryNoChange(checkDiagnostics)
     tryTouchingOther(checkDiagnostics)
+    tryTouchingBoth(checkDiagnostics)
     #endif
     tryReplacingMain(checkDiagnostics)
   }
@@ -417,6 +418,10 @@ final class IncrementalCompilationTests: XCTestCase {
         "Incremental compilation: Queuing (initial): {compile: other.o <= other.swift}",
         "Incremental compilation: not scheduling dependents of other.swift; unknown changes",
         "Incremental compilation: Skipping input: {compile: main.o <= main.swift}",
+        "Found 1 batchable job",
+        "Forming into 1 batch",
+        "Adding {compile: other.swift} to batch 0",
+        "Forming batch job from 1 constituents: other.swift",
         "Incremental compilation: Queuing Compiling other.swift",
         "Starting Compiling other.swift",
         "Finished Compiling other.swift",
@@ -425,6 +430,32 @@ final class IncrementalCompilationTests: XCTestCase {
     ],
     whenAutolinking: autolinkLifecycleExpectations)
   }
+  func tryTouchingBoth(_ checkDiagnostics: Bool) {
+    touch("main")
+    touch("other")
+    try! doABuild(
+      "non-propagating, both touched",
+      checkDiagnostics: checkDiagnostics,
+      expectingRemarks: [
+        "Incremental compilation: Scheduing changed input {compile: main.o <= main.swift}",
+        "Incremental compilation: Scheduing changed input {compile: other.o <= other.swift}",
+        "Incremental compilation: Queuing (initial): {compile: main.o <= main.swift}",
+        "Incremental compilation: Queuing (initial): {compile: other.o <= other.swift}",
+        "Incremental compilation: not scheduling dependents of main.swift; unknown changes",
+        "Incremental compilation: not scheduling dependents of other.swift; unknown changes",
+        "Found 2 batchable jobs",
+        "Forming into 1 batch",
+        "Adding {compile: main.swift} to batch 0",
+        "Adding {compile: other.swift} to batch 0",
+        "Forming batch job from 2 constituents: main.swift, other.swift",
+        "Incremental compilation: Queuing Compiling main.swift, other.swift",
+        "Starting Compiling main.swift, other.swift",
+        "Finished Compiling main.swift, other.swift",
+        "Starting Linking theModule",
+        "Finished Linking theModule",
+    ],
+    whenAutolinking: autolinkLifecycleExpectations)
+  }
 
   func tryReplacingMain(_ checkDiagnostics: Bool) {
     replace(contentsOf: "main", with: "let foo = \"hello\"")
@@ -437,6 +468,10 @@ final class IncrementalCompilationTests: XCTestCase {
         "Incremental compilation: Queuing (initial): {compile: main.o <= main.swift}",
         "Incremental compilation: not scheduling dependents of main.swift; unknown changes",
         "Incremental compilation: Skipping input: {compile: other.o <= other.swift}",
+        "Found 1 batchable job",
+        "Forming into 1 batch",
+        "Adding {compile: main.swift} to batch 0",
+        "Forming batch job from 1 constituents: main.swift",
         "Incremental compilation: Queuing Compiling main.swift",
         "Starting Compiling main.swift",
         "Incremental compilation: Traced: interface of main.swiftdeps -> interface of top-level name foo -> implementation of other.swiftdeps",