Merge pull request #580 from davidungar/incremental-post-compile

Author: David Ungar

Sources/SwiftDriver/IncrementalCompilation/BuildRecord.swift

@@ -19,24 +19,32 @@ public struct BuildRecord {
   public let swiftVersion: String
   /// When testing, the argsHash may be missing from the build record
   public let argsHash: String?
-  public let buildTime: Date
+  /// Next compile, will compare an input mod time against the start time of the previous build
+  public let buildStartTime: Date
+  /// Next compile, will compare an output mod time against the end time of the previous build
+  public let buildEndTime: Date
   /// The date is the modification time of the main input file the last time the driver ran
   public let inputInfos: [VirtualPath: InputInfo]
 
   public init(argsHash: String?,
               swiftVersion: String,
-              buildTime: Date,
+              buildStartTime: Date,
+              buildEndTime: Date,
               inputInfos: [VirtualPath: InputInfo]) {
     self.argsHash = argsHash
     self.swiftVersion = swiftVersion
-    self.buildTime = buildTime
+    self.buildStartTime = buildStartTime
+    self.buildEndTime = buildEndTime
     self.inputInfos = inputInfos
   }
 
   private enum SectionName: String, CaseIterable {
     case swiftVersion = "version"
     case argsHash = "options"
-    case buildTime = "build_time"
+    // Implement this for a smoother transition
+    case legacyBuildStartTime = "build_time"
+    case buildStartTime = "build_start_time"
+    case buildEndTime = "build_end_time"
     case inputInfos = "inputs"
 
     var serializedName: String { rawValue }
@@ -59,7 +67,8 @@ public extension BuildRecord {
     var argsHash: String?
     var swiftVersion: String?
     // Legacy driver does not disable incremental if no buildTime field.
-    var buildTime: Date = .distantPast
+    var buildStartTime: Date = .distantPast
+    var buildEndTime: Date = .distantFuture
     var inputInfos: [VirtualPath: InputInfo]?
     for (key, value) in sections {
       guard let k = key.string else {
@@ -80,14 +89,23 @@ public extension BuildRecord {
           return nil
         }
         argsHash = s
-     case SectionName.buildTime.serializedName:
-      guard let d = Self.decodeDate(value,
-                                    forInputInfo: false,
-                                    failedToReadOutOfDateMap)
-      else {
-        return nil
-      }
-      buildTime = d
+      case SectionName.buildStartTime.serializedName,
+           SectionName.legacyBuildStartTime.serializedName:
+        guard let d = Self.decodeDate(value,
+                                      forInputInfo: false,
+                                      failedToReadOutOfDateMap)
+        else {
+          return nil
+        }
+        buildStartTime = d
+      case SectionName.buildEndTime.serializedName:
+        guard let d = Self.decodeDate(value,
+                                      forInputInfo: false,
+                                      failedToReadOutOfDateMap)
+        else {
+          return nil
+        }
+        buildEndTime = d
       case SectionName.inputInfos.serializedName:
         guard let ii = Self.decodeInputInfos(value, failedToReadOutOfDateMap) else {
           return nil
@@ -111,7 +129,8 @@ public extension BuildRecord {
     }
     self.init(argsHash: argsHash,
               swiftVersion: sv,
-              buildTime: buildTime,
+              buildStartTime: buildStartTime,
+              buildEndTime: buildEndTime,
               inputInfos: iis)
   }
 
@@ -181,7 +200,8 @@ extension BuildRecord {
        compilationInputModificationDates: [TypedVirtualPath: Date],
        actualSwiftVersion: String,
        argsHash: String!,
-       timeBeforeFirstJob: Date
+       timeBeforeFirstJob: Date,
+       timeAfterLastJob: Date
   ) {
     let jobResultsByInput = Dictionary(uniqueKeysWithValues:
       finishedJobResults.flatMap { entry in
@@ -197,7 +217,8 @@ extension BuildRecord {
     self.init(
       argsHash: argsHash,
       swiftVersion: actualSwiftVersion,
-      buildTime: timeBeforeFirstJob,
+      buildStartTime: timeBeforeFirstJob,
+      buildEndTime: timeAfterLastJob,
       inputInfos: Dictionary(uniqueKeysWithValues: inputInfosArray)
     )
   }
@@ -216,10 +237,11 @@ extension BuildRecord {
           .map {(Yams.Node($0.0, .implicit, .doubleQuoted), Self.encode($0.1))}
       )
     let fieldNodes = [
-      (SectionName.swiftVersion, Yams.Node(swiftVersion,    .implicit, .doubleQuoted)),
-      (SectionName.argsHash,     Yams.Node(currentArgsHash, .implicit, .doubleQuoted)),
-      (SectionName.buildTime,    Self.encode(buildTime)),
-      (SectionName.inputInfos,   inputInfosNode )
+      (SectionName.swiftVersion,    Yams.Node(swiftVersion,    .implicit, .doubleQuoted)),
+      (SectionName.argsHash,        Yams.Node(currentArgsHash, .implicit, .doubleQuoted)),
+      (SectionName.buildStartTime,  Self.encode(buildStartTime)),
+      (SectionName.buildEndTime,    Self.encode(buildEndTime)),
+      (SectionName.inputInfos,      inputInfosNode )
       ] .map { (Yams.Node($0.0.serializedName), $0.1) }
 
     let buildRecordNode = Yams.Node(fieldNodes, .implicit, .block)

Sources/SwiftDriver/IncrementalCompilation/BuildRecordInfo.swift

@@ -167,7 +167,8 @@ import SwiftOptions
         compilationInputModificationDates: compilationInputModificationDates,
         actualSwiftVersion: actualSwiftVersion,
         argsHash: currentArgsHash,
-        timeBeforeFirstJob: timeBeforeFirstJob)
+        timeBeforeFirstJob: timeBeforeFirstJob,
+        timeAfterLastJob: Date())
     }
 
     guard let contents = buildRecord.encode(currentArgsHash: currentArgsHash,

Sources/SwiftDriver/IncrementalCompilation/IncrementalCompilationState.swift

@@ -46,6 +46,14 @@ public final class IncrementalCompilationState {
   /// Jobs to run *after* the last compile, for instance, link-editing.
   public let jobsAfterCompiles: [Job]
 
+  /// Can compare input mod times against this.
+  private let buildStartTime: Date
+
+  /// In order to decide while postCompile jobs to run, need to compare their outputs against the build time
+  private let buildEndTime: Date
+
+  private let fileSystem: FileSystem
+
   /// A  high-priority confinement queue that must be used to protect the incremental compilation state.
   private let confinementQueue: DispatchQueue = DispatchQueue(label: "com.apple.swift-driver.incremental-compilation-state", qos: .userInteractive)
 
@@ -124,6 +132,9 @@ public final class IncrementalCompilationState {
     self.mandatoryJobsInOrder = initial.mandatoryJobsInOrder
     self.jobsAfterCompiles = jobsInPhases.afterCompiles
     self.moduleDependencyGraph = initial.graph
+    self.buildStartTime = initial.buildStartTime
+    self.buildEndTime = initial.buildEndTime
+    self.fileSystem = driver.fileSystem
     self.driver = driver
   }
 }
@@ -150,6 +161,10 @@ extension IncrementalCompilationState {
     /// All of the pre-compile or compilation job (groups) known to be required
     /// for the first wave to execute.
     let mandatoryJobsInOrder: [Job]
+    /// The last time this compilation was started. Used to compare against e.g. input file mod dates.
+    let buildStartTime: Date
+    /// The last time this compilation finished. Used to compare against output file mod dates
+    let buildEndTime: Date
   }
 }
 
@@ -290,6 +305,14 @@ extension IncrementalCompilationState {
     return try driver.formBatchedJobs(unbatched, showJobLifecycle: driver.showJobLifecycle)
   }
 }
+// MARK: - Scheduling post-compile jobs
+extension IncrementalCompilationState {
+  public func canSkipPostCompile(job: Job) -> Bool {
+    job.outputs.allSatisfy {output in
+      let fileModTime = (try? fileSystem.lastModificationTime(for: output.file)) ?? .distantFuture
+      return fileModTime <= buildEndTime}
+  }
+}
 
 // MARK: - After the build
 extension IncrementalCompilationState {

Sources/SwiftDriver/IncrementalCompilation/InitialStateComputer.swift

@@ -38,7 +38,8 @@ extension IncrementalCompilationState {
     @_spi(Testing) public let dependencyDotFilesIncludeExternals: Bool = true
     @_spi(Testing) public let dependencyDotFilesIncludeAPINotes: Bool = false
 
-    @_spi(Testing) public let buildTime: Date
+    @_spi(Testing) public let buildStartTime: Date
+    @_spi(Testing) public let buildEndTime: Date
 
     @_spi(Testing) public init(
       _ options: IncrementalCompilationState.Options,
@@ -72,7 +73,8 @@ extension IncrementalCompilationState {
       self.isCrossModuleIncrementalBuildEnabled = options.contains(.enableCrossModuleIncrementalBuild)
       self.verifyDependencyGraphAfterEveryImport = options.contains(.verifyDependencyGraphAfterEveryImport)
       self.emitDependencyDotFileAfterEveryImport = options.contains(.emitDependencyDotFileAfterEveryImport)
-      self.buildTime = maybeBuildRecord?.buildTime ?? .distantPast
+      self.buildStartTime = maybeBuildRecord?.buildStartTime ?? .distantPast
+      self.buildEndTime = maybeBuildRecord?.buildEndTime ?? .distantFuture
     }
 
     func compute(batchJobFormer: inout Driver)
@@ -102,7 +104,9 @@ extension IncrementalCompilationState {
 
       return InitialState(graph: graph,
                           skippedCompileGroups: skippedCompileGroups,
-                          mandatoryJobsInOrder: mandatoryJobsInOrder)
+                          mandatoryJobsInOrder: mandatoryJobsInOrder,
+                          buildStartTime: buildStartTime,
+                          buildEndTime: buildEndTime)
     }
   }
 }

Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraph.swift

@@ -330,7 +330,7 @@ extension ModuleDependencyGraph {
       return true
     }
     let fileModTime = (try? info.fileSystem.lastModificationTime(for: depFile)) ?? .distantFuture
-    let hasChanged = fileModTime >= info.buildTime
+    let hasChanged = fileModTime >= info.buildStartTime
     externalDependencyModTimeCache[externalDependency] = hasChanged
     return hasChanged
   }

Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraphParts/Integrator.swift

@@ -189,8 +189,6 @@ extension ModuleDependencyGraph.Integrator {
     }
   }
 
-  private var buildTime: Date { destination.info.buildTime }
-
   // A `moduleGraphUseNode` is used by an externalDependency key being integrated.
   // Remember the dependency for later processing in externalDependencies, and
   // also return it in results.

Sources/SwiftDriverExecution/MultiJobExecutor.swift

@@ -390,17 +390,30 @@ class ExecuteAllJobsRule: LLBuildRule {
       let subtaskSuccess = try DriverBuildValue(value).success
       // After all compilation jobs are done, we can schedule post-compilation jobs,
       // including merge module and linking jobs.
-      if inputID == allCompilationId && !context.primaryIndices.isEmpty && subtaskSuccess {
-        context.postCompileIndices.forEach {
-          engine.taskNeedsInput(ExecuteJobRule.RuleKey(index: $0), inputID: $0)
-        }
+      if inputID == allCompilationId && subtaskSuccess {
+        schedulePostCompileJobs(engine)
       }
       allInputsSucceeded = allInputsSucceeded && subtaskSuccess
     } catch {
       allInputsSucceeded = false
     }
   }
 
+  /// After all compilation jobs have run, figure which, for instance link, jobs must run
+  private func schedulePostCompileJobs(_ engine: LLTaskBuildEngine) {
+    for postCompileIndex in context.postCompileIndices {
+      let job = context.jobs[postCompileIndex]
+      /// If any compile jobs ran, skip the expensive mod-time checks
+      if context.primaryIndices.isEmpty,
+         let incrementalCompilationState = context.incrementalCompilationState,
+         incrementalCompilationState.canSkipPostCompile(job: job) {
+        continue
+      }
+      engine.taskNeedsInput(ExecuteJobRule.RuleKey(index: postCompileIndex),
+                            inputID: postCompileIndex)
+    }
+  }
+
   override func inputsAvailable(_ engine: LLTaskBuildEngine) {
     engine.taskIsComplete(DriverBuildValue.jobExecution(success: allInputsSucceeded))
   }

Tests/SwiftDriverTests/Helpers/AssertDiagnostics.swift

@@ -15,17 +15,18 @@ import SwiftDriver
 import TSCBasic
 import TestUtilities
 
-func assertDriverDiagnostics(
+@discardableResult
+func assertDriverDiagnostics<Result> (
   args: [String],
   env: [String: String] = ProcessEnv.vars,
   file: StaticString = #file, line: UInt = #line,
-  do body: (inout Driver, DiagnosticVerifier) throws -> Void
-) throws {
+  do body: (inout Driver, DiagnosticVerifier) throws -> Result
+) throws -> Result {
   let matcher = DiagnosticVerifier()
   defer { matcher.verify(file: file, line: line) }
 
   var driver = try Driver(args: args, env: env, diagnosticsEngine: DiagnosticsEngine(handlers: [matcher.emit(_:)]))
-  try body(&driver, matcher)
+  return try body(&driver, matcher)
 }
 
 /// Asserts that the `Driver` it instantiates will only emit warnings and errors