[DNM Incremental] Start of cross-module incrementality (#458)

* Generalize DependencySource

* Add comments

* add isCrossModuleIncrementalBuildEnabled, but do nothing with it yet

Fix tests after extra message

* Generalize read interface to SourceFileDepGraph

* Rm warning, improve a diagnostic

* Redo Reporter interface

* Add more incremental reporting

* fix tests

Author: David Ungar

Sources/SwiftDriver/IncrementalCompilation/BuildRecordInfo.swift

@@ -176,14 +176,14 @@ final class BuildRecordInfo {
     do {
       contents = try fileSystem.readFileContents(buildRecordPath).cString
      } catch {
-      reporter?.report("Incremental compilation could not read build record at \(buildRecordPath)")
+      reporter?.report("Incremental compilation could not read build record at ", buildRecordPath)
       reporter?.reportDisablingIncrementalBuild("could not read build record")
       return nil
     }
     func failedToReadOutOfDateMap(_ reason: String? = nil) {
       let why = "malformed build record file\(reason.map {" " + $0} ?? "")"
       reporter?.report(
-        "Incremental compilation has been disabled due to \(why) '\(buildRecordPath)'")
+        "Incremental compilation has been disabled due to \(why)", buildRecordPath)
       reporter?.reportDisablingIncrementalBuild(why)
     }
     guard let outOfDateBuildRecord = BuildRecord(contents: contents,

Sources/SwiftDriver/IncrementalCompilation/DependencyKey.swift

@@ -152,10 +152,12 @@ public struct DependencyKey: Hashable, CustomStringConvertible {
     /// available from this node.
     case incrementalExternalDependency(ExternalDependency)
 
-    var externalDependency: ExternalDependency? {
+    var externalDependency: (ExternalDependency, isIncremental: Bool)? {
       switch self {
       case let .externalDepend(externalDependency):
-        return externalDependency
+        return (externalDependency, isIncremental: false)
+      case let .incrementalExternalDependency(externalDependency):
+        return (externalDependency, isIncremental: true)
       default:
         return nil}
     }

Sources/SwiftDriver/IncrementalCompilation/IncrementalCompilationState.swift

@@ -13,6 +13,9 @@ import TSCBasic
 import Foundation
 import SwiftOptions
 public class IncrementalCompilationState {
+  /// Whether cross-module incrementality is enabled
+  private let isCrossModuleIncrementalBuildEnabled: Bool
+
   /// The oracle for deciding what depends on what. Applies to this whole module.
   private let moduleDependencyGraph: ModuleDependencyGraph
 
@@ -56,6 +59,11 @@ public class IncrementalCompilationState {
       self.reporter = nil
     }
 
+    self.isCrossModuleIncrementalBuildEnabled =
+      driver.parsedOptions.contains(.enableExperimentalCrossModuleIncrementalBuild)
+    reporter?.report(
+      "\(self.isCrossModuleIncrementalBuildEnabled ? "Enabling" : "Disabling") incremental cross-module building")
+
 
     guard let (outputFileMap, buildRecordInfo, outOfDateBuildRecord)
             = try driver.getBuildInfo(self.reporter)
@@ -66,13 +74,13 @@ public class IncrementalCompilationState {
     guard let (
       moduleDependencyGraph,
       inputsHavingMalformedDependencySources: inputsHavingMalformedDependencySources
-    ) =
-            Self.computeModuleDependencyGraph(
-              buildRecordInfo,
-              outOfDateBuildRecord,
-              outputFileMap,
-              &driver,
-              self.reporter)
+    ) = Self.computeModuleDependencyGraph(
+      buildRecordInfo,
+      outOfDateBuildRecord,
+      outputFileMap,
+      &driver,
+      self.reporter,
+      isCrossModuleIncrementalBuildEnabled: isCrossModuleIncrementalBuildEnabled)
     else {
       return nil
     }
@@ -99,15 +107,16 @@ public class IncrementalCompilationState {
     _ outOfDateBuildRecord: BuildRecord,
     _ outputFileMap: OutputFileMap,
     _ driver: inout Driver,
-    _ reporter: Reporter?
+    _ reporter: Reporter?,
+    isCrossModuleIncrementalBuildEnabled: Bool
   )
   -> (ModuleDependencyGraph,
       inputsHavingMalformedDependencySources: [TypedVirtualPath])?
   {
     let diagnosticEngine = driver.diagnosticEngine
     guard let (
       moduleDependencyGraph,
-      inputsAndMalformedDependencySources: inputsAndMalformedDependencySources
+      inputsAndMalformedSwiftDeps: inputsAndMalformedSwiftDeps
     ) =
     ModuleDependencyGraph.buildInitialGraph(
       diagnosticEngine: diagnosticEngine,
@@ -117,20 +126,22 @@ public class IncrementalCompilationState {
       parsedOptions: &driver.parsedOptions,
       remarkDisabled: Diagnostic.Message.remark_incremental_compilation_has_been_disabled,
       reporter: reporter,
-      fileSystem: driver.fileSystem)
+      fileSystem: driver.fileSystem,
+      isCrossModuleIncrementalBuildEnabled: isCrossModuleIncrementalBuildEnabled
+      )
     else {
       return nil
     }
     // Preserve legacy behavior,
     // but someday, just ensure inputsAndMalformedDependencySources are compiled
-    if let badDependencySource = inputsAndMalformedDependencySources.first?.1 {
+    if let badSwiftDeps = inputsAndMalformedSwiftDeps.first?.1 {
       diagnosticEngine.emit(
         .remark_incremental_compilation_has_been_disabled(
-          because: "malformed dependencies file '\(badDependencySource)'")
+          because: "malformed dependencies file '\(badSwiftDeps)'")
       )
       return nil
     }
-    let inputsHavingMalformedDependencySources = inputsAndMalformedDependencySources.map {$0.0}
+    let inputsHavingMalformedDependencySources = inputsAndMalformedSwiftDeps.map {$0.0}
     return (moduleDependencyGraph,
             inputsHavingMalformedDependencySources: inputsHavingMalformedDependencySources)
   }
@@ -282,6 +293,7 @@ extension IncrementalCompilationState {
     alwaysRebuildDependents: Bool,
     reporter: IncrementalCompilationState.Reporter?
   ) -> Set<TypedVirtualPath> {
+    // Input == source file
     let changedInputs = Self.computeChangedInputs(
         groups: allGroups,
         buildRecordInfo: buildRecordInfo,
@@ -290,7 +302,15 @@ extension IncrementalCompilationState {
         fileSystem: fileSystem,
         reporter: reporter)
 
-    let externalDependents = computeExternallyDependentInputs(
+    let externallyChangedInputs = computeExternallyChangedInputs(
+      forIncrementalExternalDependencies: false,
+      buildTime: outOfDateBuildRecord.buildTime,
+      fileSystem: fileSystem,
+      moduleDependencyGraph: moduleDependencyGraph,
+      reporter: moduleDependencyGraph.reporter)
+
+    let incrementallyExternallyChangedInputs = computeExternallyChangedInputs(
+      forIncrementalExternalDependencies: true,
       buildTime: outOfDateBuildRecord.buildTime,
       fileSystem: fileSystem,
       moduleDependencyGraph: moduleDependencyGraph,
@@ -304,12 +324,13 @@ extension IncrementalCompilationState {
 
     // Combine to obtain the inputs that definitely must be recompiled.
     let definitelyRequiredInputs =
-      Set(changedInputs.map({ $0.filePath }) + externalDependents +
+      Set(changedInputs.map({ $0.filePath }) +
+            externallyChangedInputs + incrementallyExternallyChangedInputs +
             inputsHavingMalformedDependencySources
             + inputsMissingOutputs)
     if let reporter = reporter {
       for scheduledInput in definitelyRequiredInputs.sorted(by: {$0.file.name < $1.file.name}) {
-        reporter.report("Queuing (initial):", path: scheduledInput)
+        reporter.report("Queuing (initial):", scheduledInput)
       }
     }
 
@@ -319,7 +340,7 @@ extension IncrementalCompilationState {
     // as each first wave job finished.
     let speculativeInputs = computeSpeculativeInputs(
       changedInputs: changedInputs,
-      externalDependents: externalDependents,
+      externalDependents: externallyChangedInputs,
       inputsMissingOutputs: Set(inputsMissingOutputs),
       moduleDependencyGraph: moduleDependencyGraph,
       alwaysRebuildDependents: alwaysRebuildDependents,
@@ -328,7 +349,7 @@ extension IncrementalCompilationState {
 
     if let reporter = reporter {
       for dependent in speculativeInputs.sorted(by: {$0.file.name < $1.file.name}) {
-        reporter.report("Queuing because of the initial set:", path: dependent)
+        reporter.report("Queuing because of the initial set:", dependent)
       }
     }
     let immediatelyCompiledInputs = definitelyRequiredInputs.union(speculativeInputs)
@@ -337,7 +358,7 @@ extension IncrementalCompilationState {
       .subtracting(immediatelyCompiledInputs)
     if let reporter = reporter {
       for skippedInput in skippedInputs.sorted(by: {$0.file.name < $1.file.name})  {
-        reporter.report("Skipping input:", path: skippedInput)
+        reporter.report("Skipping input:", skippedInput)
       }
     }
     return skippedInputs
@@ -383,17 +404,17 @@ extension IncrementalCompilationState {
 
       switch previousCompilationStatus {
       case .upToDate where datesMatch:
-        reporter?.report("May skip current input:", path: input)
+        reporter?.report("May skip current input:", input)
         return nil
 
       case .upToDate:
-        reporter?.report("Scheduing changed input", path: input)
+        reporter?.report("Scheduing changed input", input)
       case .newlyAdded:
-        reporter?.report("Scheduling new", path: input)
+        reporter?.report("Scheduling new", input)
       case .needsCascadingBuild:
-        reporter?.report("Scheduling cascading build", path: input)
+        reporter?.report("Scheduling cascading build", input)
       case .needsNonCascadingBuild:
-        reporter?.report("Scheduling noncascading build", path: input)
+        reporter?.report("Scheduling noncascading build", input)
       }
       return ChangedInput(filePath: input,
                           status: previousCompilationStatus,
@@ -402,25 +423,28 @@ extension IncrementalCompilationState {
   }
 
   /// Any files dependent on modified files from other modules must be compiled, too.
-  private static func computeExternallyDependentInputs(
+  private static func computeExternallyChangedInputs(
+    forIncrementalExternalDependencies: Bool,
     buildTime: Date,
     fileSystem: FileSystem,
     moduleDependencyGraph: ModuleDependencyGraph,
     reporter: IncrementalCompilationState.Reporter?
  ) -> [TypedVirtualPath] {
     var externalDependencySources = Set<ModuleDependencyGraph.DependencySource>()
-    for extDep in moduleDependencyGraph.externalDependencies {
-      let extModTime = extDep.file.flatMap {
-        try? fileSystem.getFileInfo($0).modTime}
+    let extDeps = forIncrementalExternalDependencies
+      ? moduleDependencyGraph.incrementalExternalDependencies
+      : moduleDependencyGraph.externalDependencies
+    for extDep in extDeps {
+      let extModTime = extDep.file.flatMap {try? fileSystem.getFileInfo($0).modTime}
         ?? Date.distantFuture
       if extModTime >= buildTime {
-        for dependent in moduleDependencyGraph.untracedDependents(of: extDep) {
+        for dependent in moduleDependencyGraph.untracedDependents(of: extDep, isIncremental: forIncrementalExternalDependencies) {
           guard let dependencySource = dependent.dependencySource else {
-            fatalError("Dependent \(dependent) does not have dependencies source file!")
+            fatalError("Dependent \(dependent) does not have dependencies file!")
           }
           reporter?.report(
-            "Queuing because of external dependency on newer \(extDep.file?.basename ?? "extDep?")",
-            path: TypedVirtualPath(file: dependencySource.file, type: .swiftDeps))
+            "Queuing because of \(forIncrementalExternalDependencies ? "incremental " : "")external dependency on newer \(extDep.file?.basename ?? "extDep?")",
+            dependencySource.typedFile)
           externalDependencySources.insert(dependencySource)
         }
       }
@@ -457,7 +481,7 @@ extension IncrementalCompilationState {
       for dep in dependentsOfOneFile where !cascadingFileSet.contains(dep) {
         if dependentFiles.insert(dep).0 {
           reporter?.report(
-            "Immediately scheduling dependent on \(cascadingFile.file.basename)", path: dep)
+            "Immediately scheduling dependent on \(cascadingFile.file.basename)", dep)
         }
       }
     }
@@ -536,7 +560,8 @@ extension IncrementalCompilationState {
 
       if let reporter = self.reporter {
         for input in discoveredInputs {
-          reporter.report("Queuing because of dependencies discovered later:", path: input)
+          reporter.report(
+            "Queuing because of dependencies discovered later:", input)
         }
       }
       let newJobs = try getJobsFor(discoveredCompilationInputs: discoveredInputs)
@@ -561,7 +586,8 @@ extension IncrementalCompilationState {
           if let found = moduleDependencyGraph.findSourcesToCompileAfterCompiling(input, on: self.driver.fileSystem) {
             return found
           }
-          self.reporter?.report("Failed to read some dependencies source; compiling everything", path: input)
+          self.reporter?.report(
+            "Failed to read some dependencies source; compiling everything", input)
           return Array(skippedCompileGroups.keys)
         }
       )
@@ -579,11 +605,11 @@ extension IncrementalCompilationState {
         let primaryInputs = group.compileJob.primaryInputs
         assert(primaryInputs.count == 1)
         assert(primaryInputs[0] == input)
-        self.reporter?.report("Scheduling discovered", path: input)
+        self.reporter?.report("Scheduling discovered", input)
         return group.allJobs()
       }
       else {
-        self.reporter?.report("Tried to schedule discovered input again", path: input)
+        self.reporter?.report("Tried to schedule discovered input again", input)
         return []
       }
     }
@@ -627,20 +653,38 @@ extension IncrementalCompilationState {
     ///
     /// - Parameters:
     ///   - message: The message to emit in the remark.
-    ///   - path: If non-nil, the path of an output for an incremental job.
-    func report(_ message: String, path: TypedVirtualPath? = nil) {
-      guard let outputFileMap = outputFileMap,
-            let path = path,
+    ///   - path: If non-nil, the path of some file. If the output for an incremental job, will print out the
+    ///           source and object files.
+    func report(_ message: String, _ path: TypedVirtualPath?) {
+       guard let path = path,
+            let outputFileMap = outputFileMap,
             let input = path.type == .swift ? path.file : outputFileMap.getInput(outputFile: path.file)
       else {
-        diagnosticEngine.emit(.remark_incremental_compilation(because: message))
+        report(message, path?.file)
         return
       }
       let output = outputFileMap.getOutput(inputFile: path.file, outputType: .object)
       let compiling = " {compile: \(output.basename) <= \(input.basename)}"
       diagnosticEngine.emit(.remark_incremental_compilation(because: "\(message) \(compiling)"))
     }
 
+    /// Entry point for a simple path, won't print the compile job, path could be anything.
+    func report(_ message: String, _ path: VirtualPath?) {
+      guard let path = path
+      else {
+        report(message)
+        diagnosticEngine.emit(.remark_incremental_compilation(because: message))
+        return
+      }
+      diagnosticEngine.emit(.remark_incremental_compilation(because: "\(message) '\(path.name)'"))
+    }
+
+    /// Entry point if no path.
+    func report(_ message: String) {
+      diagnosticEngine.emit(.remark_incremental_compilation(because: message))
+    }
+
+
     // Emits a remark indicating incremental compilation has been disabled.
     func reportDisablingIncrementalBuild(_ why: String) {
       report("Disabling incremental build: \(why)")