[Explicit Module Builds] Serialize the inter-module dependency graph as incremental build state

On an incremental build, load a prior dependency graph, check if it is still valid:
1. import set of the current module has not changed
2. input files of every module in the prior graph are older than the output
and if it is, re-use it, skipping invoking the dependency scanner again.

Partially resolves rdar://66801475

Author: artemcm

Sources/SwiftDriver/Driver/Driver.swift

@@ -235,22 +235,6 @@ public struct Driver {
   /// Only present when the driver will be writing the record.
   /// Only used for reading when compiling incrementally.
   @_spi(Testing) public let buildRecordInfo: BuildRecordInfo?
-
-  /// A build-record-relative path to the location of a serialized copy of the
-  /// driver's dependency graph.
-  ///
-  /// FIXME: This is a little ridiculous. We could probably just replace the
-  /// build record outright with a serialized format.
-  var driverDependencyGraphPath: VirtualPath? {
-    guard let recordInfo = self.buildRecordInfo else {
-      return nil
-    }
-    let filename = recordInfo.buildRecordPath.basenameWithoutExt
-    return recordInfo
-      .buildRecordPath
-      .parentDirectory
-      .appending(component: filename + ".priors")
-  }
 
   /// Whether to consider incremental compilation.
   let shouldAttemptIncrementalCompilation: Bool
@@ -1283,10 +1267,6 @@ extension Driver {
   public mutating func run(
     jobs: [Job]
   ) throws {
-    if parsedOptions.hasArgument(.v) {
-      try printVersion(outputStream: &stderrStream)
-    }
-
     let forceResponseFiles = parsedOptions.contains(.driverForceResponseFiles)
 
     // If we're only supposed to print the jobs, do so now.
@@ -1451,6 +1431,15 @@ extension Driver {
           buildRecordInfo?.removeBuildRecord()
           return
       }
+      do {
+        try incrementalCompilationState.writeInterModuleDependencyGraph(buildRecordInfo)
+      }
+      catch {
+        diagnosticEngine.emit(
+          .warning("next compile must run a full dependency scan; could not write inter-module dependency graph: \(error.localizedDescription)"))
+        buildRecordInfo?.removeInterModuleDependencyGraph()
+        return
+      }
     }
     buildRecordInfo?.writeBuildRecord(
       jobs,

Sources/SwiftDriver/ExplicitModuleBuilds/ClangVersionedDependencyResolution.swift

@@ -178,7 +178,7 @@ private extension InterModuleDependencyGraph {
 
   /// Update the set of all PCMArgs against which a given clang module was re-scanned
   mutating func updateCapturedPCMArgClangDependencies(using pcmArgSetMap:
-                                                        [ModuleDependencyId : Set<[String]>]
+                                                      [ModuleDependencyId : Set<[String]>]
   ) throws {
     for (moduleId, newPCMArgs) in pcmArgSetMap {
       guard let moduleInfo = modules[moduleId] else {

Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyGraph.swift

@@ -11,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 import class Foundation.JSONEncoder
+import struct Foundation.Data
 
 /// A map from a module identifier to its info
 public typealias ModuleInfoMap = [ModuleDependencyId: ModuleInfo]
@@ -275,7 +276,7 @@ public struct InterModuleDependencyGraph: Codable {
 }
 
 internal extension InterModuleDependencyGraph {
-  func toJSONString() throws -> String {
+  func toJSONData() throws -> Data {
     let encoder = JSONEncoder()
 #if os(Linux) || os(Android)
     encoder.outputFormatting = [.prettyPrinted]
@@ -284,8 +285,11 @@ internal extension InterModuleDependencyGraph {
       encoder.outputFormatting = [.prettyPrinted, .withoutEscapingSlashes]
     }
 #endif
-    let data = try encoder.encode(self)
-    return String(data: data, encoding: .utf8)!
+    return try encoder.encode(self)
+  }
+
+  func toJSONString() throws -> String {
+    return try String(data: toJSONData(), encoding: .utf8)!
   }
 }
 

Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift

@@ -41,6 +41,116 @@ extension Diagnostic.Message {
   }
 }
 
+@_spi(Testing) public extension Driver {
+  /// Scan the current module's input source-files to compute its direct and transitive
+  /// module dependencies.
+  mutating func gatherModuleDependencies()
+  throws -> InterModuleDependencyGraph {
+    var dependencyGraph = try performDependencyScan()
+
+    if parsedOptions.hasArgument(.printPreprocessedExplicitDependencyGraph) {
+      try stdoutStream <<< dependencyGraph.toJSONString()
+      stdoutStream.flush()
+    }
+
+    if let externalTargetDetails = externalTargetModuleDetailsMap {
+      // Resolve external dependencies in the dependency graph, if any.
+      try dependencyGraph.resolveExternalDependencies(for: externalTargetDetails)
+    }
+
+    // Re-scan Clang modules at all the targets they will be built against.
+    // This is currently disabled because we are investigating it being unnecessary
+    // try resolveVersionedClangDependencies(dependencyGraph: &dependencyGraph)
+
+    // Set dependency modules' paths to be saved in the module cache.
+    try resolveDependencyModulePaths(dependencyGraph: &dependencyGraph)
+
+    if parsedOptions.hasArgument(.printExplicitDependencyGraph) {
+      let outputFormat = parsedOptions.getLastArgument(.explicitDependencyGraphFormat)?.asSingle
+      if outputFormat == nil || outputFormat == "json" {
+        try stdoutStream <<< dependencyGraph.toJSONString()
+      } else if outputFormat == "dot" {
+        DOTModuleDependencyGraphSerializer(dependencyGraph).writeDOT(to: &stdoutStream)
+      }
+      stdoutStream.flush()
+    }
+
+    return dependencyGraph
+  }
+
+  /// Update the given inter-module dependency graph to set module paths to be within the module cache,
+  /// if one is present, and for Swift modules to use the context hash in the file name.
+  private mutating func resolveDependencyModulePaths(dependencyGraph: inout InterModuleDependencyGraph)
+  throws {
+    // If a module cache path is specified, update all module dependencies
+    // to be output into it.
+    if let moduleCachePath = parsedOptions.getLastArgument(.moduleCachePath)?.asSingle {
+      try resolveDependencyModulePathsRelativeToModuleCache(dependencyGraph: &dependencyGraph,
+                                                            moduleCachePath: moduleCachePath)
+    }
+
+    // Set the output path to include the module's context hash
+    try resolveDependencyModuleFileNamesWithContextHash(dependencyGraph: &dependencyGraph)
+  }
+
+  /// For Swift module dependencies, set the output path to include the module's context hash
+  private mutating func resolveDependencyModuleFileNamesWithContextHash(dependencyGraph: inout InterModuleDependencyGraph)
+  throws {
+    for (moduleId, moduleInfo) in dependencyGraph.modules {
+      // Output path on the main module is determined by the invocation arguments.
+      guard moduleId.moduleName != dependencyGraph.mainModuleName else {
+        continue
+      }
+
+      let plainPath = VirtualPath.lookup(dependencyGraph.modules[moduleId]!.modulePath.path)
+      if case .swift(let swiftDetails) = moduleInfo.details {
+        guard let contextHash = swiftDetails.contextHash else {
+          throw Driver.Error.missingContextHashOnSwiftDependency(moduleId.moduleName)
+        }
+        let updatedPath = plainPath.parentDirectory.appending(component: "\(plainPath.basenameWithoutExt)-\(contextHash).\(plainPath.extension!)")
+        dependencyGraph.modules[moduleId]!.modulePath = TextualVirtualPath(path: updatedPath.intern())
+      }
+      // TODO: Remove this once toolchain is updated
+      else if case .clang(let clangDetails) = moduleInfo.details {
+        if !moduleInfo.modulePath.path.description.contains(clangDetails.contextHash) {
+          let contextHash = clangDetails.contextHash
+          let updatedPath = plainPath.parentDirectory.appending(component: "\(plainPath.basenameWithoutExt)-\(contextHash).\(plainPath.extension!)")
+          dependencyGraph.modules[moduleId]!.modulePath = TextualVirtualPath(path: updatedPath.intern())
+        }
+      }
+    }
+  }
+
+  /// Resolve all paths to dependency binary module files to be relative to the module cache path.
+  private mutating func resolveDependencyModulePathsRelativeToModuleCache(dependencyGraph: inout InterModuleDependencyGraph,
+                                                                          moduleCachePath: String)
+  throws {
+    for (moduleId, moduleInfo) in dependencyGraph.modules {
+      // Output path on the main module is determined by the invocation arguments.
+      if case .swift(let name) = moduleId {
+        if name == dependencyGraph.mainModuleName {
+          continue
+        }
+        let modulePath = VirtualPath.lookup(moduleInfo.modulePath.path)
+        // Use VirtualPath to get the OS-specific path separators right.
+        let modulePathInCache =
+            try VirtualPath(path: moduleCachePath).appending(component: modulePath.basename)
+        dependencyGraph.modules[moduleId]!.modulePath =
+            TextualVirtualPath(path: modulePathInCache.intern())
+      }
+      // TODO: Remove this once toolchain is updated
+      else if case .clang(_) = moduleId {
+        let modulePath = VirtualPath.lookup(moduleInfo.modulePath.path)
+        // Use VirtualPath to get the OS-specific path separators right.
+        let modulePathInCache =
+            try VirtualPath(path: moduleCachePath).appending(component: modulePath.basename)
+        dependencyGraph.modules[moduleId]!.modulePath =
+            TextualVirtualPath(path: modulePathInCache.intern())
+      }
+    }
+  }
+}
+
 public extension Driver {
   /// Precompute the dependencies for a given Swift compilation, producing a
   /// dependency graph including all Swift and C module files and

Sources/SwiftDriver/IncrementalCompilation/BuildRecordInfo.swift

@@ -10,6 +10,9 @@
 //
 //===----------------------------------------------------------------------===//
 
+import struct Foundation.Data
+import class Foundation.JSONDecoder
+
 import class TSCBasic.DiagnosticsEngine
 import protocol TSCBasic.FileSystem
 import struct TSCBasic.AbsolutePath
@@ -46,6 +49,7 @@ import class Dispatch.DispatchQueue
   @_spi(Testing) public let timeBeforeFirstJob: TimePoint
   let diagnosticEngine: DiagnosticsEngine
   let compilationInputModificationDates: [TypedVirtualPath: TimePoint]
+  private var explicitModuleDependencyGraph: InterModuleDependencyGraph? = nil
 
   private var finishedJobResults = [JobResult]()
   // A confinement queue that protects concurrent access to the
@@ -71,7 +75,6 @@ import class Dispatch.DispatchQueue
     self.compilationInputModificationDates = compilationInputModificationDates
   }
 
-
   convenience init?(
     actualSwiftVersion: String,
     compilerOutputType: FileType?,
@@ -193,6 +196,13 @@ import class Dispatch.DispatchQueue
     try? fileSystem.removeFileTree(absPath)
   }
 
+  func removeInterModuleDependencyGraph() {
+    guard let absPath = interModuleDependencyGraphPath.absolutePath else {
+      return
+    }
+    try? fileSystem.removeFileTree(absPath)
+  }
+
   /// Before writing to the dependencies file path, preserve any previous file
   /// that may have been there. No error handling -- this is just a nicety, it
   /// doesn't matter if it fails.
@@ -203,9 +213,8 @@ import class Dispatch.DispatchQueue
   }
 
 
-// TODO: Incremental too many names, buildRecord BuildRecord outofdatemap
+  // TODO: Incremental too many names, buildRecord BuildRecord outofdatemap
   func populateOutOfDateBuildRecord(
-    inputFiles: [TypedVirtualPath],
     reporter: IncrementalCompilationState.Reporter?
   ) -> BuildRecord? {
     let contents: String
@@ -245,6 +254,29 @@ import class Dispatch.DispatchQueue
     return outOfDateBuildRecord
   }
 
+  func readOutOfDateInterModuleDependencyGraph(
+    buildRecord: BuildRecord?,
+    reporter: IncrementalCompilationState.Reporter?
+  ) -> InterModuleDependencyGraph? {
+    // If a valid build record could not be produced, do not bother here
+    guard buildRecord != nil else {
+      reporter?.report("Incremental compilation did not attempt to read inter-module dependency graph.")
+      return nil
+    }
+
+    let decodedGraph: InterModuleDependencyGraph
+    do {
+      let contents = try fileSystem.readFileContents(interModuleDependencyGraphPath).cString
+      decodedGraph = try JSONDecoder().decode(InterModuleDependencyGraph.self,
+                                              from: Data(contents.utf8))
+    } catch {
+      return nil
+    }
+    reporter?.report("Read inter-module dependency graph", interModuleDependencyGraphPath)
+
+    return decodedGraph
+  }
+
   func jobFinished(job: Job, result: ProcessResult) {
     self.confinementQueue.sync {
       finishedJobResults.append(JobResult(job, result))
@@ -263,6 +295,15 @@ import class Dispatch.DispatchQueue
       .appending(component: filename + ".priors")
   }
 
+  /// A build-record-relative path to the location of a serialized copy of the
+  /// driver's inter-module dependency graph.
+  var interModuleDependencyGraphPath: VirtualPath {
+    let filename = buildRecordPath.basenameWithoutExt
+    return buildRecordPath
+      .parentDirectory
+      .appending(component: filename + ".moduledeps")
+  }
+
   /// Directory to emit dot files into
   var dotFileDirectory: VirtualPath {
     buildRecordPath.parentDirectory

Sources/SwiftDriver/IncrementalCompilation/IncrementalCompilationProtectedState.swift

@@ -36,7 +36,7 @@ extension IncrementalCompilationState {
     fileprivate let moduleDependencyGraph: ModuleDependencyGraph
 
     fileprivate let reporter: Reporter?
-    
+
     init(skippedCompileGroups: [TypedVirtualPath: CompileJobGroup],
          _ moduleDependencyGraph: ModuleDependencyGraph,
          _ driver: inout Driver) {