[Incremental Builds][Explicit Module Builds] Switch to using incremental dependency scanning

For Explicit Module Builds, have the driver configure the dependency scanner invocation to serialize its internal scanner cache state after a scan, and attempt to deserialize a prior build's scanner cache state, and validate its contents for an incremental re-scan.

Author: artemcm

Sources/SwiftDriver/Driver/Driver.swift

@@ -315,10 +315,6 @@ public struct Driver {
   /// Set during planning because needs the jobs to look at outputs.
   @_spi(Testing) public private(set) var incrementalCompilationState: IncrementalCompilationState? = nil
 
-  /// Nil if not running in explicit module build mode.
-  /// Set during planning.
-  var interModuleDependencyGraph: InterModuleDependencyGraph? = nil
-
   /// The path of the SDK.
   public var absoluteSDKPath: AbsolutePath? {
     guard let path = frontendTargetInfo.sdkPath?.path else {
@@ -1908,7 +1904,6 @@ extension Driver {
     try executor.execute(
       workload: .init(allJobs,
                       incrementalCompilationState,
-                      interModuleDependencyGraph,
                       continueBuildingAfterErrors: continueBuildingAfterErrors),
       delegate: jobExecutionDelegate,
       numParallelJobs: numParallelJobs ?? 1,
@@ -1936,16 +1931,6 @@ extension Driver {
         .warning("next compile won't be incremental; could not write dependency graph: \(error.localizedDescription)"))
       /// Ensure that a bogus dependency graph is not used next time.
       buildRecordInfo.removeBuildRecord()
-      buildRecordInfo.removeInterModuleDependencyGraph()
-      return
-    }
-    do {
-      try incrementalCompilationState.writeInterModuleDependencyGraph(buildRecordInfo)
-    } catch {
-      diagnosticEngine.emit(
-        .warning("next compile must run a full dependency scan; could not write inter-module dependency graph: \(error.localizedDescription)"))
-      buildRecordInfo.removeBuildRecord()
-      buildRecordInfo.removeInterModuleDependencyGraph()
       return
     }
   }

Sources/SwiftDriver/Execution/DriverExecutor.swift

@@ -61,22 +61,18 @@ public struct DriverExecutorWorkload {
   }
 
   public let kind: Kind
-  public let interModuleDependencyGraph: InterModuleDependencyGraph?
 
   public init(_ allJobs: [Job],
               _ incrementalCompilationState: IncrementalCompilationState?,
-              _ interModuleDependencyGraph: InterModuleDependencyGraph?,
               continueBuildingAfterErrors: Bool) {
     self.continueBuildingAfterErrors = continueBuildingAfterErrors
     self.kind = incrementalCompilationState
       .map {.incremental($0)}
       ?? .all(allJobs)
-    self.interModuleDependencyGraph = interModuleDependencyGraph
   }
 
-  static public func all(_ jobs: [Job],
-                         _ interModuleDependencyGraph: InterModuleDependencyGraph? = nil) -> Self {
-    .init(jobs, nil, interModuleDependencyGraph, continueBuildingAfterErrors: false)
+  static public func all(_ jobs: [Job]) -> Self {
+    .init(jobs, nil, continueBuildingAfterErrors: false)
   }
 }
 
@@ -119,7 +115,7 @@ extension DriverExecutor {
     recordedInputModificationDates: [TypedVirtualPath: TimePoint]
   ) throws {
     try execute(
-      workload: .all(jobs, nil),
+      workload: .all(jobs),
       delegate: delegate,
       numParallelJobs: numParallelJobs,
       forceResponseFiles: forceResponseFiles,

Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/CommonDependencyOperations.swift

@@ -174,7 +174,7 @@ internal extension InterModuleDependencyGraph {
                               reporter: reporter)
     }
 
-    if forRebuild {
+    if forRebuild && !modulesRequiringRebuild.isEmpty {
       reporter?.reportExplicitDependencyReBuildSet(Array(modulesRequiringRebuild))
     }
     return modulesRequiringRebuild

Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift

@@ -141,6 +141,28 @@ public extension Driver {
       }
     }
 
+    if (parsedOptions.contains(.driverShowIncremental) ||
+        parsedOptions.contains(.dependencyScanCacheRemarks)) &&
+       isFrontendArgSupported(.dependencyScanCacheRemarks) {
+      commandLine.appendFlag(.dependencyScanCacheRemarks)
+    }
+
+    if shouldAttemptIncrementalCompilation &&
+       parsedOptions.contains(.incrementalDependencyScan) {
+      if let serializationPath = buildRecordInfo?.dependencyScanSerializedResultPath {
+        if isFrontendArgSupported(.validatePriorDependencyScanCache) {
+          // Any compiler which supports "-validate-prior-dependency-scan-cache"
+          // also supports "-load-dependency-scan-cache"
+          // and "-serialize-dependency-scan-cache" and "-dependency-scan-cache-path"
+          commandLine.appendFlag(.dependencyScanCachePath)
+          commandLine.appendPath(serializationPath)
+          commandLine.appendFlag(.reuseDependencyScanCache)
+          commandLine.appendFlag(.validatePriorDependencyScanCache)
+          commandLine.appendFlag(.serializeDependencyScanCache)
+        }
+      }
+    }
+
     // Pass on the input files
     commandLine.append(contentsOf: inputFiles.filter { $0.type == .swift }.map { .path($0.file) })
     return (inputs, commandLine)

Sources/SwiftDriver/IncrementalCompilation/BuildRecordInfo.swift

@@ -177,29 +177,6 @@ import class Dispatch.DispatchQueue
     try? fileSystem.removeFileTree(absPath)
   }
 
-  func removeInterModuleDependencyGraph() {
-    guard let absPath = interModuleDependencyGraphPath.absolutePath else {
-      return
-    }
-    try? fileSystem.removeFileTree(absPath)
-  }
-
-  func readPriorInterModuleDependencyGraph(
-    reporter: IncrementalCompilationState.Reporter?
-  ) -> InterModuleDependencyGraph? {
-    let decodedGraph: InterModuleDependencyGraph
-    do {
-      let contents = try fileSystem.readFileContents(interModuleDependencyGraphPath).cString
-      decodedGraph = try JSONDecoder().decode(InterModuleDependencyGraph.self,
-                                              from: Data(contents.utf8))
-    } catch {
-      return nil
-    }
-    reporter?.report("Read inter-module dependency graph", interModuleDependencyGraphPath)
-
-    return decodedGraph
-  }
-
   func jobFinished(job: Job, result: ProcessResult) {
     self.confinementQueue.sync {
       finishedJobResults.append(JobResult(job, result))
@@ -220,11 +197,11 @@ import class Dispatch.DispatchQueue
 
   /// A build-record-relative path to the location of a serialized copy of the
   /// driver's inter-module dependency graph.
-  var interModuleDependencyGraphPath: VirtualPath {
+  var dependencyScanSerializedResultPath: VirtualPath {
     let filename = buildRecordPath.basenameWithoutExt
     return buildRecordPath
       .parentDirectory
-      .appending(component: filename + ".moduledeps")
+      .appending(component: filename + ".swiftmoduledeps")
   }
 
   /// Directory to emit dot files into

Sources/SwiftDriver/IncrementalCompilation/FirstWaveComputer.swift

@@ -26,7 +26,6 @@ extension IncrementalCompilationState {
     let showJobLifecycle: Bool
     let alwaysRebuildDependents: Bool
     let interModuleDependencyGraph: InterModuleDependencyGraph?
-    let explicitModuleDependenciesGuaranteedUpToDate: Bool
     /// If non-null outputs information for `-driver-show-incremental` for input path
     private let reporter: Reporter?
 
@@ -47,8 +46,6 @@ extension IncrementalCompilationState {
       self.alwaysRebuildDependents = initialState.incrementalOptions.contains(
         .alwaysRebuildDependents)
       self.interModuleDependencyGraph = interModuleDependencyGraph
-      self.explicitModuleDependenciesGuaranteedUpToDate =
-        initialState.upToDatePriorInterModuleDependencyGraph != nil ? true : false
       self.reporter = reporter
     }
 

Sources/SwiftDriver/IncrementalCompilation/IncrementalCompilationState+Extensions.swift

@@ -50,8 +50,6 @@ extension IncrementalCompilationState {
     let graph: ModuleDependencyGraph
     /// Information about the last known compilation, incl. the location of build artifacts such as the dependency graph.
     let buildRecordInfo: BuildRecordInfo
-    /// Record about the compiled module's explicit module dependencies from a prior compile.
-    let upToDatePriorInterModuleDependencyGraph: InterModuleDependencyGraph?
     /// A set of inputs invalidated by external changes.
     let inputsInvalidatedByExternals: TransitivelyInvalidatedSwiftSourceFileSet
     /// Compiler options related to incremental builds.
@@ -287,11 +285,6 @@ extension IncrementalCompilationState {
       report("\(message): \(externalDependency.shortDescription)")
     }
 
-    // Emits a remark indicating a need for a dependency scanning invocation
-    func reportExplicitBuildMustReScan(_ why: String) {
-      report("Incremental build must re-run dependency scan: \(why)")
-    }
-
     func reportExplicitDependencyOutOfDate(_ moduleName: String,
                                            inputPath: String) {
       report("Dependency module \(moduleName) is older than input file \(inputPath)")
@@ -413,55 +406,6 @@ extension IncrementalCompilationState {
   }
 }
 
-extension IncrementalCompilationState {
-  enum WriteInterModuleDependencyGraphError: LocalizedError {
-    case noDependencyGraph
-    var errorDescription: String? {
-      switch self {
-      case .noDependencyGraph:
-        return "No inter-module dependency graph present"
-      }
-    }
-  }
-
-  func writeInterModuleDependencyGraph(_ buildRecordInfo: BuildRecordInfo?) throws {
-    // If the explicit module build is not happening, there will not be a graph to write
-    guard info.explicitModuleBuild else {
-      return
-    }
-    guard let recordInfo = buildRecordInfo else {
-      throw WriteDependencyGraphError.noBuildRecordInfo
-    }
-    guard let interModuleDependencyGraph = self.upToDateInterModuleDependencyGraph else {
-      throw WriteInterModuleDependencyGraphError.noDependencyGraph
-    }
-    do {
-      let encoder = JSONEncoder()
-#if os(Linux) || os(Android)
-      encoder.outputFormatting = [.prettyPrinted]
-#else
-      if #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) {
-        encoder.outputFormatting = [.prettyPrinted, .withoutEscapingSlashes]
-      }
-#endif
-      let data = try encoder.encode(interModuleDependencyGraph)
-      try fileSystem.writeFileContents(recordInfo.interModuleDependencyGraphPath,
-                                       bytes: ByteString(data),
-                                       atomically: true)
-    } catch {
-      throw IncrementalCompilationState.WriteDependencyGraphError.couldNotWrite(
-        path: recordInfo.interModuleDependencyGraphPath, error: error)
-    }
-
-  }
-
-  @_spi(Testing) public static func removeInterModuleDependencyGraphFile(_ driver: Driver) {
-    if let path = driver.buildRecordInfo?.interModuleDependencyGraphPath {
-      try? driver.fileSystem.removeFileTree(path)
-    }
-  }
-}
-
 // MARK: - OutputFileMap
 extension OutputFileMap {
   func onlySourceFilesHaveSwiftDeps() -> Bool {

Sources/SwiftDriver/IncrementalCompilation/IncrementalCompilationState.swift

@@ -53,7 +53,8 @@ public final class IncrementalCompilationState {
   internal init(
     driver: inout Driver,
     jobsInPhases: JobsInPhases,
-    initialState: InitialStateForPlanning
+    initialState: InitialStateForPlanning,
+    interModuleDepGraph: InterModuleDependencyGraph?
   ) throws {
     let reporter = initialState.incrementalOptions.contains(.showIncremental)
       ? Reporter(diagnosticEngine: driver.diagnosticEngine,
@@ -64,12 +65,12 @@ public final class IncrementalCompilationState {
       initialState: initialState,
       jobsInPhases: jobsInPhases,
       driver: driver,
-      interModuleDependencyGraph: driver.interModuleDependencyGraph,
+      interModuleDependencyGraph: interModuleDepGraph,
       reporter: reporter)
       .compute(batchJobFormer: &driver)
 
     self.info = initialState.graph.info
-    self.upToDateInterModuleDependencyGraph = driver.interModuleDependencyGraph
+    self.upToDateInterModuleDependencyGraph = interModuleDepGraph
     self.protectedState = ProtectedState(
       skippedCompileJobs: firstWave.initiallySkippedCompileJobs,
       initialState.graph,

Sources/SwiftDriver/IncrementalCompilation/IncrementalDependencyAndInputSetup.swift

@@ -58,9 +58,6 @@ extension IncrementalCompilationState {
         ).computeInitialStateForPlanning(driver: &driver)
     else {
       Self.removeDependencyGraphFile(driver)
-      if options.contains(.explicitModuleBuild) {
-        Self.removeInterModuleDependencyGraphFile(driver)
-      }
       return nil
     }
 
@@ -96,42 +93,6 @@ extension IncrementalCompilationState {
   }
 }
 
-/// Validate if a prior inter-module dependency graph is still valid
-extension IncrementalCompilationState.IncrementalDependencyAndInputSetup {
-  static func readAndValidatePriorInterModuleDependencyGraph(
-    driver: inout Driver,
-    buildRecordInfo: BuildRecordInfo,
-    reporter: IncrementalCompilationState.Reporter?
-  ) throws -> InterModuleDependencyGraph? {
-    // Attempt to read a serialized inter-module dependency graph from a prior build
-    guard let priorInterModuleDependencyGraph =
-        buildRecordInfo.readPriorInterModuleDependencyGraph(reporter: reporter),
-          let priorImports = priorInterModuleDependencyGraph.mainModule.directDependencies?.map({ $0.moduleName }) else {
-      reporter?.reportExplicitBuildMustReScan("Could not read inter-module dependency graph at \(buildRecordInfo.interModuleDependencyGraphPath)")
-      return nil
-    }
-
-    // Verify that import sets match
-    let currentImports = try driver.performImportPrescan().imports
-    guard Set(priorImports) == Set(currentImports) else {
-      reporter?.reportExplicitBuildMustReScan("Target import set has changed.")
-      return nil
-    }
-
-    // Verify that each dependnecy is up-to-date with respect to its inputs
-    guard try priorInterModuleDependencyGraph.computeInvalidatedModuleDependencies(fileSystem: buildRecordInfo.fileSystem,
-                                                                                   cas: driver.cas,
-                                                                                   forRebuild: false,
-                                                                                   reporter: reporter).isEmpty else {
-      reporter?.reportExplicitBuildMustReScan("Not all dependencies are up-to-date.")
-      return nil
-    }
-
-    reporter?.report("Confirmed prior inter-module dependency graph is up-to-date at: \(buildRecordInfo.interModuleDependencyGraphPath)")
-    return priorInterModuleDependencyGraph
-  }
-}
-
 /// Builds the `InitialState`
 /// Also bundles up an bunch of configuration info
 extension IncrementalCompilationState {
@@ -196,23 +157,8 @@ extension IncrementalCompilationState {
         return nil
       }
 
-      // If a valid build record could not be produced, do not bother here
-      let priorInterModuleDependencyGraph: InterModuleDependencyGraph?
-      if options.contains(.explicitModuleBuild) {
-        if priors.graph.buildRecord.inputInfos.isEmpty {
-          reporter?.report("Incremental compilation did not attempt to read inter-module dependency graph.")
-          priorInterModuleDependencyGraph = nil
-        } else {
-          priorInterModuleDependencyGraph = try Self.readAndValidatePriorInterModuleDependencyGraph(
-            driver: &driver, buildRecordInfo: buildRecordInfo, reporter: reporter)
-        }
-      } else {
-        priorInterModuleDependencyGraph = nil
-      }
-
       return InitialStateForPlanning(
         graph: priors.graph, buildRecordInfo: buildRecordInfo,
-        upToDatePriorInterModuleDependencyGraph: priorInterModuleDependencyGraph,
         inputsInvalidatedByExternals: priors.fileSet,
         incrementalOptions: options)
     }

Sources/SwiftDriver/Jobs/Planning.swift

@@ -71,7 +71,7 @@ extension Driver {
       try IncrementalCompilationState.computeIncrementalStateForPlanning(driver: &self)
 
     // For an explicit build, compute the inter-module dependency graph
-    interModuleDependencyGraph = try computeInterModuleDependencyGraph(with: initialIncrementalState)
+    let interModuleDependencyGraph = try computeInterModuleDependencyGraph()
 
     // Compute the set of all jobs required to build this module
     let jobsInPhases = try computeJobsForPhasedStandardBuild(moduleDependencyGraph: interModuleDependencyGraph,
@@ -84,7 +84,8 @@ extension Driver {
       incrementalCompilationState =
         try IncrementalCompilationState(driver: &self,
                                         jobsInPhases: jobsInPhases,
-                                        initialState: initialState)
+                                        initialState: initialState,
+                                        interModuleDepGraph: interModuleDependencyGraph)
     } else {
       incrementalCompilationState = nil
     }
@@ -104,17 +105,12 @@ extension Driver {
   /// If performing an explicit module build, compute an inter-module dependency graph.
   /// If performing an incremental build, and the initial incremental state contains a valid
   /// graph already, it is safe to re-use without repeating the scan.
-  private mutating func computeInterModuleDependencyGraph(with initialIncrementalState:
-                                                          IncrementalCompilationState.InitialStateForPlanning?)
+  private mutating func computeInterModuleDependencyGraph()
   throws -> InterModuleDependencyGraph? {
     if (parsedOptions.contains(.driverExplicitModuleBuild) ||
         parsedOptions.contains(.explainModuleDependency)) &&
        inputFiles.contains(where: { $0.type.isPartOfSwiftCompilation }) {
-      // If the incremental build record's module dependency graph is up-to-date, we
-      // can skip dependency scanning entirely.
-      return
-        try initialIncrementalState?.upToDatePriorInterModuleDependencyGraph ??
-            gatherModuleDependencies()
+      return try gatherModuleDependencies()
     } else {
       return nil
     }
@@ -182,12 +178,6 @@ extension Driver {
     // for example when `-explain-dependency` is being used.
     guard parsedOptions.contains(.driverExplicitModuleBuild) else { return }
 
-    // If the initial incremental state deemed the entire dependency graph up-to-date,
-    // then no modules need compilation.
-    guard initialIncrementalState?.upToDatePriorInterModuleDependencyGraph == nil else {
-      return
-    }
-
     // Filter out the tasks for building module dependencies which are already up-to-date
     let mandatoryModuleCompileJobs: [Job]
     // If -always-rebuild-module-dependencies is specified, no filtering needed