Merge pull request #799 from davidungar/add-source-file-type

[Incremental] Add SwiftSourceFile type

Author: David Ungar

Sources/SwiftDriver/CMakeLists.txt

@@ -51,6 +51,7 @@ add_library(SwiftDriver
   "IncrementalCompilation/KeyAndFingerprintHolder.swift"
   "IncrementalCompilation/ModuleDependencyGraph.swift"
   "IncrementalCompilation/Multidictionary.swift"
+  "IncrementalCompilation/SwiftSourceFile.swift"
   "IncrementalCompilation/SourceFileDependencyGraph.swift"
   "IncrementalCompilation/TwoDMap.swift"
 

Sources/SwiftDriver/IncrementalCompilation/DirectAndTransitiveCollections.swift

@@ -85,5 +85,7 @@ public typealias TransitivelyInvalidatedNodeArray = InvalidatedArray<Transitivel
 public typealias TransitivelyInvalidatedSourceSet = InvalidatedSet<Transitively, DependencySource>
 public typealias TransitivelyInvalidatedInputArray = InvalidatedArray<Transitively, TypedVirtualPath>
 public typealias TransitivelyInvalidatedInputSet = InvalidatedSet<Transitively, TypedVirtualPath>
+public typealias TransitivelyInvalidatedSwiftSourceFileArray = InvalidatedArray<Transitively, SwiftSourceFile>
+public typealias TransitivelyInvalidatedSwiftSourceFileSet = InvalidatedSet<Transitively, SwiftSourceFile>
 public typealias DirectlyInvalidatedNodeArray = InvalidatedArray<Directly, ModuleDependencyGraph.Node>
 public typealias DirectlyInvalidatedNodeSet = InvalidatedSet<Directly, ModuleDependencyGraph.Node>

Sources/SwiftDriver/IncrementalCompilation/FirstWaveComputer.swift

@@ -17,7 +17,7 @@ extension IncrementalCompilationState {
   struct FirstWaveComputer {
     let moduleDependencyGraph: ModuleDependencyGraph
     let jobsInPhases: JobsInPhases
-    let inputsInvalidatedByExternals: TransitivelyInvalidatedInputSet
+    let inputsInvalidatedByExternals: TransitivelyInvalidatedSwiftSourceFileSet
     let inputFiles: [TypedVirtualPath]
     let sourceFiles: SourceFiles
     let buildRecordInfo: BuildRecordInfo
@@ -74,20 +74,26 @@ extension IncrementalCompilationState.FirstWaveComputer {
       Dictionary(uniqueKeysWithValues:
                   jobsInPhases.compileGroups.map { ($0.primaryInput, $0) })
     guard let buildRecord = maybeBuildRecord else {
-      let mandatoryCompileGroupsInOrder = sourceFiles.currentInOrder.compactMap {
-        input -> CompileJobGroup? in
-        compileGroups[input]
-      }
+      func everythingIsMandatory()
+        throws -> (skippedCompileGroups: [TypedVirtualPath: CompileJobGroup],
+                   mandatoryJobsInOrder: [Job])
+      {
+        let mandatoryCompileGroupsInOrder = sourceFiles.currentInOrder.compactMap {
+          input -> CompileJobGroup? in
+          compileGroups[input.typedFile]
+        }
 
-      let mandatoryJobsInOrder = try
+        let mandatoryJobsInOrder = try
         jobsInPhases.beforeCompiles +
         batchJobFormer.formBatchedJobs(
           mandatoryCompileGroupsInOrder.flatMap {$0.allJobs()},
           showJobLifecycle: showJobLifecycle)
 
-      moduleDependencyGraph.phase = .buildingAfterEachCompilation
-      return (skippedCompileGroups: [:],
-              mandatoryJobsInOrder: mandatoryJobsInOrder)
+        moduleDependencyGraph.phase = .buildingAfterEachCompilation
+        return (skippedCompileGroups: [:],
+                mandatoryJobsInOrder: mandatoryJobsInOrder)
+      }
+      return try everythingIsMandatory()
     }
     moduleDependencyGraph.phase = .updatingAfterCompilation
 
@@ -117,7 +123,7 @@ extension IncrementalCompilationState.FirstWaveComputer {
 
   // Figure out which compilation inputs are *not* mandatory
   private func computeSkippedCompilationInputs(
-    inputsInvalidatedByExternals: TransitivelyInvalidatedInputSet,
+    inputsInvalidatedByExternals: TransitivelyInvalidatedSwiftSourceFileSet,
     _ moduleDependencyGraph: ModuleDependencyGraph,
     _ buildRecord: BuildRecord
   ) -> Set<TypedVirtualPath> {
@@ -153,11 +159,11 @@ extension IncrementalCompilationState.FirstWaveComputer {
     }
 
     // Combine to obtain the inputs that definitely must be recompiled.
-    let definitelyRequiredInputs =
-      Set(changedInputs.map({ $0.filePath }) +
-            inputsInvalidatedByExternals +
-            inputsMissingFromGraph +
-            inputsMissingOutputs)
+    var definitelyRequiredInputs = Set(changedInputs.lazy.map {$0.typedFile})
+    definitelyRequiredInputs.formUnion(inputsInvalidatedByExternals.lazy.map {$0.typedFile})
+    definitelyRequiredInputs.formUnion(inputsMissingFromGraph.lazy.map {$0.typedFile})
+    definitelyRequiredInputs.formUnion(inputsMissingOutputs)
+
     if let reporter = reporter {
       for scheduledInput in sortByCommandLineOrder(definitelyRequiredInputs) {
         reporter.report("Queuing (initial):", scheduledInput)
@@ -173,14 +179,15 @@ extension IncrementalCompilationState.FirstWaveComputer {
       externalDependents: inputsInvalidatedByExternals,
       inputsMissingOutputs: Set(inputsMissingOutputs),
       moduleDependencyGraph)
-      .subtracting(definitelyRequiredInputs)
+      .subtracting(definitelyRequiredInputs.swiftSourceFiles)
+
 
     if let reporter = reporter {
       for dependent in sortByCommandLineOrder(speculativeInputs) {
         reporter.report("Queuing because of the initial set:", dependent)
       }
     }
-    let immediatelyCompiledInputs = definitelyRequiredInputs.union(speculativeInputs)
+    let immediatelyCompiledInputs = definitelyRequiredInputs.union(speculativeInputs.lazy.map {$0.typedFile})
 
     let skippedInputs = Set(buildRecordInfo.compilationInputModificationDates.keys)
       .subtracting(immediatelyCompiledInputs)
@@ -198,11 +205,17 @@ extension IncrementalCompilationState.FirstWaveComputer {
       inputFiles.lazy.filter(inputs.contains)
   }
 
+  private func sortByCommandLineOrder(
+    _ inputs: Set<SwiftSourceFile>
+  ) -> LazyFilterSequence<[TypedVirtualPath]> {
+    inputFiles.lazy.filter {inputs.contains(SwiftSourceFile($0))}
+  }
+
   /// Encapsulates information about an input the driver has determined has
   /// changed in a way that requires an incremental rebuild.
   struct ChangedInput {
     /// The path to the input file.
-    let filePath: TypedVirtualPath
+    let typedFile: TypedVirtualPath
     /// The status of the input file.
     let status: InputInfo.Status
     /// If `true`, the modification time of this input matches the modification
@@ -244,7 +257,7 @@ extension IncrementalCompilationState.FirstWaveComputer {
       case .needsNonCascadingBuild:
         reporter?.report("Scheduling noncascading build", input)
       }
-      return ChangedInput(filePath: input,
+      return ChangedInput(typedFile: input,
                           status: previousCompilationStatus,
                           datesMatch: datesMatch)
     }
@@ -256,22 +269,29 @@ extension IncrementalCompilationState.FirstWaveComputer {
   // before the whole frontend job finished.
   private func collectInputsToBeSpeculativelyRecompiled(
     changedInputs: [ChangedInput],
-    externalDependents: TransitivelyInvalidatedInputSet,
+    externalDependents: TransitivelyInvalidatedSwiftSourceFileSet,
     inputsMissingOutputs: Set<TypedVirtualPath>,
     _ moduleDependencyGraph: ModuleDependencyGraph
-  ) -> Set<TypedVirtualPath> {
+  ) -> Set<SwiftSourceFile> {
     let cascadingChangedInputs = computeCascadingChangedInputs(
       from: changedInputs,
       inputsMissingOutputs: inputsMissingOutputs)
 
-    var inputsToBeCertainlyRecompiled = alwaysRebuildDependents ? externalDependents : TransitivelyInvalidatedInputSet()
-    inputsToBeCertainlyRecompiled.formUnion(cascadingChangedInputs)
+    var inputsToBeCertainlyRecompiled = Set(cascadingChangedInputs)
+    if alwaysRebuildDependents {
+      inputsToBeCertainlyRecompiled.formUnion(externalDependents.lazy.map {$0.typedFile})
+    }
 
     return inputsToBeCertainlyRecompiled.reduce(into: Set()) {
       speculativelyRecompiledInputs, certainlyRecompiledInput in
-      let speculativeDependents = moduleDependencyGraph.collectInputsInvalidatedBy(changedInput: certainlyRecompiledInput)
+      guard let certainlyRecompiledSwiftSourceFile = SwiftSourceFile(ifSource: certainlyRecompiledInput)
+      else {
+        return
+      }
+      let speculativeDependents = moduleDependencyGraph.collectInputsInvalidatedBy(changedInput: certainlyRecompiledSwiftSourceFile)
+
       for speculativeDependent in speculativeDependents
-      where !inputsToBeCertainlyRecompiled.contains(speculativeDependent) {
+      where !inputsToBeCertainlyRecompiled.contains(speculativeDependent.typedFile) {
         if speculativelyRecompiledInputs.insert(speculativeDependent).inserted {
           reporter?.report(
             "Immediately scheduling dependent on \(certainlyRecompiledInput.file.basename)",
@@ -288,8 +308,8 @@ extension IncrementalCompilationState.FirstWaveComputer {
   ) -> [TypedVirtualPath] {
     changedInputs.compactMap { changedInput in
       let inputIsUpToDate =
-        changedInput.datesMatch && !inputsMissingOutputs.contains(changedInput.filePath)
-      let basename = changedInput.filePath.file.basename
+        changedInput.datesMatch && !inputsMissingOutputs.contains(changedInput.typedFile)
+      let basename = changedInput.typedFile.file.basename
 
       // If we're asked to always rebuild dependents, all we need to do is
       // return inputs whose modification times have changed.
@@ -301,15 +321,15 @@ extension IncrementalCompilationState.FirstWaveComputer {
         } else {
           reporter?.report(
             "scheduling dependents of \(basename); -driver-always-rebuild-dependents")
-          return changedInput.filePath
+          return changedInput.typedFile
         }
       }
 
       switch changedInput.status {
       case .needsCascadingBuild:
         reporter?.report(
           "scheduling dependents of \(basename); needed cascading build")
-        return changedInput.filePath
+        return changedInput.typedFile
       case .upToDate:
         reporter?.report(
           "not scheduling dependents of \(basename); unknown changes")

Sources/SwiftDriver/IncrementalCompilation/IncrementalCompilationState.swift

@@ -126,7 +126,7 @@ extension IncrementalCompilationState {
     /// Record about existence and time of the last compile.
     let maybeBuildRecord: BuildRecord?
     /// A set of inputs invalidated by external chagnes.
-    let inputsInvalidatedByExternals: TransitivelyInvalidatedInputSet
+    let inputsInvalidatedByExternals: TransitivelyInvalidatedSwiftSourceFileSet
     /// Compiler options related to incremental builds.
     let incrementalOptions: IncrementalCompilationState.Options
     /// The last time this compilation was started. Used to compare against e.g. input file mod dates.
@@ -146,6 +146,7 @@ extension IncrementalCompilationState {
     let skippedCompileGroups: [TypedVirtualPath: CompileJobGroup]
     /// All of the pre-compile or compilation job (groups) known to be required
     /// for the first wave to execute.
+    /// The primaries could be other than .swift files, i.e. .sib
     let mandatoryJobsInOrder: [Job]
   }
 }
@@ -218,7 +219,7 @@ extension IncrementalCompilationState {
     return try self.confinementQueue.sync {
       // Find and deal with inputs that now need to be compiled
       let invalidatedInputs = collectInputsInvalidatedByRunning(finishedJob)
-      assert(Set(invalidatedInputs).isDisjoint(with: finishedJob.primaryInputs),
+      assert(invalidatedInputs.isDisjoint(with: finishedJob.primarySwiftSourceFiles),
              "Primaries should not overlap secondaries.")
 
       if let reporter = self.reporter {
@@ -239,37 +240,39 @@ extension IncrementalCompilationState {
   }
 
   /// After `job` finished find out which inputs must compiled that were not known to need compilation before
-  private func collectInputsInvalidatedByRunning(_ job: Job)-> Set<TypedVirtualPath> {
+  private func collectInputsInvalidatedByRunning(_ job: Job)-> Set<SwiftSourceFile> {
     dispatchPrecondition(condition: .onQueue(self.confinementQueue))
     guard job.kind == .compile else {
-      return Set<TypedVirtualPath>()
+      return Set<SwiftSourceFile>()
     }
     return job.primaryInputs.reduce(into: Set()) { invalidatedInputs, primaryInput in
-      invalidatedInputs.formUnion(collectInputsInvalidated(byCompiling: primaryInput))
+      if let primary = SwiftSourceFile(ifSource: primaryInput) {
+        invalidatedInputs.formUnion(collectInputsInvalidated(byCompiling: primary))
+      }
     }
-    .subtracting(job.primaryInputs) // have already compiled these
+    .subtracting(job.primarySwiftSourceFiles) // have already compiled these
   }
 
   private func collectInputsInvalidated(
-    byCompiling input: TypedVirtualPath
-  ) -> TransitivelyInvalidatedInputSet {
+    byCompiling input: SwiftSourceFile
+  ) -> TransitivelyInvalidatedSwiftSourceFileSet {
     dispatchPrecondition(condition: .onQueue(self.confinementQueue))
     if let found = moduleDependencyGraph.collectInputsRequiringCompilation(byCompiling: input) {
       return found
     }
     self.reporter?.report(
       "Failed to read some dependencies source; compiling everything", input)
-    return TransitivelyInvalidatedInputSet(skippedCompileGroups.keys)
+    return TransitivelyInvalidatedSwiftSourceFileSet(skippedCompileGroups.keys.swiftSourceFiles)
   }
 
   /// Find the jobs that now must be run that were not originally known to be needed.
   private func getJobs(
-    for invalidatedInputs: Set<TypedVirtualPath>
+    for invalidatedInputs: Set<SwiftSourceFile>
   ) throws -> [Job] {
     dispatchPrecondition(condition: .onQueue(self.confinementQueue))
     let unbatched = invalidatedInputs.flatMap { input -> [Job] in
-      if let group = skippedCompileGroups.removeValue(forKey: input) {
-        let primaryInputs = group.compileJob.primaryInputs
+      if let group = skippedCompileGroups.removeValue(forKey: input.typedFile) {
+        let primaryInputs = group.compileJob.primarySwiftSourceFiles
         assert(primaryInputs.count == 1)
         assert(primaryInputs[0] == input)
         self.reporter?.report("Scheduling invalidated", input)
@@ -399,6 +402,10 @@ extension IncrementalCompilationState {
       diagnosticEngine.emit(.remark_incremental_compilation(because: "\(message) \(compiling)"))
     }
 
+    public func report(_ message: String, _ ifh: SwiftSourceFile) {
+      report(message, ifh.typedFile)
+    }
+
     /// Entry point for a simple path, won't print the compile job, path could be anything.
     public func report(_ message: String, _ path: VirtualPath?) {
       guard let path = path
@@ -545,43 +552,41 @@ extension OutputFileMap {
 
 /// Handy information about the source files in the current invocation
 @_spi(Testing) public struct SourceFiles {
-  /// The current files in same order as the invocation
-  let currentInOrder: [TypedVirtualPath]
+  /// The current (.swift) files in same order as the invocation
+  let currentInOrder: [SwiftSourceFile]
 
   /// The set of current files (actually the handles)
-  let currentSet: Set<VirtualPath.Handle>
+  let currentSet: Set<SwiftSourceFile>
 
   /// Handles of the input files in the previous invocation
-  private let previousSet: Set<VirtualPath.Handle>
+  private let previousSet: Set<SwiftSourceFile>
 
   /// The files that were in the previous but not in the current invocation
-  let disappeared: [VirtualPath]
+  let disappeared: [SwiftSourceFile]
 
   init(inputFiles: [TypedVirtualPath], buildRecord: BuildRecord?) {
-    self.currentInOrder = inputFiles.filter {$0.type == .swift}
-    self.currentSet = currentInOrder.reduce(into: Set()) {
-      currentSet, currentFile in
-      currentSet.insert(currentFile.fileHandle)
-    }
+    self.currentInOrder = inputFiles.swiftSourceFiles
+    self.currentSet = Set(currentInOrder)
     guard let buildRecord = buildRecord else {
       self.previousSet = Set()
       self.disappeared = []
       return
     }
-    var previous = Set<VirtualPath.Handle>()
-    var disappeared = [VirtualPath]()
+    var previous = Set<SwiftSourceFile>()
+    var disappeared = [SwiftSourceFile]()
     for prevPath in buildRecord.inputInfos.keys {
-      let handle = prevPath.intern()
+      let handle = SwiftSourceFile(prevPath)
       previous.insert(handle)
       if !currentSet.contains(handle) {
-        disappeared.append(prevPath)
+        disappeared.append(handle)
       }
     }
     self.previousSet = previous
-    self.disappeared = disappeared.sorted {$0.name < $1.name}
+    self.disappeared = disappeared.sorted {
+      VirtualPath.lookup($0.fileHandle).name < VirtualPath.lookup($1.fileHandle).name}
   }
 
-  func isANewInput(_ file: VirtualPath) -> Bool {
-    !previousSet.contains(file.intern())
+  func isANewInput(_ file: SwiftSourceFile) -> Bool {
+    !previousSet.contains(file)
   }
 }

Sources/SwiftDriver/IncrementalCompilation/IncrementalDependencyAndInputSetup.swift

@@ -166,8 +166,8 @@ extension IncrementalCompilationState {
         if let reporter = reporter {
           reporter.report(
             "Incremental compilation has been disabled, "
-              + "because the following inputs were used in the previous compilation but not in this one: "
-              + sourceFiles.disappeared.map { $0.basename }.joined(separator: ", "))
+            + "because the following inputs were used in the previous compilation but not in this one: "
+            + sourceFiles.disappeared.map { $0.typedFile.file.basename }.joined(separator: ", "))
         }
         return nil
       }
@@ -197,7 +197,7 @@ extension IncrementalCompilationState.IncrementalDependencyAndInputSetup {
   /// For inputs with swiftDeps in OFM, but no readable file, puts input in graph map, but no nodes in graph:
   ///   caller must ensure scheduling of those
   private func computeGraphAndInputsInvalidatedByExternals()
-    -> (ModuleDependencyGraph, TransitivelyInvalidatedInputSet)?
+    -> (ModuleDependencyGraph, TransitivelyInvalidatedSwiftSourceFileSet)?
   {
     precondition(
       sourceFiles.disappeared.isEmpty,
@@ -211,7 +211,7 @@ extension IncrementalCompilationState.IncrementalDependencyAndInputSetup {
   }
 
   private func readPriorGraphAndCollectInputsInvalidatedByChangedOrAddedExternals(
-  ) -> (ModuleDependencyGraph, TransitivelyInvalidatedInputSet)?
+  ) -> (ModuleDependencyGraph, TransitivelyInvalidatedSwiftSourceFileSet)?
   {
     let dependencyGraphPath = buildRecordInfo.dependencyGraphPath
     let graphIfPresent: ModuleDependencyGraph?
@@ -269,10 +269,10 @@ extension IncrementalCompilationState.IncrementalDependencyAndInputSetup {
   /// For externalDependencies, puts then in graph.fingerprintedExternalDependencies, but otherwise
   /// does nothing special.
   private func buildInitialGraphFromSwiftDepsAndCollectInputsInvalidatedByChangedExternals()
-  -> (ModuleDependencyGraph, TransitivelyInvalidatedInputSet)?
+  -> (ModuleDependencyGraph, TransitivelyInvalidatedSwiftSourceFileSet)?
   {
     let graph = ModuleDependencyGraph(self, .buildingFromSwiftDeps)
-    var inputsInvalidatedByChangedExternals = TransitivelyInvalidatedInputSet()
+    var inputsInvalidatedByChangedExternals = TransitivelyInvalidatedSwiftSourceFileSet()
     for input in sourceFiles.currentInOrder {
        guard let invalidatedInputs =
               graph.collectInputsRequiringCompilationFromExternalsFoundByCompiling(input: input)
@@ -286,8 +286,8 @@ extension IncrementalCompilationState.IncrementalDependencyAndInputSetup {
   }
 
   private func bulidEmptyGraphAndCompileEverything()
-  -> (ModuleDependencyGraph, TransitivelyInvalidatedInputSet) {
+  -> (ModuleDependencyGraph, TransitivelyInvalidatedSwiftSourceFileSet) {
     let graph = ModuleDependencyGraph(self, .buildingAfterEachCompilation)
-    return (graph, TransitivelyInvalidatedInputSet())
+    return (graph, TransitivelyInvalidatedSwiftSourceFileSet())
   }
 }