Remove unparsed ranges and use diffs to schedule dependents earlier.

Author: David Ungar

include/swift/AST/IncrementalRanges.h

@@ -26,7 +26,6 @@
 #include <vector>
 
 namespace swift {
-class PersistentParserState;
 class SourceManager;
 class DiagnosticEngine;
 class SourceFile;
@@ -170,15 +169,12 @@ struct SwiftRangesFileContents {
   /// For each non-primary, the unparsed ranges in it.
   /// At present these represent the bodies of types defined in the nonprimary
   /// that are not used in the primary.
-  RangesByFilename unparsedRangesByNonPrimary;
   Ranges noninlinableFunctionBodies;
 
-  SwiftRangesFileContents() : SwiftRangesFileContents({}, {}) {}
+  SwiftRangesFileContents() = default;
 
-  SwiftRangesFileContents(RangesByFilename &&unparsedRangesByNonPrimary,
-                          Ranges &&noninlinableFunctionBodies)
-      : unparsedRangesByNonPrimary(std::move(unparsedRangesByNonPrimary)),
-        noninlinableFunctionBodies(std::move(noninlinableFunctionBodies)) {}
+  SwiftRangesFileContents(Ranges &&noninlinableFunctionBodies)
+      : noninlinableFunctionBodies(std::move(noninlinableFunctionBodies)) {}
 
   /// Return None for error.
   static Optional<SwiftRangesFileContents>
@@ -198,8 +194,6 @@ struct llvm::yaml::MappingTraits<
   static void
   mapping(llvm::yaml::IO &io,
           swift::incremental_ranges::SwiftRangesFileContents &srfc) {
-    io.mapRequired("unparsedRangesByNonPrimary",
-                   srfc.unparsedRangesByNonPrimary);
     io.mapRequired("noninlinableFunctionBodies",
                    srfc.noninlinableFunctionBodies);
   }
@@ -218,16 +212,14 @@ namespace incremental_ranges {
 class SwiftRangesEmitter {
   const StringRef outputPath;
   SourceFile *const primaryFile;
-  const PersistentParserState &persistentState;
   const SourceManager &sourceMgr;
   DiagnosticEngine &diags;
 
 public:
   SwiftRangesEmitter(StringRef outputPath, SourceFile *primaryFile,
-                     const PersistentParserState &persistentState,
                      const SourceManager &sourceMgr, DiagnosticEngine &diags)
-      : outputPath(outputPath), primaryFile(primaryFile),
-        persistentState(persistentState), sourceMgr(sourceMgr), diags(diags) {}
+      : outputPath(outputPath), primaryFile(primaryFile), sourceMgr(sourceMgr),
+        diags(diags) {}
 
   /// True for error
   bool emit() const;
@@ -236,13 +228,9 @@ class SwiftRangesEmitter {
   void emitRanges(llvm::raw_ostream &out) const;
 
 private:
-  RangesByFilename collectSerializedUnparsedRangesByNonPrimary() const;
-
   Ranges collectSortedSerializedNoninlinableFunctionBodies() const;
   std::vector<CharSourceRange> collectNoninlinableFunctionBodies() const;
 
-  std::map<std::string, std::vector<CharSourceRange>>
-  collectUnparsedRanges() const;
   std::vector<CharSourceRange>
   sortRanges(std::vector<CharSourceRange> ranges) const;
 

include/swift/Driver/Compilation.h

@@ -82,7 +82,6 @@ class Compilation {
     const bool EnableIncrementalBuildWhenConstructed;
     const bool &EnableIncrementalBuild;
     const bool EnableSourceRangeDependencies;
-    const bool &UseSourceRangeDependencies;
 
     /// If not empty, the path to use to log the comparision.
     const StringRef CompareIncrementalSchemesPath;
@@ -95,38 +94,28 @@ class Compilation {
   private:
     DiagnosticEngine &Diags;
 
-    CommandSet DependencyCompileJobs;
-    CommandSet SourceRangeCompileJobs;
-    CommandSet SourceRangeLackingSuppJobs;
+    CommandSet JobsWithoutRanges;
+    CommandSet JobsWithRanges;
 
-    unsigned DependencyCompileStages = 0;
-    unsigned SourceRangeCompileStages = 0;
+    unsigned CompileStagesWithoutRanges = 0;
+    unsigned CompileStagesWithRanges = 0;
 
   public:
     IncrementalSchemeComparator(const bool &EnableIncrementalBuild,
                                 bool EnableSourceRangeDependencies,
-                                const bool &UseSourceRangeDependencies,
                                 const StringRef CompareIncrementalSchemesPath,
                                 unsigned SwiftInputCount,
                                 DiagnosticEngine &Diags)
         : EnableIncrementalBuildWhenConstructed(EnableIncrementalBuild),
           EnableIncrementalBuild(EnableIncrementalBuild),
           EnableSourceRangeDependencies(EnableSourceRangeDependencies),
-          UseSourceRangeDependencies(UseSourceRangeDependencies),
           CompareIncrementalSchemesPath(CompareIncrementalSchemesPath),
           SwiftInputCount(SwiftInputCount), Diags(Diags) {}
 
     /// Record scheduled jobs in support of the
     /// -compare-incremental-schemes[-path] options
-    ///
-    /// \param depJobs A vector-like collection of jobs that the dependency
-    /// scheme would run \param rangeJobs A vector-like collection of jobs that
-    /// the range scheme would run because of changes \param lackingSuppJobs A
-    /// vector-like collection of jobs that the range scheme would run because
-    /// there are no incremental supplementary outputs such as swiftdeps,
-    /// swiftranges, compiledsource
-    void update(const CommandSet &depJobs, const CommandSet &rangeJobs,
-                const CommandSet &lackingSuppJobs);
+    void update(const CommandSet &withoutRangeJobs,
+                const CommandSet &withRangeJobs);
 
     /// Write the information for the -compare-incremental-schemes[-path]
     /// options
@@ -285,9 +274,6 @@ class Compilation {
   /// Experiment with source-range-based dependencies
   const bool EnableSourceRangeDependencies;
 
-  /// May not actually use them if e.g. there is a new input
-  bool UseSourceRangeDependencies;
-
 public:
   /// Will contain a comparator if an argument demands it.
   Optional<IncrementalSchemeComparator> IncrementalComparator;
@@ -408,14 +394,6 @@ class Compilation {
     return EnableSourceRangeDependencies;
   }
 
-  bool getUseSourceRangeDependencies() const {
-    return UseSourceRangeDependencies;
-  }
-
-  void setUseSourceRangeDependencies(bool use) {
-    UseSourceRangeDependencies = use;
-  }
-
   bool getBatchModeEnabled() const {
     return EnableBatchMode;
   }
@@ -515,14 +493,6 @@ class Compilation {
   /// How many .swift input files?
   unsigned countSwiftInputs() const;
 
-  void updateIncrementalComparison(const CommandSet &depJobs,
-                                   const CommandSet &rangeJobs,
-                                   const CommandSet &lackingSuppJobs) {
-    if (IncrementalComparator.hasValue())
-      IncrementalComparator.getValue().update(depJobs, rangeJobs,
-                                              lackingSuppJobs);
-  }
-
 private:
   /// Perform all jobs.
   ///

include/swift/Driver/DependencyGraph.h

@@ -165,9 +165,7 @@ class DependencyGraphImpl {
   void markExternal(SmallVectorImpl<const void *> &visited,
                     StringRef externalDependency);
 
-public: // for ranges
-  size_t countTopLevelProvides(const void *) const;
-
+public:
   void forEachUnmarkedJobDirectlyDependentOnExternalSwiftdeps(
       StringRef externalDependency, function_ref<void(const void *)> fn);
 

include/swift/Driver/DriverIncrementalRanges.h

@@ -36,34 +36,34 @@ namespace incremental_ranges {
 /// A per-primary collection of information about its source ranges.
 
 class SourceRangeBasedInfo {
-  SwiftRangesFileContents swiftRangesFileContents;
+  const std::string primaryInputPath;
+  const SwiftRangesFileContents swiftRangesFileContents;
   /// All changed ranges in the primary as computed by the diff in the driver
   /// Both relative to the previously-compiled and the current source.
-  SourceComparator::LRRanges changedRanges;
+  const SourceComparator::LRRanges changedRanges;
   /// All of the non-local changes in the previously-compiled code:  those
   /// residing outside function bodies.
   /// (We only have and only need function-body ranges for the
   /// previously-compiled source.)
-  Ranges nonlocalChangedRanges;
+  const Ranges nonlocalChangedRanges;
 
   //==============================================================================
   // MARK: construction
   //==============================================================================
 
 public:
-  ///  return hadError and info
-  static llvm::StringMap<SourceRangeBasedInfo>
-  loadAllInfo(const driver::Compilation &);
+  static Optional<SourceRangeBasedInfo>
+  loadInfoForOneJob(const driver::Job *cmd,
+                    const bool showIncrementalBuildDecisions,
+                    DiagnosticEngine &diags);
 
   SourceRangeBasedInfo(SourceRangeBasedInfo &&);
 
 private:
-  SourceRangeBasedInfo(SwiftRangesFileContents &&,
+  SourceRangeBasedInfo(StringRef primaryInputPath, SwiftRangesFileContents &&,
                        SourceComparator::LRRanges &&changedRanges,
                        Ranges &&nonlocalChangedRanges);
 
-  static Optional<SourceRangeBasedInfo> wholeFileChanged();
-
   /// Using supplied paths, interrogate the supplementary outputs and update the
   /// two references. Return None if a file was missing or corrupted.
   static Optional<SourceRangeBasedInfo>
@@ -93,37 +93,29 @@ class SourceRangeBasedInfo {
   //==============================================================================
   // MARK: scheduling jobs
   //==============================================================================
-
 public:
-  static bool shouldScheduleCompileJob(
-      const llvm::StringMap<SourceRangeBasedInfo> &allInfos,
-      const driver::Job *, function_ref<void(bool, Twine)>);
+  bool
+  didInputChangeAtAll(DiagnosticEngine &,
+                      function_ref<void(bool, StringRef)> noteBuilding) const;
+  bool didInputChangeNonlocally(
+      DiagnosticEngine &,
+      function_ref<void(bool, StringRef)> noteInitiallyCascading) const;
 
 private:
   static Optional<bool> isFileNewerThan(StringRef lhs, StringRef rhs,
                                         DiagnosticEngine&);
 
-private:
-  /// Return hadError
-
-  bool didPrimaryParseAnyNonlocalNonprimaryChanges(
-      StringRef primary, const llvm::StringMap<SourceRangeBasedInfo> &,
-      function_ref<void(bool, Twine)>) const;
-
-  bool wasEveryNonprimaryNonlocalChangeUnparsed(
-      StringRef primary, const llvm::StringMap<SourceRangeBasedInfo> &,
-      function_ref<void(bool, Twine)>) const;
 
   //==============================================================================
   // MARK: printing
   //==============================================================================
 
 public:
-  static void dumpAllInfo(const llvm::StringMap<SourceRangeBasedInfo> &,
-                          bool dumpCompiledSourceDiffs, bool dumpSwiftRanges);
+  void dump(bool dumpCompiledSourceDiffs = true,
+            bool dumpSwiftRanges = true) const;
 
 private:
-  void dumpChangedRanges(StringRef primary) const;
+  void dumpChangedRanges() const;
 };
 
 } // namespace incremental_ranges

include/swift/Driver/ExperimentalDependencyDriverGraph.h

@@ -267,6 +267,9 @@ class ModuleDepGraph {
     assert(verify() && "ModuleDepGraph should be fine when created");
   }
 
+  /// Unlike the standard \c DependencyGraph, returns \c
+  /// DependencyGraphImpl::LoadResult::AffectsDownstream when loading a new
+  /// file, i.e. when determining the initial set. Caller compensates.
   DependencyGraphImpl::LoadResult loadFromPath(const driver::Job *, StringRef,
                                                DiagnosticEngine &);
 
@@ -286,10 +289,6 @@ class ModuleDepGraph {
   forEachMatchingNode(const DependencyKey &key,
                       function_ref<void(const ModuleDepGraphNode *)>) const;
 
-public:
-  // Interface to source-range incremental system
-  size_t countTopLevelProvides(const driver::Job *);
-
 public:
   // This section contains the interface to the status quo code in the driver.
 

include/swift/Parse/PersistentParserState.h

@@ -128,9 +128,6 @@ class PersistentParserState {
     MarkedPos = ParserPosition();
     return Pos;
   }
-
-  void forEachDelayedSourceRange(const SourceFile *primaryFile,
-                                 function_ref<void(SourceRange)>) const;
 };
 
 } // end namespace swift

lib/AST/IncrementalRanges.cpp

@@ -25,7 +25,6 @@
 #include "swift/AST/SourceFile.h"
 #include "swift/Basic/SourceManager.h"
 #include "swift/Parse/Lexer.h"
-#include "swift/Parse/PersistentParserState.h"
 #include "llvm/Support/YAMLParser.h"
 
 using namespace swift;
@@ -194,26 +193,11 @@ bool SwiftRangesEmitter::emit() const {
 
 void SwiftRangesEmitter::emitRanges(llvm::raw_ostream &out) const {
   SwiftRangesFileContents wholeFileContents(
-      collectSerializedUnparsedRangesByNonPrimary(),
       collectSortedSerializedNoninlinableFunctionBodies());
   llvm::yaml::Output yamlWriter(out);
   yamlWriter << wholeFileContents;
 }
 
-RangesByFilename
-SwiftRangesEmitter::collectSerializedUnparsedRangesByNonPrimary() const {
-  auto rangesByNonprimary = collectUnparsedRanges();
-  std::map<std::string, std::vector<SerializableSourceRange>>
-      serializedRangesByNonprimary;
-  // The driver counts on getting coalescedSortedRanges because it
-  // does binary chop searches
-  for (auto &nonPriAndRanges : rangesByNonprimary)
-    serializedRangesByNonprimary.insert(
-        {nonPriAndRanges.first, serializeRanges(coalesceSortedRanges(sortRanges(
-                                    std::move(nonPriAndRanges.second))))});
-  return serializedRangesByNonprimary;
-}
-
 Ranges
 SwiftRangesEmitter::collectSortedSerializedNoninlinableFunctionBodies() const {
   return serializeRanges(
@@ -249,20 +233,6 @@ SwiftRangesEmitter::collectNoninlinableFunctionBodies() const {
   return collector.ranges;
 }
 
-std::map<std::string, std::vector<CharSourceRange>>
-SwiftRangesEmitter::collectUnparsedRanges() const {
-  std::map<std::string, std::vector<CharSourceRange>> rangesByNonprimaryFile;
-  persistentState.forEachDelayedSourceRange(
-      primaryFile, [&](const SourceRange sr) {
-        const auto filename = sourceMgr.getIdentifierForBuffer(
-            sourceMgr.findBufferContainingLoc(sr.Start));
-        const auto csr =
-            Lexer::getCharSourceRangeFromSourceRange(sourceMgr, sr);
-        rangesByNonprimaryFile[filename].push_back(csr);
-      });
-  return rangesByNonprimaryFile;
-}
-
 std::vector<CharSourceRange>
 SwiftRangesEmitter::sortRanges(std::vector<CharSourceRange> ranges) const {
   std::sort(ranges.begin(), ranges.end(),
@@ -331,9 +301,9 @@ bool CompiledSourceEmitter::emit() {
 // MARK: SwiftRangesFileContents
 //==============================================================================
 
-void SwiftRangesFileContents::dump(const StringRef primaryFilename) const {
-  llvm::errs() << "\n*** Swift range file contents for '" << primaryFilename
-               << "': ***\n";
+void SwiftRangesFileContents::dump(const StringRef primaryInputFilename) const {
+  llvm::errs() << "\n*** Swift range file contents for '"
+               << primaryInputFilename << "': ***\n";
   llvm::yaml::Output dumper(llvm::errs());
   dumper << *const_cast<SwiftRangesFileContents *>(this);
 }