Merge pull request swiftlang#18078 from ahoppen/01#2-record-reused-node-ids

ahoppen · web-flow · commit dc958d670b1a · 2018-07-19T14:37:18.000-07:00
[libSyntax] Record reused node IDs
diff --git a/include/swift/Parse/SyntaxParsingCache.h b/include/swift/Parse/SyntaxParsingCache.h
@@ -16,6 +16,7 @@
 #include "swift/Syntax/SyntaxNodes.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/raw_ostream.h"
+#include <unordered_set>
 
 namespace {
 
@@ -48,10 +49,8 @@ namespace swift {
 using namespace swift::syntax;
 
 struct SyntaxReuseRegion {
-  /// The byte offset at which the range begins
-  uintptr_t Start;
-  /// The byte offset at which the end ends
-  uintptr_t End;
+  AbsolutePosition Start;
+  AbsolutePosition End;
 };
 
 class SyntaxParsingCache {
@@ -62,13 +61,8 @@ class SyntaxParsingCache {
   /// the source file that is now parsed incrementally
   llvm::SmallVector<SourceEdit, 4> Edits;
 
-  /// Whether or not information about reused nodes shall be recored in
-  /// \c ReusedRanges
-  bool RecordReuseInformation = false;
-
-  /// If \c RecordReuseInformation buffer offsets of ranges that have been
-  /// successfully looked up in this cache are stored.
-  std::vector<SyntaxReuseRegion> ReusedRanges;
+  /// The IDs of all syntax nodes that got reused are collected in this vector.
+  std::unordered_set<SyntaxNodeId> ReusedNodeIds;
 
 public:
   SyntaxParsingCache(SourceFileSyntax OldSyntaxTree)
@@ -86,16 +80,15 @@ class SyntaxParsingCache {
   /// reused for a new syntax tree.
   llvm::Optional<Syntax> lookUp(size_t NewPosition, SyntaxKind Kind);
 
-  /// Turn recording of reused ranges on
-  void setRecordReuseInformation() { RecordReuseInformation = true; }
-
-  /// Return the ranges of the new source file that have been successfully
-  /// looked up in this cache as a (start, end) pair of byte offsets in the
-  /// post-edit file.
-  std::vector<SyntaxReuseRegion> getReusedRanges() const {
-    return ReusedRanges;
+  const std::unordered_set<SyntaxNodeId> &getReusedNodeIds() const {
+    return ReusedNodeIds;
   }
 
+  /// Get the source regions of the new source file, represented by
+  /// \p SyntaxTree that have been reused as part of the incremental parse.
+  std::vector<SyntaxReuseRegion>
+  getReusedRegions(const SourceFileSyntax &SyntaxTree) const;
+
 private:
   llvm::Optional<Syntax> lookUpFrom(const Syntax &Node, size_t Position,
                                     SyntaxKind Kind);
diff --git a/include/swift/Syntax/RawSyntax.h b/include/swift/Syntax/RawSyntax.h
@@ -212,6 +212,8 @@ struct SyntaxPrintOptions {
   bool PrintTrivialNodeKind = false;
 };
 
+typedef unsigned SyntaxNodeId;
+
 /// RawSyntax - the strictly immutable, shared backing nodes for all syntax.
 ///
 /// This is implementation detail - do not expose it in public API.
@@ -223,10 +225,10 @@ class RawSyntax final
 
   /// The ID that shall be used for the next node that is created and does not
   /// have a manually specified id
-  static unsigned NextFreeNodeId;
+  static SyntaxNodeId NextFreeNodeId;
 
   /// An ID of this node that is stable across incremental parses
-  unsigned NodeId;
+  SyntaxNodeId NodeId;
 
   union {
     uint64_t OpaqueBits;
@@ -283,13 +285,13 @@ class RawSyntax final
   /// the caller needs to assure that the node ID has not been used yet.
   RawSyntax(SyntaxKind Kind, ArrayRef<RC<RawSyntax>> Layout,
             SourcePresence Presence, bool ManualMemory,
-            llvm::Optional<unsigned> NodeId);
+            llvm::Optional<SyntaxNodeId> NodeId);
   /// Constructor for creating token nodes
   /// If \p NodeId is \c None, the next free NodeId is used, if it is passed,
   /// the caller needs to assure that the NodeId has not been used yet.
   RawSyntax(tok TokKind, OwnedString Text, ArrayRef<TriviaPiece> LeadingTrivia,
             ArrayRef<TriviaPiece> TrailingTrivia, SourcePresence Presence,
-            bool ManualMemory, llvm::Optional<unsigned> NodeId);
+            bool ManualMemory, llvm::Optional<SyntaxNodeId> NodeId);
 
 public:
   ~RawSyntax();
@@ -312,15 +314,15 @@ class RawSyntax final
   static RC<RawSyntax> make(SyntaxKind Kind, ArrayRef<RC<RawSyntax>> Layout,
                             SourcePresence Presence,
                             SyntaxArena *Arena = nullptr,
-                            llvm::Optional<unsigned> NodeId = llvm::None);
+                            llvm::Optional<SyntaxNodeId> NodeId = llvm::None);
 
   /// Make a raw "token" syntax node.
   static RC<RawSyntax> make(tok TokKind, OwnedString Text,
                             ArrayRef<TriviaPiece> LeadingTrivia,
                             ArrayRef<TriviaPiece> TrailingTrivia,
                             SourcePresence Presence,
                             SyntaxArena *Arena = nullptr,
-                            llvm::Optional<unsigned> NodeId = llvm::None);
+                            llvm::Optional<SyntaxNodeId> NodeId = llvm::None);
 
   /// Make a missing raw "layout" syntax node.
   static RC<RawSyntax> missing(SyntaxKind Kind, SyntaxArena *Arena = nullptr) {
@@ -349,7 +351,7 @@ class RawSyntax final
   }
 
   /// Get an ID for this node that is stable across incremental parses
-  unsigned getId() const { return NodeId; }
+  SyntaxNodeId getId() const { return NodeId; }
 
   /// Returns true if the node is "missing" in the source (i.e. it was
   /// expected (or optional) but not written.
@@ -523,4 +525,8 @@ class RawSyntax final
 } // end namespace syntax
 } // end namespace swift
 
+namespace llvm {
+raw_ostream &operator<<(raw_ostream &OS, swift::syntax::AbsolutePosition Pos);
+} // end namespace llvm
+
 #endif // SWIFT_SYNTAX_RAWSYNTAX_H
diff --git a/include/swift/Syntax/Syntax.h b/include/swift/Syntax/Syntax.h
@@ -85,7 +85,7 @@ class Syntax {
   RC<RawSyntax> getRaw() const;
 
   /// Get an ID for this node that is stable across incremental parses
-  unsigned getId() const { return getRaw()->getId(); }
+  SyntaxNodeId getId() const { return getRaw()->getId(); }
 
   /// Get the number of child nodes in this piece of syntax, not including
   /// tokens.
diff --git a/include/swift/Syntax/SyntaxClassifier.h.gyb b/include/swift/Syntax/SyntaxClassifier.h.gyb
@@ -69,7 +69,7 @@ class SyntaxClassifier: public SyntaxVisitor {
         ForceClassification(ForceClassification) {}
   };
 
-  std::map<unsigned, SyntaxClassification> ClassifiedTokens;
+  std::map<SyntaxNodeId, SyntaxClassification> ClassifiedTokens;
   /// The top classification of this stack determines the color of identifiers
   std::stack<ContextStackEntry, llvm::SmallVector<ContextStackEntry, 16>> ContextStack;
 
@@ -101,7 +101,7 @@ class SyntaxClassifier: public SyntaxVisitor {
 % end
 
 public:
-  std::map<unsigned, SyntaxClassification> classify(Syntax Node) {
+  std::map<SyntaxNodeId, SyntaxClassification> classify(Syntax Node) {
     // Clean up the environment
     ContextStack = std::stack<ContextStackEntry, llvm::SmallVector<ContextStackEntry, 16>>();
     ContextStack.push({SyntaxClassification::None, false});
diff --git a/lib/Parse/SyntaxParsingCache.cpp b/lib/Parse/SyntaxParsingCache.cpp
@@ -11,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "swift/Parse/SyntaxParsingCache.h"
+#include "swift/Syntax/SyntaxVisitor.h"
 
 using namespace swift;
 using namespace swift::syntax;
@@ -84,10 +85,54 @@ llvm::Optional<Syntax> SyntaxParsingCache::lookUp(size_t NewPosition,
 
   auto Node = lookUpFrom(OldSyntaxTree, OldPosition, Kind);
   if (Node.hasValue()) {
-    if (RecordReuseInformation) {
-      ReusedRanges.push_back(
-          {NewPosition, NewPosition + Node->getTextLength()});
-    }
+    ReusedNodeIds.insert(Node->getId());
   }
   return Node;
 }
+
+std::vector<SyntaxReuseRegion>
+SyntaxParsingCache::getReusedRegions(const SourceFileSyntax &SyntaxTree) const {
+  /// Determines the reused source regions from reused syntax node IDs
+  class ReusedRegionsCollector : public SyntaxVisitor {
+    std::unordered_set<SyntaxNodeId> ReusedNodeIds;
+    std::vector<SyntaxReuseRegion> ReusedRegions;
+
+    bool didReuseNode(SyntaxNodeId NodeId) {
+      return ReusedNodeIds.count(NodeId) > 0;
+    }
+
+  public:
+    ReusedRegionsCollector(std::unordered_set<SyntaxNodeId> ReusedNodeIds)
+        : ReusedNodeIds(ReusedNodeIds) {}
+
+    const std::vector<SyntaxReuseRegion> &getReusedRegions() {
+      std::sort(ReusedRegions.begin(), ReusedRegions.end(),
+                [](const SyntaxReuseRegion &Lhs,
+                   const SyntaxReuseRegion &Rhs) -> bool {
+                  return Lhs.Start.getOffset() < Rhs.Start.getOffset();
+                });
+      return ReusedRegions;
+    }
+
+    void visit(Syntax Node) override {
+      if (didReuseNode(Node.getId())) {
+        // Node has been reused, add it to the list
+        auto Start = Node.getAbsolutePositionBeforeLeadingTrivia();
+        auto End = Node.getAbsoluteEndPositionAfterTrailingTrivia();
+        ReusedRegions.push_back({Start, End});
+      } else {
+        SyntaxVisitor::visit(Node);
+      }
+    }
+
+    void collectReusedRegions(SourceFileSyntax Node) {
+      assert(ReusedRegions.empty() &&
+             "ReusedRegionsCollector cannot be reused");
+      Node.accept(*this);
+    }
+  };
+
+  ReusedRegionsCollector ReuseRegionsCollector(getReusedNodeIds());
+  ReuseRegionsCollector.collectReusedRegions(SyntaxTree);
+  return ReuseRegionsCollector.getReusedRegions();
+}
diff --git a/lib/Syntax/RawSyntax.cpp b/lib/Syntax/RawSyntax.cpp
@@ -343,3 +343,8 @@ void RawSyntax::Profile(llvm::FoldingSetNodeID &ID, tok TokKind,
   for (auto &Piece : TrailingTrivia)
     Piece.Profile(ID);
 }
+
+llvm::raw_ostream &llvm::operator<<(raw_ostream &OS, AbsolutePosition Pos) {
+  Pos.printLineAndColumn(OS);
+  return OS;
+}
diff --git a/tools/SourceKit/lib/SwiftLang/SwiftEditor.cpp b/tools/SourceKit/lib/SwiftLang/SwiftEditor.cpp
@@ -562,12 +562,12 @@ struct SwiftSyntaxMap {
 class SyntaxToSyntaxMapConverter : public SyntaxVisitor {
   SwiftSyntaxMap &SyntaxMap;
 
-  std::map<unsigned, SyntaxClassification> TokenClassifications;
+  std::map<SyntaxNodeId, SyntaxClassification> TokenClassifications;
 
 public:
   SyntaxToSyntaxMapConverter(
       SwiftSyntaxMap &SyntaxMap,
-      std::map<unsigned, SyntaxClassification> TokenClassifications)
+      std::map<SyntaxNodeId, SyntaxClassification> TokenClassifications)
       : SyntaxMap(SyntaxMap), TokenClassifications(TokenClassifications) {}
 
 private:
@@ -2013,18 +2013,6 @@ SwiftEditorDocument::getSyntaxTree() const {
   return Impl.SyntaxTree;
 }
 
-const SourceManager &SwiftEditorDocument::getSourceManager() const {
-  return Impl.SyntaxInfo->getSourceManager();
-}
-
-SourceManager &SwiftEditorDocument::getSourceManager() {
-  return Impl.SyntaxInfo->getSourceManager();
-}
-
-unsigned SwiftEditorDocument::getBufferID() const {
-  return Impl.SyntaxInfo->getBufferID();
-}
-
 std::string SwiftEditorDocument::getFilePath() const { return Impl.FilePath; }
 
 bool SwiftEditorDocument::hasUpToDateAST() const {
@@ -2371,7 +2359,6 @@ void SwiftLangSupport::editorReplaceText(StringRef Name,
     if (EditorDoc->getSyntaxTree().hasValue()) {
       SyntaxCache.emplace(EditorDoc->getSyntaxTree().getValue());
       SyntaxCache->addEdit(Offset, Offset + Length, Buf->getBufferSize());
-      SyntaxCache->setRecordReuseInformation();
     }
 
     SyntaxParsingCache *SyntaxCachePtr = nullptr;
@@ -2386,37 +2373,35 @@ void SwiftLangSupport::editorReplaceText(StringRef Name,
       // Avoid computing the reused ranges if the consumer doesn't care about
       // them
       if (Consumer.syntaxReuseInfoEnabled()) {
-        auto ReuseRegions = SyntaxCache->getReusedRanges();
+        auto &SyntaxTree = EditorDoc->getSyntaxTree();
+        auto ReuseRegions = SyntaxCache->getReusedRegions(*SyntaxTree);
+
+        // Abstract away from SyntaxReuseRegions to std::pair<unsigned, unsigned>
+        // so that SourceKit doesn't have to import swiftParse
         std::vector<SourceFileRange> ReuseRegionOffsets;
         ReuseRegionOffsets.reserve(ReuseRegions.size());
         for (auto ReuseRegion : ReuseRegions) {
-          auto Start = ReuseRegion.Start;
-          auto End = ReuseRegion.End;
+          auto Start = ReuseRegion.Start.getOffset();
+          auto End = ReuseRegion.End.getOffset();
           ReuseRegionOffsets.push_back({Start, End});
         }
         Consumer.handleSyntaxReuseRegions(ReuseRegionOffsets);
       }
       if (LogReuseRegions) {
+        auto &SyntaxTree = EditorDoc->getSyntaxTree();
+        auto ReuseRegions = SyntaxCache->getReusedRegions(*SyntaxTree);
         LOG_SECTION("SyntaxCache", InfoHighPrio) {
           Log->getOS() << "Reused ";
 
           bool FirstIteration = true;
-          unsigned LastPrintedBufferID;
-          for (auto ReuseRegion : SyntaxCache->getReusedRanges()) {
+          for (auto ReuseRegion : ReuseRegions) {
             if (!FirstIteration) {
               Log->getOS() << ", ";
             } else {
               FirstIteration = false;
             }
 
-            const SourceManager &SM = EditorDoc->getSourceManager();
-            unsigned BufferID = EditorDoc->getBufferID();
-            auto Start = SM.getLocForOffset(BufferID, ReuseRegion.Start);
-            auto End = SM.getLocForOffset(BufferID, ReuseRegion.End);
-
-            Start.print(Log->getOS(), SM, LastPrintedBufferID);
-            Log->getOS() << " - ";
-            End.print(Log->getOS(), SM, LastPrintedBufferID);
+            Log->getOS() << ReuseRegion.Start << " - " << ReuseRegion.End;
           }
         }
       }
diff --git a/tools/SourceKit/lib/SwiftLang/SwiftLangSupport.h b/tools/SourceKit/lib/SwiftLang/SwiftLangSupport.h
@@ -110,12 +110,6 @@ class SwiftEditorDocument :
 
   const llvm::Optional<swift::SourceFileSyntax> &getSyntaxTree() const;
 
-  const swift::SourceManager &getSourceManager() const;
-  swift::SourceManager &getSourceManager();
-
-  /// Get the buffer ID of this file in its source manager
-  unsigned getBufferID() const;
-
   std::string getFilePath() const;
 
   /// Whether or not the AST stored for this document is up-to-date or just an
diff --git a/tools/swift-ide-test/swift-ide-test.cpp b/tools/swift-ide-test/swift-ide-test.cpp
@@ -960,7 +960,7 @@ class PrintSyntaxColorWalker : public ide::SyntaxModelWalker {
 class ColoredSyntaxTreePrinter : public SyntaxVisitor {
   llvm::raw_ostream &OS;
 
-  std::map<unsigned, SyntaxClassification> TokenClassifications;
+  std::map<SyntaxNodeId, SyntaxClassification> TokenClassifications;
 
   /// The name of the tag that is currently open
   StringRef CurrentTag;
@@ -972,7 +972,7 @@ class ColoredSyntaxTreePrinter : public SyntaxVisitor {
 public:
   ColoredSyntaxTreePrinter(
       llvm::raw_ostream &OS,
-      std::map<unsigned, SyntaxClassification> TokenClassifications)
+      std::map<SyntaxNodeId, SyntaxClassification> TokenClassifications)
       : OS(OS), TokenClassifications(TokenClassifications) {}
 
 private:
diff --git a/tools/swift-syntax-test/swift-syntax-test.cpp b/tools/swift-syntax-test/swift-syntax-test.cpp

Original file line number	Diff line number	Diff line change
`@@ -343,3 +343,8 @@ void RawSyntax::Profile(llvm::FoldingSetNodeID &ID, tok TokKind,`
`343`	`343`	`for (auto &Piece : TrailingTrivia)`
`344`	`344`	`Piece.Profile(ID);`
`345`	`345`	`}`
	`346`	`+`
	`347`	`+llvm::raw_ostream &llvm::operator<<(raw_ostream &OS, AbsolutePosition Pos) {`
	`348`	`+ Pos.printLineAndColumn(OS);`
	`349`	`+ return OS;`
	`350`	`+}`