[SourceKit] Syntax parsing performance improvements

include/swift/Parse/SyntaxParsingCache.h

@@ -90,10 +90,10 @@ class SyntaxParsingCache {
   getReusedRegions(const SourceFileSyntax &SyntaxTree) const;
 
 private:
-  llvm::Optional<Syntax> lookUpFrom(const Syntax &Node, size_t Position,
-                                    SyntaxKind Kind);
+  llvm::Optional<Syntax> lookUpFrom(const Syntax &Node, size_t NodeStart,
+                                    size_t Position, SyntaxKind Kind);
 
-  bool nodeCanBeReused(const Syntax &Node, size_t Position,
+  bool nodeCanBeReused(const Syntax &Node, size_t Position, size_t NodeStart,
                        SyntaxKind Kind) const;
 };
 

lib/Parse/SyntaxParsingCache.cpp

@@ -16,9 +16,14 @@
 using namespace swift;
 using namespace swift::syntax;
 
-bool SyntaxParsingCache::nodeCanBeReused(const Syntax &Node, size_t Position,
+bool SyntaxParsingCache::nodeCanBeReused(const Syntax &Node, size_t NodeStart,
+                                         size_t Position,
                                          SyntaxKind Kind) const {
-  auto NodeStart = Node.getAbsolutePositionBeforeLeadingTrivia().getOffset();
+  // Computing the value of NodeStart on the fly is faster than determining a
+  // node's absolute position, but make sure the values match in an assertion
+  // build
+  assert(NodeStart == Node.getAbsolutePositionBeforeLeadingTrivia().getOffset());
+
   if (NodeStart != Position)
     return false;
   if (Node.getKind() != Kind)
@@ -50,23 +55,26 @@ bool SyntaxParsingCache::nodeCanBeReused(const Syntax &Node, size_t Position,
 }
 
 llvm::Optional<Syntax> SyntaxParsingCache::lookUpFrom(const Syntax &Node,
+                                                      size_t NodeStart,
                                                       size_t Position,
                                                       SyntaxKind Kind) {
-  if (nodeCanBeReused(Node, Position, Kind)) {
+  if (nodeCanBeReused(Node, NodeStart, Position, Kind)) {
     return Node;
   }
 
+  // Compute the child's position on the fly
+  size_t ChildStart = NodeStart;
   for (size_t I = 0, E = Node.getNumChildren(); I < E; ++I) {
     llvm::Optional<Syntax> Child = Node.getChild(I);
     if (!Child.hasValue()) {
       continue;
     }
-    auto ChildStart =
-        Child->getAbsolutePositionBeforeLeadingTrivia().getOffset();
     auto ChildEnd = ChildStart + Child->getTextLength();
     if (ChildStart <= Position && Position < ChildEnd) {
-      return lookUpFrom(Child.getValue(), Position, Kind);
+      return lookUpFrom(Child.getValue(), ChildStart, Position, Kind);
     }
+    // The next child starts where the previous child ended
+    ChildStart = ChildEnd;
   }
   return llvm::None;
 }
@@ -83,7 +91,7 @@ llvm::Optional<Syntax> SyntaxParsingCache::lookUp(size_t NewPosition,
     }
   }
 
-  auto Node = lookUpFrom(OldSyntaxTree, OldPosition, Kind);
+  auto Node = lookUpFrom(OldSyntaxTree, /*NodeStart=*/0, OldPosition, Kind);
   if (Node.hasValue()) {
     ReusedNodeIds.insert(Node->getId());
   }

tools/SourceKit/include/SourceKit/Core/LangSupport.h

@@ -265,7 +265,7 @@ class EditorConsumer {
 
   virtual void
   handleSyntaxTree(const swift::syntax::SourceFileSyntax &SyntaxTree,
-                   std::unordered_set<unsigned> ReusedNodeIds) = 0;
+                   std::unordered_set<unsigned> &ReusedNodeIds) = 0;
   virtual bool syntaxTreeEnabled() {
     return syntaxTreeTransferMode() != SyntaxTreeTransferMode::Off;
   }

tools/SourceKit/lib/SwiftLang/SwiftEditor.cpp

@@ -1796,12 +1796,6 @@ void SwiftEditorDocument::readSyntaxInfo(EditorConsumer &Consumer) {
 
   Impl.ParserDiagnostics = Impl.SyntaxInfo->getDiagnostics();
 
-  if (Consumer.syntaxTreeEnabled()) {
-    std::unordered_set<unsigned> ReusedNodeIds;
-    Consumer.handleSyntaxTree(Impl.SyntaxInfo->getSourceFile().getSyntaxRoot(),
-                              ReusedNodeIds);
-  }
-
   SwiftSyntaxMap NewMap = SwiftSyntaxMap(Impl.SyntaxMap.Tokens.size() + 16);
 
   if (Consumer.syntaxTreeEnabled()) {
@@ -2107,6 +2101,13 @@ void SwiftLangSupport::editorOpen(StringRef Name, llvm::MemoryBuffer *Buf,
 
   EditorDoc->readSyntaxInfo(Consumer);
   EditorDoc->readSemanticInfo(Snapshot, Consumer);
+
+  if (Consumer.syntaxTreeEnabled()) {
+    assert(EditorDoc->getSyntaxTree().hasValue());
+    std::unordered_set<unsigned> ReusedNodeIds;
+    Consumer.handleSyntaxTree(EditorDoc->getSyntaxTree().getValue(), 
+                              ReusedNodeIds);
+  }
 }
 
 

tools/SourceKit/tools/sourcekitd/lib/API/Requests.cpp

@@ -37,6 +37,7 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/NativeFormatting.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/PrettyStackTrace.h"
 #include "llvm/Support/raw_ostream.h"
@@ -2088,7 +2089,7 @@ class SKEditorConsumer : public EditorConsumer {
   void handleSourceText(StringRef Text) override;
 
   void handleSyntaxTree(const swift::syntax::SourceFileSyntax &SyntaxTree,
-                        std::unordered_set<unsigned> ReusedNodeIds) override;
+                        std::unordered_set<unsigned> &ReusedNodeIds) override;
 
   bool syntaxReuseInfoEnabled() override { return Opts.EnableSyntaxReuseInfo; }
   void
@@ -2419,22 +2420,44 @@ void SKEditorConsumer::handleSourceText(StringRef Text) {
   Dict.set(KeySourceText, Text);
 }
 
-void SKEditorConsumer::handleSyntaxTree(
+void serializeSyntaxTreeAsJson(
     const swift::syntax::SourceFileSyntax &SyntaxTree,
-    std::unordered_set<unsigned> ReusedNodeIds) {
-  if (Opts.SyntaxTransferMode == SyntaxTreeTransferMode::Off)
-    return;
-
+    std::unordered_set<unsigned> &ReusedNodeIds,
+    ResponseBuilder::Dictionary &Dict) {
+  auto StartClock = clock();
+  // 4096 is a heuristic buffer size that appears to usually be able to fit an
+  // incremental syntax tree
+  size_t ReserveBufferSize = 4096;
   std::string SyntaxTreeString;
+  SyntaxTreeString.reserve(ReserveBufferSize);
   {
     llvm::raw_string_ostream SyntaxTreeStream(SyntaxTreeString);
+    SyntaxTreeStream.SetBufferSize(ReserveBufferSize);
     swift::json::Output::UserInfoMap JsonUserInfo;
     JsonUserInfo[swift::json::OmitNodesUserInfoKey] = &ReusedNodeIds;
     swift::json::Output SyntaxTreeOutput(SyntaxTreeStream, JsonUserInfo,
                                          /*PrettyPrint=*/false);
     SyntaxTreeOutput << *SyntaxTree.getRaw();
   }
   Dict.set(KeySerializedSyntaxTree, SyntaxTreeString);
+
+  auto EndClock = clock();
+  LOG_SECTION("incrParse Performance", InfoLowPrio) {
+    Log->getOS() << "Serialized " << SyntaxTreeString.size() << " bytes in ";
+    llvm::write_double(Log->getOS(),
+                       (double)(EndClock - StartClock) * 1000 / CLOCKS_PER_SEC,
+                       llvm::FloatStyle::Fixed, 2);
+    Log->getOS() << "ms";
+  }
+}
+
+void SKEditorConsumer::handleSyntaxTree(
+    const swift::syntax::SourceFileSyntax &SyntaxTree,
+    std::unordered_set<unsigned> &ReusedNodeIds) {
+  if (Opts.SyntaxTransferMode == SyntaxTreeTransferMode::Off)
+    return;
+
+  serializeSyntaxTreeAsJson(SyntaxTree, ReusedNodeIds, Dict);
 }
 
 void SKEditorConsumer::handleSyntaxReuseRegions(

unittests/SourceKit/SwiftLang/CursorInfoTest.cpp

@@ -79,7 +79,7 @@ class NullEditorConsumer : public EditorConsumer {
 
   void handleSourceText(StringRef Text) override {}
   void handleSyntaxTree(const swift::syntax::SourceFileSyntax &SyntaxTree,
-                        std::unordered_set<unsigned> ReusedNodeIds) override {}
+                        std::unordered_set<unsigned> &ReusedNodeIds) override {}
 
   SyntaxTreeTransferMode syntaxTreeTransferMode() override {
     return SyntaxTreeTransferMode::Off;

unittests/SourceKit/SwiftLang/EditingTest.cpp

@@ -87,7 +87,7 @@ class DiagConsumer : public EditorConsumer {
 
   void handleSourceText(StringRef Text) override {}
   void handleSyntaxTree(const swift::syntax::SourceFileSyntax &SyntaxTree,
-                        std::unordered_set<unsigned> ReusedNodeIds) override {}
+                        std::unordered_set<unsigned> &ReusedNodeIds) override {}
 
   SyntaxTreeTransferMode syntaxTreeTransferMode() override {
     return SyntaxTreeTransferMode::Off;