Merge pull request #16340 from ahoppen/incremental-syntax-parsing

[libSyntax] Incremental Syntax Parsing

Author: ahoppen

include/swift/AST/Module.h

@@ -28,6 +28,7 @@
 #include "swift/Basic/OptionSet.h"
 #include "swift/Basic/STLExtras.h"
 #include "swift/Basic/SourceLoc.h"
+#include "swift/Parse/SyntaxParsingCache.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/STLExtras.h"
@@ -838,6 +839,10 @@ class SourceFile final : public FileUnit {
   /// The list of top-level declarations in the source file.
   std::vector<Decl*> Decls;
 
+  /// A cache of syntax nodes that can be reused when creating the syntax tree
+  /// for this file.
+  SyntaxParsingCache *SyntaxParsingCache = nullptr;
+
   /// The list of local type declarations in the source file.
   llvm::SetVector<TypeDecl *> LocalTypeDecls;
 

include/swift/Basic/LangOptions.h

@@ -261,7 +261,9 @@ namespace swift {
     /// Whether collect tokens during parsing for syntax coloring.
     bool CollectParsedToken = false;
 
-    /// Whether to parse syntax tree.
+    /// Whether to parse syntax tree. If the syntax tree is built, the generated
+    /// AST may not be correct when syntax nodes are reused as part of
+    /// incrementals parsing.
     bool BuildSyntaxTree = false;
 
     /// Whether to verify the parsed syntax tree and emit related diagnostics.

include/swift/Basic/SourceLoc.h

@@ -68,7 +68,8 @@ class SourceLoc {
   void print(raw_ostream &OS, const SourceManager &SM,
              unsigned &LastBufferID) const;
 
-  void printLineAndColumn(raw_ostream &OS, const SourceManager &SM) const;
+  void printLineAndColumn(raw_ostream &OS, const SourceManager &SM,
+                          unsigned BufferID = 0) const;
 
   void print(raw_ostream &OS, const SourceManager &SM) const {
     unsigned Tmp = ~0U;

include/swift/Frontend/Frontend.h

@@ -34,6 +34,7 @@
 #include "swift/Migrator/MigratorOptions.h"
 #include "swift/Parse/CodeCompletionCallbacks.h"
 #include "swift/Parse/Parser.h"
+#include "swift/Parse/SyntaxParsingCache.h"
 #include "swift/Sema/SourceLoader.h"
 #include "swift/Serialization/Validation.h"
 #include "swift/Subsystems.h"
@@ -68,6 +69,9 @@ class CompilerInvocation {
   MigratorOptions MigratorOpts;
   SILOptions SILOpts;
   IRGenOptions IRGenOpts;
+  /// The \c SyntaxParsingCache to use when parsing the main file of this
+  /// invocation
+  SyntaxParsingCache *MainFileSyntaxParsingCache = nullptr;
 
   llvm::MemoryBuffer *CodeCompletionBuffer = nullptr;
 
@@ -217,6 +221,14 @@ class CompilerInvocation {
   IRGenOptions &getIRGenOptions() { return IRGenOpts; }
   const IRGenOptions &getIRGenOptions() const { return IRGenOpts; }
 
+  void setMainFileSyntaxParsingCache(SyntaxParsingCache *Cache) {
+    MainFileSyntaxParsingCache = Cache;
+  }
+
+  SyntaxParsingCache *getMainFileSyntaxParsingCache() const {
+    return MainFileSyntaxParsingCache;
+  }
+
   void setParseStdlib() {
     FrontendOpts.ParseStdlib = true;
   }

include/swift/Parse/Lexer.h

@@ -192,6 +192,15 @@ class Lexer {
     lex(Result, LeadingTrivia, TrailingTrivia);
   }
 
+  /// Reset the lexer's buffer pointer to \p Offset bytes after the buffer
+  /// start.
+  void resetToOffset(size_t Offset) {
+    assert(BufferStart + Offset <= BufferEnd && "Offset after buffer end");
+
+    CurPtr = BufferStart + Offset;
+    lexImpl();
+  }
+
   bool isKeepingComments() const {
     return RetainComments == CommentRetentionMode::ReturnAsTokens;
   }

include/swift/Parse/Parser.h

@@ -521,6 +521,13 @@ class Parser {
   /// \brief Skip until the next '#else', '#endif' or until eof.
   void skipUntilConditionalBlockClose();
 
+  /// If the parser is generating only a syntax tree, try loading the current
+  /// node from a previously generated syntax tree.
+  /// Returns \c true if the node has been loaded and inserted into the current
+  /// syntax tree. In this case the parser should behave as if the node has
+  /// successfully been created.
+  bool loadCurrentSyntaxNodeFromCache();
+
   /// Parse an #endif.
   bool parseEndIfDirective(SourceLoc &Loc);
 

include/swift/Parse/SyntaxParsingCache.h

@@ -0,0 +1,102 @@
+//===----------- SyntaxParsingCache.h -================----------*- C++ -*-===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2018 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SWIFT_PARSE_SYNTAXPARSINGCACHE_H
+#define SWIFT_PARSE_SYNTAXPARSINGCACHE_H
+
+#include "swift/Syntax/SyntaxNodes.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/raw_ostream.h"
+
+namespace {
+
+/// A single edit to the original source file in which a continuous range of
+/// characters have been replaced by a new string
+struct SourceEdit {
+  /// The byte offset from which on characters were replaced.
+  size_t Start;
+
+  /// The byte offset to which on characters were replaced.
+  size_t End;
+
+  /// The length of the string that replaced the range described above.
+  size_t ReplacementLength;
+
+  /// The length of the range that has been replaced
+  size_t originalLength() { return End - Start; }
+
+  /// Check if the characters replaced by this edit fall into the given range
+  /// or are directly adjacent to it
+  bool intersectsOrTouchesRange(size_t RangeStart, size_t RangeEnd) {
+    return !(End <= RangeStart || Start >= RangeEnd);
+  }
+};
+
+} // anonymous namespace
+
+namespace swift {
+
+using namespace swift::syntax;
+
+class SyntaxParsingCache {
+  /// The syntax tree prior to the edit
+  SourceFileSyntax OldSyntaxTree;
+
+  /// The edits that were made from the source file that created this cache to
+  /// the source file that is now parsed incrementally
+  llvm::SmallVector<SourceEdit, 4> Edits;
+
+  /// Whether or not information about reused nodes shall be recored in
+  /// \c ReusedRanges
+  bool RecordReuseInformation = false;
+
+  /// If \c RecordReuseInformation buffer offsets of ranges that have been
+  /// successfully looked up in this cache are stored.
+  std::vector<std::pair<unsigned, unsigned>> ReusedRanges;
+
+public:
+  SyntaxParsingCache(SourceFileSyntax OldSyntaxTree)
+      : OldSyntaxTree(OldSyntaxTree) {}
+
+  /// Add an edit that transformed the source file which created this cache into
+  /// the source file that is now being parsed incrementally. The order in which
+  /// the edits are added using this method needs to be the same order in which
+  /// the edits were applied to the source file.
+  void addEdit(size_t Start, size_t End, size_t ReplacementLength) {
+    Edits.push_back({Start, End, ReplacementLength});
+  }
+
+  /// Check if a syntax node of the given kind at the given position can be
+  /// reused for a new syntax tree.
+  llvm::Optional<Syntax> lookUp(size_t NewPosition, SyntaxKind Kind);
+
+  /// Turn recording of reused ranges on
+  void setRecordReuseInformation() { RecordReuseInformation = true; }
+
+  /// Return the ranges of the new source file that have been successfully
+  /// looked up in this cache as a (start, end) pair of byte offsets in the
+  /// post-edit file.
+  std::vector<std::pair<unsigned, unsigned>> getReusedRanges() const {
+    return ReusedRanges;
+  }
+
+private:
+  llvm::Optional<Syntax> lookUpFrom(const Syntax &Node, size_t Position,
+                                    SyntaxKind Kind);
+
+  bool nodeCanBeReused(const Syntax &Node, size_t Position,
+                       SyntaxKind Kind) const;
+};
+
+} // namespace swift
+
+#endif // SWIFT_SYNTAX_PARSING_CACHE_H

include/swift/Parse/SyntaxParsingContext.h

@@ -20,6 +20,7 @@
 
 namespace swift {
 class SourceFile;
+class SyntaxParsingCache;
 class Token;
 class DiagnosticEngine;
 
@@ -74,9 +75,17 @@ class alignas(1 << SyntaxAlignInBits) SyntaxParsingContext {
     // Storage for Collected parts.
     std::vector<RC<RawSyntax>> Storage;
 
+    SyntaxArena &Arena;
+
+    /// A cache of nodes that can be reused when creating the current syntax
+    /// tree
+    SyntaxParsingCache *SyntaxCache = nullptr;
+
     RootContextData(SourceFile &SF, DiagnosticEngine &Diags,
-                    SourceManager &SourceMgr, unsigned BufferID)
-        : SF(SF), Diags(Diags), SourceMgr(SourceMgr), BufferID(BufferID) {}
+                    SourceManager &SourceMgr, unsigned BufferID,
+                    SyntaxArena &Arena, SyntaxParsingCache *SyntaxCache)
+        : SF(SF), Diags(Diags), SourceMgr(SourceMgr), BufferID(BufferID),
+          Arena(Arena), SyntaxCache(SyntaxCache) {}
   };
 
 private:
@@ -97,6 +106,9 @@ class alignas(1 << SyntaxAlignInBits) SyntaxParsingContext {
     // Discard all parts in the context.
     Discard,
 
+    // The node has been loaded from the cache and all parts shall be discarded.
+    LoadedFromCache,
+
     // Construct SourceFile syntax to the specified SF.
     Root,
 
@@ -112,9 +124,7 @@ class alignas(1 << SyntaxAlignInBits) SyntaxParsingContext {
   // Reference to the
   SyntaxParsingContext *&CtxtHolder;
 
-  SyntaxArena &Arena;
-
-  std::vector<RC<RawSyntax>> &Storage;
+  RootContextData *RootData;
 
   // Offet for 'Storage' this context owns from.
   const size_t Offset;
@@ -138,7 +148,7 @@ class alignas(1 << SyntaxAlignInBits) SyntaxParsingContext {
   void createNodeInPlace(SyntaxKind Kind, size_t N);
 
   ArrayRef<RC<RawSyntax>> getParts() const {
-    return makeArrayRef(Storage).drop_front(Offset);
+    return makeArrayRef(getStorage()).drop_front(Offset);
   }
 
   RC<RawSyntax> makeUnknownSyntax(SyntaxKind Kind,
@@ -154,11 +164,12 @@ class alignas(1 << SyntaxAlignInBits) SyntaxParsingContext {
   /// Designated constructor for child context.
   SyntaxParsingContext(SyntaxParsingContext *&CtxtHolder)
       : RootDataOrParent(CtxtHolder), CtxtHolder(CtxtHolder),
-        Arena(CtxtHolder->Arena),
-        Storage(CtxtHolder->Storage), Offset(Storage.size()),
+        RootData(CtxtHolder->RootData), Offset(RootData->Storage.size()),
         Enabled(CtxtHolder->isEnabled()) {
     assert(CtxtHolder->isTopOfContextStack() &&
            "SyntaxParsingContext cannot have multiple children");
+    assert(CtxtHolder->Mode != AccumulationMode::LoadedFromCache &&
+           "Cannot create child context for a node loaded from the cache");
     CtxtHolder = this;
   }
 
@@ -174,20 +185,41 @@ class alignas(1 << SyntaxAlignInBits) SyntaxParsingContext {
 
   ~SyntaxParsingContext();
 
+  /// Try loading the current node from the \c SyntaxParsingCache by looking up
+  /// if an unmodified node exists at \p LexerOffset of the same kind. If a node
+  /// is found, replace the node that is currently being constructed by the
+  /// parsing context with the node from the cache and return the number of
+  /// bytes the loaded node took up in the original source. The lexer should
+  /// pretend it has read these bytes and continue from the advanced offset.
+  /// If nothing is found \c 0 is returned.
+  size_t loadFromCache(size_t LexerOffset);
+
   void disable() { Enabled = false; }
   bool isEnabled() const { return Enabled; }
   bool isRoot() const { return RootDataOrParent.is<RootContextData*>(); }
   bool isTopOfContextStack() const { return this == CtxtHolder; }
 
-  SyntaxParsingContext *getParent() {
+  SyntaxParsingContext *getParent() const {
     return RootDataOrParent.get<SyntaxParsingContext*>();
   }
 
-  RootContextData &getRootData() {
-    return *getRoot()->RootDataOrParent.get<RootContextData*>();
+  RootContextData *getRootData() { return RootData; }
+
+  const RootContextData *getRootData() const { return RootData; }
+
+  std::vector<RC<RawSyntax>> &getStorage() { return getRootData()->Storage; }
+
+  const std::vector<RC<RawSyntax>> &getStorage() const {
+    return getRootData()->Storage;
+  }
+
+  SyntaxParsingCache *getSyntaxParsingCache() const {
+    return getRootData()->SyntaxCache;
   }
 
-  SyntaxParsingContext *getRoot();
+  SyntaxArena &getArena() const { return getRootData()->Arena; }
+
+  const SyntaxParsingContext *getRoot() const;
 
   /// Add RawSyntax to the parts.
   void addRawSyntax(RC<RawSyntax> Raw);
@@ -201,6 +233,7 @@ class alignas(1 << SyntaxAlignInBits) SyntaxParsingContext {
 
   template<typename SyntaxNode>
   llvm::Optional<SyntaxNode> popIf() {
+    auto &Storage = getStorage();
     assert(Storage.size() > Offset);
     if (auto Node = make<Syntax>(Storage.back()).getAs<SyntaxNode>()) {
       Storage.pop_back();
@@ -210,6 +243,7 @@ class alignas(1 << SyntaxAlignInBits) SyntaxParsingContext {
   }
 
   TokenSyntax popToken() {
+    auto &Storage = getStorage();
     assert(Storage.size() > Offset);
     assert(Storage.back()->getKind() == SyntaxKind::Token);
     auto Node = make<TokenSyntax>(std::move(Storage.back()));
@@ -263,6 +297,10 @@ class alignas(1 << SyntaxAlignInBits) SyntaxParsingContext {
   /// Make a missing node corresponding to the given node kind, and
   /// push this node into the context.
   void synthesize(SyntaxKind Kind);
+
+  /// Dump the nodes that are in the storage stack of the SyntaxParsingContext
+  LLVM_ATTRIBUTE_DEPRECATED(void dumpStorage() const LLVM_ATTRIBUTE_USED,
+                            "Only meant for use in the debugger");
 };
 
 } // namespace swift