Merge pull request #21955 from akyrtzi/syntax-parser-opt

[Parser] Optimize syntactic parsing for libSyntax

Author: akyrtzi

include/swift/Parse/Lexer.h

@@ -22,8 +22,7 @@
 #include "swift/Basic/SourceManager.h"
 #include "swift/Parse/LexerState.h"
 #include "swift/Parse/Token.h"
-#include "swift/Syntax/References.h"
-#include "swift/Syntax/Trivia.h"
+#include "swift/Parse/ParsedTrivia.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Support/SaveAndRestore.h"
 
@@ -118,13 +117,13 @@ class Lexer {
   ///
   /// This is only preserved if this Lexer was constructed with
   /// `TriviaRetentionMode::WithTrivia`.
-  syntax::Trivia LeadingTrivia;
+  ParsedTrivia LeadingTrivia;
 
   /// The current trailing trivia for the next token.
   ///
   /// This is only preserved if this Lexer was constructed with
   /// `TriviaRetentionMode::WithTrivia`.
-  syntax::Trivia TrailingTrivia;
+  ParsedTrivia TrailingTrivia;
 
   Lexer(const Lexer&) = delete;
   void operator=(const Lexer&) = delete;
@@ -185,8 +184,8 @@ class Lexer {
 
   /// Lex a token. If \c TriviaRetentionMode is \c WithTrivia, passed pointers
   /// to trivias are populated.
-  void lex(Token &Result, syntax::Trivia &LeadingTriviaResult,
-           syntax::Trivia &TrailingTriviaResult) {
+  void lex(Token &Result, ParsedTrivia &LeadingTriviaResult,
+           ParsedTrivia &TrailingTriviaResult) {
     Result = NextToken;
     if (TriviaRetention == TriviaRetentionMode::WithTrivia) {
       LeadingTriviaResult = {LeadingTrivia};
@@ -197,7 +196,7 @@ class Lexer {
   }
 
   void lex(Token &Result) {
-    syntax::Trivia LeadingTrivia, TrailingTrivia;
+    ParsedTrivia LeadingTrivia, TrailingTrivia;
     lex(Result, LeadingTrivia, TrailingTrivia);
   }
 
@@ -229,7 +228,7 @@ class Lexer {
   /// After restoring the state, lexer will return this token and continue from
   /// there.
   State getStateForBeginningOfToken(const Token &Tok,
-                                    const syntax::Trivia &LeadingTrivia = {}) const {
+                                    const ParsedTrivia &LeadingTrivia = {}) const {
 
     // If the token has a comment attached to it, rewind to before the comment,
     // not just the start of the token.  This ensures that we will re-lex and
@@ -529,7 +528,7 @@ class Lexer {
   void lexOperatorIdentifier();
   void lexHexNumber();
   void lexNumber();
-  void lexTrivia(syntax::Trivia &T, bool IsForTrailingTrivia);
+  void lexTrivia(ParsedTrivia &T, bool IsForTrailingTrivia);
   static unsigned lexUnicodeEscape(const char *&CurPtr, Lexer *Diags);
 
   unsigned lexCharacter(const char *&CurPtr, char StopQuote,

include/swift/Parse/LexerState.h

@@ -19,7 +19,7 @@
 
 #include "llvm/ADT/Optional.h"
 #include "swift/Basic/SourceLoc.h"
-#include "swift/Syntax/Trivia.h"
+#include "swift/Parse/ParsedTrivia.h"
 
 namespace swift {
 class Lexer;
@@ -39,7 +39,7 @@ class LexerState {
 private:
   explicit LexerState(SourceLoc Loc) : Loc(Loc) {}
   SourceLoc Loc;
-  llvm::Optional<syntax::Trivia> LeadingTrivia;
+  llvm::Optional<ParsedTrivia> LeadingTrivia;
   friend class Lexer;
 };
 

include/swift/Parse/ParsedRawSyntaxNode.h

@@ -14,14 +14,15 @@
 #define SWIFT_PARSE_PARSEDRAWSYNTAXNODE_H
 
 #include "swift/Basic/SourceLoc.h"
+#include "swift/Parse/ParsedTrivia.h"
 #include "swift/Parse/Token.h"
 #include "swift/Syntax/SyntaxKind.h"
-#include "swift/Syntax/Trivia.h"
 #include <vector>
 
 namespace swift {
 
 typedef void *OpaqueSyntaxNode;
+class SyntaxParsingContext;
 
 /// Represents a raw syntax node formed by the parser.
 ///
@@ -50,12 +51,14 @@ class ParsedRawSyntaxNode {
     CharSourceRange Range;
   };
   struct DeferredLayoutNode {
-    std::vector<ParsedRawSyntaxNode> Children;
+    ArrayRef<ParsedRawSyntaxNode> Children;
   };
   struct DeferredTokenNode {
-    Token Tok;
-    syntax::Trivia LeadingTrivia;
-    syntax::Trivia TrailingTrivia;
+    const ParsedTriviaPiece *TriviaPieces;
+    SourceLoc TokLoc;
+    unsigned TokLength;
+    uint16_t NumLeadingTrivia;
+    uint16_t NumTrailingTrivia;
   };
 
   union {
@@ -77,16 +80,22 @@ class ParsedRawSyntaxNode {
     assert(getKind() == k && "Syntax kind with too large value!");
   }
 
-  ParsedRawSyntaxNode(Token tok,
-                      syntax::Trivia leadingTrivia,
-                      syntax::Trivia trailingTrivia)
-    : DeferredToken{std::move(tok),
-                    std::move(leadingTrivia),
-                    std::move(trailingTrivia)},
+  ParsedRawSyntaxNode(tok tokKind, SourceLoc tokLoc, unsigned tokLength,
+                      const ParsedTriviaPiece *triviaPieces,
+                      unsigned numLeadingTrivia,
+                      unsigned numTrailingTrivia)
+    : DeferredToken{triviaPieces,
+                    tokLoc, tokLength,
+                    uint16_t(numLeadingTrivia),
+                    uint16_t(numTrailingTrivia)},
       SynKind(uint16_t(syntax::SyntaxKind::Token)),
-      TokKind(uint16_t(tok.getKind())),
+      TokKind(uint16_t(tokKind)),
       DK(DataKind::DeferredToken) {
-    assert(getTokenKind() == tok.getKind() && "Token kind is too large value!");
+    assert(getTokenKind() == tokKind && "Token kind is too large value!");
+    assert(DeferredToken.NumLeadingTrivia == numLeadingTrivia &&
+           "numLeadingTrivia is too large value!");
+    assert(DeferredToken.NumTrailingTrivia == numTrailingTrivia &&
+           "numLeadingTrivia is too large value!");
   }
 
 public:
@@ -106,63 +115,6 @@ class ParsedRawSyntaxNode {
     assert(getTokenKind() == tokKind && "Token kind with too large value!");
   }
 
-  ParsedRawSyntaxNode(const ParsedRawSyntaxNode &other) {
-    switch (other.DK) {
-    case DataKind::Null:
-      break;
-    case DataKind::Recorded:
-      new(&this->RecordedData)RecordedSyntaxNode(other.RecordedData);
-      break;
-    case DataKind::DeferredLayout:
-      new(&this->DeferredLayout)DeferredLayoutNode(other.DeferredLayout);
-      break;
-    case DataKind::DeferredToken:
-      new(&this->DeferredToken)DeferredTokenNode(other.DeferredToken);
-      break;
-    }
-    this->SynKind = other.SynKind;
-    this->TokKind = other.TokKind;
-    this->DK = other.DK;
-  }
-
-  ParsedRawSyntaxNode(ParsedRawSyntaxNode &&other) {
-    switch (other.DK) {
-    case DataKind::Null:
-      break;
-    case DataKind::Recorded:
-      new(&this->RecordedData)RecordedSyntaxNode(
-          std::move(other.RecordedData));
-      break;
-    case DataKind::DeferredLayout:
-      new(&this->DeferredLayout)DeferredLayoutNode(
-          std::move(other.DeferredLayout));
-      break;
-    case DataKind::DeferredToken:
-      new(&this->DeferredToken)DeferredTokenNode(
-          std::move(other.DeferredToken));
-      break;
-    }
-    this->SynKind = other.SynKind;
-    this->TokKind = other.TokKind;
-    this->DK = other.DK;
-  }
-
-  ~ParsedRawSyntaxNode() {
-    releaseMemory();
-  }
-
-  ParsedRawSyntaxNode &operator=(const ParsedRawSyntaxNode &other) {
-    releaseMemory();
-    new (this)ParsedRawSyntaxNode(other);
-    return *this;
-  }
-
-  ParsedRawSyntaxNode &operator=(ParsedRawSyntaxNode &&other) {
-    releaseMemory();
-    new (this)ParsedRawSyntaxNode(std::move(other));
-    return *this;
-  }
-
   syntax::SyntaxKind getKind() const { return syntax::SyntaxKind(SynKind); }
   tok getTokenKind() const { return tok(TokKind); }
 
@@ -201,44 +153,44 @@ class ParsedRawSyntaxNode {
     assert(DK == DataKind::DeferredLayout);
     return DeferredLayout.Children;
   }
-  void addDeferredChild(ParsedRawSyntaxNode subnode) {
-    assert(DK == DataKind::DeferredLayout);
-    DeferredLayout.Children.push_back(std::move(subnode));
-  }
 
   // Deferred Token Data =====================================================//
 
-  const Token &getToken() const {
+  CharSourceRange getDeferredTokenRangeWithoutBackticks() const {
     assert(DK == DataKind::DeferredToken);
-    return DeferredToken.Tok;
+    return CharSourceRange{DeferredToken.TokLoc, DeferredToken.TokLength};
   }
-  const syntax::Trivia &getLeadingTrivia() const {
+  ArrayRef<ParsedTriviaPiece> getDeferredLeadingTriviaPieces() const {
     assert(DK == DataKind::DeferredToken);
-    return DeferredToken.LeadingTrivia;
+    return ArrayRef<ParsedTriviaPiece>(DeferredToken.TriviaPieces,
+                                       DeferredToken.NumLeadingTrivia);
   }
-  const syntax::Trivia &getTrailingTrivia() const {
+  ArrayRef<ParsedTriviaPiece> getDeferredTrailingTriviaPieces() const {
     assert(DK == DataKind::DeferredToken);
-    return DeferredToken.TrailingTrivia;
+    return ArrayRef<ParsedTriviaPiece>(
+      DeferredToken.TriviaPieces + DeferredToken.NumLeadingTrivia,
+      DeferredToken.NumTrailingTrivia);
   }
 
   //==========================================================================//
 
   /// Form a deferred syntax layout node.
   static ParsedRawSyntaxNode makeDeferred(syntax::SyntaxKind k,
-                        ArrayRef<ParsedRawSyntaxNode> deferredNodes) {
-    return ParsedRawSyntaxNode{k, deferredNodes};
-  }
+                        ArrayRef<ParsedRawSyntaxNode> deferredNodes,
+                                          SyntaxParsingContext &ctx);
 
   /// Form a deferred token node.
   static ParsedRawSyntaxNode makeDeferred(Token tok,
-                                          syntax::Trivia leadingTrivia,
-                                          syntax::Trivia trailingTrivia) {
-    return ParsedRawSyntaxNode{std::move(tok), std::move(leadingTrivia),
-        std::move(trailingTrivia)};
-  }
+                                          const ParsedTrivia &leadingTrivia,
+                                          const ParsedTrivia &trailingTrivia,
+                                          SyntaxParsingContext &ctx);
 
   /// Form a deferred missing token node.
-  static ParsedRawSyntaxNode makeDeferredMissing(tok tokKind, SourceLoc loc);
+  static ParsedRawSyntaxNode makeDeferredMissing(tok tokKind, SourceLoc loc) {
+    auto raw = ParsedRawSyntaxNode(tokKind, loc, 0, nullptr, 0, 0);
+    raw.IsMissing = true;
+    return raw;
+  }
 
   /// Dump this piece of syntax recursively for debugging or testing.
   LLVM_ATTRIBUTE_DEPRECATED(
@@ -251,20 +203,6 @@ class ParsedRawSyntaxNode {
   static ParsedRawSyntaxNode null() {
     return ParsedRawSyntaxNode{};
   }
-
-private:
-  void releaseMemory() {
-    switch (DK) {
-    case DataKind::Null:
-      break;
-    case DataKind::Recorded:
-      RecordedData.~RecordedSyntaxNode(); break;
-    case DataKind::DeferredLayout:
-      DeferredLayout.~DeferredLayoutNode(); break;
-    case DataKind::DeferredToken:
-      DeferredToken.~DeferredTokenNode(); break;
-    }
-  }
 };
 
 } // end namespace swift

include/swift/Parse/ParsedRawSyntaxRecorder.h

@@ -24,15 +24,17 @@
 
 namespace swift {
 
+class CharSourceRange;
 class ParsedRawSyntaxNode;
+struct ParsedTrivia;
+class ParsedTriviaPiece;
 class SyntaxParseActions;
 class SourceLoc;
 class Token;
 enum class tok;
 
 namespace syntax {
   enum class SyntaxKind;
-  struct Trivia;
 }
 
 class ParsedRawSyntaxRecorder {
@@ -43,8 +45,12 @@ class ParsedRawSyntaxRecorder {
     : SPActions(std::move(spActions)) {}
 
   ParsedRawSyntaxNode recordToken(const Token &tok,
-                                  const syntax::Trivia &leadingTrivia,
-                                  const syntax::Trivia &trailingTrivia);
+                                  const ParsedTrivia &leadingTrivia,
+                                  const ParsedTrivia &trailingTrivia);
+
+  ParsedRawSyntaxNode recordToken(tok tokenKind, CharSourceRange tokenRange,
+                                  ArrayRef<ParsedTriviaPiece> leadingTrivia,
+                                  ArrayRef<ParsedTriviaPiece> trailingTrivia);
 
   /// Record a missing token. \p loc can be invalid or an approximate location
   /// of where the token would be if not missing.

include/swift/Parse/ParsedSyntaxBuilders.h.gyb

@@ -34,17 +34,20 @@ class SyntaxParsingContext;
 %   if node.is_buildable():
 %     child_count = len(node.children)
 class Parsed${node.name}Builder {
-  ParsedRawSyntaxRecorder &Rec;
-  bool IsBacktracking;
-  ParsedRawSyntaxNode Layout[${child_count}] = {
+  SyntaxParsingContext &SPCtx;
+  ParsedRawSyntaxNode Layout[${child_count}];
 %     for child in node.children:
-    ParsedRawSyntaxNode::null(),
+%       child_node = NODE_MAP.get(child.syntax_kind)
+%       if child_node and child_node.is_syntax_collection():
+%         child_elt = child_node.collection_element_name
+%         child_elt_type = child_node.collection_element_type
+  SmallVector<ParsedRawSyntaxNode, 8> ${child_elt}Nodes;
+%       end
 %     end
-  };
 
 public:
   explicit Parsed${node.name}Builder(SyntaxParsingContext &SPCtx)
-    : Rec(SPCtx.getRecorder()), IsBacktracking(SPCtx.isBacktracking()) {}
+    : SPCtx(SPCtx) {}
 
 %     for child in node.children:
   Parsed${node.name}Builder &use${child.name}(Parsed${child.type_name} ${child.name});
@@ -61,7 +64,7 @@ public:
 
 private:
   Parsed${node.name} record();
-  void finishLayout();
+  void finishLayout(bool deferred);
 };
 
 %   end

include/swift/Parse/ParsedSyntaxRecorder.h.gyb

@@ -46,7 +46,8 @@ private:
   static Parsed${node.name} record${node.syntax_kind}(${child_params},
                                               ParsedRawSyntaxRecorder &rec);
 public:
-  static Parsed${node.name} defer${node.syntax_kind}(${child_params});
+  static Parsed${node.name} defer${node.syntax_kind}(${child_params},
+                                              SyntaxParsingContext &SPCtx);
   static Parsed${node.name} make${node.syntax_kind}(${child_params},
                                               SyntaxParsingContext &SPCtx);
 %   elif node.is_syntax_collection():
@@ -57,7 +58,8 @@ private:
 
 public:
   static Parsed${node.name} defer${node.syntax_kind}(
-      ArrayRef<Parsed${node.collection_element_type}> elts);
+      ArrayRef<Parsed${node.collection_element_type}> elts,
+      SyntaxParsingContext &SPCtx);
   static Parsed${node.name} make${node.syntax_kind}(
       ArrayRef<Parsed${node.collection_element_type}> elts,
       SyntaxParsingContext &SPCtx);