[Parse] Optimize syntax parsing: Speed-up Lexer::lexTrivia()

Introduce ParsedTrivia which is a more efficient structure to use during lexing than syntax::Trivia.

Author: akyrtzi

include/swift/Parse/Lexer.h

@@ -22,8 +22,7 @@
 #include "swift/Basic/SourceManager.h"
 #include "swift/Parse/LexerState.h"
 #include "swift/Parse/Token.h"
-#include "swift/Syntax/References.h"
-#include "swift/Syntax/Trivia.h"
+#include "swift/Parse/ParsedTrivia.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Support/SaveAndRestore.h"
 
@@ -118,13 +117,13 @@ class Lexer {
   ///
   /// This is only preserved if this Lexer was constructed with
   /// `TriviaRetentionMode::WithTrivia`.
-  syntax::Trivia LeadingTrivia;
+  ParsedTrivia LeadingTrivia;
 
   /// The current trailing trivia for the next token.
   ///
   /// This is only preserved if this Lexer was constructed with
   /// `TriviaRetentionMode::WithTrivia`.
-  syntax::Trivia TrailingTrivia;
+  ParsedTrivia TrailingTrivia;
 
   Lexer(const Lexer&) = delete;
   void operator=(const Lexer&) = delete;
@@ -185,8 +184,8 @@ class Lexer {
 
   /// Lex a token. If \c TriviaRetentionMode is \c WithTrivia, passed pointers
   /// to trivias are populated.
-  void lex(Token &Result, syntax::Trivia &LeadingTriviaResult,
-           syntax::Trivia &TrailingTriviaResult) {
+  void lex(Token &Result, ParsedTrivia &LeadingTriviaResult,
+           ParsedTrivia &TrailingTriviaResult) {
     Result = NextToken;
     if (TriviaRetention == TriviaRetentionMode::WithTrivia) {
       LeadingTriviaResult = {LeadingTrivia};
@@ -197,7 +196,7 @@ class Lexer {
   }
 
   void lex(Token &Result) {
-    syntax::Trivia LeadingTrivia, TrailingTrivia;
+    ParsedTrivia LeadingTrivia, TrailingTrivia;
     lex(Result, LeadingTrivia, TrailingTrivia);
   }
 
@@ -229,7 +228,7 @@ class Lexer {
   /// After restoring the state, lexer will return this token and continue from
   /// there.
   State getStateForBeginningOfToken(const Token &Tok,
-                                    const syntax::Trivia &LeadingTrivia = {}) const {
+                                    const ParsedTrivia &LeadingTrivia = {}) const {
 
     // If the token has a comment attached to it, rewind to before the comment,
     // not just the start of the token.  This ensures that we will re-lex and
@@ -529,7 +528,7 @@ class Lexer {
   void lexOperatorIdentifier();
   void lexHexNumber();
   void lexNumber();
-  void lexTrivia(syntax::Trivia &T, bool IsForTrailingTrivia);
+  void lexTrivia(ParsedTrivia &T, bool IsForTrailingTrivia);
   static unsigned lexUnicodeEscape(const char *&CurPtr, Lexer *Diags);
 
   unsigned lexCharacter(const char *&CurPtr, char StopQuote,

include/swift/Parse/LexerState.h

@@ -19,7 +19,7 @@
 
 #include "llvm/ADT/Optional.h"
 #include "swift/Basic/SourceLoc.h"
-#include "swift/Syntax/Trivia.h"
+#include "swift/Parse/ParsedTrivia.h"
 
 namespace swift {
 class Lexer;
@@ -39,7 +39,7 @@ class LexerState {
 private:
   explicit LexerState(SourceLoc Loc) : Loc(Loc) {}
   SourceLoc Loc;
-  llvm::Optional<syntax::Trivia> LeadingTrivia;
+  llvm::Optional<ParsedTrivia> LeadingTrivia;
   friend class Lexer;
 };
 

include/swift/Parse/ParsedRawSyntaxNode.h

@@ -14,9 +14,9 @@
 #define SWIFT_PARSE_PARSEDRAWSYNTAXNODE_H
 
 #include "swift/Basic/SourceLoc.h"
+#include "swift/Parse/ParsedTrivia.h"
 #include "swift/Parse/Token.h"
 #include "swift/Syntax/SyntaxKind.h"
-#include "swift/Syntax/Trivia.h"
 #include <vector>
 
 namespace swift {
@@ -54,8 +54,8 @@ class ParsedRawSyntaxNode {
   };
   struct DeferredTokenNode {
     Token Tok;
-    syntax::Trivia LeadingTrivia;
-    syntax::Trivia TrailingTrivia;
+    ParsedTrivia LeadingTrivia;
+    ParsedTrivia TrailingTrivia;
   };
 
   union {
@@ -78,8 +78,8 @@ class ParsedRawSyntaxNode {
   }
 
   ParsedRawSyntaxNode(Token tok,
-                      syntax::Trivia leadingTrivia,
-                      syntax::Trivia trailingTrivia)
+                      ParsedTrivia leadingTrivia,
+                      ParsedTrivia trailingTrivia)
     : DeferredToken{std::move(tok),
                     std::move(leadingTrivia),
                     std::move(trailingTrivia)},
@@ -212,11 +212,11 @@ class ParsedRawSyntaxNode {
     assert(DK == DataKind::DeferredToken);
     return DeferredToken.Tok;
   }
-  const syntax::Trivia &getLeadingTrivia() const {
+  const ParsedTrivia &getLeadingTrivia() const {
     assert(DK == DataKind::DeferredToken);
     return DeferredToken.LeadingTrivia;
   }
-  const syntax::Trivia &getTrailingTrivia() const {
+  const ParsedTrivia &getTrailingTrivia() const {
     assert(DK == DataKind::DeferredToken);
     return DeferredToken.TrailingTrivia;
   }
@@ -231,8 +231,8 @@ class ParsedRawSyntaxNode {
 
   /// Form a deferred token node.
   static ParsedRawSyntaxNode makeDeferred(Token tok,
-                                          syntax::Trivia leadingTrivia,
-                                          syntax::Trivia trailingTrivia) {
+                                          ParsedTrivia leadingTrivia,
+                                          ParsedTrivia trailingTrivia) {
     return ParsedRawSyntaxNode{std::move(tok), std::move(leadingTrivia),
         std::move(trailingTrivia)};
   }

include/swift/Parse/ParsedRawSyntaxRecorder.h

@@ -25,14 +25,14 @@
 namespace swift {
 
 class ParsedRawSyntaxNode;
+struct ParsedTrivia;
 class SyntaxParseActions;
 class SourceLoc;
 class Token;
 enum class tok;
 
 namespace syntax {
   enum class SyntaxKind;
-  struct Trivia;
 }
 
 class ParsedRawSyntaxRecorder {
@@ -43,8 +43,8 @@ class ParsedRawSyntaxRecorder {
     : SPActions(std::move(spActions)) {}
 
   ParsedRawSyntaxNode recordToken(const Token &tok,
-                                  const syntax::Trivia &leadingTrivia,
-                                  const syntax::Trivia &trailingTrivia);
+                                  const ParsedTrivia &leadingTrivia,
+                                  const ParsedTrivia &trailingTrivia);
 
   /// Record a missing token. \p loc can be invalid or an approximate location
   /// of where the token would be if not missing.

include/swift/Parse/ParsedTrivia.h

@@ -0,0 +1,96 @@
+//===--- ParsedTrivia.h - ParsedTrivia API ----------------------*- C++ -*-===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2019 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SWIFT_PARSE_PARSEDTRIVIA_H
+#define SWIFT_PARSE_PARSEDTRIVIA_H
+
+#include "swift/Basic/LLVM.h"
+#include "llvm/ADT/SmallVector.h"
+
+namespace swift {
+  class SourceLoc;
+  class SourceManager;
+
+namespace syntax {
+  enum class TriviaKind;
+  struct Trivia;
+}
+
+class ParsedTriviaPiece {
+  syntax::TriviaKind Kind;
+  unsigned Length;
+
+public:
+  ParsedTriviaPiece(syntax::TriviaKind kind, unsigned length)
+      : Kind(kind), Length(length) {}
+
+  syntax::TriviaKind getKind() const { return Kind; }
+
+  /// Return the text of the trivia.
+  unsigned getLength() const { return Length; }
+
+  void extendLength(unsigned len) {
+    Length += len;
+  }
+};
+
+using ParsedTriviaList = SmallVector<ParsedTriviaPiece, 3>;
+
+struct ParsedTrivia {
+  ParsedTriviaList Pieces;
+
+  /// Get the begin iterator of the pieces.
+  ParsedTriviaList::const_iterator begin() const {
+    return Pieces.begin();
+  }
+
+  /// Get the end iterator of the pieces.
+  ParsedTriviaList::const_iterator end() const {
+    return Pieces.end();
+  }
+
+  /// Clear pieces.
+  void clear() {
+    Pieces.clear();
+  }
+
+  /// Returns true if there are no pieces in this Trivia collection.
+  bool empty() const {
+    return Pieces.empty();
+  }
+
+  size_t getLength() const {
+    size_t Len = 0;
+    for (auto &P : Pieces)
+      Len += P.getLength();
+    return Len;
+  }
+
+  void push_back(syntax::TriviaKind kind, unsigned length) {
+    Pieces.emplace_back(kind, length);
+  }
+
+  void appendOrSquash(syntax::TriviaKind kind, unsigned length) {
+    if (empty() || Pieces.back().getKind() != kind) {
+      push_back(kind, length);
+    } else {
+      Pieces.back().extendLength(length);
+    }
+  }
+
+  syntax::Trivia convertToSyntaxTrivia(SourceLoc loc, const SourceManager &SM,
+                                       unsigned bufferID) const;
+};
+
+} // end namespace swift
+
+#endif

include/swift/Parse/Parser.h

@@ -33,6 +33,7 @@
 #include "swift/Parse/ParserResult.h"
 #include "swift/Parse/SyntaxParserResult.h"
 #include "swift/Parse/SyntaxParsingContext.h"
+#include "swift/Syntax/References.h"
 #include "swift/Config.h"
 #include "llvm/ADT/SetVector.h"
 
@@ -200,11 +201,11 @@ class Parser {
 
   /// leading trivias for \c Tok.
   /// Always empty if !SF.shouldBuildSyntaxTree().
-  syntax::Trivia LeadingTrivia;
+  ParsedTrivia LeadingTrivia;
 
   /// trailing trivias for \c Tok.
   /// Always empty if !SF.shouldBuildSyntaxTree().
-  syntax::Trivia TrailingTrivia;
+  ParsedTrivia TrailingTrivia;
 
   /// Whether we should disable delayed parsing.
   bool DisableDelayedParsing;

include/swift/Parse/SyntaxParseActions.h

@@ -23,13 +23,13 @@
 namespace swift {
 
 class CharSourceRange;
+struct ParsedTrivia;
 class SourceLoc;
 class Token;
 enum class tok;
 
 namespace syntax {
   enum class SyntaxKind;
-  struct Trivia;
 }
 
 typedef void *OpaqueSyntaxNode;
@@ -41,8 +41,8 @@ class SyntaxParseActions {
   virtual ~SyntaxParseActions() = default;
 
   virtual OpaqueSyntaxNode recordToken(const Token &tok,
-                                       const syntax::Trivia &leadingTrivia,
-                                       const syntax::Trivia &trailingTrivia,
+                                       const ParsedTrivia &leadingTrivia,
+                                       const ParsedTrivia &trailingTrivia,
                                        CharSourceRange range) = 0;
 
   /// Record a missing token. \c loc can be invalid or an approximate location

include/swift/Parse/SyntaxParsingContext.h

@@ -24,6 +24,7 @@ using namespace swift::syntax;
 
 class ParsedSyntax;
 class ParsedTokenSyntax;
+struct ParsedTrivia;
 class SourceFile;
 enum class tok;
 class Token;
@@ -245,8 +246,8 @@ class alignas(1 << SyntaxAlignInBits) SyntaxParsingContext {
   void addRawSyntax(ParsedRawSyntaxNode Raw);
 
   /// Add Token with Trivia to the parts.
-  void addToken(Token &Tok, Trivia &LeadingTrivia,
-                Trivia &TrailingTrivia);
+  void addToken(Token &Tok, ParsedTrivia &LeadingTrivia,
+                ParsedTrivia &TrailingTrivia);
 
   /// Add Syntax to the parts.
   void addSyntax(ParsedSyntax Node);

include/swift/Parse/Token.h

@@ -248,6 +248,17 @@ class Token {
     return CharSourceRange(getLoc(), getLength());
   }
 
+  CharSourceRange getRangeWithoutBackticks() const {
+    SourceLoc TokLoc = getLoc();
+    unsigned TokLength = getLength();
+    if (isEscapedIdentifier()) {
+      // Adjust to account for the backticks.
+      TokLoc = TokLoc.getAdvancedLoc(1);
+      TokLength -= 2;
+    }
+    return CharSourceRange(TokLoc, TokLength);
+  }
+
   bool hasComment() const {
     return CommentLength != 0;
   }

include/swift/Syntax/Trivia.h.gyb

@@ -109,6 +109,8 @@ enum class TriviaKind {
 % end
 };
 
+bool isCommentTriviaKind(TriviaKind kind);
+
 /// A contiguous stretch of a single kind of trivia. The constituent part of
 /// a `Trivia` collection.
 ///
@@ -150,6 +152,8 @@ public:
 % end
 % end
 
+  static TriviaPiece fromText(TriviaKind kind, StringRef text);
+
   /// Return kind of the trivia.
   TriviaKind getKind() const { return Kind; }
 
@@ -174,7 +178,9 @@ public:
     llvm_unreachable("unhandled kind");
   }
 
-  bool isComment() const;
+  bool isComment() const {
+    return isCommentTriviaKind(getKind());
+  }
 
   void accumulateAbsolutePosition(AbsolutePosition &Pos) const;