Merge pull request #15152 from omochi/lex-remove-fields

[Parse] Remove unnecessary Lexer fields

Author: rintaro

include/swift/Parse/Lexer.h

@@ -92,22 +92,6 @@ class Lexer {
   /// Pointer to the next not consumed character.
   const char *CurPtr;
 
-  /// @{
-  /// Members that are *not* permanent lexer state.  The values only make sense
-  /// during the lexImpl() invocation.  These variables are declared as members
-  /// rather than locals so that we don't have to thread them through to all
-  /// lexing helpers.
-
-  /// Points to the point in the source buffer where we started scanning for
-  /// the current token.  Thus, the range [LastCommentBlockStart, CurPtr)
-  /// covers all comments and whitespace that we skipped, and the token itself.
-  const char *LastCommentBlockStart = nullptr;
-
-  /// True if we have seen a comment while scanning for the current token.
-  bool SeenComment = false;
-
-  /// @}
-
   Token NextToken;
 
   /// \brief This is true if we're lexing a .sil file instead of a .swift

include/swift/Syntax/Trivia.h.gyb

@@ -170,6 +170,8 @@ public:
     }
   }
 
+  bool isComment() const;
+
   void accumulateAbsolutePosition(AbsolutePosition &Pos) const;
 
   /// Try to compose this and Next to one TriviaPiece.

lib/Parse/Lexer.cpp

@@ -278,8 +278,14 @@ void Lexer::formToken(tok Kind, const char *TokStart, bool MultilineString) {
     Kind = tok::eof;
   }
   unsigned CommentLength = 0;
-  if (RetainComments == CommentRetentionMode::AttachToNextToken && SeenComment)
-    CommentLength = TokStart - LastCommentBlockStart;
+  if (RetainComments == CommentRetentionMode::AttachToNextToken) {
+    auto Iter = llvm::find_if(LeadingTrivia, [](const TriviaPiece &Piece) {
+      return Piece.isComment();
+    });
+    for (auto End = LeadingTrivia.end(); Iter != End; Iter++) {
+      CommentLength += Iter->getTextLength();
+    }
+  }
 
   StringRef TokenText { TokStart, static_cast<size_t>(CurPtr - TokStart) };
 
@@ -2160,10 +2166,6 @@ void Lexer::lexImpl() {
     NextToken.setAtStartOfLine(false);
   }
 
-  // Remember where we started so that we can find the comment range.
-  LastCommentBlockStart = CurPtr;
-  SeenComment = false;
-
   lexTrivia(LeadingTrivia, /* IsForTrailingTrivia */ false);
 
   // Remember the start of the token so we can form the text range.
@@ -2253,14 +2255,12 @@ void Lexer::lexImpl() {
   case '/':
     if (CurPtr[0] == '/') {  // "//"
       skipSlashSlashComment(/*EatNewline=*/true);
-      SeenComment = true;
       assert(isKeepingComments() &&
              "Non token comment should be eaten by lexTrivia as LeadingTrivia");
       return formToken(tok::comment, TokStart);
     }
     if (CurPtr[0] == '*') { // "/*"
       skipSlashStarComment();
-      SeenComment = true;
       assert(isKeepingComments() &&
              "Non token comment should be eaten by lexTrivia as LeadingTrivia");
       return formToken(tok::comment, TokStart);
@@ -2393,7 +2393,6 @@ void Lexer::lexTrivia(syntax::Trivia &Pieces, bool IsForTrailingTrivia) {
       break;
     } else if (*CurPtr == '/') {
       // '// ...' comment.
-      SeenComment = true;
       bool isDocComment = CurPtr[1] == '/';
       skipSlashSlashComment(/*EatNewline=*/false);
       size_t Length = CurPtr - TriviaStart;
@@ -2403,7 +2402,6 @@ void Lexer::lexTrivia(syntax::Trivia &Pieces, bool IsForTrailingTrivia) {
       goto Restart;
     } else if (*CurPtr == '*') {
       // '/* ... */' comment.
-      SeenComment = true;
       bool isDocComment = CurPtr[1] == '*';
       skipSlashStarComment();
       size_t Length = CurPtr - TriviaStart;

lib/Syntax/Trivia.cpp.gyb

@@ -44,6 +44,19 @@ void TriviaPiece::dump(llvm::raw_ostream &OS, unsigned Indent) const {
   OS << ')';
 }
 
+bool TriviaPiece::isComment() const {
+  switch (Kind) {
+% for trivia in TRIVIAS:
+  case TriviaKind::${trivia.name}:
+% if trivia.is_comment:
+    return true;
+% else:
+    return false;
+% end
+% end
+  }
+}
+
 void TriviaPiece::accumulateAbsolutePosition(AbsolutePosition &Pos) const {
   switch (Kind) {
 % for trivia in TRIVIAS:

utils/gyb_syntax_support/Trivia.py

@@ -3,12 +3,13 @@
 
 class Trivia(object):
     def __init__(self, name, comment, characters=[], swift_characters=[],
-                 is_new_line=False):
+                 is_new_line=False, is_comment=False):
         self.name = name
         self.comment = comment
         self.characters = characters
         self.lower_name = lowercase_first_word(name)
         self.is_new_line = is_new_line
+        self.is_comment = is_comment
 
         # Swift sometimes doesn't support escaped characters like \f or \v;
         # we should allow specifying alternatives explicitly.
@@ -44,14 +45,18 @@ def is_collection(self):
            'A backtick \'`\' character, used to escape identifiers.',
            characters=['`']),
 
-    Trivia('LineComment', 'A developer line comment, starting with \'//\''),
+    Trivia('LineComment', 'A developer line comment, starting with \'//\'',
+           is_comment=True),
     Trivia('BlockComment',
            'A developer block comment, starting with \'/*\' and ending with'
-           ' \'*/\'.'),
+           ' \'*/\'.',
+           is_comment=True),
     Trivia('DocLineComment',
-           'A documentation line comment, starting with \'///\'.'),
+           'A documentation line comment, starting with \'///\'.',
+           is_comment=True),
     Trivia('DocBlockComment',
            'A documentation block comment, starting with \'/**\' and ending '
-           'with \'*/\'.'),
+           'with \'*/\'.',
+           is_comment=True),
     Trivia('GarbageText', 'Any skipped garbage text.'),
 ]