Merge pull request #13631 from omochi/lex-escape-backtick

[Parse] Lexer build backtick trivia around espaced identifier token

Author: rintaro

include/swift/Parse/Lexer.h

@@ -484,6 +484,7 @@ class Lexer {
   }
 
   void formToken(tok Kind, const char *TokStart, bool MultilineString = false);
+  void formEscapedIdentifierToken(const char *TokStart);
 
   /// Advance to the end of the line.
   /// If EatNewLine is true, CurPtr will be at end of newline character.

lib/Parse/Lexer.cpp

@@ -271,6 +271,19 @@ void Lexer::formToken(tok Kind, const char *TokStart, bool MultilineString) {
   NextToken.setToken(Kind, TokenText, CommentLength, MultilineString);
 }
 
+void Lexer::formEscapedIdentifierToken(const char *TokStart) {
+  assert(CurPtr - TokStart >= 3 && "escaped identifier must be longer than or equal 3 bytes");
+  assert(TokStart[0] == '`' && "escaped identifier starts with backtick");
+  assert(CurPtr[-1] == '`' && "escaped identifier ends with backtick");
+  if (TriviaRetention == TriviaRetentionMode::WithTrivia) {
+    LeadingTrivia.push_back(TriviaPiece::backtick());
+    assert(TrailingTrivia.size() == 0 && "TrailingTrivia is empty here");
+    TrailingTrivia.push_back(TriviaPiece::backtick());
+  }
+  formToken(tok::identifier, TokStart);
+  NextToken.setEscapedIdentifier(true);
+}
+
 Lexer::State Lexer::getStateForBeginningOfTokenLoc(SourceLoc Loc) const {
   const char *Ptr = getBufferPtrForSourceLoc(Loc);
   // Skip whitespace backwards until we hit a newline.  This is needed to
@@ -1770,17 +1783,15 @@ void Lexer::lexEscapedIdentifier() {
     // If we have the terminating "`", it's an escaped identifier.
     if (*CurPtr == '`') {
       ++CurPtr;
-      formToken(tok::identifier, Quote);
-      NextToken.setEscapedIdentifier(true);
+      formEscapedIdentifierToken(Quote);
       return;
     }
   }
 
   // Special case; allow '`$`'.
   if (Quote[1] == '$' && Quote[2] == '`') {
     CurPtr = Quote + 3;
-    formToken(tok::identifier, Quote);
-    NextToken.setEscapedIdentifier(true);
+    formEscapedIdentifierToken(Quote);
     return;
   }
 

lib/Parse/Parser.cpp

@@ -286,10 +286,6 @@ swift::tokenizeWithTrivia(const LangOptions &LangOpts,
   Trivia LeadingTrivia, TrailingTrivia;
   do {
     L.lex(Tok, LeadingTrivia, TrailingTrivia);
-    if (Tok.isEscapedIdentifier()) {
-      LeadingTrivia.push_back(TriviaPiece::backtick());
-      TrailingTrivia.push_front(TriviaPiece::backtick());
-    }
     auto ThisToken = RawTokenSyntax::make(Tok.getKind(), Tok.getText(),
                                       SourcePresence::Present, LeadingTrivia,
                                       TrailingTrivia);

lib/Syntax/SyntaxParsingContext.cpp

@@ -77,10 +77,6 @@ void SyntaxParsingContext::addToken(Token &Tok, Trivia &LeadingTrivia,
   if (!Enabled)
     return;
 
-  if (Tok.isEscapedIdentifier()) {
-    LeadingTrivia.push_back(TriviaPiece::backtick());
-    TrailingTrivia.push_front(TriviaPiece::backtick());
-  }
   addRawSyntax(RawTokenSyntax::make(Tok.getKind(), Tok.getText(),
                                     SourcePresence::Present, LeadingTrivia,
                                     TrailingTrivia));

test/Syntax/tokens_escaped_identifier.swift

@@ -0,0 +1,21 @@
+// RUN: %swift-syntax-test -input-source-filename %s -dump-full-tokens | %FileCheck %s
+let /*leading trivia*/ `if` = 3
+print(/*leading trivia*/ `if` )
+
+// CHECK-LABEL: 2:25
+// CHECK-NEXT:(token identifier
+// CHECK-NEXT: (trivia block_comment/*leading trivia*/)
+// CHECK-NEXT: (trivia space 1)
+// CHECK-NEXT: (trivia backtick 1)
+// CHECK-NEXT: (text="if")
+// CHECK-NEXT: (trivia backtick 1)
+// CHECK-NEXT: (trivia space 1))
+
+// CHECK-LABEL: 3:27
+// CHECK-NEXT:(token identifier
+// CHECK-NEXT: (trivia block_comment/*leading trivia*/)
+// CHECK-NEXT: (trivia space 1)
+// CHECK-NEXT: (trivia backtick 1)
+// CHECK-NEXT: (text="if")
+// CHECK-NEXT: (trivia backtick 1)
+// CHECK-NEXT: (trivia space 1))