[Parse] Lexer build backtick trivia around espaced identifier token

Author: omochi

include/swift/Parse/Lexer.h

@@ -496,6 +496,7 @@ class Lexer {
   }
 
   void formToken(tok Kind, const char *TokStart, bool MultilineString = false);
+  void formEscapedIdentifierToken(const char *TokStart);
 
   /// Advance to the end of the line but leave the current buffer pointer
   /// at that newline character.

lib/Parse/Lexer.cpp

@@ -34,6 +34,7 @@
 #include <limits>
 
 using namespace swift;
+using namespace swift::syntax;
 
 // clang::isIdentifierHead and clang::isIdentifierBody are deliberately not in
 // this list as a reminder that they are using C rules for identifiers.
@@ -269,6 +270,19 @@ void Lexer::formToken(tok Kind, const char *TokStart, bool MultilineString) {
   NextToken.setToken(Kind, TokenText, CommentLength, MultilineString);
 }
 
+void Lexer::formEscapedIdentifierToken(const char *TokStart) {
+  assert(CurPtr - TokStart >= 3 && "escaped identifier must be longer than or equal 3 bytes");
+  assert(TokStart[0] == '`' && "escaped identifier starts with backtick");
+  assert(CurPtr[-1] == '`' && "escaped identifier ends with backtick");
+  if (TriviaRetention == TriviaRetentionMode::WithTrivia) {
+    LeadingTrivia.push_back(TriviaPiece::backtick());
+    assert(TrailingTrivia.size() == 0 && "TrailingTrivia is empty here");
+    TrailingTrivia.push_back(TriviaPiece::backtick());
+  }
+  formToken(tok::identifier, TokStart);
+  NextToken.setEscapedIdentifier(true);
+}
+
 Lexer::State Lexer::getStateForBeginningOfTokenLoc(SourceLoc Loc) const {
   const char *Ptr = getBufferPtrForSourceLoc(Loc);
   // Skip whitespace backwards until we hit a newline.  This is needed to
@@ -1795,17 +1809,15 @@ void Lexer::lexEscapedIdentifier() {
     // If we have the terminating "`", it's an escaped identifier.
     if (*CurPtr == '`') {
       ++CurPtr;
-      formToken(tok::identifier, Quote);
-      NextToken.setEscapedIdentifier(true);
+      formEscapedIdentifierToken(Quote);
       return;
     }
   }
 
   // Special case; allow '`$`'.
   if (Quote[1] == '$' && Quote[2] == '`') {
     CurPtr = Quote + 3;
-    formToken(tok::identifier, Quote);
-    NextToken.setEscapedIdentifier(true);
+    formEscapedIdentifierToken(Quote);
     return;
   }
 

lib/Parse/Parser.cpp

@@ -286,10 +286,6 @@ swift::tokenizeWithTrivia(const LangOptions &LangOpts,
   Trivia LeadingTrivia, TrailingTrivia;
   do {
     L.lex(Tok, LeadingTrivia, TrailingTrivia);
-    if (Tok.isEscapedIdentifier()) {
-      LeadingTrivia.push_back(TriviaPiece::backtick());
-      TrailingTrivia.push_front(TriviaPiece::backtick());
-    }
     auto ThisToken = RawTokenSyntax::make(Tok.getKind(), Tok.getText(),
                                       SourcePresence::Present, LeadingTrivia,
                                       TrailingTrivia);

lib/Syntax/SyntaxParsingContext.cpp

@@ -77,10 +77,6 @@ void SyntaxParsingContext::addToken(Token &Tok, Trivia &LeadingTrivia,
   if (!Enabled)
     return;
 
-  if (Tok.isEscapedIdentifier()) {
-    LeadingTrivia.push_back(TriviaPiece::backtick());
-    TrailingTrivia.push_front(TriviaPiece::backtick());
-  }
   addRawSyntax(RawTokenSyntax::make(Tok.getKind(), Tok.getText(),
                                     SourcePresence::Present, LeadingTrivia,
                                     TrailingTrivia));