[clang-format] Fix a bug in lexing C++ UDL ending in $

Author: owenca

clang/lib/Format/FormatTokenLexer.cpp

@@ -128,6 +128,12 @@ void FormatTokenLexer::tryMergePreviousTokens() {
   if (Style.isCpp() && tryTransformTryUsageForC())
     return;
 
+  if ((Style.Language == FormatStyle::LK_Cpp ||
+       Style.Language == FormatStyle::LK_ObjC) &&
+      tryMergeUserDefinedLiteral()) {
+    return;
+  }
+
   if (Style.isJavaScript() || Style.isCSharp()) {
     static const tok::TokenKind NullishCoalescingOperator[] = {tok::question,
                                                                tok::question};
@@ -559,6 +565,26 @@ bool FormatTokenLexer::tryMergeGreaterGreater() {
   return true;
 }
 
+bool FormatTokenLexer::tryMergeUserDefinedLiteral() {
+  if (Tokens.size() < 2)
+    return false;
+
+  auto *First = Tokens.end() - 2;
+  auto &Suffix = First[1];
+  if (Suffix->hasWhitespaceBefore() || Suffix->TokenText != "$")
+    return false;
+
+  auto &Literal = First[0];
+  auto &Text = Literal->TokenText;
+  if (!Literal->Tok.isLiteral() || !Text.ends_with("_"))
+    return false;
+
+  Text = StringRef(Text.data(), Text.size() + 1);
+  ++Literal->ColumnWidth;
+  Tokens.erase(&Suffix);
+  return true;
+}
+
 bool FormatTokenLexer::tryMergeTokens(ArrayRef<tok::TokenKind> Kinds,
                                       TokenType NewType) {
   if (Tokens.size() < Kinds.size())

clang/lib/Format/FormatTokenLexer.h

@@ -48,6 +48,7 @@ class FormatTokenLexer {
 
   bool tryMergeLessLess();
   bool tryMergeGreaterGreater();
+  bool tryMergeUserDefinedLiteral();
   bool tryMergeNSStringLiteral();
   bool tryMergeJSPrivateIdentifier();
   bool tryMergeCSharpStringLiteral();

clang/unittests/Format/TokenAnnotatorTest.cpp

@@ -3969,6 +3969,12 @@ TEST_F(TokenAnnotatorTest, IdentifierPackage) {
   EXPECT_FALSE(Tokens[0]->isObjCAccessSpecifier());
 }
 
+TEST_F(TokenAnnotatorTest, UserDefinedLiteral) {
+  auto Tokens = annotate("auto dollars = 2_$;");
+  ASSERT_EQ(Tokens.size(), 6u) << Tokens;
+  EXPECT_EQ(Tokens[3]->TokenText, "2_$");
+}
+
 } // namespace
 } // namespace format
 } // namespace clang