Merge pull request #19356 from rintaro/lexer-stringliteral

[Lexer] Tweaks for string literal lexing

Author: rintaro

include/swift/Parse/Lexer.h

@@ -506,9 +506,10 @@ class Lexer {
     return diagnose(Loc, Diagnostic(DiagID, std::forward<ArgTypes>(Args)...));
   }
 
-  void formToken(tok Kind, const char *TokStart, bool IsMultilineString = false,
-                 unsigned CustomDelimiterLen = 0);
+  void formToken(tok Kind, const char *TokStart);
   void formEscapedIdentifierToken(const char *TokStart);
+  void formStringLiteralToken(const char *TokStart, bool IsMultilineString,
+                              unsigned CustomDelimiterLen);
 
   /// Advance to the end of the line.
   /// If EatNewLine is true, CurPtr will be at end of newline character.

include/swift/Parse/Token.h

@@ -220,6 +220,21 @@ class Token {
     default: return false;
     }
   }
+
+  /// \brief True if the string literal token is multiline.
+  bool isMultilineString() const {
+    return MultilineString;
+  }
+  /// \brief Count of extending escaping '#'.
+  unsigned getCustomDelimiterLen() const {
+    return CustomDelimiterLen;
+  }
+  /// \brief Set characteristics of string literal token.
+  void setStringLiteral(bool IsMultilineString, unsigned CustomDelimiterLen) {
+    assert(Kind == tok::string_literal);
+    this->MultilineString = IsMultilineString;
+    this->CustomDelimiterLen = CustomDelimiterLen;
+  }
 
   /// getLoc - Return a source location identifier for the specified
   /// offset in the current file.
@@ -268,25 +283,16 @@ class Token {
   void setText(StringRef T) { Text = T; }
 
   /// \brief Set the token to the specified kind and source range.
-  void setToken(tok K, StringRef T, unsigned CommentLength = 0,
-                bool IsMultilineString = false, unsigned CustomDelimiterLen = 0) {
+  void setToken(tok K, StringRef T, unsigned CommentLength = 0) {
     Kind = K;
     Text = T;
     this->CommentLength = CommentLength;
     EscapedIdentifier = false;
-    this->MultilineString = IsMultilineString;
-    this->CustomDelimiterLen = CustomDelimiterLen;
+    this->MultilineString = false;
+    this->CustomDelimiterLen = 0;
     assert(this->CustomDelimiterLen == CustomDelimiterLen &&
            "custom string delimiter length > 255");
   }
-
-  bool isMultilineString() const {
-    return MultilineString;
-  }
-
-  unsigned getCustomDelimiterLen() const {
-    return CustomDelimiterLen;
-  }
 };
 
 } // end namespace swift