[Syntax] Introduce TOKEN macro in TokenKinds.def

Author: rintaro

include/swift/Syntax/Serialization/SyntaxSerialization.h

@@ -40,13 +40,8 @@ struct ScalarEnumerationTraits<syntax::SourcePresence> {
 template <>
 struct ScalarEnumerationTraits<tok> {
   static void enumeration(Output &out, tok &value) {
-#define EXPAND(Str, Case) \
-    out.enumCase(value, Str, Case);
-#define LITERAL(X) EXPAND(#X, tok::X)
-#define MISC(X) EXPAND(#X, tok::X)
-#define KEYWORD(X) EXPAND("kw_" #X, tok::kw_##X)
-#define PUNCTUATOR(X, Y) EXPAND(#X, tok::X)
-#define POUND_KEYWORD(X) EXPAND("pound_" #X, tok::pound_##X)
+#define TOKEN(name) \
+    out.enumCase(value, #name, tok::name);
 #include "swift/Syntax/TokenKinds.def"
   }
 };

include/swift/Syntax/TokenKinds.def

@@ -12,29 +12,36 @@
 ///
 /// This file defines x-macros used for metaprogramming with lexer tokens.
 ///
-/// KEYWORD(kw)
-///   SWIFT_KEYWORD(kw)
-///     DECL_KEYWORD(kw)
-///     STMT_KEYWORD(kw)
-///     EXPR_KEYWORD(kw)
-///     PAT_KEYWORD(kw)
-///   SIL_KEYWORD(kw)
-/// POUND_KEYWORD(kw)
-///   POUND_OBJECT_LITERAL(kw, desc, proto)
-///   POUND_OLD_OBJECT_LITERAL(kw, new_kw, old_arg, new_arg)
-///   POUND_CONFIG(kw)
-/// PUNCTUATOR(name, str)
-/// LITERAL(name)
-/// MISC(name)
+/// TOKEN(name)
+///   KEYWORD(kw)
+///     SWIFT_KEYWORD(kw)
+///       DECL_KEYWORD(kw)
+///       STMT_KEYWORD(kw)
+///       EXPR_KEYWORD(kw)
+///       PAT_KEYWORD(kw)
+///     SIL_KEYWORD(kw)
+///   POUND_KEYWORD(kw)
+///     POUND_OBJECT_LITERAL(kw, desc, proto)
+///     POUND_OLD_OBJECT_LITERAL(kw, new_kw, old_arg, new_arg)
+///     POUND_CONFIG(kw)
+///   PUNCTUATOR(name, str)
+///   LITERAL(name)
+///   MISC(name)
 ///
 //===----------------------------------------------------------------------===//
 
+/// TOKEN(name)
+///   Expands by default for every token kind.
+#ifndef TOKEN
+#define TOKEN(name)
+#endif
+
 /// KEYWORD(kw)
 ///   Expands by default for every Swift keyword and every SIL keyword, such as
 ///   'if', 'else', 'sil_global', etc. If you only want to use Swift keywords
 ///   see SWIFT_KEYWORD.
 #ifndef KEYWORD
-#define KEYWORD(kw)
+#define KEYWORD(kw) TOKEN(kw_ ## kw)
 #endif
 
 /// SWIFT_KEYWORD(kw)
@@ -78,7 +85,7 @@
 /// POUND_KEYWORD(kw)
 ///   Every keyword prefixed with a '#'.
 #ifndef POUND_KEYWORD
-#define POUND_KEYWORD(kw)
+#define POUND_KEYWORD(kw) TOKEN(pound_ ## kw)
 #endif
 
 /// POUND_OBJECT_LITERAL(kw, desc, proto)
@@ -107,19 +114,19 @@
 ///   \param str   A string literal containing the spelling of the punctuator,
 ///                such as '"("' or '"->"'.
 #ifndef PUNCTUATOR
-#define PUNCTUATOR(name, str)
+#define PUNCTUATOR(name, str) TOKEN(name)
 #endif
 
 /// LITERAL(name)
 ///   Tokens representing literal values, e.g. 'integer_literal'.
 #ifndef LITERAL
-#define LITERAL(name)
+#define LITERAL(name) TOKEN(name)
 #endif
 
 /// MISC(name)
 ///   Miscellaneous tokens, e.g. 'eof' and 'unknown'.
 #ifndef MISC
-#define MISC(name)
+#define MISC(name) TOKEN(name)
 #endif
 
 // Keywords that start decls.
@@ -291,6 +298,7 @@ MISC(string_quote)
 MISC(multiline_string_quote)
 MISC(string_segment)
 
+#undef TOKEN
 #undef KEYWORD
 #undef SWIFT_KEYWORD
 #undef DECL_KEYWORD

include/swift/Syntax/TokenKinds.h

@@ -19,11 +19,7 @@
 
 namespace swift {
 enum class tok {
-#define LITERAL(X) X,
-#define MISC(X) X,
-#define KEYWORD(X) kw_ ## X,
-#define PUNCTUATOR(X, Y) X,
-#define POUND_KEYWORD(X) pound_ ## X,
+#define TOKEN(X) X,
 #include "swift/Syntax/TokenKinds.def"
 
   NUM_TOKENS

lib/Syntax/RawTokenSyntax.cpp

@@ -55,79 +55,14 @@ RawTokenSyntax::accumulateAbsolutePosition(AbsolutePosition &Pos) const {
 
 void RawTokenSyntax::dumpKind(llvm::raw_ostream &OS) const {
   switch (getTokenKind()) {
-  case tok::unknown:
-    OS << "unknown";
-    break;
-  case tok::eof:
-    OS << "eof";
-    break;
-  case tok::code_complete:
-    OS << "code_complete";
-    break;
-  case tok::identifier:
-    OS << "identifier";
-    break;
-  case tok::oper_binary_unspaced:
-    OS << "oper_binary_unspaced";
-    break;
-  case tok::oper_binary_spaced:
-    OS << "oper_binary_spaced";
-    break;
-  case tok::string_interpolation_anchor:
-    OS << "string_interpolation_anchor";
-    break;
-  case tok::string_quote:
-    OS << "string_quote";
-    break;
-  case tok::multiline_string_quote:
-    OS << "multiline_string_quote";
-    break;
-  case tok::string_segment:
-    OS << "string_segment";
-    break;
-  case tok::contextual_keyword:
-    OS << "contextual_keyword";
-    break;
-  case tok::oper_postfix:
-    OS << "oper_postfix";
-    break;
-  case tok::oper_prefix:
-    OS << "oper_prefix";
-    break;
-  case tok::dollarident:
-    OS << "dollarident";
-    break;
-  case tok::integer_literal:
-    OS << "integer_literal";
-    break;
-  case tok::floating_literal:
-    OS << "floating_literal";
-    break;
-  case tok::string_literal:
-    OS << "string_literal";
-    break;
-  case tok::sil_local_name:
-    OS << "sil_local_name";
-    break;
-  case tok::comment:
-    OS << "comment";
-    break;
-  case tok::NUM_TOKENS:
-    OS << "NUM_TOKENS (unset)";
-    break;
-#define KEYWORD(X)                                                             \
-  case tok::kw_##X:                                                            \
-    OS << "kw_" << #X;                                                         \
-    break;
-#define PUNCTUATOR(X, Y)                                                       \
+#define TOKEN(X)                                                               \
   case tok::X:                                                                 \
     OS << #X;                                                                  \
     break;
-#define POUND_KEYWORD(X)                                                       \
-  case tok::pound_##X:                                                         \
-    OS << "pound_" << #X;                                                      \
-    break;
 #include "swift/Syntax/TokenKinds.def"
+  case tok::NUM_TOKENS:
+    OS << "NUM_TOKENS (unset)";
+    break;
   }
 }