Simplify TokenSpec in CodeGeneration

CodeGeneration/Sources/SyntaxSupport/GrammarGenerator.swift

@@ -23,7 +23,7 @@ struct GrammarGenerator {
       if let tokenText = token.text {
         return "`'\(tokenText)'`"
       } else {
-        return "`<\(token.swiftKind)>`"
+        return "`<\(token.varOrCaseName)>`"
       }
     }
   }

CodeGeneration/Sources/SyntaxSupport/TokenSpec.swift

@@ -10,157 +10,114 @@
 //
 //===----------------------------------------------------------------------===//
 
+import SwiftSyntax
+
 /// Represents the specification for a Token in the TokenSyntax file.
-public class TokenSpec {
-  public let name: String
+public struct TokenSpec {
+  public enum Kind {
+    case punctuation
+    /// The `keyword` TokenKind that contains the actual keyword as an associated value
+    case keyword
+    case other
+  }
+
+  public let varOrCaseName: TokenSyntax
   public let nameForDiagnostics: String
   public let text: String?
-  public let associatedValueClass: String?
-
-  public var swiftKind: String {
-    return lowercaseFirstWord(name: self.name)
-  }
+  public let kind: Kind
 
-  init(
+  fileprivate init(
     name: String,
     nameForDiagnostics: String,
     text: String? = nil,
-    associatedValueClass: String? = nil
+    kind: Kind
   ) {
-    self.name = name
+    self.varOrCaseName = .identifier(name)
     self.nameForDiagnostics = nameForDiagnostics
     self.text = text
-    self.associatedValueClass = associatedValueClass
-  }
-}
-
-public class PoundSpec: TokenSpec {
-  init(
-    name: String,
-    nameForDiagnostics: String? = nil,
-    text: String
-  ) {
-    super.init(
-      name: name,
-      nameForDiagnostics: nameForDiagnostics ?? text,
-      text: text
-    )
-  }
-}
-
-public class PoundObjectLiteralSpec: PoundSpec {
-  let `protocol`: String
-
-  init(
-    name: String,
-    text: String,
-    nameForDiagnostics: String,
-    `protocol`: String
-  ) {
-    self.`protocol` = `protocol`
-    super.init(
-      name: name,
-      nameForDiagnostics: nameForDiagnostics,
-      text: text
-    )
+    self.kind = kind
   }
-}
-
-public class PoundConfigSpec: PoundSpec {}
 
-public class PoundDirectiveSpec: PoundSpec {
-  init(
-    name: String,
-    text: String
-  ) {
-    super.init(
+  static func punctuator(name: String, text: String) -> TokenSpec {
+    return TokenSpec(
       name: name,
-      text: text
+      nameForDiagnostics: text,
+      text: text,
+      kind: .punctuation
     )
   }
-}
 
-public class PoundConditionalDirectiveSpec: PoundDirectiveSpec {
-  override init(
-    name: String,
-    text: String
-  ) {
-    super.init(
+  static func poundKeyword(name: String, text: String) -> TokenSpec {
+    return TokenSpec(
       name: name,
-      text: text
+      nameForDiagnostics: text,
+      text: text,
+      kind: .other
     )
   }
-}
 
-public class PunctuatorSpec: TokenSpec {
-  init(
-    name: String,
-    text: String
-  ) {
-    super.init(
+  static func other(name: String, nameForDiagnostics: String, text: String? = nil) -> TokenSpec {
+    TokenSpec(
       name: name,
-      nameForDiagnostics: text,
-      text: text
+      nameForDiagnostics: nameForDiagnostics,
+      text: text,
+      kind: .other
     )
   }
 }
 
-public class LiteralSpec: TokenSpec {}
-
-public class MiscSpec: TokenSpec {}
-
 public let SYNTAX_TOKENS: [TokenSpec] = [
-  PunctuatorSpec(name: "Arrow", text: "->"),
-  PunctuatorSpec(name: "AtSign", text: "@"),
-  PunctuatorSpec(name: "Backslash", text: "\\"),
-  PunctuatorSpec(name: "Backtick", text: "`"),
-  MiscSpec(name: "BinaryOperator", nameForDiagnostics: "binary operator"),
-  PunctuatorSpec(name: "Colon", text: ":"),
-  PunctuatorSpec(name: "Comma", text: ","),
-  MiscSpec(name: "DollarIdentifier", nameForDiagnostics: "dollar identifier"),
-  PunctuatorSpec(name: "Ellipsis", text: "..."),
-  MiscSpec(name: "EndOfFile", nameForDiagnostics: "end of file", text: ""),
-  PunctuatorSpec(name: "Equal", text: "="),
-  PunctuatorSpec(name: "ExclamationMark", text: "!"),
-  MiscSpec(name: "ExtendedRegexDelimiter", nameForDiagnostics: "extended delimiter"),
-  LiteralSpec(name: "FloatingLiteral", nameForDiagnostics: "floating literal"),
-  MiscSpec(name: "Identifier", nameForDiagnostics: "identifier"),
-  PunctuatorSpec(name: "InfixQuestionMark", text: "?"),
-  LiteralSpec(name: "IntegerLiteral", nameForDiagnostics: "integer literal"),
-  MiscSpec(name: "Keyword", nameForDiagnostics: "keyword", associatedValueClass: "Keyword"),
-  PunctuatorSpec(name: "LeftAngle", text: "<"),
-  PunctuatorSpec(name: "LeftBrace", text: "{"),
-  PunctuatorSpec(name: "LeftParen", text: "("),
-  PunctuatorSpec(name: "LeftSquare", text: "["),
-  PunctuatorSpec(name: "MultilineStringQuote", text: "\"\"\""),
-  PunctuatorSpec(name: "Period", text: "."),
-  MiscSpec(name: "PostfixOperator", nameForDiagnostics: "postfix operator"),
-  PunctuatorSpec(name: "PostfixQuestionMark", text: "?"),
-  PunctuatorSpec(name: "Pound", text: "#"),
-  PoundConfigSpec(name: "PoundAvailable", text: "#available"),
-  PoundConditionalDirectiveSpec(name: "PoundElse", text: "#else"),
-  PoundConditionalDirectiveSpec(name: "PoundElseif", text: "#elseif"),
-  PoundConditionalDirectiveSpec(name: "PoundEndif", text: "#endif"),
-  PoundConditionalDirectiveSpec(name: "PoundIf", text: "#if"),
-  PoundDirectiveSpec(name: "PoundSourceLocation", text: "#sourceLocation"),
-  PoundConfigSpec(name: "PoundUnavailable", text: "#unavailable"),
-  PunctuatorSpec(name: "PrefixAmpersand", text: "&"),
-  MiscSpec(name: "PrefixOperator", nameForDiagnostics: "prefix operator"),
-  MiscSpec(name: "RawStringDelimiter", nameForDiagnostics: "raw string delimiter"),
-  MiscSpec(name: "RegexLiteralPattern", nameForDiagnostics: "regex pattern"),
-  PunctuatorSpec(name: "RegexSlash", text: "/"),
-  PunctuatorSpec(name: "RightAngle", text: ">"),
-  PunctuatorSpec(name: "RightBrace", text: "}"),
-  PunctuatorSpec(name: "RightParen", text: ")"),
-  PunctuatorSpec(name: "RightSquare", text: "]"),
-  PunctuatorSpec(name: "Semicolon", text: ";"),
-  PunctuatorSpec(name: "SingleQuote", text: "\'"),
-  PunctuatorSpec(name: "StringQuote", text: "\""),
-  MiscSpec(name: "StringSegment", nameForDiagnostics: "string segment"),
-  MiscSpec(name: "Unknown", nameForDiagnostics: "token"),
-  MiscSpec(name: "Wildcard", nameForDiagnostics: "wildcard", text: "_"),
+  .punctuator(name: "arrow", text: "->"),
+  .punctuator(name: "atSign", text: "@"),
+  .punctuator(name: "backslash", text: "\\"),
+  .punctuator(name: "backtick", text: "`"),
+  .other(name: "binaryOperator", nameForDiagnostics: "binary operator"),
+  .punctuator(name: "colon", text: ":"),
+  .punctuator(name: "comma", text: ","),
+  .other(name: "dollarIdentifier", nameForDiagnostics: "dollar identifier"),
+  .punctuator(name: "ellipsis", text: "..."),
+  .other(name: "endOfFile", nameForDiagnostics: "end of file", text: ""),
+  .punctuator(name: "equal", text: "="),
+  .punctuator(name: "exclamationMark", text: "!"),
+  .other(name: "extendedRegexDelimiter", nameForDiagnostics: "extended delimiter"),
+  .other(name: "floatingLiteral", nameForDiagnostics: "floating literal"),
+  .other(name: "identifier", nameForDiagnostics: "identifier"),
+  .punctuator(name: "infixQuestionMark", text: "?"),
+  .other(name: "integerLiteral", nameForDiagnostics: "integer literal"),
+  TokenSpec(name: "keyword", nameForDiagnostics: "keyword", text: nil, kind: .keyword),
+  .punctuator(name: "leftAngle", text: "<"),
+  .punctuator(name: "leftBrace", text: "{"),
+  .punctuator(name: "leftParen", text: "("),
+  .punctuator(name: "leftSquare", text: "["),
+  .punctuator(name: "multilineStringQuote", text: "\"\"\""),
+  .punctuator(name: "period", text: "."),
+  .other(name: "postfixOperator", nameForDiagnostics: "postfix operator"),
+  .punctuator(name: "postfixQuestionMark", text: "?"),
+  .punctuator(name: "pound", text: "#"),
+  .poundKeyword(name: "poundAvailable", text: "#available"),
+  .poundKeyword(name: "poundElse", text: "#else"),
+  .poundKeyword(name: "poundElseif", text: "#elseif"),
+  .poundKeyword(name: "poundEndif", text: "#endif"),
+  .poundKeyword(name: "poundIf", text: "#if"),
+  .poundKeyword(name: "poundSourceLocation", text: "#sourceLocation"),
+  .poundKeyword(name: "poundUnavailable", text: "#unavailable"),
+  .punctuator(name: "prefixAmpersand", text: "&"),
+  .other(name: "prefixOperator", nameForDiagnostics: "prefix operator"),
+  .other(name: "rawStringDelimiter", nameForDiagnostics: "raw string delimiter"),
+  .other(name: "regexLiteralPattern", nameForDiagnostics: "regex pattern"),
+  .punctuator(name: "regexSlash", text: "/"),
+  .punctuator(name: "rightAngle", text: ">"),
+  .punctuator(name: "rightBrace", text: "}"),
+  .punctuator(name: "rightParen", text: ")"),
+  .punctuator(name: "rightSquare", text: "]"),
+  .punctuator(name: "semicolon", text: ";"),
+  .punctuator(name: "singleQuote", text: "\'"),
+  .punctuator(name: "stringQuote", text: "\""),
+  .other(name: "stringSegment", nameForDiagnostics: "string segment"),
+  .other(name: "unknown", nameForDiagnostics: "token"),
+  .other(name: "wildcard", nameForDiagnostics: "wildcard", text: "_"),
 ]
 
 public let SYNTAX_TOKEN_MAP = Dictionary(
-  uniqueKeysWithValues: SYNTAX_TOKENS.map { ("\($0.name)Token", $0) }
+  uniqueKeysWithValues: SYNTAX_TOKENS.map { ("\($0.varOrCaseName.description.withFirstCharacterUppercased)Token", $0) }
 )

CodeGeneration/Sources/Utils/SyntaxBuildableChild.swift

@@ -69,9 +69,9 @@ public extension Child {
       return type.defaultValue
     }
     if token.text != nil {
-      return ExprSyntax(".\(raw: token.swiftKind)Token()")
+      return ExprSyntax(".\(token.varOrCaseName)Token()")
     }
-    guard case .token(let choices, _, _) = kind, choices.count == 1, token.associatedValueClass != nil else {
+    guard case .token(let choices, _, _) = kind, choices.count == 1, token.kind == .keyword else {
       return nil
     }
     var textChoice: String
@@ -82,7 +82,7 @@ public extension Child {
     if textChoice == "init" {
       textChoice = "`init`"
     }
-    return ExprSyntax(".\(raw: token.swiftKind)(.\(raw: textChoice))")
+    return ExprSyntax(".\(token.varOrCaseName)(.\(raw: textChoice))")
   }
 
   /// If the child node has a default value, return an expression of the form

CodeGeneration/Sources/Utils/SyntaxBuildableType.swift

@@ -64,7 +64,7 @@ public struct SyntaxBuildableType: Hashable {
       return ExprSyntax(NilLiteralExprSyntax())
     } else if let token = token {
       if token.text != nil {
-        return ExprSyntax(".\(raw: lowercaseFirstWord(name: token.name))Token()")
+        return ExprSyntax(".\(token.varOrCaseName)Token()")
       }
     }
     return nil

CodeGeneration/Sources/generate-swiftsyntax/templates/swiftparser/ParserTokenSpecSetFile.swift

@@ -30,7 +30,7 @@ let parserTokenSpecSetFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
                 DeclSyntax("case \(raw: keyword.escapedName)")
               case .token(let tokenText):
                 let token = SYNTAX_TOKEN_MAP[tokenText]!
-                DeclSyntax("case \(raw: token.swiftKind)")
+                DeclSyntax("case \(token.varOrCaseName)")
               }
             }
 
@@ -44,7 +44,7 @@ let parserTokenSpecSetFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
                   case .token(let tokenText):
                     let token = SYNTAX_TOKEN_MAP[tokenText]!
                     SwitchCaseSyntax(
-                      "case TokenSpec(.\(raw: token.swiftKind)): self = .\(raw: token.swiftKind)"
+                      "case TokenSpec(.\(token.varOrCaseName)): self = .\(token.varOrCaseName)"
                     )
                   }
                 }
@@ -64,7 +64,7 @@ let parserTokenSpecSetFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
                   case .token(let tokenText):
                     let token = SYNTAX_TOKEN_MAP[tokenText]!
                     SwitchCaseSyntax(
-                      "case .\(raw: token.swiftKind): return .\(raw: token.swiftKind)"
+                      "case .\(token.varOrCaseName): return .\(token.varOrCaseName)"
                     )
                   }
                 }

CodeGeneration/Sources/generate-swiftsyntax/templates/swiftparser/TokenSpecStaticMembersFile.swift

@@ -19,8 +19,8 @@ let tokenSpecStaticMembersFile = SourceFileSyntax(leadingTrivia: copyrightHeader
   DeclSyntax("@_spi(RawSyntax) import SwiftSyntax")
 
   try! ExtensionDeclSyntax("extension TokenSpec") {
-    for token in SYNTAX_TOKENS where token.swiftKind != "keyword" {
-      DeclSyntax("static var \(raw: token.swiftKind): TokenSpec { return TokenSpec(.\(raw: token.swiftKind)) }")
+    for token in SYNTAX_TOKENS where token.kind != .keyword {
+      DeclSyntax("static var \(token.varOrCaseName): TokenSpec { return TokenSpec(.\(token.varOrCaseName)) }")
     }
 
     DeclSyntax("static func keyword(_ keyword: Keyword) -> TokenSpec { return TokenSpec(keyword) }")

CodeGeneration/Sources/generate-swiftsyntax/templates/swiftparserdiagnostics/TokenNameForDiagnosticsFile.swift

@@ -21,9 +21,9 @@ let tokenNameForDiagnosticFile = SourceFileSyntax(leadingTrivia: copyrightHeader
   try! ExtensionDeclSyntax("extension TokenKind") {
     try! VariableDeclSyntax("var nameForDiagnostics: String") {
       try! SwitchExprSyntax("switch self") {
-        for token in SYNTAX_TOKENS where token.swiftKind != "keyword" {
-          SwitchCaseSyntax("case .\(raw: token.swiftKind):") {
-            StmtSyntax("return #\"\(raw: token.nameForDiagnostics)\"#")
+        for token in SYNTAX_TOKENS where token.kind != .keyword {
+          SwitchCaseSyntax("case .\(token.varOrCaseName):") {
+            StmtSyntax("return \(literal: token.nameForDiagnostics)")
           }
         }
         SwitchCaseSyntax("case .keyword(let keyword):") {

CodeGeneration/Sources/generate-swiftsyntax/templates/swiftsyntax/RawSyntaxValidationFile.swift

@@ -216,7 +216,7 @@ let rawSyntaxValidationFile = try! SourceFileSyntax(leadingTrivia: copyrightHead
                               case .keyword(text: let text):
                                 ArrayElementSyntax(expression: ExprSyntax(#".keyword("\#(raw: text)")"#))
                               case .token(tokenKind: let tokenKind):
-                                ArrayElementSyntax(expression: ExprSyntax(".tokenKind(.\(raw: SYNTAX_TOKEN_MAP[tokenKind]!.swiftKind))"))
+                                ArrayElementSyntax(expression: ExprSyntax(".tokenKind(.\(SYNTAX_TOKEN_MAP[tokenKind]!.varOrCaseName))"))
                               }
                             }
                           }