Preparation to switch RawTokenKind to be a trivial enum again

CodeGeneration/Sources/generate-swiftsyntax/templates/swiftparser/TokenSpecStaticMembersFile.swift

@@ -27,6 +27,6 @@ let tokenSpecStaticMembersFile = SourceFileSyntax(
       DeclSyntax("static var \(raw: token.swiftKind): TokenSpec { return TokenSpec(.\(raw: token.swiftKind)) }")
     }
 
-    DeclSyntax("static func keyword(_ keyword: Keyword) -> TokenSpec { return TokenSpec(.keyword(keyword)) }")
+    DeclSyntax("static func keyword(_ keyword: Keyword) -> TokenSpec { return TokenSpec(keyword) }")
   }
 }

CodeGeneration/Sources/generate-swiftsyntax/templates/swiftsyntax/TokenKindFile.swift

@@ -96,6 +96,23 @@ let tokenKindFile = SourceFileSyntax {
       }
     }
 
+    try! VariableDeclSyntax("public var nameForDiagnostics: String") {
+      try! SwitchExprSyntax("switch self") {
+        SwitchCaseSyntax("case .eof:") {
+          StmtSyntax(#"return "end of file""#)
+        }
+
+        for token in SYNTAX_TOKENS where token.swiftKind != "keyword" {
+          SwitchCaseSyntax("case .\(raw: token.swiftKind):") {
+            StmtSyntax("return #\"\(raw: token.nameForDiagnostics)\"#")
+          }
+        }
+        SwitchCaseSyntax("case .keyword(let keyword):") {
+          StmtSyntax("return String(syntaxText: keyword.defaultText)")
+        }
+      }
+    }
+
     try VariableDeclSyntax(
       """
       /// Returns `true` if the token is a Swift keyword.
@@ -287,49 +304,6 @@ let tokenKindFile = SourceFileSyntax {
       }
     }
 
-    try! VariableDeclSyntax("public var nameForDiagnostics: String") {
-      try! SwitchExprSyntax("switch self.base") {
-        SwitchCaseSyntax("case .eof:") {
-          StmtSyntax(#"return "end of file""#)
-        }
-
-        for token in SYNTAX_TOKENS where token.swiftKind != "keyword" {
-          SwitchCaseSyntax("case .\(raw: token.swiftKind):") {
-            StmtSyntax("return #\"\(raw: token.nameForDiagnostics)\"#")
-          }
-        }
-        SwitchCaseSyntax("case .keyword:") {
-          StmtSyntax("return String(syntaxText: self.keyword.defaultText)")
-        }
-      }
-    }
-
-    try! VariableDeclSyntax(
-      """
-      /// Returns `true` if the token is a Swift keyword.
-      ///
-      /// Keywords are reserved unconditionally for use by Swift and may not
-      /// appear as identifiers in any position without being escaped. For example,
-      /// `class`, `func`, or `import`.
-      public var isLexerClassifiedKeyword: Bool
-      """
-    ) {
-      try! SwitchExprSyntax("switch self.base") {
-        SwitchCaseSyntax("case .eof:") {
-          StmtSyntax("return false")
-        }
-
-        for token in SYNTAX_TOKENS where token.swiftKind != "keyword" {
-          SwitchCaseSyntax("case .\(raw: token.swiftKind):") {
-            StmtSyntax("return \(raw: token.isKeyword)")
-          }
-        }
-        SwitchCaseSyntax("case .keyword:") {
-          StmtSyntax("return self.keyword.isLexerClassified")
-        }
-      }
-    }
-
     try! VariableDeclSyntax(
       """
       /// Returns `true` if the token is a Swift punctuator.

Sources/SwiftParser/Attributes.swift

@@ -32,6 +32,7 @@ extension Parser {
   /// Compiler-known attributes that take arguments.
   enum DeclarationAttributeWithSpecialSyntax: TokenSpecSet {
     case _alignment
+    case _backDeploy
     case _cdecl
     case _documentation
     case _dynamicReplacement
@@ -66,7 +67,7 @@ extension Parser {
     init?(lexeme: Lexer.Lexeme) {
       switch lexeme {
       case TokenSpec(._alignment): self = ._alignment
-      case TokenSpec(._backDeploy): self = .backDeployed
+      case TokenSpec(._backDeploy): self = ._backDeploy
       case TokenSpec(._cdecl): self = ._cdecl
       case TokenSpec(._documentation): self = ._documentation
       case TokenSpec(._dynamicReplacement): self = ._dynamicReplacement
@@ -105,6 +106,7 @@ extension Parser {
     var spec: TokenSpec {
       switch self {
       case ._alignment: return .keyword(._alignment)
+      case ._backDeploy: return .keyword(._backDeploy)
       case ._cdecl: return .keyword(._cdecl)
       case ._documentation: return .keyword(._documentation)
       case ._dynamicReplacement: return .keyword(._dynamicReplacement)
@@ -223,7 +225,7 @@ extension Parser {
       return parseAttribute(argumentMode: .required) { parser in
         return .availability(parser.parseAvailabilityArgumentSpecList())
       }
-    case .backDeployed:
+    case .backDeployed, ._backDeploy:
       return parseAttribute(argumentMode: .required) { parser in
         return .backDeployedArguments(parser.parseBackDeployedArguments())
       }
@@ -313,7 +315,7 @@ extension Parser {
       }
     case .rethrows:
       let (unexpectedBeforeAtSign, atSign) = self.expect(.atSign)
-      let (unexpectedBeforeAttributeName, attributeName) = self.expect(TokenSpec(.keyword(.rethrows), remapping: .identifier))
+      let (unexpectedBeforeAttributeName, attributeName) = self.expect(TokenSpec(.rethrows, remapping: .identifier))
       return .attribute(
         RawAttributeSyntax(
           unexpectedBeforeAtSign,
@@ -1156,15 +1158,15 @@ extension Parser.Lookahead {
     // Alternatively, we might have a token that illustrates we're not going to
     // get anything following the attribute, which means the parentheses describe
     // what follows the attribute.
-    switch lookahead.currentToken.rawTokenKind {
-    case .arrow,
-      .keyword(.throw),
-      .keyword(.throws),
-      .keyword(.rethrows),
-      .rightParen,
-      .rightBrace,
-      .rightSquareBracket,
-      .rightAngle:
+    switch lookahead.currentToken {
+    case TokenSpec(.arrow),
+      TokenSpec(.throw),
+      TokenSpec(.throws),
+      TokenSpec(.rethrows),
+      TokenSpec(.rightParen),
+      TokenSpec(.rightBrace),
+      TokenSpec(.rightSquareBracket),
+      TokenSpec(.rightAngle):
       return false
     case _ where lookahead.at(.keyword(.async)):
       return false

Sources/SwiftParser/Declarations.swift

@@ -1477,7 +1477,7 @@ extension Parser {
     inMemberDeclList: Bool = false
   ) -> RawVariableDeclSyntax {
     let (unexpectedBeforeIntroducer, introducer) = self.eat(handle)
-    let hasTryBeforeIntroducer = unexpectedBeforeIntroducer?.containsToken(where: { $0.tokenKind == .keyword(.try) }) ?? false
+    let hasTryBeforeIntroducer = unexpectedBeforeIntroducer?.containsToken(where: { TokenSpec(.try) ~= $0 }) ?? false
 
     var elements = [RawPatternBindingSyntax]()
     do {

Sources/SwiftParser/Expressions.swift

@@ -356,7 +356,7 @@ extension Parser {
       return parseUnresolvedAsExpr(handle: handle)
 
     case (.async, _)?:
-      if self.peek().rawTokenKind == .arrow || self.peek().rawTokenKind == .keyword(.throws) {
+      if self.peek().rawTokenKind == .arrow || TokenSpec(.throws) ~= self.peek() {
         fallthrough
       } else {
         return nil
@@ -494,20 +494,20 @@ extension Parser {
     // First check to see if we have the start of a regex literal `/.../`.
     //    tryLexRegexLiteral(/*forUnappliedOperator*/ false)
 
-    switch self.currentToken.rawTokenKind {
-    case .keyword(.repeat):
+    switch self.currentToken {
+    case TokenSpec(.repeat):
       // 'repeat' is the start of a pack expansion expression.
       return RawExprSyntax(parsePackExpansionExpr(flavor, pattern: pattern))
 
     // Try parse an 'if' or 'switch' as an expression. Note we do this here in
     // parseUnaryExpression as we don't allow postfix syntax to hang off such
     // expressions to avoid ambiguities such as postfix '.member', which can
     // currently be parsed as a static dot member for a result builder.
-    case .keyword(.switch):
+    case TokenSpec(.switch):
       return RawExprSyntax(
         parseSwitchExpression(switchHandle: .constant(.keyword(.switch)))
       )
-    case .keyword(.if):
+    case TokenSpec(.if):
       return RawExprSyntax(
         parseIfExpression(ifHandle: .constant(.keyword(.if)))
       )
@@ -2063,7 +2063,7 @@ extension Parser.Lookahead {
     }
 
     // If this is the start of a switch body, this isn't a trailing closure.
-    if self.peek().rawTokenKind == .keyword(.case) {
+    if TokenSpec(.case) ~= self.peek() {
       return false
     }
 
@@ -2104,23 +2104,23 @@ extension Parser.Lookahead {
       return false
     }
 
-    switch backtrack.currentToken.rawTokenKind {
-    case .leftBrace,
-      .keyword(.where),
-      .comma:
+    switch backtrack.currentToken {
+    case TokenSpec(.leftBrace),
+      TokenSpec(.where),
+      TokenSpec(.comma):
       return true
-    case .leftSquareBracket,
-      .leftParen,
-      .period,
-      .keyword(.is),
-      .keyword(.as),
-      .postfixQuestionMark,
-      .infixQuestionMark,
-      .exclamationMark,
-      .colon,
-      .equal,
-      .postfixOperator,
-      .binaryOperator:
+    case TokenSpec(.leftSquareBracket),
+      TokenSpec(.leftParen),
+      TokenSpec(.period),
+      TokenSpec(.is),
+      TokenSpec(.as),
+      TokenSpec(.postfixQuestionMark),
+      TokenSpec(.infixQuestionMark),
+      TokenSpec(.exclamationMark),
+      TokenSpec(.colon),
+      TokenSpec(.equal),
+      TokenSpec(.postfixOperator),
+      TokenSpec(.binaryOperator):
       return !backtrack.currentToken.isAtStartOfLine
     default:
       return false
@@ -2225,7 +2225,7 @@ extension Parser {
   public mutating func parseSwitchCases(allowStandaloneStmtRecovery: Bool) -> RawSwitchCaseListSyntax {
     var elements = [RawSwitchCaseListSyntax.Element]()
     var elementsProgress = LoopProgressCondition()
-    while !self.at(any: [.eof, .rightBrace, .poundEndifKeyword, .poundElseifKeyword, .poundElseKeyword])
+    while !self.at(.eof, .rightBrace) && !self.at(.poundEndifKeyword, .poundElseifKeyword, .poundElseKeyword)
       && elementsProgress.evaluate(currentToken)
     {
       if self.withLookahead({ $0.isAtStartOfSwitchCase(allowRecovery: false) }) {
@@ -2294,7 +2294,7 @@ extension Parser {
   mutating func parseSwitchCaseBody() -> RawCodeBlockItemListSyntax {
     var items = [RawCodeBlockItemSyntax]()
     var loopProgress = LoopProgressCondition()
-    while !self.at(any: [.rightBrace, .poundEndifKeyword, .poundElseifKeyword, .poundElseKeyword])
+    while !self.at(.rightBrace) && !self.at(.poundEndifKeyword, .poundElseifKeyword, .poundElseKeyword)
       && !self.withLookahead({ $0.isStartOfConditionalSwitchCases() }),
       let newItem = self.parseCodeBlockItem(),
       loopProgress.evaluate(currentToken)
@@ -2559,14 +2559,7 @@ extension Parser.Lookahead {
       return true
     }
 
-    if self.peek().rawTokenKind != .identifier,
-      self.peek().rawTokenKind != .keyword(.Self),
-      self.peek().rawTokenKind != .keyword(.self),
-      !self.peek().rawTokenKind.isLexerClassifiedKeyword
-    {
-      return false
-    }
-    return true
+    return self.peek().isLexerClassifiedKeyword || TokenSpec(.identifier) ~= self.peek()
   }
 
   fileprivate func isNextTokenCallPattern() -> Bool {

Sources/SwiftParser/Lookahead.swift

@@ -254,7 +254,7 @@ extension Parser.Lookahead {
     // If we have a 'didSet' or a 'willSet' label, disambiguate immediately as
     // an accessor block.
     let nextToken = self.peek()
-    if TokenSpec(.keyword(.didSet)) ~= nextToken || TokenSpec(.keyword(.willSet)) ~= nextToken {
+    if TokenSpec(.didSet) ~= nextToken || TokenSpec(.willSet) ~= nextToken {
       return true
     }
 
@@ -286,7 +286,7 @@ extension Parser.Lookahead {
 // MARK: Skipping Tokens
 
 extension Parser.Lookahead {
-  mutating func skipUntil(_ t1: RawTokenKind, _ t2: RawTokenKind) {
+  mutating func skipUntil(_ t1: TokenSpec, _ t2: TokenSpec) {
     return skip(initialState: .skipUntil(t1, t2))
   }
 
@@ -346,7 +346,7 @@ extension Parser.Lookahead {
     /// Execute code after skipping bracketed tokens detected from `skipSingle`.
     case skipSinglePost(start: BracketedTokens)
     /// Skip until either `t1` or `t2`.
-    case skipUntil(_ t1: RawTokenKind, _ t2: RawTokenKind)
+    case skipUntil(_ t1: TokenSpec, _ t2: TokenSpec)
   }
 
   /// A non-recursie function to skip tokens.

Sources/SwiftParser/Modifiers.swift

@@ -196,7 +196,7 @@ extension Parser {
 
     let unexpectedBeforeDetail: RawUnexpectedNodesSyntax?
     let detail: RawTokenSyntax
-    if let setHandle = canRecoverTo(TokenSpec(.keyword(.set), recoveryPrecedence: .weakBracketClose)) {
+    if let setHandle = canRecoverTo(TokenSpec(.set, recoveryPrecedence: .weakBracketClose)) {
       (unexpectedBeforeDetail, detail) = eat(setHandle)
     } else {
       unexpectedBeforeDetail = nil

Sources/SwiftParser/Names.swift

@@ -68,7 +68,7 @@ extension Parser {
       ident = self.expectIdentifierWithoutRecovery()
     } else if flags.contains(.operators), let (_, _) = self.at(anyIn: Operator.self) {
       ident = self.consumeAnyToken(remapping: .binaryOperator)
-    } else if flags.contains(.keywords) && self.currentToken.rawTokenKind.isLexerClassifiedKeyword {
+    } else if flags.contains(.keywords) && self.currentToken.isLexerClassifiedKeyword {
       ident = self.consumeAnyToken(remapping: .identifier)
     } else {
       ident = self.expectIdentifierWithoutRecovery()
@@ -279,7 +279,10 @@ extension Lexer.Lexeme {
   }
 
   var isLexerClassifiedKeyword: Bool {
-    self.rawTokenKind.isLexerClassifiedKeyword
+    // Only lexer-classified lexemes have `RawTokenKind` of `keyword.
+    // Contextual keywords will only be made keywords when a `RawTokenSyntax` is
+    // constructed from them.
+    return self.rawTokenKind.base == .keyword
   }
 
   func starts(with symbol: SyntaxText) -> Bool {

Sources/SwiftParser/Parser.swift

@@ -427,7 +427,7 @@ extension Parser {
         self.missingToken(.identifier, text: nil)
       )
     } else if keywordRecovery,
-      (self.currentToken.rawTokenKind.isLexerClassifiedKeyword || self.currentToken.rawTokenKind == .wildcard),
+      (self.currentToken.isLexerClassifiedKeyword || self.currentToken.rawTokenKind == .wildcard),
       !self.currentToken.isAtStartOfLine
     {
       let keyword = self.consumeAnyToken()

Sources/SwiftParser/Patterns.swift

@@ -55,13 +55,13 @@ extension Parser {
       case varKeyword
 
       init?(lexeme: Lexer.Lexeme) {
-        switch lexeme.rawTokenKind {
-        case .leftParen: self = .leftParen
-        case .wildcard: self = .wildcard
-        case .identifier: self = .identifier
-        case .dollarIdentifier: self = .dollarIdentifier
-        case .keyword(.let): self = .letKeyword
-        case .keyword(.var): self = .varKeyword
+        switch lexeme {
+        case TokenSpec(.leftParen): self = .leftParen
+        case TokenSpec(.wildcard): self = .wildcard
+        case TokenSpec(.identifier): self = .identifier
+        case TokenSpec(.dollarIdentifier): self = .dollarIdentifier
+        case TokenSpec(.let): self = .letKeyword
+        case TokenSpec(.var): self = .varKeyword
         default: return nil
         }
       }
@@ -131,7 +131,7 @@ extension Parser {
         )
       )
     case nil:
-      if self.currentToken.rawTokenKind.isLexerClassifiedKeyword, !self.currentToken.isAtStartOfLine {
+      if self.currentToken.isLexerClassifiedKeyword, !self.currentToken.isAtStartOfLine {
         // Recover if a keyword was used instead of an identifier
         let keyword = self.consumeAnyToken()
         return RawPatternSyntax(
@@ -296,12 +296,12 @@ extension Parser.Lookahead {
       case leftParen
 
       init?(lexeme: Lexer.Lexeme) {
-        switch lexeme.rawTokenKind {
-        case .identifier: self = .identifier
-        case .wildcard: self = .wildcard
-        case .keyword(.let): self = .letKeyword
-        case .keyword(.var): self = .varKeyword
-        case .leftParen: self = .leftParen
+        switch lexeme {
+        case TokenSpec(.identifier): self = .identifier
+        case TokenSpec(.wildcard): self = .wildcard
+        case TokenSpec(.let): self = .letKeyword
+        case TokenSpec(.var): self = .varKeyword
+        case TokenSpec(.leftParen): self = .leftParen
         default: return nil
         }
       }