Assert that tokens in the syntax tree match token_choices and text_choices

CodeGeneration/Sources/SyntaxSupport/KeywordSpec.swift

@@ -130,6 +130,7 @@ public let KEYWORDS: [KeywordSpec] = [
   KeywordSpec("fileprivate", isLexerClassified: true, requiresTrailingSpace: true),
   KeywordSpec("final"),
   KeywordSpec("for", isLexerClassified: true, requiresTrailingSpace: true),
+  KeywordSpec("forward"),
   KeywordSpec("func", isLexerClassified: true, requiresTrailingSpace: true),
   KeywordSpec("get"),
   KeywordSpec("guard", isLexerClassified: true, requiresTrailingSpace: true),
@@ -151,6 +152,7 @@ public let KEYWORDS: [KeywordSpec] = [
   KeywordSpec("left"),
   KeywordSpec("let", isLexerClassified: true, requiresTrailingSpace: true),
   KeywordSpec("line"),
+  KeywordSpec("linear"),
   KeywordSpec("lowerThan"),
   KeywordSpec("macro"),
   KeywordSpec("message"),
@@ -163,6 +165,7 @@ public let KEYWORDS: [KeywordSpec] = [
   KeywordSpec("noasync"),
   KeywordSpec("noDerivative"),
   KeywordSpec("noescape"),
+  KeywordSpec("none"),
   KeywordSpec("nonisolated"),
   KeywordSpec("nonmutating"),
   KeywordSpec("objc"),
@@ -187,6 +190,7 @@ public let KEYWORDS: [KeywordSpec] = [
   KeywordSpec("rethrows", isLexerClassified: true, requiresTrailingSpace: true),
   KeywordSpec("return", isLexerClassified: true, requiresTrailingSpace: true),
   KeywordSpec("reverse"),
+  KeywordSpec("right"),
   KeywordSpec("safe"),
   KeywordSpec("self", isLexerClassified: true),
   KeywordSpec("Self", isLexerClassified: true),

CodeGeneration/Sources/SyntaxSupport/gyb_generated/AttributeNodes.swift

@@ -103,7 +103,7 @@ public let ATTRIBUTE_NODES: [Node] = [
        kind: "Syntax",
        children: [
          Child(name: "Label",
-               kind: .token(choices: [.token(tokenKind: "IdentifierToken")]),
+               kind: .token(choices: [.keyword(text: "availability")]),
                description: "The label of the argument"),
          Child(name: "Colon",
                kind: .token(choices: [.token(tokenKind: "ColonToken")]),
@@ -123,7 +123,7 @@ public let ATTRIBUTE_NODES: [Node] = [
        ],
        children: [
          Child(name: "Label",
-               kind: .token(choices: [.token(tokenKind: "IdentifierToken")]),
+               kind: .token(choices: [.token(tokenKind: "IdentifierToken"), .keyword(text: "available"), .keyword(text: "exported"), .keyword(text: "kind"), .keyword(text: "spi"), .keyword(text: "spiModule")]),
                description: "The label of the argument"),
          Child(name: "Colon",
                kind: .token(choices: [.token(tokenKind: "ColonToken")]),
@@ -146,7 +146,7 @@ public let ATTRIBUTE_NODES: [Node] = [
        ],
        children: [
          Child(name: "Label",
-               kind: .token(choices: [.token(tokenKind: "IdentifierToken")]),
+               kind: .token(choices: [.keyword(text: "target")]),
                description: "The label of the argument"),
          Child(name: "Colon",
                kind: .token(choices: [.token(tokenKind: "ColonToken")]),
@@ -165,7 +165,7 @@ public let ATTRIBUTE_NODES: [Node] = [
        kind: "Syntax",
        children: [
          Child(name: "DeclBaseName",
-               kind: .token(choices: [.token(tokenKind: "IdentifierToken"), .token(tokenKind: "PrefixOperatorToken")]),
+               kind: .token(choices: [.token(tokenKind: "IdentifierToken"), .token(tokenKind: "PrefixOperatorToken"), .keyword(text: "init")]),
                description: "The base name of the protocol's requirement."),
          Child(name: "DeclNameArguments",
                kind: .node(kind: "DeclNameArguments"),
@@ -396,7 +396,7 @@ public let ATTRIBUTE_NODES: [Node] = [
        kind: "Syntax",
        children: [
          Child(name: "ConventionLabel",
-               kind: .token(choices: [.keyword(text: "block"), .keyword(text: "c"), .keyword(text: "objc_method"), .keyword(text: "thin"), .keyword(text: "thick")]),
+               kind: .token(choices: [.token(tokenKind: "IdentifierToken")]),
                description: "The convention label."),
          Child(name: "Comma",
                kind: .token(choices: [.token(tokenKind: "CommaToken")]),
@@ -418,7 +418,7 @@ public let ATTRIBUTE_NODES: [Node] = [
        kind: "Syntax",
        children: [
          Child(name: "WitnessMethodLabel",
-               kind: .token(choices: [.token(tokenKind: "IdentifierToken")])),
+               kind: .token(choices: [.keyword(text: "witness_method")])),
          Child(name: "Colon",
                kind: .token(choices: [.token(tokenKind: "ColonToken")])),
          Child(name: "ProtocolName",

CodeGeneration/Sources/SyntaxSupport/gyb_generated/AvailabilityNodes.swift

@@ -45,7 +45,7 @@ public let AVAILABILITY_NODES: [Node] = [
        kind: "Syntax",
        children: [
          Child(name: "Label",
-               kind: .token(choices: [.token(tokenKind: "IdentifierToken")]),
+               kind: .token(choices: [.keyword(text: "message"), .keyword(text: "renamed"), .keyword(text: "introduced"), .keyword(text: "obsoleted"), .keyword(text: "deprecated")]),
                description: "The label of the argument"),
          Child(name: "Colon",
                kind: .token(choices: [.token(tokenKind: "ColonToken")]),

CodeGeneration/Sources/SyntaxSupport/gyb_generated/DeclNodes.swift

@@ -216,7 +216,7 @@ public let DECL_NODES: [Node] = [
          Child(name: "LeftParen",
                kind: .token(choices: [.token(tokenKind: "LeftParenToken")])),
          Child(name: "Detail",
-               kind: .token(choices: [.token(tokenKind: "IdentifierToken")])),
+               kind: .token(choices: [.token(tokenKind: "IdentifierToken"), .keyword(text: "set")])),
          Child(name: "RightParen",
                kind: .token(choices: [.token(tokenKind: "RightParenToken")]))
        ]),
@@ -226,7 +226,7 @@ public let DECL_NODES: [Node] = [
        kind: "Syntax",
        children: [
          Child(name: "Name",
-               kind: .token(choices: [.keyword(text: "class"), .keyword(text: "convenience"), .keyword(text: "dynamic"), .keyword(text: "final"), .keyword(text: "infix"), .keyword(text: "lazy"), .keyword(text: "optional"), .keyword(text: "override"), .keyword(text: "postfix"), .keyword(text: "prefix"), .keyword(text: "required"), .keyword(text: "static"), .keyword(text: "unowned"), .keyword(text: "weak"), .keyword(text: "private"), .keyword(text: "fileprivate"), .keyword(text: "internal"), .keyword(text: "public"), .keyword(text: "open"), .keyword(text: "mutating"), .keyword(text: "nonmutating"), .keyword(text: "indirect"), .keyword(text: "__consuming"), .keyword(text: "actor"), .keyword(text: "async"), .keyword(text: "distributed"), .keyword(text: "isolated"), .keyword(text: "nonisolated"), .keyword(text: "_const"), .keyword(text: "_local")]),
+               kind: .token(choices: [.keyword(text: "class"), .keyword(text: "convenience"), .keyword(text: "dynamic"), .keyword(text: "final"), .keyword(text: "infix"), .keyword(text: "lazy"), .keyword(text: "optional"), .keyword(text: "override"), .keyword(text: "postfix"), .keyword(text: "prefix"), .keyword(text: "required"), .keyword(text: "static"), .keyword(text: "unowned"), .keyword(text: "weak"), .keyword(text: "private"), .keyword(text: "fileprivate"), .keyword(text: "internal"), .keyword(text: "public"), .keyword(text: "open"), .keyword(text: "mutating"), .keyword(text: "nonmutating"), .keyword(text: "indirect"), .keyword(text: "__consuming"), .keyword(text: "actor"), .keyword(text: "async"), .keyword(text: "distributed"), .keyword(text: "isolated"), .keyword(text: "nonisolated"), .keyword(text: "_const"), .keyword(text: "_local"), .keyword(text: "package")]),
                classification: "Attribute"),
          Child(name: "Detail",
                kind: .node(kind: "DeclModifierDetail"),
@@ -934,7 +934,7 @@ public let DECL_NODES: [Node] = [
          Child(name: "LeadingComma",
                kind: .token(choices: [.token(tokenKind: "CommaToken")])),
          Child(name: "Name",
-               kind: .token(choices: [.token(tokenKind: "IdentifierToken")]))
+               kind: .node(kind: "Token"))
        ]),
 
   Node(name: "OperatorPrecedenceAndTypes",

CodeGeneration/Sources/SyntaxSupport/gyb_generated/ExprNodes.swift

@@ -737,11 +737,11 @@ public let EXPR_NODES: [Node] = [
                kind: .token(choices: [.token(tokenKind: "RawStringDelimiterToken")]),
                isOptional: true),
          Child(name: "OpenQuote",
-               kind: .token(choices: [.token(tokenKind: "StringQuoteToken"), .token(tokenKind: "MultilineStringQuoteToken")])),
+               kind: .token(choices: [.token(tokenKind: "StringQuoteToken"), .token(tokenKind: "MultilineStringQuoteToken"), .token(tokenKind: "SingleQuoteToken")])),
          Child(name: "Segments",
                kind: .collection(kind: "StringLiteralSegments", collectionElementName: "Segment")),
          Child(name: "CloseQuote",
-               kind: .token(choices: [.token(tokenKind: "StringQuoteToken"), .token(tokenKind: "MultilineStringQuoteToken")])),
+               kind: .token(choices: [.token(tokenKind: "StringQuoteToken"), .token(tokenKind: "MultilineStringQuoteToken"), .token(tokenKind: "SingleQuoteToken")])),
          Child(name: "CloseDelimiter",
                kind: .token(choices: [.token(tokenKind: "RawStringDelimiterToken")]),
                isOptional: true)

CodeGeneration/Sources/SyntaxSupport/gyb_generated/GenericNodes.swift

@@ -71,7 +71,7 @@ public let GENERIC_NODES: [Node] = [
          Child(name: "Colon",
                kind: .token(choices: [.token(tokenKind: "ColonToken")])),
          Child(name: "LayoutConstraint",
-               kind: .token(choices: [.token(tokenKind: "IdentifierToken")])),
+               kind: .token(choices: [.keyword(text: "_Trivial"), .keyword(text: "_TrivialAtMost"), .keyword(text: "_UnknownLayout"), .keyword(text: "_RefCountedObject"), .keyword(text: "_NativeRefCountedObject"), .keyword(text: "_Class"), .keyword(text: "_NativeClass")])),
          Child(name: "LeftParen",
                kind: .token(choices: [.token(tokenKind: "LeftParenToken")]),
                isOptional: true),

CodeGeneration/Sources/SyntaxSupport/gyb_generated/TypeNodes.swift

@@ -18,7 +18,7 @@ public let TYPE_NODES: [Node] = [
        kind: "Type",
        children: [
          Child(name: "Name",
-               kind: .token(choices: [.token(tokenKind: "IdentifierToken"), .token(tokenKind: "KeywordToken")]),
+               kind: .token(choices: [.token(tokenKind: "IdentifierToken"), .token(tokenKind: "KeywordToken"), .token(tokenKind: "WildcardToken")]),
                classification: "TypeIdentifier"),
          Child(name: "GenericArgumentClause",
                kind: .node(kind: "GenericArgumentClause"),
@@ -245,7 +245,7 @@ public let TYPE_NODES: [Node] = [
        ],
        children: [
          Child(name: "Specifier",
-               kind: .token(choices: [.keyword(text: "inout"), .keyword(text: "__shared"), .keyword(text: "__owned")]),
+               kind: .token(choices: [.keyword(text: "inout"), .keyword(text: "__shared"), .keyword(text: "__owned"), .keyword(text: "isolated"), .keyword(text: "_const")]),
                isOptional: true),
          Child(name: "Attributes",
                kind: .collection(kind: "AttributeList", collectionElementName: "Attribute"),

Sources/SwiftParser/Attributes.swift

@@ -418,7 +418,7 @@ extension Parser {
   }
 
   mutating func parseDifferentiabilityParameters() -> RawDifferentiabilityParamsClauseSyntax {
-    let (unexpectedBeforeWrt, wrt) = self.expectIdentifier(keywordRecovery: true)
+    let (unexpectedBeforeWrt, wrt) = self.expect(.keyword(.wrt))
     let (unexpectedBeforeColon, colon) = self.expect(.colon)
 
     guard let leftParen = self.consume(if: .leftParen) else {
@@ -879,31 +879,35 @@ extension Parser {
         )
       )
     } else {
-      let label = self.consumeAnyToken()
+      let (unexpectedBeforeLabel, label) = self.expect(.identifier)
       let unexpectedBeforeComma: RawUnexpectedNodesSyntax?
       let comma: RawTokenSyntax?
+      let unexpectedBeforeCTypeLabel: RawUnexpectedNodesSyntax?
       let cTypeLabel: RawTokenSyntax?
       let unexpectedBeforeColon: RawUnexpectedNodesSyntax?
       let colon: RawTokenSyntax?
       let cTypeString: RawStringLiteralExprSyntax?
       if self.at(.comma) {
         (unexpectedBeforeComma, comma) = self.expect(.comma)
-        cTypeLabel = self.consumeAnyToken()
+        (unexpectedBeforeCTypeLabel, cTypeLabel) = self.expect(.keyword(.cType))
         (unexpectedBeforeColon, colon) = self.expect(.colon)
         cTypeString = self.parseStringLiteral()
       } else {
         unexpectedBeforeComma = nil
         comma = nil
+        unexpectedBeforeCTypeLabel = nil
         cTypeLabel = nil
         unexpectedBeforeColon = nil
         colon = nil
         cTypeString = nil
       }
       return .conventionArguments(
         RawConventionAttributeArgumentsSyntax(
+          unexpectedBeforeLabel,
           conventionLabel: label,
           unexpectedBeforeComma,
           comma: comma,
+          unexpectedBeforeCTypeLabel,
           cTypeLabel: cTypeLabel,
           unexpectedBeforeColon,
           colon: colon,
@@ -917,7 +921,7 @@ extension Parser {
 
 extension Parser {
   mutating func parseBackDeployArguments() -> RawBackDeployAttributeSpecListSyntax {
-    let (unexpectedBeforeLabel, label) = self.expect(.keyword(.before), remapping: .identifier)
+    let (unexpectedBeforeLabel, label) = self.expect(.keyword(.before))
     let (unexpectedBeforeColon, colon) = self.expect(.colon)
     var elements: [RawAvailabilityVersionRestrictionListEntrySyntax] = []
     var keepGoing: RawTokenSyntax? = nil
@@ -974,7 +978,7 @@ extension Parser {
 
 extension Parser {
   mutating func parseOriginallyDefinedInArguments() -> RawOriginallyDefinedInArgumentsSyntax {
-    let (unexpectedBeforeModuleLabel, moduleLabel) = self.expect(.keyword(.module), remapping: .identifier)
+    let (unexpectedBeforeModuleLabel, moduleLabel) = self.expect(.keyword(.module))
     let (unexpectedBeforeColon, colon) = self.expect(.colon)
     let moduleName = self.parseStringLiteral()
     let (unexpectedBeforeComma, comma) = self.expect(.comma)
@@ -1009,7 +1013,7 @@ extension Parser {
 
 extension Parser {
   mutating func parseUnderscorePrivateAttributeArguments() -> RawUnderscorePrivateAttributeArgumentsSyntax {
-    let (unexpectedBeforeLabel, label) = self.expect(.keyword(.sourceFile), remapping: .identifier)
+    let (unexpectedBeforeLabel, label) = self.expect(.keyword(.sourceFile))
     let (unexpectedBeforeColon, colon) = self.expect(.colon)
     let filename = self.parseStringLiteral()
     return RawUnderscorePrivateAttributeArgumentsSyntax(
@@ -1025,7 +1029,7 @@ extension Parser {
 
 extension Parser {
   mutating func parseDynamicReplacementArguments() -> RawDynamicReplacementArgumentsSyntax {
-    let (unexpectedBeforeLabel, label) = self.expect(.keyword(.for), remapping: .identifier)
+    let (unexpectedBeforeLabel, label) = self.expect(.keyword(.for))
     let (unexpectedBeforeColon, colon) = self.expect(.colon)
     let base: RawTokenSyntax
     let args: RawDeclNameArgumentsSyntax?
@@ -1051,7 +1055,7 @@ extension Parser {
 
 extension Parser {
   mutating func parseUnavailableFromAsyncArguments() -> RawUnavailableFromAsyncArgumentsSyntax {
-    let (unexpectedBeforeLabel, label) = self.expect(.keyword(.message), remapping: .identifier)
+    let (unexpectedBeforeLabel, label) = self.expect(.keyword(.message))
     let (unexpectedBeforeColon, colon) = self.expect(.colon)
     let message = self.parseStringLiteral()
     return RawUnavailableFromAsyncArgumentsSyntax(

Sources/SwiftParser/Declarations.swift

@@ -695,7 +695,7 @@ extension Parser {
           requirement = .sameTypeRequirement(
             RawSameTypeRequirementSyntax(
               leftTypeIdentifier: firstType,
-              equalityToken: RawTokenSyntax(missing: .equal, arena: self.arena),
+              equalityToken: RawTokenSyntax(missing: .binaryOperator, text: "==", arena: self.arena),
               rightTypeIdentifier: RawTypeSyntax(RawMissingTypeSyntax(arena: self.arena)),
               arena: self.arena
             )
@@ -981,7 +981,7 @@ extension Parser {
     let failable: RawTokenSyntax?
     if let parsedFailable = self.consume(ifAny: [.exclamationMark, .postfixQuestionMark, .infixQuestionMark]) {
       failable = parsedFailable
-    } else if let parsedFailable = self.consumeIfContextualPunctuator("!") {
+    } else if let parsedFailable = self.consumeIfContextualPunctuator("!", remapping: .exclamationMark) {
       failable = parsedFailable
     } else {
       failable = nil
@@ -1563,8 +1563,10 @@ extension Parser {
     // get and set.
     let modifier: RawDeclModifierSyntax?
     if hasModifier {
+      let (unexpectedBeforeName, name) = self.expectAny([.keyword(.mutating), .keyword(.nonmutating), .keyword(.__consuming)], default: .keyword(.mutating))
       modifier = RawDeclModifierSyntax(
-        name: self.consumeAnyToken(),
+        unexpectedBeforeName,
+        name: name,
         detail: nil,
         arena: self.arena
       )
@@ -1980,7 +1982,10 @@ extension Parser {
         case (.associativity, let handle)?:
           let associativity = self.eat(handle)
           let (unexpectedBeforeColon, colon) = self.expect(.colon)
-          let (unexpectedBeforeValue, value) = self.expectIdentifier()
+          var (unexpectedBeforeValue, value) = self.expectAny([.keyword(.left), .keyword(.right), .keyword(.none)], default: .keyword(.none))
+          if value.isMissing, let identifier = self.consume(if: .identifier) {
+            unexpectedBeforeValue = RawUnexpectedNodesSyntax(combining: unexpectedBeforeValue, identifier, arena: self.arena)
+          }
           elements.append(
             .precedenceGroupAssociativity(
               RawPrecedenceGroupAssociativitySyntax(

Sources/SwiftParser/Directives.swift

@@ -152,12 +152,12 @@ extension Parser {
     let (unexpectedBeforeLParen, lparen) = self.expect(.leftParen)
     let args: RawPoundSourceLocationArgsSyntax?
     if !self.at(.rightParen) {
-      let (unexpectedBeforeFile, file) = self.expectIdentifier()
+      let (unexpectedBeforeFile, file) = self.expect(.keyword(.file))
       let (unexpectedBeforeFileColon, fileColon) = self.expect(.colon)
       let fileName = self.parseStringLiteral()
       let (unexpectedBeforeComma, comma) = self.expect(.comma)
 
-      let (unexpectedBeforeLine, line) = self.expectIdentifier()
+      let (unexpectedBeforeLine, line) = self.expect(.keyword(.line))
       let (unexpectedBeforeLineColon, lineColon) = self.expect(.colon)
       let lineNumber = self.expectWithoutRecovery(.integerLiteral)
 

Sources/SwiftParser/RawTokenKindSubset.swift

@@ -467,6 +467,18 @@ enum OperatorLike: RawTokenKindSubset {
     default: return nil
     }
   }
+
+  var remappedKind: RawTokenKind? {
+    switch self {
+    case .operator(_): return nil
+    case .exclamationMark: return .postfixOperator
+    case .infixQuestionMark: return .binaryOperator
+    case .postfixQuestionMark: return .postfixOperator
+    case .equal: return .binaryOperator
+    case .arrow: return .binaryOperator
+    case .regexLiteral: return .binaryOperator
+    }
+  }
 }
 
 enum PoundDeclarationStart: RawTokenKindSubset {

Sources/SwiftParser/TokenConsumer.swift

@@ -162,9 +162,13 @@ extension TokenConsumer {
 
   /// Consumes and returns the current token is an operator with the given `name`.
   @_spi(RawSyntax)
-  public mutating func consumeIfContextualPunctuator(_ name: SyntaxText) -> Token? {
+  public mutating func consumeIfContextualPunctuator(_ name: SyntaxText, remapping: RawTokenKind? = nil) -> Token? {
     if self.atContextualPunctuator(name) {
-      return self.consumeAnyToken()
+      if let remapping = remapping {
+        return self.consumeAnyToken(remapping: remapping)
+      } else {
+        return self.consumeAnyToken()
+      }
     }
     return nil
   }

Sources/SwiftParser/Types.swift

@@ -548,7 +548,7 @@ extension Parser {
         }
         // Parse the type annotation.
         let type = self.parseType(misplacedSpecifiers: misplacedSpecifiers)
-        let ellipsis = self.currentToken.isEllipsis ? self.consumeAnyToken() : nil
+        let ellipsis = self.consumeIfContextualPunctuator("...", remapping: .ellipsis)
         var trailingComma = self.consume(if: .comma)
         if trailingComma == nil && self.withLookahead({ $0.canParseType() }) {
           // If the next token does not close the tuple, it is very likely the user forgot the comma.