Explicitly model all supported contextual keywords in the contextualKeyword RawTokenKind

CONTRIBUTING.md

@@ -64,7 +64,7 @@ If you are seeing issues regarding a mismatched parser library, try the followin
 Tip: Running SwiftSyntax’s self-parse tests takes the majority of testing time. If you want to iterate quickly, you can skip these tests using the following steps:
 1. Product -> Scheme -> Edit Scheme…
 2. Select the Arguments tab in the Run section
-3. Add a `SKIP_SELF_PARSE` environment variable with value `1`
+3. Add a `SKIP_LONG_TESTS` environment variable with value `1`
 
 ### `lit`-based tests
 

CodeGeneration/Sources/SyntaxSupport/TokenSpec.swift.gyb

@@ -29,6 +29,7 @@ public class TokenSpec {
   public let isKeyword: Bool
   public let requiresLeadingSpace: Bool
   public let requiresTrailingSpace: Bool
+  public let associatedValueClass: String?
 
   public var swiftKind: String {
     let name = lowercaseFirstWord(name: self.name)
@@ -49,7 +50,8 @@ public class TokenSpec {
     classification: String = "None",
     isKeyword: Bool = false,
     requiresLeadingSpace: Bool = false,
-    requiresTrailingSpace: Bool = false
+    requiresTrailingSpace: Bool = false,
+    associatedValueClass: String? = nil
   ) {
     self.name = name
     self.kind = kind
@@ -64,6 +66,7 @@ public class TokenSpec {
     self.isKeyword = isKeyword
     self.requiresLeadingSpace = requiresLeadingSpace
     self.requiresTrailingSpace = requiresTrailingSpace
+    self.associatedValueClass = associatedValueClass
   }
 }
 
@@ -243,6 +246,9 @@ public let SYNTAX_TOKENS: [TokenSpec] = [
 %     if token.requires_trailing_space:
 %       parameters += ['requiresTrailingSpace: true']
 %     end
+%   end
+%   if token.associated_value_class:
+%     parameters += [f'associatedValueClass: "{token.associated_value_class}"']
 %   end
   ${class_name}Spec(${", ".join(parameters)}),
 % end

CodeGeneration/Sources/SyntaxSupport/gyb_generated/TokenSpec.swift

@@ -23,6 +23,7 @@ public class TokenSpec {
   public let isKeyword: Bool
   public let requiresLeadingSpace: Bool
   public let requiresTrailingSpace: Bool
+  public let associatedValueClass: String?
 
   public var swiftKind: String {
     let name = lowercaseFirstWord(name: self.name)
@@ -43,7 +44,8 @@ public class TokenSpec {
     classification: String = "None",
     isKeyword: Bool = false,
     requiresLeadingSpace: Bool = false,
-    requiresTrailingSpace: Bool = false
+    requiresTrailingSpace: Bool = false,
+    associatedValueClass: String? = nil
   ) {
     self.name = name
     self.kind = kind
@@ -58,6 +60,7 @@ public class TokenSpec {
     self.isKeyword = isKeyword
     self.requiresLeadingSpace = requiresLeadingSpace
     self.requiresTrailingSpace = requiresTrailingSpace
+    self.associatedValueClass = associatedValueClass
   }
 }
 
@@ -315,7 +318,7 @@ public let SYNTAX_TOKENS: [TokenSpec] = [
   MiscSpec(name: "PostfixOperator", kind: "oper_postfix", nameForDiagnostics: "postfix operator", classification: "OperatorIdentifier"),
   MiscSpec(name: "PrefixOperator", kind: "oper_prefix", nameForDiagnostics: "prefix operator", classification: "OperatorIdentifier"),
   MiscSpec(name: "DollarIdentifier", kind: "dollarident", nameForDiagnostics: "dollar identifier", classification: "DollarIdentifier"),
-  MiscSpec(name: "ContextualKeyword", kind: "contextual_keyword", nameForDiagnostics: "keyword", classification: "Keyword"),
+  MiscSpec(name: "ContextualKeyword", kind: "contextual_keyword", nameForDiagnostics: "keyword", classification: "Keyword", associatedValueClass: "Keyword"),
   MiscSpec(name: "RawStringDelimiter", kind: "raw_string_delimiter", nameForDiagnostics: "raw string delimiter"),
   MiscSpec(name: "StringSegment", kind: "string_segment", nameForDiagnostics: "string segment", classification: "StringLiteral"),
   MiscSpec(name: "Yield", kind: "kw_yield", nameForDiagnostics: "yield", text: "yield"),

CodeGeneration/Sources/generate-swiftsyntax/templates/basicformat/BasicFormatFile.swift

@@ -182,7 +182,7 @@ let basicFormatFile = SourceFile {
             }
           }
         }
-        SwitchCase(#"case .contextualKeyword("async"):"#) {
+        SwitchCase("case .contextualKeyword(.async):") {
           ReturnStmt("return true")
         }
         SwitchCase("default:") {

CodeGeneration/Sources/generate-swiftsyntax/templates/swiftparser/DeclarationAttributeFile.swift

@@ -25,11 +25,49 @@ let declarationAttributeFile = SourceFile {
   )
 
   ExtensionDecl("extension Parser") {
-    EnumDecl("enum DeclarationAttribute: SyntaxText, ContextualKeywords") {
+    EnumDecl("enum DeclarationAttribute: RawTokenKindSubset") {
       for attribute in DECL_ATTR_KINDS {
-        EnumCaseDecl("case \(raw: attribute.swiftName) = \"\(raw: attribute.name)\"")
+        EnumCaseDecl("case \(raw: attribute.swiftName)")
+      }
+      EnumCaseDecl("case _spi_available")
+
+      InitializerDecl("init?(lexeme: Lexer.Lexeme)") {
+        SwitchStmt(switchKeyword: .switch, expression: Expr("lexeme")) {
+          for attribute in DECL_ATTR_KINDS {
+            SwitchCase("case RawTokenKindMatch(.\(raw: attribute.name)):") {
+              SequenceExpr("self = .\(raw: attribute.swiftName)")
+            }
+          }
+          SwitchCase("case RawTokenKindMatch(.rethrowsKeyword):") {
+            SequenceExpr("self = .atRethrows")
+          }
+          SwitchCase("case RawTokenKindMatch(._spi_available):") {
+            SequenceExpr("self = ._spi_available")
+          }
+          SwitchCase("default:") {
+            ReturnStmt("return nil")
+          }
+        }
+      }
+
+      VariableDecl(
+        name: IdentifierPattern("rawTokenKind"),
+        type: TypeAnnotation(
+          colon: .colon,
+          type: SimpleTypeIdentifier("RawTokenKind")
+        )
+      ) {
+        SwitchStmt(switchKeyword: .switch, expression: Expr("self")) {
+          for attribute in DECL_ATTR_KINDS {
+            SwitchCase("case .\(raw: attribute.swiftName):") {
+              ReturnStmt("return .contextualKeyword(.\(raw: attribute.name))")
+            }
+          }
+          SwitchCase("case ._spi_available:") {
+            ReturnStmt("return .contextualKeyword(._spi_available)")
+          }
+        }
       }
-      EnumCaseDecl(#"case _spi_available = "_spi_available""#)
     }
   }
 }

CodeGeneration/Sources/generate-swiftsyntax/templates/swiftparser/DeclarationModifierFile.swift

@@ -24,20 +24,17 @@ let declarationModifierFile = SourceFile {
     """
   )
 
-  EnumDecl("enum DeclarationModifier: SyntaxText, ContextualKeywords, RawTokenKindSubset") {
+  EnumDecl("enum DeclarationModifier: RawTokenKindSubset") {
     for attribute in DECL_MODIFIER_KINDS {
-      EnumCaseDecl("case \(raw: attribute.swiftName) = \"\(raw: attribute.name)\"")
+      EnumCaseDecl("case \(raw: attribute.swiftName)")
     }
     InitializerDecl("init?(lexeme: Lexer.Lexeme)") {
-      SwitchStmt(switchKeyword: .switch, expression: Expr("lexeme.tokenKind")) {
-        for attribute in DECL_MODIFIER_KINDS where attribute.swiftName.hasSuffix("Keyword") {
-          SwitchCase("case .\(raw: attribute.swiftName):") {
+      SwitchStmt(expression: Expr("lexeme")) {
+        for attribute in DECL_MODIFIER_KINDS {
+          SwitchCase("case RawTokenKindMatch(.\(raw: attribute.swiftName)):") {
             SequenceExpr("self = .\(raw: attribute.swiftName)")
           }
         }
-        SwitchCase("case .identifier:") {
-          FunctionCallExpr("self.init(rawValue: lexeme.tokenText)")
-        }
         SwitchCase("default:") {
           ReturnStmt("return nil")
         }
@@ -51,38 +48,17 @@ let declarationModifierFile = SourceFile {
         type: SimpleTypeIdentifier("RawTokenKind")
       )
     ) {
-      SwitchStmt(switchKeyword: .switch, expression: Expr("self")) {
+      SwitchStmt(expression: Expr("self")) {
         for attribute in DECL_MODIFIER_KINDS {
           SwitchCase("case .\(raw: attribute.swiftName):") {
             if attribute.swiftName.hasSuffix("Keyword") {
               ReturnStmt("return .\(raw: attribute.swiftName)")
             } else {
-              ReturnStmt("return .identifier")
+              ReturnStmt("return .contextualKeyword(.\(raw: attribute.swiftName))")
             }
           }
         }
       }
     }
-
-
-    VariableDecl(
-      name: IdentifierPattern("contextualKeyword"),
-      type: TypeAnnotation(
-        colon: .colon,
-        type: OptionalType("SyntaxText?")
-      )
-    ) {
-      SwitchStmt(switchKeyword: .switch, expression: Expr("self")) {
-        for attribute in DECL_MODIFIER_KINDS where !attribute.swiftName.hasSuffix("Keyword") {
-          SwitchCase("case .\(raw: attribute.swiftName):") {
-            ReturnStmt("return \"\(raw: attribute.name)\"")
-          }
-        }
-
-        SwitchCase("default:") {
-          ReturnStmt("return nil")
-        }
-      }
-    }
   }
 }

CodeGeneration/Sources/generate-swiftsyntax/templates/swiftparser/TypeAttributeFile.swift

@@ -25,9 +25,38 @@ let typeAttributeFile = SourceFile {
   )
 
   ExtensionDecl("extension Parser") {
-    EnumDecl("enum TypeAttribute: SyntaxText, ContextualKeywords") {
+    EnumDecl("enum TypeAttribute: RawTokenKindSubset") {
       for attribute in TYPE_ATTR_KINDS {
-        EnumCaseDecl("case \(raw: attribute.name) = \"\(raw: attribute.name)\"")
+        EnumCaseDecl("case \(raw: attribute.name)")
+      }
+
+      InitializerDecl("init?(lexeme: Lexer.Lexeme)") {
+        SwitchStmt(switchKeyword: .switch, expression: Expr("lexeme")) {
+          for attribute in TYPE_ATTR_KINDS {
+            SwitchCase("case RawTokenKindMatch(.\(raw: attribute.name)):") {
+              SequenceExpr("self = .\(raw: attribute.swiftName)")
+            }
+          }
+          SwitchCase("default:") {
+            ReturnStmt("return nil")
+          }
+        }
+      }
+
+      VariableDecl(
+        name: IdentifierPattern("rawTokenKind"),
+        type: TypeAnnotation(
+          colon: .colon,
+          type: SimpleTypeIdentifier("RawTokenKind")
+        )
+      ) {
+        SwitchStmt(switchKeyword: .switch, expression: Expr("self")) {
+          for attribute in TYPE_ATTR_KINDS {
+            SwitchCase("case .\(raw: attribute.swiftName):") {
+              ReturnStmt("return .contextualKeyword(.\(raw: attribute.name))")
+            }
+          }
+        }
       }
     }
   }

CodeGeneration/Sources/generate-swiftsyntax/templates/swiftsyntax/SyntaxTraitsFile.swift

@@ -25,7 +25,7 @@ let syntaxTraitsFile = SourceFileSyntax {
 
       for child in trait.children {
         VariableDeclSyntax("var \(raw: child.swiftName): \(raw: child.typeName)\(raw: child.isOptional ? "?" : "") { get }")
-        FunctionDeclSyntax("func with\(raw: child.name)(_ newChild: \(raw: child.typeName)?) -> Self")
+        FunctionDeclSyntax("func with\(raw: child.name)(_ newChild: \(raw: child.typeName)\(raw: child.isOptional ? "?" : "")) -> Self")
       }
     }