Remove text_choices and model the text choices in token_choices

Author: ahoppen

CodeGeneration/Sources/SyntaxSupport/KeywordSpec.swift

@@ -98,13 +98,15 @@ public let KEYWORDS: [KeywordSpec] = [
   KeywordSpec("available"),
   KeywordSpec("await"),
   KeywordSpec("before"),
+  KeywordSpec("block"),
   KeywordSpec("break", isLexerClassified: true, requiresTrailingSpace: true),
   KeywordSpec("case", isLexerClassified: true, requiresTrailingSpace: true),
   KeywordSpec("catch", isLexerClassified: true, requiresLeadingSpace: true),
   KeywordSpec("class", isLexerClassified: true, requiresTrailingSpace: true),
   KeywordSpec("continue", isLexerClassified: true, requiresTrailingSpace: true),
   KeywordSpec("convenience"),
   KeywordSpec("convention"),
+  KeywordSpec("cType"),
   KeywordSpec("default", isLexerClassified: true),
   KeywordSpec("defer", isLexerClassified: true, requiresTrailingSpace: true),
   KeywordSpec("deinit", isLexerClassified: true, requiresTrailingSpace: true),
@@ -124,6 +126,7 @@ public let KEYWORDS: [KeywordSpec] = [
   KeywordSpec("extension", isLexerClassified: true, requiresTrailingSpace: true),
   KeywordSpec("fallthrough", isLexerClassified: true, requiresTrailingSpace: true),
   KeywordSpec("false", isLexerClassified: true),
+  KeywordSpec("file"),
   KeywordSpec("fileprivate", isLexerClassified: true, requiresTrailingSpace: true),
   KeywordSpec("final"),
   KeywordSpec("for", isLexerClassified: true, requiresTrailingSpace: true),
@@ -145,7 +148,9 @@ public let KEYWORDS: [KeywordSpec] = [
   KeywordSpec("isolated"),
   KeywordSpec("kind"),
   KeywordSpec("lazy"),
+  KeywordSpec("left"),
   KeywordSpec("let", isLexerClassified: true, requiresTrailingSpace: true),
+  KeywordSpec("line"),
   KeywordSpec("lowerThan"),
   KeywordSpec("macro"),
   KeywordSpec("message"),

CodeGeneration/Sources/SyntaxSupport/gyb_generated/TypeNodes.swift

@@ -104,8 +104,7 @@ public let TYPE_NODES: [Node] = [
        kind: "Type",
        children: [
          Child(name: "SomeOrAnySpecifier",
-               kind: .token(choices: [.keyword(text: "some"), .keyword(text: "any")]),
-               classification: "Keyword"),
+               kind: .token(choices: [.keyword(text: "some"), .keyword(text: "any")])),
          Child(name: "BaseType",
                kind: .node(kind: "Type"))
        ]),
@@ -127,7 +126,7 @@ public let TYPE_NODES: [Node] = [
          Child(name: "Type",
                kind: .node(kind: "Type")),
          Child(name: "Ampersand",
-               kind: .token(choices: [.keyword(text: "&")]),
+               kind: .node(kind: "Token"),
                isOptional: true)
        ]),
 

CodeGeneration/Sources/SyntaxSupport/gyb_helpers/utils.py

@@ -55,11 +55,7 @@ def make_swift_child(child, spaces):
   parameter_spaces = 8
   parameters = ['name: "%s"' % child.name]
 
-  if child.text_choices:
-    mapped_choices = [f'.keyword(text: "{text}")' for text in child.text_choices]
-    joined_choices = ', '.join(mapped_choices)
-    kind = f'.token(choices: [{joined_choices}])'
-  elif child.token_choices:
+  if child.token_choices:
     mapped_choices = [f'.token(tokenKind: "{choice.name}Token")' for (choice, text) in child.token_choices if text is None]
     mapped_choices += [f'.keyword(text: "{text}")' for (choice, text) in child.token_choices if text is not None]
     joined_choices = ', '.join(mapped_choices)

Sources/IDEUtils/generated/SyntaxClassification.swift

@@ -91,8 +91,6 @@ extension SyntaxClassification {
       switch (parentKind, indexInParent) {
       case (.availabilityVersionRestriction, 1): 
         return (.keyword, false)
-      case (.constrainedSugarType, 1): 
-        return (.keyword, false)
       case (.declModifier, 1): 
         return (.attribute, false)
       case (.expressionSegment, 5): 

Sources/SwiftSyntax/generated/Keyword.swift

@@ -147,6 +147,7 @@ public enum Keyword: UInt8, Hashable {
 
   case before
 
+<<<<<<< HEAD
   case `break`
 
   case `case`
@@ -390,6 +391,261 @@ public enum Keyword: UInt8, Hashable {
   case wrt
 
   case yield
+=======
+  case block = 61
+  
+  case `break` = 62
+  
+  case `case` = 63
+  
+  case `catch` = 64
+  
+  case `class` = 65
+  
+  case `continue` = 66
+  
+  case convenience = 67
+  
+  case convention = 68
+  
+  case cType = 69
+  
+  case `default` = 70
+  
+  case `defer` = 71
+  
+  case `deinit` = 72
+  
+  case deprecated = 73
+  
+  case derivative = 74
+  
+  case didSet = 75
+  
+  case differentiable = 76
+  
+  case distributed = 77
+  
+  case `do` = 78
+  
+  case dynamic = 79
+  
+  case each = 80
+  
+  case `else` = 81
+  
+  case `enum` = 82
+  
+  case escaping = 83
+  
+  case exclusivity = 84
+  
+  case exported = 85
+  
+  case `extension` = 86
+  
+  case `fallthrough` = 87
+  
+  case `false` = 88
+  
+  case file = 89
+  
+  case `fileprivate` = 90
+  
+  case final = 91
+  
+  case `for` = 92
+  
+  case `func` = 93
+  
+  case get = 94
+  
+  case `guard` = 95
+  
+  case higherThan = 96
+  
+  case `if` = 97
+  
+  case `import` = 98
+  
+  case `in` = 99
+  
+  case indirect = 100
+  
+  case infix = 101
+  
+  case `init` = 102
+  
+  case inline = 103
+  
+  case `inout` = 104
+  
+  case `internal` = 105
+  
+  case introduced = 106
+  
+  case `is` = 107
+  
+  case isolated = 108
+  
+  case kind = 109
+  
+  case lazy = 110
+  
+  case left = 111
+  
+  case `let` = 112
+  
+  case line = 113
+  
+  case lowerThan = 114
+  
+  case macro = 115
+  
+  case message = 116
+  
+  case metadata = 117
+  
+  case module = 118
+  
+  case mutableAddressWithNativeOwner = 119
+  
+  case mutableAddressWithOwner = 120
+  
+  case mutating = 121
+  
+  case `nil` = 122
+  
+  case noasync = 123
+  
+  case noDerivative = 124
+  
+  case noescape = 125
+  
+  case nonisolated = 126
+  
+  case nonmutating = 127
+  
+  case objc = 128
+  
+  case obsoleted = 129
+  
+  case of = 130
+  
+  case open = 131
+  
+  case `operator` = 132
+  
+  case optional = 133
+  
+  case override = 134
+  
+  case package = 135
+  
+  case postfix = 136
+  
+  case `precedencegroup` = 137
+  
+  case prefix = 138
+  
+  case `private` = 139
+  
+  case `Protocol` = 140
+  
+  case `protocol` = 141
+  
+  case `public` = 142
+  
+  case reasync = 143
+  
+  case renamed = 144
+  
+  case `repeat` = 145
+  
+  case required = 146
+  
+  case `rethrows` = 147
+  
+  case `return` = 148
+  
+  case reverse = 149
+  
+  case safe = 150
+  
+  case `self` = 151
+  
+  case `Self` = 152
+  
+  case Sendable = 153
+  
+  case set = 154
+  
+  case some = 155
+  
+  case sourceFile = 156
+  
+  case spi = 157
+  
+  case spiModule = 158
+  
+  case `static` = 159
+  
+  case `struct` = 160
+  
+  case `subscript` = 161
+  
+  case `super` = 162
+  
+  case swift = 163
+  
+  case `switch` = 164
+  
+  case target = 165
+  
+  case `throw` = 166
+  
+  case `throws` = 167
+  
+  case transpose = 168
+  
+  case `true` = 169
+  
+  case `try` = 170
+  
+  case `Type` = 171
+  
+  case `typealias` = 172
+  
+  case unavailable = 173
+  
+  case unchecked = 174
+  
+  case unowned = 175
+  
+  case unsafe = 176
+  
+  case unsafeAddress = 177
+  
+  case unsafeMutableAddress = 178
+  
+  case `var` = 179
+  
+  case visibility = 180
+  
+  case weak = 181
+  
+  case `where` = 182
+  
+  case `while` = 183
+  
+  case willSet = 184
+  
+  case witness_method = 185
+  
+  case wrt = 186
+  
+  case yield = 187
+>>>>>>> a7a0ff37 (Remove text_choices and model the text choides in token_choices)
 
   @_spi(RawSyntax) public init?(_ text: SyntaxText) {
     switch text.count {
@@ -449,6 +705,8 @@ public enum Keyword: UInt8, Hashable {
         self = .`else`
       case "enum": 
         self = .`enum`
+      case "file": 
+        self = .file
       case "func": 
         self = .`func`
       case "init": 
@@ -457,6 +715,10 @@ public enum Keyword: UInt8, Hashable {
         self = .kind
       case "lazy": 
         self = .lazy
+      case "left": 
+        self = .left
+      case "line": 
+        self = .line
       case "objc": 
         self = .objc
       case "open": 
@@ -490,12 +752,16 @@ public enum Keyword: UInt8, Hashable {
         self = .async
       case "await": 
         self = .await
+      case "block": 
+        self = .block
       case "break": 
         self = .`break`
       case "catch": 
         self = .`catch`
       case "class": 
         self = .`class`
+      case "cType": 
+        self = .cType
       case "defer": 
         self = .`defer`
       case "false": 
@@ -999,6 +1265,7 @@ public enum Keyword: UInt8, Hashable {
   /// table is significantly faster. Ideally, we could also get the compiler to
   /// constant-evaluate `Keyword.spi.defaultText` to a `SyntaxText` but I don't
   /// see how that's possible right now.
+<<<<<<< HEAD
   private static let keywordTextLookupTable: [SyntaxText] = [
     "__consuming", 
     "__owned", 
@@ -1184,6 +1451,9 @@ public enum Keyword: UInt8, Hashable {
     "wrt", 
     "yield", 
   ]
+=======
+  private static let keywordTextLookupTable: [SyntaxText] = ["__consuming", "__owned", "__setter_access", "__shared", "_alignment", "_backDeploy", "_borrow", "_cdecl", "_Class", "_compilerInitialized", "_const", "_documentation", "_dynamicReplacement", "_effects", "_expose", "_forward", "_implements", "_linear", "_local", "_modify", "_move", "_NativeClass", "_NativeRefCountedObject", "_noMetadata", "_nonSendable", "_objcImplementation", "_objcRuntimeName", "_opaqueReturnTypeOf", "_optimize", "_originallyDefinedIn", "_PackageDescription", "_private", "_projectedValueProperty", "_read", "_RefCountedObject", "_semantics", "_silgen_name", "_specialize", "_spi", "_spi_available", "_swift_native_objc_runtime_base", "_Trivial", "_TrivialAtMost", "_typeEraser", "_unavailableFromAsync", "_UnknownLayout", "actor", "addressWithNativeOwner", "addressWithOwner", "any", "Any", "as", "assignment", "associatedtype", "associativity", "async", "autoclosure", "availability", "available", "await", "before", "block", "break", "case", "catch", "class", "continue", "convenience", "convention", "cType", "default", "defer", "deinit", "deprecated", "derivative", "didSet", "differentiable", "distributed", "do", "dynamic", "each", "else", "enum", "escaping", "exclusivity", "exported", "extension", "fallthrough", "false", "file", "fileprivate", "final", "for", "func", "get", "guard", "higherThan", "if", "import", "in", "indirect", "infix", "init", "inline", "inout", "internal", "introduced", "is", "isolated", "kind", "lazy", "left", "let", "line", "lowerThan", "macro", "message", "metadata", "module", "mutableAddressWithNativeOwner", "mutableAddressWithOwner", "mutating", "nil", "noasync", "noDerivative", "noescape", "nonisolated", "nonmutating", "objc", "obsoleted", "of", "open", "operator", "optional", "override", "package", "postfix", "precedencegroup", "prefix", "private", "Protocol", "protocol", "public", "reasync", "renamed", "repeat", "required", "rethrows", "return", "reverse", "safe", "self", "Self", "Sendable", "set", "some", "sourceFile", "spi", "spiModule", "static", "struct", "subscript", "super", "swift", "switch", "target", "throw", "throws", "transpose", "true", "try", "Type", "typealias", "unavailable", "unchecked", "unowned", "unsafe", "unsafeAddress", "unsafeMutableAddress", "var", "visibility", "weak", "where", "while", "willSet", "witness_method", "wrt", "yield"]
+>>>>>>> a7a0ff37 (Remove text_choices and model the text choides in token_choices)
 
   @_spi(RawSyntax)
   public var defaultText: SyntaxText {