Stop using TokenList and NonEmptyTokenList

CodeGeneration/Sources/SyntaxSupport/gyb_generated/AttributeNodes.swift

@@ -13,11 +13,6 @@
 //===----------------------------------------------------------------------===//
 
 public let ATTRIBUTE_NODES: [Node] = [
-  Node(name: "TokenList",
-       nameForDiagnostics: "token list",
-       kind: "SyntaxCollection",
-       element: "Token"),
-
   Node(name: "Attribute",
        nameForDiagnostics: "attribute",
        description: "An `@` attribute.",

CodeGeneration/Sources/SyntaxSupport/gyb_generated/CommonNodes.swift

@@ -89,20 +89,15 @@ public let COMMON_NODES: [Node] = [
                  Child(name: "Stmt",
                        kind: "Stmt"),
                  Child(name: "Expr",
-                       kind: "Expr"),
-                 Child(name: "TokenList",
-                       kind: "TokenList")
+                       kind: "Expr")
                ]),
          Child(name: "Semicolon",
                kind: "SemicolonToken",
                description: "If present, the trailing semicolon at the end of the item.",
                isOptional: true,
                tokenChoices: [
                  "Semicolon"
-               ]),
-         Child(name: "ErrorTokens",
-               kind: "Syntax",
-               isOptional: true)
+               ])
        ],
        omitWhenEmpty: true),
 

CodeGeneration/Sources/SyntaxSupport/gyb_generated/ExprNodes.swift

@@ -659,6 +659,37 @@ public let EXPR_NODES: [Node] = [
                kind: "Type")
        ]),
 
+  Node(name: "ClosureCaptureItemSpecifier",
+       nameForDiagnostics: "closure capture specifier",
+       kind: "Syntax",
+       children: [
+         Child(name: "Specifier",
+               kind: "ContextualKeywordToken",
+               textChoices: [
+                 "weak",
+                 "unowned"
+               ]),
+         Child(name: "LeftParen",
+               kind: "LeftParenToken",
+               isOptional: true,
+               tokenChoices: [
+                 "LeftParen"
+               ]),
+         Child(name: "Detail",
+               kind: "ContextualKeywordToken",
+               isOptional: true,
+               textChoices: [
+                 "safe",
+                 "unsafe"
+               ]),
+         Child(name: "RightParen",
+               kind: "RightParenToken",
+               isOptional: true,
+               tokenChoices: [
+                 "RightParen"
+               ])
+       ]),
+
   Node(name: "ClosureCaptureItem",
        nameForDiagnostics: "closure capture item",
        kind: "Syntax",
@@ -667,9 +698,8 @@ public let EXPR_NODES: [Node] = [
        ],
        children: [
          Child(name: "Specifier",
-               kind: "TokenList",
-               isOptional: true,
-               collectionElementName: "SpecifierToken"),
+               kind: "ClosureCaptureItemSpecifier",
+               isOptional: true),
          Child(name: "Name",
                kind: "IdentifierToken",
                isOptional: true,

Sources/SwiftParser/Expressions.swift

@@ -2333,39 +2333,42 @@ extension Parser {
   }
 
   @_spi(RawSyntax)
-  public mutating func parseClosureCaptureSpecifiers() -> RawTokenListSyntax? {
-    var specifiers = [RawTokenSyntax]()
-    do {
-      // Check for the strength specifier: "weak", "unowned", or
-      // "unowned(safe/unsafe)".
-      if let weakKeyword = self.consume(if: .keyword(.weak)) {
-        specifiers.append(weakKeyword)
-      } else if let unownedKeyword = self.consume(if: .keyword(.unowned)) {
-        specifiers.append(unownedKeyword)
-        if let lparen = self.consume(if: .leftParen) {
-          specifiers.append(lparen)
-          if self.currentToken.tokenText == "safe" {
-            specifiers.append(self.expectWithoutRecovery(.keyword(.safe)))
-          } else {
-            specifiers.append(self.expectWithoutRecovery(.keyword(.unsafe)))
-          }
-          specifiers.append(self.expectWithoutRecovery(.rightParen))
-        }
-      } else if self.at(.identifier) || self.at(.keyword(.self)) {
-        let next = self.peek()
-        // "x = 42", "x," and "x]" are all strong captures of x.
-        guard
-          next.rawTokenKind == .equal || next.rawTokenKind == .comma
-            || next.rawTokenKind == .rightSquareBracket || next.rawTokenKind == .period
-        else {
-          return nil
-        }
+  public mutating func parseClosureCaptureSpecifiers() -> RawClosureCaptureItemSpecifierSyntax? {
+    // Check for the strength specifier: "weak", "unowned", or
+    // "unowned(safe/unsafe)".
+    if let weakContextualKeyword = self.consume(if: .keyword(.weak)) {
+      return RawClosureCaptureItemSpecifierSyntax(
+        specifier: weakContextualKeyword,
+        leftParen: nil,
+        detail: nil,
+        rightParen: nil,
+        arena: self.arena
+      )
+    } else if let unownedContextualKeyword = self.consume(if: .keyword(.unowned)) {
+      if let lparen = self.consume(if: .leftParen) {
+        let (unexpectedBeforeDetail, detail) = self.expectAny([.keyword(.safe), .keyword(.unsafe)], default: .keyword(.safe))
+        let (unexpectedBeforeRParen, rparen) = self.expect(.rightParen)
+        return RawClosureCaptureItemSpecifierSyntax(
+          specifier: unownedContextualKeyword,
+          leftParen: lparen,
+          unexpectedBeforeDetail,
+          detail: detail,
+          unexpectedBeforeRParen,
+          rightParen: rparen,
+          arena: self.arena
+        )
       } else {
-        return nil
+        return RawClosureCaptureItemSpecifierSyntax(
+          specifier: unownedContextualKeyword,
+          leftParen: nil,
+          detail: nil,
+          rightParen: nil,
+          arena: self.arena
+        )
       }
+    } else {
+      return nil
     }
-    // Squash all tokens, if any, as the specifier of the captured item.
-    return RawTokenListSyntax(elements: specifiers, arena: self.arena)
   }
 }
 

Sources/SwiftParser/TopLevel.swift

@@ -73,9 +73,7 @@ extension Parser {
           item: .init(lastItem.item)!,
           lastItem.unexpectedBetweenItemAndSemicolon,
           semicolon: self.missingToken(.semicolon, text: nil),
-          lastItem.unexpectedBetweenSemicolonAndErrorTokens,
-          errorTokens: lastItem.errorTokens,
-          lastItem.unexpectedAfterErrorTokens,
+          lastItem.unexpectedAfterSemicolon,
           arena: self.arena
         )
       }
@@ -156,7 +154,6 @@ extension Parser {
         remainingTokens,
         item: .expr(RawExprSyntax(RawMissingExprSyntax(arena: self.arena))),
         semicolon: nil,
-        errorTokens: nil,
         arena: self.arena
       )
     }
@@ -168,7 +165,6 @@ extension Parser {
         RawUnexpectedNodesSyntax(elements: [RawSyntax(switchCase)], arena: self.arena),
         item: .expr(RawExprSyntax(RawMissingExprSyntax(arena: self.arena))),
         semicolon: nil,
-        errorTokens: nil,
         arena: self.arena
       )
     }
@@ -189,7 +185,6 @@ extension Parser {
       item: item,
       semicolon: semi,
       RawUnexpectedNodesSyntax(trailingSemis, arena: self.arena),
-      errorTokens: nil,
       arena: self.arena
     )
   }
@@ -210,9 +205,7 @@ extension Parser {
             item: .init(lastElement.item)!,
             lastElement.unexpectedBetweenItemAndSemicolon,
             semicolon: parser.missingToken(.semicolon, text: nil),
-            lastElement.unexpectedBetweenSemicolonAndErrorTokens,
-            errorTokens: lastElement.errorTokens,
-            lastElement.unexpectedAfterErrorTokens,
+            lastElement.unexpectedAfterSemicolon,
             arena: parser.arena
           )
         } else {

Sources/SwiftSyntax/Documentation.docc/gyb_generated/SwiftSyntax.md

@@ -283,6 +283,7 @@ allows Swift tools to parse, inspect, generate, and transform Swift source code.
 - <doc:SwiftSyntax/TupleExprElementSyntax>
 - <doc:SwiftSyntax/ArrayElementSyntax>
 - <doc:SwiftSyntax/DictionaryElementSyntax>
+- <doc:SwiftSyntax/ClosureCaptureItemSpecifierSyntax>
 - <doc:SwiftSyntax/ClosureCaptureItemSyntax>
 - <doc:SwiftSyntax/ClosureCaptureItemListSyntax>
 - <doc:SwiftSyntax/ClosureCaptureSignatureSyntax>
@@ -338,7 +339,6 @@ allows Swift tools to parse, inspect, generate, and transform Swift source code.
 - <doc:SwiftSyntax/PrecedenceGroupNameElementSyntax>
 - <doc:SwiftSyntax/PrecedenceGroupAssignmentSyntax>
 - <doc:SwiftSyntax/PrecedenceGroupAssociativitySyntax>
-- <doc:SwiftSyntax/TokenListSyntax>
 - <doc:SwiftSyntax/AttributeSyntax>
 - <doc:SwiftSyntax/AttributeListSyntax>
 - <doc:SwiftSyntax/SpecializeAttributeSpecListSyntax>