Use a dedicated Token type in SwiftSyntaxBuilder

At the moment, tokens were a little bit of an oddity in SwiftSyntaxBuilder: All nodes were represented by dedicated types in SwiftSyntaxBuilder but tokens were represented by the `TokenSyntax` from the SwiftSyntax module. This has lead to a few problems:
- We needed to add extensions on `TokenSyntax` to add convienience properties that are only designed for SwiftSyntaxBuilder
- `TokenSyntax` was not `SyntaxBuildable` and thus couldn’t be used as elements in unexpected node collections

In general, it seems cleaner if we just add a custom `Token` type in SwiftSyntaxBuilder.

rdar://99030016

Author: ahoppen

Package.swift

@@ -140,7 +140,7 @@ let package = Package(
           "PatternNodes.swift.gyb",
           "StmtNodes.swift.gyb",
           "SyntaxBaseKinds.swift.gyb",
-          "Tokens.swift.gyb",
+          "TokenSpec.swift.gyb",
           "Traits.swift.gyb",
           "Trivia.swift.gyb",
           "TypeNodes.swift.gyb"

Sources/SwiftSyntaxBuilder/Convenience Initializers/CatchClauseConvenienceInitializer.swift

@@ -21,7 +21,7 @@ extension CatchClause {
   ) {
     self.init(
       leadingTrivia: leadingTrivia,
-      catchKeyword: .catchKeyword(trailingTrivia: catchItems.elements.isEmpty ? [] : .space),
+      catchKeyword: .catch.withTrailingTrivia(catchItems.elements.isEmpty ? [] : .space),
       catchItems: catchItems,
       body: bodyBuilder()
     )

Sources/SwiftSyntaxBuilder/Convenience Initializers/DictionaryExprConvenienceInitializers.swift

@@ -16,14 +16,14 @@ extension DictionaryExpr {
   /// A convenience initializer that allows passing in members using a result builder
   /// instead of having to wrap them in a `DictionaryElementList`.
   public init(
-    leftSquare: TokenSyntax = .`leftSquareBracket`,
-    rightSquare: TokenSyntax = .`rightSquareBracket`,
+    leftSquare: Token = .`leftSquareBracket`,
+    rightSquare: Token = .`rightSquareBracket`,
     @DictionaryElementListBuilder contentBuilder: () -> ExpressibleAsDictionaryElementList = { [] }
   ) {
     let elementList = contentBuilder().createDictionaryElementList()
     self.init(
       leftSquare: leftSquare,
-      content: elementList.elements.isEmpty ? TokenSyntax.colonToken(trailingTrivia: []) : elementList,
+      content: elementList.elements.isEmpty ? Token.colon.withTrailingTrivia([]) : elementList,
       rightSquare: rightSquare
     )
   }

Sources/SwiftSyntaxBuilder/Convenience Initializers/IfStmtConvenienceInitializers.swift

@@ -27,7 +27,7 @@ public extension IfStmt {
       leadingTrivia: leadingTrivia,
       conditions: conditions,
       body: body(),
-      elseKeyword: generatedElseBody == nil ? nil : TokenSyntax.elseKeyword(leadingTrivia: .space, trailingTrivia: []),
+      elseKeyword: generatedElseBody == nil ? nil : Token.else.withLeadingTrivia(.space).withTrailingTrivia([]),
       elseBody: generatedElseBody.map { CodeBlock(statements: $0) }
     )
   }

Sources/SwiftSyntaxBuilder/Convenience Initializers/MemberAccessExprConvenienceInitializers.swift

@@ -16,7 +16,7 @@ extension MemberAccessExpr {
   /// Creates a `MemberAccessExpr` using the provided parameters.
   public init(
     base: ExpressibleAsExprBuildable? = nil,
-    dot: TokenSyntax = .period,
+    dot: Token = .period,
     name: String,
     declNameArguments: ExpressibleAsDeclNameArguments? = nil
   ) {

Sources/SwiftSyntaxBuilder/Convenience Initializers/StringLiteralExprConvenienceInitializers.swift

@@ -26,13 +26,13 @@ private let rawStringPotentialEscapesPattern = try! NSRegularExpression(
 extension StringLiteralExpr {
   /// Creates a string literal, optionally specifying quotes and delimiters.
   public init(
-    openDelimiter: TokenSyntax? = nil,
-    openQuote: TokenSyntax = .stringQuote,
+    openDelimiter: Token? = nil,
+    openQuote: Token = .stringQuote,
     _ value: String,
-    closeQuote: TokenSyntax = .stringQuote,
-    closeDelimiter: TokenSyntax? = nil
+    closeQuote: Token = .stringQuote,
+    closeDelimiter: Token? = nil
   ) {
-    let content = TokenSyntax.stringSegment(value)
+    let content = Token.stringSegment(value)
     let segment = StringSegment(content: content)
     let segments = StringLiteralSegments([segment])
 
@@ -57,7 +57,7 @@ extension StringLiteralExpr {
       .max() ?? 0
 
     // Use a delimiter that is exactly one longer
-    let delimiter = TokenSyntax.rawStringDelimiter(String(repeating: "#", count: 1 + maxPoundCount))
+    let delimiter = Token.rawStringDelimiter(String(repeating: "#", count: 1 + maxPoundCount))
 
     self.init(
       openDelimiter: delimiter,

Sources/SwiftSyntaxBuilder/Convenience Initializers/TernaryExprConvenienceInitializers.swift

@@ -20,9 +20,9 @@ extension TernaryExpr {
   ) {
     self.init(
       conditionExpression: condition,
-      questionMark: .infixQuestionMarkToken(leadingTrivia: .space, trailingTrivia: .space),
+      questionMark: .infixQuestionMark.withLeadingTrivia(.space).withTrailingTrivia(.space),
       firstChoice: firstChoice,
-      colonMark: .colonToken(leadingTrivia: .space),
+      colonMark: .colon.withLeadingTrivia(.space),
       secondChoice: secondChoice
     )
   }

Sources/SwiftSyntaxBuilder/Convenience Initializers/TupleExprElementConvenienceInitializers.swift

@@ -17,6 +17,6 @@ public extension TupleExprElement {
   /// The presence of the colon will be inferred based on the presence of the label.
   init(label: String? = nil, expression: ExpressibleAsExprBuildable) {
     self.init(
-      label: label.map { TokenSyntax.identifier($0) }, colon: label == nil ? nil : .colon, expression: expression)
+      label: label.map { Token.identifier($0) }, colon: label == nil ? nil : Token.colon, expression: expression)
   }
 }

Sources/SwiftSyntaxBuilder/Convenience Initializers/VariableDeclConvenienceInitializers.swift

@@ -17,7 +17,7 @@ extension VariableDecl {
   public init(
     leadingTrivia: Trivia = [],
     modifiers: ModifierList? = nil,
-    _ letOrVarKeyword: TokenSyntax,
+    _ letOrVarKeyword: Token,
     name: ExpressibleAsIdentifierPattern,
     type: ExpressibleAsTypeAnnotation? = nil,
     initializer: ExpressibleAsInitializerClause? = nil

Sources/SwiftSyntaxBuilder/HasTrailingComma.swift

@@ -13,7 +13,7 @@
 import SwiftSyntax
 
 protocol HasTrailingComma {
-  var trailingComma: TokenSyntax? { get }
+  var trailingComma: Token? { get }
 
   /// Returns this node overriding presence of the trailing comma
   func withTrailingComma(_ withComma: Bool) -> Self

Sources/SwiftSyntaxBuilder/generated/TokenSyntax.swift renamed to Sources/SwiftSyntaxBuilder/Token.swift

@@ -1,6 +1,3 @@
-
-//// Automatically Generated by generate-swift-syntax-builder
-//// Do Not Edit Directly!
 //===----------------------------------------------------------------------===//
 //
 // This source file is part of the Swift.org open source project
@@ -14,34 +11,65 @@
 //===----------------------------------------------------------------------===//
 
 import SwiftSyntax
-extension TokenSyntax: ExpressibleAsTokenList, ExpressibleAsNonEmptyTokenList, ExpressibleAsBinaryOperatorExpr, ExpressibleAsDeclModifier, ExpressibleAsIdentifierExpr {
+
+/// Represents `TokenSyntax` in `SwiftSyntaxBuilder`.
+/// At the moment, this just wraps `TokenSyntax`, but we can make it store just the information necessary to build a `TokenSyntax` in the future.
+public struct Token: SyntaxBuildable, ExpressibleAsTokenList, ExpressibleAsNonEmptyTokenList, ExpressibleAsBinaryOperatorExpr, ExpressibleAsDeclModifier, ExpressibleAsIdentifierExpr {
+  let tokenSyntax: TokenSyntax
+
+  var text: String {
+    return tokenSyntax.text
+  }
+
+  init(tokenSyntax: TokenSyntax) {
+    self.tokenSyntax = tokenSyntax
+  }
+
+  public func withLeadingTrivia(_ leadingTrivia: Trivia) -> Token {
+    return Token(tokenSyntax: tokenSyntax.withLeadingTrivia(leadingTrivia))
+  }
+
+  public func withTrailingTrivia(_ leadingTrivia: Trivia) -> Token {
+    return Token(tokenSyntax: tokenSyntax.withTrailingTrivia(leadingTrivia))
+  }
+
+  public func buildToken() -> TokenSyntax {
+    return tokenSyntax
+  }
+
+  public func buildSyntax(format: Format, leadingTrivia: Trivia?) -> Syntax {
+    return Syntax(tokenSyntax)
+  }
+
   /// Conformance to ExpressibleAsTokenList
   public func createTokenList() -> TokenList {
     return TokenList([self])
   }
+
   /// Conformance to ExpressibleAsNonEmptyTokenList
   public func createNonEmptyTokenList() -> NonEmptyTokenList {
     return NonEmptyTokenList([self])
   }
+
   /// Conformance to ExpressibleAsBinaryOperatorExpr
   public func createBinaryOperatorExpr() -> BinaryOperatorExpr {
     return BinaryOperatorExpr(operatorToken: self)
   }
+
   /// Conformance to ExpressibleAsDeclModifier
   public func createDeclModifier() -> DeclModifier {
     return DeclModifier(name: self)
   }
+
   /// Conformance to ExpressibleAsIdentifierExpr
   public func createIdentifierExpr() -> IdentifierExpr {
     return IdentifierExpr(identifier: self)
   }
-}
 
-/// `TokenSyntax` conforms to `SyntaxBuildable` and `ExprBuildable` via different paths, so we need to pick one default conversion path to create an `ExprSyntax` (and `Syntax`) from a `String`. We choose `IdentifierExpr`.
-extension TokenSyntax {
   public func createSyntaxBuildable() -> SyntaxBuildable {
     return createIdentifierExpr()
   }
+
   public func createExprBuildable() -> ExprBuildable {
     return createIdentifierExpr()
   }

Sources/SwiftSyntaxBuilder/generated/BuildableCollectionNodes.swift

@@ -1039,13 +1039,13 @@ extension Array: ExpressibleAsEnumCaseElementList where Element == ExpressibleAs
     return EnumCaseElementList(self)
   }
 }
-/// `IdentifierList` represents a collection of `TokenSyntax`
+/// `IdentifierList` represents a collection of `Token`
 public struct IdentifierList: ExpressibleByArrayLiteral, SyntaxBuildable, ExpressibleAsIdentifierList {
-  let elements: [TokenSyntax]
+  let elements: [Token]
   /// Creates a `IdentifierList` with the provided list of elements.
   /// - Parameters:
-  ///   - elements: A list of `TokenSyntax`
-  public init (_ elements: [TokenSyntax]) {
+  ///   - elements: A list of `Token`
+  public init (_ elements: [Token]) {
     self.elements = elements
   }
   /// Creates a new `IdentifierList` by flattening the elements in `lists`
@@ -1054,11 +1054,13 @@ public struct IdentifierList: ExpressibleByArrayLiteral, SyntaxBuildable, Expres
       $0.createIdentifierList().elements
     }
   }
-  public init (arrayLiteral elements: TokenSyntax...) {
+  public init (arrayLiteral elements: Token...) {
     self.init(elements)
   }
   public func buildIdentifierList(format: Format, leadingTrivia: Trivia? = nil) -> IdentifierListSyntax {
-    let result = IdentifierListSyntax(elements)
+    let result = IdentifierListSyntax(elements.map {
+      $0.buildToken()
+    })
     if let leadingTrivia = leadingTrivia {
       return result.withLeadingTrivia((leadingTrivia + (result.leadingTrivia ?? [])).addingSpacingAfterNewlinesIfNeeded())
     } else {
@@ -1080,7 +1082,7 @@ public struct IdentifierList: ExpressibleByArrayLiteral, SyntaxBuildable, Expres
     return self
   }
 }
-extension Array: ExpressibleAsIdentifierList where Element == TokenSyntax {
+extension Array: ExpressibleAsIdentifierList where Element == Token {
   public func createIdentifierList() -> IdentifierList {
     return IdentifierList(self)
   }
@@ -1185,13 +1187,13 @@ extension Array: ExpressibleAsPrecedenceGroupNameList where Element == Expressib
     return PrecedenceGroupNameList(self)
   }
 }
-/// `TokenList` represents a collection of `TokenSyntax`
+/// `TokenList` represents a collection of `Token`
 public struct TokenList: ExpressibleByArrayLiteral, SyntaxBuildable, ExpressibleAsTokenList {
-  let elements: [TokenSyntax]
+  let elements: [Token]
   /// Creates a `TokenList` with the provided list of elements.
   /// - Parameters:
-  ///   - elements: A list of `TokenSyntax`
-  public init (_ elements: [TokenSyntax]) {
+  ///   - elements: A list of `Token`
+  public init (_ elements: [Token]) {
     self.elements = elements
   }
   /// Creates a new `TokenList` by flattening the elements in `lists`
@@ -1200,11 +1202,13 @@ public struct TokenList: ExpressibleByArrayLiteral, SyntaxBuildable, Expressible
       $0.createTokenList().elements
     }
   }
-  public init (arrayLiteral elements: TokenSyntax...) {
+  public init (arrayLiteral elements: Token...) {
     self.init(elements)
   }
   public func buildTokenList(format: Format, leadingTrivia: Trivia? = nil) -> TokenListSyntax {
-    let result = TokenListSyntax(elements)
+    let result = TokenListSyntax(elements.map {
+      $0.buildToken()
+    })
     if let leadingTrivia = leadingTrivia {
       return result.withLeadingTrivia((leadingTrivia + (result.leadingTrivia ?? [])).addingSpacingAfterNewlinesIfNeeded())
     } else {
@@ -1226,18 +1230,18 @@ public struct TokenList: ExpressibleByArrayLiteral, SyntaxBuildable, Expressible
     return self
   }
 }
-extension Array: ExpressibleAsTokenList where Element == TokenSyntax {
+extension Array: ExpressibleAsTokenList where Element == Token {
   public func createTokenList() -> TokenList {
     return TokenList(self)
   }
 }
-/// `NonEmptyTokenList` represents a collection of `TokenSyntax`
+/// `NonEmptyTokenList` represents a collection of `Token`
 public struct NonEmptyTokenList: ExpressibleByArrayLiteral, SyntaxBuildable, ExpressibleAsNonEmptyTokenList {
-  let elements: [TokenSyntax]
+  let elements: [Token]
   /// Creates a `NonEmptyTokenList` with the provided list of elements.
   /// - Parameters:
-  ///   - elements: A list of `TokenSyntax`
-  public init (_ elements: [TokenSyntax]) {
+  ///   - elements: A list of `Token`
+  public init (_ elements: [Token]) {
     self.elements = elements
   }
   /// Creates a new `NonEmptyTokenList` by flattening the elements in `lists`
@@ -1246,11 +1250,13 @@ public struct NonEmptyTokenList: ExpressibleByArrayLiteral, SyntaxBuildable, Exp
       $0.createNonEmptyTokenList().elements
     }
   }
-  public init (arrayLiteral elements: TokenSyntax...) {
+  public init (arrayLiteral elements: Token...) {
     self.init(elements)
   }
   public func buildNonEmptyTokenList(format: Format, leadingTrivia: Trivia? = nil) -> NonEmptyTokenListSyntax {
-    let result = NonEmptyTokenListSyntax(elements)
+    let result = NonEmptyTokenListSyntax(elements.map {
+      $0.buildToken()
+    })
     if let leadingTrivia = leadingTrivia {
       return result.withLeadingTrivia((leadingTrivia + (result.leadingTrivia ?? [])).addingSpacingAfterNewlinesIfNeeded())
     } else {
@@ -1272,7 +1278,7 @@ public struct NonEmptyTokenList: ExpressibleByArrayLiteral, SyntaxBuildable, Exp
     return self
   }
 }
-extension Array: ExpressibleAsNonEmptyTokenList where Element == TokenSyntax {
+extension Array: ExpressibleAsNonEmptyTokenList where Element == Token {
   public func createNonEmptyTokenList() -> NonEmptyTokenList {
     return NonEmptyTokenList(self)
   }