Move Tokens from gyb to codegen

Author: kimdv

CodeGeneration/Sources/generate-swiftsyntax/GenerateSwiftSyntax.swift

@@ -45,7 +45,7 @@ struct GenerateSwiftSyntax: ParsableCommand {
 
   @Flag(help: "Enable verbose output")
   var verbose: Bool = false
-    
+
   func run() throws {
     let templates: [TemplateSpec] = [
       TemplateSpec(sourceFile: basicFormatFile, module: swiftBasicFormatDir, filename: "BasicFormat.swift"),
@@ -62,6 +62,7 @@ struct GenerateSwiftSyntax: ParsableCommand {
       TemplateSpec(sourceFile: syntaxTraitsFile, module: swiftSyntaxDir, filename: "SyntaxTraits.swift"),
       TemplateSpec(sourceFile: syntaxTransformFile, module: swiftSyntaxDir, filename: "SyntaxTransform.swift"),
       TemplateSpec(sourceFile: syntaxVisitorFile, module: swiftSyntaxDir, filename: "SyntaxVisitor.swift"),
+      TemplateSpec(sourceFile: tokensFile, module: swiftSyntaxDir, filename: "Tokens.swift"),
       TemplateSpec(sourceFile: buildableCollectionNodesFile, module: swiftSyntaxBuilderDir, filename: "BuildableCollectionNodes.swift"),
       TemplateSpec(sourceFile: buildableNodesFile, module: swiftSyntaxBuilderDir, filename: "BuildableNodes.swift"),
       TemplateSpec(sourceFile: resultBuildersFile, module: swiftSyntaxBuilderDir, filename: "ResultBuilders.swift"),
@@ -91,7 +92,7 @@ struct GenerateSwiftSyntax: ParsableCommand {
       throw firstError
     }
   }
-  
+
   private func generateTemplate(
     sourceFile: SourceFileSyntax,
     module: String,
@@ -103,12 +104,12 @@ struct GenerateSwiftSyntax: ParsableCommand {
         withIntermediateDirectories: true,
         attributes: nil
       )
-      
+
       let fileURL = destination
         .appendingPathComponent(module)
         .appendingPathComponent("generated")
         .appendingPathComponent(filename)
-      
+
       if verbose {
         print("Generating \(fileURL.path)...")
       }

CodeGeneration/Sources/generate-swiftsyntax/templates/swiftsyntax/TokensFile.swift

@@ -0,0 +1,96 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2022 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import SwiftSyntax
+import SwiftSyntaxBuilder
+import SyntaxSupport
+import Utils
+
+let tokensFile = SourceFile(
+  leadingTrivia: .docLineComment(generateCopyrightHeader(for: "generate-swiftsyntax"))) {
+    FunctionDecl("""
+    fileprivate func defaultTrivia(presence: SourcePresence, trivia: Trivia) -> Trivia {
+      switch presence {
+      case .present:
+        return trivia
+      case .missing:
+        return []
+      }
+    }
+    """)
+    
+    ExtensionDecl("extension TokenSyntax") {
+      for token in SYNTAX_TOKENS {
+        if token.isKeyword {
+          FunctionDecl("""
+          public static func \(raw: token.swiftKind)(
+            leadingTrivia: Trivia = [],
+            trailingTrivia: Trivia = [],
+            presence: SourcePresence = .present
+          ) -> TokenSyntax {
+            return TokenSyntax(
+              .\(raw: token.swiftKind),
+              leadingTrivia: leadingTrivia,
+              trailingTrivia: trailingTrivia,
+              presence: presence
+            )
+          }
+          """)
+        } else if let text = token.text {
+          FunctionDecl("""
+          public static func \(raw: token.swiftKind)Token(
+            leadingTrivia: Trivia = [],
+            trailingTrivia: Trivia = [],
+            presence: SourcePresence = .present
+          ) -> TokenSyntax {
+            return TokenSyntax(
+              .\(raw: token.swiftKind),
+              leadingTrivia: leadingTrivia,
+              trailingTrivia: trailingTrivia,
+              presence: presence
+            )
+          }
+          """)
+        } else {
+          FunctionDecl("""
+          public static func \(raw: token.swiftKind)(
+            _ text: String,
+            leadingTrivia: Trivia = [],
+            trailingTrivia: Trivia = [],
+            presence: SourcePresence = .present
+          ) -> TokenSyntax {
+            return TokenSyntax(
+              .\(raw: token.swiftKind)(text),
+              leadingTrivia: leadingTrivia,
+              trailingTrivia: trailingTrivia,
+              presence: presence
+            )
+          }
+          """)
+        }
+      }
+      
+      FunctionDecl("""
+      public static func eof(
+        leadingTrivia: Trivia = [],
+        presence: SourcePresence = .present
+      ) -> TokenSyntax {
+        return TokenSyntax(
+          .eof,
+          leadingTrivia: leadingTrivia,
+          trailingTrivia: [],
+          presence: presence
+        )
+      }
+      """)
+    }
+  }

Package.swift

@@ -70,7 +70,6 @@ let package = Package(
         "SyntaxNodes.swift.gyb.template",
         "SyntaxRewriter.swift.gyb",
         "TokenKind.swift.gyb",
-        "Tokens.swift.gyb",
         "Trivia.swift.gyb",
       ],
       swiftSettings: swiftSyntaxSwiftSettings

Sources/SwiftSyntax/CMakeLists.txt

@@ -43,7 +43,7 @@ add_swift_host_library(SwiftSyntax
   generated/SyntaxTransform.swift
   generated/SyntaxVisitor.swift
   gyb_generated/TokenKind.swift
-  gyb_generated/Tokens.swift
+  generated/Tokens.swift
   gyb_generated/Trivia.swift
 
   gyb_generated/syntax_nodes/SyntaxDeclNodes.swift