Merge pull request #839 from akyrtzi/pr/created-parsed-tokentext

Introduce `RawTokenSyntax(kind:, tokenText:, ...)` for a convenient way to create a token without any trivia

Author: akyrtzi

Sources/SwiftParser/Expressions.swift

@@ -1474,7 +1474,6 @@ extension Parser {
       let segmentToken = RawTokenSyntax(
         kind: .stringSegment,
         text: SyntaxText(rebasing: text[stringLiteralSegmentStart..<slashIndex]),
-        leadingTriviaPieces: [], trailingTriviaPieces: [],
         presence: .present,
         arena: self.arena)
       segments.append(RawSyntax(RawStringSegmentSyntax(content: segmentToken, arena: self.arena)))
@@ -1490,7 +1489,6 @@ extension Parser {
         let slashToken = RawTokenSyntax(
           kind: .backslash,
           text: SyntaxText(rebasing: text[slashIndex..<text.index(after: slashIndex)]),
-          leadingTriviaPieces: [], trailingTriviaPieces: [],
           presence: .present,
           arena: self.arena)
 
@@ -1500,7 +1498,6 @@ extension Parser {
           delim = RawTokenSyntax(
             kind: .rawStringDelimiter,
             text: SyntaxText(rebasing: text[delimiterStart..<contentStart]),
-            leadingTriviaPieces: [], trailingTriviaPieces: [],
             presence: .present,
             arena: self.arena)
         } else {
@@ -1567,7 +1564,6 @@ extension Parser {
     let segmentToken = RawTokenSyntax(
       kind: .stringSegment,
       text: SyntaxText(rebasing: segment),
-      leadingTriviaPieces: [], trailingTriviaPieces: [],
       presence: .present,
       arena: self.arena)
     segments.append(RawSyntax(RawStringSegmentSyntax(content: segmentToken,

Sources/SwiftSyntax/Raw/RawSyntaxNodeProtocol.swift

@@ -140,6 +140,37 @@ public struct RawTokenSyntax: RawSyntaxToSyntax, RawSyntaxNodeProtocol {
   public init(
     kind: RawTokenKind,
     text: SyntaxText,
+    leadingTriviaPieces: [RawTriviaPiece] = [],
+    trailingTriviaPieces: [RawTriviaPiece] = [],
+    presence: SourcePresence,
+    arena: __shared SyntaxArena
+  ) {
+    if leadingTriviaPieces.isEmpty && trailingTriviaPieces.isEmpty {
+      // Create it via `RawSyntax.parsedToken()`.
+      self.init(
+        kind: kind,
+        wholeText: text,
+        textRange: 0 ..< text.count,
+        presence: presence,
+        arena: arena
+      )
+    } else {
+      // Create it via `RawSyntax.makeMaterializedToken()`.
+      self.init(
+        materialized: kind,
+        text: text,
+        leadingTriviaPieces: leadingTriviaPieces,
+        trailingTriviaPieces: trailingTriviaPieces,
+        presence: presence,
+        arena: arena
+      )
+    }
+  }
+
+  /// Creates a `MaterializedToken`. Trivia must be managed by the same `arena`.
+  fileprivate init(
+    materialized kind: RawTokenKind,
+    text: SyntaxText,
     leadingTriviaPieces: [RawTriviaPiece],
     trailingTriviaPieces: [RawTriviaPiece],
     presence: SourcePresence,
@@ -169,8 +200,11 @@ public struct RawTokenSyntax: RawSyntaxToSyntax, RawSyntaxNodeProtocol {
     text: SyntaxText? = nil,
     arena: __shared SyntaxArena
   ) {
+    // FIXME: Allow creating a `RawSyntax.parsedToken()` with a string literal
+    // for text. Currently it asserts that the string buffer is not contained
+    // within the `arena`.
     self.init(
-      kind: kind,
+      materialized: kind,
       text: text ?? kind.defaultText ?? "",
       leadingTriviaPieces: [],
       trailingTriviaPieces: [],