Add basic RawSyntaxData.ParsedToken test

Author: rintaro

Sources/SwiftSyntax/RawSyntax.swift

@@ -24,7 +24,7 @@ internal struct RawSyntaxData {
   /// Token with lazy trivia parsing.
   ///
   /// The RawSyntax's `arena` must have a valid trivia parsing function to
-  /// lazily materialize the leading/trailing trivia pieces.
+  /// lazily materialize the leading/trailing trivia xpieces.
   struct ParsedToken {
     var tokenKind: RawTokenKind
 
@@ -307,13 +307,13 @@ extension RawSyntax {
   }
 
   /// Returns the leading `Trivia`, assuming this node is a token.
-  func formTokenLeadingTrivia() -> Trivia? {
+  func formTokenLeadingTrivia() -> Trivia {
     return Trivia(pieces: tokenLeadingRawTriviaPieces.map({ TriviaPiece(raw: $0) }))
   }
 
   /// Returns the trailing `Trivia`, assuming this node is a token.
   /// - Returns: nil if called on a layout node.
-  func formTokenTrailingTrivia() -> Trivia? {
+  func formTokenTrailingTrivia() -> Trivia {
     return Trivia(pieces: tokenTrailingRawTriviaPieces.map({ TriviaPiece(raw: $0) }))
   }
 
@@ -334,10 +334,11 @@ extension RawSyntax {
   func withTokenKind(_ newValue: TokenKind) -> RawSyntax {
     switch rawData.payload {
     case .parsedToken(_):
+      // The wholeText can't be continuous anymore. Make a materialized token.
       return .makeMaterializedToken(
         kind: newValue,
-        leadingTrivia: formLeadingTrivia()!,
-        trailingTrivia: formTokenTrailingTrivia()!,
+        leadingTrivia: formTokenLeadingTrivia(),
+        trailingTrivia: formTokenTrailingTrivia(),
         arena: arena)
     case .materializedToken(var payload):
       let decomposed = newValue.decomposeToRaw()
@@ -362,7 +363,7 @@ extension RawSyntax {
       return .makeMaterializedToken(
         kind: formTokenKind()!,
         leadingTrivia: leadingTrivia,
-        trailingTrivia: formTokenTrailingTrivia()!,
+        trailingTrivia: formTokenTrailingTrivia(),
         arena: arena)
     }
 
@@ -382,7 +383,7 @@ extension RawSyntax {
     if isToken {
       return .makeMaterializedToken(
         kind: formTokenKind()!,
-        leadingTrivia: formTokenLeadingTrivia()!,
+        leadingTrivia: formTokenLeadingTrivia(),
         trailingTrivia: trailingTrivia,
         arena: arena)
     }

Sources/SwiftSyntax/Syntax.swift

@@ -118,13 +118,14 @@ public protocol SyntaxProtocol: CustomStringConvertible,
   var syntaxNodeType: SyntaxProtocol.Type { get }
 }
 
-internal extension SyntaxProtocol {
+extension SyntaxProtocol {
   var data: SyntaxData {
     return _syntaxNode.data
   }
 
   /// Access the raw syntax assuming the node is a Syntax.
-  var raw: RawSyntax {
+  @_spi(RawSyntax)
+  public var raw: RawSyntax {
     return _syntaxNode.data.raw
   }
 }

Sources/SwiftSyntax/SyntaxOtherNodes.swift

@@ -144,7 +144,7 @@ public struct TokenSyntax: SyntaxProtocol, SyntaxHashable {
   /// The leading trivia (spaces, newlines, etc.) associated with this token.
   public var leadingTrivia: Trivia {
     get {
-      return raw.formTokenLeadingTrivia()!
+      return raw.formTokenLeadingTrivia()
     }
     set {
       self = withLeadingTrivia(newValue)
@@ -154,7 +154,7 @@ public struct TokenSyntax: SyntaxProtocol, SyntaxHashable {
   /// The trailing trivia (spaces, newlines, etc.) associated with this token.
   public var trailingTrivia: Trivia {
     get {
-      return raw.formTokenTrailingTrivia()!
+      return raw.formTokenTrailingTrivia()
     }
     set {
       self = withTrailingTrivia(newValue)

Tests/SwiftSyntaxTest/RawSyntaxTests.swift

@@ -80,5 +80,33 @@ final class RawSyntaxTests: XCTestCase {
     }
   }
 
+  func testParsedToken() throws {
+    // Dummy trivia parsing function.
+    func dummyParseToken(source: SyntaxText, isTrailing: Bool) -> [RawTriviaPiece] {
+      // Emit a single `unexpectedText` trivia of the whole trivia text.
+      return [.unexpectedText(source)]
+    }
 
+    withExtendedLifetime(SyntaxArena(parseTriviaFunction: dummyParseToken)) { arena in
+      let ident = RawTokenSyntax(
+        kind: .identifier, wholeText: arena.intern("\nfoo "),
+        textRange: 1..<4, arena: arena)
+      XCTAssertEqual(ident.tokenKind, .identifier)
+      XCTAssertEqual(ident.tokenText, "foo")
+      XCTAssertEqual(ident.presence, .present)
+      XCTAssertEqual(ident.leadingTriviaPieces, [.unexpectedText("\n")])
+      XCTAssertEqual(ident.trailingTriviaPieces, [.unexpectedText(" ")])
+      XCTAssertEqual(ident.description, "\nfoo ")
+
+      let identSyntax = Syntax(raw: ident.raw).as(TokenSyntax.self)!
+      let barIdentSyntax = identSyntax.withKind(.contextualKeyword("open"))
+      let barIdent = barIdentSyntax.raw.as(RawTokenSyntax.self)!
+
+      XCTAssertEqual(barIdent.tokenKind, .contextualKeyword)
+      XCTAssertEqual(barIdent.tokenText, "open")
+      XCTAssertEqual(barIdent.leadingTriviaPieces, [.unexpectedText("\n")])
+      XCTAssertEqual(barIdent.trailingTriviaPieces, [.unexpectedText(" ")])
+      XCTAssertEqual(barIdent.description, "\nopen ")
+    }
+  }
 }