Introduce a source accurate and structural view of a SwiftSyntax tree

`SyntaxTreeViewMode` allows two different views on a SwiftSyntax tree:

1. Either a purely `sourceAccurate` view, which skips over all missing children but traverses garbage syntax (once implemented). This mode is intended for clients that operate at the source code level, like swift-format
2. For clients that don’t care about 100% about source fidelity but would like to view the syntax tree for structural analysis with fixes from the parser applied, there’s the `fixedUp` view. This includes missing nodes (because the user probably meant to write these) and skips over garbage nodes (because the client will likely have a hard time reasoning about these nodes if the parser couldn’t make sense of them)

Author: ahoppen

Snippets/AddOneToIntegerLiterals.swift renamed to Examples/AddOneToIntegerLiterals.swift

@@ -31,7 +31,7 @@ class AddOneToIntegerLiterals: SyntaxRewriter {
 
     // Create a new integer literal token with `int + 1` as its text.
     let newIntegerLiteralToken = token.withKind(.integerLiteral("\(int + 1)"))
-    
+
     // Return the new integer literal.
     return Syntax(newIntegerLiteralToken)
   }

Snippets/CodeGenerationUsingSwiftSyntaxBuilder.swift renamed to Examples/CodeGenerationUsingSwiftSyntaxBuilder.swift

@@ -1,7 +1,7 @@
 import SwiftSyntaxBuilder
 
 /// This example will print the following example:
-/// 
+///
 ///```
 /// import Foundation
 /// import UIKit

Snippets/README.md renamed to Examples/README.md

@@ -304,7 +304,7 @@ public struct IncrementalParseLookup {
       if !edits.edits.isEmpty && edits.edits.first!.range.offset > nextSibling.endPosition.utf8Offset {
         return true
       }
-      if let nextToken = nextSibling.raw.firstPresentToken {
+      if let nextToken = nextSibling.raw.firstToken(viewMode: .sourceAccurate) {
         nextLeafNodeLength = nextToken.totalLength - nextToken.trailingTriviaLength
       }
     }
@@ -350,6 +350,7 @@ fileprivate struct SyntaxCursor {
   var parents: [AbsoluteRawSyntax]
   var node: AbsoluteRawSyntax
   var finished: Bool
+  let viewMode = SyntaxTreeViewMode.sourceAccurate
 
   init(root: AbsoluteRawSyntax) {
     self.node = root
@@ -368,7 +369,7 @@ fileprivate struct SyntaxCursor {
     var parents = ArraySlice(self.parents)
     var node = self.node
     while !parents.isEmpty {
-      if let sibling = node.nextSibling(parent: parents.last!) {
+      if let sibling = node.nextSibling(parent: parents.last!, viewMode: viewMode) {
         return sibling
       }
       node = parents.removeLast()
@@ -380,7 +381,7 @@ fileprivate struct SyntaxCursor {
   /// Moves to the first child of the current node.
   /// - Returns: False if the node has no children.
   mutating func advanceToFirstChild() -> Bool {
-    guard let child = node.firstChild else { return false }
+    guard let child = node.firstChild(viewMode: viewMode) else { return false }
     parents.append(node)
     node = child
     return true
@@ -391,7 +392,7 @@ fileprivate struct SyntaxCursor {
   /// - Returns: False if it run out of nodes to walk to.
   mutating func advanceToNextSibling() -> Bool {
     while !parents.isEmpty {
-      if let sibling = node.nextSibling(parent: parents.last!) {
+      if let sibling = node.nextSibling(parent: parents.last!, viewMode: viewMode) {
         node = sibling
         return true
       }

Sources/SwiftSyntax/IncrementalParseTransition.swift

@@ -993,18 +993,8 @@ final class RawSyntax: ManagedBuffer<RawSyntaxBase, RawSyntaxDataElement> {
     return header.totalNodes
   }
 
-  /// Whether this node is present in the original source.
-  var isPresent: Bool {
-    return header.isPresent
-  }
-
-  /// Whether this node is missing from the original source.
-  var isMissing: Bool {
-    return !isPresent
-  }
-
   var presence: SourcePresence {
-    return isPresent ? .present : .missing
+    return header.isPresent ? .present : .missing
   }
 
   var totalLength: SourceLength {
@@ -1247,7 +1237,7 @@ extension RawSyntax: TextOutputStreamable, CustomStringConvertible {
   /// - Parameter stream: The stream on which to output this node.
   func write<Target>(to target: inout Target)
     where Target: TextOutputStream {
-    guard isPresent else { return }
+    guard SyntaxTreeViewMode.sourceAccurate.shouldTraverse(node: self) else { return }
     if isToken {
       withUnsafeMutablePointers {
         $0.pointee.writeToken(to: &target, extraPtr: $1)
@@ -1269,48 +1259,48 @@ extension RawSyntax: TextOutputStreamable, CustomStringConvertible {
 
 extension RawSyntax {
   /// Return the first `present` token of a layout node or self if it is a token.
-  var firstPresentToken: RawSyntax? {
-    guard isPresent else { return nil }
+  func firstToken(viewMode: SyntaxTreeViewMode) -> RawSyntax? {
+    guard viewMode.shouldTraverse(node: self) else { return nil }
     if isToken { return self }
     for i in 0..<self.numberOfChildren {
-      if let token = self.child(at: i)?.firstPresentToken {
+      if let token = self.child(at: i)?.firstToken(viewMode: viewMode) {
         return token
       }
     }
     return nil
   }
 
   /// Return the last `present` token of a layout node or self if it is a token.
-  var lastPresentToken: RawSyntax? {
-    guard isPresent else { return nil }
+  func lastToken(viewMode: SyntaxTreeViewMode) -> RawSyntax? {
+    guard viewMode.shouldTraverse(node: self) else { return nil }
     if isToken { return self }
     for i in (0..<self.numberOfChildren).reversed() {
-      if let token = self.child(at: i)?.lastPresentToken {
+      if let token = self.child(at: i)?.lastToken(viewMode: viewMode) {
         return token
       }
     }
     return nil
   }
 
   func formLeadingTrivia() -> Trivia? {
-    guard let token = self.firstPresentToken else { return nil }
+    guard let token = self.firstToken(viewMode: .sourceAccurate) else { return nil }
     return token.formTokenLeadingTrivia()
   }
 
   func formTrailingTrivia() -> Trivia? {
-    guard let token = self.lastPresentToken else { return nil }
+    guard let token = self.lastToken(viewMode: .sourceAccurate) else { return nil }
     return token.formTokenTrailingTrivia()
   }
 }
 
 extension RawSyntax {
   var leadingTriviaLength: SourceLength {
-    guard let token = self.firstPresentToken else { return .zero }
+    guard let token = self.firstToken(viewMode: .sourceAccurate) else { return .zero }
     return token.tokenLeadingTriviaLength
   }
 
   var trailingTriviaLength: SourceLength {
-    guard let token = self.lastPresentToken else { return .zero }
+    guard let token = self.lastToken(viewMode: .sourceAccurate) else { return .zero }
     return token.tokenTrailingTriviaLength
   }
 

Sources/SwiftSyntax/RawSyntax.swift

@@ -346,7 +346,7 @@ fileprivate func computeLines(
     lines.append(position)
   }
   var curPrefix: SourceLength = .zero
-  for token in tree.tokens {
+  for token in tree.tokens(viewMode: .sourceAccurate) {
     curPrefix = token.forEachLineLength(prefix: curPrefix, body: addLine)
   }
   position += curPrefix

Sources/SwiftSyntax/SourceLocation.swift

@@ -135,9 +135,14 @@ internal extension SyntaxProtocol {
 }
 
 public extension SyntaxProtocol {
-  /// A sequence over the `present` children of this node.
+  @available(*, deprecated, message: "Use children(viewMode:) instead")
   var children: SyntaxChildren {
-    return SyntaxChildren(_syntaxNode)
+    return children(viewMode: .sourceAccurate)
+  }
+
+  /// A sequence over the `present` children of this node.
+  func children(viewMode: SyntaxTreeViewMode) -> SyntaxChildren {
+    return SyntaxChildren(_syntaxNode, viewMode: viewMode)
   }
 
   /// The index of this node in a `SyntaxChildren` collection.
@@ -147,12 +152,12 @@ public extension SyntaxProtocol {
 
   /// Whether or not this node is marked as `present`.
   var isPresent: Bool {
-    return raw.isPresent
+    return raw.presence == .present
   }
 
   /// Whether or not this node is marked as `missing`.
   var isMissing: Bool {
-    return raw.isMissing
+    return raw.presence == .missing
   }
 
   /// Whether or not this node is a token one.
@@ -191,45 +196,63 @@ public extension SyntaxProtocol {
   /// Recursively walks through the tree to find the token semantically before
   /// this node.
   var previousToken: TokenSyntax? {
+    return self.previousToken(viewMode: .sourceAccurate)
+  }
+
+  /// Recursively walks through the tree to find the token semantically before
+  /// this node.
+  func previousToken(viewMode: SyntaxTreeViewMode) -> TokenSyntax? {
     guard let parent = self.parent else {
       return nil
     }
-    let siblings = PresentRawSyntaxChildren(parent)
+    let siblings = NonNilRawSyntaxChildren(parent, viewMode: viewMode)
     for absoluteRaw in siblings[..<self.index].reversed() {
       let child = Syntax(SyntaxData(absoluteRaw, parent: parent))
-      if let token = child.lastToken {
+      if let token = child.lastToken(viewMode: viewMode) {
         return token
       }
     }
-    return parent.previousToken
+    return parent.previousToken(viewMode: viewMode)
   }
 
   /// Recursively walks through the tree to find the next token semantically
   /// after this node.
   var nextToken: TokenSyntax? {
+    return self.nextToken(viewMode: .sourceAccurate)
+  }
+
+  /// Recursively walks through the tree to find the next token semantically
+  /// after this node.
+  func nextToken(viewMode: SyntaxTreeViewMode) -> TokenSyntax? {
     guard let parent = self.parent else {
       return nil
     }
-    let siblings = PresentRawSyntaxChildren(parent)
+    let siblings = NonNilRawSyntaxChildren(parent, viewMode: viewMode)
     let nextSiblingIndex = siblings.index(after: self.index)
     for absoluteRaw in siblings[nextSiblingIndex...] {
       let child = Syntax(SyntaxData(absoluteRaw, parent: parent))
-      if let token = child.firstToken {
+      if let token = child.firstToken(viewMode: viewMode) {
         return token
       }
     }
-    return parent.nextToken
+    return parent.nextToken(viewMode: viewMode)
   }
 
-  /// Returns the first token node that is part of this syntax node.
+  /// Returns the first token in this syntax node in the source accurate view of
+  /// the syntax tree.
   var firstToken: TokenSyntax? {
-    if isMissing { return nil }
+    return self.firstToken(viewMode: .sourceAccurate)
+  }
+
+  /// Returns the first token node that is part of this syntax node.
+  func firstToken(viewMode: SyntaxTreeViewMode) -> TokenSyntax? {
+    guard viewMode.shouldTraverse(node: raw) else { return nil }
     if let token = _syntaxNode.as(TokenSyntax.self) {
       return token
     }
 
-    for child in children {
-      if let token = child.firstToken {
+    for child in children(viewMode: viewMode) {
+      if let token = child.firstToken(viewMode: viewMode) {
         return token
       }
     }
@@ -238,13 +261,18 @@ public extension SyntaxProtocol {
 
   /// Returns the last token node that is part of this syntax node.
   var lastToken: TokenSyntax? {
-    if isMissing { return nil }
+    return self.lastToken(viewMode: .sourceAccurate)
+  }
+
+  /// Returns the last token node that is part of this syntax node.
+  func lastToken(viewMode: SyntaxTreeViewMode) -> TokenSyntax? {
+    guard viewMode.shouldTraverse(node: raw) else { return nil }
     if let token = _syntaxNode.as(TokenSyntax.self) {
       return token
     }
 
-    for child in children.reversed() {
-      if let tok = child.lastToken {
+    for child in children(viewMode: viewMode).reversed() {
+      if let tok = child.lastToken(viewMode: viewMode) {
         return tok
       }
     }
@@ -377,8 +405,14 @@ public extension SyntaxProtocol {
   }
 
   /// Sequence of tokens that are part of this Syntax node.
+  @available(*, deprecated, message: "Use tokens(viewMode:) instead")
   var tokens: TokenSequence {
-    return TokenSequence(_syntaxNode)
+    return tokens(viewMode: .sourceAccurate)
+  }
+
+  /// Sequence of tokens that are part of this Syntax node.
+  func tokens(viewMode: SyntaxTreeViewMode) -> TokenSequence {
+    return TokenSequence(_syntaxNode, viewMode: viewMode)
   }
 
   /// Sequence of `SyntaxClassifiedRange`s for this syntax node.
@@ -466,15 +500,17 @@ public struct TokenSequence: Sequence {
   public struct Iterator: IteratorProtocol {
     var nextToken: TokenSyntax?
     let endPosition: AbsolutePosition
+    let viewMode: SyntaxTreeViewMode
 
-    init(_ token: TokenSyntax?, endPosition: AbsolutePosition) {
+    init(_ token: TokenSyntax?, endPosition: AbsolutePosition, viewMode: SyntaxTreeViewMode) {
       self.nextToken = token
       self.endPosition = endPosition
+      self.viewMode = viewMode
     }
 
     public mutating func next() -> TokenSyntax? {
       guard let token = self.nextToken else { return nil }
-      self.nextToken = token.nextToken
+      self.nextToken = token.nextToken(viewMode: viewMode)
       // Make sure we stop once we reach the end of the containing node.
       if let nextTok = self.nextToken, nextTok.position >= self.endPosition {
         self.nextToken = nil
@@ -484,17 +520,19 @@ public struct TokenSequence: Sequence {
   }
 
   let node: Syntax
+  let viewMode: SyntaxTreeViewMode
 
-  public init(_ node: Syntax) {
+  public init(_ node: Syntax, viewMode: SyntaxTreeViewMode) {
     self.node = node
+    self.viewMode = viewMode
   }
 
   public func makeIterator() -> Iterator {
-    return Iterator(node.firstToken, endPosition: node.endPosition)
+    return Iterator(node.firstToken(viewMode: viewMode), endPosition: node.endPosition, viewMode: viewMode)
   }
 
   public func reversed() -> ReversedTokenSequence {
-    return ReversedTokenSequence(node)
+    return ReversedTokenSequence(node, viewMode: viewMode)
   }
 }
 
@@ -509,15 +547,17 @@ public struct ReversedTokenSequence: Sequence {
   public struct Iterator: IteratorProtocol {
     var nextToken: TokenSyntax?
     let startPosition: AbsolutePosition
+    let viewMode: SyntaxTreeViewMode
 
-    init(_ token: TokenSyntax?, startPosition: AbsolutePosition) {
+    init(_ token: TokenSyntax?, startPosition: AbsolutePosition, viewMode: SyntaxTreeViewMode) {
       self.nextToken = token
       self.startPosition = startPosition
+      self.viewMode = viewMode
     }
 
     public mutating func next() -> TokenSyntax? {
       guard let token = self.nextToken else { return nil }
-      self.nextToken = token.previousToken
+      self.nextToken = token.previousToken(viewMode: viewMode)
       // Make sure we stop once we went beyond the start of the containing node.
       if let nextTok = self.nextToken, nextTok.position < self.startPosition {
         self.nextToken = nil
@@ -527,17 +567,19 @@ public struct ReversedTokenSequence: Sequence {
   }
 
   let node: Syntax
+  let viewMode: SyntaxTreeViewMode
 
-  public init(_ node: Syntax) {
+  public init(_ node: Syntax, viewMode: SyntaxTreeViewMode) {
     self.node = node
+    self.viewMode = viewMode
   }
 
   public func makeIterator() -> Iterator {
-    return Iterator(node.lastToken, startPosition: node.position)
+    return Iterator(node.lastToken(viewMode: viewMode), startPosition: node.position, viewMode: viewMode)
   }
 
   public func reversed() -> TokenSequence {
-    return TokenSequence(node)
+    return TokenSequence(node, viewMode: viewMode)
   }
 }