Introduce a source accurate and structural view of a SwiftSyntax tree

`SyntaxTreeViewMode` allows two different views on a SwiftSyntax tree:

1. Either a purely `sourceAccurate` view, which skips over all missing children but traverses garbage syntax (once implemented). This mode is intended for clients that operate at the source code level, like swift-format
2. For clients that don’t care about 100% about source fidelity but would like to view the syntax tree for structural analysis with fixes from the parser applied, there’s the `fixedUp` view. This includes missing nodes (because the user probably meant to write these) and skips over garbage nodes (because the client will likely have a hard time reasoning about these nodes if the parser couldn’t make sense of them)

Author: ahoppen

Sources/SwiftSyntax/IncrementalParseTransition.swift

@@ -304,7 +304,7 @@ public struct IncrementalParseLookup {
       if !edits.edits.isEmpty && edits.edits.first!.range.offset > nextSibling.endPosition.utf8Offset {
         return true
       }
-      if let nextToken = nextSibling.raw.firstPresentToken {
+      if let nextToken = nextSibling.raw.firstToken(viewMode: .sourceAccurate) {
         nextLeafNodeLength = nextToken.totalLength - nextToken.trailingTriviaLength
       }
     }
@@ -350,6 +350,7 @@ fileprivate struct SyntaxCursor {
   var parents: [AbsoluteRawSyntax]
   var node: AbsoluteRawSyntax
   var finished: Bool
+  let viewMode = SyntaxTreeViewMode.sourceAccurate
 
   init(root: AbsoluteRawSyntax) {
     self.node = root
@@ -368,7 +369,7 @@ fileprivate struct SyntaxCursor {
     var parents = ArraySlice(self.parents)
     var node = self.node
     while !parents.isEmpty {
-      if let sibling = node.nextSibling(parent: parents.last!) {
+      if let sibling = node.nextSibling(parent: parents.last!, viewMode: viewMode) {
         return sibling
       }
       node = parents.removeLast()
@@ -380,7 +381,7 @@ fileprivate struct SyntaxCursor {
   /// Moves to the first child of the current node.
   /// - Returns: False if the node has no children.
   mutating func advanceToFirstChild() -> Bool {
-    guard let child = node.firstChild else { return false }
+    guard let child = node.firstChild(viewMode: viewMode) else { return false }
     parents.append(node)
     node = child
     return true
@@ -391,7 +392,7 @@ fileprivate struct SyntaxCursor {
   /// - Returns: False if it run out of nodes to walk to.
   mutating func advanceToNextSibling() -> Bool {
     while !parents.isEmpty {
-      if let sibling = node.nextSibling(parent: parents.last!) {
+      if let sibling = node.nextSibling(parent: parents.last!, viewMode: viewMode) {
         node = sibling
         return true
       }

Sources/SwiftSyntax/RawSyntax.swift

@@ -993,18 +993,8 @@ final class RawSyntax: ManagedBuffer<RawSyntaxBase, RawSyntaxDataElement> {
     return header.totalNodes
   }
 
-  /// Whether this node is present in the original source.
-  var isPresent: Bool {
-    return header.isPresent
-  }
-
-  /// Whether this node is missing from the original source.
-  var isMissing: Bool {
-    return !isPresent
-  }
-
   var presence: SourcePresence {
-    return isPresent ? .present : .missing
+    return header.isPresent ? .present : .missing
   }
 
   var totalLength: SourceLength {
@@ -1247,7 +1237,7 @@ extension RawSyntax: TextOutputStreamable, CustomStringConvertible {
   /// - Parameter stream: The stream on which to output this node.
   func write<Target>(to target: inout Target)
     where Target: TextOutputStream {
-    guard isPresent else { return }
+    guard SyntaxTreeViewMode.sourceAccurate.shouldTraverse(node: self) else { return }
     if isToken {
       withUnsafeMutablePointers {
         $0.pointee.writeToken(to: &target, extraPtr: $1)
@@ -1269,48 +1259,48 @@ extension RawSyntax: TextOutputStreamable, CustomStringConvertible {
 
 extension RawSyntax {
   /// Return the first `present` token of a layout node or self if it is a token.
-  var firstPresentToken: RawSyntax? {
-    guard isPresent else { return nil }
+  func firstToken(viewMode: SyntaxTreeViewMode) -> RawSyntax? {
+    guard viewMode.shouldTraverse(node: self) else { return nil }
     if isToken { return self }
     for i in 0..<self.numberOfChildren {
-      if let token = self.child(at: i)?.firstPresentToken {
+      if let token = self.child(at: i)?.firstToken(viewMode: viewMode) {
         return token
       }
     }
     return nil
   }
 
   /// Return the last `present` token of a layout node or self if it is a token.
-  var lastPresentToken: RawSyntax? {
-    guard isPresent else { return nil }
+  func lastToken(viewMode: SyntaxTreeViewMode) -> RawSyntax? {
+    guard viewMode.shouldTraverse(node: self) else { return nil }
     if isToken { return self }
     for i in (0..<self.numberOfChildren).reversed() {
-      if let token = self.child(at: i)?.lastPresentToken {
+      if let token = self.child(at: i)?.lastToken(viewMode: viewMode) {
         return token
       }
     }
     return nil
   }
 
   func formLeadingTrivia() -> Trivia? {
-    guard let token = self.firstPresentToken else { return nil }
+    guard let token = self.firstToken(viewMode: .sourceAccurate) else { return nil }
     return token.formTokenLeadingTrivia()
   }
 
   func formTrailingTrivia() -> Trivia? {
-    guard let token = self.lastPresentToken else { return nil }
+    guard let token = self.lastToken(viewMode: .sourceAccurate) else { return nil }
     return token.formTokenTrailingTrivia()
   }
 }
 
 extension RawSyntax {
   var leadingTriviaLength: SourceLength {
-    guard let token = self.firstPresentToken else { return .zero }
+    guard let token = self.firstToken(viewMode: .sourceAccurate) else { return .zero }
     return token.tokenLeadingTriviaLength
   }
 
   var trailingTriviaLength: SourceLength {
-    guard let token = self.lastPresentToken else { return .zero }
+    guard let token = self.lastToken(viewMode: .sourceAccurate) else { return .zero }
     return token.tokenTrailingTriviaLength
   }
 

Sources/SwiftSyntax/SourceLocation.swift

@@ -346,7 +346,7 @@ fileprivate func computeLines(
     lines.append(position)
   }
   var curPrefix: SourceLength = .zero
-  for token in tree.tokens {
+  for token in tree.tokens(viewMode: .sourceAccurate) {
     curPrefix = token.forEachLineLength(prefix: curPrefix, body: addLine)
   }
   position += curPrefix

Sources/SwiftSyntax/Syntax.swift

@@ -136,25 +136,15 @@ internal extension SyntaxProtocol {
 
 public extension SyntaxProtocol {
   /// A sequence over the `present` children of this node.
-  var children: SyntaxChildren {
-    return SyntaxChildren(_syntaxNode)
+  func children(viewMode: SyntaxTreeViewMode) -> SyntaxChildren {
+    return SyntaxChildren(_syntaxNode, viewMode: viewMode)
   }
 
   /// The index of this node in a `SyntaxChildren` collection.
   var index: SyntaxChildrenIndex {
     return SyntaxChildrenIndex(self.data.absoluteRaw.info)
   }
 
-  /// Whether or not this node is marked as `present`.
-  var isPresent: Bool {
-    return raw.isPresent
-  }
-
-  /// Whether or not this node is marked as `missing`.
-  var isMissing: Bool {
-    return raw.isMissing
-  }
-
   /// Whether or not this node is a token one.
   var isToken: Bool {
     return raw.isToken
@@ -191,45 +181,63 @@ public extension SyntaxProtocol {
   /// Recursively walks through the tree to find the token semantically before
   /// this node.
   var previousToken: TokenSyntax? {
+    return self.previousToken(viewMode: .sourceAccurate)
+  }
+
+  /// Recursively walks through the tree to find the token semantically before
+  /// this node.
+  func previousToken(viewMode: SyntaxTreeViewMode) -> TokenSyntax? {
     guard let parent = self.parent else {
       return nil
     }
-    let siblings = PresentRawSyntaxChildren(parent)
+    let siblings = NonNilRawSyntaxChildren(parent, viewMode: viewMode)
     for absoluteRaw in siblings[..<self.index].reversed() {
       let child = Syntax(SyntaxData(absoluteRaw, parent: parent))
-      if let token = child.lastToken {
+      if let token = child.lastToken(viewMode: viewMode) {
         return token
       }
     }
-    return parent.previousToken
+    return parent.previousToken(viewMode: viewMode)
   }
 
   /// Recursively walks through the tree to find the next token semantically
   /// after this node.
   var nextToken: TokenSyntax? {
+    return self.nextToken(viewMode: .sourceAccurate)
+  }
+
+  /// Recursively walks through the tree to find the next token semantically
+  /// after this node.
+  func nextToken(viewMode: SyntaxTreeViewMode) -> TokenSyntax? {
     guard let parent = self.parent else {
       return nil
     }
-    let siblings = PresentRawSyntaxChildren(parent)
+    let siblings = NonNilRawSyntaxChildren(parent, viewMode: viewMode)
     let nextSiblingIndex = siblings.index(after: self.index)
     for absoluteRaw in siblings[nextSiblingIndex...] {
       let child = Syntax(SyntaxData(absoluteRaw, parent: parent))
-      if let token = child.firstToken {
+      if let token = child.firstToken(viewMode: viewMode) {
         return token
       }
     }
-    return parent.nextToken
+    return parent.nextToken(viewMode: viewMode)
   }
 
-  /// Returns the first token node that is part of this syntax node.
+  /// Returns the first token in this syntax node in the source accurate view of
+  /// the syntax tree.
   var firstToken: TokenSyntax? {
-    if isMissing { return nil }
+    return self.firstToken(viewMode: .sourceAccurate)
+  }
+
+  /// Returns the first token node that is part of this syntax node.
+  func firstToken(viewMode: SyntaxTreeViewMode) -> TokenSyntax? {
+    guard viewMode.shouldTraverse(node: raw) else { return nil }
     if let token = _syntaxNode.as(TokenSyntax.self) {
       return token
     }
 
-    for child in children {
-      if let token = child.firstToken {
+    for child in children(viewMode: viewMode) {
+      if let token = child.firstToken(viewMode: viewMode) {
         return token
       }
     }
@@ -238,13 +246,18 @@ public extension SyntaxProtocol {
 
   /// Returns the last token node that is part of this syntax node.
   var lastToken: TokenSyntax? {
-    if isMissing { return nil }
+    return self.lastToken(viewMode: .sourceAccurate)
+  }
+
+  /// Returns the last token node that is part of this syntax node.
+  func lastToken(viewMode: SyntaxTreeViewMode) -> TokenSyntax? {
+    guard viewMode.shouldTraverse(node: raw) else { return nil }
     if let token = _syntaxNode.as(TokenSyntax.self) {
       return token
     }
 
-    for child in children.reversed() {
-      if let tok = child.lastToken {
+    for child in children(viewMode: viewMode).reversed() {
+      if let tok = child.lastToken(viewMode: viewMode) {
         return tok
       }
     }
@@ -377,8 +390,8 @@ public extension SyntaxProtocol {
   }
 
   /// Sequence of tokens that are part of this Syntax node.
-  var tokens: TokenSequence {
-    return TokenSequence(_syntaxNode)
+  func tokens(viewMode: SyntaxTreeViewMode) -> TokenSequence {
+    return TokenSequence(_syntaxNode, viewMode: viewMode)
   }
 
   /// Sequence of `SyntaxClassifiedRange`s for this syntax node.
@@ -466,15 +479,17 @@ public struct TokenSequence: Sequence {
   public struct Iterator: IteratorProtocol {
     var nextToken: TokenSyntax?
     let endPosition: AbsolutePosition
+    let viewMode: SyntaxTreeViewMode
 
-    init(_ token: TokenSyntax?, endPosition: AbsolutePosition) {
+    init(_ token: TokenSyntax?, endPosition: AbsolutePosition, viewMode: SyntaxTreeViewMode) {
       self.nextToken = token
       self.endPosition = endPosition
+      self.viewMode = viewMode
     }
 
     public mutating func next() -> TokenSyntax? {
       guard let token = self.nextToken else { return nil }
-      self.nextToken = token.nextToken
+      self.nextToken = token.nextToken(viewMode: viewMode)
       // Make sure we stop once we reach the end of the containing node.
       if let nextTok = self.nextToken, nextTok.position >= self.endPosition {
         self.nextToken = nil
@@ -484,17 +499,19 @@ public struct TokenSequence: Sequence {
   }
 
   let node: Syntax
+  let viewMode: SyntaxTreeViewMode
 
-  public init(_ node: Syntax) {
+  public init(_ node: Syntax, viewMode: SyntaxTreeViewMode) {
     self.node = node
+    self.viewMode = viewMode
   }
 
   public func makeIterator() -> Iterator {
-    return Iterator(node.firstToken, endPosition: node.endPosition)
+    return Iterator(node.firstToken(viewMode: viewMode), endPosition: node.endPosition, viewMode: viewMode)
   }
 
   public func reversed() -> ReversedTokenSequence {
-    return ReversedTokenSequence(node)
+    return ReversedTokenSequence(node, viewMode: viewMode)
   }
 }
 
@@ -509,15 +526,17 @@ public struct ReversedTokenSequence: Sequence {
   public struct Iterator: IteratorProtocol {
     var nextToken: TokenSyntax?
     let startPosition: AbsolutePosition
+    let viewMode: SyntaxTreeViewMode
 
-    init(_ token: TokenSyntax?, startPosition: AbsolutePosition) {
+    init(_ token: TokenSyntax?, startPosition: AbsolutePosition, viewMode: SyntaxTreeViewMode) {
       self.nextToken = token
       self.startPosition = startPosition
+      self.viewMode = viewMode
     }
 
     public mutating func next() -> TokenSyntax? {
       guard let token = self.nextToken else { return nil }
-      self.nextToken = token.previousToken
+      self.nextToken = token.previousToken(viewMode: viewMode)
       // Make sure we stop once we went beyond the start of the containing node.
       if let nextTok = self.nextToken, nextTok.position < self.startPosition {
         self.nextToken = nil
@@ -527,17 +546,19 @@ public struct ReversedTokenSequence: Sequence {
   }
 
   let node: Syntax
+  let viewMode: SyntaxTreeViewMode
 
-  public init(_ node: Syntax) {
+  public init(_ node: Syntax, viewMode: SyntaxTreeViewMode) {
     self.node = node
+    self.viewMode = viewMode
   }
 
   public func makeIterator() -> Iterator {
-    return Iterator(node.lastToken, startPosition: node.position)
+    return Iterator(node.lastToken(viewMode: viewMode), startPosition: node.position, viewMode: viewMode)
   }
 
   public func reversed() -> TokenSequence {
-    return TokenSequence(node)
+    return TokenSequence(node, viewMode: viewMode)
   }
 }