Make presence a token-only property

It doesn’t really make sense to talk about presence for layout nodes. Are they missing if all of their tokens are missing? What if all tokens are missing but one has trivia attached to it?

To simplify the semantics, define `presence` for tokens only.

This slightly changes the semantics because layout nodes without any present children are now walked in the `sourceAccurate` view, but I think that’s a good change.

rdar://98818987

Author: ahoppen

Sources/SwiftParser/Diagnostics/ParseDiagnosticsGenerator.swift

@@ -78,7 +78,7 @@ public class ParseDiagnosticsGenerator: SyntaxAnyVisitor {
     if shouldSkip(node) {
       return .skipChildren
     }
-    if node.isMissing {
+    if node.presence == .missing {
       addDiagnostic(node, MissingTokenDiagnostic(missingToken: node))
     }
     return .skipChildren

Sources/SwiftSyntax/Raw/RawSyntax.swift

@@ -158,15 +158,15 @@ extension RawSyntax {
   }
 
   var recursiveFlags: RecursiveRawSyntaxFlags {
-    switch rawData.payload {
-    case .materializedToken, .parsedToken:
+    switch view {
+    case .token(let tokenView):
       var recursiveFlags: RecursiveRawSyntaxFlags = []
-      if presence == .missing {
+      if tokenView.presence == .missing {
         recursiveFlags.insert(.hasError)
       }
       return recursiveFlags
-    case .layout(let dat):
-      return dat.recursiveFlags
+    case .layout(let layoutView):
+      return layoutView.layoutData.recursiveFlags
     }
   }
 
@@ -181,26 +181,6 @@ extension RawSyntax {
     }
   }
 
-  var presence: SourcePresence {
-    if self.byteLength != 0 {
-      // The node has source text associated with it. It's present.
-      return .present
-    }
-    if self.isCollection || self.isUnknown {
-      // We always consider collections 'present' because they can just be empty.
-      return .present
-    }
-    if isToken && (self.tokenView!.rawKind == .eof || self.tokenView!.rawKind == .stringSegment) {
-      // The end of file token never has source code associated with it but we
-      // still consider it valid.
-      // String segments can be empty if they occur in an empty string literal or in between two interpolation segments.
-      return .present
-    }
-
-    // If none of the above apply, the node is missing.
-    return .missing
-  }
-
   /// The "width" of the node.
   ///
   /// Sum of text byte lengths of all descendant token nodes.

Sources/SwiftSyntax/Raw/RawSyntaxLayoutView.swift

@@ -37,6 +37,16 @@ struct RawSyntaxLayoutView {
     }
   }
 
+  var layoutData: RawSyntaxData.Layout {
+    switch raw.rawData.payload {
+    case .parsedToken(_),
+        .materializedToken(_):
+      preconditionFailure("RawSyntax must be a layout")
+    case .layout(let dat):
+      return dat
+    }
+  }
+
   /// Creates a new node of the same kind but with children replaced by `elements`.
   func replacingLayout<C: Collection>(
     with elements: C,
@@ -140,13 +150,7 @@ struct RawSyntaxLayoutView {
 
   /// Child nodes.
   var children: RawSyntaxBuffer {
-    switch raw.rawData.payload {
-    case .parsedToken(_),
-         .materializedToken(_):
-      preconditionFailure("RawSyntax must be a layout")
-    case .layout(let dat):
-      return dat.layout
-    }
+    layoutData.layout
   }
 
   /// Returns the child at the provided cursor in the layout.

Sources/SwiftSyntax/Raw/RawSyntaxNodeProtocol.swift

@@ -95,7 +95,7 @@ public struct RawTokenSyntax: RawSyntaxNodeProtocol {
   }
 
   public var presence: SourcePresence {
-    raw.presence
+    tokenView.presence
   }
 
   public var isMissing: Bool {

Sources/SwiftSyntax/Raw/RawSyntaxTokenView.swift

@@ -187,4 +187,21 @@ struct RawSyntaxTokenView {
       preconditionFailure("Must be invoked on a token")
     }
   }
+
+  var presence: SourcePresence {
+    if self.raw.byteLength != 0 {
+      // The node has source text associated with it. It's present.
+      return .present
+    }
+    if rawKind == .eof || rawKind == .stringSegment {
+      // The end of file token never has source code associated with it but we
+      // still consider it valid.
+      // String segments can be empty if they occur in an empty string literal or in between two interpolation segments.
+      return .present
+    }
+
+    // If none of the above apply, the node is missing.
+    return .missing
+  }
+
 }

Sources/SwiftSyntax/Syntax.swift

@@ -146,16 +146,6 @@ public extension SyntaxProtocol {
     return SyntaxChildrenIndex(self.data.absoluteRaw.info)
   }
 
-  /// Whether or not this node is marked as `present`.
-  var isPresent: Bool {
-    return raw.presence == .present
-  }
-
-  /// Whether or not this node is marked as `missing`.
-  var isMissing: Bool {
-    return raw.presence == .missing
-  }
-
   /// Whether or not this node is a token one.
   var isToken: Bool {
     return raw.isToken
@@ -553,7 +543,7 @@ public extension SyntaxProtocol {
       target.write(" [\(range.start)...\(range.end)]")
     }
 
-    if isMissing {
+    if let tokenView = raw.tokenView, tokenView.presence == .missing {
       target.write(" MISSING")
     }
 

Sources/SwiftSyntax/SyntaxOtherNodes.swift

@@ -98,7 +98,7 @@ public struct TokenSyntax: SyntaxProtocol, SyntaxHashable {
   }
 
   public var presence: SourcePresence {
-    return raw.presence
+    return tokenView.presence
   }
 
   /// The text of the token as written in the source code.

Sources/SwiftSyntax/SyntaxTreeViewMode.swift

@@ -31,7 +31,10 @@ public enum SyntaxTreeViewMode {
   func shouldTraverse(node: RawSyntax) -> Bool {
     switch self {
     case .sourceAccurate:
-      return node.presence == .present
+      if let tokenView = node.tokenView {
+        return tokenView.presence == .present
+      }
+      return true
     case .fixedUp:
       return node.kind != .unexpectedNodes
     case .all:

Sources/_SwiftSyntaxTestSupport/SyntaxComparison.swift

@@ -119,12 +119,11 @@ public extension SyntaxProtocol {
       return .nodeType
     }
 
-    if isPresent != baseline.isPresent {
-      return .presence
-    }
-
     if isToken {
       if let token = Syntax(self).as(TokenSyntax.self), let baselineToken = Syntax(baseline).as(TokenSyntax.self) {
+        if token.presence != baselineToken.presence {
+          return .presence
+        }
         if token.tokenKind != baselineToken.tokenKind {
           return .token
         }

Tests/SwiftSyntaxParserTest/SyntaxComparisonTests.swift

@@ -88,8 +88,8 @@ public class SyntaxComparisonTests: XCTestCase {
     func expectations(_ diff: TreeDifference?, file: StaticString = #filePath, line: UInt = #line) throws {
       let diff = try XCTUnwrap(diff, file: file, line: line)
       XCTAssertEqual(diff.reason, .presence)
-      XCTAssertTrue(diff.baseline.isMissing)
-      XCTAssertFalse(diff.node.isMissing)
+      XCTAssertEqual(diff.baseline.as(TokenSyntax.self)?.presence, .missing)
+      XCTAssertEqual(diff.node.as(TokenSyntax.self)?.presence, .present)
     }
 
     let actual = Syntax(makeFunc(identifier: .identifier("f")))

Tests/SwiftSyntaxTest/SyntaxChildrenTests.swift

@@ -56,7 +56,7 @@ public class SyntaxChildrenTests: XCTestCase {
     )))
 
     var sourceAccurateIt = node.children(viewMode: .sourceAccurate).makeIterator()
-    XCTAssertNextIsNil(&sourceAccurateIt)
+    try XCTAssertNext(&sourceAccurateIt) { $0.is(MissingDeclSyntax.self) }
 
     var fixedUpIt = node.children(viewMode: .fixedUp).makeIterator()
     try XCTAssertNext(&fixedUpIt) { $0.is(MissingDeclSyntax.self) }

Tests/SwiftSyntaxTest/VisitorTests.swift

@@ -23,10 +23,35 @@ public class VisitorTests: XCTestCase {
       }
     }
 
-    XCTAssertFalse(MissingDeclChecker.check(node, viewMode: .sourceAccurate))
+    XCTAssertTrue(MissingDeclChecker.check(node, viewMode: .sourceAccurate))
     XCTAssertTrue(MissingDeclChecker.check(node, viewMode: .fixedUp))
   }
 
+  public func testVisitMissingToken() throws {
+    let node = ReturnStmtSyntax(
+      returnKeyword: .returnKeyword(presence: .missing),
+      expression: nil
+    )
+
+    class MissingTokenChecker: SyntaxVisitor {
+      var didSeeToken = false
+
+      override func visit(_ node: TokenSyntax) -> SyntaxVisitorContinueKind {
+        didSeeToken = true
+        return .visitChildren
+      }
+
+      static func check<Tree: SyntaxProtocol>(_ tree: Tree, viewMode: SyntaxTreeViewMode) -> Bool {
+        let visitor = MissingTokenChecker(viewMode: viewMode)
+        visitor.walk(tree)
+        return visitor.didSeeToken
+      }
+    }
+
+    XCTAssertFalse(MissingTokenChecker.check(node, viewMode: .sourceAccurate))
+    XCTAssertTrue(MissingTokenChecker.check(node, viewMode: .fixedUp))
+  }
+
   public func testVisitUnexpected() throws {
     // This is just bunch of unexpected
     let unexpectedReturnStmt = ReturnStmtSyntax(

lit_tests/coloring.swift

@@ -232,7 +232,7 @@ func f(x: Int) -> Int {
    )
    """
 
-  // CHECK: <str>"</str>\<anchor>(</anchor><int>1</int><anchor>)</anchor>\<anchor>(</anchor><int>1</int><anchor>)</anchor><str>"</str>
+  // CHECK: <str>"</str>\<anchor>(</anchor><int>1</int><anchor>)</anchor><str></str>\<anchor>(</anchor><int>1</int><anchor>)</anchor><str>"</str>
   "\(1)\(1)"
 }