Merge pull request #609 from ahoppen/pr/presence-token-only

Make `presence` a token-only property

Author: ahoppen

Sources/SwiftParser/Diagnostics/ParseDiagnosticsGenerator.swift

@@ -78,7 +78,7 @@ public class ParseDiagnosticsGenerator: SyntaxAnyVisitor {
     if shouldSkip(node) {
       return .skipChildren
     }
-    if node.isMissing {
+    if node.presence == .missing {
       addDiagnostic(node, MissingTokenDiagnostic(missingToken: node))
     }
     return .skipChildren

Sources/SwiftSyntax/Raw/RawSyntax.swift

@@ -158,15 +158,15 @@ extension RawSyntax {
   }
 
   var recursiveFlags: RecursiveRawSyntaxFlags {
-    switch rawData.payload {
-    case .materializedToken, .parsedToken:
+    switch view {
+    case .token(let tokenView):
       var recursiveFlags: RecursiveRawSyntaxFlags = []
-      if presence == .missing {
+      if tokenView.presence == .missing {
         recursiveFlags.insert(.hasError)
       }
       return recursiveFlags
-    case .layout(let dat):
-      return dat.recursiveFlags
+    case .layout(let layoutView):
+      return layoutView.layoutData.recursiveFlags
     }
   }
 
@@ -181,26 +181,6 @@ extension RawSyntax {
     }
   }
 
-  var presence: SourcePresence {
-    if self.byteLength != 0 {
-      // The node has source text associated with it. It's present.
-      return .present
-    }
-    if self.isCollection || self.isUnknown {
-      // We always consider collections 'present' because they can just be empty.
-      return .present
-    }
-    if isToken && (self.tokenView!.rawKind == .eof || self.tokenView!.rawKind == .stringSegment) {
-      // The end of file token never has source code associated with it but we
-      // still consider it valid.
-      // String segments can be empty if they occur in an empty string literal or in between two interpolation segments.
-      return .present
-    }
-
-    // If none of the above apply, the node is missing.
-    return .missing
-  }
-
   /// The "width" of the node.
   ///
   /// Sum of text byte lengths of all descendant token nodes.

Sources/SwiftSyntax/Raw/RawSyntaxLayoutView.swift

@@ -37,6 +37,16 @@ struct RawSyntaxLayoutView {
     }
   }
 
+  var layoutData: RawSyntaxData.Layout {
+    switch raw.rawData.payload {
+    case .parsedToken(_),
+        .materializedToken(_):
+      preconditionFailure("RawSyntax must be a layout")
+    case .layout(let dat):
+      return dat
+    }
+  }
+
   /// Creates a new node of the same kind but with children replaced by `elements`.
   func replacingLayout<C: Collection>(
     with elements: C,
@@ -140,13 +150,7 @@ struct RawSyntaxLayoutView {
 
   /// Child nodes.
   var children: RawSyntaxBuffer {
-    switch raw.rawData.payload {
-    case .parsedToken(_),
-         .materializedToken(_):
-      preconditionFailure("RawSyntax must be a layout")
-    case .layout(let dat):
-      return dat.layout
-    }
+    layoutData.layout
   }
 
   /// Returns the child at the provided cursor in the layout.

Sources/SwiftSyntax/Raw/RawSyntaxNodeProtocol.swift

@@ -95,7 +95,7 @@ public struct RawTokenSyntax: RawSyntaxNodeProtocol {
   }
 
   public var presence: SourcePresence {
-    raw.presence
+    tokenView.presence
   }
 
   public var isMissing: Bool {

Sources/SwiftSyntax/Raw/RawSyntaxTokenView.swift

@@ -187,4 +187,21 @@ struct RawSyntaxTokenView {
       preconditionFailure("Must be invoked on a token")
     }
   }
+
+  var presence: SourcePresence {
+    if self.raw.byteLength != 0 {
+      // The node has source text associated with it. It's present.
+      return .present
+    }
+    if rawKind == .eof || rawKind == .stringSegment {
+      // The end of file token never has source code associated with it but we
+      // still consider it valid.
+      // String segments can be empty if they occur in an empty string literal or in between two interpolation segments.
+      return .present
+    }
+
+    // If none of the above apply, the node is missing.
+    return .missing
+  }
+
 }

Sources/SwiftSyntax/Syntax.swift

@@ -146,16 +146,6 @@ public extension SyntaxProtocol {
     return SyntaxChildrenIndex(self.data.absoluteRaw.info)
   }
 
-  /// Whether or not this node is marked as `present`.
-  var isPresent: Bool {
-    return raw.presence == .present
-  }
-
-  /// Whether or not this node is marked as `missing`.
-  var isMissing: Bool {
-    return raw.presence == .missing
-  }
-
   /// Whether or not this node is a token one.
   var isToken: Bool {
     return raw.isToken
@@ -553,7 +543,7 @@ public extension SyntaxProtocol {
       target.write(" [\(range.start)...\(range.end)]")
     }
 
-    if isMissing {
+    if let tokenView = raw.tokenView, tokenView.presence == .missing {
       target.write(" MISSING")
     }
 

Sources/SwiftSyntax/SyntaxOtherNodes.swift

@@ -98,7 +98,7 @@ public struct TokenSyntax: SyntaxProtocol, SyntaxHashable {
   }
 
   public var presence: SourcePresence {
-    return raw.presence
+    return tokenView.presence
   }
 
   /// The text of the token as written in the source code.

Sources/SwiftSyntax/SyntaxTreeViewMode.swift

@@ -31,7 +31,10 @@ public enum SyntaxTreeViewMode {
   func shouldTraverse(node: RawSyntax) -> Bool {
     switch self {
     case .sourceAccurate:
-      return node.presence == .present
+      if let tokenView = node.tokenView {
+        return tokenView.presence == .present
+      }
+      return true
     case .fixedUp:
       return node.kind != .unexpectedNodes
     case .all:

Sources/_SwiftSyntaxTestSupport/SyntaxComparison.swift

@@ -119,12 +119,11 @@ public extension SyntaxProtocol {
       return .nodeType
     }
 
-    if isPresent != baseline.isPresent {
-      return .presence
-    }
-
     if isToken {
       if let token = Syntax(self).as(TokenSyntax.self), let baselineToken = Syntax(baseline).as(TokenSyntax.self) {
+        if token.presence != baselineToken.presence {
+          return .presence
+        }
         if token.tokenKind != baselineToken.tokenKind {
           return .token
         }

Tests/SwiftSyntaxParserTest/SyntaxComparisonTests.swift

@@ -88,8 +88,8 @@ public class SyntaxComparisonTests: XCTestCase {
     func expectations(_ diff: TreeDifference?, file: StaticString = #filePath, line: UInt = #line) throws {
       let diff = try XCTUnwrap(diff, file: file, line: line)
       XCTAssertEqual(diff.reason, .presence)
-      XCTAssertTrue(diff.baseline.isMissing)
-      XCTAssertFalse(diff.node.isMissing)
+      XCTAssertEqual(diff.baseline.as(TokenSyntax.self)?.presence, .missing)
+      XCTAssertEqual(diff.node.as(TokenSyntax.self)?.presence, .present)
     }
 
     let actual = Syntax(makeFunc(identifier: .identifier("f")))

Tests/SwiftSyntaxTest/SyntaxChildrenTests.swift

@@ -56,7 +56,7 @@ public class SyntaxChildrenTests: XCTestCase {
     )))
 
     var sourceAccurateIt = node.children(viewMode: .sourceAccurate).makeIterator()
-    XCTAssertNextIsNil(&sourceAccurateIt)
+    try XCTAssertNext(&sourceAccurateIt) { $0.is(MissingDeclSyntax.self) }
 
     var fixedUpIt = node.children(viewMode: .fixedUp).makeIterator()
     try XCTAssertNext(&fixedUpIt) { $0.is(MissingDeclSyntax.self) }

Tests/SwiftSyntaxTest/VisitorTests.swift

@@ -23,10 +23,35 @@ public class VisitorTests: XCTestCase {
       }
     }
 
-    XCTAssertFalse(MissingDeclChecker.check(node, viewMode: .sourceAccurate))
+    XCTAssertTrue(MissingDeclChecker.check(node, viewMode: .sourceAccurate))
     XCTAssertTrue(MissingDeclChecker.check(node, viewMode: .fixedUp))
   }
 
+  public func testVisitMissingToken() throws {
+    let node = ReturnStmtSyntax(
+      returnKeyword: .returnKeyword(presence: .missing),
+      expression: nil
+    )
+
+    class MissingTokenChecker: SyntaxVisitor {
+      var didSeeToken = false
+
+      override func visit(_ node: TokenSyntax) -> SyntaxVisitorContinueKind {
+        didSeeToken = true
+        return .visitChildren
+      }
+
+      static func check<Tree: SyntaxProtocol>(_ tree: Tree, viewMode: SyntaxTreeViewMode) -> Bool {
+        let visitor = MissingTokenChecker(viewMode: viewMode)
+        visitor.walk(tree)
+        return visitor.didSeeToken
+      }
+    }
+
+    XCTAssertFalse(MissingTokenChecker.check(node, viewMode: .sourceAccurate))
+    XCTAssertTrue(MissingTokenChecker.check(node, viewMode: .fixedUp))
+  }
+
   public func testVisitUnexpected() throws {
     // This is just bunch of unexpected
     let unexpectedReturnStmt = ReturnStmtSyntax(

lit_tests/coloring.swift

@@ -232,7 +232,7 @@ func f(x: Int) -> Int {
    )
    """
 
-  // CHECK: <str>"</str>\<anchor>(</anchor><int>1</int><anchor>)</anchor>\<anchor>(</anchor><int>1</int><anchor>)</anchor><str>"</str>
+  // CHECK: <str>"</str>\<anchor>(</anchor><int>1</int><anchor>)</anchor><str></str>\<anchor>(</anchor><int>1</int><anchor>)</anchor><str>"</str>
   "\(1)\(1)"
 }