Store presence of a token in an explicit field

We recently changed a token’s presence to be implicitly determined by whether it contains any source code. It turns out that this has two downsides:
- We cannot synthesize missing tokens with explicitly known text, like a missing `for` identifier token for the `@_dynamicReplacable` attribute
- We cannot synthesize missing tokens with trivia

Author: ahoppen

Sources/SwiftParser/Expressions.swift

@@ -1004,7 +1004,12 @@ extension Parser {
       textRange = wholeText.startIndex ..< wholeText.startIndex + text.count
     }
     return RawTokenSyntax(
-      kind: kind, wholeText: wholeText, textRange: textRange, arena: self.arena)
+      kind: kind,
+      wholeText: wholeText,
+      textRange: textRange,
+      presence: .present,
+      arena: self.arena
+    )
   }
 
   mutating func parseStringLiteralDelimiter(
@@ -1118,6 +1123,7 @@ extension Parser {
         kind: .stringSegment,
         text: SyntaxText(rebasing: text[stringLiteralSegmentStart..<slashIndex]),
         leadingTriviaPieces: [], trailingTriviaPieces: [],
+        presence: .present,
         arena: self.arena)
       segments.append(RawSyntax(RawStringSegmentSyntax(content: segmentToken, arena: self.arena)))
 
@@ -1133,6 +1139,7 @@ extension Parser {
           kind: .backslash,
           text: SyntaxText(rebasing: text[slashIndex..<text.index(after: slashIndex)]),
           leadingTriviaPieces: [], trailingTriviaPieces: [],
+          presence: .present,
           arena: self.arena)
 
         // `###`
@@ -1142,6 +1149,7 @@ extension Parser {
             kind: .rawStringDelimiter,
             text: SyntaxText(rebasing: text[delimiterStart..<contentStart]),
             leadingTriviaPieces: [], trailingTriviaPieces: [],
+            presence: .present,
             arena: self.arena)
         } else {
           delim = nil
@@ -1207,6 +1215,7 @@ extension Parser {
       kind: .stringSegment,
       text: SyntaxText(rebasing: segment),
       leadingTriviaPieces: [], trailingTriviaPieces: [],
+      presence: .present,
       arena: self.arena)
     segments.append(RawSyntax(RawStringSegmentSyntax(content: segmentToken,
                                                      arena: self.arena)))

Sources/SwiftParser/Parser.swift

@@ -180,8 +180,12 @@ extension Parser {
     let tok = self.currentToken
     self.currentToken = self.lexemes.advance()
     return RawTokenSyntax(
-      kind: tok.tokenKind, wholeText: tok.wholeText, textRange: tok.textRange,
-      arena: arena)
+      kind: tok.tokenKind,
+      wholeText: tok.wholeText,
+      textRange: tok.textRange,
+      presence: .present,
+      arena: arena
+    )
   }
 
   /// Consumes the current token and sets its kind to the given `TokenKind`,
@@ -298,7 +302,9 @@ extension Parser {
       kind: tokenKind,
       wholeText: SyntaxText(rebasing: current.wholeText[..<endIndex]),
       textRange: current.textRange.lowerBound ..< endIndex,
-      arena: self.arena)
+      presence: .present,
+      arena: self.arena
+    )
 
     // ... or a multi-character token with the first N characters being the one
     // that we want to consume as a separate token.

Sources/SwiftSyntax/Raw/RawSyntax.swift

@@ -50,6 +50,8 @@ internal struct RawSyntaxData {
     /// Text in `wholeText` before `textRange.lowerBound` is leading trivia and
     /// after `textRange.upperBound` is trailing trivia.
     var textRange: Range<SyntaxText.Index>
+
+    var presence: SourcePresence
   }
 
   /// Token typically created with `TokenSyntax.<someToken>`.
@@ -59,6 +61,7 @@ internal struct RawSyntaxData {
     var triviaPieces: RawTriviaPieceBuffer
     var numLeadingTrivia: UInt32
     var byteLength: UInt32
+    var presence: SourcePresence
   }
 
   /// Layout node including collections.
@@ -183,12 +186,23 @@ extension RawSyntax {
 
   /// The "width" of the node.
   ///
-  /// Sum of text byte lengths of all descendant token nodes.
+  /// Sum of text byte lengths of all present descendant token nodes.
   var byteLength: Int {
     switch rawData.payload {
-    case .parsedToken(let dat): return dat.wholeText.count
-    case .materializedToken(let dat): return Int(dat.byteLength)
-    case .layout(let dat): return dat.byteLength
+    case .parsedToken(let dat):
+      if dat.presence == .present {
+        return dat.wholeText.count
+      } else {
+        return 0
+      }
+    case .materializedToken(let dat):
+      if dat.presence == .present {
+        return Int(dat.byteLength)
+      } else {
+        return 0
+      }
+    case .layout(let dat):
+      return dat.byteLength
     }
   }
 
@@ -258,18 +272,19 @@ extension RawSyntax: TextOutputStreamable, CustomStringConvertible {
   public func write<Target: TextOutputStream>(to target: inout Target) {
     switch rawData.payload {
     case .parsedToken(let dat):
-      String(syntaxText: dat.wholeText).write(to: &target)
-      break
+      if dat.presence == .present {
+        String(syntaxText: dat.wholeText).write(to: &target)
+      }
     case .materializedToken(let dat):
-      for p in dat.leadingTrivia { p.write(to: &target) }
-      String(syntaxText: dat.tokenText).write(to: &target)
-      for p in dat.trailingTrivia { p.write(to: &target) }
-      break
+      if dat.presence == .present {
+        for p in dat.leadingTrivia { p.write(to: &target) }
+        String(syntaxText: dat.tokenText).write(to: &target)
+        for p in dat.trailingTrivia { p.write(to: &target) }
+      }
     case .layout(let dat):
       for case let child? in dat.layout {
         child.write(to: &target)
       }
-      break
     }
   }
 
@@ -364,17 +379,23 @@ extension RawSyntax {
   ///   - kind: Token kind.
   ///   - wholeText: Whole text of this token including trailing/leading trivia.
   ///   - textRange: Range of the token text in `wholeText`.
+  ///   - presence: Whether the token appeared in the source code or if it was synthesized.
   ///   - arena: SyntaxArea to the result node data resides.
   internal static func parsedToken(
     kind: RawTokenKind,
     wholeText: SyntaxText,
     textRange: Range<SyntaxText.Index>,
+    presence: SourcePresence,
     arena: SyntaxArena
   ) -> RawSyntax {
     assert(arena.contains(text: wholeText),
            "token text must be managed by the arena")
     let payload = RawSyntaxData.ParsedToken(
-      tokenKind: kind, wholeText: wholeText, textRange: textRange)
+      tokenKind: kind,
+      wholeText: wholeText,
+      textRange: textRange,
+      presence: presence
+    )
     return RawSyntax(arena: arena, payload: .parsedToken(payload))
   }
 
@@ -385,29 +406,34 @@ extension RawSyntax {
   /// `makeMissingToken(arena:kind:)` instead.
   ///
   /// - Parameters:
-  ///   - arena: SyntaxArea to the result node data resides.
   ///   - kind: Token kind.
   ///   - text: Token text.
   ///   - triviaPieces: Raw trivia pieces including leading and trailing trivia.
   ///   - numLeadingTrivia: Number of leading trivia pieces in `triviaPieces`.
   ///   - byteLength: Byte length of this token including trivia.
+  ///   - presence: Whether the token appeared in the source code or if it was synthesized.
+  ///   - arena: SyntaxArea to the result node data resides.
   internal static func materializedToken(
     kind: RawTokenKind,
     text: SyntaxText,
     triviaPieces: RawTriviaPieceBuffer,
     numLeadingTrivia: UInt32,
     byteLength: UInt32,
+    presence: SourcePresence,
     arena: SyntaxArena
   ) -> RawSyntax {
     assert(arena.contains(text: text) || kind.defaultText?.baseAddress == text.baseAddress,
            "token text must be managed by the arena, or known default text for the token")
     assert(triviaPieces.allSatisfy({$0.storedText.map({arena.contains(text: $0)}) ?? true}),
            "trivia text must be managed by the arena")
     let payload = RawSyntaxData.MaterializedToken(
-      tokenKind: kind, tokenText: text,
+      tokenKind: kind,
+      tokenText: text,
       triviaPieces: triviaPieces,
       numLeadingTrivia: numLeadingTrivia,
-      byteLength: byteLength)
+      byteLength: byteLength,
+      presence: presence
+    )
     return RawSyntax(arena: arena, payload: .materializedToken(payload))
   }
 
@@ -418,6 +444,7 @@ extension RawSyntax {
   ///   - text: Token text.
   ///   - leadingTriviaPieceCount: Number of leading trivia pieces.
   ///   - trailingTriviaPieceCount: Number of trailing trivia pieces.
+  ///   - presence: Whether the token appeared in the source code or if it was synthesized.
   ///   - arena: SyntaxArea to the result node data resides.
   ///   - initializingLeadingTriviaWith: A closure that initializes leading trivia pieces.
   ///   - initializingTrailingTriviaWith: A closure that initializes trailing trivia pieces.
@@ -426,6 +453,7 @@ extension RawSyntax {
     text: SyntaxText,
     leadingTriviaPieceCount: Int,
     trailingTriviaPieceCount: Int,
+    presence: SourcePresence,
     arena: SyntaxArena,
     initializingLeadingTriviaWith: (UnsafeMutableBufferPointer<RawTriviaPiece>) -> Void,
     initializingTrailingTriviaWith : (UnsafeMutableBufferPointer<RawTriviaPiece>) -> Void
@@ -442,7 +470,9 @@ extension RawSyntax {
       kind: kind, text: text, triviaPieces: RawTriviaPieceBuffer(triviaBuffer),
       numLeadingTrivia: numericCast(leadingTriviaPieceCount),
       byteLength: numericCast(byteLength),
-      arena: arena)
+      presence: presence,
+      arena: arena
+    )
   }
 
   /// Factory method to create a materialized token node.
@@ -462,20 +492,15 @@ extension RawSyntax {
   ) -> RawSyntax {
     let decomposed = kind.decomposeToRaw()
     let rawKind = decomposed.rawKind
-    let text: SyntaxText
-    switch presence {
-    case .present:
-      text = (decomposed.string.map({arena.intern($0)}) ??
-              decomposed.rawKind.defaultText ??
-              "")
-    case .missing:
-      text = SyntaxText()
-    }
+    let text = (decomposed.string.map({arena.intern($0)}) ??
+                decomposed.rawKind.defaultText ??
+                "")
 
     return .makeMaterializedToken(
       kind: rawKind, text: text,
       leadingTriviaPieceCount: leadingTrivia.count,
       trailingTriviaPieceCount: trailingTrivia.count,
+      presence: presence,
       arena: arena,
       initializingLeadingTriviaWith: { buffer in
         guard var ptr = buffer.baseAddress else { return }
@@ -499,8 +524,10 @@ extension RawSyntax {
   ) -> RawSyntax {
     let (rawKind, _) = kind.decomposeToRaw()
     return .materializedToken(
-      kind: rawKind, text: "", triviaPieces: .init(start: nil, count: 0),
+      kind: rawKind, text: rawKind.defaultText ?? "",
+      triviaPieces: .init(start: nil, count: 0),
       numLeadingTrivia: 0, byteLength: 0,
+      presence: .missing,
       arena: arena)
   }
 }

Sources/SwiftSyntax/Raw/RawSyntaxNodeProtocol.swift

@@ -116,10 +116,16 @@ public struct RawTokenSyntax: RawSyntaxNodeProtocol {
     kind: RawTokenKind,
     wholeText: SyntaxText,
     textRange: Range<SyntaxText.Index>,
+    presence: SourcePresence,
     arena: __shared SyntaxArena
   ) {
     let raw = RawSyntax.parsedToken(
-      kind: kind, wholeText: wholeText, textRange: textRange, arena: arena)
+      kind: kind,
+      wholeText: wholeText,
+      textRange: textRange,
+      presence: presence,
+      arena: arena
+    )
     self = RawTokenSyntax(raw: raw)
   }
 
@@ -130,12 +136,15 @@ public struct RawTokenSyntax: RawSyntaxNodeProtocol {
     text: SyntaxText,
     leadingTriviaPieces: [RawTriviaPiece],
     trailingTriviaPieces: [RawTriviaPiece],
+    presence: SourcePresence,
     arena: __shared SyntaxArena
   ) {
     let raw = RawSyntax.makeMaterializedToken(
-      kind: kind, text: text,
+      kind: kind,
+      text: text,
       leadingTriviaPieceCount: leadingTriviaPieces.count,
       trailingTriviaPieceCount: trailingTriviaPieces.count,
+      presence: presence,
       arena: arena,
       initializingLeadingTriviaWith: { buffer in
         _ = buffer.initialize(from: leadingTriviaPieces)
@@ -146,10 +155,18 @@ public struct RawTokenSyntax: RawSyntaxNodeProtocol {
   }
 
   /// Creates a missing `TokenSyntax` with the specified kind.
+  /// If `text` is passed, it will be used to represent the missing token's text.
+  /// If `text` is `nil`, the `kind`'s default text will be used.
+  /// If that is also `nil`, the token will have empty text.
   public init(missing kind: RawTokenKind, arena: __shared SyntaxArena) {
     self.init(
-      kind: kind, text: "", leadingTriviaPieces: [], trailingTriviaPieces: [],
-      arena: arena)
+      kind: kind,
+      text: kind.defaultText ?? "",
+      leadingTriviaPieces: [],
+      trailingTriviaPieces: [],
+      presence: .missing,
+      arena: arena
+    )
   }
 }
 

Sources/SwiftSyntax/Raw/RawSyntaxTokenView.swift

@@ -189,19 +189,14 @@ struct RawSyntaxTokenView {
   }
 
   var presence: SourcePresence {
-    if self.raw.byteLength != 0 {
-      // The node has source text associated with it. It's present.
-      return .present
-    }
-    if rawKind == .eof || rawKind == .stringSegment {
-      // The end of file token never has source code associated with it but we
-      // still consider it valid.
-      // String segments can be empty if they occur in an empty string literal or in between two interpolation segments.
-      return .present
+    switch raw.rawData.payload {
+    case .parsedToken(let dat):
+      return dat.presence
+    case .materializedToken(let dat):
+      return dat.presence
+    case .layout(_):
+      preconditionFailure("'presence' is a token-only property")
     }
-
-    // If none of the above apply, the node is missing.
-    return .missing
   }
 
 }

Sources/SwiftSyntaxParser/RawSyntax+CNodes.swift

@@ -65,6 +65,7 @@ extension RawSyntax {
       kind: tokenKind, text: tokenText,
       leadingTriviaPieceCount: leadingTriviaInfo.count,
       trailingTriviaPieceCount: trailingTriviaInfo.count,
+      presence: cnode.present ? .present : .missing,
       arena: arena,
       initializingLeadingTriviaWith: {
         initializeRawTriviaBuffer($0, 0, leadingTriviaInfo)

Tests/SwiftParserTest/Assertions.swift

@@ -208,8 +208,8 @@ func AssertParse<Node: RawSyntaxNodeProtocol>(
 
       // Applying Fix-Its
       if let expectedFixedSource = expectedFixedSource {
-        let fixedSource = FixItApplier.applyFixes(in: diags, to: tree).description
-        AssertStringsEqualWithDiff(fixedSource, expectedFixedSource, file: file, line: line)
+        let fixedTree = FixItApplier.applyFixes(in: diags, to: tree)
+        AssertStringsEqualWithDiff(fixedTree.description, expectedFixedSource, file: file, line: line)
       }
     }
   }