Make SyntaxIndexInTree public and serializable into a UInt64

If you have two copies of the same syntax tree within two processes, this allows us to communicate the identifier of a node in one process the the equivalent node in the other process.

Author: ahoppen

Release Notes/600.md

@@ -75,6 +75,22 @@
   - Description: With the change to parse `#if canImport(MyModule, _version: 1.2.3)` as a function call instead of a dedicated syntax node, `1.2.3` natively gets parsed as a member access `3` to the `1.2` float literal. This property allows the reinterpretation of such an expression as a version tuple.
   - Pull request: https://github.com/apple/swift-syntax/pull/2025
 
+- `SyntaxProtocol.node(at:)` 
+  - Description: Given a `SyntaxIdentifier`, returns the `Syntax` node with that identifier
+  - Pull request: https://github.com/apple/swift-syntax/pull/2594
+
+- `SyntaxIdentifier.IndexInTree`
+  - Description: Uniquely identifies a syntax node within a tree. This is similar to ``SyntaxIdentifier`` but does not store the root ID of the tree. It can thus be transferred across trees that are structurally equivalent, for example two copies of the same tree that live in different processes. The only public functions on this type are `toOpaque` and `init(fromOpaque:)`, which allow serialization of the `IndexInTree`.
+  - Pull request: https://github.com/apple/swift-syntax/pull/2594
+  
+- `SyntaxIdentifier` conformance to `Comparable`:
+  - Description: A `SyntaxIdentifier` compares less than another `SyntaxIdentifier` if the node at that identifier occurs first during a depth-first traversal of the tree.
+  - Pull request: https://github.com/apple/swift-syntax/pull/2594
+
+- `SyntaxIdentifier.indexInTree` and `SyntaxIdentifier.fromIndexInTree` 
+  - Description: `SyntaxIdentifier.indexInTree` allows the retrieval of a `SyntaxIdentifier` that identifies the syntax node independent of the syntax tree. `SyntaxIdentifier.fromIndexInTree` allows the creation for a `SyntaxIdentifier` from a tree-agnostic `SyntaxIdentifier.IndexInTree` and the tree's root node.
+  - Pull request: https://github.com/apple/swift-syntax/pull/2594
+
 ## API Behavior Changes
 
 ## Deprecations

Sources/SwiftParserDiagnostics/ParseDiagnosticsGenerator.swift

@@ -13,11 +13,11 @@
 #if swift(>=6)
 public import SwiftDiagnostics
 @_spi(Diagnostics) import SwiftParser
-@_spi(RawSyntax) @_spi(ExperimentalLanguageFeatures) public import SwiftSyntax
+@_spi(ExperimentalLanguageFeatures) public import SwiftSyntax
 #else
 import SwiftDiagnostics
 @_spi(Diagnostics) import SwiftParser
-@_spi(RawSyntax) @_spi(ExperimentalLanguageFeatures) import SwiftSyntax
+@_spi(ExperimentalLanguageFeatures) import SwiftSyntax
 #endif
 
 fileprivate func getTokens(between first: TokenSyntax, and second: TokenSyntax) -> [TokenSyntax] {
@@ -119,7 +119,7 @@ public class ParseDiagnosticsGenerator: SyntaxAnyVisitor {
         return false
       } else {
         // If multiple tokens are missing at the same location, emit diagnostics about nodes that occur earlier in the tree first.
-        return $0.node.id.indexInTree < $1.node.id.indexInTree
+        return $0.node.id < $1.node.id
       }
     }
     return diagProducer.diagnostics

Sources/SwiftSyntax/CommonAncestor.swift

@@ -18,11 +18,11 @@ public func findCommonAncestorOrSelf(_ lhs: Syntax, _ rhs: Syntax) -> Syntax? {
     if lhs == rhs {
       return lhs
     }
-    if let lhsIndex = lhs?.indexInParent.data?.indexInTree, let rhsIndex = rhs?.indexInParent.data?.indexInTree {
-      if lhsIndex < rhsIndex {
-        rhs = rhs?.parent
+    if let lhsUnwrapped = lhs, let rhsUnwrapped = rhs {
+      if lhsUnwrapped.id < rhsUnwrapped.id {
+        rhs = rhsUnwrapped.parent
       } else {
-        lhs = lhs?.parent
+        lhs = lhsUnwrapped.parent
       }
     }
   }

Sources/SwiftSyntax/SyntaxChildren.swift

@@ -22,8 +22,8 @@ struct SyntaxChildrenIndexData: Hashable, Comparable, Sendable {
   /// See `AbsoluteSyntaxPosition.indexInParent`
   let indexInParent: UInt32
   /// Unique value for a node within its own tree.
-  /// See `SyntaxIdentifier.indexIntree`
-  let indexInTree: SyntaxIndexInTree
+  /// See ``SyntaxIdentifier/indexInTree``
+  let indexInTree: SyntaxIdentifier.SyntaxIndexInTree
 
   static func < (
     lhs: SyntaxChildrenIndexData,
@@ -35,7 +35,7 @@ struct SyntaxChildrenIndexData: Hashable, Comparable, Sendable {
   fileprivate init(
     offset: UInt32,
     indexInParent: UInt32,
-    indexInTree: SyntaxIndexInTree
+    indexInTree: SyntaxIdentifier.SyntaxIndexInTree
   ) {
     self.offset = offset
     self.indexInParent = indexInParent
@@ -72,7 +72,7 @@ public struct SyntaxChildrenIndex: Hashable, Comparable, ExpressibleByNilLiteral
   fileprivate init(
     offset: UInt32,
     indexInParent: UInt32,
-    indexInTree: SyntaxIndexInTree
+    indexInTree: SyntaxIdentifier.SyntaxIndexInTree
   ) {
     self.data = SyntaxChildrenIndexData(
       offset: offset,
@@ -222,7 +222,7 @@ struct RawSyntaxChildren: BidirectionalCollection, Sendable {
       let offset = startIndex.offset + UInt32(parent.totalLength.utf8Length)
       let indexInParent = startIndex.indexInParent + UInt32(parentLayoutView.children.count)
       let indexInTree = startIndex.indexInTree.indexInTree + UInt32(parent.totalNodes) - 1
-      let syntaxIndexInTree = SyntaxIndexInTree(indexInTree: indexInTree)
+      let syntaxIndexInTree = SyntaxIdentifier.SyntaxIndexInTree(indexInTree: indexInTree)
       let materialized = SyntaxChildrenIndex(
         offset: offset,
         indexInParent: indexInParent,

Sources/SwiftSyntax/SyntaxIdentifier.swift

@@ -10,42 +10,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-/// Represents a unique value for a node within its own tree.
-@_spi(RawSyntax)
-public struct SyntaxIndexInTree: Comparable, Hashable, Sendable {
-  let indexInTree: UInt32
-
-  static let zero: SyntaxIndexInTree = SyntaxIndexInTree(indexInTree: 0)
-
-  /// Assuming that this index points to the start of ``Raw``, so that it points
-  /// to the next sibling of ``Raw``.
-  func advancedBy(_ raw: RawSyntax?) -> SyntaxIndexInTree {
-    let newIndexInTree = self.indexInTree + UInt32(truncatingIfNeeded: raw?.totalNodes ?? 0)
-    return .init(indexInTree: newIndexInTree)
-  }
-
-  /// Assuming that this index points to the next sibling of ``Raw``, reverse it
-  /// so that it points to the start of ``Raw``.
-  func reversedBy(_ raw: RawSyntax?) -> SyntaxIndexInTree {
-    let newIndexInTree = self.indexInTree - UInt32(truncatingIfNeeded: raw?.totalNodes ?? 0)
-    return .init(indexInTree: newIndexInTree)
-  }
-
-  func advancedToFirstChild() -> SyntaxIndexInTree {
-    let newIndexInTree = self.indexInTree + 1
-    return .init(indexInTree: newIndexInTree)
-  }
-
-  init(indexInTree: UInt32) {
-    self.indexInTree = indexInTree
-  }
-
-  /// Returns `true` if `lhs` occurs before `rhs` in the tree.
-  public static func < (lhs: SyntaxIndexInTree, rhs: SyntaxIndexInTree) -> Bool {
-    return lhs.indexInTree < rhs.indexInTree
-  }
-}
-
 /// Provides a stable and unique identity for ``Syntax`` nodes.
 ///
 /// Note that two nodes might have the same contents even if their IDs are
@@ -57,7 +21,53 @@ public struct SyntaxIndexInTree: Comparable, Hashable, Sendable {
 /// different syntax tree. Modifying any node in the syntax tree a node is
 /// contained in generates a copy of that tree and thus changes the IDs of all
 /// nodes in the tree, not just the modified node's children.
-public struct SyntaxIdentifier: Hashable, Sendable {
+public struct SyntaxIdentifier: Comparable, Hashable, Sendable {
+  /// Represents a unique value for a node within its own tree.
+  ///
+  /// This is similar to ``SyntaxIdentifier`` but does not store the root ID of the tree.
+  /// It can thus be transferred across trees that are structurally equivalent, for example two copies of the same tree
+  /// that live in different processes.
+  public struct SyntaxIndexInTree: Hashable, Sendable {
+    /// When traversing the syntax tree using a depth-first traversal, the index at which the node will be visited.
+    let indexInTree: UInt32
+
+    /// Assuming that this index points to the start of `raw`, advance it so that it points to the next sibling of
+    /// `raw`.
+    func advancedBy(_ raw: RawSyntax?) -> SyntaxIndexInTree {
+      let newIndexInTree = self.indexInTree + UInt32(truncatingIfNeeded: raw?.totalNodes ?? 0)
+      return .init(indexInTree: newIndexInTree)
+    }
+
+    /// Assuming that this index points to the next sibling of `raw`, reverse it so that it points to the start of
+    /// `raw`.
+    func reversedBy(_ raw: RawSyntax?) -> SyntaxIndexInTree {
+      let newIndexInTree = self.indexInTree - UInt32(truncatingIfNeeded: raw?.totalNodes ?? 0)
+      return .init(indexInTree: newIndexInTree)
+    }
+
+    func advancedToFirstChild() -> SyntaxIndexInTree {
+      let newIndexInTree = self.indexInTree + 1
+      return .init(indexInTree: newIndexInTree)
+    }
+
+    init(indexInTree: UInt32) {
+      self.indexInTree = indexInTree
+    }
+
+    /// Converts the ``SyntaxIdentifier/SyntaxIndexInTree`` to an opaque value that can be serialized.
+    /// The opaque value can be restored to a ``SyntaxIdentifier/SyntaxIndexInTree`` using ``init(fromOpaque:)``.
+    ///
+    /// - Note: The contents of the opaque value are not specified and clients should not rely on them.
+    public func toOpaque() -> UInt64 {
+      return UInt64(indexInTree)
+    }
+
+    /// Creates a ``SyntaxIdentifier/SyntaxIndexInTree`` from an opaque value obtained using ``toOpaque()``.
+    public init(fromOpaque opaque: UInt64) {
+      self.indexInTree = UInt32(opaque)
+    }
+  }
+
   /// Unique value for the root node.
   ///
   /// Multiple trees may have the same 'rootId' if their root RawSyntax is the
@@ -67,23 +77,65 @@ public struct SyntaxIdentifier: Hashable, Sendable {
   let rootId: UInt
 
   /// Unique value for a node within its own tree.
-  @_spi(RawSyntax)
   public let indexInTree: SyntaxIndexInTree
 
+  /// Returns the `UInt` that is used as the root ID for the given raw syntax node.
+  private static func rootId(of raw: RawSyntax) -> UInt {
+    return UInt(bitPattern: raw.pointer.unsafeRawPointer)
+  }
+
   func advancedBySibling(_ raw: RawSyntax?) -> SyntaxIdentifier {
     let newIndexInTree = indexInTree.advancedBy(raw)
-    return .init(rootId: self.rootId, indexInTree: newIndexInTree)
+    return SyntaxIdentifier(rootId: self.rootId, indexInTree: newIndexInTree)
   }
 
   func advancedToFirstChild() -> SyntaxIdentifier {
     let newIndexInTree = self.indexInTree.advancedToFirstChild()
-    return .init(rootId: self.rootId, indexInTree: newIndexInTree)
+    return SyntaxIdentifier(rootId: self.rootId, indexInTree: newIndexInTree)
   }
 
   static func forRoot(_ raw: RawSyntax) -> SyntaxIdentifier {
-    return .init(
-      rootId: UInt(bitPattern: raw.pointer.unsafeRawPointer),
-      indexInTree: .zero
+    return SyntaxIdentifier(
+      rootId: Self.rootId(of: raw),
+      indexInTree: SyntaxIndexInTree(indexInTree: 0)
     )
   }
+
+  /// Forms a ``SyntaxIdentifier`` from an ``SyntaxIdentifier/SyntaxIndexInTree`` inside a ``Syntax`` node that
+  /// constitutes the tree's root.
+  ///
+  /// Returns `nil` if `root` is not the root of a syntax tree or if `indexInTree` points to a node that is not within
+  /// the tree spanned up by `root`.
+  ///
+  /// - Warning: ``SyntaxIdentifier/SyntaxIndexInTree`` is not stable with regard to insertion or deletions of nodes
+  ///   into a syntax tree. There are only two scenarios where it is valid to share ``SyntaxIndexInTree`` between syntax
+  ///   trees with different nodes:
+  ///   (1) If two trees are guaranteed to be exactly the same eg. because they were parsed using the same version of
+  ///       `SwiftParser` from the same source code.
+  ///   (2) If a tree was mutated by only replacing tokens with other tokens. No nodes must have been inserted or
+  ///       removed during the process, including tokens that are marked as ``SourcePresence/missing``.
+  public static func fromIndexInTree(
+    _ indexInTree: SyntaxIndexInTree,
+    relativeToRoot root: some SyntaxProtocol
+  ) -> SyntaxIdentifier? {
+    guard !root.hasParent else {
+      return nil
+    }
+    guard indexInTree.indexInTree < SyntaxIndexInTree(indexInTree: 0).advancedBy(root.raw).indexInTree else {
+      return nil
+    }
+
+    return SyntaxIdentifier(rootId: Self.rootId(of: root.raw), indexInTree: indexInTree)
+  }
+
+  /// A ``SyntaxIdentifier`` compares less than another ``SyntaxIdentifier`` if the node at that identifier occurs first
+  /// during a depth-first traversal of the tree. This implies that nodes with an earlier ``AbsolutePosition`` also
+  /// have a lower ``SyntaxIdentifier``.
+  public static func < (lhs: SyntaxIdentifier, rhs: SyntaxIdentifier) -> Bool {
+    guard lhs.rootId == rhs.rootId else {
+      // Nodes in different trees are not comparable.
+      return false
+    }
+    return lhs.indexInTree.indexInTree < rhs.indexInTree.indexInTree
+  }
 }

Sources/SwiftSyntax/SyntaxProtocol.swift

@@ -352,6 +352,28 @@ public extension SyntaxProtocol {
 
     fatalError("Children of syntax node do not cover all positions in it")
   }
+
+  /// If the node with the given `syntaxIdentifier` is a (recursive) child of this node, return the node with that
+  /// identifier.
+  ///
+  /// If the identifier references a node from a different tree (ie. one that has a different root ID in the
+  /// ``SyntaxIdentifier``) or if no node with the given identifier is a child of this syntax node, returns `nil`.
+  func node(at syntaxIdentifier: SyntaxIdentifier) -> Syntax? {
+    guard self.id <= syntaxIdentifier && syntaxIdentifier < self.id.advancedBySibling(self.raw) else {
+      // The syntax identifier is not part of this tree.
+      return nil
+    }
+    if self.id == syntaxIdentifier {
+      return Syntax(self)
+    }
+    for child in children(viewMode: .all) {
+      if let node = child.node(at: syntaxIdentifier) {
+        return node
+      }
+    }
+
+    preconditionFailure("syntaxIdentifier is covered by this node but not any of its children?")
+  }
 }
 
 // MARK: Recursive flags

Tests/SwiftSyntaxTest/SyntaxTests.swift

@@ -146,4 +146,26 @@ class SyntaxTests: XCTestCase {
     let node = ClosureCaptureSyntax(name: "test", expression: ExprSyntax("123"))
     XCTAssertEqual(node.formatted().description, "test = 123")
   }
+
+  func testShareSyntaxIndexInTreeBetweenTrees() throws {
+    let source = "func foo() {}"
+
+    let tree1 = DeclSyntax(stringLiteral: source)
+    let tree2 = DeclSyntax(stringLiteral: source)
+
+    let funcKeywordInTree1 = try XCTUnwrap(tree1.firstToken(viewMode: .sourceAccurate))
+    XCTAssertEqual(funcKeywordInTree1.tokenKind, .keyword(.func))
+
+    let opaqueIndexInTree1 = funcKeywordInTree1.id.indexInTree.toOpaque()
+
+    let funcKeywordIdentifierInTree2 = try XCTUnwrap(
+      SyntaxIdentifier.fromIndexInTree(
+        SyntaxIdentifier.SyntaxIndexInTree(fromOpaque: opaqueIndexInTree1),
+        relativeToRoot: tree2
+      )
+    )
+    let funcKeywordInTree2 = tree2.node(at: funcKeywordIdentifierInTree2)
+    XCTAssertEqual(funcKeywordInTree2?.as(TokenSyntax.self)?.tokenKind, .keyword(.func))
+    XCTAssertNotEqual(funcKeywordInTree1.id, funcKeywordInTree2?.id)
+  }
 }