[swiftSyntax] Incremental syntax classification

tools/SwiftSyntax/RawSyntax.swift

@@ -26,7 +26,7 @@ extension CodingUserInfoKey {
 
 /// A ID that uniquely identifies a syntax node and stays stable across multiple
 /// incremental parses
-struct SyntaxNodeId: Hashable, Codable {
+public struct SyntaxNodeId: Hashable, Codable {
   private let rawValue: UInt
 
   // Start creating fresh node IDs for user generated nodes on in the upper
@@ -43,11 +43,11 @@ struct SyntaxNodeId: Hashable, Codable {
     self.rawValue = rawValue
   }
 
-  init(from decoder: Decoder) throws {
+  public init(from decoder: Decoder) throws {
     self.rawValue = try decoder.singleValueContainer().decode(UInt.self)
   }
 
-  func encode(to encoder: Encoder) throws {
+  public func encode(to encoder: Encoder) throws {
     var container = encoder.singleValueContainer()
     try container.encode(rawValue)
   }

tools/SwiftSyntax/SwiftSyntax.swift

@@ -53,6 +53,10 @@ public final class SyntaxTreeDeserializer {
   /// they were omitted in an incremental syntax tree transfer
   private var nodeLookupTable: [SyntaxNodeId: RawSyntax] = [:]
 
+  /// The IDs of the nodes that were reused as part of incremental syntax
+  /// parsing during the last deserialization
+  public var reusedNodeIds: Set<SyntaxNodeId> = []
+
   public init() {
   }
 
@@ -61,6 +65,7 @@ public final class SyntaxTreeDeserializer {
   /// - Returns: A top-level Syntax node representing the contents of the tree,
   ///            if the parse was successful.
   public func deserialize(_ data: Data) throws -> SourceFileSyntax {
+    reusedNodeIds = []
     let decoder = JSONDecoder()
     decoder.userInfo[.rawSyntaxDecodedCallback] = self.addToLookupTable
     decoder.userInfo[.omittedNodeLookupFunction] = self.lookupNode
@@ -74,6 +79,7 @@ public final class SyntaxTreeDeserializer {
   // MARK: Incremental deserialization helper functions
 
   private func lookupNode(id: SyntaxNodeId) -> RawSyntax? {
+    reusedNodeIds.insert(id)
     return nodeLookupTable[id]
   }
 

tools/SwiftSyntax/SyntaxClassifier.swift.gyb

@@ -30,9 +30,18 @@ public enum SyntaxClassification {
 
 class _SyntaxClassifier: SyntaxVisitor {
 
+  /// Don't classify nodes with these IDs or any of their children
+  var skipNodeIds: Set<SyntaxNodeId> = []
+
+  /// The top of the `contextStack` determines the classification for all tokens
+  /// encountered that do not have a native classification. If `force` is `true`
+  /// the top of the stack also determines the classification of tokens with a
+  /// native classification
   private var contextStack: [(classification: SyntaxClassification, force: Bool)] = 
       [(classification: .none, force: false)]
 
+  /// The classifications that have determined so far are collected in this
+  /// dictionary
   var classifications: [TokenSyntax: SyntaxClassification] = [:]
 
   private func visit(
@@ -82,6 +91,9 @@ class _SyntaxClassifier: SyntaxVisitor {
 % for node in SYNTAX_NODES:
 %   if is_visitable(node):
   override func visit(_ node: ${node.name}) {
+    if skipNodeIds.contains(node.raw.id) {
+      return
+    }
 %     if node.is_unknown() or node.is_syntax_collection():
     super.visit(node)
 %     else:
@@ -114,10 +126,15 @@ class _SyntaxClassifier: SyntaxVisitor {
 }
 
 public enum SyntaxClassifier {
-  /// Classify all tokens in the given syntax tree for syntax highlighting
-  public static func classifyTokensInTree(_ syntaxTree: SourceFileSyntax) 
-      -> [TokenSyntax: SyntaxClassification] {
+  /// Classify all tokens in the given syntax tree for syntax highlighting.
+  /// If a `IncrementalTreeTransferSession` is passed, only nodes that have 
+  /// changed since the last transfer will be classified.
+  public static func classifyTokensInTree(
+    _ syntaxTree: SourceFileSyntax,
+    skipNodes: Set<SyntaxNodeId> = []
+  ) -> [TokenSyntax: SyntaxClassification] {
     let classifier = _SyntaxClassifier()
+    classifier.skipNodeIds = skipNodes
     classifier.visit(syntaxTree)
     return classifier.classifications
   }