Move SyntaxClassifier to a separate module

rdar://98318240

Author: ahoppen

Package.swift

@@ -45,6 +45,7 @@ let package = Package(
     .macCatalyst(.v13),
   ],
   products: [
+    .library(name: "IDEUtils", type: .static, targets: ["IDEUtils"]),
     .library(name: "SwiftOperators", type: .static, targets: ["SwiftOperators"]),
     .library(name: "SwiftParser", type: .static, targets: ["SwiftParser"]),
     .library(name: "SwiftSyntax", type: .static, targets: ["SwiftSyntax"]),
@@ -77,7 +78,6 @@ let package = Package(
         "Raw/RawSyntaxValidation.swift.gyb",
         "SyntaxAnyVisitor.swift.gyb",
         "SyntaxBaseNodes.swift.gyb",
-        "SyntaxClassification.swift.gyb",
         "SyntaxCollections.swift.gyb",
         "SyntaxEnum.swift.gyb",
         "SyntaxFactory.swift.gyb",
@@ -97,6 +97,7 @@ let package = Package(
       name: "SwiftSyntaxBuilder",
       dependencies: ["SwiftBasicFormat", "SwiftSyntax", "SwiftParser"],
       exclude: [
+        "CMakeLists.txt",
         "gyb_helpers",
         "SyntaxExpressibleByStringInterpolationConformances.swift.gyb",
       ]
@@ -114,6 +115,13 @@ let package = Package(
       name: "_SwiftSyntaxTestSupport",
       dependencies: ["SwiftBasicFormat", "SwiftSyntax", "SwiftSyntaxBuilder"]
     ),
+    .target(
+      name: "IDEUtils",
+      dependencies: ["SwiftSyntax"],
+      exclude: [
+        "SyntaxClassification.swift.gyb",
+      ]
+    ),
     .target(
       name: "SwiftParser",
       dependencies: ["SwiftBasicFormat", "SwiftDiagnostics", "SwiftSyntax"],
@@ -151,13 +159,14 @@ let package = Package(
       ]),
     .executableTarget(
       name: "lit-test-helper",
-      dependencies: ["SwiftSyntax", "SwiftSyntaxParser"]
+      dependencies: ["IDEUtils", "SwiftSyntax", "SwiftSyntaxParser"]
     ),
     .executableTarget(
       name: "swift-parser-cli",
       dependencies: ["SwiftDiagnostics", "SwiftSyntax", "SwiftParser", "SwiftOperators", "_SwiftSyntaxMacros",
                      .product(name: "ArgumentParser", package: "swift-argument-parser")]
     ),
+    .testTarget(name: "IDEUtilsTest", dependencies: ["_SwiftSyntaxTestSupport", "SwiftParser", "SwiftSyntax", "IDEUtils"]),
     .testTarget(
       name: "SwiftDiagnosticsTest",
       dependencies: ["_SwiftSyntaxTestSupport", "SwiftDiagnostics", "SwiftParser"]
@@ -183,7 +192,7 @@ let package = Package(
     ),
     .testTarget(
       name: "PerformanceTest",
-      dependencies: ["SwiftSyntax", "SwiftSyntaxParser", "SwiftParser"],
+      dependencies: ["IDEUtils", "SwiftSyntax", "SwiftSyntaxParser", "SwiftParser"],
       exclude: ["Inputs"]
     ),
     .testTarget(

Sources/IDEUtils/Syntax+Classifications.swift

@@ -0,0 +1,66 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2022 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import SwiftSyntax
+
+public extension SyntaxProtocol {
+
+  /// Sequence of `SyntaxClassifiedRange`s for this syntax node.
+  ///
+  /// The provided classified ranges are consecutive and cover the full source
+  /// text of the node. The ranges may also span multiple tokens, if multiple
+  /// consecutive tokens would have the same classification then a single classified
+  /// range is provided for all of them.
+  var classifications: SyntaxClassifications {
+    let fullRange = ByteSourceRange(offset: 0, length: byteSize)
+    return SyntaxClassifications(_syntaxNode, in: fullRange)
+  }
+
+  /// Sequence of `SyntaxClassifiedRange`s contained in this syntax node within
+  /// a relative range.
+  ///
+  /// The provided classified ranges may extend beyond the provided `range`.
+  /// Active classifications (non-`none`) will extend the range to include the
+  /// full classified range (e.g. from the beginning of the comment block), while
+  /// `none` classified ranges will extend to the beginning or end of the token
+  /// that the `range` touches.
+  /// It is guaranteed that no classified range will be provided that doesn't
+  /// intersect the provided `range`.
+  ///
+  /// - Parameters:
+  ///   - in: The relative byte range to pull `SyntaxClassifiedRange`s from.
+  /// - Returns: Sequence of `SyntaxClassifiedRange`s.
+  func classifications(in range: ByteSourceRange) -> SyntaxClassifications {
+    return SyntaxClassifications(_syntaxNode, in: range)
+  }
+
+  /// The `SyntaxClassifiedRange` for a relative byte offset.
+  /// - Parameters:
+  ///   - at: The relative to the node byte offset.
+  /// - Returns: The `SyntaxClassifiedRange` for the offset or nil if the source text
+  ///   at the given offset is unclassified.
+  func classification(at offset: Int) -> SyntaxClassifiedRange? {
+    let classifications = SyntaxClassifications(_syntaxNode, in: ByteSourceRange(offset: offset, length: 1))
+    var iterator = classifications.makeIterator()
+    return iterator.next()
+  }
+
+  /// The `SyntaxClassifiedRange` for an absolute position.
+  /// - Parameters:
+  ///   - at: The absolute position.
+  /// - Returns: The `SyntaxClassifiedRange` for the position or nil if the source text
+  ///   at the given position is unclassified.
+  func classification(at position: AbsolutePosition) -> SyntaxClassifiedRange? {
+    let relativeOffset = position.utf8Offset - self.position.utf8Offset
+    return self.classification(at: relativeOffset)
+  }
+}

Sources/SwiftSyntax/SyntaxClassification.swift.gyb renamed to Sources/IDEUtils/SyntaxClassification.swift.gyb

@@ -35,6 +35,8 @@
 //
 //===----------------------------------------------------------------------===//
 
+@_spi(RawSyntax) import SwiftSyntax
+
 public enum SyntaxClassification {
 % for classification in SYNTAX_CLASSIFICATIONS:
 %   for line in dedented_lines(classification.description):

Sources/SwiftSyntax/SyntaxClassifier.swift renamed to Sources/IDEUtils/SyntaxClassifier.swift

@@ -10,10 +10,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-extension SyntaxData {
+@_spi(RawSyntax) import SwiftSyntax
+
+fileprivate extension SyntaxProtocol {
   var contextualClassification: (SyntaxClassification, Bool)? {
     var contextualClassif: (SyntaxClassification, Bool)? = nil
-    var curData = self
+    var curData = Syntax(self)
     repeat {
       guard let parent = curData.parent else { break }
       contextualClassif = SyntaxClassification.classify(parentKind: parent.raw.kind,
@@ -27,10 +29,10 @@ extension SyntaxData {
 extension TokenSyntax {
   /// The `SyntaxClassifiedRange` for the token text, excluding trivia.
   public var tokenClassification: SyntaxClassifiedRange {
-    let contextualClassification = self.data.contextualClassification
-    let relativeOffset = tokenView.leadingTriviaLength.utf8Length
+    let contextualClassification = self.contextualClassification
+    let relativeOffset = leadingTriviaLength.utf8Length
     let absoluteOffset = position.utf8Offset + relativeOffset
-    return TokenKindAndText(kind: tokenView.rawKind, text: tokenView.rawText).classify(
+    return TokenKindAndText(kind: rawTokenKind, text: tokenView.rawText).classify(
       offset: absoluteOffset, contextualClassification: contextualClassification)
   }
 }
@@ -122,7 +124,7 @@ private struct ClassificationVisitor {
       _ = self.visit(Descriptor(
         node: node.raw,
         byteOffset: node.position.utf8Offset,
-        contextualClassification: node.data.contextualClassification))
+        contextualClassification: node.contextualClassification))
     }
   }
 

Sources/SwiftSyntax/gyb_generated/SyntaxClassification.swift renamed to Sources/IDEUtils/gyb_generated/SyntaxClassification.swift

@@ -12,6 +12,8 @@
 //
 //===----------------------------------------------------------------------===//
 
+@_spi(RawSyntax) import SwiftSyntax
+
 public enum SyntaxClassification {
   /// The token should not receive syntax coloring. 
   case none

Sources/SwiftSyntax/CMakeLists.txt

@@ -18,7 +18,6 @@ add_library(SwiftSyntax STATIC
   Syntax.swift
   SyntaxArena.swift
   SyntaxChildren.swift
-  SyntaxClassifier.swift
   SyntaxData.swift
   SyntaxOtherNodes.swift
   SyntaxText.swift
@@ -37,7 +36,6 @@ add_library(SwiftSyntax STATIC
   gyb_generated/Misc.swift
   gyb_generated/SyntaxAnyVisitor.swift
   gyb_generated/SyntaxBaseNodes.swift
-  gyb_generated/SyntaxClassification.swift
   gyb_generated/SyntaxCollections.swift
   gyb_generated/SyntaxEnum.swift
   gyb_generated/SyntaxFactory.swift

Sources/SwiftSyntax/Syntax.swift

@@ -456,56 +456,6 @@ public extension SyntaxProtocol {
     return TokenSequence(_syntaxNode, viewMode: viewMode)
   }
 
-  /// Sequence of `SyntaxClassifiedRange`s for this syntax node.
-  ///
-  /// The provided classified ranges are consecutive and cover the full source
-  /// text of the node. The ranges may also span multiple tokens, if multiple
-  /// consecutive tokens would have the same classification then a single classified
-  /// range is provided for all of them.
-  var classifications: SyntaxClassifications {
-    let fullRange = ByteSourceRange(offset: 0, length: byteSize)
-    return SyntaxClassifications(_syntaxNode, in: fullRange)
-  }
-
-  /// Sequence of `SyntaxClassifiedRange`s contained in this syntax node within
-  /// a relative range.
-  ///
-  /// The provided classified ranges may extend beyond the provided `range`.
-  /// Active classifications (non-`none`) will extend the range to include the
-  /// full classified range (e.g. from the beginning of the comment block), while
-  /// `none` classified ranges will extend to the beginning or end of the token
-  /// that the `range` touches.
-  /// It is guaranteed that no classified range will be provided that doesn't
-  /// intersect the provided `range`.
-  ///
-  /// - Parameters:
-  ///   - in: The relative byte range to pull `SyntaxClassifiedRange`s from.
-  /// - Returns: Sequence of `SyntaxClassifiedRange`s.
-  func classifications(in range: ByteSourceRange) -> SyntaxClassifications {
-    return SyntaxClassifications(_syntaxNode, in: range)
-  }
-
-  /// The `SyntaxClassifiedRange` for a relative byte offset.
-  /// - Parameters:
-  ///   - at: The relative to the node byte offset.
-  /// - Returns: The `SyntaxClassifiedRange` for the offset or nil if the source text
-  ///   at the given offset is unclassified.
-  func classification(at offset: Int) -> SyntaxClassifiedRange? {
-    let classifications = SyntaxClassifications(_syntaxNode, in: ByteSourceRange(offset: offset, length: 1))
-    var iterator = classifications.makeIterator()
-    return iterator.next()
-  }
-
-  /// The `SyntaxClassifiedRange` for an absolute position.
-  /// - Parameters:
-  ///   - at: The absolute position.
-  /// - Returns: The `SyntaxClassifiedRange` for the position or nil if the source text
-  ///   at the given position is unclassified.
-  func classification(at position: AbsolutePosition) -> SyntaxClassifiedRange? {
-    let relativeOffset = position.utf8Offset - self.position.utf8Offset
-    return self.classification(at: relativeOffset)
-  }
-
   /// Returns a value representing the unique identity of the node.
   var id: SyntaxIdentifier {
     return data.nodeId

Sources/SwiftSyntax/SyntaxTreeViewMode.swift

@@ -28,7 +28,8 @@ public enum SyntaxTreeViewMode {
   case all
 
   /// Returns whether this traversal node should visit `node` or ignore it.
-  func shouldTraverse(node: RawSyntax) -> Bool {
+  @_spi(RawSyntax)
+  public func shouldTraverse(node: RawSyntax) -> Bool {
     switch self {
     case .sourceAccurate:
       if let tokenView = node.tokenView {

Sources/lit-test-helper/ClassifiedSyntaxTreePrinter.swift

@@ -10,6 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+import IDEUtils
 import SwiftSyntax
 import Foundation
 

Tests/SwiftParserTest/ClassificationTests.swift renamed to Tests/IDEUtilsTest/ClassificationTests.swift

@@ -11,8 +11,10 @@
 //===----------------------------------------------------------------------===//
 
 import XCTest
+import IDEUtils
 import SwiftSyntax
 import SwiftParser
+import _SwiftSyntaxTestSupport
 
 public class ClassificationTests: XCTestCase {
 

Tests/PerformanceTest/SyntaxClassifierPerformanceTests.swift

@@ -11,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 import XCTest
+import IDEUtils
 import SwiftSyntax
 import SwiftSyntaxParser
 

build-script.py

@@ -16,6 +16,7 @@
 PACKAGE_DIR = os.path.dirname(os.path.realpath(__file__))
 WORKSPACE_DIR = os.path.dirname(PACKAGE_DIR)
 SOURCES_DIR = os.path.join(PACKAGE_DIR, "Sources")
+IDEUTILS_DIR = os.path.join(SOURCES_DIR, "IDEUtils")
 SWIFTSYNTAX_DIR = os.path.join(SOURCES_DIR, "SwiftSyntax")
 SWIFTSYNTAX_DOCUMENTATION_DIR = \
         os.path.join(SWIFTSYNTAX_DIR, "Documentation.docc")
@@ -299,6 +300,7 @@ def generate_syntax_node_template_gyb_files(
 def gyb_dir_mapping(temp_directories: bool) -> Dict[str, str]:
     source_dirs = [
         SYNTAXSUPPORT_DIR,
+        IDEUTILS_DIR,
         SWIFTSYNTAX_DIR,
         os.path.join(SWIFTSYNTAX_DIR, "Raw"),
         SWIFTSYNTAXBUILDER_DIR,