Merge pull request #999 from ahoppen/ahoppen/move-syntax-classifier-to-separate-package

Move SyntaxClassifier to a separate module

Author: ahoppen

Package.swift

@@ -45,6 +45,7 @@ let package = Package(
     .macCatalyst(.v13),
   ],
   products: [
+    .library(name: "IDEUtils", type: .static, targets: ["IDEUtils"]),
     .library(name: "SwiftDiagnostics", type: .static, targets: ["SwiftDiagnostics"]),
     .library(name: "SwiftOperators", type: .static, targets: ["SwiftOperators"]),
     .library(name: "SwiftParser", type: .static, targets: ["SwiftParser"]),
@@ -79,7 +80,6 @@ let package = Package(
         "Raw/RawSyntaxValidation.swift.gyb",
         "SyntaxAnyVisitor.swift.gyb",
         "SyntaxBaseNodes.swift.gyb",
-        "SyntaxClassification.swift.gyb",
         "SyntaxCollections.swift.gyb",
         "SyntaxEnum.swift.gyb",
         "SyntaxFactory.swift.gyb",
@@ -116,6 +116,13 @@ let package = Package(
       name: "_SwiftSyntaxTestSupport",
       dependencies: ["SwiftBasicFormat", "SwiftSyntax", "SwiftSyntaxBuilder"]
     ),
+    .target(
+      name: "IDEUtils",
+      dependencies: ["SwiftSyntax"],
+      exclude: [
+        "SyntaxClassification.swift.gyb",
+      ]
+    ),
     .target(
       name: "SwiftParser",
       dependencies: ["SwiftDiagnostics", "SwiftSyntax"],
@@ -160,13 +167,14 @@ let package = Package(
       ]),
     .executableTarget(
       name: "lit-test-helper",
-      dependencies: ["SwiftSyntax", "SwiftSyntaxParser"]
+      dependencies: ["IDEUtils", "SwiftSyntax", "SwiftSyntaxParser"]
     ),
     .executableTarget(
       name: "swift-parser-cli",
       dependencies: ["SwiftDiagnostics", "SwiftSyntax", "SwiftParser", "SwiftParserDiagnostics", "SwiftOperators", "_SwiftSyntaxMacros",
                      .product(name: "ArgumentParser", package: "swift-argument-parser")]
     ),
+    .testTarget(name: "IDEUtilsTest", dependencies: ["_SwiftSyntaxTestSupport", "SwiftParser", "SwiftSyntax", "IDEUtils"]),
     .testTarget(
       name: "SwiftDiagnosticsTest",
       dependencies: ["_SwiftSyntaxTestSupport", "SwiftDiagnostics", "SwiftParser", "SwiftParserDiagnostics"]
@@ -192,7 +200,7 @@ let package = Package(
     ),
     .testTarget(
       name: "PerformanceTest",
-      dependencies: ["SwiftSyntax", "SwiftSyntaxParser", "SwiftParser"],
+      dependencies: ["IDEUtils", "SwiftSyntax", "SwiftSyntaxParser", "SwiftParser"],
       exclude: ["Inputs"]
     ),
     .testTarget(

Sources/IDEUtils/Syntax+Classifications.swift

@@ -0,0 +1,66 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2022 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import SwiftSyntax
+
+public extension SyntaxProtocol {
+
+  /// Sequence of `SyntaxClassifiedRange`s for this syntax node.
+  ///
+  /// The provided classified ranges are consecutive and cover the full source
+  /// text of the node. The ranges may also span multiple tokens, if multiple
+  /// consecutive tokens would have the same classification then a single classified
+  /// range is provided for all of them.
+  var classifications: SyntaxClassifications {
+    let fullRange = ByteSourceRange(offset: 0, length: byteSize)
+    return SyntaxClassifications(_syntaxNode, in: fullRange)
+  }
+
+  /// Sequence of `SyntaxClassifiedRange`s contained in this syntax node within
+  /// a relative range.
+  ///
+  /// The provided classified ranges may extend beyond the provided `range`.
+  /// Active classifications (non-`none`) will extend the range to include the
+  /// full classified range (e.g. from the beginning of the comment block), while
+  /// `none` classified ranges will extend to the beginning or end of the token
+  /// that the `range` touches.
+  /// It is guaranteed that no classified range will be provided that doesn't
+  /// intersect the provided `range`.
+  ///
+  /// - Parameters:
+  ///   - in: The relative byte range to pull `SyntaxClassifiedRange`s from.
+  /// - Returns: Sequence of `SyntaxClassifiedRange`s.
+  func classifications(in range: ByteSourceRange) -> SyntaxClassifications {
+    return SyntaxClassifications(_syntaxNode, in: range)
+  }
+
+  /// The `SyntaxClassifiedRange` for a relative byte offset.
+  /// - Parameters:
+  ///   - at: The relative to the node byte offset.
+  /// - Returns: The `SyntaxClassifiedRange` for the offset or nil if the source text
+  ///   at the given offset is unclassified.
+  func classification(at offset: Int) -> SyntaxClassifiedRange? {
+    let classifications = SyntaxClassifications(_syntaxNode, in: ByteSourceRange(offset: offset, length: 1))
+    var iterator = classifications.makeIterator()
+    return iterator.next()
+  }
+
+  /// The `SyntaxClassifiedRange` for an absolute position.
+  /// - Parameters:
+  ///   - at: The absolute position.
+  /// - Returns: The `SyntaxClassifiedRange` for the position or nil if the source text
+  ///   at the given position is unclassified.
+  func classification(at position: AbsolutePosition) -> SyntaxClassifiedRange? {
+    let relativeOffset = position.utf8Offset - self.position.utf8Offset
+    return self.classification(at: relativeOffset)
+  }
+}

Sources/SwiftSyntax/SyntaxClassification.swift.gyb renamed to Sources/IDEUtils/SyntaxClassification.swift.gyb

@@ -35,6 +35,8 @@
 //
 //===----------------------------------------------------------------------===//
 
+@_spi(RawSyntax) import SwiftSyntax
+
 public enum SyntaxClassification {
 % for classification in SYNTAX_CLASSIFICATIONS:
 %   for line in dedented_lines(classification.description):

Sources/SwiftSyntax/SyntaxClassifier.swift renamed to Sources/IDEUtils/SyntaxClassifier.swift

@@ -10,10 +10,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-extension SyntaxData {
+@_spi(RawSyntax) import SwiftSyntax
+
+fileprivate extension SyntaxProtocol {
   var contextualClassification: (SyntaxClassification, Bool)? {
     var contextualClassif: (SyntaxClassification, Bool)? = nil
-    var curData = self
+    var curData = Syntax(self)
     repeat {
       guard let parent = curData.parent else { break }
       contextualClassif = SyntaxClassification.classify(parentKind: parent.raw.kind,
@@ -27,10 +29,10 @@ extension SyntaxData {
 extension TokenSyntax {
   /// The `SyntaxClassifiedRange` for the token text, excluding trivia.
   public var tokenClassification: SyntaxClassifiedRange {
-    let contextualClassification = self.data.contextualClassification
-    let relativeOffset = tokenView.leadingTriviaLength.utf8Length
+    let contextualClassification = self.contextualClassification
+    let relativeOffset = leadingTriviaLength.utf8Length
     let absoluteOffset = position.utf8Offset + relativeOffset
-    return TokenKindAndText(kind: tokenView.rawKind, text: tokenView.rawText).classify(
+    return TokenKindAndText(kind: rawTokenKind, text: tokenView.rawText).classify(
       offset: absoluteOffset, contextualClassification: contextualClassification)
   }
 }
@@ -122,7 +124,7 @@ private struct ClassificationVisitor {
       _ = self.visit(Descriptor(
         node: node.raw,
         byteOffset: node.position.utf8Offset,
-        contextualClassification: node.data.contextualClassification))
+        contextualClassification: node.contextualClassification))
     }
   }
 

Sources/SwiftSyntax/gyb_generated/SyntaxClassification.swift renamed to Sources/IDEUtils/gyb_generated/SyntaxClassification.swift

@@ -12,6 +12,8 @@
 //
 //===----------------------------------------------------------------------===//
 
+@_spi(RawSyntax) import SwiftSyntax
+
 public enum SyntaxClassification {
   /// The token should not receive syntax coloring. 
   case none

Sources/SwiftSyntax/CMakeLists.txt

@@ -18,7 +18,6 @@ add_library(SwiftSyntax STATIC
   Syntax.swift
   SyntaxArena.swift
   SyntaxChildren.swift
-  SyntaxClassifier.swift
   SyntaxData.swift
   SyntaxOtherNodes.swift
   SyntaxText.swift
@@ -37,7 +36,6 @@ add_library(SwiftSyntax STATIC
   gyb_generated/Misc.swift
   gyb_generated/SyntaxAnyVisitor.swift
   gyb_generated/SyntaxBaseNodes.swift
-  gyb_generated/SyntaxClassification.swift
   gyb_generated/SyntaxCollections.swift
   gyb_generated/SyntaxEnum.swift
   gyb_generated/SyntaxFactory.swift

Sources/SwiftSyntax/Raw/RawSyntax.swift

@@ -10,7 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-typealias RawSyntaxBuffer = UnsafeBufferPointer<RawSyntax?>
+@_spi(RawSyntax) public typealias RawSyntaxBuffer = UnsafeBufferPointer<RawSyntax?>
 typealias RawTriviaPieceBuffer = UnsafeBufferPointer<RawTriviaPiece>
 
 fileprivate extension SyntaxKind {
@@ -148,7 +148,8 @@ extension RawSyntax {
   }
 
   /// Whether or not this node is a token one.
-  var isToken: Bool {
+  @_spi(RawSyntax)
+  public var isToken: Bool {
     kind == .token
   }
 
@@ -171,7 +172,7 @@ extension RawSyntax {
       }
       return recursiveFlags
     case .layout(let layoutView):
-      return layoutView.layoutData.recursiveFlags
+      return layoutView.recursiveFlags
     }
   }
 

Sources/SwiftSyntax/Raw/RawSyntaxLayoutView.swift

@@ -13,7 +13,8 @@
 extension RawSyntax {
   /// A view into the `RawSyntax` that exposes functionality that's specific to layout nodes.
   /// The token's payload must be a layout, otherwise this traps.
-  var layoutView: RawSyntaxLayoutView? {
+  @_spi(RawSyntax)
+  public var layoutView: RawSyntaxLayoutView? {
     switch raw.payload {
     case .parsedToken, .materializedToken:
       return nil
@@ -24,7 +25,8 @@ extension RawSyntax {
 }
 
 /// A view into `RawSyntax` that exposes functionality that only applies to layout nodes.
-struct RawSyntaxLayoutView {
+@_spi(RawSyntax)
+public struct RawSyntaxLayoutView {
   private let raw: RawSyntax
 
   fileprivate init(raw: RawSyntax) {
@@ -37,7 +39,7 @@ struct RawSyntaxLayoutView {
     }
   }
 
-  var layoutData: RawSyntaxData.Layout {
+  private var layoutData: RawSyntaxData.Layout {
     switch raw.rawData.payload {
     case .parsedToken(_),
         .materializedToken(_):
@@ -47,8 +49,13 @@ struct RawSyntaxLayoutView {
     }
   }
 
+  var recursiveFlags: RecursiveRawSyntaxFlags {
+    return layoutData.recursiveFlags
+  }
+
   /// Creates a new node of the same kind but with children replaced by `elements`.
-  func replacingLayout<C: Collection>(
+  @_spi(RawSyntax)
+  public func replacingLayout<C: Collection>(
     with elements: C,
     arena: SyntaxArena
   ) -> RawSyntax where C.Element == RawSyntax? {
@@ -60,7 +67,8 @@ struct RawSyntaxLayoutView {
     }
   }
 
-  func insertingChild(
+  @_spi(RawSyntax)
+  public func insertingChild(
     _ newChild: RawSyntax?,
     at index: Int,
     arena: SyntaxArena
@@ -78,7 +86,8 @@ struct RawSyntaxLayoutView {
     }
   }
 
-  func removingChild(
+  @_spi(RawSyntax)
+  public func removingChild(
     at index: Int,
     arena: SyntaxArena
   ) -> RawSyntax {
@@ -101,11 +110,13 @@ struct RawSyntaxLayoutView {
     }
   }
 
-  func appending(_ newChild: RawSyntax?, arena: SyntaxArena) -> RawSyntax {
+  @_spi(RawSyntax)
+  public func appending(_ newChild: RawSyntax?, arena: SyntaxArena) -> RawSyntax {
     insertingChild(newChild, at: children.count, arena: arena)
   }
 
-  func replacingChildSubrange<C: Collection>(
+  @_spi(RawSyntax)
+  public func replacingChildSubrange<C: Collection>(
     _ range: Range<Int>,
     with elements: C,
     arena: SyntaxArena
@@ -130,7 +141,8 @@ struct RawSyntaxLayoutView {
     }
   }
 
-  func replacingChild(
+  @_spi(RawSyntax)
+  public func replacingChild(
     at index: Int,
     with newChild: RawSyntax?,
     arena: SyntaxArena
@@ -144,17 +156,20 @@ struct RawSyntaxLayoutView {
     }
   }
 
-  func formLayoutArray() -> [RawSyntax?] {
+  @_spi(RawSyntax)
+  public func formLayoutArray() -> [RawSyntax?] {
     Array(children)
   }
 
   /// Child nodes.
-  var children: RawSyntaxBuffer {
+  @_spi(RawSyntax)
+  public var children: RawSyntaxBuffer {
     layoutData.layout
   }
 
   /// The number of children, `present` or `missing`, in this node.
-  var numberOfChildren: Int {
+  @_spi(RawSyntax)
+  public var numberOfChildren: Int {
     return children.count
   }
 }

Sources/SwiftSyntax/Raw/RawSyntaxNodeProtocol.swift

@@ -68,7 +68,8 @@ extension RawSyntax: RawSyntaxNodeProtocol {
 public struct RawTokenSyntax: RawSyntaxToSyntax, RawSyntaxNodeProtocol {
   public typealias SyntaxType = TokenSyntax
 
-  var tokenView: RawSyntaxTokenView {
+  @_spi(RawSyntax)
+  public var tokenView: RawSyntaxTokenView {
     return raw.tokenView!
   }
 

Sources/SwiftSyntax/Raw/RawSyntaxNodes.swift.gyb

@@ -31,7 +31,8 @@ public protocol Raw${node.name}NodeProtocol: Raw${node.base_type}NodeProtocol {}
 public struct Raw${node.name}: Raw${node.name if node.is_base() else node.base_type}NodeProtocol, RawSyntaxToSyntax {
   public typealias SyntaxType = ${node.name}
 
-  var layoutView: RawSyntaxLayoutView {
+  @_spi(RawSyntax)
+  public var layoutView: RawSyntaxLayoutView {
     return raw.layoutView!
   }