Merge pull request #1012 from krzyzanowskim/marcin/semantictokentypes

ahoppen · web-flow · commit e0fe9d6aa165 · 2024-01-08T18:14:28.000-08:00
Refactor SemanticTokenTypes, SemanticTokenModifiers
diff --git a/Sources/LanguageServerProtocol/CMakeLists.txt b/Sources/LanguageServerProtocol/CMakeLists.txt
@@ -116,7 +116,9 @@ add_library(LanguageServerProtocol STATIC
   SupportTypes/PositionEncoding.swift
   SupportTypes/ProgressToken.swift
   SupportTypes/RegistrationOptions.swift
+  SupportTypes/SemanticTokenModifiers.swift
   SupportTypes/SemanticTokens.swift
+  SupportTypes/SemanticTokenTypes.swift
   SupportTypes/ServerCapabilities.swift
   SupportTypes/SKCompletionOptions.swift
   SupportTypes/StringOrMarkupContent.swift
diff --git a/Sources/LanguageServerProtocol/SupportTypes/SemanticTokenModifiers.swift b/Sources/LanguageServerProtocol/SupportTypes/SemanticTokenModifiers.swift
@@ -0,0 +1,67 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import Foundation
+
+/// Additional metadata about a token.
+///
+/// Similar to `SemanticTokenTypes`, the bit indices should
+/// be numbered starting at 0.
+public struct SemanticTokenModifiers: OptionSet, Hashable {
+  public let rawValue: UInt32
+
+  public init(rawValue: UInt32) {
+    self.rawValue = rawValue
+  }
+
+  public static let declaration = Self(rawValue: 1 << 0)
+  public static let definition = Self(rawValue: 1 << 1)
+  public static let readonly = Self(rawValue: 1 << 2)
+  public static let `static` = Self(rawValue: 1 << 3)
+  public static let deprecated = Self(rawValue: 1 << 4)
+  public static let abstract = Self(rawValue: 1 << 5)
+  public static let async = Self(rawValue: 1 << 6)
+  public static let modification = Self(rawValue: 1 << 7)
+  public static let documentation = Self(rawValue: 1 << 8)
+  public static let defaultLibrary = Self(rawValue: 1 << 9)
+
+  public var name: String? {
+    switch self {
+    case .declaration: return "declaration"
+    case .definition: return "definition"
+    case .readonly: return "readonly"
+    case .static: return "static"
+    case .deprecated: return "deprecated"
+    case .abstract: return "abstract"
+    case .async: return "async"
+    case .modification: return "modification"
+    case .documentation: return "documentation"
+    case .defaultLibrary: return "defaultLibrary"
+    default: return nil
+    }
+  }
+
+  /// All available modifiers, in ascending order of the bit index
+  /// they are represented with (starting at the rightmost bit).
+  public static let predefined: [Self] = [
+    .declaration,
+    .definition,
+    .readonly,
+    .static,
+    .deprecated,
+    .abstract,
+    .async,
+    .modification,
+    .documentation,
+    .defaultLibrary,
+  ]
+}
diff --git a/Sources/LanguageServerProtocol/SupportTypes/SemanticTokenTypes.swift b/Sources/LanguageServerProtocol/SupportTypes/SemanticTokenTypes.swift
@@ -0,0 +1,77 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import Foundation
+
+/// The predefined token type values
+///
+/// The protocol defines a set of token types and modifiers but clients are
+/// allowed to extend these and announce the values they support in the
+/// corresponding client capability.
+public struct SemanticTokenTypes: Hashable {
+  public let name: String
+  public init(_ name: String) {
+    self.name = name
+  }
+
+  public static let namespace = Self("namespace")
+  /// Represents a generic type. Acts as a fallback for types which
+  /// can't be mapped to a specific type like class or enum.
+  public static let type = Self("type")
+  public static let `class` = Self("class")
+  public static let `enum` = Self("enum")
+  public static let interface = Self("interface")
+  public static let `struct` = Self("struct")
+  public static let typeParameter = Self("typeParameter")
+  public static let parameter = Self("parameter")
+  public static let variable = Self("variable")
+  public static let property = Self("property")
+  public static let enumMember = Self("enumMember")
+  public static let event = Self("event")
+  public static let function = Self("function")
+  public static let method = Self("method")
+  public static let macro = Self("macro")
+  public static let keyword = Self("keyword")
+  public static let modifier = Self("modifier")
+  public static let comment = Self("comment")
+  public static let string = Self("string")
+  public static let number = Self("number")
+  public static let regexp = Self("regexp")
+  public static let `operator` = Self("operator")
+  /// since 3.17.0
+  public static let decorator = Self("decorator")
+
+  public static var predefined: [Self] = [
+    .namespace,
+    .type,
+    .class,
+    .enum,
+    .interface,
+    .struct,
+    .typeParameter,
+    .parameter,
+    .variable,
+    .property,
+    .enumMember,
+    .event,
+    .function,
+    .method,
+    .macro,
+    .keyword,
+    .modifier,
+    .comment,
+    .string,
+    .number,
+    .regexp,
+    .operator,
+  ]
+}
diff --git a/Sources/SKTestSupport/Array+SyntaxHighlightingToken.swift b/Sources/SKTestSupport/Array+SyntaxHighlightingToken.swift
@@ -37,8 +37,8 @@ extension Array where Element == SyntaxHighlightingToken {
         current.utf16index = charDelta
       }
 
-      guard let kind = SyntaxHighlightingToken.Kind(rawValue: rawKind) else { continue }
-      let modifiers = SyntaxHighlightingToken.Modifiers(rawValue: rawModifiers)
+      let kind = SemanticTokenTypes.all[Int(rawKind)]
+      let modifiers = SemanticTokenModifiers(rawValue: rawModifiers)
 
       append(
         SyntaxHighlightingToken(
diff --git a/Sources/SourceKitLSP/Swift/SemanticTokens.swift b/Sources/SourceKitLSP/Swift/SemanticTokens.swift
@@ -129,7 +129,7 @@ extension SyntaxClassifiedRange {
 }
 
 extension SyntaxClassification {
-  fileprivate var highlightingKindAndModifiers: (SyntaxHighlightingToken.Kind, SyntaxHighlightingToken.Modifiers)? {
+  fileprivate var highlightingKindAndModifiers: (SemanticTokenTypes, SemanticTokenModifiers)? {
     switch self {
     case .none:
       return nil
diff --git a/Sources/SourceKitLSP/Swift/SwiftLanguageServer.swift b/Sources/SourceKitLSP/Swift/SwiftLanguageServer.swift
@@ -231,8 +231,8 @@ extension SwiftLanguageServer {
         ),
         semanticTokensProvider: SemanticTokensOptions(
           legend: SemanticTokensLegend(
-            tokenTypes: SyntaxHighlightingToken.Kind.allCases.map(\.lspName),
-            tokenModifiers: SyntaxHighlightingToken.Modifiers.allModifiers.map { $0.lspName! }
+            tokenTypes: SemanticTokenTypes.all.map(\.name),
+            tokenModifiers: SemanticTokenModifiers.all.compactMap(\.name)
           ),
           range: .bool(true),
           full: .bool(true)
diff --git a/Sources/SourceKitLSP/Swift/SyntaxHighlightingToken.swift b/Sources/SourceKitLSP/Swift/SyntaxHighlightingToken.swift
@@ -23,9 +23,9 @@ public struct SyntaxHighlightingToken: Hashable {
     }
   }
   /// The token type.
-  public var kind: Kind
+  public var kind: SemanticTokenTypes
   /// Additional metadata about the token.
-  public var modifiers: Modifiers
+  public var modifiers: SemanticTokenModifiers
 
   /// The (inclusive) start position of the token.
   public var start: Position { range.lowerBound }
@@ -34,159 +34,18 @@ public struct SyntaxHighlightingToken: Hashable {
   /// The length of the token in UTF-16 code units.
   public var utf16length: Int { end.utf16index - start.utf16index }
 
-  public init(range: Range<Position>, kind: Kind, modifiers: Modifiers = []) {
+  public init(range: Range<Position>, kind: SemanticTokenTypes, modifiers: SemanticTokenModifiers = []) {
     assert(range.lowerBound.line == range.upperBound.line)
 
     self.range = range
     self.kind = kind
     self.modifiers = modifiers
   }
 
-  public init(start: Position, utf16length: Int, kind: Kind, modifiers: Modifiers = []) {
+  public init(start: Position, utf16length: Int, kind: SemanticTokenTypes, modifiers: SemanticTokenModifiers = []) {
     let range = start..<Position(line: start.line, utf16index: start.utf16index + utf16length)
     self.init(range: range, kind: kind, modifiers: modifiers)
   }
-
-  /// The token type.
-  ///
-  /// Represented using an int to make the conversion to
-  /// LSP tokens efficient. The order of this enum does not have to be
-  /// stable, since we provide a `SemanticTokensLegend` during initialization.
-  /// It is, however, important that the values are numbered from 0 due to
-  /// the way the kinds are encoded in LSP.
-  /// Also note that we intentionally use an enum here instead of e.g. a
-  /// `RawRepresentable` struct, since we want to have a conversion to
-  /// strings for known kinds and since these kinds are only provided by the
-  /// server, i.e. there is no need to handle cases where unknown kinds
-  /// have to be decoded.
-  public enum Kind: UInt32, CaseIterable, Hashable {
-    case namespace = 0
-    case type
-    case actor
-    case `class`
-    case `enum`
-    case interface
-    case `struct`
-    case typeParameter
-    case parameter
-    case variable
-    case property
-    case enumMember
-    case event
-    case function
-    case method
-    case macro
-    case keyword
-    case modifier
-    case comment
-    case string
-    case number
-    case regexp
-    case `operator`
-    case decorator
-    /// **(LSP Extension)**
-    case identifier
-
-    /// The name of the token type used by LSP.
-    var lspName: String {
-      switch self {
-      case .namespace: return "namespace"
-      case .type: return "type"
-      case .actor: return "class"  // LSP doesn’t know about actors. Display actors as classes.
-      case .class: return "class"
-      case .enum: return "enum"
-      case .interface: return "interface"
-      case .struct: return "struct"
-      case .typeParameter: return "typeParameter"
-      case .parameter: return "parameter"
-      case .variable: return "variable"
-      case .property: return "property"
-      case .enumMember: return "enumMember"
-      case .event: return "event"
-      case .function: return "function"
-      case .method: return "method"
-      case .macro: return "macro"
-      case .keyword: return "keyword"
-      case .modifier: return "modifier"
-      case .comment: return "comment"
-      case .string: return "string"
-      case .number: return "number"
-      case .regexp: return "regexp"
-      case .operator: return "operator"
-      case .decorator: return "decorator"
-      case .identifier: return "identifier"
-      }
-    }
-
-    /// **Public for testing**
-    public var _lspName: String {
-      lspName
-    }
-  }
-
-  /// Additional metadata about a token.
-  ///
-  /// Similar to `Kind`, the raw values do not actually have
-  /// to be stable, do note however that the bit indices should
-  /// be numbered starting at 0 and that the ordering should
-  /// correspond to `allModifiers`.
-  public struct Modifiers: OptionSet, Hashable {
-    public static let declaration = Self(rawValue: 1 << 0)
-    public static let definition = Self(rawValue: 1 << 1)
-    public static let readonly = Self(rawValue: 1 << 2)
-    public static let `static` = Self(rawValue: 1 << 3)
-    public static let deprecated = Self(rawValue: 1 << 4)
-    public static let abstract = Self(rawValue: 1 << 5)
-    public static let async = Self(rawValue: 1 << 6)
-    public static let modification = Self(rawValue: 1 << 7)
-    public static let documentation = Self(rawValue: 1 << 8)
-    public static let defaultLibrary = Self(rawValue: 1 << 9)
-
-    /// All available modifiers, in ascending order of the bit index
-    /// they are represented with (starting at the rightmost bit).
-    public static let allModifiers: [Self] = [
-      .declaration,
-      .definition,
-      .readonly,
-      .static,
-      .deprecated,
-      .abstract,
-      .async,
-      .modification,
-      .documentation,
-      .defaultLibrary,
-    ]
-
-    public let rawValue: UInt32
-
-    /// The name of the modifier used by LSP, if this
-    /// is a single modifier. Note that every modifier
-    /// in `allModifiers` must have an associated `lspName`.
-    var lspName: String? {
-      switch self {
-      case .declaration: return "declaration"
-      case .definition: return "definition"
-      case .readonly: return "readonly"
-      case .static: return "static"
-      case .deprecated: return "deprecated"
-      case .abstract: return "abstract"
-      case .async: return "async"
-      case .modification: return "modification"
-      case .documentation: return "documentation"
-      case .defaultLibrary: return "defaultLibrary"
-      default: return nil
-      }
-    }
-
-    /// **Public for testing**
-    public var _lspName: String? {
-      lspName
-    }
-
-    public init(rawValue: UInt32) {
-      self.rawValue = rawValue
-    }
-  }
 }
 
 extension Array where Element == SyntaxHighlightingToken {
@@ -214,7 +73,7 @@ extension Array where Element == SyntaxHighlightingToken {
         UInt32(lineDelta),
         UInt32(charDelta),
         UInt32(token.utf16length),
-        token.kind.rawValue,
+        token.kind.tokenType,
         token.modifiers.rawValue,
       ]
     }
@@ -230,3 +89,24 @@ extension Array where Element == SyntaxHighlightingToken {
     return filter { !otherRanges.contains($0.range) } + other
   }
 }
+
+extension SemanticTokenTypes {
+  /// **(LSP Extension)**
+  public static let identifier = Self("identifier")
+
+  // LSP doesn’t know about actors. Display actors as classes.
+  public static let actor = Self("class")
+
+  /// All tokens supported by sourcekit-lsp
+  public static let all: [Self] = predefined + [.identifier, .actor]
+
+  /// Token types are looked up by index
+  public var tokenType: UInt32 {
+    UInt32(Self.all.firstIndex(of: self)!)
+  }
+}
+
+extension SemanticTokenModifiers {
+  /// All tokens supported by sourcekit-lsp
+  public static let all: [Self] = predefined
+}
diff --git a/Sources/SourceKitLSP/Swift/SyntaxHighlightingTokenParser.swift b/Sources/SourceKitLSP/Swift/SyntaxHighlightingTokenParser.swift
diff --git a/Tests/SourceKitLSPTests/SemanticTokensTests.swift b/Tests/SourceKitLSPTests/SemanticTokensTests.swift

Original file line number	Diff line number	Diff line change
`@@ -37,8 +37,8 @@ extension Array where Element == SyntaxHighlightingToken {`
`37`	`37`	`current.utf16index = charDelta`
`38`	`38`	`}`
`39`	`39`
`40`		`- guard let kind = SyntaxHighlightingToken.Kind(rawValue: rawKind) else { continue }`
`41`		`- let modifiers = SyntaxHighlightingToken.Modifiers(rawValue: rawModifiers)`
	`40`	`+ let kind = SemanticTokenTypes.all[Int(rawKind)]`
	`41`	`+ let modifiers = SemanticTokenModifiers(rawValue: rawModifiers)`
`42`	`42`
`43`	`43`	`append(`
`44`	`44`	`SyntaxHighlightingToken(`
Original file line number	Diff line number	Diff line change
`@@ -129,7 +129,7 @@ extension SyntaxClassifiedRange {`
`129`	`129`	`}`
`130`	`130`
`131`	`131`	`extension SyntaxClassification {`
`132`		`- fileprivate var highlightingKindAndModifiers: (SyntaxHighlightingToken.Kind, SyntaxHighlightingToken.Modifiers)? {`
	`132`	`+ fileprivate var highlightingKindAndModifiers: (SemanticTokenTypes, SemanticTokenModifiers)? {`
`133`	`133`	`switch self {`
`134`	`134`	`case .none:`
`135`	`135`	`return nil`