Add support for clangd's semantic highlighting (#388)

- Add LSP types for semantic highlighting
- Limited to clients which support dynamic registration for semantic highlighting
- Requires clangd 11 or later

Author: DavidGoldman

Sources/LanguageServerProtocol/CMakeLists.txt

@@ -31,6 +31,9 @@ add_library(LanguageServerProtocol
   Requests/DefinitionRequest.swift
   Requests/DocumentColorRequest.swift
   Requests/DocumentHighlightRequest.swift
+  Requests/DocumentSemanticTokensDeltaRequest.swift
+  Requests/DocumentSemanticTokensRangeRequest.swift
+  Requests/DocumentSemanticTokensRequest.swift
   Requests/DocumentSymbolRequest.swift
   Requests/ExecuteCommandRequest.swift
   Requests/FoldingRangeRequest.swift
@@ -46,6 +49,7 @@ add_library(LanguageServerProtocol
   Requests/SymbolInfoRequest.swift
   Requests/UnregisterCapabilityRequest.swift
   Requests/WorkspaceFoldersRequest.swift
+  Requests/WorkspaceSemanticTokensRefreshRequest.swift
   Requests/WorkspaceSymbolsRequest.swift
 
   SupportTypes/CallHierarchyItem.swift
@@ -67,6 +71,7 @@ add_library(LanguageServerProtocol
   SupportTypes/MarkupContent.swift
   SupportTypes/Position.swift
   SupportTypes/RegistrationOptions.swift
+  SupportTypes/SemanticTokens.swift
   SupportTypes/ServerCapabilities.swift
   SupportTypes/SKCompletionOptions.swift
   SupportTypes/SymbolKind.swift

Sources/LanguageServerProtocol/Messages.swift

@@ -21,6 +21,7 @@ public let builtinRequests: [_RequestType.Type] = [
   WorkspaceFoldersRequest.self,
   CompletionRequest.self,
   HoverRequest.self,
+  WorkspaceSemanticTokensRefreshRequest.self,
   WorkspaceSymbolsRequest.self,
   CallHierarchyIncomingCallsRequest.self,
   CallHierarchyOutgoingCallsRequest.self,
@@ -31,6 +32,9 @@ public let builtinRequests: [_RequestType.Type] = [
   DocumentHighlightRequest.self,
   DocumentFormattingRequest.self,
   DocumentRangeFormattingRequest.self,
+  DocumentSemanticTokensDeltaRequest.self,
+  DocumentSemanticTokensRangeRequest.self,
+  DocumentSemanticTokensRequest.self,
   DocumentOnTypeFormattingRequest.self,
   FoldingRangeRequest.self,
   DocumentSymbolRequest.self,

Sources/LanguageServerProtocol/Requests/DocumentSemanticTokensDeltaRequest.swift

@@ -0,0 +1,87 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2021 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+public struct DocumentSemanticTokensDeltaRequest: TextDocumentRequest, Hashable {
+  public static let method: String = "textDocument/semanticTokens/full/delta"
+  public typealias Response = DocumentSemanticTokensDeltaResponse?
+
+  /// The document to fetch semantic tokens for.
+  public var textDocument: TextDocumentIdentifier
+
+  /// The result identifier of a previous response, which acts as the diff base for the delta.
+  /// This can either point to a full response or a delta response, depending on what was
+  /// last received by the client.
+  public var previousResultId: String
+
+  public init(textDocument: TextDocumentIdentifier, previousResultId: String) {
+    self.textDocument = textDocument
+    self.previousResultId = previousResultId
+  }
+}
+
+public enum DocumentSemanticTokensDeltaResponse: ResponseType, Codable, Equatable {
+  case tokens(DocumentSemanticTokensResponse)
+  case delta(SemanticTokensDelta)
+
+  public init(from decoder: Decoder) throws {
+    let container = try decoder.singleValueContainer()
+    if let tokens = try? container.decode(DocumentSemanticTokensResponse.self) {
+      self = .tokens(tokens)
+    } else if let delta = try? container.decode(SemanticTokensDelta.self) {
+      self = .delta(delta)
+    } else {
+      let error = "DocumentSemanticTokensDeltaResponse has neither SemanticTokens or SemanticTokensDelta."
+      throw DecodingError.dataCorruptedError(in: container, debugDescription: error)
+    }
+  }
+
+  public func encode(to encoder: Encoder) throws {
+    var container = encoder.singleValueContainer()
+    switch self {
+    case .tokens(let tokens):
+      try container.encode(tokens)
+    case .delta(let delta):
+      try container.encode(delta)
+    }
+  }
+}
+
+public struct SemanticTokensDelta: Codable, Hashable {
+  /// An optional result identifier which enables supporting clients to request semantic token deltas
+  /// subsequent requests.
+  public var resultId: String?
+
+  /// The edits to transform a previous result into a new result.
+  public var edits: [SemanticTokensEdit]
+
+  public init(resultId: String? = nil, edits: [SemanticTokensEdit]) {
+    self.resultId = resultId
+    self.edits = edits
+  }
+}
+
+public struct SemanticTokensEdit: Codable, Hashable {
+  /// Start offset of the edit.
+  public var start: Int
+
+  /// The number of elements to remove.
+  public var deleteCount: Int
+
+  /// The elements to insert.
+  public var data: [UInt32]?
+
+  public init(start: Int, deleteCount: Int, data: [UInt32]? = nil) {
+    self.start = start
+    self.deleteCount = deleteCount
+    self.data = data
+  }
+}

Sources/LanguageServerProtocol/Requests/DocumentSemanticTokensRangeRequest.swift

@@ -0,0 +1,28 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2021 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+public struct DocumentSemanticTokensRangeRequest: TextDocumentRequest, Hashable {
+  public static let method: String = "textDocument/semanticTokens/range"
+  public typealias Response = DocumentSemanticTokensResponse?
+
+  /// The document to fetch semantic tokens for.
+  public var textDocument: TextDocumentIdentifier
+
+  /// The range to fetch semantic tokens for.
+  @CustomCodable<PositionRange>
+  public var range: Range<Position>
+
+  public init(textDocument: TextDocumentIdentifier, range: Range<Position>) {
+    self.textDocument = textDocument
+    self.range = range
+  }
+}

Sources/LanguageServerProtocol/Requests/DocumentSemanticTokensRequest.swift

@@ -0,0 +1,37 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2021 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+public struct DocumentSemanticTokensRequest: TextDocumentRequest, Hashable {
+  public static let method: String = "textDocument/semanticTokens/full"
+  public typealias Response = DocumentSemanticTokensResponse?
+
+  /// The document to fetch semantic tokens for.
+  public var textDocument: TextDocumentIdentifier
+
+  public init(textDocument: TextDocumentIdentifier) {
+    self.textDocument = textDocument
+  }
+}
+
+public struct DocumentSemanticTokensResponse: ResponseType, Hashable {
+  /// An optional result identifier which enables supporting clients to request semantic token deltas
+  /// subsequent requests.
+  public var resultId: String?
+
+  /// Raw tokens data.
+  public var data: [UInt32]
+
+  public init(resultId: String? = nil, data: [UInt32]) {
+    self.resultId = resultId
+    self.data = data
+  }
+}

Sources/LanguageServerProtocol/Requests/WorkspaceSemanticTokensRefreshRequest.swift

@@ -0,0 +1,19 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2021 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+/// Sent from the server to the client. Servers can use this to ask clients to
+/// refresh semantic tokens for editors for which this server provides semantic
+/// tokens, useful in cases of project wide configuration changes.
+public struct WorkspaceSemanticTokensRefreshRequest: RequestType, Hashable {
+  public static let method: String = "workspace/semanticTokens/refresh"
+  public typealias Response = VoidResponse
+}

Sources/LanguageServerProtocol/SupportTypes/ClientCapabilities.swift

@@ -80,6 +80,23 @@ public struct WorkspaceClientCapabilities: Hashable, Codable {
     }
   }
 
+  /// Capabilities specific to the `workspace/semanticTokens/refresh` request.
+  public struct SemanticTokensWorkspace: Hashable, Codable {
+
+    /// Whether the client implementation supports a refresh request sent from
+    /// the server to the client.
+    ///
+    /// Note that this event is global and will force the client to refresh all
+    /// semantic tokens currently shown. It should be used with absolute care
+    /// and is useful for situation where a server, for example, detects a project
+    /// wide change that requires such a calculation.
+    public var refreshSupport: Bool?
+
+    public init(refreshSupport: Bool? = nil) {
+      self.refreshSupport = refreshSupport
+    }
+  }
+
   // MARK: Properties
 
   /// Whether the client can apply text edits via the `workspace/applyEdit` request.
@@ -103,7 +120,19 @@ public struct WorkspaceClientCapabilities: Hashable, Codable {
   /// Whether the client supports the `workspace/configuration` request.
   public var configuration: Bool? = nil
 
-  public init(applyEdit: Bool? = nil, workspaceEdit: WorkspaceEdit? = nil, didChangeConfiguration: DynamicRegistrationCapability? = nil, didChangeWatchedFiles: DynamicRegistrationCapability? = nil, symbol: Symbol? = nil, executeCommand: DynamicRegistrationCapability? = nil, workspaceFolders: Bool? = nil, configuration: Bool? = nil) {
+  public var semanticTokens: SemanticTokensWorkspace? = nil
+
+  public init(
+    applyEdit: Bool? = nil,
+    workspaceEdit: WorkspaceEdit? = nil,
+    didChangeConfiguration: DynamicRegistrationCapability? = nil,
+    didChangeWatchedFiles: DynamicRegistrationCapability? = nil,
+    symbol: Symbol? = nil,
+    executeCommand: DynamicRegistrationCapability? = nil,
+    workspaceFolders: Bool? = nil,
+    configuration: Bool? = nil,
+    semanticTokens: SemanticTokensWorkspace? = nil
+  ) {
     self.applyEdit = applyEdit
     self.workspaceEdit = workspaceEdit
     self.didChangeConfiguration = didChangeConfiguration
@@ -112,6 +141,7 @@ public struct WorkspaceClientCapabilities: Hashable, Codable {
     self.executeCommand = executeCommand
     self.workspaceFolders = workspaceFolders
     self.configuration = configuration
+    self.semanticTokens = semanticTokens
   }
 }
 
@@ -394,6 +424,81 @@ public struct TextDocumentClientCapabilities: Hashable, Codable {
     }
   }
 
+  public struct SemanticTokensRangeClientCapabilities: Equatable, Hashable, Codable {
+    // Empty in the LSP 3.16 spec.
+    public init() {}
+  }
+
+  public struct SemanticTokensFullClientCapabilities: Equatable, Hashable, Codable {
+    /// The client will also send the `textDocument/semanticTokens/full/delta`
+    /// request if the server provides a corresponding handler.
+    public var delta: Bool?
+
+    public init(delta: Bool? = nil) {
+      self.delta = delta
+    }
+  }
+
+  public struct SemanticTokensRequestsClientCapabilities: Equatable, Hashable, Codable {
+    /// The client will send the `textDocument/semanticTokens/range` request
+    /// if the server provides a corresponding handler.
+    public var range: ValueOrBool<SemanticTokensRangeClientCapabilities>?
+
+    /// The client will send the `textDocument/semanticTokens/full` request
+    /// if the server provides a corresponding handler.
+    public var full: ValueOrBool<SemanticTokensFullClientCapabilities>?
+
+    public init(
+      range: ValueOrBool<SemanticTokensRangeClientCapabilities>?,
+      full: ValueOrBool<SemanticTokensFullClientCapabilities>?
+    ) {
+      self.range = range
+      self.full = full
+    }
+  }
+
+  /// Capabilities specific to `textDocument/semanticTokens`.
+  public struct SemanticTokens: Equatable, Hashable, Codable {
+
+    /// Whether the client supports dynamic registration of this request.
+    public var dynamicRegistration: Bool? = nil
+
+    public var requests: SemanticTokensRequestsClientCapabilities
+
+    /// The token types that the client supports.
+    public var tokenTypes: [String]
+
+    /// The token modifiers that the client supports.
+    public var tokenModifiers: [String]
+
+    /// The formats the clients supports.
+    public var formats: [TokenFormat]
+
+    /// Whether the client supports tokens that can overlap each other.
+    public var overlappingTokenSupport: Bool? = nil
+
+    /// Whether the client supports tokens that can span multiple lines.
+    public var multilineTokenSupport: Bool? = nil
+
+    public init(
+      dynamicRegistration: Bool? = nil,
+      requests: SemanticTokensRequestsClientCapabilities,
+      tokenTypes: [String],
+      tokenModifiers: [String],
+      formats: [TokenFormat],
+      overlappingTokenSupport: Bool? = nil,
+      multilineTokenSupport: Bool? = nil
+    ) {
+      self.dynamicRegistration = dynamicRegistration
+      self.requests = requests
+      self.tokenTypes = tokenTypes
+      self.tokenModifiers = tokenModifiers
+      self.formats = formats
+      self.overlappingTokenSupport = overlappingTokenSupport
+      self.multilineTokenSupport = multilineTokenSupport
+    }
+  }
+
   // MARK: Properties
 
   public var synchronization: Synchronization? = nil
@@ -440,6 +545,8 @@ public struct TextDocumentClientCapabilities: Hashable, Codable {
 
   public var callHierarchy: DynamicRegistrationCapability? = nil
 
+  public var semanticTokens: SemanticTokens? = nil
+
   public init(synchronization: Synchronization? = nil,
               completion: Completion? = nil,
               hover: Hover? = nil,
@@ -461,7 +568,8 @@ public struct TextDocumentClientCapabilities: Hashable, Codable {
               rename: DynamicRegistrationCapability? = nil,
               publishDiagnostics: PublishDiagnostics? = nil,
               foldingRange: FoldingRange? = nil,
-              callHierarchy: DynamicRegistrationCapability? = nil) {
+              callHierarchy: DynamicRegistrationCapability? = nil,
+              semanticTokens: SemanticTokens? = nil) {
     self.synchronization = synchronization
     self.completion = completion
     self.hover = hover
@@ -484,5 +592,6 @@ public struct TextDocumentClientCapabilities: Hashable, Codable {
     self.publishDiagnostics = publishDiagnostics
     self.foldingRange = foldingRange
     self.callHierarchy = callHierarchy
+    self.semanticTokens = semanticTokens
   }
 }