Merge pull request #414 from fwcd/semantic-tokens

Author: ahoppen

README.md

@@ -48,7 +48,7 @@ SourceKit-LSP is still in early development, so you may run into rough edges wit
 | Local Refactoring | ✅ | |
 | Formatting | ❌ | |
 | Folding | ✅ | |
-| Syntax Highlighting | ❌ | Not currently part of LSP. |
+| Syntax Highlighting | ✅ | Both syntactic and semantic tokens |
 | Document Symbols | ✅ |  |
 
 

Sources/LSPLogging/Logging.swift

@@ -62,6 +62,23 @@ public func orLog<R>(
   }
 }
 
+/// Logs the time that the given block takes to execute in milliseconds.
+public func logExecutionTime<R>(
+  _ prefix: String = #function,
+  level: LogLevel = .default,
+  logger: Logger = Logger.shared,
+  _ block: () throws -> R
+) rethrows -> R {
+  let start = Date()
+  let result = try block()
+  let deltaMs = -start.timeIntervalSinceNow * 1000
+  logger.log(
+    "\(prefix)\(prefix.isEmpty ? "" : " ")took \(String(format: "%.2f", deltaMs)) ms to execute",
+    level: level
+  )
+  return result
+}
+
 public protocol LogHandler: AnyObject {
   func handle(_ message: String, level: LogLevel)
 }

Sources/LanguageServerProtocol/Requests/DocumentSymbolRequest.swift

@@ -91,7 +91,7 @@ public struct DocumentSymbol: Hashable, Codable {
   public var selectionRange: Range<Position>
 
   /// Children of this symbol, e.g. properties of a class.
-  public var children: [DocumentSymbol]?
+  public var children: [DocumentSymbol]
 
   public init(
     name: String,
@@ -100,7 +100,7 @@ public struct DocumentSymbol: Hashable, Codable {
     deprecated: Bool? = nil,
     range: Range<Position>,
     selectionRange: Range<Position>,
-    children: [DocumentSymbol]? = nil)
+    children: [DocumentSymbol] = [])
   {
     self.name = name
     self.detail = detail

Sources/LanguageServerProtocol/Requests/WorkspaceSemanticTokensRefreshRequest.swift

@@ -16,4 +16,6 @@
 public struct WorkspaceSemanticTokensRefreshRequest: RequestType, Hashable {
   public static let method: String = "workspace/semanticTokens/refresh"
   public typealias Response = VoidResponse
+
+  public init() {}
 }

Sources/SKTestSupport/Array+SyntaxHighlightingToken.swift

@@ -0,0 +1,51 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2021 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import SourceKitLSP
+import LanguageServerProtocol
+
+extension Array where Element == SyntaxHighlightingToken {
+  /// Decodes the LSP representation of syntax highlighting tokens
+  public init(lspEncodedTokens rawTokens: [UInt32]) {
+    self.init()
+    assert(rawTokens.count.isMultiple(of: 5))
+    reserveCapacity(rawTokens.count / 5)
+
+    var current = Position(line: 0, utf16index: 0)
+
+    for i in stride(from: 0, to: rawTokens.count, by: 5) {
+      let lineDelta = Int(rawTokens[i])
+      let charDelta = Int(rawTokens[i + 1])
+      let length = Int(rawTokens[i + 2])
+      let rawKind = rawTokens[i + 3]
+      let rawModifiers = rawTokens[i + 4]
+
+      current.line += lineDelta
+
+      if lineDelta == 0 {
+        current.utf16index += charDelta
+      } else {
+        current.utf16index = charDelta
+      }
+
+      guard let kind = SyntaxHighlightingToken.Kind(rawValue: rawKind) else { continue }
+      let modifiers = SyntaxHighlightingToken.Modifiers(rawValue: rawModifiers)
+
+      append(SyntaxHighlightingToken(
+        start: current,
+        utf16length: length,
+        kind: kind,
+        modifiers: modifiers
+      ))
+    }
+  }
+}

Sources/SourceKitD/SKDResponseDictionary.swift

@@ -30,16 +30,31 @@ public final class SKDResponseDictionary {
     return sourcekitd.api.variant_dictionary_get_string(dict, key).map(String.init(cString:))
   }
   public subscript(key: sourcekitd_uid_t?) -> Int? {
-    return Int(sourcekitd.api.variant_dictionary_get_int64(dict, key))
+    let value = sourcekitd.api.variant_dictionary_get_value(dict, key)
+    if sourcekitd.api.variant_get_type(value) == SOURCEKITD_VARIANT_TYPE_INT64 {
+      return Int(sourcekitd.api.variant_int64_get_value(value))
+    } else {
+      return nil
+    }
   }
   public subscript(key: sourcekitd_uid_t?) -> Bool? {
-    return sourcekitd.api.variant_dictionary_get_bool(dict, key)
+    let value = sourcekitd.api.variant_dictionary_get_value(dict, key)
+    if sourcekitd.api.variant_get_type(value) == SOURCEKITD_VARIANT_TYPE_BOOL {
+      return sourcekitd.api.variant_bool_get_value(value)
+    } else {
+      return nil
+    }
   }
   public subscript(key: sourcekitd_uid_t?) -> sourcekitd_uid_t? {
     return sourcekitd.api.variant_dictionary_get_uid(dict, key)
   }
   public subscript(key: sourcekitd_uid_t?) -> SKDResponseArray? {
-    return SKDResponseArray(sourcekitd.api.variant_dictionary_get_value(dict, key), response: resp)
+    let value = sourcekitd.api.variant_dictionary_get_value(dict, key)
+    if sourcekitd.api.variant_get_type(value) == SOURCEKITD_VARIANT_TYPE_ARRAY {
+      return SKDResponseArray(value, response: resp)
+    } else {
+      return nil
+    }
   }
 }
 

Sources/SourceKitD/sourcekitd_uids.swift

@@ -16,6 +16,7 @@ public struct sourcekitd_keys {
   public let actionname: sourcekitd_uid_t
   public let actionuid: sourcekitd_uid_t
   public let annotated_decl: sourcekitd_uid_t
+  public let annotations: sourcekitd_uid_t
   public let associated_usrs: sourcekitd_uid_t
   public let bodylength: sourcekitd_uid_t
   public let bodyoffset: sourcekitd_uid_t
@@ -40,6 +41,7 @@ public struct sourcekitd_keys {
   public let filepath: sourcekitd_uid_t
   public let fixits: sourcekitd_uid_t
   public let id: sourcekitd_uid_t
+  public let is_system: sourcekitd_uid_t
   public let kind: sourcekitd_uid_t
   public let length: sourcekitd_uid_t
   public let line: sourcekitd_uid_t
@@ -61,6 +63,7 @@ public struct sourcekitd_keys {
   public let substructure: sourcekitd_uid_t
   public let syntactic_only: sourcekitd_uid_t
   public let syntaxmap: sourcekitd_uid_t
+  public let enablesyntaxmap: sourcekitd_uid_t
   public let text: sourcekitd_uid_t
   public let typename: sourcekitd_uid_t
   public let usr: sourcekitd_uid_t
@@ -86,6 +89,7 @@ public struct sourcekitd_keys {
     actionname = api.uid_get_from_cstr("key.actionname")!
     actionuid = api.uid_get_from_cstr("key.actionuid")!
     annotated_decl = api.uid_get_from_cstr("key.annotated_decl")!
+    annotations = api.uid_get_from_cstr("key.annotations")!
     associated_usrs = api.uid_get_from_cstr("key.associated_usrs")!
     bodylength = api.uid_get_from_cstr("key.bodylength")!
     bodyoffset = api.uid_get_from_cstr("key.bodyoffset")!
@@ -110,6 +114,7 @@ public struct sourcekitd_keys {
     filepath = api.uid_get_from_cstr("key.filepath")!
     fixits = api.uid_get_from_cstr("key.fixits")!
     id = api.uid_get_from_cstr("key.id")!
+    is_system = api.uid_get_from_cstr("key.is_system")!
     kind = api.uid_get_from_cstr("key.kind")!
     length = api.uid_get_from_cstr("key.length")!
     line = api.uid_get_from_cstr("key.line")!
@@ -131,6 +136,7 @@ public struct sourcekitd_keys {
     substructure = api.uid_get_from_cstr("key.substructure")!
     syntactic_only = api.uid_get_from_cstr("key.syntactic_only")!
     syntaxmap = api.uid_get_from_cstr("key.syntaxmap")!
+    enablesyntaxmap = api.uid_get_from_cstr("key.enablesyntaxmap")!
     text = api.uid_get_from_cstr("key.text")!
     typename = api.uid_get_from_cstr("key.typename")!
     usr = api.uid_get_from_cstr("key.usr")!
@@ -272,12 +278,20 @@ public struct sourcekitd_values {
   public let decl_generic_type_param: sourcekitd_uid_t
   public let ref_generic_type_param: sourcekitd_uid_t
   public let ref_module: sourcekitd_uid_t
+  public let syntaxtype_attribute_builtin: sourcekitd_uid_t
   public let syntaxtype_comment: sourcekitd_uid_t
   public let syntaxtype_comment_marker: sourcekitd_uid_t
   public let syntaxtype_comment_url: sourcekitd_uid_t
   public let syntaxtype_doccomment: sourcekitd_uid_t
   public let syntaxtype_doccomment_field: sourcekitd_uid_t
+  public let syntaxtype_keyword: sourcekitd_uid_t
+  public let syntaxtype_number: sourcekitd_uid_t
+  public let syntaxtype_string: sourcekitd_uid_t
+  public let syntaxtype_string_interpolation_anchor: sourcekitd_uid_t
+  public let syntaxtype_type_identifier: sourcekitd_uid_t
+  public let syntaxtype_identifier: sourcekitd_uid_t
   public let expr_object_literal: sourcekitd_uid_t
+  public let expr_call: sourcekitd_uid_t
 
   public let kind_keyword: sourcekitd_uid_t
 
@@ -367,12 +381,20 @@ public struct sourcekitd_values {
     decl_generic_type_param = api.uid_get_from_cstr("source.lang.swift.decl.generic_type_param")!
     ref_generic_type_param = api.uid_get_from_cstr("source.lang.swift.ref.generic_type_param")!
     ref_module = api.uid_get_from_cstr("source.lang.swift.ref.module")!
+    syntaxtype_attribute_builtin = api.uid_get_from_cstr("source.lang.swift.syntaxtype.attribute.builtin")!
     syntaxtype_comment = api.uid_get_from_cstr("source.lang.swift.syntaxtype.comment")!
     syntaxtype_comment_marker = api.uid_get_from_cstr("source.lang.swift.syntaxtype.comment.mark")!
     syntaxtype_comment_url = api.uid_get_from_cstr("source.lang.swift.syntaxtype.comment.url")!
     syntaxtype_doccomment = api.uid_get_from_cstr("source.lang.swift.syntaxtype.doccomment")!
     syntaxtype_doccomment_field = api.uid_get_from_cstr("source.lang.swift.syntaxtype.doccomment.field")!
+    syntaxtype_keyword = api.uid_get_from_cstr("source.lang.swift.syntaxtype.keyword")!
+    syntaxtype_number = api.uid_get_from_cstr("source.lang.swift.syntaxtype.number")!
+    syntaxtype_string = api.uid_get_from_cstr("source.lang.swift.syntaxtype.string")!
+    syntaxtype_string_interpolation_anchor = api.uid_get_from_cstr("source.lang.swift.syntaxtype.string_interpolation_anchor")!
+    syntaxtype_type_identifier = api.uid_get_from_cstr("source.lang.swift.syntaxtype.typeidentifier")!
+    syntaxtype_identifier = api.uid_get_from_cstr("source.lang.swift.syntaxtype.identifier")!
     expr_object_literal = api.uid_get_from_cstr("source.lang.swift.expr.object_literal")!
+    expr_call = api.uid_get_from_cstr("source.lang.swift.expr.call")!
 
     kind_keyword = api.uid_get_from_cstr("source.lang.swift.keyword")!
   }

Sources/SourceKitLSP/CMakeLists.txt

@@ -8,6 +8,7 @@ endif()
 add_library(SourceKitLSP
   CapabilityRegistry.swift
   DocumentManager.swift
+  DocumentTokens.swift
   IndexStoreDB+MainFilesProvider.swift
   SourceKitIndexDelegate.swift
   SourceKitLSPCommandMetadata.swift
@@ -30,6 +31,8 @@ target_sources(SourceKitLSP PRIVATE
   Swift/SourceKitD+ResponseError.swift
   Swift/SwiftCommand.swift
   Swift/SwiftLanguageServer.swift
+  Swift/SyntaxHighlightingToken.swift
+  Swift/SyntaxHighlightingTokenParser.swift
   Swift/VariableTypeInfo.swift)
 set_target_properties(SourceKitLSP PROPERTIES
   INTERFACE_INCLUDE_DIRECTORIES ${CMAKE_Swift_MODULE_DIRECTORY})