Apply PR suggestions regarding semantic tokens

- Make makeToken a fileprivate initializer instead
- Fix doc comments on withMutableTokensOfEachKind
- Rename `previousLineCount` to `replacedLineCount`
- Fix token shifting logic
- Add another test case for inserting tokens
- Return empty document tokens if we don't get an edit response
- Update DocumentSymbol.children to be non-optional
- Rename DocumentSymbolTest to DocumentSymbolTests
  ...to be consistent with the file name and the other test modules.
- Fix indentation in DocumentSymbolTests
- Fix doc comment on Modifier.lspName
- Remove no-longer-needed useName from token parser
  These were only needed for syntactic tokens, which we decided not to
  include in #414 for now.
- Add test case for identifiers only backticked on one side
- Parse tokens into inout array

Author: fwcd

Sources/LanguageServerProtocol/Requests/DocumentSymbolRequest.swift

@@ -91,7 +91,7 @@ public struct DocumentSymbol: Hashable, Codable {
   public var selectionRange: Range<Position>
 
   /// Children of this symbol, e.g. properties of a class.
-  public var children: [DocumentSymbol]?
+  public var children: [DocumentSymbol]
 
   public init(
     name: String,
@@ -100,7 +100,7 @@ public struct DocumentSymbol: Hashable, Codable {
     deprecated: Bool? = nil,
     range: Range<Position>,
     selectionRange: Range<Position>,
-    children: [DocumentSymbol]? = nil)
+    children: [DocumentSymbol] = [])
   {
     self.name = name
     self.detail = detail

Sources/SourceKitLSP/DocumentManager.swift

@@ -150,13 +150,13 @@ public final class DocumentManager {
 
           // Remove all tokens in the updated range and shift later ones.
 
-          let previousLineCount = 1 + range.upperBound.line - range.lowerBound.line
+          let replacedLineCount = 1 + range.upperBound.line - range.lowerBound.line
           let newLines = edit.text.split(separator: "\n", omittingEmptySubsequences: false)
-          let lastLineReplaceLength = (
-            range.lowerBound.line == range.upperBound.line ? range.upperBound.utf16index : 0
-          ) - range.lowerBound.utf16index
-          let lastLineLengthDelta = newLines.last!.count - lastLineReplaceLength
-          let lineDelta = newLines.count - previousLineCount
+          let upperUtf16IndexAfterEdit = (
+            newLines.count == 1 ? range.lowerBound.utf16index : 0
+          ) + newLines.last!.utf16.count
+          let charDelta = upperUtf16IndexAfterEdit - range.upperBound.utf16index
+          let lineDelta = newLines.count - replacedLineCount // may be negative
 
           document.latestTokens.withMutableTokensOfEachKind { tokens in
             tokens = Array(tokens.lazy
@@ -169,7 +169,7 @@ public final class DocumentManager {
                 var token = $0
                 if token.start.line == range.upperBound.line
                   && token.start.utf16index >= range.upperBound.utf16index {
-                  token.move(lineDelta: lineDelta, utf16indexDelta: lastLineLengthDelta)
+                  token.move(lineDelta: lineDelta, utf16indexDelta: charDelta)
                 } else if token.start.line > range.upperBound.line {
                   token.move(lineDelta: lineDelta)
                 }

Sources/SourceKitLSP/DocumentTokens.swift

@@ -27,7 +27,7 @@ public struct DocumentTokens {
   }
 
   /// Modifies the syntax highlighting tokens of each kind
-  /// (lexical, syntactic, semantic) according to `action`.
+  /// (lexical and semantic) according to `action`.
   public mutating func withMutableTokensOfEachKind(_ action: (inout [SyntaxHighlightingToken]) -> Void) {
     action(&lexical)
     action(&semantic)

Sources/SourceKitLSP/Swift/SwiftLanguageServer.swift

@@ -200,7 +200,8 @@ public final class SwiftLanguageServer: ToolchainLanguageServer {
 
       if let syntaxMap: SKDResponseArray = response[keys.syntaxmap] {
         let tokenParser = SyntaxHighlightingTokenParser(sourcekitd: sourcekitd)
-        let tokens = tokenParser.parseTokens(syntaxMap, in: snapshot)
+        var tokens: [SyntaxHighlightingToken] = []
+        tokenParser.parseTokens(syntaxMap, in: snapshot, into: &tokens)
 
         docTokens.replaceLexical(in: range, with: tokens)
       }
@@ -237,7 +238,8 @@ public final class SwiftLanguageServer: ToolchainLanguageServer {
 
       if let skTokens: SKDResponseArray = response[keys.annotations] {
         let tokenParser = SyntaxHighlightingTokenParser(sourcekitd: sourcekitd)
-        let tokens = tokenParser.parseTokens(skTokens, in: snapshot)
+        var tokens: [SyntaxHighlightingToken] = []
+        tokenParser.parseTokens(skTokens, in: snapshot, into: &tokens)
 
         docTokens.semantic = tokens
       }
@@ -531,7 +533,7 @@ extension SwiftLanguageServer {
         if let dict = lastResponse {
           return self.updatedLexicalTokens(response: dict, for: after)
         } else {
-          return nil
+          return DocumentTokens()
         }
       }
 

Sources/SourceKitLSP/Swift/SyntaxHighlightingToken.swift

@@ -171,7 +171,7 @@ public struct SyntaxHighlightingToken: Hashable {
 
     /// The name of the modifier used by LSP, if this
     /// is a single modifier. Note that every modifier
-    /// in `allCases` must have an associated `lspName`.
+    /// in `allModifiers` must have an associated `lspName`.
     var lspName: String? {
       switch self {
       case .declaration: return "declaration"

Sources/SourceKitLSP/Swift/SyntaxHighlightingTokenParser.swift

@@ -17,37 +17,20 @@ import LSPLogging
 /// Parses tokens from sourcekitd response dictionaries.
 struct SyntaxHighlightingTokenParser {
   private let sourcekitd: SourceKitD
-  /// Use `keys.nameoffset` and `keys.namelength` instead of `keys.offset`
-  /// and `keys.length`. This is useful in AST structures provided by
-  /// sourcekitd, since these contain both a name range and a normal range,
-  /// where the latter refers to the entire node (e.g. the entire function
-  /// declaration).
-  private let useName: Bool
-
-  init(sourcekitd: SourceKitD, useName: Bool = false) {
+
+  init(sourcekitd: SourceKitD) {
     self.sourcekitd = sourcekitd
-    self.useName = useName
   }
 
-  func parseTokens(_ response: SKDResponseDictionary, in snapshot: DocumentSnapshot) -> [SyntaxHighlightingToken] {
+  func parseTokens(_ response: SKDResponseDictionary, in snapshot: DocumentSnapshot, into tokens: inout [SyntaxHighlightingToken]) {
     let keys = sourcekitd.keys
-    var tokens: [SyntaxHighlightingToken] = []
 
-    if let offset: Int = useName ? response[keys.nameoffset] : response[keys.offset],
-       var length: Int = useName ? response[keys.namelength] : response[keys.length],
+    if let offset: Int = response[keys.offset],
+       var length: Int = response[keys.length],
        let start: Position = snapshot.positionOf(utf8Offset: offset),
        let skKind: sourcekitd_uid_t = response[keys.kind],
        case (let kind, var modifiers)? = parseKindAndModifiers(skKind) {
 
-      // We treat function declaration and enum member name tokens as a special
-      // case, e.g. SourceKit returns `f(x: Int, y: Int)` as a name instead of just `f`.
-      if useName && [.function, .method, .enumMember].contains(kind) && modifiers.contains(.declaration),
-         let name: String = response[keys.name],
-         name.contains("("),
-         let funcNameLength: Int = name.split(separator: "(").first?.utf8.count {
-        length = funcNameLength
-      }
-
       // If the name is escaped in backticks, we need to add two characters to the
       // length for the backticks.
       if modifiers.contains(.declaration),
@@ -74,19 +57,15 @@ struct SyntaxHighlightingTokenParser {
     }
 
     if let substructure: SKDResponseArray = response[keys.substructure] {
-      tokens += parseTokens(substructure, in: snapshot)
+      parseTokens(substructure, in: snapshot, into: &tokens)
     }
-
-    return tokens
   }
 
-  func parseTokens(_ response: SKDResponseArray, in snapshot: DocumentSnapshot) -> [SyntaxHighlightingToken] {
-    var result: [SyntaxHighlightingToken] = []
+  func parseTokens(_ response: SKDResponseArray, in snapshot: DocumentSnapshot, into tokens: inout [SyntaxHighlightingToken]) {
     response.forEach { (_, value) in
-      result += parseTokens(value, in: snapshot)
+      parseTokens(value, in: snapshot, into: &tokens)
       return true
     }
-    return result
   }
 
   private func parseKindAndModifiers(_ uid: sourcekitd_uid_t) -> (SyntaxHighlightingToken.Kind, SyntaxHighlightingToken.Modifiers)? {

Tests/SourceKitLSPTests/DocumentSymbolTests.swift

@@ -16,7 +16,7 @@ import SKTestSupport
 import SourceKitLSP
 import XCTest
 
-final class DocumentSymbolTest: XCTestCase {
+final class DocumentSymbolTests: XCTestCase {
   typealias DocumentSymbolCapabilities = TextDocumentClientCapabilities.DocumentSymbol
 
   /// Connection and lifetime management for the service.
@@ -25,12 +25,12 @@ final class DocumentSymbolTest: XCTestCase {
   /// The primary interface to make requests to the SourceKitServer.
   var sk: TestClient! = nil
 
-override func tearDown() {
-  sk = nil
-  connection = nil
-}
+  override func tearDown() {
+    sk = nil
+    connection = nil
+  }
 
-func initialize(capabilities: DocumentSymbolCapabilities) {
+  func initialize(capabilities: DocumentSymbolCapabilities) {
     connection = TestSourceKitServer()
     sk = connection.client
     var documentCapabilities = TextDocumentClientCapabilities()