Apply mostly stylistic suggestions regarding tokens

- Make token parser more robust
  Continue adding line and char deltas from LSP-encoded tokens even if the
  kind is not recognized.
- Move DocumentTokens to its own file
- Rename edit callbacks for clarity
  beforeCallback -> willEditDocument
  afterCallback -> updateDocumentTokens
- Move token start setter into mutating function
  Since this setter has some subtle semantics, this might be expressed
  more clearly in a separate method.
- Remove CaseIterable conformance from Token.Modifiers
  Since allCases didn't actually contain all combinations of modifiers, we
  remove this conformance and rename allCases to allModifiers.
- Move SyntaxHighlightingTokenParser into separate file
- Update doc comments on SyntaxHighlightingToken
  Reformat them to distinguish between brief and long documentation.
- Add Token.move(lineDelta:utf16indexDelta:)
- Remove redundant range emptiness check

Author: fwcd

Sources/SKTestSupport/Array+SyntaxHighlightingToken.swift

@@ -29,9 +29,6 @@ extension Array where Element == SyntaxHighlightingToken {
       let rawKind = rawTokens[i + 3]
       let rawModifiers = rawTokens[i + 4]
 
-      guard let kind = SyntaxHighlightingToken.Kind(rawValue: rawKind) else { continue }
-      let modifiers = SyntaxHighlightingToken.Modifiers(rawValue: rawModifiers)
-
       current.line += lineDelta
 
       if lineDelta == 0 {
@@ -40,6 +37,9 @@ extension Array where Element == SyntaxHighlightingToken {
         current.utf16index = charDelta
       }
 
+      guard let kind = SyntaxHighlightingToken.Kind(rawValue: rawKind) else { continue }
+      let modifiers = SyntaxHighlightingToken.Modifiers(rawValue: rawModifiers)
+
       append(SyntaxHighlightingToken(
         start: current,
         utf16length: length,

Sources/SourceKitLSP/CMakeLists.txt

@@ -8,6 +8,7 @@ endif()
 add_library(SourceKitLSP
   CapabilityRegistry.swift
   DocumentManager.swift
+  DocumentTokens.swift
   IndexStoreDB+MainFilesProvider.swift
   SourceKitIndexDelegate.swift
   SourceKitLSPCommandMetadata.swift
@@ -31,6 +32,7 @@ target_sources(SourceKitLSP PRIVATE
   Swift/SwiftCommand.swift
   Swift/SwiftLanguageServer.swift
   Swift/SyntaxHighlightingToken.swift
+  Swift/SyntaxHighlightingTokenParser.swift
   Swift/VariableTypeInfo.swift)
 set_target_properties(SourceKitLSP PROPERTIES
   INTERFACE_INCLUDE_DIRECTORIES ${CMAKE_Swift_MODULE_DIRECTORY})

Sources/SourceKitLSP/DocumentManager.swift

@@ -15,36 +15,6 @@ import LanguageServerProtocol
 import LSPLogging
 import SKSupport
 
-public struct DocumentTokens {
-  /// Lexical tokens, e.g. keywords, raw identifiers, ...
-  public var lexical: [SyntaxHighlightingToken] = []
-  /// Syntactic tokens, e.g. declarations, etc.
-  public var syntactic: [SyntaxHighlightingToken] = []
-  /// Semantic tokens, e.g. variable references, type references, ...
-  public var semantic: [SyntaxHighlightingToken] = []
-
-  private var merged: [SyntaxHighlightingToken] {
-    [lexical, syntactic, semantic].reduce([]) { $0.mergingTokens(with: $1) }
-  }
-  public var mergedAndSorted: [SyntaxHighlightingToken] {
-    merged.sorted { $0.start < $1.start }
-  }
-
-  /// Modifies the syntax highlighting tokens of each kind
-  /// (lexical, syntactic, semantic) according to `action`.
-  public mutating func withMutableTokensOfEachKind(_ action: (inout [SyntaxHighlightingToken]) -> Void) {
-    action(&lexical)
-    action(&syntactic)
-    action(&semantic)
-  }
-
-  // Replace all lexical tokens in `range`.
-  public mutating func replaceLexical(in range: Range<Position>, with newTokens: [SyntaxHighlightingToken]) {
-    lexical.removeAll { $0.range.overlaps(range) }
-    lexical += newTokens
-  }
-}
-
 public struct DocumentSnapshot {
   public var document: Document
   public var version: Int
@@ -146,8 +116,8 @@ public final class DocumentManager {
 
   /// Applies the given edits to the document.
   ///
-  /// - parameter beforeCallback: Optional closure to call before each edit.
-  /// - parameter afterCallback: Optional closure to call after each edit.
+  /// - parameter willEditDocument: Optional closure to call before each edit.
+  /// - parameter updateDocumentTokens: Optional closure to call after each edit.
   /// - parameter before: The document contents *before* the edit is applied.
   /// - parameter after: The document contents *after* the edit is applied.
   /// - returns: The contents of the file after all the edits are applied.
@@ -157,16 +127,16 @@ public final class DocumentManager {
     _ uri: DocumentURI,
     newVersion: Int,
     edits: [TextDocumentContentChangeEvent],
-    beforeCallback: ((_ before: DocumentSnapshot, TextDocumentContentChangeEvent) -> Void)? = nil,
-    afterCallback: ((_ after: DocumentSnapshot) -> DocumentTokens?)? = nil
+    willEditDocument: ((_ before: DocumentSnapshot, TextDocumentContentChangeEvent) -> Void)? = nil,
+    updateDocumentTokens: ((_ after: DocumentSnapshot) -> DocumentTokens?)? = nil
   ) throws -> DocumentSnapshot {
     return try queue.sync {
       guard let document = documents[uri] else {
         throw Error.missingDocument(uri)
       }
 
       for edit in edits {
-        if let f = beforeCallback {
+        if let f = willEditDocument {
           f(document.latestSnapshot, edit)
         }
 
@@ -199,10 +169,9 @@ public final class DocumentManager {
                 var token = $0
                 if token.start.line == range.upperBound.line
                   && token.start.utf16index >= range.upperBound.utf16index {
-                  token.start.utf16index += lastLineLengthDelta
-                  token.start.line += lineDelta
+                  token.move(lineDelta: lineDelta, utf16indexDelta: lastLineLengthDelta)
                 } else if token.start.line > range.upperBound.line {
-                  token.start.line += lineDelta
+                  token.move(lineDelta: lineDelta)
                 }
                 return token
               })
@@ -213,7 +182,7 @@ public final class DocumentManager {
           document.latestTokens = DocumentTokens()
         }
 
-        if let f = afterCallback, let tokens = f(document.latestSnapshot) {
+        if let f = updateDocumentTokens, let tokens = f(document.latestSnapshot) {
           document.latestTokens = tokens
         }
       }
@@ -270,21 +239,21 @@ extension DocumentManager {
     }
   }
 
-  /// Convenience wrapper for `edit(_:newVersion:edits:beforeCallback:afterCallback:)`
+  /// Convenience wrapper for `edit(_:newVersion:edits:willEditDocument:updateDocumentTokens:)`
   /// that logs on failure.
   @discardableResult
   func edit(
     _ note: DidChangeTextDocumentNotification,
-    beforeCallback: ((_ before: DocumentSnapshot, TextDocumentContentChangeEvent) -> Void)? = nil,
-    afterCallback: ((_ after: DocumentSnapshot) -> DocumentTokens?)? = nil
+    willEditDocument: ((_ before: DocumentSnapshot, TextDocumentContentChangeEvent) -> Void)? = nil,
+    updateDocumentTokens: ((_ after: DocumentSnapshot) -> DocumentTokens?)? = nil
   ) -> DocumentSnapshot? {
     return orLog("failed to edit document", level: .error) {
       try edit(
         note.textDocument.uri,
         newVersion: note.textDocument.version ?? -1,
         edits: note.contentChanges,
-        beforeCallback: beforeCallback,
-        afterCallback: afterCallback
+        willEditDocument: willEditDocument,
+        updateDocumentTokens: updateDocumentTokens
       )
     }
   }

Sources/SourceKitLSP/DocumentTokens.swift

@@ -0,0 +1,44 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2021 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import LanguageServerProtocol
+
+/// Syntax highlighting tokens for a particular document.
+public struct DocumentTokens {
+  /// Lexical tokens, e.g. keywords, raw identifiers, ...
+  public var lexical: [SyntaxHighlightingToken] = []
+  /// Syntactic tokens, e.g. declarations, etc.
+  public var syntactic: [SyntaxHighlightingToken] = []
+  /// Semantic tokens, e.g. variable references, type references, ...
+  public var semantic: [SyntaxHighlightingToken] = []
+
+  private var merged: [SyntaxHighlightingToken] {
+    [lexical, syntactic, semantic].reduce([]) { $0.mergingTokens(with: $1) }
+  }
+  public var mergedAndSorted: [SyntaxHighlightingToken] {
+    merged.sorted { $0.start < $1.start }
+  }
+
+  /// Modifies the syntax highlighting tokens of each kind
+  /// (lexical, syntactic, semantic) according to `action`.
+  public mutating func withMutableTokensOfEachKind(_ action: (inout [SyntaxHighlightingToken]) -> Void) {
+    action(&lexical)
+    action(&syntactic)
+    action(&semantic)
+  }
+
+  // Replace all lexical tokens in `range`.
+  public mutating func replaceLexical(in range: Range<Position>, with newTokens: [SyntaxHighlightingToken]) {
+    lexical.removeAll { $0.range.overlaps(range) }
+    lexical += newTokens
+  }
+}

Sources/SourceKitLSP/Swift/SwiftLanguageServer.swift

@@ -190,19 +190,13 @@ public final class SwiftLanguageServer: ToolchainLanguageServer {
           let length: Int = response[keys.length],
           let start: Position = snapshot.positionOf(utf8Offset: offset),
           let end: Position = snapshot.positionOf(utf8Offset: offset + length) else {
-      log("updateLexicalAndSyntacticTokens failed, no range found", level: .error)
+      // This e.g. happens in the case of empty edits
+      log("did not update lexical/syntactic tokens, no range found", level: .warning)
       return docTokens
     }
 
     let range = start..<end
 
-    // If the range is empty we don't have to (and shouldn't) update anything.
-    // This is important, since the substructure may be empty, causing us to
-    // unnecessarily remove all syntactic tokens.
-    guard !range.isEmpty else {
-      return docTokens
-    }
-
     if let syntaxMap: SKDResponseArray = response[keys.syntaxmap] {
       let tokenParser = SyntaxHighlightingTokenParser(sourcekitd: sourcekitd)
       let tokens = tokenParser.parseTokens(syntaxMap, in: snapshot)
@@ -365,7 +359,7 @@ extension SwiftLanguageServer {
       semanticTokensProvider: SemanticTokensOptions(
         legend: SemanticTokensLegend(
           tokenTypes: SyntaxHighlightingToken.Kind.allCases.map(\.lspName),
-          tokenModifiers: SyntaxHighlightingToken.Modifiers.allCases.map { $0.lspName! }),
+          tokenModifiers: SyntaxHighlightingToken.Modifiers.allModifiers.map { $0.lspName! }),
         range: .bool(true),
         full: .bool(true))
     ))
@@ -536,7 +530,7 @@ extension SwiftLanguageServer {
 
         req[keys.sourcetext] = edit.text
         lastResponse = try? self.sourcekitd.sendSync(req)
-      } afterCallback: { (after: DocumentSnapshot) in
+      } updateDocumentTokens: { (after: DocumentSnapshot) in
         if let dict = lastResponse {
           return self.updatedLexicalAndSyntacticTokens(response: dict, for: after)
         } else {