Add tests for semantic tokens after edits

- Factor out openDocument in SemanticTokensTests
- Test tokens with empty edits
- Test replace until middle of token
- Test replace until end of token
- Test inserting spaces with tokens
- Test inserting a newline with tokens
- Add test removing newline with tokens
- Add test for semantic multi-edit (variable renaming)

Author: fwcd

Sources/SourceKitLSP/DocumentManager.swift

@@ -177,8 +177,10 @@ public final class DocumentManager {
           document.latestTokens.withMutableTokensOfEachKind { tokens in
             tokens = Array(tokens.lazy
               .filter {
-                // Only keep tokens that don't overlap with the edit range
-                $0.start >= range.upperBound || range.lowerBound >= $0.sameLineEnd
+                // Only keep tokens that don't overlap or bound with the edit range
+                $0.start >= range.upperBound
+                || range.lowerBound >= $0.sameLineEnd
+                || range.isEmpty
               }
               .map {
                 // Shift tokens after the edit range
@@ -264,7 +266,9 @@ public final class DocumentManager {
       // Remove all tokens in `range` (or the entire document if `range` is `nil`)
       document.latestTokens.lexical.removeAll { token in
         range.map {
-          token.start <= $0.upperBound && $0.lowerBound <= token.sameLineEnd
+          token.start <= $0.upperBound
+          && $0.lowerBound <= token.sameLineEnd
+          && !$0.isEmpty
         } ?? true
       }
 

Sources/SourceKitLSP/Swift/SwiftLanguageServer.swift

@@ -519,7 +519,11 @@ extension SwiftLanguageServer {
             lastResponse = try? self.sourcekitd.sendSync(req)
           }
 
-          if let dict = lastResponse, let snapshot = self.documentManager.latestSnapshot(uri) {
+          // SourceKit seems to respond with an empty substructure after sending an
+          // empty range for an edit, causing all syntactic tokens to get removed
+          // therefore we only update them if the range is non-empty.
+
+          if !(edit.range?.isEmpty ?? false), let dict = lastResponse, let snapshot = self.documentManager.latestSnapshot(uri) {
             self.updateLexicalAndSyntacticTokens(response: dict, for: snapshot)
           }
         }