Skip semantic tokens tests if sourcekitd doesn't support the semantic tokens request

When running sourcekit-lsp’s tests using Xcode 15, they fail because the sourcekitd in Xcode 15 does not contain the semantic tokens request. The intended workaround/fix is to dowload a recent Swift development snapshot from swift.org and set it as `SOURCEKIT_TOOLCHAIN_PATH` when running tests. But that’s not a great experience for new contributors. Instead, if sourcekitd doesn’t support the semantic tokens request, simply skip the test.

This also changes the implementation of the semantic tokens LSP request slightly: When the sourcekitd request to get semantic tokens fails, we now fail the entire LSP request, instead of returning the tokens from the synax tree. I think that’s reasonable because the editor did ask for semantic tokens, not syntactic tokens.

Fixes #940
rdar://117590581

Author: ahoppen

Sources/SourceKitLSP/Swift/SemanticTokens.swift

@@ -50,9 +50,9 @@ extension SwiftLanguageServer {
   private func mergedAndSortedTokens(
     for snapshot: DocumentSnapshot,
     in range: Range<Position>? = nil
-  ) async -> [SyntaxHighlightingToken] {
+  ) async throws -> [SyntaxHighlightingToken] {
     async let tree = syntaxTreeManager.syntaxTree(for: snapshot)
-    async let semanticTokens = await orLog("Loading semantic tokens") { try await semanticHighlightingTokens(for: snapshot) }
+    async let semanticTokens = semanticHighlightingTokens(for: snapshot)
 
     let range =
       if let range = range.flatMap({ $0.byteSourceRange(in: snapshot) }) {
@@ -64,7 +64,7 @@ extension SwiftLanguageServer {
       await tree
       .classifications(in: range)
       .flatMap({ $0.highlightingTokens(in: snapshot) })
-      .mergingTokens(with: semanticTokens ?? [])
+      .mergingTokens(with: try semanticTokens ?? [])
       .sorted { $0.start < $1.start }
   }
 
@@ -73,7 +73,7 @@ extension SwiftLanguageServer {
   ) async throws -> DocumentSemanticTokensResponse? {
     let snapshot = try self.documentManager.latestSnapshot(req.textDocument.uri)
 
-    let tokens = await mergedAndSortedTokens(for: snapshot)
+    let tokens = try await mergedAndSortedTokens(for: snapshot)
     let encodedTokens = tokens.lspEncoded
 
     return DocumentSemanticTokensResponse(data: encodedTokens)
@@ -89,8 +89,7 @@ extension SwiftLanguageServer {
     _ req: DocumentSemanticTokensRangeRequest
   ) async throws -> DocumentSemanticTokensResponse? {
     let snapshot = try self.documentManager.latestSnapshot(req.textDocument.uri)
-
-    let tokens = await mergedAndSortedTokens(for: snapshot, in: req.range)
+    let tokens = try await mergedAndSortedTokens(for: snapshot, in: req.range)
     let encodedTokens = tokens.lspEncoded
 
     return DocumentSemanticTokensResponse(data: encodedTokens)

Tests/SourceKitLSPTests/SemanticTokensTests.swift

@@ -14,6 +14,7 @@ import LSPTestSupport
 import LanguageServerProtocol
 import SKTestSupport
 import SourceKitLSP
+import SourceKitD
 import XCTest
 
 private typealias Token = SyntaxHighlightingToken
@@ -29,7 +30,7 @@ final class SemanticTokensTests: XCTestCase {
   /// - Note: This URI is set to a unique value before each test case in `setUp`.
   private var uri: DocumentURI!
 
-  /// The current verion of the document being opened.
+  /// The current version of the document being opened.
   ///
   /// - Note: This gets reset to 0 in `setUp` and incremented on every call to
   ///   `openDocument` and `editDocument`.
@@ -141,7 +142,16 @@ final class SemanticTokensTests: XCTestCase {
     range: Range<Position>? = nil
   ) async throws -> [Token] {
     openDocument(text: text)
-    return try await performSemanticTokensRequest(range: range)
+    do {
+      return try await performSemanticTokensRequest(range: range)
+    } catch let error as ResponseError {
+      // FIXME: Remove when the semantic tokens request is widely available in sourcekitd
+      if error.message.contains("unknown request: source.request.semantic_tokens") {
+        throw XCTSkip("semantic tokens request not supported by sourcekitd")
+      } else {
+        throw error
+      }
+    }
   }
 
   func testIntArrayCoding() {