Merge pull request #1986 from ahoppen/pause-background-indexing

Add an experimental request to pause background indexing

Author: ahoppen

Contributor Documentation/LSP Extensions.md

@@ -451,13 +451,38 @@ logName?: string;
 
 New request to wait until the index is up-to-date.
 
+> [!IMPORTANT]
+> This request is experimental and may be modified or removed in future versions of SourceKit-LSP without notice. Do not rely on it.
+
 - params: `PollIndexParams`
 - result: `void`
 
 ```ts
 export interface PollIndexParams {}
 ```
 
+## `workspace/_setOptions`
+
+New request to modify runtime options of SourceKit-LSP.
+
+Any options not specified in this request will be left as-is.
+
+> [!IMPORTANT]
+> This request is experimental, guarded behind the `set-options-request` experimental feature, and may be modified or removed in future versions of SourceKit-LSP without notice. Do not rely on it.
+
+- params: `SetOptionsParams`
+- result: `void`
+
+```ts
+export interface SetOptionsParams {
+  /**
+   * `true` to pause background indexing or `false` to resume background indexing.
+   */
+  backgroundIndexingPaused?: bool;
+}
+```
+
+
 ## `workspace/getReferenceDocument`
 
 Request from the client to the server asking for contents of a URI having a custom scheme.

Documentation/Configuration File.md

@@ -55,7 +55,7 @@ The structure of the file is currently not guaranteed to be stable. Options may
   - `noLazy`: Prepare a target without generating object files but do not do lazy type checking and function body skipping. This uses SwiftPM's `--experimental-prepare-for-indexing-no-lazy` flag.
   - `enabled`: Prepare a target without generating object files.
 - `cancelTextDocumentRequestsOnEditAndClose: boolean`: Whether sending a `textDocument/didChange` or `textDocument/didClose` notification for a document should cancel all pending requests for that document.
-- `experimentalFeatures: ("on-type-formatting")[]`: Experimental features that are enabled.
+- `experimentalFeatures: ("on-type-formatting"|"set-options-request")[]`: Experimental features that are enabled.
 - `swiftPublishDiagnosticsDebounceDuration: number`: The time that `SwiftLanguageService` should wait after an edit before starting to compute diagnostics and sending a `PublishDiagnosticsNotification`.
 - `workDoneProgressDebounceDuration: number`: When a task is started that should be displayed to the client as a work done progress, how many milliseconds to wait before actually starting the work done progress. This prevents flickering of the work done progress in the client for short-lived index tasks which end within this duration.
 - `sourcekitdRequestTimeout: number`: The maximum duration that a sourcekitd request should be allowed to execute before being declared as timed out. In general, editors should cancel requests that they are no longer interested in, but in case editors don't cancel requests, this ensures that a long-running non-cancelled request is not blocking sourcekitd and thus most semantic functionality. In particular, VS Code does not cancel the semantic tokens request, which can cause a long-running AST build that blocks sourcekitd.

Sources/LanguageServerProtocol/CMakeLists.txt

@@ -75,6 +75,7 @@ add_library(LanguageServerProtocol STATIC
   Requests/RegisterCapabilityRequest.swift
   Requests/RenameRequest.swift
   Requests/SelectionRangeRequest.swift
+  Requests/SetOptionsRequest.swift
   Requests/ShowDocumentRequest.swift
   Requests/ShowMessageRequest.swift
   Requests/ShutdownRequest.swift

Sources/LanguageServerProtocol/Messages.swift

@@ -67,6 +67,7 @@ public let builtinRequests: [_RequestType.Type] = [
   RegisterCapabilityRequest.self,
   RenameRequest.self,
   SelectionRangeRequest.self,
+  SetOptionsRequest.self,
   ShowDocumentRequest.self,
   ShowMessageRequest.self,
   ShutdownRequest.self,

Sources/LanguageServerProtocol/Requests/SetOptionsRequest.swift

@@ -0,0 +1,26 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2025 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+/// New request to modify runtime options of SourceKit-LSP.
+//
+/// Any options not specified in this request will be left as-is.
+public struct SetOptionsRequest: RequestType {
+  public static let method: String = "workspace/_setOptions"
+  public typealias Response = VoidResponse
+
+  /// `true` to pause background indexing or `false` to resume background indexing.
+  public var backgroundIndexingPaused: Bool?
+
+  public init(backgroundIndexingPaused: Bool?) {
+    self.backgroundIndexingPaused = backgroundIndexingPaused
+  }
+}

Sources/SKOptions/ExperimentalFeatures.swift

@@ -14,5 +14,9 @@
 /// to `sourcekit-lsp` on the command line or through the configuration file.
 /// The raw value of this feature is how it is named on the command line and in the configuration file.
 public enum ExperimentalFeature: String, Codable, Sendable, CaseIterable {
+  /// Enable support for the `textDocument/onTypeFormatting` request.
   case onTypeFormatting = "on-type-formatting"
+
+  /// Enable support for the `workspace/_setOptions` request.
+  case setOptionsRequest = "set-options-request"
 }

Sources/SKTestSupport/Assertions.swift

@@ -134,13 +134,13 @@ package func unwrap<T>(
   return try XCTUnwrap(expression, file: file, line: line)
 }
 
-private struct ExpectationNotFulfilledError: Error, CustomStringConvertible {
-  var expecatations: [XCTestExpectation]
+package struct ExpectationNotFulfilledError: Error, CustomStringConvertible {
+  var expectations: [XCTestExpectation]
 
-  var description: String {
+  package var description: String {
     return """
       One of the expectation was not fulfilled within timeout: \
-      \(expecatations.map(\.description).joined(separator: ", "))
+      \(expectations.map(\.description).joined(separator: ", "))
       """
   }
 }
@@ -162,6 +162,6 @@ package nonisolated func fulfillmentOfOrThrow(
     enforceOrder: enforceOrderOfFulfillment
   )
   if started != .completed {
-    throw ExpectationNotFulfilledError(expecatations: expectations)
+    throw ExpectationNotFulfilledError(expectations: expectations)
   }
 }

Sources/SemanticIndex/TaskScheduler.swift

@@ -188,9 +188,10 @@ package actor QueuedTask<TaskDescription: TaskDescriptionProtocol> {
   /// execution.
   private let executionStateChangedCallback: (@Sendable (QueuedTask, TaskExecutionState) async -> Void)?
 
-  init(
+  fileprivate init(
     priority: TaskPriority,
     description: TaskDescription,
+    taskPriorityChangedCallback: @escaping @Sendable (_ newPriority: TaskPriority) -> Void,
     executionStateChangedCallback: (@Sendable (QueuedTask, TaskExecutionState) async -> Void)?
   ) async {
     self._priority = AtomicUInt8(initialValue: priority.rawValue)
@@ -223,6 +224,7 @@ package actor QueuedTask<TaskDescription: TaskDescriptionProtocol> {
             )
           }
           self.priority = Task.currentPriority
+          taskPriorityChangedCallback(self.priority)
         }
       } onCancel: {
         self.resultTaskCancelled.value = true
@@ -349,13 +351,66 @@ package actor TaskScheduler<TaskDescription: TaskDescriptionProtocol> {
   ///  - `.medium`: 4
   ///  - `.low`: 2
   ///  - `.background`: 2
-  private let maxConcurrentTasksByPriority: [(priority: TaskPriority, maxConcurrentTasks: Int)]
+  private var maxConcurrentTasksByPriority: [(priority: TaskPriority, maxConcurrentTasks: Int)] {
+    didSet {
+      // These preconditions need to match the ones in `init`
+      maxConcurrentTasksByPriority = maxConcurrentTasksByPriority.sorted(by: { $0.priority > $1.priority })
+      precondition(maxConcurrentTasksByPriority.allSatisfy { $0.maxConcurrentTasks >= 0 })
+      precondition(maxConcurrentTasksByPriority.map(\.maxConcurrentTasks).isSorted(descending: true))
+      precondition(!maxConcurrentTasksByPriority.isEmpty)
+
+      // Check we are over-subscribed in currently executing tasks. If we are, cancel currently executing task to be
+      // rescheduled until we are within the new limit.
+      var tasksToReschedule: [QueuedTask<TaskDescription>] = []
+      for (priority, maxConcurrentTasks) in maxConcurrentTasksByPriority {
+        var tasksInPrioritySlot = currentlyExecutingTasks.filter { $0.priority <= priority }
+        if tasksInPrioritySlot.count <= maxConcurrentTasks {
+          // We have enough available slots. Nothing to do.
+          continue
+        }
+        tasksInPrioritySlot = tasksInPrioritySlot.sorted { $0.priority > $1.priority }
+        while tasksInPrioritySlot.count > maxConcurrentTasks {
+          // Cancel the task with the lowest priority (because it is least important and also takes a slot in the lower
+          // priority execution buckets) and the among those the most recent one because it has probably made the least
+          // progress.
+          guard let mostRecentTaskInSlot = tasksInPrioritySlot.popLast() else {
+            // Should never happen because `tasksInPrioritySlot.count > maxConcurrentTasks >= 0`
+            logger.fault("Unexpectedly unable to pop last task from tasksInPrioritySlot")
+            break
+          }
+          tasksToReschedule.append(mostRecentTaskInSlot)
+        }
+      }
+
+      // Poke the scheduler to schedule new jobs if new execution slots became available.
+      poke()
+
+      // Cancel any tasks that didn't fit into the new execution slots anymore. Do this on a separate task is fine
+      // because even if we extend the number of execution slots before the task gets executed (which is unlikely), we
+      // would cancel the tasks and then immediately reschedule it – while that's doing unnecessary work, it's still
+      // correct.
+      Task.detached(priority: .high) {
+        for tasksToReschedule in tasksToReschedule {
+          await tasksToReschedule.cancelToBeRescheduled()
+        }
+      }
+    }
+  }
+
+  /// Modify the number of tasks that are allowed to run concurrently at each priority level.
+  ///
+  /// If there are more tasks executing currently that fit within the new execution limits, tasks will be cancelled and
+  /// rescheduled again when execution slots become available.
+  package func setMaxConcurrentTasksByPriority(_ newValue: [(priority: TaskPriority, maxConcurrentTasks: Int)]) {
+    self.maxConcurrentTasksByPriority = newValue
+  }
 
   package init(maxConcurrentTasksByPriority: [(priority: TaskPriority, maxConcurrentTasks: Int)]) {
+    // These preconditions need to match the ones in `maxConcurrentTasksByPriority:didSet`
     self.maxConcurrentTasksByPriority = maxConcurrentTasksByPriority.sorted(by: { $0.priority > $1.priority })
+    precondition(maxConcurrentTasksByPriority.allSatisfy { $0.maxConcurrentTasks >= 0 })
     precondition(maxConcurrentTasksByPriority.map(\.maxConcurrentTasks).isSorted(descending: true))
     precondition(!maxConcurrentTasksByPriority.isEmpty)
-    precondition(maxConcurrentTasksByPriority.last!.maxConcurrentTasks >= 1)
   }
 
   /// Enqueue a new task to be executed.
@@ -374,6 +429,13 @@ package actor TaskScheduler<TaskDescription: TaskDescriptionProtocol> {
     let queuedTask = await QueuedTask(
       priority: priority ?? Task.currentPriority,
       description: taskDescription,
+      taskPriorityChangedCallback: { [weak self] (newPriority) in
+        Task.detached(priority: newPriority) {
+          // If the task's priority got elevated, there might be an execution slot for it now. Poke the scheduler
+          // to run the task if possible.
+          await self?.poke()
+        }
+      },
       executionStateChangedCallback: executionStateChangedCallback
     )
     pendingTasks.append(queuedTask)

Sources/SourceKitLSP/MessageHandlingDependencyTracker.swift

@@ -211,6 +211,10 @@ package enum MessageHandlingDependencyTracker: QueueBasedMessageHandlerDependenc
       self = .globalConfigurationChange
     case is RegisterCapabilityRequest:
       self = .globalConfigurationChange
+    case is SetOptionsRequest:
+      // The request does not modify any global state in an observable way, so we can treat it as a freestanding
+      // request.
+      self = .freestanding
     case is ShowMessageRequest:
       self = .freestanding
     case is ShutdownRequest:

Sources/SourceKitLSP/SourceKitLSPServer.swift

@@ -159,6 +159,26 @@ package actor SourceKitLSPServer {
   /// The files that we asked the client to watch.
   private var watchers: Set<FileSystemWatcher> = []
 
+  private static func maxConcurrentIndexingTasksByPriority(
+    isIndexingPaused: Bool,
+    options: SourceKitLSPOptions
+  ) -> [(priority: TaskPriority, maxConcurrentTasks: Int)] {
+    let processorCount = ProcessInfo.processInfo.processorCount
+    let lowPriorityCores =
+      if isIndexingPaused {
+        0
+      } else {
+        max(
+          Int(options.indexOrDefault.maxCoresPercentageToUseForBackgroundIndexingOrDefault * Double(processorCount)),
+          1
+        )
+      }
+    return [
+      (TaskPriority.medium, processorCount),
+      (TaskPriority.low, lowPriorityCores),
+    ]
+  }
+
   /// Creates a language server for the given client.
   package init(
     client: Connection,
@@ -173,13 +193,9 @@ package actor SourceKitLSPServer {
     self.onExit = onExit
 
     self.client = client
-    let processorCount = ProcessInfo.processInfo.processorCount
-    let lowPriorityCores =
-      options.indexOrDefault.maxCoresPercentageToUseForBackgroundIndexingOrDefault * Double(processorCount)
-    self.indexTaskScheduler = TaskScheduler(maxConcurrentTasksByPriority: [
-      (TaskPriority.medium, processorCount),
-      (TaskPriority.low, max(Int(lowPriorityCores), 1)),
-    ])
+    self.indexTaskScheduler = TaskScheduler(
+      maxConcurrentTasksByPriority: Self.maxConcurrentIndexingTasksByPriority(isIndexingPaused: false, options: options)
+    )
     self.indexProgressManager = nil
     #if canImport(SwiftDocC)
     self.documentationManager = nil
@@ -822,6 +838,8 @@ extension SourceKitLSPServer: QueueBasedMessageHandler {
       await self.handleRequest(for: request, requestHandler: self.references)
     case let request as RequestAndReply<RenameRequest>:
       await request.reply { try await rename(request.params) }
+    case let request as RequestAndReply<SetOptionsRequest>:
+      await request.reply { try await self.setBackgroundIndexingPaused(request.params) }
     case let request as RequestAndReply<ShutdownRequest>:
       await request.reply { try await shutdown(request.params) }
     case let request as RequestAndReply<SymbolInfoRequest>:
@@ -1434,6 +1452,19 @@ extension SourceKitLSPServer {
     }
   }
 
+  func setBackgroundIndexingPaused(_ request: SetOptionsRequest) async throws -> VoidResponse {
+    guard self.options.hasExperimentalFeature(.setOptionsRequest) else {
+      throw ResponseError.unknown("Pausing background indexing is an experimental feature")
+    }
+    if let backgroundIndexingPaused = request.backgroundIndexingPaused {
+      await self.indexTaskScheduler.setMaxConcurrentTasksByPriority(
+        Self.maxConcurrentIndexingTasksByPriority(isIndexingPaused: backgroundIndexingPaused, options: self.options)
+      )
+    }
+
+    return VoidResponse()
+  }
+
   // MARK: - Language features
 
   func completion(