Add an option to show the files that are currently being index / targets being prepared in the work done progress

Author: ahoppen

.gitignore

@@ -2,8 +2,7 @@
 default.profraw
 Package.resolved
 /.build
-/.index-build
-/.linux-build
+/.*-build
 /Packages
 /*.xcodeproj
 /*.sublime-project

Sources/SemanticIndex/SemanticIndexManager.swift

@@ -55,6 +55,12 @@ private enum InProgressIndexStore {
   case updatingIndexStore(updateIndexStoreTask: OpaqueQueuedIndexTask, indexTask: Task<Void, Never>)
 }
 
+/// Status of document indexing / target preparation in `inProgressIndexAndPreparationTasks`.
+public enum IndexTaskStatus: Comparable {
+  case scheduled
+  case executing
+}
+
 /// Schedules index tasks and keeps track of the index status of files.
 public final actor SemanticIndexManager {
   /// The underlying index. This is used to check if the index of a file is already up-to-date, in which case it doesn't
@@ -107,42 +113,36 @@ public final actor SemanticIndexManager {
   /// The parameter is the number of files that were scheduled to be indexed.
   private let indexTasksWereScheduled: @Sendable (_ numberOfFileScheduled: Int) -> Void
 
-  /// Callback that is called when an index task has finished.
+  /// Callback that is called when the progress status of an update indexstore or preparation task finishes.
   ///
   /// An object observing this property probably wants to check `inProgressIndexTasks` when the callback is called to
   /// get the current list of in-progress index tasks.
   ///
-  /// The number of `indexTaskDidFinish` calls does not have to relate to the number of `indexTasksWereScheduled` calls.
-  private let indexTaskDidFinish: @Sendable () -> Void
+  /// The number of `indexStatusDidChange` calls does not have to relate to the number of `indexTasksWereScheduled` calls.
+  private let indexStatusDidChange: @Sendable () -> Void
 
   // MARK: - Public API
 
-  /// The files that still need to be indexed.
-  ///
-  /// Scheduled tasks are files that are waiting for their target to be prepared or whose index store update task is
-  /// waiting to be scheduled by the task scheduler.
-  ///
-  /// `executing` are the files that currently have an active index store update task running.
-  public var inProgressIndexFiles: (scheduled: [DocumentURI], executing: [DocumentURI]) {
-    var scheduled: [DocumentURI] = []
-    var executing: [DocumentURI] = []
-    for (uri, status) in inProgressIndexTasks {
-      let isExecuting: Bool
+  /// A summary of the tasks that this `SemanticIndexManager` has currently scheduled or is currently indexing.
+  public var inProgressTasks:
+    (
+      isGeneratingBuildGraph: Bool,
+      indexTasks: [DocumentURI: IndexTaskStatus],
+      preparationTasks: [ConfiguredTarget: IndexTaskStatus]
+    )
+  {
+    let indexTasks = inProgressIndexTasks.mapValues { status in
       switch status {
       case .waitingForPreparation:
-        isExecuting = false
+        return IndexTaskStatus.scheduled
       case .updatingIndexStore(updateIndexStoreTask: let updateIndexStoreTask, indexTask: _):
-        isExecuting = updateIndexStoreTask.isExecuting
-      }
-
-      if isExecuting {
-        executing.append(uri)
-      } else {
-        scheduled.append(uri)
+        return updateIndexStoreTask.isExecuting ? IndexTaskStatus.executing : IndexTaskStatus.scheduled
       }
     }
-
-    return (scheduled, executing)
+    let preparationTasks = inProgressPreparationTasks.mapValues { queuedTask in
+      return queuedTask.isExecuting ? IndexTaskStatus.executing : IndexTaskStatus.scheduled
+    }
+    return (generateBuildGraphTask != nil, indexTasks, preparationTasks)
   }
 
   public init(
@@ -151,14 +151,14 @@ public final actor SemanticIndexManager {
     testHooks: IndexTestHooks,
     indexTaskScheduler: TaskScheduler<AnyIndexTaskDescription>,
     indexTasksWereScheduled: @escaping @Sendable (Int) -> Void,
-    indexTaskDidFinish: @escaping @Sendable () -> Void
+    indexStatusDidChange: @escaping @Sendable () -> Void
   ) {
     self.index = index
     self.buildSystemManager = buildSystemManager
     self.testHooks = testHooks
     self.indexTaskScheduler = indexTaskScheduler
     self.indexTasksWereScheduled = indexTasksWereScheduled
-    self.indexTaskDidFinish = indexTaskDidFinish
+    self.indexStatusDidChange = indexStatusDidChange
   }
 
   /// Schedules a task to index `files`. Files that are known to be up-to-date based on `indexStatus` will
@@ -358,14 +358,15 @@ public final actor SemanticIndexManager {
     }
     let preparationTask = await indexTaskScheduler.schedule(priority: priority, taskDescription) { task, newState in
       guard case .finished = newState else {
+        self.indexStatusDidChange()
         return
       }
       for target in targetsToPrepare {
         if self.inProgressPreparationTasks[target] == OpaqueQueuedIndexTask(task) {
           self.inProgressPreparationTasks[target] = nil
         }
       }
-      self.indexTaskDidFinish()
+      self.indexStatusDidChange()
     }
     for target in targetsToPrepare {
       inProgressPreparationTasks[target] = OpaqueQueuedIndexTask(preparationTask)
@@ -400,6 +401,7 @@ public final actor SemanticIndexManager {
     )
     let updateIndexTask = await indexTaskScheduler.schedule(priority: priority, taskDescription) { task, newState in
       guard case .finished = newState else {
+        self.indexStatusDidChange()
         return
       }
       for fileAndTarget in filesAndTargets {
@@ -409,7 +411,7 @@ public final actor SemanticIndexManager {
           self.inProgressIndexTasks[fileAndTarget.file.sourceFile] = nil
         }
       }
-      self.indexTaskDidFinish()
+      self.indexStatusDidChange()
     }
     for fileAndTarget in filesAndTargets {
       if case .waitingForPreparation(preparationTaskID, let indexTask) = inProgressIndexTasks[

Sources/SourceKitLSP/IndexProgressManager.swift

@@ -11,19 +11,13 @@
 //===----------------------------------------------------------------------===//
 
 import LanguageServerProtocol
+import SKCore
 import SKSupport
 import SemanticIndex
 
 /// Listens for index status updates from `SemanticIndexManagers`. From that information, it manages a
 /// `WorkDoneProgress` that communicates the index progress to the editor.
 actor IndexProgressManager {
-  /// A queue on which `indexTaskWasQueued` and `indexStatusDidChange` are handled.
-  ///
-  /// This allows the two functions two be `nonisolated` (and eg. the caller of `indexStatusDidChange` doesn't have to
-  /// wait for the work done progress to be updated) while still guaranteeing that we handle them in the order they
-  /// were called.
-  private let queue = AsyncQueue<Serial>()
-
   /// The `SourceKitLSPServer` for which this manages the index progress. It gathers all `SemanticIndexManagers` from
   /// the workspaces in the `SourceKitLSPServer`.
   private weak var sourceKitLSPServer: SourceKitLSPServer?
@@ -47,20 +41,20 @@ actor IndexProgressManager {
   }
 
   /// Called when a new file is scheduled to be indexed. Increments the target index count, eg. the 3 in `1/3`.
-  nonisolated func indexTaskWasQueued(count: Int) {
-    queue.async {
-      await self.indexTaskWasQueuedImpl(count: count)
+  nonisolated func indexTasksWereScheduled(count: Int) {
+    Task {
+      await self.indexTasksWereScheduledImpl(count: count)
     }
   }
 
-  private func indexTaskWasQueuedImpl(count: Int) async {
+  private func indexTasksWereScheduledImpl(count: Int) async {
     queuedIndexTasks += count
     await indexStatusDidChangeImpl()
   }
 
   /// Called when a `SemanticIndexManager` finishes indexing a file. Adjusts the done index count, eg. the 1 in `1/3`.
   nonisolated func indexStatusDidChange() {
-    queue.async {
+    Task {
       await self.indexStatusDidChangeImpl()
     }
   }
@@ -70,23 +64,47 @@ actor IndexProgressManager {
       workDoneProgress = nil
       return
     }
-    var scheduled: [DocumentURI] = []
-    var executing: [DocumentURI] = []
+    var isGeneratingBuildGraph = false
+    var indexTasks: [DocumentURI: IndexTaskStatus] = [:]
+    var preparationTasks: [ConfiguredTarget: IndexTaskStatus] = [:]
     for indexManager in await sourceKitLSPServer.workspaces.compactMap({ $0.semanticIndexManager }) {
-      let inProgress = await indexManager.inProgressIndexFiles
-      scheduled += inProgress.scheduled
-      executing += inProgress.executing
+      let inProgress = await indexManager.inProgressTasks
+      isGeneratingBuildGraph = isGeneratingBuildGraph || inProgress.isGeneratingBuildGraph
+      indexTasks.merge(inProgress.indexTasks) { lhs, rhs in
+        return max(lhs, rhs)
+      }
+      preparationTasks.merge(inProgress.preparationTasks) { lhs, rhs in
+        return max(lhs, rhs)
+      }
     }
 
-    if scheduled.isEmpty && executing.isEmpty {
+    if indexTasks.isEmpty {
       // Nothing left to index. Reset the target count and dismiss the work done progress.
       queuedIndexTasks = 0
       workDoneProgress = nil
       return
     }
 
-    let finishedTasks = queuedIndexTasks - scheduled.count - executing.count
-    let message = "\(finishedTasks) / \(queuedIndexTasks)"
+    // We can get into a situation where queuedIndexTasks < indexTasks.count if we haven't processed all
+    // `indexTasksWereScheduled` calls yet but the semantic index managers already track them in their in-progress tasks.
+    // Clip the finished tasks to 0 because showing a negative number there looks stupid.
+    let finishedTasks = max(queuedIndexTasks - indexTasks.count, 0)
+    var message = "\(finishedTasks) / \(queuedIndexTasks)"
+
+    if await sourceKitLSPServer.options.indexOptions.showActivePreparationTasksInProgress {
+      var inProgressTasks: [String] = []
+      if isGeneratingBuildGraph {
+        inProgressTasks.append("- Generating build graph")
+      }
+      inProgressTasks += preparationTasks.filter { $0.value == .executing }
+        .map { "- Preparing \($0.key.targetID)" }
+        .sorted()
+      inProgressTasks += indexTasks.filter { $0.value == .executing }
+        .map { "- Indexing \($0.key.fileURL?.lastPathComponent ?? $0.key.pseudoPath)" }
+        .sorted()
+
+      message += "\n\n" + inProgressTasks.joined(separator: "\n")
+    }
 
     let percentage = Int(Double(finishedTasks) / Double(queuedIndexTasks) * 100)
     if let workDoneProgress {

Sources/SourceKitLSP/SourceKitLSPServer.swift

@@ -1233,9 +1233,9 @@ extension SourceKitLSPServer {
         }
       },
       indexTasksWereScheduled: { [weak self] count in
-        self?.indexProgressManager.indexTaskWasQueued(count: count)
+        self?.indexProgressManager.indexTasksWereScheduled(count: count)
       },
-      indexTaskDidFinish: { [weak self] in
+      indexStatusDidChange: { [weak self] in
         self?.indexProgressManager.indexStatusDidChange()
       }
     )
@@ -1296,9 +1296,9 @@ extension SourceKitLSPServer {
           indexDelegate: nil,
           indexTaskScheduler: self.indexTaskScheduler,
           indexTasksWereScheduled: { [weak self] count in
-            self?.indexProgressManager.indexTaskWasQueued(count: count)
+            self?.indexProgressManager.indexTasksWereScheduled(count: count)
           },
-          indexTaskDidFinish: { [weak self] in
+          indexStatusDidChange: { [weak self] in
             self?.indexProgressManager.indexStatusDidChange()
           }
         )

Sources/SourceKitLSP/Workspace.swift

@@ -95,7 +95,7 @@ public final class Workspace: Sendable {
     indexDelegate: SourceKitIndexDelegate?,
     indexTaskScheduler: TaskScheduler<AnyIndexTaskDescription>,
     indexTasksWereScheduled: @escaping @Sendable (Int) -> Void,
-    indexTaskDidFinish: @escaping @Sendable () -> Void
+    indexStatusDidChange: @escaping @Sendable () -> Void
   ) async {
     self.documentManager = documentManager
     self.buildSetup = options.buildSetup
@@ -115,7 +115,7 @@ public final class Workspace: Sendable {
         testHooks: options.indexTestHooks,
         indexTaskScheduler: indexTaskScheduler,
         indexTasksWereScheduled: indexTasksWereScheduled,
-        indexTaskDidFinish: indexTaskDidFinish
+        indexStatusDidChange: indexStatusDidChange
       )
     } else {
       self.semanticIndexManager = nil
@@ -153,7 +153,7 @@ public final class Workspace: Sendable {
     indexTaskScheduler: TaskScheduler<AnyIndexTaskDescription>,
     reloadPackageStatusCallback: @Sendable @escaping (ReloadPackageStatus) async -> Void,
     indexTasksWereScheduled: @Sendable @escaping (Int) -> Void,
-    indexTaskDidFinish: @Sendable @escaping () -> Void
+    indexStatusDidChange: @Sendable @escaping () -> Void
   ) async throws {
     var buildSystem: BuildSystem? = nil
 
@@ -259,7 +259,7 @@ public final class Workspace: Sendable {
       indexDelegate: indexDelegate,
       indexTaskScheduler: indexTaskScheduler,
       indexTasksWereScheduled: indexTasksWereScheduled,
-      indexTaskDidFinish: indexTaskDidFinish
+      indexStatusDidChange: indexStatusDidChange
     )
   }
 
@@ -316,19 +316,29 @@ public struct IndexOptions: Sendable {
   /// Setting this to a value < 1 ensures that background indexing doesn't use all CPU resources.
   public var maxCoresPercentageToUseForBackgroundIndexing: Double
 
+  /// Whether to show the files that are currently being indexed / the targets that are currently being prepared in the
+  /// work done progress.
+  ///
+  /// This is an option because VS Code tries to render a multi-line work done progress into a single line text field in
+  /// the status bar, which looks broken. But at the same time, it is very useful to get a feeling about what's
+  /// currently happening indexing-wise.
+  public var showActivePreparationTasksInProgress: Bool
+
   public init(
     indexStorePath: AbsolutePath? = nil,
     indexDatabasePath: AbsolutePath? = nil,
     indexPrefixMappings: [PathPrefixMapping]? = nil,
     listenToUnitEvents: Bool = true,
     enableBackgroundIndexing: Bool = false,
-    maxCoresPercentageToUseForBackgroundIndexing: Double = 1
+    maxCoresPercentageToUseForBackgroundIndexing: Double = 1,
+    showActivePreparationTasksInProgress: Bool = false
   ) {
     self.indexStorePath = indexStorePath
     self.indexDatabasePath = indexDatabasePath
     self.indexPrefixMappings = indexPrefixMappings
     self.listenToUnitEvents = listenToUnitEvents
     self.enableBackgroundIndexing = enableBackgroundIndexing
     self.maxCoresPercentageToUseForBackgroundIndexing = maxCoresPercentageToUseForBackgroundIndexing
+    self.showActivePreparationTasksInProgress = showActivePreparationTasksInProgress
   }
 }

Sources/sourcekit-lsp/SourceKitLSP.swift

@@ -203,7 +203,15 @@ struct SourceKitLSP: AsyncParsableCommand {
   @Flag(
     help: "Enable background indexing. This feature is still under active development and may be incomplete."
   )
-  var enableExperimentalBackgroundIndexing = false
+  var experimentalEnableBackgroundIndexing = false
+
+  @Flag(
+    help: """
+      Show which index tasks are currently running in the indexing work done progress. \
+      This produces a multi-line work done progress, which might render incorrectly depending in the editor.
+      """
+  )
+  var experimentalShowActivePreparationTasksInProgress = false
 
   func mapOptions() -> SourceKitLSPServer.Options {
     var serverOptions = SourceKitLSPServer.Options()
@@ -220,7 +228,8 @@ struct SourceKitLSP: AsyncParsableCommand {
     serverOptions.indexOptions.indexStorePath = indexStorePath
     serverOptions.indexOptions.indexDatabasePath = indexDatabasePath
     serverOptions.indexOptions.indexPrefixMappings = indexPrefixMappings
-    serverOptions.indexOptions.enableBackgroundIndexing = enableExperimentalBackgroundIndexing
+    serverOptions.indexOptions.enableBackgroundIndexing = experimentalEnableBackgroundIndexing
+    serverOptions.indexOptions.showActivePreparationTasksInProgress = experimentalShowActivePreparationTasksInProgress
     serverOptions.completionOptions.maxResults = completionMaxResults
     serverOptions.generatedInterfacesPath = generatedInterfacesPath
 

Tests/SourceKitLSPTests/BackgroundIndexingTests.swift

@@ -462,7 +462,7 @@ final class BackgroundIndexingTests: XCTestCase {
       return
     }
     var didGetEndWorkDoneProgress = false
-    for _ in 0..<3 {
+    for _ in 0..<5 {
       let workEndProgress = try await project.testClient.nextNotification(ofType: WorkDoneProgress.self)
       switch workEndProgress.value {
       case .begin:

Tests/SourceKitLSPTests/BuildSystemTests.swift

@@ -137,7 +137,7 @@ final class BuildSystemTests: XCTestCase {
       indexDelegate: nil,
       indexTaskScheduler: .forTesting,
       indexTasksWereScheduled: { _ in },
-      indexTaskDidFinish: {}
+      indexStatusDidChange: {}
     )
 
     await server.setWorkspaces([(workspace: workspace, isImplicit: false)])