Merge branch 'apple:main' into rename-note-to-notification-wherever-necessary

Author: lokesh-tr

Package.swift

@@ -367,6 +367,7 @@ let package = Package(
       name: "SourceKitLSPTests",
       dependencies: [
         "BuildServerProtocol",
+        "CAtomics",
         "LSPLogging",
         "LSPTestSupport",
         "LanguageServerProtocol",

Sources/SKCore/BuildServerBuildSystem.swift

@@ -279,7 +279,7 @@ extension BuildServerBuildSystem: BuildSystem {
     return [ConfiguredTarget(targetID: "dummy", runDestinationID: "dummy")]
   }
 
-  public func generateBuildGraph() {}
+  public func generateBuildGraph(allowFileSystemWrites: Bool) {}
 
   public func topologicalSort(of targets: [ConfiguredTarget]) async -> [ConfiguredTarget]? {
     return nil

Sources/SKCore/BuildSystem.swift

@@ -135,9 +135,14 @@ public protocol BuildSystem: AnyObject, Sendable {
   /// Return the list of targets and run destinations that the given document can be built for.
   func configuredTargets(for document: DocumentURI) async -> [ConfiguredTarget]
 
-  /// Re-generate the build graph including all the tasks that are necessary for building the entire build graph, like
-  /// resolving package versions.
-  func generateBuildGraph() async throws
+  /// Re-generate the build graph.
+  ///
+  /// If `allowFileSystemWrites` is `true`, this should include all the tasks that are necessary for building the entire
+  /// build graph, like resolving package versions.
+  ///
+  /// If `allowFileSystemWrites` is `false`, no files must be written to disk. This mode is used to determine whether
+  /// the build system can handle a source file, and decide whether a workspace should be opened with this build system
+  func generateBuildGraph(allowFileSystemWrites: Bool) async throws
 
   /// Sort the targets so that low-level targets occur before high-level targets.
   ///

Sources/SKCore/BuildSystemManager.swift

@@ -219,8 +219,8 @@ extension BuildSystemManager {
     return settings
   }
 
-  public func generateBuildGraph() async throws {
-    try await self.buildSystem?.generateBuildGraph()
+  public func generateBuildGraph(allowFileSystemWrites: Bool) async throws {
+    try await self.buildSystem?.generateBuildGraph(allowFileSystemWrites: allowFileSystemWrites)
   }
 
   public func topologicalSort(of targets: [ConfiguredTarget]) async throws -> [ConfiguredTarget]? {

Sources/SKCore/CompilationDatabaseBuildSystem.swift

@@ -132,7 +132,7 @@ extension CompilationDatabaseBuildSystem: BuildSystem {
     throw PrepareNotSupportedError()
   }
 
-  public func generateBuildGraph() {}
+  public func generateBuildGraph(allowFileSystemWrites: Bool) {}
 
   public func topologicalSort(of targets: [ConfiguredTarget]) -> [ConfiguredTarget]? {
     return nil

Sources/SKCore/TaskScheduler.swift

@@ -182,11 +182,11 @@ public actor QueuedTask<TaskDescription: TaskDescriptionProtocol> {
   private let executionStateChangedCallback: (@Sendable (QueuedTask, TaskExecutionState) async -> Void)?
 
   init(
-    priority: TaskPriority? = nil,
+    priority: TaskPriority,
     description: TaskDescription,
     executionStateChangedCallback: (@Sendable (QueuedTask, TaskExecutionState) async -> Void)?
   ) async {
-    self._priority = AtomicUInt8(initialValue: priority?.rawValue ?? Task.currentPriority.rawValue)
+    self._priority = AtomicUInt8(initialValue: priority.rawValue)
     self.description = description
     self.executionStateChangedCallback = executionStateChangedCallback
 
@@ -277,6 +277,10 @@ public actor QueuedTask<TaskDescription: TaskDescriptionProtocol> {
           "Elevating priority of \(self.description.forLogging) from \(self.priority.rawValue) to \(targetPriority.rawValue)"
         )
       }
+      // Awaiting the result task from a higher-priority task will eventually update `priority` through
+      // `withTaskPriorityChangedHandler` but that might take a while because `withTaskPriorityChangedHandler` polls.
+      // Since we know that the priority will be elevated, set it now. That way we don't try to elevate it again.
+      self.priority = targetPriority
       Task(priority: targetPriority) {
         await self.resultTask.value
       }
@@ -352,7 +356,7 @@ public actor TaskScheduler<TaskDescription: TaskDescriptionProtocol> {
     )? = nil
   ) async -> QueuedTask<TaskDescription> {
     let queuedTask = await QueuedTask(
-      priority: priority,
+      priority: priority ?? Task.currentPriority,
       description: taskDescription,
       executionStateChangedCallback: executionStateChangedCallback
     )

Sources/SKSupport/AsyncUtils.swift

@@ -166,3 +166,28 @@ extension Collection where Element: Sendable {
     }
   }
 }
+
+public struct TimeoutError: Error, CustomStringConvertible {
+  public var description: String { "Timed out" }
+}
+
+/// Executes `body`. If it doesn't finish after `duration`, throws a `TimeoutError`.
+public func withTimeout<T: Sendable>(
+  _ duration: Duration,
+  _ body: @escaping @Sendable () async throws -> T
+) async throws -> T {
+  try await withThrowingTaskGroup(of: T.self) { taskGroup in
+    taskGroup.addTask {
+      try await Task.sleep(for: duration)
+      throw TimeoutError()
+    }
+    taskGroup.addTask {
+      return try await body()
+    }
+    for try await value in taskGroup {
+      taskGroup.cancelAll()
+      return value
+    }
+    throw CancellationError()
+  }
+}

Sources/SKSwiftPMWorkspace/SwiftPMBuildSystem.swift

@@ -178,7 +178,9 @@ public actor SwiftPMBuildSystem {
       forRootPackage: AbsolutePath(packageRoot),
       fileSystem: fileSystem
     )
-    if let scratchDirectory = buildSetup.path {
+    if isForIndexBuild {
+      location.scratchDirectory = AbsolutePath(packageRoot.appending(component: ".index-build"))
+    } else if let scratchDirectory = buildSetup.path {
       location.scratchDirectory = AbsolutePath(scratchDirectory)
     }
 
@@ -227,7 +229,6 @@ public actor SwiftPMBuildSystem {
       }
       await delegate.filesDependenciesUpdated(filesWithUpdatedDependencies)
     }
-    try await reloadPackage()
   }
 
   /// Creates a build system using the Swift Package Manager, if this workspace is a package.
@@ -261,13 +262,9 @@ public actor SwiftPMBuildSystem {
 }
 
 extension SwiftPMBuildSystem {
-  public func generateBuildGraph() async throws {
-    try await self.reloadPackage()
-  }
-
   /// (Re-)load the package settings by parsing the manifest and resolving all the targets and
   /// dependencies.
-  func reloadPackage() async throws {
+  public func reloadPackage(forceResolvedVersions: Bool) async throws {
     await reloadPackageStatusCallback(.start)
     defer {
       Task {
@@ -277,7 +274,7 @@ extension SwiftPMBuildSystem {
 
     let modulesGraph = try self.workspace.loadPackageGraph(
       rootInput: PackageGraphRootInput(packages: [AbsolutePath(projectRoot)]),
-      forceResolvedVersions: !isForIndexBuild,
+      forceResolvedVersions: forceResolvedVersions,
       observabilityScope: observabilitySystem.topScope
     )
 
@@ -430,6 +427,10 @@ extension SwiftPMBuildSystem: SKCore.BuildSystem {
     return []
   }
 
+  public func generateBuildGraph(allowFileSystemWrites: Bool) async throws {
+    try await self.reloadPackage(forceResolvedVersions: !isForIndexBuild || !allowFileSystemWrites)
+  }
+
   public func topologicalSort(of targets: [ConfiguredTarget]) -> [ConfiguredTarget]? {
     return targets.sorted { (lhs: ConfiguredTarget, rhs: ConfiguredTarget) -> Bool in
       let lhsIndex = self.targets[lhs]?.index ?? self.targets.count
@@ -590,7 +591,7 @@ extension SwiftPMBuildSystem: SKCore.BuildSystem {
       logger.log("Reloading package because of file change")
       await orLog("Reloading package") {
         // TODO: It should not be necessary to reload the entire package just to get build settings for one file.
-        try await self.reloadPackage()
+        try await self.reloadPackage(forceResolvedVersions: !isForIndexBuild)
       }
     }
 

Sources/SKTestSupport/MultiFileTestProject.swift

@@ -82,6 +82,7 @@ public class MultiFileTestProject {
     workspaces: (URL) async throws -> [WorkspaceFolder] = { [WorkspaceFolder(uri: DocumentURI($0))] },
     capabilities: ClientCapabilities = ClientCapabilities(),
     serverOptions: SourceKitLSPServer.Options = .testDefault,
+    enableBackgroundIndexing: Bool = false,
     usePullDiagnostics: Bool = true,
     preInitialization: ((TestSourceKitLSPClient) -> Void)? = nil,
     cleanUp: (() -> Void)? = nil,
@@ -117,6 +118,7 @@ public class MultiFileTestProject {
       serverOptions: serverOptions,
       capabilities: capabilities,
       usePullDiagnostics: usePullDiagnostics,
+      enableBackgroundIndexing: enableBackgroundIndexing,
       workspaceFolders: workspaces(scratchDirectory),
       preInitialization: preInitialization,
       cleanUp: { [scratchDirectory] in

Sources/SKTestSupport/SkipUnless.swift

@@ -103,7 +103,7 @@ public actor SkipUnless {
   ) async throws {
     try await shared.skipUnlessSupportedByToolchain(swiftVersion: SwiftVersion(5, 11), file: file, line: line) {
       let testClient = try await TestSourceKitLSPClient()
-      let uri = DocumentURI.for(.swift)
+      let uri = DocumentURI(for: .swift)
       testClient.openDocument("0.bitPattern", uri: uri)
       let response = try unwrap(
         await testClient.send(DocumentSemanticTokensRequest(textDocument: TextDocumentIdentifier(uri)))
@@ -134,7 +134,7 @@ public actor SkipUnless {
   ) async throws {
     try await shared.skipUnlessSupportedByToolchain(swiftVersion: SwiftVersion(5, 11), file: file, line: line) {
       let testClient = try await TestSourceKitLSPClient()
-      let uri = DocumentURI.for(.swift)
+      let uri = DocumentURI(for: .swift)
       let positions = testClient.openDocument("func 1️⃣test() {}", uri: uri)
       do {
         _ = try await testClient.send(
@@ -154,7 +154,7 @@ public actor SkipUnless {
   ) async throws {
     try await shared.skipUnlessSupportedByToolchain(swiftVersion: SwiftVersion(5, 11), file: file, line: line) {
       let testClient = try await TestSourceKitLSPClient()
-      let uri = DocumentURI.for(.c)
+      let uri = DocumentURI(for: .c)
       let positions = testClient.openDocument("void 1️⃣test() {}", uri: uri)
       do {
         _ = try await testClient.send(
@@ -214,7 +214,7 @@ public actor SkipUnless {
     return try await shared.skipUnlessSupportedByToolchain(swiftVersion: SwiftVersion(6, 0), file: file, line: line) {
       // The XML-based doc comment conversion did not preserve `Precondition`.
       let testClient = try await TestSourceKitLSPClient()
-      let uri = DocumentURI.for(.swift)
+      let uri = DocumentURI(for: .swift)
       let positions = testClient.openDocument(
         """
         /// - Precondition: Must have an apple

Sources/SKTestSupport/SwiftPMTestProject.swift

@@ -44,9 +44,10 @@ public class SwiftPMTestProject: MultiFileTestProject {
     allowBuildFailure: Bool = false,
     capabilities: ClientCapabilities = ClientCapabilities(),
     serverOptions: SourceKitLSPServer.Options = .testDefault,
+    enableBackgroundIndexing: Bool = false,
+    usePullDiagnostics: Bool = true,
     pollIndex: Bool = true,
     preInitialization: ((TestSourceKitLSPClient) -> Void)? = nil,
-    usePullDiagnostics: Bool = true,
     cleanUp: (() -> Void)? = nil,
     testName: String = #function
   ) async throws {
@@ -71,6 +72,7 @@ public class SwiftPMTestProject: MultiFileTestProject {
       workspaces: workspaces,
       capabilities: capabilities,
       serverOptions: serverOptions,
+      enableBackgroundIndexing: enableBackgroundIndexing,
       usePullDiagnostics: usePullDiagnostics,
       preInitialization: preInitialization,
       cleanUp: cleanUp,

Sources/SKTestSupport/TestSourceKitLSPClient.swift

@@ -88,7 +88,8 @@ public final class TestSourceKitLSPClient: MessageHandler {
   ///     `true` by default
   ///   - initializationOptions: Initialization options to pass to the SourceKit-LSP server.
   ///   - capabilities: The test client's capabilities.
-  ///   - usePullDiagnostics: Whether to use push diagnostics or use push-based diagnostics
+  ///   - usePullDiagnostics: Whether to use push diagnostics or use push-based diagnostics.
+  ///   - enableBackgroundIndexing: Whether background indexing should be enabled in the project.
   ///   - workspaceFolders: Workspace folders to open.
   ///   - preInitialization: A closure that is called after the test client is created but before SourceKit-LSP is
   ///     initialized. This can be used to eg. register request handlers.
@@ -102,6 +103,7 @@ public final class TestSourceKitLSPClient: MessageHandler {
     initializationOptions: LSPAny? = nil,
     capabilities: ClientCapabilities = ClientCapabilities(),
     usePullDiagnostics: Bool = true,
+    enableBackgroundIndexing: Bool = false,
     workspaceFolders: [WorkspaceFolder]? = nil,
     preInitialization: ((TestSourceKitLSPClient) -> Void)? = nil,
     cleanUp: @Sendable @escaping () -> Void = {}
@@ -115,6 +117,7 @@ public final class TestSourceKitLSPClient: MessageHandler {
     if let moduleCache {
       serverOptions.buildSetup.flags.swiftCompilerFlags += ["-module-cache-path", moduleCache.path]
     }
+    serverOptions.indexOptions.enableBackgroundIndexing = enableBackgroundIndexing
 
     var notificationYielder: AsyncStream<any NotificationType>.Continuation!
     self.notifications = AsyncStream { continuation in
@@ -155,8 +158,8 @@ public final class TestSourceKitLSPClient: MessageHandler {
         XCTAssertEqual(request.registrations.only?.method, DocumentDiagnosticsRequest.method)
         return VoidResponse()
       }
-      preInitialization?(self)
     }
+    preInitialization?(self)
     if initialize {
       _ = try await self.send(
         InitializeRequest(
@@ -193,12 +196,22 @@ public final class TestSourceKitLSPClient: MessageHandler {
   /// Send the request to `server` and return the request result.
   public func send<R: RequestType>(_ request: R) async throws -> R.Response {
     return try await withCheckedThrowingContinuation { continuation in
-      server.handle(request, id: .number(Int(nextRequestID.fetchAndIncrement()))) { result in
+      self.send(request) { result in
         continuation.resume(with: result)
       }
     }
   }
 
+  /// Send the request to `server` and return the result via a completion handler.
+  ///
+  /// This version of the `send` function should only be used if some action needs to be performed after the request is
+  /// sent but before it returns a result.
+  public func send<R: RequestType>(_ request: R, completionHandler: @escaping (LSPResult<R.Response>) -> Void) {
+    server.handle(request, id: .number(Int(nextRequestID.fetchAndIncrement()))) { result in
+      completionHandler(result)
+    }
+  }
+
   /// Send the notification to `server`.
   public func send(_ notification: some NotificationType) {
     server.handle(notification)

Sources/SKTestSupport/Utils.swift

@@ -36,16 +36,17 @@ extension Language {
 }
 
 extension DocumentURI {
-  /// Create a unique URI for a document of the given language.
-  public static func `for`(_ language: Language, testName: String = #function) -> DocumentURI {
+  /// Construct a `DocumentURI` by creating a unique URI for a document of the given language.
+  public init(for language: Language, testName: String = #function) {
     let testBaseName = testName.prefix(while: \.isLetter)
 
     #if os(Windows)
     let url = URL(fileURLWithPath: "C:/\(testBaseName)/\(UUID())/test.\(language.fileExtension)")
     #else
     let url = URL(fileURLWithPath: "/\(testBaseName)/\(UUID())/test.\(language.fileExtension)")
     #endif
-    return DocumentURI(url)
+
+    self.init(url)
   }
 }
 

Sources/SemanticIndex/CMakeLists.txt

@@ -2,12 +2,12 @@
 add_library(SemanticIndex STATIC
   CheckedIndex.swift
   CompilerCommandLineOption.swift
-  IndexStatusManager.swift
   IndexTaskDescription.swift
   PreparationTaskDescription.swift
   SemanticIndexManager.swift
   TestHooks.swift
   UpdateIndexStoreTaskDescription.swift
+  UpToDateTracker.swift
 )
 set_target_properties(SemanticIndex PROPERTIES
   INTERFACE_INCLUDE_DIRECTORIES ${CMAKE_Swift_MODULE_DIRECTORY})

Sources/SemanticIndex/PreparationTaskDescription.swift

@@ -35,7 +35,7 @@ public struct PreparationTaskDescription: IndexTaskDescription {
   /// The build system manager that is used to get the toolchain and build settings for the files to index.
   private let buildSystemManager: BuildSystemManager
 
-  private let preparationUpToDateStatus: IndexUpToDateStatusManager<ConfiguredTarget>
+  private let preparationUpToDateTracker: UpToDateTracker<ConfiguredTarget>
 
   /// See `SemanticIndexManager.indexProcessDidProduceResult`
   private let indexProcessDidProduceResult: @Sendable (IndexProcessResult) -> Void
@@ -59,13 +59,13 @@ public struct PreparationTaskDescription: IndexTaskDescription {
   init(
     targetsToPrepare: [ConfiguredTarget],
     buildSystemManager: BuildSystemManager,
-    preparationUpToDateStatus: IndexUpToDateStatusManager<ConfiguredTarget>,
+    preparationUpToDateTracker: UpToDateTracker<ConfiguredTarget>,
     indexProcessDidProduceResult: @escaping @Sendable (IndexProcessResult) -> Void,
     testHooks: IndexTestHooks
   ) {
     self.targetsToPrepare = targetsToPrepare
     self.buildSystemManager = buildSystemManager
-    self.preparationUpToDateStatus = preparationUpToDateStatus
+    self.preparationUpToDateTracker = preparationUpToDateTracker
     self.indexProcessDidProduceResult = indexProcessDidProduceResult
     self.testHooks = testHooks
   }
@@ -76,7 +76,7 @@ public struct PreparationTaskDescription: IndexTaskDescription {
     // The last 2 digits should be sufficient to differentiate between multiple concurrently running preparation operations
     await withLoggingSubsystemAndScope(subsystem: indexLoggingSubsystem, scope: "preparation-\(id % 100)") {
       let targetsToPrepare = await targetsToPrepare.asyncFilter {
-        await !preparationUpToDateStatus.isUpToDate($0)
+        await !preparationUpToDateTracker.isUpToDate($0)
       }.sorted(by: {
         ($0.targetID, $0.runDestinationID) < ($1.targetID, $1.runDestinationID)
       })
@@ -114,7 +114,7 @@ public struct PreparationTaskDescription: IndexTaskDescription {
       }
       await testHooks.preparationTaskDidFinish?(self)
       if !Task.isCancelled {
-        await preparationUpToDateStatus.markUpToDate(targetsToPrepare, updateOperationStartDate: startDate)
+        await preparationUpToDateTracker.markUpToDate(targetsToPrepare, updateOperationStartDate: startDate)
       }
     }
   }