Add a development subcommand to index a project

.gitignore

@@ -2,8 +2,7 @@
 default.profraw
 Package.resolved
 /.build
-/.index-build
-/.linux-build
+/.*-build
 /Packages
 /*.xcodeproj
 /*.sublime-project

Package.swift

@@ -74,9 +74,12 @@ let package = Package(
     .target(
       name: "Diagnose",
       dependencies: [
+        "InProcessClient",
         "LSPLogging",
-        "SourceKitD",
         "SKCore",
+        "SKSupport",
+        "SourceKitD",
+        "SourceKitLSP",
         .product(name: "ArgumentParser", package: "swift-argument-parser"),
         .product(name: "SwiftIDEUtils", package: "swift-syntax"),
         .product(name: "SwiftSyntax", package: "swift-syntax"),
@@ -99,6 +102,20 @@ let package = Package(
       ]
     ),
 
+    // MARK: InProcessClient
+
+    .target(
+      name: "InProcessClient",
+      dependencies: [
+        "CAtomics",
+        "LanguageServerProtocol",
+        "LSPLogging",
+        "SKCore",
+        "SourceKitLSP",
+      ],
+      exclude: ["CMakeLists.txt"]
+    ),
+
     // MARK: LanguageServerProtocol
     // The core LSP types, suitable for any LSP implementation.
     .target(
@@ -162,6 +179,7 @@ let package = Package(
     .target(
       name: "LSPTestSupport",
       dependencies: [
+        "InProcessClient",
         "LanguageServerProtocol",
         "LanguageServerProtocolJSONRPC",
         "SKSupport",
@@ -278,6 +296,7 @@ let package = Package(
       name: "SKTestSupport",
       dependencies: [
         "CSKTestSupport",
+        "InProcessClient",
         "LanguageServerProtocol",
         "LSPTestSupport",
         "LSPLogging",

Sources/CMakeLists.txt

@@ -2,6 +2,7 @@ add_subdirectory(BuildServerProtocol)
 add_subdirectory(CAtomics)
 add_subdirectory(Csourcekitd)
 add_subdirectory(Diagnose)
+add_subdirectory(InProcessClient)
 add_subdirectory(LanguageServerProtocol)
 add_subdirectory(LanguageServerProtocolJSONRPC)
 add_subdirectory(LSPLogging)

Sources/Diagnose/CMakeLists.txt

@@ -2,6 +2,7 @@ add_library(Diagnose STATIC
   CommandConfiguration+Sendable.swift
   CommandLineArgumentsReducer.swift
   DiagnoseCommand.swift
+  IndexCommand.swift
   MergeSwiftFiles.swift
   OSLogScraper.swift
   ReduceCommand.swift
@@ -23,6 +24,8 @@ set_target_properties(Diagnose PROPERTIES
   INTERFACE_INCLUDE_DIRECTORIES ${CMAKE_Swift_MODULE_DIRECTORY})
 
 target_link_libraries(Diagnose PUBLIC
+  InProcessClient
+  LSPLogging
   SKCore
   SourceKitD
   ArgumentParser

Sources/Diagnose/IndexCommand.swift

@@ -0,0 +1,111 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2024 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import ArgumentParser
+import Foundation
+import InProcessClient
+import LanguageServerProtocol
+import SKCore
+import SKSupport
+import SourceKitLSP
+
+import struct TSCBasic.AbsolutePath
+import class TSCBasic.Process
+import var TSCBasic.stderrStream
+import class TSCUtility.PercentProgressAnimation
+
+private actor IndexLogMessageHandler: MessageHandler {
+  var hasSeenError: Bool = false
+
+  /// Queue to ensure that we don't have two interleaving `print` calls.
+  let queue = AsyncQueue<Serial>()
+
+  nonisolated func handle(_ notification: some NotificationType) {
+    if let notification = notification as? LogMessageNotification {
+      queue.async {
+        await self.handle(notification)
+      }
+    }
+  }
+
+  func handle(_ notification: LogMessageNotification) {
+    self.hasSeenError = notification.type == .warning
+    print(notification.message)
+  }
+
+  nonisolated func handle<Request: RequestType>(
+    _ request: Request,
+    id: RequestID,
+    reply: @escaping @Sendable (LSPResult<Request.Response>) -> Void
+  ) {
+    reply(.failure(.methodNotFound(Request.method)))
+  }
+
+}
+
+public struct IndexCommand: AsyncParsableCommand {
+  public static let configuration: CommandConfiguration = CommandConfiguration(
+    commandName: "index",
+    abstract: "Index a project and print all the processes executed for it as well as their outputs",
+    shouldDisplay: false
+  )
+
+  @Option(
+    name: .customLong("toolchain"),
+    help: """
+      The toolchain used to reduce the sourcekitd issue. \
+      If not specified, the toolchain is found in the same way that sourcekit-lsp finds it
+      """
+  )
+  var toolchainOverride: String?
+
+  @Option(help: "The path to the project that should be indexed")
+  var project: String
+
+  public init() {}
+
+  public func run() async throws {
+    var serverOptions = SourceKitLSPServer.Options()
+    serverOptions.indexOptions.enableBackgroundIndexing = true
+
+    let installPath =
+      if let toolchainOverride, let toolchain = Toolchain(try AbsolutePath(validating: toolchainOverride)) {
+        toolchain.path
+      } else {
+        try AbsolutePath(validating: Bundle.main.bundlePath)
+      }
+
+    let messageHandler = IndexLogMessageHandler()
+    let inProcessClient = try await InProcessSourceKitLSPClient(
+      toolchainRegistry: ToolchainRegistry(installPath: installPath),
+      serverOptions: serverOptions,
+      workspaceFolders: [WorkspaceFolder(uri: DocumentURI(URL(fileURLWithPath: project)))],
+      messageHandler: messageHandler
+    )
+    let start = ContinuousClock.now
+    _ = try await inProcessClient.send(PollIndexRequest())
+    print("Indexing finished in \(start.duration(to: .now))")
+    if await messageHandler.hasSeenError {
+      throw ExitCode(1)
+    }
+  }
+}
+
+fileprivate extension SourceKitLSPServer {
+  func handle<R: RequestType>(_ request: R, requestID: RequestID) async throws -> R.Response {
+    return try await withCheckedThrowingContinuation { continuation in
+      self.handle(request, id: requestID) { result in
+        continuation.resume(with: result)
+      }
+    }
+  }
+}

Sources/InProcessClient/CMakeLists.txt

@@ -0,0 +1,14 @@
+add_library(InProcessClient STATIC
+  InProcessSourceKitLSPClient.swift
+  LocalConnection.swift)
+
+set_target_properties(InProcessClient PROPERTIES
+  INTERFACE_INCLUDE_DIRECTORIES ${CMAKE_Swift_MODULE_DIRECTORY})
+
+target_link_libraries(InProcessClient PUBLIC
+  CAtomics
+  LanguageServerProtocol
+  LSPLogging
+  SKCore
+  SourceKitLSP
+)

Sources/InProcessClient/InProcessSourceKitLSPClient.swift

@@ -0,0 +1,81 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2024 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import CAtomics
+import LanguageServerProtocol
+import SKCore
+import SourceKitLSP
+
+/// Launches a `SourceKitLSPServer` in-process and allows sending messages to it.
+public final class InProcessSourceKitLSPClient: Sendable {
+  private let server: SourceKitLSPServer
+
+  /// `nonisolated(unsafe)` if fine because `nextRequestID` is atomic.
+  private nonisolated(unsafe) var nextRequestID = AtomicUInt32(initialValue: 0)
+
+  /// Create a new `SourceKitLSPServer`. An `InitializeRequest` is automatically sent to the server.
+  ///
+  /// `messageHandler` handles notifications and requests sent from the SourceKit-LSP server to the client.
+  public init(
+    toolchainRegistry: ToolchainRegistry,
+    serverOptions: SourceKitLSPServer.Options = SourceKitLSPServer.Options(),
+    capabilities: ClientCapabilities = ClientCapabilities(),
+    workspaceFolders: [WorkspaceFolder],
+    messageHandler: any MessageHandler
+  ) async throws {
+    let serverToClientConnection = LocalConnection(name: "client")
+    self.server = SourceKitLSPServer(
+      client: serverToClientConnection,
+      toolchainRegistry: toolchainRegistry,
+      options: serverOptions,
+      onExit: {
+        serverToClientConnection.close()
+      }
+    )
+    serverToClientConnection.start(handler: messageHandler)
+    _ = try await self.send(
+      InitializeRequest(
+        processId: nil,
+        rootPath: nil,
+        rootURI: nil,
+        initializationOptions: nil,
+        capabilities: capabilities,
+        trace: .off,
+        workspaceFolders: workspaceFolders
+      )
+    )
+  }
+
+  /// Send the request to `server` and return the request result.
+  ///
+  /// - Important: Because this is an async function, Swift concurrency makes no guarantees about the execution ordering
+  ///   of this request with regard to other requests to the server. If execution of requests in a particular order is
+  ///   necessary and the response of the request is not awaited, use the version of the function that takes a
+  ///   completion handler
+  public func send<R: RequestType>(_ request: R) async throws -> R.Response {
+    return try await withCheckedThrowingContinuation { continuation in
+      self.send(request) {
+        continuation.resume(with: $0)
+      }
+    }
+  }
+
+  /// Send the request to `server` and return the request result via a completion handler.
+  public func send<R: RequestType>(_ request: R, reply: @Sendable @escaping (LSPResult<R.Response>) -> Void) {
+    server.handle(request, id: .number(Int(nextRequestID.fetchAndIncrement())), reply: reply)
+  }
+
+  /// Send the notification to `server`.
+  public func send(_ notification: some NotificationType) {
+    server.handle(notification)
+  }
+}

Sources/LSPTestSupport/TestJSONRPCConnection.swift

@@ -10,6 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+import InProcessClient
 import LanguageServerProtocol
 import LanguageServerProtocolJSONRPC
 import SKSupport

Sources/SKCore/BuildServerBuildSystem.swift

@@ -289,7 +289,10 @@ extension BuildServerBuildSystem: BuildSystem {
     return nil
   }
 
-  public func prepare(targets: [ConfiguredTarget]) async throws {
+  public func prepare(
+    targets: [ConfiguredTarget],
+    indexProcessDidProduceResult: @Sendable (IndexProcessResult) -> Void
+  ) async throws {
     throw PrepareNotSupportedError()
   }
 

Sources/SKCore/BuildSystem.swift

@@ -158,7 +158,10 @@ public protocol BuildSystem: AnyObject, Sendable {
 
   /// Prepare the given targets for indexing and semantic functionality. This should build all swift modules of target
   /// dependencies.
-  func prepare(targets: [ConfiguredTarget]) async throws
+  func prepare(
+    targets: [ConfiguredTarget],
+    indexProcessDidProduceResult: @Sendable (IndexProcessResult) -> Void
+  ) async throws
 
   /// If the build system has knowledge about the language that this document should be compiled in, return it.
   ///

Sources/SKCore/BuildSystemManager.swift

@@ -231,8 +231,11 @@ extension BuildSystemManager {
     return await buildSystem?.targets(dependingOn: targets)
   }
 
-  public func prepare(targets: [ConfiguredTarget]) async throws {
-    try await buildSystem?.prepare(targets: targets)
+  public func prepare(
+    targets: [ConfiguredTarget],
+    indexProcessDidProduceResult: @Sendable (IndexProcessResult) -> Void
+  ) async throws {
+    try await buildSystem?.prepare(targets: targets, indexProcessDidProduceResult: indexProcessDidProduceResult)
   }
 
   public func registerForChangeNotifications(for uri: DocumentURI, language: Language) async {

Sources/SKCore/CMakeLists.txt

@@ -10,6 +10,7 @@ add_library(SKCore STATIC
   Debouncer.swift
   FallbackBuildSystem.swift
   FileBuildSettings.swift
+  IndexProcessResult.swift
   MainFilesProvider.swift
   PathPrefixMapping.swift
   SplitShellCommand.swift

Sources/SKCore/CompilationDatabaseBuildSystem.swift

@@ -125,7 +125,10 @@ extension CompilationDatabaseBuildSystem: BuildSystem {
     return [ConfiguredTarget(targetID: "dummy", runDestinationID: "dummy")]
   }
 
-  public func prepare(targets: [ConfiguredTarget]) async throws {
+  public func prepare(
+    targets: [ConfiguredTarget],
+    indexProcessDidProduceResult: @Sendable (IndexProcessResult) -> Void
+  ) async throws {
     throw PrepareNotSupportedError()
   }