Miscellaneous commits in preparation for background indexing

Package.swift

@@ -168,6 +168,18 @@ let package = Package(
       ]
     ),
 
+    // MARK: SemanticIndex
+
+    .target(
+      name: "SemanticIndex",
+      dependencies: [
+        "LSPLogging",
+        "SKCore",
+        .product(name: "IndexStoreDB", package: "indexstore-db"),
+      ],
+      exclude: ["CMakeLists.txt"]
+    ),
+
     // MARK: SKCore
     // Data structures and algorithms useful across the project, but not necessarily
     // suitable for use in other packages.
@@ -300,9 +312,11 @@ let package = Package(
       name: "SourceKitLSP",
       dependencies: [
         "BuildServerProtocol",
+        "CAtomics",
         "LanguageServerProtocol",
         "LanguageServerProtocolJSONRPC",
         "LSPLogging",
+        "SemanticIndex",
         "SKCore",
         "SKSupport",
         "SKSwiftPMWorkspace",

Sources/CAtomics/include/CAtomics.h

@@ -63,4 +63,32 @@ static inline void atomic_uint8_set(AtomicUInt8 *atomic, uint8_t newValue) {
   atomic->value = newValue;
 }
 
+// MARK: AtomicInt
+
+typedef struct {
+  _Atomic(int) value;
+} AtomicUInt32;
+
+__attribute__((swift_name("AtomicUInt32.init(initialValue:)")))
+static inline AtomicUInt32 atomic_int_create(uint8_t initialValue) {
+  AtomicUInt32 atomic;
+  atomic.value = initialValue;
+  return atomic;
+}
+
+__attribute__((swift_name("getter:AtomicUInt32.value(self:)")))
+static inline uint32_t atomic_int_get(AtomicUInt32 *atomic) {
+  return atomic->value;
+}
+
+__attribute__((swift_name("setter:AtomicUInt32.value(self:_:)")))
+static inline void atomic_uint32_set(AtomicUInt32 *atomic, uint32_t newValue) {
+  atomic->value = newValue;
+}
+
+__attribute__((swift_name("AtomicUInt32.fetchAndIncrement(self:)")))
+static inline uint32_t atomic_uint32_fetch_and_increment(AtomicUInt32 *atomic) {
+  return atomic->value++;
+}
+
 #endif // SOURCEKITLSP_CATOMICS_H

Sources/CMakeLists.txt

@@ -5,6 +5,7 @@ add_subdirectory(Diagnose)
 add_subdirectory(LanguageServerProtocol)
 add_subdirectory(LanguageServerProtocolJSONRPC)
 add_subdirectory(LSPLogging)
+add_subdirectory(SemanticIndex)
 add_subdirectory(SKCore)
 add_subdirectory(SKSupport)
 add_subdirectory(SKSwiftPMWorkspace)

Sources/LSPLogging/Logging.swift

@@ -21,9 +21,6 @@
 
 import Foundation
 
-/// The subsystem that should be used for any logging by default.
-public let subsystem = "org.swift.sourcekit-lsp"
-
 #if canImport(os) && !SOURCEKITLSP_FORCE_NON_DARWIN_LOGGER
 import os  // os_log
 
@@ -44,5 +41,5 @@ public typealias Signposter = NonDarwinSignposter
 
 /// The logger that is used to log any messages.
 public var logger: Logger {
-  Logger(subsystem: subsystem, category: LoggingScope.scope)
+  Logger(subsystem: LoggingScope.subsystem, category: LoggingScope.scope)
 }

Sources/LSPLogging/LoggingScope.swift

@@ -13,15 +13,50 @@
 import Foundation
 
 public final class LoggingScope {
+  /// The name of the current logging subsystem or `nil` if no logging scope is set.
+  @TaskLocal fileprivate static var _subsystem: String?
+
   /// The name of the current logging scope or `nil` if no logging scope is set.
   @TaskLocal fileprivate static var _scope: String?
 
+  /// The name of the current logging subsystem.
+  public static var subsystem: String {
+    return _subsystem ?? "org.swift.sourcekit-lsp"
+  }
+
   /// The name of the current logging scope.
   public static var scope: String {
     return _scope ?? "default"
   }
 }
 
+/// Logs all messages created from the operation to the given subsystem.
+///
+/// This overrides the current logging subsystem.
+///
+/// - Note: Since this stores the logging subsystem in a task-local value, it only works when run inside a task.
+///   Outside a task, this is a no-op.
+public func withLoggingSubsystemAndScope<Result>(
+  subsystem: String,
+  scope: String?,
+  _ operation: () throws -> Result
+) rethrows -> Result {
+  return try LoggingScope.$_subsystem.withValue(subsystem) {
+    return try LoggingScope.$_scope.withValue(scope, operation: operation)
+  }
+}
+
+/// Same as `withLoggingSubsystemAndScope` but allows the operation to be `async`.
+public func withLoggingSubsystemAndScope<Result>(
+  subsystem: String,
+  scope: String?,
+  _ operation: () async throws -> Result
+) async rethrows -> Result {
+  return try await LoggingScope.$_subsystem.withValue(subsystem) {
+    return try await LoggingScope.$_scope.withValue(scope, operation: operation)
+  }
+}
+
 /// Create a new logging scope, which will be used as the category in any log messages created from the operation.
 ///
 /// This overrides the current logging scope.

Sources/SKCore/BuildServerBuildSystem.swift

@@ -263,10 +263,14 @@ extension BuildServerBuildSystem: BuildSystem {
   ///
   /// Returns `nil` if no build settings have been received from the build
   /// server yet or if no build settings are available for this file.
-  public func buildSettings(for document: DocumentURI, language: Language) async throws -> FileBuildSettings? {
+  public func buildSettings(for document: DocumentURI, language: Language) async -> FileBuildSettings? {
     return buildSettings[document]
   }
 
+  public func defaultLanguage(for document: DocumentURI) async -> Language? {
+    return nil
+  }
+
   public func registerForChangeNotifications(for uri: DocumentURI) {
     let request = RegisterForChanges(uri: uri, action: .register)
     _ = self.buildServer?.send(request) { result in
@@ -317,14 +321,14 @@ extension BuildServerBuildSystem: BuildSystem {
     return .unhandled
   }
 
-  public func testFiles() async -> [DocumentURI] {
-    // BuildServerBuildSystem does not support syntactic test discovery
+  public func sourceFiles() async -> [SourceFileInfo] {
+    // BuildServerBuildSystem does not support syntactic test discovery or background indexing.
     // (https://github.com/apple/sourcekit-lsp/issues/1173).
     return []
   }
 
-  public func addTestFilesDidChangeCallback(_ callback: @escaping () async -> Void) {
-    // BuildServerBuildSystem does not support syntactic test discovery
+  public func addSourceFilesDidChangeCallback(_ callback: @escaping () async -> Void) {
+    // BuildServerBuildSystem does not support syntactic test discovery or background indexing.
     // (https://github.com/apple/sourcekit-lsp/issues/1173).
   }
 }

Sources/SKCore/BuildSystem.swift

@@ -27,6 +27,22 @@ public enum FileHandlingCapability: Comparable, Sendable {
   case handled
 }
 
+public struct SourceFileInfo: Sendable {
+  /// The URI of the source file.
+  public let uri: DocumentURI
+
+  /// Whether the file might contain test cases. This property is an over-approximation. It might be true for files
+  /// from non-test targets or files that don't actually contain any tests. Keeping this list of files with
+  /// `mayContainTets` minimal as possible helps reduce the amount of work that the syntactic test indexer needs to
+  /// perform.
+  public let mayContainTests: Bool
+
+  public init(uri: DocumentURI, mayContainTests: Bool) {
+    self.uri = uri
+    self.mayContainTests = mayContainTests
+  }
+}
+
 /// Provider of FileBuildSettings and other build-related information.
 ///
 /// The primary role of the build system is to answer queries for
@@ -71,6 +87,13 @@ public protocol BuildSystem: AnyObject, Sendable {
   /// file or if it hasn't computed build settings for the file yet.
   func buildSettings(for document: DocumentURI, language: Language) async throws -> FileBuildSettings?
 
+  /// If the build system has knowledge about the language that this document should be compiled in, return it.
+  ///
+  /// This is used to determine the language in which a source file should be background indexed.
+  ///
+  /// If `nil` is returned, the language based on the file's extension.
+  func defaultLanguage(for document: DocumentURI) async -> Language?
+
   /// Register the given file for build-system level change notifications, such
   /// as command line flag changes, dependency changes, etc.
   ///
@@ -88,18 +111,13 @@ public protocol BuildSystem: AnyObject, Sendable {
 
   func fileHandlingCapability(for uri: DocumentURI) async -> FileHandlingCapability
 
-  /// Returns the list of files that might contain test cases.
-  ///
-  /// The returned file list is an over-approximation. It might contain tests from non-test targets or files that don't
-  /// actually contain any tests. Keeping this list as minimal as possible helps reduce the amount of work that the
-  /// syntactic test indexer needs to perform.
-  func testFiles() async -> [DocumentURI]
+  /// Returns the list of source files in the project.
+  func sourceFiles() async -> [SourceFileInfo]
 
-  /// Adds a callback that should be called when the value returned by `testFiles()` changes.
+  /// Adds a callback that should be called when the value returned by `sourceFiles()` changes.
   ///
-  /// The callback might also be called without an actual change to `testFiles`.
-  func addTestFilesDidChangeCallback(_ callback: @Sendable @escaping () async -> Void) async
+  /// The callback might also be called without an actual change to `sourceFiles`.
+  func addSourceFilesDidChangeCallback(_ callback: @Sendable @escaping () async -> Void) async
 }
 
-public let buildTargetsNotSupported =
-  ResponseError.methodNotFound(BuildTargets.method)
+public let buildTargetsNotSupported = ResponseError.methodNotFound(BuildTargets.method)

Sources/SKCore/BuildSystemManager.swift

@@ -50,6 +50,11 @@ public actor BuildSystemManager {
   /// Build system delegate that will receive notifications about setting changes, etc.
   var delegate: BuildSystemDelegate?
 
+  /// The list of toolchains that are available.
+  ///
+  /// Used to determine which toolchain to use for a given document.
+  private let toolchainRegistry: ToolchainRegistry
+
   /// The root of the project that this build system manages. For example, for SwiftPM packages, this is the folder
   /// containing Package.swift. For compilation databases it is the root folder based on which the compilation database
   /// was found.
@@ -65,13 +70,15 @@ public actor BuildSystemManager {
     buildSystem: BuildSystem?,
     fallbackBuildSystem: FallbackBuildSystem?,
     mainFilesProvider: MainFilesProvider?,
+    toolchainRegistry: ToolchainRegistry,
     fallbackSettingsTimeout: DispatchTimeInterval = .seconds(3)
   ) async {
     let buildSystemHasDelegate = await buildSystem?.delegate != nil
     precondition(!buildSystemHasDelegate)
     self.buildSystem = buildSystem
     self.fallbackBuildSystem = fallbackBuildSystem
     self.mainFilesProvider = mainFilesProvider
+    self.toolchainRegistry = toolchainRegistry
     self.fallbackSettingsTimeout = fallbackSettingsTimeout
     await self.buildSystem?.setDelegate(self)
   }
@@ -87,11 +94,34 @@ extension BuildSystemManager {
     self.delegate = delegate
   }
 
+  /// Returns the toolchain that should be used to process the given document.
+  public func toolchain(for uri: DocumentURI, _ language: Language) async -> Toolchain? {
+    // To support multiple toolchains within a single workspace, we need to ask the build system which toolchain to use
+    // for this document.
+    return await toolchainRegistry.defaultToolchain(for: language)
+  }
+
   /// - Note: Needed so we can set the delegate from a different isolation context.
   public func setMainFilesProvider(_ mainFilesProvider: MainFilesProvider?) {
     self.mainFilesProvider = mainFilesProvider
   }
 
+  /// Returns the language that a document should be interpreted in for background tasks where the editor doesn't
+  /// specify the document's language.
+  public func defaultLanguage(for document: DocumentURI) async -> Language? {
+    if let defaultLanguage = await buildSystem?.defaultLanguage(for: document) {
+      return defaultLanguage
+    }
+    switch document.fileURL?.pathExtension {
+    case "c": return .c
+    case "cpp", "cc", "cxx": return .cpp
+    case "m": return .objective_c
+    case "mm", "h": return .objective_cpp
+    case "swift": return .swift
+    default: return nil
+    }
+  }
+
   private func buildSettings(
     for document: DocumentURI,
     language: Language
@@ -177,8 +207,17 @@ extension BuildSystemManager {
     )
   }
 
+  public func sourceFiles() async -> [SourceFileInfo] {
+    return await buildSystem?.sourceFiles() ?? []
+  }
+
   public func testFiles() async -> [DocumentURI] {
-    return await buildSystem?.testFiles() ?? []
+    return await sourceFiles().compactMap { (info: SourceFileInfo) -> DocumentURI? in
+      guard info.mayContainTests else {
+        return nil
+      }
+      return info.uri
+    }
   }
 }
 

Sources/SKCore/CompilationDatabaseBuildSystem.swift

@@ -114,6 +114,10 @@ extension CompilationDatabaseBuildSystem: BuildSystem {
     )
   }
 
+  public func defaultLanguage(for document: DocumentURI) async -> Language? {
+    return nil
+  }
+
   public func registerForChangeNotifications(for uri: DocumentURI) async {
     self.watchedFiles.insert(uri)
   }
@@ -192,11 +196,16 @@ extension CompilationDatabaseBuildSystem: BuildSystem {
     }
   }
 
-  public func testFiles() async -> [DocumentURI] {
-    return compdb?.allCommands.map { DocumentURI($0.url) } ?? []
+  public func sourceFiles() async -> [SourceFileInfo] {
+    guard let compdb else {
+      return []
+    }
+    return compdb.allCommands.map {
+      SourceFileInfo(uri: DocumentURI($0.url), mayContainTests: true)
+    }
   }
 
-  public func addTestFilesDidChangeCallback(_ callback: @escaping () async -> Void) async {
+  public func addSourceFilesDidChangeCallback(_ callback: @escaping () async -> Void) async {
     testFilesDidChangeCallbacks.append(callback)
   }
 }