Set plugin paths in all tests

Otherwise, we were trying to load sourcekitd without the plugins for the normal SourceKIt-LSP tests and with the plugins for the plugin tests.

Author: ahoppen

Sources/Diagnose/RunSourcekitdRequestCommand.swift

@@ -67,7 +67,8 @@ package struct RunSourceKitdRequestCommand: AsyncParsableCommand {
         throw ExitCode(1)
       }
     let sourcekitd = try await DynamicallyLoadedSourceKitD.getOrCreate(
-      dylibPath: sourcekitdPath
+      dylibPath: sourcekitdPath,
+      pluginPaths: nil
     )
 
     var lastResponse: SKDResponse?

Sources/SKOptions/SourceKitLSPOptions.swift

@@ -242,6 +242,26 @@ public struct SourceKitLSPOptions: Sendable, Codable, Equatable {
     }
   }
 
+  public struct SourceKitDOptions: Sendable, Codable, Equatable {
+    /// When set, load the SourceKit client plugin from this path instead of locating it inside the toolchain.
+    public var clientPlugin: String?
+
+    /// When set, load the SourceKit service plugin from this path instead of locating it inside the toolchain.
+    public var servicePlugin: String?
+
+    public init(clientPlugin: String? = nil, servicePlugin: String? = nil) {
+      self.clientPlugin = clientPlugin
+      self.servicePlugin = servicePlugin
+    }
+
+    static func merging(base: SourceKitDOptions, override: SourceKitDOptions?) -> SourceKitDOptions {
+      return SourceKitDOptions(
+        clientPlugin: override?.clientPlugin ?? base.clientPlugin,
+        servicePlugin: override?.servicePlugin ?? base.servicePlugin
+      )
+    }
+  }
+
   public enum BackgroundPreparationMode: String, Sendable, Codable, Equatable {
     /// Build a target to prepare it.
     case build
@@ -303,6 +323,13 @@ public struct SourceKitLSPOptions: Sendable, Codable, Equatable {
     set { logging = newValue }
   }
 
+  /// Options modifying the behavior of sourcekitd.
+  private var sourcekitd: SourceKitDOptions?
+  public var sourcekitdOrDefault: SourceKitDOptions {
+    get { sourcekitd ?? .init() }
+    set { sourcekitd = newValue }
+  }
+
   /// Default workspace type. Overrides workspace type selection logic.
   public var defaultWorkspaceType: WorkspaceType?
   /// Directory in which generated interfaces and macro expansions should be stored.
@@ -381,6 +408,7 @@ public struct SourceKitLSPOptions: Sendable, Codable, Equatable {
     clangdOptions: [String]? = nil,
     index: IndexOptions? = .init(),
     logging: LoggingOptions? = .init(),
+    sourcekitd: SourceKitDOptions? = .init(),
     defaultWorkspaceType: WorkspaceType? = nil,
     generatedFilesPath: String? = nil,
     backgroundIndexing: Bool? = nil,
@@ -398,6 +426,7 @@ public struct SourceKitLSPOptions: Sendable, Codable, Equatable {
     self.clangdOptions = clangdOptions
     self.index = index
     self.logging = logging
+    self.sourcekitd = sourcekitd
     self.generatedFilesPath = generatedFilesPath
     self.defaultWorkspaceType = defaultWorkspaceType
     self.backgroundIndexing = backgroundIndexing
@@ -454,6 +483,7 @@ public struct SourceKitLSPOptions: Sendable, Codable, Equatable {
       clangdOptions: override?.clangdOptions ?? base.clangdOptions,
       index: IndexOptions.merging(base: base.indexOrDefault, override: override?.index),
       logging: LoggingOptions.merging(base: base.loggingOrDefault, override: override?.logging),
+      sourcekitd: SourceKitDOptions.merging(base: base.sourcekitdOrDefault, override: override?.sourcekitd),
       defaultWorkspaceType: override?.defaultWorkspaceType ?? base.defaultWorkspaceType,
       generatedFilesPath: override?.generatedFilesPath ?? base.generatedFilesPath,
       backgroundIndexing: override?.backgroundIndexing ?? base.backgroundIndexing,

Tests/SwiftSourceKitPluginTests/PluginPaths.swift renamed to Sources/SKTestSupport/PluginPaths.swift

@@ -11,7 +11,12 @@
 //===----------------------------------------------------------------------===//
 
 import Foundation
+
+#if compiler(>=6)
+package import SourceKitD
+#else
 import SourceKitD
+#endif
 
 /// The path to the `SwiftSourceKitPluginTests` test bundle. This gives us a hook into the the build directory.
 private let xctestBundle: URL = {
@@ -29,7 +34,7 @@ private let xctestBundle: URL = {
 }()
 
 /// When running tests from Xcode, determine the build configuration of the package.
-var inferedXcodeBuildConfiguration: String? {
+var inferredXcodeBuildConfiguration: String? {
   if let xcodeBuildDirectory = ProcessInfo.processInfo.environment["__XCODE_BUILT_PRODUCTS_DIR_PATHS"] {
     return URL(fileURLWithPath: xcodeBuildDirectory).lastPathComponent
   } else {
@@ -47,7 +52,7 @@ private func fileExists(at url: URL) -> Bool {
 /// Implementation detail of `sourceKitPluginPaths` which walks up the directory structure, repeatedly calling this method.
 private func pluginPaths(relativeTo base: URL) -> PluginPaths? {
   // When building in Xcode
-  if let buildConfiguration = inferedXcodeBuildConfiguration {
+  if let buildConfiguration = inferredXcodeBuildConfiguration {
     let frameworksDir = base.appendingPathComponent("Products")
       .appendingPathComponent(buildConfiguration)
       .appendingPathComponent("PackageFrameworks")
@@ -99,16 +104,25 @@ private func pluginPaths(relativeTo base: URL) -> PluginPaths? {
   return nil
 }
 
-/// Returns the path the the client plugin and the server plugin within the current build directory.
-///
-/// Returns `nil` if either of the plugins can't be found in the build directory.
-let sourceKitPluginPaths: PluginPaths? = {
-  var base = xctestBundle
-  while base.pathComponents.count > 1 {
-    if let paths = pluginPaths(relativeTo: base) {
-      return paths
+/// Returns the paths from which the SourceKit plugins should be loaded or throws an error if the plugins cannot be
+/// found.
+package var sourceKitPluginPaths: PluginPaths {
+  get throws {
+    struct PluginLoadingError: Error, CustomStringConvertible {
+      var description: String =
+        "Could not find SourceKit plugin. Ensure that you build the entire SourceKit-LSP package before running tests."
+    }
+
+    var base = xctestBundle
+    while base.pathComponents.count > 1 {
+      if let paths = pluginPaths(relativeTo: base) {
+        return paths
+      }
+      base = base.deletingLastPathComponent()
     }
-    base = base.deletingLastPathComponent()
+
+    // If we couldn't find the plugins, keep `didLoadPlugin = false`, which will throw an error in each test case's
+    // `setUp` function. We can't throw an error from the class `setUp` function.
+    throw PluginLoadingError()
   }
-  return nil
-}()
+}

Sources/SKTestSupport/TestSourceKitLSPClient.swift

@@ -18,6 +18,7 @@ import LanguageServerProtocolJSONRPC
 import LanguageServerProtocolExtensions
 package import SKOptions
 import SKUtilities
+import SourceKitD
 package import SourceKitLSP
 import SwiftExtensions
 import SwiftSyntax
@@ -31,6 +32,7 @@ import LanguageServerProtocolJSONRPC
 import LanguageServerProtocolExtensions
 import SKOptions
 import SKUtilities
+import SourceKitD
 import SourceKitLSP
 import SwiftExtensions
 import SwiftSyntax
@@ -44,6 +46,10 @@ extension SourceKitLSPOptions {
     experimentalFeatures: Set<ExperimentalFeature>? = nil
   ) -> SourceKitLSPOptions {
     return SourceKitLSPOptions(
+      sourcekitd: SourceKitDOptions(
+        clientPlugin: try! sourceKitPluginPaths.clientPlugin.filePath,
+        servicePlugin: try! sourceKitPluginPaths.servicePlugin.filePath
+      ),
       backgroundIndexing: backgroundIndexing,
       experimentalFeatures: experimentalFeatures,
       swiftPublishDiagnosticsDebounceDuration: 0,

Sources/SourceKitD/DynamicallyLoadedSourceKitD.swift

@@ -101,7 +101,7 @@ package actor DynamicallyLoadedSourceKitD: SourceKitD {
 
   package static func getOrCreate(
     dylibPath: URL,
-    pluginPaths: PluginPaths? = nil
+    pluginPaths: PluginPaths?
   ) async throws -> SourceKitD {
     try await SourceKitDRegistry.shared
       .getOrAdd(

Sources/SourceKitD/SourceKitD.swift

@@ -30,8 +30,8 @@ fileprivate struct SourceKitDRequestHandle: Sendable {
 }
 
 package struct PluginPaths: Equatable, CustomLogStringConvertible {
-  let clientPlugin: URL
-  let servicePlugin: URL
+  package let clientPlugin: URL
+  package let servicePlugin: URL
 
   package init(clientPlugin: URL, servicePlugin: URL) {
     self.clientPlugin = clientPlugin

Sources/SourceKitLSP/Swift/SwiftLanguageService.swift

@@ -208,7 +208,21 @@ package actor SwiftLanguageService: LanguageService, Sendable {
     guard let sourcekitd = toolchain.sourcekitd else { return nil }
     self.sourceKitLSPServer = sourceKitLSPServer
     self.swiftFormat = toolchain.swiftFormat
-    self.sourcekitd = try await DynamicallyLoadedSourceKitD.getOrCreate(dylibPath: sourcekitd)
+    let pluginPaths: PluginPaths?
+    if let clientPlugin = options.sourcekitdOrDefault.clientPlugin,
+      let servicePlugin = options.sourcekitdOrDefault.servicePlugin
+    {
+      pluginPaths = PluginPaths(
+        clientPlugin: URL(fileURLWithPath: clientPlugin),
+        servicePlugin: URL(fileURLWithPath: servicePlugin)
+      )
+    } else {
+      pluginPaths = nil
+    }
+    self.sourcekitd = try await DynamicallyLoadedSourceKitD.getOrCreate(
+      dylibPath: sourcekitd,
+      pluginPaths: pluginPaths
+    )
     self.capabilityRegistry = workspace.capabilityRegistry
     self.semanticIndexManager = workspace.semanticIndexManager
     self.testHooks = testHooks

Sources/SwiftSourceKitPlugin/Plugin.swift

@@ -168,16 +168,74 @@ public func sourcekitd_plugin_initialize(_ params: sourcekitd_api_plugin_initial
   #endif
 }
 
+#if canImport(Darwin)
+private extension DynamicallyLoadedSourceKitD {
+  /// When a plugin is initialized, it gets passed the library it was loaded from to `sourcekitd_plugin_initialize_2`.
+  ///
+  /// Since the plugin wants to interact with sourcekitd in-process, it needs to load `sourcekitdInProc`. This function
+  /// loads `sourcekitdInProc` relative to the parent library path, if it exists, or `sourcekitd` if `sourcekitdInProc`
+  /// doesn't exist (eg. on Linux where `sourcekitd` is already in-process).
+  static func inProcLibrary(relativeTo parentLibraryPath: URL) throws -> DynamicallyLoadedSourceKitD {
+    var frameworkUrl = parentLibraryPath
+
+    // Remove path components until we reach the `sourcekitd.framework` directory. The plugin might have been loaded
+    // from an XPC service, in which case `parentLibraryPath` is
+    // `sourcekitd.framework/XPCServices/SourceKitService.xpc/Contents/MacOS/SourceKitService`.
+    while frameworkUrl.pathExtension != "framework" {
+      guard frameworkUrl.pathComponents.count > 1 else {
+        struct NoFrameworkPathError: Error, CustomStringConvertible {
+          var parentLibraryPath: URL
+          var description: String { "Could not find .framework directory relative to '\(parentLibraryPath)'" }
+        }
+        throw NoFrameworkPathError(parentLibraryPath: parentLibraryPath)
+      }
+      frameworkUrl.deleteLastPathComponent()
+    }
+    frameworkUrl.deleteLastPathComponent()
+
+    let inProcUrl =
+      frameworkUrl
+      .appendingPathComponent("sourcekitdInProc.framework")
+      .appendingPathComponent("sourcekitdInProc")
+    if FileManager.default.fileExists(at: inProcUrl) {
+      return try DynamicallyLoadedSourceKitD(
+        dylib: inProcUrl,
+        pluginPaths: nil,
+        initialize: false
+      )
+    }
+
+    let sourcekitdUrl =
+      frameworkUrl
+      .appendingPathComponent("sourcekitd.framework")
+      .appendingPathComponent("sourcekitd")
+    return try DynamicallyLoadedSourceKitD(
+      dylib: sourcekitdUrl,
+      pluginPaths: nil,
+      initialize: false
+    )
+  }
+}
+#endif
+
 @_cdecl("sourcekitd_plugin_initialize_2")
 public func sourcekitd_plugin_initialize_2(
   _ params: sourcekitd_api_plugin_initialize_params_t,
-  _ sourcekitdPath: UnsafePointer<CChar>
+  _ parentLibraryPath: UnsafePointer<CChar>
 ) {
+  #if canImport(Darwin)
+  // On macOS, we need to find sourcekitdInProc relative to the library the plugin was loaded from.
+  DynamicallyLoadedSourceKitD.forPlugin = try! DynamicallyLoadedSourceKitD.inProcLibrary(
+    relativeTo: URL(fileURLWithPath: String(cString: parentLibraryPath))
+  )
+  #else
+  // On other platforms, sourcekitd is always in process, so we can load it straight away.
   DynamicallyLoadedSourceKitD.forPlugin = try! DynamicallyLoadedSourceKitD(
     dylib: URL(fileURLWithPath: String(cString: parentLibraryPath)),
     pluginPaths: nil,
     initialize: false
   )
+  #endif
   let sourcekitd = DynamicallyLoadedSourceKitD.forPlugin
 
   let completionResultsBufferKind = sourcekitd.pluginApi.plugin_initialize_custom_buffer_start(params)

Tests/DiagnoseTests/DiagnoseTests.swift

@@ -317,7 +317,10 @@ private class InProcessSourceKitRequestExecutor: SourceKitRequestExecutor {
     let requestString = try request.request(for: temporarySourceFile)
     logger.info("Sending request: \(requestString)")
 
-    let sourcekitd = try await DynamicallyLoadedSourceKitD.getOrCreate(dylibPath: sourcekitd)
+    let sourcekitd = try await DynamicallyLoadedSourceKitD.getOrCreate(
+      dylibPath: sourcekitd,
+      pluginPaths: sourceKitPluginPaths
+    )
     let response = try await sourcekitd.run(requestYaml: requestString)
 
     logger.info("Received response: \(response.description)")

Tests/SourceKitDTests/SourceKitDTests.swift

@@ -26,7 +26,10 @@ import class TSCBasic.Process
 final class SourceKitDTests: XCTestCase {
   func testMultipleNotificationHandlers() async throws {
     let sourcekitdPath = await ToolchainRegistry.forTesting.default!.sourcekitd!
-    let sourcekitd = try await DynamicallyLoadedSourceKitD.getOrCreate(dylibPath: sourcekitdPath)
+    let sourcekitd = try await DynamicallyLoadedSourceKitD.getOrCreate(
+      dylibPath: sourcekitdPath,
+      pluginPaths: sourceKitPluginPaths
+    )
     let keys = sourcekitd.keys
     let path = DocumentURI(for: .swift).pseudoPath
 

Tests/SourceKitLSPTests/LifecycleTests.swift

@@ -45,7 +45,14 @@ final class LifecycleTests: XCTestCase {
     // Check that none of the keys in `SourceKitLSPOptions` are required.
     XCTAssertEqual(
       try JSONDecoder().decode(SourceKitLSPOptions.self, from: XCTUnwrap("{}".data(using: .utf8))),
-      SourceKitLSPOptions(swiftPM: nil, fallbackBuildSystem: nil, compilationDatabase: nil, index: nil, logging: nil)
+      SourceKitLSPOptions(
+        swiftPM: nil,
+        fallbackBuildSystem: nil,
+        compilationDatabase: nil,
+        index: nil,
+        logging: nil,
+        sourcekitd: nil
+      )
     )
   }
 

Tests/SwiftSourceKitPluginTests/SwiftSourceKitPluginTests.swift

@@ -20,23 +20,6 @@ import ToolchainRegistry
 import XCTest
 
 final class SwiftSourceKitPluginTests: XCTestCase {
-  /// Returns the paths from which the SourceKit plugins should be loaded or throws an error if the plugins cannot be
-  /// found.
-  private var pluginPaths: PluginPaths {
-    get throws {
-      struct PluginLoadingError: Error, CustomStringConvertible {
-        var description: String = "Could not find SourceKit plugin"
-      }
-
-      guard let sourceKitPluginPaths else {
-        // If we couldn't find the plugins, keep `didLoadPlugin = false`, which will throw an error in each test case's
-        // `setUp` function. We can't throw an error from the class `setUp` function.
-        throw PluginLoadingError()
-      }
-      return sourceKitPluginPaths
-    }
-  }
-
   /// Returns a path to a file name that is unique to this test execution.
   ///
   /// The file does not actually exist on disk.
@@ -51,7 +34,10 @@ final class SwiftSourceKitPluginTests: XCTestCase {
       }
       throw NoSourceKitdFound()
     }
-    return try await DynamicallyLoadedSourceKitD.getOrCreate(dylibPath: sourcekitd, pluginPaths: try self.pluginPaths)
+    return try await DynamicallyLoadedSourceKitD.getOrCreate(
+      dylibPath: sourcekitd,
+      pluginPaths: try sourceKitPluginPaths
+    )
   }
 
   func testBasicCompletion() async throws {