Merge main into release/6.0

.gitignore

@@ -3,6 +3,7 @@ default.profraw
 Package.resolved
 /.build
 /.index-build
+/.linux-build
 /Packages
 /*.xcodeproj
 /*.sublime-project

Documentation/Files_To_Reindex.md

@@ -0,0 +1,35 @@
+# Which files to re-index when a file is modified
+
+## `.swift`
+
+Obviously affects the file itself, which we do re-index.
+
+If an internal declaration was changed, all files within the module might be also affected. If a public declaration was changed, all modules that depend on it might be affected. The effect can only really be in three ways:
+1. It might change overload resolution in another file, which is fairly unlikely
+2. A new declaration is introduced in this file that is already referenced by another file
+3. A declaration is removed in this file that was referenced from other files. In those cases the other files now have an invalid reference.
+
+We decided to not re-index any files other than the file itself because naively re-indexing all files that might depend on the modified file requires too much processing power that will likely have no or very little effect on the index – we are trading accuracy for CPU time here.
+We mark the targets of the changed file as well as any dependent targets as out-of-date. The assumption is that most likely the user will go back to any of the affected files shortly afterwards and modify them again. When that happens, the affected file will get re-indexed and bring the index back up to date.
+
+Alternatives would be:
+- We could we check the file’s interface hash and re-index other files based on whether it has changed. But at that point we are somewhat implementing a build system. And even if a new public method was introduced it’s very likely that the user hasn’t actually used it anywhere yet, which means that re-indexing all dependent modules would still be doing unnecessary work.
+- To cover case (2) from above, we could re-index only dependencies that previously indexed with errors. This is an alternative that hasn’t been fully explored.
+
+## `.h`
+
+All files that include the header (including via other headers) might be affected by the change, similar to how all `.swift` files that import a module might be affected. Similar to modules, we choose to not re-index all files that include the header because of the same considerations mentioned above.
+
+To re-index the header, we pick one main file that includes the header and re-index that, which will effectively update the index for the header. For existing headers, we know which files import a header from the existing index. For new header files, we assume that it hasn’t been included in any file yet and thus don't index it. If the user wrote an include to the new header before creating the header itself, we don't know about that include from the existing index. But it’s likely that the user will modify the file that includes the new header file shortly after, which will index the header and establish the header to main file connection.
+
+## `.c` / `.cpp` / `.m`
+
+This is the easy case since only the file itself is affected.
+
+## Compiler settings (`compile_commands.json` / `Package.swift`)
+
+Ideally, we would like to consider a file as changed when its compile commands have changed, if they changed in a meaningful way (ie. in a way that would also trigger re-compilation in an incremental build). Non-meaningful changes would be:
+- If compiler arguments that aren't order dependent are shuffled around. We could have a really quick check for compiler arguments equality by comparing them unordered. Any real compiler argument change will most likely do more than rearranging the arguments.
+- The addition of a new Swift file to a target is equivalent to that file being modified and shouldn’t trigger a re-index of the entire target.
+
+At the moment, unit files don’t include information about the compiler arguments with which they were created, so it’s impossible to know whether the compiler arguments have changed when a project is opened. Thus, for now, we don’t re-index any files on compiler settings changing.

Editors/README.md

@@ -22,58 +22,7 @@ The [Swift for Visual Studio Code extension](https://marketplace.visualstudio.co
 
 ## Sublime Text
 
-Before using SourceKit-LSP with Sublime Text, you will need to install the LSP package from Package Control. To configure SourceKit-LSP, open the LSP package's settings. The following snippet should be enough to get started with Swift.
-
-You will need the path to the `sourcekit-lsp` executable for the "command" section.
-
-```json
-{
-  "clients":
-  {
-    "SourceKit-LSP":
-    {
-      "enabled": true,
-      "command": [
-        "<sourcekit-lsp command>"
-      ],
-      "env": {
-        // To override the toolchain, uncomment the following:
-        // "SOURCEKIT_TOOLCHAIN_PATH": "<path to toolchain>",
-      },
-      "languages": [
-        {
-          "scopes": ["source.swift"],
-          "syntaxes": [
-            "Packages/Swift/Syntaxes/Swift.tmLanguage",
-            "Packages/Decent Swift Syntax/Swift.sublime-syntax",
-          ],
-          "languageId": "swift"
-        },
-        {
-          "scopes": ["source.c"],
-          "syntaxes": ["Packages/C++/C.sublime-syntax"],
-          "languageId": "c"
-        },
-        {
-          "scopes": ["source.c++"],
-          "syntaxes": ["Packages/C++/C++.sublime-syntax"],
-          "languageId": "cpp"
-        },
-        {
-          "scopes": ["source.objc"],
-          "syntaxes": ["Packages/Objective-C/Objective-C.sublime-syntax"],
-          "languageId": "objective-c"
-        },
-        {
-          "scopes": ["source.objc++"],
-          "syntaxes": ["Packages/Objective-C/Objective-C++.sublime-syntax"],
-          "languageId": "objective-cpp"
-        },
-      ]
-    }
-  }
-}
-```
+Before using SourceKit-LSP with Sublime Text, you will need to install the [LSP](https://packagecontrol.io/packages/LSP), [LSP-SourceKit](https://github.com/sublimelsp/LSP-SourceKit) and [Swift-Next](https://github.com/Swift-Next/Swift-Next) packages from Package Control. Then toggle the server on by typing in command palette `LSP: Enable Language Server Globally` or `LSP: Enable Language Server in Project`.
 
 ## Emacs
 

Package.swift

@@ -182,6 +182,13 @@ let package = Package(
       exclude: ["CMakeLists.txt"]
     ),
 
+    .testTarget(
+      name: "SemanticIndexTests",
+      dependencies: [
+        "SemanticIndex"
+      ]
+    ),
+
     // MARK: SKCore
     // Data structures and algorithms useful across the project, but not necessarily
     // suitable for use in other packages.

Sources/LSPLogging/NonDarwinLogging.swift

@@ -195,6 +195,17 @@ public struct NonDarwinLogInterpolation: StringInterpolationProtocol, Sendable {
     append(description: message.description, redactedDescription: message.redactedDescription, privacy: privacy)
   }
 
+  public mutating func appendInterpolation(
+    _ message: (some CustomLogStringConvertibleWrapper & Sendable)?,
+    privacy: NonDarwinLogPrivacy = .private
+  ) {
+    if let message {
+      self.appendInterpolation(message, privacy: privacy)
+    } else {
+      self.appendLiteral("<nil>")
+    }
+  }
+
   public mutating func appendInterpolation(_ type: Any.Type, privacy: NonDarwinLogPrivacy = .public) {
     append(description: String(reflecting: type), redactedDescription: "<private>", privacy: privacy)
   }

Sources/LSPTestSupport/LocalConnection.swift

@@ -0,0 +1,133 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import Dispatch
+import LSPLogging
+import LanguageServerProtocol
+
+/// A connection between two message handlers in the same process.
+///
+/// You must call `start(handler:)` before sending any messages, and must call `close()` when finished to avoid a memory leak.
+///
+/// ```
+/// let client: MessageHandler = ...
+/// let server: MessageHandler = ...
+/// let conn = LocalConnection()
+/// conn.start(handler: server)
+/// conn.send(...) // handled by server
+/// conn.close()
+/// ```
+///
+/// - Note: Unchecked sendable conformance because shared state is guarded by `queue`.
+public final class LocalConnection: Connection, @unchecked Sendable {
+  enum State {
+    case ready, started, closed
+  }
+
+  /// A name of the endpoint for this connection, used for logging, e.g. `clangd`.
+  private let name: String
+
+  /// The queue guarding `_nextRequestID`.
+  let queue: DispatchQueue = DispatchQueue(label: "local-connection-queue")
+
+  var _nextRequestID: Int = 0
+
+  var state: State = .ready
+
+  var handler: MessageHandler? = nil
+
+  public init(name: String) {
+    self.name = name
+  }
+
+  deinit {
+    if state != .closed {
+      close()
+    }
+  }
+
+  public func start(handler: MessageHandler) {
+    precondition(state == .ready)
+    state = .started
+    self.handler = handler
+  }
+
+  public func close() {
+    precondition(state != .closed)
+    handler = nil
+    state = .closed
+  }
+
+  func nextRequestID() -> RequestID {
+    return queue.sync {
+      _nextRequestID += 1
+      return .number(_nextRequestID)
+    }
+  }
+
+  public func send<Notification: NotificationType>(_ notification: Notification) {
+    logger.info(
+      """
+      Sending notification to \(self.name, privacy: .public)
+      \(notification.forLogging)
+      """
+    )
+    self.handler?.handle(notification)
+  }
+
+  public func send<Request: RequestType>(
+    _ request: Request,
+    reply: @Sendable @escaping (LSPResult<Request.Response>) -> Void
+  ) -> RequestID {
+    let id = nextRequestID()
+
+    logger.info(
+      """
+      Sending request to \(self.name, privacy: .public) (id: \(id, privacy: .public)):
+      \(request.forLogging)
+      """
+    )
+
+    guard let handler = self.handler else {
+      logger.info(
+        """
+        Replying to request \(id, privacy: .public) with .serverCancelled because no handler is specified in \(self.name, privacy: .public)
+        """
+      )
+      reply(.failure(.serverCancelled))
+      return id
+    }
+
+    precondition(self.state == .started)
+    handler.handle(request, id: id) { result in
+      switch result {
+      case .success(let response):
+        logger.info(
+          """
+          Received reply for request \(id, privacy: .public) from \(self.name, privacy: .public)
+          \(response.forLogging)
+          """
+        )
+      case .failure(let error):
+        logger.error(
+          """
+          Received error for request \(id, privacy: .public) from \(self.name, privacy: .public)
+          \(error.forLogging)
+          """
+        )
+      }
+      reply(result)
+    }
+
+    return id
+  }
+}

Sources/LSPTestSupport/TestJSONRPCConnection.swift

@@ -17,7 +17,7 @@ import XCTest
 
 import class Foundation.Pipe
 
-public final class TestJSONRPCConnection {
+public final class TestJSONRPCConnection: Sendable {
   public let clientToServer: Pipe = Pipe()
   public let serverToClient: Pipe = Pipe()
 
@@ -76,9 +76,9 @@ public final class TestJSONRPCConnection {
 
 public struct TestLocalConnection {
   public let client: TestClient
-  public let clientConnection: LocalConnection = LocalConnection()
+  public let clientConnection: LocalConnection = LocalConnection(name: "Test")
   public let server: TestServer
-  public let serverConnection: LocalConnection = LocalConnection()
+  public let serverConnection: LocalConnection = LocalConnection(name: "Test")
 
   public init(allowUnexpectedNotification: Bool = true) {
     client = TestClient(connectionToServer: serverConnection, allowUnexpectedNotification: allowUnexpectedNotification)
@@ -151,7 +151,7 @@ public actor TestClient: MessageHandler {
   /// Send a request to the LSP server and (asynchronously) receive a reply.
   public nonisolated func send<Request: RequestType>(
     _ request: Request,
-    reply: @escaping (LSPResult<Request.Response>) -> Void
+    reply: @Sendable @escaping (LSPResult<Request.Response>) -> Void
   ) -> RequestID {
     return connectionToServer.send(request, reply: reply)
   }

Sources/LanguageServerProtocol/Connection.swift

@@ -10,8 +10,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-import Dispatch
-
 /// An abstract connection, allow messages to be sent to a (potentially remote) `MessageHandler`.
 public protocol Connection: AnyObject, Sendable {
 
@@ -47,83 +45,3 @@ public protocol MessageHandler: AnyObject, Sendable {
     reply: @Sendable @escaping (LSPResult<Request.Response>) -> Void
   )
 }
-
-/// A connection between two message handlers in the same process.
-///
-/// You must call `start(handler:)` before sending any messages, and must call `close()` when finished to avoid a memory leak.
-///
-/// ```
-/// let client: MessageHandler = ...
-/// let server: MessageHandler = ...
-/// let conn = LocalConnection()
-/// conn.start(handler: server)
-/// conn.send(...) // handled by server
-/// conn.close()
-/// ```
-///
-/// - Note: Unchecked sendable conformance because shared state is guarded by `queue`.
-public final class LocalConnection: Connection, @unchecked Sendable {
-
-  enum State {
-    case ready, started, closed
-  }
-
-  /// The queue guarding `_nextRequestID`.
-  let queue: DispatchQueue = DispatchQueue(label: "local-connection-queue")
-
-  var _nextRequestID: Int = 0
-
-  var state: State = .ready
-
-  var handler: MessageHandler? = nil
-
-  public init() {}
-
-  deinit {
-    if state != .closed {
-      close()
-    }
-  }
-
-  public func start(handler: MessageHandler) {
-    precondition(state == .ready)
-    state = .started
-    self.handler = handler
-  }
-
-  public func close() {
-    precondition(state != .closed)
-    handler = nil
-    state = .closed
-  }
-
-  func nextRequestID() -> RequestID {
-    return queue.sync {
-      _nextRequestID += 1
-      return .number(_nextRequestID)
-    }
-  }
-
-  public func send<Notification>(_ notification: Notification) where Notification: NotificationType {
-    self.handler?.handle(notification)
-  }
-
-  public func send<Request: RequestType>(
-    _ request: Request,
-    reply: @Sendable @escaping (LSPResult<Request.Response>) -> Void
-  ) -> RequestID {
-    let id = nextRequestID()
-
-    guard let handler = self.handler else {
-      reply(.failure(.serverCancelled))
-      return id
-    }
-
-    precondition(self.state == .started)
-    handler.handle(request, id: id) { result in
-      reply(result)
-    }
-
-    return id
-  }
-}