Restart sourcekitd and clangd after they have crashed

Author: ahoppen

Sources/LanguageServerProtocol/Error.swift

@@ -35,6 +35,9 @@ public struct ErrorCode: RawRepresentable, Codable, Hashable {
 
   // LSP
   public static let cancelled: ErrorCode = ErrorCode(rawValue: -32800)
+
+  // sourcekitd
+  public static let connectionInterrupted = ErrorCode(rawValue: -13000)
 }
 
 /// An error response represented by a code and message.
@@ -63,6 +66,10 @@ extension ResponseError {
   public static func unknown(_ message: String) -> ResponseError {
     return ResponseError(code: .unknownErrorCode, message: message)
   }
+
+  public static func connectionInterrupted(_ message: String) -> ResponseError {
+    return ResponseError(code: .connectionInterrupted, message: message)
+  }
 }
 
 /// An error during message decoding.

Sources/SKTestSupport/SKTibsTestWorkspace.swift

@@ -200,4 +200,5 @@ extension TibsToolchain {
 
 extension TestLocation {
   public var docIdentifier: TextDocumentIdentifier { TextDocumentIdentifier(url) }
+  public var docUri: DocumentURI { DocumentURI(url) }
 }

Sources/SourceKit/SourceKitServer.swift

@@ -185,6 +185,36 @@ public final class SourceKitServer: LanguageServer {
     return nil
   }
 
+  func reopenDocuments(for languages: Set<Language>) {
+    guard let workspace = workspace else {
+      return
+    }
+
+    for documentUri in workspace.documentManager.openDocuments {
+      guard let snapshot = workspace.documentManager.latestSnapshot(documentUri) else {
+        // The document has been closed since we retrieved its URI. We don't care about it anymore.
+        continue
+      }
+      guard languages.contains(snapshot.document.language) else {
+        // The document is a different language
+        continue
+      }
+
+      do {
+        // Close the document in the document manager so we can reopen it. If the document has been
+        // closed since, the try will fail and we won't try to reopen it.
+        try workspace.documentManager.close(documentUri)
+        let textDocument = TextDocumentItem(uri: documentUri,
+                                            language: snapshot.document.language,
+                                            version: snapshot.version,
+                                            text: snapshot.text)
+        openDocument(DidOpenTextDocumentNotification(textDocument: textDocument), workspace: workspace)
+      } catch {
+        // The document was no longer open. Ignore it
+      }
+    }
+  }
+
   func languageService(for toolchain: Toolchain, _ language: Language) -> ToolchainLanguageServer? {
     let key = LanguageServiceKey(toolchain: toolchain.identifier, language: language)
     if let service = languageService[key] {
@@ -193,7 +223,16 @@ public final class SourceKitServer: LanguageServer {
 
     // Start a new service.
     return orLog("failed to start language service", level: .error) {
-      guard let service = try SourceKit.languageService(for: toolchain, language, options: options, client: self) else {
+      let reopenDocuments = {
+        switch language {
+        case .c, .cpp, .objective_c, .objective_cpp:
+          // Re-open all C-based languages if clangd crashed
+          self.reopenDocuments(for: [.c, .cpp, .objective_c, .objective_cpp])
+        default:
+          self.reopenDocuments(for: [language])
+        }
+      }
+      guard let service = try SourceKit.languageService(for: toolchain, language, options: options, client: self, reopenDocuments: reopenDocuments) else {
         return nil
       }
 
@@ -220,7 +259,8 @@ public final class SourceKitServer: LanguageServer {
     }
   }
 
-  func languageService(for uri: DocumentURI, _ language: Language, in workspace: Workspace) -> ToolchainLanguageServer? {
+  /// **Public for testing purposes only**
+  public func languageService(for uri: DocumentURI, _ language: Language, in workspace: Workspace) -> ToolchainLanguageServer? {
     if let service = workspace.documentService[uri] {
       return service
     }
@@ -418,13 +458,17 @@ extension SourceKitServer {
   // MARK: - Text synchronization
 
   func openDocument(_ note: Notification<DidOpenTextDocumentNotification>, workspace: Workspace) {
-    workspace.documentManager.open(note.params)
+    openDocument(note.params, workspace: workspace)
+  }
+
+  private func openDocument(_ note: DidOpenTextDocumentNotification, workspace: Workspace) {
+    workspace.documentManager.open(note)
 
-    let textDocument = note.params.textDocument
+    let textDocument = note.textDocument
     workspace.buildSettings.registerForChangeNotifications(for: textDocument.uri)
 
     if let service = languageService(for: textDocument.uri, textDocument.language, in: workspace) {
-      service.openDocument(note.params)
+      service.openDocument(note)
     }
   }
 
@@ -762,17 +806,18 @@ public func languageService(
   for toolchain: Toolchain,
   _ language: Language,
   options: SourceKitServer.Options,
-  client: MessageHandler) throws -> ToolchainLanguageServer?
+  client: MessageHandler,
+  reopenDocuments: @escaping () -> Void) throws -> ToolchainLanguageServer?
 {
   switch language {
 
   case .c, .cpp, .objective_c, .objective_cpp:
     guard toolchain.clangd != nil else { return nil }
-    return try makeJSONRPCClangServer(client: client, toolchain: toolchain, buildSettings: (client as? SourceKitServer)?.workspace?.buildSettings, clangdOptions: options.clangdOptions)
+    return try makeJSONRPCClangServer(client: client, toolchain: toolchain, buildSettings: (client as? SourceKitServer)?.workspace?.buildSettings, clangdOptions: options.clangdOptions, reopenDocuments: reopenDocuments)
 
   case .swift:
     guard let sourcekitd = toolchain.sourcekitd else { return nil }
-    return try makeLocalSwiftServer(client: client, sourcekitd: sourcekitd, buildSettings: (client as? SourceKitServer)?.workspace?.buildSettings, clientCapabilities: (client as? SourceKitServer)?.workspace?.clientCapabilities)
+    return try makeLocalSwiftServer(client: client, sourcekitd: sourcekitd, buildSettings: (client as? SourceKitServer)?.workspace?.buildSettings, clientCapabilities: (client as? SourceKitServer)?.workspace?.clientCapabilities, reopenDocuments: reopenDocuments)
 
   default:
     return nil

Sources/SourceKit/ToolchainLanguageServer.swift

@@ -13,6 +13,18 @@
 import Foundation
 import LanguageServerProtocol
 
+/// The state of a `ToolchainLanguageServer`
+public enum LanguageServerState {
+  /// The language server is running with semantic functionality enabled
+  case connected
+  /// The language server server has crashed and we are waiting for it to relaunch
+  case connectionInterrupted
+  /// The language server has relaunched but semantic functionality is currently disabled
+  case semanticFunctionalityDisabled
+  /// The langauge server has been shut down gracefully
+  case shutDown
+}
+
 /// A `LanguageServer` that exists within the context of the current process.
 public protocol ToolchainLanguageServer: AnyObject {
 
@@ -21,6 +33,9 @@ public protocol ToolchainLanguageServer: AnyObject {
   func initializeSync(_ initialize: InitializeRequest) throws -> InitializeResult
   func clientInitialized(_ initialized: InitializedNotification)
 
+  /// Add a handler that is called whenever the state of the language server changes.
+  func addStateChangeHandler(handler: @escaping (_ oldState: LanguageServerState, _ newState: LanguageServerState) -> Void)
+
   // MARK: - Text synchronization
 
   func openDocument(_ note: DidOpenTextDocumentNotification)

Sources/SourceKit/clangd/ClangLanguageServer.swift

@@ -17,29 +17,136 @@ import LSPLogging
 import SKCore
 import SKSupport
 import TSCBasic
+import Dispatch
 
 /// A thin wrapper over a connection to a clangd server providing build setting handling.
 final class ClangLanguageServerShim: ToolchainLanguageServer {
 
   /// The server's request queue, used to serialize requests and responses to `clangd`.
   public let queue: DispatchQueue = DispatchQueue(label: "clangd-language-server-queue", qos: .userInitiated)
 
-  let clangd: Connection
+  /// The connection to `clangd`. `nil` before `initialize` has been called.
+  private var clangd: Connection!
 
-  var capabilities: ServerCapabilities? = nil
+  private var capabilities: ServerCapabilities? = nil
 
-  let buildSystem: BuildSystem
+  private let buildSystem: BuildSystem
 
-  let clang: AbsolutePath?
+  private let clangdPath: AbsolutePath
 
-  /// Creates a language server for the given client using the sourcekitd dylib at the specified path.
-  public init(client: Connection, clangd: Connection, buildSystem: BuildSystem,
-              clang: AbsolutePath?) throws {
-    self.clangd = clangd
-    self.buildSystem = buildSystem
-    self.clang = clang
+  private let clangdOptions: [String]
+
+  private let client: MessageHandler
+
+  private var state: LanguageServerState {
+     didSet {
+       for handler in stateChangeHandlers {
+         handler(oldValue, state)
+       }
+     }
+   }
+
+  private var stateChangeHandlers: [(_ oldState: LanguageServerState, _ newState: LanguageServerState) -> Void] = []
+
+  /// Ask the parent of this language server to re-open all documents in this language server.
+  private let reopenDocuments: () -> Void
+
+  /// The `InitializeRequest` with which `clangd` was originally initialized.
+  /// Stored so we can replay the initialization when clangd crashes.
+  private var initializeRequest: InitializeRequest?
+
+  public init(client: MessageHandler,
+              clangdPath: AbsolutePath,
+              buildSettings: BuildSystem?,
+              clangdOptions: [String],
+              reopenDocuments: @escaping () -> Void
+  ) throws {
+    self.client = client
+    self.buildSystem = buildSettings ?? BuildSystemList()
+    self.clangdPath = clangdPath
+    self.clangdOptions = clangdOptions
+    self.state = .connected
+    self.reopenDocuments = reopenDocuments
+  }
+
+  func initialize() throws {
+    let clientToServer: Pipe = Pipe()
+    let serverToClient: Pipe = Pipe()
+
+    let connection = JSONRPCConnection(
+      protocol: MessageRegistry.lspProtocol,
+      inFD: serverToClient.fileHandleForReading.fileDescriptor,
+      outFD: clientToServer.fileHandleForWriting.fileDescriptor
+    )
+
+    self.clangd = connection
+
+    connection.start(receiveHandler: client)
+
+    let process = Foundation.Process()
+
+    if #available(OSX 10.13, *) {
+      process.executableURL = clangdPath.asURL
+    } else {
+      process.launchPath = clangdPath.pathString
+    }
+
+    process.arguments = [
+      "-compile_args_from=lsp",   // Provide compiler args programmatically.
+      "-background-index=false",  // Disable clangd indexing, we use the build
+      "-index=false",             // system index store instead.
+    ] + clangdOptions
+
+    process.standardOutput = serverToClient
+    process.standardInput = clientToServer
+    process.terminationHandler = { [weak self] process in
+      log("clangd exited: \(process.terminationReason) \(process.terminationStatus)")
+      connection.close()
+      if process.terminationStatus != 0 {
+        if let self = self {
+          self.state = .connectionInterrupted
+          self.restartClangd()
+        }
+      }
+    }
+
+    if #available(OSX 10.13, *) {
+      try process.run()
+    } else {
+      process.launch()
+    }
+  }
+
+  private func restartClangd() {
+    precondition(self.state == .connectionInterrupted)
+
+    guard let initializeRequest = initializeRequest else {
+      log("clangd crashed before it was sent an InitializeRequest. *Not* trying to restart.", level: .error)
+      return
+    }
+
+    do {
+      try self.initialize()
+      // FIXME: We assume that clangd will return the same capabilites after restarting.
+      // Theoretically they could have changed and we would need to inform SourceKitServer about them.
+      // But since SourceKitServer more or less ignores them right now anyway, this should be fine for now.
+      _ = try self.initializeSync(initializeRequest)
+      self.clientInitialized(InitializedNotification())
+      self.reopenDocuments()
+      self.state = .connected
+    } catch {
+      log("Failed to restart clangd after a crash, retrying in 5 seconds: \(error)", level: .error)
+      DispatchQueue.main.asyncAfter(deadline: .now() + .seconds(5)) { [weak self] in
+        self?.restartClangd()
+      }
+    }
   }
 
+  func addStateChangeHandler(handler: @escaping (LanguageServerState, LanguageServerState) -> Void) {
+    stateChangeHandlers.append(handler)
+  }
+
+
   /// Forwards a request to the given connection, taking care of replying to the original request
   /// and cancellation, while providing a callback with the response for additional processing.
   ///
@@ -73,6 +180,7 @@ final class ClangLanguageServerShim: ToolchainLanguageServer {
 extension ClangLanguageServerShim {
 
   func initializeSync(_ initialize: InitializeRequest) throws -> InitializeResult {
+    self.initializeRequest = initialize
     let result = try clangd.sendSync(initialize)
     self.capabilities = result.capabilities
     return result
@@ -124,7 +232,7 @@ extension ClangLanguageServerShim {
     if let settings = settings {
       clangd.send(DidChangeConfigurationNotification(settings: .clangd(
         ClangWorkspaceSettings(
-          compilationDatabaseChanges: [url.path: ClangCompileCommand(settings, clang: clang)]))))
+          compilationDatabaseChanges: [url.path: ClangCompileCommand(settings, clang: clangdPath)]))))
     }
   }
 
@@ -191,60 +299,16 @@ func makeJSONRPCClangServer(
   client: MessageHandler,
   toolchain: Toolchain,
   buildSettings: BuildSystem?,
-  clangdOptions: [String]
+  clangdOptions: [String],
+  reopenDocuments: @escaping () -> Void
 ) throws -> ToolchainLanguageServer {
   guard let clangd = toolchain.clangd else {
     preconditionFailure("missing clang from toolchain \(toolchain.identifier)")
   }
 
-  let clientToServer: Pipe = Pipe()
-  let serverToClient: Pipe = Pipe()
-
-  let connection = JSONRPCConnection(
-    protocol: MessageRegistry.lspProtocol,
-    inFD: serverToClient.fileHandleForReading.fileDescriptor,
-    outFD: clientToServer.fileHandleForWriting.fileDescriptor
-  )
-
-  let connectionToClient = LocalConnection()
-
-  let shim = try ClangLanguageServerShim(
-    client: connectionToClient,
-    clangd: connection,
-    buildSystem: buildSettings ?? BuildSystemList(),
-    clang: toolchain.clang)
-
-  connectionToClient.start(handler: client)
-  connection.start(receiveHandler: client)
-
-  let process = Foundation.Process()
-
-  if #available(OSX 10.13, *) {
-    process.executableURL = clangd.asURL
-  } else {
-    process.launchPath = clangd.pathString
-  }
-
-  process.arguments = [
-    "-compile_args_from=lsp",   // Provide compiler args programmatically.
-    "-background-index=false",  // Disable clangd indexing, we use the build
-    "-index=false",             // system index store instead.
-  ] + clangdOptions
-
-  process.standardOutput = serverToClient
-  process.standardInput = clientToServer
-  process.terminationHandler = { process in
-    log("clangd exited: \(process.terminationReason) \(process.terminationStatus)")
-    connection.close()
-  }
-
-  if #available(OSX 10.13, *) {
-    try process.run()
-  } else {
-    process.launch()
-  }
-
-  return shim
+  let server = try ClangLanguageServerShim(client: client, clangdPath: clangd, buildSettings: buildSettings, clangdOptions: clangdOptions, reopenDocuments: reopenDocuments)
+  try server.initialize()
+  return server
 }
 
 extension ClangCompileCommand {