Restart sourcekitd and clangd after they have crashed

Documentation/Development.md

@@ -175,6 +175,10 @@ Location(ws.testLoc("aaa:call"))
 Position(ws.testLoc("aaa:call"))
 ```
 
+### Long tests
+
+Tests that run longer than approx. 1 second are only executed if the the `SOURCEKIT_LSP_ENABLE_LONG_TESTS` environment variable is set to `YES` or `1`. This, in particular, includes the crash recovery tests.
+
 ## Tibs
 
 We use Tibs, the "Test Index Build System" from the IndexStoreDB project to provide build system support for test projects, including getting compiler arguments and building an index. 

Sources/LanguageServerProtocol/Error.swift

@@ -35,6 +35,9 @@ public struct ErrorCode: RawRepresentable, Codable, Hashable {
 
   // LSP
   public static let cancelled: ErrorCode = ErrorCode(rawValue: -32800)
+
+  // sourcekitd
+  public static let connectionInterrupted = ErrorCode(rawValue: -13000)
 }
 
 /// An error response represented by a code and message.
@@ -63,6 +66,10 @@ extension ResponseError {
   public static func unknown(_ message: String) -> ResponseError {
     return ResponseError(code: .unknownErrorCode, message: message)
   }
+
+  public static func connectionInterrupted(_ message: String) -> ResponseError {
+    return ResponseError(code: .connectionInterrupted, message: message)
+  }
 }
 
 /// An error during message decoding.

Sources/SKTestSupport/LongTestsEnabled.swift

@@ -0,0 +1,20 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+import Foundation
+
+public var longTestsEnabled: Bool {
+  if let value = ProcessInfo.processInfo.environment["SOURCEKIT_LSP_ENABLE_LONG_TESTS"] {
+    return value == "1" || value == "YES"
+  }
+  return false
+}

Sources/SKTestSupport/SKTibsTestWorkspace.swift

@@ -216,4 +216,5 @@ extension TibsToolchain {
 
 extension TestLocation {
   public var docIdentifier: TextDocumentIdentifier { TextDocumentIdentifier(url) }
+  public var docUri: DocumentURI { DocumentURI(url) }
 }

Sources/SourceKit/SourceKitServer.swift

@@ -181,6 +181,36 @@ public final class SourceKitServer: LanguageServer {
     return nil
   }
 
+  func reopenDocuments(for languageService: ToolchainLanguageServer) {
+    queue.async {
+      guard let workspace = self.workspace else {
+        return
+      }
+
+      for documentUri in workspace.documentManager.openDocuments where workspace.documentService[documentUri] === languageService {
+        guard let snapshot = workspace.documentManager.latestSnapshot(documentUri) else {
+          // The document has been closed since we retrieved its URI. We don't care about it anymore.
+          continue
+        }
+
+        do {
+          // Close the document in the document manager so we can reopen it. If the document has been
+          // closed since, the try will fail and we won't try to reopen it.
+          try workspace.documentManager.close(documentUri)
+          let textDocument = TextDocumentItem(uri: documentUri,
+                                              language: snapshot.document.language,
+                                              version: snapshot.version,
+                                              text: snapshot.text)
+          // Re-open the document but don't register for build system notifications again since
+          // the document is still open in the build system server from before the sourcektid crash.
+          self.openDocument(DidOpenTextDocumentNotification(textDocument: textDocument), workspace: workspace, registerForBuildSystemNotifications: false)
+        } catch {
+          // The document was no longer open. Ignore it
+        }
+      }
+    }
+  }
+
   func languageService(for toolchain: Toolchain, _ language: Language) -> ToolchainLanguageServer? {
     let key = LanguageServiceKey(toolchain: toolchain.identifier, language: language)
     if let service = languageService[key] {
@@ -189,7 +219,7 @@ public final class SourceKitServer: LanguageServer {
 
     // Start a new service.
     return orLog("failed to start language service", level: .error) {
-      guard let service = try SourceKit.languageService(for: toolchain, language, options: options, client: self) else {
+      guard let service = try SourceKit.languageService(for: toolchain, language, options: options, client: self, reopenDocuments: reopenDocuments) else {
         return nil
       }
 
@@ -216,7 +246,8 @@ public final class SourceKitServer: LanguageServer {
     }
   }
 
-  func languageService(for uri: DocumentURI, _ language: Language, in workspace: Workspace) -> ToolchainLanguageServer? {
+  /// **Public for testing purposes only**
+  public func languageService(for uri: DocumentURI, _ language: Language, in workspace: Workspace) -> ToolchainLanguageServer? {
     if let service = workspace.documentService[uri] {
       return service
     }
@@ -424,14 +455,21 @@ extension SourceKitServer {
   // MARK: - Text synchronization
 
   func openDocument(_ note: Notification<DidOpenTextDocumentNotification>, workspace: Workspace) {
-    workspace.documentManager.open(note.params)
+    openDocument(note.params, workspace: workspace)
+  }
+
+  private func openDocument(_ note: DidOpenTextDocumentNotification, workspace: Workspace, registerForBuildSystemNotifications: Bool = true) {
+    workspace.documentManager.open(note)
 
-    let textDocument = note.params.textDocument
-    workspace.buildSettings.registerForChangeNotifications(
-      for: textDocument.uri, language: textDocument.language)
+    let textDocument = note.textDocument
+
+    if registerForBuildSystemNotifications {
+      workspace.buildSettings.registerForChangeNotifications(
+        for: textDocument.uri, language: textDocument.language)
+    }
 
     if let service = languageService(for: textDocument.uri, textDocument.language, in: workspace) {
-      service.openDocument(note.params)
+      service.openDocument(note)
     }
   }
 
@@ -771,17 +809,18 @@ public func languageService(
   for toolchain: Toolchain,
   _ language: Language,
   options: SourceKitServer.Options,
-  client: MessageHandler) throws -> ToolchainLanguageServer?
+  client: MessageHandler,
+  reopenDocuments: @escaping (ToolchainLanguageServer) -> Void) throws -> ToolchainLanguageServer?
 {
   switch language {
 
   case .c, .cpp, .objective_c, .objective_cpp:
     guard toolchain.clangd != nil else { return nil }
-    return try makeJSONRPCClangServer(client: client, toolchain: toolchain, buildSettings: (client as? SourceKitServer)?.workspace?.buildSettings, clangdOptions: options.clangdOptions)
+    return try makeJSONRPCClangServer(client: client, toolchain: toolchain, buildSettings: (client as? SourceKitServer)?.workspace?.buildSettings, clangdOptions: options.clangdOptions, reopenDocuments: reopenDocuments)
 
   case .swift:
     guard let sourcekitd = toolchain.sourcekitd else { return nil }
-    return try makeLocalSwiftServer(client: client, sourcekitd: sourcekitd, buildSettings: (client as? SourceKitServer)?.workspace?.buildSettings, clientCapabilities: (client as? SourceKitServer)?.workspace?.clientCapabilities)
+    return try makeLocalSwiftServer(client: client, sourcekitd: sourcekitd, buildSettings: (client as? SourceKitServer)?.workspace?.buildSettings, clientCapabilities: (client as? SourceKitServer)?.workspace?.clientCapabilities, reopenDocuments: reopenDocuments)
 
   default:
     return nil

Sources/SourceKit/ToolchainLanguageServer.swift

@@ -13,6 +13,18 @@
 import Foundation
 import LanguageServerProtocol
 
+/// The state of a `ToolchainLanguageServer`
+public enum LanguageServerState {
+  /// The language server is running with semantic functionality enabled
+  case connected
+  /// The language server server has crashed and we are waiting for it to relaunch
+  case connectionInterrupted
+  /// The language server has relaunched but semantic functionality is currently disabled
+  case semanticFunctionalityDisabled
+  /// The langauge server has been shut down gracefully
+  case shutDown
+}
+
 /// A `LanguageServer` that exists within the context of the current process.
 public protocol ToolchainLanguageServer: AnyObject {
 
@@ -21,6 +33,9 @@ public protocol ToolchainLanguageServer: AnyObject {
   func initializeSync(_ initialize: InitializeRequest) throws -> InitializeResult
   func clientInitialized(_ initialized: InitializedNotification)
 
+  /// Add a handler that is called whenever the state of the language server changes.
+  func addStateChangeHandler(handler: @escaping (_ oldState: LanguageServerState, _ newState: LanguageServerState) -> Void)
+
   // MARK: - Text synchronization
 
   func openDocument(_ note: DidOpenTextDocumentNotification)