Delay restarting cland in case of a crash loop

Author: ahoppen

Sources/SourceKit/clangd/ClangLanguageServer.swift

@@ -44,6 +44,12 @@ final class ClangLanguageServerShim: ToolchainLanguageServer {
        }
      }
    }
+  
+  /// The date at which `clangd` was last restarted. Used to delay restarting in case of a crash loop.
+  private var lastClangdRestart: Date?
+  
+  /// Whether or not a restart of `clangd` has been scheduled. Used to make sure we are not restarting `clangd` twice.
+  private var clangRestartScheduled = false
 
   private var stateChangeHandlers: [(_ oldState: LanguageServerState, _ newState: LanguageServerState) -> Void] = []
 
@@ -118,23 +124,39 @@ final class ClangLanguageServerShim: ToolchainLanguageServer {
 
   private func restartClangd() {
     precondition(self.state == .connectionInterrupted)
+    
+    precondition(clangRestartScheduled == false)
+    clangRestartScheduled = true
 
     guard let initializeRequest = initializeRequest else {
       log("clangd crashed before it was sent an InitializeRequest.", level: .error)
       return
     }
-
-    do {
-      try self.initialize()
-      // FIXME: We assume that clangd will return the same capabilites after restarting.
-      // Theoretically they could have changed and we would need to inform SourceKitServer about them.
-      // But since SourceKitServer more or less ignores them right now anyway, this should be fine for now.
-      _ = try self.initializeSync(initializeRequest)
-      self.clientInitialized(InitializedNotification())
-      self.reopenDocuments(self)
-      self.state = .connected
-    } catch {
-      log("Failed to restart clangd after a crash.", level: .error)
+    
+    let restartDelay: Int
+    if let lastClangdRestart = lastClangdRestart, Date().timeIntervalSince(lastClangdRestart) < 30 {
+      // We have crashed in the last 30 seconds. Delay any further restart by 10 seconds
+      log("clangd crashed in the last 30 seconds. Delaying another restart by 10 seconds.", level: .info)
+      restartDelay = 10
+    } else {
+      restartDelay = 0
+    }
+    lastClangdRestart = Date()
+    
+    DispatchQueue.global(qos: .default).asyncAfter(deadline: .now() + .seconds(restartDelay)) {
+      self.clangRestartScheduled = false
+      do {
+        try self.initialize()
+        // FIXME: We assume that clangd will return the same capabilites after restarting.
+        // Theoretically they could have changed and we would need to inform SourceKitServer about them.
+        // But since SourceKitServer more or less ignores them right now anyway, this should be fine for now.
+        _ = try self.initializeSync(initializeRequest)
+        self.clientInitialized(InitializedNotification())
+        self.reopenDocuments(self)
+        self.state = .connected
+      } catch {
+        log("Failed to restart clangd after a crash.", level: .error)
+      }
     }
   }
 

Tests/SourceKitTests/CrashRecoveryTests.swift

@@ -206,5 +206,64 @@ final class CrashRecoveryTests: XCTestCase {
       XCTAssertEqual(postCrashHighlightResponse, expectedHighlightResponse)
     }())
   }
+  
+  func testPreventClangdCrashLoop() {
+    guard longTestsEnabled else {
+      return
+    }
+
+    let ws = try! staticSourceKitTibsWorkspace(name: "ClangCrashRecovery")!
+    let loc = ws.testLoc("loc")
+
+    try! ws.openDocument(loc.url, language: .cpp)
+
+    // Send a non-sential request to wait for clangd to start up
+    
+    let hoverRequest = HoverRequest(textDocument: loc.docIdentifier, position: Position(line: 1, utf16index: 6))
+    _ = try! ws.sk.sendSync(hoverRequest)
+    
+    // Start crashing clangd
+    
+    let clangdServer = ws.testServer.server!.languageService(for: loc.docUri, .cpp, in: ws.testServer.server!.workspace!)!
+    
+    let clangdCrashed = self.expectation(description: "clangd crashed")
+    clangdCrashed.assertForOverFulfill = false
+    let clangdRestartedFirstTime = self.expectation(description: "clangd restarted for the first time")
+    let clangdRestartedSecondTime = self.expectation(description: "clangd restarted for the second time")
+    clangdRestartedSecondTime.assertForOverFulfill = false
+    var clangdHasRestartedFirstTime = false
+
+    clangdServer.addStateChangeHandler { (oldState, newState) in
+      switch newState {
+      case .connectionInterrupted:
+        clangdCrashed.fulfill()
+      case .connected:
+        if !clangdHasRestartedFirstTime {
+          clangdRestartedFirstTime.fulfill()
+          clangdHasRestartedFirstTime = true
+        } else {
+          clangdRestartedSecondTime.fulfill()
+        }
+      default:
+        break
+      }
+    }
+
+    // Add a pragma to crash clang
+    
+    let addCrashPragma = TextDocumentContentChangeEvent(range: loc.position..<loc.position, rangeLength: 0, text: "#pragma clang __debug crash\n")
+    ws.sk.send(DidChangeTextDocumentNotification(textDocument: VersionedTextDocumentIdentifier(loc.docUri, version: 3), contentChanges: [addCrashPragma]))
+
+    self.wait(for: [clangdCrashed], timeout: 5)
+
+    // Clangd has crashed for the first time. Leave the pragma there to crash clangd once again.
+    
+    self.wait(for: [clangdRestartedFirstTime], timeout: 30)
+    // Clangd has restarted. Note the date so we can check that the second restart doesn't happen too quickly.
+    let firstRestartDate = Date()
+    
+    self.wait(for: [clangdRestartedSecondTime], timeout: 30)
+    XCTAssert(Date().timeIntervalSince(firstRestartDate) > 5, "Clangd restarted too quickly after crashing twice in a row. We are not preventing crash loops.")
+  }
 }