New ThreadPlanSingleThreadTimeout to resolve potential deadlock in single thread stepping (#90930)

This PR introduces a new `ThreadPlanSingleThreadTimeout` that will be
used to address potential deadlock during single-thread stepping.

While debugging a target with a non-trivial number of threads (around
5000 threads in one example target), we noticed that a simple step over
can take as long as 10 seconds. Enabling single-thread stepping mode
significantly reduces the stepping time to around 3 seconds. However,
this can introduce deadlock if we try to step over a method that depends
on other threads to release a lock.

To address this issue, we introduce a new
`ThreadPlanSingleThreadTimeout` that can be controlled by the
`target.process.thread.single-thread-plan-timeout` setting during
single-thread stepping mode. The concept involves counting the elapsed
time since the last internal stop to detect overall stepping progress.
Once a timeout occurs, we assume the target is not making progress due
to a potential deadlock, as mentioned above. We then send a new async
interrupt, resume all threads, and `ThreadPlanSingleThreadTimeout`
completes its task.

To support this design, the major changes made in this PR are:
1. `ThreadPlanSingleThreadTimeout` is popped during every internal stop
and reset (re-pushed) to the top of the stack (as a leaf node) during
resume. This is achieved by always returning `true` from
`ThreadPlanSingleThreadTimeout::DoPlanExplainsStop()` and
`ThreadPlanSingleThreadTimeout::MischiefManaged()`.
2. A new thread-specific async interrupt stop is introduced, which can
be detected/consumed by `ThreadPlanSingleThreadTimeout`.
3. The clearing of branch breakpoints in the range thread plan has been
moved from `DoPlanExplainsStop()` to `ShouldStop()`, as it is not
guaranteed that it will be called.

The detailed design is discussed in the RFC below:

[https://discourse.llvm.org/t/improve-single-thread-stepping/74599](https://discourse.llvm.org/t/improve-single-thread-stepping/74599)

---------

Co-authored-by: jeffreytan81 <[email protected]>

Author: jeffreytan81

lldb/include/lldb/Target/Process.h

@@ -1314,10 +1314,16 @@ class Process : public std::enable_shared_from_this<Process>,
 
   size_t GetThreadStatus(Stream &ostrm, bool only_threads_with_stop_reason,
                          uint32_t start_frame, uint32_t num_frames,
-                         uint32_t num_frames_with_source,
-                         bool stop_format);
+                         uint32_t num_frames_with_source, bool stop_format);
 
-  void SendAsyncInterrupt();
+  /// Send an async interrupt request.
+  ///
+  /// If \a thread is specified the async interrupt stop will be attributed to
+  /// the specified thread.
+  ///
+  /// \param[in] thread
+  ///     The thread the async interrupt will be attributed to.
+  void SendAsyncInterrupt(Thread *thread = nullptr);
 
   // Notify this process class that modules got loaded.
   //
@@ -2867,6 +2873,17 @@ void PruneThreadPlans();
     return std::nullopt;
   }
 
+  /// Handle thread specific async interrupt and return the original thread
+  /// that requested the async interrupt. It can be null if original thread
+  /// has exited.
+  ///
+  /// \param[in] description
+  ///     Returns the stop reason description of the async interrupt.
+  virtual lldb::ThreadSP
+  HandleThreadAsyncInterrupt(uint8_t signo, const std::string &description) {
+    return lldb::ThreadSP();
+  }
+
   lldb::StateType GetPrivateState();
 
   /// The "private" side of resuming a process.  This doesn't alter the state
@@ -3153,6 +3170,11 @@ void PruneThreadPlans();
                            // Resume will only request a resume, using this
                            // flag to check.
 
+  lldb::tid_t m_interrupt_tid; /// The tid of the thread that issued the async
+                               /// interrupt, used by thread plan timeout. It
+                               /// can be LLDB_INVALID_THREAD_ID to indicate
+                               /// user level async interrupt.
+
   /// This is set at the beginning of Process::Finalize() to stop functions
   /// from looking up or creating things during or after a finalize call.
   std::atomic<bool> m_finalizing;

lldb/include/lldb/Target/StopInfo.h

@@ -123,6 +123,10 @@ class StopInfo : public std::enable_shared_from_this<StopInfo> {
                              const char *description = nullptr,
                              std::optional<int> code = std::nullopt);
 
+  static lldb::StopInfoSP
+  CreateStopReasonWithInterrupt(Thread &thread, int signo,
+                                const char *description);
+
   static lldb::StopInfoSP CreateStopReasonToTrace(Thread &thread);
 
   static lldb::StopInfoSP

lldb/include/lldb/Target/Thread.h

@@ -58,6 +58,8 @@ class ThreadProperties : public Properties {
   bool GetStepOutAvoidsNoDebug() const;
 
   uint64_t GetMaxBacktraceDepth() const;
+
+  uint64_t GetSingleThreadPlanTimeout() const;
 };
 
 class Thread : public std::enable_shared_from_this<Thread>,

lldb/include/lldb/Target/ThreadPlan.h

@@ -302,7 +302,8 @@ class ThreadPlan : public std::enable_shared_from_this<ThreadPlan>,
     eKindStepInRange,
     eKindRunToAddress,
     eKindStepThrough,
-    eKindStepUntil
+    eKindStepUntil,
+    eKindSingleThreadTimeout,
   };
 
   virtual ~ThreadPlan();
@@ -395,6 +396,11 @@ class ThreadPlan : public std::enable_shared_from_this<ThreadPlan>,
 
   bool IsControllingPlan() { return m_is_controlling_plan; }
 
+  // Returns true if this plan is a leaf plan, meaning the plan will be popped
+  // during each stop if it does not explain the stop and re-pushed before
+  // resuming to stay at the top of the stack.
+  virtual bool IsLeafPlan() { return false; }
+
   bool SetIsControllingPlan(bool value) {
     bool old_value = m_is_controlling_plan;
     m_is_controlling_plan = value;
@@ -483,6 +489,8 @@ class ThreadPlan : public std::enable_shared_from_this<ThreadPlan>,
     return m_takes_iteration_count;
   }
 
+  virtual lldb::StateType GetPlanRunState() = 0;
+
 protected:
   // Constructors and Destructors
   ThreadPlan(ThreadPlanKind kind, const char *name, Thread &thread,
@@ -522,8 +530,6 @@ class ThreadPlan : public std::enable_shared_from_this<ThreadPlan>,
     GetThread().SetStopInfo(stop_reason_sp);
   }
 
-  virtual lldb::StateType GetPlanRunState() = 0;
-
   bool IsUsuallyUnexplainedStopReason(lldb::StopReason);
 
   Status m_status;

lldb/include/lldb/Target/ThreadPlanSingleThreadTimeout.h

@@ -0,0 +1,107 @@
+//===-- ThreadPlanSingleThreadTimeout.h -------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLDB_TARGET_THREADPLANSINGLETHREADTIMEOUT_H
+#define LLDB_TARGET_THREADPLANSINGLETHREADTIMEOUT_H
+
+#include "lldb/Target/Thread.h"
+#include "lldb/Target/ThreadPlan.h"
+#include "lldb/Utility/Event.h"
+#include "lldb/Utility/LLDBLog.h"
+#include "lldb/Utility/State.h"
+
+#include <chrono>
+#include <thread>
+
+namespace lldb_private {
+
+//
+// Thread plan used by single thread execution to issue timeout. This is useful
+// to detect potential deadlock in single thread execution. The timeout measures
+// the elapsed time from the last internal stop and gets reset by each internal
+// stop to ensure we are accurately detecting execution not moving forward.
+// This means this thread plan may be created/destroyed multiple times by the
+// parent execution plan.
+//
+// When a timeout happens, the thread plan resolves the potential deadlock by
+// issuing a thread specific async interrupt to enter stop state, then execution
+// is resumed with all threads running to resolve the potential deadlock
+//
+class ThreadPlanSingleThreadTimeout : public ThreadPlan {
+  enum class State {
+    WaitTimeout,    // Waiting for timeout.
+    AsyncInterrupt, // Async interrupt has been issued.
+    Done,           // Finished resume all threads.
+  };
+
+public:
+  // TODO: allow timeout to be set on per thread plan basis.
+  struct TimeoutInfo {
+    // Whether there is a ThreadPlanSingleThreadTimeout instance alive.
+    bool m_isAlive = false;
+    ThreadPlanSingleThreadTimeout::State m_last_state = State::WaitTimeout;
+  };
+
+  ~ThreadPlanSingleThreadTimeout() override;
+
+  // If input \param thread is running in single thread mode, push a
+  // new ThreadPlanSingleThreadTimeout based on timeout setting from fresh new
+  // state. The reference of \param info is passed in so that when
+  // ThreadPlanSingleThreadTimeout got popped its last state can be stored
+  // in it for future resume.
+  static void PushNewWithTimeout(Thread &thread, TimeoutInfo &info);
+
+  // Push a new ThreadPlanSingleThreadTimeout by restoring state from
+  // input \param info and resume execution.
+  static void ResumeFromPrevState(Thread &thread, TimeoutInfo &info);
+
+  void GetDescription(Stream *s, lldb::DescriptionLevel level) override;
+  bool ValidatePlan(Stream *error) override { return true; }
+  bool WillStop() override;
+  void DidPop() override;
+
+  bool IsLeafPlan() override { return true; }
+  bool DoPlanExplainsStop(Event *event_ptr) override;
+
+  lldb::StateType GetPlanRunState() override;
+  static void TimeoutThreadFunc(ThreadPlanSingleThreadTimeout *self);
+
+  bool MischiefManaged() override;
+
+  bool ShouldStop(Event *event_ptr) override;
+  void SetStopOthers(bool new_value) override;
+  bool StopOthers() override;
+
+private:
+  ThreadPlanSingleThreadTimeout(Thread &thread, TimeoutInfo &info);
+
+  bool IsTimeoutAsyncInterrupt(Event *event_ptr);
+  bool HandleEvent(Event *event_ptr);
+  void HandleTimeout();
+  uint64_t GetRemainingTimeoutMilliSeconds();
+
+  static std::string StateToString(State state);
+
+  ThreadPlanSingleThreadTimeout(const ThreadPlanSingleThreadTimeout &) = delete;
+  const ThreadPlanSingleThreadTimeout &
+  operator=(const ThreadPlanSingleThreadTimeout &) = delete;
+
+  TimeoutInfo &m_info; // Reference to controlling ThreadPlan's TimeoutInfo.
+  State m_state;
+
+  // Lock for m_wakeup_cv and m_exit_flag between thread plan thread and timer
+  // thread
+  std::mutex m_mutex;
+  std::condition_variable m_wakeup_cv;
+  std::thread m_timer_thread;
+  std::chrono::steady_clock::time_point m_timeout_start;
+};
+
+} // namespace lldb_private
+
+#endif // LLDB_TARGET_THREADPLANSINGLETHREADTIMEOUT_H

lldb/include/lldb/Target/ThreadPlanStepOut.h

@@ -30,6 +30,7 @@ class ThreadPlanStepOut : public ThreadPlan, public ThreadPlanShouldStopHere {
   bool ValidatePlan(Stream *error) override;
   bool ShouldStop(Event *event_ptr) override;
   bool StopOthers() override;
+  void SetStopOthers(bool new_value) override { m_stop_others = new_value; }
   lldb::StateType GetPlanRunState() override;
   bool WillStop() override;
   bool MischiefManaged() override;

lldb/include/lldb/Target/ThreadPlanStepOverRange.h

@@ -13,11 +13,13 @@
 #include "lldb/Target/StackID.h"
 #include "lldb/Target/Thread.h"
 #include "lldb/Target/ThreadPlanStepRange.h"
+#include "lldb/Target/TimeoutResumeAll.h"
 
 namespace lldb_private {
 
 class ThreadPlanStepOverRange : public ThreadPlanStepRange,
-                                ThreadPlanShouldStopHere {
+                                ThreadPlanShouldStopHere,
+                                TimeoutResumeAll {
 public:
   ThreadPlanStepOverRange(Thread &thread, const AddressRange &range,
                           const SymbolContext &addr_context,
@@ -27,7 +29,9 @@ class ThreadPlanStepOverRange : public ThreadPlanStepRange,
   ~ThreadPlanStepOverRange() override;
 
   void GetDescription(Stream *s, lldb::DescriptionLevel level) override;
+  void SetStopOthers(bool new_value) override;
   bool ShouldStop(Event *event_ptr) override;
+  void DidPush() override;
 
 protected:
   bool DoPlanExplainsStop(Event *event_ptr) override;
@@ -44,6 +48,7 @@ class ThreadPlanStepOverRange : public ThreadPlanStepRange,
   bool IsEquivalentContext(const SymbolContext &context);
 
   bool m_first_resume;
+  lldb::RunMode m_run_mode;
 
   ThreadPlanStepOverRange(const ThreadPlanStepOverRange &) = delete;
   const ThreadPlanStepOverRange &

lldb/include/lldb/Target/ThreadPlanStepRange.h

@@ -58,8 +58,15 @@ class ThreadPlanStepRange : public ThreadPlan {
   // run' plan, then just single step.
   bool SetNextBranchBreakpoint();
 
+  // Whether the input stop info is caused by the next branch breakpoint.
+  // Note: this does not check if branch breakpoint site is shared by other
+  // breakpoints or not.
+  bool IsNextBranchBreakpointStop(lldb::StopInfoSP stop_info_sp);
+
   void ClearNextBranchBreakpoint();
 
+  void ClearNextBranchBreakpointExplainedStop();
+
   bool NextRangeBreakpointExplainsStop(lldb::StopInfoSP stop_info_sp);
 
   SymbolContext m_addr_context;

lldb/include/lldb/Target/TimeoutResumeAll.h

@@ -0,0 +1,40 @@
+//===-- TimeoutResumeAll.h -------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLDB_TARGET_TIMEOUTRESUMEALL_H
+#define LLDB_TARGET_TIMEOUTRESUMEALL_H
+
+#include "lldb/Target/ThreadPlanSingleThreadTimeout.h"
+
+namespace lldb_private {
+
+// Mixin class that provides the capability for ThreadPlan to support single
+// thread execution that resumes all threads after a timeout.
+// Opt-in thread plan should call PushNewTimeout() in its DidPush() and
+// ResumeWithTimeout() during DoWillResume().
+class TimeoutResumeAll {
+public:
+  TimeoutResumeAll(Thread &thread) : m_thread(thread) {}
+
+  void PushNewTimeout() {
+    ThreadPlanSingleThreadTimeout::PushNewWithTimeout(m_thread, m_timeout_info);
+  }
+
+  void ResumeWithTimeout() {
+    ThreadPlanSingleThreadTimeout::ResumeFromPrevState(m_thread,
+                                                       m_timeout_info);
+  }
+
+private:
+  Thread &m_thread;
+  ThreadPlanSingleThreadTimeout::TimeoutInfo m_timeout_info;
+};
+
+} // namespace lldb_private
+
+#endif // LLDB_TARGET_TIMEOUTRESUMEALL_H

lldb/include/lldb/lldb-enumerations.h

@@ -253,6 +253,7 @@ enum StopReason {
   eStopReasonFork,
   eStopReasonVFork,
   eStopReasonVForkDone,
+  eStopReasonInterrupt, ///< Thread requested interrupt
 };
 
 /// Command Return Status Types.

lldb/source/API/SBThread.cpp

@@ -192,6 +192,9 @@ size_t SBThread::GetStopReasonDataCount() {
         case eStopReasonSignal:
           return 1;
 
+        case eStopReasonInterrupt:
+          return 1;
+
         case eStopReasonException:
           return 1;
 
@@ -261,6 +264,9 @@ uint64_t SBThread::GetStopReasonDataAtIndex(uint32_t idx) {
         case eStopReasonSignal:
           return stop_info_sp->GetValue();
 
+        case eStopReasonInterrupt:
+          return stop_info_sp->GetValue();
+
         case eStopReasonException:
           return stop_info_sp->GetValue();
 

lldb/source/Interpreter/CommandInterpreter.cpp

@@ -2513,7 +2513,7 @@ bool CommandInterpreter::DidProcessStopAbnormally() const {
     const StopReason reason = stop_info->GetStopReason();
     if (reason == eStopReasonException ||
         reason == eStopReasonInstrumentation ||
-        reason == eStopReasonProcessorTrace)
+        reason == eStopReasonProcessorTrace || reason == eStopReasonInterrupt)
       return true;
 
     if (reason == eStopReasonSignal) {

lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerLLGS.cpp

@@ -714,6 +714,8 @@ static const char *GetStopReasonString(StopReason stop_reason) {
     return "vfork";
   case eStopReasonVForkDone:
     return "vforkdone";
+  case eStopReasonInterrupt:
+    return "async interrupt";
   case eStopReasonInstrumentation:
   case eStopReasonInvalid:
   case eStopReasonPlanComplete: