[libunwind][Haiku] Improve support (#115462)

* Signal frame unwinding on x86_64 from X512
* Header search for commpage_defs.h on non-standard paths

Unwind supported tests pass on Haiku x86_64

---------

Co-authored-by: Trung Nguyen <[email protected]>

Author: korli

libunwind/src/CMakeLists.txt

@@ -113,6 +113,22 @@ if (HAIKU)
 
   add_compile_flags("-D_DEFAULT_SOURCE")
   add_compile_flags("-DPT_GNU_EH_FRAME=PT_EH_FRAME")
+
+  find_path(LIBUNWIND_HAIKU_PRIVATE_HEADERS
+            "commpage_defs.h"
+            PATHS ${CMAKE_SYSTEM_INCLUDE_PATH}
+            PATH_SUFFIXES "/private/system"
+            NO_DEFAULT_PATH
+            REQUIRED)
+
+  include_directories(SYSTEM "${LIBUNWIND_HAIKU_PRIVATE_HEADERS}")
+  if (LIBUNWIND_TARGET_TRIPLE)
+    if (${LIBUNWIND_TARGET_TRIPLE} MATCHES "^x86_64")
+      include_directories(SYSTEM "${LIBUNWIND_HAIKU_PRIVATE_HEADERS}/arch/x86_64")
+    endif()
+  else()
+    include_directories(SYSTEM "${LIBUNWIND_HAIKU_PRIVATE_HEADERS}/arch/${CMAKE_SYSTEM_PROCESSOR}")
+  endif()
 endif ()
 
 string(REPLACE ";" " " LIBUNWIND_COMPILE_FLAGS "${LIBUNWIND_COMPILE_FLAGS}")

libunwind/src/UnwindCursor.hpp

@@ -1010,6 +1010,9 @@ class UnwindCursor : public AbstractUnwindCursor{
   template <typename Registers> int stepThroughSigReturn(Registers &) {
     return UNW_STEP_END;
   }
+#elif defined(_LIBUNWIND_TARGET_HAIKU)
+  bool setInfoForSigReturn();
+  int stepThroughSigReturn();
 #endif
 
 #if defined(_LIBUNWIND_SUPPORT_DWARF_UNWIND)
@@ -1313,7 +1316,8 @@ class UnwindCursor : public AbstractUnwindCursor{
   unw_proc_info_t  _info;
   bool             _unwindInfoMissing;
   bool             _isSignalFrame;
-#if defined(_LIBUNWIND_CHECK_LINUX_SIGRETURN)
+#if defined(_LIBUNWIND_CHECK_LINUX_SIGRETURN) ||                               \
+    defined(_LIBUNWIND_TARGET_HAIKU)
   bool             _isSigReturn = false;
 #endif
 };
@@ -2549,7 +2553,8 @@ int UnwindCursor<A, R>::stepWithTBTable(pint_t pc, tbtable *TBTable,
 
 template <typename A, typename R>
 void UnwindCursor<A, R>::setInfoBasedOnIPRegister(bool isReturnAddress) {
-#if defined(_LIBUNWIND_CHECK_LINUX_SIGRETURN)
+#if defined(_LIBUNWIND_CHECK_LINUX_SIGRETURN) ||                               \
+    defined(_LIBUNWIND_TARGET_HAIKU)
   _isSigReturn = false;
 #endif
 
@@ -2673,7 +2678,8 @@ void UnwindCursor<A, R>::setInfoBasedOnIPRegister(bool isReturnAddress) {
   }
 #endif // #if defined(_LIBUNWIND_SUPPORT_DWARF_UNWIND)
 
-#if defined(_LIBUNWIND_CHECK_LINUX_SIGRETURN)
+#if defined(_LIBUNWIND_CHECK_LINUX_SIGRETURN) ||                               \
+    defined(_LIBUNWIND_TARGET_HAIKU)
   if (setInfoForSigReturn())
     return;
 #endif
@@ -2749,6 +2755,63 @@ int UnwindCursor<A, R>::stepThroughSigReturn(Registers_arm64 &) {
   _isSignalFrame = true;
   return UNW_STEP_SUCCESS;
 }
+
+#elif defined(_LIBUNWIND_TARGET_HAIKU) && defined(_LIBUNWIND_TARGET_X86_64)
+#include <commpage_defs.h>
+#include <signal.h>
+
+extern "C" {
+extern void *__gCommPageAddress;
+}
+
+template <typename A, typename R>
+bool UnwindCursor<A, R>::setInfoForSigReturn() {
+#if defined(_LIBUNWIND_TARGET_X86_64)
+  addr_t signal_handler =
+      (((addr_t *)__gCommPageAddress)[COMMPAGE_ENTRY_X86_SIGNAL_HANDLER] +
+       (addr_t)__gCommPageAddress);
+  addr_t signal_handler_ret = signal_handler + 45;
+#endif
+  pint_t pc = static_cast<pint_t>(this->getReg(UNW_REG_IP));
+  if (pc == signal_handler_ret) {
+    _info = {};
+    _info.start_ip = signal_handler;
+    _info.end_ip = signal_handler_ret;
+    _isSigReturn = true;
+    return true;
+  }
+  return false;
+}
+
+template <typename A, typename R>
+int UnwindCursor<A, R>::stepThroughSigReturn() {
+  _isSignalFrame = true;
+  pint_t sp = _registers.getSP();
+#if defined(_LIBUNWIND_TARGET_X86_64)
+  vregs *regs = (vregs *)(sp + 0x70);
+
+  _registers.setRegister(UNW_REG_IP, regs->rip);
+  _registers.setRegister(UNW_REG_SP, regs->rsp);
+  _registers.setRegister(UNW_X86_64_RAX, regs->rax);
+  _registers.setRegister(UNW_X86_64_RDX, regs->rdx);
+  _registers.setRegister(UNW_X86_64_RCX, regs->rcx);
+  _registers.setRegister(UNW_X86_64_RBX, regs->rbx);
+  _registers.setRegister(UNW_X86_64_RSI, regs->rsi);
+  _registers.setRegister(UNW_X86_64_RDI, regs->rdi);
+  _registers.setRegister(UNW_X86_64_RBP, regs->rbp);
+  _registers.setRegister(UNW_X86_64_R8, regs->r8);
+  _registers.setRegister(UNW_X86_64_R9, regs->r9);
+  _registers.setRegister(UNW_X86_64_R10, regs->r10);
+  _registers.setRegister(UNW_X86_64_R11, regs->r11);
+  _registers.setRegister(UNW_X86_64_R12, regs->r12);
+  _registers.setRegister(UNW_X86_64_R13, regs->r13);
+  _registers.setRegister(UNW_X86_64_R14, regs->r14);
+  _registers.setRegister(UNW_X86_64_R15, regs->r15);
+  // TODO: XMM
+#endif
+
+  return UNW_STEP_SUCCESS;
+}
 #endif // defined(_LIBUNWIND_CHECK_LINUX_SIGRETURN) &&
        // defined(_LIBUNWIND_TARGET_AARCH64)
 
@@ -2917,7 +2980,8 @@ template <typename A, typename R> int UnwindCursor<A, R>::step(bool stage2) {
 
   // Use unwinding info to modify register set as if function returned.
   int result;
-#if defined(_LIBUNWIND_CHECK_LINUX_SIGRETURN)
+#if defined(_LIBUNWIND_CHECK_LINUX_SIGRETURN) ||                               \
+    defined(_LIBUNWIND_TARGET_HAIKU)
   if (_isSigReturn) {
     result = this->stepThroughSigReturn();
   } else