Adds priority-inheritance futexes for mutexex

This uses FUTEX_LOCK_PI and FUTEX_UNLOCK_PI
on Linux.

Author: ruihe774

library/std/src/sys/pal/unix/mod.rs

@@ -23,6 +23,7 @@ pub mod net;
 #[cfg(target_os = "l4re")]
 pub use self::l4re::net;
 pub mod os;
+pub mod pi_futex;
 pub mod pipe;
 pub mod process;
 pub mod stack_overflow;

library/std/src/sys/pal/unix/pi_futex.rs

@@ -0,0 +1,58 @@
+#![cfg(any(target_os = "linux", target_os = "android"))]
+
+use crate::sync::atomic::AtomicU32;
+use crate::sys::cvt;
+use crate::sys::os::errno;
+use crate::{io, ptr};
+
+pub enum FutexLockError {
+    TryAgain,
+    DeadLock,
+    Other(io::Error),
+}
+
+pub const fn unlocked() -> u32 {
+    0
+}
+
+pub fn locked() -> u32 {
+    (unsafe { libc::gettid() }) as _
+}
+
+pub fn is_contended(futex_val: u32) -> bool {
+    (futex_val & libc::FUTEX_WAITERS) != 0
+}
+
+pub fn futex_lock(futex: &AtomicU32) -> Result<(), FutexLockError> {
+    if (unsafe {
+        libc::syscall(
+            libc::SYS_futex,
+            futex as *const AtomicU32,
+            libc::FUTEX_LOCK_PI | libc::FUTEX_PRIVATE_FLAG,
+            0,
+            ptr::null::<u32>(),
+            // remaining args are unused
+        )
+    } == -1)
+    {
+        Err(match errno() {
+            libc::EAGAIN | libc::EINTR => FutexLockError::TryAgain,
+            libc::EDEADLK => FutexLockError::DeadLock,
+            errno => FutexLockError::Other(io::Error::from_raw_os_error(errno)),
+        })
+    } else {
+        Ok(())
+    }
+}
+
+pub fn futex_unlock(futex: &AtomicU32) -> io::Result<()> {
+    cvt(unsafe {
+        libc::syscall(
+            libc::SYS_futex,
+            futex as *const AtomicU32,
+            libc::FUTEX_UNLOCK_PI | libc::FUTEX_PRIVATE_FLAG,
+            // remaining args are unused
+        )
+    })
+    .map(|_| ())
+}

library/std/src/sys/sync/mutex/mod.rs

@@ -1,8 +1,12 @@
 cfg_if::cfg_if! {
     if #[cfg(any(
-        all(target_os = "windows", not(target_vendor = "win7")),
         target_os = "linux",
         target_os = "android",
+    ))] {
+        mod pi_futex;
+        pub use pi_futex::Mutex;
+    } else if #[cfg(any(
+        all(target_os = "windows", not(target_vendor = "win7")),
         target_os = "freebsd",
         target_os = "openbsd",
         target_os = "dragonfly",

library/std/src/sys/sync/mutex/pi_futex.rs

@@ -0,0 +1,86 @@
+use crate::sync::atomic::AtomicU32;
+use crate::sync::atomic::Ordering::{Acquire, Relaxed, Release};
+use crate::sys::pi_futex as pi;
+pub struct Mutex {
+    futex: AtomicU32,
+}
+
+impl Mutex {
+    #[inline]
+    pub const fn new() -> Self {
+        Self { futex: AtomicU32::new(pi::unlocked()) }
+    }
+
+    #[inline]
+    pub fn try_lock(&self) -> bool {
+        self.futex.compare_exchange(pi::unlocked(), pi::locked(), Acquire, Relaxed).is_ok()
+    }
+
+    #[inline]
+    pub fn lock(&self) {
+        if self.futex.compare_exchange(pi::unlocked(), pi::locked(), Acquire, Relaxed).is_err() {
+            self.lock_contended();
+        }
+    }
+
+    #[cold]
+    fn lock_contended(&self) {
+        loop {
+            // Spin first to speed things up if the lock is released quickly.
+            let state = self.spin();
+
+            // If it's unlocked now, attempt to take the lock.
+            if state == pi::unlocked() {
+                if self.try_lock() {
+                    return;
+                }
+            };
+
+            if let Err(e) = pi::futex_lock(&self.futex) {
+                const MSG_PREFIX: &str = "failed to lock mutex";
+                match e {
+                    pi::FutexLockError::TryAgain => (), // Try again in this case.
+                    pi::FutexLockError::DeadLock => panic!("{MSG_PREFIX}: deadlock detected"),
+                    pi::FutexLockError::Other(e) => panic!("{MSG_PREFIX}: {e}"),
+                }
+            } else {
+                return;
+            }
+        }
+    }
+
+    fn spin(&self) -> u32 {
+        let mut spin = 100;
+        loop {
+            // We only use `load` (and not `swap` or `compare_exchange`)
+            // while spinning, to be easier on the caches.
+            let state = self.futex.load(Relaxed);
+
+            // We stop spinning when the mutex is unlocked,
+            // but also when it's contended.
+            if state == pi::unlocked() || pi::is_contended(state) || spin == 0 {
+                return state;
+            }
+
+            crate::hint::spin_loop();
+            spin -= 1;
+        }
+    }
+
+    #[inline]
+    pub unsafe fn unlock(&self) {
+        if self.futex.compare_exchange(pi::locked(), pi::unlocked(), Release, Relaxed).is_err() {
+            // We only wake up one thread. When that thread locks the mutex,
+            // the kernel will mark the mutex as contended automatically
+            // (futex != pi::locked() in this case),
+            // which makes sure that any other waiting threads will also be
+            // woken up eventually.
+            self.wake();
+        }
+    }
+
+    #[cold]
+    fn wake(&self) {
+        pi::futex_unlock(&self.futex).unwrap();
+    }
+}