Use Mutex#owned? to correctly check if the Mutex is owned by the current Thread or Fiber

Changelog.md

@@ -4,6 +4,9 @@ Breaking Changes:
 
 * Ruby < 2.3 is no longer supported. (Phil Pirozhkov, #436)
 
+Bug Fixes:
+* Fix reentrant mutex for Ruby 3.0. (Benoit Daloze, #503)
+
 ### 3.10.0 / 2020-10-30
 
 No changes. Released to support other RSpec releases.

lib/rspec/support/reentrant_mutex.rb

@@ -17,7 +17,6 @@ class << self
     # @private
     class ReentrantMutex
       def initialize
-        @owner = nil
         @count = 0
         @mutex = Mutex.new
       end
@@ -32,16 +31,16 @@ def synchronize
     private
 
       def enter
-        @mutex.lock if @owner != Thread.current
-        @owner = Thread.current
+        @mutex.lock unless @mutex.owned?
         @count += 1
       end
 
       def exit
+        unless @mutex.owned?
+          raise ThreadError, "Attempt to unlock a mutex which is locked by another thread/fiber"
+        end
         @count -= 1
-        return unless @count == 0
-        @owner = nil
-        @mutex.unlock
+        @mutex.unlock if @count == 0
       end
     end
   end

spec/rspec/support/reentrant_mutex_spec.rb

@@ -34,4 +34,28 @@
     mutex.synchronize { order.pass_to :thread, :resume_on => :sleep }
     order.join_all
   end
+
+  if RUBY_VERSION >= '3.0'
+    it 'waits when trying to lock from another Fiber' do
+      mutex.synchronize do
+        ready = false
+        f = Fiber.new do
+          expect {
+            ready = true
+            mutex.send(:enter)
+            raise 'should reach here: mutex is already locked on different Fiber'
+          }.to raise_error(Exception, 'waited correctly')
+        end
+
+        main_thread = Thread.current
+
+        t = Thread.new do
+          Thread.pass until ready and main_thread.stop?
+          main_thread.raise Exception, 'waited correctly'
+        end
+        f.resume
+        t.join
+      end
+    end
+  end
 end