Skip to content

Commit f5e29a2

Browse files
committed
Merge tag 'locking-urgent-2021-09-19' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull locking fixes from Thomas Gleixner: "A set of updates for the RT specific reader/writer locking base code: - Make the fast path reader ordering guarantees correct. - Code reshuffling to make the fix simpler" [ This plays ugly games with atomic_add_return_release() because we don't have a plain atomic_add_release(), and should really be cleaned up, I think - Linus ] * tag 'locking-urgent-2021-09-19' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: locking/rwbase: Take care of ordering guarantee for fastpath reader locking/rwbase: Extract __rwbase_write_trylock() locking/rwbase: Properly match set_and_save_state() to restore_state()
2 parents 62453a4 + 8112152 commit f5e29a2

File tree

1 file changed

+45
-20
lines changed

1 file changed

+45
-20
lines changed

kernel/locking/rwbase_rt.c

Lines changed: 45 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,12 @@
4141
* The risk of writer starvation is there, but the pathological use cases
4242
* which trigger it are not necessarily the typical RT workloads.
4343
*
44+
* Fast-path orderings:
45+
* The lock/unlock of readers can run in fast paths: lock and unlock are only
46+
* atomic ops, and there is no inner lock to provide ACQUIRE and RELEASE
47+
* semantics of rwbase_rt. Atomic ops should thus provide _acquire()
48+
* and _release() (or stronger).
49+
*
4450
* Common code shared between RT rw_semaphore and rwlock
4551
*/
4652

@@ -53,6 +59,7 @@ static __always_inline int rwbase_read_trylock(struct rwbase_rt *rwb)
5359
* set.
5460
*/
5561
for (r = atomic_read(&rwb->readers); r < 0;) {
62+
/* Fully-ordered if cmpxchg() succeeds, provides ACQUIRE */
5663
if (likely(atomic_try_cmpxchg(&rwb->readers, &r, r + 1)))
5764
return 1;
5865
}
@@ -162,6 +169,8 @@ static __always_inline void rwbase_read_unlock(struct rwbase_rt *rwb,
162169
/*
163170
* rwb->readers can only hit 0 when a writer is waiting for the
164171
* active readers to leave the critical section.
172+
*
173+
* dec_and_test() is fully ordered, provides RELEASE.
165174
*/
166175
if (unlikely(atomic_dec_and_test(&rwb->readers)))
167176
__rwbase_read_unlock(rwb, state);
@@ -172,7 +181,11 @@ static inline void __rwbase_write_unlock(struct rwbase_rt *rwb, int bias,
172181
{
173182
struct rt_mutex_base *rtm = &rwb->rtmutex;
174183

175-
atomic_add(READER_BIAS - bias, &rwb->readers);
184+
/*
185+
* _release() is needed in case that reader is in fast path, pairing
186+
* with atomic_try_cmpxchg() in rwbase_read_trylock(), provides RELEASE
187+
*/
188+
(void)atomic_add_return_release(READER_BIAS - bias, &rwb->readers);
176189
raw_spin_unlock_irqrestore(&rtm->wait_lock, flags);
177190
rwbase_rtmutex_unlock(rtm);
178191
}
@@ -196,6 +209,23 @@ static inline void rwbase_write_downgrade(struct rwbase_rt *rwb)
196209
__rwbase_write_unlock(rwb, WRITER_BIAS - 1, flags);
197210
}
198211

212+
static inline bool __rwbase_write_trylock(struct rwbase_rt *rwb)
213+
{
214+
/* Can do without CAS because we're serialized by wait_lock. */
215+
lockdep_assert_held(&rwb->rtmutex.wait_lock);
216+
217+
/*
218+
* _acquire is needed in case the reader is in the fast path, pairing
219+
* with rwbase_read_unlock(), provides ACQUIRE.
220+
*/
221+
if (!atomic_read_acquire(&rwb->readers)) {
222+
atomic_set(&rwb->readers, WRITER_BIAS);
223+
return 1;
224+
}
225+
226+
return 0;
227+
}
228+
199229
static int __sched rwbase_write_lock(struct rwbase_rt *rwb,
200230
unsigned int state)
201231
{
@@ -210,34 +240,30 @@ static int __sched rwbase_write_lock(struct rwbase_rt *rwb,
210240
atomic_sub(READER_BIAS, &rwb->readers);
211241

212242
raw_spin_lock_irqsave(&rtm->wait_lock, flags);
213-
/*
214-
* set_current_state() for rw_semaphore
215-
* current_save_and_set_rtlock_wait_state() for rwlock
216-
*/
217-
rwbase_set_and_save_current_state(state);
243+
if (__rwbase_write_trylock(rwb))
244+
goto out_unlock;
218245

219-
/* Block until all readers have left the critical section. */
220-
for (; atomic_read(&rwb->readers);) {
246+
rwbase_set_and_save_current_state(state);
247+
for (;;) {
221248
/* Optimized out for rwlocks */
222249
if (rwbase_signal_pending_state(state, current)) {
223-
__set_current_state(TASK_RUNNING);
250+
rwbase_restore_current_state();
224251
__rwbase_write_unlock(rwb, 0, flags);
225252
return -EINTR;
226253
}
254+
255+
if (__rwbase_write_trylock(rwb))
256+
break;
257+
227258
raw_spin_unlock_irqrestore(&rtm->wait_lock, flags);
259+
rwbase_schedule();
260+
raw_spin_lock_irqsave(&rtm->wait_lock, flags);
228261

229-
/*
230-
* Schedule and wait for the readers to leave the critical
231-
* section. The last reader leaving it wakes the waiter.
232-
*/
233-
if (atomic_read(&rwb->readers) != 0)
234-
rwbase_schedule();
235262
set_current_state(state);
236-
raw_spin_lock_irqsave(&rtm->wait_lock, flags);
237263
}
238-
239-
atomic_set(&rwb->readers, WRITER_BIAS);
240264
rwbase_restore_current_state();
265+
266+
out_unlock:
241267
raw_spin_unlock_irqrestore(&rtm->wait_lock, flags);
242268
return 0;
243269
}
@@ -253,8 +279,7 @@ static inline int rwbase_write_trylock(struct rwbase_rt *rwb)
253279
atomic_sub(READER_BIAS, &rwb->readers);
254280

255281
raw_spin_lock_irqsave(&rtm->wait_lock, flags);
256-
if (!atomic_read(&rwb->readers)) {
257-
atomic_set(&rwb->readers, WRITER_BIAS);
282+
if (__rwbase_write_trylock(rwb)) {
258283
raw_spin_unlock_irqrestore(&rtm->wait_lock, flags);
259284
return 1;
260285
}

0 commit comments

Comments
 (0)