@@ -112,7 +112,7 @@ class WaitingQueue final : private RawMutex {
112
112
}
113
113
};
114
114
115
- // The State of the RwLock is stored in an integer word, consisting of the
115
+ // The RwState of the RwLock is stored in an integer word, consisting of the
116
116
// following components:
117
117
// -----------------------------------------------
118
118
// | Range | Description |
@@ -125,8 +125,7 @@ class WaitingQueue final : private RawMutex {
125
125
// -----------------------------------------------
126
126
// | MSB | Active Writer Bit |
127
127
// -----------------------------------------------
128
- class State {
129
-
128
+ class RwState {
130
129
// Shift amounts to access the components of the state.
131
130
LIBC_INLINE_VAR static constexpr int PENDING_READER_SHIFT = 0 ;
132
131
LIBC_INLINE_VAR static constexpr int PENDING_WRITER_SHIFT = 1 ;
@@ -153,7 +152,7 @@ class State {
153
152
154
153
public:
155
154
// Construction and conversion functions.
156
- LIBC_INLINE constexpr State (int state = 0 ) : state(state) {}
155
+ LIBC_INLINE constexpr RwState (int state = 0 ) : state(state) {}
157
156
LIBC_INLINE constexpr operator int () const { return state; }
158
157
159
158
// Utilities to check the state of the RwLock.
@@ -174,8 +173,8 @@ class State {
174
173
return state & PENDING_MASK;
175
174
}
176
175
177
- LIBC_INLINE constexpr State set_writer_bit () const {
178
- return State (state | ACTIVE_WRITER_BIT);
176
+ LIBC_INLINE constexpr RwState set_writer_bit () const {
177
+ return RwState (state | ACTIVE_WRITER_BIT);
179
178
}
180
179
181
180
// The preference parameter changes the behavior of the lock acquisition
@@ -196,61 +195,52 @@ class State {
196
195
197
196
// This function check if it is possible to grow the reader count without
198
197
// overflowing the state.
199
- LIBC_INLINE cpp::optional<State > try_increase_reader_count () const {
198
+ LIBC_INLINE cpp::optional<RwState > try_increase_reader_count () const {
200
199
LIBC_ASSERT (!has_active_writer () &&
201
200
" try_increase_reader_count shall only be called when there "
202
201
" is no active writer." );
203
- State res;
202
+ RwState res;
204
203
if (LIBC_UNLIKELY (__builtin_sadd_overflow (state, ACTIVE_READER_COUNT_UNIT,
205
204
&res.state )))
206
205
return cpp::nullopt;
207
206
return res;
208
207
}
209
208
210
209
// Utilities to do atomic operations on the state.
211
- LIBC_INLINE static State
212
- fetch_sub_reader_count (cpp::Atomic<int > &target,
213
- cpp::MemoryOrder order = cpp::MemoryOrder::SEQ_CST) {
214
- return State (target.fetch_sub (ACTIVE_READER_COUNT_UNIT, order));
210
+ LIBC_INLINE static RwState fetch_sub_reader_count (cpp::Atomic<int > &target,
211
+ cpp::MemoryOrder order) {
212
+ return RwState (target.fetch_sub (ACTIVE_READER_COUNT_UNIT, order));
215
213
}
216
214
217
- LIBC_INLINE static State
218
- load (cpp::Atomic<int > &target,
219
- cpp::MemoryOrder order = cpp::MemoryOrder::SEQ_CST) {
220
- return State (target.load (order));
215
+ LIBC_INLINE static RwState load (cpp::Atomic<int > &target,
216
+ cpp::MemoryOrder order) {
217
+ return RwState (target.load (order));
221
218
}
222
219
223
220
template <Role role>
224
- LIBC_INLINE static State
225
- fetch_set_pending_bit (cpp::Atomic<int > &target,
226
- cpp::MemoryOrder order = cpp::MemoryOrder::SEQ_CST) {
221
+ LIBC_INLINE static RwState fetch_set_pending_bit (cpp::Atomic<int > &target,
222
+ cpp::MemoryOrder order) {
227
223
if constexpr (role == Role::Reader)
228
- return State (target.fetch_or (PENDING_READER_BIT, order));
224
+ return RwState (target.fetch_or (PENDING_READER_BIT, order));
229
225
else
230
- return State (target.fetch_or (PENDING_WRITER_BIT, order));
226
+ return RwState (target.fetch_or (PENDING_WRITER_BIT, order));
231
227
}
232
228
template <Role role>
233
- LIBC_INLINE static State
234
- fetch_clear_pending_bit (cpp::Atomic<int > &target,
235
- cpp::MemoryOrder order = cpp::MemoryOrder::SEQ_CST) {
229
+ LIBC_INLINE static RwState fetch_clear_pending_bit (cpp::Atomic<int > &target,
230
+ cpp::MemoryOrder order) {
236
231
if constexpr (role == Role::Reader)
237
- return State (target.fetch_and (~PENDING_READER_BIT, order));
232
+ return RwState (target.fetch_and (~PENDING_READER_BIT, order));
238
233
else
239
- return State (target.fetch_and (~PENDING_WRITER_BIT, order));
240
- }
241
- LIBC_INLINE static State
242
- fetch_set_active_writer (cpp::Atomic<int > &target,
243
- cpp::MemoryOrder order = cpp::MemoryOrder::SEQ_CST) {
244
- return State (target.fetch_or (ACTIVE_WRITER_BIT, order));
234
+ return RwState (target.fetch_and (~PENDING_WRITER_BIT, order));
245
235
}
246
- LIBC_INLINE static State fetch_clear_active_writer (
247
- cpp::Atomic<int > &target,
248
- cpp::MemoryOrder order = cpp::MemoryOrder::SEQ_CST ) {
249
- return State (target.fetch_and (~ACTIVE_WRITER_BIT, order));
236
+
237
+ LIBC_INLINE static RwState fetch_clear_active_writer ( cpp::Atomic<int > &target,
238
+ cpp::MemoryOrder order ) {
239
+ return RwState (target.fetch_and (~ACTIVE_WRITER_BIT, order));
250
240
}
251
241
252
242
LIBC_INLINE bool compare_exchange_weak_with (cpp::Atomic<int > &target,
253
- State desired,
243
+ RwState desired,
254
244
cpp::MemoryOrder success_order,
255
245
cpp::MemoryOrder failure_order) {
256
246
return target.compare_exchange_weak (state, desired, success_order,
@@ -260,10 +250,10 @@ class State {
260
250
// Utilities to spin and reload the state.
261
251
private:
262
252
template <class F >
263
- LIBC_INLINE static State spin_reload_until (cpp::Atomic<int > &target, F &&func ,
264
- unsigned spin_count) {
253
+ LIBC_INLINE static RwState spin_reload_until (cpp::Atomic<int > &target,
254
+ F &&func, unsigned spin_count) {
265
255
for (;;) {
266
- auto state = State ::load (target, cpp::MemoryOrder::RELAXED);
256
+ auto state = RwState ::load (target, cpp::MemoryOrder::RELAXED);
267
257
if (func (state) || spin_count == 0 )
268
258
return state;
269
259
sleep_briefly ();
@@ -273,38 +263,38 @@ class State {
273
263
274
264
public:
275
265
template <Role role>
276
- LIBC_INLINE static State spin_reload (cpp::Atomic<int > &target,
277
- Role preference, unsigned spin_count) {
266
+ LIBC_INLINE static RwState spin_reload (cpp::Atomic<int > &target,
267
+ Role preference, unsigned spin_count) {
278
268
if constexpr (role == Role::Reader) {
279
269
// Return the reader state if either the lock is available or there is
280
- // any
281
- // ongoing contention.
270
+ // any ongoing contention.
282
271
return spin_reload_until (
283
272
target,
284
- [=](State state) {
273
+ [=](RwState state) {
285
274
return state.can_acquire <Role::Reader>(preference) ||
286
275
state.has_pending ();
287
276
},
288
277
spin_count);
289
278
} else {
290
279
// Return the writer state if either the lock is available or there is
291
- // any
292
- // contention *between writers*. Since writers can be way less than
280
+ // any contention *between writers*. Since writers can be way less than
293
281
// readers, we allow them to spin more to improve the fairness.
294
282
return spin_reload_until (
295
283
target,
296
- [=](State state) {
284
+ [=](RwState state) {
297
285
return state.can_acquire <Role::Writer>(preference) ||
298
286
state.has_pending_writer ();
299
287
},
300
288
spin_count);
301
289
}
302
290
}
291
+
292
+ friend class RwLockTester ;
303
293
};
304
294
} // namespace rwlock
305
295
306
296
class RwLock {
307
- using State = rwlock::State ;
297
+ using RwState = rwlock::RwState ;
308
298
using Role = rwlock::Role;
309
299
using WaitingQueue = rwlock::WaitingQueue;
310
300
@@ -330,13 +320,13 @@ class RwLock {
330
320
// Reader/Writer preference.
331
321
LIBC_PREFERED_TYPE (Role)
332
322
unsigned preference : 1 ;
333
- // State to keep track of the RwLock.
323
+ // RwState to keep track of the RwLock.
334
324
cpp::Atomic<int > state;
335
325
// writer_tid is used to keep track of the thread id of the writer. Notice
336
326
// that TLS address is not a good idea here since it may remains the same
337
327
// across forked processes.
338
328
cpp::Atomic<pid_t > writer_tid;
339
- // Waiting queue to keep track of the pending readers and writers.
329
+ // Waiting queue to keep track of the readers and writers.
340
330
WaitingQueue queue;
341
331
342
332
private:
@@ -347,10 +337,10 @@ class RwLock {
347
337
// TODO: use cached thread id once implemented.
348
338
LIBC_INLINE static pid_t gettid () { return syscall_impl<pid_t >(SYS_gettid); }
349
339
350
- template <Role role> LIBC_INLINE LockResult try_lock (State &old) {
340
+ template <Role role> LIBC_INLINE LockResult try_lock (RwState &old) {
351
341
if constexpr (role == Role::Reader) {
352
342
while (LIBC_LIKELY (old.can_acquire <Role::Reader>(get_preference ()))) {
353
- cpp::optional<State > next = old.try_increase_reader_count ();
343
+ cpp::optional<RwState > next = old.try_increase_reader_count ();
354
344
if (!next)
355
345
return LockResult::Overflow;
356
346
if (LIBC_LIKELY (old.compare_exchange_weak_with (
@@ -385,12 +375,12 @@ class RwLock {
385
375
386
376
[[nodiscard]]
387
377
LIBC_INLINE LockResult try_read_lock () {
388
- State old = State ::load (state, cpp::MemoryOrder::RELAXED);
378
+ RwState old = RwState ::load (state, cpp::MemoryOrder::RELAXED);
389
379
return try_lock<Role::Reader>(old);
390
380
}
391
381
[[nodiscard]]
392
382
LIBC_INLINE LockResult try_write_lock () {
393
- State old = State ::load (state, cpp::MemoryOrder::RELAXED);
383
+ RwState old = RwState ::load (state, cpp::MemoryOrder::RELAXED);
394
384
return try_lock<Role::Writer>(old);
395
385
}
396
386
@@ -412,7 +402,8 @@ class RwLock {
412
402
413
403
// Phase 3: spin to get the initial state. We ignore the timing due to
414
404
// spin since it should end quickly.
415
- State old = State::spin_reload<role>(state, get_preference (), spin_count);
405
+ RwState old =
406
+ RwState::spin_reload<role>(state, get_preference (), spin_count);
416
407
417
408
// Enter the main acquisition loop.
418
409
for (;;) {
@@ -421,7 +412,7 @@ class RwLock {
421
412
if (result != LockResult::Busy)
422
413
return result;
423
414
424
- // Phase 5: register ourselves as a pending reader.
415
+ // Phase 5: register ourselves as a reader.
425
416
int serial_number;
426
417
{
427
418
// The queue need to be protected by a mutex since the operations in
@@ -436,8 +427,8 @@ class RwLock {
436
427
// on the state. The pending flag update should be visible to any
437
428
// succeeding unlock events. Or, if a unlock does happen before we
438
429
// sleep on the futex, we can avoid such waiting.
439
- old = State ::fetch_set_pending_bit<role>(state,
440
- cpp::MemoryOrder::RELAXED);
430
+ old = RwState ::fetch_set_pending_bit<role>(state,
431
+ cpp::MemoryOrder::RELAXED);
441
432
// no need to use atomic since it is already protected by the mutex.
442
433
serial_number = guard.serialization <role>();
443
434
}
@@ -449,7 +440,7 @@ class RwLock {
449
440
timeout_flag = (queue.wait <role>(serial_number, timeout, is_pshared) ==
450
441
-ETIMEDOUT);
451
442
452
- // Phase 7: unregister ourselves as a pending reader.
443
+ // Phase 7: unregister ourselves as a pending reader/writer .
453
444
{
454
445
// Similarly, the unregister operation should also be an atomic
455
446
// transaction.
@@ -459,16 +450,16 @@ class RwLock {
459
450
// cleared otherwise operations like trylock may fail even though
460
451
// there is no competitors.
461
452
if (guard.pending_count <role>() == 0 )
462
- State ::fetch_clear_pending_bit<role>(state,
463
- cpp::MemoryOrder::RELAXED);
453
+ RwState ::fetch_clear_pending_bit<role>(state,
454
+ cpp::MemoryOrder::RELAXED);
464
455
}
465
456
466
457
// Phase 8: exit the loop is timeout is reached.
467
458
if (timeout_flag)
468
459
return LockResult::TimedOut;
469
460
470
461
// Phase 9: reload the state and retry the acquisition.
471
- old = State ::spin_reload<role>(state, get_preference (), spin_count);
462
+ old = RwState ::spin_reload<role>(state, get_preference (), spin_count);
472
463
}
473
464
}
474
465
@@ -522,7 +513,7 @@ class RwLock {
522
513
public:
523
514
[[nodiscard]]
524
515
LIBC_INLINE LockResult unlock () {
525
- State old = State ::load (state, cpp::MemoryOrder::RELAXED);
516
+ RwState old = RwState ::load (state, cpp::MemoryOrder::RELAXED);
526
517
if (old.has_active_writer ()) {
527
518
// The lock is held by a writer.
528
519
// Check if we are the owner of the lock.
@@ -531,14 +522,15 @@ class RwLock {
531
522
// clear writer tid.
532
523
writer_tid.store (0 , cpp::MemoryOrder::RELAXED);
533
524
// clear the writer bit.
534
- old = State::fetch_clear_active_writer (state, cpp::MemoryOrder::RELEASE);
525
+ old =
526
+ RwState::fetch_clear_active_writer (state, cpp::MemoryOrder::RELEASE);
535
527
// If there is no pending readers or writers, we are done.
536
528
if (!old.has_pending ())
537
529
return LockResult::Success;
538
530
} else if (old.has_active_reader ()) {
539
531
// The lock is held by readers.
540
532
// Decrease the reader count.
541
- old = State ::fetch_sub_reader_count (state, cpp::MemoryOrder::RELEASE);
533
+ old = RwState ::fetch_sub_reader_count (state, cpp::MemoryOrder::RELEASE);
542
534
// If there is no pending readers or writers, we are done.
543
535
if (!old.has_last_reader () || !old.has_pending ())
544
536
return LockResult::Success;
@@ -553,7 +545,7 @@ class RwLock {
553
545
// will only check if the lock is currently held by any thread.
554
546
[[nodiscard]]
555
547
LIBC_INLINE LockResult check_for_destroy () {
556
- State old = State ::load (state, cpp::MemoryOrder::RELAXED);
548
+ RwState old = RwState ::load (state, cpp::MemoryOrder::RELAXED);
557
549
if (old.has_acitve_owner ())
558
550
return LockResult::Busy;
559
551
return LockResult::Success;
0 commit comments