Skip to content

Extract RefCount implementation from header to cpp file #6827

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

Extract RefCount implementation from header to cpp file #6827

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
297 changes: 26 additions & 271 deletions stdlib/public/SwiftShims/RefCount.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,26 +91,14 @@ class StrongRefCount {
}

// Increment the reference count.
void increment() {
__atomic_fetch_add(&refCount, RC_ONE, __ATOMIC_RELAXED);
}
void increment();

void incrementNonAtomic() {
uint32_t val = __atomic_load_n(&refCount, __ATOMIC_RELAXED);
val += RC_ONE;
__atomic_store_n(&refCount, val, __ATOMIC_RELAXED);
}
void incrementNonAtomic();

// Increment the reference count by n.
void increment(uint32_t n) {
__atomic_fetch_add(&refCount, n << RC_FLAGS_COUNT, __ATOMIC_RELAXED);
}
void increment(uint32_t n);

void incrementNonAtomic(uint32_t n) {
uint32_t val = __atomic_load_n(&refCount, __ATOMIC_RELAXED);
val += n << RC_FLAGS_COUNT;
__atomic_store_n(&refCount, val, __ATOMIC_RELAXED);
}
void incrementNonAtomic(uint32_t n);

// Try to simultaneously set the pinned flag and increment the
// reference count. If the flag is already set, don't increment the
Expand All @@ -121,93 +109,39 @@ class StrongRefCount {
// Returns true if the flag was set by this operation.
//
// Postcondition: the flag is set.
bool tryIncrementAndPin() {
uint32_t oldval = __atomic_load_n(&refCount, __ATOMIC_RELAXED);
while (true) {
// If the flag is already set, just fail.
if (oldval & RC_PINNED_FLAG) {
return false;
}

// Try to simultaneously set the flag and increment the reference count.
uint32_t newval = oldval + (RC_PINNED_FLAG + RC_ONE);
if (__atomic_compare_exchange(&refCount, &oldval, &newval, 0,
__ATOMIC_RELAXED, __ATOMIC_RELAXED)) {
return true;
}

// Try again; oldval has been updated with the value we saw.
}
}
bool tryIncrementAndPin();

bool tryIncrementAndPinNonAtomic() {
uint32_t oldval = __atomic_load_n(&refCount, __ATOMIC_RELAXED);
// If the flag is already set, just fail.
if (oldval & RC_PINNED_FLAG) {
return false;
}

// Try to simultaneously set the flag and increment the reference count.
uint32_t newval = oldval + (RC_PINNED_FLAG + RC_ONE);
__atomic_store_n(&refCount, newval, __ATOMIC_RELAXED);
return true;
}
bool tryIncrementAndPinNonAtomic();

// Increment the reference count, unless the object is deallocating.
bool tryIncrement() {
// FIXME: this could be better on LL/SC architectures like arm64
uint32_t oldval = __atomic_fetch_add(&refCount, RC_ONE, __ATOMIC_RELAXED);
if (oldval & RC_DEALLOCATING_FLAG) {
__atomic_fetch_sub(&refCount, RC_ONE, __ATOMIC_RELAXED);
return false;
} else {
return true;
}
}
bool tryIncrement();

// Simultaneously clear the pinned flag and decrement the reference
// count.
//
// Precondition: the pinned flag is set.
bool decrementAndUnpinShouldDeallocate() {
return doDecrementShouldDeallocate<true>();
}
bool decrementAndUnpinShouldDeallocate();

bool decrementAndUnpinShouldDeallocateNonAtomic() {
return doDecrementShouldDeallocateNonAtomic<true>();
}
bool decrementAndUnpinShouldDeallocateNonAtomic();

// Decrement the reference count.
// Return true if the caller should now deallocate the object.
bool decrementShouldDeallocate() {
return doDecrementShouldDeallocate<false>();
}
bool decrementShouldDeallocate();

bool decrementShouldDeallocateNonAtomic() {
return doDecrementShouldDeallocateNonAtomic<false>();
}
bool decrementShouldDeallocateNonAtomic();

bool decrementShouldDeallocateN(uint32_t n) {
return doDecrementShouldDeallocateN<false>(n);
}
bool decrementShouldDeallocateN(uint32_t n);

// Set the RC_DEALLOCATING_FLAG flag non-atomically.
//
// Precondition: the reference count must be 1
void decrementFromOneAndDeallocateNonAtomic() {
assert(refCount == RC_ONE && "Expect a count of 1");
__atomic_store_n(&refCount, RC_DEALLOCATING_FLAG, __ATOMIC_RELAXED);
}
void decrementFromOneAndDeallocateNonAtomic();

bool decrementShouldDeallocateNNonAtomic(uint32_t n) {
return doDecrementShouldDeallocateNNonAtomic<false>(n);
}
bool decrementShouldDeallocateNNonAtomic(uint32_t n);

// Return the reference count.
// During deallocation the reference count is undefined.
uint32_t getCount() const {
return __atomic_load_n(&refCount, __ATOMIC_RELAXED) >> RC_FLAGS_COUNT;
}
uint32_t getCount() const;

// Return whether the reference count is exactly 1.
// During deallocation the reference count is undefined.
Expand All @@ -217,178 +151,23 @@ class StrongRefCount {

// Return whether the reference count is exactly 1 or the pin flag
// is set. During deallocation the reference count is undefined.
bool isUniquelyReferencedOrPinned() const {
auto value = __atomic_load_n(&refCount, __ATOMIC_RELAXED);
// Rotating right by one sets the sign bit to the pinned bit. After
// rotation, the dealloc flag is the least significant bit followed by the
// reference count. A reference count of two or higher means that our value
// is bigger than 3 if the pinned bit is not set. If the pinned bit is set
// the value is negative.
// Note: Because we are using the sign bit for testing pinnedness it
// is important to do a signed comparison below.
static_assert(RC_PINNED_FLAG == 1,
"The pinned flag must be the lowest bit");
auto rotateRightByOne = ((value >> 1) | (value << 31));
return (int32_t)rotateRightByOne < (int32_t)RC_ONE;
}
bool isUniquelyReferencedOrPinned() const;

// Return true if the object is inside deallocation.
bool isDeallocating() const {
return __atomic_load_n(&refCount, __ATOMIC_RELAXED) & RC_DEALLOCATING_FLAG;
}
bool isDeallocating() const;

private:
template <bool ClearPinnedFlag>
bool doDecrementShouldDeallocate() {
// If we're being asked to clear the pinned flag, we can assume
// it's already set.
constexpr uint32_t quantum =
(ClearPinnedFlag ? RC_ONE + RC_PINNED_FLAG : RC_ONE);
uint32_t newval = __atomic_sub_fetch(&refCount, quantum, __ATOMIC_RELEASE);

assert((!ClearPinnedFlag || !(newval & RC_PINNED_FLAG)) &&
"unpinning reference that was not pinned");
assert(newval + quantum >= RC_ONE &&
"releasing reference with a refcount of zero");

// If we didn't drop the reference count to zero, or if the
// deallocating flag is already set, we're done; don't start
// deallocation. We can assume that the pinned flag isn't set
// unless the refcount is nonzero, and or'ing it in gives us a
// more efficient mask: the check just becomes "is newval nonzero".
if ((newval & (RC_COUNT_MASK | RC_PINNED_FLAG | RC_DEALLOCATING_FLAG))
!= 0) {
// Refcount is not zero. We definitely do not need to deallocate.
return false;
}

// Refcount is now 0 and is not already deallocating. Try to set
// the deallocating flag. This must be atomic because it can race
// with weak retains.
//
// This also performs the before-deinit acquire barrier if we set the flag.
static_assert(RC_FLAGS_COUNT == 2,
"fix decrementShouldDeallocate() if you add more flags");
uint32_t oldval = 0;
newval = RC_DEALLOCATING_FLAG;
return __atomic_compare_exchange(&refCount, &oldval, &newval, 0,
__ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
}
bool doDecrementShouldDeallocate();

template <bool ClearPinnedFlag>
bool doDecrementShouldDeallocateNonAtomic() {
// If we're being asked to clear the pinned flag, we can assume
// it's already set.
constexpr uint32_t quantum =
(ClearPinnedFlag ? RC_ONE + RC_PINNED_FLAG : RC_ONE);
uint32_t val = __atomic_load_n(&refCount, __ATOMIC_RELAXED);
val -= quantum;
__atomic_store_n(&refCount, val, __ATOMIC_RELEASE);
uint32_t newval = refCount;

assert((!ClearPinnedFlag || !(newval & RC_PINNED_FLAG)) &&
"unpinning reference that was not pinned");
assert(newval + quantum >= RC_ONE &&
"releasing reference with a refcount of zero");

// If we didn't drop the reference count to zero, or if the
// deallocating flag is already set, we're done; don't start
// deallocation. We can assume that the pinned flag isn't set
// unless the refcount is nonzero, and or'ing it in gives us a
// more efficient mask: the check just becomes "is newval nonzero".
if ((newval & (RC_COUNT_MASK | RC_PINNED_FLAG | RC_DEALLOCATING_FLAG))
!= 0) {
// Refcount is not zero. We definitely do not need to deallocate.
return false;
}

// Refcount is now 0 and is not already deallocating. Try to set
// the deallocating flag. This must be atomic because it can race
// with weak retains.
//
// This also performs the before-deinit acquire barrier if we set the flag.
static_assert(RC_FLAGS_COUNT == 2,
"fix decrementShouldDeallocate() if you add more flags");
uint32_t oldval = 0;
newval = RC_DEALLOCATING_FLAG;
return __atomic_compare_exchange(&refCount, &oldval, &newval, 0,
__ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
}
bool doDecrementShouldDeallocateNonAtomic();

template <bool ClearPinnedFlag>
bool doDecrementShouldDeallocateN(uint32_t n) {
// If we're being asked to clear the pinned flag, we can assume
// it's already set.
uint32_t delta = (n << RC_FLAGS_COUNT) + (ClearPinnedFlag ? RC_PINNED_FLAG : 0);
uint32_t newval = __atomic_sub_fetch(&refCount, delta, __ATOMIC_RELEASE);

assert((!ClearPinnedFlag || !(newval & RC_PINNED_FLAG)) &&
"unpinning reference that was not pinned");
assert(newval + delta >= RC_ONE &&
"releasing reference with a refcount of zero");

// If we didn't drop the reference count to zero, or if the
// deallocating flag is already set, we're done; don't start
// deallocation. We can assume that the pinned flag isn't set
// unless the refcount is nonzero, and or'ing it in gives us a
// more efficient mask: the check just becomes "is newval nonzero".
if ((newval & (RC_COUNT_MASK | RC_PINNED_FLAG | RC_DEALLOCATING_FLAG))
!= 0) {
// Refcount is not zero. We definitely do not need to deallocate.
return false;
}

// Refcount is now 0 and is not already deallocating. Try to set
// the deallocating flag. This must be atomic because it can race
// with weak retains.
//
// This also performs the before-deinit acquire barrier if we set the flag.
static_assert(RC_FLAGS_COUNT == 2,
"fix decrementShouldDeallocate() if you add more flags");
uint32_t oldval = 0;
newval = RC_DEALLOCATING_FLAG;
return __atomic_compare_exchange(&refCount, &oldval, &newval, 0,
__ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
}
bool doDecrementShouldDeallocateN(uint32_t n);

template <bool ClearPinnedFlag>
bool doDecrementShouldDeallocateNNonAtomic(uint32_t n) {
// If we're being asked to clear the pinned flag, we can assume
// it's already set.
uint32_t delta = (n << RC_FLAGS_COUNT) + (ClearPinnedFlag ? RC_PINNED_FLAG : 0);
uint32_t val = __atomic_load_n(&refCount, __ATOMIC_RELAXED);
val -= delta;
__atomic_store_n(&refCount, val, __ATOMIC_RELEASE);
uint32_t newval = val;

assert((!ClearPinnedFlag || !(newval & RC_PINNED_FLAG)) &&
"unpinning reference that was not pinned");
assert(newval + delta >= RC_ONE &&
"releasing reference with a refcount of zero");

// If we didn't drop the reference count to zero, or if the
// deallocating flag is already set, we're done; don't start
// deallocation. We can assume that the pinned flag isn't set
// unless the refcount is nonzero, and or'ing it in gives us a
// more efficient mask: the check just becomes "is newval nonzero".
if ((newval & (RC_COUNT_MASK | RC_PINNED_FLAG | RC_DEALLOCATING_FLAG))
!= 0) {
// Refcount is not zero. We definitely do not need to deallocate.
return false;
}

// Refcount is now 0 and is not already deallocating. Try to set
// the deallocating flag. This must be atomic because it can race
// with weak retains.
//
// This also performs the before-deinit acquire barrier if we set the flag.
static_assert(RC_FLAGS_COUNT == 2,
"fix decrementShouldDeallocate() if you add more flags");
uint32_t oldval = 0;
newval = RC_DEALLOCATING_FLAG;
return __atomic_compare_exchange(&refCount, &oldval, &newval, 0,
__ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
}
bool doDecrementShouldDeallocateNNonAtomic(uint32_t n);
};


Expand Down Expand Up @@ -436,46 +215,22 @@ class WeakRefCount {
}

// Increment the weak reference count.
void increment() {
uint32_t newval = __atomic_add_fetch(&refCount, RC_ONE, __ATOMIC_RELAXED);
assert(newval >= RC_ONE && "weak refcount overflow");
(void)newval;
}
void increment();

/// Increment the weak reference count by n.
void increment(uint32_t n) {
uint32_t addval = (n << RC_FLAGS_COUNT);
uint32_t newval = __atomic_add_fetch(&refCount, addval, __ATOMIC_RELAXED);
assert(newval >= addval && "weak refcount overflow");
(void)newval;
}
void increment(uint32_t n);

// Decrement the weak reference count.
// Return true if the caller should deallocate the object.
bool decrementShouldDeallocate() {
uint32_t oldval = __atomic_fetch_sub(&refCount, RC_ONE, __ATOMIC_RELAXED);
assert(oldval >= RC_ONE && "weak refcount underflow");

// Should dealloc if count was 1 before decrementing (i.e. it is zero now)
return (oldval & RC_COUNT_MASK) == RC_ONE;
}
bool decrementShouldDeallocate();

/// Decrement the weak reference count.
/// Return true if the caller should deallocate the object.
bool decrementShouldDeallocateN(uint32_t n) {
uint32_t subval = (n << RC_FLAGS_COUNT);
uint32_t oldval = __atomic_fetch_sub(&refCount, subval, __ATOMIC_RELAXED);
assert(oldval >= subval && "weak refcount underflow");

// Should dealloc if count was subval before decrementing (i.e. it is zero now)
return (oldval & RC_COUNT_MASK) == subval;
}
bool decrementShouldDeallocateN(uint32_t n);

// Return weak reference count.
// Note that this is not equal to the number of outstanding weak pointers.
uint32_t getCount() const {
return __atomic_load_n(&refCount, __ATOMIC_RELAXED) >> RC_FLAGS_COUNT;
}
uint32_t getCount() const;
};

static_assert(swift::IsTriviallyConstructible<StrongRefCount>::value,
Expand Down
1 change: 1 addition & 0 deletions stdlib/public/runtime/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ set(swift_runtime_sources
Once.cpp
Portability.cpp
ProtocolConformance.cpp
RefCount.cpp
RuntimeEntrySymbols.cpp)

# Acknowledge that the following sources are known.
Expand Down
Loading