Skip to content

Commit 69768ad

Browse files
mtakemtake
committed
Update for the new reference counting mechanism
1 parent 10163ae commit 69768ad

File tree

3 files changed

+137
-39
lines changed

3 files changed

+137
-39
lines changed

‎stdlib/public/SwiftShims/RefCount.h

Lines changed: 93 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -745,6 +745,9 @@ class RefCounts {
745745
LLVM_ATTRIBUTE_NOINLINE
746746
bool tryIncrementSlow(RefCountBits oldbits);
747747

748+
LLVM_ATTRIBUTE_NOINLINE
749+
bool tryIncrementNonAtomicSlow(RefCountBits oldbits);
750+
748751
public:
749752
enum Initialized_t { Initialized };
750753

@@ -859,16 +862,17 @@ class RefCounts {
859862
return true;
860863
}
861864

862-
// Increment the reference count, unless the object is deallocating.
863865
bool tryIncrementNonAtomic() {
864-
uint32_t oldval = __atomic_load_n(&refCount, __ATOMIC_RELAXED);
865-
if (oldval & RC_DEALLOCATING_FLAG) {
866+
auto oldbits = refCounts.load(SWIFT_MEMORY_ORDER_CONSUME);
867+
if (!oldbits.hasSideTable() && oldbits.getIsDeiniting())
866868
return false;
867-
} else {
868-
uint32_t newval = oldval + RC_ONE;
869-
__atomic_store_n(&refCount, newval, __ATOMIC_RELAXED);
870-
return true;
871-
}
869+
870+
auto newbits = oldbits;
871+
bool fast = newbits.incrementStrongExtraRefCount(1);
872+
if (!fast)
873+
return tryIncrementNonAtomicSlow(oldbits);
874+
refCounts.store(newbits, std::memory_order_relaxed);
875+
return true;
872876
}
873877

874878
// Simultaneously clear the pinned flag and decrement the reference
@@ -1089,6 +1093,18 @@ class RefCounts {
10891093
std::memory_order_relaxed));
10901094
}
10911095

1096+
void incrementUnownedNonAtomic(uint32_t inc) {
1097+
auto oldbits = refCounts.load(SWIFT_MEMORY_ORDER_CONSUME);
1098+
if (oldbits.hasSideTable())
1099+
return oldbits.getSideTable()->incrementUnownedNonAtomic(inc);
1100+
1101+
auto newbits = oldbits;
1102+
assert(newbits.getUnownedRefCount() != 0);
1103+
newbits.incrementUnownedRefCount(inc);
1104+
// FIXME: overflow check?
1105+
refCounts.store(newbits, std::memory_order_relaxed);
1106+
}
1107+
10921108
// Decrement the unowned reference count.
10931109
// Return true if the caller should free the object.
10941110
bool decrementUnownedShouldFree(uint32_t dec) {
@@ -1117,6 +1133,29 @@ class RefCounts {
11171133
return performFree;
11181134
}
11191135

1136+
bool decrementUnownedShouldFreeNonAtomic(uint32_t dec) {
1137+
auto oldbits = refCounts.load(SWIFT_MEMORY_ORDER_CONSUME);
1138+
1139+
if (oldbits.hasSideTable())
1140+
return oldbits.getSideTable()->decrementUnownedShouldFreeNonAtomic(dec);
1141+
1142+
bool performFree;
1143+
auto newbits = oldbits;
1144+
newbits.decrementUnownedRefCount(dec);
1145+
if (newbits.getUnownedRefCount() == 0) {
1146+
// DEINITED -> FREED or DEINITED -> DEAD
1147+
// Caller will free the object. Weak decrement is handled by
1148+
// HeapObjectSideTableEntry::decrementUnownedShouldFreeNonAtomic.
1149+
assert(newbits.getIsDeiniting());
1150+
performFree = true;
1151+
} else {
1152+
performFree = false;
1153+
}
1154+
// FIXME: underflow check?
1155+
refCounts.store(newbits, std::memory_order_relaxed);
1156+
return performFree;
1157+
}
1158+
11201159
// Return unowned reference count.
11211160
// Note that this is not equal to the number of outstanding unowned pointers.
11221161
uint32_t getUnownedCount() const {
@@ -1163,6 +1202,16 @@ class RefCounts {
11631202

11641203
return performFree;
11651204
}
1205+
1206+
bool decrementWeakShouldCleanUpNonAtomic() {
1207+
auto oldbits = refCounts.load(SWIFT_MEMORY_ORDER_CONSUME);
1208+
1209+
auto newbits = oldbits;
1210+
auto performFree = newbits.decrementWeakRefCount();
1211+
refCounts.store(newbits, std::memory_order_relaxed);
1212+
1213+
return performFree;
1214+
}
11661215

11671216
// Return weak reference count.
11681217
// Note that this is not equal to the number of outstanding weak pointers.
@@ -1260,6 +1309,14 @@ class HeapObjectSideTableEntry {
12601309
return refCounts.tryIncrementAndPin();
12611310
}
12621311

1312+
bool tryIncrementNonAtomic() {
1313+
return refCounts.tryIncrementNonAtomic();
1314+
}
1315+
1316+
bool tryIncrementAndPinNonAtomic() {
1317+
return refCounts.tryIncrementAndPinNonAtomic();
1318+
}
1319+
12631320
// Return weak reference count.
12641321
// Note that this is not equal to the number of outstanding weak pointers.
12651322
uint32_t getCount() const {
@@ -1276,6 +1333,10 @@ class HeapObjectSideTableEntry {
12761333
return refCounts.incrementUnowned(inc);
12771334
}
12781335

1336+
void incrementUnownedNonAtomic(uint32_t inc) {
1337+
return refCounts.incrementUnownedNonAtomic(inc);
1338+
}
1339+
12791340
bool decrementUnownedShouldFree(uint32_t dec) {
12801341
bool shouldFree = refCounts.decrementUnownedShouldFree(dec);
12811342
if (shouldFree) {
@@ -1287,6 +1348,17 @@ class HeapObjectSideTableEntry {
12871348
return shouldFree;
12881349
}
12891350

1351+
bool decrementUnownedShouldFreeNonAtomic(uint32_t dec) {
1352+
bool shouldFree = refCounts.decrementUnownedShouldFreeNonAtomic(dec);
1353+
if (shouldFree) {
1354+
// DEINITED -> FREED
1355+
// Caller will free the object.
1356+
decrementWeakNonAtomic();
1357+
}
1358+
1359+
return shouldFree;
1360+
}
1361+
12901362
uint32_t getUnownedCount() const {
12911363
return refCounts.getUnownedCount();
12921364
}
@@ -1320,6 +1392,19 @@ class HeapObjectSideTableEntry {
13201392
delete this;
13211393
}
13221394

1395+
void decrementWeakNonAtomic() {
1396+
// FIXME: assertions
1397+
// FIXME: optimize barriers
1398+
bool cleanup = refCounts.decrementWeakShouldCleanUpNonAtomic();
1399+
if (!cleanup)
1400+
return;
1401+
1402+
// Weak ref count is now zero. Delete the side table entry.
1403+
// FREED -> DEAD
1404+
assert(refCounts.getUnownedCount() == 0);
1405+
delete this;
1406+
}
1407+
13231408
uint32_t getWeakCount() const {
13241409
return refCounts.getWeakCount();
13251410
}

‎stdlib/public/runtime/HeapObject.cpp

Lines changed: 30 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -346,23 +346,24 @@ void swift::swift_unownedRelease(HeapObject *object)
346346

347347
void swift::swift_nonatomic_unownedRetain(HeapObject *object)
348348
SWIFT_CC(RegisterPreservingCC_IMPL) {
349-
if (!object)
349+
if (!isValidPointerForNativeRetain(object))
350350
return;
351351

352-
object->weakRefCount.incrementNonAtomic();
352+
object->refCounts.incrementUnownedNonAtomic(1);
353353
}
354354

355355
void swift::swift_nonatomic_unownedRelease(HeapObject *object)
356356
SWIFT_CC(RegisterPreservingCC_IMPL) {
357-
if (!object)
357+
if (!isValidPointerForNativeRetain(object))
358358
return;
359359

360-
if (object->weakRefCount.decrementShouldDeallocateNonAtomic()) {
361-
// Only class objects can be weak-retained and weak-released.
362-
auto metadata = object->metadata;
363-
assert(metadata->isClassObject());
364-
auto classMetadata = static_cast<const ClassMetadata*>(metadata);
365-
assert(classMetadata->isTypeMetadata());
360+
// Only class objects can be unowned-retained and unowned-released.
361+
assert(object->metadata->isClassObject());
362+
assert(static_cast<const ClassMetadata*>(object->metadata)->isTypeMetadata());
363+
364+
if (object->refCounts.decrementUnownedShouldFreeNonAtomic(1)) {
365+
auto classMetadata = static_cast<const ClassMetadata*>(object->metadata);
366+
366367
SWIFT_RT_ENTRY_CALL(swift_slowDealloc)
367368
(object, classMetadata->getInstanceSize(),
368369
classMetadata->getInstanceAlignMask());
@@ -396,23 +397,23 @@ void swift::swift_unownedRelease_n(HeapObject *object, int n)
396397

397398
void swift::swift_nonatomic_unownedRetain_n(HeapObject *object, int n)
398399
SWIFT_CC(RegisterPreservingCC_IMPL) {
399-
if (!object)
400+
if (!isValidPointerForNativeRetain(object))
400401
return;
401402

402-
object->weakRefCount.incrementNonAtomic(n);
403+
object->refCounts.incrementUnownedNonAtomic(n);
403404
}
404405

405406
void swift::swift_nonatomic_unownedRelease_n(HeapObject *object, int n)
406407
SWIFT_CC(RegisterPreservingCC_IMPL) {
407-
if (!object)
408+
if (!isValidPointerForNativeRetain(object))
408409
return;
409410

410-
if (object->weakRefCount.decrementShouldDeallocateNNonAtomic(n)) {
411-
// Only class objects can be weak-retained and weak-released.
412-
auto metadata = object->metadata;
413-
assert(metadata->isClassObject());
414-
auto classMetadata = static_cast<const ClassMetadata*>(metadata);
415-
assert(classMetadata->isTypeMetadata());
411+
// Only class objects can be unowned-retained and unowned-released.
412+
assert(object->metadata->isClassObject());
413+
assert(static_cast<const ClassMetadata*>(object->metadata)->isTypeMetadata());
414+
415+
if (object->refCounts.decrementUnownedShouldFreeNonAtomic(n)) {
416+
auto classMetadata = static_cast<const ClassMetadata*>(object->metadata);
416417
SWIFT_RT_ENTRY_CALL(swift_slowDealloc)
417418
(object, classMetadata->getInstanceSize(),
418419
classMetadata->getInstanceAlignMask());
@@ -501,13 +502,13 @@ void swift::swift_unownedRetainStrong(HeapObject *object)
501502

502503
void swift::swift_nonatomic_unownedRetainStrong(HeapObject *object)
503504
SWIFT_CC(RegisterPreservingCC_IMPL) {
504-
if (!object)
505+
if (!isValidPointerForNativeRetain(object))
505506
return;
506-
assert(object->weakRefCount.getCount() &&
507-
"object is not currently weakly retained");
507+
assert(object->refCounts.getUnownedCount() &&
508+
"object is not currently unowned-retained");
508509

509-
if (! object->refCount.tryIncrementNonAtomic())
510-
_swift_abortRetainUnowned(object);
510+
if (! object->refCounts.tryIncrementNonAtomic())
511+
swift::swift_abortRetainUnowned(object);
511512
}
512513

513514
void swift::swift_unownedRetainStrongAndRelease(HeapObject *object)
@@ -528,16 +529,16 @@ void swift::swift_unownedRetainStrongAndRelease(HeapObject *object)
528529

529530
void swift::swift_nonatomic_unownedRetainStrongAndRelease(HeapObject *object)
530531
SWIFT_CC(RegisterPreservingCC_IMPL) {
531-
if (!object)
532+
if (!isValidPointerForNativeRetain(object))
532533
return;
533-
assert(object->weakRefCount.getCount() &&
534-
"object is not currently weakly retained");
534+
assert(object->refCounts.getUnownedCount() &&
535+
"object is not currently unowned-retained");
535536

536-
if (! object->refCount.tryIncrementNonAtomic())
537-
_swift_abortRetainUnowned(object);
537+
if (! object->refCounts.tryIncrementNonAtomic())
538+
swift::swift_abortRetainUnowned(object);
538539

539540
// This should never cause a deallocation.
540-
bool dealloc = object->weakRefCount.decrementShouldDeallocateNonAtomic();
541+
bool dealloc = object->refCounts.decrementUnownedShouldFreeNonAtomic(1);
541542
assert(!dealloc && "retain-strong-and-release caused dealloc?");
542543
(void) dealloc;
543544
}

‎stdlib/public/runtime/RefCount.cpp

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,16 @@ bool RefCounts<RefCountBits>::tryIncrementSlow(RefCountBits oldbits) {
5454
template bool RefCounts<InlineRefCountBits>::tryIncrementSlow(InlineRefCountBits oldbits);
5555
template bool RefCounts<SideTableRefCountBits>::tryIncrementSlow(SideTableRefCountBits oldbits);
5656

57+
template <typename RefCountBits>
58+
bool RefCounts<RefCountBits>::tryIncrementNonAtomicSlow(RefCountBits oldbits) {
59+
if (oldbits.hasSideTable())
60+
return oldbits.getSideTable()->tryIncrementNonAtomic();
61+
else
62+
swift::swift_abortRetainOverflow();
63+
}
64+
template bool RefCounts<InlineRefCountBits>::tryIncrementNonAtomicSlow(InlineRefCountBits oldbits);
65+
template bool RefCounts<SideTableRefCountBits>::tryIncrementNonAtomicSlow(SideTableRefCountBits oldbits);
66+
5767
template <typename RefCountBits>
5868
bool RefCounts<RefCountBits>::tryIncrementAndPinSlow(RefCountBits oldbits) {
5969
if (oldbits.hasSideTable())
@@ -66,8 +76,10 @@ template bool RefCounts<SideTableRefCountBits>::tryIncrementAndPinSlow(SideTable
6676

6777
template <typename RefCountBits>
6878
bool RefCounts<RefCountBits>::tryIncrementAndPinNonAtomicSlow(RefCountBits oldbits) {
69-
// No nonatomic implementation provided.
70-
return tryIncrementAndPinSlow(oldbits);
79+
if (oldbits.hasSideTable())
80+
return oldbits.getSideTable()->tryIncrementAndPinNonAtomic();
81+
else
82+
swift::swift_abortRetainOverflow();
7183
}
7284
template bool RefCounts<InlineRefCountBits>::tryIncrementAndPinNonAtomicSlow(InlineRefCountBits oldbits);
7385
template bool RefCounts<SideTableRefCountBits>::tryIncrementAndPinNonAtomicSlow(SideTableRefCountBits oldbits);

0 commit comments

Comments
 (0)