Merge pull request #35122 from mikeash/refcount-always-inline

[Runtime] Add SWIFT_ALWAYS_INLINE to various refcounting helpers.

Author: mikeash

stdlib/public/SwiftShims/RefCount.h

@@ -793,6 +793,7 @@ class RefCounts {
   }
 
   // Increment the reference count.
+  SWIFT_ALWAYS_INLINE
   void increment(uint32_t inc = 1) {
     auto oldbits = refCounts.load(SWIFT_MEMORY_ORDER_CONSUME);
 
@@ -815,6 +816,7 @@ class RefCounts {
                                               std::memory_order_relaxed));
   }
 
+  SWIFT_ALWAYS_INLINE
   void incrementNonAtomic(uint32_t inc = 1) {
     auto oldbits = refCounts.load(SWIFT_MEMORY_ORDER_CONSUME);
 
@@ -835,6 +837,7 @@ class RefCounts {
  }
 
   // Increment the reference count, unless the object is deiniting.
+  SWIFT_ALWAYS_INLINE
   bool tryIncrement() {
     auto oldbits = refCounts.load(SWIFT_MEMORY_ORDER_CONSUME);
     RefCountBits newbits;
@@ -854,6 +857,7 @@ class RefCounts {
     return true;
   }
 
+  SWIFT_ALWAYS_INLINE
   bool tryIncrementNonAtomic() {
     auto oldbits = refCounts.load(SWIFT_MEMORY_ORDER_CONSUME);
     if (!oldbits.hasSideTable() && oldbits.getIsDeiniting())
@@ -896,6 +900,7 @@ class RefCounts {
   // object as deiniting.
   //
   // Precondition: the reference count must be 1
+  SWIFT_ALWAYS_INLINE
   void decrementFromOneNonAtomic() {
     auto bits = refCounts.load(SWIFT_MEMORY_ORDER_CONSUME);
     if (bits.isImmortal(true)) {
@@ -1078,6 +1083,7 @@ class RefCounts {
   // Deinit is optionally handled directly instead of always deferring to 
   // the caller because the compiler can optimize this arrangement better.
   template <PerformDeinit performDeinit>
+  SWIFT_ALWAYS_INLINE
   bool doDecrement(uint32_t dec) {
     auto oldbits = refCounts.load(SWIFT_MEMORY_ORDER_CONSUME);
     RefCountBits newbits;

stdlib/public/runtime/HeapObject.cpp

@@ -60,6 +60,7 @@ using namespace swift;
 
 /// Returns true if the pointer passed to a native retain or release is valid.
 /// If false, the operation should immediately return.
+SWIFT_ALWAYS_INLINE
 static inline bool isValidPointerForNativeRetain(const void *p) {
 #if defined(__x86_64__) || defined(__arm64__) || defined(__aarch64__) || defined(_M_ARM64) || defined(__s390x__) || (defined(__powerpc64__) && defined(__LITTLE_ENDIAN__))
   // On these platforms, except s390x, the upper half of address space is reserved for the
@@ -330,6 +331,7 @@ HeapObject *swift::swift_allocEmptyBox() {
 extern "C" SWIFT_LIBRARY_VISIBILITY SWIFT_NOINLINE SWIFT_USED void
 _swift_release_dealloc(HeapObject *object);
 
+SWIFT_ALWAYS_INLINE
 static HeapObject *_swift_retain_(HeapObject *object) {
   SWIFT_RT_TRACK_INVOCATION(object, swift_retain);
   if (isValidPointerForNativeRetain(object))
@@ -356,6 +358,7 @@ HeapObject *swift::swift_nonatomic_retain(HeapObject *object) {
   return object;
 }
 
+SWIFT_ALWAYS_INLINE
 static HeapObject *_swift_retain_n_(HeapObject *object, uint32_t n) {
   SWIFT_RT_TRACK_INVOCATION(object, swift_retain_n);
   if (isValidPointerForNativeRetain(object))
@@ -382,6 +385,7 @@ HeapObject *swift::swift_nonatomic_retain_n(HeapObject *object, uint32_t n) {
   return object;
 }
 
+SWIFT_ALWAYS_INLINE
 static void _swift_release_(HeapObject *object) {
   SWIFT_RT_TRACK_INVOCATION(object, swift_release);
   if (isValidPointerForNativeRetain(object))
@@ -406,6 +410,7 @@ void swift::swift_nonatomic_release(HeapObject *object) {
     object->refCounts.decrementAndMaybeDeinitNonAtomic(1);
 }
 
+SWIFT_ALWAYS_INLINE
 static void _swift_release_n_(HeapObject *object, uint32_t n) {
   SWIFT_RT_TRACK_INVOCATION(object, swift_release_n);
   if (isValidPointerForNativeRetain(object))
@@ -562,6 +567,7 @@ void swift::swift_nonatomic_unownedRelease_n(HeapObject *object, int n) {
   }
 }
 
+SWIFT_ALWAYS_INLINE
 static HeapObject *_swift_tryRetain_(HeapObject *object) {
   SWIFT_RT_TRACK_INVOCATION(object, swift_tryRetain);
   if (!isValidPointerForNativeRetain(object))