Track the identity of the task draining the actor in the

ActiveActorStatus

Radar-Id: rdar://problem/86100521

Author: rokhinip

stdlib/public/Concurrency/Actor.cpp

@@ -528,8 +528,64 @@ class JobRef {
   }
 };
 
-/// This is designed to fit into two words, which can generally be
-/// done lock-free on all our supported platforms.
+/// Similar to the ActiveTaskStatus, this denotes the ActiveActorState for
+/// tracking the atomic state of the actor
+///
+/// The runtime needs to track the following state about the actor within the
+/// same atomic:
+///
+/// * The current status of the actor - scheduled, running, idle, etc
+/// * The current maximum priority of the jobs enqueued in the actor
+/// * The identity of the thread currently holding the actor lock
+/// * Pointer to list of jobs enqueued in actor
+///
+/// It is important for all of this information to be in the same atomic so that
+/// when the actor's state changes, the information is visible to all threads that
+/// may be modifying the actor, allowing the algorithm to eventually converge.
+///
+/// In order to provide priority escalation support with actors, deeper integration is
+/// required with the OS in order to have the intended side effects. On Darwin, Swift
+/// Concurrency Tasks runs on dispatch's queues. As such, we need to use an
+/// encoding of thread identity vended by libdispatch called dispatch_lock_t,
+/// and a futex-style dispatch API in order to escalate the priority of a
+/// thread. Henceforth, the dispatch_lock_t tracked in the ActiveActorStatus
+/// will be called the DrainLock.
+///
+/// When a thread starts running on an actor, it's identity is recorded in the
+/// ActiveActorStatus. This way, if a higher priority job is enqueued behind the
+/// thread executing the actor, we can escalate the thread holding the actor
+/// lock, thereby resolving the priority inversion. When a thread hops off of
+/// the actor, any priority boosts it may have gotten as a result of contention
+/// on the actor, is removed as well.
+///
+/// In order to keep the entire ActiveActorStatus size to 2 words, the thread
+/// identity is only tracked on platforms which can support 128 bit atomic
+/// operations. The ActiveActorStatus's layout has thus been changed to have the
+/// following layout depending on the system configuration supported:
+///
+/// 32 bit systems with SWIFT_CONCURRENCY_ENABLE_PRIORITY_ESCALATION=1
+///
+///          Flags               Drain Lock               Unused                JobRef
+/// |----------------------|----------------------|----------------------|-------------------|
+///          32 bits                32 bits                32 bits              32 bits
+///
+/// 64 bit systems with SWIFT_CONCURRENCY_ENABLE_PRIORITY_ESCALATION=1
+///
+///         Flags                Drain Lock             JobRef
+/// |----------------------|-------------------|----------------------|
+///          32 bits                32 bits             64 bits
+///
+/// 32 bit systems with SWIFT_CONCURRENCY_ENABLE_PRIORITY_ESCALATION=0
+///
+///          Flags                  JobRef
+/// |----------------------|----------------------|
+///          32 bits                32 bits
+//
+/// 64 bit systems with SWIFT_CONCURRENCY_ENABLE_PRIORITY_ESCALATION=0
+///
+///         Flags                  Unused                 JobRef
+/// |----------------------|----------------------|---------------------|
+///         32 bits                 32 bits               64 bits
 class alignas(sizeof(void *) * 2) ActiveActorStatus {
   enum : uint32_t {
     // Bits 0-2: Actor state
@@ -567,11 +623,29 @@ class alignas(sizeof(void *) * 2) ActiveActorStatus {
     PriorityMask = 0xFF00,
     PriorityShift = 0x8,
   };
+
+#if SWIFT_CONCURRENCY_ENABLE_PRIORITY_ESCALATION && SWIFT_POINTER_IS_4_BYTES
+  uint32_t Flags;
+  dispatch_lock_t DrainLock;
+  uint32_t Unused;
+#elif SWIFT_CONCURRENCY_ENABLE_PRIORITY_ESCALATION && SWIFT_POINTER_IS_8_BYTES
   uint32_t Flags;
+  dispatch_lock_t DrainLock;
+#elif !SWIFT_CONCURRENCY_ENABLE_PRIORITY_ESCALATION && SWIFT_POINTER_IS_4_BYTES
+  uint32_t Flags;
+#else /* !SWIFT_CONCURRENCY_ENABLE_PRIORITY_ESCALATION && SWIFT_POINTER_IS_8_BYTES */
+  uint32_t Flags;
+  uint32_t Unused;
+#endif
   JobRef FirstJob;
 
+#if SWIFT_CONCURRENCY_ENABLE_PRIORITY_ESCALATION
+  ActiveActorStatus(uint32_t flags, dispatch_lock_t drainLockValue, JobRef job)
+    : Flags(flags), DrainLock(drainLockValue), FirstJob(job) {}
+#else
   ActiveActorStatus(uint32_t flags, JobRef job)
     : Flags(flags), FirstJob(job) {}
+#endif
 
 public:
 #ifdef __GLIBCXX__
@@ -582,11 +656,15 @@ class alignas(sizeof(void *) * 2) ActiveActorStatus {
 #endif
 
   constexpr ActiveActorStatus()
-      : Flags(), FirstJob(JobRef()) {}
+      : Flags(), DrainLock(DLOCK_OWNER_NULL), FirstJob(JobRef()) {}
 
   bool isDistributedRemote() const { return Flags & DistributedRemote; }
   ActiveActorStatus withDistributedRemote() const {
+#if SWIFT_CONCURRENCY_ENABLE_PRIORITY_ESCALATION
+    return ActiveActorStatus(Flags | DistributedRemote, DrainLock, FirstJob);
+#else
     return ActiveActorStatus(Flags | DistributedRemote, FirstJob);
+#endif
   }
 
   bool isIdle() const {
@@ -597,7 +675,11 @@ class alignas(sizeof(void *) * 2) ActiveActorStatus {
     return isIdle;
   }
   ActiveActorStatus withIdle() const {
+#if SWIFT_CONCURRENCY_ENABLE_PRIORITY_ESCALATION
+    return ActiveActorStatus((Flags & ~ActorStateMask) | Idle, DLOCK_OWNER_NULL, FirstJob);
+#else
     return ActiveActorStatus((Flags & ~ActorStateMask) | Idle, FirstJob);
+#endif
   }
 
   bool isAnyRunning() const {
@@ -609,44 +691,78 @@ class alignas(sizeof(void *) * 2) ActiveActorStatus {
   }
   ActiveActorStatus withRunning() const {
     uint32_t flags = (Flags & ~ActorStateMask) | Running;
+#if SWIFT_CONCURRENCY_ENABLE_PRIORITY_ESCALATION
+    return ActiveActorStatus(flags, dispatch_lock_value_for_self(), FirstJob);
+#else
     return ActiveActorStatus(flags, FirstJob);
+#endif
   }
 
   bool isScheduled() const {
     return (Flags & ActorStateMask) == Scheduled;
   }
+
   ActiveActorStatus withScheduled() const {
+#if SWIFT_CONCURRENCY_ENABLE_PRIORITY_ESCALATION
+    return ActiveActorStatus((Flags & ~ActorStateMask) | Scheduled, DLOCK_OWNER_NULL, FirstJob);
+#else
     return ActiveActorStatus((Flags & ~ActorStateMask) | Scheduled, FirstJob);
+#endif
   }
 
   bool isZombie_ReadyForDeallocation() const {
     return (Flags & ActorStateMask) == Zombie_ReadyForDeallocation;
   }
   ActiveActorStatus withZombie_ReadyForDeallocation() const {
+#if SWIFT_CONCURRENCY_ENABLE_PRIORITY_ESCALATION
+    assert(dispatch_lock_owner(DrainLock) != DLOCK_OWNER_NULL);
+    return ActiveActorStatus((Flags & ~ActorStateMask) | Zombie_ReadyForDeallocation, DrainLock, FirstJob);
+#else
     return ActiveActorStatus((Flags & ~ActorStateMask) | Zombie_ReadyForDeallocation, FirstJob);
+#endif
   }
 
   JobPriority getMaxPriority() const {
     return (JobPriority) ((Flags & PriorityMask) >> PriorityShift);
   }
   ActiveActorStatus withMaxPriority(JobPriority priority) const {
-    return ActiveActorStatus((Flags & ~PriorityMask) | (uint32_t(priority) << PriorityShift) , FirstJob);
+#if SWIFT_CONCURRENCY_ENABLE_PRIORITY_ESCALATION
+    return ActiveActorStatus((Flags & ~PriorityMask) | (uint32_t(priority) << PriorityShift), DrainLock, FirstJob);
+#else
+    return ActiveActorStatus((Flags & ~PriorityMask) | (uint32_t(priority) << PriorityShift), FirstJob);
+#endif
   }
 
   JobRef getFirstJob() const {
     return FirstJob;
   }
   ActiveActorStatus withFirstJob(JobRef firstJob) const {
+#if SWIFT_CONCURRENCY_ENABLE_PRIORITY_ESCALATION
+    return ActiveActorStatus(Flags, DrainLock, firstJob);
+#else
     return ActiveActorStatus(Flags, firstJob);
+#endif
   }
 
   uint32_t getOpaqueFlags() const {
     return Flags;
   }
+  uint32_t currentDrainer() const {
+#if SWIFT_CONCURRENCY_ENABLE_PRIORITY_ESCALATION
+    return dispatch_lock_owner(DrainLock);
+#else
+    return 0;
+#endif
+  }
 };
 
+#if SWIFT_CONCURRENCY_ENABLE_PRIORITY_ESCALATION && SWIFT_POINTER_IS_4_BYTES
+static_assert(sizeof(ActiveActorStatus) == 4 * sizeof(uintptr_t),
+  "ActiveActorStatus is 4 words large");
+#else
 static_assert(sizeof(ActiveActorStatus) == 2 * sizeof(uintptr_t),
-  "ActiveTaskStatus is 2 words large");
+  "ActiveActorStatus is 2 words large");
+#endif
 
 /// The default actor implementation.
 ///