[CoroutineAccessors] Dispatch and PtrAuth.

docs/ABI/Mangling.rst

@@ -235,10 +235,10 @@ types where the metadata itself has unknown layout.)
   global ::= global 'TF'                 // distributed method accessor
   global ::= global 'TI'                 // implementation of a dynamic_replaceable function
   global ::= global 'Tu'                 // async function pointer of a function
-  global ::= global 'Tv'                 // coro function pointer of a function
   global ::= global 'TX'                 // function pointer of a dynamic_replaceable function
   global ::= global 'Twb'                // back deployment thunk
   global ::= global 'TwB'                // back deployment fallback function
+  global ::= global 'Twc'                // coro function pointer of a function
   global ::= entity entity 'TV'          // vtable override thunk, derived followed by base
   global ::= type label-list? 'D'        // type mangling for the debugger with label list for function types.
   global ::= type 'TC'                   // continuation prototype (not actually used for real symbols)

include/swift/ABI/Metadata.h

@@ -5094,9 +5094,9 @@ struct DynamicReplacementKey {
   uint16_t getExtraDiscriminator() const {
     return flags & 0x0000FFFF;
   }
-  bool isAsync() const {
-    return ((flags >> 16 ) & 0x1);
-  }
+  bool isAsync() const { return ((flags >> 16) & 0x1); }
+  bool isCalleeAllocatedCoroutine() const { return ((flags >> 16) & 0x2); }
+  bool isData() const { return isAsync() || isCalleeAllocatedCoroutine(); }
 };
 
 /// A record describing a dynamic function replacement.

include/swift/ABI/MetadataValues.h

@@ -438,6 +438,23 @@ class MethodDescriptorFlags {
 
   bool isAsync() const { return Value & IsAsyncMask; }
 
+  bool isCalleeAllocatedCoroutine() const {
+    switch (getKind()) {
+    case Kind::Method:
+    case Kind::Init:
+    case Kind::Getter:
+    case Kind::Setter:
+    case Kind::ModifyCoroutine:
+    case Kind::ReadCoroutine:
+      return false;
+    case Kind::Read2Coroutine:
+    case Kind::Modify2Coroutine:
+      return true;
+    }
+  }
+
+  bool isData() const { return isAsync() || isCalleeAllocatedCoroutine(); }
+
   uint16_t getExtraDiscriminator() const {
     return (Value >> ExtraDiscriminatorShift);
   }
@@ -649,6 +666,26 @@ class ProtocolRequirementFlags {
 
   bool isAsync() const { return Value & IsAsyncMask; }
 
+  bool isCalleeAllocatedCoroutine() const {
+    switch (getKind()) {
+    case Kind::BaseProtocol:
+    case Kind::Method:
+    case Kind::Init:
+    case Kind::Getter:
+    case Kind::Setter:
+    case Kind::ReadCoroutine:
+    case Kind::ModifyCoroutine:
+    case Kind::AssociatedTypeAccessFunction:
+    case Kind::AssociatedConformanceAccessFunction:
+      return false;
+    case Kind::Read2Coroutine:
+    case Kind::Modify2Coroutine:
+      return true;
+    }
+  }
+
+  bool isData() const { return isAsync() || isCalleeAllocatedCoroutine(); }
+
   bool isSignedWithAddress() const {
     return getKind() != Kind::BaseProtocol;
   }

include/swift/AST/IRGenOptions.h

@@ -230,6 +230,26 @@ struct PointerAuthOptions : clang::PointerAuthOptions {
 
   /// Type layout string descriminator.
   PointerAuthSchema TypeLayoutString;
+
+  /// Like SwiftFunctionPointers but for use with CoroFunctionPointer values.
+  PointerAuthSchema CoroSwiftFunctionPointers;
+
+  /// Like SwiftClassMethods but for use with CoroFunctionPointer values.
+  PointerAuthSchema CoroSwiftClassMethods;
+
+  /// Like ProtocolWitnesses but for use with CoroFunctionPointer values.
+  PointerAuthSchema CoroProtocolWitnesses;
+
+  /// Like SwiftClassMethodPointers but for use with CoroFunctionPointer
+  /// values.
+  PointerAuthSchema CoroSwiftClassMethodPointers;
+
+  /// Like SwiftDynamicReplacements but for use with CoroFunctionPointer
+  /// values.
+  PointerAuthSchema CoroSwiftDynamicReplacements;
+
+  /// Like PartialApplyCapture but for use with CoroFunctionPointer values.
+  PointerAuthSchema CoroPartialApplyCapture;
 };
 
 enum class JITDebugArtifact : unsigned {
@@ -502,9 +522,6 @@ class IRGenOptions {
   // Whether to emit typed malloc during coroutine frame allocation.
   unsigned EmitTypeMallocForCoroFrame : 1;
 
-  // Whether to use the yield_once ABI when emitting yield_once_2 coroutines.
-  unsigned EmitYieldOnce2AsYieldOnce : 1;
-
   // Whether to force emission of a frame for all async functions
   // (LLVM's 'frame-pointer=all').
   unsigned AsyncFramePointerAll : 1;
@@ -621,9 +638,9 @@ class IRGenOptions {
         ColocateTypeDescriptors(true), UseRelativeProtocolWitnessTables(false),
         UseFragileResilientProtocolWitnesses(false), EnableHotColdSplit(false),
         EmitAsyncFramePushPopMetadata(true), EmitTypeMallocForCoroFrame(false),
-        EmitYieldOnce2AsYieldOnce(true), AsyncFramePointerAll(false),
-        UseProfilingMarkerThunks(false), UseCoroCCX8664(false),
-        UseCoroCCArm64(false), DebugInfoForProfiling(false), CmdArgs(),
+        AsyncFramePointerAll(false), UseProfilingMarkerThunks(false),
+        UseCoroCCX8664(false), UseCoroCCArm64(false),
+        DebugInfoForProfiling(false), CmdArgs(),
         SanitizeCoverage(llvm::SanitizerCoverageOptions()),
         TypeInfoFilter(TypeInfoDumpFilter::All),
         PlatformCCallingConvention(llvm::CallingConv::C), UseCASBackend(false),

include/swift/Basic/Features.def

@@ -441,9 +441,6 @@ SUPPRESSIBLE_EXPERIMENTAL_FEATURE(CoroutineAccessors, true)
 /// modify/read single-yield coroutines always execute code post-yield code
 EXPERIMENTAL_FEATURE(CoroutineAccessorsUnwindOnCallerError, false)
 
-/// modify/read coroutines use the callee-allocated ABI
-EXPERIMENTAL_FEATURE(CoroutineAccessorsAllocateInCallee, false)
-
 /// When a parameter has unspecified isolation, infer it as main actor isolated.
 EXPERIMENTAL_FEATURE(GenerateForceToMainActorThunks, false)
 

include/swift/Demangling/DemangleNodes.def

@@ -408,6 +408,7 @@ NODE(DependentGenericInverseConformanceRequirement)
 NODE(Integer)
 NODE(NegativeInteger)
 NODE(DependentGenericParamValueMarker)
+NODE(CoroFunctionPointer)
 
 #undef CONTEXT_NODE
 #undef NODE

include/swift/IRGen/Linking.h

@@ -585,6 +585,11 @@ class LinkEntity {
     /// The pointer is the llvm::Function* for a partial apply forwarder.
     PartialApplyForwarderCoroFunctionPointer,
 
+    /// An coro function pointer to a function which is known to exist whose
+    /// name is known.
+    /// The pointer is a const char* of the name.
+    KnownCoroFunctionPointer,
+
     /// An coro function pointer for a distributed accessor (method or
     /// property).
     /// The pointer is a SILFunction*.
@@ -1553,6 +1558,15 @@ class LinkEntity {
     return entity;
   }
 
+  static LinkEntity forKnownCoroFunctionPointer(const char *name) {
+    LinkEntity entity;
+    entity.Pointer = const_cast<char *>(name);
+    entity.SecondaryPointer = nullptr;
+    entity.Data =
+        LINKENTITY_SET_FIELD(Kind, unsigned(Kind::KnownCoroFunctionPointer));
+    return entity;
+  }
+
   LinkEntity getUnderlyingEntityForCoroFunctionPointer() const {
     LinkEntity entity;
     entity.Pointer = Pointer;

include/swift/SIL/SILDeclRef.h

@@ -612,6 +612,7 @@ struct SILDeclRef {
   }
 
   bool hasAsync() const;
+  bool isCalleeAllocatedCoroutine() const;
 
 private:
   friend struct llvm::DenseMapInfo<swift::SILDeclRef>;

lib/AST/Decl.cpp

@@ -10931,7 +10931,7 @@ bool AccessorDecl::isRequirementWithSynthesizedDefaultImplementation() const {
   if (!requiresFeatureCoroutineAccessors(getAccessorKind())) {
     return false;
   }
-  if (getStorage()->getOverrideLoc()) {
+  if (!requiresNewWitnessTableEntry()) {
     return false;
   }
   return getStorage()->requiresCorrespondingUnderscoredCoroutineAccessor(

lib/AST/FeatureSet.cpp

@@ -392,7 +392,6 @@ UNINTERESTING_FEATURE(SafeInteropWrappers)
 UNINTERESTING_FEATURE(AssumeResilientCxxTypes)
 UNINTERESTING_FEATURE(ImportNonPublicCxxMembers)
 UNINTERESTING_FEATURE(CoroutineAccessorsUnwindOnCallerError)
-UNINTERESTING_FEATURE(CoroutineAccessorsAllocateInCallee)
 
 bool swift::usesFeatureIsolatedDeinit(const Decl *decl) {
   if (auto cd = dyn_cast<ClassDecl>(decl)) {

lib/Demangling/Demangler.cpp

@@ -155,6 +155,7 @@ bool swift::Demangle::isFunctionAttr(Node::Kind kind) {
     case Node::Kind::BackDeploymentThunk:
     case Node::Kind::BackDeploymentFallback:
     case Node::Kind::HasSymbolQuery:
+    case Node::Kind::CoroFunctionPointer:
       return true;
     default:
       return false;
@@ -3134,6 +3135,8 @@ NodePointer Demangler::demangleThunkOrSpecialization() {
       switch (nextChar()) {
       case 'b': return createNode(Node::Kind::BackDeploymentThunk);
       case 'B': return createNode(Node::Kind::BackDeploymentFallback);
+      case 'c':
+        return createNode(Node::Kind::CoroFunctionPointer);
       case 'S': return createNode(Node::Kind::HasSymbolQuery);
       default:
         return nullptr;

lib/Demangling/NodePrinter.cpp

@@ -657,6 +657,7 @@ class NodePrinter {
     case Node::Kind::ObjectiveCProtocolSymbolicReference:
     case Node::Kind::DependentGenericInverseConformanceRequirement:
     case Node::Kind::DependentGenericParamValueMarker:
+    case Node::Kind::CoroFunctionPointer:
       return false;
     }
     printer_unreachable("bad node kind");
@@ -3478,6 +3479,9 @@ NodePointer NodePrinter::print(NodePointer Node, unsigned depth,
     Printer << signedValue;
     return nullptr;
   }
+  case Node::Kind::CoroFunctionPointer:
+    Printer << "coro function pointer to ";
+    return nullptr;
   }
 
   printer_unreachable("bad node kind!");

lib/Demangling/OldRemangler.cpp

@@ -1198,6 +1198,11 @@ ManglingError Remangler::mangleAsyncFunctionPointer(Node *node,
   return ManglingError::Success;
 }
 
+ManglingError Remangler::mangleCoroFunctionPointer(Node *node, unsigned depth) {
+  Buffer << "Twc";
+  return ManglingError::Success;
+}
+
 ManglingError Remangler::mangleDeallocator(Node *node, EntityContext &ctx,
                                            unsigned depth) {
   return mangleSimpleEntity(node, 'F', "D", ctx, depth + 1);

lib/Demangling/Remangler.cpp

@@ -1051,6 +1051,11 @@ ManglingError Remangler::mangleAsyncFunctionPointer(Node *node,
   return ManglingError::Success;
 }
 
+ManglingError Remangler::mangleCoroFunctionPointer(Node *node, unsigned depth) {
+  Buffer << "Twc";
+  return ManglingError::Success;
+}
+
 ManglingError Remangler::mangleDependentAssociatedTypeRef(Node *node,
                                                           unsigned depth) {
   RETURN_IF_ERROR(mangleIdentifier(node->getFirstChild(), depth));
@@ -1836,6 +1841,7 @@ ManglingError Remangler::mangleGlobal(Node *node, unsigned depth) {
       case Node::Kind::BackDeploymentThunk:
       case Node::Kind::BackDeploymentFallback:
       case Node::Kind::HasSymbolQuery:
+      case Node::Kind::CoroFunctionPointer:
         mangleInReverseOrder = true;
         break;
       default:

lib/Frontend/CompilerInvocation.cpp

@@ -3635,12 +3635,9 @@ static bool ParseIRGenArgs(IRGenOptions &Opts, ArgList &Args,
     Args.hasFlag(OPT_enable_fragile_resilient_protocol_witnesses,
                  OPT_disable_fragile_resilient_protocol_witnesses,
                  Opts.UseFragileResilientProtocolWitnesses);
-  Opts.UseProfilingMarkerThunks =
-    Args.hasFlag(OPT_enable_profiling_marker_thunks,
-                 OPT_disable_profiling_marker_thunks,
-                 Opts.UseProfilingMarkerThunks);
-  Opts.EmitYieldOnce2AsYieldOnce =
-      !LangOpts.hasFeature(Feature::CoroutineAccessorsAllocateInCallee);
+  Opts.UseProfilingMarkerThunks = Args.hasFlag(
+      OPT_enable_profiling_marker_thunks, OPT_disable_profiling_marker_thunks,
+      Opts.UseProfilingMarkerThunks);
   Opts.EnableHotColdSplit =
       Args.hasFlag(OPT_enable_split_cold_code,
                    OPT_disable_split_cold_code,

lib/IRGen/Callee.h

@@ -206,9 +206,11 @@ namespace irgen {
     FunctionPointerKind(SpecialKind kind)
       : value(unsigned(kind) + SpecialOffset) {}
     FunctionPointerKind(CanSILFunctionType fnType)
-      : FunctionPointerKind(fnType->isAsync()
-                              ? BasicKind::AsyncFunctionPointer
-                              : BasicKind::Function) {}
+        : FunctionPointerKind(fnType->isAsync()
+                                  ? BasicKind::AsyncFunctionPointer
+                              : fnType->isCalleeAllocatedCoroutine()
+                                  ? BasicKind::CoroFunctionPointer
+                                  : BasicKind::Function) {}
 
     static FunctionPointerKind defaultSync() {
       return BasicKind::Function;