[sending] Change CheckedContinuation.resume and Async{Throwing,}Stream.yield to use transferring parameters.

include/swift/AST/ASTMangler.h

@@ -445,7 +445,8 @@ class ASTMangler : public Mangler {
   void appendRetroactiveConformances(SubstitutionMap subMap,
                                      GenericSignature sig);
   void appendImplFunctionType(SILFunctionType *fn, GenericSignature sig,
-                              const ValueDecl *forDecl = nullptr);
+                              const ValueDecl *forDecl = nullptr,
+                              bool isInRecursion = true);
   void appendOpaqueTypeArchetype(ArchetypeType *archetype,
                                  OpaqueTypeDecl *opaqueDecl,
                                  SubstitutionMap subs,
@@ -521,25 +522,29 @@ class ASTMangler : public Mangler {
     FunctionMangling,
   };
 
-  void appendFunction(AnyFunctionType *fn, GenericSignature sig,
-                    FunctionManglingKind functionMangling = NoFunctionMangling,
-                    const ValueDecl *forDecl = nullptr);
+  void
+  appendFunction(AnyFunctionType *fn, GenericSignature sig,
+                 FunctionManglingKind functionMangling = NoFunctionMangling,
+                 const ValueDecl *forDecl = nullptr,
+                 bool isRecursedInto = true);
   void appendFunctionType(AnyFunctionType *fn, GenericSignature sig,
                           bool isAutoClosure = false,
-                          const ValueDecl *forDecl = nullptr);
+                          const ValueDecl *forDecl = nullptr,
+                          bool isRecursedInto = true);
   void appendClangType(AnyFunctionType *fn);
   template <typename FnType>
   void appendClangType(FnType *fn, llvm::raw_svector_ostream &os);
 
-  void appendFunctionSignature(AnyFunctionType *fn,
-                               GenericSignature sig,
+  void appendFunctionSignature(AnyFunctionType *fn, GenericSignature sig,
                                const ValueDecl *forDecl,
-                               FunctionManglingKind functionMangling);
+                               FunctionManglingKind functionMangling,
+                               bool isRecursedInto = true);
 
   void appendFunctionInputType(ArrayRef<AnyFunctionType::Param> params,
                                LifetimeDependenceInfo lifetimeDependenceInfo,
                                GenericSignature sig,
-                               const ValueDecl *forDecl = nullptr);
+                               const ValueDecl *forDecl = nullptr,
+                               bool isRecursedInto = true);
   void appendFunctionResultType(Type resultType,
                                 GenericSignature sig,
                                 const ValueDecl *forDecl = nullptr);

include/swift/AST/Builtins.def

@@ -517,14 +517,14 @@ BUILTIN_SIL_OPERATION(ApplyDerivative, "applyDerivative", Special)
 /// applyTranspose
 BUILTIN_SIL_OPERATION(ApplyTranspose, "applyTranspose", Special)
 
-/// withUnsafeContinuation<T> : (Builtin.RawUnsafeContinuation -> ()) async -> T
+/// withUnsafeContinuation<T> : (Builtin.RawUnsafeContinuation -> ()) async -> sending T
 ///
 /// Unsafely capture the current continuation and pass it to the given
 /// function value. Returns a value of type T when the continuation is
 /// resumed.
 BUILTIN_SIL_OPERATION(WithUnsafeContinuation, "withUnsafeContinuation", Special)
 
-/// withUnsafeThrowingContinuation<T> : (Builtin.RawUnsafeContinuation -> ()) async throws -> T
+/// withUnsafeThrowingContinuation<T> : (Builtin.RawUnsafeContinuation -> ()) async throws -> sending T
 ///
 /// Unsafely capture the current continuation and pass it to the given
 /// function value. Returns a value of type T or throws an error when

include/swift/Basic/Features.def

@@ -196,6 +196,8 @@ SUPPRESSIBLE_LANGUAGE_FEATURE(NoncopyableGenerics, 427, "Noncopyable generics")
 SUPPRESSIBLE_LANGUAGE_FEATURE(ConformanceSuppression, 426, "Suppressible inferred conformances")
 SUPPRESSIBLE_LANGUAGE_FEATURE(BitwiseCopyable2, 426, "BitwiseCopyable feature")
 LANGUAGE_FEATURE(BodyMacros, 415, "Function body macros")
+LANGUAGE_FEATURE(TransferringArgsAndResults, 430, "Transferring args and results")
+SUPPRESSIBLE_LANGUAGE_FEATURE(SendingArgsAndResults, 430, "Sending arg and results")
 
 // Swift 6
 UPCOMING_FEATURE(ConciseMagicFile, 274, 6)
@@ -214,8 +216,6 @@ UPCOMING_FEATURE(DynamicActorIsolation, 423, 6)
 UPCOMING_FEATURE(NonfrozenEnumExhaustivity, 192, 6)
 UPCOMING_FEATURE(GlobalActorIsolatedTypesUsability, 0434, 6)
 UPCOMING_FEATURE(BorrowingSwitch, 432, 6)
-UPCOMING_FEATURE(TransferringArgsAndResults, 430, 6)
-SUPPRESSIBLE_UPCOMING_FEATURE(SendingArgsAndResults, 430, 6)
 
 // Swift 7
 UPCOMING_FEATURE(ExistentialAny, 335, 7)

include/swift/Demangling/TypeDecoder.h

@@ -183,6 +183,14 @@ class ImplFunctionParam {
     return {};
   }
 
+  static OptionsType getSending() {
+    OptionsType result;
+
+    result |= ImplParameterInfoFlags::Sending;
+
+    return result;
+  }
+
   ImplFunctionParam(BuiltType type, ImplParameterConvention convention,
                     OptionsType options)
       : Type(type), Convention(convention), Options(options) {}
@@ -256,6 +264,12 @@ class ImplFunctionResult {
     return {};
   }
 
+  static OptionsType getSending() {
+    OptionsType result;
+    result |= ImplResultInfoFlags::IsSending;
+    return result;
+  }
+
   ImplFunctionResult(BuiltType type, ImplResultConvention convention,
                      ImplResultInfoOptions options = {})
       : Type(type), Convention(convention), Options(options) {}
@@ -1594,8 +1608,9 @@ class TypeDecoder {
     if (depth > TypeDecoder::MaxDepth)
       return true;
 
-    // Children: `convention, differentiability?, type`
-    if (node->getNumChildren() != 2 && node->getNumChildren() != 3)
+    // Children: `convention, differentiability?, sending?, type`
+    if (node->getNumChildren() != 2 && node->getNumChildren() != 3 &&
+        node->getNumChildren() != 4)
       return true;
 
     auto *conventionNode = node->getChild(0);
@@ -1613,7 +1628,7 @@ class TypeDecoder {
       return true;
 
     typename T::OptionsType options;
-    if (node->getNumChildren() == 3) {
+    if (node->getNumChildren() == 3 || node->getNumChildren() == 4) {
       auto diffKindNode = node->getChild(1);
       if (diffKindNode->getKind() !=
           Node::Kind::ImplParameterResultDifferentiability)
@@ -1625,6 +1640,13 @@ class TypeDecoder {
       options |= *optDiffOptions;
     }
 
+    if (node->getNumChildren() == 4) {
+      auto sendingKindNode = node->getChild(2);
+      if (sendingKindNode->getKind() != Node::Kind::ImplParameterSending)
+        return true;
+      options |= T::getSending();
+    }
+
     results.emplace_back(result.getType(), *convention, options);
 
     return false;

include/swift/SIL/ApplySite.h

@@ -470,6 +470,21 @@ class ApplySite {
     llvm_unreachable("covered switch");
   }
 
+  /// Return the sil_isolated operand if we have one.
+  Operand *getIsolatedArgumentOperandOrNullPtr() {
+    switch (ApplySiteKind(Inst->getKind())) {
+    case ApplySiteKind::ApplyInst:
+      return cast<ApplyInst>(Inst)->getIsolatedArgumentOperandOrNullPtr();
+    case ApplySiteKind::BeginApplyInst:
+      return cast<BeginApplyInst>(Inst)->getIsolatedArgumentOperandOrNullPtr();
+    case ApplySiteKind::TryApplyInst:
+      return cast<TryApplyInst>(Inst)->getIsolatedArgumentOperandOrNullPtr();
+    case ApplySiteKind::PartialApplyInst:
+      llvm_unreachable("Unhandled case");
+    }
+    llvm_unreachable("covered switch");
+  }
+
   /// Return a list of applied arguments without self.
   OperandValueArrayRef getArgumentsWithoutSelf() const {
     switch (ApplySiteKind(Inst->getKind())) {

include/swift/SIL/SILInstruction.h

@@ -2972,14 +2972,15 @@ class ApplyInstBase<Impl, Base, true>
   const Impl &asImpl() const { return static_cast<const Impl &>(*this); }
 
 public:
+  using super::getArgument;
+  using super::getArgumentOperands;
+  using super::getArguments;
   using super::getCallee;
-  using super::getSubstCalleeType;
+  using super::getCalleeOperand;
+  using super::getNumArguments;
   using super::getSubstCalleeConv;
+  using super::getSubstCalleeType;
   using super::hasSubstitutions;
-  using super::getNumArguments;
-  using super::getArgument;
-  using super::getArguments;
-  using super::getArgumentOperands;
 
   /// The collection of following routines wrap the representation difference in
   /// between the self substitution being first, but the self parameter of a
@@ -3064,6 +3065,21 @@ class ApplyInstBase<Impl, Base, true>
     return getSubstCalleeType()->hasSelfParam();
   }
 
+  Operand *getIsolatedArgumentOperandOrNullPtr() {
+    SILFunctionConventions conv = getSubstCalleeConv();
+    for (Operand &argOp : getOperandsWithoutIndirectResults()) {
+      // Skip the callee.
+      if (getCalleeOperand() == &argOp)
+        continue;
+
+      auto opNum = argOp.getOperandNumber() - 1;
+      auto paramInfo = conv.getParamInfoForSILArg(opNum);
+      if (paramInfo.getOptions().contains(SILParameterInfo::Isolated))
+        return &argOp;
+    }
+    return nullptr;
+  }
+
   bool hasGuaranteedSelfArgument() const {
     auto C = getSubstCalleeType()->getSelfParameter().getConvention();
     return C == ParameterConvention::Direct_Guaranteed;
@@ -3077,7 +3093,7 @@ class ApplyInstBase<Impl, Base, true>
     return getArguments().slice(getNumIndirectResults());
   }
 
-  MutableArrayRef<Operand> getOperandsWithoutIndirectResults() const {
+  MutableArrayRef<Operand> getOperandsWithoutIndirectResults() {
     return getArgumentOperands().slice(getNumIndirectResults());
   }
 

lib/AST/ASTMangler.cpp

@@ -2039,7 +2039,8 @@ getResultDifferentiability(SILResultInfo::Options options) {
 
 void ASTMangler::appendImplFunctionType(SILFunctionType *fn,
                                         GenericSignature outerGenericSig,
-                                        const ValueDecl *forDecl) {
+                                        const ValueDecl *forDecl,
+                                        bool isInRecursion) {
 
   llvm::SmallVector<char, 32> OpArgs;
 
@@ -2164,7 +2165,7 @@ void ASTMangler::appendImplFunctionType(SILFunctionType *fn,
   }
 
   // Mangle if we have a transferring result.
-  if (fn->hasSendingResult())
+  if (isInRecursion && fn->hasSendingResult())
     OpArgs.push_back('T');
 
   // Mangle the results.
@@ -2947,7 +2948,7 @@ void ASTMangler::appendAnyGenericType(const GenericTypeDecl *decl,
 
 void ASTMangler::appendFunction(AnyFunctionType *fn, GenericSignature sig,
                                 FunctionManglingKind functionMangling,
-                                const ValueDecl *forDecl) {
+                                const ValueDecl *forDecl, bool isRecursedInto) {
   // Append parameter labels right before the signature/type.
   auto parameters = fn->getParams();
   auto firstLabel = std::find_if(
@@ -2967,19 +2968,20 @@ void ASTMangler::appendFunction(AnyFunctionType *fn, GenericSignature sig,
   }
 
   if (functionMangling != NoFunctionMangling) {
-    appendFunctionSignature(fn, sig, forDecl, functionMangling);
+    appendFunctionSignature(fn, sig, forDecl, functionMangling, isRecursedInto);
   } else {
-    appendFunctionType(fn, sig, /*autoclosure*/ false, forDecl);
+    appendFunctionType(fn, sig, /*autoclosure*/ false, forDecl, isRecursedInto);
   }
 }
 
 void ASTMangler::appendFunctionType(AnyFunctionType *fn, GenericSignature sig,
                                     bool isAutoClosure,
-                                    const ValueDecl *forDecl) {
+                                    const ValueDecl *forDecl,
+                                    bool isRecursedInto) {
   assert((DWARFMangling || fn->isCanonical()) &&
          "expecting canonical types when not mangling for the debugger");
 
-  appendFunctionSignature(fn, sig, forDecl, NoFunctionMangling);
+  appendFunctionSignature(fn, sig, forDecl, NoFunctionMangling, isRecursedInto);
 
   bool mangleClangType = fn->getASTContext().LangOpts.UseClangFunctionTypes &&
                          fn->hasNonDerivableClangType();
@@ -3047,10 +3049,11 @@ void ASTMangler::appendClangType(AnyFunctionType *fn) {
 void ASTMangler::appendFunctionSignature(AnyFunctionType *fn,
                                          GenericSignature sig,
                                          const ValueDecl *forDecl,
-                                         FunctionManglingKind functionMangling) {
+                                         FunctionManglingKind functionMangling,
+                                         bool isRecursedInto) {
   appendFunctionResultType(fn->getResult(), sig, forDecl);
   appendFunctionInputType(fn->getParams(), fn->getLifetimeDependenceInfo(), sig,
-                          forDecl);
+                          forDecl, isRecursedInto);
   if (fn->isAsync())
     appendOperator("Ya");
   if (fn->isSendable())
@@ -3096,7 +3099,7 @@ void ASTMangler::appendFunctionSignature(AnyFunctionType *fn,
     break;
   }
 
-  if (fn->hasSendingResult()) {
+  if (isRecursedInto && fn->hasSendingResult()) {
     appendOperator("YT");
   }
 
@@ -3142,19 +3145,25 @@ getDefaultOwnership(const ValueDecl *forDecl) {
 
 static ParameterTypeFlags
 getParameterFlagsForMangling(ParameterTypeFlags flags,
-                             ParamSpecifier defaultSpecifier) {
+                             ParamSpecifier defaultSpecifier,
+                             bool isInRecursion = true) {
+  bool initiallySending = flags.isSending();
+
+  // If we have been recursed into, then remove sending from our flags.
+  if (!isInRecursion) {
+    flags = flags.withSending(false);
+  }
+
   switch (auto specifier = flags.getOwnershipSpecifier()) {
   // If no parameter specifier was provided, mangle as-is, because we are by
   // definition using the default convention.
   case ParamSpecifier::Default:
   // If the legacy `__shared` or `__owned` modifier was provided, mangle as-is,
   // because we need to maintain compatibility with their existing behavior.
-  case ParamSpecifier::LegacyShared:
   case ParamSpecifier::LegacyOwned:
   // `inout` should already be specified in the flags.
   case ParamSpecifier::InOut:
     return flags;
-
   case ParamSpecifier::ImplicitlyCopyableConsuming:
   case ParamSpecifier::Consuming:
   case ParamSpecifier::Borrowing:
@@ -3163,13 +3172,23 @@ getParameterFlagsForMangling(ParameterTypeFlags flags,
       flags = flags.withOwnershipSpecifier(ParamSpecifier::Default);
     }
     return flags;
+  case ParamSpecifier::LegacyShared:
+    // If we were originally sending and by default we are borrowing, suppress
+    // this and set ownership specifier to default so we do not mangle in
+    // __shared.
+    //
+    // This is a work around in the short term since shared borrow is not
+    // supported.
+    if (initiallySending && ParamSpecifier::Borrowing == defaultSpecifier)
+      return flags.withOwnershipSpecifier(ParamSpecifier::Default);
+    return flags;
   }
 }
 
 void ASTMangler::appendFunctionInputType(
     ArrayRef<AnyFunctionType::Param> params,
     LifetimeDependenceInfo lifetimeDependenceInfo, GenericSignature sig,
-    const ValueDecl *forDecl) {
+    const ValueDecl *forDecl, bool isRecursedInto) {
   auto defaultSpecifier = getDefaultOwnership(forDecl);
 
   switch (params.size()) {
@@ -3192,7 +3211,7 @@ void ASTMangler::appendFunctionInputType(
       appendParameterTypeListElement(
           Identifier(), type,
           getParameterFlagsForMangling(param.getParameterFlags(),
-                                       defaultSpecifier),
+                                       defaultSpecifier, isRecursedInto),
           lifetimeDependenceInfo.getLifetimeDependenceOnParam(/*paramIndex*/ 0),
           sig, nullptr);
       break;
@@ -3214,7 +3233,7 @@ void ASTMangler::appendFunctionInputType(
       appendParameterTypeListElement(
           Identifier(), param.getPlainType(),
           getParameterFlagsForMangling(param.getParameterFlags(),
-                                       defaultSpecifier),
+                                       defaultSpecifier, isRecursedInto),
           lifetimeDependenceInfo.getLifetimeDependenceOnParam(paramIndex), sig,
           nullptr);
       appendListSeparator(isFirstParam);
@@ -3878,7 +3897,8 @@ void ASTMangler::appendDeclType(const ValueDecl *decl,
               : decl->getDeclContext()->getGenericSignatureOfContext());
 
   if (AnyFunctionType *FuncTy = type->getAs<AnyFunctionType>()) {
-    appendFunction(FuncTy, sig, functionMangling, decl);
+    appendFunction(FuncTy, sig, functionMangling, decl,
+                   false /*is recursed into*/);
   } else {
     appendType(type, sig, decl);
   }