[region-isolation] Add support for transferring parameters.

include/swift/AST/ASTBridging.h

@@ -1235,6 +1235,7 @@ enum ENUM_EXTENSIBILITY_ATTR(open) BridgedAttributedTypeSpecifier : size_t {
   BridgedAttributedTypeSpecifierConst,
   BridgedAttributedTypeSpecifierIsolated,
   BridgedAttributedTypeSpecifierResultDependsOn,
+  BridgedAttributedTypeSpecifierTransferring,
 };
 
 SWIFT_NAME("BridgedSimpleIdentTypeRepr.createParsed(_:loc:name:)")

include/swift/AST/Attr.def

@@ -549,6 +549,10 @@ SIMPLE_DECL_ATTR(_noObjCBridging, NoObjCBridging,
 CONTEXTUAL_SIMPLE_DECL_ATTR(_resultDependsOn, ResultDependsOn,
   OnParam | DeclModifier | UserInaccessible | ABIBreakingToAdd | ABIStableToRemove | APIBreakingToAdd | APIStableToRemove,
   156)
+CONTEXTUAL_SIMPLE_DECL_ATTR(transferring, Transferring,
+  OnParam | DeclModifier | UserInaccessible | NotSerialized | ABIBreakingToAdd | ABIBreakingToRemove | APIBreakingToAdd | APIStableToRemove,
+  157)
+
 #undef TYPE_ATTR
 #undef DECL_ATTR_ALIAS
 #undef CONTEXTUAL_DECL_ATTR_ALIAS

include/swift/AST/Decl.h

@@ -6727,6 +6727,7 @@ class ParamDecl : public VarDecl {
       return true;
     case Specifier::Consuming:
     case Specifier::InOut:
+    case Specifier::Transferring:
       return false;
     }
     llvm_unreachable("unhandled specifier");
@@ -6744,6 +6745,7 @@ class ParamDecl : public VarDecl {
     case ParamSpecifier::LegacyShared:
       return ValueOwnership::Shared;
     case ParamSpecifier::Consuming:
+    case ParamSpecifier::Transferring:
     case ParamSpecifier::LegacyOwned:
       return ValueOwnership::Owned;
     case ParamSpecifier::Default:

include/swift/AST/DiagnosticsSIL.def

@@ -898,6 +898,12 @@ WARNING(regionbasedisolation_transfer_yields_race_no_isolation, none,
 WARNING(regionbasedisolation_transfer_yields_race_with_isolation, none,
         "passing argument of non-sendable type %0 from %1 context to %2 context at this call site could yield a race with accesses later in this function",
         (Type, ActorIsolation, ActorIsolation))
+WARNING(regionbasedisolation_transfer_yields_race_transferring_parameter, none,
+        "transferred value of non-Sendable type %0 into transferring parameter; later accesses could result in races",
+        (Type))
+WARNING(regionbasedisolation_transfer_yields_race_stronglytransferred_binding, none,
+        "binding of non-Sendable type %0 accessed after being transferred; later accesses could result in races",
+        (Type))
 NOTE(regionbasedisolation_maybe_race, none,
      "access here could race", ())
 ERROR(regionbasedisolation_unknown_pattern, none,

include/swift/AST/Types.h

@@ -2211,7 +2211,11 @@ enum class ParamSpecifier : uint8_t {
   /// `__shared`, a legacy spelling of `borrowing`.
   LegacyShared = 4,
   /// `__owned`, a legacy spelling of `consuming`.
-  LegacyOwned = 5
+  LegacyOwned = 5,
+
+  /// `transferring`. Indicating the transfer of a value from one isolation
+  /// domain to another.
+  Transferring = 6,
 };
 
 /// Provide parameter type relevant flags, i.e. variadic, autoclosure, and
@@ -2228,7 +2232,8 @@ class ParameterTypeFlags {
     Isolated = 1 << 7,
     CompileTimeConst = 1 << 8,
     ResultDependsOn = 1 << 9,
-    NumBits = 10
+    Transferring = 1 << 10,
+    NumBits = 11
   };
   OptionSet<ParameterFlags> value;
   static_assert(NumBits <= 8*sizeof(OptionSet<ParameterFlags>), "overflowed");
@@ -2243,20 +2248,22 @@ class ParameterTypeFlags {
 
   ParameterTypeFlags(bool variadic, bool autoclosure, bool nonEphemeral,
                      ParamSpecifier specifier, bool isolated, bool noDerivative,
-                     bool compileTimeConst, bool hasResultDependsOn)
+                     bool compileTimeConst, bool hasResultDependsOn,
+                     bool isTransferring)
       : value((variadic ? Variadic : 0) | (autoclosure ? AutoClosure : 0) |
               (nonEphemeral ? NonEphemeral : 0) |
               uint8_t(specifier) << SpecifierShift | (isolated ? Isolated : 0) |
               (noDerivative ? NoDerivative : 0) |
               (compileTimeConst ? CompileTimeConst : 0) |
-              (hasResultDependsOn ? ResultDependsOn : 0)) {}
+              (hasResultDependsOn ? ResultDependsOn : 0) |
+              (isTransferring ? Transferring : 0)) {}
 
   /// Create one from what's present in the parameter type
   inline static ParameterTypeFlags
   fromParameterType(Type paramTy, bool isVariadic, bool isAutoClosure,
                     bool isNonEphemeral, ParamSpecifier ownership,
                     bool isolated, bool isNoDerivative, bool compileTimeConst,
-                    bool hasResultDependsOn);
+                    bool hasResultDependsOn, bool isTransferring);
 
   bool isNone() const { return !value; }
   bool isVariadic() const { return value.contains(Variadic); }
@@ -2269,6 +2276,7 @@ class ParameterTypeFlags {
   bool isCompileTimeConst() const { return value.contains(CompileTimeConst); }
   bool isNoDerivative() const { return value.contains(NoDerivative); }
   bool hasResultDependsOn() const { return value.contains(ResultDependsOn); }
+  bool isTransferring() const { return value.contains(Transferring); }
 
   /// Get the spelling of the parameter specifier used on the parameter.
   ParamSpecifier getOwnershipSpecifier() const {
@@ -2331,6 +2339,12 @@ class ParameterTypeFlags {
                                   : value - ParameterTypeFlags::NoDerivative);
   }
 
+  ParameterTypeFlags withTransferring(bool withTransferring) const {
+    return ParameterTypeFlags(withTransferring
+                                  ? value | ParameterTypeFlags::Transferring
+                                  : value - ParameterTypeFlags::Transferring);
+  }
+
   bool operator ==(const ParameterTypeFlags &other) const {
     return value.toRaw() == other.value.toRaw();
   }
@@ -2424,7 +2438,8 @@ class YieldTypeFlags {
                               /*nonEphemeral*/ false, getOwnershipSpecifier(),
                               /*isolated*/ false, /*noDerivative*/ false,
                               /*compileTimeConst*/ false,
-                              /*hasResultDependsOn*/ false);
+                              /*hasResultDependsOn*/ false,
+                              /*is transferring*/ false);
   }
 
   bool operator ==(const YieldTypeFlags &other) const {
@@ -4111,6 +4126,9 @@ class SILParameterInfo {
     /// - If the function type is `@differentiable`, the function is
     ///   differentiable with respect to this parameter.
     NotDifferentiable = 0x1,
+
+    /// Set if the given parameter is transferring.
+    Transferring = 0x2,
   };
 
   using Options = OptionSet<Flag>;
@@ -7627,7 +7645,7 @@ inline TupleTypeElt TupleTypeElt::getWithType(Type T) const {
 inline ParameterTypeFlags ParameterTypeFlags::fromParameterType(
     Type paramTy, bool isVariadic, bool isAutoClosure, bool isNonEphemeral,
     ParamSpecifier ownership, bool isolated, bool isNoDerivative,
-    bool compileTimeConst, bool hasResultDependsOn) {
+    bool compileTimeConst, bool hasResultDependsOn, bool isTransferring) {
   // FIXME(Remove InOut): The last caller that needs this is argument
   // decomposition.  Start by enabling the assertion there and fixing up those
   // callers, then remove this, then remove
@@ -7637,8 +7655,9 @@ inline ParameterTypeFlags ParameterTypeFlags::fromParameterType(
            ownership == ParamSpecifier::InOut);
     ownership = ParamSpecifier::InOut;
   }
-  return {isVariadic, isAutoClosure,  isNonEphemeral,   ownership,
-          isolated,   isNoDerivative, compileTimeConst, hasResultDependsOn};
+  return {isVariadic,       isAutoClosure,      isNonEphemeral,
+          ownership,        isolated,           isNoDerivative,
+          compileTimeConst, hasResultDependsOn, isTransferring};
 }
 
 inline const Type *BoundGenericType::getTrailingObjectsPointer() const {

include/swift/SIL/ApplySite.h

@@ -721,6 +721,18 @@ class FullApplySite : public ApplySite {
                                 getNumIndirectSILErrorResults());
   }
 
+  MutableArrayRef<Operand> getOperandsWithoutIndirectResults() const {
+    return getArgumentOperands().slice(getNumIndirectSILResults() +
+                                       getNumIndirectSILErrorResults());
+  }
+
+  MutableArrayRef<Operand> getOperandsWithoutIndirectResultsOrSelf() const {
+    auto ops = getOperandsWithoutIndirectResults();
+    if (!hasSelfArgument())
+      return ops;
+    return ops.drop_back();
+  }
+
   InoutArgumentRange getInoutArguments() const {
     switch (getKind()) {
     case FullApplySiteKind::ApplyInst:
@@ -789,6 +801,15 @@ class FullApplySite : public ApplySite {
     }
   }
 
+  SILParameterInfo getArgumentParameterInfo(const Operand &oper) const {
+    assert(!getArgumentConvention(oper).isIndirectOutParameter() &&
+           "Can only be applied to non-out parameters");
+
+    // The ParameterInfo is going to be the parameter in the caller.
+    unsigned calleeArgIndex = getCalleeArgIndex(oper);
+    return getSubstCalleeConv().getParamInfoForSILArg(calleeArgIndex);
+  }
+
   static FullApplySite getFromOpaqueValue(void *p) { return FullApplySite(p); }
 
   static bool classof(const SILInstruction *inst) {

include/swift/SIL/SILArgument.h

@@ -434,6 +434,10 @@ class SILFunctionArgument : public SILArgument {
     sharedUInt32().SILFunctionArgument.hasResultDependsOn = flag;
   }
 
+  bool isTransferring() const {
+    return getKnownParameterInfo().hasOption(SILParameterInfo::Transferring);
+  }
+
   Lifetime getLifetime() const {
     return getType()
         .getLifetime(*getFunction())

include/swift/SIL/SILInstruction.h

@@ -3009,6 +3009,10 @@ class ApplyInstBase<Impl, Base, true>
     return getArguments().slice(getNumIndirectResults());
   }
 
+  MutableArrayRef<Operand> getOperandsWithoutIndirectResults() const {
+    return getArgumentOperands().slice(getNumIndirectResults());
+  }
+
   /// Returns all `@inout` and `@inout_aliasable` arguments passed to the
   /// instruction.
   InoutArgumentRange getInoutArguments() const {

include/swift/SIL/SILValue.h

@@ -1655,6 +1655,15 @@ namespace llvm {
     enum { NumLowBitsAvailable = swift::SILValue::NumLowBitsAvailable };
   };
 
+  /// A SILValue can be checked if a value is present, so we can use it with
+  /// dyn_cast_or_null.
+  template <>
+  struct ValueIsPresent<swift::SILValue> {
+    using SILValue = swift::SILValue;
+    using UnwrappedType = SILValue;
+    static inline bool isPresent(const SILValue &t) { return bool(t); }
+    static inline decltype(auto) unwrapValue(SILValue &t) { return t; }
+  };
 } // end namespace llvm
 
 #endif

include/swift/SILOptimizer/Analysis/RegionAnalysis.h

@@ -254,6 +254,10 @@ class regionanalysisimpl::TrackableValue {
 
   TrackableValueState getValueState() const { return valueState; }
 
+  /// Returns true if this TrackableValue is an alloc_stack from a transferring
+  /// parameter.
+  bool isTransferringParameter() const;
+
   void print(llvm::raw_ostream &os) const {
     os << "TrackableValue. State: ";
     valueState.print(os);

lib/AST/ASTBridging.cpp

@@ -1704,6 +1704,10 @@ BridgedSpecifierTypeRepr BridgedSpecifierTypeRepr_createParsed(
     return new (context)
         OwnershipTypeRepr(baseType, ParamSpecifier::LegacyOwned, loc);
   }
+  case BridgedAttributedTypeSpecifierTransferring: {
+    return new (context)
+        OwnershipTypeRepr(baseType, ParamSpecifier::Transferring, loc);
+  }
   case BridgedAttributedTypeSpecifierConst: {
     return new (context) CompileTimeConstTypeRepr(baseType, loc);
   }

lib/AST/ASTMangler.cpp

@@ -2875,7 +2875,8 @@ getParameterFlagsForMangling(ParameterTypeFlags flags,
   // `inout` should already be specified in the flags.
   case ParamSpecifier::InOut:
     return flags;
-
+
+  case ParamSpecifier::Transferring:
   case ParamSpecifier::Consuming:
   case ParamSpecifier::Borrowing:
     // Only mangle the ownership if it diverges from the default.

lib/AST/ASTPrinter.cpp

@@ -3906,7 +3906,15 @@ static bool usesFeatureExtractConstantsFromMembers(Decl *decl) {
 
 static bool usesFeatureBitwiseCopyable(Decl *decl) { return false; }
 
-static bool usesFeatureTransferringArgsAndResults(Decl *decl) { return false; }
+static bool usesFeatureTransferringArgsAndResults(Decl *decl) {
+  if (auto *pd = dyn_cast<ParamDecl>(decl))
+    if (pd->getSpecifier() == ParamSpecifier::Transferring)
+      return true;
+
+  // TODO: Results.
+
+  return false;
+}
 
 static bool usesFeaturePreconcurrencyConformances(Decl *decl) {
   auto usesPreconcurrencyConformance = [&](const InheritedTypes &inherited) {
@@ -4567,6 +4575,8 @@ static void printParameterFlags(ASTPrinter &printer,
     printer.printAttrName("@autoclosure ");
   if (!options.excludeAttrKind(TAK_noDerivative) && flags.isNoDerivative())
     printer.printAttrName("@noDerivative ");
+  if (flags.isTransferring())
+    printer.printAttrName("@transferring ");
 
   switch (flags.getOwnershipSpecifier()) {
   case ParamSpecifier::Default:
@@ -4587,6 +4597,9 @@ static void printParameterFlags(ASTPrinter &printer,
   case ParamSpecifier::LegacyOwned:
     printer.printKeyword("__owned", options, " ");
     break;
+  case ParamSpecifier::Transferring:
+    printer.printKeyword("transferring", options, " ");
+    break;
   }
 
   if (flags.isIsolated())
@@ -8068,6 +8081,11 @@ void SILParameterInfo::print(ASTPrinter &Printer,
     Printer << "@noDerivative ";
   }
 
+  if (options.contains(SILParameterInfo::Transferring)) {
+    options -= SILParameterInfo::Transferring;
+    Printer << "@transferring ";
+  }
+
   // If we did not handle a case in Options, this code was not updated
   // appropriately.
   assert(!bool(options) && "Code not updated for introduced option");

lib/AST/Decl.cpp

@@ -7746,6 +7746,7 @@ void ParamDecl::setSpecifier(Specifier specifier) {
   // `inout` and `consuming` parameters are locally mutable.
   case ParamSpecifier::InOut:
   case ParamSpecifier::Consuming:
+  case ParamSpecifier::Transferring:
     introducer = VarDecl::Introducer::Var;
     break;
   }
@@ -7809,6 +7810,8 @@ StringRef ParamDecl::getSpecifierSpelling(ParamSpecifier specifier) {
     return "__shared";
   case ParamSpecifier::LegacyOwned:
     return "__owned";
+  case ParamSpecifier::Transferring:
+    return "transferring";
   }
   llvm_unreachable("invalid ParamSpecifier");
 }
@@ -8410,7 +8413,8 @@ AnyFunctionType::Param ParamDecl::toFunctionParam(Type type) const {
   auto flags = ParameterTypeFlags::fromParameterType(
       type, isVariadic(), isAutoClosure(), isNonEphemeral(), getSpecifier(),
       isIsolated(), /*isNoDerivative*/ false, isCompileTimeConst(),
-      hasResultDependsOn());
+      hasResultDependsOn(),
+      getSpecifier() == ParamDecl::Specifier::Transferring /*is transferring*/);
   return AnyFunctionType::Param(type, label, flags, internalLabel);
 }
 

lib/Parse/ParseDecl.cpp

@@ -5137,6 +5137,7 @@ ParserStatus Parser::ParsedTypeAttributeList::slowParse(Parser &P) {
            Tok.isContextualKeyword("isolated") ||
            Tok.isContextualKeyword("consuming") ||
            Tok.isContextualKeyword("borrowing") ||
+           Tok.isContextualKeyword("transferring") ||
            Tok.isContextualKeyword("_const") ||
            Tok.isContextualKeyword("_resultDependsOn")))) {
 
@@ -5164,6 +5165,15 @@ ParserStatus Parser::ParsedTypeAttributeList::slowParse(Parser &P) {
       continue;
     }
 
+    // Perform an extra check for transferring. Since it is a specifier, we use
+    // the actual parsing logic below.
+    if (Tok.isContextualKeyword("transferring")) {
+      if (!P.Context.LangOpts.hasFeature(Feature::TransferringArgsAndResults)) {
+        P.diagnose(Tok, diag::requires_experimental_feature, "transferring",
+                   false, getFeatureName(Feature::TransferringArgsAndResults));
+      }
+    }
+
     if (SpecifierLoc.isValid()) {
       P.diagnose(Tok, diag::parameter_specifier_repeated)
           .fixItRemove(SpecifierLoc);
@@ -5177,6 +5187,8 @@ ParserStatus Parser::ParsedTypeAttributeList::slowParse(Parser &P) {
           Specifier = ParamDecl::Specifier::LegacyOwned;
         } else if (Tok.getRawText().equals("borrowing")) {
           Specifier = ParamDecl::Specifier::Borrowing;
+        } else if (Tok.getRawText().equals("transferring")) {
+          Specifier = ParamDecl::Specifier::Transferring;
         } else if (Tok.getRawText().equals("consuming")) {
           Specifier = ParamDecl::Specifier::Consuming;
         }
@@ -7539,6 +7551,8 @@ static ParserStatus parseAccessorIntroducer(Parser &P,
       P.parseNewDeclAttribute(Attributes, /*AtLoc*/ {}, DAK_Consuming);
     } else if (P.Tok.isContextualKeyword("borrowing")) {
       P.parseNewDeclAttribute(Attributes, /*AtLoc*/ {}, DAK_Borrowing);
+    } else if (P.Tok.isContextualKeyword("transferring")) {
+      P.parseNewDeclAttribute(Attributes, /*AtLoc*/ {}, DAK_Transferring);
     }
   }