[sil] Add a new CastConsumptionKind called BorrowAlways.

include/swift/SIL/Consumption.h

@@ -43,16 +43,41 @@ enum class CastConsumptionKind : uint8_t {
   /// The source value is always left in place, and the destination
   /// value is copied into on success.
   CopyOnSuccess,
+
+  /// The source value is never taken, regardless of whether the cast
+  /// succeeds. Instead, we always borrow the source value and feed it through.
+  ///
+  /// NOTE: This can only be used with objects. We do not support borrowing of
+  /// addresses. If an address is needed for a cast operation, a BorrowAlways
+  /// value must be copied into a temporary and operated upon. If the result of
+  /// the cast is a loadable type then the value is loaded using a
+  /// load_borrow. If an address only value is returned, we continue processing
+  /// the value as an owned TakeAlways value.
+  BorrowAlways,
 };
 
 /// Should the source value be destroyed if the cast fails?
 inline bool shouldDestroyOnFailure(CastConsumptionKind kind) {
-  return (kind == CastConsumptionKind::TakeAlways);
+  switch (kind) {
+  case CastConsumptionKind::TakeAlways:
+    return true;
+  case CastConsumptionKind::TakeOnSuccess:
+  case CastConsumptionKind::CopyOnSuccess:
+  case CastConsumptionKind::BorrowAlways:
+    return false;
+  }
 }
 
 /// Should the source value be taken if the cast succeeds?
 inline IsTake_t shouldTakeOnSuccess(CastConsumptionKind kind) {
-  return IsTake_t(kind != CastConsumptionKind::CopyOnSuccess);
+  switch (kind) {
+  case CastConsumptionKind::TakeAlways:
+  case CastConsumptionKind::TakeOnSuccess:
+    return IsTake;
+  case CastConsumptionKind::CopyOnSuccess:
+  case CastConsumptionKind::BorrowAlways:
+    return IsNotTake;
+  }
 }
 
 } // end namespace swift

include/swift/SIL/SILInstruction.h

@@ -7402,7 +7402,10 @@ class CheckedCastAddrBranchInst
       : InstructionBase(DebugLoc), ConsumptionKind(consumptionKind),
         Operands{this, src, dest}, DestBBs{{this, successBB, Target1Count},
                                            {this, failureBB, Target2Count}},
-        SourceType(srcType), TargetType(targetType) {}
+        SourceType(srcType), TargetType(targetType) {
+    assert(ConsumptionKind != CastConsumptionKind::BorrowAlways &&
+           "BorrowAlways is not supported on addresses");
+  }
 
 public:
   enum {

include/swift/Serialization/ModuleFormat.h

@@ -55,7 +55,7 @@ const uint16_t VERSION_MAJOR = 0;
 /// describe what change you made. The content of this comment isn't important;
 /// it just ensures a conflict if two people change the module format.
 /// Don't worry about adhering to the 80-column limit for this line.
-const uint16_t VERSION_MINOR = 452; // Last change: nominal types for operators
+const uint16_t VERSION_MINOR = 453; // Last change: borrow always
 
 using DeclIDField = BCFixed<31>;
 

lib/ParseSIL/ParseSIL.cpp

@@ -3155,6 +3155,8 @@ bool SILParser::parseSILInstruction(SILBuilder &B) {
                            "cast consumption kind")) {
       return true;
     }
+    // NOTE: BorrowAlways is not a supported cast kind for address types, so we
+    // purposely do not parse it here.
     auto kind = llvm::StringSwitch<Optional<CastConsumptionKind>>(
                     consumptionKindToken.str())
                     .Case("take_always", CastConsumptionKind::TakeAlways)

lib/SIL/SILPrinter.cpp

@@ -398,6 +398,7 @@ static StringRef getCastConsumptionKindName(CastConsumptionKind kind) {
   case CastConsumptionKind::TakeAlways: return "take_always";
   case CastConsumptionKind::TakeOnSuccess: return "take_on_success";
   case CastConsumptionKind::CopyOnSuccess: return "copy_on_success";
+  case CastConsumptionKind::BorrowAlways: return "borrow_always";
   }
   llvm_unreachable("bad cast consumption kind");
 }

lib/SIL/SILVerifier.cpp

@@ -2860,9 +2860,15 @@ class SILVerifier : public SILVerifierBase<SILVerifier> {
           return true;
         }
         case SILInstructionKind::CheckedCastAddrBranchInst:
-          if (cast<CheckedCastAddrBranchInst>(inst)->getConsumptionKind() !=
-              CastConsumptionKind::CopyOnSuccess)
+          switch (cast<CheckedCastAddrBranchInst>(inst)->getConsumptionKind()) {
+          case CastConsumptionKind::BorrowAlways:
+            llvm_unreachable("checked_cast_addr_br cannot have BorrowAlways");
+          case CastConsumptionKind::CopyOnSuccess:
+            break;
+          case CastConsumptionKind::TakeAlways:
+          case CastConsumptionKind::TakeOnSuccess:
             return true;
+          }
           break;
         case SILInstructionKind::LoadInst:
           // A 'non-taking' value load is harmless.

lib/SILGen/SILGenDynamicCast.cpp

@@ -229,11 +229,16 @@ namespace {
           return;
         }
 
-        if (consumption == CastConsumptionKind::CopyOnSuccess) {
+        switch (consumption) {
+        case CastConsumptionKind::BorrowAlways:
+        case CastConsumptionKind::CopyOnSuccess:
           SGF.B.createGuaranteedPhiArgument(operandValue.getType());
           handleFalse(None);
-        } else {
+          break;
+        case CastConsumptionKind::TakeAlways:
+        case CastConsumptionKind::TakeOnSuccess:
           handleFalse(SGF.B.createOwnedPhiArgument(operandValue.getType()));
+          break;
         }
 
         assert(!SGF.B.hasValidInsertionPoint() && "handler did not end block");

lib/SILGen/SILGenPattern.cpp

@@ -1165,6 +1165,7 @@ static bool shouldTake(ConsumableManagedValue value, bool isIrrefutable) {
   case CastConsumptionKind::TakeAlways: return true;
   case CastConsumptionKind::TakeOnSuccess: return isIrrefutable;
   case CastConsumptionKind::CopyOnSuccess: return false;
+  case CastConsumptionKind::BorrowAlways: return false;
   }
   llvm_unreachable("bad consumption kind");
 }
@@ -1349,14 +1350,17 @@ static ConsumableManagedValue
 getManagedSubobject(SILGenFunction &SGF, SILValue value,
                     const TypeLowering &valueTL,
                     CastConsumptionKind consumption) {
-  if (consumption == CastConsumptionKind::CopyOnSuccess) {
+  switch (consumption) {
+  case CastConsumptionKind::BorrowAlways:
+  case CastConsumptionKind::CopyOnSuccess:
     return {ManagedValue::forUnmanaged(value), consumption};
+  case CastConsumptionKind::TakeAlways:
+  case CastConsumptionKind::TakeOnSuccess:
+    assert((!SGF.F.getModule().getOptions().EnableSILOwnership ||
+            consumption != CastConsumptionKind::TakeOnSuccess) &&
+           "TakeOnSuccess should never be used when sil ownership is enabled");
+    return {SGF.emitManagedRValueWithCleanup(value, valueTL), consumption};
   }
-
-  assert((!SGF.F.getModule().getOptions().EnableSILOwnership ||
-          consumption != CastConsumptionKind::TakeOnSuccess) &&
-         "TakeOnSuccess should never be used when sil ownership is enabled");
-  return {SGF.emitManagedRValueWithCleanup(value, valueTL), consumption};
 }
 
 static ConsumableManagedValue
@@ -1873,6 +1877,7 @@ void PatternMatchEmission::emitEnumElementDispatch(
     switch (src.getFinalConsumption()) {
     case CastConsumptionKind::TakeAlways:
     case CastConsumptionKind::CopyOnSuccess:
+    case CastConsumptionKind::BorrowAlways:
       // No change to src necessary.
       break;
 
@@ -1977,7 +1982,13 @@ void PatternMatchEmission::emitEnumElementDispatch(
           eltValue = SGF.B.createUncheckedTakeEnumDataAddr(loc, srcValue,
                                                            elt, eltTy);
           break;
-
+        case CastConsumptionKind::BorrowAlways:
+          // If we reach this point, we know that we have a loadable
+          // element type from an enum with mixed address
+          // only/loadable cases. Since we had an address only type,
+          // we assume that we will not have BorrowAlways since
+          // address only types do not support BorrowAlways.
+          llvm_unreachable("not allowed");
         case CastConsumptionKind::CopyOnSuccess: {
           auto copy = SGF.emitTemporaryAllocation(loc, srcValue->getType());
           SGF.B.createCopyAddr(loc, srcValue, copy,

lib/SILOptimizer/Utils/CastOptimizer.cpp

@@ -242,13 +242,19 @@ SILInstruction *CastOptimizer::optimizeBridgedObjCToSwiftCast(
   }
 
   if (auto *CCABI = dyn_cast<CheckedCastAddrBranchInst>(Inst)) {
-    if (CCABI->getConsumptionKind() == CastConsumptionKind::TakeAlways) {
+    switch (CCABI->getConsumptionKind()) {
+    case CastConsumptionKind::TakeAlways:
       Builder.createReleaseValue(Loc, SrcOp, Builder.getDefaultAtomicity());
-    } else if (CCABI->getConsumptionKind() ==
-               CastConsumptionKind::TakeOnSuccess) {
+      break;
+    case CastConsumptionKind::TakeOnSuccess:
       // Insert a release in the success BB.
       Builder.setInsertionPoint(SuccessBB->begin());
       Builder.createReleaseValue(Loc, SrcOp, Builder.getDefaultAtomicity());
+      break;
+    case CastConsumptionKind::BorrowAlways:
+      llvm_unreachable("checked_cast_addr_br never has BorrowAlways");
+    case CastConsumptionKind::CopyOnSuccess:
+      break;
     }
   }
 
@@ -442,6 +448,7 @@ SILInstruction *CastOptimizer::optimizeBridgedSwiftToObjCCast(
     case CastConsumptionKind::TakeOnSuccess:
       needReleaseInSuccess = true;
       break;
+    case CastConsumptionKind::BorrowAlways:
     case CastConsumptionKind::CopyOnSuccess:
       // Conservatively insert a retain/release pair around the conversion
       // function because the conversion function could decrement the
@@ -771,12 +778,21 @@ SILInstruction *CastOptimizer::simplifyCheckedCastAddrBranchInst(
     if (!Src->getType().isAddress() || !Dest->getType().isAddress()) {
       return nullptr;
     }
+
     // For CopyOnSuccess casts, we could insert an explicit copy here, but this
     // case does not happen in practice.
+    //
     // Both TakeOnSuccess and TakeAlways can be reduced to an
     // UnconditionalCheckedCast, since the failure path is irrelevant.
-    if (Inst->getConsumptionKind() == CastConsumptionKind::CopyOnSuccess)
+    switch (Inst->getConsumptionKind()) {
+    case CastConsumptionKind::BorrowAlways:
+      llvm_unreachable("checked_cast_addr_br never has BorrowAlways");
+    case CastConsumptionKind::CopyOnSuccess:
       return nullptr;
+    case CastConsumptionKind::TakeAlways:
+    case CastConsumptionKind::TakeOnSuccess:
+      break;
+    }
 
     if (!emitSuccessfulIndirectUnconditionalCast(Builder, Mod.getSwiftModule(),
                                                  Loc, Src, SourceType, Dest,

lib/Serialization/DeserializeSIL.cpp

@@ -789,6 +789,8 @@ static CastConsumptionKind getCastConsumptionKind(unsigned attr) {
     return CastConsumptionKind::TakeOnSuccess;
   case SIL_CAST_CONSUMPTION_COPY_ON_SUCCESS:
     return CastConsumptionKind::CopyOnSuccess;
+  case SIL_CAST_CONSUMPTION_BORROW_ALWAYS:
+    return CastConsumptionKind::BorrowAlways;
   default:
     llvm_unreachable("not a valid CastConsumptionKind for SIL");
   }

lib/Serialization/SILFormat.h

@@ -76,6 +76,7 @@ enum CastConsumptionKindEncoding : uint8_t {
   SIL_CAST_CONSUMPTION_TAKE_ALWAYS,
   SIL_CAST_CONSUMPTION_TAKE_ON_SUCCESS,
   SIL_CAST_CONSUMPTION_COPY_ON_SUCCESS,
+  SIL_CAST_CONSUMPTION_BORROW_ALWAYS,
 };
 
 enum class KeyPathComponentKindEncoding : uint8_t {

lib/Serialization/SerializeSIL.cpp

@@ -85,6 +85,8 @@ static unsigned toStableCastConsumptionKind(CastConsumptionKind kind) {
     return SIL_CAST_CONSUMPTION_TAKE_ON_SUCCESS;
   case CastConsumptionKind::CopyOnSuccess:
     return SIL_CAST_CONSUMPTION_COPY_ON_SUCCESS;
+  case CastConsumptionKind::BorrowAlways:
+    return SIL_CAST_CONSUMPTION_BORROW_ALWAYS;
   }
   llvm_unreachable("bad cast consumption kind");
 }