6.0: [SILGen] Store_borrow into in_guaranteed.

lib/SILGen/Initialization.h

@@ -185,13 +185,16 @@ class Initialization {
   /// of last resort: it is generally better to split tuples or evaluate
   /// in-place when the initialization supports that.
   ///
-  /// If this is an *copy* of the rvalue into this initialization then isInit is
+  /// If this is a *copy* of the rvalue into this initialization then isInit is
   /// false.  If it is an *initialization* of the memory in the initialization,
   /// then isInit is true.
   virtual void copyOrInitValueInto(SILGenFunction &SGF, SILLocation loc,
                                    ManagedValue explodedElement,
                                    bool isInit) = 0;
 
+  /// Whether the storage owns what's stored or merely borrows it.
+  virtual bool isBorrow() { return false; }
+
   /// Whether to emit a debug value during initialization.
   void setEmitDebugValueOnInit(bool emit) { EmitDebugValueOnInit = emit; }
 
@@ -217,7 +220,7 @@ class Initialization {
 
 /// Abstract base class for single-buffer initializations.  These are
 /// initializations that have an addressable memory object to be stored into.
-class SingleBufferInitialization : public Initialization {
+class SingleBufferInitialization : virtual public Initialization {
   llvm::TinyPtrVector<CleanupHandle::AsPointer> SplitCleanups;
 public:
   SingleBufferInitialization() {}
@@ -262,7 +265,7 @@ class SingleBufferInitialization : public Initialization {
                                      SmallVectorImpl<InitializationPtr> &buf,
                        TinyPtrVector<CleanupHandle::AsPointer> &splitCleanups);
 };
-  
+
 /// This is an initialization for a specific address in memory.
 class KnownAddressInitialization : public SingleBufferInitialization {
   /// The physical address of the global.
@@ -285,7 +288,13 @@ class KnownAddressInitialization : public SingleBufferInitialization {
   void finishUninitialized(SILGenFunction &SGF) override {}
 };
 
-class TemporaryInitialization : public SingleBufferInitialization {
+class AnyTemporaryInitialization : virtual public Initialization {
+public:
+  virtual ManagedValue getManagedAddress() const = 0;
+};
+
+class TemporaryInitialization : public SingleBufferInitialization,
+                                public AnyTemporaryInitialization {
   SILValue Addr;
   CleanupHandle Cleanup;
 public:
@@ -312,12 +321,38 @@ class TemporaryInitialization : public SingleBufferInitialization {
   /// Returns the cleanup corresponding to the value of the temporary.
   CleanupHandle getInitializedCleanup() const { return Cleanup; }
 
-  ManagedValue getManagedAddress() const  {
+  ManagedValue getManagedAddress() const override {
     return ManagedValue::forOwnedAddressRValue(getAddress(),
                                                getInitializedCleanup());
   }
 };
 
+class StoreBorrowInitialization final : public AnyTemporaryInitialization {
+  SILValue address;
+  ManagedValue storeBorrow;
+
+public:
+  StoreBorrowInitialization(SILValue address);
+
+  void copyOrInitValueInto(SILGenFunction &SGF, SILLocation loc,
+                           ManagedValue mv, bool isInit) override;
+
+  void finishInitialization(SILGenFunction &SGF) override {}
+
+  void finishUninitialized(SILGenFunction &SGF) override {}
+
+  bool isBorrow() override { return true; }
+
+  SILValue getAddress() const;
+
+  bool isInPlaceInitializationOfGlobal() const override {
+    // Can't store_borrow to a global.
+    return false;
+  }
+
+  ManagedValue getManagedAddress() const override;
+};
+
 /// An initialization which accumulates several other initializations
 /// into a tuple.
 class TupleInitialization : public Initialization {

lib/SILGen/ManagedValue.cpp

@@ -170,7 +170,7 @@ void ManagedValue::assignInto(SILGenFunction &SGF, SILLocation loc,
 
 void ManagedValue::forwardInto(SILGenFunction &SGF, SILLocation loc,
                                Initialization *dest) {
-  assert(isPlusOneOrTrivial(SGF));
+  assert(isPlusOneOrTrivial(SGF) || dest->isBorrow());
   dest->copyOrInitValueInto(SGF, loc, *this, /*isInit*/ true);
   dest->finishInitialization(SGF);
 }

lib/SILGen/SILGenDecl.cpp

@@ -367,6 +367,43 @@ void TemporaryInitialization::finishInitialization(SILGenFunction &SGF) {
     SGF.Cleanups.setCleanupState(Cleanup, CleanupState::Active);
 }
 
+StoreBorrowInitialization::StoreBorrowInitialization(SILValue address)
+    : address(address) {
+  assert(isa<AllocStackInst>(address) ||
+         isa<MarkUnresolvedNonCopyableValueInst>(address) &&
+             "invalid destination for store_borrow initialization!?");
+}
+
+void StoreBorrowInitialization::copyOrInitValueInto(SILGenFunction &SGF,
+                                                    SILLocation loc,
+                                                    ManagedValue mv,
+                                                    bool isInit) {
+  auto value = mv.getValue();
+  auto &lowering = SGF.getTypeLowering(value->getType());
+  if (lowering.isAddressOnly() && SGF.silConv.useLoweredAddresses()) {
+    llvm::report_fatal_error(
+        "Attempting to store_borrow an address-only value!?");
+  }
+  if (value->getType().isAddress()) {
+    value = SGF.emitManagedLoadBorrow(loc, value).getValue();
+  }
+  if (!isInit) {
+    value = lowering.emitCopyValue(SGF.B, loc, value);
+  }
+  storeBorrow = SGF.emitManagedStoreBorrow(loc, value, address);
+}
+
+SILValue StoreBorrowInitialization::getAddress() const {
+  if (storeBorrow) {
+    return storeBorrow.getValue();
+  }
+  return address;
+}
+
+ManagedValue StoreBorrowInitialization::getManagedAddress() const {
+  return storeBorrow;
+}
+
 namespace {
 class ReleaseValueCleanup : public Cleanup {
   SILValue v;

lib/SILGen/SILGenPoly.cpp

@@ -1080,6 +1080,33 @@ class ParamInfo {
   IndirectSlot slot;
   ParameterConvention convention;
 
+  bool temporaryShouldBorrow(SILGenFunction &SGF, bool forceAllocation) {
+    if (slot.hasAddress() && !forceAllocation) {
+      // An address has already been allocated via some projection.  It is not
+      // currently supported to store_borrow to such projections.
+      return false;
+    }
+    auto &tl = getTypeLowering(SGF);
+    if (tl.isAddressOnly() && SGF.silConv.useLoweredAddresses()) {
+      // In address-lowered mode, address-only types can't be loaded in the
+      // first place before being store_borrow'd.
+      return false;
+    }
+    if (convention != ParameterConvention::Indirect_In_Guaranteed) {
+      // Can only store_borrow into a temporary allocation for @in_guaranteed.
+      return false;
+    }
+    if (tl.isTrivial()) {
+      // Can't store_borrow a trivial type.
+      return false;
+    }
+    return true;
+  }
+
+  TypeLowering const &getTypeLowering(SILGenFunction &SGF) {
+    return SGF.getTypeLowering(getType());
+  }
+
 public:
   ParamInfo(IndirectSlot slot, ParameterConvention convention)
     : slot(slot), convention(convention) {}
@@ -1088,11 +1115,26 @@ class ParamInfo {
     return slot.allocate(SGF, loc);
   }
 
-  std::unique_ptr<TemporaryInitialization>
-  allocateForInitialization(SILGenFunction &SGF, SILLocation loc) const {
-    auto addr = slot.allocate(SGF, loc);
-    auto &addrTL = SGF.getTypeLowering(addr->getType());
-    return SGF.useBufferAsTemporary(addr, addrTL);
+  std::unique_ptr<AnyTemporaryInitialization>
+  allocateForInitialization(SILGenFunction &SGF, SILLocation loc,
+                            bool forceAllocation = false) {
+    auto &lowering = getTypeLowering(SGF);
+    SILValue address;
+    if (forceAllocation) {
+      address = SGF.emitTemporaryAllocation(loc, lowering.getLoweredType());
+    } else {
+      address = allocate(SGF, loc);
+    }
+    if (address->getType().isMoveOnly())
+      address = SGF.B.createMarkUnresolvedNonCopyableValueInst(
+          loc, address,
+          MarkUnresolvedNonCopyableValueInst::CheckKind::
+              ConsumableAndAssignable);
+    if (temporaryShouldBorrow(SGF, forceAllocation)) {
+      return std::make_unique<StoreBorrowInitialization>(address);
+    }
+    auto innerTemp = SGF.useBufferAsTemporary(address, lowering);
+    return innerTemp;
   }
 
   SILType getType() const {
@@ -1904,8 +1946,8 @@ class TranslateArguments : public ExpanderBase<TranslateArguments, ParamInfo> {
                                ManagedValue outerArg,
                                ParamInfo innerSlot) {
     auto innerResultTy = innerSlot.getType();
-    auto &innerTL = SGF.getTypeLowering(innerResultTy);
-    auto innerTemp = SGF.emitTemporary(Loc, innerTL);
+    auto innerTemp =
+        innerSlot.allocateForInitialization(SGF, Loc, /*forceAllocation=*/true);
     processSingleInto(innerOrigType, innerSubstType,
                       outerOrigType, outerSubstType,
                       outerArg, innerResultTy.getAddressType(),
@@ -2139,7 +2181,7 @@ class TranslateArguments : public ExpanderBase<TranslateArguments, ParamInfo> {
                                      outer, innerTy,
                                      SGFContext(&init));
     if (!innerArg.isInContext()) {
-      if (innerArg.isPlusOneOrTrivial(SGF)) {
+      if (innerArg.isPlusOneOrTrivial(SGF) || init.isBorrow()) {
         innerArg.forwardInto(SGF, Loc, &init);
       } else {
         innerArg.copyInto(SGF, Loc, &init);

lib/SILOptimizer/Mandatory/MoveOnlyAddressCheckerUtils.cpp

@@ -3043,6 +3043,13 @@ static void insertDestroyBeforeInstruction(UseState &addressUseState,
                                            SILValue baseAddress,
                                            SmallBitVector &bv,
                                            ConsumeInfo &consumes) {
+  if (!baseAddress->getUsersOfType<StoreBorrowInst>().empty()) {
+    // If there are _any_ store_borrow users, then all users of the address are
+    // store_borrows (and dealloc_stacks).  Nothing is stored, there's nothing
+    // to destroy.
+    return;
+  }
+
   // If we need all bits...
   if (bv.all()) {
     // And our next instruction is a destroy_addr on the base address, just

test/SILGen/protocol_resilience.swift

@@ -220,15 +220,14 @@ extension ReabstractSelfBase {
 
 // CHECK-LABEL: sil private [transparent] [thunk] [ossa] @$s19protocol_resilience21ReabstractSelfRefinedP8callbackyxxcvg :
 // CHECK: [[SELF_BOX:%.*]] = alloc_stack $τ_0_0
-// CHECK-NEXT: [[SELF_COPY:%.*]] = copy_value %0 : $τ_0_0
-// CHECK-NEXT: store [[SELF_COPY]] to [init] [[SELF_BOX]] : $*τ_0_0
+// CHECK-NEXT: [[SELF_BOX_BORROW:%.*]] = store_borrow %0 to [[SELF_BOX]]
 // CHECK: [[WITNESS:%.*]] = function_ref @$s19protocol_resilience18ReabstractSelfBasePAAE8callbackyxxcvg
-// CHECK-NEXT: [[RESULT:%.*]] = apply [[WITNESS]]<τ_0_0>([[SELF_BOX]])
+// CHECK-NEXT: [[RESULT:%.*]] = apply [[WITNESS]]<τ_0_0>([[SELF_BOX_BORROW]])
 // CHECK-NEXT: [[RESULT_CONV:%.*]] = convert_function [[RESULT]]
 // CHECK: [[THUNK_FN:%.*]] = function_ref
 // CHECK-NEXT: [[THUNK:%.*]] = partial_apply [callee_guaranteed] [[THUNK_FN]]<τ_0_0>([[RESULT_CONV]])
 // CHECK-NEXT: [[THUNK_CONV:%.*]] = convert_function [[THUNK]]
-// CHECK-NEXT: destroy_addr [[SELF_BOX]]
+// CHECK-NEXT: end_borrow [[SELF_BOX_BORROW]]
 // CHECK-NEXT: dealloc_stack [[SELF_BOX]]
 // CHECK-NEXT: return [[THUNK_CONV]]
 

test/SILGen/variadic-generic-reabstract-tuple-arg.swift

@@ -36,15 +36,14 @@ func test1() {
 // CHECK-NEXT:    [[PACK:%.*]] = alloc_pack $Pack{String}
 // CHECK-NEXT:    [[ARG_TEMP:%.*]] = alloc_stack $String
 //   It'd be nice to avoid this unnecessary copy.
-// CHECK-NEXT:    [[ARG_COPY:%.*]] = copy_value %0 : $String
-// CHECK-NEXT:    store [[ARG_COPY]] to [init] [[ARG_TEMP]] : $*String
+// CHECK-NEXT:    [[ARG_TEMP_BORROW:%.*]] = store_borrow %0 to [[ARG_TEMP]]
 // CHECK-NEXT:    [[INDEX:%.*]] = scalar_pack_index 0 of $Pack{String}
-// CHECK-NEXT:    pack_element_set [[ARG_TEMP]] : $*String into [[INDEX]] of [[PACK]] : $*Pack{String}
+// CHECK-NEXT:    pack_element_set [[ARG_TEMP_BORROW]] : $*String into [[INDEX]] of [[PACK]] : $*Pack{String}
 // CHECK-NEXT:    [[RESULT_TEMP:%.*]] = alloc_stack $Array<String>
 // CHECK-NEXT:    apply %1([[RESULT_TEMP]], [[PACK]]) : $@callee_guaranteed (@pack_guaranteed Pack{String}) -> @out Array<String>
 // CHECK-NEXT:    [[RESULT:%.*]] = load [take] [[RESULT_TEMP]] : $*Array<String>
 // CHECK-NEXT:    dealloc_stack [[RESULT_TEMP]] : $*Array<String>
-// CHECK-NEXT:    destroy_addr [[ARG_TEMP]] : $*String
+// CHECK-NEXT:    end_borrow [[ARG_TEMP_BORROW]]
 // CHECK-NEXT:    dealloc_stack [[ARG_TEMP]] : $*String
 // CHECK-NEXT:    dealloc_pack [[PACK]] : $*Pack{String}
 // CHECK-NEXT:    return [[RESULT]] : $Array<String>

test/SILGen/witnesses.swift

@@ -250,15 +250,14 @@ struct ConformsWithMoreGeneric : X, Y {
   // CHECK-LABEL: sil private [transparent] [thunk] [ossa] @$s9witnesses23ConformsWithMoreGenericVAA1XA2aDP7classes{{[_0-9a-zA-Z]*}}FTW :
   // CHECK:       bb0([[ARG0:%.*]] : @guaranteed $τ_0_0, [[ARG1:%.*]] : $*ConformsWithMoreGeneric):
   // CHECK-NEXT:    [[SELF_BOX:%.*]] = alloc_stack $τ_0_0
-  // CHECK-NEXT:    [[ARG0_COPY:%.*]] = copy_value [[ARG0]]
-  // CHECK-NEXT:    store [[ARG0_COPY]] to [init] [[SELF_BOX]] : $*τ_0_0
+  // CHECK-NEXT:    [[SELF_BOX_BORROW:%.*]] = store_borrow [[ARG0]] to [[SELF_BOX]]
   // CHECK-NEXT:    // function_ref witnesses.ConformsWithMoreGeneric.classes
   // CHECK-NEXT:    [[WITNESS_FN:%.*]] = function_ref @$s9witnesses23ConformsWithMoreGenericV7classes{{[_0-9a-zA-Z]*}}F : $@convention(method) <τ_0_0> (@in_guaranteed τ_0_0, @inout ConformsWithMoreGeneric) -> @out τ_0_0
   // CHECK-NEXT:    [[RESULT_BOX:%.*]] = alloc_stack $τ_0_0
-  // CHECK-NEXT:    [[RESULT:%.*]] = apply [[WITNESS_FN]]<τ_0_0>([[RESULT_BOX]], [[SELF_BOX]], %1) : $@convention(method) <τ_0_0> (@in_guaranteed τ_0_0, @inout ConformsWithMoreGeneric) -> @out τ_0_0
+  // CHECK-NEXT:    [[RESULT:%.*]] = apply [[WITNESS_FN]]<τ_0_0>([[RESULT_BOX]], [[SELF_BOX_BORROW]], %1) : $@convention(method) <τ_0_0> (@in_guaranteed τ_0_0, @inout ConformsWithMoreGeneric) -> @out τ_0_0
   // CHECK-NEXT:    [[RESULT:%.*]] = load [take] [[RESULT_BOX]] : $*τ_0_0
   // CHECK-NEXT:    dealloc_stack [[RESULT_BOX]] : $*τ_0_0
-  // CHECK-NEXT:    destroy_addr [[SELF_BOX]]
+  // CHECK-NEXT:    end_borrow [[SELF_BOX_BORROW]]
   // CHECK-NEXT:    dealloc_stack [[SELF_BOX]] : $*τ_0_0
   // CHECK-NEXT:    return [[RESULT]] : $τ_0_0
   // CHECK-NEXT:  }

test/SILOptimizer/moveonly_addresschecker_unmaximized.sil

@@ -23,6 +23,7 @@ struct M4: ~Copyable {
 
 sil @get_M4 : $@convention(thin) () -> @owned M4
 sil @end_2 : $@convention(thin) (@owned M, @owned M) -> ()
+sil @see_addr : $@convention(thin) (@in_guaranteed M) -> ()
 sil @see_addr_2 : $@convention(thin) (@in_guaranteed M, @in_guaranteed M) -> ()
 sil @replace_2 : $@convention(thin) (@inout M, @inout M) -> ()
 sil @get_out_2 : $@convention(thin) () -> (@out M, @out M)
@@ -235,3 +236,20 @@ bb0(%m_in : @owned $M):
   apply %die(%pa) : $@convention(thin) (@guaranteed @noescape @callee_guaranteed () -> @owned String) -> Never
   unreachable
 }
+
+// CHECK-LABEL: sil [ossa] @dont_destroy_store_borrowed_addr : {{.*}} {
+// CHECK-NOT:     destroy_addr
+// CHECK-LABEL: } // end sil function 'dont_destroy_store_borrowed_addr'
+sil [ossa] @dont_destroy_store_borrowed_addr : $@convention(thin) (@guaranteed M) -> () {
+bb0(%0 : @guaranteed $M):
+  %stack = alloc_stack $M
+  %mark = mark_unresolved_non_copyable_value [consumable_and_assignable] %stack : $*M
+  %borrow = store_borrow %0 to %mark : $*M
+  %see_addr = function_ref @see_addr : $@convention(thin) (@in_guaranteed M) -> ()
+  apply %see_addr(%borrow) : $@convention(thin) (@in_guaranteed M) -> ()
+  end_borrow %borrow : $*M
+  dealloc_stack %stack : $*M
+  %retval = tuple ()
+  return %retval : $()
+}
+

validation-test/SILGen/rdar128710064.swift

@@ -0,0 +1,25 @@
+// RUN: %target-swift-emit-silgen %s | %FileCheck %s
+// RUN: %target-swift-emit-sil %s -verify
+
+struct NC : ~Copyable {}
+
+func loadClosure(_ umrp: UnsafeMutableRawPointer) {
+  typealias Enumerator = (borrowing NC) -> Void
+  let body = umrp.load(as: Enumerator.self)
+  _ = body
+}
+
+// CHECK-LABEL: sil {{.*}} @$s13rdar1287100642NCVytIegnr_ACIegg_TR : {{.*}} {
+// CHECK:       bb0(
+// CHECK-SAME:      [[NC:%[^,]+]] :
+// CHECK-SAME:    , [[CLOSURE:%[^,]+]] :
+// CHECK-SAME:  ):
+// CHECK:         [[NC_INDIRECT_ADDR:%[^,]+]] = alloc_stack $NC
+// CHECK:         [[NC_INDIRECT_CHECK:%[^,]+]] = mark_unresolved_non_copyable_value [consumable_and_assignable] [[NC_INDIRECT_ADDR]]
+// CHECK:         [[NC_INDIRECT:%[^,]+]] = store_borrow [[NC]] to [[NC_INDIRECT_CHECK]]
+// CHECK:         [[OUT:%[^,]+]] = alloc_stack $()
+// CHECK:         apply [[CLOSURE]]([[OUT]], [[NC_INDIRECT]])
+// CHECK:         dealloc_stack [[OUT]] : $*()
+// CHECK:         end_borrow [[NC_INDIRECT]]
+// CHECK:         dealloc_stack [[NC_INDIRECT_ADDR]]
+// CHECK-LABEL: } // end sil function '$s13rdar1287100642NCVytIegnr_ACIegg_TR'