[SILGen] Used move_value for more lexical values.

include/swift/SIL/InstWrappers.h

@@ -257,8 +257,12 @@ class ForwardingOperation {
   // ForwardingInstruction.swift mirrors this implementation.
   Operand *getSingleForwardingOperand() const {
     switch (forwardingInst->getKind()) {
-    case SILInstructionKind::StructInst:
     case SILInstructionKind::TupleInst:
+    case SILInstructionKind::StructInst: {
+      if (forwardingInst->getNumRealOperands() != 1)
+        return nullptr;
+      return *forwardingInst->getRealOperands().begin();
+    }
     case SILInstructionKind::LinearFunctionInst:
     case SILInstructionKind::DifferentiableFunctionInst:
       return nullptr;

include/swift/SIL/SILBuilder.h

@@ -829,11 +829,14 @@ class SILBuilder {
     return createLoadBorrow(loc, v);
   }
 
-  SILValue emitBeginBorrowOperation(SILLocation loc, SILValue v) {
+  SILValue emitBeginBorrowOperation(SILLocation loc, SILValue v,
+                                    bool isLexical = false,
+                                    bool hasPointerEscape = false,
+                                    bool fromVarDecl = false) {
     if (!hasOwnership() ||
         v->getOwnershipKind().isCompatibleWith(OwnershipKind::Guaranteed))
       return v;
-    return createBeginBorrow(loc, v);
+    return createBeginBorrow(loc, v, isLexical, hasPointerEscape, fromVarDecl);
   }
 
   void emitEndBorrowOperation(SILLocation loc, SILValue v) {
@@ -2817,13 +2820,16 @@ class SILBuilder {
 
   /// Convenience function that is a no-op for trivial values and inserts a
   /// move_value on non-trivial instructions.
-  SILValue emitMoveValueOperation(SILLocation Loc, SILValue v) {
+  SILValue emitMoveValueOperation(SILLocation Loc, SILValue v,
+                                  bool isLexical = false,
+                                  bool hasPointerEscape = false,
+                                  bool fromVarDecl = false) {
     assert(!v->getType().isAddress());
     if (v->getType().isTrivial(*getInsertionBB()->getParent()))
       return v;
     assert(v->getOwnershipKind() == OwnershipKind::Owned &&
            "move_value consumes its argument");
-    return createMoveValue(Loc, v);
+    return createMoveValue(Loc, v, isLexical, hasPointerEscape, fromVarDecl);
   }
 
   SILValue emitTupleExtract(SILLocation Loc, SILValue Operand, unsigned FieldNo,

include/swift/SIL/SILInstruction.h

@@ -612,6 +612,16 @@ class SILInstruction : public llvm::ilist_node<SILInstruction> {
   /// Return the array of mutable type dependent operands for this instruction.
   MutableArrayRef<Operand> getTypeDependentOperands();
 
+private:
+  struct FilterOperandToRealOperand;
+
+public:
+  using RealOperandRange =
+      OptionalTransformRange<ArrayRef<Operand>, FilterOperandToRealOperand>;
+
+  /// The array of real (i.e. not type-dependent) operands.
+  RealOperandRange getRealOperands() const;
+
   unsigned getNumOperands() const { return getAllOperands().size(); }
 
   unsigned getNumTypeDependentOperands() const {
@@ -1005,6 +1015,18 @@ struct SILInstruction::OperandRefToValue {
   }
 };
 
+struct SILInstruction::FilterOperandToRealOperand {
+  const SILInstruction &i;
+
+  FilterOperandToRealOperand(const SILInstruction &i) : i(i) {}
+
+  llvm::Optional<Operand *> operator()(const Operand &use) const {
+    if (i.isTypeDependentOperand(use))
+      return llvm::None;
+    return {const_cast<Operand *>(&use)};
+  }
+};
+
 struct SILInstruction::NonTypeDependentOperandToValue {
   const SILInstruction &i;
 
@@ -1036,6 +1058,11 @@ struct SILInstruction::OperandToTransformedValue {
   }
 };
 
+inline SILInstruction::RealOperandRange
+SILInstruction::getRealOperands() const {
+  return RealOperandRange(getAllOperands(), FilterOperandToRealOperand(*this));
+}
+
 inline SILInstruction::OperandValueRange
 SILInstruction::getOperandValues(ArrayRef<Operand*> operands) {
   return OperandValueRange(operands, OperandToValue());

include/swift/SILOptimizer/Utils/CanonicalizeOSSALifetime.h

@@ -414,6 +414,8 @@ class CanonicalizeOSSALifetime final {
   bool respectsDeinitBarriers() const {
     if (!currentDef->isLexical())
       return false;
+    if (currentDef->getFunction()->forceEnableLexicalLifetimes())
+      return true;
     auto &module = currentDef->getFunction()->getModule();
     return module.getASTContext().SILOpts.supportsLexicalLifetimes(module);
   }

lib/SILGen/SILGenDecl.cpp

@@ -832,25 +832,33 @@ class LetValueInitialization : public Initialization {
               ConsumableAndAssignable);
     }
 
-    // Otherwise, if we do not have a no implicit copy variable, just follow
-    // the "normal path": perform a lexical borrow if the lifetime is lexical.
-    if (!vd->isNoImplicitCopy()) {
-      return SGF.B.createBeginBorrow(
+    // If we have a no implicit copy lexical, emit the instruction stream so
+    // that the move checker knows to check this variable.
+    if (vd->isNoImplicitCopy()) {
+      value = SGF.B.createMoveValue(PrologueLoc, value, /*isLexical*/ true,
+                                    /*hasPointerEscape=*/false,
+                                    /*fromVarDecl=*/true);
+      value =
+          SGF.B.createOwnedCopyableToMoveOnlyWrapperValue(PrologueLoc, value);
+      return SGF.B.createMarkUnresolvedNonCopyableValueInst(
           PrologueLoc, value,
-          /*isLexical=*/SGF.F.getLifetime(vd, value->getType()).isLexical(),
-          /*hasPointerEscape=*/false, /*fromVarDecl=*/true);
+          MarkUnresolvedNonCopyableValueInst::CheckKind::
+              ConsumableAndAssignable);
     }
 
-    // If we have a no implicit copy lexical, emit the instruction stream so
-    // that the move checker knows to check this variable.
-    value = SGF.B.createBeginBorrow(
-        PrologueLoc, value,
-        /*isLexical*/ true, /*hasPointerEscape=*/false, /*fromVarDecl=*/true);
-    value = SGF.B.createCopyValue(PrologueLoc, value);
-    value = SGF.B.createOwnedCopyableToMoveOnlyWrapperValue(PrologueLoc, value);
-    return SGF.B.createMarkUnresolvedNonCopyableValueInst(
-        PrologueLoc, value,
-        MarkUnresolvedNonCopyableValueInst::CheckKind::ConsumableAndAssignable);
+    // Otherwise, if we do not have a no implicit copy variable, just follow
+    // the "normal path".
+
+    auto isLexical = SGF.F.getLifetime(vd, value->getType()).isLexical();
+
+    if (value->getOwnershipKind() == OwnershipKind::Owned)
+      return SGF.B.createMoveValue(PrologueLoc, value, /*isLexical*/ isLexical,
+                                   /*hasPointerEscape=*/false,
+                                   /*fromVarDecl=*/true);
+
+    return SGF.B.createBeginBorrow(PrologueLoc, value, /*isLexical*/ isLexical,
+                                   /*hasPointerEscape=*/false,
+                                   /*fromVarDecl=*/true);
   }
 
   void bindValue(SILValue value, SILGenFunction &SGF, bool wasPlusOne,
@@ -2176,6 +2184,11 @@ void SILGenFunction::destroyLocalVariable(SILLocation silLoc, VarDecl *vd) {
     return;
   }
 
+  if (auto *mvi = dyn_cast<MoveValueInst>(Val.getDefiningInstruction())) {
+    B.emitDestroyValueOperation(silLoc, mvi);
+    return;
+  }
+
   if (auto *mvi = dyn_cast<MarkUnresolvedNonCopyableValueInst>(
           Val.getDefiningInstruction())) {
     if (mvi->hasMoveCheckerKind()) {
@@ -2192,14 +2205,10 @@ void SILGenFunction::destroyLocalVariable(SILLocation silLoc, VarDecl *vd) {
 
       if (auto *copyToMove = dyn_cast<CopyableToMoveOnlyWrapperValueInst>(
               mvi->getOperand())) {
-        if (auto *cvi = dyn_cast<CopyValueInst>(copyToMove->getOperand())) {
-          if (auto *bbi = dyn_cast<BeginBorrowInst>(cvi->getOperand())) {
-            if (bbi->isLexical()) {
-              B.emitDestroyValueOperation(silLoc, mvi);
-              B.createEndBorrow(silLoc, bbi);
-              B.emitDestroyValueOperation(silLoc, bbi->getOperand());
-              return;
-            }
+        if (auto *move = dyn_cast<MoveValueInst>(copyToMove->getOperand())) {
+          if (move->isLexical()) {
+            B.emitDestroyValueOperation(silLoc, mvi);
+            return;
           }
         }
       }

lib/SILOptimizer/Differentiation/JVPCloner.cpp

@@ -777,7 +777,9 @@ class JVPCloner::Implementation final
     auto &diffBuilder = getDifferentialBuilder();
     auto loc = bbi->getLoc();
     auto tanVal = materializeTangent(getTangentValue(bbi->getOperand()), loc);
-    auto tanValBorrow = diffBuilder.emitBeginBorrowOperation(loc, tanVal);
+    auto tanValBorrow = diffBuilder.emitBeginBorrowOperation(
+        loc, tanVal, bbi->isLexical(), bbi->hasPointerEscape(),
+        bbi->isFromVarDecl());
     setTangentValue(bbi->getParent(), bbi,
                     makeConcreteTangentValue(tanValBorrow));
   }
@@ -805,6 +807,17 @@ class JVPCloner::Implementation final
                     makeConcreteTangentValue(tanValCopy));
   }
 
+  CLONE_AND_EMIT_TANGENT(MoveValue, mvi) {
+    auto &diffBuilder = getDifferentialBuilder();
+    auto tan = getTangentValue(mvi->getOperand());
+    auto tanVal = materializeTangent(tan, mvi->getLoc());
+    auto tanValMove = diffBuilder.emitMoveValueOperation(
+        mvi->getLoc(), tanVal, mvi->isLexical(), mvi->hasPointerEscape(),
+        mvi->isFromVarDecl());
+    setTangentValue(mvi->getParent(), mvi,
+                    makeConcreteTangentValue(tanValMove));
+  }
+
   /// Handle `load` instruction.
   ///   Original: y = load x
   ///    Tangent: tan[y] = load tan[x]

lib/SILOptimizer/LoopTransforms/ForEachLoopUnroll.cpp

@@ -307,7 +307,7 @@ void ArrayInfo::classifyUsesOfArray(SILValue arrayValue) {
       forEachCalls.insert(forEachCall);
       continue;
     }
-    // Recursively classify begin_borrow and copy_value uses.
+    // Recursively classify begin_borrow, copy_value, and move_value uses.
     if (BeginBorrowInst *beginBorrow = dyn_cast<BeginBorrowInst>(user)) {
       classifyUsesOfArray(beginBorrow);
       continue;
@@ -316,6 +316,10 @@ void ArrayInfo::classifyUsesOfArray(SILValue arrayValue) {
       classifyUsesOfArray(copyValue);
       continue;
     }
+    if (MoveValueInst *moveValue = dyn_cast<MoveValueInst>(user)) {
+      classifyUsesOfArray(moveValue);
+      continue;
+    }
     if (DestroyValueInst *destroyValue = dyn_cast<DestroyValueInst>(user)) {
       destroys.push_back(destroyValue);
       continue;

lib/SILOptimizer/Mandatory/ConsumeOperatorCopyableValuesChecker.cpp

@@ -409,6 +409,13 @@ bool ConsumeOperatorCopyableValuesChecker::check() {
 
   for (auto &block : *fn) {
     for (auto &ii : block) {
+      if (auto *mvi = dyn_cast<MoveValueInst>(&ii)) {
+        if (mvi->isFromVarDecl() && !mvi->getType().isMoveOnly()) {
+          LLVM_DEBUG(llvm::dbgs()
+                     << "Found lexical lifetime to check: " << *mvi);
+          valuesToCheck.insert(mvi);
+        }
+      }
       if (auto *bbi = dyn_cast<BeginBorrowInst>(&ii)) {
         if (bbi->isFromVarDecl() && !bbi->getType().isMoveOnly()) {
           LLVM_DEBUG(llvm::dbgs()

lib/SILOptimizer/Mandatory/OSLogOptimization.cpp

@@ -313,6 +313,7 @@ static bool isSILValueFoldable(SILValue value) {
   return (!silType.isAddress() && !isa<LiteralInst>(definingInst) &&
           !isa<LoadBorrowInst>(definingInst) &&
           !isa<BeginBorrowInst>(definingInst) &&
+          !isa<MoveValueInst>(definingInst) &&
           !isa<CopyValueInst>(definingInst) &&
           (isFoldableIntOrBool(value, astContext) ||
            isFoldableString(value, astContext) ||

lib/SILOptimizer/SemanticARC/RedundantMoveValueElimination.cpp

@@ -33,9 +33,6 @@ using namespace semanticarc;
 //===----------------------------------------------------------------------===//
 
 bool SemanticARCOptVisitor::visitMoveValueInst(MoveValueInst *mvi) {
-  if (ctx.onlyMandatoryOpts)
-    return false;
-
   if (!ctx.shouldPerform(ARCTransformKind::RedundantMoveValueElim))
     return false;
 

lib/SILOptimizer/Transforms/CopyPropagation.cpp

@@ -503,7 +503,7 @@ void CopyPropagation::run() {
     // Run the sequence of utilities:
     // - ShrinkBorrowScope
     // - CanonicalizeOSSALifetime
-    // - LexicalDestroyFolder
+    // - LexicalDestroyFolding
     // at least once and then until each stops making changes.
     while (true) {
       SmallVector<CopyValueInst *, 4> modifiedCopyValueInsts;

lib/SILOptimizer/Utils/CanonicalizeOSSALifetime.cpp

@@ -978,9 +978,10 @@ void CanonicalizeOSSALifetime::insertDestroysOnBoundary(
           auto *insertionPoint = &*successor->begin();
           insertDestroyBeforeInstruction(insertionPoint, getCurrentDef(),
                                          consumes, getCallbacks());
-          LLVM_DEBUG(llvm::dbgs()
-                     << "  Destroy after terminator " << instruction
-                     << " at beginning of " << successor << "\n");
+          LLVM_DEBUG(llvm::dbgs() << "  Destroy after terminator "
+                                  << *instruction << " at beginning of ";
+                     successor->printID(llvm::dbgs(), false);
+                     llvm::dbgs() << "\n";);
         }
         continue;
       }

stdlib/private/SwiftReflectionTest/SwiftReflectionTest.swift

@@ -508,6 +508,7 @@ public func reflect<T: Error>(error: T) {
   let error: Error = error
   let errorPointerValue = unsafeBitCast(error, to: UInt.self)
   reflect(instanceAddress: errorPointerValue, kind: .ErrorExistential)
+  withExtendedLifetime(error) {}
 }
 
 // Like reflect<T: Error>(error: T), but calls projectExistentialAndUnwrapClass 
@@ -523,6 +524,7 @@ public func reflectUnwrappingClassExistential<T: Error>(error: T) {
           kind: .ErrorExistential, 
           shouldUnwrapClassExistential: true)
   anyPointer.deallocate()
+  withExtendedLifetime(error) {}
 }
 
 // Reflect an `Enum`

test/AutoDiff/SILGen/autodiff_builtins.swift

@@ -96,8 +96,10 @@ func test_context_builtins_with_type<T>(t: T) {
 
 // CHECK-LABEL: sil{{.*}}@test_context_builtins_with_type : $@convention(thin) <T> (@in_guaranteed T) -> () {
 // CHECK: bb0({{%.*}} : $*T):
-// CHECK:   [[CTX:%.*]] = builtin "autoDiffCreateLinearMapContextWithType"<T>({{%.*}} : $@thick T.Type) : $Builtin.NativeObject // users: {{.*}}
-// CHECK:   [[BORROWED_CTX:%.*]] = begin_borrow [lexical] [var_decl] [[CTX]] : $Builtin.NativeObject // users: {{.*}}
+// CHECK:   [[CTX:%.*]] = builtin "autoDiffCreateLinearMapContextWithType"<T>({{%.*}} : $@thick T.Type) : $Builtin.NativeObject
+// CHECK:   [[CTX_LIFETIME:%.*]] = move_value [lexical] [var_decl] [[CTX]]
+// CHECK:   [[BORROWED_CTX:%.*]] = begin_borrow [[CTX_LIFETIME]]
 // CHECK:   [[BUF:%.*]] = builtin "autoDiffProjectTopLevelSubcontext"([[BORROWED_CTX]] : $Builtin.NativeObject) : $Builtin.RawPointer // users: {{.*}}
+// CHECK:   [[BORROWED_CTX:%.*]] = begin_borrow [[CTX_LIFETIME]]
 // CHECK:   [[BUF:%.*]] = builtin "autoDiffAllocateSubcontextWithType"<T>([[BORROWED_CTX]] : $Builtin.NativeObject, {{.*}} : $@thick T.Type) : $Builtin.RawPointer // users: {{.*}}
-// CHECK:   destroy_value [[CTX]]
+// CHECK:   destroy_value [[CTX_LIFETIME]]

test/DebugInfo/guard-let-scope3.swift

@@ -15,7 +15,7 @@ public class S {
       // CHECK: debug_value {{.*}} : $Optional<C>, let, name "x", {{.*}}, scope [[X1]]
       // CHECK: debug_value {{.*}} : $C, let, name "x", {{.*}}, scope [[X2]]
       // FIXME: This source location is a little wild.
-      // CHECK-NEXT: strong_retain{{.*}}:[[@LINE+4]]:12, scope [[X2]]
+      // CHECK-NEXT: release_value{{.*}}:[[@LINE+5]]:3, scope [[X2]]
       throw MyError()
       // CHECK: function_ref {{.*}}MyError{{.*}}:[[@LINE-1]]:13, scope [[GUARD]]
     }

test/SILGen/cf_members.swift

@@ -68,7 +68,8 @@ public func foo(_ x: Double) {
   // CHECK: [[ZVAL:%.*]] = load [trivial] [[READ]]
   // CHECK: [[THUNK:%.*]] = function_ref @$s10cf_members3fooyySdFSo10IAMStruct1VSdcADcfu0_ : $@convention(thin) (Struct1) -> @owned @callee_guaranteed (Double) -> Struct1
   // CHECK: [[C:%.*]] = apply [[THUNK]]([[ZVAL]])
-  // CHECK: [[BORROWED_C:%.*]] = begin_borrow [lexical] [var_decl] [[C]]
+  // CHECK: [[C_MOVE:%.*]] = move_value [lexical] [var_decl] [[C]]
+  // CHECK: [[BORROWED_C:%.*]] = begin_borrow [[C_MOVE]]
   // CHECK: [[C_COPY:%.*]] = copy_value [[BORROWED_C]]
   // CHECK: [[BORROWED_C2:%.*]] = begin_borrow [[C_COPY]]
   let c: (Double) -> Struct1 = z.translate(radians:)
@@ -102,7 +103,8 @@ public func foo(_ x: Double) {
   // CHECK: [[ZVAL:%.*]] = load [trivial] [[READ]]
   // CHECK: [[THUNK:%.*]] = function_ref @$s10cf_members3fooyySdFSo10IAMStruct1VSdcADcfu4_ : $@convention(thin) (Struct1) -> @owned @callee_guaranteed (Double) -> Struct1
   // CHECK: [[F:%.*]] = apply [[THUNK]]([[ZVAL]])
-  // CHECK: [[BORROWED_F:%.*]] = begin_borrow [lexical] [var_decl] [[F]]
+  // CHECK: [[MOVED_F:%.*]] = move_value [lexical] [var_decl] [[F]]
+  // CHECK: [[BORROWED_F:%.*]] = begin_borrow [[MOVED_F]]
   // CHECK: [[F_COPY:%.*]] = copy_value [[BORROWED_F]]
   // CHECK: [[BORROWED_F2:%.*]] = begin_borrow [[F_COPY]]
   let f = z.scale