Merge pull request #60670 from nate-chandler/lexical_lifetimes/owned_arguments

[SIL] Maintain owned argument lifetimes at inlining.

Author: nate-chandler

SwiftCompilerSources/Sources/Optimizer/Utilities/WalkUtils.swift

@@ -341,7 +341,7 @@ extension ValueDefUseWalker {
         return unmatchedPath(value: operand, path: path)
       }
     case is InitExistentialRefInst, is OpenExistentialRefInst,
-      is BeginBorrowInst, is CopyValueInst,
+      is BeginBorrowInst, is CopyValueInst, is MoveValueInst,
       is UpcastInst, is UncheckedRefCastInst, is EndCOWMutationInst,
       is RefToBridgeObjectInst, is BridgeObjectToRefInst, is MarkMustCheckInst:
       return walkDownUses(ofValue: (instruction as! SingleValueInstruction), path: path)
@@ -618,7 +618,7 @@ extension ValueUseDefWalker {
         return rootDef(value: mvr, path: path)
       }
     case is InitExistentialRefInst, is OpenExistentialRefInst,
-      is BeginBorrowInst, is CopyValueInst,
+      is BeginBorrowInst, is CopyValueInst, is MoveValueInst,
       is UpcastInst, is UncheckedRefCastInst, is EndCOWMutationInst,
       is RefToBridgeObjectInst, is BridgeObjectToRefInst, is MarkMustCheckInst:
       return walkUp(value: (def as! Instruction).operands[0].value, path: path)

SwiftCompilerSources/Sources/SIL/Instruction.swift

@@ -609,6 +609,8 @@ final public class ProjectBoxInst : SingleValueInstruction, UnaryInstruction {
 
 final public class CopyValueInst : SingleValueInstruction, UnaryInstruction {}
 
+final public class MoveValueInst : SingleValueInstruction, UnaryInstruction {}
+
 final public class StrongCopyUnownedValueInst : SingleValueInstruction, UnaryInstruction {}
 
 final public class StrongCopyUnmanagedValueInst : SingleValueInstruction, UnaryInstruction  {}

SwiftCompilerSources/Sources/SIL/Registration.swift

@@ -123,6 +123,7 @@ public func registerSILClasses() {
   register(BeginBorrowInst.self)
   register(ProjectBoxInst.self)
   register(CopyValueInst.self)
+  register(MoveValueInst.self)
   register(EndCOWMutationInst.self)
   register(ClassifyBridgeObjectInst.self)
   register(PartialApplyInst.self)

docs/SIL.rst

@@ -2414,27 +2414,55 @@ Variable Lifetimes
 In order for programmer intended lifetimes to be maintained under optimization,
 the lifetimes of SIL values which correspond to named source-level values can
 only be modified in limited ways.  Generally, the behavior is that the lifetime
-of a named source-level value cannot _observably_ end before the end of the
-lexical scope in which that value is defined.  Specifically, code motion may
-not move the ends of these lifetimes across a **deinit barrier**.
-
-A few sorts of SIL value have lifetimes that are constrained that way:
+of a named source-level value is anchored to the variable's lexical scope and
+confined by **deinit barriers**.  Specifically, code motion may not move the
+ends of these lifetimes across a deinit barrier.
+
+Source level variables (lets, vars, ...) and function arguments will result in
+SIL-level lexical lifetimes if either of the two sets of circumstances apply:
+(1) Inferred lexicality.
+- the type is non-trivial
+- the type is not eager-move
+- the variable or argument is not annotated to be eager-move
+OR
+(2) Explicit lexicality.
+- the type, variable, or argument is annotated `@_lexical`
+
+A type is eager-move by satisfying one of two conditions:
+(1) Inferred: An aggregate is inferred to be eager-move if all of its fields are
+    eager-move.
+(2) Annotated: Any type can be eager-move if it is annotated with an attribute
+    that explicitly specifies it to be: `@_eagerMove`, `@_noImplicitCopy`.
+
+A variable or argument is eager-move by satisfying one of two conditions:
+(1) Inferred: Its type is eager-move.
+(2) Annotated: The variable or argument is annotated with an attribute that
+    specifies it to be: `@_eagerMove`, `@_noImplicitCopy`.
+
+These source-level rules result in a few sorts of SIL value whose destroys must
+not be moved across deinit barriers:
 
 1: `begin_borrow [lexical]`
 2: `move_value [lexical]`
-3: @owned function arguments
+3: function arguments
 4: `alloc_stack [lexical]`
 
-That these three have constrained lifetimes is encoded in ValueBase::isLexical,
-which should be checked before changing the lifetime of a value.
+To translate from the source-level representation of lexicality to the
+SIL-level representation, for source-level variables (vars, lets, ...) SILGen
+generates `begin_borrow [lexical]`, `move_value [lexical]`, `alloc_stack
+[lexical]` .  For function arguments, there is no work to do:
+a `SILFunctionArgument` itself can be lexical
+(`SILFunctionArgument::isLexical`).
+
+That the first three have constrained lifetimes is encoded in
+ValueBase::isLexical, which should be checked before changing the lifetime of a
+value.
 
-The reason that only @owned function arguments are constrained is that a
-@guaranteed function argument is guaranteed by the function's caller to live for
-the full duration of the function already.  Optimization of the function alone
-can't shorten it.  When such a function is inlined into its caller, though, a
-lexical borrow scope is added for each of its @guaranteed arguments, ensuring
-that the lifetime of the corresponding source-level value is not shortened in a
-way that doesn't respect deinit barriers.
+When a function is inlined into its caller, a lexical borrow scope is added for
+each of its @guaranteed arguments, and a lexical move is added for each of its
+@owned arguments, (unless the values being passed are already lexical
+themselves) ensuring that the lifetimes of the corresponding source-level
+values are not shortened in a way that doesn't respect deinit barriers.
 
 Unlike the other sorts, `alloc_stack [lexical]` isn't a SILValue.  Instead, it
 constrains the lifetime of an addressable variable.  Since the constraint is

include/swift/SIL/InstructionUtils.h

@@ -88,8 +88,8 @@ SILValue stripBorrow(SILValue V);
 //===----------------------------------------------------------------------===//
 
 /// Return a non-null SingleValueInstruction if the given instruction merely
-/// copies the value of its first operand, possibly changing its type or
-/// ownership state, but otherwise having no effect.
+/// copies or moves the value of its first operand, possibly changing its type
+/// or ownership state, but otherwise having no effect.
 ///
 /// The returned instruction may have additional "incidental" operands;
 /// mark_dependence for example.

include/swift/SILOptimizer/Utils/CanonicalizeOSSALifetime.h

@@ -109,7 +109,7 @@
 
 namespace swift {
 
-extern llvm::Statistic NumCopiesEliminated;
+extern llvm::Statistic NumCopiesAndMovesEliminated;
 extern llvm::Statistic NumCopiesGenerated;
 
 /// Insert a copy on this operand. Trace and update stats.

lib/SIL/IR/SILValue.cpp

@@ -116,11 +116,8 @@ ValueBase::getDefiningInstructionResult() {
 }
 
 bool ValueBase::isLexical() const {
-  if (auto *argument = dyn_cast<SILFunctionArgument>(this)) {
-    // TODO: Recognize guaranteed arguments as lexical too.
-    return argument->getOwnershipKind() == OwnershipKind::Owned &&
-           argument->getLifetime().isLexical();
-  }
+  if (auto *argument = dyn_cast<SILFunctionArgument>(this))
+    return argument->getLifetime().isLexical();
   if (auto *bbi = dyn_cast<BeginBorrowInst>(this))
     return bbi->isLexical();
   if (auto *mvi = dyn_cast<MoveValueInst>(this))

lib/SIL/Utils/InstructionUtils.cpp

@@ -31,6 +31,7 @@ SILValue swift::lookThroughOwnershipInsts(SILValue v) {
     switch (v->getKind()) {
     default:
       return v;
+    case ValueKind::MoveValueInst:
     case ValueKind::CopyValueInst:
     case ValueKind::BeginBorrowInst:
       v = cast<SingleValueInstruction>(v)->getOperand(0);
@@ -255,6 +256,7 @@ SingleValueInstruction *swift::getSingleValueCopyOrCast(SILInstruction *I) {
   case SILInstructionKind::BeginBorrowInst:
   case SILInstructionKind::BeginAccessInst:
   case SILInstructionKind::MarkDependenceInst:
+  case SILInstructionKind::MoveValueInst:
     return cast<SingleValueInstruction>(I);
   }
 }

lib/SIL/Utils/OwnershipUtils.cpp

@@ -2414,16 +2414,25 @@ void swift::visitTransitiveEndBorrows(
 ///
 /// A begin_borrow [lexical] is nested if the borrowed value's lifetime is
 /// guaranteed by another lexical scope.  That happens if:
-/// - the value is a guaranteed argument to the function
-/// - the value is itself a begin_borrow [lexical]
+/// - the non-guaranteed borrowee's value is lexical
+/// - the guaranteed borrowee's value's reference roots are lexical
+///   - for example, the borrowee is itself a begin_borrow [lexical]
 bool swift::isNestedLexicalBeginBorrow(BeginBorrowInst *bbi) {
   assert(bbi->isLexical());
   auto value = bbi->getOperand();
-  if (auto *outerBBI = dyn_cast<BeginBorrowInst>(value)) {
-    return outerBBI->isLexical();
+  if (value->getOwnershipKind() != OwnershipKind::Guaranteed) {
+    return value->isLexical();
   }
-  if (auto *arg = dyn_cast<SILFunctionArgument>(value)) {
-    return arg->getOwnershipKind() == OwnershipKind::Guaranteed;
-  }
-  return false;
+  SmallVector<SILValue, 8> roots;
+  findGuaranteedReferenceRoots(value, /*lookThroughNestedBorrows=*/false,
+                               roots);
+  return llvm::all_of(roots, [](auto root) {
+    if (auto *outerBBI = dyn_cast<BeginBorrowInst>(root)) {
+      return outerBBI->isLexical();
+    }
+    if (auto *arg = dyn_cast<SILFunctionArgument>(root)) {
+      return arg->getOwnershipKind() == OwnershipKind::Guaranteed;
+    }
+    return false;
+  });
 }

lib/SIL/Utils/Projection.cpp

@@ -387,7 +387,7 @@ Optional<ProjectionPath> ProjectionPath::getProjectionPath(SILValue Start,
     // TODO: migrate users to getProjectionPath to the AccessPath utility to
     // avoid this hack.
     if (!isa<EndCOWMutationInst>(Iter) && !isa<BeginAccessInst>(Iter) &&
-        !isa<BeginBorrowInst>(Iter)) {
+        !isa<BeginBorrowInst>(Iter) && !isa<MoveValueInst>(Iter)) {
       Projection AP(Iter);
       if (!AP.isValid())
         break;

lib/SILGen/SILGenProlog.cpp

@@ -303,15 +303,8 @@ struct ArgumentInitHelper {
     // form.
     if (!isNoImplicitCopy) {
       if (!value->getType().isMoveOnly()) {
-        // Follow the "normal path": perform a lexical borrow if the lifetime is
-        // lexical.
-        if (value->getOwnershipKind() == OwnershipKind::Owned) {
-          if (lifetime.isLexical()) {
-            value = SILValue(
-                SGF.B.createBeginBorrow(loc, value, /*isLexical*/ true));
-            SGF.Cleanups.pushCleanup<EndBorrowCleanup>(value);
-          }
-        }
+        // Follow the normal path.  The value's lifetime will be enforced based
+        // on its ownership.
         return value;
       }
 

lib/SILOptimizer/Mandatory/MandatoryInlining.cpp

@@ -67,13 +67,6 @@ static void diagnose(ASTContext &Context, SourceLoc loc, Diag<T...> diag,
   Context.Diags.diagnose(loc, diag, std::forward<U>(args)...);
 }
 
-static SILValue stripCopiesAndBorrows(SILValue v) {
-  while (isa<CopyValueInst>(v) || isa<BeginBorrowInst>(v)) {
-    v = cast<SingleValueInstruction>(v)->getOperand(0);
-  }
-  return v;
-}
-
 /// Fixup reference counts after inlining a function call (which is a no-op
 /// unless the function is a thick function).
 ///
@@ -388,7 +381,7 @@ static void cleanupCalleeValue(SILValue calleeValue,
   if (auto loadedValue = cleanupLoadedCalleeValue(calleeValue))
     calleeValue = loadedValue;
 
-  calleeValue = stripCopiesAndBorrows(calleeValue);
+  calleeValue = lookThroughOwnershipInsts(calleeValue);
 
   // Inline constructor
   auto calleeSource = ([&]() -> SILValue {
@@ -398,12 +391,12 @@ static void cleanupCalleeValue(SILValue calleeValue,
     // will delete any uses of the closure, including a
     // convert_escape_to_noescape conversion.
     if (auto *cfi = dyn_cast<ConvertFunctionInst>(calleeValue))
-      return stripCopiesAndBorrows(cfi->getOperand());
+      return lookThroughOwnershipInsts(cfi->getOperand());
 
     if (auto *cvt = dyn_cast<ConvertEscapeToNoEscapeInst>(calleeValue))
-      return stripCopiesAndBorrows(cvt->getOperand());
+      return lookThroughOwnershipInsts(cvt->getOperand());
 
-    return stripCopiesAndBorrows(calleeValue);
+    return lookThroughOwnershipInsts(calleeValue);
   })();
 
   if (auto *pai = dyn_cast<PartialApplyInst>(calleeSource)) {
@@ -419,7 +412,7 @@ static void cleanupCalleeValue(SILValue calleeValue,
   }
   invalidatedStackNesting = true;
 
-  calleeValue = stripCopiesAndBorrows(calleeValue);
+  calleeValue = lookThroughOwnershipInsts(calleeValue);
 
   // Handle function_ref -> convert_function -> partial_apply/thin_to_thick.
   if (auto *cfi = dyn_cast<ConvertFunctionInst>(calleeValue)) {
@@ -599,7 +592,7 @@ static SILValue getLoadedCalleeValue(LoadInst *li) {
 // a cast.
 static SILValue stripFunctionConversions(SILValue CalleeValue) {
   // Skip any copies that we see.
-  CalleeValue = stripCopiesAndBorrows(CalleeValue);
+  CalleeValue = lookThroughOwnershipInsts(CalleeValue);
 
   // We can also allow a thin @escape to noescape conversion as such:
   // %1 = function_ref @thin_closure_impl : $@convention(thin) () -> ()
@@ -626,7 +619,7 @@ static SILValue stripFunctionConversions(SILValue CalleeValue) {
     if (FromCalleeTy != EscapingCalleeTy)
       return CalleeValue;
 
-    return stripCopiesAndBorrows(ConvertFn->getOperand());
+    return lookThroughOwnershipInsts(ConvertFn->getOperand());
   }
 
   // Ignore mark_dependence users. A partial_apply [stack] uses them to mark
@@ -642,7 +635,7 @@ static SILValue stripFunctionConversions(SILValue CalleeValue) {
 
   auto *CFI = dyn_cast<ConvertEscapeToNoEscapeInst>(CalleeValue);
   if (!CFI)
-    return stripCopiesAndBorrows(CalleeValue);
+    return lookThroughOwnershipInsts(CalleeValue);
 
   // TODO: Handle argument conversion. All the code in this file needs to be
   // cleaned up and generalized. The argument conversion handling in
@@ -658,9 +651,9 @@ static SILValue stripFunctionConversions(SILValue CalleeValue) {
   auto EscapingCalleeTy =
     ToCalleeTy->getWithExtInfo(ToCalleeTy->getExtInfo().withNoEscape(false));
   if (FromCalleeTy != EscapingCalleeTy)
-    return stripCopiesAndBorrows(CalleeValue);
+    return lookThroughOwnershipInsts(CalleeValue);
 
-  return stripCopiesAndBorrows(CFI->getOperand());
+  return lookThroughOwnershipInsts(CFI->getOperand());
 }
 
 /// Returns the callee SILFunction called at a call site, in the case
@@ -687,7 +680,7 @@ getCalleeFunction(SILFunction *F, FullApplySite AI, bool &IsThick,
 
   // Then grab a first approximation of our apply by stripping off all copy
   // operations.
-  SILValue CalleeValue = stripCopiesAndBorrows(AI.getCallee());
+  SILValue CalleeValue = lookThroughOwnershipInsts(AI.getCallee());
 
   // If after stripping off copy_values, we have a load then see if we the
   // function we want to inline has a simple available value through a simple
@@ -696,7 +689,7 @@ getCalleeFunction(SILFunction *F, FullApplySite AI, bool &IsThick,
     CalleeValue = getLoadedCalleeValue(li);
     if (!CalleeValue)
       return nullptr;
-    CalleeValue = stripCopiesAndBorrows(CalleeValue);
+    CalleeValue = lookThroughOwnershipInsts(CalleeValue);
   }
 
   // Look through a escape to @noescape conversion.
@@ -709,11 +702,11 @@ getCalleeFunction(SILFunction *F, FullApplySite AI, bool &IsThick,
     // Collect the applied arguments and their convention.
     collectPartiallyAppliedArguments(PAI, CapturedArgConventions, FullArgs);
 
-    CalleeValue = stripCopiesAndBorrows(PAI->getCallee());
+    CalleeValue = lookThroughOwnershipInsts(PAI->getCallee());
     IsThick = true;
     PartialApply = PAI;
   } else if (auto *TTTFI = dyn_cast<ThinToThickFunctionInst>(CalleeValue)) {
-    CalleeValue = stripCopiesAndBorrows(TTTFI->getOperand());
+    CalleeValue = lookThroughOwnershipInsts(TTTFI->getOperand());
     IsThick = true;
   }
 

lib/SILOptimizer/Mandatory/OwnershipModelEliminator.cpp

@@ -135,6 +135,10 @@ struct OwnershipModelEliminatorVisitor
   bool visitExplicitCopyValueInst(ExplicitCopyValueInst *cvi);
   bool visitDestroyValueInst(DestroyValueInst *dvi);
   bool visitLoadBorrowInst(LoadBorrowInst *lbi);
+  bool visitMoveValueInst(MoveValueInst *mvi) {
+    eraseInstructionAndRAUW(mvi, mvi->getOperand());
+    return true;
+  }
   bool visitBeginBorrowInst(BeginBorrowInst *bbi) {
     eraseInstructionAndRAUW(bbi, bbi->getOperand());
     return true;

lib/SILOptimizer/Transforms/ObjectOutliner.cpp

@@ -252,6 +252,11 @@ bool ObjectOutliner::getObjectInitVals(SILValue Val,
       if (!getObjectInitVals(UC, MemberStores, TailStores, NumTailTupleElements,
                              toIgnore))
         return false;
+    } else if (auto *mvi = dyn_cast<MoveValueInst>(User)) {
+      // move_value is transparent.
+      if (!getObjectInitVals(mvi, MemberStores, TailStores,
+                             NumTailTupleElements, toIgnore))
+        return false;
     } else if (auto *REA = dyn_cast<RefElementAddrInst>(User)) {
       // The address of a stored property.
       for (Operand *ElemAddrUse : REA->getUses()) {
@@ -316,6 +321,9 @@ static EndCOWMutationInst *getEndCOWMutation(SILValue object) {
       // Look through upcast instructions.
       if (EndCOWMutationInst *ecm = getEndCOWMutation(upCast))
         return ecm;
+    } else if (auto *mvi = dyn_cast<MoveValueInst>(user)) {
+      if (auto *ecm = getEndCOWMutation(mvi))
+        return ecm;
     } else if (auto *ecm = dyn_cast<EndCOWMutationInst>(use->getUser())) {
       return ecm;
     }