[TempRValueOpt] Invalidate insts when completing.

include/swift/SILOptimizer/PassManager/PassManager.h

@@ -444,8 +444,9 @@ class SILPassManager {
 
   void executePassPipelinePlan(const SILPassPipelinePlan &Plan);
 
-  bool continueWithNextSubpassRun(SILInstruction *forInst, SILFunction *function,
-                                  SILTransform *trans);
+  using Transformee = llvm::PointerUnion<SILValue, SILInstruction *>;
+  bool continueWithNextSubpassRun(std::optional<Transformee> forTransformee,
+                                  SILFunction *function, SILTransform *trans);
 
   static bool isPassDisabled(StringRef passName);
   static bool isInstructionPassDisabled(StringRef instName);

include/swift/SILOptimizer/PassManager/Transforms.h

@@ -133,6 +133,10 @@ namespace swift {
       return PM->continueWithNextSubpassRun(forInst, F, this);
     }
 
+    bool continueWithNextSubpassRun(SILValue forValue) {
+      return PM->continueWithNextSubpassRun(forValue, F, this);
+    }
+
     void invalidateAnalysis(SILAnalysis::InvalidationKind K) {
       PM->invalidateAnalysis(F, K);
     }

lib/SIL/IR/SILInstruction.cpp

@@ -1833,41 +1833,35 @@ visitRecursivelyLifetimeEndingUses(
   llvm::function_ref<bool(Operand *)> visitScopeEnd,
   llvm::function_ref<bool(Operand *)> visitUnknownUse) {
 
-  for (Operand *use : i->getConsumingUses()) {
-    noUsers = false;
-    if (isa<DestroyValueInst>(use->getUser())) {
-      if (!visitScopeEnd(use)) {
-        return false;
+  StackList<SILValue> values(i->getFunction());
+  values.push_back(i);
+
+  while (!values.empty()) {
+    auto value = values.pop_back_val();
+    for (Operand *use : value->getConsumingUses()) {
+      noUsers = false;
+      if (isa<DestroyValueInst>(use->getUser())) {
+        if (!visitScopeEnd(use)) {
+          return false;
+        }
+        continue;
       }
-      continue;
-    }
-    if (auto *ret = dyn_cast<ReturnInst>(use->getUser())) {
-      auto fnTy = ret->getFunction()->getLoweredFunctionType();
-      assert(!fnTy->getLifetimeDependencies().empty());
-      if (!visitScopeEnd(use)) {
-        return false;
+      if (auto *ret = dyn_cast<ReturnInst>(use->getUser())) {
+        auto fnTy = ret->getFunction()->getLoweredFunctionType();
+        assert(!fnTy->getLifetimeDependencies().empty());
+        if (!visitScopeEnd(use)) {
+          return false;
+        }
+        continue;
       }
-      continue;
-    }
-    // FIXME: Handle store to indirect result
-
-    // There shouldn't be any dead-end consumptions of a nonescaping
-    // partial_apply that don't forward it along, aside from destroy_value.
-    //
-    // On-stack partial_apply cannot be cloned, so it should never be used by a
-    // BranchInst.
-    //
-    // This is a fatal error because it performs SIL verification that is not
-    // separately checked in the verifier. It is the only check that verifies
-    // the structural requirements of on-stack partial_apply uses.
-    auto *user = use->getUser();
-    if (user->getNumResults() == 0) {
-      return visitUnknownUse(use);
-    }
-    for (auto res : use->getUser()->getResults()) {
-      if (!visitRecursivelyLifetimeEndingUses(res, noUsers, visitScopeEnd,
-                                              visitUnknownUse)) {
-        return false;
+      // FIXME: Handle store to indirect result
+
+      auto *user = use->getUser();
+      if (user->getNumResults() == 0) {
+        return visitUnknownUse(use);
+      }
+      for (auto res : use->getUser()->getResults()) {
+        values.push_back(res);
       }
     }
   }
@@ -1882,7 +1876,16 @@ PartialApplyInst::visitOnStackLifetimeEnds(
          && "only meaningful for OSSA stack closures");
   bool noUsers = true;
 
-  auto visitUnknownUse = [](Operand *unknownUse){
+  auto visitUnknownUse = [](Operand *unknownUse) {
+    // There shouldn't be any dead-end consumptions of a nonescaping
+    // partial_apply that don't forward it along, aside from destroy_value.
+    //
+    // On-stack partial_apply cannot be cloned, so it should never be used by a
+    // BranchInst.
+    //
+    // This is a fatal error because it performs SIL verification that is not
+    // separately checked in the verifier. It is the only check that verifies
+    // the structural requirements of on-stack partial_apply uses.
     llvm::errs() << "partial_apply [on_stack] use:\n";
     auto *user = unknownUse->getUser();
     user->printInContext(llvm::errs());

lib/SILOptimizer/PassManager/PassManager.cpp

@@ -488,16 +488,35 @@ bool SILPassManager::continueTransforming() {
   return NumPassesRun < maxNumPassesToRun;
 }
 
-bool SILPassManager::continueWithNextSubpassRun(SILInstruction *forInst,
-                                                SILFunction *function,
-                                                SILTransform *trans) {
+bool SILPassManager::continueWithNextSubpassRun(
+    std::optional<Transformee> origTransformee, SILFunction *function,
+    SILTransform *trans) {
+  // Rewrite .some(nullptr) as .none.
+  std::optional<llvm::PointerUnion<SILValue, SILInstruction *>> forTransformee;
+  if (origTransformee) {
+    auto forValue = dyn_cast<SILValue>(*origTransformee);
+    if (forValue) {
+      forTransformee = forValue;
+    } else if (auto *forInst = cast<SILInstruction *>(*origTransformee)) {
+      forTransformee = forInst;
+    }
+  }
+
   unsigned subPass = numSubpassesRun++;
 
-  if (forInst && isFunctionSelectedForPrinting(function) &&
-      SILPrintEverySubpass) {
+  if (isFunctionSelectedForPrinting(function) && SILPrintEverySubpass) {
     dumpPassInfo("*** SIL function before ", trans, function);
-    if (forInst) {
-      llvm::dbgs() << "  *** sub-pass " << subPass << " for " << *forInst;
+    llvm::dbgs() << "  *** sub-pass " << subPass << " for ";
+    if (forTransformee) {
+      auto forValue = dyn_cast<SILValue>(*forTransformee);
+      if (forValue) {
+        llvm::dbgs() << forValue;
+      } else {
+        auto *forInst = cast<SILInstruction *>(*forTransformee);
+        llvm::dbgs() << *forInst;
+      }
+    } else {
+      llvm::dbgs() << "???\n";
     }
     function->dump(getOptions().EmitVerboseSIL);
   }
@@ -509,8 +528,16 @@ bool SILPassManager::continueWithNextSubpassRun(SILInstruction *forInst,
 
   if (subPass == maxNumSubpassesToRun - 1 && SILPrintLast) {
     dumpPassInfo("*** SIL function before ", trans, function);
-    if (forInst) {
-      llvm::dbgs() << "  *** sub-pass " << subPass << " for " << *forInst;
+    if (forTransformee) {
+      auto forValue = dyn_cast<SILValue>(*forTransformee);
+      if (forValue) {
+        llvm::dbgs() << forValue;
+      } else {
+        auto *forInst = cast<SILInstruction *>(*forTransformee);
+        llvm::dbgs() << *forInst;
+      }
+    } else {
+      llvm::dbgs() << "???\n";
     }
     function->dump(getOptions().EmitVerboseSIL);
   }

lib/SILOptimizer/Transforms/CopyPropagation.cpp

@@ -443,38 +443,26 @@ class CopyPropagation : public SILFunctionTransform {
   /// The entry point to this function transformation.
   void run() override;
 
+  void propagateCopies(CanonicalDefWorklist &defWorklist, bool &changed,
+                       NonLocalAccessBlockAnalysis *accessBlockAnalysis,
+                       InstructionDeleter &deleter);
+
   void verifyOwnership();
 };
 
 } // end anonymous namespace
 
-/// Top-level pass driver.
-void CopyPropagation::run() {
+void CopyPropagation::propagateCopies(
+    CanonicalDefWorklist &defWorklist, bool &changed,
+    NonLocalAccessBlockAnalysis *accessBlockAnalysis,
+    InstructionDeleter &deleter) {
   auto *f = getFunction();
   auto *postOrderAnalysis = getAnalysis<PostOrderAnalysis>();
-  auto *accessBlockAnalysis = getAnalysis<NonLocalAccessBlockAnalysis>();
   auto *deadEndBlocksAnalysis = getAnalysis<DeadEndBlocksAnalysis>();
   auto *dominanceAnalysis = getAnalysis<DominanceAnalysis>();
   auto *calleeAnalysis = getAnalysis<BasicCalleeAnalysis>();
   DominanceInfo *domTree = dominanceAnalysis->get(f);
 
-  // Label for unit testing with debug output.
-  LLVM_DEBUG(llvm::dbgs() << "*** CopyPropagation: " << f->getName() << "\n");
-
-  // This algorithm fundamentally assumes ownership.
-  if (!f->hasOwnership())
-    return;
-
-  CanonicalDefWorklist defWorklist(canonicalizeBorrows);
-  auto callbacks =
-      InstModCallbacks().onDelete([&](SILInstruction *instToDelete) {
-        defWorklist.erase(instToDelete);
-        instToDelete->eraseFromParent();
-      });
-
-  InstructionDeleter deleter(std::move(callbacks));
-  bool changed = false;
-
   StackList<BeginBorrowInst *> beginBorrowsToShrink(f);
   StackList<MoveValueInst *> moveValues(f);
 
@@ -514,6 +502,8 @@ void CopyPropagation::run() {
     // at least once and then until each stops making changes.
     while (true) {
       SmallVector<CopyValueInst *, 4> modifiedCopyValueInsts;
+      if (!continueWithNextSubpassRun(bbi))
+        return;
       auto shrunk = shrinkBorrowScope(*bbi, deleter, calleeAnalysis,
                                       modifiedCopyValueInsts);
       for (auto *cvi : modifiedCopyValueInsts)
@@ -528,25 +518,35 @@ void CopyPropagation::run() {
       if (borrowee->getOwnershipKind() != OwnershipKind::Owned)
         break;
 
+      if (!continueWithNextSubpassRun(borrowee))
+        return;
       auto canonicalized = canonicalizer.canonicalizeValueLifetime(borrowee);
       if (!canonicalized && !firstRun)
         break;
 
+      if (!continueWithNextSubpassRun(bbi))
+        return;
       auto folded = foldDestroysOfCopiedLexicalBorrow(bbi, *domTree, deleter);
       if (!folded)
         break;
       auto hoisted = canonicalizer.canonicalizeValueLifetime(folded);
       // Keep running even if the new move's destroys can't be hoisted.
       (void)hoisted;
+      if (!continueWithNextSubpassRun(folded))
+        return;
       eliminateRedundantMove(folded, deleter, defWorklist);
       firstRun = false;
     }
   }
   for (auto *mvi : moveValues) {
+    if (!continueWithNextSubpassRun(mvi))
+      return;
     eliminateRedundantMove(mvi, deleter, defWorklist);
   }
   for (auto *argument : f->getArguments()) {
     if (argument->getOwnershipKind() == OwnershipKind::Owned) {
+      if (!continueWithNextSubpassRun(argument))
+        return;
       canonicalizer.canonicalizeValueLifetime(argument);
     }
   }
@@ -584,8 +584,12 @@ void CopyPropagation::run() {
       // they may be chained, and CanonicalizeBorrowScopes pushes them
       // top-down.
       for (auto result : ownedForward->getResults()) {
+        if (!continueWithNextSubpassRun(result))
+          return;
         canonicalizer.canonicalizeValueLifetime(result);
       }
+      if (!continueWithNextSubpassRun(ownedForward))
+        return;
       if (sinkOwnedForward(ownedForward, postOrderAnalysis, domTree)) {
         changed = true;
         // Sinking 'ownedForward' may create an opportunity to sink its
@@ -607,6 +611,8 @@ void CopyPropagation::run() {
     BorrowedValue borrow(defWorklist.borrowedValues.pop_back_val());
     assert(canonicalizeBorrows || !borrow.isLocalScope());
 
+    if (!continueWithNextSubpassRun(borrow.value))
+      return;
     borrowCanonicalizer.canonicalizeBorrowScope(borrow);
     for (CopyValueInst *copy : borrowCanonicalizer.getUpdatedCopies()) {
       defWorklist.updateForCopy(copy);
@@ -623,13 +629,41 @@ void CopyPropagation::run() {
   // Canonicalize all owned defs.
   while (!defWorklist.ownedValues.empty()) {
     SILValue def = defWorklist.ownedValues.pop_back_val();
+    if (!continueWithNextSubpassRun(def))
+      return;
     auto canonicalized = canonicalizer.canonicalizeValueLifetime(def);
     if (!canonicalized)
       continue;
     // Copies of borrowed values may be dead.
     if (auto *inst = def->getDefiningInstruction())
       deleter.trackIfDead(inst);
   }
+}
+
+/// Top-level pass driver.
+void CopyPropagation::run() {
+  auto *f = getFunction();
+  // This algorithm fundamentally assumes ownership.
+  if (!f->hasOwnership())
+    return;
+
+  // Label for unit testing with debug output.
+  LLVM_DEBUG(llvm::dbgs() << "*** CopyPropagation: " << f->getName() << "\n");
+
+  auto *accessBlockAnalysis = getAnalysis<NonLocalAccessBlockAnalysis>();
+
+  CanonicalDefWorklist defWorklist(canonicalizeBorrows);
+
+  auto callbacks =
+      InstModCallbacks().onDelete([&](SILInstruction *instToDelete) {
+        defWorklist.erase(instToDelete);
+        instToDelete->eraseFromParent();
+      });
+  InstructionDeleter deleter(std::move(callbacks));
+
+  bool changed = false;
+  propagateCopies(defWorklist, changed, accessBlockAnalysis, deleter);
+
   // Recursively cleanup dead defs after removing uses.
   deleter.cleanupDeadInstructions();