[CanonOSSALifetime] Run on lexical lifetimes.

Previously, the utility bailed out on lexical lifetimes because it
didn't respect deinit barriers.  Here, deinit barriers are found and
added to liveness if the value is lexical.  This enables copies to be
propagated without hoisting destroys over deinit barriers.

Author: nate-chandler

include/swift/SILOptimizer/Utils/CanonicalizeOSSALifetime.h

@@ -109,6 +109,8 @@
 
 namespace swift {
 
+class BasicCalleeAnalysis;
+
 extern llvm::Statistic NumCopiesAndMovesEliminated;
 extern llvm::Statistic NumCopiesGenerated;
 
@@ -225,6 +227,8 @@ class CanonicalizeOSSALifetime final {
 
   DominanceInfo *domTree;
 
+  BasicCalleeAnalysis *calleeAnalysis;
+
   InstructionDeleter &deleter;
 
   /// Original points in the CFG where the current value's lifetime is consumed
@@ -282,11 +286,13 @@ class CanonicalizeOSSALifetime final {
 
   CanonicalizeOSSALifetime(bool pruneDebugMode, bool maximizeLifetime,
                            NonLocalAccessBlockAnalysis *accessBlockAnalysis,
-                           DominanceInfo *domTree, InstructionDeleter &deleter)
+                           DominanceInfo *domTree,
+                           BasicCalleeAnalysis *calleeAnalysis,
+                           InstructionDeleter &deleter)
       : pruneDebugMode(pruneDebugMode), maximizeLifetime(maximizeLifetime),
         accessBlockAnalysis(accessBlockAnalysis), domTree(domTree),
-        deleter(deleter),
-        liveness(maximizeLifetime ? &discoveredBlocks : nullptr) {}
+        calleeAnalysis(calleeAnalysis), deleter(deleter),
+        liveness(&discoveredBlocks) {}
 
   SILValue getCurrentDef() const { return liveness.getDef(); }
 
@@ -383,6 +389,9 @@ class CanonicalizeOSSALifetime final {
   void findExtendedBoundary(PrunedLivenessBoundary const &originalBoundary,
                             PrunedLivenessBoundary &boundary);
 
+  void findFinalDestroys(SmallVectorImpl<SILInstruction *> &destroys);
+  void extendLivenessToDeinitBarriers();
+
   void extendUnconsumedLiveness(PrunedLivenessBoundary const &boundary);
 
   void insertDestroysOnBoundary(PrunedLivenessBoundary const &boundary);

lib/SILOptimizer/Mandatory/MoveOnlyObjectCheckerUtils.h

@@ -62,7 +62,8 @@ struct OSSACanonicalizer {
             InstructionDeleter &deleter) {
     canonicalizer.emplace(false /*pruneDebugMode*/,
                           !fn->shouldOptimize() /*maximizeLifetime*/,
-                          nullptr /*accessBlockAnalysis*/, domTree, deleter);
+                          nullptr /*accessBlockAnalysis*/, domTree,
+                          nullptr /*calleeAnalysis*/, deleter);
   }
 
   void clear() {

lib/SILOptimizer/SILCombiner/SILCombine.cpp

@@ -166,6 +166,7 @@ SILCombiner::SILCombiner(SILFunctionTransform *trans,
                          bool removeCondFails, bool enableCopyPropagation) :
   parentTransform(trans),
   AA(trans->getPassManager()->getAnalysis<AliasAnalysis>(trans->getFunction())),
+  CA(trans->getPassManager()->getAnalysis<BasicCalleeAnalysis>()),
   DA(trans->getPassManager()->getAnalysis<DominanceAnalysis>()),
   PCA(trans->getPassManager()->getAnalysis<ProtocolConformanceAnalysis>()),
   CHA(trans->getPassManager()->getAnalysis<ClassHierarchyAnalysis>()),
@@ -353,7 +354,7 @@ void SILCombiner::canonicalizeOSSALifetimes(SILInstruction *currentInst) {
   CanonicalizeOSSALifetime canonicalizer(
       false /*prune debug*/,
       !parentTransform->getFunction()->shouldOptimize() /*maximize lifetime*/,
-      NLABA, domTree, deleter);
+      NLABA, domTree, CA, deleter);
   CanonicalizeBorrowScope borrowCanonicalizer(deleter);
 
   while (!defsToCanonicalize.empty()) {

lib/SILOptimizer/SILCombiner/SILCombiner.h

@@ -23,21 +23,22 @@
 
 #include "swift/Basic/Defer.h"
 #include "swift/SIL/BasicBlockUtils.h"
+#include "swift/SIL/InstructionUtils.h"
 #include "swift/SIL/SILBuilder.h"
 #include "swift/SIL/SILInstruction.h"
 #include "swift/SIL/SILInstructionWorklist.h"
 #include "swift/SIL/SILValue.h"
 #include "swift/SIL/SILVisitor.h"
-#include "swift/SIL/InstructionUtils.h"
+#include "swift/SILOptimizer/Analysis/BasicCalleeAnalysis.h"
 #include "swift/SILOptimizer/Analysis/ClassHierarchyAnalysis.h"
 #include "swift/SILOptimizer/Analysis/NonLocalAccessBlockAnalysis.h"
 #include "swift/SILOptimizer/Analysis/ProtocolConformanceAnalysis.h"
 #include "swift/SILOptimizer/OptimizerBridging.h"
+#include "swift/SILOptimizer/PassManager/PassManager.h"
 #include "swift/SILOptimizer/Utils/CastOptimizer.h"
 #include "swift/SILOptimizer/Utils/Existential.h"
 #include "swift/SILOptimizer/Utils/InstOptUtils.h"
 #include "swift/SILOptimizer/Utils/OwnershipOptUtils.h"
-#include "swift/SILOptimizer/PassManager/PassManager.h"
 
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/SmallVector.h"
@@ -55,6 +56,8 @@ class SILCombiner :
 
   AliasAnalysis *AA;
 
+  BasicCalleeAnalysis *CA;
+
   DominanceAnalysis *DA;
 
   /// Determine the set of types a protocol conforms to in whole-module

lib/SILOptimizer/Transforms/CopyPropagation.cpp

@@ -400,6 +400,7 @@ void CopyPropagation::run() {
   auto *postOrderAnalysis = getAnalysis<PostOrderAnalysis>();
   auto *accessBlockAnalysis = getAnalysis<NonLocalAccessBlockAnalysis>();
   auto *dominanceAnalysis = getAnalysis<DominanceAnalysis>();
+  auto *calleeAnalysis = getAnalysis<BasicCalleeAnalysis>();
   DominanceInfo *domTree = dominanceAnalysis->get(f);
 
   // Label for unit testing with debug output.
@@ -440,9 +441,7 @@ void CopyPropagation::run() {
   // don't need to explicitly check for changes.
   CanonicalizeOSSALifetime canonicalizer(
       pruneDebug, /*maximizeLifetime=*/!getFunction()->shouldOptimize(),
-      accessBlockAnalysis, domTree, deleter);
-  auto *calleeAnalysis = getAnalysis<BasicCalleeAnalysis>();
-
+      accessBlockAnalysis, domTree, calleeAnalysis, deleter);
   // NOTE: We assume that the function is in reverse post order so visiting the
   //       blocks and pushing begin_borrows as we see them and then popping them
   //       off the end will result in shrinking inner borrow scopes first.

lib/SILOptimizer/UtilityPasses/UnitTestRunner.cpp

@@ -404,13 +404,15 @@ struct CanonicalizeOSSALifetimeTest : UnitTest {
     auto *accessBlockAnalysis = getAnalysis<NonLocalAccessBlockAnalysis>();
     auto *dominanceAnalysis = getAnalysis<DominanceAnalysis>();
     DominanceInfo *domTree = dominanceAnalysis->get(getFunction());
+    auto *calleeAnalysis = getAnalysis<BasicCalleeAnalysis>();
     auto pruneDebug = arguments.takeBool();
     auto maximizeLifetimes = arguments.takeBool();
     auto respectAccessScopes = arguments.takeBool();
     InstructionDeleter deleter;
     CanonicalizeOSSALifetime canonicalizer(
         pruneDebug, maximizeLifetimes,
-        respectAccessScopes ? accessBlockAnalysis : nullptr, domTree, deleter);
+        respectAccessScopes ? accessBlockAnalysis : nullptr, domTree,
+        calleeAnalysis, deleter);
     auto value = arguments.takeValue();
     canonicalizer.canonicalizeValueLifetime(value);
     getFunction()->dump();
@@ -776,6 +778,7 @@ void UnitTestRunner::withTest(StringRef name, Doit doit) {
     ADD_UNIT_TEST_SUBCLASS("function-get-self-argument-index", FunctionGetSelfArgumentIndex)
     ADD_UNIT_TEST_SUBCLASS("interior-liveness", InteriorLivenessTest)
     ADD_UNIT_TEST_SUBCLASS("is-deinit-barrier", IsDeinitBarrierTest)
+    ADD_UNIT_TEST_SUBCLASS("lexical-destroy-folding", LexicalDestroyFoldingTest)
     ADD_UNIT_TEST_SUBCLASS("linear-liveness", LinearLivenessTest)
     ADD_UNIT_TEST_SUBCLASS("multidef-liveness", MultiDefLivenessTest)
     ADD_UNIT_TEST_SUBCLASS("pruned-liveness-boundary-with-list-of-last-users-insertion-points", PrunedLivenessBoundaryWithListOfLastUsersInsertionPointsTest)

lib/SILOptimizer/Utils/CanonicalizeOSSALifetime.cpp

@@ -70,6 +70,8 @@
 #include "swift/SIL/NodeDatastructures.h"
 #include "swift/SIL/OwnershipUtils.h"
 #include "swift/SIL/PrunedLiveness.h"
+#include "swift/SILOptimizer/Analysis/BasicCalleeAnalysis.h"
+#include "swift/SILOptimizer/Analysis/Reachability.h"
 #include "swift/SILOptimizer/Utils/CFGOptUtils.h"
 #include "swift/SILOptimizer/Utils/DebugOptUtils.h"
 #include "swift/SILOptimizer/Utils/InstructionDeleter.h"
@@ -222,6 +224,82 @@ bool CanonicalizeOSSALifetime::computeCanonicalLiveness() {
   return true;
 }
 
+static bool findDestroyInBlockAfter(SILInstruction *start,
+                                    ArrayRef<SILInstruction *> candidates) {
+  for (auto *inst = start; inst; inst = inst->getNextInstruction()) {
+    if (llvm::is_contained(candidates, inst))
+      return true;
+  }
+  return false;
+}
+
+static bool isFinalDestroy(SILInstruction *inst,
+                           ArrayRef<SILInstruction *> candidates) {
+  if (findDestroyInBlockAfter(inst->getNextInstruction(), candidates))
+    return false;
+  BasicBlockWorklist worklist(inst->getFunction());
+  auto addSuccessorsToWorklist = [&](SILBasicBlock *block) {
+    for (auto *successor : block->getSuccessorBlocks()) {
+      worklist.pushIfNotVisited(successor);
+    }
+  };
+  addSuccessorsToWorklist(inst->getParent());
+  while (auto *block = worklist.pop()) {
+    if (findDestroyInBlockAfter(&block->front(), candidates))
+      return false;
+    addSuccessorsToWorklist(block);
+  }
+  return true;
+}
+
+void CanonicalizeOSSALifetime::findFinalDestroys(
+    SmallVectorImpl<SILInstruction *> &finalDestroys) {
+  SmallVector<SILInstruction *, 4> finalDestroyCandidates;
+  for (auto destroy : destroys) {
+    if (liveness.isWithinBoundary(destroy))
+      continue;
+    finalDestroyCandidates.push_back(destroy);
+  }
+  // TODO: Make more efficient.
+  for (auto *candidate : finalDestroyCandidates) {
+    if (!isFinalDestroy(candidate, finalDestroyCandidates))
+      continue;
+    finalDestroys.push_back(candidate);
+  }
+}
+
+void CanonicalizeOSSALifetime::extendLivenessToDeinitBarriers() {
+  SmallVector<SILInstruction *, 4> finalDestroys;
+  findFinalDestroys(finalDestroys);
+
+  auto *def = getCurrentDef()->getDefiningInstruction();
+  ReachableBarriers barriers;
+  findBarriersBeforeInstructions(
+      finalDestroys, getCurrentDef()->getParentBlock(),
+      *getCurrentDef()->getFunction(), barriers, [&](auto *inst) {
+        if (inst == def)
+          return true;
+        if (liveness.isInterestingUser(inst) != PrunedLiveness::NonUser)
+          return true;
+        return isDeinitBarrier(inst, calleeAnalysis);
+      });
+  for (auto *barrier : barriers.instructions) {
+    liveness.updateForUse(barrier, /*lifetimeEnding*/ false);
+  }
+  for (auto *barrier : barriers.phis) {
+    for (auto *predecessor : barrier->getPredecessorBlocks()) {
+      liveness.updateForUse(predecessor->getTerminator(),
+                            /*lifetimeEnding*/ false);
+    }
+  }
+  for (auto *barrier : barriers.blocks) {
+    for (auto *predecessor : barrier->getPredecessorBlocks()) {
+      liveness.updateForUse(predecessor->getTerminator(),
+                            /*lifetimeEnding*/ false);
+    }
+  }
+}
+
 // Return true if \p inst is an end_access whose access scope overlaps the end
 // of the pruned live range. This means that a hoisted destroy might execute
 // within the access scope which previously executed outside the access scope.
@@ -994,9 +1072,6 @@ bool CanonicalizeOSSALifetime::computeLiveness(SILValue def) {
   if (def->getOwnershipKind() != OwnershipKind::Owned)
     return false;
 
-  if (def->isLexical())
-    return false;
-
   LLVM_DEBUG(llvm::dbgs() << "  Canonicalizing: " << def);
 
   // Note: There is no need to register callbacks with this utility. 'onDelete'
@@ -1020,6 +1095,9 @@ bool CanonicalizeOSSALifetime::computeLiveness(SILValue def) {
     clearLiveness();
     return false;
   }
+  if (def->isLexical()) {
+    extendLivenessToDeinitBarriers();
+  }
   if (accessBlockAnalysis) {
     extendLivenessThroughOverlappingAccess();
   }

test/SILOptimizer/copy_propagation.sil

@@ -894,10 +894,10 @@ exit:
 
 // CHECK-LABEL: sil [ossa] @dont_extend_beyond_nonoverlapping_end_access_after_store_in_consuming_block : {{.*}} {
 // CHECK:       {{bb[0-9]+}}([[INSTANCE:%[^,]+]] : @owned $C
-// CHECK:         [[COPY:%[^,]+]] = copy_value [[INSTANCE]]
+// CHECK-NOT:     copy_value
 // CHECK:         begin_access
 // CHECK-NOT:     copy_value
-// CHECK:         store [[COPY]] to [init] {{%[^,]+}}
+// CHECK:         store [[INSTANCE]] to [init] {{%[^,]+}}
 // CHECK-LABEL: } // end sil function 'dont_extend_beyond_nonoverlapping_end_access_after_store_in_consuming_block'
 sil [ossa] @dont_extend_beyond_nonoverlapping_end_access_after_store_in_consuming_block : $@convention(thin) (@owned C) -> () {
 bb0(%instance : @owned $C):

test/SILOptimizer/copy_propagation_opaque.sil

@@ -440,9 +440,7 @@ bb0:
 //
 // CHECK-LABEL: sil [ossa] @testCopyBorrow : $@convention(thin) <T> (@in T) -> () {
 // CHECK:       bb0(%0 : @owned $T):
-// CHECK:       [[COPY:%[^,]+]] = copy_value %0
 // CHECK:       destroy_value %0 : $T
-// CHECK:       destroy_value [[COPY]]
 // CHECK-NEXT:  tuple
 // CHECK-NEXT:  return
 // CHECK-LABEL: } // end sil function 'testCopyBorrow'

test/SILOptimizer/lexical_destroy_hoisting.sil

@@ -332,10 +332,8 @@ exit:
 //
 // CHECK-LABEL: sil [ossa] @dont_hoist_over_apply_at_copy : {{.*}} {
 // CHECK:       {{bb[0-9]+}}([[REGISTER_0:%[^,]+]] :
-// CHECK:         [[COPY:%[^,]+]] = copy_value [[INSTANCE]]
 // CHECK:         apply
 // CHECK:         destroy_value [[INSTANCE]]
-// CHECK:         destroy_value [[COPY]]
 // CHECK-LABEL: } // end sil function 'dont_hoist_over_apply_at_copy'
 sil [ossa] @dont_hoist_over_apply_at_copy : $@convention(thin) (@owned C) -> () {
 entry(%instance : @owned $C):

test/SILOptimizer/ossa_rauw_tests.sil

@@ -189,23 +189,21 @@ bb0(%0 : @owned $Klass):
 // CHECK-NOT: destroy_value
 //
 // CHECK: bb2:
-// CHECK-NEXT: [[COPY:%.*]] = copy_value [[ARG]]
 // CHECK-NEXT: cond_br undef, bb3, bb4
 //
 // CHECK: bb3:
-// CHECK-NEXT: destroy_value [[COPY]]
 // CHECK-NEXT: br bb2
 //
 // CHECK: bb4:
-// CHECK-NEXT: [[ENUM_SOME_RESULT:%.*]] = enum $FakeOptional<Klass>, #FakeOptional.some!enumelt, [[COPY]]
+// CHECK-NEXT: [[ENUM_SOME_RESULT:%.*]] = enum $FakeOptional<Klass>, #FakeOptional.some!enumelt, [[ARG]]
 // CHECK-NEXT: br bb6([[ENUM_SOME_RESULT]] : $FakeOptional<Klass>)
 //
 // CHECK: bb5:
+// CHECK-NEXT: destroy_value [[ARG]]
 // CHECK-NEXT: [[ENUM_NONE_RESULT:%.*]] = enum $FakeOptional<Klass>, #FakeOptional.none!enumelt
 // CHECK-NEXT: br bb6([[ENUM_NONE_RESULT]] :
 //
 // CHECK: bb6([[RESULT:%.*]] : @owned $FakeOptional<Klass>):
-// CHECK-NEXT: destroy_value [[ARG]]
 // CHECK-NEXT: return [[RESULT]]
 // CHECK: } // end sil function 'unowned_to_owned_rauw_loop'
 sil [ossa] @unowned_to_owned_rauw_loop : $@convention(thin) (@owned Klass) -> @owned FakeOptional<Klass> {