[CanonOSSALifetime] Run on lexical lifetimes.

Previously, the utility bailed out on lexical lifetimes because it
didn't respect deinit barriers.  Here, deinit barriers are found and
added to liveness if the value is lexical.  This enables copies to be
propagated without hoisting destroys over deinit barriers.

rdar://104630103

Author: nate-chandler

include/swift/SILOptimizer/Utils/CanonicalizeOSSALifetime.h

@@ -109,6 +109,8 @@
 
 namespace swift {
 
+class BasicCalleeAnalysis;
+
 extern llvm::Statistic NumCopiesAndMovesEliminated;
 extern llvm::Statistic NumCopiesGenerated;
 
@@ -225,6 +227,8 @@ class CanonicalizeOSSALifetime final {
 
   DominanceInfo *domTree;
 
+  BasicCalleeAnalysis *calleeAnalysis;
+
   InstructionDeleter &deleter;
 
   /// Original points in the CFG where the current value's lifetime is consumed
@@ -282,11 +286,13 @@ class CanonicalizeOSSALifetime final {
 
   CanonicalizeOSSALifetime(bool pruneDebugMode, bool maximizeLifetime,
                            NonLocalAccessBlockAnalysis *accessBlockAnalysis,
-                           DominanceInfo *domTree, InstructionDeleter &deleter)
+                           DominanceInfo *domTree,
+                           BasicCalleeAnalysis *calleeAnalysis,
+                           InstructionDeleter &deleter)
       : pruneDebugMode(pruneDebugMode), maximizeLifetime(maximizeLifetime),
         accessBlockAnalysis(accessBlockAnalysis), domTree(domTree),
-        deleter(deleter),
-        liveness(maximizeLifetime ? &discoveredBlocks : nullptr) {}
+        calleeAnalysis(calleeAnalysis), deleter(deleter),
+        liveness(&discoveredBlocks) {}
 
   SILValue getCurrentDef() const { return liveness.getDef(); }
 
@@ -383,6 +389,9 @@ class CanonicalizeOSSALifetime final {
   void findExtendedBoundary(PrunedLivenessBoundary const &originalBoundary,
                             PrunedLivenessBoundary &boundary);
 
+  void findDestroysOutsideBoundary(SmallVectorImpl<SILInstruction *> &destroys);
+  void extendLivenessToDeinitBarriers();
+
   void extendUnconsumedLiveness(PrunedLivenessBoundary const &boundary);
 
   void insertDestroysOnBoundary(PrunedLivenessBoundary const &boundary);

lib/SILOptimizer/Mandatory/MoveOnlyObjectCheckerUtils.h

@@ -62,7 +62,8 @@ struct OSSACanonicalizer {
             InstructionDeleter &deleter) {
     canonicalizer.emplace(false /*pruneDebugMode*/,
                           !fn->shouldOptimize() /*maximizeLifetime*/,
-                          nullptr /*accessBlockAnalysis*/, domTree, deleter);
+                          nullptr /*accessBlockAnalysis*/, domTree,
+                          nullptr /*calleeAnalysis*/, deleter);
   }
 
   void clear() {

lib/SILOptimizer/SILCombiner/SILCombine.cpp

@@ -166,6 +166,7 @@ SILCombiner::SILCombiner(SILFunctionTransform *trans,
                          bool removeCondFails, bool enableCopyPropagation) :
   parentTransform(trans),
   AA(trans->getPassManager()->getAnalysis<AliasAnalysis>(trans->getFunction())),
+  CA(trans->getPassManager()->getAnalysis<BasicCalleeAnalysis>()),
   DA(trans->getPassManager()->getAnalysis<DominanceAnalysis>()),
   PCA(trans->getPassManager()->getAnalysis<ProtocolConformanceAnalysis>()),
   CHA(trans->getPassManager()->getAnalysis<ClassHierarchyAnalysis>()),
@@ -353,7 +354,7 @@ void SILCombiner::canonicalizeOSSALifetimes(SILInstruction *currentInst) {
   CanonicalizeOSSALifetime canonicalizer(
       false /*prune debug*/,
       !parentTransform->getFunction()->shouldOptimize() /*maximize lifetime*/,
-      NLABA, domTree, deleter);
+      NLABA, domTree, CA, deleter);
   CanonicalizeBorrowScope borrowCanonicalizer(deleter);
 
   while (!defsToCanonicalize.empty()) {

lib/SILOptimizer/SILCombiner/SILCombiner.h

@@ -23,21 +23,22 @@
 
 #include "swift/Basic/Defer.h"
 #include "swift/SIL/BasicBlockUtils.h"
+#include "swift/SIL/InstructionUtils.h"
 #include "swift/SIL/SILBuilder.h"
 #include "swift/SIL/SILInstruction.h"
 #include "swift/SIL/SILInstructionWorklist.h"
 #include "swift/SIL/SILValue.h"
 #include "swift/SIL/SILVisitor.h"
-#include "swift/SIL/InstructionUtils.h"
+#include "swift/SILOptimizer/Analysis/BasicCalleeAnalysis.h"
 #include "swift/SILOptimizer/Analysis/ClassHierarchyAnalysis.h"
 #include "swift/SILOptimizer/Analysis/NonLocalAccessBlockAnalysis.h"
 #include "swift/SILOptimizer/Analysis/ProtocolConformanceAnalysis.h"
 #include "swift/SILOptimizer/OptimizerBridging.h"
+#include "swift/SILOptimizer/PassManager/PassManager.h"
 #include "swift/SILOptimizer/Utils/CastOptimizer.h"
 #include "swift/SILOptimizer/Utils/Existential.h"
 #include "swift/SILOptimizer/Utils/InstOptUtils.h"
 #include "swift/SILOptimizer/Utils/OwnershipOptUtils.h"
-#include "swift/SILOptimizer/PassManager/PassManager.h"
 
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/SmallVector.h"
@@ -55,6 +56,8 @@ class SILCombiner :
 
   AliasAnalysis *AA;
 
+  BasicCalleeAnalysis *CA;
+
   DominanceAnalysis *DA;
 
   /// Determine the set of types a protocol conforms to in whole-module

lib/SILOptimizer/Transforms/CopyPropagation.cpp

@@ -429,6 +429,7 @@ void CopyPropagation::run() {
   auto *postOrderAnalysis = getAnalysis<PostOrderAnalysis>();
   auto *accessBlockAnalysis = getAnalysis<NonLocalAccessBlockAnalysis>();
   auto *dominanceAnalysis = getAnalysis<DominanceAnalysis>();
+  auto *calleeAnalysis = getAnalysis<BasicCalleeAnalysis>();
   DominanceInfo *domTree = dominanceAnalysis->get(f);
 
   // Label for unit testing with debug output.
@@ -472,9 +473,7 @@ void CopyPropagation::run() {
   // don't need to explicitly check for changes.
   CanonicalizeOSSALifetime canonicalizer(
       pruneDebug, /*maximizeLifetime=*/!getFunction()->shouldOptimize(),
-      accessBlockAnalysis, domTree, deleter);
-  auto *calleeAnalysis = getAnalysis<BasicCalleeAnalysis>();
-
+      accessBlockAnalysis, domTree, calleeAnalysis, deleter);
   // NOTE: We assume that the function is in reverse post order so visiting the
   //       blocks and pushing begin_borrows as we see them and then popping them
   //       off the end will result in shrinking inner borrow scopes first.

lib/SILOptimizer/UtilityPasses/UnitTestRunner.cpp

@@ -422,13 +422,15 @@ struct CanonicalizeOSSALifetimeTest : UnitTest {
     auto *accessBlockAnalysis = getAnalysis<NonLocalAccessBlockAnalysis>();
     auto *dominanceAnalysis = getAnalysis<DominanceAnalysis>();
     DominanceInfo *domTree = dominanceAnalysis->get(getFunction());
+    auto *calleeAnalysis = getAnalysis<BasicCalleeAnalysis>();
     auto pruneDebug = arguments.takeBool();
     auto maximizeLifetimes = arguments.takeBool();
     auto respectAccessScopes = arguments.takeBool();
     InstructionDeleter deleter;
     CanonicalizeOSSALifetime canonicalizer(
         pruneDebug, maximizeLifetimes,
-        respectAccessScopes ? accessBlockAnalysis : nullptr, domTree, deleter);
+        respectAccessScopes ? accessBlockAnalysis : nullptr, domTree,
+        calleeAnalysis, deleter);
     auto value = arguments.takeValue();
     canonicalizer.canonicalizeValueLifetime(value);
     getFunction()->dump();
@@ -827,6 +829,7 @@ void UnitTestRunner::withTest(StringRef name, Doit doit) {
     ADD_UNIT_TEST_SUBCLASS("interior-liveness", InteriorLivenessTest)
     ADD_UNIT_TEST_SUBCLASS("is-deinit-barrier", IsDeinitBarrierTest)
     ADD_UNIT_TEST_SUBCLASS("is-lexical", IsLexicalTest)
+    ADD_UNIT_TEST_SUBCLASS("lexical-destroy-folding", LexicalDestroyFoldingTest)
     ADD_UNIT_TEST_SUBCLASS("linear-liveness", LinearLivenessTest)
     ADD_UNIT_TEST_SUBCLASS("multidef-liveness", MultiDefLivenessTest)
     ADD_UNIT_TEST_SUBCLASS("ossa-lifetime-completion", OSSALifetimeCompletionTest)

lib/SILOptimizer/Utils/CanonicalizeOSSALifetime.cpp

@@ -70,6 +70,8 @@
 #include "swift/SIL/NodeDatastructures.h"
 #include "swift/SIL/OwnershipUtils.h"
 #include "swift/SIL/PrunedLiveness.h"
+#include "swift/SILOptimizer/Analysis/BasicCalleeAnalysis.h"
+#include "swift/SILOptimizer/Analysis/Reachability.h"
 #include "swift/SILOptimizer/Utils/CFGOptUtils.h"
 #include "swift/SILOptimizer/Utils/DebugOptUtils.h"
 #include "swift/SILOptimizer/Utils/InstructionDeleter.h"
@@ -222,6 +224,42 @@ bool CanonicalizeOSSALifetime::computeCanonicalLiveness() {
   return true;
 }
 
+void CanonicalizeOSSALifetime::findDestroysOutsideBoundary(
+    SmallVectorImpl<SILInstruction *> &outsideDestroys) {
+  for (auto destroy : destroys) {
+    if (liveness.isWithinBoundary(destroy))
+      continue;
+    outsideDestroys.push_back(destroy);
+  }
+}
+
+void CanonicalizeOSSALifetime::extendLivenessToDeinitBarriers() {
+  SmallVector<SILInstruction *, 4> outsideDestroys;
+  findDestroysOutsideBoundary(outsideDestroys);
+
+  auto *def = getCurrentDef()->getDefiningInstruction();
+  ReachableBarriers barriers;
+  findBarriersBeforeInstructions(
+      outsideDestroys, getCurrentDef()->getParentBlock(),
+      *getCurrentDef()->getFunction(), barriers, [&](auto *inst) {
+        if (inst == def)
+          return true;
+        return isDeinitBarrier(inst, calleeAnalysis);
+      });
+  for (auto *barrier : barriers.instructions) {
+    liveness.updateForUse(barrier, /*lifetimeEnding*/ false);
+  }
+  for (auto *barrier : barriers.phis) {
+    for (auto *predecessor : barrier->getPredecessorBlocks()) {
+      liveness.updateForUse(predecessor->getTerminator(),
+                            /*lifetimeEnding*/ false);
+    }
+  }
+  // Ignore barriers.edges.  The beginning of the targets of such edges should
+  // not be added to liveness.  These edges will be rediscovered when computing
+  // the liveness boundary.
+}
+
 // Return true if \p inst is an end_access whose access scope overlaps the end
 // of the pruned live range. This means that a hoisted destroy might execute
 // within the access scope which previously executed outside the access scope.
@@ -1005,9 +1043,6 @@ bool CanonicalizeOSSALifetime::computeLiveness(SILValue def) {
   if (def->getOwnershipKind() != OwnershipKind::Owned)
     return false;
 
-  if (def->isLexical())
-    return false;
-
   LLVM_DEBUG(llvm::dbgs() << "  Canonicalizing: " << def);
 
   // Note: There is no need to register callbacks with this utility. 'onDelete'
@@ -1031,6 +1066,9 @@ bool CanonicalizeOSSALifetime::computeLiveness(SILValue def) {
     clearLiveness();
     return false;
   }
+  if (def->isLexical()) {
+    extendLivenessToDeinitBarriers();
+  }
   if (accessBlockAnalysis) {
     extendLivenessThroughOverlappingAccess();
   }

test/SILOptimizer/canonicalize_ossa_lifetime_unit.sil

@@ -2,6 +2,7 @@
 
 class C {}
 sil @getOwned : $@convention(thin) () -> @owned C
+sil @barrier : $@convention(thin) () -> ()
 
 // When access scopes are respected, the lifetime which previously extended
 // beyond the access scope still extends beyond it.
@@ -74,3 +75,43 @@ exit(%out : @owned $C):
   destroy_value %instance : $C
   return %out : $C
 }
+
+// CHECK-LABEL: begin running test 1 of 1 on store_arg_to_out_addr: canonicalize-ossa-lifetime with: true, false, true, @trace
+// CHECK-LABEL: sil [ossa] @store_arg_to_out_addr : $@convention(thin) (@owned C) -> @out C {
+// CHECK:       {{bb[0-9]+}}([[ADDR:%[^,]+]] : $*C, [[INSTANCE:%[^,]+]] :
+// CHECK:       store [[INSTANCE]] to [init] [[ADDR]]
+// CHECK-LABEL: } // end sil function 'store_arg_to_out_addr'
+// CHECK-LABEL: end running test 1 of 1 on store_arg_to_out_addr: canonicalize-ossa-lifetime with: true, false, true, @trace
+sil [ossa] @store_arg_to_out_addr : $@convention(thin) (@owned C) -> @out C {
+bb0(%0 : $*C, %instance : @owned $C):
+  debug_value [trace] %instance : $C
+  test_specification "canonicalize-ossa-lifetime true false true @trace"
+  %copy = copy_value %instance : $C
+  store %copy to [init] %0 : $*C
+  destroy_value %instance : $C
+  %retval = tuple ()
+  return %retval : $()
+}
+
+// CHECK-LABEL: begin running test 1 of 1 on store_arg_to_out_addr_with_barrier: canonicalize-ossa-lifetime with: true, false, true, @trace
+// CHECK-LABEL: sil [ossa] @store_arg_to_out_addr_with_barrier : $@convention(thin) (@owned C) -> @out C {
+// CHECK:       {{bb[0-9]+}}([[ADDR:%[^,]+]] : $*C, [[INSTANCE:%[^,]+]] :
+// CHECK:         [[BARRIER:%[^,]+]] = function_ref @barrier
+// CHECK:         [[REGISTER_3:%[^,]+]] = copy_value [[INSTANCE]]
+// CHECK:         store [[REGISTER_3]] to [init] [[ADDR]]
+// CHECK:         apply [[BARRIER]]()
+// CHECK:         destroy_value [[INSTANCE]]
+// CHECK-LABEL: } // end sil function 'store_arg_to_out_addr_with_barrier'
+// CHECK-LABEL: end running test 1 of 1 on store_arg_to_out_addr_with_barrier: canonicalize-ossa-lifetime with: true, false, true, @trace
+sil [ossa] @store_arg_to_out_addr_with_barrier : $@convention(thin) (@owned C) -> @out C {
+bb0(%0 : $*C, %instance : @owned $C):
+  %barrier = function_ref @barrier : $@convention(thin) () -> ()
+  debug_value [trace] %instance : $C
+  test_specification "canonicalize-ossa-lifetime true false true @trace"
+  %copy = copy_value %instance : $C
+  store %copy to [init] %0 : $*C
+  apply %barrier() : $@convention(thin) () -> ()
+  destroy_value %instance : $C
+  %retval = tuple ()
+  return %retval : $()
+}

test/SILOptimizer/copy_propagation.sil

@@ -897,10 +897,10 @@ exit:
 
 // CHECK-LABEL: sil [ossa] @dont_extend_beyond_nonoverlapping_end_access_after_store_in_consuming_block : {{.*}} {
 // CHECK:       {{bb[0-9]+}}([[INSTANCE:%[^,]+]] : @owned $C
-// CHECK:         [[COPY:%[^,]+]] = copy_value [[INSTANCE]]
+// CHECK-NOT:     copy_value
 // CHECK:         begin_access
 // CHECK-NOT:     copy_value
-// CHECK:         store [[COPY]] to [init] {{%[^,]+}}
+// CHECK:         store [[INSTANCE]] to [init] {{%[^,]+}}
 // CHECK-LABEL: } // end sil function 'dont_extend_beyond_nonoverlapping_end_access_after_store_in_consuming_block'
 sil [ossa] @dont_extend_beyond_nonoverlapping_end_access_after_store_in_consuming_block : $@convention(thin) (@owned C) -> () {
 bb0(%instance : @owned $C):

test/SILOptimizer/copy_propagation_opaque.sil

@@ -440,9 +440,7 @@ bb0:
 //
 // CHECK-LABEL: sil [ossa] @testCopyBorrow : $@convention(thin) <T> (@in T) -> () {
 // CHECK:       bb0(%0 : @owned $T):
-// CHECK:       [[COPY:%[^,]+]] = copy_value %0
 // CHECK:       destroy_value %0 : $T
-// CHECK:       destroy_value [[COPY]]
 // CHECK-NEXT:  tuple
 // CHECK-NEXT:  return
 // CHECK-LABEL: } // end sil function 'testCopyBorrow'

test/SILOptimizer/lexical_destroy_hoisting.sil

@@ -332,10 +332,8 @@ exit:
 //
 // CHECK-LABEL: sil [ossa] @dont_hoist_over_apply_at_copy : {{.*}} {
 // CHECK:       {{bb[0-9]+}}([[REGISTER_0:%[^,]+]] :
-// CHECK:         [[COPY:%[^,]+]] = copy_value [[INSTANCE]]
 // CHECK:         apply
 // CHECK:         destroy_value [[INSTANCE]]
-// CHECK:         destroy_value [[COPY]]
 // CHECK-LABEL: } // end sil function 'dont_hoist_over_apply_at_copy'
 sil [ossa] @dont_hoist_over_apply_at_copy : $@convention(thin) (@owned C) -> () {
 entry(%instance : @owned $C):

test/SILOptimizer/ossa_rauw_tests.sil

@@ -189,23 +189,21 @@ bb0(%0 : @owned $Klass):
 // CHECK-NOT: destroy_value
 //
 // CHECK: bb2:
-// CHECK-NEXT: [[COPY:%.*]] = copy_value [[ARG]]
 // CHECK-NEXT: cond_br undef, bb3, bb4
 //
 // CHECK: bb3:
-// CHECK-NEXT: destroy_value [[COPY]]
 // CHECK-NEXT: br bb2
 //
 // CHECK: bb4:
-// CHECK-NEXT: [[ENUM_SOME_RESULT:%.*]] = enum $FakeOptional<Klass>, #FakeOptional.some!enumelt, [[COPY]]
+// CHECK-NEXT: [[ENUM_SOME_RESULT:%.*]] = enum $FakeOptional<Klass>, #FakeOptional.some!enumelt, [[ARG]]
 // CHECK-NEXT: br bb6([[ENUM_SOME_RESULT]] : $FakeOptional<Klass>)
 //
 // CHECK: bb5:
+// CHECK-NEXT: destroy_value [[ARG]]
 // CHECK-NEXT: [[ENUM_NONE_RESULT:%.*]] = enum $FakeOptional<Klass>, #FakeOptional.none!enumelt
 // CHECK-NEXT: br bb6([[ENUM_NONE_RESULT]] :
 //
 // CHECK: bb6([[RESULT:%.*]] : @owned $FakeOptional<Klass>):
-// CHECK-NEXT: destroy_value [[ARG]]
 // CHECK-NEXT: return [[RESULT]]
 // CHECK: } // end sil function 'unowned_to_owned_rauw_loop'
 sil [ossa] @unowned_to_owned_rauw_loop : $@convention(thin) (@owned Klass) -> @owned FakeOptional<Klass> {