SILOptimizer: a new optimization to hoist destroys of memory locations

include/swift/SIL/MemoryLifetime.h

@@ -183,6 +183,11 @@ class MemoryLocations {
     return &locations[index];
   }
 
+  /// Registers an address projection instruction for a location.
+  void registerProjection(SingleValueInstruction *projection, unsigned locIdx) {
+    addr2LocIdx[projection] = locIdx;
+  }
+
   /// Sets the location bits os \p addr in \p bits, if \p addr is associated
   /// with a location.
   void setBits(Bits &bits, SILValue addr) {
@@ -341,13 +346,29 @@ class MemoryDataflow {
   /// Calculates the BlockState::exitReachable flags.
   void exitReachableAnalysis();
 
+  using JoinOperation = std::function<void (Bits &dest, const Bits &src)>;
+
   /// Derives the block exit sets from the entry sets by applying the gen and
   /// kill sets.
-  void solveDataflowForward();
+  /// At control flow joins, the \p join operation is applied.
+  void solveForward(JoinOperation join);
+
+  /// Calls solveForward() with a bit-intersection as join operation.
+  void solveForwardWithIntersect();
+
+  /// Calls solveForward() with a bit-union as join operation.
+  void solveForwardWithUnion();
 
   /// Derives the block entry sets from the exit sets by applying the gen and
   /// kill sets.
-  void solveDataflowBackward();
+  /// At control flow joins, the \p join operation is applied.
+  void solveBackward(JoinOperation join);
+
+  /// Calls solveBackward() with a bit-intersection as join operation.
+  void solveBackwardWithIntersect();
+
+  /// Calls solveBackward() with a bit-union as join operation.
+  void solveBackwardWithUnion();
 
   /// Debug dump the MemoryLifetime internals.
   void dump() const;

include/swift/SIL/SILInstruction.h

@@ -3240,6 +3240,9 @@ class LoadInst
     return LoadOwnershipQualifier(
       SILInstruction::Bits.LoadInst.OwnershipQualifier);
   }
+  void setOwnershipQualifier(LoadOwnershipQualifier qualifier) {
+    SILInstruction::Bits.LoadInst.OwnershipQualifier = unsigned(qualifier);
+  }
 };
 
 // *NOTE* When serializing, we can only represent up to 4 values here. If more
@@ -3279,6 +3282,9 @@ class StoreInst
     return StoreOwnershipQualifier(
       SILInstruction::Bits.StoreInst.OwnershipQualifier);
   }
+  void setOwnershipQualifier(StoreOwnershipQualifier qualifier) {
+    SILInstruction::Bits.StoreInst.OwnershipQualifier = unsigned(qualifier);
+  }
 };
 
 class EndBorrowInst;

include/swift/SILOptimizer/PassManager/Passes.def

@@ -132,6 +132,8 @@ PASS(DeadObjectElimination, "deadobject-elim",
      "Dead Object Elimination for Classes with Trivial Destruction")
 PASS(DefiniteInitialization, "definite-init",
      "Definite Initialization for Diagnostics")
+PASS(DestroyHoisting, "destroy-hoisting",
+     "Hoisting of value destroys")
 PASS(Devirtualizer, "devirtualizer",
      "Indirect Call Devirtualization")
 PASS(DiagnoseInfiniteRecursion, "diagnose-infinite-recursion",

lib/SIL/MemoryLifetime.cpp

@@ -299,7 +299,7 @@ bool MemoryLocations::analyzeAddrProjection(
                                       subLocationMap)) {
     return false;
   }
-  addr2LocIdx[projection] = subLocIdx;
+  registerProjection(projection, subLocIdx);
   collectedVals.push_back(projection);
   return true;
 }
@@ -395,7 +395,7 @@ void MemoryDataflow::exitReachableAnalysis() {
   }
 }
 
-void MemoryDataflow::solveDataflowForward() {
+void MemoryDataflow::solveForward(JoinOperation join) {
   // Pretty standard data flow solving.
   bool changed = false;
   bool firstRound = true;
@@ -405,7 +405,7 @@ void MemoryDataflow::solveDataflowForward() {
       Bits bits = st.entrySet;
       assert(!bits.empty());
       for (SILBasicBlock *pred : st.block->getPredecessorBlocks()) {
-        bits &= block2State[pred]->exitSet;
+        join(bits, block2State[pred]->exitSet);
       }
       if (firstRound || bits != st.entrySet) {
         changed = true;
@@ -419,7 +419,19 @@ void MemoryDataflow::solveDataflowForward() {
   } while (changed);
 }
 
-void MemoryDataflow::solveDataflowBackward() {
+void MemoryDataflow::solveForwardWithIntersect() {
+  solveForward([](Bits &entry, const Bits &predExit){
+    entry &= predExit;
+  });
+}
+
+void MemoryDataflow::solveForwardWithUnion() {
+  solveForward([](Bits &entry, const Bits &predExit){
+    entry |= predExit;
+  });
+}
+
+void MemoryDataflow::solveBackward(JoinOperation join) {
   // Pretty standard data flow solving.
   bool changed = false;
   bool firstRound = true;
@@ -429,7 +441,7 @@ void MemoryDataflow::solveDataflowBackward() {
       Bits bits = st.exitSet;
       assert(!bits.empty());
       for (SILBasicBlock *succ : st.block->getSuccessorBlocks()) {
-        bits &= block2State[succ]->entrySet;
+        join(bits, block2State[succ]->entrySet);
       }
       if (firstRound || bits != st.exitSet) {
         changed = true;
@@ -443,6 +455,18 @@ void MemoryDataflow::solveDataflowBackward() {
   } while (changed);
 }
 
+void MemoryDataflow::solveBackwardWithIntersect() {
+  solveBackward([](Bits &entry, const Bits &predExit){
+    entry &= predExit;
+  });
+}
+
+void MemoryDataflow::solveBackwardWithUnion() {
+  solveBackward([](Bits &entry, const Bits &predExit){
+    entry |= predExit;
+  });
+}
+
 void MemoryDataflow::dump() const {
   for (const BlockState &st : blockStates) {
     llvm::dbgs() << "bb" << st.block->getDebugID() << ":\n"
@@ -894,7 +918,7 @@ void MemoryLifetimeVerifier::verify() {
     MemoryDataflow dataFlow(function, locations.getNumLocations());
     dataFlow.entryReachabilityAnalysis();
     initDataflow(dataFlow);
-    dataFlow.solveDataflowForward();
+    dataFlow.solveForwardWithIntersect();
     checkFunction(dataFlow);
   }
   // Second step: handle single-block locations.

lib/SILOptimizer/PassManager/PassPipeline.cpp

@@ -104,6 +104,7 @@ static void addMandatoryOptPipeline(SILPassPipelinePlan &P) {
   P.addClosureLifetimeFixup();
   if (Options.shouldOptimize()) {
     P.addSemanticARCOpts();
+    P.addDestroyHoisting();
   }
   if (!Options.StripOwnershipAfterSerialization)
     P.addOwnershipModelEliminator();

lib/SILOptimizer/Transforms/CMakeLists.txt

@@ -15,6 +15,7 @@ silopt_register_sources(
   DeadCodeElimination.cpp
   DeadObjectElimination.cpp
   DeadStoreElimination.cpp
+  DestroyHoisting.cpp
   Devirtualizer.cpp
   GenericSpecializer.cpp
   MergeCondFail.cpp

lib/SILOptimizer/Transforms/CopyForwarding.cpp

@@ -1604,10 +1604,7 @@ void TempRValueOptPass::run() {
       ++II;
 
       // Remove identity copies which are a result of this optimization.
-      if (CopyInst && CopyInst->getSrc() == CopyInst->getDest() &&
-          // Identity copies cannot take the source. This check is just here
-          // to be on the safe side.
-          !CopyInst->isTakeOfSrc()) {
+      if (CopyInst && CopyInst->getSrc() == CopyInst->getDest()) {
         // This is either the CopyInst which just got optimized or it is a
         // follow-up from an earlier iteration, where another copy_addr copied
         // the temporary back to the source location.
@@ -1725,9 +1722,8 @@ bool TempRValueOptPass::collectLoads(
 
   case SILInstructionKind::CopyAddrInst: {
     // copy_addr which read from the temporary are like loads.
-    // TODO: Handle copy_addr [take]. But this doesn't seem to be important.
     auto *copyFromTmp = cast<CopyAddrInst>(user);
-    if (copyFromTmp->getDest() == address || copyFromTmp->isTakeOfSrc()) {
+    if (copyFromTmp->getDest() == address) {
       LLVM_DEBUG(llvm::dbgs() << "  Temp written or taken" << *user);
       return false;
     }
@@ -1774,7 +1770,7 @@ bool TempRValueOptPass::checkNoSourceModification(CopyAddrInst *copyInst,
 
 /// Tries to perform the temporary rvalue copy elimination for \p copyInst
 bool TempRValueOptPass::tryOptimizeCopyIntoTemp(CopyAddrInst *copyInst) {
-  if (copyInst->isTakeOfSrc() || !copyInst->isInitializationOfDest())
+  if (!copyInst->isInitializationOfDest())
     return false;
 
   auto *tempObj = dyn_cast<AllocStackInst>(copyInst->getDest());
@@ -1818,10 +1814,25 @@ bool TempRValueOptPass::tryOptimizeCopyIntoTemp(CopyAddrInst *copyInst) {
     SILInstruction *user = use->getUser();
     switch (user->getKind()) {
     case SILInstructionKind::DestroyAddrInst:
+      if (copyInst->isTakeOfSrc()) {
+        use->set(copyInst->getSrc());
+      } else {
+        user->eraseFromParent();
+      }
+      break;
     case SILInstructionKind::DeallocStackInst:
       user->eraseFromParent();
       break;
-    case SILInstructionKind::CopyAddrInst:
+    case SILInstructionKind::CopyAddrInst: {
+      auto *CAI = cast<CopyAddrInst>(user);
+      if (CAI != copyInst) {
+        assert(CAI->getSrc() == tempObj);
+        if (CAI->isTakeOfSrc() && !copyInst->isTakeOfSrc())
+          CAI->setIsTakeOfSrc(IsNotTake);
+      }
+      use->set(copyInst->getSrc());
+      break;
+    }
     case SILInstructionKind::StructElementAddrInst:
     case SILInstructionKind::TupleElementAddrInst:
     case SILInstructionKind::LoadInst: