Support @guaranteed forwarding phis

Author: meg-gupta

include/swift/SIL/LinearLifetimeChecker.h

@@ -53,7 +53,7 @@ class LinearLifetimeChecker {
   class ErrorBuilder;
 
 private:
-  friend class ReborrowVerifier;
+  friend class GuaranteedPhiVerifier;
   friend class SILOwnershipVerifier;
   friend class SILValueOwnershipChecker;
 

include/swift/SIL/OwnershipUtils.h

@@ -90,6 +90,32 @@ inline bool isGuaranteedForwarding(SILValue value) {
   return canOpcodeForwardGuaranteedValues(value);
 }
 
+/// Returns true if it is a forwarding phi
+inline bool isGuaranteedForwardingPhi(SILValue value) {
+  if (value->getOwnershipKind() != OwnershipKind::Guaranteed) {
+    return false;
+  }
+  if (isa<BeginBorrowInst>(value) || isa<LoadBorrowInst>(value)) {
+    return false;
+  }
+  auto *phi = dyn_cast<SILPhiArgument>(value);
+  if (!phi || !phi->isPhi()) {
+    return true;
+  }
+  bool isGuaranteedForwardingPhi = true;
+  phi->visitTransitiveIncomingPhiOperands(
+      [&](auto *phi, auto *operand) -> bool {
+        if (isa<BeginBorrowInst>(operand->get()) ||
+            isa<LoadBorrowInst>(operand->get())) {
+          isGuaranteedForwardingPhi = false;
+          return false;
+        }
+        return true;
+      });
+
+  return isGuaranteedForwardingPhi;
+}
+
 /// Is the opcode that produces \p value capable of forwarding owned values?
 ///
 /// This may be true even if the current instance of the instruction is not a
@@ -215,6 +241,11 @@ bool findExtendedTransitiveGuaranteedUses(
 bool findUsesOfSimpleValue(SILValue value,
                            SmallVectorImpl<Operand *> *usePoints = nullptr);
 
+/// Visit all GuaranteedForwardingPhis of \p value, not looking through
+/// reborrows.
+bool visitGuaranteedForwardingPhisForSSAValue(
+    SILValue value, function_ref<bool(Operand *)> func);
+
 /// An operand that forwards ownership to one or more results.
 class ForwardingOperand {
   Operand *use = nullptr;
@@ -1249,14 +1280,19 @@ OwnedValueIntroducer getSingleOwnedValueIntroducer(SILValue value);
 
 using BaseValueSet = SmallPtrSet<SILValue, 8>;
 
-/// Starting from \p initialScopeOperand, find all reborrows and their
-/// corresponding base values, and run the visitor function \p
-/// visitReborrowBaseValuePair on them.
-///  Note that a reborrow phi, can have different base values based on different
-/// control flow paths.
-void findTransitiveReborrowBaseValuePairs(
-    BorrowingOperand initialScopeOperand, SILValue origBaseValue,
-    function_ref<void(SILPhiArgument *, SILValue)> visitReborrowBaseValuePair);
+/// Starting from \p borrowInst, find all reborrows along with their base
+/// values, and run the visitor function \p visitReborrowPhiBaseValuePair on
+/// them.
+void visitExtendedReborrowPhiBaseValuePairs(
+    BeginBorrowInst *borrowInst, function_ref<void(SILPhiArgument *, SILValue)>
+                                     visitReborrowPhiBaseValuePair);
+
+/// Starting from \p borrow, find all GuaranteedForwardingPhi uses along with
+/// their base values, and run the visitor function \p
+/// visitGuaranteedForwardingPhiBaseValuePair on them.
+void visitExtendedGuaranteedForwardingPhiBaseValuePairs(
+    BorrowedValue borrow, function_ref<void(SILPhiArgument *, SILValue)>
+                              visitGuaranteedForwardingPhiBaseValuePair);
 
 /// Visit the phis in the same block as \p phi which are reborrows of a borrow
 /// of one of the values reaching \p phi.

include/swift/SIL/SILValue.h

@@ -833,6 +833,8 @@ struct OperandOwnership {
     /// borrow scope.
     /// (tuple_extract, struct_extract, cast, switch)
     GuaranteedForwarding,
+    /// A GuaranteedForwarding value passed as a phi operand.
+    GuaranteedForwardingPhi,
     /// End Borrow. End the borrow scope opened directly by the operand.
     /// The operand must be a begin_borrow, begin_apply, or function argument.
     /// (end_borrow, end_apply)
@@ -925,6 +927,7 @@ inline OwnershipConstraint OperandOwnership::getOwnershipConstraint() {
     return {OwnershipKind::Owned, UseLifetimeConstraint::LifetimeEnding};
   case OperandOwnership::InteriorPointer:
   case OperandOwnership::GuaranteedForwarding:
+  case OperandOwnership::GuaranteedForwardingPhi:
     return {OwnershipKind::Guaranteed,
             UseLifetimeConstraint::NonLifetimeEnding};
   case OperandOwnership::EndBorrow:
@@ -953,6 +956,7 @@ inline bool canAcceptUnownedValue(OperandOwnership operandOwnership) {
   case OperandOwnership::ForwardingConsume:
   case OperandOwnership::InteriorPointer:
   case OperandOwnership::GuaranteedForwarding:
+  case OperandOwnership::GuaranteedForwardingPhi:
   case OperandOwnership::EndBorrow:
   case OperandOwnership::Reborrow:
     return false;

lib/SIL/IR/OperandOwnership.cpp

@@ -27,6 +27,9 @@ bool swift::checkOperandOwnershipInvariants(const Operand *operand,
     // Must be a valid BorrowingOperand.
     return bool(BorrowingOperand(const_cast<Operand *>(operand)));
   }
+  if (opOwnership == OperandOwnership::GuaranteedForwarding) {
+    return canOpcodeForwardGuaranteedValues(const_cast<Operand *>(operand));
+  }
   return true;
 }
 
@@ -440,10 +443,10 @@ OperandOwnership OperandOwnershipClassifier::visitBranchInst(BranchInst *bi) {
   ValueOwnershipKind destBlockArgOwnershipKind =
       bi->getDestBB()->getArgument(getOperandIndex())->getOwnershipKind();
 
-  // FIXME: remove this special case once all aggregate operations behave just
-  // like phis.
   if (destBlockArgOwnershipKind == OwnershipKind::Guaranteed) {
-    return OperandOwnership::Reborrow;
+    return isGuaranteedForwardingPhi(getValue())
+               ? OperandOwnership::GuaranteedForwardingPhi
+               : OperandOwnership::Reborrow;
   }
   return destBlockArgOwnershipKind.getForwardingOperandOwnership(
     /*allowUnowned*/true);

lib/SIL/IR/SILValue.cpp

@@ -432,6 +432,8 @@ StringRef OperandOwnership::asString() const {
     return "interior-pointer";
   case OperandOwnership::GuaranteedForwarding:
     return "guaranteed-forwarding";
+  case OperandOwnership::GuaranteedForwardingPhi:
+    return "guaranteed-forwarding-phi";
   case OperandOwnership::EndBorrow:
     return "end-borrow";
   case OperandOwnership::Reborrow: