Add new flags "reborrow" and "escaping" to SILArgument.

include/swift/SIL/OwnershipUseVisitor.h

@@ -317,14 +317,11 @@ bool OwnershipUseVisitor<Impl>::visitInteriorUses(SILValue ssaDef) {
   // Inner adjacent reborrows are considered inner borrow scopes.
   if (auto phi = SILArgument::asPhi(ssaDef)) {
     if (!visitInnerAdjacentPhis(phi, [&](SILArgument *innerPhi) {
-      // TODO: Remove this call to isGuaranteedForwarding.
-      // The phi itself should know if it is a reborrow.
-      if (isGuaranteedForwarding(innerPhi)) {
-        return visitGuaranteedUses(innerPhi);
-      } else {
-        return visitInnerAdjacentReborrow(innerPhi);
-      }
-    })) {
+          if (innerPhi->isGuaranteedForwarding()) {
+            return visitGuaranteedUses(innerPhi);
+          }
+          return visitInnerAdjacentReborrow(innerPhi);
+        })) {
       return false;
     }
   }

include/swift/SIL/OwnershipUtils.h

@@ -57,45 +57,13 @@ bool canOpcodeForwardInnerGuaranteedValues(SILValue value);
 /// the operation may be trivially rewritten with Guaranteed ownership.
 bool canOpcodeForwardInnerGuaranteedValues(Operand *use);
 
+bool computeIsScoped(SILArgument *arg);
+
+bool computeIsReborrow(SILArgument *arg);
+
 // This is the use-def equivalent of use->getOperandOwnership() ==
 // OperandOwnership::GuaranteedForwarding.
-inline bool isGuaranteedForwarding(SILValue value) {
-  if (value->getOwnershipKind() != OwnershipKind::Guaranteed) {
-    return false;
-  }
-  // NOTE: canOpcodeForwardInnerGuaranteedValues returns true for transformation
-  // terminator results.
-  if (canOpcodeForwardInnerGuaranteedValues(value) ||
-      isa<SILFunctionArgument>(value)) {
-    return true;
-  }
-  // If not a phi, return false
-  auto *phi = dyn_cast<SILPhiArgument>(value);
-  if (!phi || !phi->isPhi()) {
-    return false;
-  }
-  // For a phi, if we find GuaranteedForwarding phi operand on any incoming
-  // path, we return true. Additional verification is added to ensure
-  // GuaranteedForwarding phi operands are found on zero or all paths in the
-  // OwnershipVerifier.
-  bool isGuaranteedForwardingPhi = false;
-  phi->visitTransitiveIncomingPhiOperands([&](auto *, auto *op) -> bool {
-    auto opValue = op->get();
-    assert(opValue->getOwnershipKind().isCompatibleWith(
-        OwnershipKind::Guaranteed));
-    if (canOpcodeForwardInnerGuaranteedValues(opValue) ||
-        isa<SILFunctionArgument>(opValue)) {
-      isGuaranteedForwardingPhi = true;
-      return false;
-    }
-    auto *phi = dyn_cast<SILPhiArgument>(opValue);
-    if (!phi || !phi->isPhi()) {
-      return false;
-    }
-    return true;
-  });
-  return isGuaranteedForwardingPhi;
-}
+bool computeIsGuaranteedForwarding(SILValue value);
 
 /// Is the opcode that produces \p value capable of forwarding owned values?
 ///
@@ -523,7 +491,7 @@ class BorrowedValueKind {
                        })) {
         return Kind::Invalid;
       }
-      if (isGuaranteedForwarding(value)) {
+      if (cast<SILArgument>(value)->isGuaranteedForwarding()) {
         return Kind::Invalid;
       }
       return Kind::Phi;

include/swift/SIL/SILArgument.h

@@ -75,16 +75,21 @@ class SILArgument : public ValueBase {
 protected:
   SILArgument(ValueKind subClassKind, SILBasicBlock *inputParentBlock,
               SILType type, ValueOwnershipKind ownershipKind,
-              const ValueDecl *inputDecl = nullptr);
+              const ValueDecl *inputDecl = nullptr, bool reborrow = false,
+              bool escaping = false);
 
   // A special constructor, only intended for use in
   // SILBasicBlock::replacePHIArg and replaceFunctionArg.
   explicit SILArgument(ValueKind subClassKind, SILType type,
                        ValueOwnershipKind ownershipKind,
-                       const ValueDecl *inputDecl = nullptr)
-      : ValueBase(subClassKind, type),
-        parentBlock(nullptr), decl(inputDecl) {
+                       const ValueDecl *inputDecl = nullptr,
+                       bool reborrow = false, bool escaping = false)
+      : ValueBase(subClassKind, type), parentBlock(nullptr), decl(inputDecl) {
     sharedUInt8().SILArgument.valueOwnershipKind = uint8_t(ownershipKind);
+    // When the optimizer creates reborrows, reborrow flag needs to be set by
+    // calling setReborrow.
+    sharedUInt8().SILArgument.reborrow = false;
+    sharedUInt8().SILArgument.escaping = false;
   }
 
 public:
@@ -95,10 +100,41 @@ class SILArgument : public ValueBase {
     return ValueOwnershipKind(sharedUInt8().SILArgument.valueOwnershipKind);
   }
 
+  bool isScoped() const {
+    auto ownershipKind = getOwnershipKind();
+    if (ownershipKind == OwnershipKind::Owned) {
+      return true;
+    }
+    if (ownershipKind != OwnershipKind::Guaranteed) {
+      return false;
+    }
+    return isReborrow();
+  }
+
+  bool isReborrow() const {
+    return ValueOwnershipKind(sharedUInt8().SILArgument.reborrow);
+  }
+
+  bool isGuaranteedForwarding() const {
+    return getOwnershipKind() == OwnershipKind::Guaranteed && !isReborrow();
+  }
+
+  bool isEscaping() const {
+    return ValueOwnershipKind(sharedUInt8().SILArgument.escaping);
+  }
+
   void setOwnershipKind(ValueOwnershipKind newKind) {
     sharedUInt8().SILArgument.valueOwnershipKind = uint8_t(newKind);
   }
 
+  void setReborrow(bool isReborrow) {
+    sharedUInt8().SILArgument.reborrow = isReborrow;
+  }
+
+  void setEscaping(bool isEscaping) {
+    sharedUInt8().SILArgument.escaping = isEscaping;
+  }
+
   SILBasicBlock *getParent() const { return parentBlock; }
 
   /// Returns true if this argument is erased from a basic block.
@@ -223,14 +259,18 @@ class SILPhiArgument : public SILArgument {
 
   SILPhiArgument(SILBasicBlock *parentBlock, SILType type,
                  ValueOwnershipKind ownershipKind,
-                 const ValueDecl *decl = nullptr)
+                 const ValueDecl *decl = nullptr, bool isReborrow = false,
+                 bool isEscaping = false)
       : SILArgument(ValueKind::SILPhiArgument, parentBlock, type, ownershipKind,
-                    decl) {}
+                    decl, isReborrow, isEscaping) {}
+
   // A special constructor, only intended for use in
   // SILBasicBlock::replacePHIArg.
   explicit SILPhiArgument(SILType type, ValueOwnershipKind ownershipKind,
-                          const ValueDecl *decl = nullptr)
-      : SILArgument(ValueKind::SILPhiArgument, type, ownershipKind, decl) {}
+                          const ValueDecl *decl = nullptr,
+                          bool isReborrow = false, bool isEscaping = false)
+      : SILArgument(ValueKind::SILPhiArgument, type, ownershipKind, decl,
+                    isReborrow, isEscaping) {}
 
 public:
   /// Return true if this is block argument is a phi, as opposed to a terminator

include/swift/SIL/SILBasicBlock.h

@@ -350,24 +350,31 @@ public SwiftObjectHeader {
   /// replacePhiArgumentAndRAUW.
   SILPhiArgument *replacePhiArgument(unsigned i, SILType type,
                                      ValueOwnershipKind kind,
-                                     const ValueDecl *decl = nullptr);
+                                     const ValueDecl *decl = nullptr,
+                                     bool isReborrow = false,
+                                     bool isEscaping = false);
 
   /// Replace phi argument \p i and RAUW all uses.
-  SILPhiArgument *
-  replacePhiArgumentAndReplaceAllUses(unsigned i, SILType type,
-                                      ValueOwnershipKind kind,
-                                      const ValueDecl *decl = nullptr);
+  SILPhiArgument *replacePhiArgumentAndReplaceAllUses(
+      unsigned i, SILType type, ValueOwnershipKind kind,
+      const ValueDecl *decl = nullptr, bool isReborrow = false,
+      bool isEscaping = false);
 
   /// Allocate a new argument of type \p Ty and append it to the argument
-  /// list. Optionally you can pass in a value decl parameter.
+  /// list. Optionally you can pass in a value decl parameter, reborrow flag and
+  /// escaping flag.
   SILPhiArgument *createPhiArgument(SILType Ty, ValueOwnershipKind Kind,
-                                    const ValueDecl *D = nullptr);
+                                    const ValueDecl *D = nullptr,
+                                    bool isReborrow = false,
+                                    bool isEscaping = false);
 
   /// Insert a new SILPhiArgument with type \p Ty and \p Decl at position \p
   /// AtArgPos.
   SILPhiArgument *insertPhiArgument(unsigned AtArgPos, SILType Ty,
                                     ValueOwnershipKind Kind,
-                                    const ValueDecl *D = nullptr);
+                                    const ValueDecl *D = nullptr,
+                                    bool isReborrow = false,
+                                    bool isEscaping = false);
 
   /// Remove all block arguments.
   void dropAllArguments();

include/swift/SIL/SILCloner.h

@@ -734,7 +734,8 @@ void SILCloner<ImplClass>::clonePhiArgs(SILBasicBlock *oldBB) {
   // Create new arguments for each of the original block's arguments.
   for (auto *Arg : oldBB->getSILPhiArguments()) {
     SILValue mappedArg = mappedBB->createPhiArgument(
-      getOpType(Arg->getType()), Arg->getOwnershipKind());
+        getOpType(Arg->getType()), Arg->getOwnershipKind(), Arg->getDecl(),
+        Arg->isReborrow(), Arg->isEscaping());
 
     asImpl().mapValue(Arg, mappedArg);
   }

include/swift/SIL/SILNode.h

@@ -181,7 +181,6 @@ class alignas(8) SILNode :
     SHARED_TEMPLATE_FIELD(typename, SwitchEnumInstBase, bool hasDefault);
     SHARED_TEMPLATE_FIELD(SILInstructionKind, LoadReferenceInstBase, bool isTake);
     SHARED_TEMPLATE_FIELD(SILInstructionKind, StoreReferenceInstBase, bool isInitializationOfDest);
-    SHARED_FIELD(SILArgument, uint8_t valueOwnershipKind);
     SHARED_FIELD(MultipleValueInstructionResult, uint8_t valueOwnershipKind);
     SHARED_FIELD(UncheckedOwnershipConversionInst, uint8_t valueOwnershipKind);
     SHARED_FIELD(StoreInst, uint8_t ownershipQualifier);
@@ -203,6 +202,11 @@ class alignas(8) SILNode :
     SHARED_FIELD(ConvertFunctionInst, bool withoutActuallyEscaping);
     SHARED_FIELD(BeginCOWMutationInst, bool native);
 
+    SHARED_FIELD(SILArgument, uint8_t
+                 valueOwnershipKind : NumVOKindBits,
+                 reborrow : 1,
+                 escaping : 1);
+
     SHARED_FIELD(DebugValueInst, uint8_t
                  poisonRefs : 1,
                  usesMoveableValueDebugInfo : 1,

include/swift/SIL/SILValue.h

@@ -570,6 +570,8 @@ class ValueBase : public SILNode, public SILAllocated<ValueBase> {
 
   bool isLexical() const;
 
+  bool isGuaranteedForwarding() const;
+
   /// Unsafely eliminate moveonly from this value's type. Returns true if the
   /// value's underlying type was move only and thus was changed. Returns false
   /// otherwise.

lib/SIL/IR/OperandOwnership.cpp

@@ -435,11 +435,11 @@ OperandOwnershipClassifier::visitSelectEnumInst(SelectEnumInst *i) {
 }
 
 OperandOwnership OperandOwnershipClassifier::visitBranchInst(BranchInst *bi) {
-  ValueOwnershipKind destBlockArgOwnershipKind =
-      bi->getDestBB()->getArgument(getOperandIndex())->getOwnershipKind();
+  auto *destArg = bi->getDestBB()->getArgument(getOperandIndex());
+  ValueOwnershipKind destBlockArgOwnershipKind = destArg->getOwnershipKind();
 
   if (destBlockArgOwnershipKind == OwnershipKind::Guaranteed) {
-    return isGuaranteedForwarding(getValue())
+    return destArg->isGuaranteedForwarding()
                ? OperandOwnership::GuaranteedForwarding
                : OperandOwnership::Reborrow;
   }

lib/SIL/IR/SILArgument.cpp

@@ -27,10 +27,13 @@ using namespace swift;
 SILArgument::SILArgument(ValueKind subClassKind,
                          SILBasicBlock *inputParentBlock, SILType type,
                          ValueOwnershipKind ownershipKind,
-                         const ValueDecl *inputDecl)
-    : ValueBase(subClassKind, type),
-      parentBlock(inputParentBlock), decl(inputDecl) {
+                         const ValueDecl *inputDecl, bool reborrow,
+                         bool escaping)
+    : ValueBase(subClassKind, type), parentBlock(inputParentBlock),
+      decl(inputDecl) {
   sharedUInt8().SILArgument.valueOwnershipKind = uint8_t(ownershipKind);
+  sharedUInt8().SILArgument.reborrow = reborrow;
+  sharedUInt8().SILArgument.escaping = escaping;
   inputParentBlock->insertArgument(inputParentBlock->args_end(), this);
 }
 

lib/SIL/IR/SILBasicBlock.cpp

@@ -203,7 +203,9 @@ SILFunctionArgument *SILBasicBlock::replaceFunctionArgument(
 /// ValueDecl D).
 SILPhiArgument *SILBasicBlock::replacePhiArgument(unsigned i, SILType Ty,
                                                   ValueOwnershipKind Kind,
-                                                  const ValueDecl *D) {
+                                                  const ValueDecl *D,
+                                                  bool isReborrow,
+                                                  bool isEscaping) {
   assert(!isEntry() && "PHI Arguments can not be in the entry block");
   SILFunction *F = getParent();
   SILModule &M = F->getModule();
@@ -212,7 +214,8 @@ SILPhiArgument *SILBasicBlock::replacePhiArgument(unsigned i, SILType Ty,
 
   assert(ArgumentList[i]->use_empty() && "Expected no uses of the old BB arg!");
 
-  SILPhiArgument *NewArg = new (M) SILPhiArgument(Ty, Kind, D);
+  SILPhiArgument *NewArg =
+      new (M) SILPhiArgument(Ty, Kind, D, isReborrow, isEscaping);
   NewArg->setParent(this);
   ArgumentList[i]->parentBlock = nullptr;
 
@@ -224,7 +227,8 @@ SILPhiArgument *SILBasicBlock::replacePhiArgument(unsigned i, SILType Ty,
 }
 
 SILPhiArgument *SILBasicBlock::replacePhiArgumentAndReplaceAllUses(
-    unsigned i, SILType ty, ValueOwnershipKind kind, const ValueDecl *d) {
+    unsigned i, SILType ty, ValueOwnershipKind kind, const ValueDecl *d,
+    bool isReborrow, bool isEscaping) {
   // Put in an undef placeholder before we do the replacement since
   // replacePhiArgument() expects the replaced argument to not have
   // any uses.
@@ -238,7 +242,7 @@ SILPhiArgument *SILBasicBlock::replacePhiArgumentAndReplaceAllUses(
   }
 
   // Perform the replacement.
-  auto *newArg = replacePhiArgument(i, ty, kind, d);
+  auto *newArg = replacePhiArgument(i, ty, kind, d, isReborrow, isEscaping);
 
   // Wire back up the uses.
   while (!operands.empty()) {
@@ -250,20 +254,26 @@ SILPhiArgument *SILBasicBlock::replacePhiArgumentAndReplaceAllUses(
 
 SILPhiArgument *SILBasicBlock::createPhiArgument(SILType Ty,
                                                  ValueOwnershipKind Kind,
-                                                 const ValueDecl *D) {
+                                                 const ValueDecl *D,
+                                                 bool isReborrow,
+                                                 bool isEscaping) {
   assert(!isEntry() && "PHI Arguments can not be in the entry block");
   if (Ty.isTrivial(*getParent()))
     Kind = OwnershipKind::None;
-  return new (getModule()) SILPhiArgument(this, Ty, Kind, D);
+  return new (getModule())
+      SILPhiArgument(this, Ty, Kind, D, isReborrow, isEscaping);
 }
 
 SILPhiArgument *SILBasicBlock::insertPhiArgument(unsigned AtArgPos, SILType Ty,
                                                  ValueOwnershipKind Kind,
-                                                 const ValueDecl *D) {
+                                                 const ValueDecl *D,
+                                                 bool isReborrow,
+                                                 bool isEscaping) {
   assert(!isEntry() && "PHI Arguments can not be in the entry block");
   if (Ty.isTrivial(*getParent()))
     Kind = OwnershipKind::None;
-  auto *arg = new (getModule()) SILPhiArgument(Ty, Kind, D);
+  auto *arg =
+      new (getModule()) SILPhiArgument(Ty, Kind, D, isReborrow, isEscaping);
   arg->parentBlock = this;
   insertArgument(ArgumentList.begin() + AtArgPos, arg);
   return arg;