Handle mark_dependence [nonescaping] like a borrowing instruction.

Author: atrick

SwiftCompilerSources/Sources/Optimizer/Utilities/BorrowUtils.swift

@@ -145,6 +145,11 @@ import SIL
 ///
 /// Note: This must handle all instructions with a .borrow operand ownership.
 ///
+/// Note: mark_dependence is a BorrowingInstruction because it creates
+/// a borrow scope for its base operand. Its result, however, is not a
+/// BeginBorrowValue. It is instead a ForwardingInstruction relative
+/// to its value operand.
+///
 /// TODO: replace BorrowIntroducingInstruction
 ///
 /// TODO: Add non-escaping MarkDependence.
@@ -153,15 +158,21 @@ enum BorrowingInstruction : CustomStringConvertible, Hashable {
   case storeBorrow(StoreBorrowInst)
   case beginApply(BeginApplyInst)
   case partialApply(PartialApplyInst)
+  case markDependence(MarkDependenceInst)
   case startAsyncLet(BuiltinInst)
 
   init?(_ inst: Instruction) {
     switch inst {
-    case let bbi as BeginBorrowInst: self = .beginBorrow(bbi)
-    case let sbi as StoreBorrowInst: self = .storeBorrow(sbi)
-    case let bai as BeginApplyInst: self = .beginApply(bai)
+    case let bbi as BeginBorrowInst:
+      self = .beginBorrow(bbi)
+    case let sbi as StoreBorrowInst:
+      self = .storeBorrow(sbi)
+    case let bai as BeginApplyInst:
+      self = .beginApply(bai)
     case let pai as PartialApplyInst where pai.isOnStack:
       self = .partialApply(pai)
+    case let mdi as MarkDependenceInst:
+      self = .markDependence(mdi)
     case let bi as BuiltinInst
            where bi.id == .StartAsyncLetWithLocalBuffer:
       self = .startAsyncLet(bi)
@@ -172,11 +183,18 @@ enum BorrowingInstruction : CustomStringConvertible, Hashable {
 
   var instruction: Instruction {
     switch self {
-    case .beginBorrow(let bbi): return bbi
-    case .storeBorrow(let sbi): return sbi
-    case .beginApply(let bai): return bai
-    case .partialApply(let pai): return pai
-    case .startAsyncLet(let bi): return bi
+    case .beginBorrow(let bbi):
+      return bbi
+    case .storeBorrow(let sbi):
+      return sbi
+    case .beginApply(let bai):
+      return bai
+    case .partialApply(let pai):
+      return pai
+    case .markDependence(let mdi):
+      return mdi
+    case .startAsyncLet(let bi):
+      return bi
     }
   }
 
@@ -188,9 +206,17 @@ enum BorrowingInstruction : CustomStringConvertible, Hashable {
   /// incoming value dominates or is consumed by an outer adjacent
   /// phi. See InteriorLiveness.
   ///
-  /// TODO: to hande reborrow-extended uses migrate ExtendedLiveness
+  /// TODO: to hande reborrow-extended uses, migrate ExtendedLiveness
   /// to SwiftCompilerSources.
   ///
+  /// TODO: Handle .partialApply and .markDependence forwarded uses
+  /// that are phi operands. Currently, partial_apply [on_stack]
+  /// and mark_dependence [nonescaping] cannot be cloned, so walking
+  /// through the phi safely returns dominated scope-ending operands.
+  /// Instead, this could report the phi as a scope-ending use, and
+  /// the client could decide whether to walk through them or to
+  /// construct reborrow-extended liveness.
+  ///
   /// TODO: For instructions that are not a BeginBorrowValue, verify
   /// that scope ending instructions exist on all paths. These
   /// instructions should be complete after SILGen and never cloned to
@@ -206,8 +232,10 @@ enum BorrowingInstruction : CustomStringConvertible, Hashable {
       }
     case .beginApply(let bai):
       return bai.token.uses.walk { return visitor($0) }
-    case .partialApply(let pai):
-      return visitForwardedUses(introducer: pai, context) {
+    case .partialApply, .markDependence:
+      let svi = instruction as! SingleValueInstruction
+      assert(svi.ownership == .owned)
+      return visitForwardedUses(introducer: svi, context) {
         switch $0 {
         case let .operand(operand):
           if operand.endsLifetime {
@@ -306,7 +334,7 @@ enum BeginBorrowValue {
       self = BeginBorrowValue(beginBorrow)!
     case let .beginApply(beginApply):
       self = BeginBorrowValue(beginApply.token)!
-    case .storeBorrow, .partialApply, .startAsyncLet:
+    case .storeBorrow, .partialApply, .markDependence, .startAsyncLet:
       return nil
     }
   }

SwiftCompilerSources/Sources/Optimizer/Utilities/OwnershipLiveness.swift

@@ -142,6 +142,7 @@ func computeInteriorLiveness(for definingValue: Value,
 /// - forwardingUse(of:isInnerlifetime:)
 /// - interiorPointerUse(of:into:)
 /// - pointerEscapingUse(of:)
+/// - dependentUse(of:into:)
 /// - borrowingUse(of:by:)
 /// - reborrowingUse(of:isInnerlifetime:)
 ///
@@ -195,9 +196,6 @@ protocol OwnershipUseVisitor {
   mutating func interiorPointerUse(of: Operand, into address: Value)
     -> WalkResult
 
-  /// A use that creates a dependent value.
-  mutating func dependentUse(of: Operand, into value: Value) -> WalkResult
-
   /// A use that escapes information from its operand's value.
   ///
   /// Note: this may not find all relevant pointer escapes, such as
@@ -206,6 +204,14 @@ protocol OwnershipUseVisitor {
   /// implement this as a fatalError.
   mutating func pointerEscapingUse(of operand: Operand) -> WalkResult
 
+  /// A use that creates an implicit borrow scope over the lifetime of
+  /// an owned dependent value. The operand owership is .borrow, but
+  /// there are no explicit scope-ending operations. Instead
+  /// BorrowingInstruction.scopeEndingOperands will return the final
+  /// consumes in the dependent value's forwaring chain.
+  mutating func dependentUse(of operand: Operand, into value: Value)
+    -> WalkResult
+
   /// A use that is scoped to an inner borrow scope.
   ///
   /// Call `visitInnerBorrowUses(of:)` to recursively classify any
@@ -334,8 +340,7 @@ extension OwnershipUseVisitor {
       return ownershipLeafUse(of: operand, isInnerLifetime: false)
 
     case .borrow:
-      return borrowingUse(of: operand,
-                          by: BorrowingInstruction(operand.instruction)!)
+      return visitBorrowingUse(of: operand)
 
     // TODO: Eventually, visit owned InteriorPointers as implicit borrows.
     case .interiorPointer, .trivialUse, .endBorrow, .reborrow,
@@ -365,8 +370,7 @@ extension OwnershipUseVisitor {
       return forwardingUse(of: operand, isInnerLifetime: false)
 
     case .borrow:
-      return borrowingUse(of: operand,
-                          by: BorrowingInstruction(operand.instruction)!)
+      return visitBorrowingUse(of: operand)
 
     case .interiorPointer:
       return visitInteriorPointerUse(of: operand)
@@ -376,6 +380,21 @@ extension OwnershipUseVisitor {
     }
   }
 
+  private mutating func visitBorrowingUse(of operand: Operand)
+    -> WalkResult {
+    switch operand.instruction {
+    case let pai as PartialApplyInst:
+      assert(pai.isOnStack)
+      return dependentUse(of: operand, into: pai)
+    case let mdi as MarkDependenceInst:
+      assert(operand == mdi.baseOperand && mdi.isNonEscaping)
+      return dependentUse(of: operand, into: mdi)
+    default:
+      return borrowingUse(of: operand,
+                          by: BorrowingInstruction(operand.instruction)!)
+    }
+  }
+
   // TODO: Change ProjectBox ownership to InteriorPointer and allow
   // owned interior pointers.
   private mutating func visitInteriorPointerUse(of operand: Operand)
@@ -385,9 +404,6 @@ extension OwnershipUseVisitor {
          is OpenExistentialBoxInst:
       let svi = operand.instruction as! SingleValueInstruction
       return interiorPointerUse(of: operand, into: svi)
-    case let mdi as MarkDependenceInst:
-      assert(operand == mdi.baseOperand && mdi.isNonEscaping)
-      return dependentUse(of: operand, into: mdi)
     default:
       return pointerEscapingUse(of: operand)
     }
@@ -554,6 +570,19 @@ extension InteriorUseWalker: OwnershipUseVisitor {
     return walkDownAddressUses(of: address)
   }  
 
+  // Handle partial_apply [on_stack] and mark_dependence [nonescaping].
+  //
+  // TODO: Rather than walking down the owned uses, this could call
+  // visitInnerBorrowUses, but we need to ensure all dependent values
+  // are complete first:
+  //
+  //   if let svi = borrowInst as! SingleValueInstruction,
+  //          svi.ownership == .owned {
+  //     if handleInner(borrowed: beginBorrow.value) == .abortWalk {
+  //       return .abortWalk
+  //     }
+  //     return visitInnerBorrowUses(of: borrowInst)
+  //   }
   mutating func dependentUse(of operand: Operand, into value: Value)
     -> WalkResult {
     // OSSA lifetime ignores trivial types.
@@ -665,7 +694,8 @@ extension InteriorUseWalker: AddressUseVisitor {
   }
 
   private mutating func walkDownAddressUses(of address: Value) -> WalkResult {
-    address.uses.ignoreTypeDependence.walk {
+    assert(address.type.isAddress)
+    return address.uses.ignoreTypeDependence.walk {
       // Record all uses
       if useVisitor($0) == .abortWalk {
         return .abortWalk
@@ -690,9 +720,9 @@ extension InteriorUseWalker {
       if phi.value.ownership == .guaranteed {
         return walkDown(guaranteedPhi: phi)
       }
-      // This is a phi of a dependent value.
-      // On-stack partial apply cannot be cloned, so all dependent
-      // phis must be dominated.
+      // This is a phi of a dependent value. partial_apply [on_stack]
+      // and mark_dependence [nonescaping] cannot be cloned, so all
+      // dependent phis must be dominated.
       assert(definingValue.parentBlock.dominates(phi.successor,
                                                  functionContext.dominatorTree),
              "on-stack partial apply cannot be cloned")

docs/SIL.rst

@@ -5523,15 +5523,14 @@ the dependency is on the current value stored in the address.
 The optional ``nonescaping`` attribute indicates that no value derived
 from ``%value`` escapes the lifetime of ``%base``. As with escaping
 ``mark_dependence``, all values transitively forwarded from ``%value``
-must be destroyed within the lifetime of ``%base``. Unlike escaping
+must be destroyed within the lifetime of ` `%base``. Unlike escaping
 ``mark_dependence``, this must be statically verifiable. Additionally,
 unlike escaping ``mark_dependence``, derived values include copies of
 ``%value`` and values transitively forwarded from those copies. If
-``%base`` is identical to ``%value`` this simply means that copies of
-``%value`` do not outlive the original OSSA lifetime of
-``%value``. Furthermore, unlike escaping ``mark_dependence``, no value
-derived from ``%value`` may have a bitwise escape (conversion to
-UnsafePointer) or pointer escape (unknown use).
+``%base`` must not be identical to ``%value``. Unlike escaping
+``mark_dependence``, no value derived from ``%value`` may have a
+bitwise escape (conversion to UnsafePointer) or pointer escape
+(unknown use).
 
 is_unique
 `````````

include/swift/SIL/OwnershipUtils.h

@@ -261,6 +261,7 @@ class BorrowingOperandKind {
     TryApply,
     Yield,
     PartialApplyStack,
+    MarkDependenceNonEscaping,
     BeginAsyncLet,
   };
 
@@ -292,6 +293,8 @@ class BorrowingOperandKind {
       return Kind::Yield;
     case SILInstructionKind::PartialApplyInst:
       return Kind::PartialApplyStack;
+    case SILInstructionKind::MarkDependenceInst:
+      return Kind::MarkDependenceNonEscaping;
     case SILInstructionKind::BuiltinInst: {
       auto bi = cast<BuiltinInst>(i);
       if (bi->getBuiltinKind() == BuiltinValueKind::StartAsyncLetWithLocalBuffer) {
@@ -395,6 +398,7 @@ struct BorrowingOperand {
     case BorrowingOperandKind::TryApply:
     case BorrowingOperandKind::Yield:
     case BorrowingOperandKind::PartialApplyStack:
+    case BorrowingOperandKind::MarkDependenceNonEscaping:
     case BorrowingOperandKind::BeginAsyncLet:
       return false;
     case BorrowingOperandKind::Branch:
@@ -428,6 +432,7 @@ struct BorrowingOperand {
     case BorrowingOperandKind::TryApply:
     case BorrowingOperandKind::Yield:
     case BorrowingOperandKind::PartialApplyStack:
+    case BorrowingOperandKind::MarkDependenceNonEscaping:
     case BorrowingOperandKind::BeginAsyncLet:
       return false;
     }

include/swift/SIL/SILInstruction.h

@@ -8370,6 +8370,10 @@ class MarkDependenceInst
   bool isNonEscaping() const {
     return sharedUInt8().MarkDependenceInst.nonEscaping;
   }
+
+  /// Visit the instructions that end the lifetime of an OSSA on-stack closure.
+  bool visitNonEscapingLifetimeEnds(llvm::function_ref<bool (Operand*)> func)
+    const;
 };
 
 /// Promote an Objective-C block that is on the stack to the heap, or simply

lib/SIL/IR/OperandOwnership.cpp

@@ -654,10 +654,15 @@ OperandOwnership OperandOwnershipClassifier::visitCopyBlockWithoutEscapingInst(
 OperandOwnership
 OperandOwnershipClassifier::visitMarkDependenceInst(MarkDependenceInst *mdi) {
   // If we are analyzing "the value", we forward ownership.
-  if (getValue() == mdi->getValue()) {
+  if (getOperandIndex() == MarkDependenceInst::Value) {
     return getOwnershipKind().getForwardingOperandOwnership(
       /*allowUnowned*/true);
   }
+  if (getOperandIndex() == MarkDependenceInst::Base && mdi->isNonEscaping()) {
+    // This creates a "dependent value", just like on-stack partial_apply, which
+    // we treat like a borrow.
+    return OperandOwnership::Borrow;
+  }
   // FIXME: Add an end_dependence instruction so we can treat mark_dependence as
   // a borrow of the base (mark_dependence %base -> end_dependence is analogous
   // to a borrow scope).

lib/SIL/IR/SILInstruction.cpp

@@ -1401,6 +1401,12 @@ bool SILInstruction::isTriviallyDuplicatable() const {
   if (auto *PA = dyn_cast<PartialApplyInst>(this)) {
     return !PA->isOnStack();
   }
+  // Like partial_apply [onstack], mark_dependence [nonescaping] creates a
+  // borrow scope. We currently assume that a set of dominated scope-ending uses
+  // can be found.
+  if (auto *MD = dyn_cast<MarkDependenceInst>(this)) {
+    return !MD->isNonEscaping();
+  }
 
   if (isa<OpenExistentialAddrInst>(this) || isa<OpenExistentialRefInst>(this) ||
       isa<OpenExistentialMetatypeInst>(this) ||
@@ -1831,6 +1837,18 @@ PartialApplyInst::visitOnStackLifetimeEnds(
   return !noUsers;
 }
 
+bool MarkDependenceInst::
+visitNonEscapingLifetimeEnds(llvm::function_ref<bool (Operand *)> func) const {
+  assert(getFunction()->hasOwnership() && isNonEscaping()
+         && "only meaningful for nonescaping dependencies");
+  bool noUsers = true;
+
+  if (!visitRecursivelyLifetimeEndingUses(this, noUsers, func)) {
+    return false;
+  }
+  return !noUsers;
+}
+
 PartialApplyInst *
 DestroyValueInst::getNonescapingClosureAllocation() const {
   SILValue operand = getOperand();

lib/SIL/Utils/OwnershipUtils.cpp

@@ -621,6 +621,9 @@ void BorrowingOperandKind::print(llvm::raw_ostream &os) const {
   case Kind::PartialApplyStack:
     os << "PartialApply [stack]";
     return;
+  case Kind::MarkDependenceNonEscaping:
+    os << "MarkDependence [nonescaping]";
+    return;
   case Kind::BeginAsyncLet:
     os << "BeginAsyncLet";
     return;
@@ -655,7 +658,8 @@ bool BorrowingOperand::hasEmptyRequiredEndingUses() const {
   case BorrowingOperandKind::StoreBorrow:
   case BorrowingOperandKind::BeginApply:
   case BorrowingOperandKind::BeginAsyncLet:
-  case BorrowingOperandKind::PartialApplyStack: {
+  case BorrowingOperandKind::PartialApplyStack:
+  case BorrowingOperandKind::MarkDependenceNonEscaping: {
     return op->getUser()->hasUsesOfAnyResult();
   }
   case BorrowingOperandKind::Branch: {
@@ -716,13 +720,18 @@ bool BorrowingOperand::visitScopeEndingUses(
     return !deadApply;
   }
   case BorrowingOperandKind::PartialApplyStack: {
-    auto user = cast<PartialApplyInst>(op->getUser());
+    auto *user = cast<PartialApplyInst>(op->getUser());
     assert(user->isOnStack() && "escaping closures can't borrow");
     // The closure's borrow lifetimes end when the closure itself ends its
     // lifetime. That may happen transitively through conversions that forward
     // ownership of the closure.
     return user->visitOnStackLifetimeEnds(func);
   }
+  case BorrowingOperandKind::MarkDependenceNonEscaping: {
+    auto *user = cast<MarkDependenceInst>(op->getUser());
+    assert(user->isNonEscaping() && "escaping dependencies don't borrow");
+    return user->visitNonEscapingLifetimeEnds(func);
+  }
   case BorrowingOperandKind::BeginAsyncLet: {
     auto user = cast<BuiltinInst>(op->getUser());
     // The async let ends its borrow when the task is ended.
@@ -780,6 +789,7 @@ BorrowedValue BorrowingOperand::getBorrowIntroducingUserResult() const {
   case BorrowingOperandKind::BeginApply:
   case BorrowingOperandKind::Yield:
   case BorrowingOperandKind::PartialApplyStack:
+  case BorrowingOperandKind::MarkDependenceNonEscaping:
   case BorrowingOperandKind::BeginAsyncLet:
   case BorrowingOperandKind::StoreBorrow:
     return BorrowedValue();
@@ -810,6 +820,7 @@ SILValue BorrowingOperand::getScopeIntroducingUserResult() {
 
   case BorrowingOperandKind::BeginAsyncLet:
   case BorrowingOperandKind::PartialApplyStack:
+  case BorrowingOperandKind::MarkDependenceNonEscaping:
   case BorrowingOperandKind::BeginBorrow:
   case BorrowingOperandKind::StoreBorrow:
     return cast<SingleValueInstruction>(op->getUser());