Merge pull request #66201 from nate-chandler/dealloc-on-stack-packs

[IRGen] Dealloc on-stack metadata/wtable packs on the dominance frontier.

Author: swift-ci

docs/SIL.rst

@@ -3797,7 +3797,21 @@ address of the allocated memory.
 
 ``alloc_pack`` is a stack allocation instruction.  See the section above
 on stack discipline.  The corresponding stack deallocation instruction is
-``dealloc_stack``.
+``dealloc_pack``.
+
+alloc_pack_metadata
+```````````````````
+
+::
+
+  sil-instruction ::= 'alloc_pack_metadata' $()
+
+Inserted as the last SIL lowering pass of IRGen, indicates that the next instruction
+may have on-stack pack metadata allocated on its behalf.
+
+Notionally, ``alloc_pack_metadata`` is a stack allocation instruction.  See the
+section above on stack discipline.  The corresponding stack deallocation
+instruction is ``dealloc_pack_metadata``.
 
 alloc_ref
 `````````
@@ -4056,6 +4070,23 @@ prior to being deallocated.
 on Stack Discipline above.  The operand must be an ``alloc_pack``
 instruction.
 
+dealloc_pack_metadata
+`````````````````````
+
+::
+
+  sil-instruction ::= 'dealloc_pack_metadata' sil-operand
+
+  dealloc_pack_metadata $0 : $*()
+
+Inserted as the last SIL lowering pass of IRGen, indicates that the on-stack
+pack metadata emitted on behalf of its operand (actually on behalf of the
+instruction after its operand) must be cleaned up here.
+
+``dealloc_pack_metadata`` is a stack deallocation instruction.  See the section
+on Stack Discipline above.  The operand must be an ``alloc_pack_metadata``
+instruction.
+
 dealloc_box
 ```````````
 ::

include/swift/AST/SILOptions.h

@@ -173,6 +173,12 @@ class SILOptions {
   /// Require linear OSSA lifetimes after SILGen
   bool OSSACompleteLifetimes = false;
 
+  /// Enable pack metadata stack "promotion".
+  ///
+  /// More accurately, enable skipping mandatory heapification of pack metadata
+  /// when possible.
+  bool EnablePackMetadataStackPromotion = true;
+
   // The kind of function bodies to skip emitting.
   FunctionBodySkipping SkipFunctionBodies = FunctionBodySkipping::None;
 

include/swift/AST/Types.h

@@ -179,7 +179,10 @@ class RecursiveTypeProperties {
     /// they have a type variable originator.
     SolverAllocated = 0x8000,
 
-    Last_Property = SolverAllocated
+    /// This type contains a concrete pack.
+    HasConcretePack = 0x10000,
+
+    Last_Property = HasConcretePack
   };
   enum { BitWidth = countBitsUsed(Property::Last_Property) };
 
@@ -255,6 +258,8 @@ class RecursiveTypeProperties {
 
   bool hasParameterPack() const { return Bits & HasParameterPack; }
 
+  bool hasConcretePack() const { return Bits & HasConcretePack; }
+
   /// Does a type with these properties structurally contain a
   /// parameterized existential type?
   bool hasParameterizedExistential() const {
@@ -412,12 +417,12 @@ class alignas(1 << TypeAlignInBits) TypeBase
     NumProtocols : 16
   );
 
-  SWIFT_INLINE_BITFIELD_FULL(TypeVariableType, TypeBase, 7+31,
+  SWIFT_INLINE_BITFIELD_FULL(TypeVariableType, TypeBase, 7+30,
     /// Type variable options.
     Options : 7,
     : NumPadBits,
     /// The unique number assigned to this type variable.
-    ID : 31
+    ID : 30
   );
 
   SWIFT_INLINE_BITFIELD(SILFunctionType, TypeBase, NumSILExtInfoBits+1+4+1+2+1+1,
@@ -684,6 +689,13 @@ class alignas(1 << TypeAlignInBits) TypeBase
     return getRecursiveProperties().hasParameterPack();
   }
 
+  bool hasConcretePack() const {
+    return getRecursiveProperties().hasConcretePack();
+  }
+
+  /// Whether the type has some flavor of pack.
+  bool hasPack() const { return hasParameterPack() || hasConcretePack(); }
+
   /// Determine whether the type involves a parameterized existential type.
   bool hasParameterizedExistential() const {
     return getRecursiveProperties().hasParameterizedExistential();

include/swift/IRGen/IRGenSILPasses.h

@@ -19,6 +19,7 @@ namespace irgen {
 /// Create a pass to hoist alloc_stack instructions with non-fixed size.
 SILTransform *createAllocStackHoisting();
 SILTransform *createLoadableByAddress();
+SILTransform *createPackMetadataMarkerInserter();
 
 } // end namespace irgen
 } // end namespace swift

include/swift/Option/FrontendOptions.td

@@ -1028,6 +1028,16 @@ def force_struct_type_layouts : Flag<["-"], "force-struct-type-layouts">,
 def enable_layout_string_value_witnesses : Flag<["-"], "enable-layout-string-value-witnesses">,
   HelpText<"Enable layout string based value witnesses">;
 
+let Flags = [FrontendOption, NoDriverOption, HelpHidden, ModuleInterfaceOptionIgnorable] in {
+  def enable_pack_metadata_stack_promotion :
+    Joined<["-"], "enable-pack-metadata-stack-promotion=">,
+    HelpText<"Whether to skip heapifying stack metadata packs when possible.">,
+    MetaVarName<"true|false">;
+  def enable_pack_metadata_stack_promotion_noArg :
+    Flag<["-"], "enable-pack-metadata-stack-promotion">,
+    HelpText<"Skip heapifying stack metadata packs when possible.">;
+}
+
 def disable_layout_string_value_witnesses : Flag<["-"], "disable-layout-string-value-witnesses">,
   HelpText<"Disable layout string based value witnesses">;
 

include/swift/SIL/Dominance.h

@@ -94,7 +94,7 @@ class DominanceInfo : public DominatorTreeBase {
 
 /// Compute a single block's dominance frontier.
 ///
-/// Precondition: no critical edges (OSSA)
+/// Precondition: no critical edges
 ///
 /// Postcondition: each block in \p boundary is dominated by \p root and either
 /// exits the function or has a single successor which has a predecessor that is

include/swift/SIL/SILBuilder.h

@@ -436,6 +436,15 @@ class SILBuilder {
     return insert(AllocPackInst::create(getSILDebugLocation(loc), packType,
                                         getFunction()));
   }
+  AllocPackMetadataInst *
+  createAllocPackMetadata(SILLocation loc,
+                          Optional<SILType> elementType = llvm::None) {
+    return insert(new (getModule()) AllocPackMetadataInst(
+        getSILDebugLocation(loc),
+        elementType.value_or(
+            SILType::getEmptyTupleType(getModule().getASTContext())
+                .getAddressType())));
+  }
 
   AllocRefInst *createAllocRef(SILLocation Loc, SILType ObjectType,
                                bool objc, bool canAllocOnStack,
@@ -2226,6 +2235,11 @@ class SILBuilder {
     return insert(new (getModule())
                       DeallocPackInst(getSILDebugLocation(loc), operand));
   }
+  DeallocPackMetadataInst *createDeallocPackMetadata(SILLocation loc,
+                                                     SILValue alloc) {
+    return insert(new (getModule())
+                      DeallocPackMetadataInst(getSILDebugLocation(loc), alloc));
+  }
   DeallocStackRefInst *createDeallocStackRef(SILLocation Loc,
                                                      SILValue operand) {
     return insert(new (getModule())

include/swift/SIL/SILCloner.h

@@ -892,6 +892,14 @@ SILCloner<ImplClass>::visitAllocStackInst(AllocStackInst *Inst) {
   recordClonedInstruction(Inst, NewInst);
 }
 
+template <typename ImplClass>
+void SILCloner<ImplClass>::visitAllocPackMetadataInst(
+    AllocPackMetadataInst *Inst) {
+  getBuilder().setCurrentDebugScope(getOpScope(Inst->getDebugScope()));
+  recordClonedInstruction(Inst, getBuilder().createAllocPackMetadata(
+                                    getOpLocation(Inst->getLoc())));
+}
+
 template<typename ImplClass>
 void
 SILCloner<ImplClass>::visitAllocPackInst(AllocPackInst *Inst) {
@@ -2854,6 +2862,15 @@ SILCloner<ImplClass>::visitDeallocPackInst(DeallocPackInst *Inst) {
                                            getOpValue(Inst->getOperand())));
 }
 
+template <typename ImplClass>
+void SILCloner<ImplClass>::visitDeallocPackMetadataInst(
+    DeallocPackMetadataInst *Inst) {
+  getBuilder().setCurrentDebugScope(getOpScope(Inst->getDebugScope()));
+  recordClonedInstruction(
+      Inst, getBuilder().createDeallocPackMetadata(
+                getOpLocation(Inst->getLoc()), Inst->getOperand()));
+}
+
 template<typename ImplClass>
 void
 SILCloner<ImplClass>::visitDeallocRefInst(DeallocRefInst *Inst) {

include/swift/SIL/SILFunction.h

@@ -76,6 +76,11 @@ enum ForceEnableLexicalLifetimes_t {
   DoForceEnableLexicalLifetimes
 };
 
+enum UseStackForPackMetadata_t {
+  DoNotUseStackForPackMetadata,
+  DoUseStackForPackMetadata,
+};
+
 enum class PerformanceConstraints : uint8_t {
   None = 0,
   NoAllocation = 1,
@@ -418,6 +423,10 @@ class SILFunction
   /// If true, the function has lexical lifetimes even if the module does not.
   unsigned ForceEnableLexicalLifetimes : 1;
 
+  /// If true, the function contains an instruction that prevents stack nesting
+  /// from running with pack metadata markers in place.
+  unsigned UseStackForPackMetadata : 1;
+
   static void
   validateSubclassScope(SubclassScope scope, IsThunk_t isThunk,
                         const GenericSpecializationInformation *genericInfo) {
@@ -693,6 +702,14 @@ class SILFunction
     ForceEnableLexicalLifetimes = value;
   }
 
+  UseStackForPackMetadata_t useStackForPackMetadata() const {
+    return UseStackForPackMetadata_t(UseStackForPackMetadata);
+  }
+
+  void setUseStackForPackMetadata(UseStackForPackMetadata_t value) {
+    UseStackForPackMetadata = value;
+  }
+
   /// Returns true if this is a reabstraction thunk of escaping function type
   /// whose single argument is a potentially non-escaping closure. i.e. the
   /// thunks' function argument may itself have @inout_aliasable parameters.

include/swift/SIL/SILInstruction.h

@@ -821,6 +821,10 @@ class SILInstruction : public llvm::ilist_node<SILInstruction> {
   /// The first operand must be the allocating instruction.
   bool isDeallocatingStack() const;
 
+  /// Whether IRGen lowering of this instruction may result in emitting packs of
+  /// metadata or witness tables.
+  bool mayRequirePackMetadata() const;
+
   /// Create a new copy of this instruction, which retains all of the operands
   /// and other information of this one.  If an insertion point is specified,
   /// then the new instruction is inserted before the specified point, otherwise
@@ -2275,6 +2279,27 @@ class AllocPackInst final
   }
 };
 
+/// AllocPackMetadataInst - Marker instruction indicating that the next
+///                         instruction might allocate on-stack pack metadata
+///                         during IRGen.
+///
+/// Only valid in lowered SIL.
+class AllocPackMetadataInst final
+    : public NullaryInstructionWithTypeDependentOperandsBase<
+          SILInstructionKind::AllocPackMetadataInst, AllocPackMetadataInst,
+          AllocationInst> {
+  friend SILBuilder;
+
+  AllocPackMetadataInst(SILDebugLocation loc, SILType elementType)
+      : NullaryInstructionWithTypeDependentOperandsBase(
+            loc, {}, elementType.getAddressType()) {}
+
+public:
+  /// The instruction which may trigger on-stack pack metadata when IRGen
+  /// lowering.
+  SILInstruction *getIntroducer() { return getNextInstruction(); }
+};
+
 /// The base class for AllocRefInst and AllocRefDynamicInst.
 ///
 /// The first NumTailTypes operands are counts for the tail allocated
@@ -8685,6 +8710,27 @@ class DeallocPackInst :
       : UnaryInstructionBase(debugLoc, operand) {}
 };
 
+/// DeallocPackMetadataInst - Deallocate stack memory allocated on behalf of the
+///                           operand by IRGen.
+///
+/// Only valid in lowered SIL.
+class DeallocPackMetadataInst final
+    : public UnaryInstructionBase<SILInstructionKind::DeallocPackMetadataInst,
+                                  DeallocationInst> {
+  friend SILBuilder;
+
+  DeallocPackMetadataInst(SILDebugLocation debugLoc, SILValue alloc)
+      : UnaryInstructionBase(debugLoc, alloc) {}
+
+public:
+  AllocPackMetadataInst *getAllocation() {
+    return cast<AllocPackMetadataInst>(getOperand().getDefiningInstruction());
+  }
+  /// The instruction which may trigger on-stack pack metadata when IRGen
+  /// lowering.
+  SILInstruction *getIntroducer() { return getAllocation()->getIntroducer(); }
+};
+
 /// Like DeallocStackInst, but for `alloc_ref [stack]`.
 class DeallocStackRefInst
     : public UnaryInstructionBase<SILInstructionKind::DeallocStackRefInst,

include/swift/SIL/SILNodes.def

@@ -310,6 +310,8 @@ ABSTRACT_VALUE_AND_INST(SingleValueInstruction, ValueBase, SILInstruction)
                               AllocationInst, None, DoesNotRelease)
     SINGLE_VALUE_INST(AllocPackInst, alloc_pack,
                               AllocationInst, None, DoesNotRelease)
+    SINGLE_VALUE_INST(AllocPackMetadataInst, alloc_pack_metadata,
+                      AllocationInst, None, DoesNotRelease)
     SINGLE_VALUE_INST(AllocRefInst, alloc_ref,
                               AllocationInst, None, DoesNotRelease)
     SINGLE_VALUE_INST(AllocRefDynamicInst, alloc_ref_dynamic,
@@ -727,6 +729,8 @@ ABSTRACT_INST(DeallocationInst, SILInstruction)
                  DeallocationInst, MayHaveSideEffects, DoesNotRelease)
   NON_VALUE_INST(DeallocPackInst, dealloc_pack,
                  DeallocationInst, MayHaveSideEffects, DoesNotRelease)
+  NON_VALUE_INST(DeallocPackMetadataInst, dealloc_pack_metadata,
+                 DeallocationInst, MayHaveSideEffects, DoesNotRelease)
   NON_VALUE_INST(DeallocStackRefInst, dealloc_stack_ref,
                  DeallocationInst, MayHaveSideEffects, DoesNotRelease)
   NON_VALUE_INST(DeallocRefInst, dealloc_ref,

include/swift/SIL/SILType.h

@@ -373,6 +373,16 @@ class SILType {
   /// An efficient implementation of `!isTrivial() && isOrContainsRawPointer()`.
   bool isNonTrivialOrContainsRawPointer(const SILFunction *f) const;
 
+  /// Whether the type contains a generic parameter declared as a parameter
+  /// pack.
+  bool hasParameterPack() const { return getASTType()->hasParameterPack(); }
+
+  /// Whether the type contains a concrete pack.
+  bool hasConcretePack() const { return getASTType()->hasConcretePack(); }
+
+  /// Whether the type contains some flavor of pack.
+  bool hasPack() const { return getASTType()->hasPack(); }
+
   /// True if the type is an empty tuple or an empty struct or a tuple or
   /// struct containing only empty types.
   bool isEmpty(const SILFunction &F) const;

include/swift/SILOptimizer/PassManager/Passes.def

@@ -353,6 +353,8 @@ IRGEN_PASS(LoadableByAddress, "loadable-address",
      "SIL Large Loadable type by-address lowering.")
 PASS(MandatorySILLinker, "mandatory-linker",
      "Deserialize all referenced SIL functions that are shared or transparent")
+IRGEN_PASS(PackMetadataMarkerInserter, "pack-metadata-marker-inserter",
+     "Insert markers where pack metadata might be de/allocated.")
 PASS(PerformanceSILLinker, "performance-linker",
      "Deserialize all referenced SIL functions")
 PASS(RawSILInstLowering, "raw-sil-inst-lowering",

include/swift/SILOptimizer/Utils/StackNesting.h

@@ -88,10 +88,10 @@ class StackNesting {
   ///
   /// Each stack location is allocated by a single allocation instruction.
   struct StackLoc {
-    StackLoc(SingleValueInstruction *Alloc) : Alloc(Alloc) { }
+    StackLoc(SILInstruction *Alloc) : Alloc(Alloc) {}
 
     /// Back-link to the allocation instruction.
-    SingleValueInstruction *Alloc;
+    SILInstruction *Alloc;
 
     /// Bit-set which represents all alive locations at this allocation.
     /// It obviously includes this location itself. And it includes all "outer"
@@ -100,7 +100,7 @@ class StackNesting {
   };
 
   /// Mapping from stack allocations (= locations) to bit numbers.
-  llvm::DenseMap<SingleValueInstruction *, unsigned> StackLoc2BitNumbers;
+  llvm::DenseMap<SILInstruction *, unsigned> StackLoc2BitNumbers;
 
   /// The list of stack locations. The index into this array is also the bit
   /// number in the bit-sets.
@@ -147,16 +147,22 @@ class StackNesting {
   /// Returns the location bit number for a stack allocation instruction.
   int bitNumberForAlloc(SILInstruction *AllocInst) {
     assert(AllocInst->isAllocatingStack());
-    return StackLoc2BitNumbers[cast<SingleValueInstruction>(AllocInst)];
+    return StackLoc2BitNumbers[AllocInst];
   }
 
   /// Returns the location bit number for a stack deallocation instruction.
   int bitNumberForDealloc(SILInstruction *DeallocInst) {
     assert(DeallocInst->isDeallocatingStack());
-    auto *AllocInst = cast<SingleValueInstruction>(DeallocInst->getOperand(0));
+    auto *AllocInst = getAllocForDealloc(DeallocInst);
     return bitNumberForAlloc(AllocInst);
   }
 
+  /// Returns the stack allocation instruction for a stack deallocation
+  /// instruction.
+  SILInstruction *getAllocForDealloc(SILInstruction *Dealloc) const {
+    return Dealloc->getOperand(0)->getDefiningInstruction();
+  }
+
   /// Insert deallocations at block boundaries.
   Changes insertDeallocsAtBlockBoundaries();