Code Size: Pass large loadable types by address instead of by value - Updated Version

docs/ABI.rst

@@ -794,6 +794,8 @@ Globals
 
   global ::= type 'Wy' // Outlined Copy Function Type
   global ::= type 'We' // Outlined Consume Function Type
+  global ::= type 'Wr' // Outlined Retain Function Type
+  global ::= type 'Ws' // Outlined Release Function Type
 
   DIRECTNESS ::= 'd'                         // direct
   DIRECTNESS ::= 'i'                         // indirect
@@ -1101,6 +1103,7 @@ mangled in to disambiguate.
   FUNC-REPRESENTATION ::= 'W'                // protocol witness
 
   PARAM-CONVENTION ::= 'i'                   // indirect in
+  PARAM-CONVENTION ::= 'c'                   // indirect in constant
   PARAM-CONVENTION ::= 'l'                   // indirect inout
   PARAM-CONVENTION ::= 'b'                   // indirect inout aliasable
   PARAM-CONVENTION ::= 'n'                   // indirect in guaranteed

docs/SIL.rst

@@ -479,6 +479,9 @@ number of ways:
     initialized object; both the caller and callee promise not to mutate the
     pointee, allowing the callee to read it.
 
+  - An ``@in_constant`` parameter is indirect.  The address must be of an
+    initialized object; the function will treat the value held there as read-only.
+
   - Otherwise, the parameter is an unowned direct parameter.
 
 - A SIL function type declares the conventions for its results.
@@ -3296,6 +3299,18 @@ For aggregate types, especially enums, it is typically both easier
 and more efficient to reason about aggregate copies than it is to
 reason about copies of the subobjects.
 
+retain_value_addr
+`````````````````
+
+::
+
+  sil-instruction ::= 'retain_value_addr' sil-operand
+
+  retain_value_addr %0 : $*A
+
+Retains a loadable value inside given address,
+which simply retains any references it holds.
+
 unmanaged_retain_value
 ``````````````````````
 
@@ -3362,6 +3377,18 @@ For aggregate types, especially enums, it is typically both easier
 and more efficient to reason about aggregate destroys than it is to
 reason about destroys of the subobjects.
 
+release_value_addr
+``````````````````
+
+::
+
+  sil-instruction ::= 'release_value_addr' sil-operand
+
+  release_value_addr %0 : $*A
+
+Destroys a loadable value inside given address,
+by releasing any retainable pointers within it.
+
 unmanaged_release_value
 ```````````````````````
 

include/swift/AST/Attr.def

@@ -50,6 +50,7 @@ TYPE_ATTR(in)
 TYPE_ATTR(inout)
 TYPE_ATTR(inout_aliasable)
 TYPE_ATTR(in_guaranteed)
+TYPE_ATTR(in_constant)
 TYPE_ATTR(owned)
 TYPE_ATTR(unowned_inner_pointer)
 TYPE_ATTR(guaranteed)

include/swift/AST/Types.h

@@ -2663,6 +2663,11 @@ enum class ParameterConvention {
   /// object.
   Indirect_In,
 
+  /// This argument is passed indirectly, i.e. by directly passing the address
+  /// of an object in memory.  The callee must treat the object as read-only
+  /// The callee may assume that the address does not alias any valid object.
+  Indirect_In_Constant,
+
   /// This argument is passed indirectly, i.e. by directly passing the address
   /// of an object in memory.  The callee may not modify and does not destroy
   /// the object.
@@ -2675,7 +2680,7 @@ enum class ParameterConvention {
   /// single-threaded aliasing may produce inconsistent results, but should
   /// remain memory safe.
   Indirect_Inout,
-  
+
   /// This argument is passed indirectly, i.e. by directly passing the address
   /// of an object in memory. The object is allowed to be aliased by other
   /// well-typed references, but is not allowed to be escaped. This is the
@@ -2705,6 +2710,7 @@ static_assert(unsigned(ParameterConvention::Direct_Guaranteed) < (1<<3),
 inline bool isIndirectFormalParameter(ParameterConvention conv) {
   switch (conv) {
   case ParameterConvention::Indirect_In:
+  case ParameterConvention::Indirect_In_Constant:
   case ParameterConvention::Indirect_Inout:
   case ParameterConvention::Indirect_InoutAliasable:
   case ParameterConvention::Indirect_In_Guaranteed:
@@ -2720,6 +2726,7 @@ inline bool isIndirectFormalParameter(ParameterConvention conv) {
 inline bool isConsumedParameter(ParameterConvention conv) {
   switch (conv) {
   case ParameterConvention::Indirect_In:
+  case ParameterConvention::Indirect_In_Constant:
   case ParameterConvention::Direct_Owned:
     return true;
 
@@ -2745,6 +2752,7 @@ inline bool isGuaranteedParameter(ParameterConvention conv) {
   case ParameterConvention::Indirect_Inout:
   case ParameterConvention::Indirect_InoutAliasable:
   case ParameterConvention::Indirect_In:
+  case ParameterConvention::Indirect_In_Constant:
   case ParameterConvention::Direct_Unowned:
   case ParameterConvention::Direct_Owned:
     return false;

include/swift/Demangling/DemangleNodes.def

@@ -181,5 +181,7 @@ NODE(FirstElementMarker)
 NODE(VariadicMarker)
 NODE(OutlinedCopy)
 NODE(OutlinedConsume)
+NODE(OutlinedRetain)
+NODE(OutlinedRelease)
 #undef CONTEXT_NODE
 #undef NODE

include/swift/IRGen/IRGenSILPasses.h

@@ -12,12 +12,13 @@
 
 namespace swift {
 
-class SILFunctionTransform;
+class SILTransform;
 
 namespace irgen {
 
 /// Create a pass to hoist alloc_stack instructions with non-fixed size.
-SILFunctionTransform *createAllocStackHoisting();
+SILTransform *createAllocStackHoisting();
+SILTransform *createLoadableByAddress();
 
 } // end namespace irgen
 } // end namespace swift

include/swift/SIL/PatternMatch.h

@@ -345,7 +345,9 @@ UNARY_OP_MATCH_WITH_ARG_MATCHER(ObjCMetatypeToObjectInst)
 UNARY_OP_MATCH_WITH_ARG_MATCHER(ObjCExistentialMetatypeToObjectInst)
 UNARY_OP_MATCH_WITH_ARG_MATCHER(IsNonnullInst)
 UNARY_OP_MATCH_WITH_ARG_MATCHER(RetainValueInst)
+UNARY_OP_MATCH_WITH_ARG_MATCHER(RetainValueAddrInst)
 UNARY_OP_MATCH_WITH_ARG_MATCHER(ReleaseValueInst)
+UNARY_OP_MATCH_WITH_ARG_MATCHER(ReleaseValueAddrInst)
 UNARY_OP_MATCH_WITH_ARG_MATCHER(AutoreleaseValueInst)
 UNARY_OP_MATCH_WITH_ARG_MATCHER(UncheckedEnumDataInst)
 UNARY_OP_MATCH_WITH_ARG_MATCHER(InitEnumDataAddrInst)

include/swift/SIL/SILArgumentConvention.h

@@ -37,6 +37,7 @@ enum class InoutAliasingAssumption {
 struct SILArgumentConvention {
   enum ConventionType : uint8_t {
     Indirect_In,
+    Indirect_In_Constant,
     Indirect_In_Guaranteed,
     Indirect_Inout,
     Indirect_InoutAliasable,
@@ -55,6 +56,9 @@ struct SILArgumentConvention {
     case ParameterConvention::Indirect_In:
       Value = SILArgumentConvention::Indirect_In;
       return;
+    case ParameterConvention::Indirect_In_Constant:
+      Value = SILArgumentConvention::Indirect_In_Constant;
+      return;
     case ParameterConvention::Indirect_Inout:
       Value = SILArgumentConvention::Indirect_Inout;
       return;
@@ -90,6 +94,7 @@ struct SILArgumentConvention {
   bool isNotAliasedIndirectParameter(InoutAliasingAssumption isInoutAliasing) {
     switch (Value) {
     case SILArgumentConvention::Indirect_In:
+    case SILArgumentConvention::Indirect_In_Constant:
     case SILArgumentConvention::Indirect_Out:
     case SILArgumentConvention::Indirect_In_Guaranteed:
       return true;

include/swift/SIL/SILBuilder.h

@@ -864,13 +864,28 @@ class SILBuilder {
                                                       operand, atomicity));
   }
 
+  RetainValueAddrInst *createRetainValueAddr(SILLocation Loc, SILValue operand,
+                                             Atomicity atomicity) {
+    assert(isParsing || F.hasUnqualifiedOwnership());
+    return insert(new (F.getModule()) RetainValueAddrInst(
+        getSILDebugLocation(Loc), operand, atomicity));
+  }
+
   ReleaseValueInst *createReleaseValue(SILLocation Loc, SILValue operand,
                                        Atomicity atomicity) {
     assert(isParsing || F.hasUnqualifiedOwnership());
     return insert(new (F.getModule()) ReleaseValueInst(getSILDebugLocation(Loc),
                                                        operand, atomicity));
   }
 
+  ReleaseValueAddrInst *createReleaseValueAddr(SILLocation Loc,
+                                               SILValue operand,
+                                               Atomicity atomicity) {
+    assert(isParsing || F.hasUnqualifiedOwnership());
+    return insert(new (F.getModule()) ReleaseValueAddrInst(
+        getSILDebugLocation(Loc), operand, atomicity));
+  }
+
   UnmanagedRetainValueInst *createUnmanagedRetainValue(SILLocation Loc,
                                                        SILValue operand,
                                                        Atomicity atomicity) {

include/swift/SIL/SILCloner.h

@@ -1232,6 +1232,15 @@ void SILCloner<ImplClass>::visitRetainValueInst(RetainValueInst *Inst) {
                                    Inst->getAtomicity()));
 }
 
+template <typename ImplClass>
+void SILCloner<ImplClass>::visitRetainValueAddrInst(RetainValueAddrInst *Inst) {
+  getBuilder().setCurrentDebugScope(getOpScope(Inst->getDebugScope()));
+  doPostProcess(
+      Inst, getBuilder().createRetainValueAddr(getOpLocation(Inst->getLoc()),
+                                               getOpValue(Inst->getOperand()),
+                                               Inst->getAtomicity()));
+}
+
 template <typename ImplClass>
 void SILCloner<ImplClass>::visitUnmanagedRetainValueInst(
     UnmanagedRetainValueInst *Inst) {
@@ -1268,6 +1277,16 @@ void SILCloner<ImplClass>::visitReleaseValueInst(ReleaseValueInst *Inst) {
                                     Inst->getAtomicity()));
 }
 
+template <typename ImplClass>
+void SILCloner<ImplClass>::visitReleaseValueAddrInst(
+    ReleaseValueAddrInst *Inst) {
+  getBuilder().setCurrentDebugScope(getOpScope(Inst->getDebugScope()));
+  doPostProcess(
+      Inst, getBuilder().createReleaseValueAddr(getOpLocation(Inst->getLoc()),
+                                                getOpValue(Inst->getOperand()),
+                                                Inst->getAtomicity()));
+}
+
 template <typename ImplClass>
 void SILCloner<ImplClass>::visitUnmanagedReleaseValueInst(
     UnmanagedReleaseValueInst *Inst) {

include/swift/SIL/SILFunction.h

@@ -239,9 +239,9 @@ class SILFunction
   /// or return instructions; you need to do that yourself
   /// if you care.
   ///
-  /// This is a hack and should be removed!
+  /// This routine does not update all the references in the module
+  /// You have to do that yourself
   void rewriteLoweredTypeUnsafe(CanSILFunctionType newType) {
-    assert(canBeDeleted());
     LoweredType = newType;
   }
 

include/swift/SIL/SILFunctionConventions.h

@@ -352,6 +352,7 @@ inline bool SILModuleConventions::isIndirectSILParam(SILParameterInfo param,
     return false;
 
   case ParameterConvention::Indirect_In:
+  case ParameterConvention::Indirect_In_Constant:
   case ParameterConvention::Indirect_In_Guaranteed:
     return (loweredAddresses ||
             param.getType()->isOpenedExistentialWithError());

include/swift/SIL/SILInstruction.h

@@ -3667,6 +3667,19 @@ class RetainValueInst : public UnaryInstructionBase<ValueKind::RetainValueInst,
   }
 };
 
+/// RetainValueAddrInst - Copies a loadable value by address.
+class RetainValueAddrInst
+    : public UnaryInstructionBase<ValueKind::RetainValueAddrInst,
+                                  RefCountingInst, /*HasValue*/ false> {
+  friend SILBuilder;
+
+  RetainValueAddrInst(SILDebugLocation DebugLoc, SILValue operand,
+                      Atomicity atomicity)
+      : UnaryInstructionBase(DebugLoc, operand) {
+    setAtomicity(atomicity);
+  }
+};
+
 /// ReleaseValueInst - Destroys a loadable value.
 class ReleaseValueInst : public UnaryInstructionBase<ValueKind::ReleaseValueInst,
                                                      RefCountingInst,
@@ -3680,6 +3693,19 @@ class ReleaseValueInst : public UnaryInstructionBase<ValueKind::ReleaseValueInst
   }
 };
 
+/// ReleaseValueInst - Destroys a loadable value by address.
+class ReleaseValueAddrInst
+    : public UnaryInstructionBase<ValueKind::ReleaseValueAddrInst,
+                                  RefCountingInst, /*HasValue*/ false> {
+  friend SILBuilder;
+
+  ReleaseValueAddrInst(SILDebugLocation DebugLoc, SILValue operand,
+                       Atomicity atomicity)
+      : UnaryInstructionBase(DebugLoc, operand) {
+    setAtomicity(atomicity);
+  }
+};
+
 /// Copies a loadable value in an unmanaged, unbalanced way. Only meant for use
 /// in ownership qualified SIL. Please do not use this EVER unless you are
 /// implementing a part of the stdlib called Unmanaged.

include/swift/SIL/SILNodes.def

@@ -152,7 +152,9 @@ ABSTRACT_VALUE(SILInstruction, ValueBase)
     INST(UnownedReleaseInst, RefCountingInst, unowned_release, MayHaveSideEffects,
          MayRelease)
     INST(RetainValueInst, RefCountingInst, retain_value, MayHaveSideEffects, DoesNotRelease)
+    INST(RetainValueAddrInst, RefCountingInst, retain_value_addr, MayHaveSideEffects, DoesNotRelease)
     INST(ReleaseValueInst, RefCountingInst, release_value, MayHaveSideEffects, MayRelease)
+    INST(ReleaseValueAddrInst, RefCountingInst, release_value_addr, MayHaveSideEffects, MayRelease)
     INST(SetDeallocatingInst, RefCountingInst, set_deallocating, MayHaveSideEffects,
          DoesNotRelease)
     INST(AutoreleaseValueInst, RefCountingInst, autorelease_value, MayHaveSideEffects,

include/swift/SILOptimizer/PassManager/PassManager.h

@@ -99,7 +99,7 @@ class SILPassManager {
 
 
   /// The IRGen SIL passes. These have to be dynamically added by IRGen.
-  llvm::DenseMap<unsigned, SILFunctionTransform *> IRGenPasses;
+  llvm::DenseMap<unsigned, SILTransform *> IRGenPasses;
 
 public:
   /// C'tor. It creates and registers all analysis passes, which are defined
@@ -247,7 +247,7 @@ class SILPassManager {
     }
   }
 
-  void registerIRGenPass(PassKind Kind, SILFunctionTransform *Transform) {
+  void registerIRGenPass(PassKind Kind, SILTransform *Transform) {
     assert(IRGenPasses.find(unsigned(Kind)) == IRGenPasses.end() &&
            "Pass already registered");
     assert(

include/swift/SILOptimizer/PassManager/Passes.def

@@ -222,6 +222,8 @@ PASS(ReleaseHoisting, "release-hoisting",
      "SIL release Hoisting")
 PASS(LateReleaseHoisting, "late-release-hoisting",
      "Late SIL release Hoisting Preserving Epilogues")
+IRGEN_PASS(LoadableByAddress, "loadable-address",
+           "SIL Loadable type by-address lowering.")
 PASS(RemovePins, "remove-pins",
      "Remove SIL pin/unpin pairs")
 PASS(SideEffectsDumper, "side-effects-dump",

include/swift/SILOptimizer/PassManager/Transforms.h

@@ -66,6 +66,12 @@ namespace swift {
     /// Inject the pass manager running this pass.
     void injectPassManager(SILPassManager *PMM) { PM = PMM; }
 
+    irgen::IRGenModule *getIRGenModule() {
+      auto *Mod = PM->getIRGenModule();
+      assert(Mod && "Expecting a valid module");
+      return Mod;
+    }
+
     /// Get the name of the transform.
     llvm::StringRef getName() { return PassKindName(getPassKind()); }
 
@@ -116,12 +122,6 @@ namespace swift {
   protected:
     SILFunction *getFunction() { return F; }
 
-    irgen::IRGenModule *getIRGenModule() {
-      auto *Mod = PM->getIRGenModule();
-      assert(Mod && "Expecting a valid module");
-      return Mod;
-    }
-
     void invalidateAnalysis(SILAnalysis::InvalidationKind K) {
       PM->invalidateAnalysis(F, K);
     }

include/swift/Serialization/ModuleFormat.h

@@ -54,7 +54,7 @@ const uint16_t VERSION_MAJOR = 0;
 /// in source control, you should also update the comment to briefly
 /// describe what change you made. The content of this comment isn't important;
 /// it just ensures a conflict if two people change the module format.
-const uint16_t VERSION_MINOR = 339; // Last change: member canonical types
+const uint16_t VERSION_MINOR = 341; // Last change: retain/release addr
 
 using DeclID = PointerEmbeddedInt<unsigned, 31>;
 using DeclIDField = BCFixed<31>;
@@ -199,6 +199,7 @@ enum class ParameterConvention : uint8_t {
   Direct_Unowned,
   Direct_Guaranteed,
   Indirect_In_Guaranteed,
+  Indirect_In_Constant,
 };
 using ParameterConventionField = BCFixed<4>;
 

lib/AST/ASTMangler.cpp

@@ -947,6 +947,8 @@ static char getParamConvention(ParameterConvention conv) {
   // different places.
   switch (conv) {
     case ParameterConvention::Indirect_In: return 'i';
+    case ParameterConvention::Indirect_In_Constant:
+      return 'c';
     case ParameterConvention::Indirect_Inout: return 'l';
     case ParameterConvention::Indirect_InoutAliasable: return 'b';
     case ParameterConvention::Indirect_In_Guaranteed: return 'n';