Merge pull request #81649 from eeckstein/global-inlinearray-initialization

Allow more complex InlineArray initializations to end up in a statically initialized global

Author: eeckstein

SwiftCompilerSources/Sources/Optimizer/FunctionPasses/InitializeStaticGlobals.swift

@@ -376,3 +376,75 @@ private extension Instruction {
     return shiftValue > 0
   }
 }
+
+/// Analyses the global initializer function and returns the `alloc_global` and `store`
+/// instructions which initialize the global.
+/// Returns nil if `function` has any side-effects beside initializing the global.
+///
+/// The function's single basic block must contain following code pattern:
+/// ```
+///   alloc_global @the_global
+///   %a = global_addr @the_global
+///   %i = some_const_initializer_insts
+///   store %i to %a
+/// ```
+///
+/// For all other instructions `handleUnknownInstruction` is called and such an instruction
+/// is accepted if `handleUnknownInstruction` returns true.
+private func getGlobalInitialization(
+  of function: Function,
+  _ context: some Context,
+  handleUnknownInstruction: (Instruction) -> Bool
+) -> (allocInst: AllocGlobalInst, storeToGlobal: StoreInst)? {
+  guard let block = function.blocks.singleElement else {
+    return nil
+  }
+
+  var allocInst: AllocGlobalInst? = nil
+  var globalAddr: GlobalAddrInst? = nil
+  var store: StoreInst? = nil
+
+  for inst in block.instructions {
+    switch inst {
+    case is ReturnInst,
+         is DebugValueInst,
+         is DebugStepInst,
+         is BeginAccessInst,
+         is EndAccessInst:
+      continue
+    case let agi as AllocGlobalInst:
+      if allocInst == nil {
+        allocInst = agi
+        continue
+      }
+    case let ga as GlobalAddrInst:
+      if let agi = allocInst, agi.global == ga.global {
+        globalAddr = ga
+      }
+      continue
+    case let si as StoreInst:
+      if store == nil,
+         let ga = globalAddr,
+         si.destination == ga
+      {
+        store = si
+        continue
+      }
+    // Note that the initializer must not contain a `global_value` because `global_value` needs to
+    // initialize the class metadata at runtime.
+    default:
+      if inst.isValidInStaticInitializerOfGlobal(context) {
+        continue
+      }
+    }
+    if handleUnknownInstruction(inst) {
+      continue
+    }
+    return nil
+  }
+  if let store = store {
+    return (allocInst: allocInst!, storeToGlobal: store)
+  }
+  return nil
+}
+

SwiftCompilerSources/Sources/Optimizer/InstructionSimplification/SimplifyLoad.swift

@@ -293,13 +293,6 @@ private extension Value {
       return self
     }
   }
-
-  var lookThroughTruncOrBitCast: Value {
-    if let truncOrBitCast = self as? BuiltinInst, truncOrBitCast.id == .TruncOrBitCast {
-      return truncOrBitCast.arguments[0]
-    }
-    return self
-  }
 }
 
 private extension BuiltinInst {

SwiftCompilerSources/Sources/Optimizer/InstructionSimplification/SimplifyPointerToAddress.swift

@@ -159,7 +159,7 @@ extension AddressUseVisitor {
            .GenericFDiv, .GenericMul, .GenericFMul, .GenericSDiv,
            .GenericExactSDiv, .GenericShl, .GenericSRem, .GenericSub,
            .GenericFSub, .GenericUDiv, .GenericExactUDiv, .GenericURem,
-           .GenericFRem, .GenericXor, .TaskRunInline, .ZeroInitializer,
+           .GenericFRem, .GenericXor, .TaskRunInline, .ZeroInitializer, .PrepareInitialization,
            .GetEnumTag, .InjectEnumTag:
         return leafAddressUse(of: operand)
       default:

SwiftCompilerSources/Sources/Optimizer/Utilities/AddressUtils.swift

@@ -45,6 +45,13 @@ extension Value {
     }
   }
 
+  var lookThroughTruncOrBitCast: Value {
+    if let truncOrBitCast = self as? BuiltinInst, truncOrBitCast.id == .TruncOrBitCast {
+      return truncOrBitCast.arguments[0]
+    }
+    return self
+  }
+
   func isInLexicalLiverange(_ context: some Context) -> Bool {
     var worklist = ValueWorklist(context)
     defer { worklist.deinitialize() }
@@ -846,77 +853,6 @@ extension InstructionRange {
   }
 }
 
-/// Analyses the global initializer function and returns the `alloc_global` and `store`
-/// instructions which initialize the global.
-/// Returns nil if `function` has any side-effects beside initializing the global.
-///
-/// The function's single basic block must contain following code pattern:
-/// ```
-///   alloc_global @the_global
-///   %a = global_addr @the_global
-///   %i = some_const_initializer_insts
-///   store %i to %a
-/// ```
-/// 
-/// For all other instructions `handleUnknownInstruction` is called and such an instruction
-/// is accepted if `handleUnknownInstruction` returns true.
-func getGlobalInitialization(
-  of function: Function,
-  _ context: some Context,
-  handleUnknownInstruction: (Instruction) -> Bool
-) -> (allocInst: AllocGlobalInst, storeToGlobal: StoreInst)? {
-  guard let block = function.blocks.singleElement else {
-    return nil
-  }
-
-  var allocInst: AllocGlobalInst? = nil
-  var globalAddr: GlobalAddrInst? = nil
-  var store: StoreInst? = nil
-
-  for inst in block.instructions {
-    switch inst {
-    case is ReturnInst,
-         is DebugValueInst,
-         is DebugStepInst,
-         is BeginAccessInst,
-         is EndAccessInst:
-      continue
-    case let agi as AllocGlobalInst:
-      if allocInst == nil {
-        allocInst = agi
-        continue
-      }
-    case let ga as GlobalAddrInst:
-      if let agi = allocInst, agi.global == ga.global {
-        globalAddr = ga
-      }
-      continue
-    case let si as StoreInst:
-      if store == nil,
-         let ga = globalAddr,
-         si.destination == ga
-      {
-        store = si
-        continue
-      }
-    // Note that the initializer must not contain a `global_value` because `global_value` needs to
-    // initialize the class metadata at runtime.
-    default:
-      if inst.isValidInStaticInitializerOfGlobal(context) {
-        continue
-      }
-    }
-    if handleUnknownInstruction(inst) {
-      continue
-    }
-    return nil
-  }
-  if let store = store {
-    return (allocInst: allocInst!, storeToGlobal: store)
-  }
-  return nil
-}
-
 func canDynamicallyCast(from sourceType: CanonicalType, to destType: CanonicalType,
                         in function: Function, sourceTypeIsExact: Bool
 ) -> Bool? {

SwiftCompilerSources/Sources/Optimizer/Utilities/OptUtils.swift

@@ -29,6 +29,8 @@ final public class GlobalVariable : CustomStringConvertible, HasShortDescription
 
   public var shortDescription: String { name.string }
 
+  public var type: Type { Type(bridged: bridged.getType()) }
+
   public var isLet: Bool { bridged.isLet() }
 
   public var linkage: Linkage { bridged.getLinkage().linkage }

SwiftCompilerSources/Sources/SIL/GlobalVariable.swift

@@ -851,6 +851,10 @@ BUILTIN_MISC_OPERATION_WITH_SILGEN(Alignof, "alignof", "n", Special)
 /// own rules.
 BUILTIN_MISC_OPERATION_WITH_SILGEN(ZeroInitializer, "zeroInitializer", "n", Special)
 
+/// Like `zeroInitializer`, but does not actually initialize the memory.
+/// It only indicates to mandatory passes that the memory is going to be initialized.
+BUILTIN_MISC_OPERATION(PrepareInitialization, "prepareInitialization", "n", Special)
+
 // getCurrentExecutor: () async -> Builtin.Executor?
 //
 // Retrieve the SerialExecutorRef on which the current asynchronous

include/swift/AST/Builtins.def

@@ -288,6 +288,7 @@ TransitiveAddressWalker<Impl>::walk(SILValue projectedAddress) {
         case BuiltinValueKind::GenericXor:
         case BuiltinValueKind::TaskRunInline:
         case BuiltinValueKind::ZeroInitializer:
+        case BuiltinValueKind::PrepareInitialization:
         case BuiltinValueKind::GetEnumTag:
         case BuiltinValueKind::InjectEnumTag:
         case BuiltinValueKind::AddressOfRawLayout:

include/swift/SIL/AddressWalker.h

@@ -587,6 +587,7 @@ struct BridgedGlobalVar {
   SWIFT_IMPORT_UNSAFE BRIDGED_INLINE BridgedStringRef getName() const;
   BRIDGED_INLINE bool isLet() const;
   BRIDGED_INLINE void setLet(bool value) const;
+  SWIFT_IMPORT_UNSAFE BRIDGED_INLINE BridgedType getType() const;
   BRIDGED_INLINE BridgedLinkage getLinkage() const;
   SWIFT_IMPORT_UNSAFE BRIDGED_INLINE BridgedSourceLoc getSourceLocation() const;
   BRIDGED_INLINE bool isPossiblyUsedExternally() const;

include/swift/SIL/SILBridging.h

@@ -918,6 +918,10 @@ bool BridgedGlobalVar::isLet() const { return getGlobal()->isLet(); }
 
 void BridgedGlobalVar::setLet(bool value) const { getGlobal()->setLet(value); }
 
+BridgedType BridgedGlobalVar::getType() const {
+  return getGlobal()->getLoweredType();
+}
+
 BridgedLinkage BridgedGlobalVar::getLinkage() const {
   return (BridgedLinkage)getGlobal()->getLinkage();
 }

include/swift/SIL/SILBridgingImpl.h

@@ -3132,6 +3132,7 @@ ValueDecl *swift::getBuiltinValueDecl(ASTContext &Context, Identifier Id) {
     return getUnreachableOperation(Context, Id);
 
   case BuiltinValueKind::ZeroInitializer:
+  case BuiltinValueKind::PrepareInitialization:
     return getZeroInitializerOperation(Context, Id);
 
   case BuiltinValueKind::Once:

lib/AST/Builtins.cpp

@@ -1353,7 +1353,13 @@ void irgen::emitBuiltinCall(IRGenFunction &IGF, const BuiltinInfo &Builtin,
     }
     return;
   }
-  
+
+  if (Builtin.ID == BuiltinValueKind::PrepareInitialization) {
+    ASSERT(args.size() > 0 && "only address-variant of prepareInitialization is supported");
+    (void)args.claimNext();
+    return;
+  }
+
   if (Builtin.ID == BuiltinValueKind::GetObjCTypeEncoding) {
     (void)args.claimAll();
     Type valueTy = substitutions.getReplacementTypes()[0];

lib/IRGen/GenBuiltin.cpp

@@ -917,6 +917,7 @@ BUILTIN_OPERAND_OWNERSHIP(InstantaneousUse, GenericXor)
 BUILTIN_OPERAND_OWNERSHIP(InstantaneousUse, ZExt)
 BUILTIN_OPERAND_OWNERSHIP(InstantaneousUse, ZExtOrBitCast)
 BUILTIN_OPERAND_OWNERSHIP(InstantaneousUse, ZeroInitializer)
+BUILTIN_OPERAND_OWNERSHIP(InstantaneousUse, PrepareInitialization)
 BUILTIN_OPERAND_OWNERSHIP(InstantaneousUse, PoundAssert)
 BUILTIN_OPERAND_OWNERSHIP(InstantaneousUse, GlobalStringTablePointer)
 BUILTIN_OPERAND_OWNERSHIP(InstantaneousUse, TypePtrAuthDiscriminator)

lib/SIL/IR/OperandOwnership.cpp

@@ -1009,7 +1009,8 @@ MemoryBehavior SILInstruction::getMemoryBehavior() const {
   if (auto *BI = dyn_cast<BuiltinInst>(this)) {
     // Handle Swift builtin functions.
     const BuiltinInfo &BInfo = BI->getBuiltinInfo();
-    if (BInfo.ID == BuiltinValueKind::ZeroInitializer) {
+    if (BInfo.ID == BuiltinValueKind::ZeroInitializer ||
+        BInfo.ID == BuiltinValueKind::PrepareInitialization) {
       // The address form of `zeroInitializer` writes to its argument to
       // initialize it. The value form has no side effects.
       return BI->getArguments().size() > 0

lib/SIL/IR/SILInstruction.cpp

@@ -694,6 +694,7 @@ UNOWNED_OR_NONE_DEPENDING_ON_RESULT(ShuffleVector)
 // fields. The initialized value is immediately consumed by an assignment, so it
 // must be owned.
 OWNED_OR_NONE_DEPENDING_ON_RESULT(ZeroInitializer)
+OWNED_OR_NONE_DEPENDING_ON_RESULT(PrepareInitialization)
 #undef OWNED_OR_NONE_DEPENDING_ON_RESULT
 
 #define BUILTIN(X,Y,Z)

lib/SIL/IR/ValueOwnership.cpp

@@ -2657,6 +2657,7 @@ static void visitBuiltinAddress(BuiltinInst *builtin,
 
     // zeroInitializer with an address operand zeroes the address.
     case BuiltinValueKind::ZeroInitializer:
+    case BuiltinValueKind::PrepareInitialization:
       if (builtin->getAllOperands().size() > 0) {
         visitor(&builtin->getAllOperands()[0]);
       }

lib/SIL/Utils/MemAccessUtils.cpp

@@ -2387,7 +2387,8 @@ class SILVerifier : public SILVerifierBase<SILVerifier> {
     auto builtinKind = BI->getBuiltinKind();
     auto arguments = BI->getArguments();
 
-    if (builtinKind == BuiltinValueKind::ZeroInitializer) {
+    if (builtinKind == BuiltinValueKind::ZeroInitializer ||
+        builtinKind == BuiltinValueKind::PrepareInitialization) {
       require(!BI->getSubstitutions(),
               "zeroInitializer has no generic arguments as a SIL builtin");
       if (arguments.size() == 0) {

lib/SIL/Verifier/SILVerifier.cpp

@@ -2165,11 +2165,13 @@ static ManagedValue emitBuiltinEmplace(SILGenFunction &SGF,
 
   auto buffer = dest->getAddressForInPlaceInitialization(SGF, loc);
 
-  // Zero-initialize the buffer.
-  // Aside from providing a modicum of predictability if the memory isn't
-  // actually initialized, this also serves to communicate to DI that the memory
+  // Mark the buffer as initializedto communicate to DI that the memory
   // is considered initialized from this point.
-  SGF.B.createZeroInitAddr(loc, buffer);
+  auto markInit = getBuiltinValueDecl(Ctx, Ctx.getIdentifier("prepareInitialization"));
+  SGF.B.createBuiltin(loc, markInit->getBaseIdentifier(),
+                       SILType::getEmptyTupleType(Ctx),
+                       SubstitutionMap(),
+                       buffer);
 
   SILValue bufferPtr = SGF.B.createAddressToPointer(loc, buffer,
         SILType::getPrimitiveObjectType(SGF.getASTContext().TheRawPointerType),

lib/SILGen/SILGenBuiltin.cpp

@@ -218,7 +218,8 @@ bool noncopyable::memInstMustInitialize(Operand *memOper) {
   }
   case SILInstructionKind::BuiltinInst: {
     auto bi = cast<BuiltinInst>(memInst);
-    if (bi->getBuiltinKind() == BuiltinValueKind::ZeroInitializer) {
+    if (bi->getBuiltinKind() == BuiltinValueKind::ZeroInitializer ||
+        bi->getBuiltinKind() == BuiltinValueKind::PrepareInitialization) {
       // `zeroInitializer` with an address operand zeroes out the address operand
       return true;
     }

lib/SILOptimizer/Mandatory/MoveOnlyUtils.cpp

@@ -135,6 +135,7 @@ static bool isBarrier(SILInstruction *inst) {
     case BuiltinValueKind::IsNegative:
     case BuiltinValueKind::WordAtIndex:
     case BuiltinValueKind::ZeroInitializer:
+    case BuiltinValueKind::PrepareInitialization:
     case BuiltinValueKind::Once:
     case BuiltinValueKind::OnceWithContext:
     case BuiltinValueKind::GetObjCTypeEncoding:

lib/SILOptimizer/Transforms/AccessEnforcementReleaseSinking.cpp

@@ -145,6 +145,10 @@ llvm::cl::opt<int> OSizeClassMethodBenefit(
     llvm::cl::desc("The benefit of inlining class methods with -Osize. We only "
                    "inline very small class methods with -Osize."));
 
+llvm::cl::opt<int> GlobalInitBenefit(
+    "sil-inline-global-init-benefit", llvm::cl::init(100),
+    llvm::cl::desc("The benefit of inlining constructors into global initializers."));
+
 llvm::cl::opt<int> TrivialFunctionThreshold(
     "sil-inline-trivial-function-threshold", llvm::cl::init(18),
     llvm::cl::desc("Approximately up to this cost level a function can be "
@@ -445,6 +449,14 @@ bool isFunctionAutodiffVJP(SILFunction *callee) {
   return false;
 }
 
+bool isAllocator(SILFunction *callee) {
+  swift::Demangle::Context Ctx;
+  if (auto *Root = Ctx.demangleSymbolAsNode(callee->getName())) {
+    return Root->findByKind(swift::Demangle::Node::Kind::Allocator, 3) != nullptr;
+  }
+  return false;
+}
+
 bool isProfitableToInlineAutodiffVJP(SILFunction *vjp, SILFunction *caller,
                                      InlineSelection whatToInline,
                                      StringRef stageName) {
@@ -785,6 +797,12 @@ bool SILPerformanceInliner::isProfitableToInline(
     Benefit = std::max(Benefit, ExclusivityBenefitWeight);
   }
 
+  if (AI.getFunction()->isGlobalInitOnceFunction() && isAllocator(Callee)) {
+    // Inlining constructors into global initializers increase the changes that
+    // the global can be initialized statically.
+    CallerWeight.updateBenefit(Benefit, GlobalInitBenefit);
+  }
+
   if (AI.getFunction()->isThunk()) {
     // Only inline trivial functions into thunks (which will not increase the
     // code size).