IRGen: re-enable generate static arrays in read-only data sections.

include/swift/AST/ASTContext.h

@@ -880,6 +880,10 @@ class ASTContext final {
   /// for extended existential types.
   AvailabilityContext getParameterizedExistentialRuntimeAvailability();
 
+  /// Get the runtime availability of immortal ref-count symbols, which are
+  /// needed to place array buffers into constant data sections.
+  AvailabilityContext getImmortalRefCountSymbolsAvailability();
+
   /// Get the runtime availability of features introduced in the Swift 5.2
   /// compiler for the target platform.
   AvailabilityContext getSwift52Availability();

include/swift/AST/IRGenOptions.h

@@ -402,6 +402,8 @@ class IRGenOptions {
   /// and protocol conformance caches.
   unsigned NoPreallocatedInstantiationCaches : 1;
 
+  unsigned DisableReadonlyStaticObjects : 1;
+
   /// The number of threads for multi-threaded code generation.
   unsigned NumThreads = 0;
 
@@ -470,7 +472,8 @@ class IRGenOptions {
         EnableGlobalISel(false), VirtualFunctionElimination(false),
         WitnessMethodElimination(false), ConditionalRuntimeRecords(false),
         InternalizeAtLink(false), InternalizeSymbols(false),
-        NoPreallocatedInstantiationCaches(false), CmdArgs(),
+        NoPreallocatedInstantiationCaches(false),
+        DisableReadonlyStaticObjects(false), CmdArgs(),
         SanitizeCoverage(llvm::SanitizerCoverageOptions()),
         TypeInfoFilter(TypeInfoDumpFilter::All) {
 #ifndef NDEBUG

include/swift/IRGen/Linking.h

@@ -364,6 +364,9 @@ class LinkEntity {
     /// A SIL global variable. The pointer is a SILGlobalVariable*.
     SILGlobalVariable,
 
+    /// An outlined read-only global object. The pointer is a SILGlobalVariable*.
+    ReadOnlyGlobalObject,
+
     // These next few are protocol-conformance kinds.
 
     /// A direct protocol witness table. The secondary pointer is a
@@ -993,13 +996,7 @@ class LinkEntity {
     return entity;
   }
 
-  static LinkEntity forSILGlobalVariable(SILGlobalVariable *G) {
-    LinkEntity entity;
-    entity.Pointer = G;
-    entity.SecondaryPointer = nullptr;
-    entity.Data = LINKENTITY_SET_FIELD(Kind, unsigned(Kind::SILGlobalVariable));
-    return entity;
-  }
+  static LinkEntity forSILGlobalVariable(SILGlobalVariable *G, IRGenModule &IGM);
 
   static LinkEntity
   forDifferentiabilityWitness(const SILDifferentiabilityWitness *witness) {
@@ -1420,7 +1417,8 @@ class LinkEntity {
   }
 
   SILGlobalVariable *getSILGlobalVariable() const {
-    assert(getKind() == Kind::SILGlobalVariable);
+    assert(getKind() == Kind::SILGlobalVariable ||
+           getKind() == Kind::ReadOnlyGlobalObject);
     return reinterpret_cast<SILGlobalVariable*>(Pointer);
   }
 

include/swift/Option/FrontendOptions.td

@@ -641,6 +641,10 @@ def disable_preallocated_instantiation_caches : Flag<["-"], "disable-preallocate
   Flags<[FrontendOption, NoInteractiveOption, HelpHidden]>,
   HelpText<"Avoid preallocating metadata instantiation caches in globals">;
 
+def disable_readonly_static_objects : Flag<["-"], "disable-readonly-static-objects">,
+  Flags<[FrontendOption, NoInteractiveOption, HelpHidden]>,
+  HelpText<"Avoid allocating static objects in a read-only data section">;
+
 def trap_function
   : Separate<["-"], "trap-function">, MetaVarName<"<name>">,
     HelpText<"Lower traps to calls to this function instead of trap instructions">;

lib/AST/Availability.cpp

@@ -399,6 +399,13 @@ ASTContext::getParameterizedExistentialRuntimeAvailability() {
   return getSwift57Availability();
 }
 
+AvailabilityContext
+ASTContext::getImmortalRefCountSymbolsAvailability() {
+  // TODO: replace this with a concrete swift version once we have it.
+  // rdar://94185998
+  return getSwiftFutureAvailability();
+}
+
 AvailabilityContext ASTContext::getSwift52Availability() {
   auto target = LangOpts.Target;
 

lib/Frontend/CompilerInvocation.cpp

@@ -2329,6 +2329,10 @@ static bool ParseIRGenArgs(IRGenOptions &Opts, ArgList &Args,
     Opts.NoPreallocatedInstantiationCaches = true;
   }
 
+  if (Args.hasArg(OPT_disable_readonly_static_objects)) {
+    Opts.DisableReadonlyStaticObjects = true;
+  }
+
   // Default to disabling swift async extended frame info on anything but
   // darwin. Other platforms are unlikely to have support for extended frame
   // pointer information.

lib/IRGen/GenConstant.cpp

@@ -278,9 +278,27 @@ llvm::Constant *irgen::emitConstantObject(IRGenModule &IGM, ObjectInst *OI,
     }
   }
   // Construct the object header.
-  llvm::Type *ObjectHeaderTy = sTy->getElementType(0);
-  assert(ObjectHeaderTy->isStructTy());
-  elts[0] = llvm::Constant::getNullValue(ObjectHeaderTy);
+  llvm::StructType *ObjectHeaderTy = cast<llvm::StructType>(sTy->getElementType(0));
+
+  if (IGM.canMakeStaticObjectsReadOnly()) {
+    if (!IGM.swiftImmortalRefCount) {
+      auto *var = new llvm::GlobalVariable(IGM.Module, IGM.Int8Ty,
+                                        /*constant*/ true, llvm::GlobalValue::ExternalLinkage,
+                                        /*initializer*/ nullptr, "_swiftImmortalRefCount");
+      IGM.swiftImmortalRefCount = var;
+    }
+    if (!IGM.swiftStaticArrayMetadata) {
+      auto *var = new llvm::GlobalVariable(IGM.Module, IGM.TypeMetadataStructTy,
+                                        /*constant*/ true, llvm::GlobalValue::ExternalLinkage,
+                                        /*initializer*/ nullptr, "_swiftStaticArrayMetadata");
+      IGM.swiftStaticArrayMetadata = var;
+    }
+    elts[0] = llvm::ConstantStruct::get(ObjectHeaderTy, {
+      IGM.swiftStaticArrayMetadata,
+      llvm::ConstantExpr::getPtrToInt(IGM.swiftImmortalRefCount, IGM.IntPtrTy)});
+  } else {
+    elts[0] = llvm::Constant::getNullValue(ObjectHeaderTy);
+  }
   insertPadding(elts, sTy);
   return llvm::ConstantStruct::get(sTy, elts);
 }

lib/IRGen/GenDecl.cpp

@@ -2482,7 +2482,7 @@ Address IRGenModule::getAddrOfSILGlobalVariable(SILGlobalVariable *var,
     return Address(addr, alignment);
   }
 
-  LinkEntity entity = LinkEntity::forSILGlobalVariable(var);
+  LinkEntity entity = LinkEntity::forSILGlobalVariable(var, *this);
   ResilienceExpansion expansion = getResilienceExpansionForLayout(var);
 
   llvm::Type *storageType;
@@ -2528,28 +2528,33 @@ Address IRGenModule::getAddrOfSILGlobalVariable(SILGlobalVariable *var,
 
   // Check whether we've created the global variable already.
   // FIXME: We should integrate this into the LinkEntity cache more cleanly.
-  auto gvar = Module.getGlobalVariable(var->getName(), /*allowInternal*/ true);
+  LinkInfo link = LinkInfo::get(*this, entity, forDefinition);
+  auto gvar = Module.getGlobalVariable(link.getName(), /*allowInternal*/ true);
   if (gvar) {
     if (forDefinition)
       updateLinkageForDefinition(*this, gvar, entity);
   } else {
-    LinkInfo link = LinkInfo::get(*this, entity, forDefinition);
     llvm::Type *storageTypeWithContainer = storageType;
     if (var->isInitializedObject()) {
-      // A statically initialized object must be placed into a container struct
-      // because the swift_initStaticObject needs a swift_once_t at offset -1:
-      //     struct Container {
-      //       swift_once_t token[fixedAlignment / sizeof(swift_once_t)];
-      //       HeapObject object;
-      //     };
-      std::string typeName = storageType->getStructName().str() + 'c';
-      assert(fixedAlignment >= getPointerAlignment());
-      unsigned numTokens = fixedAlignment.getValue() /
-        getPointerAlignment().getValue();
-      storageTypeWithContainer = llvm::StructType::create(getLLVMContext(),
-              {llvm::ArrayType::get(OnceTy, numTokens), storageType}, typeName);
-      gvar = createVariable(*this, link, storageTypeWithContainer,
-                            fixedAlignment);
+      if (canMakeStaticObjectsReadOnly()) {
+        gvar = createVariable(*this, link, storageType, fixedAlignment);
+        gvar->setConstant(true);
+      } else {
+        // A statically initialized object must be placed into a container struct
+        // because the swift_initStaticObject needs a swift_once_t at offset -1:
+        //     struct Container {
+        //       swift_once_t token[fixedAlignment / sizeof(swift_once_t)];
+        //       HeapObject object;
+        //     };
+        std::string typeName = storageType->getStructName().str() + 'c';
+        assert(fixedAlignment >= getPointerAlignment());
+        unsigned numTokens = fixedAlignment.getValue() /
+          getPointerAlignment().getValue();
+        storageTypeWithContainer = llvm::StructType::create(getLLVMContext(),
+                {llvm::ArrayType::get(OnceTy, numTokens), storageType}, typeName);
+        gvar = createVariable(*this, link, storageTypeWithContainer,
+                              fixedAlignment);
+      }
     } else {
       StringRef name;
       Optional<SILLocation> loc;
@@ -2574,7 +2579,7 @@ Address IRGenModule::getAddrOfSILGlobalVariable(SILGlobalVariable *var,
       gvar->setComdat(nullptr);
   }
   llvm::Constant *addr = gvar;
-  if (var->isInitializedObject()) {
+  if (var->isInitializedObject() && !canMakeStaticObjectsReadOnly()) {
     // Project out the object from the container.
     llvm::Constant *Indices[2] = {
       llvm::ConstantExpr::getIntegerValue(Int32Ty, APInt(32, 0)),

lib/IRGen/IRGenModule.cpp

@@ -1785,6 +1785,19 @@ bool IRGenModule::shouldPrespecializeGenericMetadata() {
          canPrespecializeTarget;
 }
 
+bool IRGenModule::canMakeStaticObjectsReadOnly() {
+  if (getOptions().DisableReadonlyStaticObjects)
+    return false;
+
+  // TODO: Support constant static arrays on other platforms, too.
+  // See also the comment in GlobalObjects.cpp.
+  if (!Triple.isOSDarwin())
+    return false;
+
+  return getAvailabilityContext().isContainedIn(
+          Context.getImmortalRefCountSymbolsAvailability());
+}
+
 void IRGenerator::addGenModule(SourceFile *SF, IRGenModule *IGM) {
   assert(GenModules.count(SF) == 0);
   GenModules[SF] = IGM;

lib/IRGen/IRGenModule.h

@@ -763,6 +763,9 @@ class IRGenModule {
 
   llvm::GlobalVariable *TheTrivialPropertyDescriptor = nullptr;
 
+  llvm::GlobalVariable *swiftImmortalRefCount = nullptr;
+  llvm::GlobalVariable *swiftStaticArrayMetadata = nullptr;
+
   /// Used to create unique names for class layout types with tail allocated
   /// elements.
   unsigned TailElemTypeID = 0;
@@ -886,6 +889,8 @@ class IRGenModule {
 
   bool shouldPrespecializeGenericMetadata();
 
+  bool canMakeStaticObjectsReadOnly();
+
   Size getAtomicBoolSize() const { return AtomicBoolSize; }
   Alignment getAtomicBoolAlignment() const { return AtomicBoolAlign; }
 

lib/IRGen/IRGenSIL.cpp

@@ -2765,7 +2765,7 @@ void IRGenSILFunction::visitGlobalValueInst(GlobalValueInst *i) {
                                                 NotForDefinition).getAddress();
   // We don't need to initialize the global object if it's never used for
   // something which can access the object header.
-  if (!hasOnlyProjections(i)) {
+  if (!hasOnlyProjections(i) && !IGM.canMakeStaticObjectsReadOnly()) {
     auto ClassType = loweredTy.getASTType();
     llvm::Value *Metadata =
       emitClassHeapMetadataRef(*this, ClassType, MetadataValueType::TypeMetadata,
@@ -7099,8 +7099,11 @@ void IRGenModule::emitSILStaticInitializers() {
            "cannot emit a static initializer for dynamically-sized global");
 #endif
 
+    LinkEntity entity = LinkEntity::forSILGlobalVariable(&Global, *this);
+    std::string name = entity.mangleAsString();
+
     auto *IRGlobal =
-        Module.getGlobalVariable(Global.getName(), true /* = AllowLocal */);
+        Module.getGlobalVariable(name, true /* = AllowLocal */);
 
     // A check for multi-threaded compilation: Is this the llvm module where the
     // global is defined and not only referenced (or not referenced at all).
@@ -7110,10 +7113,14 @@ void IRGenModule::emitSILStaticInitializers() {
     if (auto *OI = dyn_cast<ObjectInst>(InitValue)) {
       StructLayout *Layout = StaticObjectLayouts[&Global].get();
       llvm::Constant *InitVal = emitConstantObject(*this, OI, Layout);
-      auto *ContainerTy = cast<llvm::StructType>(IRGlobal->getValueType());
-      auto *zero = llvm::ConstantAggregateZero::get(ContainerTy->getElementType(0));
-      IRGlobal->setInitializer(llvm::ConstantStruct::get(ContainerTy,
-                                                         {zero , InitVal}));
+      if (canMakeStaticObjectsReadOnly()) {
+        IRGlobal->setInitializer(InitVal);
+      } else {
+        auto *ContainerTy = cast<llvm::StructType>(IRGlobal->getValueType());
+        auto *zero = llvm::ConstantAggregateZero::get(ContainerTy->getElementType(0));
+        IRGlobal->setInitializer(llvm::ConstantStruct::get(ContainerTy,
+                                                           {zero , InitVal}));
+      }
       continue;
     }
 

lib/IRGen/Linking.cpp

@@ -91,6 +91,18 @@ UniversalLinkageInfo::UniversalLinkageInfo(const llvm::Triple &triple,
       UseDLLStorage(useDllStorage(triple)), Internalize(isStaticLibrary),
       HasMultipleIGMs(hasMultipleIGMs), ForcePublicDecls(forcePublicDecls) {}
 
+LinkEntity LinkEntity::forSILGlobalVariable(SILGlobalVariable *G,
+                                            IRGenModule &IGM) {
+  LinkEntity entity;
+  entity.Pointer = G;
+  entity.SecondaryPointer = nullptr;
+  auto kind = (G->isInitializedObject() && IGM.canMakeStaticObjectsReadOnly() ?
+                Kind::ReadOnlyGlobalObject : Kind::SILGlobalVariable);
+  entity.Data = unsigned(kind) << KindShift;
+  return entity;
+}
+
+
 /// Mangle this entity into the given buffer.
 void LinkEntity::mangle(SmallVectorImpl<char> &buffer) const {
   llvm::raw_svector_ostream stream(buffer);
@@ -454,6 +466,9 @@ std::string LinkEntity::mangleAsString() const {
   case Kind::SILGlobalVariable:
     return getSILGlobalVariable()->getName().str();
 
+  case Kind::ReadOnlyGlobalObject:
+    return getSILGlobalVariable()->getName().str() + "r";
+
   case Kind::ReflectionBuiltinDescriptor:
     return mangler.mangleReflectionBuiltinDescriptor(getType());
   case Kind::ReflectionFieldDescriptor:
@@ -787,6 +802,7 @@ SILLinkage LinkEntity::getLinkage(ForDefinition_t forDefinition) const {
     return getSILLinkage(getDeclLinkage(getDecl()), forDefinition);
 
   case Kind::SILGlobalVariable:
+  case Kind::ReadOnlyGlobalObject:
     return getSILGlobalVariable()->getLinkage();
 
   case Kind::ReflectionBuiltinDescriptor:
@@ -882,6 +898,7 @@ bool LinkEntity::isContextDescriptor() const {
   case Kind::DefaultAssociatedConformanceAccessor:
   case Kind::SILFunction:
   case Kind::SILGlobalVariable:
+  case Kind::ReadOnlyGlobalObject:
   case Kind::ProtocolWitnessTable:
   case Kind::ProtocolWitnessTablePattern:
   case Kind::GenericProtocolWitnessTableInstantiationFunction:
@@ -1129,6 +1146,7 @@ Alignment LinkEntity::getAlignment(IRGenModule &IGM) const {
 bool LinkEntity::isWeakImported(ModuleDecl *module) const {
   switch (getKind()) {
   case Kind::SILGlobalVariable:
+  case Kind::ReadOnlyGlobalObject:
     if (getSILGlobalVariable()->getDecl()) {
       return getSILGlobalVariable()->getDecl()->isWeakImported(module);
     }
@@ -1317,6 +1335,7 @@ DeclContext *LinkEntity::getDeclContextForEmission() const {
     return getSILFunction()->getDeclContext();
 
   case Kind::SILGlobalVariable:
+  case Kind::ReadOnlyGlobalObject:
     if (auto decl = getSILGlobalVariable()->getDecl())
       return decl->getDeclContext();