IRGen: generate static arrays in read-only data sections.

docs/ABI/Mangling.rst

@@ -349,6 +349,7 @@ Entities
   entity-spec ::= 'fE'                       // ivar destroyer; untyped
   entity-spec ::= 'fe'                       // ivar initializer; untyped
   entity-spec ::= 'Tv' NATURAL               // outlined global variable (from context function)
+  entity-spec ::= 'Tv' NATURAL 'r'           // outlined global read-only object
   entity-spec ::= 'Te' bridge-spec           // outlined objective c method call
 
   entity-spec ::= decl-name label-list function-signature generic-signature? 'F'    // function

include/swift/AST/ASTContext.h

@@ -880,6 +880,10 @@ class ASTContext final {
   /// for extended existential types.
   AvailabilityContext getParameterizedExistentialRuntimeAvailability();
 
+  /// Get the runtime availability of immortal ref-count symbols, which are
+  /// needed to place array buffers into constant data sections.
+  AvailabilityContext getImmortalRefCountSymbolsAvailability();
+
   /// Get the runtime availability of features introduced in the Swift 5.2
   /// compiler for the target platform.
   AvailabilityContext getSwift52Availability();

include/swift/Demangling/DemangleNodes.def

@@ -279,6 +279,7 @@ NODE(OutlinedAssignWithTake)
 NODE(OutlinedAssignWithCopy)
 NODE(OutlinedDestroy)
 NODE(OutlinedVariable)
+NODE(OutlinedReadOnlyObject)
 NODE(AssocTypePath)
 NODE(LabelList)
 NODE(ModuleDescriptor)

include/swift/IRGen/Linking.h

@@ -364,6 +364,9 @@ class LinkEntity {
     /// A SIL global variable. The pointer is a SILGlobalVariable*.
     SILGlobalVariable,
 
+    /// An outlined read-only global object. The pointer is a SILGlobalVariable*.
+    ReadOnlyGlobalObject,
+
     // These next few are protocol-conformance kinds.
 
     /// A direct protocol witness table. The secondary pointer is a
@@ -993,13 +996,7 @@ class LinkEntity {
     return entity;
   }
 
-  static LinkEntity forSILGlobalVariable(SILGlobalVariable *G) {
-    LinkEntity entity;
-    entity.Pointer = G;
-    entity.SecondaryPointer = nullptr;
-    entity.Data = LINKENTITY_SET_FIELD(Kind, unsigned(Kind::SILGlobalVariable));
-    return entity;
-  }
+  static LinkEntity forSILGlobalVariable(SILGlobalVariable *G, IRGenModule &IGM);
 
   static LinkEntity
   forDifferentiabilityWitness(const SILDifferentiabilityWitness *witness) {
@@ -1420,7 +1417,8 @@ class LinkEntity {
   }
 
   SILGlobalVariable *getSILGlobalVariable() const {
-    assert(getKind() == Kind::SILGlobalVariable);
+    assert(getKind() == Kind::SILGlobalVariable ||
+           getKind() == Kind::ReadOnlyGlobalObject);
     return reinterpret_cast<SILGlobalVariable*>(Pointer);
   }
 

lib/AST/Availability.cpp

@@ -399,6 +399,13 @@ ASTContext::getParameterizedExistentialRuntimeAvailability() {
   return getSwift57Availability();
 }
 
+AvailabilityContext
+ASTContext::getImmortalRefCountSymbolsAvailability() {
+  // TODO: replace this with a concrete swift version once we have it.
+  // rdar://94185998
+  return getSwiftFutureAvailability();
+}
+
 AvailabilityContext ASTContext::getSwift52Availability() {
   auto target = LangOpts.Target;
 

lib/Demangling/Demangler.cpp

@@ -131,6 +131,7 @@ bool swift::Demangle::isFunctionAttr(Node::Kind kind) {
     case Node::Kind::PartialApplyForwarder:
     case Node::Kind::PartialApplyObjCForwarder:
     case Node::Kind::OutlinedVariable:
+    case Node::Kind::OutlinedReadOnlyObject:
     case Node::Kind::OutlinedBridgedMethod:
     case Node::Kind::MergedFunction:
     case Node::Kind::DistributedThunk:
@@ -2615,6 +2616,8 @@ NodePointer Demangler::demangleThunkOrSpecialization() {
       int Idx = demangleIndex();
       if (Idx < 0)
         return nullptr;
+      if (nextChar() == 'r')
+        return createNode(Node::Kind::OutlinedReadOnlyObject, Idx);
       return createNode(Node::Kind::OutlinedVariable, Idx);
     }
     case 'e': {

lib/Demangling/NodePrinter.cpp

@@ -547,6 +547,7 @@ class NodePrinter {
     case Node::Kind::OutlinedAssignWithCopy:
     case Node::Kind::OutlinedDestroy:
     case Node::Kind::OutlinedVariable:
+    case Node::Kind::OutlinedReadOnlyObject:
     case Node::Kind::AssocTypePath:
     case Node::Kind::ModuleDescriptor:
     case Node::Kind::AnonymousDescriptor:
@@ -1265,6 +1266,9 @@ NodePointer NodePrinter::print(NodePointer Node, unsigned depth,
   case Node::Kind::OutlinedVariable:
     Printer << "outlined variable #" << Node->getIndex() << " of ";
     return nullptr;
+  case Node::Kind::OutlinedReadOnlyObject:
+    Printer << "outlined read-only object #" << Node->getIndex() << " of ";
+    return nullptr;
   case Node::Kind::Directness:
     Printer << toString(Directness(Node->getIndex())) << " ";
     return nullptr;

lib/Demangling/OldRemangler.cpp

@@ -2467,6 +2467,11 @@ ManglingError Remangler::mangleOutlinedVariable(Node *node, unsigned depth) {
   return mangleSingleChildNode(node, depth + 1);
 }
 
+ManglingError Remangler::mangleOutlinedReadOnlyObject(Node *node, unsigned depth) {
+  Buffer << "Tv" << node->getIndex() << 'r';
+  return mangleSingleChildNode(node, depth + 1);
+}
+
 ManglingError Remangler::mangleOutlinedBridgedMethod(Node *node,
                                                      unsigned depth) {
   Buffer << "Te" << node->getText();

lib/Demangling/Remangler.cpp

@@ -1628,6 +1628,7 @@ ManglingError Remangler::mangleGlobal(Node *node, unsigned depth) {
       case Node::Kind::GenericPartialSpecializationNotReAbstracted:
       case Node::Kind::OutlinedBridgedMethod:
       case Node::Kind::OutlinedVariable:
+      case Node::Kind::OutlinedReadOnlyObject:
       case Node::Kind::ObjCAttribute:
       case Node::Kind::NonObjCAttribute:
       case Node::Kind::DynamicAttribute:
@@ -3153,6 +3154,13 @@ ManglingError Remangler::mangleOutlinedVariable(Node *node, unsigned depth) {
   return ManglingError::Success;
 }
 
+ManglingError Remangler::mangleOutlinedReadOnlyObject(Node *node, unsigned depth) {
+  Buffer << "Tv";
+  mangleIndex(node->getIndex());
+  Buffer << 'r';
+  return ManglingError::Success;
+}
+
 ManglingError Remangler::mangleOutlinedBridgedMethod(Node *node,
                                                      unsigned depth) {
   Buffer << "Te";

lib/IRGen/GenConstant.cpp

@@ -278,9 +278,27 @@ llvm::Constant *irgen::emitConstantObject(IRGenModule &IGM, ObjectInst *OI,
     }
   }
   // Construct the object header.
-  llvm::Type *ObjectHeaderTy = sTy->getElementType(0);
-  assert(ObjectHeaderTy->isStructTy());
-  elts[0] = llvm::Constant::getNullValue(ObjectHeaderTy);
+  llvm::StructType *ObjectHeaderTy = cast<llvm::StructType>(sTy->getElementType(0));
+
+  if (IGM.canMakeStaticObjectsReadOnly()) {
+    if (!IGM.swiftImmortalRefCount) {
+      auto *var = new llvm::GlobalVariable(IGM.Module, IGM.Int8Ty,
+                                        /*constant*/ true, llvm::GlobalValue::ExternalLinkage,
+                                        /*initializer*/ nullptr, "_swiftImmortalRefCount");
+      IGM.swiftImmortalRefCount = var;
+    }
+    if (!IGM.swiftStaticArrayMetadata) {
+      auto *var = new llvm::GlobalVariable(IGM.Module, IGM.TypeMetadataStructTy,
+                                        /*constant*/ true, llvm::GlobalValue::ExternalLinkage,
+                                        /*initializer*/ nullptr, "_swiftStaticArrayMetadata");
+      IGM.swiftStaticArrayMetadata = var;
+    }
+    elts[0] = llvm::ConstantStruct::get(ObjectHeaderTy, {
+      IGM.swiftStaticArrayMetadata,
+      llvm::ConstantExpr::getPtrToInt(IGM.swiftImmortalRefCount, IGM.IntPtrTy)});
+  } else {
+    elts[0] = llvm::Constant::getNullValue(ObjectHeaderTy);
+  }
   insertPadding(elts, sTy);
   return llvm::ConstantStruct::get(sTy, elts);
 }

lib/IRGen/GenDecl.cpp

@@ -2482,7 +2482,7 @@ Address IRGenModule::getAddrOfSILGlobalVariable(SILGlobalVariable *var,
     return Address(addr, alignment);
   }
 
-  LinkEntity entity = LinkEntity::forSILGlobalVariable(var);
+  LinkEntity entity = LinkEntity::forSILGlobalVariable(var, *this);
   ResilienceExpansion expansion = getResilienceExpansionForLayout(var);
 
   llvm::Type *storageType;
@@ -2528,28 +2528,33 @@ Address IRGenModule::getAddrOfSILGlobalVariable(SILGlobalVariable *var,
 
   // Check whether we've created the global variable already.
   // FIXME: We should integrate this into the LinkEntity cache more cleanly.
-  auto gvar = Module.getGlobalVariable(var->getName(), /*allowInternal*/ true);
+  LinkInfo link = LinkInfo::get(*this, entity, forDefinition);
+  auto gvar = Module.getGlobalVariable(link.getName(), /*allowInternal*/ true);
   if (gvar) {
     if (forDefinition)
       updateLinkageForDefinition(*this, gvar, entity);
   } else {
-    LinkInfo link = LinkInfo::get(*this, entity, forDefinition);
     llvm::Type *storageTypeWithContainer = storageType;
     if (var->isInitializedObject()) {
-      // A statically initialized object must be placed into a container struct
-      // because the swift_initStaticObject needs a swift_once_t at offset -1:
-      //     struct Container {
-      //       swift_once_t token[fixedAlignment / sizeof(swift_once_t)];
-      //       HeapObject object;
-      //     };
-      std::string typeName = storageType->getStructName().str() + 'c';
-      assert(fixedAlignment >= getPointerAlignment());
-      unsigned numTokens = fixedAlignment.getValue() /
-        getPointerAlignment().getValue();
-      storageTypeWithContainer = llvm::StructType::create(getLLVMContext(),
-              {llvm::ArrayType::get(OnceTy, numTokens), storageType}, typeName);
-      gvar = createVariable(*this, link, storageTypeWithContainer,
-                            fixedAlignment);
+      if (canMakeStaticObjectsReadOnly()) {
+        gvar = createVariable(*this, link, storageType, fixedAlignment);
+        gvar->setConstant(true);
+      } else {
+        // A statically initialized object must be placed into a container struct
+        // because the swift_initStaticObject needs a swift_once_t at offset -1:
+        //     struct Container {
+        //       swift_once_t token[fixedAlignment / sizeof(swift_once_t)];
+        //       HeapObject object;
+        //     };
+        std::string typeName = storageType->getStructName().str() + 'c';
+        assert(fixedAlignment >= getPointerAlignment());
+        unsigned numTokens = fixedAlignment.getValue() /
+          getPointerAlignment().getValue();
+        storageTypeWithContainer = llvm::StructType::create(getLLVMContext(),
+                {llvm::ArrayType::get(OnceTy, numTokens), storageType}, typeName);
+        gvar = createVariable(*this, link, storageTypeWithContainer,
+                              fixedAlignment);
+      }
     } else {
       StringRef name;
       Optional<SILLocation> loc;
@@ -2574,7 +2579,7 @@ Address IRGenModule::getAddrOfSILGlobalVariable(SILGlobalVariable *var,
       gvar->setComdat(nullptr);
   }
   llvm::Constant *addr = gvar;
-  if (var->isInitializedObject()) {
+  if (var->isInitializedObject() && !canMakeStaticObjectsReadOnly()) {
     // Project out the object from the container.
     llvm::Constant *Indices[2] = {
       llvm::ConstantExpr::getIntegerValue(Int32Ty, APInt(32, 0)),

lib/IRGen/IRGenModule.cpp

@@ -1785,6 +1785,16 @@ bool IRGenModule::shouldPrespecializeGenericMetadata() {
          canPrespecializeTarget;
 }
 
+bool IRGenModule::canMakeStaticObjectsReadOnly() {
+  // TODO: Support constant static arrays on other platforms, too.
+  // See also the comment in GlobalObjects.cpp.
+  if (!Triple.isOSDarwin())
+    return false;
+
+  return getAvailabilityContext().isContainedIn(
+          Context.getImmortalRefCountSymbolsAvailability());
+}
+
 void IRGenerator::addGenModule(SourceFile *SF, IRGenModule *IGM) {
   assert(GenModules.count(SF) == 0);
   GenModules[SF] = IGM;

lib/IRGen/IRGenModule.h

@@ -763,6 +763,9 @@ class IRGenModule {
 
   llvm::GlobalVariable *TheTrivialPropertyDescriptor = nullptr;
 
+  llvm::GlobalVariable *swiftImmortalRefCount = nullptr;
+  llvm::GlobalVariable *swiftStaticArrayMetadata = nullptr;
+
   /// Used to create unique names for class layout types with tail allocated
   /// elements.
   unsigned TailElemTypeID = 0;
@@ -886,6 +889,8 @@ class IRGenModule {
 
   bool shouldPrespecializeGenericMetadata();
 
+  bool canMakeStaticObjectsReadOnly();
+
   Size getAtomicBoolSize() const { return AtomicBoolSize; }
   Alignment getAtomicBoolAlignment() const { return AtomicBoolAlign; }
 

lib/IRGen/IRGenSIL.cpp

@@ -2765,7 +2765,7 @@ void IRGenSILFunction::visitGlobalValueInst(GlobalValueInst *i) {
                                                 NotForDefinition).getAddress();
   // We don't need to initialize the global object if it's never used for
   // something which can access the object header.
-  if (!hasOnlyProjections(i)) {
+  if (!hasOnlyProjections(i) && !IGM.canMakeStaticObjectsReadOnly()) {
     auto ClassType = loweredTy.getASTType();
     llvm::Value *Metadata =
       emitClassHeapMetadataRef(*this, ClassType, MetadataValueType::TypeMetadata,
@@ -7099,8 +7099,11 @@ void IRGenModule::emitSILStaticInitializers() {
            "cannot emit a static initializer for dynamically-sized global");
 #endif
 
+    LinkEntity entity = LinkEntity::forSILGlobalVariable(&Global, *this);
+    std::string name = entity.mangleAsString();
+
     auto *IRGlobal =
-        Module.getGlobalVariable(Global.getName(), true /* = AllowLocal */);
+        Module.getGlobalVariable(name, true /* = AllowLocal */);
 
     // A check for multi-threaded compilation: Is this the llvm module where the
     // global is defined and not only referenced (or not referenced at all).
@@ -7110,10 +7113,14 @@ void IRGenModule::emitSILStaticInitializers() {
     if (auto *OI = dyn_cast<ObjectInst>(InitValue)) {
       StructLayout *Layout = StaticObjectLayouts[&Global].get();
       llvm::Constant *InitVal = emitConstantObject(*this, OI, Layout);
-      auto *ContainerTy = cast<llvm::StructType>(IRGlobal->getValueType());
-      auto *zero = llvm::ConstantAggregateZero::get(ContainerTy->getElementType(0));
-      IRGlobal->setInitializer(llvm::ConstantStruct::get(ContainerTy,
-                                                         {zero , InitVal}));
+      if (canMakeStaticObjectsReadOnly()) {
+        IRGlobal->setInitializer(InitVal);
+      } else {
+        auto *ContainerTy = cast<llvm::StructType>(IRGlobal->getValueType());
+        auto *zero = llvm::ConstantAggregateZero::get(ContainerTy->getElementType(0));
+        IRGlobal->setInitializer(llvm::ConstantStruct::get(ContainerTy,
+                                                           {zero , InitVal}));
+      }
       continue;
     }
 

lib/IRGen/Linking.cpp

@@ -91,6 +91,18 @@ UniversalLinkageInfo::UniversalLinkageInfo(const llvm::Triple &triple,
       UseDLLStorage(useDllStorage(triple)), Internalize(isStaticLibrary),
       HasMultipleIGMs(hasMultipleIGMs), ForcePublicDecls(forcePublicDecls) {}
 
+LinkEntity LinkEntity::forSILGlobalVariable(SILGlobalVariable *G,
+                                            IRGenModule &IGM) {
+  LinkEntity entity;
+  entity.Pointer = G;
+  entity.SecondaryPointer = nullptr;
+  auto kind = (G->isInitializedObject() && IGM.canMakeStaticObjectsReadOnly() ?
+                Kind::ReadOnlyGlobalObject : Kind::SILGlobalVariable);
+  entity.Data = unsigned(kind) << KindShift;
+  return entity;
+}
+
+
 /// Mangle this entity into the given buffer.
 void LinkEntity::mangle(SmallVectorImpl<char> &buffer) const {
   llvm::raw_svector_ostream stream(buffer);
@@ -454,6 +466,9 @@ std::string LinkEntity::mangleAsString() const {
   case Kind::SILGlobalVariable:
     return getSILGlobalVariable()->getName().str();
 
+  case Kind::ReadOnlyGlobalObject:
+    return getSILGlobalVariable()->getName().str() + "r";
+
   case Kind::ReflectionBuiltinDescriptor:
     return mangler.mangleReflectionBuiltinDescriptor(getType());
   case Kind::ReflectionFieldDescriptor:
@@ -787,6 +802,7 @@ SILLinkage LinkEntity::getLinkage(ForDefinition_t forDefinition) const {
     return getSILLinkage(getDeclLinkage(getDecl()), forDefinition);
 
   case Kind::SILGlobalVariable:
+  case Kind::ReadOnlyGlobalObject:
     return getSILGlobalVariable()->getLinkage();
 
   case Kind::ReflectionBuiltinDescriptor:
@@ -882,6 +898,7 @@ bool LinkEntity::isContextDescriptor() const {
   case Kind::DefaultAssociatedConformanceAccessor:
   case Kind::SILFunction:
   case Kind::SILGlobalVariable:
+  case Kind::ReadOnlyGlobalObject:
   case Kind::ProtocolWitnessTable:
   case Kind::ProtocolWitnessTablePattern:
   case Kind::GenericProtocolWitnessTableInstantiationFunction:
@@ -1129,6 +1146,7 @@ Alignment LinkEntity::getAlignment(IRGenModule &IGM) const {
 bool LinkEntity::isWeakImported(ModuleDecl *module) const {
   switch (getKind()) {
   case Kind::SILGlobalVariable:
+  case Kind::ReadOnlyGlobalObject:
     if (getSILGlobalVariable()->getDecl()) {
       return getSILGlobalVariable()->getDecl()->isWeakImported(module);
     }
@@ -1317,6 +1335,7 @@ DeclContext *LinkEntity::getDeclContextForEmission() const {
     return getSILFunction()->getDeclContext();
 
   case Kind::SILGlobalVariable:
+  case Kind::ReadOnlyGlobalObject:
     if (auto decl = getSILGlobalVariable()->getDecl())
       return decl->getDeclContext();