[5.3] [metadata prespecialization] Zero trailing flags field.

include/swift/ABI/Metadata.h

@@ -3373,6 +3373,7 @@ class TargetGenericMetadataPatternTrailingObjects :
   /// class metadata.
   ///
   /// See also: [pre-5.2-extra-data-zeroing]
+  /// See also: [pre-5.3-extra-data-zeroing]
   const GenericMetadataPartialPattern *getExtraDataPattern() const {
     assert(asSelf()->hasExtraDataPattern());
     return this->template getTrailingObjects<GenericMetadataPartialPattern>();

lib/IRGen/ConstantBuilder.h

@@ -77,6 +77,10 @@ class ConstantAggregateBuilderBase
     addInt(IGM().Int32Ty, value);
   }
 
+  void addInt64(uint64_t value) { addInt(IGM().Int64Ty, value); }
+
+  void addSize(Size size) { addInt(IGM().SizeTy, size.getValue()); }
+
   void addRelativeAddressOrNull(llvm::Constant *target) {
     if (target) {
       addRelativeAddress(target);

lib/IRGen/EnumMetadataVisitor.h

@@ -94,12 +94,13 @@ class EnumMetadataScanner : public EnumMetadataVisitor<Impl> {
   void addGenericWitnessTable(GenericRequirement requirement) { addPointer(); }
   void addPayloadSize() { addPointer(); }
   void noteStartOfTypeSpecificMembers() {}
-  void addTrailingFlags() { addPointer(); }
+  void addTrailingFlags() { addInt64(); }
 
 private:
   void addPointer() {
     NextOffset += super::IGM.getPointerSize();
   }
+  void addInt64() { NextOffset += Size(8); }
 };
 
 } // end namespace irgen

lib/IRGen/GenMeta.cpp

@@ -2317,6 +2317,7 @@ namespace {
       asImpl().layoutHeader();
 
       // See also: [pre-5.2-extra-data-zeroing]
+      // See also: [pre-5.3-extra-data-zeroing]
       if (asImpl().hasExtraDataPattern()) {
         asImpl().addExtraDataPattern();
       }
@@ -3730,7 +3731,7 @@ namespace {
     GenericMetadataPatternFlags getPatternFlags() {
       auto flags = super::getPatternFlags();
 
-      if (IGM.shouldPrespecializeGenericMetadata()) {
+      if (hasTrailingFlags()) {
         flags.setHasTrailingFlags(true);
       }
 
@@ -3743,7 +3744,11 @@ namespace {
                                                      HasDependentVWT);
     }
 
-    bool hasExtraDataPattern() {
+    bool hasTrailingFlags() {
+      return IGM.shouldPrespecializeGenericMetadata();
+    }
+
+    bool hasKnownFieldOffsets() {
       auto &ti = IGM.getTypeInfo(getLoweredType());
       if (!isa<FixedTypeInfo>(ti))
         return false;
@@ -3754,19 +3759,31 @@ namespace {
       return true;
     }
 
-    /// Fill in a constant field offset vector if possible.
+    bool hasExtraDataPattern() {
+      return hasKnownFieldOffsets() || hasTrailingFlags();
+    }
+
+    /// If present, the extra data pattern consists of one or both of the
+    /// following:
+    ///
+    /// - the field offset vector
+    /// - the trailing flags
     PartialPattern buildExtraDataPattern() {
       ConstantInitBuilder builder(IGM);
-      auto init = builder.beginArray(IGM.Int32Ty);
+      auto init = builder.beginStruct();
 
       struct Scanner : StructMetadataScanner<Scanner> {
+        GenericStructMetadataBuilder &Outer;
         SILType Type;
-        ConstantArrayBuilder &B;
-        Scanner(IRGenModule &IGM, StructDecl *target, SILType type,
-                ConstantArrayBuilder &B)
-          : StructMetadataScanner(IGM, target), Type(type), B(B) {}
+        ConstantStructBuilder &B;
+        Scanner(GenericStructMetadataBuilder &outer, IRGenModule &IGM, StructDecl *target, SILType type,
+                ConstantStructBuilder &B)
+            : StructMetadataScanner(IGM, target), Outer(outer), Type(type), B(B) {}
 
         void addFieldOffset(VarDecl *field) {
+          if (!Outer.hasKnownFieldOffsets()) {
+            return;
+          }
           auto offset = emitPhysicalStructMemberFixedOffset(IGM, Type, field);
           if (offset) {
             B.add(offset);
@@ -3782,22 +3799,28 @@ namespace {
         void noteEndOfFieldOffsets() {
           B.addAlignmentPadding(IGM.getPointerAlignment());
         }
+
+        void addTrailingFlags() { B.addInt64(0); }
       };
-      Scanner(IGM, Target, getLoweredType(), init).layout();
-      Size vectorSize = init.getNextOffsetFromGlobal();
+      Scanner(*this, IGM, Target, getLoweredType(), init).layout();
+      Size structSize = init.getNextOffsetFromGlobal();
 
       auto global = init.finishAndCreateGlobal("", IGM.getPointerAlignment(),
                                                /*constant*/ true);
 
       auto &layout = IGM.getMetadataLayout(Target);
+
+      bool offsetUpToTrailingFlags = hasTrailingFlags() && !hasKnownFieldOffsets(); 
+      Size zeroingStart = IGM.getOffsetOfStructTypeSpecificMetadataMembers();
+      Offset zeroingEnd = offsetUpToTrailingFlags 
+                            ? layout.getTrailingFlagsOffset()
+                            : layout.getFieldOffsetVectorOffset();
       return { global,
-               layout.getFieldOffsetVectorOffset().getStatic()
-                 - IGM.getOffsetOfStructTypeSpecificMetadataMembers(),
-               vectorSize };
+               zeroingEnd.getStatic()
+                 - zeroingStart,
+               structSize };
     }
 
-    void addTrailingFlags() { this->B.addInt(IGM.Int64Ty, 0); }
-
     bool hasCompletionFunction() {
       return !isa<FixedTypeInfo>(IGM.getTypeInfo(getLoweredType()));
     }
@@ -4171,24 +4194,58 @@ namespace {
         descriptor = emitPointerAuthSign(IGF, descriptor, authInfo);
       }
 
-      auto metadata =
+      return
         IGF.Builder.CreateCall(IGM.getAllocateGenericValueMetadataFn(),
                                {descriptor, arguments, templatePointer,
                                 extraSizeV});
+    }
+
+    bool hasTrailingFlags() {
+      return IGM.shouldPrespecializeGenericMetadata();
+    }
+
+    bool hasKnownPayloadSize() {
+      auto &layout = IGM.getMetadataLayout(Target);
+      return layout.hasPayloadSizeOffset() && (bool)getConstantPayloadSize(IGM, Target);
+    }
+
+    bool hasExtraDataPattern() {
+      return hasKnownPayloadSize() || hasTrailingFlags();
+    }
+
+    /// If present, the extra data pattern consists of one or both of the
+    /// following:
+    ///
+    /// - the payload-size
+    /// - the trailing flags
+    PartialPattern buildExtraDataPattern() {
+      ConstantInitBuilder builder(IGM);
+      auto init = builder.beginStruct();
+
+      auto &layout = IGM.getMetadataLayout(Target);
 
-      // Initialize the payload-size field if we have a constant value for it.
-      // This is so small that we just do it inline instead of bothering
-      // with a pattern.
       if (layout.hasPayloadSizeOffset()) {
         if (auto size = getConstantPayloadSize(IGM, Target)) {
-          auto offset = layout.getPayloadSizeOffset();
-          auto slot = IGF.emitAddressAtOffset(metadata, offset, IGM.SizeTy,
-                                              IGM.getPointerAlignment());
-          IGF.Builder.CreateStore(IGM.getSize(*size), slot);
+          init.addSize(*size);
         }
       }
+      if (hasTrailingFlags()) {
+        init.addInt64(0);
+      }
+      Size structSize = init.getNextOffsetFromGlobal();
 
-      return metadata;
+      auto global = init.finishAndCreateGlobal("", IGM.getPointerAlignment(),
+                                               /*constant*/ true);
+
+      bool offsetUpToTrailingFlags = hasTrailingFlags() && !hasKnownPayloadSize(); 
+      Size zeroingStart = IGM.getOffsetOfEnumTypeSpecificMetadataMembers();
+      Offset zeroingEnd = offsetUpToTrailingFlags 
+                            ? layout.getTrailingFlagsOffset()
+                            : layout.getPayloadSizeOffset();
+      return { global,
+               zeroingEnd.getStatic()
+                 - zeroingStart,
+               structSize };
     }
 
     llvm::Constant *emitNominalTypeDescriptor() {
@@ -4198,7 +4255,7 @@ namespace {
     GenericMetadataPatternFlags getPatternFlags() {
       auto flags = super::getPatternFlags();
 
-      if (IGM.shouldPrespecializeGenericMetadata()) {
+      if (hasTrailingFlags()) {
         flags.setHasTrailingFlags(true);
       }
 

lib/IRGen/MetadataLayout.cpp

@@ -543,6 +543,11 @@ EnumMetadataLayout::EnumMetadataLayout(IRGenModule &IGM, EnumDecl *decl)
       super::noteStartOfGenericRequirements();
     }
 
+    void addTrailingFlags() {
+      Layout.TrailingFlagsOffset = getNextOffset();
+      super::addTrailingFlags();
+    }
+
     void layout() {
       super::layout();
       Layout.TheSize = getMetadataSize();
@@ -558,6 +563,11 @@ EnumMetadataLayout::getPayloadSizeOffset() const {
   return Offset(PayloadSizeOffset.getStaticOffset());
 }
 
+Offset EnumMetadataLayout::getTrailingFlagsOffset() const {
+  assert(TrailingFlagsOffset.isStatic());
+  return Offset(TrailingFlagsOffset.getStaticOffset());
+}
+
 /********************************** STRUCTS ***********************************/
 
 StructMetadataLayout::StructMetadataLayout(IRGenModule &IGM, StructDecl *decl)
@@ -594,6 +604,11 @@ StructMetadataLayout::StructMetadataLayout(IRGenModule &IGM, StructDecl *decl)
       super::noteEndOfFieldOffsets();
     }
 
+    void addTrailingFlags() {
+      Layout.TrailingFlagsOffset = getNextOffset();
+      super::addTrailingFlags();
+    }
+
     void layout() {
       super::layout();
       Layout.TheSize = getMetadataSize();
@@ -620,6 +635,11 @@ StructMetadataLayout::getFieldOffsetVectorOffset() const {
   return Offset(FieldOffsetVector.getStaticOffset());
 }
 
+Offset 
+StructMetadataLayout::getTrailingFlagsOffset() const {
+  assert(TrailingFlagsOffset.isStatic());
+  return Offset(TrailingFlagsOffset.getStaticOffset());
+}
 /****************************** FOREIGN CLASSES *******************************/
 ForeignClassMetadataLayout::ForeignClassMetadataLayout(IRGenModule &IGM,
                                                        ClassDecl *theClass)

lib/IRGen/MetadataLayout.h

@@ -284,6 +284,7 @@ class ClassMetadataLayout : public NominalMetadataLayout {
 class EnumMetadataLayout : public NominalMetadataLayout {
   /// The offset of the payload size field, if there is one.
   StoredOffset PayloadSizeOffset;
+  StoredOffset TrailingFlagsOffset;
 
   // TODO: presumably it would be useful to store *something* here
   // for resilience.
@@ -301,6 +302,7 @@ class EnumMetadataLayout : public NominalMetadataLayout {
   }
 
   Offset getPayloadSizeOffset() const;
+  Offset getTrailingFlagsOffset() const;
 
   static bool classof(const MetadataLayout *layout) {
     return layout->getKind() == Kind::Enum;
@@ -310,6 +312,7 @@ class EnumMetadataLayout : public NominalMetadataLayout {
 /// Layout for struct type metadata.
 class StructMetadataLayout : public NominalMetadataLayout {
   llvm::DenseMap<VarDecl*, StoredOffset> FieldOffsets;
+  StoredOffset TrailingFlagsOffset;
 
   /// The start of the field-offset vector.
   StoredOffset FieldOffsetVector;
@@ -338,6 +341,7 @@ class StructMetadataLayout : public NominalMetadataLayout {
   Size getStaticFieldOffset(VarDecl *field) const;
 
   Offset getFieldOffsetVectorOffset() const;
+  Offset getTrailingFlagsOffset() const;
 
   static bool classof(const MetadataLayout *layout) {
     return layout->getKind() == Kind::Struct;

lib/IRGen/StructMetadataVisitor.h

@@ -104,13 +104,14 @@ class StructMetadataScanner : public StructMetadataVisitor<Impl> {
     NextOffset = NextOffset.roundUpToAlignment(super::IGM.getPointerAlignment());
   }
 
-  void addTrailingFlags() { addPointer(); }
+  void addTrailingFlags() { addInt64(); }
 
 private:
   void addPointer() {
     NextOffset += super::IGM.getPointerSize();
   }
   void addInt32() { NextOffset += Size(4); }
+  void addInt64() { NextOffset += Size(8); }
 };
 
 } // end namespace irgen

stdlib/public/runtime/Metadata.cpp

@@ -436,6 +436,18 @@ SWIFT_ALLOWED_RUNTIME_GLOBAL_CTOR_END
 extern "C" void *_objc_empty_cache;
 #endif
 
+template <>
+bool Metadata::isCanonicalStaticallySpecializedGenericMetadata() const {
+  if (auto *metadata = dyn_cast<StructMetadata>(this))
+    return metadata->isCanonicalStaticallySpecializedGenericMetadata();
+  if (auto *metadata = dyn_cast<EnumMetadata>(this))
+    return metadata->isCanonicalStaticallySpecializedGenericMetadata();
+  if (auto *metadata = dyn_cast<ClassMetadata>(this))
+    return metadata->isCanonicalStaticallySpecializedGenericMetadata();
+
+  return false;
+}
+
 static void copyMetadataPattern(void **section,
                                 const GenericMetadataPartialPattern *pattern) {
   memcpy(section + pattern->OffsetInWords,
@@ -605,7 +617,10 @@ initializeValueMetadataFromPattern(ValueMetadata *metadata,
     auto extraDataPattern = pattern->getExtraDataPattern();
 
     // Zero memory up to the offset.
-    memset(metadataExtraData, 0, size_t(extraDataPattern->OffsetInWords));
+    // [pre-5.3-extra-data-zeroing] Before Swift 5.3, the runtime did not
+    // correctly zero the zero-prefix of the extra-data pattern.
+    memset(metadataExtraData, 0,
+           size_t(extraDataPattern->OffsetInWords) * sizeof(void *));
 
     // Copy the pattern into the rest of the extra data.
     copyMetadataPattern(metadataExtraData, extraDataPattern);
@@ -648,6 +663,11 @@ swift::swift_allocateGenericValueMetadata(const ValueTypeDescriptor *description
 
   auto bytes = (char*) cache.getAllocator().Allocate(totalSize, alignof(void*));
 
+#ifndef NDEBUG
+  // Fill the metadata record with garbage.
+  memset(bytes, 0xAA, totalSize);
+#endif
+
   auto addressPoint = bytes + sizeof(ValueMetadata::HeaderType);
   auto metadata = reinterpret_cast<ValueMetadata *>(addressPoint);
 
@@ -672,6 +692,9 @@ swift::swift_getGenericMetadata(MetadataRequest request,
                               arguments);
   auto result = cache.getOrInsert(key, request, description, arguments);
 
+  assert(
+      !result.second.Value->isCanonicalStaticallySpecializedGenericMetadata());
+
   return result.second;
 }
 
@@ -5521,18 +5544,6 @@ bool Metadata::satisfiesClassConstraint() const {
   return isAnyClass();
 }
 
-template <>
-bool Metadata::isCanonicalStaticallySpecializedGenericMetadata() const {
-  if (auto *metadata = dyn_cast<StructMetadata>(this))
-    return metadata->isCanonicalStaticallySpecializedGenericMetadata();
-  if (auto *metadata = dyn_cast<EnumMetadata>(this))
-    return metadata->isCanonicalStaticallySpecializedGenericMetadata();
-  if (auto *metadata = dyn_cast<ClassMetadata>(this))
-    return metadata->isCanonicalStaticallySpecializedGenericMetadata();
-
-  return false;
-}
-
 #if !NDEBUG
 static bool referencesAnonymousContext(Demangle::Node *node) {
   if (node->getKind() == Demangle::Node::Kind::AnonymousContext)

test/IRGen/enum_future.sil

@@ -2660,7 +2660,7 @@ entry(%x : $*MyOptional):
 // CHECK-LABEL: define{{( dllexport)?}}{{( protected)?}} internal %swift.type* @"$s11enum_future16DynamicSingletonOMi"(%swift.type_descriptor* %0, i8** %1, i8* %2) {{.*}} {
 // CHECK:   [[T0:%.*]] = bitcast i8** %1 to %swift.type**
 // CHECK:   [[T:%T]] = load %swift.type*, %swift.type** [[T0]],
-// CHECK:   [[METADATA:%.*]] = call %swift.type* @swift_allocateGenericValueMetadata(%swift.type_descriptor* %0, i8** %1, i8* %2, [[WORD]] {{4|8|16}})
+// CHECK:   [[METADATA:%.*]] = call %swift.type* @swift_allocateGenericValueMetadata(%swift.type_descriptor* %0, i8** %1, i8* %2, [[WORD]] {{4|8|12|16}})
 // CHECK-NEXT: ret %swift.type* [[METADATA]]
 
 // CHECK-LABEL: define{{( protected)?}} internal swiftcc %swift.metadata_response @"$s11enum_future16DynamicSingletonOMr"