[IRGen] Generate compressed representation of value witnesses (#63813)

rdar://105837040

* WIP: Store layout string in type metadata

* WIP: More cases working

* WIP: Layout strings almost working

* Add layout string pointer to struct metadata

* Fetch bytecode layout strings from metadata in runtime

* More efficient bytecode layout

* Add support for interpreted generics in layout strings

* Layout string instantiation, take and more

* Remove duplicate information from layout strings

* Include size of previous object in next objects offset to reduce number of increments at runtime

* Add support for existentials

* Build type layout strings with StructBuilder to support target sizes and metadata pointers

* Add support for resilient types

* Properly cache layout strings in compiler

* Generic resilient types working

* Non-generic resilient types working

* Instantiate resilient type in layout when possible

* Fix a few issues around alignment and signing

* Disable generics, fix static alignment

* Fix MultiPayloadEnum size when no extra tag is necessary

* Fixes after rebase

* Cleanup

* Fix most tests

* Fix objcImplementattion and non-Darwin builds

* Fix BytecodeLayouts on non-Darwin

* Fix Linux build

* Fix sizes in linux tests

* Sign layout string pointers

* Use nullptr instead of debug value

Author: drexin

docs/proposals/RuntimeValueWitness.rst

@@ -58,6 +58,7 @@ copied, but do not need to be released or retained.
    I16 = 's',
    I32 = 'l',
    I64 = 'L',
+   I128 = 'Q',
 
 We also have reference types. While they are all 64bit sized, we need to
 differentiate between them because they have different ways of being

include/swift/ABI/Metadata.h

@@ -116,6 +116,15 @@ struct MetadataDependency {
   }
 };
 
+/// Prefix of a metadata header, containing a pointer to the
+/// type layout string.
+template <typename Runtime>
+struct TargetTypeMetadataLayoutPrefix {
+    TargetSignedPointer<Runtime, const uint8_t *
+                                      __ptrauth_swift_type_layout_string>
+        layoutString;
+};
+
 /// The header before a metadata object which appears on all type
 /// metadata.  Note that heap metadata are not necessarily type
 /// metadata, even for objects of a heap type: for example, objects of
@@ -124,11 +133,25 @@ struct MetadataDependency {
 /// This case can be distinguished using the isTypeMetadata() flag
 /// on ClassMetadata.
 template <typename Runtime>
-struct TargetTypeMetadataHeader {
+struct TargetTypeMetadataHeaderBase {
   /// A pointer to the value-witnesses for this type.  This is only
   /// present for type metadata.
   TargetPointer<Runtime, const ValueWitnessTable> ValueWitnesses;
 };
+
+template <typename Runtime>
+struct TargetTypeMetadataHeader
+    : TargetTypeMetadataLayoutPrefix<Runtime>,
+      TargetTypeMetadataHeaderBase<Runtime> {
+
+  TargetTypeMetadataHeader() = default;
+  constexpr TargetTypeMetadataHeader(
+    const TargetTypeMetadataLayoutPrefix<Runtime> &layout,
+    const TargetTypeMetadataHeaderBase<Runtime> &header)
+      : TargetTypeMetadataLayoutPrefix<Runtime>(layout),
+        TargetTypeMetadataHeaderBase<Runtime>(header) {}
+};
+
 using TypeMetadataHeader = TargetTypeMetadataHeader<InProcess>;
 
 /// A "full" metadata pointer is simply an adjusted address point on a
@@ -284,6 +307,17 @@ struct TargetMetadata {
     return isAnyKindOfClass(getKind());
   }
 
+  const uint8_t *getLayoutString() const {
+    assert(hasLayoutString());
+    if (isAnyClass()) {
+      return asFullMetadata(
+                 reinterpret_cast<const TargetAnyClassMetadata<Runtime> *>(
+                     this))
+          ->layoutString;
+    }
+    return asFullMetadata(this)->layoutString;
+  }
+
   const ValueWitnessTable *getValueWitnesses() const {
     return asFullMetadata(this)->ValueWitnesses;
   }
@@ -295,6 +329,19 @@ struct TargetMetadata {
   void setValueWitnesses(const ValueWitnessTable *table) {
     asFullMetadata(this)->ValueWitnesses = table;
   }
+
+  void setLayoutString(const uint8_t *layoutString) {
+    if (isAnyClass()) {
+      asFullMetadata(reinterpret_cast<TargetAnyClassMetadata<Runtime> *>(this))
+          ->layoutString = layoutString;
+    } else {
+      asFullMetadata(this)->layoutString = layoutString;
+    }
+  }
+
+  bool hasLayoutString() const {
+    return getTypeContextDescriptor()->hasLayoutString();
+  }
 
   // Define forwarders for value witnesses. These invoke this metadata's value
   // witness table with itself as the 'self' parameter.
@@ -445,7 +492,7 @@ struct TargetMetadata {
 /// The common structure of opaque metadata.  Adds nothing.
 template <typename Runtime>
 struct TargetOpaqueMetadata {
-  typedef TargetTypeMetadataHeader<Runtime> HeaderType;
+  typedef TargetTypeMetadataHeaderBase<Runtime> HeaderType;
 
   // We have to represent this as a member so we can list-initialize it.
   TargetMetadata<Runtime> base;
@@ -469,13 +516,16 @@ using HeapMetadataHeaderPrefix =
 /// The header present on all heap metadata.
 template <typename Runtime>
 struct TargetHeapMetadataHeader
-    : TargetHeapMetadataHeaderPrefix<Runtime>,
-      TargetTypeMetadataHeader<Runtime> {
+    : TargetTypeMetadataLayoutPrefix<Runtime>,
+      TargetHeapMetadataHeaderPrefix<Runtime>,
+      TargetTypeMetadataHeaderBase<Runtime> {
   constexpr TargetHeapMetadataHeader(
+      const TargetTypeMetadataLayoutPrefix<Runtime> &typeLayoutPrefix,
       const TargetHeapMetadataHeaderPrefix<Runtime> &heapPrefix,
-      const TargetTypeMetadataHeader<Runtime> &typePrefix)
-    : TargetHeapMetadataHeaderPrefix<Runtime>(heapPrefix),
-      TargetTypeMetadataHeader<Runtime>(typePrefix) {}
+      const TargetTypeMetadataHeaderBase<Runtime> &typePrefix)
+    : TargetTypeMetadataLayoutPrefix<Runtime>(typeLayoutPrefix),
+      TargetHeapMetadataHeaderPrefix<Runtime>(heapPrefix),
+      TargetTypeMetadataHeaderBase<Runtime>(typePrefix) {}
 };
 using HeapMetadataHeader =
   TargetHeapMetadataHeader<InProcess>;
@@ -1511,6 +1561,7 @@ using MetatypeMetadata = TargetMetatypeMetadata<InProcess>;
 template <typename Runtime>
 struct TargetTupleTypeMetadata : public TargetMetadata<Runtime> {
   using StoredSize = typename Runtime::StoredSize;
+  using HeaderType = TargetTypeMetadataHeaderBase<Runtime>;
   TargetTupleTypeMetadata() = default;
   constexpr TargetTupleTypeMetadata(const TargetMetadata<Runtime> &base,
                                     uint32_t numElements,
@@ -1696,6 +1747,7 @@ struct TargetExistentialTypeMetadata
       TargetExistentialTypeMetadata<Runtime>,
       ConstTargetMetadataPointer<Runtime, TargetMetadata>,
       TargetProtocolDescriptorRef<Runtime>> {
+  using HeaderType = TargetTypeMetadataHeaderBase<Runtime>;
 
 private:
   using ProtocolDescriptorRef = TargetProtocolDescriptorRef<Runtime>;
@@ -3642,6 +3694,10 @@ class TargetTypeContextDescriptor
     return getTypeContextDescriptorFlags().hasCanonicalMetadataPrespecializations();
   }
 
+  bool hasLayoutString() const {
+    return getTypeContextDescriptorFlags().hasLayoutString();
+  }
+
   /// Given that this type has foreign metadata initialization, return the
   /// control structure for it.
   const TargetForeignMetadataInitialization<Runtime> &

include/swift/ABI/MetadataValues.h

@@ -1445,6 +1445,8 @@ namespace SpecialPointerAuthDiscriminators {
 
   // Relative protocol witness table descriminator
   const uint16_t RelativeProtocolWitnessTable = 0xb830; // = 47152
+
+  const uint16_t TypeLayoutString = 0x8b65; // = 35685
 }
 
 /// The number of arguments that will be passed directly to a generic
@@ -1598,6 +1600,9 @@ class TypeContextDescriptorFlags : public FlagSet<uint16_t> {
     /// prespecializations.
     HasCanonicalMetadataPrespecializations = 3,
 
+    /// Set if the metadata contains a pointer to a layout string
+    HasLayoutString = 4,
+
     // Type-specific flags:
 
     /// Set if the class is an actor.
@@ -1680,6 +1685,10 @@ class TypeContextDescriptorFlags : public FlagSet<uint16_t> {
 
   FLAGSET_DEFINE_FLAG_ACCESSORS(HasCanonicalMetadataPrespecializations, hasCanonicalMetadataPrespecializations, setHasCanonicalMetadataPrespecializations)
 
+  FLAGSET_DEFINE_FLAG_ACCESSORS(HasLayoutString,
+                                hasLayoutString,
+                                setHasLayoutString)
+
   FLAGSET_DEFINE_FLAG_ACCESSORS(Class_HasVTable,
                                 class_hasVTable,
                                 class_setHasVTable)

include/swift/AST/IRGenOptions.h

@@ -207,6 +207,9 @@ struct PointerAuthOptions : clang::PointerAuthOptions {
 
   /// Relative protocol witness table descriminator.
   PointerAuthSchema RelativeProtocolWitnessTable;
+
+  /// Type layout string descriminator.
+  PointerAuthSchema TypeLayoutString;
 };
 
 enum class JITDebugArtifact : unsigned {

include/swift/Basic/Features.def

@@ -118,6 +118,9 @@ EXPERIMENTAL_FEATURE(TypeWitnessSystemInference, false)
 EXPERIMENTAL_FEATURE(LayoutPrespecialization, true)
 EXPERIMENTAL_FEATURE(ModuleInterfaceExportAs, true)
 
+/// Whether to enable experimental layout string value witnesses
+EXPERIMENTAL_FEATURE(LayoutStringValueWitnesses, true)
+
 /// Whether to enable experimental differentiable programming features:
 /// `@differentiable` declaration attribute, etc.
 EXPERIMENTAL_FEATURE(DifferentiableProgramming, false)

include/swift/Runtime/Config.h

@@ -310,7 +310,7 @@ extern uintptr_t __COMPATIBILITY_LIBRARIES_CANNOT_CHECK_THE_IS_SWIFT_BIT_DIRECTL
 #define __ptrauth_swift_objc_superclass                                        \
   __ptrauth(ptrauth_key_process_independent_data, 1,                           \
             swift::SpecialPointerAuthDiscriminators::ObjCSuperclass)
-#define __ptrauth_swift_nonunique_extended_existential_type_shape                        \
+#define __ptrauth_swift_nonunique_extended_existential_type_shape              \
   __ptrauth(ptrauth_key_process_independent_data, 1,                           \
             SpecialPointerAuthDiscriminators::NonUniqueExtendedExistentialTypeShape)
 #define swift_ptrauth_sign_opaque_read_resume_function(__fn, __buffer)         \
@@ -323,6 +323,9 @@ extern uintptr_t __COMPATIBILITY_LIBRARIES_CANNOT_CHECK_THE_IS_SWIFT_BIT_DIRECTL
                           ptrauth_key_process_independent_code,                \
                           ptrauth_blend_discriminator(__buffer,                \
             SpecialPointerAuthDiscriminators::OpaqueModifyResumeFunction))
+#define __ptrauth_swift_type_layout_string                                     \
+  __ptrauth(ptrauth_key_process_independent_data, 1,                           \
+            SpecialPointerAuthDiscriminators::TypeLayoutString)
 #else
 #define SWIFT_PTRAUTH 0
 #define __ptrauth_swift_function_pointer(__typekey)
@@ -350,6 +353,7 @@ extern uintptr_t __COMPATIBILITY_LIBRARIES_CANNOT_CHECK_THE_IS_SWIFT_BIT_DIRECTL
 #define __ptrauth_swift_dynamic_replacement_key
 #define swift_ptrauth_sign_opaque_read_resume_function(__fn, __buffer) (__fn)
 #define swift_ptrauth_sign_opaque_modify_resume_function(__fn, __buffer) (__fn)
+#define __ptrauth_swift_type_layout_string
 #endif
 
 #ifdef __cplusplus

include/swift/Runtime/RuntimeFunctions.def

@@ -2160,34 +2160,61 @@ FUNCTION(StoreMultiPayloadEnumTagSinglePayload,
          ATTRS(NoUnwind),
          EFFECT(NoEffect))
 
-// void *swift_generic_destroy(opaque*, const Metadata* type, const char*);
+// void swift_generic_destroy(opaque*, const Metadata* type);
 FUNCTION(GenericDestroy,
          swift_generic_destroy,
          C_CC, AlwaysAvailable,
          RETURNS(VoidTy),
-         ARGS(Int8PtrTy, TypeMetadataPtrTy, Int8PtrTy),
+         ARGS(Int8PtrTy, TypeMetadataPtrTy),
          ATTRS(NoUnwind),
          EFFECT(Deallocating))
 
 
-// void *swift_generic_assign(opaque* dest, opaque* src, const Metadata* type, const char*, bool isTake);
-FUNCTION(GenericAssign,
-         swift_generic_assign,
+// void *swift_generic_assignWithCopy(opaque* dest, opaque* src, const Metadata* type);
+FUNCTION(GenericAssignWithCopy,
+         swift_generic_assignWithCopy,
          C_CC, AlwaysAvailable,
-         RETURNS(VoidTy),
-         ARGS(Int8PtrTy, Int8PtrTy, TypeMetadataPtrTy, Int8PtrTy, Int1Ty),
+         RETURNS(Int8PtrTy),
+         ARGS(Int8PtrTy, Int8PtrTy, TypeMetadataPtrTy),
          ATTRS(NoUnwind),
          EFFECT(Refcounting, Deallocating))
 
-// void *swift_generic_initialize(opaque* dest, opaque* src, const Metadata* type, const char*, bool isTake);
-FUNCTION(GenericInitialize,
-         swift_generic_initialize,
+// void *swift_generic_assignWithTake(opaque* dest, opaque* src, const Metadata* type);
+FUNCTION(GenericAssignWithTake,
+         swift_generic_assignWithTake,
          C_CC, AlwaysAvailable,
-         RETURNS(VoidTy),
-         ARGS(Int8PtrTy, Int8PtrTy, TypeMetadataPtrTy, Int8PtrTy, Int1Ty),
+         RETURNS(Int8PtrTy),
+         ARGS(Int8PtrTy, Int8PtrTy, TypeMetadataPtrTy),
+         ATTRS(NoUnwind),
+         EFFECT(Refcounting, Deallocating))
+
+// void *swift_generic_initWithCopy(opaque* dest, opaque* src, const Metadata* type);
+FUNCTION(GenericInitWithCopy,
+         swift_generic_initWithCopy,
+         C_CC, AlwaysAvailable,
+         RETURNS(Int8PtrTy),
+         ARGS(Int8PtrTy, Int8PtrTy, TypeMetadataPtrTy),
          ATTRS(NoUnwind),
          EFFECT(Refcounting))
 
+// void *swift_generic_initWithTake(opaque* dest, opaque* src, const Metadata* type);
+FUNCTION(GenericInitWithTake,
+         swift_generic_initWithTake,
+         C_CC, AlwaysAvailable,
+         RETURNS(Int8PtrTy),
+         ARGS(Int8PtrTy, Int8PtrTy, TypeMetadataPtrTy),
+         ATTRS(NoUnwind),
+         EFFECT(Refcounting))
+
+// void swift_generic_instantiateLayoutString(const uint8_t* layoutStr, Metadata* type);
+FUNCTION(GenericInstantiateLayoutString,
+         swift_generic_instantiateLayoutString,
+         C_CC, AlwaysAvailable,
+         RETURNS(VoidTy),
+         ARGS(Int8PtrTy, TypeMetadataPtrTy),
+         ATTRS(NoUnwind),
+         EFFECT(MetaData))
+
 #undef RETURNS
 #undef ARGS
 #undef ATTRS

lib/AST/ASTPrinter.cpp

@@ -3110,6 +3110,10 @@ static bool usesFeatureLayoutPrespecialization(Decl *decl) {
   return false;
 }
 
+static bool usesFeatureLayoutStringValueWitnesses(Decl *decl) {
+  return false;
+}
+
 static bool usesFeatureModuleInterfaceExportAs(Decl *decl) {
   return false;
 }

lib/IRGen/ClassMetadataVisitor.h

@@ -65,9 +65,12 @@ template <class Impl> class ClassMetadataVisitor
 
 public:
   void layout() {
-    static_assert(MetadataAdjustmentIndex::Class == 2,
+    static_assert(MetadataAdjustmentIndex::Class == 3,
                   "Adjustment index must be synchronized with this layout");
 
+    // Pointer to layout string
+    asImpl().addLayoutStringPointer();
+
     // HeapMetadata header.
     asImpl().addDestructorFunction();
 
@@ -225,6 +228,7 @@ class ClassMetadataScanner : public ClassMetadataVisitor<Impl> {
   void addNominalTypeDescriptor() { addPointer(); }
   void addIVarDestroyer() { addPointer(); }
   void addValueWitnessTable() { addPointer(); }
+  void addLayoutStringPointer() { addPointer(); }
   void addDestructorFunction() { addPointer(); }
   void addSuperclass() { addPointer(); }
   void addClassFlags() { addInt32(); }

lib/IRGen/EnumMetadataVisitor.h

@@ -44,9 +44,11 @@ template <class Impl> class EnumMetadataVisitor
 
 public:
   void layout() {
-    static_assert(MetadataAdjustmentIndex::ValueType == 1,
+    static_assert(MetadataAdjustmentIndex::ValueType == 2,
                   "Adjustment index must be synchronized with this layout");
 
+    asImpl().addLayoutStringPointer();
+
     // Metadata header.
     super::layout();
 
@@ -91,6 +93,7 @@ class EnumMetadataScanner : public EnumMetadataVisitor<Impl> {
 
 public:
   void addMetadataFlags() { addPointer(); }
+  void addLayoutStringPointer() { addPointer(); }
   void addValueWitnessTable() { addPointer(); }
   void addNominalTypeDescriptor() { addPointer(); }
   void addGenericRequirement(GenericRequirement requirement) { addPointer(); }

lib/IRGen/ForeignClassMetadataVisitor.h

@@ -35,6 +35,7 @@ class ForeignClassMetadataVisitor
     : super(IGM), Target(target) {}
 
   void layout() {
+    asImpl().addLayoutStringPointer();
     super::layout();
     asImpl().addNominalTypeDescriptor();
     asImpl().addSuperclass();
@@ -60,6 +61,7 @@ class ForeignClassMetadataScanner : public ForeignClassMetadataVisitor<Impl> {
 
 public:
   void addMetadataFlags() { addPointer(); }
+  void addLayoutStringPointer() { addPointer(); }
   void addValueWitnessTable() { addPointer(); }
   void addNominalTypeDescriptor() { addPointer(); }
   void addSuperclass() { addPointer(); }
@@ -83,6 +85,7 @@ class ForeignReferenceTypeMetadataVisitor
       : super(IGM), Target(target) {}
 
   void layout() {
+    asImpl().addLayoutStringPointer();
     super::layout();
     asImpl().addNominalTypeDescriptor();
     asImpl().addReservedWord();

lib/IRGen/GenArchetype.cpp

@@ -116,8 +116,10 @@ class OpaqueArchetypeTypeInfo
     collector.collectTypeMetadataForLayout(T);
   }
 
-  TypeLayoutEntry *buildTypeLayoutEntry(IRGenModule &IGM,
-                                        SILType T) const override {
+  TypeLayoutEntry
+  *buildTypeLayoutEntry(IRGenModule &IGM,
+                        SILType T,
+                        bool useStructLayouts) const override {
     return IGM.typeLayoutCache.getOrCreateArchetypeEntry(T.getObjectType());
   }
 };