[Runtime] Create an external generic metadata builder.

Create a version of the metadata specialization code which is abstracted so that it can work in different contexts, such as building specialized metadata from dylibs on disk rather than from inside a running process.

The GenericMetadataBuilder class is templatized on a ReaderWriter. The ReaderWriter abstracts out everything that's different between in-process and external construction of this data. Instead of reading and writing pointers directly, the builder calls the ReaderWriter to resolve and write pointers. The ReaderWriter also handles symbol lookups and looking up other Swift types by name.

This is accompanied by a simple implementation of the ReaderWriter which works in-process. The abstracted calls to resolve and write pointers are implemented using standard pointer dereferencing.

A new SWIFT_DEBUG_VALIDATE_EXTERNAL_GENERIC_METADATA_BUILDER environment variable uses the in-process ReaderWriter to validate the builder by running it in parallel with the existing metadata builder code in the runtime. When enabled, the GenericMetadataBuilder is used to build a second copy of metadata built by the runtime, and the two are compared to ensure that they match. When this environment variable is not set, the new builder code is inactive.

The builder is incomplete, and this initial version only works on structs. Any unsupported type produces an error, and skips the validation.

rdar://116592420

Author: mikeash

include/swift/ABI/ValueWitnessTable.h

@@ -29,7 +29,8 @@
 
 namespace swift {
 
-struct TypeLayout;
+template <typename Runtime>
+struct TargetTypeLayout;
 template <class Runtime> struct TargetEnumValueWitnessTable;
 template <typename Runtime> struct TargetMetadata;
 using Metadata = TargetMetadata<InProcess>;
@@ -120,7 +121,7 @@ class TargetValueWitnessTypes {
   // types for the flags enums.
   typedef size_t size;
   typedef size_t stride;
-  typedef ValueWitnessFlags flags;
+  typedef TargetValueWitnessFlags<typename Runtime::StoredSize> flags;
   typedef uint32_t extraInhabitantCount;
 };
 
@@ -214,8 +215,8 @@ template <typename Runtime> struct TargetValueWitnessTable {
   const TargetEnumValueWitnessTable<Runtime> *_asEVWT() const;
 
   /// Get the type layout record within this value witness table.
-  const TypeLayout *getTypeLayout() const {
-    return reinterpret_cast<const TypeLayout *>(&size);
+  const TargetTypeLayout<Runtime> *getTypeLayout() const {
+    return reinterpret_cast<const TargetTypeLayout<Runtime> *>(&size);
   }
 
   /// Check whether this metadata is complete.
@@ -224,7 +225,7 @@ template <typename Runtime> struct TargetValueWitnessTable {
   /// "Publish" the layout of this type to other threads.  All other stores
   /// to the value witness table (including its extended header) should have
   /// happened before this is called.
-  void publishLayout(const TypeLayout &layout);
+  void publishLayout(const TargetTypeLayout<Runtime> &layout);
 };
 
 /// A value-witness table with enum entry points.
@@ -275,23 +276,27 @@ TargetValueWitnessTable<Runtime>::_asEVWT() const {
 /// necessary to perform dependent layout of generic value types. It excludes
 /// the value witness functions and includes only the size, alignment,
 /// extra inhabitants, and miscellaneous flags about the type.
-struct TypeLayout {
-  ValueWitnessTypes::size size;
-  ValueWitnessTypes::stride stride;
-  ValueWitnessTypes::flags flags;
-  ValueWitnessTypes::extraInhabitantCount extraInhabitantCount;
+template <typename Runtime>
+struct TargetTypeLayout {
+  typename TargetValueWitnessTypes<Runtime>::size size;
+  typename TargetValueWitnessTypes<Runtime>::stride stride;
+  typename TargetValueWitnessTypes<Runtime>::flags flags;
+  typename TargetValueWitnessTypes<Runtime>::extraInhabitantCount
+      extraInhabitantCount;
 
 private:
   void _static_assert_layout();
 public:
-  TypeLayout() = default;
-  constexpr TypeLayout(ValueWitnessTypes::size size,
-                       ValueWitnessTypes::stride stride,
-                       ValueWitnessTypes::flags flags,
-                       ValueWitnessTypes::extraInhabitantCount xiCount)
-    : size(size), stride(stride), flags(flags), extraInhabitantCount(xiCount) {}
+  TargetTypeLayout() = default;
+  constexpr TargetTypeLayout(
+      typename TargetValueWitnessTypes<Runtime>::size size,
+      typename TargetValueWitnessTypes<Runtime>::stride stride,
+      typename TargetValueWitnessTypes<Runtime>::flags flags,
+      typename TargetValueWitnessTypes<Runtime>::extraInhabitantCount xiCount)
+      : size(size), stride(stride), flags(flags),
+        extraInhabitantCount(xiCount) {}
 
-  const TypeLayout *getTypeLayout() const { return this; }
+  const TargetTypeLayout *getTypeLayout() const { return this; }
 
   /// The number of extra inhabitants, that is, bit patterns that do not form
   /// valid values of the type, in this type's binary representation.
@@ -303,14 +308,16 @@ struct TypeLayout {
     return extraInhabitantCount != 0;
   }
 };
+using TypeLayout = TargetTypeLayout<InProcess>;
 
-inline void TypeLayout::_static_assert_layout() {
-  #define CHECK_TYPE_LAYOUT_OFFSET(FIELD)                               \
-    static_assert(offsetof(ValueWitnessTable, FIELD)                    \
-                    - offsetof(ValueWitnessTable, size)                 \
-                  == offsetof(TypeLayout, FIELD),                       \
-                  "layout of " #FIELD " in TypeLayout doesn't match "   \
-                  "value witness table")
+template <typename Runtime>
+inline void TargetTypeLayout<Runtime>::_static_assert_layout() {
+#define CHECK_TYPE_LAYOUT_OFFSET(FIELD)                                        \
+  static_assert(offsetof(TargetValueWitnessTable<Runtime>, FIELD) -            \
+                        offsetof(TargetValueWitnessTable<Runtime>, size) ==    \
+                    offsetof(TargetTypeLayout<Runtime>, FIELD),                \
+                "layout of " #FIELD " in TypeLayout doesn't match "            \
+                "value witness table")
   CHECK_TYPE_LAYOUT_OFFSET(size);
   CHECK_TYPE_LAYOUT_OFFSET(flags);
   CHECK_TYPE_LAYOUT_OFFSET(extraInhabitantCount);

include/swift/Demangling/TypeLookupError.h

@@ -153,13 +153,13 @@ class TypeLookupError {
   /// Get the error string from the error value. The value must be passed to
   /// `freeErrorString` when done. (Unless you're just calling a `fatalError`
   /// in which case there's no point.)
-  char *copyErrorString() {
+  char *copyErrorString() const {
     return reinterpret_cast<char *>(
         Fn(Context, Command::CopyErrorString, nullptr));
   }
 
   /// Free an error string previously obtained from `copyErrorString`.
-  void freeErrorString(char *str) {
+  void freeErrorString(char *str) const {
     Fn(Context, Command::DestroyErrorString, str);
   }
 };

include/swift/Remote/MetadataReader.h

@@ -24,6 +24,7 @@
 #include "swift/Demangling/TypeDecoder.h"
 #include "swift/Basic/Defer.h"
 #include "swift/Basic/ExternalUnion.h"
+#include "swift/Basic/MathUtils.h"
 #include "swift/Basic/Range.h"
 #include "swift/Basic/LLVM.h"
 #include "swift/ABI/TypeIdentity.h"
@@ -3419,11 +3420,6 @@ class MetadataReader {
 
     return finish(TaggedPointerEncodingKind::Extended);
   }
-
-  template <class T>
-  static constexpr T roundUpToAlignment(T offset, T alignment) {
-    return (offset + alignment - 1) & ~(alignment - 1);
-  }
 };
 
 } // end namespace remote

include/swift/Runtime/Enum.h

@@ -32,7 +32,9 @@ using Metadata = TargetMetadata<InProcess>;
 
 template <typename Runtime> struct TargetEnumMetadata;
 using EnumMetadata = TargetEnumMetadata<InProcess>;
-struct TypeLayout;
+template <typename Runtime>
+struct TargetTypeLayout;
+using TypeLayout = TargetTypeLayout<InProcess>;
 
 /// Initialize the type metadata for a single-case enum type.
 ///