[Tools] Add a library to build specialized generic metadata out of process.

This library uses GenericMetadataBuilder with a ReaderWriter that can read data and resolve pointers from MachO files, and emit a JSON representation of a dylib containing the built metadata.

We use LLVM's binary file readers to parse the MachO files and resolve fixups so we can follow pointers. This code is somewhat MachO specific, but could be generalized to other formats that LLVM supports.

rdar://116592577

Author: mikeash

CMakeLists.txt

@@ -215,18 +215,22 @@ option(SWIFT_BUILD_REMOTE_MIRROR
     "Build the Swift Remote Mirror Library"
     TRUE)
 
+option(SWIFT_BUILD_TYPE_METADATA_EXTRACTOR
+    "Build the Swift Type Metadata Extractor Library"
+    TRUE)
+
 option(SWIFT_BUILD_DYNAMIC_STDLIB
     "Build dynamic variants of the Swift standard library"
     TRUE)
 
 option(SWIFT_BUILD_STATIC_STDLIB
     "Build static variants of the Swift standard library"
     FALSE)
-    
+
 option(SWIFT_STDLIB_STATIC_PRINT
     "Build compile-time evaluated vprintf()"
     FALSE)
-    
+
 option(SWIFT_STDLIB_ENABLE_UNICODE_DATA
     "Include Unicode data files in the standard library.
     NOTE: Disabling this will cause many String methods to crash."
@@ -1245,6 +1249,11 @@ if(SWIFT_BUILD_REMOTE_MIRROR)
   message(STATUS "")
 endif()
 
+if(SWIFT_BUILD_TYPE_METADATA_EXTRACTOR)
+  message(STATUS "Building Swift Type Metadata Extractor for SDKs: ${SWIFT_SDKS}")
+  message(STATUS "")
+endif()
+
 #
 # Find required dependencies.
 #
@@ -1338,6 +1347,10 @@ else()
     add_subdirectory(stdlib/public/SwiftRemoteMirror)
   endif()
 
+  if(SWIFT_BUILD_TYPE_METADATA_EXTRACTOR)
+    add_subdirectory(stdlib/public/SwiftTypeMetadataExtractor)
+  endif()
+
   # We might also want to build the things in libexec seaprately
   if(SWIFT_BUILD_LIBEXEC)
     add_subdirectory(stdlib/public/libexec)

cmake/modules/SwiftComponents.cmake

@@ -70,7 +70,7 @@
 # * llvm-toolchain-dev-tools -- install LLVM development tools useful in a shared toolchain
 # * dev -- headers and libraries required to use Swift compiler as a library.
 set(_SWIFT_DEFINED_COMPONENTS
-  "autolink-driver;back-deployment;compiler;clang-builtin-headers;clang-resource-dir-symlink;clang-builtin-headers-in-clang-resource-dir;libexec;stdlib;stdlib-experimental;sdk-overlay;static-mirror-lib;swift-syntax-lib;editor-integration;tools;testsuite-tools;toolchain-tools;toolchain-dev-tools;llvm-toolchain-dev-tools;dev;license;sourcekit-xpc-service;sourcekit-inproc;swift-remote-mirror;swift-remote-mirror-headers")
+  "autolink-driver;back-deployment;compiler;clang-builtin-headers;clang-resource-dir-symlink;clang-builtin-headers-in-clang-resource-dir;libexec;stdlib;stdlib-experimental;sdk-overlay;static-mirror-lib;swift-syntax-lib;editor-integration;tools;testsuite-tools;toolchain-tools;toolchain-dev-tools;llvm-toolchain-dev-tools;dev;license;sourcekit-xpc-service;sourcekit-inproc;swift-remote-mirror;swift-remote-mirror-headers;swift-type-metadata-extractor;swift-type-metadata-extractor-headers")
 
 # The default install components include all of the defined components, except
 # for the following exceptions.

include/swift/ABI/Metadata.h

@@ -151,7 +151,7 @@ template <typename Runtime>
 struct TargetTypeMetadataHeaderBase {
   /// A pointer to the value-witnesses for this type.  This is only
   /// present for type metadata.
-  TargetPointer<Runtime, const ValueWitnessTable> ValueWitnesses;
+  TargetPointer<Runtime, const TargetValueWitnessTable<Runtime>> ValueWitnesses;
 };
 
 template <typename Runtime>
@@ -329,7 +329,7 @@ struct TargetMetadata {
     return asFullMetadata(this)->layoutString;
   }
 
-  const ValueWitnessTable *getValueWitnesses() const {
+  const TargetValueWitnessTable<Runtime> *getValueWitnesses() const {
     return asFullMetadata(this)->ValueWitnesses;
   }
 
@@ -2979,7 +2979,7 @@ template <typename Runtime>
 struct swift_ptrauth_struct_context_descriptor(ModuleContextDescriptor)
     TargetModuleContextDescriptor final : TargetContextDescriptor<Runtime> {
   /// The module name.
-  RelativeDirectPointer<const char, /*nullable*/ false> Name;
+  TargetRelativeDirectPointer<Runtime, const char, /*nullable*/ false> Name;
 
   /// Is this module a special C-imported module?
   bool isCImportedContext() const {
@@ -3036,7 +3036,7 @@ struct swift_ptrauth_struct_context_descriptor(ExtensionContextDescriptor)
   ///
   /// Note that the Parent of the extension will be the module context the
   /// extension is declared inside.
-  RelativeDirectPointer<const char> ExtendedContext;
+  TargetRelativeDirectPointer<Runtime, const char> ExtendedContext;
 
   using TrailingGenericContextObjects::getGenericContext;
 
@@ -3527,10 +3527,10 @@ struct TargetGenericValueMetadataPattern final :
 
   /// The value-witness table.  Indirectable so that we can re-use tables
   /// from other libraries if that seems wise.
-  TargetRelativeIndirectablePointer<Runtime, const ValueWitnessTable>
+  TargetRelativeIndirectablePointer<Runtime, const TargetValueWitnessTable<Runtime>>
     ValueWitnesses;
 
-  const ValueWitnessTable *getValueWitnessesPattern() const {
+  const TargetValueWitnessTable<Runtime> *getValueWitnessesPattern() const {
     return ValueWitnesses.get();
   }
 
@@ -3825,9 +3825,11 @@ class swift_ptrauth_struct_context_descriptor(TypeContextDescriptor)
                               /*Nullable*/ true> AccessFunctionPtr;
 
   /// A pointer to the field descriptor for the type, if any.
-  TargetRelativeDirectPointer<Runtime, const reflection::FieldDescriptor,
-                              /*nullable*/ true> Fields;
-      
+  TargetRelativeDirectPointer<Runtime,
+                              const reflection::TargetFieldDescriptor<Runtime>,
+                              /*nullable*/ true>
+      Fields;
+
   bool isReflectable() const { return (bool)Fields; }
 
   MetadataAccessFunction getAccessFunction() const {

include/swift/ABI/TargetLayout.h

@@ -63,7 +63,8 @@ struct RuntimeTarget<8> {
 };
 
 namespace reflection {
-  class FieldDescriptor;
+  template <typename Runtime>
+  class TargetFieldDescriptor;
 }
 
 /// In-process native runtime target.

include/swift/Basic/RelativePointer.h

@@ -590,8 +590,12 @@ class RelativeDirectPointerIntPairImpl {
   using ValueTy = PointeeTy;
   using PointerTy = PointeeTy*;
 
+  Offset getOffset() const & {
+    return RelativeOffsetPlusInt & ~getMask();
+  }
+
   PointerTy getPointer() const & {
-    Offset offset = (RelativeOffsetPlusInt & ~getMask());
+    Offset offset = getOffset();
 
     // Check for null.
     if (Nullable && offset == 0)
@@ -624,6 +628,7 @@ class RelativeDirectPointerIntPair<PointeeTy, IntTy, Nullable, Offset,
 {
   using super = RelativeDirectPointerIntPairImpl<PointeeTy, IntTy, Nullable, Offset>;
 public:
+  using super::getOffset;
   using super::getPointer;
   using super::getInt;
   using super::getOpaqueValue;

include/swift/CMakeLists.txt

@@ -4,6 +4,10 @@ if(SWIFT_BUILD_REMOTE_MIRROR)
   add_subdirectory(SwiftRemoteMirror)
 endif()
 
+if(SWIFT_BUILD_TYPE_METADATA_EXTRACTOR)
+  add_subdirectory(SwiftTypeMetadataExtractor)
+endif()
+
 if(SWIFT_INCLUDE_TOOLS)
   configure_file(Config.h.in ${CMAKE_CURRENT_BINARY_DIR}/Config.h
                 ESCAPE_QUOTES @ONLY)

include/swift/Demangling/Demangle.h

@@ -264,6 +264,8 @@ class Node {
   // Only to be used by the demangler parsers.
   void removeChildAt(unsigned Pos);
 
+  void replaceChild(unsigned Pos, NodePointer Child);
+
   // Reverses the order of children.
   void reverseChildren(size_t StartingAt = 0);
 

include/swift/RemoteInspection/Records.h

@@ -17,7 +17,7 @@
 #ifndef SWIFT_REFLECTION_RECORDS_H
 #define SWIFT_REFLECTION_RECORDS_H
 
-#include "swift/Basic/RelativePointer.h"
+#include "swift/ABI/TargetLayout.h"
 #include "swift/Demangling/Demangle.h"
 #include "llvm/ADT/ArrayRef.h"
 
@@ -82,14 +82,15 @@ class FieldRecordFlags {
   }
 };
 
-class FieldRecord {
+template <typename Runtime>
+class TargetFieldRecord {
   const FieldRecordFlags Flags;
 
 public:
-  const RelativeDirectPointer<const char> MangledTypeName;
-  const RelativeDirectPointer<const char> FieldName;
+  const TargetRelativeDirectPointer<Runtime, const char> MangledTypeName;
+  const TargetRelativeDirectPointer<Runtime, const char> FieldName;
 
-  FieldRecord() = delete;
+  TargetFieldRecord() = delete;
 
   bool hasMangledTypeName() const {
     return MangledTypeName;
@@ -111,36 +112,37 @@ class FieldRecord {
     return Flags.isVar();
   }
 };
+using FieldRecord = TargetFieldRecord<InProcess>;
 
-struct FieldRecordIterator {
-  const FieldRecord *Cur;
-  const FieldRecord * const End;
+template <typename Runtime>
+struct TargetFieldRecordIterator {
+  const TargetFieldRecord<Runtime> *Cur;
+  const TargetFieldRecord<Runtime> *const End;
 
-  FieldRecordIterator(const FieldRecord *Cur, const FieldRecord * const End)
-    : Cur(Cur), End(End) {}
+  TargetFieldRecordIterator(const TargetFieldRecord<Runtime> *Cur,
+                            const TargetFieldRecord<Runtime> *const End)
+      : Cur(Cur), End(End) {}
 
-  const FieldRecord &operator*() const {
-    return *Cur;
-  }
+  const TargetFieldRecord<Runtime> &operator*() const { return *Cur; }
 
-  const FieldRecord *operator->() const {
-    return Cur;
-  }
+  const TargetFieldRecord<Runtime> *operator->() const { return Cur; }
 
-  FieldRecordIterator &operator++() {
+  TargetFieldRecordIterator &operator++() {
     ++Cur;
     return *this;
   }
 
-  bool operator==(const FieldRecordIterator &other) const {
+  bool operator==(const TargetFieldRecordIterator &other) const {
     return Cur == other.Cur && End == other.End;
   }
 
-  bool operator!=(const FieldRecordIterator &other) const {
+  bool operator!=(const TargetFieldRecordIterator &other) const {
     return !(*this == other);
   }
 };
 
+using FieldRecordIterator = TargetFieldRecordIterator<InProcess>;
+
 enum class FieldDescriptorKind : uint16_t {
   // Swift nominal types.
   Struct,
@@ -173,16 +175,17 @@ enum class FieldDescriptorKind : uint16_t {
 
 // Field descriptors contain a collection of field records for a single
 // class, struct or enum declaration.
-class FieldDescriptor {
-  const FieldRecord *getFieldRecordBuffer() const {
-    return reinterpret_cast<const FieldRecord *>(this + 1);
+template <typename Runtime>
+class TargetFieldDescriptor {
+  const TargetFieldRecord<Runtime> *getFieldRecordBuffer() const {
+    return reinterpret_cast<const TargetFieldRecord<Runtime> *>(this + 1);
   }
 
 public:
-  const RelativeDirectPointer<const char> MangledTypeName;
-  const RelativeDirectPointer<const char> Superclass;
+  const TargetRelativeDirectPointer<Runtime, const char> MangledTypeName;
+  const TargetRelativeDirectPointer<Runtime, const char> Superclass;
 
-  FieldDescriptor() = delete;
+  TargetFieldDescriptor() = delete;
 
   const FieldDescriptorKind Kind;
   const uint16_t FieldRecordSize;
@@ -222,7 +225,7 @@ class FieldDescriptor {
     return const_iterator { End, End };
   }
 
-  llvm::ArrayRef<FieldRecord> getFields() const {
+  llvm::ArrayRef<TargetFieldRecord<Runtime>> getFields() const {
     return {getFieldRecordBuffer(), NumFields};
   }
 
@@ -242,6 +245,7 @@ class FieldDescriptor {
     return Demangle::makeSymbolicMangledNameStringRef(Superclass.get());
   }
 };
+using FieldDescriptor = TargetFieldDescriptor<InProcess>;
 
 // Associated type records describe the mapping from an associated
 // type to the type witness of a conformance.

include/swift/Runtime/Enum.h

@@ -20,7 +20,7 @@
 #include "swift/Runtime/Config.h"
 
 namespace swift {
-  
+
 struct OpaqueValue;
 struct InProcess;
 
@@ -139,7 +139,7 @@ void swift_initEnumMetadataMultiPayloadWithLayoutString(EnumMetadata *enumType,
 SWIFT_RUNTIME_EXPORT
 unsigned swift_getEnumCaseMultiPayload(const OpaqueValue *value,
                                        const EnumMetadata *enumType);
-  
+
 /// Store the tag value for the given case into a multi-payload enum,
 ///        whose associated payload (if any) has already been initialized.
 SWIFT_RUNTIME_EXPORT

include/swift/Runtime/GenericMetadataBuilder.h

@@ -313,7 +313,7 @@ class GenericMetadataBuilder {
   installGenericArguments(WritableData<FullMetadata<Metadata>> data,
                           Size metadataOffset,
                           Buffer<const ValueTypeDescriptor> descriptionBuffer,
-                          const GenericArgument *arguments) {
+                          llvm::ArrayRef<GenericArgument> arguments) {
     METADATA_BUILDER_LOG("Building %s",
                          ERROR_CHECK(getDescriptorName(descriptionBuffer)));
     char *metadataBase = reinterpret_cast<char *>(data.ptr);
@@ -325,6 +325,11 @@ class GenericMetadataBuilder {
         (reinterpret_cast<Pointer<const Metadata> *>(metadata) +
          getGenericArgumentOffset(
              descriptionBuffer.template cast<const TypeContextDescriptor>()));
+
+    if (arguments.size() < header.NumKeyArguments) {
+      return BuilderError("Not enough generic arguments, %zu provided, %" PRId32 " required", arguments.size(), header.NumKeyArguments);
+    }
+
     METADATA_BUILDER_LOG(
         "Installing %" PRIu16 " generic arguments at offset %" PRId32,
         header.NumKeyArguments,
@@ -353,7 +358,7 @@ class GenericMetadataBuilder {
   /// Allocate and build a metadata structure.
   BuilderErrorOr<ConstructedMetadata>
   buildGenericMetadata(Buffer<const TypeContextDescriptor> descriptionBuffer,
-                       const GenericArgument *arguments,
+                       llvm::ArrayRef<GenericArgument> arguments,
                        Buffer<const GenericMetadataPattern> patternBuffer,
                        size_t extraDataSize) {
     auto description = descriptionBuffer.ptr;
@@ -377,7 +382,7 @@ class GenericMetadataBuilder {
 
   BuilderErrorOr<ConstructedMetadata> buildGenericValueMetadata(
       Buffer<const ValueTypeDescriptor> descriptionBuffer,
-      const GenericArgument *arguments,
+      llvm::ArrayRef<GenericArgument> arguments,
       Buffer<const GenericValueMetadataPattern> patternBuffer,
       size_t extraDataSize) {
     auto *pattern = patternBuffer.ptr;

include/swift/SwiftTypeMetadataExtractor/CMakeLists.txt

@@ -0,0 +1,10 @@
+set(swift_type_metadata_extractor_headers)
+list(APPEND swift_type_metadata_extractor_headers
+       SwiftTypeMetadataExtractor.h
+       module.modulemap)
+swift_install_in_component(FILES
+                             ${swift_type_metadata_extractor_headers}
+                           DESTINATION
+                             "include/swift/SwiftTypeMetadataExtractor"
+                           COMPONENT
+                             swift-type-metadata-extractor-headers)