[Tools] Add a library to build specialized generic metadata out of process.

This library uses GenericMetadataBuilder with a ReaderWriter that can read data and resolve pointers from MachO files, and emit a JSON representation of a dylib containing the built metadata.

We use LLVM's binary file readers to parse the MachO files and resolve fixups so we can follow pointers. This code is somewhat MachO specific, but could be generalized to other formats that LLVM supports.

rdar://116592577

Author: mikeash

CMakeLists.txt

@@ -215,18 +215,22 @@ option(SWIFT_BUILD_REMOTE_MIRROR
     "Build the Swift Remote Mirror Library"
     TRUE)
 
+option(SWIFT_BUILD_EXTERNAL_GENERIC_METADATA_BUILDER
+    "Build the Swift External Generic Metadata Builder Library"
+    TRUE)
+
 option(SWIFT_BUILD_DYNAMIC_STDLIB
     "Build dynamic variants of the Swift standard library"
     TRUE)
 
 option(SWIFT_BUILD_STATIC_STDLIB
     "Build static variants of the Swift standard library"
     FALSE)
-    
+
 option(SWIFT_STDLIB_STATIC_PRINT
     "Build compile-time evaluated vprintf()"
     FALSE)
-    
+
 option(SWIFT_STDLIB_ENABLE_UNICODE_DATA
     "Include Unicode data files in the standard library.
     NOTE: Disabling this will cause many String methods to crash."
@@ -1245,6 +1249,11 @@ if(SWIFT_BUILD_REMOTE_MIRROR)
   message(STATUS "")
 endif()
 
+if(SWIFT_BUILD_EXTERNAL_GENERIC_METADATA_BUILDER)
+  message(STATUS "Building Swift External Generic Metadata Builder for SDKs: ${SWIFT_SDKS}")
+  message(STATUS "")
+endif()
+
 #
 # Find required dependencies.
 #
@@ -1338,6 +1347,10 @@ else()
     add_subdirectory(stdlib/public/SwiftRemoteMirror)
   endif()
 
+  if(SWIFT_BUILD_EXTERNAL_GENERIC_METADATA_BUILDER)
+    add_subdirectory(stdlib/public/ExternalGenericMetadataBuilder)
+  endif()
+
   # We might also want to build the things in libexec seaprately
   if(SWIFT_BUILD_LIBEXEC)
     add_subdirectory(stdlib/public/libexec)

cmake/modules/SwiftComponents.cmake

@@ -70,7 +70,7 @@
 # * llvm-toolchain-dev-tools -- install LLVM development tools useful in a shared toolchain
 # * dev -- headers and libraries required to use Swift compiler as a library.
 set(_SWIFT_DEFINED_COMPONENTS
-  "autolink-driver;back-deployment;compiler;clang-builtin-headers;clang-resource-dir-symlink;clang-builtin-headers-in-clang-resource-dir;libexec;stdlib;stdlib-experimental;sdk-overlay;static-mirror-lib;swift-syntax-lib;editor-integration;tools;testsuite-tools;toolchain-tools;toolchain-dev-tools;llvm-toolchain-dev-tools;dev;license;sourcekit-xpc-service;sourcekit-inproc;swift-remote-mirror;swift-remote-mirror-headers")
+  "autolink-driver;back-deployment;compiler;clang-builtin-headers;clang-resource-dir-symlink;clang-builtin-headers-in-clang-resource-dir;libexec;stdlib;stdlib-experimental;sdk-overlay;static-mirror-lib;swift-syntax-lib;editor-integration;tools;testsuite-tools;toolchain-tools;toolchain-dev-tools;llvm-toolchain-dev-tools;dev;license;sourcekit-xpc-service;sourcekit-inproc;swift-remote-mirror;swift-remote-mirror-headers;swift-external-generic-metadata-builder;swift-external-generic-metadata-builder-headers")
 
 # The default install components include all of the defined components, except
 # for the following exceptions.

include/swift/ABI/Metadata.h

@@ -151,7 +151,7 @@ template <typename Runtime>
 struct TargetTypeMetadataHeaderBase {
   /// A pointer to the value-witnesses for this type.  This is only
   /// present for type metadata.
-  TargetPointer<Runtime, const ValueWitnessTable> ValueWitnesses;
+  TargetPointer<Runtime, const TargetValueWitnessTable<Runtime>> ValueWitnesses;
 };
 
 template <typename Runtime>
@@ -329,7 +329,7 @@ struct TargetMetadata {
     return asFullMetadata(this)->layoutString;
   }
 
-  const ValueWitnessTable *getValueWitnesses() const {
+  const TargetValueWitnessTable<Runtime> *getValueWitnesses() const {
     return asFullMetadata(this)->ValueWitnesses;
   }
 
@@ -2979,7 +2979,7 @@ template <typename Runtime>
 struct swift_ptrauth_struct_context_descriptor(ModuleContextDescriptor)
     TargetModuleContextDescriptor final : TargetContextDescriptor<Runtime> {
   /// The module name.
-  RelativeDirectPointer<const char, /*nullable*/ false> Name;
+  TargetRelativeDirectPointer<Runtime, const char, /*nullable*/ false> Name;
 
   /// Is this module a special C-imported module?
   bool isCImportedContext() const {
@@ -3036,7 +3036,7 @@ struct swift_ptrauth_struct_context_descriptor(ExtensionContextDescriptor)
   ///
   /// Note that the Parent of the extension will be the module context the
   /// extension is declared inside.
-  RelativeDirectPointer<const char> ExtendedContext;
+  TargetRelativeDirectPointer<Runtime, const char> ExtendedContext;
 
   using TrailingGenericContextObjects::getGenericContext;
 
@@ -3527,10 +3527,10 @@ struct TargetGenericValueMetadataPattern final :
 
   /// The value-witness table.  Indirectable so that we can re-use tables
   /// from other libraries if that seems wise.
-  TargetRelativeIndirectablePointer<Runtime, const ValueWitnessTable>
+  TargetRelativeIndirectablePointer<Runtime, const TargetValueWitnessTable<Runtime>>
     ValueWitnesses;
 
-  const ValueWitnessTable *getValueWitnessesPattern() const {
+  const TargetValueWitnessTable<Runtime> *getValueWitnessesPattern() const {
     return ValueWitnesses.get();
   }
 
@@ -3825,9 +3825,11 @@ class swift_ptrauth_struct_context_descriptor(TypeContextDescriptor)
                               /*Nullable*/ true> AccessFunctionPtr;
 
   /// A pointer to the field descriptor for the type, if any.
-  TargetRelativeDirectPointer<Runtime, const reflection::FieldDescriptor,
-                              /*nullable*/ true> Fields;
-      
+  TargetRelativeDirectPointer<Runtime,
+                              const reflection::TargetFieldDescriptor<Runtime>,
+                              /*nullable*/ true>
+      Fields;
+
   bool isReflectable() const { return (bool)Fields; }
 
   MetadataAccessFunction getAccessFunction() const {

include/swift/ABI/TargetLayout.h

@@ -63,7 +63,8 @@ struct RuntimeTarget<8> {
 };
 
 namespace reflection {
-  class FieldDescriptor;
+  template <typename Runtime>
+  class TargetFieldDescriptor;
 }
 
 /// In-process native runtime target.

include/swift/Basic/RelativePointer.h

@@ -590,8 +590,12 @@ class RelativeDirectPointerIntPairImpl {
   using ValueTy = PointeeTy;
   using PointerTy = PointeeTy*;
 
+  Offset getOffset() const & {
+    return RelativeOffsetPlusInt & ~getMask();
+  }
+
   PointerTy getPointer() const & {
-    Offset offset = (RelativeOffsetPlusInt & ~getMask());
+    Offset offset = getOffset();
 
     // Check for null.
     if (Nullable && offset == 0)
@@ -624,6 +628,7 @@ class RelativeDirectPointerIntPair<PointeeTy, IntTy, Nullable, Offset,
 {
   using super = RelativeDirectPointerIntPairImpl<PointeeTy, IntTy, Nullable, Offset>;
 public:
+  using super::getOffset;
   using super::getPointer;
   using super::getInt;
   using super::getOpaqueValue;

include/swift/CMakeLists.txt

@@ -4,6 +4,10 @@ if(SWIFT_BUILD_REMOTE_MIRROR)
   add_subdirectory(SwiftRemoteMirror)
 endif()
 
+if(SWIFT_BUILD_EXTERNAL_GENERIC_METADATA_BUILDER)
+  add_subdirectory(ExternalGenericMetadataBuilder)
+endif()
+
 if(SWIFT_INCLUDE_TOOLS)
   configure_file(Config.h.in ${CMAKE_CURRENT_BINARY_DIR}/Config.h
                 ESCAPE_QUOTES @ONLY)

include/swift/Demangling/Demangle.h

@@ -264,6 +264,8 @@ class Node {
   // Only to be used by the demangler parsers.
   void removeChildAt(unsigned Pos);
 
+  void replaceChild(unsigned Pos, NodePointer Child);
+
   // Reverses the order of children.
   void reverseChildren(size_t StartingAt = 0);
 

include/swift/ExternalGenericMetadataBuilder/CMakeLists.txt

@@ -0,0 +1,10 @@
+set(swift_external_generic_metadata_builder_headers)
+list(APPEND swift_external_generic_metadata_builder_headers
+       ExternalGenericMetadataBuilder.h
+       module.modulemap)
+swift_install_in_component(FILES
+                             ${swift_external_generic_metadata_builder_headers}
+                           DESTINATION
+                             "include/swift/ExternalGenericMetadataBuilder"
+                           COMPONENT
+                             swift-external-generic-metadata-builder-headers)

include/swift/ExternalGenericMetadataBuilder/ExternalGenericMetadataBuilder.h

@@ -0,0 +1,81 @@
+//===--- ExternalGenericMetadataBuilder.h - Public interface ----*- C++ -*-===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This header declares functions in the libswiftReflection library,
+/// which provides mechanisms for reflecting heap information in a
+/// remote Swift process.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef SWIFT_EXTERNAL_GENERIC_METADATA_BUILDER_H
+#define SWIFT_EXTERNAL_GENERIC_METADATA_BUILDER_H
+
+#include <stdint.h>
+
+#include "swift/shims/Visibility.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct SwiftExternalMetadataBuilder;
+struct mach_header;
+
+// Create a builder object with the given platform and architecture name.
+SWIFT_ATTRIBUTE_FOR_EXPORTS
+struct SwiftExternalMetadataBuilder *
+swift_externalMetadataBuilder_create(int platform, const char *arch);
+
+// Destroy a builder object.
+SWIFT_ATTRIBUTE_FOR_EXPORTS
+void swift_externalMetadataBuilder_destroy(
+    struct SwiftExternalMetadataBuilder *);
+
+// Returns an error string if the dylib could not be added
+// The builder owns the string, so the caller does not have to free it
+// The mach_header* is the raw dylib from disk/memory, before the shared cache
+// builder has created its own copy of it
+SWIFT_ATTRIBUTE_FOR_EXPORTS
+const char *swift_externalMetadataBuilder_addDylib(
+    struct SwiftExternalMetadataBuilder *, const char *install_name,
+    const struct mach_header *, uint64_t size);
+
+SWIFT_ATTRIBUTE_FOR_EXPORTS
+const char *swift_externalMetadataBuilder_readNamesJSON(
+    struct SwiftExternalMetadataBuilder *, const char *names_json);
+
+// Returns an error string if the dylib could not be added
+// The builder owns the string, so the caller does not have to free it
+SWIFT_ATTRIBUTE_FOR_EXPORTS
+const char *swift_externalMetadataBuilder_buildMetadata(
+    struct SwiftExternalMetadataBuilder *);
+
+// Get the JSON for the built metadata.
+SWIFT_ATTRIBUTE_FOR_EXPORTS
+const char *swift_externalMetadataBuilder_getMetadataJSON(
+    struct SwiftExternalMetadataBuilder *);
+
+// Convenience function that works with the filesystem and handles everything in
+// one call.
+SWIFT_ATTRIBUTE_FOR_EXPORTS
+int swift_type_metadata_extract(const char *inputPath,       // mangled names
+                                const char *dylibSearchPath, // images to add
+                                const char *arch,
+                                const char *outputPath // json output
+);
+
+#ifdef __cplusplus
+} // extern "C"
+#endif
+
+#endif // SWIFT_EXTERNAL_GENERIC_METADATA_BUILDER_H

include/swift/ExternalGenericMetadataBuilder/module.modulemap

@@ -0,0 +1,4 @@
+module ExternalGenericMetadataBuilder {
+  header "ExternalGenericMetadataBuilder.h"
+  export *
+}

include/swift/RemoteInspection/Records.h

@@ -17,7 +17,7 @@
 #ifndef SWIFT_REFLECTION_RECORDS_H
 #define SWIFT_REFLECTION_RECORDS_H
 
-#include "swift/Basic/RelativePointer.h"
+#include "swift/ABI/TargetLayout.h"
 #include "swift/Demangling/Demangle.h"
 #include "llvm/ADT/ArrayRef.h"
 
@@ -82,14 +82,15 @@ class FieldRecordFlags {
   }
 };
 
-class FieldRecord {
+template <typename Runtime>
+class TargetFieldRecord {
   const FieldRecordFlags Flags;
 
 public:
-  const RelativeDirectPointer<const char> MangledTypeName;
-  const RelativeDirectPointer<const char> FieldName;
+  const TargetRelativeDirectPointer<Runtime, const char> MangledTypeName;
+  const TargetRelativeDirectPointer<Runtime, const char> FieldName;
 
-  FieldRecord() = delete;
+  TargetFieldRecord() = delete;
 
   bool hasMangledTypeName() const {
     return MangledTypeName;
@@ -111,36 +112,37 @@ class FieldRecord {
     return Flags.isVar();
   }
 };
+using FieldRecord = TargetFieldRecord<InProcess>;
 
-struct FieldRecordIterator {
-  const FieldRecord *Cur;
-  const FieldRecord * const End;
+template <typename Runtime>
+struct TargetFieldRecordIterator {
+  const TargetFieldRecord<Runtime> *Cur;
+  const TargetFieldRecord<Runtime> *const End;
 
-  FieldRecordIterator(const FieldRecord *Cur, const FieldRecord * const End)
-    : Cur(Cur), End(End) {}
+  TargetFieldRecordIterator(const TargetFieldRecord<Runtime> *Cur,
+                            const TargetFieldRecord<Runtime> *const End)
+      : Cur(Cur), End(End) {}
 
-  const FieldRecord &operator*() const {
-    return *Cur;
-  }
+  const TargetFieldRecord<Runtime> &operator*() const { return *Cur; }
 
-  const FieldRecord *operator->() const {
-    return Cur;
-  }
+  const TargetFieldRecord<Runtime> *operator->() const { return Cur; }
 
-  FieldRecordIterator &operator++() {
+  TargetFieldRecordIterator &operator++() {
     ++Cur;
     return *this;
   }
 
-  bool operator==(const FieldRecordIterator &other) const {
+  bool operator==(const TargetFieldRecordIterator &other) const {
     return Cur == other.Cur && End == other.End;
   }
 
-  bool operator!=(const FieldRecordIterator &other) const {
+  bool operator!=(const TargetFieldRecordIterator &other) const {
     return !(*this == other);
   }
 };
 
+using FieldRecordIterator = TargetFieldRecordIterator<InProcess>;
+
 enum class FieldDescriptorKind : uint16_t {
   // Swift nominal types.
   Struct,
@@ -173,16 +175,17 @@ enum class FieldDescriptorKind : uint16_t {
 
 // Field descriptors contain a collection of field records for a single
 // class, struct or enum declaration.
-class FieldDescriptor {
-  const FieldRecord *getFieldRecordBuffer() const {
-    return reinterpret_cast<const FieldRecord *>(this + 1);
+template <typename Runtime>
+class TargetFieldDescriptor {
+  const TargetFieldRecord<Runtime> *getFieldRecordBuffer() const {
+    return reinterpret_cast<const TargetFieldRecord<Runtime> *>(this + 1);
   }
 
 public:
-  const RelativeDirectPointer<const char> MangledTypeName;
-  const RelativeDirectPointer<const char> Superclass;
+  const TargetRelativeDirectPointer<Runtime, const char> MangledTypeName;
+  const TargetRelativeDirectPointer<Runtime, const char> Superclass;
 
-  FieldDescriptor() = delete;
+  TargetFieldDescriptor() = delete;
 
   const FieldDescriptorKind Kind;
   const uint16_t FieldRecordSize;
@@ -222,7 +225,7 @@ class FieldDescriptor {
     return const_iterator { End, End };
   }
 
-  llvm::ArrayRef<FieldRecord> getFields() const {
+  llvm::ArrayRef<TargetFieldRecord<Runtime>> getFields() const {
     return {getFieldRecordBuffer(), NumFields};
   }
 
@@ -242,6 +245,7 @@ class FieldDescriptor {
     return Demangle::makeSymbolicMangledNameStringRef(Superclass.get());
   }
 };
+using FieldDescriptor = TargetFieldDescriptor<InProcess>;
 
 // Associated type records describe the mapping from an associated
 // type to the type witness of a conformance.