[Tools] Create swift-inspect, a debugging tool for dumping Swift runtime data from another process

include/swift/Reflection/ReflectionContext.h

@@ -27,6 +27,7 @@
 #include "swift/Remote/MemoryReader.h"
 #include "swift/Remote/MetadataReader.h"
 #include "swift/Reflection/Records.h"
+#include "swift/Reflection/RuntimeInternals.h"
 #include "swift/Reflection/TypeLowering.h"
 #include "swift/Reflection/TypeRef.h"
 #include "swift/Reflection/TypeRefBuilder.h"
@@ -104,7 +105,10 @@ class ReflectionContext
   using super::readMetadataAndValueOpaqueExistential;
   using super::readMetadataFromInstance;
   using super::readTypeFromMetadata;
+  using super::stripSignedPointer;
   using typename super::StoredPointer;
+  using typename super::StoredSignedPointer;
+  using typename super::StoredSize;
 
   explicit ReflectionContext(std::shared_ptr<MemoryReader> reader)
     : super(std::move(reader), *this)
@@ -768,6 +772,156 @@ class ReflectionContext
     }
   }
 
+  /// Iterate the protocol conformance cache tree rooted at NodePtr, calling
+  /// Call with the type and protocol in each node.
+  void iterateConformanceTree(StoredPointer NodePtr,
+    std::function<void(StoredPointer Type, StoredPointer Proto)> Call) {
+    if (!NodePtr)
+      return;
+    auto NodeBytes = getReader().readBytes(RemoteAddress(NodePtr), sizeof(Node));
+    auto NodeData =
+      reinterpret_cast<const ConformanceNode<Runtime> *>(NodeBytes.get());
+    if (!NodeData)
+      return;
+    Call(NodeData->Type, NodeData->Proto);
+    iterateConformanceTree(NodeData->Left, Call);
+    iterateConformanceTree(NodeData->Right, Call);
+  }
+
+  /// Iterate the protocol conformance cache in the target process, calling Call
+  /// with the type and protocol of each conformance. Returns None on success,
+  /// and a string describing the error on failure.
+  llvm::Optional<std::string> iterateConformances(
+    std::function<void(StoredPointer Type, StoredPointer Proto)> Call) {
+    std::string ConformancesPointerName =
+      "__swift_debug_protocolConformanceStatePointer";
+    auto ConformancesAddrAddr =
+      getReader().getSymbolAddress(ConformancesPointerName);
+    if (!ConformancesAddrAddr)
+      return "unable to look up debug variable " + ConformancesPointerName;
+
+    auto ConformancesAddr =
+      getReader().readPointer(ConformancesAddrAddr, sizeof(StoredPointer));
+    if (!ConformancesAddr)
+      return "unable to read value of " + ConformancesPointerName;
+
+    auto Root = getReader().readPointer(ConformancesAddr->getResolvedAddress(),
+                                        sizeof(StoredPointer));
+    iterateConformanceTree(Root->getResolvedAddress().getAddressData(), Call);
+    return llvm::None;
+  }
+
+  /// Fetch the metadata pointer from a metadata allocation, or 0 if this
+  /// allocation's tag is not handled or an error occurred.
+  StoredPointer allocationMetadataPointer(
+    MetadataAllocation<Runtime> Allocation) {
+    if (Allocation.Tag == GenericMetadataCacheTag) {
+        struct GenericMetadataCacheEntry {
+          StoredPointer Left, Right;
+          StoredPointer LockedStorage;
+          uint8_t LockedStorageKind;
+          uint8_t TrackingInfo;
+          uint16_t NumKeyParameters;
+          uint16_t NumWitnessTables;
+          uint32_t Hash;
+          StoredPointer Value;
+        };
+        auto AllocationBytes =
+          getReader().readBytes(RemoteAddress(Allocation.Ptr),
+                                              Allocation.Size);
+        auto Entry = reinterpret_cast<const GenericMetadataCacheEntry *>(
+          AllocationBytes.get());
+        if (!Entry)
+          return 0;
+        return Entry->Value;
+    }
+    return 0;
+  }
+
+  /// Iterate the metadata allocations in the target process, calling Call with
+  /// each allocation found. Returns None on success, and a string describing
+  /// the error on failure.
+  llvm::Optional<std::string> iterateMetadataAllocations(
+    std::function<void (MetadataAllocation<Runtime>)> Call) {
+    std::string IterationEnabledName =
+      "__swift_debug_metadataAllocationIterationEnabled";
+    std::string AllocationPoolPointerName =
+      "__swift_debug_allocationPoolPointer";
+
+    auto IterationEnabledAddr =
+      getReader().getSymbolAddress(IterationEnabledName);
+    if (!IterationEnabledAddr)
+      return "unable to look up debug variable " + IterationEnabledName;
+    char IterationEnabled;
+    if (!getReader().readInteger(IterationEnabledAddr, &IterationEnabled))
+      return "failed to read value of " + IterationEnabledName;
+    if (!IterationEnabled)
+      return std::string("remote process does not have metadata allocation "
+                         "iteration enabled");
+
+    auto AllocationPoolAddrAddr =
+      getReader().getSymbolAddress(AllocationPoolPointerName);
+    if (!AllocationPoolAddrAddr)
+      return "unable to look up debug variable " + AllocationPoolPointerName;
+    auto AllocationPoolAddr =
+      getReader().readPointer(AllocationPoolAddrAddr, sizeof(StoredPointer));
+    if (!AllocationPoolAddr)
+      return "failed to read value of " + AllocationPoolPointerName;
+
+    struct PoolRange {
+      StoredPointer Begin;
+      StoredSize Remaining;
+    };
+    struct PoolTrailer {
+      StoredPointer PrevTrailer;
+      StoredSize PoolSize;
+    };
+    struct alignas(StoredPointer) AllocationHeader {
+      uint16_t Size;
+      uint16_t Tag;
+    };
+
+    auto PoolBytes = getReader()
+      .readBytes(AllocationPoolAddr->getResolvedAddress(), sizeof(PoolRange));
+    auto Pool = reinterpret_cast<const PoolRange *>(PoolBytes.get());
+    if (!Pool)
+      return std::string("failure reading allocation pool contents");
+
+    auto TrailerPtr = Pool->Begin + Pool->Remaining;
+    while (TrailerPtr) {
+      auto TrailerBytes = getReader()
+        .readBytes(RemoteAddress(TrailerPtr), sizeof(PoolTrailer));
+      auto Trailer = reinterpret_cast<const PoolTrailer *>(TrailerBytes.get());
+      if (!Trailer)
+        break;
+      auto PoolStart = TrailerPtr - Trailer->PoolSize;
+      auto PoolBytes = getReader()
+        .readBytes(RemoteAddress(PoolStart), Trailer->PoolSize);
+      auto PoolPtr = (const char *)PoolBytes.get();
+      if (!PoolPtr)
+        break;
+
+      uintptr_t Offset = 0;
+      while (Offset < Trailer->PoolSize) {
+        auto AllocationPtr = PoolPtr + Offset;
+        auto Header = (const AllocationHeader *)AllocationPtr;
+        if (Header->Size == 0)
+          break;
+        auto RemoteAddr = PoolStart + Offset + sizeof(AllocationHeader);
+        MetadataAllocation<Runtime> Allocation;
+        Allocation.Tag = Header->Tag;
+        Allocation.Ptr = RemoteAddr;
+        Allocation.Size = Header->Size;
+        Call(Allocation);
+
+        Offset += sizeof(AllocationHeader) + Header->Size;
+      }
+
+      TrailerPtr = Trailer->PrevTrailer;
+    }
+    return llvm::None;
+  }
+
 private:
   const TypeInfo *getClosureContextInfo(StoredPointer Context,
                                         const ClosureContextInfo &Info) {

include/swift/Reflection/RuntimeInternals.h

@@ -0,0 +1,47 @@
+//===--- RuntimeInternals.h - Runtime Internal Structures -------*- C++ -*-===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+//
+// Runtime data structures that Reflection inspects externally.
+//
+// FIXME: Ideally the original definitions would be templatized on a Runtime
+// parameter and we could use the original definitions in both the runtime and
+// in Reflection.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SWIFT_REFLECTION_RUNTIME_INTERNALS_H
+#define SWIFT_REFLECTION_RUNTIME_INTERNALS_H
+
+namespace swift {
+
+namespace reflection {
+
+template <typename Runtime>
+struct ConformanceNode {
+  typename Runtime::StoredPointer Left, Right;
+  typename Runtime::StoredPointer Type;
+  typename Runtime::StoredPointer Proto;
+  typename Runtime::StoredPointer Description;
+  typename Runtime::StoredSize FailureGeneration;
+};
+
+template <typename Runtime>
+struct MetadataAllocation {
+  uint16_t Tag;
+  typename Runtime::StoredPointer Ptr;
+  unsigned Size;
+};
+
+} // end namespace reflection
+} // end namespace swift
+
+#endif // SWIFT_REFLECTION_RUNTIME_INTERNALS_H

include/swift/Remote/MetadataReader.h

@@ -2192,6 +2192,8 @@ class MetadataReader {
         readMangledName(RemoteAddress(extendedContextAddress),
                         MangledNameKind::Type,
                         dem);
+      if (!demangledExtendedContext)
+        return nullptr;
 
       auto demangling = dem.createNode(Node::Kind::Extension);
       demangling->addChild(parentDemangling, dem);

include/swift/Runtime/Debug.h

@@ -231,6 +231,14 @@ bool _swift_reportFatalErrorsToDebugger;
 SWIFT_RUNTIME_STDLIB_SPI
 bool _swift_shouldReportFatalErrorsToDebugger();
 
+SWIFT_RUNTIME_STDLIB_SPI
+bool _swift_debug_metadataAllocationIterationEnabled;
+
+SWIFT_RUNTIME_STDLIB_SPI
+const void * const _swift_debug_allocationPoolPointer;
+
+SWIFT_RUNTIME_STDLIB_SPI
+const void * const _swift_debug_protocolConformanceStatePointer;
 
 SWIFT_RUNTIME_ATTRIBUTE_ALWAYS_INLINE
 inline static int swift_asprintf(char **strp, const char *fmt, ...) {

include/swift/Runtime/Metadata.h

@@ -25,6 +25,35 @@ namespace swift {
 #pragma clang diagnostic push
 #pragma clang diagnostic ignored "-Wreturn-type-c-linkage"
 
+// Tags used to denote different kinds of allocations made with the metadata
+// allocator. This is encoded in a header on each allocation when metadata
+// iteration is enabled, and allows tools to know where each allocation came
+// from.
+//
+// Some of these values are also declared in SwiftRemoteMirrorTypes.h. Those
+// values must be kept stable to preserve compatibility.
+enum MetadataAllocatorTags : uint16_t {
+  UnusedTag = 0,
+  BoxesTag,
+  ObjCClassWrappersTag,
+  FunctionTypesTag,
+  MetatypeTypesTag,
+  ExistentialMetatypeValueWitnessTablesTag,
+  ExistentialMetatypesTag,
+  ExistentialTypesTag,
+  OpaqueExistentialValueWitnessTablesTag,
+  ClassExistentialValueWitnessTablesTag,
+  ForeignWitnessTablesTag,
+  ResilientMetadataAllocatorTag,
+  MetadataTag,
+  TupleCacheTag,
+  GenericMetadataCacheTag,
+  ForeignMetadataCacheTag,
+  GenericWitnessTableCacheTag,
+  GenericClassMetadataTag,
+  GenericValueMetadataTag,
+};
+
 /// The buffer used by a yield-once coroutine (such as the generalized
 /// accessors `read` and `modify`).
 struct YieldOnceBuffer {

include/swift/SwiftRemoteMirror/CMakeLists.txt

@@ -3,7 +3,8 @@ list(APPEND swift_remote_mirror_headers
        MemoryReaderInterface.h
        Platform.h
        SwiftRemoteMirror.h
-       SwiftRemoteMirrorTypes.h)
+       SwiftRemoteMirrorTypes.h
+       module.modulemap)
 swift_install_in_component(FILES
                              ${swift_remote_mirror_headers}
                            DESTINATION

include/swift/SwiftRemoteMirror/SwiftRemoteMirror.h

@@ -159,6 +159,11 @@ char *
 swift_reflection_copyDemangledNameForTypeRef(
   SwiftReflectionContextRef ContextRef, swift_typeref_t OpaqueTypeRef);
 
+SWIFT_REMOTE_MIRROR_LINKAGE
+char *
+swift_reflection_copyDemangledNameForProtocolDescriptor(
+  SwiftReflectionContextRef ContextRef, swift_reflection_ptr_t Proto);
+
 /// Returns a structure describing the layout of a value of a typeref.
 /// For classes, this returns the reference value itself.
 SWIFT_REMOTE_MIRROR_LINKAGE
@@ -283,6 +288,49 @@ SWIFT_REMOTE_MIRROR_LINKAGE
 size_t swift_reflection_demangle(const char *MangledName, size_t Length,
                                  char *OutDemangledName, size_t MaxLength);
 
+/// Iterate over the process's protocol conformance cache.
+///
+/// Calls the passed in Call function for each protocol conformance found in
+/// the conformance cache. The function is passed the type which conforms and
+/// the protocol it conforms to. The ContextPtr is passed through unchanged.
+///
+/// Returns NULL on success. On error, returns a pointer to a C string
+/// describing the error. This pointer remains valid until the next
+/// swift_reflection call on the given context.
+SWIFT_REMOTE_MIRROR_LINKAGE
+const char *swift_reflection_iterateConformanceCache(
+  SwiftReflectionContextRef ContextRef,
+  void (*Call)(swift_reflection_ptr_t Type,
+               swift_reflection_ptr_t Proto,
+               void *ContextPtr),
+  void *ContextPtr);
+
+/// Iterate over the process's metadata allocations.
+///
+/// Calls the passed in Call function for each metadata allocation. The function
+/// is passed a structure that describes the allocation. The ContextPtr is
+/// passed through unchanged.
+///
+/// Returns NULL on success. On error, returns a pointer to a C string
+/// describing the error. This pointer remains valid until the next
+/// swift_reflection call on the given context.
+SWIFT_REMOTE_MIRROR_LINKAGE
+const char *swift_reflection_iterateMetadataAllocations(
+  SwiftReflectionContextRef ContextRef,
+  void (*Call)(swift_metadata_allocation_t Allocation,
+               void *ContextPtr),
+  void *ContextPtr);
+
+/// Given a metadata allocation, return the metadata it points to. Returns NULL
+/// on failure. Despite the name, not all allocations point to metadata.
+/// Currently, this will return a metadata only for allocations with tag
+/// SWIFT_GENERIC_METADATA_CACHE_ALLOCATION. Support for additional tags may be
+/// added in the future. The caller must gracefully handle failure.
+SWIFT_REMOTE_MIRROR_LINKAGE
+swift_reflection_ptr_t swift_reflection_allocationMetadataPointer(
+  SwiftReflectionContextRef ContextRef,
+  swift_metadata_allocation_t Allocation);
+
 #ifdef __cplusplus
 } // extern "C"
 #endif

include/swift/SwiftRemoteMirror/SwiftRemoteMirrorTypes.h

@@ -141,6 +141,27 @@ typedef struct swift_childinfo {
   swift_typeref_t TR;
 } swift_childinfo_t;
 
+// Values here match the values from MetadataAllocatorTags in Metadata.h.
+enum swift_metadata_allocation_tag {
+  SWIFT_GENERIC_METADATA_CACHE_ALLOCATION = 14,
+};
+
+typedef int swift_metadata_allocation_tag_t;
+
+/// A metadata allocation made by the Swift runtime.
+typedef struct swift_metadata_allocation {
+  /// The allocation's tag, which describes what kind of allocation it is. This
+  /// may be one of the values in swift_metadata_allocation_tag, or something
+  /// else, in which case the tag should be considered unknown.
+  swift_metadata_allocation_tag_t Tag;
+
+  /// A pointer to the start of the allocation in the remote process.
+  swift_reflection_ptr_t Ptr;
+
+  /// The size of the allocation in bytes.
+  unsigned Size;
+} swift_metadata_allocation_t;
+
 /// An opaque pointer to a context which maintains state and
 /// caching of reflection structure for heap instances.
 typedef struct SwiftReflectionContext *SwiftReflectionContextRef;

include/swift/SwiftRemoteMirror/module.modulemap

@@ -0,0 +1,4 @@
+module SwiftRemoteMirror {
+  header "SwiftRemoteMirror.h"
+  export *
+}

lib/Demangling/Remangler.cpp

@@ -268,8 +268,8 @@ class Remangler : public RemanglerBase {
   }
 
   void mangleChildNode(Node *node, unsigned index) {
-    assert(index < node->getNumChildren());
-    mangle(node->begin()[index]);
+    if (index < node->getNumChildren())
+      mangle(node->begin()[index]);
   }
 
   void manglePureProtocol(Node *Proto) {