[lldb] Implement cache for swift typeref metadata

Implement a mechanism to cache field descriptor location information.
One of the main problems of using type metadata as debug info, when in
comes to field descriptors in particular, is that there's no easy way
to find the correct type given a (string key), so a linear search over
every source of reflection information must be done. This patch
implements a cache which allows for field descriptors to be found in a
more efficient manner.

Author: augusto2112

lldb/include/lldb/Core/ModuleList.h

@@ -65,6 +65,12 @@ class ModuleListProperties : public Properties {
   bool GetSwiftValidateTypeSystem() const;
   SwiftModuleLoadingMode GetSwiftModuleLoadingMode() const;
   bool SetSwiftModuleLoadingMode(SwiftModuleLoadingMode);
+
+  bool GetEnableSwiftMetadataCache() const;
+  uint64_t GetSwiftMetadataCacheMaxByteSize();
+  uint64_t GetSwiftMetadataCacheExpirationDays();
+  FileSpec GetSwiftMetadataCachePath() const;
+  bool SetSwiftMetadataCachePath(const FileSpec &path);
   // END SWIFT
 
   FileSpec GetClangModulesCachePath() const;

lldb/packages/Python/lldbsuite/test/lldbtest.py

@@ -735,6 +735,12 @@ def setUpCommands(cls):
             'settings set symbols.clang-modules-cache-path "{}"'.format(
                 configuration.lldb_module_cache_dir),
 
+            # Enable the swift metadata cache in order to speed up tests.
+            'settings set symbols.enable-swift-metadata-cache true',
+
+            'settings set symbols.swift-metadata-cache-path "{}"'.format(
+                configuration.lldb_module_cache_dir),
+
             # Enable expensive validations in TypeSystemSwiftTypeRef.
             'settings set symbols.swift-validate-typesystem true',
 

lldb/source/Core/CoreProperties.td

@@ -30,6 +30,22 @@ let Definition = "modulelist" in {
     DefaultEnumValue<"eSwiftModuleLoadingModePreferSerialized">,
     EnumValues<"OptionEnumValues(g_swift_module_loading_mode_enums)">,
     Desc<"The module loading mode to use when loading modules for Swift.">;
+  def EnableSwiftMetadataCache: Property<"enable-swift-metadata-cache", "Boolean">,
+    Global,
+    DefaultFalse,
+    Desc<"Enable caching for Swift reflection metadata in LLDB.">;
+  def SwiftMetadataCachePath: Property<"swift-metadata-cache-path", "FileSpec">,
+    Global,
+    DefaultStringValue<"">,
+    Desc<"The path to LLDB's Swift reflection cache directory.">;
+  def SwiftMetadataCacheMaxByteSize: Property<"swift-metadata-cache-max-byte-size", "UInt64">,
+    Global,
+    DefaultUnsignedValue<0>,
+    Desc<"The maximum size for LLDB's Swift reflection cache directory in bytes. A value over the amount of available space on the disk will be reduced to the amount of available space. A value of 0 disables the absolute size-based pruning.">;
+  def SwiftMetadataCacheExpirationDays: Property<"swift-metadata-cache-expiration-days", "UInt64">,
+    Global,
+    DefaultUnsignedValue<7>,
+    Desc<"The expiration time in days for a Swift reflection cache file. When a file hasn't been accessed for the specified amount of days, it is removed from the cache. A value of 0 disables the expiration-based pruning.">;
 // END SWIFT
   def SymLinkPaths: Property<"debug-info-symlink-paths", "FileSpecList">,
     Global,

lldb/source/Core/ModuleList.cpp

@@ -108,6 +108,13 @@ ModuleListProperties::ModuleListProperties() {
 
   // BEGIN SWIFT
   SetSwiftModuleLoadingMode(eSwiftModuleLoadingModePreferSerialized);
+
+  path.clear();
+  if (llvm::sys::path::cache_directory(path)) {
+    llvm::sys::path::append(path, "lldb");
+    llvm::sys::path::append(path, "SwiftMetadataCache");
+    lldbassert(SetLLDBIndexCachePath(FileSpec(path)));
+  }
   // END SWIFT
 
   path.clear();
@@ -194,6 +201,36 @@ bool ModuleListProperties::SetSwiftModuleLoadingMode(SwiftModuleLoadingMode mode
   return m_collection_sp->SetPropertyAtIndexAsEnumeration(
       nullptr, ePropertySwiftModuleLoadingMode, mode);
 }
+
+FileSpec ModuleListProperties::GetSwiftMetadataCachePath() const {
+  return m_collection_sp
+      ->GetPropertyAtIndexAsOptionValueFileSpec(nullptr, false,
+                                                ePropertySwiftMetadataCachePath)
+      ->GetCurrentValue();
+}
+
+bool ModuleListProperties::SetSwiftMetadataCachePath(const FileSpec &path) {
+  return m_collection_sp->SetPropertyAtIndexAsFileSpec(
+      nullptr, ePropertySwiftMetadataCachePath, path);
+}
+
+bool ModuleListProperties::GetEnableSwiftMetadataCache() const {
+  const uint32_t idx = ePropertyEnableSwiftMetadataCache;
+  return m_collection_sp->GetPropertyAtIndexAsBoolean(
+      nullptr, idx, g_modulelist_properties[idx].default_uint_value != 0);
+}
+
+uint64_t ModuleListProperties::GetSwiftMetadataCacheMaxByteSize() {
+  const uint32_t idx = ePropertySwiftMetadataCacheMaxByteSize;
+  return m_collection_sp->GetPropertyAtIndexAsUInt64(
+      nullptr, idx, g_modulelist_properties[idx].default_uint_value);
+}
+
+uint64_t ModuleListProperties::GetSwiftMetadataCacheExpirationDays() {
+  const uint32_t idx = ePropertySwiftMetadataCacheExpirationDays;
+  return m_collection_sp->GetPropertyAtIndexAsUInt64(
+      nullptr, idx, g_modulelist_properties[idx].default_uint_value);
+}
 // END SWIFT
 
 FileSpec ModuleListProperties::GetLLDBIndexCachePath() const {

lldb/source/Plugins/LanguageRuntime/Swift/CMakeLists.txt

@@ -3,6 +3,7 @@ add_lldb_library(lldbPluginSwiftLanguageRuntime PLUGIN
   SwiftLanguageRuntime.cpp
   SwiftLanguageRuntimeDynamicTypeResolution.cpp
   SwiftLanguageRuntimeNames.cpp
+  SwiftMetadataCache.cpp
 
   LINK_LIBS
     swiftAST

lldb/source/Plugins/LanguageRuntime/Swift/SwiftLanguageRuntime.cpp

@@ -13,6 +13,7 @@
 #include "SwiftLanguageRuntime.h"
 #include "Plugins/LanguageRuntime/Swift/LLDBMemoryReader.h"
 #include "SwiftLanguageRuntimeImpl.h"
+#include "SwiftMetadataCache.h"
 
 #include "Plugins/Process/Utility/RegisterContext_x86.h"
 #include "Plugins/TypeSystem/Clang/TypeSystemClang.h"
@@ -442,6 +443,13 @@ SwiftLanguageRuntimeImpl::GetReflectionContext() {
   return m_reflection_ctx.get();
 }
 
+SwiftMetadataCache *
+SwiftLanguageRuntimeImpl::GetSwiftMetadataCache() {
+  if (!m_swift_metadata_cache.is_enabled())
+    return {};
+  return &m_swift_metadata_cache;
+}
+
 void SwiftLanguageRuntimeImpl::SetupReflection() {
   LLDB_SCOPED_TIMER();
 
@@ -481,13 +489,13 @@ void SwiftLanguageRuntimeImpl::SetupReflection() {
               "Initializing a 64-bit reflection context (%s) for \"%s\"",
               triple.str().c_str(), objc_interop_msg);
     m_reflection_ctx = ReflectionContextInterface::CreateReflectionContext64(
-        this->GetMemoryReader(), objc_interop);
+        this->GetMemoryReader(), objc_interop, GetSwiftMetadataCache());
   } else if (triple.isArch32Bit()) {
     LLDB_LOGF(GetLog(LLDBLog::Types),
               "Initializing a 32-bit reflection context (%s) for \"%s\"",
               triple.str().c_str(), objc_interop_msg);
     m_reflection_ctx = ReflectionContextInterface::CreateReflectionContext32(
-        this->GetMemoryReader(), objc_interop);
+        this->GetMemoryReader(), objc_interop, GetSwiftMetadataCache());
   } else {
     LLDB_LOGF(GetLog(LLDBLog::Types),
               "Could not initialize reflection context for \"%s\"",
@@ -649,7 +657,7 @@ bool SwiftLanguageRuntimeImpl::AddJitObjectFileToReflectionContext(
   if (!obj_file_format)
     return false;
 
-  return m_reflection_ctx->addImage(
+  auto reflection_info_id = m_reflection_ctx->addImage(
       [&](swift::ReflectionSectionKind section_kind)
           -> std::pair<swift::remote::RemoteRef<void>, uint64_t> {
         auto section_name = obj_file_format->getSectionName(section_kind);
@@ -676,9 +684,13 @@ bool SwiftLanguageRuntimeImpl::AddJitObjectFileToReflectionContext(
         return {};
       },
       likely_module_names);
+  // We don't care to cache modules generated by the jit, because they will
+  // only be used by the current process.
+  return reflection_info_id.hasValue();
 }
 
-bool SwiftLanguageRuntimeImpl::AddObjectFileToReflectionContext(
+llvm::Optional<uint32_t>
+SwiftLanguageRuntimeImpl::AddObjectFileToReflectionContext(
     ModuleSP module,
     llvm::SmallVector<llvm::StringRef, 1> likely_module_names) {
   auto obj_format_type =
@@ -856,27 +868,35 @@ bool SwiftLanguageRuntimeImpl::AddModuleToReflectionContext(
 
   auto read_from_file_cache =
       GetMemoryReader()->readMetadataFromFileCacheEnabled();
+
+  llvm::Optional<uint32_t> info_id;
   // When dealing with ELF, we need to pass in the contents of the on-disk
   // file, since the Section Header Table is not present in the child process
   if (obj_file->GetPluginName().equals("elf")) {
     DataExtractor extractor;
     auto size = obj_file->GetData(0, obj_file->GetByteSize(), extractor);
     const uint8_t *file_data = extractor.GetDataStart();
     llvm::sys::MemoryBlock file_buffer((void *)file_data, size);
-    m_reflection_ctx->readELF(
+    info_id = m_reflection_ctx->readELF(
         swift::remote::RemoteAddress(load_ptr),
         llvm::Optional<llvm::sys::MemoryBlock>(file_buffer),
         likely_module_names);
   } else if (read_from_file_cache &&
              obj_file->GetPluginName().equals("mach-o")) {
-    if (!AddObjectFileToReflectionContext(module_sp, likely_module_names)) {
-      m_reflection_ctx->addImage(swift::remote::RemoteAddress(load_ptr),
+    info_id = AddObjectFileToReflectionContext(module_sp, likely_module_names);
+    if (!info_id)
+      info_id = m_reflection_ctx->addImage(swift::remote::RemoteAddress(load_ptr),
                                  likely_module_names);
-    }
   } else {
-    m_reflection_ctx->addImage(swift::remote::RemoteAddress(load_ptr),
+    info_id = m_reflection_ctx->addImage(swift::remote::RemoteAddress(load_ptr),
                                likely_module_names);
   }
+
+  if (info_id)
+    if (auto *swift_metadata_cache = GetSwiftMetadataCache())
+      swift_metadata_cache->registerModuleWithReflectionInfoID(module_sp,
+                                                               *info_id);
+
   return true;
 }
 

lldb/source/Plugins/LanguageRuntime/Swift/SwiftLanguageRuntimeDynamicTypeResolution.cpp

@@ -11,8 +11,9 @@
 //===----------------------------------------------------------------------===//
 
 #include "LLDBMemoryReader.h"
-#include "SwiftLanguageRuntimeImpl.h"
 #include "SwiftLanguageRuntime.h"
+#include "SwiftLanguageRuntimeImpl.h"
+#include "SwiftMetadataCache.h"
 
 #include "Plugins/ExpressionParser/Clang/ClangUtil.h"
 #include "Plugins/TypeSystem/Clang/TypeSystemClang.h"
@@ -24,12 +25,13 @@
 #include "lldb/Utility/LLDBLog.h"
 #include "lldb/Utility/Log.h"
 #include "lldb/Utility/Timer.h"
-#include "swift/AST/Types.h"
+#include "llvm/ADT/STLExtras.h"
 
 #include "swift/AST/ASTContext.h"
 #include "swift/AST/ASTMangler.h"
 #include "swift/AST/ASTWalker.h"
 #include "swift/AST/Decl.h"
+#include "swift/AST/Types.h"
 #include "swift/Demangling/Demangle.h"
 #include "swift/Reflection/ReflectionContext.h"
 #include "swift/Reflection/TypeRefBuilder.h"
@@ -241,24 +243,25 @@ class TargetReflectionContext
 
 public:
   TargetReflectionContext(
-      std::shared_ptr<swift::reflection::MemoryReader> reader)
-      : m_reflection_ctx(reader) {}
+      std::shared_ptr<swift::reflection::MemoryReader> reader,
+      SwiftMetadataCache *swift_metadata_cache)
+      : m_reflection_ctx(reader, swift_metadata_cache) {}
 
-  bool addImage(
+  llvm::Optional<uint32_t> addImage(
       llvm::function_ref<std::pair<swift::remote::RemoteRef<void>, uint64_t>(
           swift::ReflectionSectionKind)>
           find_section,
       llvm::SmallVector<llvm::StringRef, 1> likely_module_names) override {
     return m_reflection_ctx.addImage(find_section, likely_module_names);
   }
 
-  bool
+  llvm::Optional<uint32_t>
   addImage(swift::remote::RemoteAddress image_start,
            llvm::SmallVector<llvm::StringRef, 1> likely_module_names) override {
     return m_reflection_ctx.addImage(image_start, likely_module_names);
   }
 
-  bool readELF(
+  llvm::Optional<uint32_t> readELF(
       swift::remote::RemoteAddress ImageStart,
       llvm::Optional<llvm::sys::MemoryBlock> FileBuffer,
       llvm::SmallVector<llvm::StringRef, 1> likely_module_names = {}) override {
@@ -361,30 +364,36 @@ class TargetReflectionContext
 
 std::unique_ptr<SwiftLanguageRuntimeImpl::ReflectionContextInterface>
 SwiftLanguageRuntimeImpl::ReflectionContextInterface::CreateReflectionContext32(
-    std::shared_ptr<swift::remote::MemoryReader> reader, bool ObjCInterop) {
+    std::shared_ptr<swift::remote::MemoryReader> reader, bool ObjCInterop,
+    SwiftMetadataCache *swift_metadata_cache) {
   using ReflectionContext32ObjCInterop =
       TargetReflectionContext<swift::reflection::ReflectionContext<
           swift::External<swift::WithObjCInterop<swift::RuntimeTarget<4>>>>>;
   using ReflectionContext32NoObjCInterop =
       TargetReflectionContext<swift::reflection::ReflectionContext<
           swift::External<swift::NoObjCInterop<swift::RuntimeTarget<4>>>>>;
-  if (ObjCInterop)    
-    return std::make_unique<ReflectionContext32ObjCInterop>(reader);
-  return std::make_unique<ReflectionContext32NoObjCInterop>(reader);
+  if (ObjCInterop)
+    return std::make_unique<ReflectionContext32ObjCInterop>(
+        reader, swift_metadata_cache);
+  return std::make_unique<ReflectionContext32NoObjCInterop>(
+      reader, swift_metadata_cache);
 }
 
 std::unique_ptr<SwiftLanguageRuntimeImpl::ReflectionContextInterface>
 SwiftLanguageRuntimeImpl::ReflectionContextInterface::CreateReflectionContext64(
-    std::shared_ptr<swift::remote::MemoryReader> reader, bool ObjCInterop) {
+    std::shared_ptr<swift::remote::MemoryReader> reader, bool ObjCInterop,
+    SwiftMetadataCache *swift_metadata_cache) {
   using ReflectionContext64ObjCInterop =
       TargetReflectionContext<swift::reflection::ReflectionContext<
           swift::External<swift::WithObjCInterop<swift::RuntimeTarget<8>>>>>;
   using ReflectionContext64NoObjCInterop =
       TargetReflectionContext<swift::reflection::ReflectionContext<
           swift::External<swift::NoObjCInterop<swift::RuntimeTarget<8>>>>>;
   if (ObjCInterop)
-    return std::make_unique<ReflectionContext64ObjCInterop>(reader);
-  return std::make_unique<ReflectionContext64NoObjCInterop>(reader);
+    return std::make_unique<ReflectionContext64ObjCInterop>(
+        reader, swift_metadata_cache);
+  return std::make_unique<ReflectionContext64NoObjCInterop>(
+      reader, swift_metadata_cache);
 }
 
 SwiftLanguageRuntimeImpl::ReflectionContextInterface::

lldb/source/Plugins/LanguageRuntime/Swift/SwiftLanguageRuntimeImpl.h

@@ -15,6 +15,7 @@
 
 #include "LLDBMemoryReader.h"
 #include "SwiftLanguageRuntime.h"
+#include "SwiftMetadataCache.h"
 #include "swift/Reflection/TypeLowering.h"
 #include "llvm/Support/Memory.h"
 
@@ -201,24 +202,26 @@ class SwiftLanguageRuntimeImpl {
     /// Return a 32-bit reflection context.
     static std::unique_ptr<ReflectionContextInterface>
     CreateReflectionContext32(
-        std::shared_ptr<swift::remote::MemoryReader> reader, bool ObjCInterop);
+        std::shared_ptr<swift::remote::MemoryReader> reader, bool ObjCInterop,
+        SwiftMetadataCache *swift_metadata_cache);
 
     /// Return a 64-bit reflection context.
     static std::unique_ptr<ReflectionContextInterface>
     CreateReflectionContext64(
-        std::shared_ptr<swift::remote::MemoryReader> reader, bool ObjCInterop);
+        std::shared_ptr<swift::remote::MemoryReader> reader, bool ObjCInterop,
+        SwiftMetadataCache *swift_metadata_cache);
 
     virtual ~ReflectionContextInterface();
 
-    virtual bool addImage(
+    virtual llvm::Optional<uint32_t> addImage(
         llvm::function_ref<std::pair<swift::remote::RemoteRef<void>, uint64_t>(
             swift::ReflectionSectionKind)>
             find_section,
         llvm::SmallVector<llvm::StringRef, 1> likely_module_names = {}) = 0;
-    virtual bool addImage(
+    virtual llvm::Optional<uint32_t> addImage(
         swift::remote::RemoteAddress image_start,
         llvm::SmallVector<llvm::StringRef, 1> likely_module_names = {}) = 0;
-    virtual bool
+    virtual llvm::Optional<uint32_t>
     readELF(swift::remote::RemoteAddress ImageStart,
             llvm::Optional<llvm::sys::MemoryBlock> FileBuffer,
             llvm::SmallVector<llvm::StringRef, 1> likely_module_names = {}) = 0;
@@ -365,6 +368,8 @@ class SwiftLanguageRuntimeImpl {
   /// Lazily initialize and return \p m_SwiftNativeNSErrorISA.
   llvm::Optional<lldb::addr_t> GetSwiftNativeNSErrorISA();
 
+  SwiftMetadataCache *GetSwiftMetadataCache();
+
   /// These members are used to track and toggle the state of the "dynamic
   /// exclusivity enforcement flag" in the swift runtime. This flag is set to
   /// true when an LLDB expression starts running, and reset to its original
@@ -382,6 +387,8 @@ class SwiftLanguageRuntimeImpl {
   /// \{
   std::unique_ptr<ReflectionContextInterface> m_reflection_ctx;
 
+  SwiftMetadataCache m_swift_metadata_cache;
+
   /// Record modules added through ModulesDidLoad, which are to be
   /// added to the reflection context once it's being initialized.
   ModuleList m_modules_to_add;
@@ -400,8 +407,9 @@ class SwiftLanguageRuntimeImpl {
 
   /// Add the reflections sections to the reflection context by extracting
   /// the directly from the object file.
-  /// \return true on success.
-  bool AddObjectFileToReflectionContext(
+  /// \return the info id of the newly registered reflection info on success, or
+  /// llvm::None otherwise.
+  llvm::Optional<uint32_t> AddObjectFileToReflectionContext(
       lldb::ModuleSP module,
       llvm::SmallVector<llvm::StringRef, 1> likely_module_names);