Merge pull request #5100 from augusto2112/swift-meta-cache

[lldb] Implement cache for swift typeref metadata

Author: augusto2112

lldb/include/lldb/Core/ModuleList.h

@@ -65,6 +65,12 @@ class ModuleListProperties : public Properties {
   bool GetSwiftValidateTypeSystem() const;
   SwiftModuleLoadingMode GetSwiftModuleLoadingMode() const;
   bool SetSwiftModuleLoadingMode(SwiftModuleLoadingMode);
+
+  bool GetEnableSwiftMetadataCache() const;
+  uint64_t GetSwiftMetadataCacheMaxByteSize();
+  uint64_t GetSwiftMetadataCacheExpirationDays();
+  FileSpec GetSwiftMetadataCachePath() const;
+  bool SetSwiftMetadataCachePath(const FileSpec &path);
   // END SWIFT
 
   FileSpec GetClangModulesCachePath() const;

lldb/packages/Python/lldbsuite/test/lldbtest.py

@@ -735,6 +735,12 @@ def setUpCommands(cls):
             'settings set symbols.clang-modules-cache-path "{}"'.format(
                 configuration.lldb_module_cache_dir),
 
+            # Enable the swift metadata cache in order to speed up tests.
+            'settings set symbols.enable-swift-metadata-cache true',
+
+            'settings set symbols.swift-metadata-cache-path "{}"'.format(
+                configuration.lldb_module_cache_dir),
+
             # Enable expensive validations in TypeSystemSwiftTypeRef.
             'settings set symbols.swift-validate-typesystem true',
 

lldb/source/Core/CoreProperties.td

@@ -30,6 +30,22 @@ let Definition = "modulelist" in {
     DefaultEnumValue<"eSwiftModuleLoadingModePreferSerialized">,
     EnumValues<"OptionEnumValues(g_swift_module_loading_mode_enums)">,
     Desc<"The module loading mode to use when loading modules for Swift.">;
+  def EnableSwiftMetadataCache: Property<"enable-swift-metadata-cache", "Boolean">,
+    Global,
+    DefaultFalse,
+    Desc<"Enable caching for Swift reflection metadata in LLDB.">;
+  def SwiftMetadataCachePath: Property<"swift-metadata-cache-path", "FileSpec">,
+    Global,
+    DefaultStringValue<"">,
+    Desc<"The path to LLDB's Swift reflection cache directory.">;
+  def SwiftMetadataCacheMaxByteSize: Property<"swift-metadata-cache-max-byte-size", "UInt64">,
+    Global,
+    DefaultUnsignedValue<0>,
+    Desc<"The maximum size for LLDB's Swift reflection cache directory in bytes. A value over the amount of available space on the disk will be reduced to the amount of available space. A value of 0 disables the absolute size-based pruning.">;
+  def SwiftMetadataCacheExpirationDays: Property<"swift-metadata-cache-expiration-days", "UInt64">,
+    Global,
+    DefaultUnsignedValue<7>,
+    Desc<"The expiration time in days for a Swift reflection cache file. When a file hasn't been accessed for the specified amount of days, it is removed from the cache. A value of 0 disables the expiration-based pruning.">;
 // END SWIFT
   def SymLinkPaths: Property<"debug-info-symlink-paths", "FileSpecList">,
     Global,

lldb/source/Core/ModuleList.cpp

@@ -108,6 +108,13 @@ ModuleListProperties::ModuleListProperties() {
 
   // BEGIN SWIFT
   SetSwiftModuleLoadingMode(eSwiftModuleLoadingModePreferSerialized);
+
+  path.clear();
+  if (llvm::sys::path::cache_directory(path)) {
+    llvm::sys::path::append(path, "lldb");
+    llvm::sys::path::append(path, "SwiftMetadataCache");
+    lldbassert(SetLLDBIndexCachePath(FileSpec(path)));
+  }
   // END SWIFT
 
   path.clear();
@@ -194,6 +201,36 @@ bool ModuleListProperties::SetSwiftModuleLoadingMode(SwiftModuleLoadingMode mode
   return m_collection_sp->SetPropertyAtIndexAsEnumeration(
       nullptr, ePropertySwiftModuleLoadingMode, mode);
 }
+
+FileSpec ModuleListProperties::GetSwiftMetadataCachePath() const {
+  return m_collection_sp
+      ->GetPropertyAtIndexAsOptionValueFileSpec(nullptr, false,
+                                                ePropertySwiftMetadataCachePath)
+      ->GetCurrentValue();
+}
+
+bool ModuleListProperties::SetSwiftMetadataCachePath(const FileSpec &path) {
+  return m_collection_sp->SetPropertyAtIndexAsFileSpec(
+      nullptr, ePropertySwiftMetadataCachePath, path);
+}
+
+bool ModuleListProperties::GetEnableSwiftMetadataCache() const {
+  const uint32_t idx = ePropertyEnableSwiftMetadataCache;
+  return m_collection_sp->GetPropertyAtIndexAsBoolean(
+      nullptr, idx, g_modulelist_properties[idx].default_uint_value != 0);
+}
+
+uint64_t ModuleListProperties::GetSwiftMetadataCacheMaxByteSize() {
+  const uint32_t idx = ePropertySwiftMetadataCacheMaxByteSize;
+  return m_collection_sp->GetPropertyAtIndexAsUInt64(
+      nullptr, idx, g_modulelist_properties[idx].default_uint_value);
+}
+
+uint64_t ModuleListProperties::GetSwiftMetadataCacheExpirationDays() {
+  const uint32_t idx = ePropertySwiftMetadataCacheExpirationDays;
+  return m_collection_sp->GetPropertyAtIndexAsUInt64(
+      nullptr, idx, g_modulelist_properties[idx].default_uint_value);
+}
 // END SWIFT
 
 FileSpec ModuleListProperties::GetLLDBIndexCachePath() const {

lldb/source/Plugins/LanguageRuntime/Swift/CMakeLists.txt

@@ -3,6 +3,7 @@ add_lldb_library(lldbPluginSwiftLanguageRuntime PLUGIN
   SwiftLanguageRuntime.cpp
   SwiftLanguageRuntimeDynamicTypeResolution.cpp
   SwiftLanguageRuntimeNames.cpp
+  SwiftMetadataCache.cpp
 
   LINK_LIBS
     swiftAST

lldb/source/Plugins/LanguageRuntime/Swift/SwiftLanguageRuntime.cpp

@@ -13,6 +13,7 @@
 #include "SwiftLanguageRuntime.h"
 #include "Plugins/LanguageRuntime/Swift/LLDBMemoryReader.h"
 #include "SwiftLanguageRuntimeImpl.h"
+#include "SwiftMetadataCache.h"
 
 #include "Plugins/Process/Utility/RegisterContext_x86.h"
 #include "Plugins/TypeSystem/Clang/TypeSystemClang.h"
@@ -442,6 +443,13 @@ SwiftLanguageRuntimeImpl::GetReflectionContext() {
   return m_reflection_ctx.get();
 }
 
+SwiftMetadataCache *
+SwiftLanguageRuntimeImpl::GetSwiftMetadataCache() {
+  if (!m_swift_metadata_cache.is_enabled())
+    return {};
+  return &m_swift_metadata_cache;
+}
+
 void SwiftLanguageRuntimeImpl::SetupReflection() {
   LLDB_SCOPED_TIMER();
 
@@ -481,13 +489,13 @@ void SwiftLanguageRuntimeImpl::SetupReflection() {
               "Initializing a 64-bit reflection context (%s) for \"%s\"",
               triple.str().c_str(), objc_interop_msg);
     m_reflection_ctx = ReflectionContextInterface::CreateReflectionContext64(
-        this->GetMemoryReader(), objc_interop);
+        this->GetMemoryReader(), objc_interop, GetSwiftMetadataCache());
   } else if (triple.isArch32Bit()) {
     LLDB_LOGF(GetLog(LLDBLog::Types),
               "Initializing a 32-bit reflection context (%s) for \"%s\"",
               triple.str().c_str(), objc_interop_msg);
     m_reflection_ctx = ReflectionContextInterface::CreateReflectionContext32(
-        this->GetMemoryReader(), objc_interop);
+        this->GetMemoryReader(), objc_interop, GetSwiftMetadataCache());
   } else {
     LLDB_LOGF(GetLog(LLDBLog::Types),
               "Could not initialize reflection context for \"%s\"",
@@ -649,7 +657,7 @@ bool SwiftLanguageRuntimeImpl::AddJitObjectFileToReflectionContext(
   if (!obj_file_format)
     return false;
 
-  return m_reflection_ctx->addImage(
+  auto reflection_info_id = m_reflection_ctx->addImage(
       [&](swift::ReflectionSectionKind section_kind)
           -> std::pair<swift::remote::RemoteRef<void>, uint64_t> {
         auto section_name = obj_file_format->getSectionName(section_kind);
@@ -676,9 +684,13 @@ bool SwiftLanguageRuntimeImpl::AddJitObjectFileToReflectionContext(
         return {};
       },
       likely_module_names);
+  // We don't care to cache modules generated by the jit, because they will
+  // only be used by the current process.
+  return reflection_info_id.hasValue();
 }
 
-bool SwiftLanguageRuntimeImpl::AddObjectFileToReflectionContext(
+llvm::Optional<uint32_t>
+SwiftLanguageRuntimeImpl::AddObjectFileToReflectionContext(
     ModuleSP module,
     llvm::SmallVector<llvm::StringRef, 1> likely_module_names) {
   auto obj_format_type =
@@ -856,27 +868,35 @@ bool SwiftLanguageRuntimeImpl::AddModuleToReflectionContext(
 
   auto read_from_file_cache =
       GetMemoryReader()->readMetadataFromFileCacheEnabled();
+
+  llvm::Optional<uint32_t> info_id;
   // When dealing with ELF, we need to pass in the contents of the on-disk
   // file, since the Section Header Table is not present in the child process
   if (obj_file->GetPluginName().equals("elf")) {
     DataExtractor extractor;
     auto size = obj_file->GetData(0, obj_file->GetByteSize(), extractor);
     const uint8_t *file_data = extractor.GetDataStart();
     llvm::sys::MemoryBlock file_buffer((void *)file_data, size);
-    m_reflection_ctx->readELF(
+    info_id = m_reflection_ctx->readELF(
         swift::remote::RemoteAddress(load_ptr),
         llvm::Optional<llvm::sys::MemoryBlock>(file_buffer),
         likely_module_names);
   } else if (read_from_file_cache &&
              obj_file->GetPluginName().equals("mach-o")) {
-    if (!AddObjectFileToReflectionContext(module_sp, likely_module_names)) {
-      m_reflection_ctx->addImage(swift::remote::RemoteAddress(load_ptr),
+    info_id = AddObjectFileToReflectionContext(module_sp, likely_module_names);
+    if (!info_id)
+      info_id = m_reflection_ctx->addImage(swift::remote::RemoteAddress(load_ptr),
                                  likely_module_names);
-    }
   } else {
-    m_reflection_ctx->addImage(swift::remote::RemoteAddress(load_ptr),
+    info_id = m_reflection_ctx->addImage(swift::remote::RemoteAddress(load_ptr),
                                likely_module_names);
   }
+
+  if (info_id)
+    if (auto *swift_metadata_cache = GetSwiftMetadataCache())
+      swift_metadata_cache->registerModuleWithReflectionInfoID(module_sp,
+                                                               *info_id);
+
   return true;
 }
 

lldb/source/Plugins/LanguageRuntime/Swift/SwiftLanguageRuntimeDynamicTypeResolution.cpp

@@ -11,8 +11,9 @@
 //===----------------------------------------------------------------------===//
 
 #include "LLDBMemoryReader.h"
-#include "SwiftLanguageRuntimeImpl.h"
 #include "SwiftLanguageRuntime.h"
+#include "SwiftLanguageRuntimeImpl.h"
+#include "SwiftMetadataCache.h"
 
 #include "Plugins/ExpressionParser/Clang/ClangUtil.h"
 #include "Plugins/TypeSystem/Clang/TypeSystemClang.h"
@@ -24,12 +25,13 @@
 #include "lldb/Utility/LLDBLog.h"
 #include "lldb/Utility/Log.h"
 #include "lldb/Utility/Timer.h"
-#include "swift/AST/Types.h"
+#include "llvm/ADT/STLExtras.h"
 
 #include "swift/AST/ASTContext.h"
 #include "swift/AST/ASTMangler.h"
 #include "swift/AST/ASTWalker.h"
 #include "swift/AST/Decl.h"
+#include "swift/AST/Types.h"
 #include "swift/Demangling/Demangle.h"
 #include "swift/Reflection/ReflectionContext.h"
 #include "swift/Reflection/TypeRefBuilder.h"
@@ -241,24 +243,25 @@ class TargetReflectionContext
 
 public:
   TargetReflectionContext(
-      std::shared_ptr<swift::reflection::MemoryReader> reader)
-      : m_reflection_ctx(reader) {}
+      std::shared_ptr<swift::reflection::MemoryReader> reader,
+      SwiftMetadataCache *swift_metadata_cache)
+      : m_reflection_ctx(reader, swift_metadata_cache) {}
 
-  bool addImage(
+  llvm::Optional<uint32_t> addImage(
       llvm::function_ref<std::pair<swift::remote::RemoteRef<void>, uint64_t>(
           swift::ReflectionSectionKind)>
           find_section,
       llvm::SmallVector<llvm::StringRef, 1> likely_module_names) override {
     return m_reflection_ctx.addImage(find_section, likely_module_names);
   }
 
-  bool
+  llvm::Optional<uint32_t>
   addImage(swift::remote::RemoteAddress image_start,
            llvm::SmallVector<llvm::StringRef, 1> likely_module_names) override {
     return m_reflection_ctx.addImage(image_start, likely_module_names);
   }
 
-  bool readELF(
+  llvm::Optional<uint32_t> readELF(
       swift::remote::RemoteAddress ImageStart,
       llvm::Optional<llvm::sys::MemoryBlock> FileBuffer,
       llvm::SmallVector<llvm::StringRef, 1> likely_module_names = {}) override {
@@ -361,30 +364,36 @@ class TargetReflectionContext
 
 std::unique_ptr<SwiftLanguageRuntimeImpl::ReflectionContextInterface>
 SwiftLanguageRuntimeImpl::ReflectionContextInterface::CreateReflectionContext32(
-    std::shared_ptr<swift::remote::MemoryReader> reader, bool ObjCInterop) {
+    std::shared_ptr<swift::remote::MemoryReader> reader, bool ObjCInterop,
+    SwiftMetadataCache *swift_metadata_cache) {
   using ReflectionContext32ObjCInterop =
       TargetReflectionContext<swift::reflection::ReflectionContext<
           swift::External<swift::WithObjCInterop<swift::RuntimeTarget<4>>>>>;
   using ReflectionContext32NoObjCInterop =
       TargetReflectionContext<swift::reflection::ReflectionContext<
           swift::External<swift::NoObjCInterop<swift::RuntimeTarget<4>>>>>;
-  if (ObjCInterop)    
-    return std::make_unique<ReflectionContext32ObjCInterop>(reader);
-  return std::make_unique<ReflectionContext32NoObjCInterop>(reader);
+  if (ObjCInterop)
+    return std::make_unique<ReflectionContext32ObjCInterop>(
+        reader, swift_metadata_cache);
+  return std::make_unique<ReflectionContext32NoObjCInterop>(
+      reader, swift_metadata_cache);
 }
 
 std::unique_ptr<SwiftLanguageRuntimeImpl::ReflectionContextInterface>
 SwiftLanguageRuntimeImpl::ReflectionContextInterface::CreateReflectionContext64(
-    std::shared_ptr<swift::remote::MemoryReader> reader, bool ObjCInterop) {
+    std::shared_ptr<swift::remote::MemoryReader> reader, bool ObjCInterop,
+    SwiftMetadataCache *swift_metadata_cache) {
   using ReflectionContext64ObjCInterop =
       TargetReflectionContext<swift::reflection::ReflectionContext<
           swift::External<swift::WithObjCInterop<swift::RuntimeTarget<8>>>>>;
   using ReflectionContext64NoObjCInterop =
       TargetReflectionContext<swift::reflection::ReflectionContext<
           swift::External<swift::NoObjCInterop<swift::RuntimeTarget<8>>>>>;
   if (ObjCInterop)
-    return std::make_unique<ReflectionContext64ObjCInterop>(reader);
-  return std::make_unique<ReflectionContext64NoObjCInterop>(reader);
+    return std::make_unique<ReflectionContext64ObjCInterop>(
+        reader, swift_metadata_cache);
+  return std::make_unique<ReflectionContext64NoObjCInterop>(
+      reader, swift_metadata_cache);
 }
 
 SwiftLanguageRuntimeImpl::ReflectionContextInterface::

lldb/source/Plugins/LanguageRuntime/Swift/SwiftLanguageRuntimeImpl.h

@@ -15,6 +15,7 @@
 
 #include "LLDBMemoryReader.h"
 #include "SwiftLanguageRuntime.h"
+#include "SwiftMetadataCache.h"
 #include "swift/Reflection/TypeLowering.h"
 #include "llvm/Support/Memory.h"
 
@@ -201,24 +202,26 @@ class SwiftLanguageRuntimeImpl {
     /// Return a 32-bit reflection context.
     static std::unique_ptr<ReflectionContextInterface>
     CreateReflectionContext32(
-        std::shared_ptr<swift::remote::MemoryReader> reader, bool ObjCInterop);
+        std::shared_ptr<swift::remote::MemoryReader> reader, bool ObjCInterop,
+        SwiftMetadataCache *swift_metadata_cache);
 
     /// Return a 64-bit reflection context.
     static std::unique_ptr<ReflectionContextInterface>
     CreateReflectionContext64(
-        std::shared_ptr<swift::remote::MemoryReader> reader, bool ObjCInterop);
+        std::shared_ptr<swift::remote::MemoryReader> reader, bool ObjCInterop,
+        SwiftMetadataCache *swift_metadata_cache);
 
     virtual ~ReflectionContextInterface();
 
-    virtual bool addImage(
+    virtual llvm::Optional<uint32_t> addImage(
         llvm::function_ref<std::pair<swift::remote::RemoteRef<void>, uint64_t>(
             swift::ReflectionSectionKind)>
             find_section,
         llvm::SmallVector<llvm::StringRef, 1> likely_module_names = {}) = 0;
-    virtual bool addImage(
+    virtual llvm::Optional<uint32_t> addImage(
         swift::remote::RemoteAddress image_start,
         llvm::SmallVector<llvm::StringRef, 1> likely_module_names = {}) = 0;
-    virtual bool
+    virtual llvm::Optional<uint32_t>
     readELF(swift::remote::RemoteAddress ImageStart,
             llvm::Optional<llvm::sys::MemoryBlock> FileBuffer,
             llvm::SmallVector<llvm::StringRef, 1> likely_module_names = {}) = 0;
@@ -365,6 +368,8 @@ class SwiftLanguageRuntimeImpl {
   /// Lazily initialize and return \p m_SwiftNativeNSErrorISA.
   llvm::Optional<lldb::addr_t> GetSwiftNativeNSErrorISA();
 
+  SwiftMetadataCache *GetSwiftMetadataCache();
+
   /// These members are used to track and toggle the state of the "dynamic
   /// exclusivity enforcement flag" in the swift runtime. This flag is set to
   /// true when an LLDB expression starts running, and reset to its original
@@ -382,6 +387,8 @@ class SwiftLanguageRuntimeImpl {
   /// \{
   std::unique_ptr<ReflectionContextInterface> m_reflection_ctx;
 
+  SwiftMetadataCache m_swift_metadata_cache;
+
   /// Record modules added through ModulesDidLoad, which are to be
   /// added to the reflection context once it's being initialized.
   ModuleList m_modules_to_add;
@@ -400,8 +407,9 @@ class SwiftLanguageRuntimeImpl {
 
   /// Add the reflections sections to the reflection context by extracting
   /// the directly from the object file.
-  /// \return true on success.
-  bool AddObjectFileToReflectionContext(
+  /// \return the info id of the newly registered reflection info on success, or
+  /// llvm::None otherwise.
+  llvm::Optional<uint32_t> AddObjectFileToReflectionContext(
       lldb::ModuleSP module,
       llvm::SmallVector<llvm::StringRef, 1> likely_module_names);