DEBUGINFOD based DWP acquisition for LLDB (#70996)

kevinfrei · Kevin Frei · bulbazord · web-flow · commit c43c86c285a3 · 2023-12-04T11:45:40.000-08:00
I've plumbed the LLVM DebugInfoD client into LLDB, and added automatic downloading of DWP files to the SymbolFileDWARF.cpp plugin. If you have DEBUGINFOD_URLS set to a space delimited set of web servers, LLDB will try to use them as a last resort when searching for DWP files. If you do *not* have that environment variable set, nothing should be changed. There's also a setting, per @clayborg 's suggestion, that will override the environment variable, or can be used instead of the environment variable. The setting is why I also needed to add an API to the llvm-debuginfod library ### Test Plan: Suggestions are welcome here. I should probably have some positive and negative tests, but I wanted to get the diff up for people who have a clue what they're doing to rip it to pieces before spending too much time validating the initial implementation. --------- Co-authored-by: Kevin Frei <freik@meta.com> Co-authored-by: Alex Langford <nirvashtzero@gmail.com>
diff --git a/lldb/include/lldb/Core/PluginManager.h b/lldb/include/lldb/Core/PluginManager.h
@@ -355,7 +355,8 @@ class PluginManager {
           nullptr,
       SymbolLocatorDownloadObjectAndSymbolFile download_object_symbol_file =
           nullptr,
-      SymbolLocatorFindSymbolFileInBundle find_symbol_file_in_bundle = nullptr);
+      SymbolLocatorFindSymbolFileInBundle find_symbol_file_in_bundle = nullptr,
+      DebuggerInitializeCallback debugger_init_callback = nullptr);
 
   static bool UnregisterPlugin(SymbolLocatorCreateInstance create_callback);
 
@@ -528,6 +529,14 @@ class PluginManager {
       Debugger &debugger, const lldb::OptionValuePropertiesSP &properties_sp,
       llvm::StringRef description, bool is_global_property);
 
+  static lldb::OptionValuePropertiesSP
+  GetSettingForSymbolLocatorPlugin(Debugger &debugger,
+                                   llvm::StringRef setting_name);
+
+  static bool CreateSettingForSymbolLocatorPlugin(
+      Debugger &debugger, const lldb::OptionValuePropertiesSP &properties_sp,
+      llvm::StringRef description, bool is_global_property);
+
   static bool CreateSettingForTracePlugin(
       Debugger &debugger, const lldb::OptionValuePropertiesSP &properties_sp,
       llvm::StringRef description, bool is_global_property);
diff --git a/lldb/source/Core/CoreProperties.td b/lldb/source/Core/CoreProperties.td
@@ -4,7 +4,7 @@ let Definition = "modulelist" in {
   def EnableExternalLookup: Property<"enable-external-lookup", "Boolean">,
     Global,
     DefaultTrue,
-    Desc<"Control the use of external tools and repositories to locate symbol files. Directories listed in target.debug-file-search-paths and directory of the executable are always checked first for separate debug info files. Then depending on this setting: On macOS, Spotlight would be also used to locate a matching .dSYM bundle based on the UUID of the executable. On NetBSD, directory /usr/libdata/debug would be also searched. On platforms other than NetBSD directory /usr/lib/debug would be also searched.">;
+    Desc<"Control the use of external tools and repositories to locate symbol files. Directories listed in target.debug-file-search-paths and directory of the executable are always checked first for separate debug info files. Then depending on this setting: On macOS, Spotlight would be also used to locate a matching .dSYM bundle based on the UUID of the executable. On NetBSD, directory /usr/libdata/debug would be also searched. On platforms other than NetBSD directory /usr/lib/debug would be also searched. If all other methods fail there may be symbol-locator plugins that, if configured properly, will also attempt to acquire symbols. The debuginfod plugin defaults to the DEGUFINFOD_URLS environment variable which is configurable through the 'plugin.symbol-locator.debuginfod.server_urls' setting.">;
   def EnableBackgroundLookup: Property<"enable-background-lookup", "Boolean">,
     Global,
     DefaultFalse,
diff --git a/lldb/source/Core/PluginManager.cpp b/lldb/source/Core/PluginManager.cpp
@@ -1091,9 +1091,10 @@ struct SymbolLocatorInstance
       SymbolLocatorLocateExecutableObjectFile locate_executable_object_file,
       SymbolLocatorLocateExecutableSymbolFile locate_executable_symbol_file,
       SymbolLocatorDownloadObjectAndSymbolFile download_object_symbol_file,
-      SymbolLocatorFindSymbolFileInBundle find_symbol_file_in_bundle)
-      : PluginInstance<SymbolLocatorCreateInstance>(name, description,
-                                                    create_callback),
+      SymbolLocatorFindSymbolFileInBundle find_symbol_file_in_bundle,
+      DebuggerInitializeCallback debugger_init_callback)
+      : PluginInstance<SymbolLocatorCreateInstance>(
+            name, description, create_callback, debugger_init_callback),
         locate_executable_object_file(locate_executable_object_file),
         locate_executable_symbol_file(locate_executable_symbol_file),
         download_object_symbol_file(download_object_symbol_file),
@@ -1117,11 +1118,12 @@ bool PluginManager::RegisterPlugin(
     SymbolLocatorLocateExecutableObjectFile locate_executable_object_file,
     SymbolLocatorLocateExecutableSymbolFile locate_executable_symbol_file,
     SymbolLocatorDownloadObjectAndSymbolFile download_object_symbol_file,
-    SymbolLocatorFindSymbolFileInBundle find_symbol_file_in_bundle) {
+    SymbolLocatorFindSymbolFileInBundle find_symbol_file_in_bundle,
+    DebuggerInitializeCallback debugger_init_callback) {
   return GetSymbolLocatorInstances().RegisterPlugin(
       name, description, create_callback, locate_executable_object_file,
       locate_executable_symbol_file, download_object_symbol_file,
-      find_symbol_file_in_bundle);
+      find_symbol_file_in_bundle, debugger_init_callback);
 }
 
 bool PluginManager::UnregisterPlugin(
@@ -1533,6 +1535,7 @@ void PluginManager::DebuggerInitialize(Debugger &debugger) {
   GetPlatformInstances().PerformDebuggerCallback(debugger);
   GetProcessInstances().PerformDebuggerCallback(debugger);
   GetSymbolFileInstances().PerformDebuggerCallback(debugger);
+  GetSymbolLocatorInstances().PerformDebuggerCallback(debugger);
   GetOperatingSystemInstances().PerformDebuggerCallback(debugger);
   GetStructuredDataPluginInstances().PerformDebuggerCallback(debugger);
   GetTracePluginInstances().PerformDebuggerCallback(debugger);
@@ -1660,6 +1663,7 @@ static constexpr llvm::StringLiteral kProcessPluginName("process");
 static constexpr llvm::StringLiteral kTracePluginName("trace");
 static constexpr llvm::StringLiteral kObjectFilePluginName("object-file");
 static constexpr llvm::StringLiteral kSymbolFilePluginName("symbol-file");
+static constexpr llvm::StringLiteral kSymbolLocatorPluginName("symbol-locator");
 static constexpr llvm::StringLiteral kJITLoaderPluginName("jit-loader");
 static constexpr llvm::StringLiteral
     kStructuredDataPluginName("structured-data");
@@ -1708,6 +1712,20 @@ bool PluginManager::CreateSettingForProcessPlugin(
                                 description, is_global_property);
 }
 
+lldb::OptionValuePropertiesSP
+PluginManager::GetSettingForSymbolLocatorPlugin(Debugger &debugger,
+                                                llvm::StringRef setting_name) {
+  return GetSettingForPlugin(debugger, setting_name, kSymbolLocatorPluginName);
+}
+
+bool PluginManager::CreateSettingForSymbolLocatorPlugin(
+    Debugger &debugger, const lldb::OptionValuePropertiesSP &properties_sp,
+    llvm::StringRef description, bool is_global_property) {
+  return CreateSettingForPlugin(debugger, kSymbolLocatorPluginName,
+                                "Settings for symbol locator plug-ins",
+                                properties_sp, description, is_global_property);
+}
+
 bool PluginManager::CreateSettingForTracePlugin(
     Debugger &debugger, const lldb::OptionValuePropertiesSP &properties_sp,
     llvm::StringRef description, bool is_global_property) {
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
@@ -4339,6 +4339,7 @@ const std::shared_ptr<SymbolFileDWARFDwo> &SymbolFileDWARF::GetDwpSymbolFile() {
     module_spec.GetSymbolFileSpec() =
         FileSpec(m_objfile_sp->GetModule()->GetFileSpec().GetPath() + ".dwp");
 
+    module_spec.GetUUID() = m_objfile_sp->GetUUID();
     FileSpecList search_paths = Target::GetDefaultDebugFileSearchPaths();
     FileSpec dwp_filespec =
         PluginManager::LocateExecutableSymbolFile(module_spec, search_paths);
diff --git a/lldb/source/Plugins/SymbolLocator/CMakeLists.txt b/lldb/source/Plugins/SymbolLocator/CMakeLists.txt
@@ -2,3 +2,4 @@ add_subdirectory(Default)
 if (CMAKE_SYSTEM_NAME MATCHES "Darwin")
   add_subdirectory(DebugSymbols)
 endif()
+add_subdirectory(Debuginfod)
diff --git a/lldb/source/Plugins/SymbolLocator/Debuginfod/CMakeLists.txt b/lldb/source/Plugins/SymbolLocator/Debuginfod/CMakeLists.txt
@@ -0,0 +1,21 @@
+lldb_tablegen(SymbolLocatorDebuginfodProperties.inc -gen-lldb-property-defs
+  SOURCE SymbolLocatorDebuginfodProperties.td
+  TARGET LLDBPluginSymbolLocatorDebuginfodPropertiesGen)
+
+lldb_tablegen(SymbolLocatorDebuginfodPropertiesEnum.inc -gen-lldb-property-enum-defs
+  SOURCE SymbolLocatorDebuginfodProperties.td
+  TARGET LLDBPluginSymbolLocatorDebuginfodPropertiesEnumGen)
+
+add_lldb_library(lldbPluginSymbolLocatorDebuginfod PLUGIN
+  SymbolLocatorDebuginfod.cpp
+
+  LINK_LIBS
+    lldbCore
+    lldbHost
+    lldbSymbol
+    LLVMDebuginfod
+  )
+
+add_dependencies(lldbPluginSymbolLocatorDebuginfod
+  LLDBPluginSymbolLocatorDebuginfodPropertiesGen
+  LLDBPluginSymbolLocatorDebuginfodPropertiesEnumGen)
diff --git a/lldb/source/Plugins/SymbolLocator/Debuginfod/SymbolLocatorDebuginfod.cpp b/lldb/source/Plugins/SymbolLocator/Debuginfod/SymbolLocatorDebuginfod.cpp
@@ -0,0 +1,142 @@
+//===-- SymbolLocatorDebuginfod.cpp ---------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "SymbolLocatorDebuginfod.h"
+
+#include "lldb/Core/PluginManager.h"
+#include "lldb/Utility/Args.h"
+
+#include "llvm/Debuginfod/Debuginfod.h"
+#include "llvm/Debuginfod/HTTPClient.h"
+
+using namespace lldb;
+using namespace lldb_private;
+
+LLDB_PLUGIN_DEFINE(SymbolLocatorDebuginfod)
+
+namespace {
+
+#define LLDB_PROPERTIES_symbollocatordebuginfod
+#include "SymbolLocatorDebuginfodProperties.inc"
+
+enum {
+#define LLDB_PROPERTIES_symbollocatordebuginfod
+#include "SymbolLocatorDebuginfodPropertiesEnum.inc"
+};
+
+class PluginProperties : public Properties {
+public:
+  static llvm::StringRef GetSettingName() {
+    return SymbolLocatorDebuginfod::GetPluginNameStatic();
+  }
+
+  PluginProperties() {
+    m_collection_sp = std::make_shared<OptionValueProperties>(GetSettingName());
+    m_collection_sp->Initialize(g_symbollocatordebuginfod_properties);
+
+    // We need to read the default value first to read the environment variable.
+    llvm::SmallVector<llvm::StringRef> urls = llvm::getDefaultDebuginfodUrls();
+    Args arg_urls{urls};
+    m_collection_sp->SetPropertyAtIndexFromArgs(ePropertyServerURLs, arg_urls);
+
+    m_collection_sp->SetValueChangedCallback(
+        ePropertyServerURLs, [this] { ServerURLsChangedCallback(); });
+  }
+
+  Args GetDebugInfoDURLs() const {
+    Args urls;
+    m_collection_sp->GetPropertyAtIndexAsArgs(ePropertyServerURLs, urls);
+    return urls;
+  }
+
+private:
+  void ServerURLsChangedCallback() {
+    m_server_urls = GetDebugInfoDURLs();
+    llvm::SmallVector<llvm::StringRef> dbginfod_urls;
+    llvm::for_each(m_server_urls, [&](const auto &obj) {
+      dbginfod_urls.push_back(obj.ref());
+    });
+    llvm::setDefaultDebuginfodUrls(dbginfod_urls);
+  }
+  // Storage for the StringRef's used within the Debuginfod library.
+  Args m_server_urls;
+};
+
+} // namespace
+
+static PluginProperties &GetGlobalPluginProperties() {
+  static PluginProperties g_settings;
+  return g_settings;
+}
+
+SymbolLocatorDebuginfod::SymbolLocatorDebuginfod() : SymbolLocator() {}
+
+void SymbolLocatorDebuginfod::Initialize() {
+  static llvm::once_flag g_once_flag;
+
+  llvm::call_once(g_once_flag, []() {
+    PluginManager::RegisterPlugin(
+        GetPluginNameStatic(), GetPluginDescriptionStatic(), CreateInstance,
+        LocateExecutableObjectFile, LocateExecutableSymbolFile, nullptr,
+        nullptr, SymbolLocatorDebuginfod::DebuggerInitialize);
+    llvm::HTTPClient::initialize();
+  });
+}
+
+void SymbolLocatorDebuginfod::DebuggerInitialize(Debugger &debugger) {
+  if (!PluginManager::GetSettingForSymbolLocatorPlugin(
+          debugger, PluginProperties::GetSettingName())) {
+    const bool is_global_setting = true;
+    PluginManager::CreateSettingForSymbolLocatorPlugin(
+        debugger, GetGlobalPluginProperties().GetValueProperties(),
+        "Properties for the Debuginfod Symbol Locator plug-in.",
+        is_global_setting);
+  }
+}
+
+void SymbolLocatorDebuginfod::Terminate() {
+  PluginManager::UnregisterPlugin(CreateInstance);
+  llvm::HTTPClient::cleanup();
+}
+
+llvm::StringRef SymbolLocatorDebuginfod::GetPluginDescriptionStatic() {
+  return "Debuginfod symbol locator.";
+}
+
+SymbolLocator *SymbolLocatorDebuginfod::CreateInstance() {
+  return new SymbolLocatorDebuginfod();
+}
+
+static std::optional<FileSpec> GetFileForModule(
+    const ModuleSpec &module_spec,
+    std::function<llvm::Expected<std::string>(llvm::object::BuildIDRef)>
+        PullFromServer) {
+  if (!ModuleList::GetGlobalModuleListProperties().GetEnableExternalLookup())
+    return {};
+  const UUID &module_uuid = module_spec.GetUUID();
+  if (module_uuid.IsValid() && llvm::canUseDebuginfod()) {
+    llvm::object::BuildID build_id(module_uuid.GetBytes());
+    llvm::Expected<std::string> result = PullFromServer(build_id);
+    if (result)
+      return FileSpec(*result);
+    // An error here should be logged as a failure in the Debuginfod library,
+    // so just consume it here
+    consumeError(result.takeError());
+  }
+  return {};
+}
+
+std::optional<ModuleSpec> SymbolLocatorDebuginfod::LocateExecutableObjectFile(
+    const ModuleSpec &module_spec) {
+  return GetFileForModule(module_spec, llvm::getCachedOrDownloadExecutable);
+}
+
+std::optional<FileSpec> SymbolLocatorDebuginfod::LocateExecutableSymbolFile(
+    const ModuleSpec &module_spec, const FileSpecList &default_search_paths) {
+  return GetFileForModule(module_spec, llvm::getCachedOrDownloadDebuginfo);
+}
diff --git a/lldb/source/Plugins/SymbolLocator/Debuginfod/SymbolLocatorDebuginfod.h b/lldb/source/Plugins/SymbolLocator/Debuginfod/SymbolLocatorDebuginfod.h
@@ -0,0 +1,54 @@
+//===-- SymbolLocatorDebuginfod.h -------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLDB_SOURCE_PLUGINS_SYMBOLLOCATOR_DEBUGINFOD_SYMBOLLOCATORDEBUGINFOD_H
+#define LLDB_SOURCE_PLUGINS_SYMBOLLOCATOR_DEBUGINFOD_SYMBOLLOCATORDEBUGINFOD_H
+
+#include "lldb/Core/Debugger.h"
+#include "lldb/Symbol/SymbolLocator.h"
+#include "lldb/lldb-private.h"
+
+namespace lldb_private {
+
+class SymbolLocatorDebuginfod : public SymbolLocator {
+public:
+  SymbolLocatorDebuginfod();
+
+  static void Initialize();
+  static void Terminate();
+  static void DebuggerInitialize(Debugger &debugger);
+
+  static llvm::StringRef GetPluginNameStatic() { return "debuginfod"; }
+  static llvm::StringRef GetPluginDescriptionStatic();
+
+  static lldb_private::SymbolLocator *CreateInstance();
+
+  /// PluginInterface protocol.
+  /// \{
+  llvm::StringRef GetPluginName() override { return GetPluginNameStatic(); }
+  /// \}
+
+  // Locate the executable file given a module specification.
+  //
+  // Locating the file should happen only on the local computer or using the
+  // current computers global settings.
+  static std::optional<ModuleSpec>
+  LocateExecutableObjectFile(const ModuleSpec &module_spec);
+
+  // Locate the symbol file given a module specification.
+  //
+  // Locating the file should happen only on the local computer or using the
+  // current computers global settings.
+  static std::optional<FileSpec>
+  LocateExecutableSymbolFile(const ModuleSpec &module_spec,
+                             const FileSpecList &default_search_paths);
+};
+
+} // namespace lldb_private
+
+#endif // LLDB_SOURCE_PLUGINS_SYMBOLLOCATOR_DEBUGINFOD_SYMBOLLOCATORDEBUGINFOD_H
diff --git a/lldb/source/Plugins/SymbolLocator/Debuginfod/SymbolLocatorDebuginfodProperties.td b/lldb/source/Plugins/SymbolLocator/Debuginfod/SymbolLocatorDebuginfodProperties.td
@@ -0,0 +1,7 @@
+include "../../../../include/lldb/Core/PropertiesBase.td"
+
+let Definition = "symbollocatordebuginfod" in {
+  def ServerURLs : Property<"server_urls", "Array">,
+    ElementType<"String">,
+    Desc<"An ordered list of Debuginfod server URLs to query for symbols. This defaults to the contents of the DEBUGINFOD_URLS environment variable.">;
+}
diff --git a/llvm/docs/ReleaseNotes.rst b/llvm/docs/ReleaseNotes.rst
@@ -262,6 +262,13 @@ Changes to LLDB
   (SME) and Scalable Matrix Extension 2 (SME2) for both live processes and core
   files. For details refer to the
   `AArch64 Linux documentation <https://lldb.llvm.org/use/aarch64-linux.html>`_.
+* LLDB now supports symbol and binary acquisition automatically using the
+  DEBUFINFOD protocol. The standard mechanism of specifying DEBUFINOD servers in
+  the ``DEBUGINFOD_URLS`` environment variable is used by default. In addition,
+  users can specify servers to request symbols from using the LLDB setting
+  ``plugin.symbol-locator.debuginfod.server_urls``, override or adding to the
+  environment variable.
+
 
 * When running on AArch64 Linux, ``lldb-server`` now provides register
   field information for the following registers: ``cpsr``, ``fpcr``,
diff --git a/llvm/include/llvm/Debuginfod/Debuginfod.h b/llvm/include/llvm/Debuginfod/Debuginfod.h
@@ -46,6 +46,10 @@ bool canUseDebuginfod();
 /// environment variable.
 SmallVector<StringRef> getDefaultDebuginfodUrls();
 
+/// Sets the list of debuginfod server URLs to query. This overrides the
+/// environment variable DEBUGINFOD_URLS.
+void setDefaultDebuginfodUrls(const SmallVector<StringRef> &URLs);
+
 /// Finds a default local file caching directory for the debuginfod client,
 /// first checking DEBUGINFOD_CACHE_PATH.
 Expected<std::string> getDefaultDebuginfodCacheDirectory();
diff --git a/llvm/lib/Debuginfod/Debuginfod.cpp b/llvm/lib/Debuginfod/Debuginfod.cpp
