[lldb] Fetching symbols in the background with dsymForUUID

On macOS, LLDB uses the DebugSymbols.framework to locate symbol rich
dSYM bundles. [1] The framework uses a variety of methods, one of them
calling into a binary or shell script to locate (and download) dSYMs.
Internally at Apple, that tool is called dsymForUUID and for simplicity
I'm just going to refer to it that way here too, even though it can be
be an arbitrary executable.

The most common use case for dsymForUUID is to fetch symbols from the
network. This can take a long time, and because the calls to the
DebugSymbols.framework are blocking, it takes a while to launch the
process. This is expected and therefore many people don't use this
functionality, but instead use add-dsym when they want symbols for a
given frame, backtrace or module. This is a little faster because you're
only fetching symbols for the module you care about, but it's still a
slow, blocking operation.

This patch introduces a hybrid approach between the two. When
symbols.enable-background-lookup is enabled, lldb will do the equivalent
of add-dsym in the background for every module that shows up in the
backtrace but doesn't have symbols for. From the user's perspective
there is no slowdown, because the process launches immediately, with
whatever symbols are available. Meanwhile, more symbol information is
added over time as the background fetching completes.

[1] https://lldb.llvm.org/use/symbols.html

rdar://76241471

Differential revision: https://reviews.llvm.org/D131328

(cherry picked from commit 11f45f3)

Author: JDevlieghere

lldb/include/lldb/Core/Debugger.h

@@ -82,6 +82,7 @@ class Debugger : public std::enable_shared_from_this<Debugger>,
     eBroadcastBitProgress = (1 << 0),
     eBroadcastBitWarning = (1 << 1),
     eBroadcastBitError = (1 << 2),
+    eBroadcastSymbolChange = (1 << 3),
   };
 
   static ConstString GetStaticBroadcasterClass();
@@ -438,6 +439,8 @@ class Debugger : public std::enable_shared_from_this<Debugger>,
               llvm::Optional<lldb::user_id_t> debugger_id = llvm::None,
               std::once_flag *once = nullptr);
 
+  static void ReportSymbolChange(const ModuleSpec &module_spec);
+
 protected:
   friend class CommandInterpreter;
   friend class SwiftREPL;

lldb/include/lldb/Core/DebuggerEvents.h

@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "lldb/Core/ModuleSpec.h"
 #include "lldb/Utility/ConstString.h"
 #include "lldb/Utility/Event.h"
 
@@ -82,6 +83,28 @@ class DiagnosticEventData : public EventData {
   const DiagnosticEventData &operator=(const DiagnosticEventData &) = delete;
 };
 
+class SymbolChangeEventData : public EventData {
+public:
+  SymbolChangeEventData(lldb::DebuggerWP debugger_wp, ModuleSpec module_spec)
+      : m_debugger_wp(debugger_wp), m_module_spec(std::move(module_spec)) {}
+
+  static ConstString GetFlavorString();
+  ConstString GetFlavor() const override;
+
+  static const SymbolChangeEventData *
+  GetEventDataFromEvent(const Event *event_ptr);
+
+  void DoOnRemoval(Event *event_ptr) override;
+
+private:
+  lldb::DebuggerWP m_debugger_wp;
+  ModuleSpec m_module_spec;
+
+  SymbolChangeEventData(const SymbolChangeEventData &) = delete;
+  const SymbolChangeEventData &
+  operator=(const SymbolChangeEventData &) = delete;
+};
+
 } // namespace lldb_private
 
 #endif // LLDB_CORE_DEBUGGER_EVENTS_H

lldb/include/lldb/Core/ModuleList.h

@@ -71,6 +71,7 @@ class ModuleListProperties : public Properties {
   bool SetClangModulesCachePath(const FileSpec &path);
   bool GetEnableExternalLookup() const;
   bool SetEnableExternalLookup(bool new_value);
+  bool GetEnableBackgroundLookup() const;
   bool GetEnableLLDBIndexCache() const;
   bool SetEnableLLDBIndexCache(bool new_value);
   uint64_t GetLLDBIndexCacheMaxByteSize();
@@ -466,6 +467,8 @@ class ModuleList {
   static void FindSharedModules(const ModuleSpec &module_spec,
                                 ModuleList &matching_module_list);
 
+  static lldb::ModuleSP FindSharedModule(const UUID &uuid);
+
   static size_t RemoveOrphanSharedModules(bool mandatory);
 
   static bool RemoveSharedModuleIfOrphaned(const Module *module_ptr);

lldb/include/lldb/Symbol/LocateSymbolFile.h

@@ -14,6 +14,7 @@
 #include "lldb/Core/FileSpecList.h"
 #include "lldb/Utility/FileSpec.h"
 #include "lldb/Utility/Status.h"
+#include "lldb/lldb-forward.h"
 
 namespace lldb_private {
 
@@ -52,7 +53,15 @@ class Symbols {
   //
   static bool DownloadObjectAndSymbolFile(ModuleSpec &module_spec,
                                           Status &error,
-                                          bool force_lookup = true);
+                                          bool force_lookup = true,
+                                          bool copy_executable = true);
+
+  /// Locate the symbol file for the given UUID on a background thread. This
+  /// function returns immediately. Under the hood it uses the debugger's
+  /// thread pool to call DownloadObjectAndSymbolFile. If a symbol file is
+  /// found, this will notify all target which contain the module with the
+  /// given UUID.
+  static void DownloadSymbolFileAsync(const UUID &uuid);
 };
 
 } // namespace lldb_private

lldb/include/lldb/Target/Target.h

@@ -207,7 +207,7 @@ class TargetProperties : public Properties {
   bool GetEnableNotifyAboutFixIts() const;
 
   FileSpec GetSaveJITObjectsDir() const;
-  
+
   bool GetEnableSyntheticValue() const;
 
   uint32_t GetMaxZeroPaddingInFloatFormat() const;
@@ -305,7 +305,7 @@ class TargetProperties : public Properties {
   void DisableASLRValueChangedCallback();
   void InheritTCCValueChangedCallback();
   void DisableSTDIOValueChangedCallback();
-  
+
   // Settings checker for target.jit-save-objects-dir:
   void CheckJITObjectsDir();
 
@@ -543,7 +543,8 @@ class Target : public std::enable_shared_from_this<Target>,
     eBroadcastBitModulesLoaded = (1 << 1),
     eBroadcastBitModulesUnloaded = (1 << 2),
     eBroadcastBitWatchpointChanged = (1 << 3),
-    eBroadcastBitSymbolsLoaded = (1 << 4)
+    eBroadcastBitSymbolsLoaded = (1 << 4),
+    eBroadcastBitSymbolsChanged = (1 << 5),
   };
 
   // These two functions fill out the Broadcaster interface:
@@ -1045,7 +1046,7 @@ class Target : public std::enable_shared_from_this<Target>,
   ModuleIsExcludedForUnconstrainedSearches(const lldb::ModuleSP &module_sp);
 
   const ArchSpec &GetArchitecture() const { return m_arch.GetSpec(); }
-  
+
   /// Returns the name of the target's ABI plugin.
   llvm::StringRef GetABIName() const;
 
@@ -1535,30 +1536,30 @@ class Target : public std::enable_shared_from_this<Target>,
     LazyBool pass = eLazyBoolCalculate;
     LazyBool notify = eLazyBoolCalculate;
     LazyBool stop = eLazyBoolCalculate;
-    DummySignalValues(LazyBool pass, LazyBool notify, LazyBool stop) : 
-        pass(pass), notify(notify), stop(stop) {}
+    DummySignalValues(LazyBool pass, LazyBool notify, LazyBool stop)
+        : pass(pass), notify(notify), stop(stop) {}
     DummySignalValues() = default;
   };
   using DummySignalElement = llvm::StringMapEntry<DummySignalValues>;
-  static bool UpdateSignalFromDummy(lldb::UnixSignalsSP signals_sp, 
-      const DummySignalElement &element);
-  static bool ResetSignalFromDummy(lldb::UnixSignalsSP signals_sp, 
-      const DummySignalElement &element);
+  static bool UpdateSignalFromDummy(lldb::UnixSignalsSP signals_sp,
+                                    const DummySignalElement &element);
+  static bool ResetSignalFromDummy(lldb::UnixSignalsSP signals_sp,
+                                   const DummySignalElement &element);
 
 public:
   /// Add a signal to the Target's list of stored signals/actions.  These
   /// values will get copied into any processes launched from
   /// this target.
-  void AddDummySignal(llvm::StringRef name, LazyBool pass, LazyBool print, 
+  void AddDummySignal(llvm::StringRef name, LazyBool pass, LazyBool print,
                       LazyBool stop);
   /// Updates the signals in signals_sp using the stored dummy signals.
   /// If warning_stream_sp is not null, if any stored signals are not found in
   /// the current process, a warning will be emitted here.
-  void UpdateSignalsFromDummy(lldb::UnixSignalsSP signals_sp, 
+  void UpdateSignalsFromDummy(lldb::UnixSignalsSP signals_sp,
                               lldb::StreamSP warning_stream_sp);
   /// Clear the dummy signals in signal_names from the target, or all signals
   /// if signal_names is empty.  Also remove the behaviors they set from the
-  /// process's signals if it exists. 
+  /// process's signals if it exists.
   void ClearDummySignals(Args &signal_names);
   /// Print all the signals set in this target.
   void PrintDummySignals(Stream &strm, Args &signals);
@@ -1644,7 +1645,7 @@ class Target : public std::enable_shared_from_this<Target>,
   lldb::TraceSP m_trace_sp;
   /// Stores the frame recognizers of this target.
   lldb::StackFrameRecognizerManagerUP m_frame_recognizer_manager_up;
-  /// These are used to set the signal state when you don't have a process and 
+  /// These are used to set the signal state when you don't have a process and
   /// more usefully in the Dummy target where you can't know exactly what
   /// signals you will have.
   llvm::StringMap<DummySignalValues> m_dummy_signals;

lldb/source/Core/CoreProperties.td

@@ -5,6 +5,10 @@ let Definition = "modulelist" in {
     Global,
     DefaultTrue,
     Desc<"Control the use of external tools and repositories to locate symbol files. Directories listed in target.debug-file-search-paths and directory of the executable are always checked first for separate debug info files. Then depending on this setting: On macOS, Spotlight would be also used to locate a matching .dSYM bundle based on the UUID of the executable. On NetBSD, directory /usr/libdata/debug would be also searched. On platforms other than NetBSD directory /usr/lib/debug would be also searched.">;
+  def EnableBackgroundLookup: Property<"enable-background-lookup", "Boolean">,
+    Global,
+    DefaultFalse,
+    Desc<"On macOS, enable calling dsymForUUID (or an equivalent script/binary) in the background to locate symbol files that weren't found.">;
   def ClangModulesCachePath: Property<"clang-modules-cache-path", "FileSpec">,
     Global,
     DefaultStringValue<"">,

lldb/source/Core/Debugger.cpp

@@ -13,6 +13,7 @@
 #include "lldb/Core/FormatEntity.h"
 #include "lldb/Core/Mangled.h"
 #include "lldb/Core/ModuleList.h"
+#include "lldb/Core/ModuleSpec.h"
 #include "lldb/Core/PluginManager.h"
 #include "lldb/Core/StreamAsynchronousIO.h"
 #include "lldb/Core/StreamFile.h"
@@ -1452,6 +1453,18 @@ void Debugger::ReportError(std::string message,
                        debugger_id, once);
 }
 
+void Debugger::ReportSymbolChange(const ModuleSpec &module_spec) {
+  if (g_debugger_list_ptr && g_debugger_list_mutex_ptr) {
+    std::lock_guard<std::recursive_mutex> guard(*g_debugger_list_mutex_ptr);
+    for (DebuggerSP debugger_sp : *g_debugger_list_ptr) {
+      EventSP event_sp = std::make_shared<Event>(
+          Debugger::eBroadcastSymbolChange,
+          new SymbolChangeEventData(debugger_sp, module_spec));
+      debugger_sp->GetBroadcaster().BroadcastEvent(event_sp);
+    }
+  }
+}
+
 static std::shared_ptr<LogHandler>
 CreateLogHandler(LogHandlerKind log_handler_kind, int fd, bool should_close,
                  size_t buffer_size) {
@@ -1761,8 +1774,8 @@ lldb::thread_result_t Debugger::DefaultEventHandler() {
           CommandInterpreter::eBroadcastBitAsynchronousErrorData);
 
   listener_sp->StartListeningForEvents(
-      &m_broadcaster,
-      eBroadcastBitProgress | eBroadcastBitWarning | eBroadcastBitError);
+      &m_broadcaster, eBroadcastBitProgress | eBroadcastBitWarning |
+                          eBroadcastBitError | eBroadcastSymbolChange);
 
   // Let the thread that spawned us know that we have started up and that we
   // are now listening to all required events so no events get missed

lldb/source/Core/DebuggerEvents.cpp

@@ -7,9 +7,12 @@
 //===----------------------------------------------------------------------===//
 
 #include "lldb/Core/DebuggerEvents.h"
+#include "lldb/Core/Debugger.h"
+#include "lldb/Core/Module.h"
 #include "llvm/Support/WithColor.h"
 
 using namespace lldb_private;
+using namespace lldb;
 
 template <typename T>
 static const T *GetEventDataFromEventImpl(const Event *event_ptr) {
@@ -79,3 +82,37 @@ const DiagnosticEventData *
 DiagnosticEventData::GetEventDataFromEvent(const Event *event_ptr) {
   return GetEventDataFromEventImpl<DiagnosticEventData>(event_ptr);
 }
+
+ConstString SymbolChangeEventData::GetFlavorString() {
+  static ConstString g_flavor("SymbolChangeEventData");
+  return g_flavor;
+}
+
+ConstString SymbolChangeEventData::GetFlavor() const {
+  return SymbolChangeEventData::GetFlavorString();
+}
+
+const SymbolChangeEventData *
+SymbolChangeEventData::GetEventDataFromEvent(const Event *event_ptr) {
+  return GetEventDataFromEventImpl<SymbolChangeEventData>(event_ptr);
+}
+
+void SymbolChangeEventData::DoOnRemoval(Event *event_ptr) {
+  DebuggerSP debugger_sp(m_debugger_wp.lock());
+  if (!debugger_sp)
+    return;
+
+  for (TargetSP target_sp : debugger_sp->GetTargetList().Targets()) {
+    if (ModuleSP module_sp =
+            target_sp->GetImages().FindModule(m_module_spec.GetUUID())) {
+      {
+        std::lock_guard<std::recursive_mutex> guard(module_sp->GetMutex());
+        if (!module_sp->GetSymbolFileFileSpec())
+          module_sp->SetSymbolFileFileSpec(m_module_spec.GetSymbolFileSpec());
+      }
+      ModuleList module_list;
+      module_list.Append(module_sp);
+      target_sp->SymbolsDidLoad(module_list);
+    }
+  }
+}

lldb/source/Core/Module.cpp

@@ -24,6 +24,7 @@
 #include "lldb/Interpreter/ScriptInterpreter.h"
 #include "lldb/Symbol/CompileUnit.h"
 #include "lldb/Symbol/Function.h"
+#include "lldb/Symbol/LocateSymbolFile.h"
 #include "lldb/Symbol/ObjectFile.h"
 #include "lldb/Symbol/Symbol.h"
 #include "lldb/Symbol/SymbolContext.h"
@@ -1362,8 +1363,11 @@ void Module::SectionFileAddressesChanged() {
 }
 
 UnwindTable &Module::GetUnwindTable() {
-  if (!m_unwind_table)
+  if (!m_unwind_table) {
     m_unwind_table.emplace(*this);
+    if (!m_symfile_spec)
+      Symbols::DownloadSymbolFileAsync(GetUUID());
+  }
   return *m_unwind_table;
 }
 

lldb/source/Core/ModuleList.cpp

@@ -130,6 +130,12 @@ bool ModuleListProperties::SetEnableExternalLookup(bool new_value) {
       nullptr, ePropertyEnableExternalLookup, new_value);
 }
 
+bool ModuleListProperties::GetEnableBackgroundLookup() const {
+  const uint32_t idx = ePropertyEnableBackgroundLookup;
+  return m_collection_sp->GetPropertyAtIndexAsBoolean(
+      nullptr, idx, g_modulelist_properties[idx].default_uint_value != 0);
+}
+
 FileSpec ModuleListProperties::GetClangModulesCachePath() const {
   return m_collection_sp
       ->GetPropertyAtIndexAsOptionValueFileSpec(nullptr, false,
@@ -925,6 +931,10 @@ void ModuleList::FindSharedModules(const ModuleSpec &module_spec,
   GetSharedModuleList().FindModules(module_spec, matching_module_list);
 }
 
+lldb::ModuleSP ModuleList::FindSharedModule(const UUID &uuid) {
+  return GetSharedModuleList().FindModule(uuid);
+}
+
 size_t ModuleList::RemoveOrphanSharedModules(bool mandatory) {
   return GetSharedModuleList().RemoveOrphans(mandatory);
 }

lldb/source/Symbol/LocateSymbolFile.cpp

@@ -8,6 +8,8 @@
 
 #include "lldb/Symbol/LocateSymbolFile.h"
 
+#include "lldb/Core/Debugger.h"
+#include "lldb/Core/Module.h"
 #include "lldb/Core/ModuleList.h"
 #include "lldb/Core/ModuleSpec.h"
 #include "lldb/Core/Progress.h"
@@ -23,7 +25,9 @@
 #include "lldb/Utility/Timer.h"
 #include "lldb/Utility/UUID.h"
 
+#include "llvm/ADT/SmallSet.h"
 #include "llvm/Support/FileSystem.h"
+#include "llvm/Support/ThreadPool.h"
 
 // From MacOSX system header "mach/machine.h"
 typedef int cpu_type_t;
@@ -381,6 +385,35 @@ Symbols::LocateExecutableSymbolFile(const ModuleSpec &module_spec,
   return LocateExecutableSymbolFileDsym(module_spec);
 }
 
+void Symbols::DownloadSymbolFileAsync(const UUID &uuid) {
+  if (!ModuleList::GetGlobalModuleListProperties().GetEnableBackgroundLookup())
+    return;
+
+  static llvm::SmallSet<UUID, 8> g_seen_uuids;
+  static std::mutex g_mutex;
+  Debugger::GetThreadPool().async([=]() {
+    {
+      std::lock_guard<std::mutex> guard(g_mutex);
+      if (g_seen_uuids.count(uuid))
+        return;
+      g_seen_uuids.insert(uuid);
+    }
+
+    Status error;
+    ModuleSpec module_spec;
+    module_spec.GetUUID() = uuid;
+    if (!Symbols::DownloadObjectAndSymbolFile(module_spec, error,
+                                              /*force_lookup=*/true,
+                                              /*copy_executable=*/false))
+      return;
+
+    if (error.Fail())
+      return;
+
+    Debugger::ReportSymbolChange(module_spec);
+  });
+}
+
 #if !defined(__APPLE__)
 
 FileSpec Symbols::FindSymbolFileInBundle(const FileSpec &symfile_bundle,
@@ -391,7 +424,8 @@ FileSpec Symbols::FindSymbolFileInBundle(const FileSpec &symfile_bundle,
 }
 
 bool Symbols::DownloadObjectAndSymbolFile(ModuleSpec &module_spec,
-                                          Status &error, bool force_lookup) {
+                                          Status &error, bool force_lookup,
+                                          bool copy_executable) {
   // Fill in the module_spec.GetFileSpec() for the object file and/or the
   // module_spec.GetSymbolFileSpec() for the debug symbols file.
   return false;