[Reland] Report only loaded debug info in statistics dump (#81706) (#82207)

Updates:

- The previous patch changed the default behavior to not load dwos in
`DWARFUnit`
~~`SymbolFileDWARFDwo *GetDwoSymbolFile(bool load_all_debug_info =
false);`~~
`SymbolFileDWARFDwo *GetDwoSymbolFile(bool load_all_debug_info = true);`
- This broke some lldb-shell tests (see
https://green.lab.llvm.org/green/view/LLDB/job/as-lldb-cmake/16273/)
- TestDebugInfoSize.py
- with symbol on-demand, by default statistics dump only reports
skeleton debug info size
- `statistics dump -f` will load all dwos. debug info = skeleton debug
info + all dwo debug info

Currently running `statistics dump` will trigger lldb to load debug info
that's not yet loaded (eg. dwo files). Resulted in a delay in the
command return, which, can be interrupting.

This patch also added a new option `--load-all-debug-info` asking
statistics to dump all possible debug info, which will force loading all
debug info available if not yet loaded.

Author: kusmour

lldb/bindings/interface/SBStatisticsOptionsDocstrings.i

@@ -6,3 +6,9 @@
 ) lldb::SBStatisticsOptions::SetSummaryOnly;
 %feature("docstring", "Gets whether the statistics only dump a summary."
 ) lldb::SBStatisticsOptions::GetSummaryOnly;
+%feature("docstring", "
+    Sets whether the statistics will force loading all possible debug info."
+) lldb::SBStatisticsOptions::SetReportAllAvailableDebugInfo;
+%feature("docstring", "
+    Gets whether the statistics will force loading all possible debug info."
+) lldb::SBStatisticsOptions::GetReportAllAvailableDebugInfo;

lldb/include/lldb/API/SBStatisticsOptions.h

@@ -25,6 +25,14 @@ class LLDB_API SBStatisticsOptions {
   void SetSummaryOnly(bool b);
   bool GetSummaryOnly();
 
+  /// If set to true, the debugger will load all debug info that is available
+  /// and report statistics on the total amount. If this is set to false, then
+  /// only report statistics on the currently loaded debug information.
+  /// This can avoid loading debug info from separate files just so it can
+  /// report the total size which can slow down statistics reporting.
+  void SetReportAllAvailableDebugInfo(bool b);
+  bool GetReportAllAvailableDebugInfo();
+
 protected:
   friend class SBTarget;
   const lldb_private::StatisticsOptions &ref() const;

lldb/include/lldb/Symbol/SymbolFile.h

@@ -381,7 +381,8 @@ class SymbolFile : public PluginInterface {
 
   /// Metrics gathering functions
 
-  /// Return the size in bytes of all debug information in the symbol file.
+  /// Return the size in bytes of all loaded debug information or total possible
+  /// debug info in the symbol file.
   ///
   /// If the debug information is contained in sections of an ObjectFile, then
   /// this call should add the size of all sections that contain debug
@@ -391,7 +392,14 @@ class SymbolFile : public PluginInterface {
   /// entire file should be returned. The default implementation of this
   /// function will iterate over all sections in a module and add up their
   /// debug info only section byte sizes.
-  virtual uint64_t GetDebugInfoSize() = 0;
+  ///
+  /// \param load_all_debug_info
+  ///   If true, force loading any symbol files if they are not yet loaded and
+  ///   add to the total size. Default to false.
+  ///
+  /// \returns
+  ///   Total currently loaded debug info size in bytes
+  virtual uint64_t GetDebugInfoSize(bool load_all_debug_info = false) = 0;
 
   /// Return the time taken to parse the debug information.
   ///
@@ -534,7 +542,7 @@ class SymbolFileCommon : public SymbolFile {
 
   void Dump(Stream &s) override;
 
-  uint64_t GetDebugInfoSize() override;
+  uint64_t GetDebugInfoSize(bool load_all_debug_info = false) override;
 
   bool GetDebugInfoIndexWasLoadedFromCache() const override {
     return m_index_was_loaded_from_cache;

lldb/include/lldb/Symbol/SymbolFileOnDemand.h

@@ -178,7 +178,7 @@ class SymbolFileOnDemand : public lldb_private::SymbolFile {
 
   void PreloadSymbols() override;
 
-  uint64_t GetDebugInfoSize() override;
+  uint64_t GetDebugInfoSize(bool load_all_debug_info = false) override;
   lldb_private::StatsDuration::Duration GetDebugInfoParseTime() override;
   lldb_private::StatsDuration::Duration GetDebugInfoIndexTime() override;
 

lldb/include/lldb/Target/Statistics.h

@@ -132,6 +132,7 @@ struct ConstStringStats {
 
 struct StatisticsOptions {
   bool summary_only = false;
+  bool load_all_debug_info = false;
 };
 
 /// A class that represents statistics for a since lldb_private::Target.

lldb/source/API/SBStatisticsOptions.cpp

@@ -44,6 +44,14 @@ void SBStatisticsOptions::SetSummaryOnly(bool b) {
 
 bool SBStatisticsOptions::GetSummaryOnly() { return m_opaque_up->summary_only; }
 
+void SBStatisticsOptions::SetReportAllAvailableDebugInfo(bool b) {
+  m_opaque_up->load_all_debug_info = b;
+}
+
+bool SBStatisticsOptions::GetReportAllAvailableDebugInfo() {
+  return m_opaque_up->load_all_debug_info;
+}
+
 const lldb_private::StatisticsOptions &SBStatisticsOptions::ref() const {
   return *m_opaque_up;
 }

lldb/source/Commands/CommandObjectStats.cpp

@@ -78,6 +78,9 @@ class CommandObjectStatsDump : public CommandObjectParsed {
       case 's':
         m_stats_options.summary_only = true;
         break;
+      case 'f':
+        m_stats_options.load_all_debug_info = true;
+        break;
       default:
         llvm_unreachable("Unimplemented option");
       }

lldb/source/Commands/Options.td

@@ -1419,6 +1419,10 @@ let Command = "statistics dump" in {
   def statistics_dump_all: Option<"all-targets", "a">, Group<1>,
     Desc<"Include statistics for all targets.">;
   def statistics_dump_summary: Option<"summary", "s">, Group<1>,
-    Desc<"Dump only high-level summary statistics."
+    Desc<"Dump only high-level summary statistics. "
          "Exclude targets, modules, breakpoints etc... details.">;
+  def statistics_dump_force: Option<"load-all-debug-info", "f">, Group<1>,
+    Desc<"Dump the total possible debug info statistics. "
+    "Force loading all the debug information if not yet loaded, and collect "
+    "statistics with those.">;
 }

lldb/source/Plugins/SymbolFile/Breakpad/SymbolFileBreakpad.cpp

@@ -918,7 +918,7 @@ void SymbolFileBreakpad::ParseUnwindData() {
   m_unwind_data->win.Sort();
 }
 
-uint64_t SymbolFileBreakpad::GetDebugInfoSize() {
+uint64_t SymbolFileBreakpad::GetDebugInfoSize(bool load_all_debug_info) {
   // Breakpad files are all debug info.
   return m_objfile_sp->GetByteSize();
 }

lldb/source/Plugins/SymbolFile/Breakpad/SymbolFileBreakpad.h

@@ -141,7 +141,7 @@ class SymbolFileBreakpad : public SymbolFileCommon {
 
   llvm::StringRef GetPluginName() override { return GetPluginNameStatic(); }
 
-  uint64_t GetDebugInfoSize() override;
+  uint64_t GetDebugInfoSize(bool load_all_debug_info = false) override;
 
 private:
   // A class representing a position in the breakpad file. Useful for

lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.cpp

@@ -896,8 +896,9 @@ void DWARFUnit::ComputeAbsolutePath() {
     m_file_spec->MakeAbsolute(GetCompilationDirectory());
 }
 
-SymbolFileDWARFDwo *DWARFUnit::GetDwoSymbolFile() {
-  ExtractUnitDIEIfNeeded();
+SymbolFileDWARFDwo *DWARFUnit::GetDwoSymbolFile(bool load_all_debug_info) {
+  if (load_all_debug_info)
+    ExtractUnitDIEIfNeeded();
   if (m_dwo)
     return &llvm::cast<SymbolFileDWARFDwo>(m_dwo->GetSymbolFileDWARF());
   return nullptr;

lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.h

@@ -241,7 +241,7 @@ class DWARFUnit : public UserID {
   FileSpec GetFile(size_t file_idx);
   FileSpec::Style GetPathStyle();
 
-  SymbolFileDWARFDwo *GetDwoSymbolFile();
+  SymbolFileDWARFDwo *GetDwoSymbolFile(bool load_all_debug_info = true);
 
   die_iterator_range dies() {
     ExtractDIEsIfNeeded();

lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp

@@ -2672,7 +2672,7 @@ static bool UpdateCompilerContextForSimpleTemplateNames(TypeQuery &match) {
   return any_context_updated;
 }
 
-uint64_t SymbolFileDWARF::GetDebugInfoSize() {
+uint64_t SymbolFileDWARF::GetDebugInfoSize(bool load_all_debug_info) {
   DWARFDebugInfo &info = DebugInfo();
   uint32_t num_comp_units = info.GetNumUnits();
 
@@ -2687,7 +2687,7 @@ uint64_t SymbolFileDWARF::GetDebugInfoSize() {
     if (cu == nullptr)
       continue;
 
-    SymbolFileDWARFDwo *dwo = cu->GetDwoSymbolFile();
+    SymbolFileDWARFDwo *dwo = cu->GetDwoSymbolFile(load_all_debug_info);
     if (dwo)
       debug_info_size += dwo->GetDebugInfoSize();
   }

lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h

@@ -186,7 +186,7 @@ class SymbolFileDWARF : public SymbolFileCommon {
   GetMangledNamesForFunction(const std::string &scope_qualified_name,
                              std::vector<ConstString> &mangled_names) override;
 
-  uint64_t GetDebugInfoSize() override;
+  uint64_t GetDebugInfoSize(bool load_all_debug_info = false) override;
 
   void FindTypes(const lldb_private::TypeQuery &match,
                  lldb_private::TypeResults &results) override;

lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDwo.cpp

@@ -85,7 +85,7 @@ lldb::offset_t SymbolFileDWARFDwo::GetVendorDWARFOpcodeSize(
   return GetBaseSymbolFile().GetVendorDWARFOpcodeSize(data, data_offset, op);
 }
 
-uint64_t SymbolFileDWARFDwo::GetDebugInfoSize() {
+uint64_t SymbolFileDWARFDwo::GetDebugInfoSize(bool load_all_debug_info) {
   // Directly get debug info from current dwo object file's section list
   // instead of asking SymbolFileCommon::GetDebugInfo() which parses from
   // owning module which is wrong.

lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDwo.h

@@ -47,7 +47,7 @@ class SymbolFileDWARFDwo : public SymbolFileDWARF {
                                           const lldb::offset_t data_offset,
                                           const uint8_t op) const override;
 
-  uint64_t GetDebugInfoSize() override;
+  uint64_t GetDebugInfoSize(bool load_all_debug_info = false) override;
 
   bool ParseVendorDWARFOpcode(uint8_t op, const DataExtractor &opcodes,
                               lldb::offset_t &offset,

lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.cpp

@@ -2156,7 +2156,7 @@ SymbolFileNativePDB::GetTypeSystemForLanguage(lldb::LanguageType language) {
   return type_system_or_err;
 }
 
-uint64_t SymbolFileNativePDB::GetDebugInfoSize() {
+uint64_t SymbolFileNativePDB::GetDebugInfoSize(bool load_all_debug_info) {
   // PDB files are a separate file that contains all debug info.
   return m_index->pdb().getFileSize();
 }

lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.h

@@ -77,7 +77,7 @@ class SymbolFileNativePDB : public SymbolFileCommon {
 
   void InitializeObject() override;
 
-  uint64_t GetDebugInfoSize() override;
+  uint64_t GetDebugInfoSize(bool load_all_debug_info = false) override;
 
   // Compile Unit function calls
 

lldb/source/Symbol/SymbolFile.cpp

@@ -227,7 +227,7 @@ SymbolFileCommon::GetTypeSystemForLanguage(lldb::LanguageType language) {
   return type_system_or_err;
 }
 
-uint64_t SymbolFileCommon::GetDebugInfoSize() {
+uint64_t SymbolFileCommon::GetDebugInfoSize(bool load_all_debug_info) {
   if (!m_objfile_sp)
     return 0;
   ModuleSP module_sp(m_objfile_sp->GetModule());

lldb/source/Symbol/SymbolFileOnDemand.cpp

@@ -535,11 +535,11 @@ void SymbolFileOnDemand::PreloadSymbols() {
   return m_sym_file_impl->PreloadSymbols();
 }
 
-uint64_t SymbolFileOnDemand::GetDebugInfoSize() {
+uint64_t SymbolFileOnDemand::GetDebugInfoSize(bool load_all_debug_info) {
   // Always return the real debug info size.
   LLDB_LOG(GetLog(), "[{0}] {1} is not skipped", GetSymbolFileName(),
            __FUNCTION__);
-  return m_sym_file_impl->GetDebugInfoSize();
+  return m_sym_file_impl->GetDebugInfoSize(load_all_debug_info);
 }
 
 StatsDuration::Duration SymbolFileOnDemand::GetDebugInfoParseTime() {

lldb/source/Target/Statistics.cpp

@@ -224,6 +224,7 @@ llvm::json::Value DebuggerStats::ReportStatistics(
     const lldb_private::StatisticsOptions &options) {
 
   const bool summary_only = options.summary_only;
+  const bool load_all_debug_info = options.load_all_debug_info;
 
   json::Array json_targets;
   json::Array json_modules;
@@ -280,7 +281,8 @@ llvm::json::Value DebuggerStats::ReportStatistics(
         ++debug_index_saved;
       module_stat.debug_index_time = sym_file->GetDebugInfoIndexTime().count();
       module_stat.debug_parse_time = sym_file->GetDebugInfoParseTime().count();
-      module_stat.debug_info_size = sym_file->GetDebugInfoSize();
+      module_stat.debug_info_size =
+          sym_file->GetDebugInfoSize(load_all_debug_info);
       module_stat.symtab_stripped = module->GetObjectFile()->IsStripped();
       if (module_stat.symtab_stripped)
         ++num_stripped_modules;

lldb/test/API/commands/target/debuginfo/TestDebugInfoSize.py

@@ -120,6 +120,7 @@ def test_dwos_loaded_symbols_on_demand(self):
         target = self.dbg.CreateTarget(exe)
         self.assertTrue(target, lldbtest.VALID_TARGET)
 
+        # By default dwo files will not be loaded
         stats = target.GetStatistics()
         stream = lldb.SBStream()
         res = stats.GetAsJSON(stream)
@@ -129,6 +130,15 @@ def test_dwos_loaded_symbols_on_demand(self):
             debug_stats,
             'Make sure the "totalDebugInfoByteSize" key is in target.GetStatistics()',
         )
+        self.assertEqual(debug_stats["totalDebugInfoByteSize"], SKELETON_DEBUGINFO_SIZE)
+
+        # Force loading all the dwo files
+        stats_options = lldb.SBStatisticsOptions()
+        stats_options.SetReportAllAvailableDebugInfo(True)
+        stats = target.GetStatistics(stats_options)
+        stream = lldb.SBStream()
+        stats.GetAsJSON(stream)
+        debug_stats = json.loads(stream.GetData())
         self.assertEqual(
             debug_stats["totalDebugInfoByteSize"],
             SKELETON_DEBUGINFO_SIZE + MAIN_DWO_DEBUGINFO_SIZE + FOO_DWO_DEBUGINFO_SIZE,

lldb/test/API/functionalities/stats_api/TestStatisticsAPI.py

@@ -117,3 +117,42 @@ def test_command_stats_api(self):
         self.assertNotIn("bt", command_stats)
         # Verify bt's regex command is not duplicatedly captured.
         self.assertNotIn("_regexp-bt", command_stats)
+
+    @add_test_categories(["dwo"])
+    def test_command_stats_force(self):
+        """
+        Test reporting all pssible debug info stats by force loading all debug
+        info. For example, dwo files
+        """
+        src_dir = self.getSourceDir()
+        dwo_yaml_path = os.path.join(src_dir, "main-main.dwo.yaml")
+        exe_yaml_path = os.path.join(src_dir, "main.yaml")
+        dwo_path = self.getBuildArtifact("main-main.dwo")
+        exe_path = self.getBuildArtifact("main")
+        self.yaml2obj(dwo_yaml_path, dwo_path)
+        self.yaml2obj(exe_yaml_path, exe_path)
+
+        # Turn on symbols on-demand loading
+        self.runCmd("settings set symbols.load-on-demand true")
+
+        # We need the current working directory to be set to the build directory
+        os.chdir(self.getBuildDir())
+        # Create a target with the object file we just created from YAML
+        target = self.dbg.CreateTarget(exe_path)
+        self.assertTrue(target, VALID_TARGET)
+
+        # Get statistics
+        stats_options = lldb.SBStatisticsOptions()
+        stats = target.GetStatistics(stats_options)
+        stream = lldb.SBStream()
+        stats.GetAsJSON(stream)
+        debug_stats = json.loads(stream.GetData())
+        self.assertEqual(debug_stats["totalDebugInfoByteSize"], 193)
+
+        # Get statistics with force loading
+        stats_options.SetReportAllAvailableDebugInfo(True)
+        stats_force = target.GetStatistics(stats_options)
+        stream_force = lldb.SBStream()
+        stats_force.GetAsJSON(stream_force)
+        debug_stats_force = json.loads(stream_force.GetData())
+        self.assertEqual(debug_stats_force["totalDebugInfoByteSize"], 445)

lldb/test/API/functionalities/stats_api/main-main.dwo.yaml

@@ -0,0 +1,37 @@
+--- !ELF
+FileHeader:
+  Class:           ELFCLASS64
+  Data:            ELFDATA2LSB
+  Type:            ET_REL
+  Machine:         EM_X86_64
+  SectionHeaderStringTable: .strtab
+Sections:
+  - Name:            .debug_str.dwo
+    Type:            SHT_PROGBITS
+    Flags:           [ SHF_EXCLUDE, SHF_MERGE, SHF_STRINGS ]
+    AddressAlign:    0x1
+    EntSize:         0x1
+    Content:         5F5A33666F6F7600666F6F006D61696E00696E7400636C616E672076657273696F6E2031372E302E36202843656E744F532031372E302E362D342E656C3929006D61696E2E637070006D61696E2D6D61696E2E64776F00
+  - Name:            .debug_str_offsets.dwo
+    Type:            SHT_PROGBITS
+    Flags:           [ SHF_EXCLUDE ]
+    AddressAlign:    0x1
+    Content:         00000000080000000C00000011000000150000004000000049000000
+  - Name:            .debug_info.dwo
+    Type:            SHT_PROGBITS
+    Flags:           [ SHF_EXCLUDE ]
+    AddressAlign:    0x1
+    Content:         3500000004000000000008010421000506939F5FCB7816797B02000600000001560001010103011C0000000156020105340000000403050400
+  - Name:            .debug_abbrev.dwo
+    Type:            SHT_PROGBITS
+    Flags:           [ SHF_EXCLUDE ]
+    AddressAlign:    0x1
+    Content:         01110125823E130503823EB042823EB142070000022E0011813E120640186E823E03823E3A0B3B0B3F190000032E0011813E1206401803823E3A0B3B0B49133F19000004240003823E3E0B0B0B000000
+  - Type:            SectionHeaderTable
+    Sections:
+      - Name:            .strtab
+      - Name:            .debug_str.dwo
+      - Name:            .debug_str_offsets.dwo
+      - Name:            .debug_info.dwo
+      - Name:            .debug_abbrev.dwo
+...