Reland "[lldb] Add count for number of DWO files loaded in statistics #144424" (#145572)

This relands changes in #144424 for adding a count of DWO files
parsed/loaded and the total number of DWO files.

The previous PR was reverted in #145494 due to the newly added unit
tests failing on Windows and MacOS CIs since these platforms don't
support DWO. This change add an additional
`@add_test_categories(["dwo"])` to the new tests to
[skip](https://github.com/llvm/llvm-project/blob/cd46354dbd10820158edabe14dbd49d9f9010722/lldb/packages/Python/lldbsuite/test/test_categories.py#L56)
these tests on Windows/MacOS.

Original PR: #144424

### Testing
Ran unit tests
```
$ bin/lldb-dotest -p TestStats.py llvm-project/lldb/test/API/commands/statistics/basic/
----------------------------------------------------------------------
Ran 24 tests in 211.391s

OK (skipped=3)
```

Author: qxy11

lldb/include/lldb/Symbol/SymbolFile.h

@@ -472,6 +472,14 @@ class SymbolFile : public PluginInterface {
     return false;
   };
 
+  /// Get number of loaded/parsed DWO files. This is emitted in "statistics
+  /// dump"
+  ///
+  /// \returns
+  ///     A pair containing (loaded_dwo_count, total_dwo_count). If this
+  ///     symbol file doesn't support DWO files, both counts will be 0.
+  virtual std::pair<uint32_t, uint32_t> GetDwoFileCounts() { return {0, 0}; }
+
   virtual lldb::TypeSP
   MakeType(lldb::user_id_t uid, ConstString name,
            std::optional<uint64_t> byte_size, SymbolContextScope *context,

lldb/include/lldb/Target/Statistics.h

@@ -153,6 +153,8 @@ struct ModuleStats {
   bool symtab_stripped = false;
   bool debug_info_had_variable_errors = false;
   bool debug_info_had_incomplete_types = false;
+  uint32_t dwo_file_count = 0;
+  uint32_t loaded_dwo_file_count = 0;
 };
 
 struct ConstStringStats {

lldb/packages/Python/lldbsuite/test/builders/builder.py

@@ -247,13 +247,25 @@ def getLLDBObjRoot(self):
     def _getDebugInfoArgs(self, debug_info):
         if debug_info is None:
             return []
-        if debug_info == "dwarf":
-            return ["MAKE_DSYM=NO"]
-        if debug_info == "dwo":
-            return ["MAKE_DSYM=NO", "MAKE_DWO=YES"]
-        if debug_info == "gmodules":
-            return ["MAKE_DSYM=NO", "MAKE_GMODULES=YES"]
-        return None
+
+        debug_options = debug_info if isinstance(debug_info, list) else [debug_info]
+        option_flags = {
+            "dwarf": {"MAKE_DSYM": "NO"},
+            "dwo": {"MAKE_DSYM": "NO", "MAKE_DWO": "YES"},
+            "gmodules": {"MAKE_DSYM": "NO", "MAKE_GMODULES": "YES"},
+            "debug_names": {"MAKE_DEBUG_NAMES": "YES"},
+            "dwp": {"MAKE_DSYM": "NO", "MAKE_DWP": "YES"},
+        }
+
+        # Collect all flags, with later options overriding earlier ones
+        flags = {}
+
+        for option in debug_options:
+            if not option or option not in option_flags:
+                return None  # Invalid options
+            flags.update(option_flags[option])
+
+        return [f"{key}={value}" for key, value in flags.items()]
 
     def getBuildCommand(
         self,

lldb/packages/Python/lldbsuite/test/make/Makefile.rules

@@ -276,6 +276,10 @@ ifeq "$(MAKE_DWO)" "YES"
 	CFLAGS += -gsplit-dwarf
 endif
 
+ifeq "$(MAKE_DEBUG_NAMES)" "YES"
+	CFLAGS += -gpubnames
+endif
+
 ifeq "$(USE_PRIVATE_MODULE_CACHE)" "YES"
 THE_CLANG_MODULE_CACHE_DIR := $(BUILDDIR)/private-module-cache
 else

lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp

@@ -4420,3 +4420,32 @@ void SymbolFileDWARF::GetCompileOptions(
     args.insert({comp_unit, Args(flags)});
   }
 }
+
+std::pair<uint32_t, uint32_t> SymbolFileDWARF::GetDwoFileCounts() {
+  uint32_t total_dwo_count = 0;
+  uint32_t loaded_dwo_count = 0;
+
+  DWARFDebugInfo &info = DebugInfo();
+  const size_t num_cus = info.GetNumUnits();
+  for (size_t cu_idx = 0; cu_idx < num_cus; cu_idx++) {
+    DWARFUnit *dwarf_cu = info.GetUnitAtIndex(cu_idx);
+    if (dwarf_cu == nullptr)
+      continue;
+
+    // Check if this is a DWO unit by checking if it has a DWO ID.
+    if (!dwarf_cu->GetDWOId().has_value())
+      continue;
+
+    total_dwo_count++;
+
+    // If we have a DWO symbol file, that means we were able to successfully
+    // load it.
+    SymbolFile *dwo_symfile =
+        dwarf_cu->GetDwoSymbolFile(/*load_all_debug_info=*/false);
+    if (dwo_symfile) {
+      loaded_dwo_count++;
+    }
+  }
+
+  return {loaded_dwo_count, total_dwo_count};
+}

lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h

@@ -282,6 +282,11 @@ class SymbolFileDWARF : public SymbolFileCommon {
   bool GetSeparateDebugInfo(StructuredData::Dictionary &d,
                             bool errors_only) override;
 
+  // Gets a pair of loaded and total dwo file counts.
+  // For split-dwarf files, this reports the counts for successfully loaded DWO
+  // CUs and total DWO CUs. For non-split-dwarf files, this reports 0 for both.
+  std::pair<uint32_t, uint32_t> GetDwoFileCounts() override;
+
   DWARFContext &GetDWARFContext() { return m_context; }
 
   const std::shared_ptr<SymbolFileDWARFDwo> &GetDwpSymbolFile();

lldb/source/Target/Statistics.cpp

@@ -73,6 +73,8 @@ json::Value ModuleStats::ToJSON() const {
                      debug_info_had_incomplete_types);
   module.try_emplace("symbolTableStripped", symtab_stripped);
   module.try_emplace("symbolTableSymbolCount", symtab_symbol_count);
+  module.try_emplace("dwoFileCount", dwo_file_count);
+  module.try_emplace("loadedDwoFileCount", loaded_dwo_file_count);
 
   if (!symbol_locator_time.map.empty()) {
     json::Object obj;
@@ -86,7 +88,7 @@ json::Value ModuleStats::ToJSON() const {
 
   if (!symfile_modules.empty()) {
     json::Array symfile_ids;
-    for (const auto symfile_id: symfile_modules)
+    for (const auto symfile_id : symfile_modules)
       symfile_ids.emplace_back(symfile_id);
     module.try_emplace("symbolFileModuleIdentifiers", std::move(symfile_ids));
   }
@@ -322,6 +324,8 @@ llvm::json::Value DebuggerStats::ReportStatistics(
   uint32_t num_modules_with_incomplete_types = 0;
   uint32_t num_stripped_modules = 0;
   uint32_t symtab_symbol_count = 0;
+  uint32_t total_loaded_dwo_file_count = 0;
+  uint32_t total_dwo_file_count = 0;
   for (size_t image_idx = 0; image_idx < num_modules; ++image_idx) {
     Module *module = target != nullptr
                          ? target->GetImages().GetModuleAtIndex(image_idx).get()
@@ -353,6 +357,10 @@ llvm::json::Value DebuggerStats::ReportStatistics(
         for (const auto &symbol_module : symbol_modules.Modules())
           module_stat.symfile_modules.push_back((intptr_t)symbol_module.get());
       }
+      std::tie(module_stat.loaded_dwo_file_count, module_stat.dwo_file_count) =
+          sym_file->GetDwoFileCounts();
+      total_dwo_file_count += module_stat.dwo_file_count;
+      total_loaded_dwo_file_count += module_stat.loaded_dwo_file_count;
       module_stat.debug_info_index_loaded_from_cache =
           sym_file->GetDebugInfoIndexWasLoadedFromCache();
       if (module_stat.debug_info_index_loaded_from_cache)
@@ -427,6 +435,8 @@ llvm::json::Value DebuggerStats::ReportStatistics(
       {"totalDebugInfoEnabled", num_debug_info_enabled_modules},
       {"totalSymbolTableStripped", num_stripped_modules},
       {"totalSymbolTableSymbolCount", symtab_symbol_count},
+      {"totalLoadedDwoFileCount", total_loaded_dwo_file_count},
+      {"totalDwoFileCount", total_dwo_file_count},
   };
 
   if (include_targets) {

lldb/test/API/commands/statistics/basic/TestStats.py

@@ -177,6 +177,8 @@ def test_default_no_run(self):
             "totalDebugInfoIndexLoadedFromCache",
             "totalDebugInfoIndexSavedToCache",
             "totalDebugInfoParseTime",
+            "totalDwoFileCount",
+            "totalLoadedDwoFileCount",
         ]
         self.verify_keys(debug_stats, '"debug_stats"', debug_stat_keys, None)
         if self.getPlatform() != "windows":
@@ -287,6 +289,8 @@ def test_default_with_run(self):
             "totalDebugInfoIndexLoadedFromCache",
             "totalDebugInfoIndexSavedToCache",
             "totalDebugInfoParseTime",
+            "totalDwoFileCount",
+            "totalLoadedDwoFileCount",
         ]
         self.verify_keys(debug_stats, '"debug_stats"', debug_stat_keys, None)
         stats = debug_stats["targets"][0]
@@ -325,6 +329,8 @@ def test_memory(self):
             "totalDebugInfoIndexLoadedFromCache",
             "totalDebugInfoIndexSavedToCache",
             "totalDebugInfoByteSize",
+            "totalDwoFileCount",
+            "totalLoadedDwoFileCount",
         ]
         self.verify_keys(debug_stats, '"debug_stats"', debug_stat_keys, None)
 
@@ -377,6 +383,8 @@ def test_modules(self):
             "totalDebugInfoIndexLoadedFromCache",
             "totalDebugInfoIndexSavedToCache",
             "totalDebugInfoByteSize",
+            "totalDwoFileCount",
+            "totalLoadedDwoFileCount",
         ]
         self.verify_keys(debug_stats, '"debug_stats"', debug_stat_keys, None)
         stats = debug_stats["targets"][0]
@@ -397,6 +405,8 @@ def test_modules(self):
             "symbolTableLoadedFromCache",
             "symbolTableParseTime",
             "symbolTableSavedToCache",
+            "dwoFileCount",
+            "loadedDwoFileCount",
             "triple",
             "uuid",
         ]
@@ -485,6 +495,8 @@ def test_breakpoints(self):
             "totalDebugInfoIndexLoadedFromCache",
             "totalDebugInfoIndexSavedToCache",
             "totalDebugInfoByteSize",
+            "totalDwoFileCount",
+            "totalLoadedDwoFileCount",
         ]
         self.verify_keys(debug_stats, '"debug_stats"', debug_stat_keys, None)
         target_stats = debug_stats["targets"][0]
@@ -513,6 +525,136 @@ def test_breakpoints(self):
                 breakpoint, 'target_stats["breakpoints"]', bp_keys_exist, None
             )
 
+    @add_test_categories(["dwo"])
+    def test_non_split_dwarf_has_no_dwo_files(self):
+        """
+        Test "statistics dump" and the dwo file count.
+        Builds a binary without split-dwarf mode, and then
+        verifies the dwo file count is zero after running "statistics dump"
+        """
+        da = {"CXX_SOURCES": "third.cpp baz.cpp", "EXE": self.getBuildArtifact("a.out")}
+        self.build(dictionary=da, debug_info=["debug_names"])
+        self.addTearDownCleanup(dictionary=da)
+        exe = self.getBuildArtifact("a.out")
+        target = self.createTestTarget(file_path=exe)
+        debug_stats = self.get_stats()
+        self.assertIn("totalDwoFileCount", debug_stats)
+        self.assertIn("totalLoadedDwoFileCount", debug_stats)
+
+        # Verify that the dwo file count is zero
+        self.assertEqual(debug_stats["totalDwoFileCount"], 0)
+        self.assertEqual(debug_stats["totalLoadedDwoFileCount"], 0)
+
+    @add_test_categories(["dwo"])
+    def test_no_debug_names_eager_loads_dwo_files(self):
+        """
+        Test the eager loading behavior of DWO files when debug_names is absent by
+        building a split-dwarf binary without debug_names and then running "statistics dump".
+        DWO file loading behavior:
+        - With debug_names: DebugNamesDWARFIndex allows for lazy loading.
+          DWO files are loaded on-demand when symbols are actually looked up
+        - Without debug_names: ManualDWARFIndex uses eager loading.
+          All DWO files are loaded upfront during the first symbol lookup to build a manual index.
+        """
+        da = {"CXX_SOURCES": "third.cpp baz.cpp", "EXE": self.getBuildArtifact("a.out")}
+        self.build(dictionary=da, debug_info=["dwo"])
+        self.addTearDownCleanup(dictionary=da)
+        exe = self.getBuildArtifact("a.out")
+        target = self.createTestTarget(file_path=exe)
+        debug_stats = self.get_stats()
+        self.assertIn("totalDwoFileCount", debug_stats)
+        self.assertIn("totalLoadedDwoFileCount", debug_stats)
+
+        # Verify that all DWO files are loaded
+        self.assertEqual(debug_stats["totalDwoFileCount"], 2)
+        self.assertEqual(debug_stats["totalLoadedDwoFileCount"], 2)
+
+    @add_test_categories(["dwo"])
+    def test_split_dwarf_dwo_file_count(self):
+        """
+        Test "statistics dump" and the dwo file count.
+        Builds a binary w/ separate .dwo files and debug_names, and then
+        verifies the loaded dwo file count is the expected count after running
+        various commands
+        """
+        da = {"CXX_SOURCES": "third.cpp baz.cpp", "EXE": self.getBuildArtifact("a.out")}
+        # -gsplit-dwarf creates separate .dwo files,
+        # -gpubnames enables the debug_names accelerator tables for faster symbol lookup
+        #  and lazy loading of DWO files
+        # Expected output: third.dwo (contains main) and baz.dwo (contains Baz struct/function)
+        self.build(dictionary=da, debug_info=["dwo", "debug_names"])
+        self.addTearDownCleanup(dictionary=da)
+        exe = self.getBuildArtifact("a.out")
+        target = self.createTestTarget(file_path=exe)
+        debug_stats = self.get_stats()
+
+        # 1) 2 DWO files available but none loaded yet
+        self.assertEqual(len(debug_stats["modules"]), 1)
+        self.assertIn("totalLoadedDwoFileCount", debug_stats)
+        self.assertIn("totalDwoFileCount", debug_stats)
+        self.assertEqual(debug_stats["totalLoadedDwoFileCount"], 0)
+        self.assertEqual(debug_stats["totalDwoFileCount"], 2)
+
+        # Since there's only one module, module stats should have the same counts as total counts
+        self.assertIn("dwoFileCount", debug_stats["modules"][0])
+        self.assertIn("loadedDwoFileCount", debug_stats["modules"][0])
+        self.assertEqual(debug_stats["modules"][0]["loadedDwoFileCount"], 0)
+        self.assertEqual(debug_stats["modules"][0]["dwoFileCount"], 2)
+
+        # 2) Setting breakpoint in main triggers loading of third.dwo (contains main function)
+        self.runCmd("b main")
+        debug_stats = self.get_stats()
+        self.assertEqual(debug_stats["totalLoadedDwoFileCount"], 1)
+        self.assertEqual(debug_stats["totalDwoFileCount"], 2)
+
+        self.assertEqual(debug_stats["modules"][0]["loadedDwoFileCount"], 1)
+        self.assertEqual(debug_stats["modules"][0]["dwoFileCount"], 2)
+
+        # 3) Type lookup forces loading of baz.dwo (contains struct Baz definition)
+        self.runCmd("type lookup Baz")
+        debug_stats = self.get_stats()
+        self.assertEqual(debug_stats["totalLoadedDwoFileCount"], 2)
+        self.assertEqual(debug_stats["totalDwoFileCount"], 2)
+
+        self.assertEqual(debug_stats["modules"][0]["loadedDwoFileCount"], 2)
+        self.assertEqual(debug_stats["modules"][0]["dwoFileCount"], 2)
+
+    @add_test_categories(["dwo"])
+    def test_dwp_dwo_file_count(self):
+        """
+        Test "statistics dump" and the loaded dwo file count.
+        Builds a binary w/ a separate .dwp file and debug_names, and then
+        verifies the loaded dwo file count is the expected count after running
+        various commands.
+
+        We expect the DWO file counters to reflect the number of compile units
+        loaded from the DWP file (each representing what was originally a separate DWO file)
+        """
+        da = {"CXX_SOURCES": "third.cpp baz.cpp", "EXE": self.getBuildArtifact("a.out")}
+        self.build(dictionary=da, debug_info=["dwp", "debug_names"])
+        self.addTearDownCleanup(dictionary=da)
+        exe = self.getBuildArtifact("a.out")
+        target = self.createTestTarget(file_path=exe)
+        debug_stats = self.get_stats()
+
+        # Initially: 2 DWO files available but none loaded yet
+        self.assertIn("totalLoadedDwoFileCount", debug_stats)
+        self.assertIn("totalDwoFileCount", debug_stats)
+        self.assertEqual(debug_stats["totalLoadedDwoFileCount"], 0)
+        self.assertEqual(debug_stats["totalDwoFileCount"], 2)
+
+        # Setting breakpoint in main triggers parsing of the CU within a.dwp corresponding to third.dwo (contains main function)
+        self.runCmd("b main")
+        debug_stats = self.get_stats()
+        self.assertEqual(debug_stats["totalLoadedDwoFileCount"], 1)
+        self.assertEqual(debug_stats["totalDwoFileCount"], 2)
+
+        # Type lookup forces parsing of the CU within a.dwp corresponding to baz.dwo (contains struct Baz definition)
+        self.runCmd("type lookup Baz")
+        debug_stats = self.get_stats()
+        self.assertEqual(debug_stats["totalDwoFileCount"], 2)
+        self.assertEqual(debug_stats["totalLoadedDwoFileCount"], 2)
+
     @skipUnlessDarwin
     @no_debug_info_test
     def test_dsym_binary_has_symfile_in_stats(self):

lldb/test/API/commands/statistics/basic/baz.cpp

@@ -0,0 +1,12 @@
+// Helper that the lldb command `statistics dump` works in split-dwarf mode.
+
+struct Baz {
+  int x;
+  bool y;
+};
+
+void baz() {
+  Baz b;
+  b.x = 1;
+  b.y = true;
+}

lldb/test/API/commands/statistics/basic/third.cpp

@@ -0,0 +1,7 @@
+// Test that the lldb command `statistics dump` works.
+
+void baz();
+int main(void) {
+  baz();
+  return 0;
+}