[LLDB] Enable 64 bit debug/type offset

This came out of from https://discourse.llvm.org/t/dwarf-dwp-4gb-limit/63902
With big binaries we can have .dwp files where .debug_info.dwo section can grow
beyond 4GB. We would like to support this in LLVM and in LLDB.

The plan is to enable manual parsing of cu/tu index in DWARF library
(https://reviews.llvm.org/D137882), and then
switch internal index data structure to 64 bit.
For the second part is to enable 64bit offset support in LLDB with
this patch.

Reviewed By: labath

Differential Revision: https://reviews.llvm.org/D138618

Author: ayermolo

lldb/include/lldb/Core/dwarf.h

@@ -30,11 +30,12 @@ typedef uint64_t dw_addr_t; // Dwarf address define that must be big enough for
                             // any addresses in the compile units that get
                             // parsed
 
-typedef uint32_t dw_offset_t; // Dwarf Debug Information Entry offset for any
+typedef uint64_t dw_offset_t; // Dwarf Debug Information Entry offset for any
                               // offset into the file
 
 /* Constants */
-#define DW_INVALID_OFFSET (~(dw_offset_t)0)
+#define DW_DIE_OFFSET_MAX_BITSIZE 40
+#define DW_INVALID_OFFSET (((uint64_t)1u << DW_DIE_OFFSET_MAX_BITSIZE) - 1)
 #define DW_INVALID_INDEX 0xFFFFFFFFul
 
 // #define DW_ADDR_none 0x0

lldb/include/lldb/Symbol/DWARFCallFrameInfo.h

@@ -128,7 +128,7 @@ class DWARFCallFrameInfo {
 
   void GetFDEIndex();
 
-  bool FDEToUnwindPlan(uint32_t offset, Address startaddr,
+  bool FDEToUnwindPlan(dw_offset_t offset, Address startaddr,
                        UnwindPlan &unwind_plan);
 
   const CIE *GetCIE(dw_offset_t cie_offset);
@@ -159,7 +159,7 @@ class DWARFCallFrameInfo {
   Type m_type;
 
   CIESP
-  ParseCIE(const uint32_t cie_offset);
+  ParseCIE(const dw_offset_t cie_offset);
 
   lldb::RegisterKind GetRegisterKind() const {
     return m_type == EH ? lldb::eRegisterKindEHFrame : lldb::eRegisterKindDWARF;

lldb/source/Plugins/SymbolFile/DWARF/AppleDWARFIndex.cpp

@@ -80,7 +80,6 @@ void AppleDWARFIndex::GetGlobalVariables(
   if (!m_apple_names_up)
     return;
 
-  lldbassert(!cu.GetSymbolFileDWARF().GetDwoNum());
   const DWARFUnit &non_skeleton_cu = cu.GetNonSkeletonUnit();
   DWARFMappedHash::DIEInfoArray hash_data;
   m_apple_names_up->AppendAllDIEsInRange(non_skeleton_cu.GetOffset(),

lldb/source/Plugins/SymbolFile/DWARF/DIERef.cpp

@@ -17,40 +17,22 @@ using namespace lldb_private;
 
 void llvm::format_provider<DIERef>::format(const DIERef &ref, raw_ostream &OS,
                                            StringRef Style) {
-  if (ref.dwo_num())
-    OS << format_hex_no_prefix(*ref.dwo_num(), 8) << "/";
+  if (ref.file_index())
+    OS << format_hex_no_prefix(*ref.file_index(), 8) << "/";
   OS << (ref.section() == DIERef::DebugInfo ? "INFO" : "TYPE");
   OS << "/" << format_hex_no_prefix(ref.die_offset(), 8);
 }
 
-constexpr uint32_t k_dwo_num_mask = 0x3FFFFFFF;
-constexpr uint32_t k_dwo_num_valid_bitmask = (1u << 30);
-constexpr uint32_t k_section_bitmask = (1u << 31);
-
 std::optional<DIERef> DIERef::Decode(const DataExtractor &data,
                                      lldb::offset_t *offset_ptr) {
-  const uint32_t bitfield_storage = data.GetU32(offset_ptr);
-  uint32_t dwo_num = bitfield_storage & k_dwo_num_mask;
-  bool dwo_num_valid = (bitfield_storage & (k_dwo_num_valid_bitmask)) != 0;
-  Section section = (Section)((bitfield_storage & (k_section_bitmask)) != 0);
+  DIERef die_ref(data.GetU64(offset_ptr));
+
   // DIE offsets can't be zero and if we fail to decode something from data,
   // it will return 0
-  dw_offset_t die_offset = data.GetU32(offset_ptr);
-  if (die_offset == 0)
+  if (!die_ref.die_offset())
     return std::nullopt;
-  if (dwo_num_valid)
-    return DIERef(dwo_num, section, die_offset);
-  else
-    return DIERef(std::nullopt, section, die_offset);
-}
 
-void DIERef::Encode(DataEncoder &encoder) const {
-  uint32_t bitfield_storage = m_dwo_num;
-  if (m_dwo_num_valid)
-    bitfield_storage |= k_dwo_num_valid_bitmask;
-  if (m_section)
-    bitfield_storage |= k_section_bitmask;
-  encoder.AppendU32(bitfield_storage);
-  static_assert(sizeof(m_die_offset) == 4, "m_die_offset must be 4 bytes");
-  encoder.AppendU32(m_die_offset);
+  return die_ref;
 }
+
+void DIERef::Encode(DataEncoder &encoder) const { encoder.AppendU64(get_id()); }

lldb/source/Plugins/SymbolFile/DWARF/DIERef.h

@@ -10,33 +10,53 @@
 #define LLDB_SOURCE_PLUGINS_SYMBOLFILE_DWARF_DIEREF_H
 
 #include "lldb/Core/dwarf.h"
-#include "llvm/Support/FormatProviders.h"
+#include "lldb/Utility/LLDBAssert.h"
 #include <cassert>
 #include <optional>
-#include <vector>
 
 /// Identifies a DWARF debug info entry within a given Module. It contains three
 /// "coordinates":
-/// - dwo_num: identifies the dwo file in the Module. If this field is not set,
-///   the DIERef references the main file.
+/// - file_index: identifies the separate stand alone debug info file
+///   that is referred to by the main debug info file. This will be the
+///   index of a DWO file for fission, or the .o file on mac when not
+///   using a dSYM file. If this field is not set, then this references
+///   a DIE inside the original object file.
 /// - section: identifies the section of the debug info entry in the given file:
 ///   debug_info or debug_types.
 /// - die_offset: The offset of the debug info entry as an absolute offset from
 ///   the beginning of the section specified in the section field.
 class DIERef {
 public:
   enum Section : uint8_t { DebugInfo, DebugTypes };
-
-  DIERef(std::optional<uint32_t> dwo_num, Section section,
+  DIERef(std::optional<uint32_t> file_index, Section section,
          dw_offset_t die_offset)
-      : m_dwo_num(dwo_num.value_or(0)), m_dwo_num_valid(bool(dwo_num)),
-        m_section(section), m_die_offset(die_offset) {
-    assert(this->dwo_num() == dwo_num && "Dwo number out of range?");
+      : m_die_offset(die_offset), m_file_index(file_index.value_or(0)),
+        m_file_index_valid(file_index ? true : false), m_section(section) {
+    assert(this->file_index() == file_index && "File Index is out of range?");
+  }
+
+  explicit DIERef(lldb::user_id_t uid) {
+    m_die_offset = uid & k_die_offset_mask;
+    m_file_index_valid = (uid & k_file_index_valid_bit) != 0;
+    m_file_index = m_file_index_valid
+                       ? (uid >> k_die_offset_bit_size) & k_file_index_mask
+                       : 0;
+    m_section =
+        (uid & k_section_bit) != 0 ? Section::DebugTypes : Section::DebugInfo;
+  }
+
+  lldb::user_id_t get_id() const {
+    if (m_die_offset == k_die_offset_mask)
+      return LLDB_INVALID_UID;
+
+    return lldb::user_id_t(file_index().value_or(0)) << k_die_offset_bit_size |
+           die_offset() | (m_file_index_valid ? k_file_index_valid_bit : 0) |
+           (section() == Section::DebugTypes ? k_section_bit : 0);
   }
 
-  std::optional<uint32_t> dwo_num() const {
-    if (m_dwo_num_valid)
-      return m_dwo_num;
+  std::optional<uint32_t> file_index() const {
+    if (m_file_index_valid)
+      return m_file_index;
     return std::nullopt;
   }
 
@@ -45,17 +65,17 @@ class DIERef {
   dw_offset_t die_offset() const { return m_die_offset; }
 
   bool operator<(DIERef other) const {
-    if (m_dwo_num_valid != other.m_dwo_num_valid)
-      return m_dwo_num_valid < other.m_dwo_num_valid;
-    if (m_dwo_num_valid && (m_dwo_num != other.m_dwo_num))
-      return m_dwo_num < other.m_dwo_num;
+    if (m_file_index_valid != other.m_file_index_valid)
+      return m_file_index_valid < other.m_file_index_valid;
+    if (m_file_index_valid && (m_file_index != other.m_file_index))
+      return m_file_index < other.m_file_index;
     if (m_section != other.m_section)
       return m_section < other.m_section;
     return m_die_offset < other.m_die_offset;
   }
 
   bool operator==(const DIERef &rhs) const {
-    return dwo_num() == rhs.dwo_num() && m_section == rhs.m_section &&
+    return file_index() == rhs.file_index() && m_section == rhs.m_section &&
            m_die_offset == rhs.m_die_offset;
   }
 
@@ -85,11 +105,28 @@ class DIERef {
   ///
   void Encode(lldb_private::DataEncoder &encoder) const;
 
+  static constexpr uint64_t k_die_offset_bit_size = DW_DIE_OFFSET_MAX_BITSIZE;
+  static constexpr uint64_t k_file_index_bit_size =
+      64 - DW_DIE_OFFSET_MAX_BITSIZE - /* size of control bits */ 2;
+
+  static constexpr uint64_t k_file_index_valid_bit =
+      (1ull << (k_file_index_bit_size + k_die_offset_bit_size));
+  static constexpr uint64_t k_section_bit =
+      (1ull << (k_file_index_bit_size + k_die_offset_bit_size + 1));
+  static constexpr uint64_t
+      k_file_index_mask = (~0ull) >> (64 - k_file_index_bit_size); // 0x3fffff;
+  static constexpr uint64_t k_die_offset_mask = (~0ull) >>
+                                                (64 - k_die_offset_bit_size);
+
 private:
-  uint32_t m_dwo_num : 30;
-  uint32_t m_dwo_num_valid : 1;
-  uint32_t m_section : 1;
-  dw_offset_t m_die_offset;
+  // Allow 2TB of .debug_info/.debug_types offset
+  dw_offset_t m_die_offset : k_die_offset_bit_size;
+  // Used for DWO index or for .o file index on mac
+  dw_offset_t m_file_index : k_file_index_bit_size;
+  // Set to 1 if m_file_index is a DWO number
+  dw_offset_t m_file_index_valid : 1;
+  // Set to 0 for .debug_info 1 for .debug_types,
+  dw_offset_t m_section : 1;
 };
 static_assert(sizeof(DIERef) == 8);
 

lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp

@@ -731,10 +731,10 @@ DWARFASTParserClang::ParseTypeModifier(const SymbolContext &sc,
     }
   }
 
-  type_sp = dwarf->MakeType(
-      die.GetID(), attrs.name, attrs.byte_size, nullptr,
-      dwarf->GetUID(attrs.type.Reference()), encoding_data_type, &attrs.decl,
-      clang_type, resolve_state, TypePayloadClang(GetOwningClangModule(die)));
+  type_sp = dwarf->MakeType(die.GetID(), attrs.name, attrs.byte_size, nullptr,
+                            attrs.type.Reference().GetID(), encoding_data_type,
+                            &attrs.decl, clang_type, resolve_state,
+                            TypePayloadClang(GetOwningClangModule(die)));
 
   dwarf->GetDIEToType()[die.GetDIE()] = type_sp.get();
   return type_sp;
@@ -834,11 +834,11 @@ TypeSP DWARFASTParserClang::ParseEnum(const SymbolContext &sc,
 
   LinkDeclContextToDIE(TypeSystemClang::GetDeclContextForType(clang_type), die);
 
-  type_sp = dwarf->MakeType(die.GetID(), attrs.name, attrs.byte_size, nullptr,
-                            dwarf->GetUID(attrs.type.Reference()),
-                            Type::eEncodingIsUID, &attrs.decl, clang_type,
-                            Type::ResolveState::Forward,
-                            TypePayloadClang(GetOwningClangModule(die)));
+  type_sp =
+      dwarf->MakeType(die.GetID(), attrs.name, attrs.byte_size, nullptr,
+                      attrs.type.Reference().GetID(), Type::eEncodingIsUID,
+                      &attrs.decl, clang_type, Type::ResolveState::Forward,
+                      TypePayloadClang(GetOwningClangModule(die)));
 
   if (TypeSystemClang::StartTagDeclarationDefinition(clang_type)) {
     if (die.HasChildren()) {
@@ -1336,7 +1336,7 @@ DWARFASTParserClang::ParseArrayType(const DWARFDIE &die,
   ConstString empty_name;
   TypeSP type_sp =
       dwarf->MakeType(die.GetID(), empty_name, array_element_bit_stride / 8,
-                      nullptr, dwarf->GetUID(type_die), Type::eEncodingIsUID,
+                      nullptr, type_die.GetID(), Type::eEncodingIsUID,
                       &attrs.decl, clang_type, Type::ResolveState::Full);
   type_sp->SetEncodingType(element_type);
   const clang::Type *type = ClangUtil::GetQualType(clang_type).getTypePtr();

lldb/source/Plugins/SymbolFile/DWARF/DWARFBaseDIE.cpp

@@ -23,8 +23,8 @@ std::optional<DIERef> DWARFBaseDIE::GetDIERef() const {
   if (!IsValid())
     return std::nullopt;
 
-  return DIERef(m_cu->GetSymbolFileDWARF().GetDwoNum(), m_cu->GetDebugSection(),
-                m_die->GetOffset());
+  return DIERef(m_cu->GetSymbolFileDWARF().GetFileIndex(),
+                m_cu->GetDebugSection(), m_die->GetOffset());
 }
 
 dw_tag_t DWARFBaseDIE::Tag() const {
@@ -70,8 +70,10 @@ uint64_t DWARFBaseDIE::GetAttributeValueAsAddress(const dw_attr_t attr,
 }
 
 lldb::user_id_t DWARFBaseDIE::GetID() const {
-  if (IsValid())
-    return GetDWARF()->GetUID(*this);
+  const std::optional<DIERef> &ref = this->GetDIERef();
+  if (ref)
+    return ref->get_id();
+
   return LLDB_INVALID_UID;
 }
 

lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugInfo.cpp

@@ -136,7 +136,7 @@ uint32_t DWARFDebugInfo::FindUnitIndex(DIERef::Section section,
       });
   uint32_t idx = std::distance(m_units.begin(), pos);
   if (idx == 0)
-    return DW_INVALID_OFFSET;
+    return DW_INVALID_INDEX;
   return idx - 1;
 }
 

lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugInfoEntry.cpp

@@ -64,7 +64,7 @@ bool DWARFDebugInfoEntry::Extract(const DWARFDataExtractor &data,
         "[{0:x16}]: invalid abbreviation code {1}, "
         "please file a bug and "
         "attach the file at the start of this error message",
-        m_offset, (unsigned)abbr_idx);
+        (uint64_t)m_offset, (unsigned)abbr_idx);
     // WE can't parse anymore if the DWARF is borked...
     *offset_ptr = UINT32_MAX;
     return false;
@@ -195,7 +195,7 @@ bool DWARFDebugInfoEntry::Extract(const DWARFDataExtractor &data,
               "[{0:x16}]: Unsupported DW_FORM_{1:x}, please file a bug "
               "and "
               "attach the file at the start of this error message",
-              m_offset, (unsigned)form);
+              (uint64_t)m_offset, (unsigned)form);
           *offset_ptr = m_offset;
           return false;
         }

lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugInfoEntry.h

@@ -36,7 +36,8 @@ class DWARFDebugInfoEntry {
   typedef collection::const_iterator const_iterator;
 
   DWARFDebugInfoEntry()
-      : m_offset(DW_INVALID_OFFSET), m_sibling_idx(0), m_has_children(false) {}
+      : m_offset(DW_INVALID_OFFSET), m_parent_idx(0), m_sibling_idx(0),
+        m_has_children(false) {}
 
   explicit operator bool() const { return m_offset != DW_INVALID_OFFSET; }
   bool operator==(const DWARFDebugInfoEntry &rhs) const;
@@ -165,14 +166,16 @@ class DWARFDebugInfoEntry {
   static DWARFDeclContext
   GetDWARFDeclContextStatic(const DWARFDebugInfoEntry *die, DWARFUnit *cu);
 
-  dw_offset_t m_offset; // Offset within the .debug_info/.debug_types
-  uint32_t m_parent_idx = 0;   // How many to subtract from "this" to get the
-                               // parent. If zero this die has no parent
-  uint32_t m_sibling_idx : 31, // How many to add to "this" to get the sibling.
-      // If it is zero, then the DIE doesn't have children, or the
-      // DWARF claimed it had children but the DIE only contained
-      // a single NULL terminating child.
-      m_has_children : 1;
+  // Up to 2TB offset within the .debug_info/.debug_types
+  dw_offset_t m_offset : DW_DIE_OFFSET_MAX_BITSIZE;
+  // How many to subtract from "this" to get the parent. If zero this die has no
+  // parent
+  dw_offset_t m_parent_idx : 64 - DW_DIE_OFFSET_MAX_BITSIZE;
+  // How many to add to "this" to get the sibling.
+  // If it is zero, then the DIE doesn't have children,
+  // or the DWARF claimed it had children but the DIE
+  // only contained a single NULL terminating child.
+  uint32_t m_sibling_idx : 31, m_has_children : 1;
   uint16_t m_abbr_idx = 0;
   /// A copy of the DW_TAG value so we don't have to go through the compile
   /// unit abbrev table

lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.cpp

@@ -341,7 +341,7 @@ void DWARFUnit::SetDwoStrOffsetsBase() {
   if (const llvm::DWARFUnitIndex::Entry *entry = m_header.GetIndexEntry()) {
     if (const auto *contribution =
             entry->getContribution(llvm::DW_SECT_STR_OFFSETS))
-      baseOffset = contribution->getOffset32();
+      baseOffset = contribution->getOffset();
     else
       return;
   }
@@ -489,7 +489,7 @@ void DWARFUnit::SetLoclistsBase(dw_addr_t loclists_base) {
           *GetDWOId());
       return;
     }
-    offset += contribution->getOffset32();
+    offset += contribution->getOffset();
   }
   m_loclists_base = loclists_base;
 
@@ -527,7 +527,7 @@ DWARFDataExtractor DWARFUnit::GetLocationData() const {
   if (const llvm::DWARFUnitIndex::Entry *entry = m_header.GetIndexEntry()) {
     if (const auto *contribution = entry->getContribution(
             GetVersion() >= 5 ? llvm::DW_SECT_LOCLISTS : llvm::DW_SECT_EXT_LOC))
-      return DWARFDataExtractor(data, contribution->getOffset32(),
+      return DWARFDataExtractor(data, contribution->getOffset(),
                                 contribution->getLength32());
     return DWARFDataExtractor();
   }
@@ -540,7 +540,7 @@ DWARFDataExtractor DWARFUnit::GetRnglistData() const {
   if (const llvm::DWARFUnitIndex::Entry *entry = m_header.GetIndexEntry()) {
     if (const auto *contribution =
             entry->getContribution(llvm::DW_SECT_RNGLISTS))
-      return DWARFDataExtractor(data, contribution->getOffset32(),
+      return DWARFDataExtractor(data, contribution->getOffset(),
                                 contribution->getLength32());
     GetSymbolFileDWARF().GetObjectFile()->GetModule()->ReportError(
         "Failed to find range list contribution for CU with signature {0:x16}",
@@ -935,7 +935,7 @@ DWARFUnitHeader::extract(const DWARFDataExtractor &data,
           llvm::inconvertibleErrorCode(),
           "DWARF package index missing abbreviation column");
     }
-    header.m_abbr_offset = abbr_entry->getOffset32();
+    header.m_abbr_offset = abbr_entry->getOffset();
   }
 
   bool length_OK = data.ValidOffset(header.GetNextUnitOffset() - 1);