Skip to content

Commit 73712c8

Browse files
ayermoloayermolo
committed
[DWARFLibrary] Add support to re-construct cu-index
According to DWARF5 specification and gnu specification for DWARF4 the offset entry in the CU/TU Index is 32 bits. This presents a problem when .debug_info.dwo in DWP file grows beyond 4GB. The CU Index becomes partially corrupted. This diff adds manual parsing of .debug_info.dwo/.debug_abbrev.dwo to reconstruct CU index in general, and TU index for DWARF5. This is a work around until DWARF6 spec is finalized. Next patch will change internal CU/TU struct to 64 bit, and change uses as necessary. The plan is to land all the patches in one go after all are approved. This patch originates from the discussion in: https://discourse.llvm.org/t/dwarf-dwp-4gb-limit/63902 Reviewed By: dblaikie Differential Revision: https://reviews.llvm.org/D137882
1 parent 3531c41 commit 73712c8

File tree

8 files changed

+226
-5
lines changed

8 files changed

+226
-5
lines changed

‎llvm/include/llvm/DebugInfo/DWARF/DWARFContext.h

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,10 @@ class DWARFContext : public DIContext {
107107
MacroDwoSection
108108
};
109109

110+
// When set parses debug_info.dwo/debug_abbrev.dwo manually and populates CU
111+
// Index, and TU Index for DWARF5.
112+
bool ParseCUTUIndexManually;
113+
110114
public:
111115
DWARFContext(std::unique_ptr<const DWARFObject> DObj,
112116
std::string DWPName = "",
@@ -443,6 +447,14 @@ class DWARFContext : public DIContext {
443447
/// into "SectionedAddress Address"
444448
DWARFCompileUnit *getCompileUnitForAddress(uint64_t Address);
445449

450+
/// Returns whether CU/TU should be populated manually. TU Index populated
451+
/// manually only for DWARF5.
452+
bool getParseCUTUIndexManually() const { return ParseCUTUIndexManually; }
453+
454+
/// Sets whether CU/TU should be populated manually. TU Index populated
455+
/// manually only for DWARF5.
456+
void setParseCUTUIndexManually(bool PCUTU) { ParseCUTUIndexManually = PCUTU; }
457+
446458
private:
447459
/// Parse a macro[.dwo] or macinfo[.dwo] section.
448460
std::unique_ptr<DWARFDebugMacro>

‎llvm/include/llvm/DebugInfo/DWARF/DWARFUnitIndex.h

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,12 +137,14 @@ class DWARFUnitIndex {
137137
public:
138138
const SectionContribution *getContribution(DWARFSectionKind Sec) const;
139139
const SectionContribution *getContribution() const;
140+
SectionContribution &getContribution();
140141

141142
const SectionContribution *getContributions() const {
142143
return Contributions.get();
143144
}
144145

145146
uint64_t getSignature() const { return Signature; }
147+
bool isValid() { return Index; }
146148
};
147149

148150
private:
@@ -183,6 +185,10 @@ class DWARFUnitIndex {
183185
ArrayRef<Entry> getRows() const {
184186
return ArrayRef(Rows.get(), Header.NumBuckets);
185187
}
188+
189+
MutableArrayRef<Entry> getMutableRows() {
190+
return makeMutableArrayRef(Rows.get(), Header.NumBuckets);
191+
}
186192
};
187193

188194
} // end namespace llvm

‎llvm/lib/DebugInfo/DWARF/DWARFContext.cpp

Lines changed: 73 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -779,14 +779,82 @@ bool DWARFContext::verify(raw_ostream &OS, DIDumpOptions DumpOpts) {
779779
return Success;
780780
}
781781

782+
void fixupIndex(const DWARFObject &DObj, DWARFContext &C,
783+
DWARFUnitIndex &Index) {
784+
using EntryType = DWARFUnitIndex::Entry::SectionContribution;
785+
using EntryMap = DenseMap<uint32_t, EntryType>;
786+
EntryMap Map;
787+
if (DObj.getCUIndexSection().empty())
788+
return;
789+
790+
uint64_t Offset = 0;
791+
uint32_t TruncOffset = 0;
792+
DObj.forEachInfoDWOSections([&](const DWARFSection &S) {
793+
if (!(C.getParseCUTUIndexManually() ||
794+
S.Data.size() >= std::numeric_limits<uint32_t>::max()))
795+
return;
796+
797+
DWARFDataExtractor Data(DObj, S, C.isLittleEndian(), 0);
798+
while (Data.isValidOffset(Offset)) {
799+
DWARFUnitHeader Header;
800+
if (!Header.extract(C, Data, &Offset, DWARFSectionKind::DW_SECT_INFO)) {
801+
logAllUnhandledErrors(
802+
createError("Failed to parse CU header in DWP file"), errs());
803+
Map.clear();
804+
break;
805+
}
806+
807+
auto Iter = Map.insert({TruncOffset,
808+
{Header.getOffset(), Header.getNextUnitOffset() -
809+
Header.getOffset()}});
810+
if (!Iter.second) {
811+
logAllUnhandledErrors(
812+
createError("Collision occured between for truncated offset 0x" +
813+
Twine::utohexstr(TruncOffset)),
814+
errs());
815+
Map.clear();
816+
return;
817+
}
818+
819+
Offset = Header.getNextUnitOffset();
820+
TruncOffset = Offset;
821+
}
822+
});
823+
824+
if (Map.empty())
825+
return;
826+
827+
for (DWARFUnitIndex::Entry &E : Index.getMutableRows()) {
828+
if (!E.isValid())
829+
continue;
830+
DWARFUnitIndex::Entry::SectionContribution &CUOff = E.getContribution();
831+
auto Iter = Map.find(CUOff.getOffset());
832+
if (Iter == Map.end()) {
833+
logAllUnhandledErrors(createError("Could not find CU offset 0x" +
834+
Twine::utohexstr(CUOff.getOffset()) +
835+
" in the Map"),
836+
errs());
837+
break;
838+
}
839+
CUOff.setOffset(Iter->second.getOffset());
840+
if (CUOff.getOffset() != Iter->second.getOffset())
841+
logAllUnhandledErrors(createError("Length of CU in CU index doesn't "
842+
"match calculated length at offset 0x" +
843+
Twine::utohexstr(CUOff.getOffset())),
844+
errs());
845+
}
846+
847+
return;
848+
}
849+
782850
const DWARFUnitIndex &DWARFContext::getCUIndex() {
783851
if (CUIndex)
784852
return *CUIndex;
785853

786854
DataExtractor CUIndexData(DObj->getCUIndexSection(), isLittleEndian(), 0);
787-
788855
CUIndex = std::make_unique<DWARFUnitIndex>(DW_SECT_INFO);
789856
CUIndex->parse(CUIndexData);
857+
fixupIndex(*DObj, *this, *CUIndex.get());
790858
return *CUIndex;
791859
}
792860

@@ -795,9 +863,12 @@ const DWARFUnitIndex &DWARFContext::getTUIndex() {
795863
return *TUIndex;
796864

797865
DataExtractor TUIndexData(DObj->getTUIndexSection(), isLittleEndian(), 0);
798-
799866
TUIndex = std::make_unique<DWARFUnitIndex>(DW_SECT_EXT_TYPES);
800867
TUIndex->parse(TUIndexData);
868+
// If we are parsing TU-index and for .debug_types section we don't need
869+
// to do anything.
870+
if (TUIndex->getVersion() != 2)
871+
fixupIndex(*DObj, *this, *TUIndex.get());
801872
return *TUIndex;
802873
}
803874

‎llvm/lib/DebugInfo/DWARF/DWARFUnitIndex.cpp

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -269,6 +269,11 @@ DWARFUnitIndex::Entry::getContribution(DWARFSectionKind Sec) const {
269269
return nullptr;
270270
}
271271

272+
DWARFUnitIndex::Entry::SectionContribution &
273+
DWARFUnitIndex::Entry::getContribution() {
274+
return Contributions[Index->InfoColumn];
275+
}
276+
272277
const DWARFUnitIndex::Entry::SectionContribution *
273278
DWARFUnitIndex::Entry::getContribution() const {
274279
return &Contributions[Index->InfoColumn];

‎llvm/test/tools/llvm-dwp/X86/cu_tu_units_manual_v5.s

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
# This test checks if we can correctly parse manull cu and tu index for DWARF5.
2+
3+
# RUN: llvm-mc -triple x86_64-unknown-linux %s -filetype=obj -o %t.o \
4+
# RUN: -split-dwarf-file=%t.dwo -dwarf-version=5
5+
# RUN: llvm-dwp %t.dwo -o %t.dwp
6+
# RUN: llvm-dwarfdump -debug-info -debug-cu-index -debug-tu-index %t.dwp | FileCheck -check-prefix=CHECK %s
7+
# RUN: llvm-dwarfdump -debug-info -debug-cu-index -debug-tu-index -manaully-generate-unit-index %t.dwp | FileCheck -check-prefix=CHECK2 %s
8+
9+
## Note: In order to check whether the type unit index is generated
10+
## there is no need to add the missing DIEs for the structure type of the type unit.
11+
12+
# CHECK-DAG: .debug_info.dwo contents:
13+
# CHECK: 0x00000000: Type Unit: length = 0x00000017, format = DWARF32, version = 0x0005, unit_type = DW_UT_split_type, abbr_offset = 0x0000, addr_size = 0x08, name = '', type_signature = [[TUID1:.*]], type_offset = 0x0019 (next unit at 0x0000001b)
14+
# CHECK: 0x0000001b: Type Unit: length = 0x00000017, format = DWARF32, version = 0x0005, unit_type = DW_UT_split_type, abbr_offset = 0x0000, addr_size = 0x08, name = '', type_signature = [[TUID2:.*]], type_offset = 0x0019 (next unit at 0x00000036)
15+
# CHECK: 0x00000036: Compile Unit: length = 0x00000011, format = DWARF32, version = 0x0005, unit_type = DW_UT_split_compile, abbr_offset = 0x0000, addr_size = 0x08, DWO_id = [[CUID1:.*]] (next unit at 0x0000004b)
16+
# CHECK-DAG: .debug_cu_index contents:
17+
# CHECK: version = 5, units = 1, slots = 2
18+
# CHECK: Index Signature INFO ABBREV
19+
# CHECK: 1 [[CUID1]] [0x0000000000000036, 0x000000000000004b) [0x00000000, 0x00000010)
20+
# CHECK-DAG: .debug_tu_index contents:
21+
# CHECK: version = 5, units = 2, slots = 4
22+
# CHECK: Index Signature INFO ABBREV
23+
# CHECK: 1 [[TUID1]] [0x0000000000000000, 0x000000000000001b) [0x00000000, 0x00000010)
24+
# CHECK: 4 [[TUID2]] [0x000000000000001b, 0x0000000000000036) [0x00000000, 0x00000010)
25+
26+
# CHECK2-DAG: .debug_info.dwo contents:
27+
# CHECK2: 0x00000000: Type Unit: length = 0x00000017, format = DWARF32, version = 0x0005, unit_type = DW_UT_split_type, abbr_offset = 0x0000, addr_size = 0x08, name = '', type_signature = [[TUID1:.*]], type_offset = 0x0019 (next unit at 0x0000001b)
28+
# CHECK2: 0x0000001b: Type Unit: length = 0x00000017, format = DWARF32, version = 0x0005, unit_type = DW_UT_split_type, abbr_offset = 0x0000, addr_size = 0x08, name = '', type_signature = [[TUID2:.*]], type_offset = 0x0019 (next unit at 0x00000036)
29+
# CHECK2: 0x00000036: Compile Unit: length = 0x00000011, format = DWARF32, version = 0x0005, unit_type = DW_UT_split_compile, abbr_offset = 0x0000, addr_size = 0x08, DWO_id = [[CUID1:.*]] (next unit at 0x0000004b)
30+
# CHECK2-DAG: .debug_cu_index contents:
31+
# CHECK2: version = 5, units = 1, slots = 2
32+
# CHECK2: Index Signature INFO ABBREV
33+
# CHECK2: 1 [[CUID1]] [0x0000000000000036, 0x000000000000004b) [0x00000000, 0x00000010)
34+
# CHECK2-DAG: .debug_tu_index contents:
35+
# CHECK2: version = 5, units = 2, slots = 4
36+
# CHECK2: Index Signature INFO ABBREV
37+
# CHECK2: 1 [[TUID1]] [0x0000000000000000, 0x000000000000001b) [0x00000000, 0x00000010)
38+
# CHECK2: 4 [[TUID2]] [0x000000000000001b, 0x0000000000000036) [0x00000000, 0x00000010)
39+
40+
.section .debug_info.dwo,"e",@progbits
41+
.long .Ldebug_info_dwo_end0-.Ldebug_info_dwo_start0 # Length of Unit
42+
.Ldebug_info_dwo_start0:
43+
.short 5 # DWARF version number
44+
.byte 6 # DWARF Unit Type (DW_UT_split_type)
45+
.byte 8 # Address Size (in bytes)
46+
.long 0 # Offset Into Abbrev. Section
47+
.quad 5657452045627120676 # Type Signature
48+
.long 25 # Type DIE Offset
49+
.byte 2 # Abbrev [2] DW_TAG_type_unit
50+
.byte 3 # Abbrev [3] DW_TAG_structure_type
51+
.byte 0 # End Of Children Mark
52+
.Ldebug_info_dwo_end0:
53+
.section .debug_info.dwo,"e",@progbits
54+
.long .Ldebug_info_dwo_end1-.Ldebug_info_dwo_start1 # Length of Unit
55+
.Ldebug_info_dwo_start1:
56+
.short 5 # DWARF version number
57+
.byte 6 # DWARF Unit Type (DW_UT_split_type)
58+
.byte 8 # Address Size (in bytes)
59+
.long 0 # Offset Into Abbrev. Section
60+
.quad -8528522068957683993 # Type Signature
61+
.long 25 # Type DIE Offset
62+
.byte 4 # Abbrev [4] DW_TAG_type_unit
63+
.byte 5 # Abbrev [5] DW_TAG_structure_type
64+
.byte 0 # End Of Children Mark
65+
.Ldebug_info_dwo_end1:
66+
.section .debug_info.dwo,"e",@progbits
67+
.long .Ldebug_info_dwo_end2-.Ldebug_info_dwo_start2 # Length of Unit
68+
.Ldebug_info_dwo_start2:
69+
.short 5 # DWARF version number
70+
.byte 5 # DWARF Unit Type (DW_UT_split_compile)
71+
.byte 8 # Address Size (in bytes)
72+
.long 0 # Offset Into Abbrev. Section
73+
.quad 1152943841751211454
74+
.byte 1 # Abbrev [1] DW_TAG_compile_unit
75+
.Ldebug_info_dwo_end2:
76+
.section .debug_abbrev.dwo,"e",@progbits
77+
.byte 1 # Abbreviation Code
78+
.byte 17 # DW_TAG_compile_unit
79+
.byte 0 # DW_CHILDREN_no
80+
.byte 0 # EOM(1)
81+
.byte 0 # EOM(2)
82+
.byte 2 # Abbreviation Code
83+
.byte 65 # DW_TAG_type_unit
84+
.byte 1 # DW_CHILDREN_yes
85+
.byte 0 # EOM
86+
.byte 0 # EOM
87+
.byte 4 # Abbreviation Code
88+
.byte 65 # DW_TAG_type_unit
89+
.byte 1 # DW_CHILDREN_yes
90+
.byte 0 # EOM
91+
.byte 0 # EOM
92+
.byte 0 # EOM

‎llvm/test/tools/llvm-dwp/X86/debug_macro_v5.s

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,8 @@
22

33
# RUN: llvm-mc -triple x86_64-unknown-linux --filetype=obj --split-dwarf-file=%t.dwo -dwarf-version=5 %s -o %t.o
44
# RUN: llvm-dwp %t.dwo -o %t.dwp 2>&1
5-
# RUN: llvm-dwarfdump -debug-macro -debug-cu-index %t.dwp | FileCheck %s
5+
# RUN: llvm-dwarfdump -debug-macro -debug-cu-index %t.dwp | FileCheck -check-prefix=CHECK %s
6+
# RUN: llvm-dwarfdump -debug-macro -debug-cu-index -manaully-generate-unit-index %t.dwp | FileCheck -check-prefix=CHECK2 %s
67

78
# CHECK-DAG: .debug_macro.dwo contents:
89
# CHECK: macro header: version = 0x0005, flags = 0x00, format = DWARF32
@@ -15,6 +16,9 @@
1516
# CHECK: Index Signature INFO ABBREV STR_OFFSETS MACRO
1617
# CHECK: 1 0x0000000000000000 [0x0000000000000000, 0x0000000000000019) [0x00000000, 0x00000008) [0x00000000, 0x0000000c) [0x00000000, 0x0000000b)
1718

19+
# CHECK2: Index Signature INFO ABBREV STR_OFFSETS MACRO
20+
# CHECK2: 1 0x0000000000000000 [0x0000000000000000, 0x0000000000000019) [0x00000000, 0x00000008) [0x00000000, 0x0000000c) [0x00000000, 0x0000000b)
21+
1822
.section .debug_info.dwo,"e",@progbits
1923
.long .Ldebug_info_dwo_end0-.Ldebug_info_dwo_start0 # Length of Unit
2024
.Ldebug_info_dwo_start0:

‎llvm/test/tools/llvm-dwp/X86/type_dedup.test

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
11
RUN: llvm-dwp %p/../Inputs/type_dedup/a.dwo %p/../Inputs/type_dedup/b.dwo -o %t
2-
RUN: llvm-dwarfdump -v %t | FileCheck %s
2+
RUN: llvm-dwarfdump -v %t | FileCheck -check-prefix=CHECK %s
3+
RUN: llvm-dwarfdump -v -manaully-generate-unit-index %t | FileCheck -check-prefix=CHECK2 %s
34
RUN: llvm-dwp %p/../Inputs/type_dedup/b.dwo -o %tb.dwp
45
RUN: llvm-dwp %p/../Inputs/type_dedup/a.dwo %tb.dwp -o %t
5-
RUN: llvm-dwarfdump -v %t | FileCheck %s
6+
RUN: llvm-dwarfdump -v %t | FileCheck -check-prefix=CHECK %s
7+
RUN: llvm-dwarfdump -v -manaully-generate-unit-index %t | FileCheck -check-prefix=CHECK2 %s
68

79
a.cpp:
810
struct common { };
@@ -36,3 +38,24 @@ CHECK: DW_TAG_type_unit
3638
CHECK: 0x00000066: DW_TAG_structure_type
3739
CHECK: DW_AT_name {{.*}} "bdistinct"
3840
CHECK-NOT: Type Unit
41+
42+
CHECK2-LABEL: .debug_types.dwo contents:
43+
CHECK2: [[COMMONUOFF:0x[0-9a-f]*]]:
44+
CHECK2-LABEL: Type Unit: length = 0x00000020, format = DWARF32, version = 0x0004, abbr_offset =
45+
CHECK2: 0x0000, addr_size = 0x08, name = 'common', type_signature = [[COMMONSIG:0x[0-9a-f]*]], type_offset = 0x[[COMMONOFF:.*]] (next unit at [[AUOFF:.*]])
46+
CHECK2: DW_TAG_type_unit
47+
CHECK2: [[COMMONOFF]]: DW_TAG_structure_type
48+
CHECK2: DW_AT_name {{.*}} "common"
49+
CHECK2: [[AUOFF]]:
50+
CHECK2-LABEL: Type Unit: length = 0x00000020, format = DWARF32, version = 0x0004, abbr_offset =
51+
CHECK2: 0x0000, addr_size = 0x08, name = 'adistinct', type_signature = [[ASIG:0x[0-9a-f]*]], type_offset = 0x[[AOFF:.*]] (next unit at [[BUOFF:.*]])
52+
CHECK2: DW_TAG_type_unit
53+
CHECK2: 0x00000042: DW_TAG_structure_type
54+
CHECK2: DW_AT_name {{.*}} "adistinct"
55+
CHECK2: [[BUOFF]]:
56+
CHECK2-LABEL: Type Unit: length = 0x00000020, format = DWARF32, version = 0x0004, abbr_offset =
57+
CHECK2: 0x{{.*}}, addr_size = 0x08, name = 'bdistinct', type_signature = [[BSIG:0x[0-9a-f]*]], type_offset = 0x[[BOFF:.*]] (next unit at [[XUOFF:.*]])
58+
CHECK2: DW_TAG_type_unit
59+
CHECK2: 0x00000066: DW_TAG_structure_type
60+
CHECK2: DW_AT_name {{.*}} "bdistinct"
61+
CHECK2-NOT: Type Unit

‎llvm/tools/llvm-dwarfdump/llvm-dwarfdump.cpp

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -249,6 +249,13 @@ static cl::opt<bool>
249249
cl::desc("Show the sizes of all debug sections, "
250250
"expressed in bytes."),
251251
cat(DwarfDumpCategory));
252+
static cl::opt<bool> ManuallyGenerateUnitIndex(
253+
"manaully-generate-unit-index",
254+
cl::desc("if the input is dwp file, parse .debug_info "
255+
"section and use it to populate "
256+
"DW_SECT_INFO contributions in cu-index. "
257+
"For DWARF5 it also populated TU Index."),
258+
cl::init(false), cl::Hidden, cl::cat(DwarfDumpCategory));
252259
static cl::opt<bool>
253260
ShowSources("show-sources",
254261
cl::desc("Show the sources across all compilation units."),
@@ -675,6 +682,7 @@ static bool handleBuffer(StringRef Filename, MemoryBufferRef Buffer,
675682
std::unique_ptr<DWARFContext> DICtx = DWARFContext::create(
676683
*Obj, DWARFContext::ProcessDebugRelocations::Process, nullptr, "",
677684
RecoverableErrorHandler);
685+
DICtx->setParseCUTUIndexManually(ManuallyGenerateUnitIndex);
678686
if (!HandleObj(*Obj, *DICtx, Filename, OS))
679687
Result = false;
680688
}

0 commit comments

Comments
 (0)