Skip to content

Commit c813fdd

Browse files
rlavaeeJaddyen
rlavaee
authored and
Jaddyen
committed
[SHT_LLVM_BB_ADDR_MAP] Encode and decode callsite offsets in a newly-introduced SHT_LLVM_BB_ADDR_MAP version. (llvm#144426)
Recently, we have been looking at some optimizations targeting individual calls. In particular, we plan to extend the address mapping technique to map to individual callsites. For example, in this piece of code for a basic blocks: ``` <BB>: 1200: lea 0x1(%rcx), %rdx 1204: callq foo 1209: cmpq 0x10, %rdx 120d: ja L1 ``` We want to emit 0x9 as the call site offset for `callq foo` (the offset from the block entry to right after the call), so that we know if a sampled address is before the call or after. This PR implements the decode/encode/emit capability. The Codegen change will be implemented in a later PR.
1 parent ecfaf08 commit c813fdd

File tree

13 files changed

+222
-96
lines changed

13 files changed

+222
-96
lines changed

‎llvm/include/llvm/Object/ELFTypes.h

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -831,6 +831,7 @@ struct BBAddrMap {
831831
bool BrProb : 1;
832832
bool MultiBBRange : 1;
833833
bool OmitBBEntries : 1;
834+
bool CallsiteOffsets : 1;
834835

835836
bool hasPGOAnalysis() const { return FuncEntryCount || BBFreq || BrProb; }
836837

@@ -842,7 +843,8 @@ struct BBAddrMap {
842843
(static_cast<uint8_t>(BBFreq) << 1) |
843844
(static_cast<uint8_t>(BrProb) << 2) |
844845
(static_cast<uint8_t>(MultiBBRange) << 3) |
845-
(static_cast<uint8_t>(OmitBBEntries) << 4);
846+
(static_cast<uint8_t>(OmitBBEntries) << 4) |
847+
(static_cast<uint8_t>(CallsiteOffsets) << 5);
846848
}
847849

848850
// Decodes from minimum bit width representation and validates no
@@ -851,7 +853,7 @@ struct BBAddrMap {
851853
Features Feat{
852854
static_cast<bool>(Val & (1 << 0)), static_cast<bool>(Val & (1 << 1)),
853855
static_cast<bool>(Val & (1 << 2)), static_cast<bool>(Val & (1 << 3)),
854-
static_cast<bool>(Val & (1 << 4))};
856+
static_cast<bool>(Val & (1 << 4)), static_cast<bool>(Val & (1 << 5))};
855857
if (Feat.encode() != Val)
856858
return createStringError(
857859
std::error_code(), "invalid encoding for BBAddrMap::Features: 0x%x",
@@ -861,9 +863,10 @@ struct BBAddrMap {
861863

862864
bool operator==(const Features &Other) const {
863865
return std::tie(FuncEntryCount, BBFreq, BrProb, MultiBBRange,
864-
OmitBBEntries) ==
866+
OmitBBEntries, CallsiteOffsets) ==
865867
std::tie(Other.FuncEntryCount, Other.BBFreq, Other.BrProb,
866-
Other.MultiBBRange, Other.OmitBBEntries);
868+
Other.MultiBBRange, Other.OmitBBEntries,
869+
Other.CallsiteOffsets);
867870
}
868871
};
869872

@@ -914,13 +917,18 @@ struct BBAddrMap {
914917
uint32_t Size = 0; // Size of the basic block.
915918
Metadata MD = {false, false, false, false,
916919
false}; // Metdata for this basic block.
920+
// Offsets of callsites (end of call instructions), relative to the basic
921+
// block start.
922+
SmallVector<uint32_t, 1> CallsiteOffsets;
917923

918-
BBEntry(uint32_t ID, uint32_t Offset, uint32_t Size, Metadata MD)
919-
: ID(ID), Offset(Offset), Size(Size), MD(MD){};
924+
BBEntry(uint32_t ID, uint32_t Offset, uint32_t Size, Metadata MD,
925+
SmallVector<uint32_t, 1> CallsiteOffsets)
926+
: ID(ID), Offset(Offset), Size(Size), MD(MD),
927+
CallsiteOffsets(std::move(CallsiteOffsets)) {}
920928

921929
bool operator==(const BBEntry &Other) const {
922930
return ID == Other.ID && Offset == Other.Offset && Size == Other.Size &&
923-
MD == Other.MD;
931+
MD == Other.MD && CallsiteOffsets == Other.CallsiteOffsets;
924932
}
925933

926934
bool hasReturn() const { return MD.HasReturn; }

‎llvm/include/llvm/ObjectYAML/ELFYAML.h

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -162,6 +162,7 @@ struct BBAddrMapEntry {
162162
llvm::yaml::Hex64 AddressOffset;
163163
llvm::yaml::Hex64 Size;
164164
llvm::yaml::Hex64 Metadata;
165+
std::optional<std::vector<llvm::yaml::Hex64>> CallsiteOffsets;
165166
};
166167
uint8_t Version;
167168
llvm::yaml::Hex8 Feature;
@@ -180,6 +181,20 @@ struct BBAddrMapEntry {
180181
return 0;
181182
return BBRanges->front().BaseAddress;
182183
}
184+
185+
// Returns if any BB entries have non-empty callsite offsets.
186+
bool hasAnyCallsiteOffsets() const {
187+
if (!BBRanges)
188+
return false;
189+
for (const ELFYAML::BBAddrMapEntry::BBRangeEntry &BBR : *BBRanges) {
190+
if (!BBR.BBEntries)
191+
continue;
192+
for (const ELFYAML::BBAddrMapEntry::BBEntry &BBE : *BBR.BBEntries)
193+
if (BBE.CallsiteOffsets && !BBE.CallsiteOffsets->empty())
194+
return true;
195+
}
196+
return false;
197+
}
183198
};
184199

185200
struct PGOAnalysisMapEntry {

‎llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1426,9 +1426,12 @@ getBBAddrMapFeature(const MachineFunction &MF, int NumMBBSectionRanges) {
14261426
"BB entries info is required for BBFreq and BrProb "
14271427
"features");
14281428
}
1429-
return {FuncEntryCountEnabled, BBFreqEnabled, BrProbEnabled,
1429+
return {FuncEntryCountEnabled,
1430+
BBFreqEnabled,
1431+
BrProbEnabled,
14301432
MF.hasBBSections() && NumMBBSectionRanges > 1,
1431-
static_cast<bool>(BBAddrMapSkipEmitBBEntries)};
1433+
static_cast<bool>(BBAddrMapSkipEmitBBEntries),
1434+
false};
14321435
}
14331436

14341437
void AsmPrinter::emitBBAddrMapSection(const MachineFunction &MF) {

‎llvm/lib/Object/ELF.cpp

Lines changed: 27 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -837,7 +837,7 @@ decodeBBAddrMapImpl(const ELFFile<ELFT> &EF,
837837
Version = Data.getU8(Cur);
838838
if (!Cur)
839839
break;
840-
if (Version > 2)
840+
if (Version > 3)
841841
return createError("unsupported SHT_LLVM_BB_ADDR_MAP version: " +
842842
Twine(static_cast<int>(Version)));
843843
Feature = Data.getU8(Cur); // Feature byte
@@ -847,12 +847,18 @@ decodeBBAddrMapImpl(const ELFFile<ELFT> &EF,
847847
if (!FeatEnableOrErr)
848848
return FeatEnableOrErr.takeError();
849849
FeatEnable = *FeatEnableOrErr;
850-
if (Feature != 0 && Version < 2 && Cur)
850+
if (FeatEnable.hasPGOAnalysis() && Version < 2)
851851
return createError(
852852
"version should be >= 2 for SHT_LLVM_BB_ADDR_MAP when "
853853
"PGO features are enabled: version = " +
854854
Twine(static_cast<int>(Version)) +
855855
" feature = " + Twine(static_cast<int>(Feature)));
856+
if (FeatEnable.CallsiteOffsets && Version < 3)
857+
return createError(
858+
"version should be >= 3 for SHT_LLVM_BB_ADDR_MAP when "
859+
"callsite offsets feature is enabled: version = " +
860+
Twine(static_cast<int>(Version)) +
861+
" feature = " + Twine(static_cast<int>(Feature)));
856862
}
857863
uint32_t NumBlocksInBBRange = 0;
858864
uint32_t NumBBRanges = 1;
@@ -893,7 +899,23 @@ decodeBBAddrMapImpl(const ELFFile<ELFT> &EF,
893899
? readULEB128As<uint32_t>(Data, Cur, ULEBSizeErr)
894900
: BlockIndex;
895901
uint32_t Offset = readULEB128As<uint32_t>(Data, Cur, ULEBSizeErr);
896-
uint32_t Size = readULEB128As<uint32_t>(Data, Cur, ULEBSizeErr);
902+
// Read the callsite offsets.
903+
uint32_t LastCallsiteOffset = 0;
904+
SmallVector<uint32_t, 1> CallsiteOffsets;
905+
if (FeatEnable.CallsiteOffsets) {
906+
uint32_t NumCallsites =
907+
readULEB128As<uint32_t>(Data, Cur, ULEBSizeErr);
908+
CallsiteOffsets.reserve(NumCallsites);
909+
for (uint32_t CallsiteIndex = 0;
910+
!ULEBSizeErr && Cur && (CallsiteIndex < NumCallsites);
911+
++CallsiteIndex) {
912+
LastCallsiteOffset +=
913+
readULEB128As<uint32_t>(Data, Cur, ULEBSizeErr);
914+
CallsiteOffsets.push_back(LastCallsiteOffset);
915+
}
916+
}
917+
uint32_t Size = readULEB128As<uint32_t>(Data, Cur, ULEBSizeErr) +
918+
LastCallsiteOffset;
897919
uint32_t MD = readULEB128As<uint32_t>(Data, Cur, ULEBSizeErr);
898920
if (Version >= 1) {
899921
// Offset is calculated relative to the end of the previous BB.
@@ -906,7 +928,8 @@ decodeBBAddrMapImpl(const ELFFile<ELFT> &EF,
906928
MetadataDecodeErr = MetadataOrErr.takeError();
907929
break;
908930
}
909-
BBEntries.push_back({ID, Offset, Size, *MetadataOrErr});
931+
BBEntries.push_back(
932+
{ID, Offset, Size, *MetadataOrErr, CallsiteOffsets});
910933
}
911934
TotalNumBlocks += BBEntries.size();
912935
}

‎llvm/lib/ObjectYAML/ELFEmitter.cpp

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1452,7 +1452,7 @@ void ELFState<ELFT>::writeSectionContent(
14521452
for (const auto &[Idx, E] : llvm::enumerate(*Section.Entries)) {
14531453
// Write version and feature values.
14541454
if (Section.Type == llvm::ELF::SHT_LLVM_BB_ADDR_MAP) {
1455-
if (E.Version > 2)
1455+
if (E.Version > 3)
14561456
WithColor::warning() << "unsupported SHT_LLVM_BB_ADDR_MAP version: "
14571457
<< static_cast<int>(E.Version)
14581458
<< "; encoding using the most recent version";
@@ -1483,6 +1483,8 @@ void ELFState<ELFT>::writeSectionContent(
14831483
if (!E.BBRanges)
14841484
continue;
14851485
uint64_t TotalNumBlocks = 0;
1486+
bool EmitCallsiteOffsets =
1487+
FeatureOrErr->CallsiteOffsets || E.hasAnyCallsiteOffsets();
14861488
for (const ELFYAML::BBAddrMapEntry::BBRangeEntry &BBR : *E.BBRanges) {
14871489
// Write the base address of the range.
14881490
CBA.write<uintX_t>(BBR.BaseAddress, ELFT::Endianness);
@@ -1500,6 +1502,15 @@ void ELFState<ELFT>::writeSectionContent(
15001502
if (Section.Type == llvm::ELF::SHT_LLVM_BB_ADDR_MAP && E.Version > 1)
15011503
SHeader.sh_size += CBA.writeULEB128(BBE.ID);
15021504
SHeader.sh_size += CBA.writeULEB128(BBE.AddressOffset);
1505+
if (EmitCallsiteOffsets) {
1506+
size_t NumCallsiteOffsets =
1507+
BBE.CallsiteOffsets ? BBE.CallsiteOffsets->size() : 0;
1508+
SHeader.sh_size += CBA.writeULEB128(NumCallsiteOffsets);
1509+
if (BBE.CallsiteOffsets) {
1510+
for (uint32_t Offset : *BBE.CallsiteOffsets)
1511+
SHeader.sh_size += CBA.writeULEB128(Offset);
1512+
}
1513+
}
15031514
SHeader.sh_size += CBA.writeULEB128(BBE.Size);
15041515
SHeader.sh_size += CBA.writeULEB128(BBE.Metadata);
15051516
}

‎llvm/lib/ObjectYAML/ELFYAML.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1883,6 +1883,7 @@ void MappingTraits<ELFYAML::BBAddrMapEntry::BBEntry>::mapping(
18831883
IO.mapRequired("AddressOffset", E.AddressOffset);
18841884
IO.mapRequired("Size", E.Size);
18851885
IO.mapRequired("Metadata", E.Metadata);
1886+
IO.mapOptional("CallsiteOffsets", E.CallsiteOffsets);
18861887
}
18871888

18881889
void MappingTraits<ELFYAML::PGOAnalysisMapEntry>::mapping(

‎llvm/test/tools/llvm-readobj/ELF/bb-addr-map.test

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,8 @@
4949
# CHECK-NEXT: {
5050
# CHECK-NEXT: ID: 2
5151
# CHECK-NEXT: Offset: 0x3
52-
# CHECK-NEXT: Size: 0x4
52+
# CHECK-NEXT: Callsite Offsets: [1, 3]
53+
# CHECK-NEXT: Size: 0x7
5354
# CHECK-NEXT: HasReturn: Yes
5455
# CHECK-NEXT: HasTailCall: No
5556
# CHECK-NEXT: IsEHPad: Yes
@@ -75,7 +76,7 @@
7576
# CHECK-NEXT: HasTailCall: No
7677
# CHECK-NEXT: IsEHPad: No
7778
# CHECK-NEXT: CanFallThrough: Yes
78-
# CHECK-NEXT: HasIndirectBranch: No
79+
# CHECK-NEXT: HasIndirectBranch: No
7980
# CHECK-NEXT: }
8081
# CHECK-NEXT: ]
8182
# CHECK-NEXT: }
@@ -143,8 +144,8 @@ Sections:
143144
ShSize: [[SIZE=<none>]]
144145
Link: .text
145146
Entries:
146-
- Version: 2
147-
Feature: 0x8
147+
- Version: 3
148+
Feature: 0x28
148149
BBRanges:
149150
- BaseAddress: [[ADDR=0x11111]]
150151
BBEntries:
@@ -158,6 +159,7 @@ Sections:
158159
AddressOffset: 0x3
159160
Size: 0x4
160161
Metadata: 0x15
162+
CallsiteOffsets: [ 0x1 , 0x2 ]
161163
- Version: 2
162164
BBRanges:
163165
- BaseAddress: 0x22222

‎llvm/test/tools/obj2yaml/ELF/bb-addr-map.yaml

Lines changed: 16 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
# VALID-NEXT: - Name: .llvm_bb_addr_map
1515
# VALID-NEXT: Type: SHT_LLVM_BB_ADDR_MAP
1616
# VALID-NEXT: Entries:
17-
# VALID-NEXT: - Version: 2
17+
# VALID-NEXT: - Version: 3
1818
# VALID-NEXT: BBRanges:
1919
## The 'BaseAddress' field is omitted when it's zero.
2020
# VALID-NEXT: - BBEntries:
@@ -30,15 +30,16 @@
3030
# VALID-NEXT: AddressOffset: 0xFFFFFFFFFFFFFFF7
3131
# VALID-NEXT: Size: 0xFFFFFFFFFFFFFFF8
3232
# VALID-NEXT: Metadata: 0xFFFFFFFFFFFFFFF9
33-
# VALID-NEXT: - Version: 2
34-
# VALID-NEXT: Feature: 0x8
33+
# VALID-NEXT: - Version: 3
34+
# VALID-NEXT: Feature: 0x28
3535
# VALID-NEXT: BBRanges:
3636
# VALID-NEXT: - BaseAddress: 0xFFFFFFFFFFFFFF20
3737
# VALID-NEXT: BBEntries:
38-
# VALID-NEXT: - ID: 6
39-
# VALID-NEXT: AddressOffset: 0xA
40-
# VALID-NEXT: Size: 0xB
41-
# VALID-NEXT: Metadata: 0xC
38+
# VALID-NEXT: - ID: 6
39+
# VALID-NEXT: AddressOffset: 0xA
40+
# VALID-NEXT: Size: 0xB
41+
# VALID-NEXT: Metadata: 0xC
42+
# VALID-NEXT: CallsiteOffsets: [ 0x1, 0x2 ]
4243

4344
--- !ELF
4445
FileHeader:
@@ -50,7 +51,7 @@ Sections:
5051
Type: SHT_LLVM_BB_ADDR_MAP
5152
ShSize: [[SIZE=<none>]]
5253
Entries:
53-
- Version: 2
54+
- Version: 3
5455
Feature: 0x0
5556
BBRanges:
5657
- BaseAddress: 0x0
@@ -67,17 +68,18 @@ Sections:
6768
AddressOffset: 0xFFFFFFFFFFFFFFF7
6869
Size: 0xFFFFFFFFFFFFFFF8
6970
Metadata: 0xFFFFFFFFFFFFFFF9
70-
- Version: 2
71-
Feature: 0x8
71+
- Version: 3
72+
Feature: 0x28
7273
NumBBRanges: [[NUMBBRANGES=<none>]]
7374
BBRanges:
7475
- BaseAddress: 0xFFFFFFFFFFFFFF20
7576
NumBlocks: [[NUMBLOCKS=<none>]]
7677
BBEntries:
77-
- ID: 6
78-
AddressOffset: 0xA
79-
Size: 0xB
80-
Metadata: 0xC
78+
- ID: 6
79+
AddressOffset: 0xA
80+
Size: 0xB
81+
Metadata: 0xC
82+
CallsiteOffsets: [ 0x1, 0x2 ]
8183

8284
## Check obj2yaml can dump empty .llvm_bb_addr_map sections.
8385

0 commit comments

Comments
 (0)