Skip to content

Commit bd01ada

Browse files
SC llvm teamSC llvm team
SC llvm team
authored and
SC llvm team
committed
Merged main:28c29fbec3057692a7985819d799a9e5d47eb2d1 into amd-gfx:2485fb73de5d
Local branch amd-gfx 2485fb7 Merged main:ed1aabef1da1b074b71ad523978ea836d6e7d2e7 into amd-gfx:0f78e4fe65f0 Remote branch main 28c29fb [RISCV] Add exact VLEN RUNs for insert_subvector and concat_vector tests. NFC
2 parents 2485fb7 + 28c29fb commit bd01ada

File tree

825 files changed

+14277
-14354
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

825 files changed

+14277
-14354
lines changed

‎bolt/include/bolt/Core/MCPlusBuilder.h

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -620,7 +620,17 @@ class MCPlusBuilder {
620620
return Info->get(Inst.getOpcode()).mayStore();
621621
}
622622

623-
virtual bool isAArch64Exclusive(const MCInst &Inst) const {
623+
virtual bool isAArch64ExclusiveLoad(const MCInst &Inst) const {
624+
llvm_unreachable("not implemented");
625+
return false;
626+
}
627+
628+
virtual bool isAArch64ExclusiveStore(const MCInst &Inst) const {
629+
llvm_unreachable("not implemented");
630+
return false;
631+
}
632+
633+
virtual bool isAArch64ExclusiveClear(const MCInst &Inst) const {
624634
llvm_unreachable("not implemented");
625635
return false;
626636
}
@@ -1173,11 +1183,16 @@ class MCPlusBuilder {
11731183
bool clearOffset(MCInst &Inst) const;
11741184

11751185
/// Return the label of \p Inst, if available.
1176-
MCSymbol *getLabel(const MCInst &Inst) const;
1186+
MCSymbol *getInstLabel(const MCInst &Inst) const;
1187+
1188+
/// Set the label of \p Inst or return the existing label for the instruction.
1189+
/// This label will be emitted right before \p Inst is emitted to MCStreamer.
1190+
MCSymbol *getOrCreateInstLabel(MCInst &Inst, const Twine &Name,
1191+
MCContext *Ctx) const;
11771192

11781193
/// Set the label of \p Inst. This label will be emitted right before \p Inst
11791194
/// is emitted to MCStreamer.
1180-
bool setLabel(MCInst &Inst, MCSymbol *Label) const;
1195+
void setInstLabel(MCInst &Inst, MCSymbol *Label) const;
11811196

11821197
/// Get instruction size specified via annotation.
11831198
std::optional<uint32_t> getSize(const MCInst &Inst) const;

‎bolt/lib/Core/BinaryContext.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1967,7 +1967,7 @@ void BinaryContext::printInstruction(raw_ostream &OS, const MCInst &Instruction,
19671967
OS << " # Offset: " << *Offset;
19681968
if (std::optional<uint32_t> Size = MIB->getSize(Instruction))
19691969
OS << " # Size: " << *Size;
1970-
if (MCSymbol *Label = MIB->getLabel(Instruction))
1970+
if (MCSymbol *Label = MIB->getInstLabel(Instruction))
19711971
OS << " # Label: " << *Label;
19721972

19731973
MIB->printAnnotations(Instruction, OS);

‎bolt/lib/Core/BinaryEmitter.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -489,7 +489,7 @@ void BinaryEmitter::emitFunctionBody(BinaryFunction &BF, FunctionFragment &FF,
489489

490490
if (!EmitCodeOnly) {
491491
// A symbol to be emitted before the instruction to mark its location.
492-
MCSymbol *InstrLabel = BC.MIB->getLabel(Instr);
492+
MCSymbol *InstrLabel = BC.MIB->getInstLabel(Instr);
493493

494494
if (opts::UpdateDebugSections && BF.getDWARFUnit()) {
495495
LastLocSeen = emitLineInfo(BF, Instr.getLoc(), LastLocSeen,

‎bolt/lib/Core/BinaryFunction.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1424,7 +1424,7 @@ Error BinaryFunction::disassemble() {
14241424
InstrMapType::iterator II = Instructions.find(Offset);
14251425
assert(II != Instructions.end() && "reference to non-existing instruction");
14261426

1427-
BC.MIB->setLabel(II->second, Label);
1427+
BC.MIB->setInstLabel(II->second, Label);
14281428
}
14291429

14301430
// Reset symbolizer for the disassembler.

‎bolt/lib/Core/Exceptions.cpp

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -408,12 +408,11 @@ void BinaryFunction::updateEHRanges() {
408408

409409
// Same symbol is used for the beginning and the end of the range.
410410
MCSymbol *EHSymbol;
411-
if (MCSymbol *InstrLabel = BC.MIB->getLabel(Instr)) {
411+
if (MCSymbol *InstrLabel = BC.MIB->getInstLabel(Instr)) {
412412
EHSymbol = InstrLabel;
413413
} else {
414414
std::unique_lock<llvm::sys::RWMutex> Lock(BC.CtxMutex);
415-
EHSymbol = BC.Ctx->createNamedTempSymbol("EH");
416-
BC.MIB->setLabel(Instr, EHSymbol);
415+
EHSymbol = BC.MIB->getOrCreateInstLabel(Instr, "EH", BC.Ctx.get());
417416
}
418417

419418
// At this point we could be in one of the following states:

‎bolt/lib/Core/MCPlusBuilder.cpp

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212

1313
#include "bolt/Core/MCPlusBuilder.h"
1414
#include "bolt/Core/MCPlus.h"
15+
#include "llvm/MC/MCContext.h"
1516
#include "llvm/MC/MCInst.h"
1617
#include "llvm/MC/MCInstrAnalysis.h"
1718
#include "llvm/MC/MCInstrDesc.h"
@@ -266,17 +267,29 @@ bool MCPlusBuilder::clearOffset(MCInst &Inst) const {
266267
return true;
267268
}
268269

269-
MCSymbol *MCPlusBuilder::getLabel(const MCInst &Inst) const {
270+
MCSymbol *MCPlusBuilder::getInstLabel(const MCInst &Inst) const {
270271
if (std::optional<int64_t> Label =
271272
getAnnotationOpValue(Inst, MCAnnotation::kLabel))
272273
return reinterpret_cast<MCSymbol *>(*Label);
273274
return nullptr;
274275
}
275276

276-
bool MCPlusBuilder::setLabel(MCInst &Inst, MCSymbol *Label) const {
277+
MCSymbol *MCPlusBuilder::getOrCreateInstLabel(MCInst &Inst, const Twine &Name,
278+
MCContext *Ctx) const {
279+
MCSymbol *Label = getInstLabel(Inst);
280+
if (Label)
281+
return Label;
282+
283+
Label = Ctx->createNamedTempSymbol(Name);
284+
setAnnotationOpValue(Inst, MCAnnotation::kLabel,
285+
reinterpret_cast<int64_t>(Label));
286+
return Label;
287+
}
288+
289+
void MCPlusBuilder::setInstLabel(MCInst &Inst, MCSymbol *Label) const {
290+
assert(!getInstLabel(Inst) && "Instruction already has assigned label.");
277291
setAnnotationOpValue(Inst, MCAnnotation::kLabel,
278292
reinterpret_cast<int64_t>(Label));
279-
return true;
280293
}
281294

282295
std::optional<uint32_t> MCPlusBuilder::getSize(const MCInst &Inst) const {

‎bolt/lib/Passes/Instrumentation.cpp

Lines changed: 86 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,9 @@
1717
#include "bolt/Utils/Utils.h"
1818
#include "llvm/Support/CommandLine.h"
1919
#include "llvm/Support/RWMutex.h"
20+
#include <queue>
2021
#include <stack>
22+
#include <unordered_set>
2123

2224
#define DEBUG_TYPE "bolt-instrumentation"
2325

@@ -86,21 +88,89 @@ cl::opt<bool> InstrumentCalls("instrument-calls",
8688
namespace llvm {
8789
namespace bolt {
8890

89-
static bool hasAArch64ExclusiveMemop(BinaryFunction &Function) {
91+
static bool hasAArch64ExclusiveMemop(
92+
BinaryFunction &Function,
93+
std::unordered_set<const BinaryBasicBlock *> &BBToSkip) {
9094
// FIXME ARMv8-a architecture reference manual says that software must avoid
9195
// having any explicit memory accesses between exclusive load and associated
92-
// store instruction. So for now skip instrumentation for functions that have
93-
// these instructions, since it might lead to runtime deadlock.
96+
// store instruction. So for now skip instrumentation for basic blocks that
97+
// have these instructions, since it might lead to runtime deadlock.
9498
BinaryContext &BC = Function.getBinaryContext();
95-
for (const BinaryBasicBlock &BB : Function)
96-
for (const MCInst &Inst : BB)
97-
if (BC.MIB->isAArch64Exclusive(Inst)) {
98-
if (opts::Verbosity >= 1)
99-
BC.outs() << "BOLT-INSTRUMENTER: Function " << Function
100-
<< " has exclusive instructions, skip instrumentation\n";
99+
std::queue<std::pair<BinaryBasicBlock *, bool>> BBQueue; // {BB, isLoad}
100+
std::unordered_set<BinaryBasicBlock *> Visited;
101+
102+
if (Function.getLayout().block_begin() == Function.getLayout().block_end())
103+
return 0;
104+
105+
BinaryBasicBlock *BBfirst = *Function.getLayout().block_begin();
106+
BBQueue.push({BBfirst, false});
107+
108+
while (!BBQueue.empty()) {
109+
BinaryBasicBlock *BB = BBQueue.front().first;
110+
bool IsLoad = BBQueue.front().second;
111+
BBQueue.pop();
112+
if (Visited.find(BB) != Visited.end())
113+
continue;
114+
Visited.insert(BB);
115+
116+
for (const MCInst &Inst : *BB) {
117+
// Two loads one after another - skip whole function
118+
if (BC.MIB->isAArch64ExclusiveLoad(Inst) && IsLoad) {
119+
if (opts::Verbosity >= 2) {
120+
outs() << "BOLT-INSTRUMENTER: function " << Function.getPrintName()
121+
<< " has two exclusive loads. Ignoring the function.\n";
122+
}
101123
return true;
102124
}
103125

126+
if (BC.MIB->isAArch64ExclusiveLoad(Inst))
127+
IsLoad = true;
128+
129+
if (IsLoad && BBToSkip.find(BB) == BBToSkip.end()) {
130+
BBToSkip.insert(BB);
131+
if (opts::Verbosity >= 2) {
132+
outs() << "BOLT-INSTRUMENTER: skip BB " << BB->getName()
133+
<< " due to exclusive instruction in function "
134+
<< Function.getPrintName() << "\n";
135+
}
136+
}
137+
138+
if (!IsLoad && BC.MIB->isAArch64ExclusiveStore(Inst)) {
139+
if (opts::Verbosity >= 2) {
140+
outs() << "BOLT-INSTRUMENTER: function " << Function.getPrintName()
141+
<< " has exclusive store without corresponding load. Ignoring "
142+
"the function.\n";
143+
}
144+
return true;
145+
}
146+
147+
if (IsLoad && (BC.MIB->isAArch64ExclusiveStore(Inst) ||
148+
BC.MIB->isAArch64ExclusiveClear(Inst)))
149+
IsLoad = false;
150+
}
151+
152+
if (IsLoad && BB->succ_size() == 0) {
153+
if (opts::Verbosity >= 2) {
154+
outs()
155+
<< "BOLT-INSTRUMENTER: function " << Function.getPrintName()
156+
<< " has exclusive load in trailing BB. Ignoring the function.\n";
157+
}
158+
return true;
159+
}
160+
161+
for (BinaryBasicBlock *BBS : BB->successors())
162+
BBQueue.push({BBS, IsLoad});
163+
}
164+
165+
if (BBToSkip.size() == Visited.size()) {
166+
if (opts::Verbosity >= 2) {
167+
outs() << "BOLT-INSTRUMENTER: all BBs are marked with true. Ignoring the "
168+
"function "
169+
<< Function.getPrintName() << "\n";
170+
}
171+
return true;
172+
}
173+
104174
return false;
105175
}
106176

@@ -307,7 +377,8 @@ void Instrumentation::instrumentFunction(BinaryFunction &Function,
307377
if (BC.isMachO() && Function.hasName("___GLOBAL_init_65535/1"))
308378
return;
309379

310-
if (BC.isAArch64() && hasAArch64ExclusiveMemop(Function))
380+
std::unordered_set<const BinaryBasicBlock *> BBToSkip;
381+
if (BC.isAArch64() && hasAArch64ExclusiveMemop(Function, BBToSkip))
311382
return;
312383

313384
SplitWorklistTy SplitWorklist;
@@ -389,6 +460,11 @@ void Instrumentation::instrumentFunction(BinaryFunction &Function,
389460

390461
for (auto BBI = Function.begin(), BBE = Function.end(); BBI != BBE; ++BBI) {
391462
BinaryBasicBlock &BB = *BBI;
463+
464+
// Skip BBs with exclusive load/stores
465+
if (BBToSkip.find(&BB) != BBToSkip.end())
466+
continue;
467+
392468
bool HasUnconditionalBranch = false;
393469
bool HasJumpTable = false;
394470
bool IsInvokeBlock = InvokeBlocks.count(&BB) > 0;

‎bolt/lib/Rewrite/LinuxKernelRewriter.cpp

Lines changed: 4 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -770,11 +770,8 @@ Error LinuxKernelRewriter::rewriteORCTables() {
770770
continue;
771771

772772
// Issue label for the instruction.
773-
MCSymbol *Label = BC.MIB->getLabel(Inst);
774-
if (!Label) {
775-
Label = BC.Ctx->createTempSymbol("__ORC_");
776-
BC.MIB->setLabel(Inst, Label);
777-
}
773+
MCSymbol *Label =
774+
BC.MIB->getOrCreateInstLabel(Inst, "__ORC_", BC.Ctx.get());
778775

779776
if (Error E = emitORCEntry(0, *ErrorOrState, Label))
780777
return E;
@@ -908,11 +905,8 @@ Error LinuxKernelRewriter::readStaticCalls() {
908905

909906
BC.MIB->addAnnotation(*Inst, "StaticCall", EntryID);
910907

911-
MCSymbol *Label = BC.MIB->getLabel(*Inst);
912-
if (!Label) {
913-
Label = BC.Ctx->createTempSymbol("__SC_");
914-
BC.MIB->setLabel(*Inst, Label);
915-
}
908+
MCSymbol *Label =
909+
BC.MIB->getOrCreateInstLabel(*Inst, "__SC_", BC.Ctx.get());
916910

917911
StaticCallEntries.push_back({EntryID, BF, Label});
918912
}

‎bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp

Lines changed: 16 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -270,32 +270,38 @@ class AArch64MCPlusBuilder : public MCPlusBuilder {
270270
return isLDRB(Inst) || isLDRH(Inst) || isLDRW(Inst) || isLDRX(Inst);
271271
}
272272

273-
bool isAArch64Exclusive(const MCInst &Inst) const override {
273+
bool isAArch64ExclusiveLoad(const MCInst &Inst) const override {
274274
return (Inst.getOpcode() == AArch64::LDXPX ||
275275
Inst.getOpcode() == AArch64::LDXPW ||
276276
Inst.getOpcode() == AArch64::LDXRX ||
277277
Inst.getOpcode() == AArch64::LDXRW ||
278278
Inst.getOpcode() == AArch64::LDXRH ||
279279
Inst.getOpcode() == AArch64::LDXRB ||
280-
Inst.getOpcode() == AArch64::STXPX ||
281-
Inst.getOpcode() == AArch64::STXPW ||
282-
Inst.getOpcode() == AArch64::STXRX ||
283-
Inst.getOpcode() == AArch64::STXRW ||
284-
Inst.getOpcode() == AArch64::STXRH ||
285-
Inst.getOpcode() == AArch64::STXRB ||
286280
Inst.getOpcode() == AArch64::LDAXPX ||
287281
Inst.getOpcode() == AArch64::LDAXPW ||
288282
Inst.getOpcode() == AArch64::LDAXRX ||
289283
Inst.getOpcode() == AArch64::LDAXRW ||
290284
Inst.getOpcode() == AArch64::LDAXRH ||
291-
Inst.getOpcode() == AArch64::LDAXRB ||
285+
Inst.getOpcode() == AArch64::LDAXRB);
286+
}
287+
288+
bool isAArch64ExclusiveStore(const MCInst &Inst) const override {
289+
return (Inst.getOpcode() == AArch64::STXPX ||
290+
Inst.getOpcode() == AArch64::STXPW ||
291+
Inst.getOpcode() == AArch64::STXRX ||
292+
Inst.getOpcode() == AArch64::STXRW ||
293+
Inst.getOpcode() == AArch64::STXRH ||
294+
Inst.getOpcode() == AArch64::STXRB ||
292295
Inst.getOpcode() == AArch64::STLXPX ||
293296
Inst.getOpcode() == AArch64::STLXPW ||
294297
Inst.getOpcode() == AArch64::STLXRX ||
295298
Inst.getOpcode() == AArch64::STLXRW ||
296299
Inst.getOpcode() == AArch64::STLXRH ||
297-
Inst.getOpcode() == AArch64::STLXRB ||
298-
Inst.getOpcode() == AArch64::CLREX);
300+
Inst.getOpcode() == AArch64::STLXRB);
301+
}
302+
303+
bool isAArch64ExclusiveClear(const MCInst &Inst) const override {
304+
return (Inst.getOpcode() == AArch64::CLREX);
299305
}
300306

301307
bool isLoadFromStack(const MCInst &Inst) const {

0 commit comments

Comments
 (0)