Skip to content

Commit 1aa05fd

Browse files
author
Yonghong Song
committed
[BPF] Do not convert atomic_fetch_and_*() to atomic_<op>()'s
In previous commit, atomic_fetch_and_*() operations are converted to atomic_<op>()'s if there are no return values. This is not what we want, we would like to preserve atomic_fetch_and_*() insn so bpf jit can add proper barrier insns. Preserving atomic_fetch_and_*() are okay for most __sync_fetch_and_*() functions, but not for __sync_fetch_and_add() since __sync_fetch_and_add() has been used to generic locked insns in cpu=v1/v2. So after preserving atomic_fetch_and_*() even if return value is not used, XFADDD in BPFInstrInfo.td is adjusted to emit locked add insn for cpu v1/v2 and to emit atomic_fetch_and_add() for cpu >= v3.
1 parent 98d3e3f commit 1aa05fd

File tree

6 files changed

+25
-94
lines changed

6 files changed

+25
-94
lines changed

‎llvm/lib/Target/BPF/BPF.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ InstructionSelector *createBPFInstructionSelector(const BPFTargetMachine &,
3737
void initializeBPFCheckAndAdjustIRPass(PassRegistry&);
3838
void initializeBPFDAGToDAGISelLegacyPass(PassRegistry &);
3939
void initializeBPFMIPeepholePass(PassRegistry &);
40-
void initializeBPFMIPreEmitCheckingPass(PassRegistry&);
40+
void initializeBPFMIPreEmitCheckingPass(PassRegistry &);
4141
void initializeBPFMIPreEmitPeepholePass(PassRegistry &);
4242
void initializeBPFMISimplifyPatchablePass(PassRegistry &);
4343

‎llvm/lib/Target/BPF/BPFInstrInfo.td

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -806,6 +806,7 @@ class XADD<BPFWidthModifer SizeOp, string OpcodeStr, PatFrag OpNode>
806806
let Constraints = "$dst = $val" in {
807807
let Predicates = [BPFNoALU32] in {
808808
def XADDW : XADD<BPF_W, "u32", atomic_load_add_i32>;
809+
def XADDD : XADD<BPF_DW, "u64", atomic_load_add_i64>;
809810
}
810811
}
811812

@@ -849,8 +850,6 @@ let Constraints = "$dst = $val" in {
849850
def XORW32 : ATOMIC32_NOFETCH<BPF_OR, "|">;
850851
def XXORW32 : ATOMIC32_NOFETCH<BPF_XOR, "^">;
851852
}
852-
853-
def XADDD : ATOMIC_NOFETCH<BPF_ADD, "+">;
854853
def XANDD : ATOMIC_NOFETCH<BPF_AND, "&">;
855854
def XORD : ATOMIC_NOFETCH<BPF_OR, "|">;
856855
def XXORD : ATOMIC_NOFETCH<BPF_XOR, "^">;
@@ -901,7 +900,9 @@ let Constraints = "$dst = $val" in {
901900
def XFXORW32 : XFALU32<BPF_W, BPF_XOR, "u32", "xor", atomic_load_xor_i32>;
902901
}
903902

904-
def XFADDD : XFALU64<BPF_DW, BPF_ADD, "u64", "add", atomic_load_add_i64>;
903+
let Predicates = [BPFHasALU32] in {
904+
def XFADDD : XFALU64<BPF_DW, BPF_ADD, "u64", "add", atomic_load_add_i64>;
905+
}
905906
def XFANDD : XFALU64<BPF_DW, BPF_AND, "u64", "and", atomic_load_and_i64>;
906907
def XFORD : XFALU64<BPF_DW, BPF_OR, "u64", "or", atomic_load_or_i64>;
907908
def XFXORD : XFALU64<BPF_DW, BPF_XOR, "u64", "xor", atomic_load_xor_i64>;

‎llvm/lib/Target/BPF/BPFMIChecking.cpp

Lines changed: 7 additions & 77 deletions
Original file line numberDiff line numberDiff line change
@@ -43,15 +43,14 @@ struct BPFMIPreEmitChecking : public MachineFunctionPass {
4343
// Initialize class variables.
4444
void initialize(MachineFunction &MFParm);
4545

46-
bool processAtomicInsts();
46+
void processAtomicInsts();
4747

4848
public:
49-
5049
// Main entry point for this pass.
5150
bool runOnMachineFunction(MachineFunction &MF) override {
5251
if (!skipFunction(MF.getFunction())) {
5352
initialize(MF);
54-
return processAtomicInsts();
53+
processAtomicInsts();
5554
}
5655
return false;
5756
}
@@ -107,7 +106,7 @@ void BPFMIPreEmitChecking::initialize(MachineFunction &MFParm) {
107106
// Dead correctly, and it is safe to use such information or our purpose.
108107
static bool hasLiveDefs(const MachineInstr &MI, const TargetRegisterInfo *TRI) {
109108
const MCRegisterClass *GPR64RegClass =
110-
&BPFMCRegisterClasses[BPF::GPRRegClassID];
109+
&BPFMCRegisterClasses[BPF::GPRRegClassID];
111110
std::vector<unsigned> GPR32LiveDefs;
112111
std::vector<unsigned> GPR64DeadDefs;
113112

@@ -153,12 +152,10 @@ static bool hasLiveDefs(const MachineInstr &MI, const TargetRegisterInfo *TRI) {
153152
return false;
154153
}
155154

156-
bool BPFMIPreEmitChecking::processAtomicInsts() {
155+
void BPFMIPreEmitChecking::processAtomicInsts() {
157156
for (MachineBasicBlock &MBB : *MF) {
158157
for (MachineInstr &MI : MBB) {
159-
if (MI.getOpcode() != BPF::XADDW &&
160-
MI.getOpcode() != BPF::XADDD &&
161-
MI.getOpcode() != BPF::XADDW32)
158+
if (MI.getOpcode() != BPF::XADDW && MI.getOpcode() != BPF::XADDD)
162159
continue;
163160

164161
LLVM_DEBUG(MI.dump());
@@ -171,81 +168,14 @@ bool BPFMIPreEmitChecking::processAtomicInsts() {
171168
}
172169
}
173170
}
174-
175-
// Check return values of atomic_fetch_and_{add,and,or,xor}.
176-
// If the return is not used, the atomic_fetch_and_<op> instruction
177-
// is replaced with atomic_<op> instruction.
178-
MachineInstr *ToErase = nullptr;
179-
bool Changed = false;
180-
const BPFInstrInfo *TII = MF->getSubtarget<BPFSubtarget>().getInstrInfo();
181-
for (MachineBasicBlock &MBB : *MF) {
182-
for (MachineInstr &MI : MBB) {
183-
if (ToErase) {
184-
ToErase->eraseFromParent();
185-
ToErase = nullptr;
186-
}
187-
188-
if (MI.getOpcode() != BPF::XFADDW32 && MI.getOpcode() != BPF::XFADDD &&
189-
MI.getOpcode() != BPF::XFANDW32 && MI.getOpcode() != BPF::XFANDD &&
190-
MI.getOpcode() != BPF::XFXORW32 && MI.getOpcode() != BPF::XFXORD &&
191-
MI.getOpcode() != BPF::XFORW32 && MI.getOpcode() != BPF::XFORD)
192-
continue;
193-
194-
if (hasLiveDefs(MI, TRI))
195-
continue;
196-
197-
LLVM_DEBUG(dbgs() << "Transforming "; MI.dump());
198-
unsigned newOpcode;
199-
switch (MI.getOpcode()) {
200-
case BPF::XFADDW32:
201-
newOpcode = BPF::XADDW32;
202-
break;
203-
case BPF::XFADDD:
204-
newOpcode = BPF::XADDD;
205-
break;
206-
case BPF::XFANDW32:
207-
newOpcode = BPF::XANDW32;
208-
break;
209-
case BPF::XFANDD:
210-
newOpcode = BPF::XANDD;
211-
break;
212-
case BPF::XFXORW32:
213-
newOpcode = BPF::XXORW32;
214-
break;
215-
case BPF::XFXORD:
216-
newOpcode = BPF::XXORD;
217-
break;
218-
case BPF::XFORW32:
219-
newOpcode = BPF::XORW32;
220-
break;
221-
case BPF::XFORD:
222-
newOpcode = BPF::XORD;
223-
break;
224-
default:
225-
llvm_unreachable("Incorrect Atomic Instruction Opcode");
226-
}
227-
228-
BuildMI(MBB, MI, MI.getDebugLoc(), TII->get(newOpcode))
229-
.add(MI.getOperand(0))
230-
.add(MI.getOperand(1))
231-
.add(MI.getOperand(2))
232-
.add(MI.getOperand(3));
233-
234-
ToErase = &MI;
235-
Changed = true;
236-
}
237-
}
238-
239-
return Changed;
240171
}
241172

242-
} // end default namespace
173+
} // namespace
243174

244175
INITIALIZE_PASS(BPFMIPreEmitChecking, "bpf-mi-pemit-checking",
245176
"BPF PreEmit Checking", false, false)
246177

247178
char BPFMIPreEmitChecking::ID = 0;
248-
FunctionPass* llvm::createBPFMIPreEmitCheckingPass()
249-
{
179+
FunctionPass* llvm::createBPFMIPreEmitCheckingPass() {
250180
return new BPFMIPreEmitChecking();
251181
}

‎llvm/test/CodeGen/BPF/atomics.ll

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,8 @@
44
; CHECK-LABEL: test_load_add_32
55
; CHECK: lock *(u32 *)(r1 + 0) += r2
66
; CHECK: encoding: [0xc3,0x21
7-
; CHECK-V3: lock *(u32 *)(r1 + 0) += w2
8-
; CHECK-V3: encoding: [0xc3,0x21,0x00,0x00,0x00,0x00,0x00,0x00]
7+
; CHECK-V3: w2 = atomic_fetch_add((u32 *)(r1 + 0), w2)
8+
; CHECK-V3: encoding: [0xc3,0x21,0x00,0x00,0x01,0x00,0x00,0x00]
99
define void @test_load_add_32(ptr %p, i32 zeroext %v) {
1010
entry:
1111
atomicrmw add ptr %p, i32 %v seq_cst
@@ -15,8 +15,8 @@ entry:
1515
; CHECK-LABEL: test_load_add_64
1616
; CHECK: lock *(u64 *)(r1 + 0) += r2
1717
; CHECK: encoding: [0xdb,0x21
18-
; CHECK-V3: lock *(u64 *)(r1 + 0) += r2
19-
; CHECK-V3: encoding: [0xdb,0x21,0x00,0x00,0x00,0x00,0x00,0x00]
18+
; CHECK-V3: r2 = atomic_fetch_add((u64 *)(r1 + 0), r2)
19+
; CHECK-V3: encoding: [0xdb,0x21,0x00,0x00,0x01,0x00,0x00,0x00]
2020
define void @test_load_add_64(ptr %p, i64 zeroext %v) {
2121
entry:
2222
atomicrmw add ptr %p, i64 %v seq_cst

‎llvm/test/CodeGen/BPF/atomics_2.ll

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -214,8 +214,8 @@ entry:
214214
}
215215

216216
; CHECK-LABEL: test_atomic_xor_32
217-
; CHECK: lock *(u32 *)(r1 + 0) ^= w2
218-
; CHECK: encoding: [0xc3,0x21,0x00,0x00,0xa0,0x00,0x00,0x00]
217+
; CHECK: w2 = atomic_fetch_xor((u32 *)(r1 + 0), w2)
218+
; CHECK: encoding: [0xc3,0x21,0x00,0x00,0xa1,0x00,0x00,0x00]
219219
; CHECK: w0 = 0
220220
define dso_local i32 @test_atomic_xor_32(ptr nocapture %p, i32 %v) local_unnamed_addr {
221221
entry:
@@ -224,8 +224,8 @@ entry:
224224
}
225225

226226
; CHECK-LABEL: test_atomic_xor_64
227-
; CHECK: lock *(u64 *)(r1 + 0) ^= r2
228-
; CHECK: encoding: [0xdb,0x21,0x00,0x00,0xa0,0x00,0x00,0x00]
227+
; CHECK: atomic_fetch_xor((u64 *)(r1 + 0), r2)
228+
; CHECK: encoding: [0xdb,0x21,0x00,0x00,0xa1,0x00,0x00,0x00]
229229
; CHECK: w0 = 0
230230
define dso_local i32 @test_atomic_xor_64(ptr nocapture %p, i64 %v) local_unnamed_addr {
231231
entry:
@@ -234,8 +234,8 @@ entry:
234234
}
235235

236236
; CHECK-LABEL: test_atomic_and_64
237-
; CHECK: lock *(u64 *)(r1 + 0) &= r2
238-
; CHECK: encoding: [0xdb,0x21,0x00,0x00,0x50,0x00,0x00,0x00]
237+
; CHECK: r2 = atomic_fetch_and((u64 *)(r1 + 0), r2)
238+
; CHECK: encoding: [0xdb,0x21,0x00,0x00,0x51,0x00,0x00,0x00]
239239
; CHECK: w0 = 0
240240
define dso_local i32 @test_atomic_and_64(ptr nocapture %p, i64 %v) local_unnamed_addr {
241241
entry:
@@ -244,8 +244,8 @@ entry:
244244
}
245245

246246
; CHECK-LABEL: test_atomic_or_64
247-
; CHECK: lock *(u64 *)(r1 + 0) |= r2
248-
; CHECK: encoding: [0xdb,0x21,0x00,0x00,0x40,0x00,0x00,0x00]
247+
; CHECK: r2 = atomic_fetch_or((u64 *)(r1 + 0), r2)
248+
; CHECK: encoding: [0xdb,0x21,0x00,0x00,0x41,0x00,0x00,0x00]
249249
; CHECK: w0 = 0
250250
define dso_local i32 @test_atomic_or_64(ptr nocapture %p, i64 %v) local_unnamed_addr {
251251
entry:

‎llvm/test/CodeGen/BPF/xadd_legal.ll

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ entry:
2020
%conv = trunc i64 %a to i32
2121
%0 = atomicrmw add ptr %ptr, i32 %conv seq_cst
2222
; CHECK-64: lock *(u32 *)(r1 + 0) += r2
23-
; CHECK-32: lock *(u32 *)(r1 + 0) += w2
23+
; CHECK-32: w2 = atomic_fetch_add((u32 *)(r1 + 0), w2)
2424
%1 = load i32, ptr %ptr, align 4
2525
ret i32 %1
2626
}

0 commit comments

Comments
 (0)