Skip to content

Commit f59cb41

Browse files
author
Abinav Puthan Purayil
committed
[AMDGPU] Select buffer_atomic_cmpswap* in tblgen
This change replaces the manual selection of buffer_atomic_cmpswap* instructions in SelectionDAG and GlobalISel with a tblgen based selection in BUFInstructions.td. This allows us to select the return and no-return variants in tblgen. Differential Revision: https://reviews.llvm.org/D121770
1 parent 557a0e7 commit f59cb41

File tree

6 files changed

+52
-148
lines changed

6 files changed

+52
-148
lines changed

‎llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp

Lines changed: 0 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -679,9 +679,6 @@ void AMDGPUDAGToDAGISel::Select(SDNode *N) {
679679
case ISD::FMA:
680680
SelectFMAD_FMA(N);
681681
return;
682-
case AMDGPUISD::ATOMIC_CMP_SWAP:
683-
SelectATOMIC_CMP_SWAP(N);
684-
return;
685682
case AMDGPUISD::CVT_PKRTZ_F16_F32:
686683
case AMDGPUISD::CVT_PKNORM_I16_F32:
687684
case AMDGPUISD::CVT_PKNORM_U16_F32:
@@ -2278,70 +2275,6 @@ void AMDGPUDAGToDAGISel::SelectFMAD_FMA(SDNode *N) {
22782275
}
22792276
}
22802277

2281-
// This is here because there isn't a way to use the generated sub0_sub1 as the
2282-
// subreg index to EXTRACT_SUBREG in tablegen.
2283-
void AMDGPUDAGToDAGISel::SelectATOMIC_CMP_SWAP(SDNode *N) {
2284-
MemSDNode *Mem = cast<MemSDNode>(N);
2285-
unsigned AS = Mem->getAddressSpace();
2286-
if (AS == AMDGPUAS::FLAT_ADDRESS) {
2287-
SelectCode(N);
2288-
return;
2289-
}
2290-
2291-
MVT VT = N->getSimpleValueType(0);
2292-
bool Is32 = (VT == MVT::i32);
2293-
SDLoc SL(N);
2294-
2295-
MachineSDNode *CmpSwap = nullptr;
2296-
if (Subtarget->hasAddr64()) {
2297-
SDValue SRsrc, VAddr, SOffset, Offset;
2298-
2299-
if (SelectMUBUFAddr64(Mem->getBasePtr(), SRsrc, VAddr, SOffset, Offset)) {
2300-
unsigned Opcode = Is32 ? AMDGPU::BUFFER_ATOMIC_CMPSWAP_ADDR64_RTN :
2301-
AMDGPU::BUFFER_ATOMIC_CMPSWAP_X2_ADDR64_RTN;
2302-
SDValue CmpVal = Mem->getOperand(2);
2303-
SDValue CPol = CurDAG->getTargetConstant(AMDGPU::CPol::GLC, SL, MVT::i32);
2304-
2305-
// XXX - Do we care about glue operands?
2306-
2307-
SDValue Ops[] = {CmpVal, VAddr, SRsrc, SOffset, Offset, CPol,
2308-
Mem->getChain()};
2309-
2310-
CmpSwap = CurDAG->getMachineNode(Opcode, SL, Mem->getVTList(), Ops);
2311-
}
2312-
}
2313-
2314-
if (!CmpSwap) {
2315-
SDValue SRsrc, SOffset, Offset;
2316-
if (SelectMUBUFOffset(Mem->getBasePtr(), SRsrc, SOffset, Offset)) {
2317-
unsigned Opcode = Is32 ? AMDGPU::BUFFER_ATOMIC_CMPSWAP_OFFSET_RTN :
2318-
AMDGPU::BUFFER_ATOMIC_CMPSWAP_X2_OFFSET_RTN;
2319-
2320-
SDValue CmpVal = Mem->getOperand(2);
2321-
SDValue CPol = CurDAG->getTargetConstant(AMDGPU::CPol::GLC, SL, MVT::i32);
2322-
SDValue Ops[] = {CmpVal, SRsrc, SOffset, Offset, CPol, Mem->getChain()};
2323-
2324-
CmpSwap = CurDAG->getMachineNode(Opcode, SL, Mem->getVTList(), Ops);
2325-
}
2326-
}
2327-
2328-
if (!CmpSwap) {
2329-
SelectCode(N);
2330-
return;
2331-
}
2332-
2333-
MachineMemOperand *MMO = Mem->getMemOperand();
2334-
CurDAG->setNodeMemRefs(CmpSwap, {MMO});
2335-
2336-
unsigned SubReg = Is32 ? AMDGPU::sub0 : AMDGPU::sub0_sub1;
2337-
SDValue Extract
2338-
= CurDAG->getTargetExtractSubreg(SubReg, SL, VT, SDValue(CmpSwap, 0));
2339-
2340-
ReplaceUses(SDValue(N, 0), Extract);
2341-
ReplaceUses(SDValue(N, 1), SDValue(CmpSwap, 1));
2342-
CurDAG->RemoveDeadNode(N);
2343-
}
2344-
23452278
void AMDGPUDAGToDAGISel::SelectDSAppendConsume(SDNode *N, unsigned IntrID) {
23462279
// The address is assumed to be uniform, so if it ends up in a VGPR, it will
23472280
// be copied to an SGPR with readfirstlane.

‎llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -249,7 +249,6 @@ class AMDGPUDAGToDAGISel : public SelectionDAGISel {
249249
bool isCBranchSCC(const SDNode *N) const;
250250
void SelectBRCOND(SDNode *N);
251251
void SelectFMAD_FMA(SDNode *N);
252-
void SelectATOMIC_CMP_SWAP(SDNode *N);
253252
void SelectDSAppendConsume(SDNode *N, unsigned IntrID);
254253
void SelectDS_GWS(SDNode *N, unsigned IntrID);
255254
void SelectInterpP1F16(SDNode *N);

‎llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp

Lines changed: 0 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -2385,65 +2385,6 @@ bool AMDGPUInstructionSelector::selectG_LOAD_STORE_ATOMICRMW(
23852385
return selectImpl(I, *CoverageInfo);
23862386
}
23872387

2388-
// TODO: No rtn optimization.
2389-
bool AMDGPUInstructionSelector::selectG_AMDGPU_ATOMIC_CMPXCHG(
2390-
MachineInstr &MI) const {
2391-
Register PtrReg = MI.getOperand(1).getReg();
2392-
const LLT PtrTy = MRI->getType(PtrReg);
2393-
if (PtrTy.getAddressSpace() == AMDGPUAS::FLAT_ADDRESS ||
2394-
STI.useFlatForGlobal())
2395-
return selectImpl(MI, *CoverageInfo);
2396-
2397-
Register DstReg = MI.getOperand(0).getReg();
2398-
const LLT Ty = MRI->getType(DstReg);
2399-
const bool Is64 = Ty.getSizeInBits() == 64;
2400-
const unsigned SubReg = Is64 ? AMDGPU::sub0_sub1 : AMDGPU::sub0;
2401-
Register TmpReg = MRI->createVirtualRegister(
2402-
Is64 ? &AMDGPU::VReg_128RegClass : &AMDGPU::VReg_64RegClass);
2403-
2404-
const DebugLoc &DL = MI.getDebugLoc();
2405-
MachineBasicBlock *BB = MI.getParent();
2406-
2407-
Register VAddr, RSrcReg, SOffset;
2408-
int64_t Offset = 0;
2409-
2410-
unsigned Opcode;
2411-
if (selectMUBUFOffsetImpl(MI.getOperand(1), RSrcReg, SOffset, Offset)) {
2412-
Opcode = Is64 ? AMDGPU::BUFFER_ATOMIC_CMPSWAP_X2_OFFSET_RTN :
2413-
AMDGPU::BUFFER_ATOMIC_CMPSWAP_OFFSET_RTN;
2414-
} else if (selectMUBUFAddr64Impl(MI.getOperand(1), VAddr,
2415-
RSrcReg, SOffset, Offset)) {
2416-
Opcode = Is64 ? AMDGPU::BUFFER_ATOMIC_CMPSWAP_X2_ADDR64_RTN :
2417-
AMDGPU::BUFFER_ATOMIC_CMPSWAP_ADDR64_RTN;
2418-
} else
2419-
return selectImpl(MI, *CoverageInfo);
2420-
2421-
auto MIB = BuildMI(*BB, &MI, DL, TII.get(Opcode), TmpReg)
2422-
.addReg(MI.getOperand(2).getReg());
2423-
2424-
if (VAddr)
2425-
MIB.addReg(VAddr);
2426-
2427-
MIB.addReg(RSrcReg);
2428-
if (SOffset)
2429-
MIB.addReg(SOffset);
2430-
else
2431-
MIB.addImm(0);
2432-
2433-
MIB.addImm(Offset);
2434-
MIB.addImm(AMDGPU::CPol::GLC);
2435-
MIB.cloneMemRefs(MI);
2436-
2437-
BuildMI(*BB, &MI, DL, TII.get(AMDGPU::COPY), DstReg)
2438-
.addReg(TmpReg, RegState::Kill, SubReg);
2439-
2440-
MI.eraseFromParent();
2441-
2442-
MRI->setRegClass(
2443-
DstReg, Is64 ? &AMDGPU::VReg_64RegClass : &AMDGPU::VGPR_32RegClass);
2444-
return constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI);
2445-
}
2446-
24472388
static bool isVCmpResult(Register Reg, MachineRegisterInfo &MRI) {
24482389
if (Reg.isPhysical())
24492390
return false;
@@ -3227,8 +3168,6 @@ bool AMDGPUInstructionSelector::select(MachineInstr &I) {
32273168
case AMDGPU::G_AMDGPU_ATOMIC_FMIN:
32283169
case AMDGPU::G_AMDGPU_ATOMIC_FMAX:
32293170
return selectG_LOAD_STORE_ATOMICRMW(I);
3230-
case AMDGPU::G_AMDGPU_ATOMIC_CMPXCHG:
3231-
return selectG_AMDGPU_ATOMIC_CMPXCHG(I);
32323171
case TargetOpcode::G_SELECT:
32333172
return selectG_SELECT(I);
32343173
case TargetOpcode::G_TRUNC:

‎llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -133,7 +133,6 @@ class AMDGPUInstructionSelector final : public InstructionSelector {
133133

134134
void initM0(MachineInstr &I) const;
135135
bool selectG_LOAD_STORE_ATOMICRMW(MachineInstr &I) const;
136-
bool selectG_AMDGPU_ATOMIC_CMPXCHG(MachineInstr &I) const;
137136
bool selectG_SELECT(MachineInstr &I) const;
138137
bool selectG_BRCOND(MachineInstr &I) const;
139138
bool selectG_GLOBAL_VALUE(MachineInstr &I) const;

‎llvm/lib/Target/AMDGPU/BUFInstructions.td

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1399,6 +1399,37 @@ multiclass BufferAtomicPat<string OpPrefix, ValueType vt, string Inst> {
13991399
} // end foreach RtnMode
14001400
}
14011401

1402+
multiclass BufferAtomicCmpSwapPat<ValueType vt, ValueType data_vt, string Inst> {
1403+
foreach RtnMode = ["ret", "noret"] in {
1404+
1405+
defvar Op = !cast<SDPatternOperator>("AMDGPUatomic_cmp_swap_global_" # RtnMode
1406+
# "_" # vt.Size);
1407+
defvar InstSuffix = !if(!eq(RtnMode, "ret"), "_RTN", "");
1408+
1409+
defvar OffsetResDag = (!cast<MUBUF_Pseudo>(Inst # "_OFFSET" # InstSuffix)
1410+
getVregSrcForVT<data_vt>.ret:$vdata_in, SReg_128:$srsrc, SCSrc_b32:$soffset,
1411+
offset:$offset);
1412+
def : Pat<
1413+
(vt (Op (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset), data_vt:$vdata_in)),
1414+
!if(!eq(RtnMode, "ret"),
1415+
(EXTRACT_SUBREG OffsetResDag, !if(!eq(vt, i32), sub0, sub0_sub1)),
1416+
OffsetResDag)
1417+
>;
1418+
1419+
defvar Addr64ResDag = (!cast<MUBUF_Pseudo>(Inst # "_ADDR64" # InstSuffix)
1420+
getVregSrcForVT<data_vt>.ret:$vdata_in, VReg_64:$vaddr, SReg_128:$srsrc,
1421+
SCSrc_b32:$soffset, offset:$offset);
1422+
def : Pat<
1423+
(vt (Op (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset),
1424+
data_vt:$vdata_in)),
1425+
!if(!eq(RtnMode, "ret"),
1426+
(EXTRACT_SUBREG Addr64ResDag, !if(!eq(vt, i32), sub0, sub0_sub1)),
1427+
Addr64ResDag)
1428+
>;
1429+
1430+
} // end foreach RtnMode
1431+
}
1432+
14021433
foreach Ty = [i32, i64] in {
14031434

14041435
defvar Suffix = !if(!eq(Ty, i64), "_X2", "");
@@ -1418,6 +1449,9 @@ defm : BufferAtomicPat<"atomic_dec_global", Ty, "BUFFER_ATOMIC_DEC" # Suffix>;
14181449

14191450
} // end foreach Ty
14201451

1452+
defm : BufferAtomicCmpSwapPat<i32, v2i32, "BUFFER_ATOMIC_CMPSWAP">;
1453+
defm : BufferAtomicCmpSwapPat<i64, v2i64, "BUFFER_ATOMIC_CMPSWAP_X2">;
1454+
14211455
multiclass SIBufferAtomicPat<string OpPrefix, ValueType vt, string Inst,
14221456
list<string> RtnModes = ["ret", "noret"]> {
14231457
foreach RtnMode = RtnModes in {

0 commit comments

Comments
 (0)