Skip to content

Commit 93ec3fa

Browse files
arsenmarsenm
committed
AMDGPU: Support atomicrmw uinc_wrap/udec_wrap
For now keep the exising intrinsics working.
1 parent 7d31d3b commit 93ec3fa

23 files changed

+925
-53
lines changed

‎llvm/lib/Target/AMDGPU/AMDGPUGISel.td

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -227,10 +227,8 @@ def : GINodeEquiv<G_AMDGPU_TBUFFER_STORE_FORMAT, SItbuffer_store>;
227227
def : GINodeEquiv<G_AMDGPU_TBUFFER_STORE_FORMAT_D16, SItbuffer_store_d16>;
228228

229229
// FIXME: Check MMO is atomic
230-
def : GINodeEquiv<G_AMDGPU_ATOMIC_INC, SIatomic_inc>;
231-
def : GINodeEquiv<G_AMDGPU_ATOMIC_DEC, SIatomic_dec>;
232-
def : GINodeEquiv<G_AMDGPU_ATOMIC_INC, atomic_inc_glue>;
233-
def : GINodeEquiv<G_AMDGPU_ATOMIC_DEC, atomic_dec_glue>;
230+
def : GINodeEquiv<G_ATOMICRMW_UINC_WRAP, atomic_load_uinc_wrap_glue>;
231+
def : GINodeEquiv<G_ATOMICRMW_UDEC_WRAP, atomic_load_udec_wrap_glue>;
234232
def : GINodeEquiv<G_AMDGPU_ATOMIC_FMIN, SIatomic_fmin>;
235233
def : GINodeEquiv<G_AMDGPU_ATOMIC_FMAX, SIatomic_fmax>;
236234
def : GINodeEquiv<G_AMDGPU_ATOMIC_FMIN, atomic_load_fmin_glue>;

‎llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -503,10 +503,8 @@ void AMDGPUDAGToDAGISel::Select(SDNode *N) {
503503
// isa<MemSDNode> almost works but is slightly too permissive for some DS
504504
// intrinsics.
505505
if (Opc == ISD::LOAD || Opc == ISD::STORE || isa<AtomicSDNode>(N) ||
506-
(Opc == AMDGPUISD::ATOMIC_INC || Opc == AMDGPUISD::ATOMIC_DEC ||
507-
Opc == ISD::ATOMIC_LOAD_FADD ||
508-
Opc == AMDGPUISD::ATOMIC_LOAD_FMIN ||
509-
Opc == AMDGPUISD::ATOMIC_LOAD_FMAX)) {
506+
Opc == AMDGPUISD::ATOMIC_LOAD_FMIN ||
507+
Opc == AMDGPUISD::ATOMIC_LOAD_FMAX) {
510508
N = glueCopyToM0LDSInit(N);
511509
SelectCode(N);
512510
return;

‎llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4508,8 +4508,6 @@ const char* AMDGPUTargetLowering::getTargetNodeName(unsigned Opcode) const {
45084508
NODE_NAME_CASE(TBUFFER_LOAD_FORMAT_D16)
45094509
NODE_NAME_CASE(DS_ORDERED_COUNT)
45104510
NODE_NAME_CASE(ATOMIC_CMP_SWAP)
4511-
NODE_NAME_CASE(ATOMIC_INC)
4512-
NODE_NAME_CASE(ATOMIC_DEC)
45134511
NODE_NAME_CASE(ATOMIC_LOAD_FMIN)
45144512
NODE_NAME_CASE(ATOMIC_LOAD_FMAX)
45154513
NODE_NAME_CASE(BUFFER_LOAD)

‎llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -505,8 +505,6 @@ enum NodeType : unsigned {
505505
TBUFFER_LOAD_FORMAT_D16,
506506
DS_ORDERED_COUNT,
507507
ATOMIC_CMP_SWAP,
508-
ATOMIC_INC,
509-
ATOMIC_DEC,
510508
ATOMIC_LOAD_FMIN,
511509
ATOMIC_LOAD_FMAX,
512510
BUFFER_LOAD,

‎llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3446,9 +3446,9 @@ bool AMDGPUInstructionSelector::select(MachineInstr &I) {
34463446
case TargetOpcode::G_ATOMICRMW_MAX:
34473447
case TargetOpcode::G_ATOMICRMW_UMIN:
34483448
case TargetOpcode::G_ATOMICRMW_UMAX:
3449+
case TargetOpcode::G_ATOMICRMW_UINC_WRAP:
3450+
case TargetOpcode::G_ATOMICRMW_UDEC_WRAP:
34493451
case TargetOpcode::G_ATOMICRMW_FADD:
3450-
case AMDGPU::G_AMDGPU_ATOMIC_INC:
3451-
case AMDGPU::G_AMDGPU_ATOMIC_DEC:
34523452
case AMDGPU::G_AMDGPU_ATOMIC_FMIN:
34533453
case AMDGPU::G_AMDGPU_ATOMIC_FMAX:
34543454
return selectG_LOAD_STORE_ATOMICRMW(I);

‎llvm/lib/Target/AMDGPU/AMDGPUInstructions.td

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -638,6 +638,8 @@ defm atomic_load_umax : binary_atomic_op_all_as<atomic_load_umax>;
638638
defm atomic_load_umin : binary_atomic_op_all_as<atomic_load_umin>;
639639
defm atomic_load_xor : binary_atomic_op_all_as<atomic_load_xor>;
640640
defm atomic_load_fadd : binary_atomic_op_all_as<atomic_load_fadd, 0>;
641+
defm atomic_load_uinc_wrap : binary_atomic_op_all_as<atomic_load_uinc_wrap>;
642+
defm atomic_load_udec_wrap : binary_atomic_op_all_as<atomic_load_udec_wrap>;
641643
let MemoryVT = v2f16 in
642644
defm atomic_load_fadd_v2f16 : binary_atomic_op_all_as<atomic_load_fadd, 0>;
643645
defm AMDGPUatomic_cmp_swap : binary_atomic_op_all_as<AMDGPUatomic_cmp_swap>;

‎llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1335,7 +1335,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
13351335
{G_ATOMICRMW_XCHG, G_ATOMICRMW_ADD, G_ATOMICRMW_SUB,
13361336
G_ATOMICRMW_AND, G_ATOMICRMW_OR, G_ATOMICRMW_XOR,
13371337
G_ATOMICRMW_MAX, G_ATOMICRMW_MIN, G_ATOMICRMW_UMAX,
1338-
G_ATOMICRMW_UMIN})
1338+
G_ATOMICRMW_UMIN, G_ATOMICRMW_UINC_WRAP, G_ATOMICRMW_UDEC_WRAP})
13391339
.legalFor({{S32, GlobalPtr}, {S32, LocalPtr},
13401340
{S64, GlobalPtr}, {S64, LocalPtr},
13411341
{S32, RegionPtr}, {S64, RegionPtr}});
@@ -4627,8 +4627,8 @@ bool AMDGPULegalizerInfo::legalizeBufferLoad(MachineInstr &MI,
46274627
bool AMDGPULegalizerInfo::legalizeAtomicIncDec(MachineInstr &MI,
46284628
MachineIRBuilder &B,
46294629
bool IsInc) const {
4630-
unsigned Opc = IsInc ? AMDGPU::G_AMDGPU_ATOMIC_INC :
4631-
AMDGPU::G_AMDGPU_ATOMIC_DEC;
4630+
unsigned Opc = IsInc ? AMDGPU::G_ATOMICRMW_UINC_WRAP :
4631+
AMDGPU::G_ATOMICRMW_UDEC_WRAP;
46324632
B.buildInstr(Opc)
46334633
.addDef(MI.getOperand(0).getReg())
46344634
.addUse(MI.getOperand(2).getReg())

‎llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4828,9 +4828,9 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
48284828
case AMDGPU::G_ATOMICRMW_UMAX:
48294829
case AMDGPU::G_ATOMICRMW_UMIN:
48304830
case AMDGPU::G_ATOMICRMW_FADD:
4831+
case AMDGPU::G_ATOMICRMW_UINC_WRAP:
4832+
case AMDGPU::G_ATOMICRMW_UDEC_WRAP:
48314833
case AMDGPU::G_AMDGPU_ATOMIC_CMPXCHG:
4832-
case AMDGPU::G_AMDGPU_ATOMIC_INC:
4833-
case AMDGPU::G_AMDGPU_ATOMIC_DEC:
48344834
case AMDGPU::G_AMDGPU_ATOMIC_FMIN:
48354835
case AMDGPU::G_AMDGPU_ATOMIC_FMAX: {
48364836
OpdsMapping[0] = getVGPROpMapping(MI.getOperand(0).getReg(), MRI, *TRI);

‎llvm/lib/Target/AMDGPU/BUFInstructions.td

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1465,8 +1465,8 @@ defm : BufferAtomicPat<"atomic_load_umax_global", Ty, "BUFFER_ATOMIC_UMAX" # Suf
14651465
defm : BufferAtomicPat<"atomic_load_and_global", Ty, "BUFFER_ATOMIC_AND" # Suffix>;
14661466
defm : BufferAtomicPat<"atomic_load_or_global", Ty, "BUFFER_ATOMIC_OR" # Suffix>;
14671467
defm : BufferAtomicPat<"atomic_load_xor_global", Ty, "BUFFER_ATOMIC_XOR" # Suffix>;
1468-
defm : BufferAtomicPat<"atomic_inc_global", Ty, "BUFFER_ATOMIC_INC" # Suffix>;
1469-
defm : BufferAtomicPat<"atomic_dec_global", Ty, "BUFFER_ATOMIC_DEC" # Suffix>;
1468+
defm : BufferAtomicPat<"atomic_load_uinc_wrap_global", Ty, "BUFFER_ATOMIC_INC" # Suffix>;
1469+
defm : BufferAtomicPat<"atomic_load_udec_wrap_global", Ty, "BUFFER_ATOMIC_DEC" # Suffix>;
14701470

14711471
} // end foreach Ty
14721472

‎llvm/lib/Target/AMDGPU/DSInstructions.td

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1069,8 +1069,8 @@ multiclass DSAtomicCmpXChg_mc<DS_Pseudo inst, DS_Pseudo noRetInst, ValueType vt,
10691069
defm : DSAtomicRetPat_mc<DS_WRXCHG_RTN_B32, i32, "atomic_swap">;
10701070
defm : DSAtomicRetNoRetPat_mc<DS_ADD_RTN_U32, DS_ADD_U32, i32, "atomic_load_add">;
10711071
defm : DSAtomicRetNoRetPat_mc<DS_SUB_RTN_U32, DS_SUB_U32, i32, "atomic_load_sub">;
1072-
defm : DSAtomicRetNoRetPat_mc<DS_INC_RTN_U32, DS_INC_U32, i32, "atomic_inc">;
1073-
defm : DSAtomicRetNoRetPat_mc<DS_DEC_RTN_U32, DS_DEC_U32, i32, "atomic_dec">;
1072+
defm : DSAtomicRetNoRetPat_mc<DS_INC_RTN_U32, DS_INC_U32, i32, "atomic_load_uinc_wrap">;
1073+
defm : DSAtomicRetNoRetPat_mc<DS_DEC_RTN_U32, DS_DEC_U32, i32, "atomic_load_udec_wrap">;
10741074
defm : DSAtomicRetNoRetPat_mc<DS_AND_RTN_B32, DS_AND_B32, i32, "atomic_load_and">;
10751075
defm : DSAtomicRetNoRetPat_mc<DS_OR_RTN_B32, DS_OR_B32, i32, "atomic_load_or">;
10761076
defm : DSAtomicRetNoRetPat_mc<DS_XOR_RTN_B32, DS_XOR_B32, i32, "atomic_load_xor">;
@@ -1097,8 +1097,8 @@ defm : DSAtomicRetNoRetPat_mc<DS_ADD_RTN_F32, DS_ADD_F32, f32, "atomic_load_fadd
10971097
defm : DSAtomicRetPat_mc<DS_WRXCHG_RTN_B64, i64, "atomic_swap">;
10981098
defm : DSAtomicRetNoRetPat_mc<DS_ADD_RTN_U64, DS_ADD_U64, i64, "atomic_load_add">;
10991099
defm : DSAtomicRetNoRetPat_mc<DS_SUB_RTN_U64, DS_SUB_U64, i64, "atomic_load_sub">;
1100-
defm : DSAtomicRetNoRetPat_mc<DS_INC_RTN_U64, DS_INC_U64, i64, "atomic_inc">;
1101-
defm : DSAtomicRetNoRetPat_mc<DS_DEC_RTN_U64, DS_DEC_U64, i64, "atomic_dec">;
1100+
defm : DSAtomicRetNoRetPat_mc<DS_INC_RTN_U64, DS_INC_U64, i64, "atomic_load_uinc_wrap">;
1101+
defm : DSAtomicRetNoRetPat_mc<DS_DEC_RTN_U64, DS_DEC_U64, i64, "atomic_load_udec_wrap">;
11021102
defm : DSAtomicRetNoRetPat_mc<DS_AND_RTN_B64, DS_AND_B64, i64, "atomic_load_and">;
11031103
defm : DSAtomicRetNoRetPat_mc<DS_OR_RTN_B64, DS_OR_B64, i64, "atomic_load_or">;
11041104
defm : DSAtomicRetNoRetPat_mc<DS_XOR_RTN_B64, DS_XOR_B64, i64, "atomic_load_xor">;

‎llvm/lib/Target/AMDGPU/FLATInstructions.td

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1160,8 +1160,8 @@ def : FlatStoreAtomicPat <FLAT_STORE_SHORT, atomic_store_16_flat, i16>;
11601160
foreach as = [ "flat", "global" ] in {
11611161
defm : FlatAtomicPat <"FLAT_ATOMIC_ADD", "atomic_load_add_"#as, i32>;
11621162
defm : FlatAtomicPat <"FLAT_ATOMIC_SUB", "atomic_load_sub_"#as, i32>;
1163-
defm : FlatAtomicPat <"FLAT_ATOMIC_INC", "atomic_inc_"#as, i32>;
1164-
defm : FlatAtomicPat <"FLAT_ATOMIC_DEC", "atomic_dec_"#as, i32>;
1163+
defm : FlatAtomicPat <"FLAT_ATOMIC_INC", "atomic_load_uinc_wrap_"#as, i32>;
1164+
defm : FlatAtomicPat <"FLAT_ATOMIC_DEC", "atomic_load_udec_wrap_"#as, i32>;
11651165
defm : FlatAtomicPat <"FLAT_ATOMIC_AND", "atomic_load_and_"#as, i32>;
11661166
defm : FlatAtomicPat <"FLAT_ATOMIC_SMAX", "atomic_load_max_"#as, i32>;
11671167
defm : FlatAtomicPat <"FLAT_ATOMIC_UMAX", "atomic_load_umax_"#as, i32>;
@@ -1174,8 +1174,8 @@ defm : FlatAtomicPat <"FLAT_ATOMIC_XOR", "atomic_load_xor_"#as, i32>;
11741174

11751175
defm : FlatAtomicPat <"FLAT_ATOMIC_ADD_X2", "atomic_load_add_"#as, i64>;
11761176
defm : FlatAtomicPat <"FLAT_ATOMIC_SUB_X2", "atomic_load_sub_"#as, i64>;
1177-
defm : FlatAtomicPat <"FLAT_ATOMIC_INC_X2", "atomic_inc_"#as, i64>;
1178-
defm : FlatAtomicPat <"FLAT_ATOMIC_DEC_X2", "atomic_dec_"#as, i64>;
1177+
defm : FlatAtomicPat <"FLAT_ATOMIC_INC_X2", "atomic_load_uinc_wrap_"#as, i64>;
1178+
defm : FlatAtomicPat <"FLAT_ATOMIC_DEC_X2", "atomic_load_udec_wrap_"#as, i64>;
11791179
defm : FlatAtomicPat <"FLAT_ATOMIC_AND_X2", "atomic_load_and_"#as, i64>;
11801180
defm : FlatAtomicPat <"FLAT_ATOMIC_SMAX_X2", "atomic_load_max_"#as, i64>;
11811181
defm : FlatAtomicPat <"FLAT_ATOMIC_UMAX_X2", "atomic_load_umax_"#as, i64>;
@@ -1429,8 +1429,8 @@ defm : GlobalFLATAtomicStorePats <GLOBAL_STORE_DWORDX2, atomic_store_64_global,
14291429

14301430
defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_ADD", "atomic_load_add_global", i32>;
14311431
defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SUB", "atomic_load_sub_global", i32>;
1432-
defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_INC", "atomic_inc_global", i32>;
1433-
defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_DEC", "atomic_dec_global", i32>;
1432+
defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_INC", "atomic_load_uinc_wrap_global", i32>;
1433+
defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_DEC", "atomic_load_udec_wrap_global", i32>;
14341434
defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_AND", "atomic_load_and_global", i32>;
14351435
defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SMAX", "atomic_load_max_global", i32>;
14361436
defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_UMAX", "atomic_load_umax_global", i32>;
@@ -1444,8 +1444,8 @@ defm : GlobalFLATAtomicPatsRtn <"GLOBAL_ATOMIC_CSUB", "int_amdgcn_global_atomic_
14441444

14451445
defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_ADD_X2", "atomic_load_add_global", i64>;
14461446
defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SUB_X2", "atomic_load_sub_global", i64>;
1447-
defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_INC_X2", "atomic_inc_global", i64>;
1448-
defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_DEC_X2", "atomic_dec_global", i64>;
1447+
defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_INC_X2", "atomic_load_uinc_wrap_global", i64>;
1448+
defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_DEC_X2", "atomic_load_udec_wrap_global", i64>;
14491449
defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_AND_X2", "atomic_load_and_global", i64>;
14501450
defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SMAX_X2", "atomic_load_max_global", i64>;
14511451
defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_UMAX_X2", "atomic_load_umax_global", i64>;

‎llvm/lib/Target/AMDGPU/R600ISelLowering.cpp

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2182,3 +2182,18 @@ SDNode *R600TargetLowering::PostISelFolding(MachineSDNode *Node,
21822182

21832183
return Node;
21842184
}
2185+
2186+
TargetLowering::AtomicExpansionKind
2187+
R600TargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const {
2188+
switch (RMW->getOperation()) {
2189+
case AtomicRMWInst::UIncWrap:
2190+
case AtomicRMWInst::UDecWrap:
2191+
// FIXME: Cayman at least appears to have instructions for this, but the
2192+
// instruction defintions appear to be missing.
2193+
return AtomicExpansionKind::CmpXChg;
2194+
default:
2195+
break;
2196+
}
2197+
2198+
return AMDGPUTargetLowering::shouldExpandAtomicRMWInIR(RMW);
2199+
}

‎llvm/lib/Target/AMDGPU/R600ISelLowering.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,9 @@ class R600TargetLowering final : public AMDGPUTargetLowering {
114114
SelectionDAG &DAG) const;
115115

116116
SDNode *PostISelFolding(MachineSDNode *N, SelectionDAG &DAG) const override;
117+
118+
TargetLowering::AtomicExpansionKind
119+
shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const override;
117120
};
118121

119122
} // End namespace llvm;

‎llvm/lib/Target/AMDGPU/SIISelLowering.cpp

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -791,6 +791,8 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
791791
ISD::ATOMIC_LOAD_UMIN,
792792
ISD::ATOMIC_LOAD_UMAX,
793793
ISD::ATOMIC_LOAD_FADD,
794+
ISD::ATOMIC_LOAD_UINC_WRAP,
795+
ISD::ATOMIC_LOAD_UDEC_WRAP,
794796
ISD::INTRINSIC_VOID,
795797
ISD::INTRINSIC_W_CHAIN});
796798

@@ -7317,31 +7319,37 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
73177319
M->getVTList(), Ops, M->getMemoryVT(),
73187320
M->getMemOperand());
73197321
}
7322+
case Intrinsic::amdgcn_atomic_inc:
7323+
case Intrinsic::amdgcn_atomic_dec:
73207324
case Intrinsic::amdgcn_ds_fadd: {
73217325
MemSDNode *M = cast<MemSDNode>(Op);
73227326
unsigned Opc;
73237327
switch (IntrID) {
73247328
case Intrinsic::amdgcn_ds_fadd:
73257329
Opc = ISD::ATOMIC_LOAD_FADD;
73267330
break;
7331+
case Intrinsic::amdgcn_atomic_inc:
7332+
Opc = ISD::ATOMIC_LOAD_UINC_WRAP;
7333+
break;
7334+
case Intrinsic::amdgcn_atomic_dec:
7335+
Opc = ISD::ATOMIC_LOAD_UDEC_WRAP;
7336+
break;
73277337
}
73287338

73297339
return DAG.getAtomic(Opc, SDLoc(Op), M->getMemoryVT(),
73307340
M->getOperand(0), M->getOperand(2), M->getOperand(3),
73317341
M->getMemOperand());
73327342
}
7333-
case Intrinsic::amdgcn_atomic_inc:
7334-
case Intrinsic::amdgcn_atomic_dec:
73357343
case Intrinsic::amdgcn_ds_fmin:
73367344
case Intrinsic::amdgcn_ds_fmax: {
73377345
MemSDNode *M = cast<MemSDNode>(Op);
73387346
unsigned Opc;
73397347
switch (IntrID) {
73407348
case Intrinsic::amdgcn_atomic_inc:
7341-
Opc = AMDGPUISD::ATOMIC_INC;
7349+
Opc = ISD::ATOMIC_LOAD_UINC_WRAP;
73427350
break;
73437351
case Intrinsic::amdgcn_atomic_dec:
7344-
Opc = AMDGPUISD::ATOMIC_DEC;
7352+
Opc = ISD::ATOMIC_LOAD_UDEC_WRAP;
73457353
break;
73467354
case Intrinsic::amdgcn_ds_fmin:
73477355
Opc = AMDGPUISD::ATOMIC_LOAD_FMIN;
@@ -12794,8 +12802,6 @@ bool SITargetLowering::isSDNodeSourceOfDivergence(
1279412802
return AMDGPU::isIntrinsicSourceOfDivergence(
1279512803
cast<ConstantSDNode>(N->getOperand(1))->getZExtValue());
1279612804
case AMDGPUISD::ATOMIC_CMP_SWAP:
12797-
case AMDGPUISD::ATOMIC_INC:
12798-
case AMDGPUISD::ATOMIC_DEC:
1279912805
case AMDGPUISD::ATOMIC_LOAD_FMIN:
1280012806
case AMDGPUISD::ATOMIC_LOAD_FMAX:
1280112807
case AMDGPUISD::BUFFER_ATOMIC_SWAP:

‎llvm/lib/Target/AMDGPU/SIInstrInfo.td

Lines changed: 2 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -50,14 +50,6 @@ def SIds_ordered_count : SDNode<"AMDGPUISD::DS_ORDERED_COUNT",
5050
[SDNPMayLoad, SDNPMayStore, SDNPMemOperand, SDNPHasChain, SDNPInGlue]
5151
>;
5252

53-
def SIatomic_inc : SDNode<"AMDGPUISD::ATOMIC_INC", SDTAtomic2,
54-
[SDNPMayLoad, SDNPMayStore, SDNPMemOperand, SDNPHasChain]
55-
>;
56-
57-
def SIatomic_dec : SDNode<"AMDGPUISD::ATOMIC_DEC", SDTAtomic2,
58-
[SDNPMayLoad, SDNPMayStore, SDNPMemOperand, SDNPHasChain]
59-
>;
60-
6153
def SDTAtomic2_f32 : SDTypeProfile<1, 2, [
6254
SDTCisSameAs<0,2>, SDTCisFP<0>, SDTCisPtrTy<1>
6355
]>;
@@ -355,8 +347,6 @@ class isPackedType<ValueType SrcVT> {
355347
// PatFrags for global memory operations
356348
//===----------------------------------------------------------------------===//
357349

358-
defm atomic_inc : binary_atomic_op_all_as<SIatomic_inc>;
359-
defm atomic_dec : binary_atomic_op_all_as<SIatomic_dec>;
360350
defm atomic_load_fmin : binary_atomic_op_all_as<SIatomic_fmin, 0>;
361351
defm atomic_load_fmax : binary_atomic_op_all_as<SIatomic_fmax, 0>;
362352

@@ -762,8 +752,8 @@ multiclass SIAtomicM0Glue2 <string op_name, bit is_amdgpu = 0,
762752

763753
defm atomic_load_add : SIAtomicM0Glue2 <"LOAD_ADD">;
764754
defm atomic_load_sub : SIAtomicM0Glue2 <"LOAD_SUB">;
765-
defm atomic_inc : SIAtomicM0Glue2 <"INC", 1>;
766-
defm atomic_dec : SIAtomicM0Glue2 <"DEC", 1>;
755+
defm atomic_load_uinc_wrap : SIAtomicM0Glue2 <"LOAD_UINC_WRAP">;
756+
defm atomic_load_udec_wrap : SIAtomicM0Glue2 <"LOAD_UDEC_WRAP">;
767757
defm atomic_load_and : SIAtomicM0Glue2 <"LOAD_AND">;
768758
defm atomic_load_min : SIAtomicM0Glue2 <"LOAD_MIN">;
769759
defm atomic_load_max : SIAtomicM0Glue2 <"LOAD_MAX">;

‎llvm/lib/Target/AMDGPU/SIInstructions.td

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3486,8 +3486,6 @@ def G_AMDGPU_ATOMIC_CMPXCHG : AMDGPUGenericInstruction {
34863486
}
34873487

34883488
let Namespace = "AMDGPU" in {
3489-
def G_AMDGPU_ATOMIC_INC : G_ATOMICRMW_OP;
3490-
def G_AMDGPU_ATOMIC_DEC : G_ATOMICRMW_OP;
34913489
def G_AMDGPU_ATOMIC_FMIN : G_ATOMICRMW_OP;
34923490
def G_AMDGPU_ATOMIC_FMAX : G_ATOMICRMW_OP;
34933491
}

0 commit comments

Comments
 (0)