Skip to content

Commit 4eaef95

Browse files
gonzalobggonzalobg
committed
[NVPTX] Cleanup SeqCst Load/Store
1 parent f6f22b3 commit 4eaef95

File tree

2 files changed

+158
-95
lines changed

2 files changed

+158
-95
lines changed

‎llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp

Lines changed: 86 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1145,6 +1145,26 @@ bool NVPTXDAGToDAGISel::tryLoad(SDNode *N) {
11451145
unsigned int PointerSize =
11461146
CurDAG->getDataLayout().getPointerSizeInBits(LD->getAddressSpace());
11471147

1148+
// If a fence is required before the operation, insert it:
1149+
SDValue Chain = N->getOperand(0);
1150+
switch (NVPTX::Ordering(FenceOrdering)) {
1151+
case NVPTX::Ordering::NotAtomic:
1152+
break;
1153+
case NVPTX::Ordering::SequentiallyConsistent: {
1154+
unsigned Op = Subtarget->hasMemoryOrdering()
1155+
? NVPTX::atomic_thread_fence_seq_cst_sys
1156+
: NVPTX::atomic_thread_fence_seq_cst_sys_membar;
1157+
Chain = SDValue(CurDAG->getMachineNode(Op, dl, MVT::Other, Chain), 0);
1158+
break;
1159+
}
1160+
default:
1161+
SmallString<256> Msg;
1162+
raw_svector_ostream OS(Msg);
1163+
OS << "Unexpected fence ordering: \"" << NVPTX::Ordering(FenceOrdering)
1164+
<< "\".";
1165+
report_fatal_error(OS.str());
1166+
}
1167+
11481168
// Type Setting: fromType + fromTypeWidth
11491169
//
11501170
// Sign : ISD::SEXTLOAD
@@ -1172,7 +1192,6 @@ bool NVPTXDAGToDAGISel::tryLoad(SDNode *N) {
11721192
fromType = getLdStRegType(ScalarVT);
11731193

11741194
// Create the machine instruction DAG
1175-
SDValue Chain = N->getOperand(0);
11761195
SDValue N1 = N->getOperand(1);
11771196
SDValue Addr;
11781197
SDValue Offset, Base;
@@ -1185,8 +1204,7 @@ bool NVPTXDAGToDAGISel::tryLoad(SDNode *N) {
11851204
NVPTX::LD_f32_avar, NVPTX::LD_f64_avar);
11861205
if (!Opcode)
11871206
return false;
1188-
SDValue Ops[] = {getI32Imm(FenceOrdering, dl),
1189-
getI32Imm(InstructionOrdering, dl),
1207+
SDValue Ops[] = {getI32Imm(InstructionOrdering, dl),
11901208
getI32Imm(CodeAddrSpace, dl),
11911209
getI32Imm(vecType, dl),
11921210
getI32Imm(fromType, dl),
@@ -1201,8 +1219,7 @@ bool NVPTXDAGToDAGISel::tryLoad(SDNode *N) {
12011219
NVPTX::LD_f32_asi, NVPTX::LD_f64_asi);
12021220
if (!Opcode)
12031221
return false;
1204-
SDValue Ops[] = {getI32Imm(FenceOrdering, dl),
1205-
getI32Imm(InstructionOrdering, dl),
1222+
SDValue Ops[] = {getI32Imm(InstructionOrdering, dl),
12061223
getI32Imm(CodeAddrSpace, dl),
12071224
getI32Imm(vecType, dl),
12081225
getI32Imm(fromType, dl),
@@ -1224,8 +1241,7 @@ bool NVPTXDAGToDAGISel::tryLoad(SDNode *N) {
12241241
NVPTX::LD_f32_ari, NVPTX::LD_f64_ari);
12251242
if (!Opcode)
12261243
return false;
1227-
SDValue Ops[] = {getI32Imm(FenceOrdering, dl),
1228-
getI32Imm(InstructionOrdering, dl),
1244+
SDValue Ops[] = {getI32Imm(InstructionOrdering, dl),
12291245
getI32Imm(CodeAddrSpace, dl),
12301246
getI32Imm(vecType, dl),
12311247
getI32Imm(fromType, dl),
@@ -1246,8 +1262,7 @@ bool NVPTXDAGToDAGISel::tryLoad(SDNode *N) {
12461262
NVPTX::LD_f32_areg, NVPTX::LD_f64_areg);
12471263
if (!Opcode)
12481264
return false;
1249-
SDValue Ops[] = {getI32Imm(FenceOrdering, dl),
1250-
getI32Imm(InstructionOrdering, dl),
1265+
SDValue Ops[] = {getI32Imm(InstructionOrdering, dl),
12511266
getI32Imm(CodeAddrSpace, dl),
12521267
getI32Imm(vecType, dl),
12531268
getI32Imm(fromType, dl),
@@ -1294,6 +1309,25 @@ bool NVPTXDAGToDAGISel::tryLoadVector(SDNode *N) {
12941309
auto [InstructionOrdering, FenceOrdering] =
12951310
getOperationOrderings(MemSD, Subtarget);
12961311

1312+
// If a fence is required before the operation, insert it:
1313+
switch (NVPTX::Ordering(FenceOrdering)) {
1314+
case NVPTX::Ordering::NotAtomic:
1315+
break;
1316+
case NVPTX::Ordering::SequentiallyConsistent: {
1317+
unsigned Op = Subtarget->hasMemoryOrdering()
1318+
? NVPTX::atomic_thread_fence_seq_cst_sys
1319+
: NVPTX::atomic_thread_fence_seq_cst_sys_membar;
1320+
Chain = SDValue(CurDAG->getMachineNode(Op, DL, MVT::Other, Chain), 0);
1321+
break;
1322+
}
1323+
default:
1324+
SmallString<256> Msg;
1325+
raw_svector_ostream OS(Msg);
1326+
OS << "Unexpected fence ordering: \"" << NVPTX::Ordering(FenceOrdering)
1327+
<< "\".";
1328+
report_fatal_error(OS.str());
1329+
}
1330+
12971331
// Vector Setting
12981332
MVT SimpleVT = LoadedVT.getSimpleVT();
12991333

@@ -1359,8 +1393,7 @@ bool NVPTXDAGToDAGISel::tryLoadVector(SDNode *N) {
13591393
}
13601394
if (!Opcode)
13611395
return false;
1362-
SDValue Ops[] = {getI32Imm(FenceOrdering, DL),
1363-
getI32Imm(InstructionOrdering, DL),
1396+
SDValue Ops[] = {getI32Imm(InstructionOrdering, DL),
13641397
getI32Imm(CodeAddrSpace, DL),
13651398
getI32Imm(VecType, DL),
13661399
getI32Imm(FromType, DL),
@@ -1389,8 +1422,7 @@ bool NVPTXDAGToDAGISel::tryLoadVector(SDNode *N) {
13891422
}
13901423
if (!Opcode)
13911424
return false;
1392-
SDValue Ops[] = {getI32Imm(FenceOrdering, DL),
1393-
getI32Imm(InstructionOrdering, DL),
1425+
SDValue Ops[] = {getI32Imm(InstructionOrdering, DL),
13941426
getI32Imm(CodeAddrSpace, DL),
13951427
getI32Imm(VecType, DL),
13961428
getI32Imm(FromType, DL),
@@ -1440,8 +1472,7 @@ bool NVPTXDAGToDAGISel::tryLoadVector(SDNode *N) {
14401472
}
14411473
if (!Opcode)
14421474
return false;
1443-
SDValue Ops[] = {getI32Imm(FenceOrdering, DL),
1444-
getI32Imm(InstructionOrdering, DL),
1475+
SDValue Ops[] = {getI32Imm(InstructionOrdering, DL),
14451476
getI32Imm(CodeAddrSpace, DL),
14461477
getI32Imm(VecType, DL),
14471478
getI32Imm(FromType, DL),
@@ -1491,8 +1522,7 @@ bool NVPTXDAGToDAGISel::tryLoadVector(SDNode *N) {
14911522
}
14921523
if (!Opcode)
14931524
return false;
1494-
SDValue Ops[] = {getI32Imm(FenceOrdering, DL),
1495-
getI32Imm(InstructionOrdering, DL),
1525+
SDValue Ops[] = {getI32Imm(InstructionOrdering, DL),
14961526
getI32Imm(CodeAddrSpace, DL),
14971527
getI32Imm(VecType, DL),
14981528
getI32Imm(FromType, DL),
@@ -1950,6 +1980,26 @@ bool NVPTXDAGToDAGISel::tryStore(SDNode *N) {
19501980
auto [InstructionOrdering, FenceOrdering] =
19511981
getOperationOrderings(ST, Subtarget);
19521982

1983+
// If a fence is required before the operation, insert it:
1984+
SDValue Chain = ST->getChain();
1985+
switch (NVPTX::Ordering(FenceOrdering)) {
1986+
case NVPTX::Ordering::NotAtomic:
1987+
break;
1988+
case NVPTX::Ordering::SequentiallyConsistent: {
1989+
unsigned Op = Subtarget->hasMemoryOrdering()
1990+
? NVPTX::atomic_thread_fence_seq_cst_sys
1991+
: NVPTX::atomic_thread_fence_seq_cst_sys_membar;
1992+
Chain = SDValue(CurDAG->getMachineNode(Op, dl, MVT::Other, Chain), 0);
1993+
break;
1994+
}
1995+
default:
1996+
SmallString<256> Msg;
1997+
raw_svector_ostream OS(Msg);
1998+
OS << "Unexpected fence ordering: \"" << NVPTX::Ordering(FenceOrdering)
1999+
<< "\".";
2000+
report_fatal_error(OS.str());
2001+
}
2002+
19532003
// Vector Setting
19542004
MVT SimpleVT = StoreVT.getSimpleVT();
19552005
unsigned vecType = NVPTX::PTXLdStInstCode::Scalar;
@@ -1969,7 +2019,6 @@ bool NVPTXDAGToDAGISel::tryStore(SDNode *N) {
19692019
unsigned int toType = getLdStRegType(ScalarVT);
19702020

19712021
// Create the machine instruction DAG
1972-
SDValue Chain = ST->getChain();
19732022
SDValue Value = PlainStore ? PlainStore->getValue() : AtomicStore->getVal();
19742023
SDValue BasePtr = ST->getBasePtr();
19752024
SDValue Addr;
@@ -1985,7 +2034,6 @@ bool NVPTXDAGToDAGISel::tryStore(SDNode *N) {
19852034
if (!Opcode)
19862035
return false;
19872036
SDValue Ops[] = {Value,
1988-
getI32Imm(FenceOrdering, dl),
19892037
getI32Imm(InstructionOrdering, dl),
19902038
getI32Imm(CodeAddrSpace, dl),
19912039
getI32Imm(vecType, dl),
@@ -2003,7 +2051,6 @@ bool NVPTXDAGToDAGISel::tryStore(SDNode *N) {
20032051
if (!Opcode)
20042052
return false;
20052053
SDValue Ops[] = {Value,
2006-
getI32Imm(FenceOrdering, dl),
20072054
getI32Imm(InstructionOrdering, dl),
20082055
getI32Imm(CodeAddrSpace, dl),
20092056
getI32Imm(vecType, dl),
@@ -2029,7 +2076,6 @@ bool NVPTXDAGToDAGISel::tryStore(SDNode *N) {
20292076
return false;
20302077

20312078
SDValue Ops[] = {Value,
2032-
getI32Imm(FenceOrdering, dl),
20332079
getI32Imm(InstructionOrdering, dl),
20342080
getI32Imm(CodeAddrSpace, dl),
20352081
getI32Imm(vecType, dl),
@@ -2052,7 +2098,6 @@ bool NVPTXDAGToDAGISel::tryStore(SDNode *N) {
20522098
if (!Opcode)
20532099
return false;
20542100
SDValue Ops[] = {Value,
2055-
getI32Imm(FenceOrdering, dl),
20562101
getI32Imm(InstructionOrdering, dl),
20572102
getI32Imm(CodeAddrSpace, dl),
20582103
getI32Imm(vecType, dl),
@@ -2096,6 +2141,25 @@ bool NVPTXDAGToDAGISel::tryStoreVector(SDNode *N) {
20962141
auto [InstructionOrdering, FenceOrdering] =
20972142
getOperationOrderings(MemSD, Subtarget);
20982143

2144+
// If a fence is required before the operation, insert it:
2145+
switch (NVPTX::Ordering(FenceOrdering)) {
2146+
case NVPTX::Ordering::NotAtomic:
2147+
break;
2148+
case NVPTX::Ordering::SequentiallyConsistent: {
2149+
unsigned Op = Subtarget->hasMemoryOrdering()
2150+
? NVPTX::atomic_thread_fence_seq_cst_sys
2151+
: NVPTX::atomic_thread_fence_seq_cst_sys_membar;
2152+
Chain = SDValue(CurDAG->getMachineNode(Op, DL, MVT::Other, Chain), 0);
2153+
break;
2154+
}
2155+
default:
2156+
SmallString<256> Msg;
2157+
raw_svector_ostream OS(Msg);
2158+
OS << "Unexpected fence ordering: \"" << NVPTX::Ordering(FenceOrdering)
2159+
<< "\".";
2160+
report_fatal_error(OS.str());
2161+
}
2162+
20992163
// Type Setting: toType + toTypeWidth
21002164
// - for integer type, always use 'u'
21012165
assert(StoreVT.isSimple() && "Store value is not simple");
@@ -2136,7 +2200,6 @@ bool NVPTXDAGToDAGISel::tryStoreVector(SDNode *N) {
21362200
ToTypeWidth = 32;
21372201
}
21382202

2139-
StOps.push_back(getI32Imm(FenceOrdering, DL));
21402203
StOps.push_back(getI32Imm(InstructionOrdering, DL));
21412204
StOps.push_back(getI32Imm(CodeAddrSpace, DL));
21422205
StOps.push_back(getI32Imm(VecType, DL));

0 commit comments

Comments
 (0)