Skip to content

Commit cff9428

Browse files
gonzalobggonzalobg
committed
[NVPTX] Cleanup SeqCst Load/Store
1 parent 34a501c commit cff9428

File tree

2 files changed

+158
-95
lines changed

2 files changed

+158
-95
lines changed

‎llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp

Lines changed: 86 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1147,6 +1147,26 @@ bool NVPTXDAGToDAGISel::tryLoad(SDNode *N) {
11471147
unsigned int PointerSize =
11481148
CurDAG->getDataLayout().getPointerSizeInBits(LD->getAddressSpace());
11491149

1150+
// If a fence is required before the operation, insert it:
1151+
SDValue Chain = N->getOperand(0);
1152+
switch (NVPTX::Ordering(FenceOrdering)) {
1153+
case NVPTX::Ordering::NotAtomic:
1154+
break;
1155+
case NVPTX::Ordering::SequentiallyConsistent: {
1156+
unsigned Op = Subtarget->hasMemoryOrdering()
1157+
? NVPTX::atomic_thread_fence_seq_cst_sys
1158+
: NVPTX::atomic_thread_fence_seq_cst_sys_membar;
1159+
Chain = SDValue(CurDAG->getMachineNode(Op, dl, MVT::Other, Chain), 0);
1160+
break;
1161+
}
1162+
default:
1163+
SmallString<256> Msg;
1164+
raw_svector_ostream OS(Msg);
1165+
OS << "Unexpected fence ordering: \"" << NVPTX::Ordering(FenceOrdering)
1166+
<< "\".";
1167+
report_fatal_error(OS.str());
1168+
}
1169+
11501170
// Type Setting: fromType + fromTypeWidth
11511171
//
11521172
// Sign : ISD::SEXTLOAD
@@ -1174,7 +1194,6 @@ bool NVPTXDAGToDAGISel::tryLoad(SDNode *N) {
11741194
fromType = getLdStRegType(ScalarVT);
11751195

11761196
// Create the machine instruction DAG
1177-
SDValue Chain = N->getOperand(0);
11781197
SDValue N1 = N->getOperand(1);
11791198
SDValue Addr;
11801199
SDValue Offset, Base;
@@ -1187,8 +1206,7 @@ bool NVPTXDAGToDAGISel::tryLoad(SDNode *N) {
11871206
NVPTX::LD_f32_avar, NVPTX::LD_f64_avar);
11881207
if (!Opcode)
11891208
return false;
1190-
SDValue Ops[] = {getI32Imm(FenceOrdering, dl),
1191-
getI32Imm(InstructionOrdering, dl),
1209+
SDValue Ops[] = {getI32Imm(InstructionOrdering, dl),
11921210
getI32Imm(CodeAddrSpace, dl),
11931211
getI32Imm(vecType, dl),
11941212
getI32Imm(fromType, dl),
@@ -1203,8 +1221,7 @@ bool NVPTXDAGToDAGISel::tryLoad(SDNode *N) {
12031221
NVPTX::LD_f32_asi, NVPTX::LD_f64_asi);
12041222
if (!Opcode)
12051223
return false;
1206-
SDValue Ops[] = {getI32Imm(FenceOrdering, dl),
1207-
getI32Imm(InstructionOrdering, dl),
1224+
SDValue Ops[] = {getI32Imm(InstructionOrdering, dl),
12081225
getI32Imm(CodeAddrSpace, dl),
12091226
getI32Imm(vecType, dl),
12101227
getI32Imm(fromType, dl),
@@ -1226,8 +1243,7 @@ bool NVPTXDAGToDAGISel::tryLoad(SDNode *N) {
12261243
NVPTX::LD_f32_ari, NVPTX::LD_f64_ari);
12271244
if (!Opcode)
12281245
return false;
1229-
SDValue Ops[] = {getI32Imm(FenceOrdering, dl),
1230-
getI32Imm(InstructionOrdering, dl),
1246+
SDValue Ops[] = {getI32Imm(InstructionOrdering, dl),
12311247
getI32Imm(CodeAddrSpace, dl),
12321248
getI32Imm(vecType, dl),
12331249
getI32Imm(fromType, dl),
@@ -1248,8 +1264,7 @@ bool NVPTXDAGToDAGISel::tryLoad(SDNode *N) {
12481264
NVPTX::LD_f32_areg, NVPTX::LD_f64_areg);
12491265
if (!Opcode)
12501266
return false;
1251-
SDValue Ops[] = {getI32Imm(FenceOrdering, dl),
1252-
getI32Imm(InstructionOrdering, dl),
1267+
SDValue Ops[] = {getI32Imm(InstructionOrdering, dl),
12531268
getI32Imm(CodeAddrSpace, dl),
12541269
getI32Imm(vecType, dl),
12551270
getI32Imm(fromType, dl),
@@ -1296,6 +1311,25 @@ bool NVPTXDAGToDAGISel::tryLoadVector(SDNode *N) {
12961311
auto [InstructionOrdering, FenceOrdering] =
12971312
getOperationOrderings(MemSD, Subtarget);
12981313

1314+
// If a fence is required before the operation, insert it:
1315+
switch (NVPTX::Ordering(FenceOrdering)) {
1316+
case NVPTX::Ordering::NotAtomic:
1317+
break;
1318+
case NVPTX::Ordering::SequentiallyConsistent: {
1319+
unsigned Op = Subtarget->hasMemoryOrdering()
1320+
? NVPTX::atomic_thread_fence_seq_cst_sys
1321+
: NVPTX::atomic_thread_fence_seq_cst_sys_membar;
1322+
Chain = SDValue(CurDAG->getMachineNode(Op, DL, MVT::Other, Chain), 0);
1323+
break;
1324+
}
1325+
default:
1326+
SmallString<256> Msg;
1327+
raw_svector_ostream OS(Msg);
1328+
OS << "Unexpected fence ordering: \"" << NVPTX::Ordering(FenceOrdering)
1329+
<< "\".";
1330+
report_fatal_error(OS.str());
1331+
}
1332+
12991333
// Vector Setting
13001334
MVT SimpleVT = LoadedVT.getSimpleVT();
13011335

@@ -1361,8 +1395,7 @@ bool NVPTXDAGToDAGISel::tryLoadVector(SDNode *N) {
13611395
}
13621396
if (!Opcode)
13631397
return false;
1364-
SDValue Ops[] = {getI32Imm(FenceOrdering, DL),
1365-
getI32Imm(InstructionOrdering, DL),
1398+
SDValue Ops[] = {getI32Imm(InstructionOrdering, DL),
13661399
getI32Imm(CodeAddrSpace, DL),
13671400
getI32Imm(VecType, DL),
13681401
getI32Imm(FromType, DL),
@@ -1391,8 +1424,7 @@ bool NVPTXDAGToDAGISel::tryLoadVector(SDNode *N) {
13911424
}
13921425
if (!Opcode)
13931426
return false;
1394-
SDValue Ops[] = {getI32Imm(FenceOrdering, DL),
1395-
getI32Imm(InstructionOrdering, DL),
1427+
SDValue Ops[] = {getI32Imm(InstructionOrdering, DL),
13961428
getI32Imm(CodeAddrSpace, DL),
13971429
getI32Imm(VecType, DL),
13981430
getI32Imm(FromType, DL),
@@ -1442,8 +1474,7 @@ bool NVPTXDAGToDAGISel::tryLoadVector(SDNode *N) {
14421474
}
14431475
if (!Opcode)
14441476
return false;
1445-
SDValue Ops[] = {getI32Imm(FenceOrdering, DL),
1446-
getI32Imm(InstructionOrdering, DL),
1477+
SDValue Ops[] = {getI32Imm(InstructionOrdering, DL),
14471478
getI32Imm(CodeAddrSpace, DL),
14481479
getI32Imm(VecType, DL),
14491480
getI32Imm(FromType, DL),
@@ -1493,8 +1524,7 @@ bool NVPTXDAGToDAGISel::tryLoadVector(SDNode *N) {
14931524
}
14941525
if (!Opcode)
14951526
return false;
1496-
SDValue Ops[] = {getI32Imm(FenceOrdering, DL),
1497-
getI32Imm(InstructionOrdering, DL),
1527+
SDValue Ops[] = {getI32Imm(InstructionOrdering, DL),
14981528
getI32Imm(CodeAddrSpace, DL),
14991529
getI32Imm(VecType, DL),
15001530
getI32Imm(FromType, DL),
@@ -1952,6 +1982,26 @@ bool NVPTXDAGToDAGISel::tryStore(SDNode *N) {
19521982
auto [InstructionOrdering, FenceOrdering] =
19531983
getOperationOrderings(ST, Subtarget);
19541984

1985+
// If a fence is required before the operation, insert it:
1986+
SDValue Chain = ST->getChain();
1987+
switch (NVPTX::Ordering(FenceOrdering)) {
1988+
case NVPTX::Ordering::NotAtomic:
1989+
break;
1990+
case NVPTX::Ordering::SequentiallyConsistent: {
1991+
unsigned Op = Subtarget->hasMemoryOrdering()
1992+
? NVPTX::atomic_thread_fence_seq_cst_sys
1993+
: NVPTX::atomic_thread_fence_seq_cst_sys_membar;
1994+
Chain = SDValue(CurDAG->getMachineNode(Op, dl, MVT::Other, Chain), 0);
1995+
break;
1996+
}
1997+
default:
1998+
SmallString<256> Msg;
1999+
raw_svector_ostream OS(Msg);
2000+
OS << "Unexpected fence ordering: \"" << NVPTX::Ordering(FenceOrdering)
2001+
<< "\".";
2002+
report_fatal_error(OS.str());
2003+
}
2004+
19552005
// Vector Setting
19562006
MVT SimpleVT = StoreVT.getSimpleVT();
19572007
unsigned vecType = NVPTX::PTXLdStInstCode::Scalar;
@@ -1971,7 +2021,6 @@ bool NVPTXDAGToDAGISel::tryStore(SDNode *N) {
19712021
unsigned int toType = getLdStRegType(ScalarVT);
19722022

19732023
// Create the machine instruction DAG
1974-
SDValue Chain = ST->getChain();
19752024
SDValue Value = PlainStore ? PlainStore->getValue() : AtomicStore->getVal();
19762025
SDValue BasePtr = ST->getBasePtr();
19772026
SDValue Addr;
@@ -1987,7 +2036,6 @@ bool NVPTXDAGToDAGISel::tryStore(SDNode *N) {
19872036
if (!Opcode)
19882037
return false;
19892038
SDValue Ops[] = {Value,
1990-
getI32Imm(FenceOrdering, dl),
19912039
getI32Imm(InstructionOrdering, dl),
19922040
getI32Imm(CodeAddrSpace, dl),
19932041
getI32Imm(vecType, dl),
@@ -2005,7 +2053,6 @@ bool NVPTXDAGToDAGISel::tryStore(SDNode *N) {
20052053
if (!Opcode)
20062054
return false;
20072055
SDValue Ops[] = {Value,
2008-
getI32Imm(FenceOrdering, dl),
20092056
getI32Imm(InstructionOrdering, dl),
20102057
getI32Imm(CodeAddrSpace, dl),
20112058
getI32Imm(vecType, dl),
@@ -2031,7 +2078,6 @@ bool NVPTXDAGToDAGISel::tryStore(SDNode *N) {
20312078
return false;
20322079

20332080
SDValue Ops[] = {Value,
2034-
getI32Imm(FenceOrdering, dl),
20352081
getI32Imm(InstructionOrdering, dl),
20362082
getI32Imm(CodeAddrSpace, dl),
20372083
getI32Imm(vecType, dl),
@@ -2054,7 +2100,6 @@ bool NVPTXDAGToDAGISel::tryStore(SDNode *N) {
20542100
if (!Opcode)
20552101
return false;
20562102
SDValue Ops[] = {Value,
2057-
getI32Imm(FenceOrdering, dl),
20582103
getI32Imm(InstructionOrdering, dl),
20592104
getI32Imm(CodeAddrSpace, dl),
20602105
getI32Imm(vecType, dl),
@@ -2098,6 +2143,25 @@ bool NVPTXDAGToDAGISel::tryStoreVector(SDNode *N) {
20982143
auto [InstructionOrdering, FenceOrdering] =
20992144
getOperationOrderings(MemSD, Subtarget);
21002145

2146+
// If a fence is required before the operation, insert it:
2147+
switch (NVPTX::Ordering(FenceOrdering)) {
2148+
case NVPTX::Ordering::NotAtomic:
2149+
break;
2150+
case NVPTX::Ordering::SequentiallyConsistent: {
2151+
unsigned Op = Subtarget->hasMemoryOrdering()
2152+
? NVPTX::atomic_thread_fence_seq_cst_sys
2153+
: NVPTX::atomic_thread_fence_seq_cst_sys_membar;
2154+
Chain = SDValue(CurDAG->getMachineNode(Op, DL, MVT::Other, Chain), 0);
2155+
break;
2156+
}
2157+
default:
2158+
SmallString<256> Msg;
2159+
raw_svector_ostream OS(Msg);
2160+
OS << "Unexpected fence ordering: \"" << NVPTX::Ordering(FenceOrdering)
2161+
<< "\".";
2162+
report_fatal_error(OS.str());
2163+
}
2164+
21012165
// Type Setting: toType + toTypeWidth
21022166
// - for integer type, always use 'u'
21032167
assert(StoreVT.isSimple() && "Store value is not simple");
@@ -2138,7 +2202,6 @@ bool NVPTXDAGToDAGISel::tryStoreVector(SDNode *N) {
21382202
ToTypeWidth = 32;
21392203
}
21402204

2141-
StOps.push_back(getI32Imm(FenceOrdering, DL));
21422205
StOps.push_back(getI32Imm(InstructionOrdering, DL));
21432206
StOps.push_back(getI32Imm(CodeAddrSpace, DL));
21442207
StOps.push_back(getI32Imm(VecType, DL));

0 commit comments

Comments
 (0)