Skip to content

Commit 3dd4aa7

Browse files
topperctopperc
committed
[RISCV] When custom iseling masked loads/stores, copy the mask into V0 instead of virtual register.
This matches what we do in our isel patterns. In our internal testing we've found this is needed to make the fast register allocator happy at -O0. Otherwise it may assign V0 to an earlier operand and find itself with no registers left when it reaches the mask operand. By using V0 explicitly, the fast register allocator will see it when it checks for phys register usages before it starts allocating vregs. I'll try to update this with a test case. Unfortunately, this does appear to prevent some instruction reordering by the pre-RA scheduler which leads to the increased spills seen in some tests. I suspect that problem could already occur for other instructions that already used V0 directly. There's a lot of repeated code here that could do with some wrapper functions. Not sure if that should be at the level of the new code that deals with V0. That would require multiple output parameters to pass the glue, chain and register back. Maybe it should be at a higher level over the entire set of push_backs. Reviewed By: frasercrmck, HsiangKai Differential Revision: https://reviews.llvm.org/D99367
1 parent a7afc8a commit 3dd4aa7

File tree

5 files changed

+209
-95
lines changed

5 files changed

+209
-95
lines changed

‎llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp

Lines changed: 116 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -134,8 +134,12 @@ void RISCVDAGToDAGISel::selectVLSEG(SDNode *Node, bool IsMasked,
134134
MVT XLenVT = Subtarget->getXLenVT();
135135
RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
136136
SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
137+
138+
SDValue Chain = Node->getOperand(0);
139+
SDValue Glue;
140+
137141
unsigned CurOp = 2;
138-
SmallVector<SDValue, 7> Operands;
142+
SmallVector<SDValue, 8> Operands;
139143
if (IsMasked) {
140144
SmallVector<SDValue, 8> Regs(Node->op_begin() + CurOp,
141145
Node->op_begin() + CurOp + NF);
@@ -148,13 +152,20 @@ void RISCVDAGToDAGISel::selectVLSEG(SDNode *Node, bool IsMasked,
148152
Operands.push_back(Base); // Base pointer.
149153
if (IsStrided)
150154
Operands.push_back(Node->getOperand(CurOp++)); // Stride.
151-
if (IsMasked)
152-
Operands.push_back(Node->getOperand(CurOp++)); // Mask.
155+
if (IsMasked) {
156+
// Mask needs to be copied to V0.
157+
SDValue Mask = Node->getOperand(CurOp++);
158+
Chain = CurDAG->getCopyToReg(Chain, DL, RISCV::V0, Mask, SDValue());
159+
Glue = Chain.getValue(1);
160+
Operands.push_back(CurDAG->getRegister(RISCV::V0, Mask.getValueType()));
161+
}
153162
SDValue VL;
154163
selectVLOp(Node->getOperand(CurOp++), VL);
155164
Operands.push_back(VL);
156165
Operands.push_back(SEW);
157-
Operands.push_back(Node->getOperand(0)); // Chain.
166+
Operands.push_back(Chain); // Chain.
167+
if (Glue)
168+
Operands.push_back(Glue);
158169
const RISCV::VLSEGPseudo *P =
159170
RISCV::getVLSEGPseudo(NF, IsMasked, IsStrided, /*FF*/ false, ScalarSize,
160171
static_cast<unsigned>(LMUL));
@@ -184,6 +195,9 @@ void RISCVDAGToDAGISel::selectVLSEGFF(SDNode *Node, bool IsMasked) {
184195
RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
185196
SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
186197

198+
SDValue Chain = Node->getOperand(0);
199+
SDValue Glue;
200+
187201
unsigned CurOp = 2;
188202
SmallVector<SDValue, 7> Operands;
189203
if (IsMasked) {
@@ -196,13 +210,20 @@ void RISCVDAGToDAGISel::selectVLSEGFF(SDNode *Node, bool IsMasked) {
196210
SDValue Base;
197211
SelectBaseAddr(Node->getOperand(CurOp++), Base);
198212
Operands.push_back(Base); // Base pointer.
199-
if (IsMasked)
200-
Operands.push_back(Node->getOperand(CurOp++)); // Mask.
213+
if (IsMasked) {
214+
// Mask needs to be copied to V0.
215+
SDValue Mask = Node->getOperand(CurOp++);
216+
Chain = CurDAG->getCopyToReg(Chain, DL, RISCV::V0, Mask, SDValue());
217+
Glue = Chain.getValue(1);
218+
Operands.push_back(CurDAG->getRegister(RISCV::V0, Mask.getValueType()));
219+
}
201220
SDValue VL;
202221
selectVLOp(Node->getOperand(CurOp++), VL);
203222
Operands.push_back(VL);
204223
Operands.push_back(SEW);
205-
Operands.push_back(Node->getOperand(0)); // Chain.
224+
Operands.push_back(Chain); // Chain.
225+
if (Glue)
226+
Operands.push_back(Glue);
206227
const RISCV::VLSEGPseudo *P =
207228
RISCV::getVLSEGPseudo(NF, IsMasked, /*Strided*/ false, /*FF*/ true,
208229
ScalarSize, static_cast<unsigned>(LMUL));
@@ -235,8 +256,12 @@ void RISCVDAGToDAGISel::selectVLXSEG(SDNode *Node, bool IsMasked,
235256
MVT XLenVT = Subtarget->getXLenVT();
236257
RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
237258
SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
259+
260+
SDValue Chain = Node->getOperand(0);
261+
SDValue Glue;
262+
238263
unsigned CurOp = 2;
239-
SmallVector<SDValue, 7> Operands;
264+
SmallVector<SDValue, 8> Operands;
240265
if (IsMasked) {
241266
SmallVector<SDValue, 8> Regs(Node->op_begin() + CurOp,
242267
Node->op_begin() + CurOp + NF);
@@ -249,13 +274,20 @@ void RISCVDAGToDAGISel::selectVLXSEG(SDNode *Node, bool IsMasked,
249274
Operands.push_back(Base); // Base pointer.
250275
Operands.push_back(Node->getOperand(CurOp++)); // Index.
251276
MVT IndexVT = Operands.back()->getSimpleValueType(0);
252-
if (IsMasked)
253-
Operands.push_back(Node->getOperand(CurOp++)); // Mask.
277+
if (IsMasked) {
278+
// Mask needs to be copied to V0.
279+
SDValue Mask = Node->getOperand(CurOp++);
280+
Chain = CurDAG->getCopyToReg(Chain, DL, RISCV::V0, Mask, SDValue());
281+
Glue = Chain.getValue(1);
282+
Operands.push_back(CurDAG->getRegister(RISCV::V0, Mask.getValueType()));
283+
}
254284
SDValue VL;
255285
selectVLOp(Node->getOperand(CurOp++), VL);
256286
Operands.push_back(VL);
257287
Operands.push_back(SEW);
258-
Operands.push_back(Node->getOperand(0)); // Chain.
288+
Operands.push_back(Chain); // Chain.
289+
if (Glue)
290+
Operands.push_back(Glue);
259291

260292
assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
261293
"Element count mismatch");
@@ -297,21 +329,32 @@ void RISCVDAGToDAGISel::selectVSSEG(SDNode *Node, bool IsMasked,
297329
SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
298330
SmallVector<SDValue, 8> Regs(Node->op_begin() + 2, Node->op_begin() + 2 + NF);
299331
SDValue StoreVal = createTuple(*CurDAG, Regs, NF, LMUL);
300-
SmallVector<SDValue, 7> Operands;
332+
333+
SDValue Chain = Node->getOperand(0);
334+
SDValue Glue;
335+
336+
SmallVector<SDValue, 8> Operands;
301337
Operands.push_back(StoreVal);
302338
unsigned CurOp = 2 + NF;
303339
SDValue Base;
304340
SelectBaseAddr(Node->getOperand(CurOp++), Base);
305341
Operands.push_back(Base); // Base pointer.
306342
if (IsStrided)
307343
Operands.push_back(Node->getOperand(CurOp++)); // Stride.
308-
if (IsMasked)
309-
Operands.push_back(Node->getOperand(CurOp++)); // Mask.
344+
if (IsMasked) {
345+
// Mask needs to be copied to V0.
346+
SDValue Mask = Node->getOperand(CurOp++);
347+
Chain = CurDAG->getCopyToReg(Chain, DL, RISCV::V0, Mask, SDValue());
348+
Glue = Chain.getValue(1);
349+
Operands.push_back(CurDAG->getRegister(RISCV::V0, Mask.getValueType()));
350+
}
310351
SDValue VL;
311352
selectVLOp(Node->getOperand(CurOp++), VL);
312353
Operands.push_back(VL);
313354
Operands.push_back(SEW);
314-
Operands.push_back(Node->getOperand(0)); // Chain.
355+
Operands.push_back(Chain); // Chain.
356+
if (Glue)
357+
Operands.push_back(Glue);
315358
const RISCV::VSSEGPseudo *P = RISCV::getVSSEGPseudo(
316359
NF, IsMasked, IsStrided, ScalarSize, static_cast<unsigned>(LMUL));
317360
MachineSDNode *Store =
@@ -334,23 +377,34 @@ void RISCVDAGToDAGISel::selectVSXSEG(SDNode *Node, bool IsMasked,
334377
MVT XLenVT = Subtarget->getXLenVT();
335378
RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
336379
SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
337-
SmallVector<SDValue, 7> Operands;
338380
SmallVector<SDValue, 8> Regs(Node->op_begin() + 2, Node->op_begin() + 2 + NF);
339381
SDValue StoreVal = createTuple(*CurDAG, Regs, NF, LMUL);
382+
383+
SDValue Chain = Node->getOperand(0);
384+
SDValue Glue;
385+
386+
SmallVector<SDValue, 8> Operands;
340387
Operands.push_back(StoreVal);
341388
unsigned CurOp = 2 + NF;
342389
SDValue Base;
343390
SelectBaseAddr(Node->getOperand(CurOp++), Base);
344391
Operands.push_back(Base); // Base pointer.
345392
Operands.push_back(Node->getOperand(CurOp++)); // Index.
346393
MVT IndexVT = Operands.back()->getSimpleValueType(0);
347-
if (IsMasked)
348-
Operands.push_back(Node->getOperand(CurOp++)); // Mask.
394+
if (IsMasked) {
395+
// Mask needs to be copied to V0.
396+
SDValue Mask = Node->getOperand(CurOp++);
397+
Chain = CurDAG->getCopyToReg(Chain, DL, RISCV::V0, Mask, SDValue());
398+
Glue = Chain.getValue(1);
399+
Operands.push_back(CurDAG->getRegister(RISCV::V0, Mask.getValueType()));
400+
}
349401
SDValue VL;
350402
selectVLOp(Node->getOperand(CurOp++), VL);
351403
Operands.push_back(VL);
352404
Operands.push_back(SEW);
353-
Operands.push_back(Node->getOperand(0)); // Chain.
405+
Operands.push_back(Chain); // Chain.
406+
if (Glue)
407+
Operands.push_back(Glue);
354408

355409
assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
356410
"Element count mismatch");
@@ -620,22 +674,32 @@ void RISCVDAGToDAGISel::Select(SDNode *Node) {
620674
MVT XLenVT = Subtarget->getXLenVT();
621675
SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
622676

677+
SDValue Chain = Node->getOperand(0);
678+
SDValue Glue;
679+
623680
unsigned CurOp = 2;
624-
SmallVector<SDValue, 7> Operands;
681+
SmallVector<SDValue, 8> Operands;
625682
if (IsMasked)
626683
Operands.push_back(Node->getOperand(CurOp++));
627684
SDValue Base;
628685
SelectBaseAddr(Node->getOperand(CurOp++), Base);
629686
Operands.push_back(Base); // Base pointer.
630687
Operands.push_back(Node->getOperand(CurOp++)); // Index.
631688
MVT IndexVT = Operands.back()->getSimpleValueType(0);
632-
if (IsMasked)
633-
Operands.push_back(Node->getOperand(CurOp++)); // Mask.
689+
if (IsMasked) {
690+
// Mask needs to be copied to V0.
691+
SDValue Mask = Node->getOperand(CurOp++);
692+
Chain = CurDAG->getCopyToReg(Chain, DL, RISCV::V0, Mask, SDValue());
693+
Glue = Chain.getValue(1);
694+
Operands.push_back(CurDAG->getRegister(RISCV::V0, Mask.getValueType()));
695+
}
634696
SDValue VL;
635697
selectVLOp(Node->getOperand(CurOp++), VL);
636698
Operands.push_back(VL);
637699
Operands.push_back(SEW);
638-
Operands.push_back(Node->getOperand(0)); // Chain.
700+
Operands.push_back(Chain); // Chain.
701+
if (Glue)
702+
Operands.push_back(Glue);
639703

640704
assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
641705
"Element count mismatch");
@@ -672,22 +736,32 @@ void RISCVDAGToDAGISel::Select(SDNode *Node) {
672736
unsigned SEWImm = (IntNo == Intrinsic::riscv_vle1) ? 8 : ScalarSize;
673737
SDValue SEW = CurDAG->getTargetConstant(SEWImm, DL, XLenVT);
674738

739+
SDValue Chain = Node->getOperand(0);
740+
SDValue Glue;
741+
675742
unsigned CurOp = 2;
676-
SmallVector<SDValue, 7> Operands;
743+
SmallVector<SDValue, 8> Operands;
677744
if (IsMasked)
678745
Operands.push_back(Node->getOperand(CurOp++));
679746
SDValue Base;
680747
SelectBaseAddr(Node->getOperand(CurOp++), Base);
681748
Operands.push_back(Base); // Base pointer.
682749
if (IsStrided)
683750
Operands.push_back(Node->getOperand(CurOp++)); // Stride.
684-
if (IsMasked)
685-
Operands.push_back(Node->getOperand(CurOp++)); // Mask.
751+
if (IsMasked) {
752+
// Mask needs to be copied to V0.
753+
SDValue Mask = Node->getOperand(CurOp++);
754+
Chain = CurDAG->getCopyToReg(Chain, DL, RISCV::V0, Mask, SDValue());
755+
Glue = Chain.getValue(1);
756+
Operands.push_back(CurDAG->getRegister(RISCV::V0, Mask.getValueType()));
757+
}
686758
SDValue VL;
687759
selectVLOp(Node->getOperand(CurOp++), VL);
688760
Operands.push_back(VL);
689761
Operands.push_back(SEW);
690-
Operands.push_back(Node->getOperand(0)); // Chain.
762+
Operands.push_back(Chain); // Chain.
763+
if (Glue)
764+
Operands.push_back(Glue);
691765

692766
RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
693767
const RISCV::VLEPseudo *P =
@@ -711,20 +785,30 @@ void RISCVDAGToDAGISel::Select(SDNode *Node) {
711785
MVT XLenVT = Subtarget->getXLenVT();
712786
SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
713787

788+
SDValue Chain = Node->getOperand(0);
789+
SDValue Glue;
790+
714791
unsigned CurOp = 2;
715792
SmallVector<SDValue, 7> Operands;
716793
if (IsMasked)
717794
Operands.push_back(Node->getOperand(CurOp++));
718795
SDValue Base;
719796
SelectBaseAddr(Node->getOperand(CurOp++), Base);
720797
Operands.push_back(Base); // Base pointer.
721-
if (IsMasked)
722-
Operands.push_back(Node->getOperand(CurOp++)); // Mask.
798+
if (IsMasked) {
799+
// Mask needs to be copied to V0.
800+
SDValue Mask = Node->getOperand(CurOp++);
801+
Chain = CurDAG->getCopyToReg(Chain, DL, RISCV::V0, Mask, SDValue());
802+
Glue = Chain.getValue(1);
803+
Operands.push_back(CurDAG->getRegister(RISCV::V0, Mask.getValueType()));
804+
}
723805
SDValue VL;
724806
selectVLOp(Node->getOperand(CurOp++), VL);
725807
Operands.push_back(VL);
726808
Operands.push_back(SEW);
727-
Operands.push_back(Node->getOperand(0)); // Chain.
809+
Operands.push_back(Chain); // Chain.
810+
if (Glue)
811+
Operands.push_back(Glue);
728812

729813
RISCVVLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
730814
const RISCV::VLEPseudo *P =
@@ -842,7 +926,7 @@ void RISCVDAGToDAGISel::Select(SDNode *Node) {
842926
SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
843927

844928
unsigned CurOp = 2;
845-
SmallVector<SDValue, 6> Operands;
929+
SmallVector<SDValue, 7> Operands;
846930
Operands.push_back(Node->getOperand(CurOp++)); // Store value.
847931
SDValue Base;
848932
SelectBaseAddr(Node->getOperand(CurOp++), Base);
@@ -893,7 +977,7 @@ void RISCVDAGToDAGISel::Select(SDNode *Node) {
893977
SDValue SEW = CurDAG->getTargetConstant(SEWImm, DL, XLenVT);
894978

895979
unsigned CurOp = 2;
896-
SmallVector<SDValue, 6> Operands;
980+
SmallVector<SDValue, 7> Operands;
897981
Operands.push_back(Node->getOperand(CurOp++)); // Store value.
898982
SDValue Base;
899983
SelectBaseAddr(Node->getOperand(CurOp++), Base);

0 commit comments

Comments
 (0)