Skip to content

Commit 3b5430e

Browse files
topperctopperc
committed
[RISCV] Add a VL output to vleff intrinsics.
The fault-only-first-load instructions can reduce VL if an element other than element 0 triggers a memory fault. This can be used to vectorize loops with data dependent exit conditions like strcmp or strlen. This patch adds a VL output to these intrinsics so that the new VL value can be captured by software. This will be expanded to 'csrr gpr, vl' after the vleff instruction during SelectionDAG. By doing this with one intrinsic we are able to guarantee that the csrr reads the VL value produced by the vleff instruction. Having it as a separate intrinsic would make it impossible to guarantee ordering without making every other vector intrinsic have side effects. The intrinsics are expanded during lowering into two ISD nodes that are glued together. These ISD nodes will go through isel separately, but should maintain the glue so that they get emitted adjacently by InstrEmitter. I've only ran the chain through the vleff instruction, allowing the READ_VL to be deleted if it is unused. Reviewed By: HsiangKai Differential Revision: https://reviews.llvm.org/D94286
1 parent 8120cfe commit 3b5430e

File tree

7 files changed

+1716
-623
lines changed

7 files changed

+1716
-623
lines changed

‎llvm/include/llvm/IR/IntrinsicsRISCV.td

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,16 @@ let TargetPrefix = "riscv" in {
101101
[LLVMPointerType<LLVMMatchType<0>>,
102102
llvm_anyint_ty],
103103
[NoCapture<ArgIndex<0>>, IntrReadMem]>, RISCVVIntrinsic;
104+
// For unit stride fault-only-first load
105+
// Input: (pointer, vl)
106+
// Output: (data, vl)
107+
// NOTE: We model this with default memory properties since we model writing
108+
// VL as a side effect. IntrReadMem, IntrHasSideEffects does not work.
109+
class RISCVUSLoadFF
110+
: Intrinsic<[llvm_anyvector_ty, llvm_anyint_ty],
111+
[LLVMPointerType<LLVMMatchType<0>>, LLVMMatchType<1>],
112+
[NoCapture<ArgIndex<0>>]>,
113+
RISCVVIntrinsic;
104114
// For unit stride load with mask
105115
// Input: (maskedoff, pointer, mask, vl)
106116
class RISCVUSLoadMask
@@ -110,6 +120,18 @@ let TargetPrefix = "riscv" in {
110120
LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
111121
llvm_anyint_ty],
112122
[NoCapture<ArgIndex<1>>, IntrReadMem]>, RISCVVIntrinsic;
123+
// For unit stride fault-only-first load with mask
124+
// Input: (maskedoff, pointer, mask, vl)
125+
// Output: (data, vl)
126+
// NOTE: We model this with default memory properties since we model writing
127+
// VL as a side effect. IntrReadMem, IntrHasSideEffects does not work.
128+
class RISCVUSLoadFFMask
129+
: Intrinsic<[llvm_anyvector_ty, llvm_anyint_ty],
130+
[LLVMMatchType<0>,
131+
LLVMPointerType<LLVMMatchType<0>>,
132+
LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
133+
LLVMMatchType<1>],
134+
[NoCapture<ArgIndex<1>>]>, RISCVVIntrinsic;
113135
// For strided load
114136
// Input: (pointer, stride, vl)
115137
class RISCVSLoad
@@ -564,6 +586,10 @@ let TargetPrefix = "riscv" in {
564586
def "int_riscv_" # NAME : RISCVUSLoad;
565587
def "int_riscv_" # NAME # "_mask" : RISCVUSLoadMask;
566588
}
589+
multiclass RISCVUSLoadFF {
590+
def "int_riscv_" # NAME : RISCVUSLoadFF;
591+
def "int_riscv_" # NAME # "_mask" : RISCVUSLoadFFMask;
592+
}
567593
multiclass RISCVSLoad {
568594
def "int_riscv_" # NAME : RISCVSLoad;
569595
def "int_riscv_" # NAME # "_mask" : RISCVSLoadMask;
@@ -680,7 +706,7 @@ let TargetPrefix = "riscv" in {
680706
}
681707

682708
defm vle : RISCVUSLoad;
683-
defm vleff : RISCVUSLoad;
709+
defm vleff : RISCVUSLoadFF;
684710
defm vse : RISCVUSStore;
685711
defm vlse: RISCVSLoad;
686712
defm vsse: RISCVSStore;

‎llvm/lib/Target/RISCV/RISCVISelLowering.cpp

Lines changed: 28 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -362,6 +362,8 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
362362
setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i32, Custom);
363363
setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i32, Custom);
364364

365+
setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom);
366+
365367
if (Subtarget.is64Bit()) {
366368
setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i64, Custom);
367369
setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i64, Custom);
@@ -1367,7 +1369,29 @@ SDValue RISCVTargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
13671369
}
13681370
}
13691371

1370-
return SDValue();
1372+
switch (IntNo) {
1373+
default:
1374+
return SDValue(); // Don't custom lower most intrinsics.
1375+
case Intrinsic::riscv_vleff: {
1376+
SDLoc DL(Op);
1377+
SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Other, MVT::Glue);
1378+
SDValue Load = DAG.getNode(RISCVISD::VLEFF, DL, VTs, Op.getOperand(0),
1379+
Op.getOperand(2), Op.getOperand(3));
1380+
VTs = DAG.getVTList(Op->getValueType(1), MVT::Other);
1381+
SDValue ReadVL = DAG.getNode(RISCVISD::READ_VL, DL, VTs, Load.getValue(2));
1382+
return DAG.getMergeValues({Load, ReadVL, Load.getValue(1)}, DL);
1383+
}
1384+
case Intrinsic::riscv_vleff_mask: {
1385+
SDLoc DL(Op);
1386+
SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Other, MVT::Glue);
1387+
SDValue Load = DAG.getNode(RISCVISD::VLEFF_MASK, DL, VTs, Op.getOperand(0),
1388+
Op.getOperand(2), Op.getOperand(3),
1389+
Op.getOperand(4), Op.getOperand(5));
1390+
VTs = DAG.getVTList(Op->getValueType(1), MVT::Other);
1391+
SDValue ReadVL = DAG.getNode(RISCVISD::READ_VL, DL, VTs, Load.getValue(2));
1392+
return DAG.getMergeValues({Load, ReadVL, Load.getValue(1)}, DL);
1393+
}
1394+
}
13711395
}
13721396

13731397
// Returns the opcode of the target-specific SDNode that implements the 32-bit
@@ -3815,6 +3839,9 @@ const char *RISCVTargetLowering::getTargetNodeName(unsigned Opcode) const {
38153839
NODE_NAME_CASE(SPLAT_VECTOR_I64)
38163840
NODE_NAME_CASE(READ_VLENB)
38173841
NODE_NAME_CASE(TRUNCATE_VECTOR)
3842+
NODE_NAME_CASE(VLEFF)
3843+
NODE_NAME_CASE(VLEFF_MASK)
3844+
NODE_NAME_CASE(READ_VL)
38183845
}
38193846
// clang-format on
38203847
return nullptr;

‎llvm/lib/Target/RISCV/RISCVISelLowering.h

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,11 @@ enum NodeType : unsigned {
9595
READ_VLENB,
9696
// Truncates a RVV integer vector by one power-of-two.
9797
TRUNCATE_VECTOR,
98+
// Unit-stride fault-only-first load
99+
VLEFF,
100+
VLEFF_MASK,
101+
// read vl CSR
102+
READ_VL,
98103
};
99104
} // namespace RISCVISD
100105

‎llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td

Lines changed: 41 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,23 @@ def riscv_vmv_x_s : SDNode<"RISCVISD::VMV_X_S",
2020
def riscv_read_vlenb : SDNode<"RISCVISD::READ_VLENB",
2121
SDTypeProfile<1, 0, [SDTCisVT<0, XLenVT>]>>;
2222

23+
def riscv_vleff : SDNode<"RISCVISD::VLEFF",
24+
SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisPtrTy<1>,
25+
SDTCisVT<2, XLenVT>]>,
26+
[SDNPHasChain, SDNPOutGlue, SDNPMayLoad,
27+
SDNPSideEffect]>;
28+
def riscv_vleff_mask : SDNode<"RISCVISD::VLEFF_MASK",
29+
SDTypeProfile<1, 4, [SDTCisVec<0>,
30+
SDTCisSameAs<0, 1>,
31+
SDTCisPtrTy<2>,
32+
SDTCVecEltisVT<3, i1>,
33+
SDTCisVT<4, XLenVT>]>,
34+
[SDNPHasChain, SDNPOutGlue, SDNPMayLoad,
35+
SDNPSideEffect]>;
36+
def riscv_read_vl : SDNode<"RISCVISD::READ_VL",
37+
SDTypeProfile<1, 0, [SDTCisVT<0, XLenVT>]>,
38+
[SDNPInGlue]>;
39+
2340
// X0 has special meaning for vsetvl/vsetvli.
2441
// rd | rs1 | AVL value | Effect on vl
2542
//--------------------------------------------------------------
@@ -1903,6 +1920,23 @@ multiclass VPatUSLoad<string intrinsic,
19031920
$rs1, (mask_type V0), (NoX0 GPR:$vl), sew)>;
19041921
}
19051922

1923+
multiclass VPatUSLoadFF<string inst,
1924+
LLVMType type,
1925+
LLVMType mask_type,
1926+
int sew,
1927+
LMULInfo vlmul,
1928+
VReg reg_class>
1929+
{
1930+
defvar Pseudo = !cast<Instruction>(inst#"_V_"#vlmul.MX);
1931+
def : Pat<(type (riscv_vleff GPR:$rs1, GPR:$vl)),
1932+
(Pseudo $rs1, (NoX0 GPR:$vl), sew)>;
1933+
defvar PseudoMask = !cast<Instruction>(inst#"_V_"#vlmul.MX#"_MASK");
1934+
def : Pat<(type (riscv_vleff_mask (type GetVRegNoV0<reg_class>.R:$merge),
1935+
GPR:$rs1, (mask_type V0), GPR:$vl)),
1936+
(PseudoMask $merge,
1937+
$rs1, (mask_type V0), (NoX0 GPR:$vl), sew)>;
1938+
}
1939+
19061940
multiclass VPatSLoad<string intrinsic,
19071941
string inst,
19081942
LLVMType type,
@@ -2817,6 +2851,11 @@ let hasSideEffects = 0, mayLoad = 0, mayStore = 0, isCodeGenOnly = 1 in {
28172851
[(set GPR:$rd, (riscv_read_vlenb))]>;
28182852
}
28192853

2854+
let hasSideEffects = 0, mayLoad = 0, mayStore = 0, isCodeGenOnly = 1,
2855+
Uses = [VL] in
2856+
def PseudoReadVL : Pseudo<(outs GPR:$rd), (ins),
2857+
[(set GPR:$rd, (riscv_read_vl))]>;
2858+
28202859
//===----------------------------------------------------------------------===//
28212860
// 6. Configuration-Setting Instructions
28222861
//===----------------------------------------------------------------------===//
@@ -3388,9 +3427,8 @@ foreach vti = AllVectors in
33883427
defm : VPatUSLoad<"int_riscv_vle",
33893428
"PseudoVLE" # vti.SEW,
33903429
vti.Vector, vti.Mask, vti.SEW, vti.LMul, vti.RegClass>;
3391-
defm : VPatUSLoad<"int_riscv_vleff",
3392-
"PseudoVLE" # vti.SEW # "FF",
3393-
vti.Vector, vti.Mask, vti.SEW, vti.LMul, vti.RegClass>;
3430+
defm : VPatUSLoadFF<"PseudoVLE" # vti.SEW # "FF",
3431+
vti.Vector, vti.Mask, vti.SEW, vti.LMul, vti.RegClass>;
33943432
defm : VPatUSStore<"int_riscv_vse",
33953433
"PseudoVSE" # vti.SEW,
33963434
vti.Vector, vti.Mask, vti.SEW, vti.LMul, vti.RegClass>;

‎llvm/lib/Target/RISCV/RISCVMCInstLower.cpp

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -219,4 +219,11 @@ void llvm::LowerRISCVMachineInstrToMCInst(const MachineInstr *MI, MCInst &OutMI,
219219
return;
220220
}
221221

222+
if (OutMI.getOpcode() == RISCV::PseudoReadVL) {
223+
OutMI.setOpcode(RISCV::CSRRS);
224+
OutMI.addOperand(MCOperand::createImm(
225+
RISCVSysReg::lookupSysRegByName("VL")->Encoding));
226+
OutMI.addOperand(MCOperand::createReg(RISCV::X0));
227+
return;
228+
}
222229
}

0 commit comments

Comments
 (0)