Skip to content

Commit 6107822

Browse files
author
Yeting Kuo
committed
[VP][RISCV] Introduce vp.splat and RISC-V.
This patch introduces a vp intrinsic for splat. It's helpful for IR-level passes to create a splat with specific vector length.
1 parent cb3bc5b commit 6107822

File tree

12 files changed

+1052
-5
lines changed

12 files changed

+1052
-5
lines changed

‎llvm/docs/LangRef.rst

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22841,6 +22841,51 @@ Examples:
2284122841
llvm.experimental.vp.splice(<A,B,C,D>, <E,F,G,H>, -2, 3, 2); ==> <B, C, poison, poison> trailing elements
2284222842

2284322843

22844+
.. _int_experimental_vp_splat:
22845+
22846+
22847+
'``llvm.experimental.vp.splat``' Intrinsic
22848+
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
22849+
22850+
Syntax:
22851+
"""""""
22852+
This is an overloaded intrinsic.
22853+
22854+
::
22855+
22856+
declare <2 x double> @llvm.experimental.vp.splat.v2f64(<2 x double> %vec, <2 x i1> %mask, i32 %evl)
22857+
declare <vscale x 4 x i32> @llvm.experimental.vp.splat.nxv4i32(<vscale x 4 x i32> %vec, <vscale x 4 x i1> %mask, i32 %evl)
22858+
22859+
Overview:
22860+
"""""""""
22861+
22862+
The '``llvm.experimental.vp.splat.*``' intrinsic is to create a prdicated splat
22863+
with specific effective vector length.
22864+
22865+
Arguments:
22866+
""""""""""
22867+
22868+
The result is a vector and it is a splat of the second scalar operand. The
22869+
second argument ``mask`` is a vector mask and has the same number of elements as
22870+
the result. The third argument is the explicit vector length of the operation.
22871+
22872+
Semantics:
22873+
""""""""""
22874+
22875+
This intrinsic splats a vector with ``evl`` elements of a scalar operand.
22876+
The lanes in the result vector disabled by ``mask`` are ``poison``. The
22877+
elements past ``evl`` are poison.
22878+
22879+
Examples:
22880+
"""""""""
22881+
22882+
.. code-block:: llvm
22883+
22884+
%r = call <4 x float> @llvm.vp.splat.v4f32(float %a, <4 x i1> %mask, i32 %evl)
22885+
;; For all lanes below %evl, %r is lane-wise equivalent to %also.r
22886+
%also.r = select <4 x i1> %mask, <4 x float> splat(float %a), <4 x float> poison
22887+
22888+
2284422889
.. _int_experimental_vp_reverse:
2284522890

2284622891

‎llvm/include/llvm/IR/Intrinsics.td

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2319,6 +2319,13 @@ def int_experimental_vp_reverse:
23192319
llvm_i32_ty],
23202320
[IntrNoMem]>;
23212321

2322+
def int_experimental_vp_splat:
2323+
DefaultAttrsIntrinsic<[llvm_anyvector_ty],
2324+
[LLVMVectorElementType<0>,
2325+
LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
2326+
llvm_i32_ty],
2327+
[IntrNoMem]>;
2328+
23222329
def int_vp_is_fpclass:
23232330
DefaultAttrsIntrinsic<[ LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>],
23242331
[ llvm_anyvector_ty,

‎llvm/include/llvm/IR/VPIntrinsics.def

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -777,6 +777,13 @@ END_REGISTER_VP(experimental_vp_reverse, EXPERIMENTAL_VP_REVERSE)
777777

778778
///// } Shuffles
779779

780+
// llvm.vp.splat(ptr,val,mask,vlen)
781+
BEGIN_REGISTER_VP_INTRINSIC(experimental_vp_splat, 1, 2)
782+
BEGIN_REGISTER_VP_SDNODE(EXPERIMENTAL_VP_SPLAT, -1, experimental_vp_splat, 1, 2)
783+
VP_PROPERTY_NO_FUNCTIONAL
784+
HELPER_MAP_VPID_TO_VPSD(experimental_vp_splat, EXPERIMENTAL_VP_SPLAT)
785+
END_REGISTER_VP(experimental_vp_splat, EXPERIMENTAL_VP_SPLAT)
786+
780787
#undef BEGIN_REGISTER_VP
781788
#undef BEGIN_REGISTER_VP_INTRINSIC
782789
#undef BEGIN_REGISTER_VP_SDNODE

‎llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,7 @@ void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
137137
break;
138138
case ISD::SPLAT_VECTOR:
139139
case ISD::SCALAR_TO_VECTOR:
140+
case ISD::EXPERIMENTAL_VP_SPLAT:
140141
Res = PromoteIntRes_ScalarOp(N);
141142
break;
142143
case ISD::STEP_VECTOR: Res = PromoteIntRes_STEP_VECTOR(N); break;
@@ -1916,6 +1917,7 @@ bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) {
19161917
break;
19171918
case ISD::SPLAT_VECTOR:
19181919
case ISD::SCALAR_TO_VECTOR:
1920+
case ISD::EXPERIMENTAL_VP_SPLAT:
19191921
Res = PromoteIntOp_ScalarOp(N);
19201922
break;
19211923
case ISD::VSELECT:
@@ -2211,10 +2213,14 @@ SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N,
22112213
}
22122214

22132215
SDValue DAGTypeLegalizer::PromoteIntOp_ScalarOp(SDNode *N) {
2216+
SDValue Op = GetPromotedInteger(N->getOperand(0));
2217+
if (N->getOpcode() == ISD::EXPERIMENTAL_VP_SPLAT)
2218+
return DAG.getNode(ISD::EXPERIMENTAL_VP_SPLAT, SDLoc(N), N->getValueType(0),
2219+
Op, N->getOperand(1), N->getOperand(2));
2220+
22142221
// Integer SPLAT_VECTOR/SCALAR_TO_VECTOR operands are implicitly truncated,
22152222
// so just promote the operand in place.
2216-
return SDValue(DAG.UpdateNodeOperands(N,
2217-
GetPromotedInteger(N->getOperand(0))), 0);
2223+
return SDValue(DAG.UpdateNodeOperands(N, Op), 0);
22182224
}
22192225

22202226
SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) {
@@ -5231,6 +5237,7 @@ bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) {
52315237
case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break;
52325238
case ISD::INSERT_VECTOR_ELT: Res = ExpandOp_INSERT_VECTOR_ELT(N); break;
52335239
case ISD::SCALAR_TO_VECTOR: Res = ExpandOp_SCALAR_TO_VECTOR(N); break;
5240+
case ISD::EXPERIMENTAL_VP_SPLAT:
52345241
case ISD::SPLAT_VECTOR: Res = ExpandIntOp_SPLAT_VECTOR(N); break;
52355242
case ISD::SELECT_CC: Res = ExpandIntOp_SELECT_CC(N); break;
52365243
case ISD::SETCC: Res = ExpandIntOp_SETCC(N); break;
@@ -5859,7 +5866,11 @@ SDValue DAGTypeLegalizer::PromoteIntRes_ScalarOp(SDNode *N) {
58595866
EVT NOutElemVT = NOutVT.getVectorElementType();
58605867

58615868
SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutElemVT, N->getOperand(0));
5862-
5869+
if (N->isVPOpcode()) {
5870+
SDValue Mask = N->getOperand(1);
5871+
SDValue VL = N->getOperand(2);
5872+
return DAG.getNode(N->getOpcode(), dl, NOutVT, Op, Mask, VL);
5873+
}
58635874
return DAG.getNode(N->getOpcode(), dl, NOutVT, Op);
58645875
}
58655876

‎llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -915,6 +915,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
915915
void SplitVecRes_Gather(MemSDNode *VPGT, SDValue &Lo, SDValue &Hi,
916916
bool SplitSETCC = false);
917917
void SplitVecRes_ScalarOp(SDNode *N, SDValue &Lo, SDValue &Hi);
918+
void SplitVecRes_VP_SPLAT(SDNode *N, SDValue &Lo, SDValue &Hi);
918919
void SplitVecRes_STEP_VECTOR(SDNode *N, SDValue &Lo, SDValue &Hi);
919920
void SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi);
920921
void SplitVecRes_VECTOR_REVERSE(SDNode *N, SDValue &Lo, SDValue &Hi);
@@ -1052,6 +1053,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
10521053
SDValue WidenVecOp_MGATHER(SDNode* N, unsigned OpNo);
10531054
SDValue WidenVecOp_MSCATTER(SDNode* N, unsigned OpNo);
10541055
SDValue WidenVecOp_VP_SCATTER(SDNode* N, unsigned OpNo);
1056+
SDValue WidenVecOp_VP_SPLAT(SDNode *N, unsigned OpNo);
10551057
SDValue WidenVecOp_SETCC(SDNode* N);
10561058
SDValue WidenVecOp_STRICT_FSETCC(SDNode* N);
10571059
SDValue WidenVecOp_VSELECT(SDNode *N);

‎llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1076,6 +1076,7 @@ void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
10761076
case ISD::FCOPYSIGN: SplitVecRes_FPOp_MultiType(N, Lo, Hi); break;
10771077
case ISD::IS_FPCLASS: SplitVecRes_IS_FPCLASS(N, Lo, Hi); break;
10781078
case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break;
1079+
case ISD::EXPERIMENTAL_VP_SPLAT: SplitVecRes_VP_SPLAT(N, Lo, Hi); break;
10791080
case ISD::SPLAT_VECTOR:
10801081
case ISD::SCALAR_TO_VECTOR:
10811082
SplitVecRes_ScalarOp(N, Lo, Hi);
@@ -1992,6 +1993,16 @@ void DAGTypeLegalizer::SplitVecRes_ScalarOp(SDNode *N, SDValue &Lo,
19921993
}
19931994
}
19941995

1996+
void DAGTypeLegalizer::SplitVecRes_VP_SPLAT(SDNode *N, SDValue &Lo,
1997+
SDValue &Hi) {
1998+
SDLoc dl(N);
1999+
auto [LoVT, HiVT] = DAG.GetSplitDestVTs(N->getValueType(0));
2000+
auto [MaskLo, MaskHi] = SplitMask(N->getOperand(1));
2001+
auto [EVLLo, EVLHi] = DAG.SplitEVL(N->getOperand(2), N->getValueType(0), dl);
2002+
Lo = DAG.getNode(N->getOpcode(), dl, LoVT, N->getOperand(0), MaskLo, EVLLo);
2003+
Hi = DAG.getNode(N->getOpcode(), dl, HiVT, N->getOperand(0), MaskHi, EVLHi);
2004+
}
2005+
19952006
void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo,
19962007
SDValue &Hi) {
19972008
assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!");
@@ -4284,6 +4295,7 @@ void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
42844295
case ISD::STEP_VECTOR:
42854296
case ISD::SPLAT_VECTOR:
42864297
case ISD::SCALAR_TO_VECTOR:
4298+
case ISD::EXPERIMENTAL_VP_SPLAT:
42874299
Res = WidenVecRes_ScalarOp(N);
42884300
break;
42894301
case ISD::SIGN_EXTEND_INREG: Res = WidenVecRes_InregOp(N); break;
@@ -5814,6 +5826,9 @@ SDValue DAGTypeLegalizer::WidenVecRes_VP_GATHER(VPGatherSDNode *N) {
58145826

58155827
SDValue DAGTypeLegalizer::WidenVecRes_ScalarOp(SDNode *N) {
58165828
EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
5829+
if (N->isVPOpcode())
5830+
return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, N->getOperand(0),
5831+
N->getOperand(1), N->getOperand(2));
58175832
return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, N->getOperand(0));
58185833
}
58195834

@@ -6353,6 +6368,10 @@ bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) {
63536368
Res = WidenVecOp_FP_TO_XINT_SAT(N);
63546369
break;
63556370

6371+
case ISD::EXPERIMENTAL_VP_SPLAT:
6372+
Res = WidenVecOp_VP_SPLAT(N, OpNo);
6373+
break;
6374+
63566375
case ISD::VECREDUCE_FADD:
63576376
case ISD::VECREDUCE_FMUL:
63586377
case ISD::VECREDUCE_ADD:
@@ -6813,6 +6832,13 @@ SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) {
68136832
report_fatal_error("Unable to widen vector store");
68146833
}
68156834

6835+
SDValue DAGTypeLegalizer::WidenVecOp_VP_SPLAT(SDNode *N, unsigned OpNo) {
6836+
assert(OpNo == 1 && "Can widen only mask operand of vp_splat");
6837+
return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0),
6838+
N->getOperand(0), GetWidenedVector(N->getOperand(1)),
6839+
N->getOperand(2));
6840+
}
6841+
68166842
SDValue DAGTypeLegalizer::WidenVecOp_VP_STORE(SDNode *N, unsigned OpNo) {
68176843
assert((OpNo == 1 || OpNo == 3) &&
68186844
"Can widen only data or mask operand of vp_store");

‎llvm/lib/IR/IntrinsicInst.cpp

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -699,6 +699,9 @@ Function *VPIntrinsic::getDeclarationForParams(Module *M, Intrinsic::ID VPID,
699699
VPFunc = Intrinsic::getDeclaration(
700700
M, VPID, {Params[0]->getType(), Params[1]->getType()});
701701
break;
702+
case Intrinsic::experimental_vp_splat:
703+
VPFunc = Intrinsic::getDeclaration(M, VPID, ReturnType);
704+
break;
702705
}
703706
assert(VPFunc && "Could not declare VP intrinsic");
704707
return VPFunc;

‎llvm/lib/Target/RISCV/RISCVISelLowering.cpp

Lines changed: 29 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -705,7 +705,8 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
705705
ISD::VP_SMAX, ISD::VP_UMIN, ISD::VP_UMAX,
706706
ISD::VP_ABS, ISD::EXPERIMENTAL_VP_REVERSE, ISD::EXPERIMENTAL_VP_SPLICE,
707707
ISD::VP_SADDSAT, ISD::VP_UADDSAT, ISD::VP_SSUBSAT,
708-
ISD::VP_USUBSAT, ISD::VP_CTTZ_ELTS, ISD::VP_CTTZ_ELTS_ZERO_UNDEF};
708+
ISD::VP_USUBSAT, ISD::VP_CTTZ_ELTS, ISD::VP_CTTZ_ELTS_ZERO_UNDEF,
709+
ISD::EXPERIMENTAL_VP_SPLAT};
709710

710711
static const unsigned FloatingPointVPOps[] = {
711712
ISD::VP_FADD, ISD::VP_FSUB, ISD::VP_FMUL,
@@ -721,7 +722,7 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
721722
ISD::VP_FMINIMUM, ISD::VP_FMAXIMUM, ISD::VP_LRINT,
722723
ISD::VP_LLRINT, ISD::EXPERIMENTAL_VP_REVERSE,
723724
ISD::EXPERIMENTAL_VP_SPLICE, ISD::VP_REDUCE_FMINIMUM,
724-
ISD::VP_REDUCE_FMAXIMUM};
725+
ISD::VP_REDUCE_FMAXIMUM, ISD::EXPERIMENTAL_VP_SPLAT};
725726

726727
static const unsigned IntegerVecReduceOps[] = {
727728
ISD::VECREDUCE_ADD, ISD::VECREDUCE_AND, ISD::VECREDUCE_OR,
@@ -7268,6 +7269,8 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
72687269
return lowerVPSpliceExperimental(Op, DAG);
72697270
case ISD::EXPERIMENTAL_VP_REVERSE:
72707271
return lowerVPReverseExperimental(Op, DAG);
7272+
case ISD::EXPERIMENTAL_VP_SPLAT:
7273+
return lowerVPSplatExperimental(Op, DAG);
72717274
case ISD::CLEAR_CACHE: {
72727275
assert(getTargetMachine().getTargetTriple().isOSLinux() &&
72737276
"llvm.clear_cache only needs custom lower on Linux targets");
@@ -11630,6 +11633,30 @@ RISCVTargetLowering::lowerVPSpliceExperimental(SDValue Op,
1163011633
return convertFromScalableVector(VT, Result, DAG, Subtarget);
1163111634
}
1163211635

11636+
SDValue
11637+
RISCVTargetLowering::lowerVPSplatExperimental(SDValue Op,
11638+
SelectionDAG &DAG) const {
11639+
SDLoc DL(Op);
11640+
SDValue Val = Op.getOperand(0);
11641+
SDValue Mask = Op.getOperand(1);
11642+
SDValue VL = Op.getOperand(2);
11643+
MVT VT = Op.getSimpleValueType();
11644+
11645+
MVT ContainerVT = VT;
11646+
if (VT.isFixedLengthVector()) {
11647+
ContainerVT = getContainerForFixedLengthVector(VT);
11648+
MVT MaskVT = getMaskTypeFor(ContainerVT);
11649+
Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
11650+
}
11651+
11652+
SDValue Result = lowerScalarSplat(SDValue(), Val, VL, ContainerVT, DL,
11653+
DAG, Subtarget);
11654+
11655+
if (!VT.isFixedLengthVector())
11656+
return Result;
11657+
return convertFromScalableVector(VT, Result, DAG, Subtarget);
11658+
}
11659+
1163311660
SDValue
1163411661
RISCVTargetLowering::lowerVPReverseExperimental(SDValue Op,
1163511662
SelectionDAG &DAG) const {

‎llvm/lib/Target/RISCV/RISCVISelLowering.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -972,6 +972,7 @@ class RISCVTargetLowering : public TargetLowering {
972972
SDValue lowerLogicVPOp(SDValue Op, SelectionDAG &DAG) const;
973973
SDValue lowerVPExtMaskOp(SDValue Op, SelectionDAG &DAG) const;
974974
SDValue lowerVPSetCCMaskOp(SDValue Op, SelectionDAG &DAG) const;
975+
SDValue lowerVPSplatExperimental(SDValue Op, SelectionDAG &DAG) const;
975976
SDValue lowerVPSpliceExperimental(SDValue Op, SelectionDAG &DAG) const;
976977
SDValue lowerVPReverseExperimental(SDValue Op, SelectionDAG &DAG) const;
977978
SDValue lowerVPFPIntConvOp(SDValue Op, SelectionDAG &DAG) const;

0 commit comments

Comments
 (0)