Skip to content

Commit 37b969b

Browse files
simeonkrsimeonkr
committed
[LV][RISCV] Introduce llvm.vp.minimum/maximum intrinsics
Although there are predicated versions of minnum/maxnum, the ones for minimum/maximum are currently missing. This patch introduces these intrinsics and implements their lowering to RISC-V.
1 parent 8bc7c0a commit 37b969b

File tree

11 files changed

+3457
-8
lines changed

11 files changed

+3457
-8
lines changed

‎llvm/docs/LangRef.rst

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20392,6 +20392,106 @@ Examples:
2039220392
%also.r = select <4 x i1> %mask, <4 x float> %t, <4 x float> poison
2039320393

2039420394

20395+
.. _int_vp_minimum:
20396+
20397+
'``llvm.vp.minimum.*``' Intrinsics
20398+
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
20399+
20400+
Syntax:
20401+
"""""""
20402+
This is an overloaded intrinsic.
20403+
20404+
::
20405+
20406+
declare <16 x float> @llvm.vp.minimum.v16f32 (<16 x float> <left_op>, <16 x float> <right_op>, <16 x i1> <mask>, i32 <vector_length>)
20407+
declare <vscale x 4 x float> @llvm.vp.minimum.nxv4f32 (<vscale x 4 x float> <left_op>, <vscale x 4 x float> <right_op>, <vscale x 4 x i1> <mask>, i32 <vector_length>)
20408+
declare <256 x double> @llvm.vp.minimum.v256f64 (<256 x double> <left_op>, <256 x double> <right_op>, <256 x i1> <mask>, i32 <vector_length>)
20409+
20410+
Overview:
20411+
"""""""""
20412+
20413+
Predicated floating-point minimum of two vectors of floating-point values,
20414+
propagating NaNs and treating -0.0 as less than +0.0.
20415+
20416+
Arguments:
20417+
""""""""""
20418+
20419+
The first two operands and the result have the same vector of floating-point type. The
20420+
third operand is the vector mask and has the same number of elements as the
20421+
result vector type. The fourth operand is the explicit vector length of the
20422+
operation.
20423+
20424+
Semantics:
20425+
""""""""""
20426+
20427+
The '``llvm.vp.minimum``' intrinsic performs floating-point minimum (:ref:`minimum <i_minimum>`)
20428+
of the first and second vector operand on each enabled lane, the result being
20429+
NaN if either operand is a NaN. -0.0 is considered to be less than +0.0 for this
20430+
intrinsic. The result on disabled lanes is a :ref:`poison value <poisonvalues>`.
20431+
The operation is performed in the default floating-point environment.
20432+
20433+
Examples:
20434+
"""""""""
20435+
20436+
.. code-block:: llvm
20437+
20438+
%r = call <4 x float> @llvm.vp.minimum.v4f32(<4 x float> %a, <4 x float> %b, <4 x i1> %mask, i32 %evl)
20439+
;; For all lanes below %evl, %r is lane-wise equivalent to %also.r
20440+
20441+
%t = call <4 x float> @llvm.minimum.v4f32(<4 x float> %a, <4 x float> %b)
20442+
%also.r = select <4 x i1> %mask, <4 x float> %t, <4 x float> poison
20443+
20444+
20445+
.. _int_vp_maximum:
20446+
20447+
'``llvm.vp.maximum.*``' Intrinsics
20448+
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
20449+
20450+
Syntax:
20451+
"""""""
20452+
This is an overloaded intrinsic.
20453+
20454+
::
20455+
20456+
declare <16 x float> @llvm.vp.maximum.v16f32 (<16 x float> <left_op>, <16 x float> <right_op>, <16 x i1> <mask>, i32 <vector_length>)
20457+
declare <vscale x 4 x float> @llvm.vp.maximum.nxv4f32 (<vscale x 4 x float> <left_op>, <vscale x 4 x float> <right_op>, <vscale x 4 x i1> <mask>, i32 <vector_length>)
20458+
declare <256 x double> @llvm.vp.maximum.v256f64 (<256 x double> <left_op>, <256 x double> <right_op>, <256 x i1> <mask>, i32 <vector_length>)
20459+
20460+
Overview:
20461+
"""""""""
20462+
20463+
Predicated floating-point maximum of two vectors of floating-point values,
20464+
propagating NaNs and treating -0.0 as less than +0.0.
20465+
20466+
Arguments:
20467+
""""""""""
20468+
20469+
The first two operands and the result have the same vector of floating-point type. The
20470+
third operand is the vector mask and has the same number of elements as the
20471+
result vector type. The fourth operand is the explicit vector length of the
20472+
operation.
20473+
20474+
Semantics:
20475+
""""""""""
20476+
20477+
The '``llvm.vp.maximum``' intrinsic performs floating-point maximum (:ref:`maximum <i_maximum>`)
20478+
of the first and second vector operand on each enabled lane, the result being
20479+
NaN if either operand is a NaN. -0.0 is considered to be less than +0.0 for this
20480+
intrinsic. The result on disabled lanes is a :ref:`poison value <poisonvalues>`.
20481+
The operation is performed in the default floating-point environment.
20482+
20483+
Examples:
20484+
"""""""""
20485+
20486+
.. code-block:: llvm
20487+
20488+
%r = call <4 x float> @llvm.vp.maximum.v4f32(<4 x float> %a, <4 x float> %b, <4 x i1> %mask, i32 %evl)
20489+
;; For all lanes below %evl, %r is lane-wise equivalent to %also.r
20490+
20491+
%t = call <4 x float> @llvm.maximum.v4f32(<4 x float> %a, <4 x float> %b, <4 x i1> %mask, i32 %evl)
20492+
%also.r = select <4 x i1> %mask, <4 x float> %t, <4 x float> poison
20493+
20494+
2039520495
.. _int_vp_fadd:
2039620496

2039720497
'``llvm.vp.fadd.*``' Intrinsics

‎llvm/include/llvm/IR/Intrinsics.td

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1991,6 +1991,16 @@ let IntrProperties = [IntrNoMem, IntrNoSync, IntrWillReturn] in {
19911991
LLVMMatchType<0>,
19921992
LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
19931993
llvm_i32_ty]>;
1994+
def int_vp_minimum : DefaultAttrsIntrinsic<[ llvm_anyvector_ty ],
1995+
[ LLVMMatchType<0>,
1996+
LLVMMatchType<0>,
1997+
LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1998+
llvm_i32_ty]>;
1999+
def int_vp_maximum : DefaultAttrsIntrinsic<[ llvm_anyvector_ty ],
2000+
[ LLVMMatchType<0>,
2001+
LLVMMatchType<0>,
2002+
LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
2003+
llvm_i32_ty]>;
19942004
def int_vp_copysign : DefaultAttrsIntrinsic<[ llvm_anyvector_ty ],
19952005
[ LLVMMatchType<0>,
19962006
LLVMMatchType<0>,

‎llvm/include/llvm/IR/VPIntrinsics.def

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -367,20 +367,34 @@ VP_PROPERTY_FUNCTIONAL_SDOPC(FCOPYSIGN)
367367
VP_PROPERTY_FUNCTIONAL_INTRINSIC(copysign)
368368
END_REGISTER_VP(vp_copysign, VP_FCOPYSIGN)
369369

370-
// llvm.vp.minnum(x, y, mask,vlen)
370+
// llvm.vp.minnum(x,y,mask,vlen)
371371
BEGIN_REGISTER_VP(vp_minnum, 2, 3, VP_FMINNUM, -1)
372372
VP_PROPERTY_BINARYOP
373373
VP_PROPERTY_FUNCTIONAL_SDOPC(FMINNUM)
374374
VP_PROPERTY_FUNCTIONAL_INTRINSIC(minnum)
375375
END_REGISTER_VP(vp_minnum, VP_FMINNUM)
376376

377-
// llvm.vp.maxnum(x, y, mask,vlen)
377+
// llvm.vp.maxnum(x,y,mask,vlen)
378378
BEGIN_REGISTER_VP(vp_maxnum, 2, 3, VP_FMAXNUM, -1)
379379
VP_PROPERTY_BINARYOP
380380
VP_PROPERTY_FUNCTIONAL_SDOPC(FMAXNUM)
381381
VP_PROPERTY_FUNCTIONAL_INTRINSIC(maxnum)
382382
END_REGISTER_VP(vp_maxnum, VP_FMAXNUM)
383383

384+
// llvm.vp.minimum(x,y,mask,vlen)
385+
BEGIN_REGISTER_VP(vp_minimum, 2, 3, VP_FMINIMUM, -1)
386+
VP_PROPERTY_BINARYOP
387+
VP_PROPERTY_FUNCTIONAL_SDOPC(FMINIMUM)
388+
VP_PROPERTY_FUNCTIONAL_INTRINSIC(minimum)
389+
END_REGISTER_VP(vp_minimum, VP_FMINIMUM)
390+
391+
// llvm.vp.maximum(x,y,mask,vlen)
392+
BEGIN_REGISTER_VP(vp_maximum, 2, 3, VP_FMAXIMUM, -1)
393+
VP_PROPERTY_BINARYOP
394+
VP_PROPERTY_FUNCTIONAL_SDOPC(FMAXIMUM)
395+
VP_PROPERTY_FUNCTIONAL_INTRINSIC(maximum)
396+
END_REGISTER_VP(vp_maximum, VP_FMAXIMUM)
397+
384398
// llvm.vp.ceil(x,mask,vlen)
385399
BEGIN_REGISTER_VP(vp_ceil, 1, 2, VP_FCEIL, -1)
386400
VP_PROPERTY_FUNCTIONAL_INTRINSIC(ceil)

‎llvm/lib/CodeGen/ExpandVectorPredication.cpp

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -729,6 +729,8 @@ Value *CachingVPExpander::expandPredication(VPIntrinsic &VPI) {
729729
case Intrinsic::vp_sqrt:
730730
case Intrinsic::vp_maxnum:
731731
case Intrinsic::vp_minnum:
732+
case Intrinsic::vp_maximum:
733+
case Intrinsic::vp_minimum:
732734
return expandPredicationToFPCall(Builder, VPI,
733735
VPI.getFunctionalIntrinsicID().value());
734736
case Intrinsic::vp_load:

‎llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1143,7 +1143,9 @@ void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
11431143
case ISD::FMINNUM: case ISD::VP_FMINNUM:
11441144
case ISD::FMAXNUM: case ISD::VP_FMAXNUM:
11451145
case ISD::FMINIMUM:
1146+
case ISD::VP_FMINIMUM:
11461147
case ISD::FMAXIMUM:
1148+
case ISD::VP_FMAXIMUM:
11471149
case ISD::SDIV: case ISD::VP_SDIV:
11481150
case ISD::UDIV: case ISD::VP_UDIV:
11491151
case ISD::FDIV: case ISD::VP_FDIV:
@@ -4131,7 +4133,9 @@ void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
41314133
case ISD::FMINNUM: case ISD::VP_FMINNUM:
41324134
case ISD::FMAXNUM: case ISD::VP_FMAXNUM:
41334135
case ISD::FMINIMUM:
4136+
case ISD::VP_FMINIMUM:
41344137
case ISD::FMAXIMUM:
4138+
case ISD::VP_FMAXIMUM:
41354139
case ISD::SMIN: case ISD::VP_SMIN:
41364140
case ISD::SMAX: case ISD::VP_SMAX:
41374141
case ISD::UMIN: case ISD::VP_UMIN:

‎llvm/lib/Target/RISCV/RISCVISelLowering.cpp

Lines changed: 24 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -687,7 +687,8 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
687687
ISD::VP_FCEIL, ISD::VP_FFLOOR, ISD::VP_FROUND,
688688
ISD::VP_FROUNDEVEN, ISD::VP_FCOPYSIGN, ISD::VP_FROUNDTOZERO,
689689
ISD::VP_FRINT, ISD::VP_FNEARBYINT, ISD::VP_IS_FPCLASS,
690-
ISD::EXPERIMENTAL_VP_REVERSE, ISD::EXPERIMENTAL_VP_SPLICE};
690+
ISD::VP_FMINIMUM, ISD::VP_FMAXIMUM, ISD::EXPERIMENTAL_VP_REVERSE,
691+
ISD::EXPERIMENTAL_VP_SPLICE};
691692

692693
static const unsigned IntegerVecReduceOps[] = {
693694
ISD::VECREDUCE_ADD, ISD::VECREDUCE_AND, ISD::VECREDUCE_OR,
@@ -927,7 +928,8 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
927928
ISD::VP_FMINNUM, ISD::VP_FMAXNUM, ISD::VP_FCEIL,
928929
ISD::VP_FFLOOR, ISD::VP_FROUND, ISD::VP_FROUNDEVEN,
929930
ISD::VP_FCOPYSIGN, ISD::VP_FROUNDTOZERO, ISD::VP_FRINT,
930-
ISD::VP_FNEARBYINT, ISD::VP_SETCC};
931+
ISD::VP_FNEARBYINT, ISD::VP_SETCC, ISD::VP_FMINIMUM,
932+
ISD::VP_FMAXIMUM};
931933

932934
// Sets common operation actions on RVV floating-point vector types.
933935
const auto SetCommonVFPActions = [&](MVT VT) {
@@ -5401,7 +5403,16 @@ static SDValue lowerFMAXIMUM_FMINIMUM(SDValue Op, SelectionDAG &DAG,
54015403
Y = convertToScalableVector(ContainerVT, Y, DAG, Subtarget);
54025404
}
54035405

5404-
auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
5406+
SDValue Mask, VL;
5407+
if (Op->isVPOpcode()) {
5408+
Mask = Op.getOperand(2);
5409+
if (VT.isFixedLengthVector())
5410+
Mask = convertToScalableVector(getMaskTypeFor(ContainerVT), Mask, DAG,
5411+
Subtarget);
5412+
VL = Op.getOperand(3);
5413+
} else {
5414+
std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
5415+
}
54055416

54065417
SDValue NewY = Y;
54075418
if (!XIsNeverNan) {
@@ -5422,7 +5433,9 @@ static SDValue lowerFMAXIMUM_FMINIMUM(SDValue Op, SelectionDAG &DAG,
54225433
}
54235434

54245435
unsigned Opc =
5425-
Op.getOpcode() == ISD::FMAXIMUM ? RISCVISD::VFMAX_VL : RISCVISD::VFMIN_VL;
5436+
Op.getOpcode() == ISD::FMAXIMUM || Op->getOpcode() == ISD::VP_FMAXIMUM
5437+
? RISCVISD::VFMAX_VL
5438+
: RISCVISD::VFMIN_VL;
54265439
SDValue Res = DAG.getNode(Opc, DL, ContainerVT, NewX, NewY,
54275440
DAG.getUNDEF(ContainerVT), Mask, VL);
54285441
if (VT.isFixedLengthVector())
@@ -6651,6 +6664,13 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
66516664
!Subtarget.hasVInstructionsF16()))
66526665
return SplitVPOp(Op, DAG);
66536666
return lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(Op, DAG, Subtarget);
6667+
case ISD::VP_FMAXIMUM:
6668+
case ISD::VP_FMINIMUM:
6669+
if (Op.getValueType() == MVT::nxv32f16 &&
6670+
(Subtarget.hasVInstructionsF16Minimal() &&
6671+
!Subtarget.hasVInstructionsF16()))
6672+
return SplitVPOp(Op, DAG);
6673+
return lowerFMAXIMUM_FMINIMUM(Op, DAG, Subtarget);
66546674
case ISD::EXPERIMENTAL_VP_SPLICE:
66556675
return lowerVPSpliceExperimental(Op, DAG);
66566676
case ISD::EXPERIMENTAL_VP_REVERSE:

0 commit comments

Comments
 (0)