Skip to content

Commit 58cfd56

Browse files
simeonkrsimeonkr
authored
[VP][RISCV] Introduce llvm.vp.minimum/maximum intrinsics (#74840)
Although there are predicated versions of minnum/maxnum, the ones for minimum/maximum are currently missing. This patch introduces these intrinsics and implements their lowering to RISC-V.
1 parent 029bfd6 commit 58cfd56

File tree

11 files changed

+4279
-8
lines changed

11 files changed

+4279
-8
lines changed

‎llvm/docs/LangRef.rst

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20394,6 +20394,106 @@ Examples:
2039420394
%also.r = select <4 x i1> %mask, <4 x float> %t, <4 x float> poison
2039520395

2039620396

20397+
.. _int_vp_minimum:
20398+
20399+
'``llvm.vp.minimum.*``' Intrinsics
20400+
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
20401+
20402+
Syntax:
20403+
"""""""
20404+
This is an overloaded intrinsic.
20405+
20406+
::
20407+
20408+
declare <16 x float> @llvm.vp.minimum.v16f32 (<16 x float> <left_op>, <16 x float> <right_op>, <16 x i1> <mask>, i32 <vector_length>)
20409+
declare <vscale x 4 x float> @llvm.vp.minimum.nxv4f32 (<vscale x 4 x float> <left_op>, <vscale x 4 x float> <right_op>, <vscale x 4 x i1> <mask>, i32 <vector_length>)
20410+
declare <256 x double> @llvm.vp.minimum.v256f64 (<256 x double> <left_op>, <256 x double> <right_op>, <256 x i1> <mask>, i32 <vector_length>)
20411+
20412+
Overview:
20413+
"""""""""
20414+
20415+
Predicated floating-point minimum of two vectors of floating-point values,
20416+
propagating NaNs and treating -0.0 as less than +0.0.
20417+
20418+
Arguments:
20419+
""""""""""
20420+
20421+
The first two operands and the result have the same vector of floating-point type. The
20422+
third operand is the vector mask and has the same number of elements as the
20423+
result vector type. The fourth operand is the explicit vector length of the
20424+
operation.
20425+
20426+
Semantics:
20427+
""""""""""
20428+
20429+
The '``llvm.vp.minimum``' intrinsic performs floating-point minimum (:ref:`minimum <i_minimum>`)
20430+
of the first and second vector operand on each enabled lane, the result being
20431+
NaN if either operand is a NaN. -0.0 is considered to be less than +0.0 for this
20432+
intrinsic. The result on disabled lanes is a :ref:`poison value <poisonvalues>`.
20433+
The operation is performed in the default floating-point environment.
20434+
20435+
Examples:
20436+
"""""""""
20437+
20438+
.. code-block:: llvm
20439+
20440+
%r = call <4 x float> @llvm.vp.minimum.v4f32(<4 x float> %a, <4 x float> %b, <4 x i1> %mask, i32 %evl)
20441+
;; For all lanes below %evl, %r is lane-wise equivalent to %also.r
20442+
20443+
%t = call <4 x float> @llvm.minimum.v4f32(<4 x float> %a, <4 x float> %b)
20444+
%also.r = select <4 x i1> %mask, <4 x float> %t, <4 x float> poison
20445+
20446+
20447+
.. _int_vp_maximum:
20448+
20449+
'``llvm.vp.maximum.*``' Intrinsics
20450+
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
20451+
20452+
Syntax:
20453+
"""""""
20454+
This is an overloaded intrinsic.
20455+
20456+
::
20457+
20458+
declare <16 x float> @llvm.vp.maximum.v16f32 (<16 x float> <left_op>, <16 x float> <right_op>, <16 x i1> <mask>, i32 <vector_length>)
20459+
declare <vscale x 4 x float> @llvm.vp.maximum.nxv4f32 (<vscale x 4 x float> <left_op>, <vscale x 4 x float> <right_op>, <vscale x 4 x i1> <mask>, i32 <vector_length>)
20460+
declare <256 x double> @llvm.vp.maximum.v256f64 (<256 x double> <left_op>, <256 x double> <right_op>, <256 x i1> <mask>, i32 <vector_length>)
20461+
20462+
Overview:
20463+
"""""""""
20464+
20465+
Predicated floating-point maximum of two vectors of floating-point values,
20466+
propagating NaNs and treating -0.0 as less than +0.0.
20467+
20468+
Arguments:
20469+
""""""""""
20470+
20471+
The first two operands and the result have the same vector of floating-point type. The
20472+
third operand is the vector mask and has the same number of elements as the
20473+
result vector type. The fourth operand is the explicit vector length of the
20474+
operation.
20475+
20476+
Semantics:
20477+
""""""""""
20478+
20479+
The '``llvm.vp.maximum``' intrinsic performs floating-point maximum (:ref:`maximum <i_maximum>`)
20480+
of the first and second vector operand on each enabled lane, the result being
20481+
NaN if either operand is a NaN. -0.0 is considered to be less than +0.0 for this
20482+
intrinsic. The result on disabled lanes is a :ref:`poison value <poisonvalues>`.
20483+
The operation is performed in the default floating-point environment.
20484+
20485+
Examples:
20486+
"""""""""
20487+
20488+
.. code-block:: llvm
20489+
20490+
%r = call <4 x float> @llvm.vp.maximum.v4f32(<4 x float> %a, <4 x float> %b, <4 x i1> %mask, i32 %evl)
20491+
;; For all lanes below %evl, %r is lane-wise equivalent to %also.r
20492+
20493+
%t = call <4 x float> @llvm.maximum.v4f32(<4 x float> %a, <4 x float> %b, <4 x i1> %mask, i32 %evl)
20494+
%also.r = select <4 x i1> %mask, <4 x float> %t, <4 x float> poison
20495+
20496+
2039720497
.. _int_vp_fadd:
2039820498

2039920499
'``llvm.vp.fadd.*``' Intrinsics

‎llvm/include/llvm/IR/Intrinsics.td

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1991,6 +1991,16 @@ let IntrProperties = [IntrNoMem, IntrNoSync, IntrWillReturn] in {
19911991
LLVMMatchType<0>,
19921992
LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
19931993
llvm_i32_ty]>;
1994+
def int_vp_minimum : DefaultAttrsIntrinsic<[ llvm_anyvector_ty ],
1995+
[ LLVMMatchType<0>,
1996+
LLVMMatchType<0>,
1997+
LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1998+
llvm_i32_ty]>;
1999+
def int_vp_maximum : DefaultAttrsIntrinsic<[ llvm_anyvector_ty ],
2000+
[ LLVMMatchType<0>,
2001+
LLVMMatchType<0>,
2002+
LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
2003+
llvm_i32_ty]>;
19942004
def int_vp_copysign : DefaultAttrsIntrinsic<[ llvm_anyvector_ty ],
19952005
[ LLVMMatchType<0>,
19962006
LLVMMatchType<0>,

‎llvm/include/llvm/IR/VPIntrinsics.def

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -367,20 +367,34 @@ VP_PROPERTY_FUNCTIONAL_SDOPC(FCOPYSIGN)
367367
VP_PROPERTY_FUNCTIONAL_INTRINSIC(copysign)
368368
END_REGISTER_VP(vp_copysign, VP_FCOPYSIGN)
369369

370-
// llvm.vp.minnum(x, y, mask,vlen)
370+
// llvm.vp.minnum(x,y,mask,vlen)
371371
BEGIN_REGISTER_VP(vp_minnum, 2, 3, VP_FMINNUM, -1)
372372
VP_PROPERTY_BINARYOP
373373
VP_PROPERTY_FUNCTIONAL_SDOPC(FMINNUM)
374374
VP_PROPERTY_FUNCTIONAL_INTRINSIC(minnum)
375375
END_REGISTER_VP(vp_minnum, VP_FMINNUM)
376376

377-
// llvm.vp.maxnum(x, y, mask,vlen)
377+
// llvm.vp.maxnum(x,y,mask,vlen)
378378
BEGIN_REGISTER_VP(vp_maxnum, 2, 3, VP_FMAXNUM, -1)
379379
VP_PROPERTY_BINARYOP
380380
VP_PROPERTY_FUNCTIONAL_SDOPC(FMAXNUM)
381381
VP_PROPERTY_FUNCTIONAL_INTRINSIC(maxnum)
382382
END_REGISTER_VP(vp_maxnum, VP_FMAXNUM)
383383

384+
// llvm.vp.minimum(x,y,mask,vlen)
385+
BEGIN_REGISTER_VP(vp_minimum, 2, 3, VP_FMINIMUM, -1)
386+
VP_PROPERTY_BINARYOP
387+
VP_PROPERTY_FUNCTIONAL_SDOPC(FMINIMUM)
388+
VP_PROPERTY_FUNCTIONAL_INTRINSIC(minimum)
389+
END_REGISTER_VP(vp_minimum, VP_FMINIMUM)
390+
391+
// llvm.vp.maximum(x,y,mask,vlen)
392+
BEGIN_REGISTER_VP(vp_maximum, 2, 3, VP_FMAXIMUM, -1)
393+
VP_PROPERTY_BINARYOP
394+
VP_PROPERTY_FUNCTIONAL_SDOPC(FMAXIMUM)
395+
VP_PROPERTY_FUNCTIONAL_INTRINSIC(maximum)
396+
END_REGISTER_VP(vp_maximum, VP_FMAXIMUM)
397+
384398
// llvm.vp.ceil(x,mask,vlen)
385399
BEGIN_REGISTER_VP(vp_ceil, 1, 2, VP_FCEIL, -1)
386400
VP_PROPERTY_FUNCTIONAL_INTRINSIC(ceil)

‎llvm/lib/CodeGen/ExpandVectorPredication.cpp

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -729,6 +729,8 @@ Value *CachingVPExpander::expandPredication(VPIntrinsic &VPI) {
729729
case Intrinsic::vp_sqrt:
730730
case Intrinsic::vp_maxnum:
731731
case Intrinsic::vp_minnum:
732+
case Intrinsic::vp_maximum:
733+
case Intrinsic::vp_minimum:
732734
return expandPredicationToFPCall(Builder, VPI,
733735
VPI.getFunctionalIntrinsicID().value());
734736
case Intrinsic::vp_load:

‎llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1143,7 +1143,9 @@ void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
11431143
case ISD::FMINNUM: case ISD::VP_FMINNUM:
11441144
case ISD::FMAXNUM: case ISD::VP_FMAXNUM:
11451145
case ISD::FMINIMUM:
1146+
case ISD::VP_FMINIMUM:
11461147
case ISD::FMAXIMUM:
1148+
case ISD::VP_FMAXIMUM:
11471149
case ISD::SDIV: case ISD::VP_SDIV:
11481150
case ISD::UDIV: case ISD::VP_UDIV:
11491151
case ISD::FDIV: case ISD::VP_FDIV:
@@ -4131,7 +4133,9 @@ void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
41314133
case ISD::FMINNUM: case ISD::VP_FMINNUM:
41324134
case ISD::FMAXNUM: case ISD::VP_FMAXNUM:
41334135
case ISD::FMINIMUM:
4136+
case ISD::VP_FMINIMUM:
41344137
case ISD::FMAXIMUM:
4138+
case ISD::VP_FMAXIMUM:
41354139
case ISD::SMIN: case ISD::VP_SMIN:
41364140
case ISD::SMAX: case ISD::VP_SMAX:
41374141
case ISD::UMIN: case ISD::VP_UMIN:

‎llvm/lib/Target/RISCV/RISCVISelLowering.cpp

Lines changed: 24 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -687,7 +687,8 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
687687
ISD::VP_FCEIL, ISD::VP_FFLOOR, ISD::VP_FROUND,
688688
ISD::VP_FROUNDEVEN, ISD::VP_FCOPYSIGN, ISD::VP_FROUNDTOZERO,
689689
ISD::VP_FRINT, ISD::VP_FNEARBYINT, ISD::VP_IS_FPCLASS,
690-
ISD::EXPERIMENTAL_VP_REVERSE, ISD::EXPERIMENTAL_VP_SPLICE};
690+
ISD::VP_FMINIMUM, ISD::VP_FMAXIMUM, ISD::EXPERIMENTAL_VP_REVERSE,
691+
ISD::EXPERIMENTAL_VP_SPLICE};
691692

692693
static const unsigned IntegerVecReduceOps[] = {
693694
ISD::VECREDUCE_ADD, ISD::VECREDUCE_AND, ISD::VECREDUCE_OR,
@@ -927,7 +928,8 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
927928
ISD::VP_FMINNUM, ISD::VP_FMAXNUM, ISD::VP_FCEIL,
928929
ISD::VP_FFLOOR, ISD::VP_FROUND, ISD::VP_FROUNDEVEN,
929930
ISD::VP_FCOPYSIGN, ISD::VP_FROUNDTOZERO, ISD::VP_FRINT,
930-
ISD::VP_FNEARBYINT, ISD::VP_SETCC};
931+
ISD::VP_FNEARBYINT, ISD::VP_SETCC, ISD::VP_FMINIMUM,
932+
ISD::VP_FMAXIMUM};
931933

932934
// Sets common operation actions on RVV floating-point vector types.
933935
const auto SetCommonVFPActions = [&](MVT VT) {
@@ -5405,7 +5407,16 @@ static SDValue lowerFMAXIMUM_FMINIMUM(SDValue Op, SelectionDAG &DAG,
54055407
Y = convertToScalableVector(ContainerVT, Y, DAG, Subtarget);
54065408
}
54075409

5408-
auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
5410+
SDValue Mask, VL;
5411+
if (Op->isVPOpcode()) {
5412+
Mask = Op.getOperand(2);
5413+
if (VT.isFixedLengthVector())
5414+
Mask = convertToScalableVector(getMaskTypeFor(ContainerVT), Mask, DAG,
5415+
Subtarget);
5416+
VL = Op.getOperand(3);
5417+
} else {
5418+
std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
5419+
}
54095420

54105421
SDValue NewY = Y;
54115422
if (!XIsNeverNan) {
@@ -5426,7 +5437,9 @@ static SDValue lowerFMAXIMUM_FMINIMUM(SDValue Op, SelectionDAG &DAG,
54265437
}
54275438

54285439
unsigned Opc =
5429-
Op.getOpcode() == ISD::FMAXIMUM ? RISCVISD::VFMAX_VL : RISCVISD::VFMIN_VL;
5440+
Op.getOpcode() == ISD::FMAXIMUM || Op->getOpcode() == ISD::VP_FMAXIMUM
5441+
? RISCVISD::VFMAX_VL
5442+
: RISCVISD::VFMIN_VL;
54305443
SDValue Res = DAG.getNode(Opc, DL, ContainerVT, NewX, NewY,
54315444
DAG.getUNDEF(ContainerVT), Mask, VL);
54325445
if (VT.isFixedLengthVector())
@@ -6655,6 +6668,13 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
66556668
!Subtarget.hasVInstructionsF16()))
66566669
return SplitVPOp(Op, DAG);
66576670
return lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(Op, DAG, Subtarget);
6671+
case ISD::VP_FMAXIMUM:
6672+
case ISD::VP_FMINIMUM:
6673+
if (Op.getValueType() == MVT::nxv32f16 &&
6674+
(Subtarget.hasVInstructionsF16Minimal() &&
6675+
!Subtarget.hasVInstructionsF16()))
6676+
return SplitVPOp(Op, DAG);
6677+
return lowerFMAXIMUM_FMINIMUM(Op, DAG, Subtarget);
66586678
case ISD::EXPERIMENTAL_VP_SPLICE:
66596679
return lowerVPSpliceExperimental(Op, DAG);
66606680
case ISD::EXPERIMENTAL_VP_REVERSE:

0 commit comments

Comments
 (0)