Skip to content

Commit 8e27c17

Browse files
author
git apple-llvm automerger
committed
Merge commit '58cfd56356ed' from llvm.org/main into next
2 parents 2923743 + 58cfd56 commit 8e27c17

File tree

11 files changed

+4279
-8
lines changed

11 files changed

+4279
-8
lines changed

‎llvm/docs/LangRef.rst

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20448,6 +20448,106 @@ Examples:
2044820448
%also.r = select <4 x i1> %mask, <4 x float> %t, <4 x float> poison
2044920449

2045020450

20451+
.. _int_vp_minimum:
20452+
20453+
'``llvm.vp.minimum.*``' Intrinsics
20454+
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
20455+
20456+
Syntax:
20457+
"""""""
20458+
This is an overloaded intrinsic.
20459+
20460+
::
20461+
20462+
declare <16 x float> @llvm.vp.minimum.v16f32 (<16 x float> <left_op>, <16 x float> <right_op>, <16 x i1> <mask>, i32 <vector_length>)
20463+
declare <vscale x 4 x float> @llvm.vp.minimum.nxv4f32 (<vscale x 4 x float> <left_op>, <vscale x 4 x float> <right_op>, <vscale x 4 x i1> <mask>, i32 <vector_length>)
20464+
declare <256 x double> @llvm.vp.minimum.v256f64 (<256 x double> <left_op>, <256 x double> <right_op>, <256 x i1> <mask>, i32 <vector_length>)
20465+
20466+
Overview:
20467+
"""""""""
20468+
20469+
Predicated floating-point minimum of two vectors of floating-point values,
20470+
propagating NaNs and treating -0.0 as less than +0.0.
20471+
20472+
Arguments:
20473+
""""""""""
20474+
20475+
The first two operands and the result have the same vector of floating-point type. The
20476+
third operand is the vector mask and has the same number of elements as the
20477+
result vector type. The fourth operand is the explicit vector length of the
20478+
operation.
20479+
20480+
Semantics:
20481+
""""""""""
20482+
20483+
The '``llvm.vp.minimum``' intrinsic performs floating-point minimum (:ref:`minimum <i_minimum>`)
20484+
of the first and second vector operand on each enabled lane, the result being
20485+
NaN if either operand is a NaN. -0.0 is considered to be less than +0.0 for this
20486+
intrinsic. The result on disabled lanes is a :ref:`poison value <poisonvalues>`.
20487+
The operation is performed in the default floating-point environment.
20488+
20489+
Examples:
20490+
"""""""""
20491+
20492+
.. code-block:: llvm
20493+
20494+
%r = call <4 x float> @llvm.vp.minimum.v4f32(<4 x float> %a, <4 x float> %b, <4 x i1> %mask, i32 %evl)
20495+
;; For all lanes below %evl, %r is lane-wise equivalent to %also.r
20496+
20497+
%t = call <4 x float> @llvm.minimum.v4f32(<4 x float> %a, <4 x float> %b)
20498+
%also.r = select <4 x i1> %mask, <4 x float> %t, <4 x float> poison
20499+
20500+
20501+
.. _int_vp_maximum:
20502+
20503+
'``llvm.vp.maximum.*``' Intrinsics
20504+
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
20505+
20506+
Syntax:
20507+
"""""""
20508+
This is an overloaded intrinsic.
20509+
20510+
::
20511+
20512+
declare <16 x float> @llvm.vp.maximum.v16f32 (<16 x float> <left_op>, <16 x float> <right_op>, <16 x i1> <mask>, i32 <vector_length>)
20513+
declare <vscale x 4 x float> @llvm.vp.maximum.nxv4f32 (<vscale x 4 x float> <left_op>, <vscale x 4 x float> <right_op>, <vscale x 4 x i1> <mask>, i32 <vector_length>)
20514+
declare <256 x double> @llvm.vp.maximum.v256f64 (<256 x double> <left_op>, <256 x double> <right_op>, <256 x i1> <mask>, i32 <vector_length>)
20515+
20516+
Overview:
20517+
"""""""""
20518+
20519+
Predicated floating-point maximum of two vectors of floating-point values,
20520+
propagating NaNs and treating -0.0 as less than +0.0.
20521+
20522+
Arguments:
20523+
""""""""""
20524+
20525+
The first two operands and the result have the same vector of floating-point type. The
20526+
third operand is the vector mask and has the same number of elements as the
20527+
result vector type. The fourth operand is the explicit vector length of the
20528+
operation.
20529+
20530+
Semantics:
20531+
""""""""""
20532+
20533+
The '``llvm.vp.maximum``' intrinsic performs floating-point maximum (:ref:`maximum <i_maximum>`)
20534+
of the first and second vector operand on each enabled lane, the result being
20535+
NaN if either operand is a NaN. -0.0 is considered to be less than +0.0 for this
20536+
intrinsic. The result on disabled lanes is a :ref:`poison value <poisonvalues>`.
20537+
The operation is performed in the default floating-point environment.
20538+
20539+
Examples:
20540+
"""""""""
20541+
20542+
.. code-block:: llvm
20543+
20544+
%r = call <4 x float> @llvm.vp.maximum.v4f32(<4 x float> %a, <4 x float> %b, <4 x i1> %mask, i32 %evl)
20545+
;; For all lanes below %evl, %r is lane-wise equivalent to %also.r
20546+
20547+
%t = call <4 x float> @llvm.maximum.v4f32(<4 x float> %a, <4 x float> %b, <4 x i1> %mask, i32 %evl)
20548+
%also.r = select <4 x i1> %mask, <4 x float> %t, <4 x float> poison
20549+
20550+
2045120551
.. _int_vp_fadd:
2045220552

2045320553
'``llvm.vp.fadd.*``' Intrinsics

‎llvm/include/llvm/IR/Intrinsics.td

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1991,6 +1991,16 @@ let IntrProperties = [IntrNoMem, IntrNoSync, IntrWillReturn] in {
19911991
LLVMMatchType<0>,
19921992
LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
19931993
llvm_i32_ty]>;
1994+
def int_vp_minimum : DefaultAttrsIntrinsic<[ llvm_anyvector_ty ],
1995+
[ LLVMMatchType<0>,
1996+
LLVMMatchType<0>,
1997+
LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1998+
llvm_i32_ty]>;
1999+
def int_vp_maximum : DefaultAttrsIntrinsic<[ llvm_anyvector_ty ],
2000+
[ LLVMMatchType<0>,
2001+
LLVMMatchType<0>,
2002+
LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
2003+
llvm_i32_ty]>;
19942004
def int_vp_copysign : DefaultAttrsIntrinsic<[ llvm_anyvector_ty ],
19952005
[ LLVMMatchType<0>,
19962006
LLVMMatchType<0>,

‎llvm/include/llvm/IR/VPIntrinsics.def

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -367,20 +367,34 @@ VP_PROPERTY_FUNCTIONAL_SDOPC(FCOPYSIGN)
367367
VP_PROPERTY_FUNCTIONAL_INTRINSIC(copysign)
368368
END_REGISTER_VP(vp_copysign, VP_FCOPYSIGN)
369369

370-
// llvm.vp.minnum(x, y, mask,vlen)
370+
// llvm.vp.minnum(x,y,mask,vlen)
371371
BEGIN_REGISTER_VP(vp_minnum, 2, 3, VP_FMINNUM, -1)
372372
VP_PROPERTY_BINARYOP
373373
VP_PROPERTY_FUNCTIONAL_SDOPC(FMINNUM)
374374
VP_PROPERTY_FUNCTIONAL_INTRINSIC(minnum)
375375
END_REGISTER_VP(vp_minnum, VP_FMINNUM)
376376

377-
// llvm.vp.maxnum(x, y, mask,vlen)
377+
// llvm.vp.maxnum(x,y,mask,vlen)
378378
BEGIN_REGISTER_VP(vp_maxnum, 2, 3, VP_FMAXNUM, -1)
379379
VP_PROPERTY_BINARYOP
380380
VP_PROPERTY_FUNCTIONAL_SDOPC(FMAXNUM)
381381
VP_PROPERTY_FUNCTIONAL_INTRINSIC(maxnum)
382382
END_REGISTER_VP(vp_maxnum, VP_FMAXNUM)
383383

384+
// llvm.vp.minimum(x,y,mask,vlen)
385+
BEGIN_REGISTER_VP(vp_minimum, 2, 3, VP_FMINIMUM, -1)
386+
VP_PROPERTY_BINARYOP
387+
VP_PROPERTY_FUNCTIONAL_SDOPC(FMINIMUM)
388+
VP_PROPERTY_FUNCTIONAL_INTRINSIC(minimum)
389+
END_REGISTER_VP(vp_minimum, VP_FMINIMUM)
390+
391+
// llvm.vp.maximum(x,y,mask,vlen)
392+
BEGIN_REGISTER_VP(vp_maximum, 2, 3, VP_FMAXIMUM, -1)
393+
VP_PROPERTY_BINARYOP
394+
VP_PROPERTY_FUNCTIONAL_SDOPC(FMAXIMUM)
395+
VP_PROPERTY_FUNCTIONAL_INTRINSIC(maximum)
396+
END_REGISTER_VP(vp_maximum, VP_FMAXIMUM)
397+
384398
// llvm.vp.ceil(x,mask,vlen)
385399
BEGIN_REGISTER_VP(vp_ceil, 1, 2, VP_FCEIL, -1)
386400
VP_PROPERTY_FUNCTIONAL_INTRINSIC(ceil)

‎llvm/lib/CodeGen/ExpandVectorPredication.cpp

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -729,6 +729,8 @@ Value *CachingVPExpander::expandPredication(VPIntrinsic &VPI) {
729729
case Intrinsic::vp_sqrt:
730730
case Intrinsic::vp_maxnum:
731731
case Intrinsic::vp_minnum:
732+
case Intrinsic::vp_maximum:
733+
case Intrinsic::vp_minimum:
732734
return expandPredicationToFPCall(Builder, VPI,
733735
VPI.getFunctionalIntrinsicID().value());
734736
case Intrinsic::vp_load:

‎llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1143,7 +1143,9 @@ void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
11431143
case ISD::FMINNUM: case ISD::VP_FMINNUM:
11441144
case ISD::FMAXNUM: case ISD::VP_FMAXNUM:
11451145
case ISD::FMINIMUM:
1146+
case ISD::VP_FMINIMUM:
11461147
case ISD::FMAXIMUM:
1148+
case ISD::VP_FMAXIMUM:
11471149
case ISD::SDIV: case ISD::VP_SDIV:
11481150
case ISD::UDIV: case ISD::VP_UDIV:
11491151
case ISD::FDIV: case ISD::VP_FDIV:
@@ -4131,7 +4133,9 @@ void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
41314133
case ISD::FMINNUM: case ISD::VP_FMINNUM:
41324134
case ISD::FMAXNUM: case ISD::VP_FMAXNUM:
41334135
case ISD::FMINIMUM:
4136+
case ISD::VP_FMINIMUM:
41344137
case ISD::FMAXIMUM:
4138+
case ISD::VP_FMAXIMUM:
41354139
case ISD::SMIN: case ISD::VP_SMIN:
41364140
case ISD::SMAX: case ISD::VP_SMAX:
41374141
case ISD::UMIN: case ISD::VP_UMIN:

‎llvm/lib/Target/RISCV/RISCVISelLowering.cpp

Lines changed: 24 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -687,7 +687,8 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
687687
ISD::VP_FCEIL, ISD::VP_FFLOOR, ISD::VP_FROUND,
688688
ISD::VP_FROUNDEVEN, ISD::VP_FCOPYSIGN, ISD::VP_FROUNDTOZERO,
689689
ISD::VP_FRINT, ISD::VP_FNEARBYINT, ISD::VP_IS_FPCLASS,
690-
ISD::EXPERIMENTAL_VP_REVERSE, ISD::EXPERIMENTAL_VP_SPLICE};
690+
ISD::VP_FMINIMUM, ISD::VP_FMAXIMUM, ISD::EXPERIMENTAL_VP_REVERSE,
691+
ISD::EXPERIMENTAL_VP_SPLICE};
691692

692693
static const unsigned IntegerVecReduceOps[] = {
693694
ISD::VECREDUCE_ADD, ISD::VECREDUCE_AND, ISD::VECREDUCE_OR,
@@ -927,7 +928,8 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
927928
ISD::VP_FMINNUM, ISD::VP_FMAXNUM, ISD::VP_FCEIL,
928929
ISD::VP_FFLOOR, ISD::VP_FROUND, ISD::VP_FROUNDEVEN,
929930
ISD::VP_FCOPYSIGN, ISD::VP_FROUNDTOZERO, ISD::VP_FRINT,
930-
ISD::VP_FNEARBYINT, ISD::VP_SETCC};
931+
ISD::VP_FNEARBYINT, ISD::VP_SETCC, ISD::VP_FMINIMUM,
932+
ISD::VP_FMAXIMUM};
931933

932934
// Sets common operation actions on RVV floating-point vector types.
933935
const auto SetCommonVFPActions = [&](MVT VT) {
@@ -5405,7 +5407,16 @@ static SDValue lowerFMAXIMUM_FMINIMUM(SDValue Op, SelectionDAG &DAG,
54055407
Y = convertToScalableVector(ContainerVT, Y, DAG, Subtarget);
54065408
}
54075409

5408-
auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
5410+
SDValue Mask, VL;
5411+
if (Op->isVPOpcode()) {
5412+
Mask = Op.getOperand(2);
5413+
if (VT.isFixedLengthVector())
5414+
Mask = convertToScalableVector(getMaskTypeFor(ContainerVT), Mask, DAG,
5415+
Subtarget);
5416+
VL = Op.getOperand(3);
5417+
} else {
5418+
std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
5419+
}
54095420

54105421
SDValue NewY = Y;
54115422
if (!XIsNeverNan) {
@@ -5426,7 +5437,9 @@ static SDValue lowerFMAXIMUM_FMINIMUM(SDValue Op, SelectionDAG &DAG,
54265437
}
54275438

54285439
unsigned Opc =
5429-
Op.getOpcode() == ISD::FMAXIMUM ? RISCVISD::VFMAX_VL : RISCVISD::VFMIN_VL;
5440+
Op.getOpcode() == ISD::FMAXIMUM || Op->getOpcode() == ISD::VP_FMAXIMUM
5441+
? RISCVISD::VFMAX_VL
5442+
: RISCVISD::VFMIN_VL;
54305443
SDValue Res = DAG.getNode(Opc, DL, ContainerVT, NewX, NewY,
54315444
DAG.getUNDEF(ContainerVT), Mask, VL);
54325445
if (VT.isFixedLengthVector())
@@ -6655,6 +6668,13 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
66556668
!Subtarget.hasVInstructionsF16()))
66566669
return SplitVPOp(Op, DAG);
66576670
return lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(Op, DAG, Subtarget);
6671+
case ISD::VP_FMAXIMUM:
6672+
case ISD::VP_FMINIMUM:
6673+
if (Op.getValueType() == MVT::nxv32f16 &&
6674+
(Subtarget.hasVInstructionsF16Minimal() &&
6675+
!Subtarget.hasVInstructionsF16()))
6676+
return SplitVPOp(Op, DAG);
6677+
return lowerFMAXIMUM_FMINIMUM(Op, DAG, Subtarget);
66586678
case ISD::EXPERIMENTAL_VP_SPLICE:
66596679
return lowerVPSpliceExperimental(Op, DAG);
66606680
case ISD::EXPERIMENTAL_VP_REVERSE:

0 commit comments

Comments
 (0)