Intrinsic: introduce minimumnum and maximumnum

llvm/include/llvm/ADT/APFloat.h

@@ -1483,6 +1483,19 @@ inline APFloat minimum(const APFloat &A, const APFloat &B) {
   return B < A ? B : A;
 }
 
+/// Implements IEEE 754-2019 minimumNumber semantics. Returns the smaller
+/// of 2 arguments, not propagating NaNs and treating -0 as less than +0.
+LLVM_READONLY
+inline APFloat minimumnum(const APFloat &A, const APFloat &B) {
+  if (A.isNaN())
+    return B.isNaN() ? B.makeQuiet() : B;
+  if (B.isNaN())
+    return A;
+  if (A.isZero() && B.isZero() && (A.isNegative() != B.isNegative()))
+    return A.isNegative() ? A : B;
+  return B < A ? B : A;
+}
+
 /// Implements IEEE 754-2019 maximum semantics. Returns the larger of 2
 /// arguments, propagating NaNs and treating -0 as less than +0.
 LLVM_READONLY
@@ -1496,6 +1509,19 @@ inline APFloat maximum(const APFloat &A, const APFloat &B) {
   return A < B ? B : A;
 }
 
+/// Implements IEEE 754-2019 maximumNumber semantics. Returns the larger
+/// of 2 arguments, not propagating NaNs and treating -0 as less than +0.
+LLVM_READONLY
+inline APFloat maximumnum(const APFloat &A, const APFloat &B) {
+  if (A.isNaN())
+    return B.isNaN() ? B.makeQuiet() : B;
+  if (B.isNaN())
+    return A;
+  if (A.isZero() && B.isZero() && (A.isNegative() != B.isNegative()))
+    return A.isNegative() ? B : A;
+  return A < B ? B : A;
+}
+
 // We want the following functions to be available in the header for inlining.
 // We cannot define them inline in the class definition of `DoubleAPFloat`
 // because doing so would instantiate `std::unique_ptr<APFloat[]>` before

llvm/include/llvm/Analysis/TargetLibraryInfo.def

@@ -1347,6 +1347,39 @@ TLI_DEFINE_ENUM_INTERNAL(fminl)
 TLI_DEFINE_STRING_INTERNAL("fminl")
 TLI_DEFINE_SIG_INTERNAL(Floating, Same, Same)
 
+// Calls to fmaximum_num and fminimum_num library functions expand to the llvm.maximumnum and
+// llvm.minimumnum intrinsics with the correct parameter types for the arguments
+// (all types must match).
+/// double fmaximum_num(double x, double y);
+TLI_DEFINE_ENUM_INTERNAL(fmaximum_num)
+TLI_DEFINE_STRING_INTERNAL("fmaximum_num")
+TLI_DEFINE_SIG_INTERNAL(Floating, Same, Same)
+
+/// float fmaximum_numf(float x, float y);
+TLI_DEFINE_ENUM_INTERNAL(fmaximum_numf)
+TLI_DEFINE_STRING_INTERNAL("fmaximum_numf")
+TLI_DEFINE_SIG_INTERNAL(Floating, Same, Same)
+
+/// long double fmaximum_numl(long double x, long double y);
+TLI_DEFINE_ENUM_INTERNAL(fmaximum_numl)
+TLI_DEFINE_STRING_INTERNAL("fmaximum_numl")
+TLI_DEFINE_SIG_INTERNAL(Floating, Same, Same)
+
+/// double fminimum_num(double x, double y);
+TLI_DEFINE_ENUM_INTERNAL(fminimum_num)
+TLI_DEFINE_STRING_INTERNAL("fminimum_num")
+TLI_DEFINE_SIG_INTERNAL(Floating, Same, Same)
+
+/// float fminimum_numf(float x, float y);
+TLI_DEFINE_ENUM_INTERNAL(fminimum_numf)
+TLI_DEFINE_STRING_INTERNAL("fminimum_numf")
+TLI_DEFINE_SIG_INTERNAL(Floating, Same, Same)
+
+/// long double fminimum_numl(long double x, long double y);
+TLI_DEFINE_ENUM_INTERNAL(fminimum_numl)
+TLI_DEFINE_STRING_INTERNAL("fminimum_numl")
+TLI_DEFINE_SIG_INTERNAL(Floating, Same, Same)
+
 /// double fmod(double x, double y);
 TLI_DEFINE_ENUM_INTERNAL(fmod)
 TLI_DEFINE_STRING_INTERNAL("fmod")

llvm/include/llvm/CodeGen/BasicTTIImpl.h

@@ -1690,6 +1690,8 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
     case Intrinsic::vector_reduce_fmin:
     case Intrinsic::vector_reduce_fmaximum:
     case Intrinsic::vector_reduce_fminimum:
+    case Intrinsic::vector_reduce_fmaximumnum:
+    case Intrinsic::vector_reduce_fminimumnum:
     case Intrinsic::vector_reduce_umax:
     case Intrinsic::vector_reduce_umin: {
       IntrinsicCostAttributes Attrs(IID, RetTy, Args[0]->getType(), FMF, I, 1);
@@ -2016,6 +2018,12 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
     case Intrinsic::maximum:
       ISD = ISD::FMAXIMUM;
       break;
+    case Intrinsic::minimumnum:
+      ISD = ISD::FMINIMUMNUM;
+      break;
+    case Intrinsic::maximumnum:
+      ISD = ISD::FMAXIMUMNUM;
+      break;
     case Intrinsic::copysign:
       ISD = ISD::FCOPYSIGN;
       break;
@@ -2097,6 +2105,8 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
     case Intrinsic::vector_reduce_fmin:
     case Intrinsic::vector_reduce_fmaximum:
     case Intrinsic::vector_reduce_fminimum:
+    case Intrinsic::vector_reduce_fmaximumnum:
+    case Intrinsic::vector_reduce_fminimumnum:
       return thisT()->getMinMaxReductionCost(getMinMaxReductionIntrinsicOp(IID),
                                              VecOpTy, ICA.getFlags(), CostKind);
     case Intrinsic::abs: {

llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h

@@ -954,8 +954,10 @@ class CombinerHelper {
   ///
   /// * G_FMAXNUM
   /// * G_FMAXIMUM
+  /// * G_FMAXIMUMNUM
   /// * G_FMINNUM
   /// * G_FMINIMUM
+  /// * G_FMINIMUMNUM
   ///
   /// Helper function for matchFPSelectToMinMax.
   unsigned getFPMinMaxOpcForSelect(CmpInst::Predicate Pred, LLT DstTy,

llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h

@@ -553,6 +553,8 @@ class GVecReduce : public GenericMachineInstr {
     case TargetOpcode::G_VECREDUCE_FMIN:
     case TargetOpcode::G_VECREDUCE_FMAXIMUM:
     case TargetOpcode::G_VECREDUCE_FMINIMUM:
+    case TargetOpcode::G_VECREDUCE_FMAXIMUMNUM:
+    case TargetOpcode::G_VECREDUCE_FMINIMUMNUM:
     case TargetOpcode::G_VECREDUCE_ADD:
     case TargetOpcode::G_VECREDUCE_MUL:
     case TargetOpcode::G_VECREDUCE_AND:
@@ -591,6 +593,12 @@ class GVecReduce : public GenericMachineInstr {
     case TargetOpcode::G_VECREDUCE_FMINIMUM:
       ScalarOpc = TargetOpcode::G_FMINIMUM;
       break;
+    case TargetOpcode::G_VECREDUCE_FMAXIMUMNUM:
+      ScalarOpc = TargetOpcode::G_FMAXIMUMNUM;
+      break;
+    case TargetOpcode::G_VECREDUCE_FMINIMUMNUM:
+      ScalarOpc = TargetOpcode::G_FMINIMUMNUM;
+      break;
     case TargetOpcode::G_VECREDUCE_ADD:
       ScalarOpc = TargetOpcode::G_ADD;
       break;
@@ -671,6 +679,8 @@ class GBinOp : public GenericMachineInstr {
     case TargetOpcode::G_FMAXNUM_IEEE:
     case TargetOpcode::G_FMINIMUM:
     case TargetOpcode::G_FMAXIMUM:
+    case TargetOpcode::G_FMINIMUMNUM:
+    case TargetOpcode::G_FMAXIMUMNUM:
     case TargetOpcode::G_FADD:
     case TargetOpcode::G_FSUB:
     case TargetOpcode::G_FMUL:
@@ -721,6 +731,8 @@ class GFBinOp : public GBinOp {
     case TargetOpcode::G_FMAXNUM_IEEE:
     case TargetOpcode::G_FMINIMUM:
     case TargetOpcode::G_FMAXIMUM:
+    case TargetOpcode::G_FMINIMUMNUM:
+    case TargetOpcode::G_FMAXIMUMNUM:
     case TargetOpcode::G_FADD:
     case TargetOpcode::G_FSUB:
     case TargetOpcode::G_FMUL:

llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h

@@ -403,6 +403,7 @@ class LegalizerHelper {
   LegalizeResult lowerMinMax(MachineInstr &MI);
   LegalizeResult lowerFCopySign(MachineInstr &MI);
   LegalizeResult lowerFMinNumMaxNum(MachineInstr &MI);
+  LegalizeResult lowerFMinimumNumMaximumNum(MachineInstr &MI);
   LegalizeResult lowerFMad(MachineInstr &MI);
   LegalizeResult lowerIntrinsicRound(MachineInstr &MI);
   LegalizeResult lowerFFloor(MachineInstr &MI);

llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h

@@ -1723,6 +1723,18 @@ class MachineIRBuilder {
     return buildInstr(TargetOpcode::G_FMAXNUM_IEEE, {Dst}, {Src0, Src1}, Flags);
   }
 
+  MachineInstrBuilder
+  buildFMinimumnum(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1,
+                   std::optional<unsigned> Flags = std::nullopt) {
+    return buildInstr(TargetOpcode::G_FMINIMUMNUM, {Dst}, {Src0, Src1}, Flags);
+  }
+
+  MachineInstrBuilder
+  buildFMaximumnum(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1,
+                   std::optional<unsigned> Flags = std::nullopt) {
+    return buildInstr(TargetOpcode::G_FMAXIMUMNUM, {Dst}, {Src0, Src1}, Flags);
+  }
+
   MachineInstrBuilder buildShl(const DstOp &Dst, const SrcOp &Src0,
                                const SrcOp &Src1,
                                std::optional<unsigned> Flags = std::nullopt) {
@@ -2074,6 +2086,18 @@ class MachineIRBuilder {
     return buildInstr(TargetOpcode::G_VECREDUCE_FMINIMUM, {Dst}, {Src});
   }
 
+  /// Build and insert \p Res = G_VECREDUCE_FMAXIMUMNUM \p Src
+  MachineInstrBuilder buildVecReduceFMaximumnum(const DstOp &Dst,
+                                                const SrcOp &Src) {
+    return buildInstr(TargetOpcode::G_VECREDUCE_FMAXIMUMNUM, {Dst}, {Src});
+  }
+
+  /// Build and insert \p Res = G_VECREDUCE_FMINIMUMNUM \p Src
+  MachineInstrBuilder buildVecReduceFMinimumnum(const DstOp &Dst,
+                                                const SrcOp &Src) {
+    return buildInstr(TargetOpcode::G_VECREDUCE_FMINIMUMNUM, {Dst}, {Src});
+  }
+
   /// Build and insert \p Res = G_VECREDUCE_ADD \p Src
   MachineInstrBuilder buildVecReduceAdd(const DstOp &Dst, const SrcOp &Src) {
     return buildInstr(TargetOpcode::G_VECREDUCE_ADD, {Dst}, {Src});

llvm/include/llvm/CodeGen/GlobalISel/Utils.h

@@ -60,6 +60,8 @@ class APFloat;
   case TargetOpcode::G_VECREDUCE_FMIN:                                         \
   case TargetOpcode::G_VECREDUCE_FMAXIMUM:                                     \
   case TargetOpcode::G_VECREDUCE_FMINIMUM:                                     \
+  case TargetOpcode::G_VECREDUCE_FMAXIMUMNUM:                                  \
+  case TargetOpcode::G_VECREDUCE_FMINIMUMNUM:                                  \
   case TargetOpcode::G_VECREDUCE_ADD:                                          \
   case TargetOpcode::G_VECREDUCE_MUL:                                          \
   case TargetOpcode::G_VECREDUCE_AND:                                          \
@@ -77,6 +79,8 @@ class APFloat;
   case TargetOpcode::G_VECREDUCE_FMIN:                                         \
   case TargetOpcode::G_VECREDUCE_FMAXIMUM:                                     \
   case TargetOpcode::G_VECREDUCE_FMINIMUM:                                     \
+  case TargetOpcode::G_VECREDUCE_FMAXIMUMNUM:                                  \
+  case TargetOpcode::G_VECREDUCE_FMINIMUMNUM:                                  \
   case TargetOpcode::G_VECREDUCE_ADD:                                          \
   case TargetOpcode::G_VECREDUCE_MUL:                                          \
   case TargetOpcode::G_VECREDUCE_AND:                                          \

llvm/include/llvm/CodeGen/ISDOpcodes.h

@@ -436,6 +436,8 @@ enum NodeType {
   STRICT_LLRINT,
   STRICT_FMAXIMUM,
   STRICT_FMINIMUM,
+  STRICT_FMAXIMUMNUM,
+  STRICT_FMINIMUMNUM,
 
   /// STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or
   /// unsigned integer. These have the same semantics as fptosi and fptoui
@@ -999,6 +1001,11 @@ enum NodeType {
   FMINIMUM,
   FMAXIMUM,
 
+  /// FMINIMUMNUM/FMAXIMUMNUM - minimumnum/maximumnum that is same with
+  /// FMINNUM_IEEE and FMAXNUM_IEEE besides if either operand is sNaN.
+  FMINIMUMNUM,
+  FMAXIMUMNUM,
+
   /// FSINCOS - Compute both fsin and fcos as a single operation.
   FSINCOS,
 
@@ -1373,6 +1380,10 @@ enum NodeType {
   /// llvm.minimum and llvm.maximum semantics.
   VECREDUCE_FMAXIMUM,
   VECREDUCE_FMINIMUM,
+  /// FMINIMUMNUM/FMAXIMUMNUM nodes don't propatate NaNs and signed zeroes using
+  /// the llvm.minimumnum and llvm.maximumnum semantics.
+  VECREDUCE_FMAXIMUMNUM,
+  VECREDUCE_FMINIMUMNUM,
   /// Integer reductions may have a result type larger than the vector element
   /// type. However, the reduction is performed using the vector element type
   /// and the value in the top bits is unspecified.

llvm/include/llvm/CodeGen/TargetLowering.h

@@ -2910,6 +2910,8 @@ class TargetLoweringBase {
     case ISD::FMAXNUM_IEEE:
     case ISD::FMINIMUM:
     case ISD::FMAXIMUM:
+    case ISD::FMINIMUMNUM:
+    case ISD::FMAXIMUMNUM:
     case ISD::AVGFLOORS:
     case ISD::AVGFLOORU:
     case ISD::AVGCEILS:
@@ -5130,6 +5132,8 @@ class TargetLowering : public TargetLoweringBase {
   /// through to the default expansion/soften to libcall, we might introduce a
   /// link-time dependency on libm into a file that originally did not have one.
   SDValue createSelectForFMINNUM_FMAXNUM(SDNode *Node, SelectionDAG &DAG) const;
+  SDValue createSelectForFMINIMUMNUM_FMAXIMUMNUM(SDNode *Node,
+                                                 SelectionDAG &DAG) const;
 
   /// Return a reciprocal estimate value for the input operand.
   /// \p Enabled is a ReciprocalEstimate enum with value either 'Unspecified' or
@@ -5261,6 +5265,9 @@ class TargetLowering : public TargetLoweringBase {
   /// Expand fminimum/fmaximum into multiple comparison with selects.
   SDValue expandFMINIMUM_FMAXIMUM(SDNode *N, SelectionDAG &DAG) const;
 
+  /// Expand fminimumnum/fmaximumnum into multiple comparison with selects.
+  SDValue expandFMINIMUMNUM_FMAXIMUMNUM(SDNode *N, SelectionDAG &DAG) const;
+
   /// Expand FP_TO_[US]INT_SAT into FP_TO_[US]INT and selects or min/max.
   /// \param N Node to expand
   /// \returns The expansion result

llvm/include/llvm/IR/ConstrainedOps.def

@@ -86,6 +86,8 @@ DAG_FUNCTION(maxnum,          2, 0, experimental_constrained_maxnum,     FMAXNUM
 DAG_FUNCTION(minnum,          2, 0, experimental_constrained_minnum,     FMINNUM)
 DAG_FUNCTION(maximum,         2, 0, experimental_constrained_maximum,    FMAXIMUM)
 DAG_FUNCTION(minimum,         2, 0, experimental_constrained_minimum,    FMINIMUM)
+DAG_FUNCTION(maximumnum,     2, 0, experimental_constrained_maximumnum,    FMAXIMUMNUM)
+DAG_FUNCTION(minimumnum,     2, 0, experimental_constrained_minimumnum,    FMINIMUMNUM)
 DAG_FUNCTION(nearbyint,       1, 1, experimental_constrained_nearbyint,  FNEARBYINT)
 DAG_FUNCTION(pow,             2, 1, experimental_constrained_pow,        FPOW)
 DAG_FUNCTION(powi,            2, 1, experimental_constrained_powi,       FPOWI)

llvm/include/llvm/IR/IRBuilder.h

@@ -1018,6 +1018,18 @@ class IRBuilderBase {
     return CreateBinaryIntrinsic(Intrinsic::maximum, LHS, RHS, nullptr, Name);
   }
 
+  /// Create call to the minimumnum intrinsic.
+  Value *CreateMinimumNum(Value *LHS, Value *RHS, const Twine &Name = "") {
+    return CreateBinaryIntrinsic(Intrinsic::minimumnum, LHS, RHS, nullptr,
+                                 Name);
+  }
+
+  /// Create call to the maximum intrinsic.
+  Value *CreateMaximumNum(Value *LHS, Value *RHS, const Twine &Name = "") {
+    return CreateBinaryIntrinsic(Intrinsic::maximumnum, LHS, RHS, nullptr,
+                                 Name);
+  }
+
   /// Create call to the copysign intrinsic.
   Value *CreateCopySign(Value *LHS, Value *RHS,
                         Instruction *FMFSource = nullptr,

llvm/include/llvm/IR/IntrinsicInst.h

@@ -76,6 +76,8 @@ class IntrinsicInst : public CallInst {
     case Intrinsic::minnum:
     case Intrinsic::maximum:
     case Intrinsic::minimum:
+    case Intrinsic::maximumnum:
+    case Intrinsic::minimumnum:
     case Intrinsic::smax:
     case Intrinsic::smin:
     case Intrinsic::umax:

llvm/include/llvm/IR/Intrinsics.td

@@ -1086,6 +1086,14 @@ def int_maximum : DefaultAttrsIntrinsic<[llvm_anyfloat_ty],
   [LLVMMatchType<0>, LLVMMatchType<0>],
   [IntrNoMem, IntrSpeculatable, IntrWillReturn, Commutative]
 >;
+def int_minimumnum : DefaultAttrsIntrinsic<[llvm_anyfloat_ty],
+  [LLVMMatchType<0>, LLVMMatchType<0>],
+  [IntrNoMem, IntrSpeculatable, IntrWillReturn, Commutative]
+>;
+def int_maximumnum : DefaultAttrsIntrinsic<[llvm_anyfloat_ty],
+  [LLVMMatchType<0>, LLVMMatchType<0>],
+  [IntrNoMem, IntrSpeculatable, IntrWillReturn, Commutative]
+>;
 
 // Internal interface for object size checking
 def int_objectsize : DefaultAttrsIntrinsic<[llvm_anyint_ty],
@@ -1280,6 +1288,14 @@ let IntrProperties = [IntrInaccessibleMemOnly, IntrWillReturn, IntrStrictFP] in
                                                        [ LLVMMatchType<0>,
                                                          LLVMMatchType<0>,
                                                          llvm_metadata_ty ]>;
+  def int_experimental_constrained_maximumnum : DefaultAttrsIntrinsic<[ llvm_anyfloat_ty ],
+                                                       [ LLVMMatchType<0>,
+                                                         LLVMMatchType<0>,
+                                                         llvm_metadata_ty ]>;
+  def int_experimental_constrained_minimumnum : DefaultAttrsIntrinsic<[ llvm_anyfloat_ty ],
+                                                       [ LLVMMatchType<0>,
+                                                         LLVMMatchType<0>,
+                                                         llvm_metadata_ty ]>;
   def int_experimental_constrained_ceil : DefaultAttrsIntrinsic<[ llvm_anyfloat_ty ],
                                                     [ LLVMMatchType<0>,
                                                       llvm_metadata_ty ]>;
@@ -2077,6 +2093,16 @@ let IntrProperties = [IntrNoMem, IntrNoSync, IntrWillReturn] in {
                                LLVMMatchType<0>,
                                LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
                                llvm_i32_ty]>;
+  def int_vp_minimumnum : DefaultAttrsIntrinsic<[ llvm_anyvector_ty ],
+                             [ LLVMMatchType<0>,
+                               LLVMMatchType<0>,
+                               LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
+                               llvm_i32_ty]>;
+  def int_vp_maximumnum : DefaultAttrsIntrinsic<[ llvm_anyvector_ty ],
+                             [ LLVMMatchType<0>,
+                               LLVMMatchType<0>,
+                               LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
+                               llvm_i32_ty]>;
   def int_vp_copysign : DefaultAttrsIntrinsic<[ llvm_anyvector_ty ],
                              [ LLVMMatchType<0>,
                                LLVMMatchType<0>,
@@ -2266,6 +2292,16 @@ let IntrProperties = [IntrNoMem, IntrNoSync, IntrWillReturn] in {
                                    llvm_anyvector_ty,
                                    LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
                                    llvm_i32_ty]>;
+  def int_vp_reduce_fmaximumnum : DefaultAttrsIntrinsic<[LLVMVectorElementType<0>],
+                                 [ LLVMVectorElementType<0>,
+                                   llvm_anyvector_ty,
+                                   LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
+                                   llvm_i32_ty]>;
+  def int_vp_reduce_fminimumnum : DefaultAttrsIntrinsic<[LLVMVectorElementType<0>],
+                                 [ LLVMVectorElementType<0>,
+                                   llvm_anyvector_ty,
+                                   LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
+                                   llvm_i32_ty]>;
 }
 
 let IntrProperties = [IntrNoMem, IntrNoSync, IntrWillReturn, ImmArg<ArgIndex<1>>] in {
@@ -2504,6 +2540,10 @@ let IntrProperties = [IntrNoMem, IntrSpeculatable] in {
                                          [llvm_anyvector_ty]>;
   def int_vector_reduce_fmaximum: DefaultAttrsIntrinsic<[LLVMVectorElementType<0>],
                                          [llvm_anyvector_ty]>;
+  def int_vector_reduce_fminimumnum: DefaultAttrsIntrinsic<[LLVMVectorElementType<0>],
+                                         [llvm_anyvector_ty]>;
+  def int_vector_reduce_fmaximumnum: DefaultAttrsIntrinsic<[LLVMVectorElementType<0>],
+                                         [llvm_anyvector_ty]>;
 }
 
 //===----- Matrix intrinsics ---------------------------------------------===//