[X86] Avoid usage constant -1 for fminimum/fmaximum lowering

Serguei Katkov · Serguei Katkov · commit 50cd2ff7bc5a · 2023-05-05T16:24:33.000+07:00
Instead of equality comparison of value to preferred zero we can check just the sign of value and if sign is set we should put this value as second operand for minimum and first operand for maximum. In this case FMIN/FMAX will choose the right result for 0.f and -0.f comparison. This allows us: 1. avoid loading of big 64-bit constant for fminimum. 2. for double on non-64-nib platform we need to check only high part of value. 3. test against zero to check sign takes less size of instruction Additionally, if we know that any of value is guaranteed to be non-zero we should not care about 0.f and -0.f comparison. Reviewed By: e-kud Differential Revision: https://reviews.llvm.org/D149812
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -30276,13 +30276,15 @@ static SDValue LowerFMINIMUM_FMAXIMUM(SDValue Op, const X86Subtarget &Subtarget,
   bool IsXNeverNaN = DAG.isKnownNeverNaN(X);
   bool IsYNeverNaN = DAG.isKnownNeverNaN(Y);
   bool IgnoreSignedZero = DAG.getTarget().Options.NoSignedZerosFPMath ||
-                          Op->getFlags().hasNoSignedZeros();
+                          Op->getFlags().hasNoSignedZeros() ||
+                          DAG.isKnownNeverZeroFloat(X) ||
+                          DAG.isKnownNeverZeroFloat(Y);
   SDValue NewX, NewY;
-  if (IgnoreSignedZero || IsPreferredZero(Y) || DAG.isKnownNeverZeroFloat(X)) {
+  if (IgnoreSignedZero || IsPreferredZero(Y)) {
     // Operands are already in right order or order does not matter.
     NewX = X;
     NewY = Y;
-  } else if (IsPreferredZero(X) || DAG.isKnownNeverZeroFloat(Y)) {
+  } else if (IsPreferredZero(X)) {
     NewX = Y;
     NewY = X;
   } else if ((VT == MVT::f16 || Subtarget.hasDQI()) &&
@@ -30307,34 +30309,32 @@ static SDValue LowerFMINIMUM_FMAXIMUM(SDValue Op, const X86Subtarget &Subtarget,
     NewY = DAG.getSelect(DL, VT, NeedSwap, X, Y);
     return DAG.getNode(MinMaxOp, DL, VT, NewX, NewY, Op->getFlags());
   } else {
-    SDValue IsXZero;
+    SDValue IsXSigned;
     if (Subtarget.is64Bit() || VT != MVT::f64) {
       SDValue XInt = DAG.getNode(ISD::BITCAST, DL, IVT, X);
-      SDValue ZeroCst = DAG.getConstant(PreferredZero, DL, IVT);
-      IsXZero = DAG.getSetCC(DL, SetCCType, XInt, ZeroCst, ISD::SETEQ);
+      SDValue ZeroCst = DAG.getConstant(0, DL, IVT);
+      IsXSigned = DAG.getSetCC(DL, SetCCType, XInt, ZeroCst, ISD::SETLT);
     } else {
       assert(VT == MVT::f64);
       SDValue Ins = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, MVT::v2f64,
                                 DAG.getConstantFP(0, DL, MVT::v2f64), X,
                                 DAG.getIntPtrConstant(0, DL));
       SDValue VX = DAG.getNode(ISD::BITCAST, DL, MVT::v4f32, Ins);
-      SDValue Lo = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, VX,
-                               DAG.getIntPtrConstant(0, DL));
-      Lo = DAG.getBitcast(MVT::i32, Lo);
       SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, VX,
                                DAG.getIntPtrConstant(1, DL));
       Hi = DAG.getBitcast(MVT::i32, Hi);
-      PreferredZero = APInt::getZero(SizeInBits / 2);
-      if (MinMaxOp == X86ISD::FMIN)
-        PreferredZero.setSignBit();
-      IsXZero = DAG.getNode(ISD::XOR, DL, MVT::i32, Hi,
-                            DAG.getConstant(PreferredZero, DL, MVT::i32));
-      IsXZero = DAG.getNode(ISD::OR, DL, MVT::i32, Lo, IsXZero);
-      IsXZero = DAG.getSetCC(DL, SetCCType, IsXZero,
-                             DAG.getConstant(0, DL, MVT::i32), ISD::SETEQ);
-    }
-    NewX = DAG.getSelect(DL, VT, IsXZero, Y, X);
-    NewY = DAG.getSelect(DL, VT, IsXZero, X, Y);
+      SDValue ZeroCst = DAG.getConstant(0, DL, MVT::i32);
+      EVT SetCCType = TLI.getSetCCResultType(DAG.getDataLayout(),
+                                             *DAG.getContext(), MVT::i32);
+      IsXSigned = DAG.getSetCC(DL, SetCCType, Hi, ZeroCst, ISD::SETLT);
+    }
+    if (MinMaxOp == X86ISD::FMAX) {
+      NewX = DAG.getSelect(DL, VT, IsXSigned, X, Y);
+      NewY = DAG.getSelect(DL, VT, IsXSigned, Y, X);
+    } else {
+      NewX = DAG.getSelect(DL, VT, IsXSigned, Y, X);
+      NewY = DAG.getSelect(DL, VT, IsXSigned, X, Y);
+    }
   }
 
   bool IgnoreNaN = DAG.getTarget().Options.NoNaNsFPMath ||
diff --git a/llvm/test/CodeGen/X86/avx512fp16-fminimum-fmaximum.ll b/llvm/test/CodeGen/X86/avx512fp16-fminimum-fmaximum.ll
@@ -10,9 +10,8 @@ define half @test_fminimum(half %x, half %y) {
 ; CHECK-LABEL: test_fminimum:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vmovw %xmm0, %eax
-; CHECK-NEXT:    movzwl %ax, %eax
-; CHECK-NEXT:    cmpl $32768, %eax # imm = 0x8000
-; CHECK-NEXT:    sete %al
+; CHECK-NEXT:    testw %ax, %ax
+; CHECK-NEXT:    sets %al
 ; CHECK-NEXT:    kmovd %eax, %k1
 ; CHECK-NEXT:    vmovaps %xmm1, %xmm2
 ; CHECK-NEXT:    vmovsh %xmm0, %xmm0, %xmm2 {%k1}
@@ -119,15 +118,14 @@ define half @test_fmaximum(half %x, half %y) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vmovw %xmm0, %eax
 ; CHECK-NEXT:    testw %ax, %ax
-; CHECK-NEXT:    sete %al
+; CHECK-NEXT:    sets %al
 ; CHECK-NEXT:    kmovd %eax, %k1
-; CHECK-NEXT:    vmovaps %xmm1, %xmm2
-; CHECK-NEXT:    vmovsh %xmm0, %xmm0, %xmm2 {%k1}
-; CHECK-NEXT:    vmovsh %xmm1, %xmm0, %xmm0 {%k1}
-; CHECK-NEXT:    vmaxsh %xmm2, %xmm0, %xmm1
-; CHECK-NEXT:    vcmpunordsh %xmm0, %xmm0, %k1
+; CHECK-NEXT:    vmovaps %xmm0, %xmm2
+; CHECK-NEXT:    vmovsh %xmm1, %xmm0, %xmm2 {%k1}
 ; CHECK-NEXT:    vmovsh %xmm0, %xmm0, %xmm1 {%k1}
-; CHECK-NEXT:    vmovaps %xmm1, %xmm0
+; CHECK-NEXT:    vmaxsh %xmm2, %xmm1, %xmm0
+; CHECK-NEXT:    vcmpunordsh %xmm1, %xmm1, %k1
+; CHECK-NEXT:    vmovsh %xmm1, %xmm0, %xmm0 {%k1}
 ; CHECK-NEXT:    retq
   %r = call half @llvm.maximum.f16(half %x, half %y)
   ret half %r
diff --git a/llvm/test/CodeGen/X86/extractelement-fp.ll b/llvm/test/CodeGen/X86/extractelement-fp.ll
@@ -677,35 +677,35 @@ define float @fmaximum_v4f32(<4 x float> %x, <4 x float> %y) nounwind {
 ; X64:       # %bb.0:
 ; X64-NEXT:    vmovd %xmm0, %eax
 ; X64-NEXT:    testl %eax, %eax
-; X64-NEXT:    je .LBB30_1
+; X64-NEXT:    js .LBB30_1
 ; X64-NEXT:  # %bb.2:
-; X64-NEXT:    vmovdqa %xmm1, %xmm2
+; X64-NEXT:    vmovdqa %xmm0, %xmm2
 ; X64-NEXT:    jmp .LBB30_3
 ; X64-NEXT:  .LBB30_1:
-; X64-NEXT:    vmovdqa %xmm0, %xmm2
-; X64-NEXT:    vmovdqa %xmm1, %xmm0
+; X64-NEXT:    vmovdqa %xmm1, %xmm2
+; X64-NEXT:    vmovdqa %xmm0, %xmm1
 ; X64-NEXT:  .LBB30_3:
-; X64-NEXT:    vmaxss %xmm2, %xmm0, %xmm1
-; X64-NEXT:    vcmpunordss %xmm0, %xmm0, %xmm2
-; X64-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; X64-NEXT:    vmaxss %xmm2, %xmm1, %xmm0
+; X64-NEXT:    vcmpunordss %xmm1, %xmm1, %xmm2
+; X64-NEXT:    vblendvps %xmm2, %xmm1, %xmm0, %xmm0
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: fmaximum_v4f32:
 ; X86:       # %bb.0:
 ; X86-NEXT:    vmovd %xmm0, %eax
 ; X86-NEXT:    testl %eax, %eax
-; X86-NEXT:    je .LBB30_1
+; X86-NEXT:    js .LBB30_1
 ; X86-NEXT:  # %bb.2:
-; X86-NEXT:    vmovdqa %xmm1, %xmm2
+; X86-NEXT:    vmovdqa %xmm0, %xmm2
 ; X86-NEXT:    jmp .LBB30_3
 ; X86-NEXT:  .LBB30_1:
-; X86-NEXT:    vmovdqa %xmm0, %xmm2
-; X86-NEXT:    vmovdqa %xmm1, %xmm0
+; X86-NEXT:    vmovdqa %xmm1, %xmm2
+; X86-NEXT:    vmovdqa %xmm0, %xmm1
 ; X86-NEXT:  .LBB30_3:
 ; X86-NEXT:    pushl %eax
-; X86-NEXT:    vmaxss %xmm2, %xmm0, %xmm1
-; X86-NEXT:    vcmpunordss %xmm0, %xmm0, %xmm2
-; X86-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; X86-NEXT:    vmaxss %xmm2, %xmm1, %xmm0
+; X86-NEXT:    vcmpunordss %xmm1, %xmm1, %xmm2
+; X86-NEXT:    vblendvps %xmm2, %xmm1, %xmm0, %xmm0
 ; X86-NEXT:    vmovss %xmm0, (%esp)
 ; X86-NEXT:    flds (%esp)
 ; X86-NEXT:    popl %eax
@@ -720,40 +720,39 @@ define double @fmaximum_v4f64(<4 x double> %x, <4 x double> %y) nounwind {
 ; X64:       # %bb.0:
 ; X64-NEXT:    vmovq %xmm0, %rax
 ; X64-NEXT:    testq %rax, %rax
-; X64-NEXT:    je .LBB31_1
+; X64-NEXT:    js .LBB31_1
 ; X64-NEXT:  # %bb.2:
-; X64-NEXT:    vmovdqa %xmm1, %xmm2
+; X64-NEXT:    vmovdqa %xmm0, %xmm2
 ; X64-NEXT:    jmp .LBB31_3
 ; X64-NEXT:  .LBB31_1:
-; X64-NEXT:    vmovdqa %xmm0, %xmm2
-; X64-NEXT:    vmovdqa %xmm1, %xmm0
+; X64-NEXT:    vmovdqa %xmm1, %xmm2
+; X64-NEXT:    vmovdqa %xmm0, %xmm1
 ; X64-NEXT:  .LBB31_3:
-; X64-NEXT:    vmaxsd %xmm2, %xmm0, %xmm1
-; X64-NEXT:    vcmpunordsd %xmm0, %xmm0, %xmm2
-; X64-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; X64-NEXT:    vmaxsd %xmm2, %xmm1, %xmm0
+; X64-NEXT:    vcmpunordsd %xmm1, %xmm1, %xmm2
+; X64-NEXT:    vblendvpd %xmm2, %xmm1, %xmm0, %xmm0
 ; X64-NEXT:    vzeroupper
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: fmaximum_v4f64:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpextrd $1, %xmm0, %eax
-; X86-NEXT:    vmovd %xmm0, %ecx
-; X86-NEXT:    orl %eax, %ecx
-; X86-NEXT:    je .LBB31_1
+; X86-NEXT:    vextractps $1, %xmm0, %eax
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    js .LBB31_1
 ; X86-NEXT:  # %bb.2:
-; X86-NEXT:    vmovdqa %xmm1, %xmm2
+; X86-NEXT:    vmovapd %xmm0, %xmm2
 ; X86-NEXT:    jmp .LBB31_3
 ; X86-NEXT:  .LBB31_1:
-; X86-NEXT:    vmovdqa %xmm0, %xmm2
-; X86-NEXT:    vmovdqa %xmm1, %xmm0
+; X86-NEXT:    vmovapd %xmm1, %xmm2
+; X86-NEXT:    vmovapd %xmm0, %xmm1
 ; X86-NEXT:  .LBB31_3:
 ; X86-NEXT:    pushl %ebp
 ; X86-NEXT:    movl %esp, %ebp
 ; X86-NEXT:    andl $-8, %esp
 ; X86-NEXT:    subl $8, %esp
-; X86-NEXT:    vmaxsd %xmm2, %xmm0, %xmm1
-; X86-NEXT:    vcmpunordsd %xmm0, %xmm0, %xmm2
-; X86-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; X86-NEXT:    vmaxsd %xmm2, %xmm1, %xmm0
+; X86-NEXT:    vcmpunordsd %xmm1, %xmm1, %xmm2
+; X86-NEXT:    vblendvpd %xmm2, %xmm1, %xmm0, %xmm0
 ; X86-NEXT:    vmovlpd %xmm0, (%esp)
 ; X86-NEXT:    fldl (%esp)
 ; X86-NEXT:    movl %ebp, %esp
@@ -769,8 +768,8 @@ define float @fminimum_v4f32(<4 x float> %x, <4 x float> %y) nounwind {
 ; X64-LABEL: fminimum_v4f32:
 ; X64:       # %bb.0:
 ; X64-NEXT:    vmovd %xmm0, %eax
-; X64-NEXT:    cmpl $-2147483648, %eax # imm = 0x80000000
-; X64-NEXT:    je .LBB32_1
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    js .LBB32_1
 ; X64-NEXT:  # %bb.2:
 ; X64-NEXT:    vmovdqa %xmm1, %xmm2
 ; X64-NEXT:    jmp .LBB32_3
@@ -786,8 +785,8 @@ define float @fminimum_v4f32(<4 x float> %x, <4 x float> %y) nounwind {
 ; X86-LABEL: fminimum_v4f32:
 ; X86:       # %bb.0:
 ; X86-NEXT:    vmovd %xmm0, %eax
-; X86-NEXT:    cmpl $-2147483648, %eax # imm = 0x80000000
-; X86-NEXT:    je .LBB32_1
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    js .LBB32_1
 ; X86-NEXT:  # %bb.2:
 ; X86-NEXT:    vmovdqa %xmm1, %xmm2
 ; X86-NEXT:    jmp .LBB32_3
@@ -812,9 +811,8 @@ define double @fminimum_v4f64(<4 x double> %x, <4 x double> %y) nounwind {
 ; X64-LABEL: fminimum_v4f64:
 ; X64:       # %bb.0:
 ; X64-NEXT:    vmovq %xmm0, %rax
-; X64-NEXT:    movabsq $-9223372036854775808, %rcx # imm = 0x8000000000000000
-; X64-NEXT:    cmpq %rcx, %rax
-; X64-NEXT:    je .LBB33_1
+; X64-NEXT:    testq %rax, %rax
+; X64-NEXT:    js .LBB33_1
 ; X64-NEXT:  # %bb.2:
 ; X64-NEXT:    vmovdqa %xmm1, %xmm2
 ; X64-NEXT:    jmp .LBB33_3
@@ -830,17 +828,15 @@ define double @fminimum_v4f64(<4 x double> %x, <4 x double> %y) nounwind {
 ;
 ; X86-LABEL: fminimum_v4f64:
 ; X86:       # %bb.0:
-; X86-NEXT:    vmovd %xmm0, %eax
-; X86-NEXT:    vpextrd $1, %xmm0, %ecx
-; X86-NEXT:    addl $-2147483648, %ecx # imm = 0x80000000
-; X86-NEXT:    orl %eax, %ecx
-; X86-NEXT:    je .LBB33_1
+; X86-NEXT:    vextractps $1, %xmm0, %eax
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    js .LBB33_1
 ; X86-NEXT:  # %bb.2:
-; X86-NEXT:    vmovdqa %xmm1, %xmm2
+; X86-NEXT:    vmovapd %xmm1, %xmm2
 ; X86-NEXT:    jmp .LBB33_3
 ; X86-NEXT:  .LBB33_1:
-; X86-NEXT:    vmovdqa %xmm0, %xmm2
-; X86-NEXT:    vmovdqa %xmm1, %xmm0
+; X86-NEXT:    vmovapd %xmm0, %xmm2
+; X86-NEXT:    vmovapd %xmm1, %xmm0
 ; X86-NEXT:  .LBB33_3:
 ; X86-NEXT:    pushl %ebp
 ; X86-NEXT:    movl %esp, %ebp
diff --git a/llvm/test/CodeGen/X86/fminimum-fmaximum.ll b/llvm/test/CodeGen/X86/fminimum-fmaximum.ll
