[X86] Add ISD::ABDS/ABDU vXi64 support on SSE41+ targets

If IMINMAX ops aren't legal, we can lower to the select(icmp(x,y),sub(x,y),sub(y,x)) pattern

Author: RKSimon

llvm/lib/Target/X86/X86ISelLowering.cpp

@@ -1256,7 +1256,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
     setOperationAction(ISD::UMIN,               MVT::v8i16, Legal);
     setOperationAction(ISD::UMIN,               MVT::v4i32, Legal);
 
-    for (auto VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32}) {
+    for (auto VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64}) {
       setOperationAction(ISD::ABDS,             VT, Custom);
       setOperationAction(ISD::ABDU,             VT, Custom);
     }
@@ -1396,14 +1396,16 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
     // In the customized shift lowering, the legal v8i32/v4i64 cases
     // in AVX2 will be recognized.
     for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {
-      setOperationAction(ISD::SRL, VT, Custom);
-      setOperationAction(ISD::SHL, VT, Custom);
-      setOperationAction(ISD::SRA, VT, Custom);
+      setOperationAction(ISD::SRL,             VT, Custom);
+      setOperationAction(ISD::SHL,             VT, Custom);
+      setOperationAction(ISD::SRA,             VT, Custom);
+      setOperationAction(ISD::ABDS,            VT, Custom);
+      setOperationAction(ISD::ABDU,            VT, Custom);
       if (VT == MVT::v4i64) continue;
-      setOperationAction(ISD::ROTL, VT, Custom);
-      setOperationAction(ISD::ROTR, VT, Custom);
-      setOperationAction(ISD::FSHL, VT, Custom);
-      setOperationAction(ISD::FSHR, VT, Custom);
+      setOperationAction(ISD::ROTL,            VT, Custom);
+      setOperationAction(ISD::ROTR,            VT, Custom);
+      setOperationAction(ISD::FSHL,            VT, Custom);
+      setOperationAction(ISD::FSHR,            VT, Custom);
     }
 
     // These types need custom splitting if their input is a 128-bit vector.
@@ -1499,8 +1501,6 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
       setOperationAction(ISD::UMAX, VT, HasInt256 ? Legal : Custom);
       setOperationAction(ISD::SMIN, VT, HasInt256 ? Legal : Custom);
       setOperationAction(ISD::UMIN, VT, HasInt256 ? Legal : Custom);
-      setOperationAction(ISD::ABDS, VT, Custom);
-      setOperationAction(ISD::ABDU, VT, Custom);
     }
 
     for (auto VT : {MVT::v16i16, MVT::v8i32, MVT::v4i64}) {
@@ -1968,8 +1968,6 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
       setOperationAction(ISD::SMIN, VT, Legal);
       setOperationAction(ISD::UMIN, VT, Legal);
       setOperationAction(ISD::ABS,  VT, Legal);
-      setOperationAction(ISD::ABDS, VT, Custom);
-      setOperationAction(ISD::ABDU, VT, Custom);
     }
 
     for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64 }) {
@@ -29659,15 +29657,30 @@ static SDValue LowerABD(SDValue Op, const X86Subtarget &Subtarget,
   if ((VT == MVT::v32i16 || VT == MVT::v64i8) && !Subtarget.useBWIRegs())
     return splitVectorIntBinary(Op, DAG);
 
-  // Default to expand: sub(smax(lhs,rhs),smin(lhs,rhs))
   // TODO: Add TargetLowering expandABD() support.
   SDLoc dl(Op);
   bool IsSigned = Op.getOpcode() == ISD::ABDS;
   SDValue LHS = DAG.getFreeze(Op.getOperand(0));
   SDValue RHS = DAG.getFreeze(Op.getOperand(1));
-  SDValue Max = DAG.getNode(IsSigned ? ISD::SMAX : ISD::UMAX, dl, VT, LHS, RHS);
-  SDValue Min = DAG.getNode(IsSigned ? ISD::SMIN : ISD::UMIN, dl, VT, LHS, RHS);
-  return DAG.getNode(ISD::SUB, dl, VT, Max, Min);
+  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+
+  // abds(lhs, rhs) -> sub(smax(lhs,rhs), smin(lhs,rhs))
+  // abdu(lhs, rhs) -> sub(umax(lhs,rhs), umin(lhs,rhs))
+  unsigned MaxOpc = IsSigned ? ISD::SMAX : ISD::UMAX;
+  unsigned MinOpc = IsSigned ? ISD::SMIN : ISD::UMIN;
+  if (TLI.isOperationLegal(MaxOpc, VT) && TLI.isOperationLegal(MinOpc, VT)) {
+    SDValue Max = DAG.getNode(MaxOpc, dl, VT, LHS, RHS);
+    SDValue Min = DAG.getNode(MinOpc, dl, VT, LHS, RHS);
+    return DAG.getNode(ISD::SUB, dl, VT, Max, Min);
+  }
+
+  // abds(lhs, rhs) -> select(sgt(lhs,rhs), sub(lhs,rhs), sub(rhs,lhs))
+  // abdu(lhs, rhs) -> select(ugt(lhs,rhs), sub(lhs,rhs), sub(rhs,lhs))
+  EVT CCVT = TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
+  ISD::CondCode CC = IsSigned ? ISD::CondCode::SETGT : ISD::CondCode::SETUGT;
+  SDValue Cmp = DAG.getSetCC(dl, CCVT, LHS, RHS, CC);
+  return DAG.getSelect(dl, VT, Cmp, DAG.getNode(ISD::SUB, dl, VT, LHS, RHS),
+                       DAG.getNode(ISD::SUB, dl, VT, RHS, LHS));
 }
 
 static SDValue LowerMUL(SDValue Op, const X86Subtarget &Subtarget,

llvm/test/CodeGen/X86/abds-vector-128.ll

@@ -501,89 +501,30 @@ define <2 x i64> @abd_ext_v2i64(<2 x i64> %a, <2 x i64> %b) nounwind {
 ;
 ; SSE42-LABEL: abd_ext_v2i64:
 ; SSE42:       # %bb.0:
-; SSE42-NEXT:    movq %xmm0, %rax
-; SSE42-NEXT:    movq %rax, %rcx
-; SSE42-NEXT:    sarq $63, %rcx
-; SSE42-NEXT:    pextrq $1, %xmm0, %rdx
-; SSE42-NEXT:    movq %rdx, %rsi
-; SSE42-NEXT:    sarq $63, %rsi
-; SSE42-NEXT:    movq %xmm1, %rdi
-; SSE42-NEXT:    movq %rdi, %r8
-; SSE42-NEXT:    sarq $63, %r8
-; SSE42-NEXT:    pextrq $1, %xmm1, %r9
-; SSE42-NEXT:    movq %r9, %r10
-; SSE42-NEXT:    sarq $63, %r10
-; SSE42-NEXT:    subq %r9, %rdx
-; SSE42-NEXT:    sbbq %r10, %rsi
-; SSE42-NEXT:    subq %rdi, %rax
-; SSE42-NEXT:    sbbq %r8, %rcx
-; SSE42-NEXT:    sarq $63, %rcx
-; SSE42-NEXT:    xorq %rcx, %rax
-; SSE42-NEXT:    subq %rcx, %rax
-; SSE42-NEXT:    sarq $63, %rsi
-; SSE42-NEXT:    xorq %rsi, %rdx
-; SSE42-NEXT:    subq %rsi, %rdx
-; SSE42-NEXT:    movq %rdx, %xmm1
-; SSE42-NEXT:    movq %rax, %xmm0
-; SSE42-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE42-NEXT:    movdqa %xmm0, %xmm2
+; SSE42-NEXT:    pcmpgtq %xmm1, %xmm2
+; SSE42-NEXT:    movdqa %xmm0, %xmm3
+; SSE42-NEXT:    psubq %xmm1, %xmm3
+; SSE42-NEXT:    psubq %xmm0, %xmm1
+; SSE42-NEXT:    movdqa %xmm2, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm3, %xmm1
+; SSE42-NEXT:    movapd %xmm1, %xmm0
 ; SSE42-NEXT:    retq
 ;
 ; AVX1-LABEL: abd_ext_v2i64:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovq %xmm0, %rax
-; AVX1-NEXT:    movq %rax, %rcx
-; AVX1-NEXT:    sarq $63, %rcx
-; AVX1-NEXT:    vpextrq $1, %xmm0, %rdx
-; AVX1-NEXT:    movq %rdx, %rsi
-; AVX1-NEXT:    sarq $63, %rsi
-; AVX1-NEXT:    vmovq %xmm1, %rdi
-; AVX1-NEXT:    movq %rdi, %r8
-; AVX1-NEXT:    sarq $63, %r8
-; AVX1-NEXT:    vpextrq $1, %xmm1, %r9
-; AVX1-NEXT:    movq %r9, %r10
-; AVX1-NEXT:    sarq $63, %r10
-; AVX1-NEXT:    subq %r9, %rdx
-; AVX1-NEXT:    sbbq %r10, %rsi
-; AVX1-NEXT:    subq %rdi, %rax
-; AVX1-NEXT:    sbbq %r8, %rcx
-; AVX1-NEXT:    sarq $63, %rcx
-; AVX1-NEXT:    xorq %rcx, %rax
-; AVX1-NEXT:    subq %rcx, %rax
-; AVX1-NEXT:    sarq $63, %rsi
-; AVX1-NEXT:    xorq %rsi, %rdx
-; AVX1-NEXT:    subq %rsi, %rdx
-; AVX1-NEXT:    vmovq %rdx, %xmm0
-; AVX1-NEXT:    vmovq %rax, %xmm1
-; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm2
+; AVX1-NEXT:    vpsubq %xmm1, %xmm0, %xmm3
+; AVX1-NEXT:    vpsubq %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vblendvpd %xmm2, %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: abd_ext_v2i64:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovq %xmm0, %rax
-; AVX2-NEXT:    movq %rax, %rcx
-; AVX2-NEXT:    sarq $63, %rcx
-; AVX2-NEXT:    vpextrq $1, %xmm0, %rdx
-; AVX2-NEXT:    movq %rdx, %rsi
-; AVX2-NEXT:    sarq $63, %rsi
-; AVX2-NEXT:    vmovq %xmm1, %rdi
-; AVX2-NEXT:    movq %rdi, %r8
-; AVX2-NEXT:    sarq $63, %r8
-; AVX2-NEXT:    vpextrq $1, %xmm1, %r9
-; AVX2-NEXT:    movq %r9, %r10
-; AVX2-NEXT:    sarq $63, %r10
-; AVX2-NEXT:    subq %r9, %rdx
-; AVX2-NEXT:    sbbq %r10, %rsi
-; AVX2-NEXT:    subq %rdi, %rax
-; AVX2-NEXT:    sbbq %r8, %rcx
-; AVX2-NEXT:    sarq $63, %rcx
-; AVX2-NEXT:    xorq %rcx, %rax
-; AVX2-NEXT:    subq %rcx, %rax
-; AVX2-NEXT:    sarq $63, %rsi
-; AVX2-NEXT:    xorq %rsi, %rdx
-; AVX2-NEXT:    subq %rsi, %rdx
-; AVX2-NEXT:    vmovq %rdx, %xmm0
-; AVX2-NEXT:    vmovq %rax, %xmm1
-; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX2-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm2
+; AVX2-NEXT:    vpsubq %xmm1, %xmm0, %xmm3
+; AVX2-NEXT:    vpsubq %xmm0, %xmm1, %xmm0
+; AVX2-NEXT:    vblendvpd %xmm2, %xmm3, %xmm0, %xmm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: abd_ext_v2i64:
@@ -634,89 +575,30 @@ define <2 x i64> @abd_ext_v2i64_undef(<2 x i64> %a, <2 x i64> %b) nounwind {
 ;
 ; SSE42-LABEL: abd_ext_v2i64_undef:
 ; SSE42:       # %bb.0:
-; SSE42-NEXT:    movq %xmm0, %rax
-; SSE42-NEXT:    movq %rax, %rcx
-; SSE42-NEXT:    sarq $63, %rcx
-; SSE42-NEXT:    pextrq $1, %xmm0, %rdx
-; SSE42-NEXT:    movq %rdx, %rsi
-; SSE42-NEXT:    sarq $63, %rsi
-; SSE42-NEXT:    movq %xmm1, %rdi
-; SSE42-NEXT:    movq %rdi, %r8
-; SSE42-NEXT:    sarq $63, %r8
-; SSE42-NEXT:    pextrq $1, %xmm1, %r9
-; SSE42-NEXT:    movq %r9, %r10
-; SSE42-NEXT:    sarq $63, %r10
-; SSE42-NEXT:    subq %r9, %rdx
-; SSE42-NEXT:    sbbq %r10, %rsi
-; SSE42-NEXT:    subq %rdi, %rax
-; SSE42-NEXT:    sbbq %r8, %rcx
-; SSE42-NEXT:    sarq $63, %rcx
-; SSE42-NEXT:    xorq %rcx, %rax
-; SSE42-NEXT:    subq %rcx, %rax
-; SSE42-NEXT:    sarq $63, %rsi
-; SSE42-NEXT:    xorq %rsi, %rdx
-; SSE42-NEXT:    subq %rsi, %rdx
-; SSE42-NEXT:    movq %rdx, %xmm1
-; SSE42-NEXT:    movq %rax, %xmm0
-; SSE42-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE42-NEXT:    movdqa %xmm0, %xmm2
+; SSE42-NEXT:    pcmpgtq %xmm1, %xmm2
+; SSE42-NEXT:    movdqa %xmm0, %xmm3
+; SSE42-NEXT:    psubq %xmm1, %xmm3
+; SSE42-NEXT:    psubq %xmm0, %xmm1
+; SSE42-NEXT:    movdqa %xmm2, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm3, %xmm1
+; SSE42-NEXT:    movapd %xmm1, %xmm0
 ; SSE42-NEXT:    retq
 ;
 ; AVX1-LABEL: abd_ext_v2i64_undef:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovq %xmm0, %rax
-; AVX1-NEXT:    movq %rax, %rcx
-; AVX1-NEXT:    sarq $63, %rcx
-; AVX1-NEXT:    vpextrq $1, %xmm0, %rdx
-; AVX1-NEXT:    movq %rdx, %rsi
-; AVX1-NEXT:    sarq $63, %rsi
-; AVX1-NEXT:    vmovq %xmm1, %rdi
-; AVX1-NEXT:    movq %rdi, %r8
-; AVX1-NEXT:    sarq $63, %r8
-; AVX1-NEXT:    vpextrq $1, %xmm1, %r9
-; AVX1-NEXT:    movq %r9, %r10
-; AVX1-NEXT:    sarq $63, %r10
-; AVX1-NEXT:    subq %r9, %rdx
-; AVX1-NEXT:    sbbq %r10, %rsi
-; AVX1-NEXT:    subq %rdi, %rax
-; AVX1-NEXT:    sbbq %r8, %rcx
-; AVX1-NEXT:    sarq $63, %rcx
-; AVX1-NEXT:    xorq %rcx, %rax
-; AVX1-NEXT:    subq %rcx, %rax
-; AVX1-NEXT:    sarq $63, %rsi
-; AVX1-NEXT:    xorq %rsi, %rdx
-; AVX1-NEXT:    subq %rsi, %rdx
-; AVX1-NEXT:    vmovq %rdx, %xmm0
-; AVX1-NEXT:    vmovq %rax, %xmm1
-; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm2
+; AVX1-NEXT:    vpsubq %xmm1, %xmm0, %xmm3
+; AVX1-NEXT:    vpsubq %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vblendvpd %xmm2, %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: abd_ext_v2i64_undef:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovq %xmm0, %rax
-; AVX2-NEXT:    movq %rax, %rcx
-; AVX2-NEXT:    sarq $63, %rcx
-; AVX2-NEXT:    vpextrq $1, %xmm0, %rdx
-; AVX2-NEXT:    movq %rdx, %rsi
-; AVX2-NEXT:    sarq $63, %rsi
-; AVX2-NEXT:    vmovq %xmm1, %rdi
-; AVX2-NEXT:    movq %rdi, %r8
-; AVX2-NEXT:    sarq $63, %r8
-; AVX2-NEXT:    vpextrq $1, %xmm1, %r9
-; AVX2-NEXT:    movq %r9, %r10
-; AVX2-NEXT:    sarq $63, %r10
-; AVX2-NEXT:    subq %r9, %rdx
-; AVX2-NEXT:    sbbq %r10, %rsi
-; AVX2-NEXT:    subq %rdi, %rax
-; AVX2-NEXT:    sbbq %r8, %rcx
-; AVX2-NEXT:    sarq $63, %rcx
-; AVX2-NEXT:    xorq %rcx, %rax
-; AVX2-NEXT:    subq %rcx, %rax
-; AVX2-NEXT:    sarq $63, %rsi
-; AVX2-NEXT:    xorq %rsi, %rdx
-; AVX2-NEXT:    subq %rsi, %rdx
-; AVX2-NEXT:    vmovq %rdx, %xmm0
-; AVX2-NEXT:    vmovq %rax, %xmm1
-; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX2-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm2
+; AVX2-NEXT:    vpsubq %xmm1, %xmm0, %xmm3
+; AVX2-NEXT:    vpsubq %xmm0, %xmm1, %xmm0
+; AVX2-NEXT:    vblendvpd %xmm2, %xmm3, %xmm0, %xmm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: abd_ext_v2i64_undef:
@@ -866,28 +748,29 @@ define <2 x i64> @abd_minmax_v2i64(<2 x i64> %a, <2 x i64> %b) nounwind {
 ; SSE42-LABEL: abd_minmax_v2i64:
 ; SSE42:       # %bb.0:
 ; SSE42-NEXT:    movdqa %xmm0, %xmm2
-; SSE42-NEXT:    pcmpgtq %xmm1, %xmm0
-; SSE42-NEXT:    movdqa %xmm2, %xmm3
-; SSE42-NEXT:    blendvpd %xmm0, %xmm1, %xmm3
-; SSE42-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
-; SSE42-NEXT:    psubq %xmm3, %xmm1
-; SSE42-NEXT:    movdqa %xmm1, %xmm0
+; SSE42-NEXT:    pcmpgtq %xmm1, %xmm2
+; SSE42-NEXT:    movdqa %xmm0, %xmm3
+; SSE42-NEXT:    psubq %xmm1, %xmm3
+; SSE42-NEXT:    psubq %xmm0, %xmm1
+; SSE42-NEXT:    movdqa %xmm2, %xmm0
+; SSE42-NEXT:    blendvpd %xmm0, %xmm3, %xmm1
+; SSE42-NEXT:    movapd %xmm1, %xmm0
 ; SSE42-NEXT:    retq
 ;
 ; AVX1-LABEL: abd_minmax_v2i64:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm2
-; AVX1-NEXT:    vblendvpd %xmm2, %xmm1, %xmm0, %xmm3
-; AVX1-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
-; AVX1-NEXT:    vpsubq %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpsubq %xmm1, %xmm0, %xmm3
+; AVX1-NEXT:    vpsubq %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vblendvpd %xmm2, %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: abd_minmax_v2i64:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm2
-; AVX2-NEXT:    vblendvpd %xmm2, %xmm1, %xmm0, %xmm3
-; AVX2-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
-; AVX2-NEXT:    vpsubq %xmm3, %xmm0, %xmm0
+; AVX2-NEXT:    vpsubq %xmm1, %xmm0, %xmm3
+; AVX2-NEXT:    vpsubq %xmm0, %xmm1, %xmm0
+; AVX2-NEXT:    vblendvpd %xmm2, %xmm3, %xmm0, %xmm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: abd_minmax_v2i64: