[AArch64] Lower fpto*i.sat intrinsics.

AArch64's fctv* instructions implement the saturating behaviour that the
fpto*i.sat intrinsics require, in cases where the destination width
matches the saturation width. Lowering them removes a lot of unnecessary
generated code.

Only scalar lowerings are supported for now.

Differential Revision: https://reviews.llvm.org/D102353

Author: jacobbramley

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

@@ -471,6 +471,11 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
   setOperationAction(ISD::STRICT_FP_ROUND, MVT::f32, Custom);
   setOperationAction(ISD::STRICT_FP_ROUND, MVT::f64, Custom);
 
+  setOperationAction(ISD::FP_TO_UINT_SAT, MVT::i32, Custom);
+  setOperationAction(ISD::FP_TO_UINT_SAT, MVT::i64, Custom);
+  setOperationAction(ISD::FP_TO_SINT_SAT, MVT::i32, Custom);
+  setOperationAction(ISD::FP_TO_SINT_SAT, MVT::i64, Custom);
+
   // Variable arguments.
   setOperationAction(ISD::VASTART, MVT::Other, Custom);
   setOperationAction(ISD::VAARG, MVT::Other, Custom);
@@ -876,6 +881,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
   setTargetDAGCombine(ISD::SINT_TO_FP);
   setTargetDAGCombine(ISD::UINT_TO_FP);
 
+  // TODO: Do the same for FP_TO_*INT_SAT.
   setTargetDAGCombine(ISD::FP_TO_SINT);
   setTargetDAGCombine(ISD::FP_TO_UINT);
   setTargetDAGCombine(ISD::FDIV);
@@ -3292,6 +3298,44 @@ SDValue AArch64TargetLowering::LowerFP_TO_INT(SDValue Op,
   return SDValue();
 }
 
+SDValue AArch64TargetLowering::LowerFP_TO_INT_SAT(SDValue Op,
+                                                  SelectionDAG &DAG) const {
+  // AArch64 FP-to-int conversions saturate to the destination register size, so
+  // we can lower common saturating conversions to simple instructions.
+  SDValue SrcVal = Op.getOperand(0);
+
+  EVT SrcVT = SrcVal.getValueType();
+  EVT DstVT = Op.getValueType();
+
+  EVT SatVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
+  uint64_t SatWidth = SatVT.getScalarSizeInBits();
+  uint64_t DstWidth = DstVT.getScalarSizeInBits();
+  assert(SatWidth <= DstWidth && "Saturation width cannot exceed result width");
+
+  // TODO: Support lowering of NEON and SVE conversions.
+  if (SrcVT.isVector())
+    return SDValue();
+
+  // TODO: Saturate to SatWidth explicitly.
+  if (SatWidth != DstWidth)
+    return SDValue();
+
+  // In the absence of FP16 support, promote f32 to f16, like LowerFP_TO_INT().
+  if (SrcVT == MVT::f16 && !Subtarget->hasFullFP16())
+    return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(),
+                       DAG.getNode(ISD::FP_EXTEND, SDLoc(Op), MVT::f32, SrcVal),
+                       Op.getOperand(1));
+
+  // Cases that we can emit directly.
+  if ((SrcVT == MVT::f64 || SrcVT == MVT::f32 ||
+       (SrcVT == MVT::f16 && Subtarget->hasFullFP16())) &&
+      (DstVT == MVT::i64 || DstVT == MVT::i32))
+    return Op;
+
+  // For all other cases, fall back on the expanded form.
+  return SDValue();
+}
+
 SDValue AArch64TargetLowering::LowerVectorINT_TO_FP(SDValue Op,
                                                     SelectionDAG &DAG) const {
   // Warning: We maintain cost tables in AArch64TargetTransformInfo.cpp.
@@ -4553,6 +4597,9 @@ SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
   case ISD::STRICT_FP_TO_SINT:
   case ISD::STRICT_FP_TO_UINT:
     return LowerFP_TO_INT(Op, DAG);
+  case ISD::FP_TO_SINT_SAT:
+  case ISD::FP_TO_UINT_SAT:
+    return LowerFP_TO_INT_SAT(Op, DAG);
   case ISD::FSINCOS:
     return LowerFSINCOS(Op, DAG);
   case ISD::FLT_ROUNDS_:

llvm/lib/Target/AArch64/AArch64ISelLowering.h

@@ -960,6 +960,7 @@ class AArch64TargetLowering : public TargetLowering {
   SDValue LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerVectorFP_TO_INT(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const;
+  SDValue LowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerINT_TO_FP(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerVectorINT_TO_FP(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerVectorOR(SDValue Op, SelectionDAG &DAG) const;

llvm/lib/Target/AArch64/AArch64InstrInfo.td

@@ -3692,6 +3692,25 @@ defm FCVTZU : FPToIntegerUnscaled<0b11, 0b001, "fcvtzu", any_fp_to_uint>;
 defm FCVTZS : FPToIntegerScaled<0b11, 0b000, "fcvtzs", any_fp_to_sint>;
 defm FCVTZU : FPToIntegerScaled<0b11, 0b001, "fcvtzu", any_fp_to_uint>;
 
+// AArch64's FCVT instructions saturate when out of range.
+multiclass FPToIntegerSatPats<SDNode to_int_sat, string INST> {
+  def : Pat<(i32 (to_int_sat f16:$Rn, i32)),
+            (!cast<Instruction>(INST # UWHr) f16:$Rn)>;
+  def : Pat<(i32 (to_int_sat f32:$Rn, i32)),
+            (!cast<Instruction>(INST # UWSr) f32:$Rn)>;
+  def : Pat<(i32 (to_int_sat f64:$Rn, i32)),
+            (!cast<Instruction>(INST # UWDr) f64:$Rn)>;
+  def : Pat<(i64 (to_int_sat f16:$Rn, i64)),
+            (!cast<Instruction>(INST # UXHr) f16:$Rn)>;
+  def : Pat<(i64 (to_int_sat f32:$Rn, i64)),
+            (!cast<Instruction>(INST # UXSr) f32:$Rn)>;
+  def : Pat<(i64 (to_int_sat f64:$Rn, i64)),
+            (!cast<Instruction>(INST # UXDr) f64:$Rn)>;
+}
+
+defm : FPToIntegerSatPats<fp_to_sint_sat, "FCVTZS">;
+defm : FPToIntegerSatPats<fp_to_uint_sat, "FCVTZU">;
+
 multiclass FPToIntegerIntPats<Intrinsic round, string INST> {
   def : Pat<(i32 (round f16:$Rn)), (!cast<Instruction>(INST # UWHr) $Rn)>;
   def : Pat<(i64 (round f16:$Rn)), (!cast<Instruction>(INST # UXHr) $Rn)>;
@@ -3717,7 +3736,7 @@ multiclass FPToIntegerIntPats<Intrinsic round, string INST> {
 defm : FPToIntegerIntPats<int_aarch64_neon_fcvtzs, "FCVTZS">;
 defm : FPToIntegerIntPats<int_aarch64_neon_fcvtzu, "FCVTZU">;
 
-multiclass FPToIntegerPats<SDNode to_int, SDNode round, string INST> {
+multiclass FPToIntegerPats<SDNode to_int, SDNode to_int_sat, SDNode round, string INST> {
   def : Pat<(i32 (to_int (round f32:$Rn))),
             (!cast<Instruction>(INST # UWSr) f32:$Rn)>;
   def : Pat<(i64 (to_int (round f32:$Rn))),
@@ -3726,16 +3745,32 @@ multiclass FPToIntegerPats<SDNode to_int, SDNode round, string INST> {
             (!cast<Instruction>(INST # UWDr) f64:$Rn)>;
   def : Pat<(i64 (to_int (round f64:$Rn))),
             (!cast<Instruction>(INST # UXDr) f64:$Rn)>;
+
+  // These instructions saturate like fp_to_[su]int_sat.
+  def : Pat<(i32 (to_int_sat (round f16:$Rn), i32)),
+            (!cast<Instruction>(INST # UWHr) f16:$Rn)>;
+  def : Pat<(i64 (to_int_sat (round f16:$Rn), i64)),
+            (!cast<Instruction>(INST # UXHr) f16:$Rn)>;
+  def : Pat<(i32 (to_int_sat (round f32:$Rn), i32)),
+            (!cast<Instruction>(INST # UWSr) f32:$Rn)>;
+  def : Pat<(i64 (to_int_sat (round f32:$Rn), i64)),
+            (!cast<Instruction>(INST # UXSr) f32:$Rn)>;
+  def : Pat<(i32 (to_int_sat (round f64:$Rn), i32)),
+            (!cast<Instruction>(INST # UWDr) f64:$Rn)>;
+  def : Pat<(i64 (to_int_sat (round f64:$Rn), i64)),
+            (!cast<Instruction>(INST # UXDr) f64:$Rn)>;
 }
 
-defm : FPToIntegerPats<fp_to_sint, fceil,  "FCVTPS">;
-defm : FPToIntegerPats<fp_to_uint, fceil,  "FCVTPU">;
-defm : FPToIntegerPats<fp_to_sint, ffloor, "FCVTMS">;
-defm : FPToIntegerPats<fp_to_uint, ffloor, "FCVTMU">;
-defm : FPToIntegerPats<fp_to_sint, ftrunc, "FCVTZS">;
-defm : FPToIntegerPats<fp_to_uint, ftrunc, "FCVTZU">;
-defm : FPToIntegerPats<fp_to_sint, fround, "FCVTAS">;
-defm : FPToIntegerPats<fp_to_uint, fround, "FCVTAU">;
+defm : FPToIntegerPats<fp_to_sint, fp_to_sint_sat, fceil,  "FCVTPS">;
+defm : FPToIntegerPats<fp_to_uint, fp_to_uint_sat, fceil,  "FCVTPU">;
+defm : FPToIntegerPats<fp_to_sint, fp_to_sint_sat, ffloor, "FCVTMS">;
+defm : FPToIntegerPats<fp_to_uint, fp_to_uint_sat, ffloor, "FCVTMU">;
+defm : FPToIntegerPats<fp_to_sint, fp_to_sint_sat, ftrunc, "FCVTZS">;
+defm : FPToIntegerPats<fp_to_uint, fp_to_uint_sat, ftrunc, "FCVTZU">;
+defm : FPToIntegerPats<fp_to_sint, fp_to_sint_sat, fround, "FCVTAS">;
+defm : FPToIntegerPats<fp_to_uint, fp_to_uint_sat, fround, "FCVTAU">;
+
+
 
 let Predicates = [HasFullFP16] in {
   def : Pat<(i32 (lround f16:$Rn)),

llvm/test/CodeGen/AArch64/fptosi-sat-scalar.ll

@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=aarch64 < %s | FileCheck %s
+; RUN: llc < %s -mtriple=aarch64 | FileCheck %s --check-prefixes=CHECK,CHECK-CVT
+; RUN: llc < %s -mtriple=aarch64 -mattr=+fullfp16 | FileCheck %s --check-prefixes=CHECK,CHECK-FP16
 
 ;
 ; 32-bit float to signed integer
@@ -106,19 +107,7 @@ define i19 @test_signed_i19_f32(float %f) nounwind {
 define i32 @test_signed_i32_f32(float %f) nounwind {
 ; CHECK-LABEL: test_signed_i32_f32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w9, #-822083584
-; CHECK-NEXT:    mov w11, #1325400063
-; CHECK-NEXT:    fmov s1, w9
-; CHECK-NEXT:    fcvtzs w8, s0
-; CHECK-NEXT:    mov w10, #-2147483648
-; CHECK-NEXT:    fcmp s0, s1
-; CHECK-NEXT:    fmov s1, w11
-; CHECK-NEXT:    mov w12, #2147483647
-; CHECK-NEXT:    csel w8, w10, w8, lt
-; CHECK-NEXT:    fcmp s0, s1
-; CHECK-NEXT:    csel w8, w12, w8, gt
-; CHECK-NEXT:    fcmp s0, s0
-; CHECK-NEXT:    csel w0, wzr, w8, vs
+; CHECK-NEXT:    fcvtzs w0, s0
 ; CHECK-NEXT:    ret
     %x = call i32 @llvm.fptosi.sat.i32.f32(float %f)
     ret i32 %x
@@ -148,19 +137,7 @@ define i50 @test_signed_i50_f32(float %f) nounwind {
 define i64 @test_signed_i64_f32(float %f) nounwind {
 ; CHECK-LABEL: test_signed_i64_f32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w9, #-553648128
-; CHECK-NEXT:    mov w11, #1593835519
-; CHECK-NEXT:    fmov s1, w9
-; CHECK-NEXT:    fcvtzs x8, s0
-; CHECK-NEXT:    mov x10, #-9223372036854775808
-; CHECK-NEXT:    fcmp s0, s1
-; CHECK-NEXT:    fmov s1, w11
-; CHECK-NEXT:    mov x12, #9223372036854775807
-; CHECK-NEXT:    csel x8, x10, x8, lt
-; CHECK-NEXT:    fcmp s0, s1
-; CHECK-NEXT:    csel x8, x12, x8, gt
-; CHECK-NEXT:    fcmp s0, s0
-; CHECK-NEXT:    csel x0, xzr, x8, vs
+; CHECK-NEXT:    fcvtzs x0, s0
 ; CHECK-NEXT:    ret
     %x = call i64 @llvm.fptosi.sat.i64.f32(float %f)
     ret i64 %x
@@ -330,16 +307,7 @@ define i19 @test_signed_i19_f64(double %f) nounwind {
 define i32 @test_signed_i32_f64(double %f) nounwind {
 ; CHECK-LABEL: test_signed_i32_f64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov x8, #-4476578029606273024
-; CHECK-NEXT:    mov x9, #281474972516352
-; CHECK-NEXT:    movk x9, #16863, lsl #48
-; CHECK-NEXT:    fmov d1, x8
-; CHECK-NEXT:    fmaxnm d1, d0, d1
-; CHECK-NEXT:    fmov d2, x9
-; CHECK-NEXT:    fminnm d1, d1, d2
-; CHECK-NEXT:    fcvtzs w8, d1
-; CHECK-NEXT:    fcmp d0, d0
-; CHECK-NEXT:    csel w0, wzr, w8, vs
+; CHECK-NEXT:    fcvtzs w0, d0
 ; CHECK-NEXT:    ret
     %x = call i32 @llvm.fptosi.sat.i32.f64(double %f)
     ret i32 %x
@@ -366,19 +334,7 @@ define i50 @test_signed_i50_f64(double %f) nounwind {
 define i64 @test_signed_i64_f64(double %f) nounwind {
 ; CHECK-LABEL: test_signed_i64_f64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov x9, #-4332462841530417152
-; CHECK-NEXT:    mov x11, #4890909195324358655
-; CHECK-NEXT:    fmov d1, x9
-; CHECK-NEXT:    fcvtzs x8, d0
-; CHECK-NEXT:    mov x10, #-9223372036854775808
-; CHECK-NEXT:    fcmp d0, d1
-; CHECK-NEXT:    fmov d1, x11
-; CHECK-NEXT:    mov x12, #9223372036854775807
-; CHECK-NEXT:    csel x8, x10, x8, lt
-; CHECK-NEXT:    fcmp d0, d1
-; CHECK-NEXT:    csel x8, x12, x8, gt
-; CHECK-NEXT:    fcmp d0, d0
-; CHECK-NEXT:    csel x0, xzr, x8, vs
+; CHECK-NEXT:    fcvtzs x0, d0
 ; CHECK-NEXT:    ret
     %x = call i64 @llvm.fptosi.sat.i64.f64(double %f)
     ret i64 %x
@@ -550,23 +506,16 @@ define i19 @test_signed_i19_f16(half %f) nounwind {
 }
 
 define i32 @test_signed_i32_f16(half %f) nounwind {
-; CHECK-LABEL: test_signed_i32_f16:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #-822083584
-; CHECK-NEXT:    fcvt s0, h0
-; CHECK-NEXT:    fmov s1, w8
-; CHECK-NEXT:    mov w8, #1325400063
-; CHECK-NEXT:    mov w9, #-2147483648
-; CHECK-NEXT:    fcmp s0, s1
-; CHECK-NEXT:    fmov s1, w8
-; CHECK-NEXT:    fcvtzs w8, s0
-; CHECK-NEXT:    csel w8, w9, w8, lt
-; CHECK-NEXT:    mov w9, #2147483647
-; CHECK-NEXT:    fcmp s0, s1
-; CHECK-NEXT:    csel w8, w9, w8, gt
-; CHECK-NEXT:    fcmp s0, s0
-; CHECK-NEXT:    csel w0, wzr, w8, vs
-; CHECK-NEXT:    ret
+; CHECK-CVT-LABEL: test_signed_i32_f16:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    fcvt s0, h0
+; CHECK-CVT-NEXT:    fcvtzs w0, s0
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-FP16-LABEL: test_signed_i32_f16:
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    fcvtzs w0, h0
+; CHECK-FP16-NEXT:    ret
     %x = call i32 @llvm.fptosi.sat.i32.f16(half %f)
     ret i32 %x
 }
@@ -594,23 +543,16 @@ define i50 @test_signed_i50_f16(half %f) nounwind {
 }
 
 define i64 @test_signed_i64_f16(half %f) nounwind {
-; CHECK-LABEL: test_signed_i64_f16:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #-553648128
-; CHECK-NEXT:    fcvt s0, h0
-; CHECK-NEXT:    fmov s1, w8
-; CHECK-NEXT:    mov w8, #1593835519
-; CHECK-NEXT:    mov x9, #-9223372036854775808
-; CHECK-NEXT:    fcmp s0, s1
-; CHECK-NEXT:    fmov s1, w8
-; CHECK-NEXT:    fcvtzs x8, s0
-; CHECK-NEXT:    csel x8, x9, x8, lt
-; CHECK-NEXT:    mov x9, #9223372036854775807
-; CHECK-NEXT:    fcmp s0, s1
-; CHECK-NEXT:    csel x8, x9, x8, gt
-; CHECK-NEXT:    fcmp s0, s0
-; CHECK-NEXT:    csel x0, xzr, x8, vs
-; CHECK-NEXT:    ret
+; CHECK-CVT-LABEL: test_signed_i64_f16:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    fcvt s0, h0
+; CHECK-CVT-NEXT:    fcvtzs x0, s0
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-FP16-LABEL: test_signed_i64_f16:
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    fcvtzs x0, h0
+; CHECK-FP16-NEXT:    ret
     %x = call i64 @llvm.fptosi.sat.i64.f16(half %f)
     ret i64 %x
 }