ConstantFolding: Do not fold fcmp of denormal without known mode (#115407)

Fixes #114947

Author: arsenm

llvm/lib/Analysis/ConstantFolding.cpp

@@ -1266,14 +1266,16 @@ Constant *llvm::ConstantFoldCompareInstOperands(
     return ConstantFoldCompareInstOperands(Predicate, Ops1, Ops0, DL, TLI);
   }
 
-  // Flush any denormal constant float input according to denormal handling
-  // mode.
-  Ops0 = FlushFPConstant(Ops0, I, /* IsOutput */ false);
-  if (!Ops0)
-    return nullptr;
-  Ops1 = FlushFPConstant(Ops1, I, /* IsOutput */ false);
-  if (!Ops1)
-    return nullptr;
+  if (CmpInst::isFPPredicate(Predicate)) {
+    // Flush any denormal constant float input according to denormal handling
+    // mode.
+    Ops0 = FlushFPConstant(Ops0, I, /*IsOutput=*/false);
+    if (!Ops0)
+      return nullptr;
+    Ops1 = FlushFPConstant(Ops1, I, /*IsOutput=*/false);
+    if (!Ops1)
+      return nullptr;
+  }
 
   return ConstantFoldCompareInstruction(Predicate, Ops0, Ops1);
 }
@@ -1298,47 +1300,110 @@ Constant *llvm::ConstantFoldBinaryOpOperands(unsigned Opcode, Constant *LHS,
   return ConstantFoldBinaryInstruction(Opcode, LHS, RHS);
 }
 
-Constant *llvm::FlushFPConstant(Constant *Operand, const Instruction *I,
-                                bool IsOutput) {
-  if (!I || !I->getParent() || !I->getFunction())
-    return Operand;
+static ConstantFP *flushDenormalConstant(Type *Ty, const APFloat &APF,
+                                         DenormalMode::DenormalModeKind Mode) {
+  switch (Mode) {
+  case DenormalMode::Dynamic:
+    return nullptr;
+  case DenormalMode::IEEE:
+    return ConstantFP::get(Ty->getContext(), APF);
+  case DenormalMode::PreserveSign:
+    return ConstantFP::get(
+        Ty->getContext(),
+        APFloat::getZero(APF.getSemantics(), APF.isNegative()));
+  case DenormalMode::PositiveZero:
+    return ConstantFP::get(Ty->getContext(),
+                           APFloat::getZero(APF.getSemantics(), false));
+  default:
+    break;
+  }
 
-  ConstantFP *CFP = dyn_cast<ConstantFP>(Operand);
-  if (!CFP)
-    return Operand;
+  llvm_unreachable("unknown denormal mode");
+}
+
+/// Return the denormal mode that can be assumed when executing a floating point
+/// operation at \p CtxI.
+static DenormalMode getInstrDenormalMode(const Instruction *CtxI, Type *Ty) {
+  if (!CtxI || !CtxI->getParent() || !CtxI->getFunction())
+    return DenormalMode::getDynamic();
+  return CtxI->getFunction()->getDenormalMode(Ty->getFltSemantics());
+}
 
+static ConstantFP *flushDenormalConstantFP(ConstantFP *CFP,
+                                           const Instruction *Inst,
+                                           bool IsOutput) {
   const APFloat &APF = CFP->getValueAPF();
-  // TODO: Should this canonicalize nans?
   if (!APF.isDenormal())
-    return Operand;
+    return CFP;
 
-  Type *Ty = CFP->getType();
-  DenormalMode DenormMode =
-      I->getFunction()->getDenormalMode(Ty->getFltSemantics());
-  DenormalMode::DenormalModeKind Mode =
-      IsOutput ? DenormMode.Output : DenormMode.Input;
-  switch (Mode) {
-  default:
-    llvm_unreachable("unknown denormal mode");
-  case DenormalMode::Dynamic:
-    return nullptr;
-  case DenormalMode::IEEE:
+  DenormalMode Mode = getInstrDenormalMode(Inst, CFP->getType());
+  return flushDenormalConstant(CFP->getType(), APF,
+                               IsOutput ? Mode.Output : Mode.Input);
+}
+
+Constant *llvm::FlushFPConstant(Constant *Operand, const Instruction *Inst,
+                                bool IsOutput) {
+  if (ConstantFP *CFP = dyn_cast<ConstantFP>(Operand))
+    return flushDenormalConstantFP(CFP, Inst, IsOutput);
+
+  if (isa<ConstantAggregateZero, UndefValue, ConstantExpr>(Operand))
     return Operand;
-  case DenormalMode::PreserveSign:
-    if (APF.isDenormal()) {
-      return ConstantFP::get(
-          Ty->getContext(),
-          APFloat::getZero(Ty->getFltSemantics(), APF.isNegative()));
+
+  Type *Ty = Operand->getType();
+  VectorType *VecTy = dyn_cast<VectorType>(Ty);
+  if (VecTy) {
+    if (auto *Splat = dyn_cast_or_null<ConstantFP>(Operand->getSplatValue())) {
+      ConstantFP *Folded = flushDenormalConstantFP(Splat, Inst, IsOutput);
+      if (!Folded)
+        return nullptr;
+      return ConstantVector::getSplat(VecTy->getElementCount(), Folded);
     }
-    return Operand;
-  case DenormalMode::PositiveZero:
-    if (APF.isDenormal()) {
-      return ConstantFP::get(Ty->getContext(),
-                             APFloat::getZero(Ty->getFltSemantics(), false));
+
+    Ty = VecTy->getElementType();
+  }
+
+  if (const auto *CV = dyn_cast<ConstantVector>(Operand)) {
+    SmallVector<Constant *, 16> NewElts;
+    for (unsigned i = 0, e = CV->getNumOperands(); i != e; ++i) {
+      Constant *Element = CV->getAggregateElement(i);
+      if (isa<UndefValue>(Element)) {
+        NewElts.push_back(Element);
+        continue;
+      }
+
+      ConstantFP *CFP = dyn_cast<ConstantFP>(Element);
+      if (!CFP)
+        return nullptr;
+
+      ConstantFP *Folded = flushDenormalConstantFP(CFP, Inst, IsOutput);
+      if (!Folded)
+        return nullptr;
+      NewElts.push_back(Folded);
     }
-    return Operand;
+
+    return ConstantVector::get(NewElts);
+  }
+
+  if (const auto *CDV = dyn_cast<ConstantDataVector>(Operand)) {
+    SmallVector<Constant *, 16> NewElts;
+    for (unsigned I = 0, E = CDV->getNumElements(); I < E; ++I) {
+      const APFloat &Elt = CDV->getElementAsAPFloat(I);
+      if (!Elt.isDenormal()) {
+        NewElts.push_back(ConstantFP::get(Ty, Elt));
+      } else {
+        DenormalMode Mode = getInstrDenormalMode(Inst, Ty);
+        ConstantFP *Folded =
+            flushDenormalConstant(Ty, Elt, IsOutput ? Mode.Output : Mode.Input);
+        if (!Folded)
+          return nullptr;
+        NewElts.push_back(Folded);
+      }
+    }
+
+    return ConstantVector::get(NewElts);
   }
-  return Operand;
+
+  return nullptr;
 }
 
 Constant *llvm::ConstantFoldFPInstOperands(unsigned Opcode, Constant *LHS,

llvm/test/Transforms/SCCP/no-fold-fcmp-dynamic-denormal-mode-issue114947.ll

@@ -0,0 +1,118 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -passes=ipsccp < %s | FileCheck %s
+
+
+define i1 @no_fold_fcmp_denormal_double_ieee_dynamic_denormal_undef() #0 {
+; CHECK-LABEL: define i1 @no_fold_fcmp_denormal_double_ieee_dynamic_denormal_undef(
+; CHECK-SAME: ) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp une double 0x8000000000000, undef
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %cmp = fcmp une double 0x8000000000000, undef
+  ret i1 %cmp
+}
+
+define i1 @no_fold_fcmp_denormal_double_ieee_dynamic_denormal_poison() #0 {
+; CHECK-LABEL: define i1 @no_fold_fcmp_denormal_double_ieee_dynamic_denormal_poison(
+; CHECK-SAME: ) #[[ATTR0]] {
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp une double 0x8000000000000, poison
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %cmp = fcmp une double 0x8000000000000, poison
+  ret i1 %cmp
+}
+
+define i1 @no_fold_fcmp_denormal_double_ieee_dynamic() #0 {
+; CHECK-LABEL: define i1 @no_fold_fcmp_denormal_double_ieee_dynamic(
+; CHECK-SAME: ) #[[ATTR0]] {
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp une double 0x8000000000000, 0.000000e+00
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %cmp = fcmp une double 0x8000000000000, 0.0
+  ret i1 %cmp
+}
+
+define i1 @fold_fcmp_nondenormal_double_ieee_dynamic() #0 {
+; CHECK-LABEL: define i1 @fold_fcmp_nondenormal_double_ieee_dynamic(
+; CHECK-SAME: ) #[[ATTR0]] {
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp = fcmp une double 2.0, 0.0
+  ret i1 %cmp
+}
+
+define <2 x i1> @no_fold_fcmp_denormal_double_ieee_dynamic_vector_splat() #0 {
+; CHECK-LABEL: define <2 x i1> @no_fold_fcmp_denormal_double_ieee_dynamic_vector_splat(
+; CHECK-SAME: ) #[[ATTR0]] {
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp une <2 x double> splat (double 0x8000000000000), zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %cmp = fcmp une <2 x double> splat (double 0x8000000000000), zeroinitializer
+  ret <2 x i1> %cmp
+}
+
+define <2 x i1> @fold_fcmp_nondenormal_double_ieee_dynamic_vector_splat() #0 {
+; CHECK-LABEL: define <2 x i1> @fold_fcmp_nondenormal_double_ieee_dynamic_vector_splat(
+; CHECK-SAME: ) #[[ATTR0]] {
+; CHECK-NEXT:    ret <2 x i1> splat (i1 true)
+;
+  %cmp = fcmp une <2 x double> splat (double 2.0), zeroinitializer
+  ret <2 x i1> %cmp
+}
+
+define <2 x i1> @fold_fcmp_nondenormal_double_ieee_dynamic_vector_nonsplat() #0 {
+; CHECK-LABEL: define <2 x i1> @fold_fcmp_nondenormal_double_ieee_dynamic_vector_nonsplat(
+; CHECK-SAME: ) #[[ATTR0]] {
+; CHECK-NEXT:    ret <2 x i1> <i1 false, i1 true>
+;
+  %cmp = fcmp une <2 x double> <double 2.0, double 4.0>, <double 2.0, double 8.0>
+  ret <2 x i1> %cmp
+}
+
+define <3 x i1> @fold_fcmp_nondenormal_double_ieee_dynamic_vector_nonsplat_undef() #0 {
+; CHECK-LABEL: define <3 x i1> @fold_fcmp_nondenormal_double_ieee_dynamic_vector_nonsplat_undef(
+; CHECK-SAME: ) #[[ATTR0]] {
+; CHECK-NEXT:    ret <3 x i1> <i1 true, i1 true, i1 false>
+;
+  %cmp = fcmp une <3 x double> <double 2.0, double undef, double 4.0>, <double 1.0, double undef, double 4.0>
+  ret <3 x i1> %cmp
+}
+
+define <2 x i1> @fold_fcmp_nondenormal_double_ieee_dynamic_zero() #0 {
+; CHECK-LABEL: define <2 x i1> @fold_fcmp_nondenormal_double_ieee_dynamic_zero(
+; CHECK-SAME: ) #[[ATTR0]] {
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %cmp = fcmp une <2 x double> zeroinitializer, zeroinitializer
+  ret <2 x i1> %cmp
+}
+
+define <2 x i1> @no_fold_fcmp_denormal_double_ieee_dynamic_vector_nonsplat() #0 {
+; CHECK-LABEL: define <2 x i1> @no_fold_fcmp_denormal_double_ieee_dynamic_vector_nonsplat(
+; CHECK-SAME: ) #[[ATTR0]] {
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp une <2 x double> <double 0x8000000000000, double 1.000000e+00>, zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %cmp = fcmp une <2 x double> <double 0x8000000000000, double 1.0>, zeroinitializer
+  ret <2 x i1> %cmp
+}
+
+define <vscale x 2 x i1> @fold_fcmp_nondenormal_double_ieee_dynamic_scalable_vector_splat() #0 {
+; CHECK-LABEL: define <vscale x 2 x i1> @fold_fcmp_nondenormal_double_ieee_dynamic_scalable_vector_splat(
+; CHECK-SAME: ) #[[ATTR0]] {
+; CHECK-NEXT:    ret <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer)
+;
+  %cmp = fcmp une <vscale x 2 x double> splat (double 2.0), zeroinitializer
+  ret <vscale x 2 x i1> %cmp
+}
+
+define <vscale x 2 x i1> @no_fold_fcmp_denormal_double_ieee_dynamic_scalaable_vector_splat() #0 {
+; CHECK-LABEL: define <vscale x 2 x i1> @no_fold_fcmp_denormal_double_ieee_dynamic_scalaable_vector_splat(
+; CHECK-SAME: ) #[[ATTR0]] {
+; CHECK-NEXT:    ret <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer)
+;
+  %cmp = fcmp une <vscale x 2 x double> splat (double 0x8000000000000), zeroinitializer
+  ret <vscale x 2 x i1> %cmp
+}
+
+attributes #0 = { "denormal-fp-math"="ieee,dynamic" }