[SimplifyLibCalls] Constant fold nan libcall

llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp

@@ -3858,6 +3858,22 @@ Value *LibCallSimplifier::optimizeStringMemoryLibCall(CallInst *CI,
   return nullptr;
 }
 
+/// Constant folding nan/nanf/nanl.
+static Value *optimizeNaN(CallInst *CI) {
+  StringRef CharSeq;
+  if (!getConstantStringInfo(CI->getArgOperand(0), CharSeq))
+    return nullptr;
+
+  APInt Fill;
+  // Treat empty strings as if they were zero.
+  if (CharSeq.empty())
+    Fill = APInt(32, 0);
+  else if (CharSeq.getAsInteger(0, Fill))
+    return nullptr;
+
+  return ConstantFP::getQNaN(CI->getType(), /*Negative=*/false, &Fill);
+}
+
 Value *LibCallSimplifier::optimizeFloatingPointLibCall(CallInst *CI,
                                                        LibFunc Func,
                                                        IRBuilderBase &Builder) {
@@ -3972,6 +3988,10 @@ Value *LibCallSimplifier::optimizeFloatingPointLibCall(CallInst *CI,
   case LibFunc_remquof:
   case LibFunc_remquol:
     return optimizeRemquo(CI, Builder);
+  case LibFunc_nan:
+  case LibFunc_nanf:
+  case LibFunc_nanl:
+    return optimizeNaN(CI);
   default:
     return nullptr;
   }

llvm/test/Transforms/InstCombine/nan.ll

@@ -0,0 +1,80 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -passes=instcombine -S | FileCheck %s
+
+@empty = private unnamed_addr constant [1 x i8] zeroinitializer, align 1
+@dec = private unnamed_addr constant [2 x i8] c"1\00", align 1
+@hex = private unnamed_addr constant [4 x i8] c"0xf\00", align 1
+
+define double @nan_empty() {
+; CHECK-LABEL: define double @nan_empty() {
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %res = call double @nan(ptr @empty)
+  ret double %res
+}
+
+define double @nan_dec() {
+; CHECK-LABEL: define double @nan_dec() {
+; CHECK-NEXT:    ret double 0x7FF8000000000001
+;
+  %res = call double @nan(ptr @dec)
+  ret double %res
+}
+
+define double @nan_hex() {
+; CHECK-LABEL: define double @nan_hex() {
+; CHECK-NEXT:    ret double 0x7FF800000000000F
+;
+  %res = call double @nan(ptr @hex)
+  ret double %res
+}
+
+define float @nanf_empty() {
+; CHECK-LABEL: define float @nanf_empty() {
+; CHECK-NEXT:    ret float 0x7FF8000000000000
+;
+  %res = call float @nanf(ptr @empty)
+  ret float %res
+}
+
+; nagative tests
+
+define double @nan_poison() {
+; CHECK-LABEL: define double @nan_poison() {
+; CHECK-NEXT:    [[RES:%.*]] = call double @nan(ptr poison)
+; CHECK-NEXT:    ret double [[RES]]
+;
+  %res = call double @nan(ptr poison)
+  ret double %res
+}
+
+define double @nan_undef() {
+; CHECK-LABEL: define double @nan_undef() {
+; CHECK-NEXT:    [[RES:%.*]] = call double @nan(ptr undef)
+; CHECK-NEXT:    ret double [[RES]]
+;
+  %res = call double @nan(ptr undef)
+  ret double %res
+}
+
+define double @nan_null() {
+; CHECK-LABEL: define double @nan_null() {
+; CHECK-NEXT:    [[RES:%.*]] = call double @nan(ptr null)
+; CHECK-NEXT:    ret double [[RES]]
+;
+  %res = call double @nan(ptr null)
+  ret double %res
+}
+
+define double @nan_non_constant(ptr %x) {
+; CHECK-LABEL: define double @nan_non_constant(
+; CHECK-SAME: ptr [[X:%.*]]) {
+; CHECK-NEXT:    [[RES:%.*]] = call double @nan(ptr [[X]])
+; CHECK-NEXT:    ret double [[RES]]
+;
+  %res = call double @nan(ptr %x)
+  ret double %res
+}
+
+declare float @nanf(ptr)
+declare double @nan(ptr)

llvm/test/Transforms/InstCombine/nanl-fp128.ll

@@ -0,0 +1,32 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -passes=instcombine -S | FileCheck %s
+
+@empty = private unnamed_addr constant [1 x i8] zeroinitializer, align 1
+@dec = private unnamed_addr constant [2 x i8] c"1\00", align 1
+@hex = private unnamed_addr constant [4 x i8] c"0xf\00", align 1
+
+define fp128 @nanl_empty() {
+; CHECK-LABEL: define fp128 @nanl_empty() {
+; CHECK-NEXT:    ret fp128 0xL00000000000000007FFF800000000000
+;
+  %res = call fp128 @nanl(ptr @empty)
+  ret fp128 %res
+}
+
+define fp128 @nanl_dec() {
+; CHECK-LABEL: define fp128 @nanl_dec() {
+; CHECK-NEXT:    ret fp128 0xL00000000000000017FFF800000000000
+;
+  %res = call fp128 @nanl(ptr @dec)
+  ret fp128 %res
+}
+
+define fp128 @nanl_hex() {
+; CHECK-LABEL: define fp128 @nanl_hex() {
+; CHECK-NEXT:    ret fp128 0xL000000000000000F7FFF800000000000
+;
+  %res = call fp128 @nanl(ptr @hex)
+  ret fp128 %res
+}
+
+declare fp128 @nanl(ptr)

llvm/test/Transforms/InstCombine/nanl-fp80.ll

@@ -0,0 +1,32 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -passes=instcombine -S | FileCheck %s
+
+@empty = private unnamed_addr constant [1 x i8] zeroinitializer, align 1
+@dec = private unnamed_addr constant [2 x i8] c"1\00", align 1
+@hex = private unnamed_addr constant [4 x i8] c"0xf\00", align 1
+
+define x86_fp80 @nanl_empty() {
+; CHECK-LABEL: define x86_fp80 @nanl_empty() {
+; CHECK-NEXT:    ret x86_fp80 0xK7FFFC000000000000000
+;
+  %res = call x86_fp80 @nanl(ptr @empty)
+  ret x86_fp80 %res
+}
+
+define x86_fp80 @nanl_dec() {
+; CHECK-LABEL: define x86_fp80 @nanl_dec() {
+; CHECK-NEXT:    ret x86_fp80 0xK7FFFC000000000000001
+;
+  %res = call x86_fp80 @nanl(ptr @dec)
+  ret x86_fp80 %res
+}
+
+define x86_fp80 @nanl_hex() {
+; CHECK-LABEL: define x86_fp80 @nanl_hex() {
+; CHECK-NEXT:    ret x86_fp80 0xK7FFFC00000000000000F
+;
+  %res = call x86_fp80 @nanl(ptr @hex)
+  ret x86_fp80 %res
+}
+
+declare x86_fp80 @nanl(ptr)

llvm/test/Transforms/InstCombine/nanl-ppc-fp128.ll

@@ -0,0 +1,32 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -passes=instcombine -S | FileCheck %s
+
+@empty = private unnamed_addr constant [1 x i8] zeroinitializer, align 1
+@dec = private unnamed_addr constant [2 x i8] c"1\00", align 1
+@hex = private unnamed_addr constant [4 x i8] c"0xf\00", align 1
+
+define ppc_fp128 @nanl_empty() {
+; CHECK-LABEL: define ppc_fp128 @nanl_empty() {
+; CHECK-NEXT:    ret ppc_fp128 0xM7FF80000000000000000000000000000
+;
+  %res = call ppc_fp128 @nanl(ptr @empty)
+  ret ppc_fp128 %res
+}
+
+define ppc_fp128 @nanl_dec() {
+; CHECK-LABEL: define ppc_fp128 @nanl_dec() {
+; CHECK-NEXT:    ret ppc_fp128 0xM7FF80000000000010000000000000000
+;
+  %res = call ppc_fp128 @nanl(ptr @dec)
+  ret ppc_fp128 %res
+}
+
+define ppc_fp128 @nanl_hex() {
+; CHECK-LABEL: define ppc_fp128 @nanl_hex() {
+; CHECK-NEXT:    ret ppc_fp128 0xM7FF800000000000F0000000000000000
+;
+  %res = call ppc_fp128 @nanl(ptr @hex)
+  ret ppc_fp128 %res
+}
+
+declare ppc_fp128 @nanl(ptr)