Skip to content

Commit a982431

Browse files
lntuelntue
committed
[libc] Add platform independent floating point rounding mode checks.
Many math functions need to check for floating point rounding modes to return correct values. Currently most of them use the internal implementation of `fegetround`, which is platform-dependent and blocking math functions to be enabled on platforms with unimplemented `fegetround`. In this change, we add platform independent rounding mode checks and switching math functions to use them instead. #63016 Reviewed By: sivachandra Differential Revision: https://reviews.llvm.org/D152280
1 parent 282324a commit a982431

28 files changed

+312
-35
lines changed

‎libc/src/__support/CMakeLists.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,7 @@ add_header_library(
133133
libc.src.__support.CPP.limits
134134
libc.src.__support.FPUtil.fenv_impl
135135
libc.src.__support.FPUtil.fp_bits
136+
libc.src.__support.FPUtil.rounding_mode
136137
libc.src.__support.builtin_wrappers
137138
libc.src.__support.common
138139
libc.src.errno.errno

‎libc/src/__support/FPUtil/CMakeLists.txt

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,15 @@ add_header_library(
22
fenv_impl
33
HDRS
44
FEnvImpl.h
5+
DEPENDS
6+
libc.include.fenv
7+
libc.src.__support.macros.attributes
8+
)
9+
10+
add_header_library(
11+
rounding_mode
12+
HDRS
13+
rounding_mode.h
514
DEPENDS
615
libc.include.fenv
716
libc.include.math
@@ -62,6 +71,7 @@ add_header_library(
6271
DEPENDS
6372
.fp_bits
6473
.fenv_impl
74+
.rounding_mode
6575
libc.src.__support.CPP.type_traits
6676
libc.src.__support.common
6777
libc.include.math
@@ -124,6 +134,7 @@ add_header_library(
124134
DEPENDS
125135
.fp_bits
126136
.fenv_impl
137+
.rounding_mode
127138
libc.src.__support.CPP.optional
128139
libc.src.__support.macros.optimization
129140
)
@@ -137,6 +148,7 @@ add_header_library(
137148
.basic_operations
138149
.fenv_impl
139150
.fp_bits
151+
.rounding_mode
140152
libc.src.__support.builtin_wrappers
141153
libc.src.__support.CPP.bit
142154
libc.src.__support.CPP.type_traits

‎libc/src/__support/FPUtil/Hypot.h

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
#include "BasicOperations.h"
1313
#include "FEnvImpl.h"
1414
#include "FPBits.h"
15+
#include "rounding_mode.h"
1516
#include "src/__support/CPP/bit.h"
1617
#include "src/__support/CPP/type_traits.h"
1718
#include "src/__support/UInt128.h"
@@ -190,7 +191,7 @@ LIBC_INLINE T hypot(T x, T y) {
190191
sum >>= 2;
191192
++out_exp;
192193
if (out_exp >= FPBits_t::MAX_EXPONENT) {
193-
if (int round_mode = get_round();
194+
if (int round_mode = quick_get_round();
194195
round_mode == FE_TONEAREST || round_mode == FE_UPWARD)
195196
return T(FPBits_t::inf());
196197
return T(FPBits_t(FPBits_t::MAX_NORMAL));
@@ -231,7 +232,7 @@ LIBC_INLINE T hypot(T x, T y) {
231232
y_new >>= 1;
232233

233234
// Round to the nearest, tie to even.
234-
int round_mode = get_round();
235+
int round_mode = quick_get_round();
235236
switch (round_mode) {
236237
case FE_TONEAREST:
237238
// Round to nearest, ties to even

‎libc/src/__support/FPUtil/NearestIntegerOperations.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111

1212
#include "FEnvImpl.h"
1313
#include "FPBits.h"
14+
#include "rounding_mode.h"
1415

1516
#include "src/__support/CPP/type_traits.h"
1617
#include "src/__support/common.h"
@@ -162,7 +163,7 @@ LIBC_INLINE T round_using_current_rounding_mode(T x) {
162163

163164
bool is_neg = bits.get_sign();
164165
int exponent = bits.get_exponent();
165-
int rounding_mode = get_round();
166+
int rounding_mode = quick_get_round();
166167

167168
// If the exponent is greater than the most negative mantissa
168169
// exponent, then x is already an integer.

‎libc/src/__support/FPUtil/except_value_utils.h

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111

1212
#include "FEnvImpl.h"
1313
#include "FPBits.h"
14+
#include "rounding_mode.h"
1415
#include "src/__support/CPP/optional.h"
1516
#include "src/__support/macros/optimization.h" // LIBC_UNLIKELY
1617

@@ -52,7 +53,7 @@ template <typename T, size_t N> struct ExceptValues {
5253
for (size_t i = 0; i < N; ++i) {
5354
if (LIBC_UNLIKELY(x_bits == values[i].input)) {
5455
UIntType out_bits = values[i].rnd_towardzero_result;
55-
switch (fputil::get_round()) {
56+
switch (fputil::quick_get_round()) {
5657
case FE_UPWARD:
5758
out_bits += values[i].rnd_upward_offset;
5859
break;
@@ -74,7 +75,7 @@ template <typename T, size_t N> struct ExceptValues {
7475
for (size_t i = 0; i < N; ++i) {
7576
if (LIBC_UNLIKELY(x_abs == values[i].input)) {
7677
UIntType out_bits = values[i].rnd_towardzero_result;
77-
switch (fputil::get_round()) {
78+
switch (fputil::quick_get_round()) {
7879
case FE_UPWARD:
7980
out_bits += sign ? values[i].rnd_downward_offset
8081
: values[i].rnd_upward_offset;

‎libc/src/__support/FPUtil/generic/CMakeLists.txt

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ add_header_library(
99
libc.src.__support.FPUtil.fenv_impl
1010
libc.src.__support.FPUtil.fp_bits
1111
libc.src.__support.FPUtil.platform_defs
12+
libc.src.__support.FPUtil.rounding_mode
1213
libc.src.__support.builtin_wrappers
1314
libc.src.__support.common
1415
libc.src.__support.uint128
@@ -25,6 +26,7 @@ add_header_library(
2526
libc.src.__support.FPUtil.fenv_impl
2627
libc.src.__support.FPUtil.float_properties
2728
libc.src.__support.FPUtil.fp_bits
29+
libc.src.__support.FPUtil.rounding_mode
2830
libc.src.__support.builtin_wrappers
2931
libc.src.__support.macros.optimization
3032
libc.src.__support.uint128
@@ -40,6 +42,7 @@ add_header_library(
4042
libc.src.__support.FPUtil.fenv_impl
4143
libc.src.__support.FPUtil.float_properties
4244
libc.src.__support.FPUtil.fp_bits
45+
libc.src.__support.FPUtil.rounding_mode
4346
libc.src.__support.builtin_wrappers
4447
libc.src.__support.macros.optimization
4548
libc.src.math.generic.math_utils

‎libc/src/__support/FPUtil/generic/FMA.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
#include "src/__support/FPUtil/FEnvImpl.h"
1414
#include "src/__support/FPUtil/FPBits.h"
1515
#include "src/__support/FPUtil/FloatProperties.h"
16+
#include "src/__support/FPUtil/rounding_mode.h"
1617
#include "src/__support/UInt128.h"
1718
#include "src/__support/builtin_wrappers.h"
1819
#include "src/__support/macros/attributes.h" // LIBC_INLINE
@@ -254,7 +255,7 @@ template <> LIBC_INLINE double fma<double>(double x, double y, double z) {
254255
}
255256

256257
// Finalize the result.
257-
int round_mode = fputil::get_round();
258+
int round_mode = fputil::quick_get_round();
258259
if (LIBC_UNLIKELY(r_exp >= FPBits::MAX_EXPONENT)) {
259260
if ((round_mode == FE_TOWARDZERO) ||
260261
(round_mode == FE_UPWARD && prod_sign) ||

‎libc/src/__support/FPUtil/generic/sqrt.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
#include "src/__support/FPUtil/FEnvImpl.h"
1616
#include "src/__support/FPUtil/FPBits.h"
1717
#include "src/__support/FPUtil/PlatformDefs.h"
18+
#include "src/__support/FPUtil/rounding_mode.h"
1819
#include "src/__support/UInt128.h"
1920
#include "src/__support/builtin_wrappers.h"
2021
#include "src/__support/common.h"
@@ -150,7 +151,7 @@ LIBC_INLINE cpp::enable_if_t<cpp::is_floating_point_v<T>, T> sqrt(T x) {
150151

151152
y = (y - ONE) | (static_cast<UIntType>(x_exp) << MantissaWidth<T>::VALUE);
152153

153-
switch (get_round()) {
154+
switch (quick_get_round()) {
154155
case FE_TONEAREST:
155156
// Round to nearest, ties to even
156157
if (rb && (lsb || (r != 0)))

‎libc/src/__support/FPUtil/generic/sqrt_80_bit_long_double.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
#include "src/__support/FPUtil/FEnvImpl.h"
1313
#include "src/__support/FPUtil/FPBits.h"
1414
#include "src/__support/FPUtil/PlatformDefs.h"
15+
#include "src/__support/FPUtil/rounding_mode.h"
1516
#include "src/__support/UInt128.h"
1617
#include "src/__support/builtin_wrappers.h"
1718
#include "src/__support/common.h"
@@ -114,7 +115,7 @@ LIBC_INLINE long double sqrt(long double x) {
114115
y |= (static_cast<UIntType>(x_exp)
115116
<< (MantissaWidth<long double>::VALUE + 1));
116117

117-
switch (get_round()) {
118+
switch (quick_get_round()) {
118119
case FE_TONEAREST:
119120
// Round to nearest, ties to even
120121
if (rb && (lsb || (r != 0)))

‎libc/src/__support/FPUtil/rounding_mode.h

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
//===---- Free-standing function to detect rounding mode --------*- C++ -*-===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
9+
#ifndef LLVM_LIBC_SRC_SUPPORT_FPUTIL_ROUNDING_MODE_H
10+
#define LLVM_LIBC_SRC_SUPPORT_FPUTIL_ROUNDING_MODE_H
11+
12+
#include "src/__support/macros/attributes.h" // LIBC_INLINE
13+
14+
#include <fenv.h>
15+
16+
namespace __llvm_libc::fputil {
17+
18+
// Quick free-standing test whether fegetround() == FE_UPWARD.
19+
// Using the following observation:
20+
// 1.0f + 2^-25 = 1.0f for FE_TONEAREST, FE_DOWNWARD, FE_TOWARDZERO
21+
// = 0x1.000002f for FE_UPWARD.
22+
LIBC_INLINE bool fenv_is_round_up() {
23+
volatile float x = 0x1.0p-25f;
24+
return (1.0f + x != 1.0f);
25+
}
26+
27+
// Quick free-standing test whether fegetround() == FE_DOWNWARD.
28+
// Using the following observation:
29+
// -1.0f - 2^-25 = -1.0f for FE_TONEAREST, FE_UPWARD, FE_TOWARDZERO
30+
// = -0x1.000002f for FE_DOWNWARD.
31+
LIBC_INLINE bool fenv_is_round_down() {
32+
volatile float x = 0x1.0p-25f;
33+
return (-1.0f - x != -1.0f);
34+
}
35+
36+
// Quick free-standing test whether fegetround() == FE_TONEAREST.
37+
// Using the following observation:
38+
// 1.5f + 2^-24 = 1.5f for FE_TONEAREST, FE_DOWNWARD, FE_TOWARDZERO
39+
// = 0x1.100002p0f for FE_UPWARD,
40+
// 1.5f - 2^-24 = 1.5f for FE_TONEAREST, FE_UPWARD
41+
// = 0x1.0ffffep-1f for FE_DOWNWARD, FE_TOWARDZERO
42+
LIBC_INLINE bool fenv_is_round_to_nearest() {
43+
static volatile float x = 0x1.0p-24f;
44+
float y = x;
45+
return (1.5f + y == 1.5f - y);
46+
}
47+
48+
// Quick free-standing test whether fegetround() == FE_TOWARDZERO.
49+
// Using the following observation:
50+
// 1.0f + 2^-23 + 2^-24 = 0x1.000002p0f for FE_DOWNWARD, FE_TOWARDZERO
51+
// = 0x1.000004p0f for FE_TONEAREST, FE_UPWARD,
52+
// -1.0f - 2^-24 = -1.0f for FE_TONEAREST, FE_UPWARD, FE_TOWARDZERO
53+
// = -0x1.000002p0f for FE_DOWNWARD
54+
// So:
55+
// (0x1.000002p0f + 2^-24) + (-1.0f - 2^-24) = 2^-23 for FE_TOWARDZERO
56+
// = 2^-22 for FE_TONEAREST, FE_UPWARD
57+
// = 0 for FE_DOWNWARD
58+
LIBC_INLINE bool fenv_is_round_to_zero() {
59+
static volatile float x = 0x1.0p-24f;
60+
float y = x;
61+
return ((0x1.000002p0f + y) + (-1.0f - y) == 0x1.0p-23f);
62+
}
63+
64+
// Quick free standing get rounding mode based on the above observations.
65+
LIBC_INLINE int quick_get_round() {
66+
static volatile float x = 0x1.0p-24f;
67+
float y = x;
68+
float z = (0x1.000002p0f + y) + (-1.0f - y);
69+
70+
if (z == 0.0f)
71+
return FE_DOWNWARD;
72+
if (z == 0x1.0p-23f)
73+
return FE_TOWARDZERO;
74+
return (2.0f + y == 2.0f) ? FE_TONEAREST : FE_UPWARD;
75+
}
76+
77+
} // namespace __llvm_libc::fputil
78+
79+
#endif // LLVM_LIBC_SRC_SUPPORT_FPUTIL_ROUNDING_MODE_H

‎libc/src/__support/str_to_float.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
#include "src/__support/CPP/optional.h"
1414
#include "src/__support/FPUtil/FEnvImpl.h"
1515
#include "src/__support/FPUtil/FPBits.h"
16+
#include "src/__support/FPUtil/rounding_mode.h"
1617
#include "src/__support/UInt128.h"
1718
#include "src/__support/builtin_wrappers.h"
1819
#include "src/__support/common.h"
@@ -1120,7 +1121,7 @@ LIBC_INLINE StrToNumResult<T> strtofloatingpoint(const char *__restrict src) {
11201121

11211122
RoundDirection round_direction = RoundDirection::Nearest;
11221123

1123-
switch (fputil::get_round()) {
1124+
switch (fputil::quick_get_round()) {
11241125
case FE_TONEAREST:
11251126
round_direction = RoundDirection::Nearest;
11261127
break;

‎libc/src/math/generic/CMakeLists.txt

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,7 @@ add_entrypoint_object(
109109
libc.src.__support.FPUtil.fma
110110
libc.src.__support.FPUtil.multiply_add
111111
libc.src.__support.FPUtil.polyeval
112+
libc.src.__support.FPUtil.rounding_mode
112113
libc.src.__support.macros.optimization
113114
COMPILE_OPTIONS
114115
-O3
@@ -130,6 +131,7 @@ add_entrypoint_object(
130131
libc.src.__support.FPUtil.fma
131132
libc.src.__support.FPUtil.multiply_add
132133
libc.src.__support.FPUtil.polyeval
134+
libc.src.__support.FPUtil.rounding_mode
133135
libc.src.__support.macros.optimization
134136
COMPILE_OPTIONS
135137
-O3
@@ -542,6 +544,7 @@ add_entrypoint_object(
542544
libc.src.__support.FPUtil.multiply_add
543545
libc.src.__support.FPUtil.nearest_integer
544546
libc.src.__support.FPUtil.polyeval
547+
libc.src.__support.FPUtil.rounding_mode
545548
libc.src.__support.macros.optimization
546549
libc.include.errno
547550
libc.src.errno.errno
@@ -563,6 +566,7 @@ add_entrypoint_object(
563566
libc.src.__support.FPUtil.multiply_add
564567
libc.src.__support.FPUtil.nearest_integer
565568
libc.src.__support.FPUtil.polyeval
569+
libc.src.__support.FPUtil.rounding_mode
566570
libc.src.__support.macros.optimization
567571
libc.include.errno
568572
libc.src.errno.errno
@@ -584,6 +588,7 @@ add_entrypoint_object(
584588
libc.src.__support.FPUtil.multiply_add
585589
libc.src.__support.FPUtil.nearest_integer
586590
libc.src.__support.FPUtil.polyeval
591+
libc.src.__support.FPUtil.rounding_mode
587592
libc.src.__support.macros.optimization
588593
libc.include.errno
589594
libc.src.errno.errno
@@ -606,6 +611,7 @@ add_entrypoint_object(
606611
libc.src.__support.FPUtil.multiply_add
607612
libc.src.__support.FPUtil.nearest_integer
608613
libc.src.__support.FPUtil.polyeval
614+
libc.src.__support.FPUtil.rounding_mode
609615
libc.src.__support.macros.optimization
610616
libc.include.errno
611617
libc.src.errno.errno
@@ -1336,6 +1342,7 @@ add_entrypoint_object(
13361342
.explogxf
13371343
libc.src.__support.FPUtil.fp_bits
13381344
libc.src.__support.FPUtil.multiply_add
1345+
libc.src.__support.FPUtil.rounding_mode
13391346
libc.src.__support.macros.optimization
13401347
COMPILE_OPTIONS
13411348
-O3
@@ -1350,6 +1357,7 @@ add_entrypoint_object(
13501357
DEPENDS
13511358
.explogxf
13521359
libc.src.__support.FPUtil.fp_bits
1360+
libc.src.__support.FPUtil.rounding_mode
13531361
libc.src.__support.macros.optimization
13541362
COMPILE_OPTIONS
13551363
-O3
@@ -1364,6 +1372,7 @@ add_entrypoint_object(
13641372
DEPENDS
13651373
.explogxf
13661374
libc.src.__support.FPUtil.fp_bits
1375+
libc.src.__support.FPUtil.rounding_mode
13671376
libc.src.__support.macros.optimization
13681377
COMPILE_OPTIONS
13691378
-O3
@@ -1483,6 +1492,7 @@ add_entrypoint_object(
14831492
.inv_trigf_utils
14841493
.math_utils
14851494
libc.src.__support.FPUtil.fp_bits
1495+
libc.src.__support.FPUtil.rounding_mode
14861496
libc.src.__support.macros.optimization
14871497
COMPILE_OPTIONS
14881498
-O3

‎libc/src/math/generic/atanf.cpp

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
#include "src/math/atanf.h"
1010
#include "math_utils.h"
1111
#include "src/__support/FPUtil/FPBits.h"
12+
#include "src/__support/FPUtil/rounding_mode.h"
1213
#include "src/__support/macros/optimization.h" // LIBC_UNLIKELY
1314
#include "src/math/generic/inv_trigf_utils.h"
1415

@@ -28,7 +29,7 @@ LLVM_LIBC_FUNCTION(float, atanf, (float x)) {
2829
}
2930
// |x| == 0.06905200332403183
3031
if (LIBC_UNLIKELY(xbits.uintval() == 0x3d8d6b23U)) {
31-
if (fputil::get_round() == FE_TONEAREST) {
32+
if (fputil::fenv_is_round_to_nearest()) {
3233
// 0.06894256919622421
3334
FPBits br(0x3d8d31c3U);
3435
br.set_sign(sign);
@@ -38,7 +39,7 @@ LLVM_LIBC_FUNCTION(float, atanf, (float x)) {
3839

3940
// |x| == 1.8670953512191772
4041
if (LIBC_UNLIKELY(xbits.uintval() == 0x3feefcfbU)) {
41-
int rounding_mode = fputil::get_round();
42+
int rounding_mode = fputil::quick_get_round();
4243
if (sign) {
4344
if (rounding_mode == FE_DOWNWARD) {
4445
// -1.0790828466415405

0 commit comments

Comments
 (0)