Skip to content

Commit e34dbb1

Browse files
overmightyovermighty
authored
[libc][math][c23] Add f16fma{,l,f128} C23 math function (#96711)
Part of #93566.
1 parent 37fe152 commit e34dbb1

29 files changed

+403
-108
lines changed

‎libc/config/linux/aarch64/entrypoints.txt

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -508,7 +508,9 @@ if(LIBC_TYPES_HAS_FLOAT16)
508508
libc.src.math.ceilf16
509509
libc.src.math.copysignf16
510510
libc.src.math.f16divf
511+
libc.src.math.f16fma
511512
libc.src.math.f16fmaf
513+
libc.src.math.f16fmal
512514
libc.src.math.f16sqrtf
513515
libc.src.math.fabsf16
514516
libc.src.math.fdimf16
@@ -560,6 +562,13 @@ if(LIBC_TYPES_HAS_FLOAT16)
560562
libc.src.math.ufromfpf16
561563
libc.src.math.ufromfpxf16
562564
)
565+
566+
if(LIBC_TYPES_HAS_FLOAT128)
567+
list(APPEND TARGET_LIBM_ENTRYPOINTS
568+
# math.h C23 mixed _Float16 and _Float128 entrypoints
569+
libc.src.math.f16fmaf128
570+
)
571+
endif()
563572
endif()
564573

565574
if(LIBC_TYPES_HAS_FLOAT128)

‎libc/config/linux/x86_64/entrypoints.txt

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -538,7 +538,9 @@ if(LIBC_TYPES_HAS_FLOAT16)
538538
libc.src.math.ceilf16
539539
libc.src.math.copysignf16
540540
libc.src.math.f16divf
541+
libc.src.math.f16fma
541542
libc.src.math.f16fmaf
543+
libc.src.math.f16fmal
542544
libc.src.math.f16sqrtf
543545
libc.src.math.fabsf16
544546
libc.src.math.fdimf16
@@ -588,6 +590,13 @@ if(LIBC_TYPES_HAS_FLOAT16)
588590
libc.src.math.ufromfpf16
589591
libc.src.math.ufromfpxf16
590592
)
593+
594+
if(LIBC_TYPES_HAS_FLOAT128)
595+
list(APPEND TARGET_LIBM_ENTRYPOINTS
596+
# math.h C23 mixed _Float16 and _Float128 entrypoints
597+
libc.src.math.f16fmaf128
598+
)
599+
endif()
591600
endif()
592601

593602
if(LIBC_TYPES_HAS_FLOAT128)

‎libc/docs/math/index.rst

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,7 @@ Basic Operations
126126
+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
127127
| f16div | |check| | | | N/A | | 7.12.14.4 | F.10.11 |
128128
+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
129-
| f16fma | |check| | | | N/A | | 7.12.14.5 | F.10.11 |
129+
| f16fma | |check| | |check| | |check| | N/A | |check| | 7.12.14.5 | F.10.11 |
130130
+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
131131
| fabs | |check| | |check| | |check| | |check| | |check| | 7.12.7.3 | F.10.4.3 |
132132
+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+

‎libc/include/llvm-libc-macros/float16-macros.h

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,18 @@
99
#ifndef LLVM_LIBC_MACROS_FLOAT16_MACROS_H
1010
#define LLVM_LIBC_MACROS_FLOAT16_MACROS_H
1111

12+
#include "../llvm-libc-types/float128.h"
13+
1214
#if defined(__FLT16_MANT_DIG__) && \
1315
(!defined(__GNUC__) || __GNUC__ >= 13 || defined(__clang__)) && \
1416
!defined(__arm__) && !defined(_M_ARM) && !defined(__riscv)
1517
#define LIBC_TYPES_HAS_FLOAT16
18+
19+
// TODO: This would no longer be required if HdrGen let us guard function
20+
// declarations with multiple macros.
21+
#ifdef LIBC_TYPES_HAS_FLOAT128
22+
#define LIBC_TYPES_HAS_FLOAT16_AND_FLOAT128
23+
#endif // LIBC_TYPES_HAS_FLOAT128
1624
#endif
1725

1826
#endif // LLVM_LIBC_MACROS_FLOAT16_MACROS_H

‎libc/spec/stdc.td

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -477,7 +477,10 @@ def StdC : StandardSpec<"stdc"> {
477477
FunctionSpec<"fma", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
478478
FunctionSpec<"fmaf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<FloatType>, ArgSpec<FloatType>]>,
479479

480+
GuardedFunctionSpec<"f16fma", RetValSpec<Float16Type>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>, ArgSpec<DoubleType>], "LIBC_TYPES_HAS_FLOAT16">,
480481
GuardedFunctionSpec<"f16fmaf", RetValSpec<Float16Type>, [ArgSpec<FloatType>, ArgSpec<FloatType>, ArgSpec<FloatType>], "LIBC_TYPES_HAS_FLOAT16">,
482+
GuardedFunctionSpec<"f16fmal", RetValSpec<Float16Type>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>], "LIBC_TYPES_HAS_FLOAT16">,
483+
GuardedFunctionSpec<"f16fmaf128", RetValSpec<Float16Type>, [ArgSpec<Float128Type>, ArgSpec<Float128Type>, ArgSpec<Float128Type>], "LIBC_TYPES_HAS_FLOAT16_AND_FLOAT128">,
481484

482485
FunctionSpec<"fmod", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
483486
FunctionSpec<"fmodf", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<FloatType>]>,

‎libc/src/__support/FPUtil/CMakeLists.txt

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -154,7 +154,6 @@ add_header_library(
154154
HDRS
155155
multiply_add.h
156156
DEPENDS
157-
.fma
158157
libc.src.__support.common
159158
)
160159

‎libc/src/__support/FPUtil/dyadic_float.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -156,13 +156,13 @@ template <size_t Bits> struct DyadicFloat {
156156
// d_lo is denormal, but the output is normal.
157157
int scale_up_exponent = 1 - exp_lo;
158158
T scale_up_factor =
159-
FPBits<T>::create_value(sign,
159+
FPBits<T>::create_value(Sign::POS,
160160
static_cast<output_bits_t>(
161161
FPBits<T>::EXP_BIAS + scale_up_exponent),
162162
IMPLICIT_MASK)
163163
.get_val();
164164
T scale_down_factor =
165-
FPBits<T>::create_value(sign,
165+
FPBits<T>::create_value(Sign::POS,
166166
static_cast<output_bits_t>(
167167
FPBits<T>::EXP_BIAS - scale_up_exponent),
168168
IMPLICIT_MASK)

‎libc/src/__support/FPUtil/generic/CMakeLists.txt

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@ add_header_library(
2424
libc.src.__support.CPP.bit
2525
libc.src.__support.CPP.limits
2626
libc.src.__support.CPP.type_traits
27+
libc.src.__support.FPUtil.basic_operations
28+
libc.src.__support.FPUtil.dyadic_float
2729
libc.src.__support.FPUtil.fenv_impl
2830
libc.src.__support.FPUtil.fp_bits
2931
libc.src.__support.FPUtil.rounding_mode

‎libc/src/__support/FPUtil/generic/FMA.h

Lines changed: 48 additions & 80 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,9 @@
1212
#include "src/__support/CPP/bit.h"
1313
#include "src/__support/CPP/limits.h"
1414
#include "src/__support/CPP/type_traits.h"
15+
#include "src/__support/FPUtil/BasicOperations.h"
1516
#include "src/__support/FPUtil/FPBits.h"
17+
#include "src/__support/FPUtil/dyadic_float.h"
1618
#include "src/__support/FPUtil/rounding_mode.h"
1719
#include "src/__support/big_int.h"
1820
#include "src/__support/macros/attributes.h" // LIBC_INLINE
@@ -106,20 +108,52 @@ LIBC_INLINE cpp::enable_if_t<cpp::is_floating_point_v<OutType> &&
106108
sizeof(OutType) <= sizeof(InType),
107109
OutType>
108110
fma(InType x, InType y, InType z) {
109-
using OutFPBits = fputil::FPBits<OutType>;
111+
using OutFPBits = FPBits<OutType>;
110112
using OutStorageType = typename OutFPBits::StorageType;
111-
using InFPBits = fputil::FPBits<InType>;
113+
using InFPBits = FPBits<InType>;
112114
using InStorageType = typename InFPBits::StorageType;
113115

114116
constexpr int IN_EXPLICIT_MANT_LEN = InFPBits::FRACTION_LEN + 1;
115117
constexpr size_t PROD_LEN = 2 * IN_EXPLICIT_MANT_LEN;
116118
constexpr size_t TMP_RESULT_LEN = cpp::bit_ceil(PROD_LEN + 1);
117119
using TmpResultType = UInt<TMP_RESULT_LEN>;
120+
using DyadicFloat = DyadicFloat<TMP_RESULT_LEN>;
118121

119-
constexpr size_t EXTRA_FRACTION_LEN =
120-
TMP_RESULT_LEN - 1 - OutFPBits::FRACTION_LEN;
121-
constexpr TmpResultType EXTRA_FRACTION_STICKY_MASK =
122-
(TmpResultType(1) << (EXTRA_FRACTION_LEN - 1)) - 1;
122+
InFPBits x_bits(x), y_bits(y), z_bits(z);
123+
124+
if (LIBC_UNLIKELY(x_bits.is_nan() || y_bits.is_nan() || z_bits.is_nan())) {
125+
if (x_bits.is_nan() || y_bits.is_nan()) {
126+
if (x_bits.is_signaling_nan() || y_bits.is_signaling_nan() ||
127+
z_bits.is_signaling_nan())
128+
raise_except_if_required(FE_INVALID);
129+
130+
if (x_bits.is_quiet_nan()) {
131+
InStorageType x_payload = static_cast<InStorageType>(getpayload(x));
132+
if ((x_payload & ~(OutFPBits::FRACTION_MASK >> 1)) == 0)
133+
return OutFPBits::quiet_nan(x_bits.sign(),
134+
static_cast<OutStorageType>(x_payload))
135+
.get_val();
136+
}
137+
138+
if (y_bits.is_quiet_nan()) {
139+
InStorageType y_payload = static_cast<InStorageType>(getpayload(y));
140+
if ((y_payload & ~(OutFPBits::FRACTION_MASK >> 1)) == 0)
141+
return OutFPBits::quiet_nan(y_bits.sign(),
142+
static_cast<OutStorageType>(y_payload))
143+
.get_val();
144+
}
145+
146+
if (z_bits.is_quiet_nan()) {
147+
InStorageType z_payload = static_cast<InStorageType>(getpayload(z));
148+
if ((z_payload & ~(OutFPBits::FRACTION_MASK >> 1)) == 0)
149+
return OutFPBits::quiet_nan(z_bits.sign(),
150+
static_cast<OutStorageType>(z_payload))
151+
.get_val();
152+
}
153+
154+
return OutFPBits::quiet_nan().get_val();
155+
}
156+
}
123157

124158
if (LIBC_UNLIKELY(x == 0 || y == 0 || z == 0))
125159
return static_cast<OutType>(x * y + z);
@@ -142,7 +176,9 @@ fma(InType x, InType y, InType z) {
142176
z *= InType(InStorageType(1) << InFPBits::FRACTION_LEN);
143177
}
144178

145-
InFPBits x_bits(x), y_bits(y), z_bits(z);
179+
x_bits = InFPBits(x);
180+
y_bits = InFPBits(y);
181+
z_bits = InFPBits(z);
146182
const Sign z_sign = z_bits.sign();
147183
Sign prod_sign = (x_bits.sign() == y_bits.sign()) ? Sign::POS : Sign::NEG;
148184
x_exp += x_bits.get_biased_exponent();
@@ -182,7 +218,6 @@ fma(InType x, InType y, InType z) {
182218
constexpr int RESULT_MIN_LEN = PROD_LEN - InFPBits::FRACTION_LEN;
183219
z_mant <<= RESULT_MIN_LEN;
184220
int z_lsb_exp = z_exp - (InFPBits::FRACTION_LEN + RESULT_MIN_LEN);
185-
bool round_bit = false;
186221
bool sticky_bits = false;
187222
bool z_shifted = false;
188223

@@ -221,85 +256,18 @@ fma(InType x, InType y, InType z) {
221256
}
222257
}
223258

224-
OutStorageType result = 0;
225-
int r_exp = 0; // Unbiased exponent of the result
226-
227-
int round_mode = fputil::quick_get_round();
228-
229-
// Normalize the result.
230-
if (prod_mant != 0) {
231-
int lead_zeros = cpp::countl_zero(prod_mant);
232-
// Move the leading 1 to the most significant bit.
233-
prod_mant <<= lead_zeros;
234-
prod_lsb_exp -= lead_zeros;
235-
r_exp = prod_lsb_exp + (cpp::numeric_limits<TmpResultType>::digits - 1) -
236-
InFPBits::EXP_BIAS + OutFPBits::EXP_BIAS;
237-
238-
if (r_exp > 0) {
239-
// The result is normal. We will shift the mantissa to the right by the
240-
// amount of extra bits compared to the length of the explicit mantissa in
241-
// the output type. The rounding bit then becomes the highest bit that is
242-
// shifted out, and the following lower bits are merged into sticky bits.
243-
round_bit =
244-
(prod_mant & (TmpResultType(1) << (EXTRA_FRACTION_LEN - 1))) != 0;
245-
sticky_bits |= (prod_mant & EXTRA_FRACTION_STICKY_MASK) != 0;
246-
result = static_cast<OutStorageType>(prod_mant >> EXTRA_FRACTION_LEN);
247-
} else {
248-
if (r_exp < -OutFPBits::FRACTION_LEN) {
249-
// The result is smaller than 1/2 of the smallest denormal number.
250-
sticky_bits = true; // since the result is non-zero.
251-
result = 0;
252-
} else {
253-
// The result is denormal.
254-
TmpResultType mask = TmpResultType(1) << (EXTRA_FRACTION_LEN - r_exp);
255-
round_bit = (prod_mant & mask) != 0;
256-
sticky_bits |= (prod_mant & (mask - 1)) != 0;
257-
if (r_exp > -OutFPBits::FRACTION_LEN)
258-
result = static_cast<OutStorageType>(
259-
prod_mant >> (EXTRA_FRACTION_LEN + 1 - r_exp));
260-
else
261-
result = 0;
262-
}
263-
264-
r_exp = 0;
265-
}
266-
} else {
259+
if (prod_mant == 0) {
267260
// When there is exact cancellation, i.e., x*y == -z exactly, return -0.0 if
268261
// rounding downward and +0.0 for other rounding modes.
269-
if (round_mode == FE_DOWNWARD)
262+
if (quick_get_round() == FE_DOWNWARD)
270263
prod_sign = Sign::NEG;
271264
else
272265
prod_sign = Sign::POS;
273266
}
274267

275-
// Finalize the result.
276-
if (LIBC_UNLIKELY(r_exp >= OutFPBits::MAX_BIASED_EXPONENT)) {
277-
if ((round_mode == FE_TOWARDZERO) ||
278-
(round_mode == FE_UPWARD && prod_sign.is_neg()) ||
279-
(round_mode == FE_DOWNWARD && prod_sign.is_pos())) {
280-
return OutFPBits::max_normal(prod_sign).get_val();
281-
}
282-
return OutFPBits::inf(prod_sign).get_val();
283-
}
284-
285-
// Remove hidden bit and append the exponent field and sign bit.
286-
result = static_cast<OutStorageType>(
287-
(result & OutFPBits::FRACTION_MASK) |
288-
(static_cast<OutStorageType>(r_exp) << OutFPBits::FRACTION_LEN));
289-
if (prod_sign.is_neg())
290-
result |= OutFPBits::SIGN_MASK;
291-
292-
// Rounding.
293-
if (round_mode == FE_TONEAREST) {
294-
if (round_bit && (sticky_bits || ((result & 1) != 0)))
295-
++result;
296-
} else if ((round_mode == FE_UPWARD && prod_sign.is_pos()) ||
297-
(round_mode == FE_DOWNWARD && prod_sign.is_neg())) {
298-
if (round_bit || sticky_bits)
299-
++result;
300-
}
301-
302-
return cpp::bit_cast<OutType>(result);
268+
DyadicFloat result(prod_sign, prod_lsb_exp - InFPBits::EXP_BIAS, prod_mant);
269+
result.mantissa |= sticky_bits;
270+
return result.template as<OutType, /*ShouldSignalExceptions=*/true>();
303271
}
304272

305273
} // namespace generic

‎libc/src/__support/FPUtil/multiply_add.h

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -39,17 +39,18 @@ multiply_add(T x, T y, T z) {
3939
#if defined(LIBC_TARGET_CPU_HAS_FMA)
4040

4141
// FMA instructions are available.
42-
#include "FMA.h"
42+
// We use builtins directly instead of including FMA.h to avoid a circular
43+
// dependency: multiply_add.h -> FMA.h -> generic/FMA.h -> dyadic_float.h.
4344

4445
namespace LIBC_NAMESPACE {
4546
namespace fputil {
4647

4748
LIBC_INLINE float multiply_add(float x, float y, float z) {
48-
return fma<float>(x, y, z);
49+
return __builtin_fmaf(x, y, z);
4950
}
5051

5152
LIBC_INLINE double multiply_add(double x, double y, double z) {
52-
return fma<double>(x, y, z);
53+
return __builtin_fma(x, y, z);
5354
}
5455

5556
} // namespace fputil

‎libc/src/math/CMakeLists.txt

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,10 @@ add_math_entrypoint_object(expm1f)
101101

102102
add_math_entrypoint_object(f16divf)
103103

104+
add_math_entrypoint_object(f16fma)
104105
add_math_entrypoint_object(f16fmaf)
106+
add_math_entrypoint_object(f16fmal)
107+
add_math_entrypoint_object(f16fmaf128)
105108

106109
add_math_entrypoint_object(f16sqrtf)
107110

‎libc/src/math/f16fma.h

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
//===-- Implementation header for f16fma ------------------------*- C++ -*-===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
9+
#ifndef LLVM_LIBC_SRC_MATH_F16FMA_H
10+
#define LLVM_LIBC_SRC_MATH_F16FMA_H
11+
12+
#include "src/__support/macros/properties/types.h"
13+
14+
namespace LIBC_NAMESPACE {
15+
16+
float16 f16fma(double x, double y, double z);
17+
18+
} // namespace LIBC_NAMESPACE
19+
20+
#endif // LLVM_LIBC_SRC_MATH_F16FMA_H

‎libc/src/math/f16fmaf128.h

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
//===-- Implementation header for f16fmaf128 --------------------*- C++ -*-===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
9+
#ifndef LLVM_LIBC_SRC_MATH_F16FMAF128_H
10+
#define LLVM_LIBC_SRC_MATH_F16FMAF128_H
11+
12+
#include "src/__support/macros/properties/types.h"
13+
14+
namespace LIBC_NAMESPACE {
15+
16+
float16 f16fmaf128(float128 x, float128 y, float128 z);
17+
18+
} // namespace LIBC_NAMESPACE
19+
20+
#endif // LLVM_LIBC_SRC_MATH_F16FMAF128_H

‎libc/src/math/f16fmal.h

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
//===-- Implementation header for f16fmal -----------------------*- C++ -*-===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
9+
#ifndef LLVM_LIBC_SRC_MATH_F16FMAL_H
10+
#define LLVM_LIBC_SRC_MATH_F16FMAL_H
11+
12+
#include "src/__support/macros/properties/types.h"
13+
14+
namespace LIBC_NAMESPACE {
15+
16+
float16 f16fmal(long double x, long double y, long double z);
17+
18+
} // namespace LIBC_NAMESPACE
19+
20+
#endif // LLVM_LIBC_SRC_MATH_F16FMAL_H

0 commit comments

Comments
 (0)