Skip to content

Commit de7fbfe

Browse files
huixie90huixie90
authored
[libc++] Floating Point Atomic (#67799)
- implement P0020R6 Floating Point Atomic Differential Revision: https://reviews.llvm.org/D153981
1 parent 956cf0e commit de7fbfe

26 files changed

+2286
-1
lines changed

‎libcxx/docs/ReleaseNotes/18.rst

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ Implemented Papers
4949
- P2614R2 - Deprecate ``numeric_limits::has_denorm``
5050
- P0053R7 - C++ Synchronized Buffered Ostream (in the experimental library)
5151
- P2467R1 - Support exclusive mode for fstreams
52+
- P0020R6 - Floating Point Atomic
5253

5354

5455
Improvements and New Features

‎libcxx/docs/Status/Cxx20Papers.csv

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
"`P0463R1 <https://wg21.link/P0463R1>`__","LWG","Endian just Endian","Toronto","|Complete|","7.0"
33
"`P0674R1 <https://wg21.link/P0674R1>`__","LWG","Extending make_shared to Support Arrays","Toronto","|Complete|","15.0"
44
"","","","","","",""
5-
"`P0020R6 <https://wg21.link/P0020R6>`__","LWG","Floating Point Atomic","Albuquerque","",""
5+
"`P0020R6 <https://wg21.link/P0020R6>`__","LWG","Floating Point Atomic","Albuquerque","|Complete|","18.0"
66
"`P0053R7 <https://wg21.link/P0053R7>`__","LWG","C++ Synchronized Buffered Ostream","Albuquerque","|Complete|","18.0"
77
"`P0202R3 <https://wg21.link/P0202R3>`__","LWG","Add constexpr modifiers to functions in <algorithm> and <utility> Headers","Albuquerque","|Complete|","12.0"
88
"`P0415R1 <https://wg21.link/P0415R1>`__","LWG","Constexpr for ``std::complex``\ ","Albuquerque","|Complete|","16.0"

‎libcxx/include/__atomic/atomic.h

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,17 @@
1414
#include <__atomic/cxx_atomic_impl.h>
1515
#include <__atomic/memory_order.h>
1616
#include <__config>
17+
#include <__functional/operations.h>
1718
#include <__memory/addressof.h>
19+
#include <__type_traits/is_floating_point.h>
1820
#include <__type_traits/is_function.h>
1921
#include <__type_traits/is_same.h>
22+
#include <__type_traits/remove_const.h>
2023
#include <__type_traits/remove_pointer.h>
24+
#include <__type_traits/remove_volatile.h>
25+
#include <__utility/forward.h>
2126
#include <cstddef>
27+
#include <cstring>
2228

2329
#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
2430
# pragma GCC system_header
@@ -136,6 +142,136 @@ struct atomic<_Tp*>
136142
atomic& operator=(const atomic&) volatile = delete;
137143
};
138144

145+
#if _LIBCPP_STD_VER >= 20
146+
template <class _Tp>
147+
requires is_floating_point_v<_Tp>
148+
struct atomic<_Tp> : __atomic_base<_Tp> {
149+
private:
150+
_LIBCPP_HIDE_FROM_ABI static constexpr bool __is_fp80_long_double() {
151+
// Only x87-fp80 long double has 64-bit mantissa
152+
return __LDBL_MANT_DIG__ == 64 && std::is_same_v<_Tp, long double>;
153+
}
154+
155+
_LIBCPP_HIDE_FROM_ABI static constexpr bool __has_rmw_builtin() {
156+
# ifndef _LIBCPP_COMPILER_CLANG_BASED
157+
return false;
158+
# else
159+
// The builtin __cxx_atomic_fetch_add errors during compilation for
160+
// long double on platforms with fp80 format.
161+
// For more details, see
162+
// lib/Sema/SemaChecking.cpp function IsAllowedValueType
163+
// LLVM Parser does not allow atomicrmw with x86_fp80 type.
164+
// if (ValType->isSpecificBuiltinType(BuiltinType::LongDouble) &&
165+
// &Context.getTargetInfo().getLongDoubleFormat() ==
166+
// &llvm::APFloat::x87DoubleExtended())
167+
// For more info
168+
// https://github.com/llvm/llvm-project/issues/68602
169+
// https://reviews.llvm.org/D53965
170+
return !__is_fp80_long_double();
171+
# endif
172+
}
173+
174+
template <class _This, class _Operation, class _BuiltinOp>
175+
_LIBCPP_HIDE_FROM_ABI static _Tp
176+
__rmw_op(_This&& __self, _Tp __operand, memory_order __m, _Operation __operation, _BuiltinOp __builtin_op) {
177+
if constexpr (__has_rmw_builtin()) {
178+
return __builtin_op(std::addressof(std::forward<_This>(__self).__a_), __operand, __m);
179+
} else {
180+
_Tp __old = __self.load(memory_order_relaxed);
181+
_Tp __new = __operation(__old, __operand);
182+
while (!__self.compare_exchange_weak(__old, __new, __m, memory_order_relaxed)) {
183+
# ifdef _LIBCPP_COMPILER_CLANG_BASED
184+
if constexpr (__is_fp80_long_double()) {
185+
// https://github.com/llvm/llvm-project/issues/47978
186+
// clang bug: __old is not updated on failure for atomic<long double>::compare_exchange_weak
187+
// Note __old = __self.load(memory_order_relaxed) will not work
188+
std::__cxx_atomic_load_inplace(std::addressof(__self.__a_), &__old, memory_order_relaxed);
189+
}
190+
# endif
191+
__new = __operation(__old, __operand);
192+
}
193+
return __old;
194+
}
195+
}
196+
197+
template <class _This>
198+
_LIBCPP_HIDE_FROM_ABI static _Tp __fetch_add(_This&& __self, _Tp __operand, memory_order __m) {
199+
auto __builtin_op = [](auto __a, auto __builtin_operand, auto __order) {
200+
return std::__cxx_atomic_fetch_add(__a, __builtin_operand, __order);
201+
};
202+
return __rmw_op(std::forward<_This>(__self), __operand, __m, std::plus<>{}, __builtin_op);
203+
}
204+
205+
template <class _This>
206+
_LIBCPP_HIDE_FROM_ABI static _Tp __fetch_sub(_This&& __self, _Tp __operand, memory_order __m) {
207+
auto __builtin_op = [](auto __a, auto __builtin_operand, auto __order) {
208+
return std::__cxx_atomic_fetch_sub(__a, __builtin_operand, __order);
209+
};
210+
return __rmw_op(std::forward<_This>(__self), __operand, __m, std::minus<>{}, __builtin_op);
211+
}
212+
213+
public:
214+
using __base = __atomic_base<_Tp>;
215+
using value_type = _Tp;
216+
using difference_type = value_type;
217+
218+
_LIBCPP_HIDE_FROM_ABI constexpr atomic() noexcept = default;
219+
_LIBCPP_HIDE_FROM_ABI constexpr atomic(_Tp __d) noexcept : __base(__d) {}
220+
221+
atomic(const atomic&) = delete;
222+
atomic& operator=(const atomic&) = delete;
223+
atomic& operator=(const atomic&) volatile = delete;
224+
225+
_LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __d) volatile noexcept
226+
requires __base::is_always_lock_free
227+
{
228+
__base::store(__d);
229+
return __d;
230+
}
231+
_LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __d) noexcept {
232+
__base::store(__d);
233+
return __d;
234+
}
235+
236+
_LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) volatile noexcept
237+
requires __base::is_always_lock_free
238+
{
239+
return __fetch_add(*this, __op, __m);
240+
}
241+
242+
_LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) noexcept {
243+
return __fetch_add(*this, __op, __m);
244+
}
245+
246+
_LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) volatile noexcept
247+
requires __base::is_always_lock_free
248+
{
249+
return __fetch_sub(*this, __op, __m);
250+
}
251+
252+
_LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) noexcept {
253+
return __fetch_sub(*this, __op, __m);
254+
}
255+
256+
_LIBCPP_HIDE_FROM_ABI _Tp operator+=(_Tp __op) volatile noexcept
257+
requires __base::is_always_lock_free
258+
{
259+
return fetch_add(__op) + __op;
260+
}
261+
262+
_LIBCPP_HIDE_FROM_ABI _Tp operator+=(_Tp __op) noexcept { return fetch_add(__op) + __op; }
263+
264+
_LIBCPP_HIDE_FROM_ABI _Tp operator-=(_Tp __op) volatile noexcept
265+
requires __base::is_always_lock_free
266+
{
267+
return fetch_sub(__op) - __op;
268+
}
269+
270+
_LIBCPP_HIDE_FROM_ABI _Tp operator-=(_Tp __op) noexcept { return fetch_sub(__op) - __op; }
271+
};
272+
273+
#endif // _LIBCPP_STD_VER >= 20
274+
139275
// atomic_is_lock_free
140276

141277
template <class _Tp>

‎libcxx/include/__atomic/cxx_atomic_impl.h

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,18 @@ _Tp __cxx_atomic_load(const volatile __cxx_atomic_base_impl<_Tp>* __a,
128128
return __ret;
129129
}
130130

131+
template <typename _Tp>
132+
_LIBCPP_HIDE_FROM_ABI void
133+
__cxx_atomic_load_inplace(const volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp* __dst, memory_order __order) {
134+
__atomic_load(std::addressof(__a->__a_value), __dst, __to_gcc_order(__order));
135+
}
136+
137+
template <typename _Tp>
138+
_LIBCPP_HIDE_FROM_ABI void
139+
__cxx_atomic_load_inplace(const __cxx_atomic_base_impl<_Tp>* __a, _Tp* __dst, memory_order __order) {
140+
__atomic_load(std::addressof(__a->__a_value), __dst, __to_gcc_order(__order));
141+
}
142+
131143
template <typename _Tp>
132144
_LIBCPP_HIDE_FROM_ABI
133145
_Tp __cxx_atomic_load(const __cxx_atomic_base_impl<_Tp>* __a, memory_order __order) {
@@ -362,6 +374,21 @@ _Tp __cxx_atomic_load(__cxx_atomic_base_impl<_Tp> const* __a, memory_order __ord
362374
const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order));
363375
}
364376

377+
template <class _Tp>
378+
_LIBCPP_HIDE_FROM_ABI void
379+
__cxx_atomic_load_inplace(__cxx_atomic_base_impl<_Tp> const volatile* __a, _Tp* __dst, memory_order __order) _NOEXCEPT {
380+
using __ptr_type = __remove_const_t<decltype(__a->__a_value)>*;
381+
*__dst = __c11_atomic_load(
382+
const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order));
383+
}
384+
template <class _Tp>
385+
_LIBCPP_HIDE_FROM_ABI void
386+
__cxx_atomic_load_inplace(__cxx_atomic_base_impl<_Tp> const* __a, _Tp* __dst, memory_order __order) _NOEXCEPT {
387+
using __ptr_type = __remove_const_t<decltype(__a->__a_value)>*;
388+
*__dst = __c11_atomic_load(
389+
const_cast<__ptr_type>(std::addressof(__a->__a_value)), static_cast<__memory_order_underlying_t>(__order));
390+
}
391+
365392
template<class _Tp>
366393
_LIBCPP_HIDE_FROM_ABI
367394
_Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __value, memory_order __order) _NOEXCEPT {
@@ -558,6 +585,16 @@ struct __cxx_atomic_lock_impl {
558585
__unlock();
559586
return __old;
560587
}
588+
_LIBCPP_HIDE_FROM_ABI void __read_inplace(_Tp* __dst) const volatile {
589+
__lock();
590+
__cxx_atomic_assign_volatile(*__dst, __a_value);
591+
__unlock();
592+
}
593+
_LIBCPP_HIDE_FROM_ABI void __read_inplace(_Tp* __dst) const {
594+
__lock();
595+
*__dst = __a_value;
596+
__unlock();
597+
}
561598
};
562599

563600
template <typename _Tp>
@@ -597,6 +634,16 @@ _Tp __cxx_atomic_load(const __cxx_atomic_lock_impl<_Tp>* __a, memory_order) {
597634
return __a->__read();
598635
}
599636

637+
template <typename _Tp>
638+
_LIBCPP_HIDE_FROM_ABI void
639+
__cxx_atomic_load(const volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp* __dst, memory_order) {
640+
__a->__read_inplace(__dst);
641+
}
642+
template <typename _Tp>
643+
_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_load(const __cxx_atomic_lock_impl<_Tp>* __a, _Tp* __dst, memory_order) {
644+
__a->__read_inplace(__dst);
645+
}
646+
600647
template <typename _Tp>
601648
_LIBCPP_HIDE_FROM_ABI
602649
_Tp __cxx_atomic_exchange(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp __value, memory_order) {

‎libcxx/include/atomic

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -262,6 +262,72 @@ struct atomic<T*>
262262
void notify_all() noexcept;
263263
};
264264
265+
template<>
266+
struct atomic<floating-point-type> { // since C++20
267+
using value_type = floating-point-type;
268+
using difference_type = value_type;
269+
270+
static constexpr bool is_always_lock_free = implementation-defined;
271+
bool is_lock_free() const volatile noexcept;
272+
bool is_lock_free() const noexcept;
273+
274+
constexpr atomic() noexcept;
275+
constexpr atomic(floating-point-type) noexcept;
276+
atomic(const atomic&) = delete;
277+
atomic& operator=(const atomic&) = delete;
278+
atomic& operator=(const atomic&) volatile = delete;
279+
280+
void store(floating-point-type, memory_order = memory_order::seq_cst) volatile noexcept;
281+
void store(floating-point-type, memory_order = memory_order::seq_cst) noexcept;
282+
floating-point-type operator=(floating-point-type) volatile noexcept;
283+
floating-point-type operator=(floating-point-type) noexcept;
284+
floating-point-type load(memory_order = memory_order::seq_cst) volatile noexcept;
285+
floating-point-type load(memory_order = memory_order::seq_cst) noexcept;
286+
operator floating-point-type() volatile noexcept;
287+
operator floating-point-type() noexcept;
288+
289+
floating-point-type exchange(floating-point-type,
290+
memory_order = memory_order::seq_cst) volatile noexcept;
291+
floating-point-type exchange(floating-point-type,
292+
memory_order = memory_order::seq_cst) noexcept;
293+
bool compare_exchange_weak(floating-point-type&, floating-point-type,
294+
memory_order, memory_order) volatile noexcept;
295+
bool compare_exchange_weak(floating-point-type&, floating-point-type,
296+
memory_order, memory_order) noexcept;
297+
bool compare_exchange_strong(floating-point-type&, floating-point-type,
298+
memory_order, memory_order) volatile noexcept;
299+
bool compare_exchange_strong(floating-point-type&, floating-point-type,
300+
memory_order, memory_order) noexcept;
301+
bool compare_exchange_weak(floating-point-type&, floating-point-type,
302+
memory_order = memory_order::seq_cst) volatile noexcept;
303+
bool compare_exchange_weak(floating-point-type&, floating-point-type,
304+
memory_order = memory_order::seq_cst) noexcept;
305+
bool compare_exchange_strong(floating-point-type&, floating-point-type,
306+
memory_order = memory_order::seq_cst) volatile noexcept;
307+
bool compare_exchange_strong(floating-point-type&, floating-point-type,
308+
memory_order = memory_order::seq_cst) noexcept;
309+
310+
floating-point-type fetch_add(floating-point-type,
311+
memory_order = memory_order::seq_cst) volatile noexcept;
312+
floating-point-type fetch_add(floating-point-type,
313+
memory_order = memory_order::seq_cst) noexcept;
314+
floating-point-type fetch_sub(floating-point-type,
315+
memory_order = memory_order::seq_cst) volatile noexcept;
316+
floating-point-type fetch_sub(floating-point-type,
317+
memory_order = memory_order::seq_cst) noexcept;
318+
319+
floating-point-type operator+=(floating-point-type) volatile noexcept;
320+
floating-point-type operator+=(floating-point-type) noexcept;
321+
floating-point-type operator-=(floating-point-type) volatile noexcept;
322+
floating-point-type operator-=(floating-point-type) noexcept;
323+
324+
void wait(floating-point-type, memory_order = memory_order::seq_cst) const volatile noexcept;
325+
void wait(floating-point-type, memory_order = memory_order::seq_cst) const noexcept;
326+
void notify_one() volatile noexcept;
327+
void notify_one() noexcept;
328+
void notify_all() volatile noexcept;
329+
void notify_all() noexcept;
330+
};
265331
266332
// [atomics.nonmembers], non-member functions
267333
template<class T>

‎libcxx/test/libcxx/atomics/atomics.types.generic/atomics.types.float/lockfree.pass.cpp

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
//===----------------------------------------------------------------------===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
// UNSUPPORTED: c++03, c++11, c++14, c++17
9+
// UNSUPPORTED: target={{.+}}-windows-gnu
10+
// ADDITIONAL_COMPILE_FLAGS(has-latomic): -latomic
11+
12+
// static constexpr bool is_always_lock_free = implementation-defined;
13+
// bool is_lock_free() const volatile noexcept;
14+
// bool is_lock_free() const noexcept;
15+
16+
#include <atomic>
17+
#include <cassert>
18+
#include <concepts>
19+
20+
#include "test_macros.h"
21+
22+
template <class T>
23+
void test() {
24+
// static constexpr bool is_always_lock_free = implementation-defined;
25+
{
26+
bool r = std::atomic<T>::is_always_lock_free;
27+
assert(r == __atomic_always_lock_free(sizeof(std::__cxx_atomic_impl<T>), 0));
28+
}
29+
30+
// bool is_lock_free() const volatile noexcept;
31+
{
32+
const volatile std::atomic<T> a;
33+
bool r = a.is_lock_free();
34+
assert(r == __cxx_atomic_is_lock_free(sizeof(std::__cxx_atomic_impl<T>)));
35+
}
36+
37+
// bool is_lock_free() const noexcept;
38+
{
39+
const std::atomic<T> a;
40+
bool r = a.is_lock_free();
41+
assert(r == __cxx_atomic_is_lock_free(sizeof(std::__cxx_atomic_impl<T>)));
42+
}
43+
}
44+
45+
int main(int, char**) {
46+
test<float>();
47+
test<double>();
48+
test<long double>();
49+
50+
return 0;
51+
}

0 commit comments

Comments
 (0)