Skip to content

Commit 2bb4045

Browse files
kjbraceykjbracey
committed
Add non-volatile atomic loads and stores
Volatile makes no real difference when we're using assembler, or locked functions, but leaving it off could be more efficient for the basic loads and stores. So add non-volatile overloads in C++ for them.
1 parent 4fe06e0 commit 2bb4045

File tree

2 files changed

+113
-13
lines changed

2 files changed

+113
-13
lines changed

‎platform/internal/mbed_atomic_impl.h

Lines changed: 97 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -613,30 +613,30 @@ MBED_FORCEINLINE bool core_util_atomic_flag_test_and_set_explicit(volatile core_
613613

614614
/* Lock-free loads and stores don't need assembler - just aligned accesses */
615615
/* Silly ordering of `T volatile` is because T can be `void *` */
616-
#define DO_MBED_LOCKFREE_LOADSTORE(T, fn_suffix) \
617-
MBED_FORCEINLINE T core_util_atomic_load_##fn_suffix(T const volatile *valuePtr) \
616+
#define DO_MBED_LOCKFREE_LOADSTORE(T, V, fn_suffix) \
617+
MBED_FORCEINLINE T core_util_atomic_load_##fn_suffix(T const V *valuePtr) \
618618
{ \
619619
T value = *valuePtr; \
620620
MBED_BARRIER(); \
621621
return value; \
622622
} \
623623
\
624-
MBED_FORCEINLINE T core_util_atomic_load_explicit_##fn_suffix(T const volatile *valuePtr, mbed_memory_order order) \
624+
MBED_FORCEINLINE T core_util_atomic_load_explicit_##fn_suffix(T const V *valuePtr, mbed_memory_order order) \
625625
{ \
626626
MBED_CHECK_LOAD_ORDER(order); \
627627
T value = *valuePtr; \
628628
MBED_ACQUIRE_BARRIER(order); \
629629
return value; \
630630
} \
631631
\
632-
MBED_FORCEINLINE void core_util_atomic_store_##fn_suffix(T volatile *valuePtr, T value) \
632+
MBED_FORCEINLINE void core_util_atomic_store_##fn_suffix(T V *valuePtr, T value) \
633633
{ \
634634
MBED_BARRIER(); \
635635
*valuePtr = value; \
636636
MBED_BARRIER(); \
637637
} \
638638
\
639-
MBED_FORCEINLINE void core_util_atomic_store_explicit_##fn_suffix(T volatile *valuePtr, T value, mbed_memory_order order) \
639+
MBED_FORCEINLINE void core_util_atomic_store_explicit_##fn_suffix(T V *valuePtr, T value, mbed_memory_order order) \
640640
{ \
641641
MBED_CHECK_STORE_ORDER(order); \
642642
MBED_RELEASE_BARRIER(order); \
@@ -658,15 +658,51 @@ MBED_FORCEINLINE void core_util_atomic_flag_clear_explicit(volatile core_util_at
658658
flagPtr->_flag = false;
659659
MBED_SEQ_CST_BARRIER(order);
660660
}
661-
DO_MBED_LOCKFREE_LOADSTORE(uint8_t, u8)
662-
DO_MBED_LOCKFREE_LOADSTORE(uint16_t, u16)
663-
DO_MBED_LOCKFREE_LOADSTORE(uint32_t, u32)
664-
DO_MBED_LOCKFREE_LOADSTORE(int8_t, s8)
665-
DO_MBED_LOCKFREE_LOADSTORE(int16_t, s16)
666-
DO_MBED_LOCKFREE_LOADSTORE(int32_t, s32)
667-
DO_MBED_LOCKFREE_LOADSTORE(bool, bool)
668-
DO_MBED_LOCKFREE_LOADSTORE(void *, ptr)
669661

662+
#ifdef __cplusplus
663+
// Temporarily turn off extern "C", so we can provide non-volatile load/store
664+
// overloads for efficiency. All these functions are static inline, so this has
665+
// no linkage effect exactly, it just permits the overloads.
666+
} // extern "C"
667+
668+
// For efficiency it's worth having non-volatile overloads
669+
MBED_FORCEINLINE void core_util_atomic_flag_clear(core_util_atomic_flag *flagPtr)
670+
{
671+
MBED_BARRIER();
672+
flagPtr->_flag = false;
673+
MBED_BARRIER();
674+
}
675+
676+
MBED_FORCEINLINE void core_util_atomic_flag_clear_explicit(core_util_atomic_flag *flagPtr, mbed_memory_order order)
677+
{
678+
MBED_RELEASE_BARRIER(order);
679+
flagPtr->_flag = false;
680+
MBED_SEQ_CST_BARRIER(order);
681+
}
682+
683+
DO_MBED_LOCKFREE_LOADSTORE(uint8_t,, u8)
684+
DO_MBED_LOCKFREE_LOADSTORE(uint16_t,, u16)
685+
DO_MBED_LOCKFREE_LOADSTORE(uint32_t,, u32)
686+
DO_MBED_LOCKFREE_LOADSTORE(int8_t,, s8)
687+
DO_MBED_LOCKFREE_LOADSTORE(int16_t,, s16)
688+
DO_MBED_LOCKFREE_LOADSTORE(int32_t,, s32)
689+
DO_MBED_LOCKFREE_LOADSTORE(bool,, bool)
690+
DO_MBED_LOCKFREE_LOADSTORE(void *,, ptr)
691+
692+
#endif
693+
694+
DO_MBED_LOCKFREE_LOADSTORE(uint8_t, volatile, u8)
695+
DO_MBED_LOCKFREE_LOADSTORE(uint16_t, volatile, u16)
696+
DO_MBED_LOCKFREE_LOADSTORE(uint32_t, volatile, u32)
697+
DO_MBED_LOCKFREE_LOADSTORE(int8_t, volatile, s8)
698+
DO_MBED_LOCKFREE_LOADSTORE(int16_t, volatile, s16)
699+
DO_MBED_LOCKFREE_LOADSTORE(int32_t, volatile, s32)
700+
DO_MBED_LOCKFREE_LOADSTORE(bool, volatile, bool)
701+
DO_MBED_LOCKFREE_LOADSTORE(void *, volatile, ptr)
702+
703+
#ifdef __cplusplus
704+
extern "C" {
705+
#endif
670706

671707
/********************* GENERIC VARIANTS - SIGNED, BOOL, POINTERS ****************/
672708

@@ -975,7 +1011,19 @@ inline T core_util_atomic_load(const volatile T *valuePtr)
9751011
} \
9761012
\
9771013
template<> \
1014+
inline T core_util_atomic_load(const T *valuePtr) \
1015+
{ \
1016+
return core_util_atomic_load_##fn_suffix(valuePtr); \
1017+
} \
1018+
\
1019+
template<> \
9781020
inline T core_util_atomic_load_explicit(const volatile T *valuePtr, mbed_memory_order order) \
1021+
{ \
1022+
return core_util_atomic_load_explicit_##fn_suffix(valuePtr, order); \
1023+
} \
1024+
\
1025+
template<> \
1026+
inline T core_util_atomic_load_explicit(const T *valuePtr, mbed_memory_order order) \
9791027
{ \
9801028
return core_util_atomic_load_explicit_##fn_suffix(valuePtr, order); \
9811029
}
@@ -986,12 +1034,24 @@ inline T *core_util_atomic_load(T *const volatile *valuePtr)
9861034
return (T *) core_util_atomic_load_ptr((void *const volatile *) valuePtr);
9871035
}
9881036

1037+
template<typename T>
1038+
inline T *core_util_atomic_load(T *const *valuePtr)
1039+
{
1040+
return (T *) core_util_atomic_load_ptr((void *const *) valuePtr);
1041+
}
1042+
9891043
template<typename T>
9901044
inline T *core_util_atomic_load_explicit(T *const volatile *valuePtr, mbed_memory_order order)
9911045
{
9921046
return (T *) core_util_atomic_load_explicit_ptr((void *const volatile *) valuePtr, order);
9931047
}
9941048

1049+
template<typename T>
1050+
inline T *core_util_atomic_load_explicit(T *const *valuePtr, mbed_memory_order order)
1051+
{
1052+
return (T *) core_util_atomic_load_explicit_ptr((void *const *) valuePtr, order);
1053+
}
1054+
9951055
DO_MBED_ATOMIC_LOAD_TEMPLATE(uint8_t, u8)
9961056
DO_MBED_ATOMIC_LOAD_TEMPLATE(uint16_t, u16)
9971057
DO_MBED_ATOMIC_LOAD_TEMPLATE(uint32_t, u32)
@@ -1010,7 +1070,19 @@ inline void core_util_atomic_store(volatile T *valuePtr, T val)
10101070
} \
10111071
\
10121072
template<> \
1073+
inline void core_util_atomic_store(T *valuePtr, T val) \
1074+
{ \
1075+
core_util_atomic_store_##fn_suffix(valuePtr, val); \
1076+
} \
1077+
\
1078+
template<> \
10131079
inline void core_util_atomic_store_explicit(volatile T *valuePtr, T val, mbed_memory_order order) \
1080+
{ \
1081+
core_util_atomic_store_explicit_##fn_suffix(valuePtr, val, order); \
1082+
} \
1083+
\
1084+
template<> \
1085+
inline void core_util_atomic_store_explicit(T *valuePtr, T val, mbed_memory_order order) \
10141086
{ \
10151087
core_util_atomic_store_explicit_##fn_suffix(valuePtr, val, order); \
10161088
}
@@ -1021,12 +1093,24 @@ inline void core_util_atomic_store(T *volatile *valuePtr, T *val)
10211093
core_util_atomic_store_ptr((void *volatile *) valuePtr, val);
10221094
}
10231095

1096+
template<typename T>
1097+
inline void core_util_atomic_store(T **valuePtr, T *val)
1098+
{
1099+
core_util_atomic_store_ptr((void **) valuePtr, val);
1100+
}
1101+
10241102
template<typename T>
10251103
inline void core_util_atomic_store_explicit(T *volatile *valuePtr, T *val, mbed_memory_order order)
10261104
{
10271105
core_util_atomic_store_ptr((void *volatile *) valuePtr, val, order);
10281106
}
10291107

1108+
template<typename T>
1109+
inline void core_util_atomic_store_explicit(T **valuePtr, T *val, mbed_memory_order order)
1110+
{
1111+
core_util_atomic_store_ptr((void **) valuePtr, val, order);
1112+
}
1113+
10301114
DO_MBED_ATOMIC_STORE_TEMPLATE(uint8_t, u8)
10311115
DO_MBED_ATOMIC_STORE_TEMPLATE(uint16_t, u16)
10321116
DO_MBED_ATOMIC_STORE_TEMPLATE(uint32_t, u32)

‎platform/mbed_atomic.h

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -896,8 +896,12 @@ MBED_FORCEINLINE uint64_t core_util_atomic_fetch_xor_explicit_u64(volatile uint6
896896

897897
/** \copydoc core_util_atomic_load_u8 */
898898
template<typename T> T core_util_atomic_load(const volatile T *valuePtr);
899+
/** \copydoc core_util_atomic_load_u8 */
900+
template<typename T> T core_util_atomic_load(const T *valuePtr);
899901
/** \copydoc core_util_atomic_store_u8 */
900902
template<typename T> void core_util_atomic_store(volatile T *valuePtr, T desiredValue);
903+
/** \copydoc core_util_atomic_store_u8 */
904+
template<typename T> void core_util_atomic_store(T *valuePtr, T desiredValue);
901905
/** \copydoc core_util_atomic_exchange_u8 */
902906
template<typename T> T core_util_atomic_exchange(volatile T *ptr, T desiredValue);
903907
/** \copydoc core_util_atomic_cas_u8 */
@@ -917,8 +921,12 @@ template<typename T> T core_util_atomic_fetch_xor(volatile T *valuePtr, T arg);
917921

918922
/** \copydoc core_util_atomic_load_explicit_u8 */
919923
template<typename T> T core_util_atomic_load_explicit(const volatile T *valuePtr, mbed_memory_order order);
924+
/** \copydoc core_util_atomic_load_explicit_u8 */
925+
template<typename T> T core_util_atomic_load_explicit(const T *valuePtr, mbed_memory_order order);
920926
/** \copydoc core_util_atomic_store_explicit_u8 */
921927
template<typename T> void core_util_atomic_store_explicit(volatile T *valuePtr, T desiredValue, mbed_memory_order order);
928+
/** \copydoc core_util_atomic_store_explicit_u8 */
929+
template<typename T> void core_util_atomic_store_explicit(T *valuePtr, T desiredValue, mbed_memory_order order);
922930
/** \copydoc core_util_atomic_exchange_explicit_u8 */
923931
template<typename T> T core_util_atomic_exchange_explicit(volatile T *ptr, T desiredValue, mbed_memory_order order);
924932
/** \copydoc core_util_atomic_cas_explicit_u8 */
@@ -938,8 +946,12 @@ template<typename T> T core_util_atomic_fetch_xor_explicit(volatile T *valuePtr,
938946

939947
/** \copydoc core_util_atomic_load_ptr */
940948
template<typename T> inline T *core_util_atomic_load(T *const volatile *valuePtr);
949+
/** \copydoc core_util_atomic_load_ptr */
950+
template<typename T> inline T *core_util_atomic_load(T *const *valuePtr);
941951
/** \copydoc core_util_atomic_store_ptr */
942952
template<typename T> inline void core_util_atomic_store(T *volatile *valuePtr, T *desiredValue);
953+
/** \copydoc core_util_atomic_store_ptr */
954+
template<typename T> inline void core_util_atomic_store(T **valuePtr, T *desiredValue);
943955
/** \copydoc core_util_atomic_exchange_ptr */
944956
template<typename T> inline T *core_util_atomic_exchange(T *volatile *valuePtr, T *desiredValue);
945957
/** \copydoc core_util_atomic_cas_ptr */
@@ -953,8 +965,12 @@ template<typename T> inline T *core_util_atomic_fetch_sub(T *volatile *valuePtr,
953965

954966
/** \copydoc core_util_atomic_load_explicit_ptr */
955967
template<typename T> inline T *core_util_atomic_load_explicit(T *const volatile *valuePtr, mbed_memory_order order);
968+
/** \copydoc core_util_atomic_load_explicit_ptr */
969+
template<typename T> inline T *core_util_atomic_load_explicit(T *const *valuePtr, mbed_memory_order order);
956970
/** \copydoc core_util_atomic_store_explicit_ptr */
957971
template<typename T> inline void core_util_atomic_store_explicit(T *volatile *valuePtr, T *desiredValue, mbed_memory_order order);
972+
/** \copydoc core_util_atomic_store_explicit_ptr */
973+
template<typename T> inline void core_util_atomic_store_explicit(T **valuePtr, T *desiredValue, mbed_memory_order order);
958974
/** \copydoc core_util_atomic_exchange_explicit_ptr */
959975
template<typename T> inline T *core_util_atomic_exchange_explicit(T *volatile *valuePtr, T *desiredValue, mbed_memory_order order);
960976
/** \copydoc core_util_atomic_cas_explicit_ptr */

0 commit comments

Comments
 (0)