Skip to content

Commit 654672d

Browse files
wildea01Ingo Molnar
wildea01
authored and
Ingo Molnar
committed
locking/atomics: Add _{acquire|release|relaxed}() variants of some atomic operations
Whilst porting the generic qrwlock code over to arm64, it became apparent that any portable locking code needs finer-grained control of the memory-ordering guarantees provided by our atomic routines. In particular: xchg, cmpxchg, {add,sub}_return are often used in situations where full barrier semantics (currently the only option available) are not required. For example, when a reader increments a reader count to obtain a lock, checking the old value to see if a writer was present, only acquire semantics are strictly needed. This patch introduces three new ordering semantics for these operations: - *_relaxed: No ordering guarantees. This is similar to what we have already for the non-return atomics (e.g. atomic_add). - *_acquire: ACQUIRE semantics, similar to smp_load_acquire. - *_release: RELEASE semantics, similar to smp_store_release. In memory-ordering speak, this means that the acquire/release semantics are RCpc as opposed to RCsc. Consequently a RELEASE followed by an ACQUIRE does not imply a full barrier, as already documented in memory-barriers.txt. Currently, all the new macros are conditionally mapped to the full-mb variants, however if the *_relaxed version is provided by the architecture, then the acquire/release variants are constructed by supplementing the relaxed routine with an explicit barrier. Signed-off-by: Will Deacon <[email protected]> Signed-off-by: Peter Zijlstra (Intel) <[email protected]> Cc: Linus Torvalds <[email protected]> Cc: Peter Zijlstra <[email protected]> Cc: Thomas Gleixner <[email protected]> Cc: [email protected] Cc: [email protected] Link: http://lkml.kernel.org/r/[email protected] Signed-off-by: Ingo Molnar <[email protected]>
1 parent ba33034 commit 654672d

File tree

1 file changed

+323
-0
lines changed

1 file changed

+323
-0
lines changed

‎include/linux/atomic.h

Lines changed: 323 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,329 @@
22
#ifndef _LINUX_ATOMIC_H
33
#define _LINUX_ATOMIC_H
44
#include <asm/atomic.h>
5+
#include <asm/barrier.h>
6+
7+
/*
8+
* Relaxed variants of xchg, cmpxchg and some atomic operations.
9+
*
10+
* We support four variants:
11+
*
12+
* - Fully ordered: The default implementation, no suffix required.
13+
* - Acquire: Provides ACQUIRE semantics, _acquire suffix.
14+
* - Release: Provides RELEASE semantics, _release suffix.
15+
* - Relaxed: No ordering guarantees, _relaxed suffix.
16+
*
17+
* For compound atomics performing both a load and a store, ACQUIRE
18+
* semantics apply only to the load and RELEASE semantics only to the
19+
* store portion of the operation. Note that a failed cmpxchg_acquire
20+
* does -not- imply any memory ordering constraints.
21+
*
22+
* See Documentation/memory-barriers.txt for ACQUIRE/RELEASE definitions.
23+
*/
24+
25+
#ifndef atomic_read_acquire
26+
#define atomic_read_acquire(v) smp_load_acquire(&(v)->counter)
27+
#endif
28+
29+
#ifndef atomic_set_release
30+
#define atomic_set_release(v, i) smp_store_release(&(v)->counter, (i))
31+
#endif
32+
33+
/*
34+
* The idea here is to build acquire/release variants by adding explicit
35+
* barriers on top of the relaxed variant. In the case where the relaxed
36+
* variant is already fully ordered, no additional barriers are needed.
37+
*/
38+
#define __atomic_op_acquire(op, args...) \
39+
({ \
40+
typeof(op##_relaxed(args)) __ret = op##_relaxed(args); \
41+
smp_mb__after_atomic(); \
42+
__ret; \
43+
})
44+
45+
#define __atomic_op_release(op, args...) \
46+
({ \
47+
smp_mb__before_atomic(); \
48+
op##_relaxed(args); \
49+
})
50+
51+
#define __atomic_op_fence(op, args...) \
52+
({ \
53+
typeof(op##_relaxed(args)) __ret; \
54+
smp_mb__before_atomic(); \
55+
__ret = op##_relaxed(args); \
56+
smp_mb__after_atomic(); \
57+
__ret; \
58+
})
59+
60+
/* atomic_add_return_relaxed */
61+
#ifndef atomic_add_return_relaxed
62+
#define atomic_add_return_relaxed atomic_add_return
63+
#define atomic_add_return_acquire atomic_add_return
64+
#define atomic_add_return_release atomic_add_return
65+
66+
#else /* atomic_add_return_relaxed */
67+
68+
#ifndef atomic_add_return_acquire
69+
#define atomic_add_return_acquire(...) \
70+
__atomic_op_acquire(atomic_add_return, __VA_ARGS__)
71+
#endif
72+
73+
#ifndef atomic_add_return_release
74+
#define atomic_add_return_release(...) \
75+
__atomic_op_release(atomic_add_return, __VA_ARGS__)
76+
#endif
77+
78+
#ifndef atomic_add_return
79+
#define atomic_add_return(...) \
80+
__atomic_op_fence(atomic_add_return, __VA_ARGS__)
81+
#endif
82+
#endif /* atomic_add_return_relaxed */
83+
84+
/* atomic_sub_return_relaxed */
85+
#ifndef atomic_sub_return_relaxed
86+
#define atomic_sub_return_relaxed atomic_sub_return
87+
#define atomic_sub_return_acquire atomic_sub_return
88+
#define atomic_sub_return_release atomic_sub_return
89+
90+
#else /* atomic_sub_return_relaxed */
91+
92+
#ifndef atomic_sub_return_acquire
93+
#define atomic_sub_return_acquire(...) \
94+
__atomic_op_acquire(atomic_sub_return, __VA_ARGS__)
95+
#endif
96+
97+
#ifndef atomic_sub_return_release
98+
#define atomic_sub_return_release(...) \
99+
__atomic_op_release(atomic_sub_return, __VA_ARGS__)
100+
#endif
101+
102+
#ifndef atomic_sub_return
103+
#define atomic_sub_return(...) \
104+
__atomic_op_fence(atomic_sub_return, __VA_ARGS__)
105+
#endif
106+
#endif /* atomic_sub_return_relaxed */
107+
108+
/* atomic_xchg_relaxed */
109+
#ifndef atomic_xchg_relaxed
110+
#define atomic_xchg_relaxed atomic_xchg
111+
#define atomic_xchg_acquire atomic_xchg
112+
#define atomic_xchg_release atomic_xchg
113+
114+
#else /* atomic_xchg_relaxed */
115+
116+
#ifndef atomic_xchg_acquire
117+
#define atomic_xchg_acquire(...) \
118+
__atomic_op_acquire(atomic_xchg, __VA_ARGS__)
119+
#endif
120+
121+
#ifndef atomic_xchg_release
122+
#define atomic_xchg_release(...) \
123+
__atomic_op_release(atomic_xchg, __VA_ARGS__)
124+
#endif
125+
126+
#ifndef atomic_xchg
127+
#define atomic_xchg(...) \
128+
__atomic_op_fence(atomic_xchg, __VA_ARGS__)
129+
#endif
130+
#endif /* atomic_xchg_relaxed */
131+
132+
/* atomic_cmpxchg_relaxed */
133+
#ifndef atomic_cmpxchg_relaxed
134+
#define atomic_cmpxchg_relaxed atomic_cmpxchg
135+
#define atomic_cmpxchg_acquire atomic_cmpxchg
136+
#define atomic_cmpxchg_release atomic_cmpxchg
137+
138+
#else /* atomic_cmpxchg_relaxed */
139+
140+
#ifndef atomic_cmpxchg_acquire
141+
#define atomic_cmpxchg_acquire(...) \
142+
__atomic_op_acquire(atomic_cmpxchg, __VA_ARGS__)
143+
#endif
144+
145+
#ifndef atomic_cmpxchg_release
146+
#define atomic_cmpxchg_release(...) \
147+
__atomic_op_release(atomic_cmpxchg, __VA_ARGS__)
148+
#endif
149+
150+
#ifndef atomic_cmpxchg
151+
#define atomic_cmpxchg(...) \
152+
__atomic_op_fence(atomic_cmpxchg, __VA_ARGS__)
153+
#endif
154+
#endif /* atomic_cmpxchg_relaxed */
155+
156+
#ifndef atomic64_read_acquire
157+
#define atomic64_read_acquire(v) smp_load_acquire(&(v)->counter)
158+
#endif
159+
160+
#ifndef atomic64_set_release
161+
#define atomic64_set_release(v, i) smp_store_release(&(v)->counter, (i))
162+
#endif
163+
164+
/* atomic64_add_return_relaxed */
165+
#ifndef atomic64_add_return_relaxed
166+
#define atomic64_add_return_relaxed atomic64_add_return
167+
#define atomic64_add_return_acquire atomic64_add_return
168+
#define atomic64_add_return_release atomic64_add_return
169+
170+
#else /* atomic64_add_return_relaxed */
171+
172+
#ifndef atomic64_add_return_acquire
173+
#define atomic64_add_return_acquire(...) \
174+
__atomic_op_acquire(atomic64_add_return, __VA_ARGS__)
175+
#endif
176+
177+
#ifndef atomic64_add_return_release
178+
#define atomic64_add_return_release(...) \
179+
__atomic_op_release(atomic64_add_return, __VA_ARGS__)
180+
#endif
181+
182+
#ifndef atomic64_add_return
183+
#define atomic64_add_return(...) \
184+
__atomic_op_fence(atomic64_add_return, __VA_ARGS__)
185+
#endif
186+
#endif /* atomic64_add_return_relaxed */
187+
188+
/* atomic64_sub_return_relaxed */
189+
#ifndef atomic64_sub_return_relaxed
190+
#define atomic64_sub_return_relaxed atomic64_sub_return
191+
#define atomic64_sub_return_acquire atomic64_sub_return
192+
#define atomic64_sub_return_release atomic64_sub_return
193+
194+
#else /* atomic64_sub_return_relaxed */
195+
196+
#ifndef atomic64_sub_return_acquire
197+
#define atomic64_sub_return_acquire(...) \
198+
__atomic_op_acquire(atomic64_sub_return, __VA_ARGS__)
199+
#endif
200+
201+
#ifndef atomic64_sub_return_release
202+
#define atomic64_sub_return_release(...) \
203+
__atomic_op_release(atomic64_sub_return, __VA_ARGS__)
204+
#endif
205+
206+
#ifndef atomic64_sub_return
207+
#define atomic64_sub_return(...) \
208+
__atomic_op_fence(atomic64_sub_return, __VA_ARGS__)
209+
#endif
210+
#endif /* atomic64_sub_return_relaxed */
211+
212+
/* atomic64_xchg_relaxed */
213+
#ifndef atomic64_xchg_relaxed
214+
#define atomic64_xchg_relaxed atomic64_xchg
215+
#define atomic64_xchg_acquire atomic64_xchg
216+
#define atomic64_xchg_release atomic64_xchg
217+
218+
#else /* atomic64_xchg_relaxed */
219+
220+
#ifndef atomic64_xchg_acquire
221+
#define atomic64_xchg_acquire(...) \
222+
__atomic_op_acquire(atomic64_xchg, __VA_ARGS__)
223+
#endif
224+
225+
#ifndef atomic64_xchg_release
226+
#define atomic64_xchg_release(...) \
227+
__atomic_op_release(atomic64_xchg, __VA_ARGS__)
228+
#endif
229+
230+
#ifndef atomic64_xchg
231+
#define atomic64_xchg(...) \
232+
__atomic_op_fence(atomic64_xchg, __VA_ARGS__)
233+
#endif
234+
#endif /* atomic64_xchg_relaxed */
235+
236+
/* atomic64_cmpxchg_relaxed */
237+
#ifndef atomic64_cmpxchg_relaxed
238+
#define atomic64_cmpxchg_relaxed atomic64_cmpxchg
239+
#define atomic64_cmpxchg_acquire atomic64_cmpxchg
240+
#define atomic64_cmpxchg_release atomic64_cmpxchg
241+
242+
#else /* atomic64_cmpxchg_relaxed */
243+
244+
#ifndef atomic64_cmpxchg_acquire
245+
#define atomic64_cmpxchg_acquire(...) \
246+
__atomic_op_acquire(atomic64_cmpxchg, __VA_ARGS__)
247+
#endif
248+
249+
#ifndef atomic64_cmpxchg_release
250+
#define atomic64_cmpxchg_release(...) \
251+
__atomic_op_release(atomic64_cmpxchg, __VA_ARGS__)
252+
#endif
253+
254+
#ifndef atomic64_cmpxchg
255+
#define atomic64_cmpxchg(...) \
256+
__atomic_op_fence(atomic64_cmpxchg, __VA_ARGS__)
257+
#endif
258+
#endif /* atomic64_cmpxchg_relaxed */
259+
260+
/* cmpxchg_relaxed */
261+
#ifndef cmpxchg_relaxed
262+
#define cmpxchg_relaxed cmpxchg
263+
#define cmpxchg_acquire cmpxchg
264+
#define cmpxchg_release cmpxchg
265+
266+
#else /* cmpxchg_relaxed */
267+
268+
#ifndef cmpxchg_acquire
269+
#define cmpxchg_acquire(...) \
270+
__atomic_op_acquire(cmpxchg, __VA_ARGS__)
271+
#endif
272+
273+
#ifndef cmpxchg_release
274+
#define cmpxchg_release(...) \
275+
__atomic_op_release(cmpxchg, __VA_ARGS__)
276+
#endif
277+
278+
#ifndef cmpxchg
279+
#define cmpxchg(...) \
280+
__atomic_op_fence(cmpxchg, __VA_ARGS__)
281+
#endif
282+
#endif /* cmpxchg_relaxed */
283+
284+
/* cmpxchg64_relaxed */
285+
#ifndef cmpxchg64_relaxed
286+
#define cmpxchg64_relaxed cmpxchg64
287+
#define cmpxchg64_acquire cmpxchg64
288+
#define cmpxchg64_release cmpxchg64
289+
290+
#else /* cmpxchg64_relaxed */
291+
292+
#ifndef cmpxchg64_acquire
293+
#define cmpxchg64_acquire(...) \
294+
__atomic_op_acquire(cmpxchg64, __VA_ARGS__)
295+
#endif
296+
297+
#ifndef cmpxchg64_release
298+
#define cmpxchg64_release(...) \
299+
__atomic_op_release(cmpxchg64, __VA_ARGS__)
300+
#endif
301+
302+
#ifndef cmpxchg64
303+
#define cmpxchg64(...) \
304+
__atomic_op_fence(cmpxchg64, __VA_ARGS__)
305+
#endif
306+
#endif /* cmpxchg64_relaxed */
307+
308+
/* xchg_relaxed */
309+
#ifndef xchg_relaxed
310+
#define xchg_relaxed xchg
311+
#define xchg_acquire xchg
312+
#define xchg_release xchg
313+
314+
#else /* xchg_relaxed */
315+
316+
#ifndef xchg_acquire
317+
#define xchg_acquire(...) __atomic_op_acquire(xchg, __VA_ARGS__)
318+
#endif
319+
320+
#ifndef xchg_release
321+
#define xchg_release(...) __atomic_op_release(xchg, __VA_ARGS__)
322+
#endif
323+
324+
#ifndef xchg
325+
#define xchg(...) __atomic_op_fence(xchg, __VA_ARGS__)
326+
#endif
327+
#endif /* xchg_relaxed */
5328

6329
/**
7330
* atomic_add_unless - add unless the number is already a given value

0 commit comments

Comments
 (0)