Merge tag 'pm+acpi-4.4-rc4' of git://git.kernel.org/pub/scm/linux/kernel/git/rafael...
[firefly-linux-kernel-4.4.55.git] / include / linux / atomic.h
1 /* Atomic operations usable in machine independent code */
2 #ifndef _LINUX_ATOMIC_H
3 #define _LINUX_ATOMIC_H
4 #include <asm/atomic.h>
5 #include <asm/barrier.h>
6
7 /*
8  * Relaxed variants of xchg, cmpxchg and some atomic operations.
9  *
10  * We support four variants:
11  *
12  * - Fully ordered: The default implementation, no suffix required.
13  * - Acquire: Provides ACQUIRE semantics, _acquire suffix.
14  * - Release: Provides RELEASE semantics, _release suffix.
15  * - Relaxed: No ordering guarantees, _relaxed suffix.
16  *
17  * For compound atomics performing both a load and a store, ACQUIRE
18  * semantics apply only to the load and RELEASE semantics only to the
19  * store portion of the operation. Note that a failed cmpxchg_acquire
20  * does -not- imply any memory ordering constraints.
21  *
22  * See Documentation/memory-barriers.txt for ACQUIRE/RELEASE definitions.
23  */
24
25 #ifndef atomic_read_acquire
26 #define  atomic_read_acquire(v)         smp_load_acquire(&(v)->counter)
27 #endif
28
29 #ifndef atomic_set_release
30 #define  atomic_set_release(v, i)       smp_store_release(&(v)->counter, (i))
31 #endif
32
33 /*
34  * The idea here is to build acquire/release variants by adding explicit
35  * barriers on top of the relaxed variant. In the case where the relaxed
36  * variant is already fully ordered, no additional barriers are needed.
37  */
38 #define __atomic_op_acquire(op, args...)                                \
39 ({                                                                      \
40         typeof(op##_relaxed(args)) __ret  = op##_relaxed(args);         \
41         smp_mb__after_atomic();                                         \
42         __ret;                                                          \
43 })
44
45 #define __atomic_op_release(op, args...)                                \
46 ({                                                                      \
47         smp_mb__before_atomic();                                        \
48         op##_relaxed(args);                                             \
49 })
50
51 #define __atomic_op_fence(op, args...)                                  \
52 ({                                                                      \
53         typeof(op##_relaxed(args)) __ret;                               \
54         smp_mb__before_atomic();                                        \
55         __ret = op##_relaxed(args);                                     \
56         smp_mb__after_atomic();                                         \
57         __ret;                                                          \
58 })
59
60 /* atomic_add_return_relaxed */
61 #ifndef atomic_add_return_relaxed
62 #define  atomic_add_return_relaxed      atomic_add_return
63 #define  atomic_add_return_acquire      atomic_add_return
64 #define  atomic_add_return_release      atomic_add_return
65
66 #else /* atomic_add_return_relaxed */
67
68 #ifndef atomic_add_return_acquire
69 #define  atomic_add_return_acquire(...)                                 \
70         __atomic_op_acquire(atomic_add_return, __VA_ARGS__)
71 #endif
72
73 #ifndef atomic_add_return_release
74 #define  atomic_add_return_release(...)                                 \
75         __atomic_op_release(atomic_add_return, __VA_ARGS__)
76 #endif
77
78 #ifndef atomic_add_return
79 #define  atomic_add_return(...)                                         \
80         __atomic_op_fence(atomic_add_return, __VA_ARGS__)
81 #endif
82 #endif /* atomic_add_return_relaxed */
83
84 /* atomic_inc_return_relaxed */
85 #ifndef atomic_inc_return_relaxed
86 #define  atomic_inc_return_relaxed      atomic_inc_return
87 #define  atomic_inc_return_acquire      atomic_inc_return
88 #define  atomic_inc_return_release      atomic_inc_return
89
90 #else /* atomic_inc_return_relaxed */
91
92 #ifndef atomic_inc_return_acquire
93 #define  atomic_inc_return_acquire(...)                                 \
94         __atomic_op_acquire(atomic_inc_return, __VA_ARGS__)
95 #endif
96
97 #ifndef atomic_inc_return_release
98 #define  atomic_inc_return_release(...)                                 \
99         __atomic_op_release(atomic_inc_return, __VA_ARGS__)
100 #endif
101
102 #ifndef atomic_inc_return
103 #define  atomic_inc_return(...)                                         \
104         __atomic_op_fence(atomic_inc_return, __VA_ARGS__)
105 #endif
106 #endif /* atomic_inc_return_relaxed */
107
108 /* atomic_sub_return_relaxed */
109 #ifndef atomic_sub_return_relaxed
110 #define  atomic_sub_return_relaxed      atomic_sub_return
111 #define  atomic_sub_return_acquire      atomic_sub_return
112 #define  atomic_sub_return_release      atomic_sub_return
113
114 #else /* atomic_sub_return_relaxed */
115
116 #ifndef atomic_sub_return_acquire
117 #define  atomic_sub_return_acquire(...)                                 \
118         __atomic_op_acquire(atomic_sub_return, __VA_ARGS__)
119 #endif
120
121 #ifndef atomic_sub_return_release
122 #define  atomic_sub_return_release(...)                                 \
123         __atomic_op_release(atomic_sub_return, __VA_ARGS__)
124 #endif
125
126 #ifndef atomic_sub_return
127 #define  atomic_sub_return(...)                                         \
128         __atomic_op_fence(atomic_sub_return, __VA_ARGS__)
129 #endif
130 #endif /* atomic_sub_return_relaxed */
131
132 /* atomic_dec_return_relaxed */
133 #ifndef atomic_dec_return_relaxed
134 #define  atomic_dec_return_relaxed      atomic_dec_return
135 #define  atomic_dec_return_acquire      atomic_dec_return
136 #define  atomic_dec_return_release      atomic_dec_return
137
138 #else /* atomic_dec_return_relaxed */
139
140 #ifndef atomic_dec_return_acquire
141 #define  atomic_dec_return_acquire(...)                                 \
142         __atomic_op_acquire(atomic_dec_return, __VA_ARGS__)
143 #endif
144
145 #ifndef atomic_dec_return_release
146 #define  atomic_dec_return_release(...)                                 \
147         __atomic_op_release(atomic_dec_return, __VA_ARGS__)
148 #endif
149
150 #ifndef atomic_dec_return
151 #define  atomic_dec_return(...)                                         \
152         __atomic_op_fence(atomic_dec_return, __VA_ARGS__)
153 #endif
154 #endif /* atomic_dec_return_relaxed */
155
156 /* atomic_xchg_relaxed */
157 #ifndef atomic_xchg_relaxed
158 #define  atomic_xchg_relaxed            atomic_xchg
159 #define  atomic_xchg_acquire            atomic_xchg
160 #define  atomic_xchg_release            atomic_xchg
161
162 #else /* atomic_xchg_relaxed */
163
164 #ifndef atomic_xchg_acquire
165 #define  atomic_xchg_acquire(...)                                       \
166         __atomic_op_acquire(atomic_xchg, __VA_ARGS__)
167 #endif
168
169 #ifndef atomic_xchg_release
170 #define  atomic_xchg_release(...)                                       \
171         __atomic_op_release(atomic_xchg, __VA_ARGS__)
172 #endif
173
174 #ifndef atomic_xchg
175 #define  atomic_xchg(...)                                               \
176         __atomic_op_fence(atomic_xchg, __VA_ARGS__)
177 #endif
178 #endif /* atomic_xchg_relaxed */
179
180 /* atomic_cmpxchg_relaxed */
181 #ifndef atomic_cmpxchg_relaxed
182 #define  atomic_cmpxchg_relaxed         atomic_cmpxchg
183 #define  atomic_cmpxchg_acquire         atomic_cmpxchg
184 #define  atomic_cmpxchg_release         atomic_cmpxchg
185
186 #else /* atomic_cmpxchg_relaxed */
187
188 #ifndef atomic_cmpxchg_acquire
189 #define  atomic_cmpxchg_acquire(...)                                    \
190         __atomic_op_acquire(atomic_cmpxchg, __VA_ARGS__)
191 #endif
192
193 #ifndef atomic_cmpxchg_release
194 #define  atomic_cmpxchg_release(...)                                    \
195         __atomic_op_release(atomic_cmpxchg, __VA_ARGS__)
196 #endif
197
198 #ifndef atomic_cmpxchg
199 #define  atomic_cmpxchg(...)                                            \
200         __atomic_op_fence(atomic_cmpxchg, __VA_ARGS__)
201 #endif
202 #endif /* atomic_cmpxchg_relaxed */
203
204 #ifndef atomic64_read_acquire
205 #define  atomic64_read_acquire(v)       smp_load_acquire(&(v)->counter)
206 #endif
207
208 #ifndef atomic64_set_release
209 #define  atomic64_set_release(v, i)     smp_store_release(&(v)->counter, (i))
210 #endif
211
212 /* atomic64_add_return_relaxed */
213 #ifndef atomic64_add_return_relaxed
214 #define  atomic64_add_return_relaxed    atomic64_add_return
215 #define  atomic64_add_return_acquire    atomic64_add_return
216 #define  atomic64_add_return_release    atomic64_add_return
217
218 #else /* atomic64_add_return_relaxed */
219
220 #ifndef atomic64_add_return_acquire
221 #define  atomic64_add_return_acquire(...)                               \
222         __atomic_op_acquire(atomic64_add_return, __VA_ARGS__)
223 #endif
224
225 #ifndef atomic64_add_return_release
226 #define  atomic64_add_return_release(...)                               \
227         __atomic_op_release(atomic64_add_return, __VA_ARGS__)
228 #endif
229
230 #ifndef atomic64_add_return
231 #define  atomic64_add_return(...)                                       \
232         __atomic_op_fence(atomic64_add_return, __VA_ARGS__)
233 #endif
234 #endif /* atomic64_add_return_relaxed */
235
236 /* atomic64_inc_return_relaxed */
237 #ifndef atomic64_inc_return_relaxed
238 #define  atomic64_inc_return_relaxed    atomic64_inc_return
239 #define  atomic64_inc_return_acquire    atomic64_inc_return
240 #define  atomic64_inc_return_release    atomic64_inc_return
241
242 #else /* atomic64_inc_return_relaxed */
243
244 #ifndef atomic64_inc_return_acquire
245 #define  atomic64_inc_return_acquire(...)                               \
246         __atomic_op_acquire(atomic64_inc_return, __VA_ARGS__)
247 #endif
248
249 #ifndef atomic64_inc_return_release
250 #define  atomic64_inc_return_release(...)                               \
251         __atomic_op_release(atomic64_inc_return, __VA_ARGS__)
252 #endif
253
254 #ifndef atomic64_inc_return
255 #define  atomic64_inc_return(...)                                       \
256         __atomic_op_fence(atomic64_inc_return, __VA_ARGS__)
257 #endif
258 #endif /* atomic64_inc_return_relaxed */
259
260
261 /* atomic64_sub_return_relaxed */
262 #ifndef atomic64_sub_return_relaxed
263 #define  atomic64_sub_return_relaxed    atomic64_sub_return
264 #define  atomic64_sub_return_acquire    atomic64_sub_return
265 #define  atomic64_sub_return_release    atomic64_sub_return
266
267 #else /* atomic64_sub_return_relaxed */
268
269 #ifndef atomic64_sub_return_acquire
270 #define  atomic64_sub_return_acquire(...)                               \
271         __atomic_op_acquire(atomic64_sub_return, __VA_ARGS__)
272 #endif
273
274 #ifndef atomic64_sub_return_release
275 #define  atomic64_sub_return_release(...)                               \
276         __atomic_op_release(atomic64_sub_return, __VA_ARGS__)
277 #endif
278
279 #ifndef atomic64_sub_return
280 #define  atomic64_sub_return(...)                                       \
281         __atomic_op_fence(atomic64_sub_return, __VA_ARGS__)
282 #endif
283 #endif /* atomic64_sub_return_relaxed */
284
285 /* atomic64_dec_return_relaxed */
286 #ifndef atomic64_dec_return_relaxed
287 #define  atomic64_dec_return_relaxed    atomic64_dec_return
288 #define  atomic64_dec_return_acquire    atomic64_dec_return
289 #define  atomic64_dec_return_release    atomic64_dec_return
290
291 #else /* atomic64_dec_return_relaxed */
292
293 #ifndef atomic64_dec_return_acquire
294 #define  atomic64_dec_return_acquire(...)                               \
295         __atomic_op_acquire(atomic64_dec_return, __VA_ARGS__)
296 #endif
297
298 #ifndef atomic64_dec_return_release
299 #define  atomic64_dec_return_release(...)                               \
300         __atomic_op_release(atomic64_dec_return, __VA_ARGS__)
301 #endif
302
303 #ifndef atomic64_dec_return
304 #define  atomic64_dec_return(...)                                       \
305         __atomic_op_fence(atomic64_dec_return, __VA_ARGS__)
306 #endif
307 #endif /* atomic64_dec_return_relaxed */
308
309 /* atomic64_xchg_relaxed */
310 #ifndef atomic64_xchg_relaxed
311 #define  atomic64_xchg_relaxed          atomic64_xchg
312 #define  atomic64_xchg_acquire          atomic64_xchg
313 #define  atomic64_xchg_release          atomic64_xchg
314
315 #else /* atomic64_xchg_relaxed */
316
317 #ifndef atomic64_xchg_acquire
318 #define  atomic64_xchg_acquire(...)                                     \
319         __atomic_op_acquire(atomic64_xchg, __VA_ARGS__)
320 #endif
321
322 #ifndef atomic64_xchg_release
323 #define  atomic64_xchg_release(...)                                     \
324         __atomic_op_release(atomic64_xchg, __VA_ARGS__)
325 #endif
326
327 #ifndef atomic64_xchg
328 #define  atomic64_xchg(...)                                             \
329         __atomic_op_fence(atomic64_xchg, __VA_ARGS__)
330 #endif
331 #endif /* atomic64_xchg_relaxed */
332
333 /* atomic64_cmpxchg_relaxed */
334 #ifndef atomic64_cmpxchg_relaxed
335 #define  atomic64_cmpxchg_relaxed       atomic64_cmpxchg
336 #define  atomic64_cmpxchg_acquire       atomic64_cmpxchg
337 #define  atomic64_cmpxchg_release       atomic64_cmpxchg
338
339 #else /* atomic64_cmpxchg_relaxed */
340
341 #ifndef atomic64_cmpxchg_acquire
342 #define  atomic64_cmpxchg_acquire(...)                                  \
343         __atomic_op_acquire(atomic64_cmpxchg, __VA_ARGS__)
344 #endif
345
346 #ifndef atomic64_cmpxchg_release
347 #define  atomic64_cmpxchg_release(...)                                  \
348         __atomic_op_release(atomic64_cmpxchg, __VA_ARGS__)
349 #endif
350
351 #ifndef atomic64_cmpxchg
352 #define  atomic64_cmpxchg(...)                                          \
353         __atomic_op_fence(atomic64_cmpxchg, __VA_ARGS__)
354 #endif
355 #endif /* atomic64_cmpxchg_relaxed */
356
357 /* cmpxchg_relaxed */
358 #ifndef cmpxchg_relaxed
359 #define  cmpxchg_relaxed                cmpxchg
360 #define  cmpxchg_acquire                cmpxchg
361 #define  cmpxchg_release                cmpxchg
362
363 #else /* cmpxchg_relaxed */
364
365 #ifndef cmpxchg_acquire
366 #define  cmpxchg_acquire(...)                                           \
367         __atomic_op_acquire(cmpxchg, __VA_ARGS__)
368 #endif
369
370 #ifndef cmpxchg_release
371 #define  cmpxchg_release(...)                                           \
372         __atomic_op_release(cmpxchg, __VA_ARGS__)
373 #endif
374
375 #ifndef cmpxchg
376 #define  cmpxchg(...)                                                   \
377         __atomic_op_fence(cmpxchg, __VA_ARGS__)
378 #endif
379 #endif /* cmpxchg_relaxed */
380
381 /* cmpxchg64_relaxed */
382 #ifndef cmpxchg64_relaxed
383 #define  cmpxchg64_relaxed              cmpxchg64
384 #define  cmpxchg64_acquire              cmpxchg64
385 #define  cmpxchg64_release              cmpxchg64
386
387 #else /* cmpxchg64_relaxed */
388
389 #ifndef cmpxchg64_acquire
390 #define  cmpxchg64_acquire(...)                                         \
391         __atomic_op_acquire(cmpxchg64, __VA_ARGS__)
392 #endif
393
394 #ifndef cmpxchg64_release
395 #define  cmpxchg64_release(...)                                         \
396         __atomic_op_release(cmpxchg64, __VA_ARGS__)
397 #endif
398
399 #ifndef cmpxchg64
400 #define  cmpxchg64(...)                                                 \
401         __atomic_op_fence(cmpxchg64, __VA_ARGS__)
402 #endif
403 #endif /* cmpxchg64_relaxed */
404
405 /* xchg_relaxed */
406 #ifndef xchg_relaxed
407 #define  xchg_relaxed                   xchg
408 #define  xchg_acquire                   xchg
409 #define  xchg_release                   xchg
410
411 #else /* xchg_relaxed */
412
413 #ifndef xchg_acquire
414 #define  xchg_acquire(...)              __atomic_op_acquire(xchg, __VA_ARGS__)
415 #endif
416
417 #ifndef xchg_release
418 #define  xchg_release(...)              __atomic_op_release(xchg, __VA_ARGS__)
419 #endif
420
421 #ifndef xchg
422 #define  xchg(...)                      __atomic_op_fence(xchg, __VA_ARGS__)
423 #endif
424 #endif /* xchg_relaxed */
425
426 /**
427  * atomic_add_unless - add unless the number is already a given value
428  * @v: pointer of type atomic_t
429  * @a: the amount to add to v...
430  * @u: ...unless v is equal to u.
431  *
432  * Atomically adds @a to @v, so long as @v was not already @u.
433  * Returns non-zero if @v was not @u, and zero otherwise.
434  */
435 static inline int atomic_add_unless(atomic_t *v, int a, int u)
436 {
437         return __atomic_add_unless(v, a, u) != u;
438 }
439
440 /**
441  * atomic_inc_not_zero - increment unless the number is zero
442  * @v: pointer of type atomic_t
443  *
444  * Atomically increments @v by 1, so long as @v is non-zero.
445  * Returns non-zero if @v was non-zero, and zero otherwise.
446  */
447 #ifndef atomic_inc_not_zero
448 #define atomic_inc_not_zero(v)          atomic_add_unless((v), 1, 0)
449 #endif
450
451 #ifndef atomic_andnot
452 static inline void atomic_andnot(int i, atomic_t *v)
453 {
454         atomic_and(~i, v);
455 }
456 #endif
457
458 static inline __deprecated void atomic_clear_mask(unsigned int mask, atomic_t *v)
459 {
460         atomic_andnot(mask, v);
461 }
462
463 static inline __deprecated void atomic_set_mask(unsigned int mask, atomic_t *v)
464 {
465         atomic_or(mask, v);
466 }
467
468 /**
469  * atomic_inc_not_zero_hint - increment if not null
470  * @v: pointer of type atomic_t
471  * @hint: probable value of the atomic before the increment
472  *
473  * This version of atomic_inc_not_zero() gives a hint of probable
474  * value of the atomic. This helps processor to not read the memory
475  * before doing the atomic read/modify/write cycle, lowering
476  * number of bus transactions on some arches.
477  *
478  * Returns: 0 if increment was not done, 1 otherwise.
479  */
480 #ifndef atomic_inc_not_zero_hint
481 static inline int atomic_inc_not_zero_hint(atomic_t *v, int hint)
482 {
483         int val, c = hint;
484
485         /* sanity test, should be removed by compiler if hint is a constant */
486         if (!hint)
487                 return atomic_inc_not_zero(v);
488
489         do {
490                 val = atomic_cmpxchg(v, c, c + 1);
491                 if (val == c)
492                         return 1;
493                 c = val;
494         } while (c);
495
496         return 0;
497 }
498 #endif
499
500 #ifndef atomic_inc_unless_negative
501 static inline int atomic_inc_unless_negative(atomic_t *p)
502 {
503         int v, v1;
504         for (v = 0; v >= 0; v = v1) {
505                 v1 = atomic_cmpxchg(p, v, v + 1);
506                 if (likely(v1 == v))
507                         return 1;
508         }
509         return 0;
510 }
511 #endif
512
513 #ifndef atomic_dec_unless_positive
514 static inline int atomic_dec_unless_positive(atomic_t *p)
515 {
516         int v, v1;
517         for (v = 0; v <= 0; v = v1) {
518                 v1 = atomic_cmpxchg(p, v, v - 1);
519                 if (likely(v1 == v))
520                         return 1;
521         }
522         return 0;
523 }
524 #endif
525
526 /*
527  * atomic_dec_if_positive - decrement by 1 if old value positive
528  * @v: pointer of type atomic_t
529  *
530  * The function returns the old value of *v minus 1, even if
531  * the atomic variable, v, was not decremented.
532  */
533 #ifndef atomic_dec_if_positive
534 static inline int atomic_dec_if_positive(atomic_t *v)
535 {
536         int c, old, dec;
537         c = atomic_read(v);
538         for (;;) {
539                 dec = c - 1;
540                 if (unlikely(dec < 0))
541                         break;
542                 old = atomic_cmpxchg((v), c, dec);
543                 if (likely(old == c))
544                         break;
545                 c = old;
546         }
547         return dec;
548 }
549 #endif
550
551 #ifdef CONFIG_GENERIC_ATOMIC64
552 #include <asm-generic/atomic64.h>
553 #endif
554
555 #ifndef atomic64_andnot
556 static inline void atomic64_andnot(long long i, atomic64_t *v)
557 {
558         atomic64_and(~i, v);
559 }
560 #endif
561
562 #include <asm-generic/atomic-long.h>
563
564 #endif /* _LINUX_ATOMIC_H */