2 This file is a part of libcds - Concurrent Data Structures library
4 (C) Copyright Maxim Khizhinsky (libcds.dev@gmail.com) 2006-2017
6 Source code repo: http://github.com/khizmax/libcds/
7 Download: http://sourceforge.net/projects/libcds/files/
9 Redistribution and use in source and binary forms, with or without
10 modification, are permitted provided that the following conditions are met:
12 * Redistributions of source code must retain the above copyright notice, this
13 list of conditions and the following disclaimer.
15 * Redistributions in binary form must reproduce the above copyright notice,
16 this list of conditions and the following disclaimer in the documentation
17 and/or other materials provided with the distribution.
19 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
22 DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
23 FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
25 SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
26 CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
27 OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31 #ifndef CDSLIB_COMPILER_CXX11_ATOMIC_H
32 #define CDSLIB_COMPILER_CXX11_ATOMIC_H
35 #include <type_traits> // make_unsigned
36 #include <cds/details/defs.h>
37 #include <cds/details/aligned_type.h>
39 namespace cds { namespace cxx11_atomic {
40 typedef enum memory_order {
49 }} // namespace cds::cxx11_atomic
52 #if CDS_COMPILER == CDS_COMPILER_MSVC || (CDS_COMPILER == CDS_COMPILER_INTEL && CDS_OS_INTERFACE == CDS_OSI_WINDOWS)
53 # if CDS_PROCESSOR_ARCH == CDS_PROCESSOR_X86
54 # include <cds/compiler/vc/x86/cxx11_atomic.h>
55 # elif CDS_PROCESSOR_ARCH == CDS_PROCESSOR_AMD64
56 # include <cds/compiler/vc/amd64/cxx11_atomic.h>
58 # error "MS VC++ compiler: unsupported processor architecture"
60 #elif CDS_COMPILER == CDS_COMPILER_GCC || CDS_COMPILER == CDS_COMPILER_CLANG || CDS_COMPILER == CDS_COMPILER_INTEL
61 # if CDS_PROCESSOR_ARCH == CDS_PROCESSOR_X86
62 # include <cds/compiler/gcc/x86/cxx11_atomic.h>
63 # elif CDS_PROCESSOR_ARCH == CDS_PROCESSOR_AMD64
64 # include <cds/compiler/gcc/amd64/cxx11_atomic.h>
65 # elif CDS_PROCESSOR_ARCH == CDS_PROCESSOR_IA64
66 # include <cds/compiler/gcc/ia64/cxx11_atomic.h>
67 # elif CDS_PROCESSOR_ARCH == CDS_PROCESSOR_SPARC
68 # include <cds/compiler/gcc/sparc/cxx11_atomic.h>
69 # elif CDS_PROCESSOR_ARCH == CDS_PROCESSOR_PPC64
70 # include <cds/compiler/gcc/ppc64/cxx11_atomic.h>
71 //# elif CDS_PROCESSOR_ARCH == CDS_PROCESSOR_ARM7
72 //# include <cds/compiler/gcc/arm7/cxx11_atomic.h>
74 # error "GCC compiler: unsupported processor architecture. Try to use native C++11 atomic or boost.atomic"
77 # error "Undefined compiler"
80 namespace cds { namespace cxx11_atomic {
82 // forward declarations
88 template <typename T, size_t Size, typename Primary = T >
89 struct atomic_generic_ops;
91 template <typename T, size_t Size>
92 struct atomic_integral_ops;
94 template <size_t TypeSize>
98 struct primary_type<1>
100 typedef std::uint8_t type;
103 struct primary_type<2>
105 typedef std::uint16_t type;
108 struct primary_type<4>
110 typedef std::uint32_t type;
113 struct primary_type<8>
115 typedef std::uint64_t type;
117 #if CDS_BUILD_BITS == 64 && CDS_DCAS_SUPPORT
119 struct primary_type<16>
121 typedef unsigned __int128_t type;
125 template <typename T, typename Primary>
126 struct make_atomic_primary
128 typedef T source_type;
129 typedef Primary primary_type;
131 static primary_type volatile * ptr( source_type volatile * p ) noexcept
133 return reinterpret_cast<primary_type volatile *>(p);
135 static primary_type const volatile * ptr( source_type const volatile * p ) noexcept
137 return reinterpret_cast<primary_type const volatile *>(p);
140 static primary_type val( source_type v ) noexcept
142 return *reinterpret_cast<primary_type*>(&v);
145 static primary_type& ref( source_type& v ) noexcept
147 return reinterpret_cast<primary_type&>(v);
150 static primary_type const& ref( source_type const& v ) noexcept
152 return reinterpret_cast<primary_type const&>(v);
155 static source_type ret( primary_type r ) noexcept
157 return *reinterpret_cast<source_type *>(&r);
161 template <typename T>
162 struct make_atomic_primary<T, T>
164 typedef T source_type;
165 typedef T primary_type;
167 static primary_type volatile * ptr( source_type volatile * p ) noexcept
171 static primary_type const volatile * ptr( source_type const volatile * p ) noexcept
176 static primary_type val( source_type v ) noexcept
181 static primary_type& ref( source_type& v ) noexcept
186 static source_type ret( primary_type r ) noexcept
192 template <typename T>
193 struct atomic_integral_bitwise_ops
196 typedef typename std::make_unsigned<T>::type unsigned_type;
197 typedef atomic_generic_ops<unsigned_type, sizeof(unsigned_type)> atomic_ops;
199 static T fetch_and(T volatile * pDest, T val, memory_order order) noexcept
201 unsigned_type cur = atomic_ops::atomic_load_explicit( reinterpret_cast<unsigned_type volatile *>(pDest), memory_order_relaxed );
202 do {} while ( !atomic_ops::atomic_compare_exchange_weak_explicit(
203 reinterpret_cast<unsigned_type volatile *>(pDest), &cur, cur & unsigned_type(val), order, memory_order_relaxed ));
207 static T fetch_or(T volatile * pDest, T val, memory_order order) noexcept
209 unsigned_type cur = atomic_ops::atomic_load_explicit( reinterpret_cast<unsigned_type volatile *>(pDest), memory_order_relaxed );
210 do {} while ( !atomic_ops::atomic_compare_exchange_weak_explicit(
211 reinterpret_cast<unsigned_type volatile *>(pDest), &cur, cur | unsigned_type(val), order, memory_order_relaxed ));
215 static T fetch_xor(T volatile * pDest, T val, memory_order order) noexcept
217 unsigned_type cur = atomic_ops::atomic_load_explicit( reinterpret_cast<unsigned_type volatile *>(pDest), memory_order_relaxed );
218 do {} while ( !atomic_ops::atomic_compare_exchange_weak_explicit(
219 reinterpret_cast<unsigned_type volatile *>(pDest), &cur, cur ^ unsigned_type(val), order, memory_order_relaxed ));
225 // 8-bit atomic operations
227 template <typename T, typename Primary>
228 struct atomic_generic_ops< T, 1, Primary >
230 typedef make_atomic_primary<T, Primary> primary;
233 static void atomic_store_explicit( T volatile * pDest, T v, memory_order order ) noexcept
235 platform::store8( primary::ptr(pDest), primary::val(v), order );
237 static void atomic_store_explicit( T * pDest, T v, memory_order order ) noexcept
239 platform::store8( primary::ptr(pDest), primary::val(v), order );
241 static void atomic_store( T volatile * pDest, T v ) noexcept
243 atomic_store_explicit( pDest, v, memory_order_seq_cst );
245 static void atomic_store( T * pDest, T v ) noexcept
247 atomic_store_explicit( pDest, v, memory_order_seq_cst );
251 static T atomic_load_explicit( T volatile const * pSrc, memory_order order ) noexcept
253 return primary::ret( platform::load8( primary::ptr(pSrc), order ));
255 static T atomic_load_explicit( T const * pSrc, memory_order order ) noexcept
257 return primary::ret( platform::load8( primary::ptr(pSrc), order ));
259 static T atomic_load( T volatile const * pSrc ) noexcept
261 return atomic_load_explicit( pSrc, memory_order_seq_cst );
263 static T atomic_load( T const * pSrc ) noexcept
265 return atomic_load_explicit( pSrc, memory_order_seq_cst );
269 static T atomic_exchange_explicit( T volatile * pDest, T val, memory_order order ) noexcept
271 return primary::ret( platform::exchange8( primary::ptr(pDest), primary::val(val), order ));
273 static T atomic_exchange_explicit( T * pDest, T val, memory_order order ) noexcept
275 return primary::ret( platform::exchange8( primary::ptr(pDest), primary::val(val), order ));
277 static T atomic_exchange( T volatile * pDest, T val ) noexcept
279 return atomic_exchange_explicit( pDest, val, memory_order_seq_cst );
281 static T atomic_exchange( T * pDest, T val ) noexcept
283 return atomic_exchange_explicit( pDest, val, memory_order_seq_cst );
287 static bool atomic_compare_exchange_weak_explicit( T volatile * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) noexcept
290 return platform::cas8_weak( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
292 static bool atomic_compare_exchange_weak_explicit( T * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) noexcept
295 return platform::cas8_weak( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
297 static bool atomic_compare_exchange_weak( T volatile * pDest, T * expected, T desired ) noexcept
299 return atomic_compare_exchange_weak_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
301 static bool atomic_compare_exchange_weak( T * pDest, T * expected, T desired ) noexcept
303 return atomic_compare_exchange_weak_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
305 static bool atomic_compare_exchange_strong_explicit( T volatile * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) noexcept
308 return platform::cas8_strong( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
310 static bool atomic_compare_exchange_strong_explicit( T * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) noexcept
313 return platform::cas8_strong( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
315 static bool atomic_compare_exchange_strong( T volatile * pDest, T * expected, T desired ) noexcept
317 return atomic_compare_exchange_strong_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
319 static bool atomic_compare_exchange_strong( T * pDest, T * expected, T desired ) noexcept
321 return atomic_compare_exchange_strong_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
325 template <typename T>
326 struct atomic_integral_ops< T, 1 >
327 : atomic_generic_ops<T, 1, T >
328 , atomic_integral_bitwise_ops<T>
330 typedef atomic_integral_bitwise_ops<T> bitwise_ops;
333 static T atomic_fetch_add_explicit(T volatile * pDest, T val, memory_order order) noexcept
335 # ifdef CDS_ATOMIC_fetch8_add_defined
336 return platform::fetch8_add( pDest, val, order );
338 T cur = atomic_load_explicit( pDest, memory_order_relaxed );
339 do {} while ( !atomic_compare_exchange_weak_explicit( pDest, &cur, cur + val, order, memory_order_relaxed ));
343 static T atomic_fetch_add_explicit(T * pDest, T val , memory_order order) noexcept
345 return atomic_fetch_add_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
347 static T atomic_fetch_add( T volatile * pDest, T val ) noexcept
349 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
351 static T atomic_fetch_add( T * pDest, T val ) noexcept
353 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
357 static T atomic_fetch_sub_explicit(T volatile * pDest, T val, memory_order order) noexcept
359 # ifdef CDS_ATOMIC_fetch8_sub_defined
360 return platform::fetch8_sub( pDest, val, order );
362 T cur = atomic_load_explicit( pDest, memory_order_relaxed );
363 do {} while ( !atomic_compare_exchange_weak_explicit( pDest, &cur, cur - val, order, memory_order_relaxed ));
367 static T atomic_fetch_sub_explicit(T * pDest, T val , memory_order order) noexcept
369 return atomic_fetch_sub_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
371 static T atomic_fetch_sub( T volatile * pDest, T val ) noexcept
373 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
375 static T atomic_fetch_sub( T * pDest, T val ) noexcept
377 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
381 static T atomic_fetch_and_explicit(T volatile * pDest, T val, memory_order order) noexcept
383 # ifdef CDS_ATOMIC_fetch8_and_defined
384 return platform::fetch8_and( pDest, val, order );
386 return bitwise_ops::fetch_and( pDest, val, order );
389 static T atomic_fetch_and_explicit(T * pDest, T val , memory_order order) noexcept
391 return atomic_fetch_and_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
393 static T atomic_fetch_and( T volatile * pDest, T val ) noexcept
395 return atomic_fetch_and_explicit( pDest, val, memory_order_seq_cst );
397 static T atomic_fetch_and( T * pDest, T val ) noexcept
399 return atomic_fetch_and_explicit( pDest, val, memory_order_seq_cst );
403 static T atomic_fetch_or_explicit(T volatile * pDest, T val, memory_order order) noexcept
405 # ifdef CDS_ATOMIC_fetch8_or_defined
406 return platform::fetch8_or( pDest, val, order );
408 return bitwise_ops::fetch_or( pDest, val, order );
411 static T atomic_fetch_or_explicit(T * pDest, T val , memory_order order) noexcept
413 return atomic_fetch_or_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
415 static T atomic_fetch_or( T volatile * pDest, T val ) noexcept
417 return atomic_fetch_or_explicit( pDest, val, memory_order_seq_cst );
419 static T atomic_fetch_or( T * pDest, T val ) noexcept
421 return atomic_fetch_or_explicit( pDest, val, memory_order_seq_cst );
425 static T atomic_fetch_xor_explicit(T volatile * pDest, T val, memory_order order) noexcept
427 # ifdef CDS_ATOMIC_fetch8_xor_defined
428 return platform::fetch8_xor( pDest, val, order );
430 return bitwise_ops::fetch_xor( pDest, val, order );
433 static T atomic_fetch_xor_explicit(T * pDest, T val , memory_order order) noexcept
435 return atomic_fetch_xor_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
437 static T atomic_fetch_xor( T volatile * pDest, T val ) noexcept
439 return atomic_fetch_xor_explicit( pDest, val, memory_order_seq_cst );
441 static T atomic_fetch_xor( T * pDest, T val ) noexcept
443 return atomic_fetch_xor_explicit( pDest, val, memory_order_seq_cst );
447 // 16-bit atomic operations
449 template <typename T, typename Primary>
450 struct atomic_generic_ops< T, 2, Primary >
452 typedef make_atomic_primary<T, Primary> primary;
455 static void atomic_store_explicit( T volatile * pDest, T v, memory_order order ) noexcept
457 platform::store16( primary::ptr(pDest), primary::val(v), order );
459 static void atomic_store_explicit( T * pDest, T v, memory_order order ) noexcept
461 platform::store16( primary::ptr(pDest), primary::val(v), order );
463 static void atomic_store( T volatile * pDest, T v ) noexcept
465 atomic_store_explicit( pDest, v, memory_order_seq_cst );
467 static void atomic_store( T * pDest, T v ) noexcept
469 atomic_store_explicit( pDest, v, memory_order_seq_cst );
473 static T atomic_load_explicit( T volatile const * pSrc, memory_order order ) noexcept
475 return primary::ret( platform::load16( primary::ptr(pSrc), order ));
477 static T atomic_load_explicit( T const * pSrc, memory_order order ) noexcept
479 return primary::ret( platform::load16( primary::ptr(pSrc), order ));
481 static T atomic_load( T volatile const * pSrc ) noexcept
483 return atomic_load_explicit( pSrc, memory_order_seq_cst );
485 static T atomic_load( T const * pSrc ) noexcept
487 return atomic_load_explicit( pSrc, memory_order_seq_cst );
491 static T atomic_exchange_explicit( T volatile * pDest, T val, memory_order order ) noexcept
493 return primary::ret( platform::exchange16( primary::ptr(pDest), primary::val(val), order ));
495 static T atomic_exchange_explicit( T * pDest, T val, memory_order order ) noexcept
497 return primary::ret( platform::exchange16( primary::ptr(pDest), primary::val(val), order ));
499 static T atomic_exchange( T volatile * pDest, T val ) noexcept
501 return atomic_exchange_explicit( pDest, val, memory_order_seq_cst );
503 static T atomic_exchange( T * pDest, T val ) noexcept
505 return atomic_exchange_explicit( pDest, val, memory_order_seq_cst );
509 static bool atomic_compare_exchange_weak_explicit( T volatile * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) noexcept
512 return platform::cas16_weak( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
514 static bool atomic_compare_exchange_weak_explicit( T * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) noexcept
517 return platform::cas16_weak( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
519 static bool atomic_compare_exchange_weak( T volatile * pDest, T * expected, T desired ) noexcept
521 return atomic_compare_exchange_weak_explicit( pDest, expected, primary::val(desired), memory_order_seq_cst, memory_order_relaxed );
523 static bool atomic_compare_exchange_weak( T * pDest, T * expected, T desired ) noexcept
525 return atomic_compare_exchange_weak_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
527 static bool atomic_compare_exchange_strong_explicit( T volatile * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) noexcept
530 return platform::cas16_strong( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
532 static bool atomic_compare_exchange_strong_explicit( T * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) noexcept
535 return platform::cas16_strong( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
537 static bool atomic_compare_exchange_strong( T volatile * pDest, T * expected, T desired ) noexcept
539 return atomic_compare_exchange_strong_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
541 static bool atomic_compare_exchange_strong( T * pDest, T * expected, T desired ) noexcept
543 return atomic_compare_exchange_strong_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
547 template <typename T>
548 struct atomic_integral_ops< T, 2 >
549 : atomic_generic_ops< T, 2, T >
550 , atomic_integral_bitwise_ops<T>
552 typedef atomic_integral_bitwise_ops<T> bitwise_ops;
555 static T atomic_fetch_add_explicit(T volatile * pDest, T val, memory_order order) noexcept
557 # ifdef CDS_ATOMIC_fetch16_add_defined
558 return platform::fetch16_add( pDest, val, order );
560 T cur = atomic_load_explicit( pDest, memory_order_relaxed );
561 do {} while ( !atomic_compare_exchange_weak_explicit( pDest, &cur, cur + val, order, memory_order_relaxed ));
565 static T atomic_fetch_add_explicit(T * pDest, T val , memory_order order) noexcept
567 return atomic_fetch_add_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
569 static T atomic_fetch_add( T volatile * pDest, T val ) noexcept
571 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
573 static T atomic_fetch_add( T * pDest, T val ) noexcept
575 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
579 static T atomic_fetch_sub_explicit(T volatile * pDest, T val, memory_order order) noexcept
581 # ifdef CDS_ATOMIC_fetch16_sub_defined
582 return platform::fetch16_sub( pDest, val, order );
584 T cur = atomic_load_explicit( pDest, memory_order_relaxed );
585 do {} while ( !atomic_compare_exchange_weak_explicit( pDest, &cur, cur - val, order, memory_order_relaxed ));
589 static T atomic_fetch_sub_explicit(T * pDest, T val , memory_order order) noexcept
591 return atomic_fetch_sub_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
593 static T atomic_fetch_sub( T volatile * pDest, T val ) noexcept
595 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
597 static T atomic_fetch_sub( T * pDest, T val ) noexcept
599 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
603 static T atomic_fetch_and_explicit(T volatile * pDest, T val, memory_order order) noexcept
605 # ifdef CDS_ATOMIC_fetch16_and_defined
606 return platform::fetch16_and( pDest, val, order );
608 return bitwise_ops::fetch_and( pDest, val, order );
611 static T atomic_fetch_and_explicit(T * pDest, T val , memory_order order) noexcept
613 return atomic_fetch_and_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
615 static T atomic_fetch_and( T volatile * pDest, T val ) noexcept
617 return atomic_fetch_and_explicit( pDest, val, memory_order_seq_cst );
619 static T atomic_fetch_and( T * pDest, T val ) noexcept
621 return atomic_fetch_and_explicit( pDest, val, memory_order_seq_cst );
625 static T atomic_fetch_or_explicit(T volatile * pDest, T val, memory_order order) noexcept
627 # ifdef CDS_ATOMIC_fetch16_or_defined
628 return platform::fetch16_or( pDest, val, order );
630 return bitwise_ops::fetch_or( pDest, val, order );
633 static T atomic_fetch_or_explicit(T * pDest, T val , memory_order order) noexcept
635 return atomic_fetch_or_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
637 static T atomic_fetch_or( T volatile * pDest, T val ) noexcept
639 return atomic_fetch_or_explicit( pDest, val, memory_order_seq_cst );
641 static T atomic_fetch_or( T * pDest, T val ) noexcept
643 return atomic_fetch_or_explicit( pDest, val, memory_order_seq_cst );
647 static T atomic_fetch_xor_explicit(T volatile * pDest, T val, memory_order order) noexcept
649 # ifdef CDS_ATOMIC_fetch16_xor_defined
650 return platform::fetch16_xor( pDest, val, order );
652 return bitwise_ops::fetch_xor( pDest, val, order );
655 static T atomic_fetch_xor_explicit(T * pDest, T val , memory_order order) noexcept
657 return atomic_fetch_xor_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
659 static T atomic_fetch_xor( T volatile * pDest, T val ) noexcept
661 return atomic_fetch_xor_explicit( pDest, val, memory_order_seq_cst );
663 static T atomic_fetch_xor( T * pDest, T val ) noexcept
665 return atomic_fetch_xor_explicit( pDest, val, memory_order_seq_cst );
669 // 32-bit atomic operations
671 template <typename T, typename Primary>
672 struct atomic_generic_ops< T, 4, Primary >
674 typedef make_atomic_primary<T, Primary> primary;
677 static void atomic_store_explicit( T volatile * pDest, T v, memory_order order ) noexcept
679 platform::store32( primary::ptr(pDest), primary::val(v), order );
681 static void atomic_store_explicit( T * pDest, T v, memory_order order ) noexcept
683 platform::store32( primary::ptr(pDest), primary::val(v), order );
685 static void atomic_store( T volatile * pDest, T v ) noexcept
687 atomic_store_explicit( pDest, v, memory_order_seq_cst );
689 static void atomic_store( T * pDest, T v ) noexcept
691 atomic_store_explicit( pDest, v, memory_order_seq_cst );
695 static T atomic_load_explicit( T volatile const * pSrc, memory_order order ) noexcept
697 return primary::ret( platform::load32( primary::ptr(pSrc), order ));
699 static T atomic_load_explicit( T const * pSrc, memory_order order ) noexcept
701 return primary::ret( platform::load32( primary::ptr(pSrc), order ));
703 static T atomic_load( T volatile const * pSrc ) noexcept
705 return atomic_load_explicit( pSrc, memory_order_seq_cst );
707 static T atomic_load( T const * pSrc ) noexcept
709 return atomic_load_explicit( pSrc, memory_order_seq_cst );
713 static T atomic_exchange_explicit( T volatile * pDest, T val, memory_order order ) noexcept
715 return primary::ret( platform::exchange32( primary::ptr(pDest), primary::val(val), order ));
717 static T atomic_exchange_explicit( T * pDest, T val, memory_order order ) noexcept
719 return primary::ret( platform::exchange32( primary::ptr(pDest), primary::val(val), order ));
721 static T atomic_exchange( T volatile * pDest, T val ) noexcept
723 return atomic_exchange_explicit( pDest, val, memory_order_seq_cst );
725 static T atomic_exchange( T * pDest, T val ) noexcept
727 return atomic_exchange_explicit( pDest, val, memory_order_seq_cst );
731 static bool atomic_compare_exchange_weak_explicit( T volatile * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) noexcept
734 return platform::cas32_weak( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
736 static bool atomic_compare_exchange_weak_explicit( T * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) noexcept
739 return platform::cas32_weak( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
741 static bool atomic_compare_exchange_weak( T volatile * pDest, T * expected, T desired ) noexcept
743 return atomic_compare_exchange_weak_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
745 static bool atomic_compare_exchange_weak( T * pDest, T * expected, T desired ) noexcept
747 return atomic_compare_exchange_weak_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
749 static bool atomic_compare_exchange_strong_explicit( T volatile * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) noexcept
752 return platform::cas32_strong( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
754 static bool atomic_compare_exchange_strong_explicit( T * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) noexcept
757 return platform::cas32_strong( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
759 static bool atomic_compare_exchange_strong( T volatile * pDest, T * expected, T desired ) noexcept
761 return atomic_compare_exchange_strong_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
763 static bool atomic_compare_exchange_strong( T * pDest, T * expected, T desired ) noexcept
765 return atomic_compare_exchange_strong_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
769 template <typename T>
770 struct atomic_integral_ops< T, 4 >
771 : atomic_generic_ops< T, 4, T >
772 , atomic_integral_bitwise_ops<T>
774 typedef atomic_integral_bitwise_ops<T> bitwise_ops;
776 static T atomic_fetch_add_explicit(T volatile * pDest, T val, memory_order order) noexcept
778 # ifdef CDS_ATOMIC_fetch32_add_defined
779 return platform::fetch32_add( pDest, val, order );
781 T cur = atomic_load_explicit( pDest, memory_order_relaxed );
782 do {} while ( !atomic_compare_exchange_weak_explicit( pDest, &cur, cur + val, order, memory_order_relaxed ));
786 static T atomic_fetch_add_explicit(T * pDest, T val , memory_order order) noexcept
788 return atomic_fetch_add_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
790 static T atomic_fetch_add( T volatile * pDest, T val ) noexcept
792 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
794 static T atomic_fetch_add( T * pDest, T val ) noexcept
796 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
800 static T atomic_fetch_sub_explicit(T volatile * pDest, T val, memory_order order) noexcept
802 # ifdef CDS_ATOMIC_fetch32_sub_defined
803 return platform::fetch32_sub( pDest, val, order );
805 T cur = atomic_load_explicit( pDest, memory_order_relaxed );
806 do {} while ( !atomic_compare_exchange_weak_explicit( pDest, &cur, cur - val, order, memory_order_relaxed ));
810 static T atomic_fetch_sub_explicit(T * pDest, T val , memory_order order) noexcept
812 return atomic_fetch_sub_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
814 static T atomic_fetch_sub( T volatile * pDest, T val ) noexcept
816 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
818 static T atomic_fetch_sub( T * pDest, T val ) noexcept
820 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
824 static T atomic_fetch_and_explicit(T volatile * pDest, T val, memory_order order) noexcept
826 # ifdef CDS_ATOMIC_fetch32_and_defined
827 return platform::fetch32_and( pDest, val, order );
829 return bitwise_ops::fetch_and( pDest, val, order );
832 static T atomic_fetch_and_explicit(T * pDest, T val , memory_order order) noexcept
834 return atomic_fetch_and_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
836 static T atomic_fetch_and( T volatile * pDest, T val ) noexcept
838 return atomic_fetch_and_explicit( pDest, val, memory_order_seq_cst );
840 static T atomic_fetch_and( T * pDest, T val ) noexcept
842 return atomic_fetch_and_explicit( pDest, val, memory_order_seq_cst );
846 static T atomic_fetch_or_explicit(T volatile * pDest, T val, memory_order order) noexcept
848 # ifdef CDS_ATOMIC_fetch32_or_defined
849 return platform::fetch32_or( pDest, val, order );
851 return bitwise_ops::fetch_or( pDest, val, order );
854 static T atomic_fetch_or_explicit(T * pDest, T val , memory_order order) noexcept
856 return atomic_fetch_or_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
858 static T atomic_fetch_or( T volatile * pDest, T val ) noexcept
860 return atomic_fetch_or_explicit( pDest, val, memory_order_seq_cst );
862 static T atomic_fetch_or( T * pDest, T val ) noexcept
864 return atomic_fetch_or_explicit( pDest, val, memory_order_seq_cst );
868 static T atomic_fetch_xor_explicit(T volatile * pDest, T val, memory_order order) noexcept
870 # ifdef CDS_ATOMIC_fetch32_xor_defined
871 return platform::fetch32_xor( pDest, val, order );
873 return bitwise_ops::fetch_xor( pDest, val, order );
876 static T atomic_fetch_xor_explicit(T * pDest, T val , memory_order order) noexcept
878 return atomic_fetch_xor_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
880 static T atomic_fetch_xor( T volatile * pDest, T val ) noexcept
882 return atomic_fetch_xor_explicit( pDest, val, memory_order_seq_cst );
884 static T atomic_fetch_xor( T * pDest, T val ) noexcept
886 return atomic_fetch_xor_explicit( pDest, val, memory_order_seq_cst );
891 // 64-bit atomic operations
893 template <typename T, typename Primary>
894 struct atomic_generic_ops< T, 8, Primary >
896 typedef make_atomic_primary<T, Primary> primary;
899 static void atomic_store_explicit( T volatile * pDest, T v, memory_order order ) noexcept
901 platform::store64( primary::ptr(pDest), primary::val(v), order );
903 static void atomic_store_explicit( T * pDest, T v, memory_order order ) noexcept
905 platform::store64( primary::ptr(pDest), primary::val(v), order );
907 static void atomic_store( T volatile * pDest, T v ) noexcept
909 atomic_store_explicit( pDest, v, memory_order_seq_cst );
911 static void atomic_store( T * pDest, T v ) noexcept
913 atomic_store_explicit( pDest, v, memory_order_seq_cst );
917 static T atomic_load_explicit( T volatile const * pSrc, memory_order order ) noexcept
919 return primary::ret( platform::load64( primary::ptr(pSrc), order ));
921 static T atomic_load_explicit( T const * pSrc, memory_order order ) noexcept
923 return primary::ret( platform::load64( primary::ptr(pSrc), order ));
925 static T atomic_load( T volatile const * pSrc ) noexcept
927 return atomic_load_explicit( pSrc, memory_order_seq_cst );
929 static T atomic_load( T const * pSrc ) noexcept
931 return atomic_load_explicit( pSrc, memory_order_seq_cst );
935 static T atomic_exchange_explicit( T volatile * pDest, T val, memory_order order ) noexcept
937 return primary::ret( platform::exchange64( primary::ptr(pDest), primary::val(val), order ));
939 static T atomic_exchange_explicit( T * pDest, T val, memory_order order ) noexcept
941 return primary::ret( platform::exchange64( primary::ptr(pDest), primary::val(val), order ));
943 static T atomic_exchange( T volatile * pDest, T val ) noexcept
945 return atomic_exchange_explicit( pDest, val, memory_order_seq_cst );
947 static T atomic_exchange( T * pDest, T val ) noexcept
949 return atomic_exchange_explicit( pDest, val, memory_order_seq_cst );
953 static bool atomic_compare_exchange_weak_explicit( T volatile * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) noexcept
956 return platform::cas64_weak( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
958 static bool atomic_compare_exchange_weak_explicit( T * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) noexcept
961 return platform::cas64_weak( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
963 static bool atomic_compare_exchange_weak( T volatile * pDest, T * expected, T desired ) noexcept
965 return atomic_compare_exchange_weak_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
967 static bool atomic_compare_exchange_weak( T * pDest, T * expected, T desired ) noexcept
969 return atomic_compare_exchange_weak_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
971 static bool atomic_compare_exchange_strong_explicit( T volatile * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) noexcept
974 return platform::cas64_strong( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
976 static bool atomic_compare_exchange_strong_explicit( T * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) noexcept
979 return platform::cas64_strong( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
981 static bool atomic_compare_exchange_strong( T volatile * pDest, T * expected, T desired ) noexcept
983 return atomic_compare_exchange_strong_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
985 static bool atomic_compare_exchange_strong( T * pDest, T * expected, T desired ) noexcept
987 return atomic_compare_exchange_strong_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
992 template <typename T>
993 struct atomic_integral_ops< T, 8 >
994 : atomic_generic_ops< T, 8, T >
995 , atomic_integral_bitwise_ops<T>
997 typedef atomic_integral_bitwise_ops<T> bitwise_ops;
998 typedef atomic_generic_ops<T, 8, T> general_ops;
1001 static T atomic_fetch_add_explicit(T volatile * pDest, T val, memory_order order) noexcept
1003 # ifdef CDS_ATOMIC_fetch64_add_defined
1004 return platform::fetch64_add( pDest, val, order );
1006 T cur = general_ops::atomic_load_explicit( pDest, memory_order_relaxed );
1007 do {} while ( !general_ops::atomic_compare_exchange_weak_explicit( pDest, &cur, cur + val, order, memory_order_relaxed ));
1011 static T atomic_fetch_add_explicit(T * pDest, T val , memory_order order) noexcept
1013 return atomic_fetch_add_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
1015 static T atomic_fetch_add( T volatile * pDest, T val ) noexcept
1017 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
1019 static T atomic_fetch_add( T * pDest, T val ) noexcept
1021 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
1025 static T atomic_fetch_sub_explicit(T volatile * pDest, T val, memory_order order) noexcept
1027 # ifdef CDS_ATOMIC_fetch64_sub_defined
1028 return platform::fetch64_sub( pDest, val, order );
1030 T cur = general_ops::atomic_load_explicit( pDest, memory_order_relaxed );
1031 do {} while ( !general_ops::atomic_compare_exchange_weak_explicit( pDest, &cur, cur - val, order, memory_order_relaxed ));
1035 static T atomic_fetch_sub_explicit(T * pDest, T val , memory_order order) noexcept
1037 return atomic_fetch_sub_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
1039 static T atomic_fetch_sub( T volatile * pDest, T val ) noexcept
1041 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
1043 static T atomic_fetch_sub( T * pDest, T val ) noexcept
1045 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
1049 static T atomic_fetch_and_explicit(T volatile * pDest, T val, memory_order order) noexcept
1051 # ifdef CDS_ATOMIC_fetch64_and_defined
1052 return platform::fetch64_and( pDest, val, order );
1054 return bitwise_ops::fetch_and( pDest, val, order );
1057 static T atomic_fetch_and_explicit(T * pDest, T val , memory_order order) noexcept
1059 return atomic_fetch_and_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
1061 static T atomic_fetch_and( T volatile * pDest, T val ) noexcept
1063 return atomic_fetch_and_explicit( pDest, val, memory_order_seq_cst );
1065 static T atomic_fetch_and( T * pDest, T val ) noexcept
1067 return atomic_fetch_and_explicit( pDest, val, memory_order_seq_cst );
1071 static T atomic_fetch_or_explicit(T volatile * pDest, T val, memory_order order) noexcept
1073 # ifdef CDS_ATOMIC_fetch64_or_defined
1074 return platform::fetch64_or( pDest, val, order );
1076 return bitwise_ops::fetch_or( pDest, val, order );
1079 static T atomic_fetch_or_explicit(T * pDest, T val , memory_order order) noexcept
1081 return atomic_fetch_or_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
1083 static T atomic_fetch_or( T volatile * pDest, T val ) noexcept
1085 return atomic_fetch_or_explicit( pDest, val, memory_order_seq_cst );
1087 static T atomic_fetch_or( T * pDest, T val ) noexcept
1089 return atomic_fetch_or_explicit( pDest, val, memory_order_seq_cst );
1093 static T atomic_fetch_xor_explicit(T volatile * pDest, T val, memory_order order) noexcept
1095 # ifdef CDS_ATOMIC_fetch64_xor_defined
1096 return platform::fetch64_xor( pDest, val, order );
1098 return bitwise_ops::fetch_xor( pDest, val, order );
1101 static T atomic_fetch_xor_explicit(T * pDest, T val , memory_order order) noexcept
1103 return atomic_fetch_xor_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
1105 static T atomic_fetch_xor( T volatile * pDest, T val ) noexcept
1107 return atomic_fetch_xor_explicit( pDest, val, memory_order_seq_cst );
1109 static T atomic_fetch_xor( T * pDest, T val ) noexcept
1111 return atomic_fetch_xor_explicit( pDest, val, memory_order_seq_cst );
1116 // atomic pointer operations
1117 template <typename T>
1118 struct atomic_pointer_base
1121 static void atomic_store_explicit( T * volatile * pDest, T * v, memory_order order ) noexcept
1123 platform::store_ptr( pDest, v, order );
1125 static void atomic_store_explicit( T * * pDest, T * v, memory_order order ) noexcept
1127 platform::store_ptr( pDest, v, order );
1129 static void atomic_store( T * volatile * pDest, T * v ) noexcept
1131 atomic_store_explicit( pDest, v, memory_order_seq_cst );
1133 static void atomic_store( T * * pDest, T * v ) noexcept
1135 atomic_store_explicit( pDest, v, memory_order_seq_cst );
1139 static T * atomic_load_explicit( T * volatile const * pSrc, memory_order order ) noexcept
1141 return platform::load_ptr( pSrc, order );
1143 static T * atomic_load_explicit( T * const * pSrc, memory_order order ) noexcept
1145 return platform::load_ptr( pSrc, order );
1147 static T * atomic_load( T * volatile const * pSrc ) noexcept
1149 return atomic_load_explicit( pSrc, memory_order_seq_cst );
1151 static T * atomic_load( T * const * pSrc ) noexcept
1153 return atomic_load_explicit( pSrc, memory_order_seq_cst );
1157 static T * atomic_exchange_explicit( T * volatile * pDest, T * val, memory_order order ) noexcept
1159 return platform::exchange_ptr( pDest, val, order );
1161 static T * atomic_exchange_explicit( T * * pDest, T * val, memory_order order ) noexcept
1163 return platform::exchange_ptr( pDest, val, order );
1165 static T * atomic_exchange( T * volatile * pDest, T * val ) noexcept
1167 return atomic_exchange_explicit( pDest, val, memory_order_seq_cst );
1169 static T * atomic_exchange( T * * pDest, T * val ) noexcept
1171 return atomic_exchange_explicit( pDest, val, memory_order_seq_cst );
1175 static bool atomic_compare_exchange_weak_explicit( T * volatile * pDest, T * * expected, T * desired, memory_order mo_success, memory_order mo_fail ) noexcept
1178 return platform::cas_ptr_weak( pDest, *expected, desired, mo_success, mo_fail );
1180 static bool atomic_compare_exchange_weak_explicit( T * * pDest, T * * expected, T * desired, memory_order mo_success, memory_order mo_fail ) noexcept
1183 return platform::cas_ptr_weak( pDest, *expected, desired, mo_success, mo_fail );
1185 static bool atomic_compare_exchange_weak( T * volatile * pDest, T ** expected, T * desired ) noexcept
1187 return atomic_compare_exchange_weak_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
1189 static bool atomic_compare_exchange_weak( T ** pDest, T ** expected, T * desired ) noexcept
1191 return atomic_compare_exchange_weak_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
1193 static bool atomic_compare_exchange_strong_explicit( T * volatile * pDest, T ** expected, T * desired, memory_order mo_success, memory_order mo_fail ) noexcept
1196 return platform::cas_ptr_strong( pDest, *expected, desired, mo_success, mo_fail );
1198 static bool atomic_compare_exchange_strong_explicit( T ** pDest, T ** expected, T * desired, memory_order mo_success, memory_order mo_fail ) noexcept
1201 return platform::cas_ptr_strong( pDest, *expected, desired, mo_success, mo_fail );
1203 static bool atomic_compare_exchange_strong( T * volatile * pDest, T ** expected, T * desired ) noexcept
1205 return atomic_compare_exchange_strong_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
1207 static bool atomic_compare_exchange_strong( T ** pDest, T ** expected, T * desired ) noexcept
1209 return atomic_compare_exchange_strong_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
1213 template <typename T>
1214 struct atomic_pointer: public atomic_pointer_base<T>
1216 typedef atomic_pointer_base<T> base_class;
1218 static T * atomic_fetch_add_explicit(T * volatile * pDest, ptrdiff_t val, memory_order order) noexcept
1220 # ifdef CDS_ATOMIC_fetch_ptr_add_defined
1221 platform::fetch_ptr_add( pDest, val, order );
1223 T * cur = base_class::atomic_load_explicit( pDest, memory_order_relaxed );
1224 do {} while ( !base_class::atomic_compare_exchange_weak_explicit( pDest, &cur, cur + val, order, memory_order_relaxed ));
1228 static T * atomic_fetch_add_explicit(T * * pDest, ptrdiff_t val , memory_order order) noexcept
1230 return atomic_fetch_add_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
1232 static T * atomic_fetch_add( T * volatile * pDest, ptrdiff_t val ) noexcept
1234 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
1236 static T * atomic_fetch_add( T ** pDest, ptrdiff_t val ) noexcept
1238 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
1242 static T * atomic_fetch_sub_explicit(T * volatile * pDest, ptrdiff_t val, memory_order order) noexcept
1244 # ifdef CDS_ATOMIC_fetch_ptr_sub_defined
1245 platform::fetch_ptr_sub( pDest, val, order );
1247 T * cur = base_class::atomic_load_explicit( pDest, memory_order_relaxed );
1248 do {} while ( !base_class::atomic_compare_exchange_weak_explicit( pDest, &cur, cur - val, order, memory_order_relaxed ));
1252 static T * atomic_fetch_sub_explicit(T ** pDest, ptrdiff_t val , memory_order order) noexcept
1254 return atomic_fetch_sub_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
1256 static T * atomic_fetch_sub( T volatile * pDest, ptrdiff_t val ) noexcept
1258 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
1260 static T * atomic_fetch_sub( T * pDest, ptrdiff_t val ) noexcept
1262 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
1267 struct atomic_pointer<void>: public atomic_pointer_base<void>
1269 typedef atomic_pointer_base<void> base_class;
1272 static void * atomic_fetch_add_explicit(void * volatile * pDest, ptrdiff_t val, memory_order order) noexcept
1274 void * cur = base_class::atomic_load_explicit( pDest, memory_order_relaxed );
1275 do {} while ( !base_class::atomic_compare_exchange_weak_explicit( pDest, &cur, reinterpret_cast<char *>(cur) + val, order, memory_order_relaxed ));
1278 static void * atomic_fetch_add_explicit(void * * pDest, ptrdiff_t val , memory_order order) noexcept
1280 return atomic_fetch_add_explicit( reinterpret_cast<void * volatile *>( pDest ), val, order );
1282 static void * atomic_fetch_add( void * volatile * pDest, ptrdiff_t val ) noexcept
1284 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
1286 static void * atomic_fetch_add( void ** pDest, ptrdiff_t val ) noexcept
1288 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
1292 static void * atomic_fetch_sub_explicit(void * volatile * pDest, ptrdiff_t val, memory_order order) noexcept
1294 void * cur = base_class::atomic_load_explicit( pDest, memory_order_relaxed );
1295 do {} while ( !base_class::atomic_compare_exchange_weak_explicit( pDest, &cur, reinterpret_cast<char *>(cur) - val, order, memory_order_relaxed ));
1298 static void * atomic_fetch_sub_explicit(void ** pDest, ptrdiff_t val , memory_order order) noexcept
1300 return atomic_fetch_sub_explicit( reinterpret_cast<void * volatile *>( pDest ), val, order );
1302 static void * atomic_fetch_sub( void * volatile * pDest, ptrdiff_t val ) noexcept
1304 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
1306 static void * atomic_fetch_sub( void ** pDest, ptrdiff_t val ) noexcept
1308 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
1312 template <typename T>
1313 struct atomic_integral
1316 typename cds::details::aligned_type<T, sizeof(T)>::type volatile m_val;
1318 typedef atomic_integral_ops<T, sizeof(T)> atomic_ops;
1320 typedef T atomic_type;
1322 bool is_lock_free() const volatile noexcept
1326 bool is_lock_free() const noexcept
1330 void store(T val, memory_order order = memory_order_seq_cst) volatile noexcept
1332 atomic_ops::atomic_store_explicit( &m_val, val, order );
1334 void store(T val, memory_order order = memory_order_seq_cst) noexcept
1336 atomic_ops::atomic_store_explicit( &m_val, val, order );
1339 T load(memory_order order = memory_order_seq_cst) const volatile noexcept
1341 return atomic_ops::atomic_load_explicit( &m_val, order );
1343 T load(memory_order order = memory_order_seq_cst) const noexcept
1345 return atomic_ops::atomic_load_explicit( &m_val, order );
1348 operator T() const volatile noexcept
1352 operator T() const noexcept
1357 T exchange(T val, memory_order order = memory_order_seq_cst) volatile noexcept
1359 return atomic_ops::atomic_exchange_explicit( &m_val, val, order );
1361 T exchange(T val, memory_order order = memory_order_seq_cst) noexcept
1363 return atomic_ops::atomic_exchange_explicit( &m_val, val, order );
1366 bool compare_exchange_weak(T& expected, T desired , memory_order success_order, memory_order failure_order) volatile noexcept
1368 return atomic_ops::atomic_compare_exchange_weak_explicit( &m_val, &expected, desired, success_order, failure_order );
1370 bool compare_exchange_weak(T& expected, T desired , memory_order success_order, memory_order failure_order) noexcept
1372 return atomic_ops::atomic_compare_exchange_weak_explicit( &m_val, &expected, desired, success_order, failure_order );
1374 bool compare_exchange_strong(T& expected, T desired , memory_order success_order, memory_order failure_order) volatile noexcept
1376 return atomic_ops::atomic_compare_exchange_strong_explicit( &m_val, &expected, desired, success_order, failure_order );
1378 bool compare_exchange_strong(T& expected, T desired , memory_order success_order, memory_order failure_order) noexcept
1380 return atomic_ops::atomic_compare_exchange_strong_explicit( &m_val, &expected, desired, success_order, failure_order );
1382 bool compare_exchange_weak(T& expected, T desired , memory_order success_order = memory_order_seq_cst) volatile noexcept
1384 return compare_exchange_weak( expected, desired, success_order, memory_order_relaxed );
1386 bool compare_exchange_weak(T& expected, T desired , memory_order success_order = memory_order_seq_cst) noexcept
1388 return compare_exchange_weak( expected, desired, success_order, memory_order_relaxed );
1390 bool compare_exchange_strong(T& expected, T desired , memory_order success_order = memory_order_seq_cst) volatile noexcept
1392 return compare_exchange_strong( expected, desired, success_order, memory_order_relaxed );
1394 bool compare_exchange_strong(T& expected, T desired , memory_order success_order = memory_order_seq_cst) noexcept
1396 return compare_exchange_strong( expected, desired, success_order, memory_order_relaxed );
1399 T fetch_add(T val, memory_order order = memory_order_seq_cst) volatile noexcept
1401 return atomic_ops::atomic_fetch_add_explicit( &m_val, val, order );
1403 T fetch_add(T val, memory_order order = memory_order_seq_cst) noexcept
1405 return atomic_ops::atomic_fetch_add_explicit( &m_val, val, order );
1407 T fetch_sub(T val, memory_order order = memory_order_seq_cst) volatile noexcept
1409 return atomic_ops::atomic_fetch_sub_explicit( &m_val, val, order );
1411 T fetch_sub(T val, memory_order order = memory_order_seq_cst) noexcept
1413 return atomic_ops::atomic_fetch_sub_explicit( &m_val, val, order );
1415 T fetch_and(T val, memory_order order = memory_order_seq_cst) volatile noexcept
1417 return atomic_ops::atomic_fetch_and_explicit( &m_val, val, order );
1419 T fetch_and(T val, memory_order order = memory_order_seq_cst) noexcept
1421 return atomic_ops::atomic_fetch_and_explicit( &m_val, val, order );
1424 T fetch_or(T val, memory_order order = memory_order_seq_cst) volatile noexcept
1426 return atomic_ops::atomic_fetch_or_explicit( &m_val, val, order );
1428 T fetch_or(T val, memory_order order = memory_order_seq_cst) noexcept
1430 return atomic_ops::atomic_fetch_or_explicit( &m_val, val, order );
1432 T fetch_xor(T val, memory_order order = memory_order_seq_cst) volatile noexcept
1434 return atomic_ops::atomic_fetch_xor_explicit( &m_val, val, order );
1436 T fetch_xor(T val, memory_order order = memory_order_seq_cst) noexcept
1438 return atomic_ops::atomic_fetch_xor_explicit( &m_val, val, order );
1441 atomic_integral() = default;
1442 constexpr atomic_integral(T val) noexcept
1446 atomic_integral(const atomic_integral&) = delete;
1447 atomic_integral& operator=(const atomic_integral&) = delete;
1448 atomic_integral& operator=(const atomic_integral&) volatile = delete;
1450 T operator=(T val) volatile noexcept
1455 T operator=(T val) noexcept
1462 T operator++(int) volatile noexcept
1464 return fetch_add( 1 );
1466 T operator++(int) noexcept
1468 return fetch_add( 1 );
1470 T operator--(int) volatile noexcept
1472 return fetch_sub( 1 );
1474 T operator--(int) noexcept
1476 return fetch_sub( 1 );
1480 T operator++() volatile noexcept
1482 return fetch_add( 1 ) + 1;
1484 T operator++() noexcept
1486 return fetch_add( 1 ) + 1;
1488 T operator--() volatile noexcept
1490 return fetch_sub( 1 ) - 1;
1492 T operator--() noexcept
1494 return fetch_sub( 1 ) - 1;
1498 T operator+=(T val) volatile noexcept
1500 return fetch_add( val ) + val;
1502 T operator+=(T val) noexcept
1504 return fetch_add( val ) + val;
1506 T operator-=(T val) volatile noexcept
1508 return fetch_sub( val ) - val;
1510 T operator-=(T val) noexcept
1512 return fetch_sub( val ) - val;
1514 T operator&=(T val) volatile noexcept
1516 return fetch_and( val ) & val;
1518 T operator&=(T val) noexcept
1520 return fetch_and( val ) & val;
1522 T operator|=(T val) volatile noexcept
1524 return fetch_or( val ) | val;
1526 T operator|=(T val) noexcept
1528 return fetch_or( val ) | val;
1530 T operator^=(T val) volatile noexcept
1532 return fetch_xor( val ) ^ val;
1534 T operator^=(T val) noexcept
1536 return fetch_xor( val ) ^ val;
1540 template <typename Type>
1541 struct select_primary_type {
1542 typedef typename details::primary_type<sizeof(Type)>::type type;
1545 struct select_primary_type<bool> {
1549 } // namespace details
1555 typedef details::atomic_generic_ops<T, sizeof(T), typename details::select_primary_type<T>::type > atomic_ops;
1559 bool is_lock_free() const volatile noexcept
1563 bool is_lock_free() const noexcept
1568 void store(T val, memory_order order = memory_order_seq_cst) volatile noexcept
1570 atomic_ops::atomic_store_explicit( &m_data, val, order );
1572 void store(T val, memory_order order = memory_order_seq_cst) noexcept
1574 atomic_ops::atomic_store_explicit( &m_data, val, order );
1577 T load(memory_order order = memory_order_seq_cst) const volatile noexcept
1579 return atomic_ops::atomic_load_explicit( &m_data, order );
1581 T load(memory_order order = memory_order_seq_cst) const noexcept
1583 return atomic_ops::atomic_load_explicit( &m_data, order );
1586 operator T() const volatile noexcept
1590 operator T() const noexcept
1595 T exchange(T val, memory_order order = memory_order_seq_cst) volatile noexcept
1597 return atomic_ops::atomic_exchange_explicit( &m_data, val, order );
1599 T exchange(T val, memory_order order = memory_order_seq_cst) noexcept
1601 return atomic_ops::atomic_exchange_explicit( &m_data, val, order );
1604 bool compare_exchange_weak(T& expected, T desired, memory_order success_order, memory_order failure_order) volatile noexcept
1606 return atomic_ops::atomic_compare_exchange_weak_explicit( &m_data, &expected, desired, success_order, failure_order );
1608 bool compare_exchange_weak(T& expected, T desired, memory_order success_order, memory_order failure_order) noexcept
1610 return atomic_ops::atomic_compare_exchange_weak_explicit( &m_data, &expected, desired, success_order, failure_order );
1612 bool compare_exchange_strong(T& expected, T desired, memory_order success_order, memory_order failure_order) volatile noexcept
1614 return atomic_ops::atomic_compare_exchange_strong_explicit( &m_data, &expected, desired, success_order, failure_order );
1616 bool compare_exchange_strong(T& expected, T desired, memory_order success_order, memory_order failure_order) noexcept
1618 return atomic_ops::atomic_compare_exchange_strong_explicit( &m_data, &expected, desired, success_order, failure_order );
1620 bool compare_exchange_weak(T& expected, T desired, memory_order success_order = memory_order_seq_cst) volatile noexcept
1622 return compare_exchange_weak( expected, desired, success_order, memory_order_relaxed );
1624 bool compare_exchange_weak(T& expected, T desired, memory_order success_order = memory_order_seq_cst) noexcept
1626 return compare_exchange_weak( expected, desired, success_order, memory_order_relaxed );
1628 bool compare_exchange_strong(T& expected, T desired, memory_order success_order = memory_order_seq_cst) volatile noexcept
1630 return compare_exchange_strong( expected, desired, success_order, memory_order_relaxed );
1632 bool compare_exchange_strong(T& expected, T desired, memory_order success_order = memory_order_seq_cst) noexcept
1634 return compare_exchange_strong( expected, desired, success_order, memory_order_relaxed );
1638 constexpr atomic(T val)
1642 atomic(const atomic&) = delete;
1643 atomic& operator=(const atomic&) = delete;
1644 atomic& operator=(const atomic&) volatile = delete;
1646 T operator=(T val) volatile noexcept
1651 T operator=(T val) noexcept
1658 # define CDS_DECLARE_ATOMIC_INTEGRAL( _type ) \
1660 struct atomic<_type>: public details::atomic_integral<_type> \
1663 typedef details::atomic_integral<_type> base_class ; \
1665 atomic() = default; \
1666 atomic(_type val) noexcept : base_class(val) {} \
1667 atomic(const atomic&) = delete; \
1668 atomic& operator=(const atomic&) = delete; \
1669 atomic& operator=(const atomic&) volatile = delete; \
1670 _type operator=(_type val) volatile noexcept { return base_class::operator=(val); } \
1671 _type operator=(_type val) noexcept { return base_class::operator=(val); } \
1674 CDS_DECLARE_ATOMIC_INTEGRAL(char)
1675 CDS_DECLARE_ATOMIC_INTEGRAL(signed char)
1676 CDS_DECLARE_ATOMIC_INTEGRAL(unsigned char)
1677 CDS_DECLARE_ATOMIC_INTEGRAL(short)
1678 CDS_DECLARE_ATOMIC_INTEGRAL(unsigned short)
1679 CDS_DECLARE_ATOMIC_INTEGRAL(int)
1680 CDS_DECLARE_ATOMIC_INTEGRAL(unsigned int)
1681 CDS_DECLARE_ATOMIC_INTEGRAL(long)
1682 CDS_DECLARE_ATOMIC_INTEGRAL(unsigned long)
1683 CDS_DECLARE_ATOMIC_INTEGRAL(long long)
1684 CDS_DECLARE_ATOMIC_INTEGRAL(unsigned long long)
1685 //#if CDS_COMPILER == CDS_COMPILER_GCC && CDS_COMPILER_VERSION >= 40400
1686 // CDS_DECLARE_ATOMIC_INTEGRAL(char16_t)
1687 // CDS_DECLARE_ATOMIC_INTEGRAL(char32_t)
1689 // CDS_DECLARE_ATOMIC_INTEGRAL(wchar_t)
1691 # undef CDS_DECLARE_ATOMIC_INTEGRAL
1694 template <typename T>
1699 typedef details::atomic_pointer<T> atomic_ops;
1701 bool is_lock_free() const volatile noexcept
1705 bool is_lock_free() const noexcept
1710 void store(T * val, memory_order order = memory_order_seq_cst) volatile noexcept
1712 atomic_ops::atomic_store_explicit( &m_ptr, val, order );
1714 void store(T * val, memory_order order = memory_order_seq_cst) noexcept
1716 atomic_ops::atomic_store_explicit( &m_ptr, val, order );
1719 T * load(memory_order order = memory_order_seq_cst) const volatile noexcept
1721 return atomic_ops::atomic_load_explicit( &m_ptr, order );
1723 T * load(memory_order order = memory_order_seq_cst) const noexcept
1725 return atomic_ops::atomic_load_explicit( &m_ptr, order );
1728 operator T *() const volatile noexcept
1732 operator T *() const noexcept
1737 T * exchange(T * val, memory_order order = memory_order_seq_cst) volatile noexcept
1739 return atomic_ops::atomic_exchange_explicit( &m_ptr, val, order );
1741 T * exchange(T * val, memory_order order = memory_order_seq_cst) noexcept
1743 return atomic_ops::atomic_exchange_explicit( &m_ptr, val, order );
1746 bool compare_exchange_weak(T *& expected, T * desired, memory_order success_order, memory_order failure_order) volatile noexcept
1748 return atomic_ops::atomic_compare_exchange_weak_explicit( &m_ptr, &expected, desired, success_order, failure_order );
1750 bool compare_exchange_weak(T *& expected, T * desired, memory_order success_order, memory_order failure_order) noexcept
1752 return atomic_ops::atomic_compare_exchange_weak_explicit( &m_ptr, &expected, desired, success_order, failure_order );
1754 bool compare_exchange_strong(T *& expected, T * desired, memory_order success_order, memory_order failure_order) volatile noexcept
1756 return atomic_ops::atomic_compare_exchange_strong_explicit( &m_ptr, &expected, desired, success_order, failure_order );
1758 bool compare_exchange_strong(T *& expected, T * desired, memory_order success_order, memory_order failure_order) noexcept
1760 return atomic_ops::atomic_compare_exchange_strong_explicit( &m_ptr, &expected, desired, success_order, failure_order );
1762 bool compare_exchange_weak(T *& expected, T * desired, memory_order success_order = memory_order_seq_cst) volatile noexcept
1764 return compare_exchange_weak( expected, desired, success_order, memory_order_relaxed );
1766 bool compare_exchange_weak(T *& expected, T * desired, memory_order success_order = memory_order_seq_cst) noexcept
1768 return compare_exchange_weak( expected, desired, success_order, memory_order_relaxed );
1770 bool compare_exchange_strong(T *& expected, T * desired, memory_order success_order = memory_order_seq_cst) volatile noexcept
1772 return compare_exchange_strong( expected, desired, success_order, memory_order_relaxed );
1774 bool compare_exchange_strong(T *& expected, T * desired, memory_order success_order = memory_order_seq_cst) noexcept
1776 return compare_exchange_strong( expected, desired, success_order, memory_order_relaxed );
1779 T * fetch_add(ptrdiff_t offset, memory_order order = memory_order_seq_cst) volatile noexcept
1781 return atomic_ops::atomic_fetch_add_explicit( &m_ptr, offset, order );
1783 T * fetch_add(ptrdiff_t offset, memory_order order = memory_order_seq_cst) noexcept
1785 return atomic_ops::atomic_fetch_add_explicit( &m_ptr, offset, order );
1788 T * fetch_sub(ptrdiff_t offset, memory_order order = memory_order_seq_cst) volatile noexcept
1790 return atomic_ops::atomic_fetch_sub_explicit( &m_ptr, offset, order );
1792 T * fetch_sub(ptrdiff_t offset, memory_order order = memory_order_seq_cst) noexcept
1794 return atomic_ops::atomic_fetch_sub_explicit( &m_ptr, offset, order );
1798 constexpr atomic(T * val) noexcept
1802 atomic(const atomic&) = delete;
1803 atomic& operator=(const atomic&) = delete;
1804 atomic& operator=(const atomic&) volatile = delete;
1806 T * operator=(T * val) volatile noexcept
1811 T * operator=(T * val) noexcept
1819 typedef atomic<bool> atomic_bool;
1820 typedef atomic<char> atomic_char;
1821 typedef atomic<signed char> atomic_schar;
1822 typedef atomic<unsigned char> atomic_uchar;
1823 typedef atomic<short> atomic_short;
1824 typedef atomic<unsigned short> atomic_ushort;
1825 typedef atomic<int> atomic_int;
1826 typedef atomic<unsigned int> atomic_uint;
1827 typedef atomic<long> atomic_long;
1828 typedef atomic<unsigned long> atomic_ulong;
1829 typedef atomic<long long> atomic_llong;
1830 typedef atomic<unsigned long long> atomic_ullong;
1831 #if ( CDS_COMPILER == CDS_COMPILER_GCC && CDS_COMPILER_VERSION >= 40400 ) || CDS_COMPILER == CDS_COMPILER_CLANG
1832 typedef atomic<char16_t> atomic_char16_t;
1833 typedef atomic<char32_t> atomic_char32_t;
1835 typedef atomic<wchar_t> atomic_wchar_t;
1838 typedef atomic<std::int_least8_t> atomic_int_least8_t;
1839 typedef atomic<std::uint_least8_t> atomic_uint_least8_t;
1840 typedef atomic<std::int_least16_t> atomic_int_least16_t;
1841 typedef atomic<std::uint_least16_t> atomic_uint_least16_t;
1842 typedef atomic<std::int_least32_t> atomic_int_least32_t;
1843 typedef atomic<std::uint_least32_t> atomic_uint_least32_t;
1844 typedef atomic<std::int_least64_t> atomic_int_least64_t;
1845 typedef atomic<std::uint_least64_t> atomic_uint_least64_t;
1846 typedef atomic<std::int_fast8_t> atomic_int_fast8_t;
1847 typedef atomic<std::uint_fast8_t> atomic_uint_fast8_t;
1848 typedef atomic<std::int_fast16_t> atomic_int_fast16_t;
1849 typedef atomic<std::uint_fast16_t> atomic_uint_fast16_t;
1850 typedef atomic<std::int_fast32_t> atomic_int_fast32_t;
1851 typedef atomic<std::uint_fast32_t> atomic_uint_fast32_t;
1852 typedef atomic<std::int_fast64_t> atomic_int_fast64_t;
1853 typedef atomic<std::uint_fast64_t> atomic_uint_fast64_t;
1854 typedef atomic<intptr_t> atomic_intptr_t;
1855 typedef atomic<uintptr_t> atomic_uintptr_t;
1856 typedef atomic<size_t> atomic_size_t;
1857 typedef atomic<ptrdiff_t> atomic_ptrdiff_t;
1858 typedef atomic<std::intmax_t> atomic_intmax_t;
1859 typedef atomic<std::uintmax_t> atomic_uintmax_t;
1862 static inline bool atomic_is_lock_free(const volatile atomic<T> * p) noexcept
1864 return p->is_lock_free();
1868 static inline bool atomic_is_lock_free(const atomic<T> * p ) noexcept
1870 return p->is_lock_free();
1875 static inline void atomic_init(volatile atomic<T> * p, T val) noexcept
1881 static inline void atomic_init( atomic<T> * p, T val) noexcept
1888 static inline void atomic_store(volatile atomic<T>* p, T val) noexcept
1893 static inline void atomic_store(atomic<T>* p, T val) noexcept
1899 static inline void atomic_store_explicit(volatile atomic<T>* p, T val, memory_order order) noexcept
1901 p->store( val, order );
1904 static inline void atomic_store_explicit(atomic<T>* p, T val, memory_order order) noexcept
1906 p->store( val, order );
1910 static inline T atomic_load(const volatile atomic<T>* p) noexcept
1915 static inline T atomic_load(const atomic<T>* p) noexcept
1921 static inline T atomic_load_explicit(const volatile atomic<T>* p, memory_order order) noexcept
1923 return p->load( order );
1926 static inline T atomic_load_explicit(const atomic<T>* p, memory_order order) noexcept
1928 return p->load( order );
1932 static inline T atomic_exchange(volatile atomic<T>* p, T val) noexcept
1934 return p->exchange( val );
1937 static inline T atomic_exchange(atomic<T>* p, T val ) noexcept
1939 return p->exchange( val );
1943 static inline T atomic_exchange_explicit(volatile atomic<T>* p, T val, memory_order order) noexcept
1945 return p->exchange( val, order );
1948 static inline T atomic_exchange_explicit(atomic<T>* p, T val, memory_order order) noexcept
1950 return p->exchange( val, order );
1954 static inline bool atomic_compare_exchange_weak(volatile atomic<T>* p, T* expected, T desired) noexcept
1956 return p->compare_exchange_weak( *expected, desired );
1959 static inline bool atomic_compare_exchange_weak(atomic<T>* p, T* expected, T desired) noexcept
1961 return p->compare_exchange_weak( *expected, desired );
1965 static inline bool atomic_compare_exchange_strong(volatile atomic<T>* p, T* expected, T desired) noexcept
1967 return p->compare_exchange_strong( *expected, desired );
1970 static inline bool atomic_compare_exchange_strong(atomic<T>* p, T* expected, T desired) noexcept
1972 return p->compare_exchange_strong( *expected, desired );
1976 static inline bool atomic_compare_exchange_weak_explicit(volatile atomic<T>* p, T* expected, T desired, memory_order success_order, memory_order failure_order) noexcept
1978 return p->compare_exchange_weak( *expected, desired, success_order, failure_order );
1981 static inline bool atomic_compare_exchange_weak_explicit(atomic<T>* p, T* expected, T desired, memory_order success_order, memory_order failure_order) noexcept
1983 return p->compare_exchange_weak( *expected, desired, success_order, failure_order );
1987 static inline bool atomic_compare_exchange_strong_explicit(volatile atomic<T>* p, T* expected, T desired, memory_order success_order, memory_order failure_order) noexcept
1989 return p->compare_exchange_strong( *expected, desired, success_order, failure_order );
1992 static inline bool atomic_compare_exchange_strong_explicit(atomic<T>* p, T* expected, T desired, memory_order success_order, memory_order failure_order) noexcept
1994 return p->compare_exchange_strong( *expected, desired, success_order, failure_order );
1998 static inline T atomic_fetch_add(volatile atomic<T>* p, T val) noexcept
2000 return p->fetch_add( val );
2003 static inline T atomic_fetch_add(atomic<T>* p, T val) noexcept
2005 return p->fetch_add( val );
2008 static inline T * atomic_fetch_add(volatile atomic<T *>* p, ptrdiff_t offset) noexcept
2010 return p->fetch_add( offset );
2013 static inline T * atomic_fetch_add(atomic<T *>* p, ptrdiff_t offset) noexcept
2015 return p->fetch_add( offset );
2019 static inline T atomic_fetch_add_explicit(volatile atomic<T>* p, T val, memory_order order) noexcept
2021 return p->fetch_add( val, order );
2024 static inline T atomic_fetch_add_explicit(atomic<T>* p, T val, memory_order order) noexcept
2026 return p->fetch_add( val, order );
2029 static inline T * atomic_fetch_add_explicit(volatile atomic<T *>* p, ptrdiff_t offset, memory_order order) noexcept
2031 return p->fetch_add( offset, order );
2034 static inline T * atomic_fetch_add_explicit(atomic<T *>* p, ptrdiff_t offset, memory_order order) noexcept
2036 return p->fetch_add( offset, order );
2040 static inline T atomic_fetch_sub(volatile atomic<T>* p, T val) noexcept
2042 return p->fetch_sub( val );
2045 static inline T atomic_fetch_sub(atomic<T>* p, T val) noexcept
2047 return p->fetch_sub( val );
2050 static inline T * atomic_fetch_sub(volatile atomic<T *>* p, ptrdiff_t offset) noexcept
2052 return p->fetch_sub( offset );
2055 static inline T * atomic_fetch_sub(atomic<T *>* p, ptrdiff_t offset) noexcept
2057 return p->fetch_sub( offset );
2061 static inline T atomic_fetch_sub_explicit(volatile atomic<T>* p, T val, memory_order order) noexcept
2063 return p->fetch_sub( val, order );
2066 static inline T atomic_fetch_sub_explicit(atomic<T>* p, T val, memory_order order) noexcept
2068 return p->fetch_sub( val, order );
2071 static inline T * atomic_fetch_sub_explicit(volatile atomic<T *>* p, ptrdiff_t offset, memory_order order) noexcept
2073 return p->fetch_sub( offset, order );
2076 static inline T * atomic_fetch_sub_explicit(atomic<T *>* p, ptrdiff_t offset, memory_order order) noexcept
2078 return p->fetch_sub( offset, order );
2082 static inline T atomic_fetch_and(volatile atomic<T>* p, T val) noexcept
2084 return p->fetch_and( val );
2087 static inline T atomic_fetch_and(atomic<T>* p, T val) noexcept
2089 return p->fetch_and( val );
2093 static inline T atomic_fetch_and_explicit(volatile atomic<T>* p, T val, memory_order order) noexcept
2095 return p->fetch_and( val, order );
2098 static inline T atomic_fetch_and_explicit(atomic<T>* p, T val, memory_order order) noexcept
2100 return p->fetch_and( val, order );
2104 static inline T atomic_fetch_or(volatile atomic<T>* p, T val) noexcept
2106 return p->fetch_or( val );
2109 static inline T atomic_fetch_or(atomic<T>* p, T val) noexcept
2111 return p->fetch_or( val );
2115 static inline T atomic_fetch_or_explicit(volatile atomic<T>* p, T val, memory_order order) noexcept
2117 return p->fetch_or( val, order );
2120 static inline T atomic_fetch_or_explicit(atomic<T>* p, T val, memory_order order) noexcept
2122 return p->fetch_or( val, order );
2126 static inline T atomic_fetch_xor(volatile atomic<T>* p, T val) noexcept
2128 return p->fetch_xor( val );
2131 static inline T atomic_fetch_xor(atomic<T>* p, T val) noexcept
2133 return p->fetch_xor( val );
2137 static inline T atomic_fetch_xor_explicit(volatile atomic<T>* p, T val, memory_order order) noexcept
2139 return p->fetch_xor( val, order );
2142 static inline T atomic_fetch_xor_explicit(atomic<T>* p, T val, memory_order order) noexcept
2144 return p->fetch_xor( val, order );
2148 typedef struct atomic_flag
2150 void clear( memory_order order = memory_order_seq_cst ) volatile noexcept
2152 assert( order != memory_order_acquire
2153 && order != memory_order_acq_rel
2154 && order != memory_order_consume
2156 platform::atomic_flag_clear( &m_Flag, order );
2158 void clear( memory_order order = memory_order_seq_cst ) noexcept
2160 assert( order != memory_order_acquire
2161 && order != memory_order_acq_rel
2162 && order != memory_order_consume
2164 platform::atomic_flag_clear( &m_Flag, order );
2167 bool test_and_set( memory_order order = memory_order_seq_cst ) volatile noexcept
2169 return platform::atomic_flag_tas( &m_Flag, order );
2171 bool test_and_set( memory_order order = memory_order_seq_cst ) noexcept
2173 return platform::atomic_flag_tas( &m_Flag, order );
2176 atomic_flag() = default;
2178 atomic_flag(const atomic_flag&) = delete;
2179 atomic_flag& operator=(const atomic_flag&) = delete;
2180 atomic_flag& operator=(const atomic_flag&) volatile = delete;
2182 platform::atomic_flag_type volatile m_Flag;
2185 static inline bool atomic_flag_test_and_set(volatile atomic_flag* p) noexcept
2187 return p->test_and_set();
2189 static inline bool atomic_flag_test_and_set(atomic_flag * p) noexcept
2191 return p->test_and_set();
2193 static inline bool atomic_flag_test_and_set_explicit(volatile atomic_flag* p, memory_order order) noexcept
2195 return p->test_and_set( order );
2197 static inline bool atomic_flag_test_and_set_explicit(atomic_flag* p, memory_order order) noexcept
2199 return p->test_and_set( order );
2201 static inline void atomic_flag_clear(volatile atomic_flag* p) noexcept
2205 static inline void atomic_flag_clear(atomic_flag* p) noexcept
2209 static inline void atomic_flag_clear_explicit(volatile atomic_flag* p, memory_order order) noexcept
2211 return p->clear( order );
2213 static inline void atomic_flag_clear_explicit(atomic_flag* p, memory_order order) noexcept
2215 return p->clear( order );
2219 static inline void atomic_thread_fence(memory_order order) noexcept
2221 platform::thread_fence( order );
2222 CDS_COMPILER_RW_BARRIER;
2224 static inline void atomic_signal_fence(memory_order order) noexcept
2226 platform::signal_fence( order );
2229 }} // namespace cds::cxx11_atomic
2232 #endif // #ifndef CDSLIB_COMPILER_CXX11_ATOMIC_H