2 This file is a part of libcds - Concurrent Data Structures library
4 (C) Copyright Maxim Khizhinsky (libcds.dev@gmail.com) 2006-2017
6 Source code repo: http://github.com/khizmax/libcds/
7 Download: http://sourceforge.net/projects/libcds/files/
9 Redistribution and use in source and binary forms, with or without
10 modification, are permitted provided that the following conditions are met:
12 * Redistributions of source code must retain the above copyright notice, this
13 list of conditions and the following disclaimer.
15 * Redistributions in binary form must reproduce the above copyright notice,
16 this list of conditions and the following disclaimer in the documentation
17 and/or other materials provided with the distribution.
19 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
22 DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
23 FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
25 SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
26 CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
27 OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31 #ifndef CDSLIB_COMPILER_CXX11_ATOMIC_H
32 #define CDSLIB_COMPILER_CXX11_ATOMIC_H
35 #include <type_traits> // make_unsigned
36 #include <cds/details/defs.h>
37 #include <cds/details/aligned_type.h>
39 namespace cds { namespace cxx11_atomic {
40 typedef enum memory_order {
49 }} // namespace cds::cxx11_atomic
52 #if CDS_COMPILER == CDS_COMPILER_MSVC || (CDS_COMPILER == CDS_COMPILER_INTEL && CDS_OS_INTERFACE == CDS_OSI_WINDOWS)
53 # if CDS_PROCESSOR_ARCH == CDS_PROCESSOR_X86
54 # include <cds/compiler/vc/x86/cxx11_atomic.h>
55 # elif CDS_PROCESSOR_ARCH == CDS_PROCESSOR_AMD64
56 # include <cds/compiler/vc/amd64/cxx11_atomic.h>
58 # error "MS VC++ compiler: unsupported processor architecture"
60 #elif CDS_COMPILER == CDS_COMPILER_GCC || CDS_COMPILER == CDS_COMPILER_CLANG || CDS_COMPILER == CDS_COMPILER_INTEL
61 # if CDS_PROCESSOR_ARCH == CDS_PROCESSOR_X86
62 # include <cds/compiler/gcc/x86/cxx11_atomic.h>
63 # elif CDS_PROCESSOR_ARCH == CDS_PROCESSOR_AMD64
64 # include <cds/compiler/gcc/amd64/cxx11_atomic.h>
65 # elif CDS_PROCESSOR_ARCH == CDS_PROCESSOR_IA64
66 # include <cds/compiler/gcc/ia64/cxx11_atomic.h>
67 # elif CDS_PROCESSOR_ARCH == CDS_PROCESSOR_SPARC
68 # include <cds/compiler/gcc/sparc/cxx11_atomic.h>
69 # elif CDS_PROCESSOR_ARCH == CDS_PROCESSOR_PPC64
70 # include <cds/compiler/gcc/ppc64/cxx11_atomic.h>
71 //# elif CDS_PROCESSOR_ARCH == CDS_PROCESSOR_ARM7
72 //# include <cds/compiler/gcc/arm7/cxx11_atomic.h>
74 # error "GCC compiler: unsupported processor architecture. Try to use native C++11 atomic or boost.atomic"
77 # error "Undefined compiler"
80 namespace cds { namespace cxx11_atomic {
82 // forward declarations
88 template <typename T, size_t Size, typename Primary = T >
89 struct atomic_generic_ops;
91 template <typename T, size_t Size>
92 struct atomic_integral_ops;
94 template <size_t TypeSize>
98 struct primary_type<1>
100 typedef std::uint8_t type;
103 struct primary_type<2>
105 typedef std::uint16_t type;
108 struct primary_type<4>
110 typedef std::uint32_t type;
113 struct primary_type<8>
115 typedef std::uint64_t type;
117 #if CDS_BUILD_BITS == 64 && CDS_DCAS_SUPPORT
119 struct primary_type<16>
121 typedef unsigned __int128_t type;
125 template <typename T, typename Primary>
126 struct make_atomic_primary
128 typedef T source_type;
129 typedef Primary primary_type;
131 static primary_type volatile * ptr( source_type volatile * p ) CDS_NOEXCEPT
133 return reinterpret_cast<primary_type volatile *>(p);
135 static primary_type const volatile * ptr( source_type const volatile * p ) CDS_NOEXCEPT
137 return reinterpret_cast<primary_type const volatile *>(p);
140 static primary_type val( source_type v ) CDS_NOEXCEPT
142 return *reinterpret_cast<primary_type*>(&v);
145 static primary_type& ref( source_type& v ) CDS_NOEXCEPT
147 return reinterpret_cast<primary_type&>(v);
150 static primary_type const& ref( source_type const& v ) CDS_NOEXCEPT
152 return reinterpret_cast<primary_type const&>(v);
155 static source_type ret( primary_type r ) CDS_NOEXCEPT
157 return *reinterpret_cast<source_type *>(&r);
161 template <typename T>
162 struct make_atomic_primary<T, T>
164 typedef T source_type;
165 typedef T primary_type;
167 static primary_type volatile * ptr( source_type volatile * p ) CDS_NOEXCEPT
171 static primary_type const volatile * ptr( source_type const volatile * p ) CDS_NOEXCEPT
176 static primary_type val( source_type v ) CDS_NOEXCEPT
181 static primary_type& ref( source_type& v ) CDS_NOEXCEPT
186 static source_type ret( primary_type r ) CDS_NOEXCEPT
192 template <typename T>
193 struct atomic_integral_bitwise_ops
196 typedef typename std::make_unsigned<T>::type unsigned_type;
197 typedef atomic_generic_ops<unsigned_type, sizeof(unsigned_type)> atomic_ops;
199 static T fetch_and(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
201 unsigned_type cur = atomic_ops::atomic_load_explicit( reinterpret_cast<unsigned_type volatile *>(pDest), memory_order_relaxed );
202 do {} while ( !atomic_ops::atomic_compare_exchange_weak_explicit(
203 reinterpret_cast<unsigned_type volatile *>(pDest), &cur, cur & unsigned_type(val), order, memory_order_relaxed ));
207 static T fetch_or(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
209 unsigned_type cur = atomic_ops::atomic_load_explicit( reinterpret_cast<unsigned_type volatile *>(pDest), memory_order_relaxed );
210 do {} while ( !atomic_ops::atomic_compare_exchange_weak_explicit(
211 reinterpret_cast<unsigned_type volatile *>(pDest), &cur, cur | unsigned_type(val), order, memory_order_relaxed ));
215 static T fetch_xor(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
217 unsigned_type cur = atomic_ops::atomic_load_explicit( reinterpret_cast<unsigned_type volatile *>(pDest), memory_order_relaxed );
218 do {} while ( !atomic_ops::atomic_compare_exchange_weak_explicit(
219 reinterpret_cast<unsigned_type volatile *>(pDest), &cur, cur ^ unsigned_type(val), order, memory_order_relaxed ));
225 // 8-bit atomic operations
227 template <typename T, typename Primary>
228 struct atomic_generic_ops< T, 1, Primary >
230 typedef make_atomic_primary<T, Primary> primary;
233 static void atomic_store_explicit( T volatile * pDest, T v, memory_order order ) CDS_NOEXCEPT
235 platform::store8( primary::ptr(pDest), primary::val(v), order );
237 static void atomic_store_explicit( T * pDest, T v, memory_order order ) CDS_NOEXCEPT
239 platform::store8( primary::ptr(pDest), primary::val(v), order );
241 static void atomic_store( T volatile * pDest, T v ) CDS_NOEXCEPT
243 atomic_store_explicit( pDest, v, memory_order_seq_cst );
245 static void atomic_store( T * pDest, T v ) CDS_NOEXCEPT
247 atomic_store_explicit( pDest, v, memory_order_seq_cst );
251 static T atomic_load_explicit( T volatile const * pSrc, memory_order order ) CDS_NOEXCEPT
253 return primary::ret( platform::load8( primary::ptr(pSrc), order ));
255 static T atomic_load_explicit( T const * pSrc, memory_order order ) CDS_NOEXCEPT
257 return primary::ret( platform::load8( primary::ptr(pSrc), order ));
259 static T atomic_load( T volatile const * pSrc ) CDS_NOEXCEPT
261 return atomic_load_explicit( pSrc, memory_order_seq_cst );
263 static T atomic_load( T const * pSrc ) CDS_NOEXCEPT
265 return atomic_load_explicit( pSrc, memory_order_seq_cst );
269 static T atomic_exchange_explicit( T volatile * pDest, T val, memory_order order ) CDS_NOEXCEPT
271 return primary::ret( platform::exchange8( primary::ptr(pDest), primary::val(val), order ));
273 static T atomic_exchange_explicit( T * pDest, T val, memory_order order ) CDS_NOEXCEPT
275 return primary::ret( platform::exchange8( primary::ptr(pDest), primary::val(val), order ));
277 static T atomic_exchange( T volatile * pDest, T val ) CDS_NOEXCEPT
279 return atomic_exchange_explicit( pDest, val, memory_order_seq_cst );
281 static T atomic_exchange( T * pDest, T val ) CDS_NOEXCEPT
283 return atomic_exchange_explicit( pDest, val, memory_order_seq_cst );
287 static bool atomic_compare_exchange_weak_explicit( T volatile * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
290 return platform::cas8_weak( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
292 static bool atomic_compare_exchange_weak_explicit( T * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
295 return platform::cas8_weak( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
297 static bool atomic_compare_exchange_weak( T volatile * pDest, T * expected, T desired ) CDS_NOEXCEPT
299 return atomic_compare_exchange_weak_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
301 static bool atomic_compare_exchange_weak( T * pDest, T * expected, T desired ) CDS_NOEXCEPT
303 return atomic_compare_exchange_weak_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
305 static bool atomic_compare_exchange_strong_explicit( T volatile * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
308 return platform::cas8_strong( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
310 static bool atomic_compare_exchange_strong_explicit( T * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
313 return platform::cas8_strong( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
315 static bool atomic_compare_exchange_strong( T volatile * pDest, T * expected, T desired ) CDS_NOEXCEPT
317 return atomic_compare_exchange_strong_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
319 static bool atomic_compare_exchange_strong( T * pDest, T * expected, T desired ) CDS_NOEXCEPT
321 return atomic_compare_exchange_strong_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
325 template <typename T>
326 struct atomic_integral_ops< T, 1 >
327 : atomic_generic_ops<T, 1, T >
328 , atomic_integral_bitwise_ops<T>
330 typedef atomic_integral_bitwise_ops<T> bitwise_ops;
333 static T atomic_fetch_add_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
335 # ifdef CDS_ATOMIC_fetch8_add_defined
336 return platform::fetch8_add( pDest, val, order );
338 T cur = atomic_load_explicit( pDest, memory_order_relaxed );
339 do {} while ( !atomic_compare_exchange_weak_explicit( pDest, &cur, cur + val, order, memory_order_relaxed ));
343 static T atomic_fetch_add_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
345 return atomic_fetch_add_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
347 static T atomic_fetch_add( T volatile * pDest, T val ) CDS_NOEXCEPT
349 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
351 static T atomic_fetch_add( T * pDest, T val ) CDS_NOEXCEPT
353 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
357 static T atomic_fetch_sub_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
359 # ifdef CDS_ATOMIC_fetch8_sub_defined
360 return platform::fetch8_sub( pDest, val, order );
362 T cur = atomic_load_explicit( pDest, memory_order_relaxed );
363 do {} while ( !atomic_compare_exchange_weak_explicit( pDest, &cur, cur - val, order, memory_order_relaxed ));
367 static T atomic_fetch_sub_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
369 return atomic_fetch_sub_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
371 static T atomic_fetch_sub( T volatile * pDest, T val ) CDS_NOEXCEPT
373 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
375 static T atomic_fetch_sub( T * pDest, T val ) CDS_NOEXCEPT
377 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
381 static T atomic_fetch_and_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
383 # ifdef CDS_ATOMIC_fetch8_and_defined
384 return platform::fetch8_and( pDest, val, order );
386 return bitwise_ops::fetch_and( pDest, val, order );
389 static T atomic_fetch_and_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
391 return atomic_fetch_and_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
393 static T atomic_fetch_and( T volatile * pDest, T val ) CDS_NOEXCEPT
395 return atomic_fetch_and_explicit( pDest, val, memory_order_seq_cst );
397 static T atomic_fetch_and( T * pDest, T val ) CDS_NOEXCEPT
399 return atomic_fetch_and_explicit( pDest, val, memory_order_seq_cst );
403 static T atomic_fetch_or_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
405 # ifdef CDS_ATOMIC_fetch8_or_defined
406 return platform::fetch8_or( pDest, val, order );
408 return bitwise_ops::fetch_or( pDest, val, order );
411 static T atomic_fetch_or_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
413 return atomic_fetch_or_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
415 static T atomic_fetch_or( T volatile * pDest, T val ) CDS_NOEXCEPT
417 return atomic_fetch_or_explicit( pDest, val, memory_order_seq_cst );
419 static T atomic_fetch_or( T * pDest, T val ) CDS_NOEXCEPT
421 return atomic_fetch_or_explicit( pDest, val, memory_order_seq_cst );
425 static T atomic_fetch_xor_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
427 # ifdef CDS_ATOMIC_fetch8_xor_defined
428 return platform::fetch8_xor( pDest, val, order );
430 return bitwise_ops::fetch_xor( pDest, val, order );
433 static T atomic_fetch_xor_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
435 return atomic_fetch_xor_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
437 static T atomic_fetch_xor( T volatile * pDest, T val ) CDS_NOEXCEPT
439 return atomic_fetch_xor_explicit( pDest, val, memory_order_seq_cst );
441 static T atomic_fetch_xor( T * pDest, T val ) CDS_NOEXCEPT
443 return atomic_fetch_xor_explicit( pDest, val, memory_order_seq_cst );
447 // 16-bit atomic operations
449 template <typename T, typename Primary>
450 struct atomic_generic_ops< T, 2, Primary >
452 typedef make_atomic_primary<T, Primary> primary;
455 static void atomic_store_explicit( T volatile * pDest, T v, memory_order order ) CDS_NOEXCEPT
457 platform::store16( primary::ptr(pDest), primary::val(v), order );
459 static void atomic_store_explicit( T * pDest, T v, memory_order order ) CDS_NOEXCEPT
461 platform::store16( primary::ptr(pDest), primary::val(v), order );
463 static void atomic_store( T volatile * pDest, T v ) CDS_NOEXCEPT
465 atomic_store_explicit( pDest, v, memory_order_seq_cst );
467 static void atomic_store( T * pDest, T v ) CDS_NOEXCEPT
469 atomic_store_explicit( pDest, v, memory_order_seq_cst );
473 static T atomic_load_explicit( T volatile const * pSrc, memory_order order ) CDS_NOEXCEPT
475 return primary::ret( platform::load16( primary::ptr(pSrc), order ));
477 static T atomic_load_explicit( T const * pSrc, memory_order order ) CDS_NOEXCEPT
479 return primary::ret( platform::load16( primary::ptr(pSrc), order ));
481 static T atomic_load( T volatile const * pSrc ) CDS_NOEXCEPT
483 return atomic_load_explicit( pSrc, memory_order_seq_cst );
485 static T atomic_load( T const * pSrc ) CDS_NOEXCEPT
487 return atomic_load_explicit( pSrc, memory_order_seq_cst );
491 static T atomic_exchange_explicit( T volatile * pDest, T val, memory_order order ) CDS_NOEXCEPT
493 return primary::ret( platform::exchange16( primary::ptr(pDest), primary::val(val), order ));
495 static T atomic_exchange_explicit( T * pDest, T val, memory_order order ) CDS_NOEXCEPT
497 return primary::ret( platform::exchange16( primary::ptr(pDest), primary::val(val), order ));
499 static T atomic_exchange( T volatile * pDest, T val ) CDS_NOEXCEPT
501 return atomic_exchange_explicit( pDest, val, memory_order_seq_cst );
503 static T atomic_exchange( T * pDest, T val ) CDS_NOEXCEPT
505 return atomic_exchange_explicit( pDest, val, memory_order_seq_cst );
509 static bool atomic_compare_exchange_weak_explicit( T volatile * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
512 return platform::cas16_weak( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
514 static bool atomic_compare_exchange_weak_explicit( T * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
517 return platform::cas16_weak( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
519 static bool atomic_compare_exchange_weak( T volatile * pDest, T * expected, T desired ) CDS_NOEXCEPT
521 return atomic_compare_exchange_weak_explicit( pDest, expected, primary::val(desired), memory_order_seq_cst, memory_order_relaxed );
523 static bool atomic_compare_exchange_weak( T * pDest, T * expected, T desired ) CDS_NOEXCEPT
525 return atomic_compare_exchange_weak_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
527 static bool atomic_compare_exchange_strong_explicit( T volatile * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
530 return platform::cas16_strong( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
532 static bool atomic_compare_exchange_strong_explicit( T * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
535 return platform::cas16_strong( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
537 static bool atomic_compare_exchange_strong( T volatile * pDest, T * expected, T desired ) CDS_NOEXCEPT
539 return atomic_compare_exchange_strong_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
541 static bool atomic_compare_exchange_strong( T * pDest, T * expected, T desired ) CDS_NOEXCEPT
543 return atomic_compare_exchange_strong_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
547 template <typename T>
548 struct atomic_integral_ops< T, 2 >
549 : atomic_generic_ops< T, 2, T >
550 , atomic_integral_bitwise_ops<T>
552 typedef atomic_integral_bitwise_ops<T> bitwise_ops;
555 static T atomic_fetch_add_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
557 # ifdef CDS_ATOMIC_fetch16_add_defined
558 return platform::fetch16_add( pDest, val, order );
560 T cur = atomic_load_explicit( pDest, memory_order_relaxed );
561 do {} while ( !atomic_compare_exchange_weak_explicit( pDest, &cur, cur + val, order, memory_order_relaxed ));
565 static T atomic_fetch_add_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
567 return atomic_fetch_add_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
569 static T atomic_fetch_add( T volatile * pDest, T val ) CDS_NOEXCEPT
571 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
573 static T atomic_fetch_add( T * pDest, T val ) CDS_NOEXCEPT
575 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
579 static T atomic_fetch_sub_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
581 # ifdef CDS_ATOMIC_fetch16_sub_defined
582 return platform::fetch16_sub( pDest, val, order );
584 T cur = atomic_load_explicit( pDest, memory_order_relaxed );
585 do {} while ( !atomic_compare_exchange_weak_explicit( pDest, &cur, cur - val, order, memory_order_relaxed ));
589 static T atomic_fetch_sub_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
591 return atomic_fetch_sub_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
593 static T atomic_fetch_sub( T volatile * pDest, T val ) CDS_NOEXCEPT
595 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
597 static T atomic_fetch_sub( T * pDest, T val ) CDS_NOEXCEPT
599 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
603 static T atomic_fetch_and_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
605 # ifdef CDS_ATOMIC_fetch16_and_defined
606 return platform::fetch16_and( pDest, val, order );
608 return bitwise_ops::fetch_and( pDest, val, order );
611 static T atomic_fetch_and_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
613 return atomic_fetch_and_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
615 static T atomic_fetch_and( T volatile * pDest, T val ) CDS_NOEXCEPT
617 return atomic_fetch_and_explicit( pDest, val, memory_order_seq_cst );
619 static T atomic_fetch_and( T * pDest, T val ) CDS_NOEXCEPT
621 return atomic_fetch_and_explicit( pDest, val, memory_order_seq_cst );
625 static T atomic_fetch_or_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
627 # ifdef CDS_ATOMIC_fetch16_or_defined
628 return platform::fetch16_or( pDest, val, order );
630 return bitwise_ops::fetch_or( pDest, val, order );
633 static T atomic_fetch_or_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
635 return atomic_fetch_or_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
637 static T atomic_fetch_or( T volatile * pDest, T val ) CDS_NOEXCEPT
639 return atomic_fetch_or_explicit( pDest, val, memory_order_seq_cst );
641 static T atomic_fetch_or( T * pDest, T val ) CDS_NOEXCEPT
643 return atomic_fetch_or_explicit( pDest, val, memory_order_seq_cst );
647 static T atomic_fetch_xor_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
649 # ifdef CDS_ATOMIC_fetch16_xor_defined
650 return platform::fetch16_xor( pDest, val, order );
652 return bitwise_ops::fetch_xor( pDest, val, order );
655 static T atomic_fetch_xor_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
657 return atomic_fetch_xor_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
659 static T atomic_fetch_xor( T volatile * pDest, T val ) CDS_NOEXCEPT
661 return atomic_fetch_xor_explicit( pDest, val, memory_order_seq_cst );
663 static T atomic_fetch_xor( T * pDest, T val ) CDS_NOEXCEPT
665 return atomic_fetch_xor_explicit( pDest, val, memory_order_seq_cst );
669 // 32-bit atomic operations
671 template <typename T, typename Primary>
672 struct atomic_generic_ops< T, 4, Primary >
674 typedef make_atomic_primary<T, Primary> primary;
677 static void atomic_store_explicit( T volatile * pDest, T v, memory_order order ) CDS_NOEXCEPT
679 platform::store32( primary::ptr(pDest), primary::val(v), order );
681 static void atomic_store_explicit( T * pDest, T v, memory_order order ) CDS_NOEXCEPT
683 platform::store32( primary::ptr(pDest), primary::val(v), order );
685 static void atomic_store( T volatile * pDest, T v ) CDS_NOEXCEPT
687 atomic_store_explicit( pDest, v, memory_order_seq_cst );
689 static void atomic_store( T * pDest, T v ) CDS_NOEXCEPT
691 atomic_store_explicit( pDest, v, memory_order_seq_cst );
695 static T atomic_load_explicit( T volatile const * pSrc, memory_order order ) CDS_NOEXCEPT
697 return primary::ret( platform::load32( primary::ptr(pSrc), order ));
699 static T atomic_load_explicit( T const * pSrc, memory_order order ) CDS_NOEXCEPT
701 return primary::ret( platform::load32( primary::ptr(pSrc), order ));
703 static T atomic_load( T volatile const * pSrc ) CDS_NOEXCEPT
705 return atomic_load_explicit( pSrc, memory_order_seq_cst );
707 static T atomic_load( T const * pSrc ) CDS_NOEXCEPT
709 return atomic_load_explicit( pSrc, memory_order_seq_cst );
713 static T atomic_exchange_explicit( T volatile * pDest, T val, memory_order order ) CDS_NOEXCEPT
715 return primary::ret( platform::exchange32( primary::ptr(pDest), primary::val(val), order ));
717 static T atomic_exchange_explicit( T * pDest, T val, memory_order order ) CDS_NOEXCEPT
719 return primary::ret( platform::exchange32( primary::ptr(pDest), primary::val(val), order ));
721 static T atomic_exchange( T volatile * pDest, T val ) CDS_NOEXCEPT
723 return atomic_exchange_explicit( pDest, val, memory_order_seq_cst );
725 static T atomic_exchange( T * pDest, T val ) CDS_NOEXCEPT
727 return atomic_exchange_explicit( pDest, val, memory_order_seq_cst );
731 static bool atomic_compare_exchange_weak_explicit( T volatile * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
734 return platform::cas32_weak( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
736 static bool atomic_compare_exchange_weak_explicit( T * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
739 return platform::cas32_weak( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
741 static bool atomic_compare_exchange_weak( T volatile * pDest, T * expected, T desired ) CDS_NOEXCEPT
743 return atomic_compare_exchange_weak_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
745 static bool atomic_compare_exchange_weak( T * pDest, T * expected, T desired ) CDS_NOEXCEPT
747 return atomic_compare_exchange_weak_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
749 static bool atomic_compare_exchange_strong_explicit( T volatile * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
752 return platform::cas32_strong( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
754 static bool atomic_compare_exchange_strong_explicit( T * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
757 return platform::cas32_strong( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
759 static bool atomic_compare_exchange_strong( T volatile * pDest, T * expected, T desired ) CDS_NOEXCEPT
761 return atomic_compare_exchange_strong_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
763 static bool atomic_compare_exchange_strong( T * pDest, T * expected, T desired ) CDS_NOEXCEPT
765 return atomic_compare_exchange_strong_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
769 template <typename T>
770 struct atomic_integral_ops< T, 4 >
771 : atomic_generic_ops< T, 4, T >
772 , atomic_integral_bitwise_ops<T>
774 typedef atomic_integral_bitwise_ops<T> bitwise_ops;
776 static T atomic_fetch_add_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
778 # ifdef CDS_ATOMIC_fetch32_add_defined
779 return platform::fetch32_add( pDest, val, order );
781 T cur = atomic_load_explicit( pDest, memory_order_relaxed );
782 do {} while ( !atomic_compare_exchange_weak_explicit( pDest, &cur, cur + val, order, memory_order_relaxed ));
786 static T atomic_fetch_add_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
788 return atomic_fetch_add_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
790 static T atomic_fetch_add( T volatile * pDest, T val ) CDS_NOEXCEPT
792 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
794 static T atomic_fetch_add( T * pDest, T val ) CDS_NOEXCEPT
796 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
800 static T atomic_fetch_sub_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
802 # ifdef CDS_ATOMIC_fetch32_sub_defined
803 return platform::fetch32_sub( pDest, val, order );
805 T cur = atomic_load_explicit( pDest, memory_order_relaxed );
806 do {} while ( !atomic_compare_exchange_weak_explicit( pDest, &cur, cur - val, order, memory_order_relaxed ));
810 static T atomic_fetch_sub_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
812 return atomic_fetch_sub_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
814 static T atomic_fetch_sub( T volatile * pDest, T val ) CDS_NOEXCEPT
816 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
818 static T atomic_fetch_sub( T * pDest, T val ) CDS_NOEXCEPT
820 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
824 static T atomic_fetch_and_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
826 # ifdef CDS_ATOMIC_fetch32_and_defined
827 return platform::fetch32_and( pDest, val, order );
829 return bitwise_ops::fetch_and( pDest, val, order );
832 static T atomic_fetch_and_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
834 return atomic_fetch_and_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
836 static T atomic_fetch_and( T volatile * pDest, T val ) CDS_NOEXCEPT
838 return atomic_fetch_and_explicit( pDest, val, memory_order_seq_cst );
840 static T atomic_fetch_and( T * pDest, T val ) CDS_NOEXCEPT
842 return atomic_fetch_and_explicit( pDest, val, memory_order_seq_cst );
846 static T atomic_fetch_or_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
848 # ifdef CDS_ATOMIC_fetch32_or_defined
849 return platform::fetch32_or( pDest, val, order );
851 return bitwise_ops::fetch_or( pDest, val, order );
854 static T atomic_fetch_or_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
856 return atomic_fetch_or_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
858 static T atomic_fetch_or( T volatile * pDest, T val ) CDS_NOEXCEPT
860 return atomic_fetch_or_explicit( pDest, val, memory_order_seq_cst );
862 static T atomic_fetch_or( T * pDest, T val ) CDS_NOEXCEPT
864 return atomic_fetch_or_explicit( pDest, val, memory_order_seq_cst );
868 static T atomic_fetch_xor_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
870 # ifdef CDS_ATOMIC_fetch32_xor_defined
871 return platform::fetch32_xor( pDest, val, order );
873 return bitwise_ops::fetch_xor( pDest, val, order );
876 static T atomic_fetch_xor_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
878 return atomic_fetch_xor_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
880 static T atomic_fetch_xor( T volatile * pDest, T val ) CDS_NOEXCEPT
882 return atomic_fetch_xor_explicit( pDest, val, memory_order_seq_cst );
884 static T atomic_fetch_xor( T * pDest, T val ) CDS_NOEXCEPT
886 return atomic_fetch_xor_explicit( pDest, val, memory_order_seq_cst );
891 // 64-bit atomic operations
893 template <typename T, typename Primary>
894 struct atomic_generic_ops< T, 8, Primary >
896 typedef make_atomic_primary<T, Primary> primary;
899 static void atomic_store_explicit( T volatile * pDest, T v, memory_order order ) CDS_NOEXCEPT
901 platform::store64( primary::ptr(pDest), primary::val(v), order );
903 static void atomic_store_explicit( T * pDest, T v, memory_order order ) CDS_NOEXCEPT
905 platform::store64( primary::ptr(pDest), primary::val(v), order );
907 static void atomic_store( T volatile * pDest, T v ) CDS_NOEXCEPT
909 atomic_store_explicit( pDest, v, memory_order_seq_cst );
911 static void atomic_store( T * pDest, T v ) CDS_NOEXCEPT
913 atomic_store_explicit( pDest, v, memory_order_seq_cst );
917 static T atomic_load_explicit( T volatile const * pSrc, memory_order order ) CDS_NOEXCEPT
919 return primary::ret( platform::load64( primary::ptr(pSrc), order ));
921 static T atomic_load_explicit( T const * pSrc, memory_order order ) CDS_NOEXCEPT
923 return primary::ret( platform::load64( primary::ptr(pSrc), order ));
925 static T atomic_load( T volatile const * pSrc ) CDS_NOEXCEPT
927 return atomic_load_explicit( pSrc, memory_order_seq_cst );
929 static T atomic_load( T const * pSrc ) CDS_NOEXCEPT
931 return atomic_load_explicit( pSrc, memory_order_seq_cst );
935 static T atomic_exchange_explicit( T volatile * pDest, T val, memory_order order ) CDS_NOEXCEPT
937 return primary::ret( platform::exchange64( primary::ptr(pDest), primary::val(val), order ));
939 static T atomic_exchange_explicit( T * pDest, T val, memory_order order ) CDS_NOEXCEPT
941 return primary::ret( platform::exchange64( primary::ptr(pDest), primary::val(val), order ));
943 static T atomic_exchange( T volatile * pDest, T val ) CDS_NOEXCEPT
945 return atomic_exchange_explicit( pDest, val, memory_order_seq_cst );
947 static T atomic_exchange( T * pDest, T val ) CDS_NOEXCEPT
949 return atomic_exchange_explicit( pDest, val, memory_order_seq_cst );
953 static bool atomic_compare_exchange_weak_explicit( T volatile * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
956 return platform::cas64_weak( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
958 static bool atomic_compare_exchange_weak_explicit( T * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
961 return platform::cas64_weak( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
963 static bool atomic_compare_exchange_weak( T volatile * pDest, T * expected, T desired ) CDS_NOEXCEPT
965 return atomic_compare_exchange_weak_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
967 static bool atomic_compare_exchange_weak( T * pDest, T * expected, T desired ) CDS_NOEXCEPT
969 return atomic_compare_exchange_weak_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
971 static bool atomic_compare_exchange_strong_explicit( T volatile * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
974 return platform::cas64_strong( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
976 static bool atomic_compare_exchange_strong_explicit( T * pDest, T * expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
979 return platform::cas64_strong( primary::ptr(pDest), primary::ref(*expected), primary::val(desired), mo_success, mo_fail );
981 static bool atomic_compare_exchange_strong( T volatile * pDest, T * expected, T desired ) CDS_NOEXCEPT
983 return atomic_compare_exchange_strong_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
985 static bool atomic_compare_exchange_strong( T * pDest, T * expected, T desired ) CDS_NOEXCEPT
987 return atomic_compare_exchange_strong_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
992 template <typename T>
993 struct atomic_integral_ops< T, 8 >
994 : atomic_generic_ops< T, 8, T >
995 , atomic_integral_bitwise_ops<T>
997 typedef atomic_integral_bitwise_ops<T> bitwise_ops;
998 typedef atomic_generic_ops<T, 8, T> general_ops;
1001 static T atomic_fetch_add_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
1003 # ifdef CDS_ATOMIC_fetch64_add_defined
1004 return platform::fetch64_add( pDest, val, order );
1006 T cur = general_ops::atomic_load_explicit( pDest, memory_order_relaxed );
1007 do {} while ( !general_ops::atomic_compare_exchange_weak_explicit( pDest, &cur, cur + val, order, memory_order_relaxed ));
1011 static T atomic_fetch_add_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
1013 return atomic_fetch_add_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
1015 static T atomic_fetch_add( T volatile * pDest, T val ) CDS_NOEXCEPT
1017 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
1019 static T atomic_fetch_add( T * pDest, T val ) CDS_NOEXCEPT
1021 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
1025 static T atomic_fetch_sub_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
1027 # ifdef CDS_ATOMIC_fetch64_sub_defined
1028 return platform::fetch64_sub( pDest, val, order );
1030 T cur = general_ops::atomic_load_explicit( pDest, memory_order_relaxed );
1031 do {} while ( !general_ops::atomic_compare_exchange_weak_explicit( pDest, &cur, cur - val, order, memory_order_relaxed ));
1035 static T atomic_fetch_sub_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
1037 return atomic_fetch_sub_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
1039 static T atomic_fetch_sub( T volatile * pDest, T val ) CDS_NOEXCEPT
1041 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
1043 static T atomic_fetch_sub( T * pDest, T val ) CDS_NOEXCEPT
1045 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
1049 static T atomic_fetch_and_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
1051 # ifdef CDS_ATOMIC_fetch64_and_defined
1052 return platform::fetch64_and( pDest, val, order );
1054 return bitwise_ops::fetch_and( pDest, val, order );
1057 static T atomic_fetch_and_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
1059 return atomic_fetch_and_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
1061 static T atomic_fetch_and( T volatile * pDest, T val ) CDS_NOEXCEPT
1063 return atomic_fetch_and_explicit( pDest, val, memory_order_seq_cst );
1065 static T atomic_fetch_and( T * pDest, T val ) CDS_NOEXCEPT
1067 return atomic_fetch_and_explicit( pDest, val, memory_order_seq_cst );
1071 static T atomic_fetch_or_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
1073 # ifdef CDS_ATOMIC_fetch64_or_defined
1074 return platform::fetch64_or( pDest, val, order );
1076 return bitwise_ops::fetch_or( pDest, val, order );
1079 static T atomic_fetch_or_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
1081 return atomic_fetch_or_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
1083 static T atomic_fetch_or( T volatile * pDest, T val ) CDS_NOEXCEPT
1085 return atomic_fetch_or_explicit( pDest, val, memory_order_seq_cst );
1087 static T atomic_fetch_or( T * pDest, T val ) CDS_NOEXCEPT
1089 return atomic_fetch_or_explicit( pDest, val, memory_order_seq_cst );
1093 static T atomic_fetch_xor_explicit(T volatile * pDest, T val, memory_order order) CDS_NOEXCEPT
1095 # ifdef CDS_ATOMIC_fetch64_xor_defined
1096 return platform::fetch64_xor( pDest, val, order );
1098 return bitwise_ops::fetch_xor( pDest, val, order );
1101 static T atomic_fetch_xor_explicit(T * pDest, T val , memory_order order) CDS_NOEXCEPT
1103 return atomic_fetch_xor_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
1105 static T atomic_fetch_xor( T volatile * pDest, T val ) CDS_NOEXCEPT
1107 return atomic_fetch_xor_explicit( pDest, val, memory_order_seq_cst );
1109 static T atomic_fetch_xor( T * pDest, T val ) CDS_NOEXCEPT
1111 return atomic_fetch_xor_explicit( pDest, val, memory_order_seq_cst );
1116 // atomic pointer operations
1117 template <typename T>
1118 struct atomic_pointer_base
1121 static void atomic_store_explicit( T * volatile * pDest, T * v, memory_order order ) CDS_NOEXCEPT
1123 platform::store_ptr( pDest, v, order );
1125 static void atomic_store_explicit( T * * pDest, T * v, memory_order order ) CDS_NOEXCEPT
1127 platform::store_ptr( pDest, v, order );
1129 static void atomic_store( T * volatile * pDest, T * v ) CDS_NOEXCEPT
1131 atomic_store_explicit( pDest, v, memory_order_seq_cst );
1133 static void atomic_store( T * * pDest, T * v ) CDS_NOEXCEPT
1135 atomic_store_explicit( pDest, v, memory_order_seq_cst );
1139 static T * atomic_load_explicit( T * volatile const * pSrc, memory_order order ) CDS_NOEXCEPT
1141 return platform::load_ptr( pSrc, order );
1143 static T * atomic_load_explicit( T * const * pSrc, memory_order order ) CDS_NOEXCEPT
1145 return platform::load_ptr( pSrc, order );
1147 static T * atomic_load( T * volatile const * pSrc ) CDS_NOEXCEPT
1149 return atomic_load_explicit( pSrc, memory_order_seq_cst );
1151 static T * atomic_load( T * const * pSrc ) CDS_NOEXCEPT
1153 return atomic_load_explicit( pSrc, memory_order_seq_cst );
1157 static T * atomic_exchange_explicit( T * volatile * pDest, T * val, memory_order order ) CDS_NOEXCEPT
1159 return platform::exchange_ptr( pDest, val, order );
1161 static T * atomic_exchange_explicit( T * * pDest, T * val, memory_order order ) CDS_NOEXCEPT
1163 return platform::exchange_ptr( pDest, val, order );
1165 static T * atomic_exchange( T * volatile * pDest, T * val ) CDS_NOEXCEPT
1167 return atomic_exchange_explicit( pDest, val, memory_order_seq_cst );
1169 static T * atomic_exchange( T * * pDest, T * val ) CDS_NOEXCEPT
1171 return atomic_exchange_explicit( pDest, val, memory_order_seq_cst );
1175 static bool atomic_compare_exchange_weak_explicit( T * volatile * pDest, T * * expected, T * desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
1178 return platform::cas_ptr_weak( pDest, *expected, desired, mo_success, mo_fail );
1180 static bool atomic_compare_exchange_weak_explicit( T * * pDest, T * * expected, T * desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
1183 return platform::cas_ptr_weak( pDest, *expected, desired, mo_success, mo_fail );
1185 static bool atomic_compare_exchange_weak( T * volatile * pDest, T ** expected, T * desired ) CDS_NOEXCEPT
1187 return atomic_compare_exchange_weak_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
1189 static bool atomic_compare_exchange_weak( T ** pDest, T ** expected, T * desired ) CDS_NOEXCEPT
1191 return atomic_compare_exchange_weak_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
1193 static bool atomic_compare_exchange_strong_explicit( T * volatile * pDest, T ** expected, T * desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
1196 return platform::cas_ptr_strong( pDest, *expected, desired, mo_success, mo_fail );
1198 static bool atomic_compare_exchange_strong_explicit( T ** pDest, T ** expected, T * desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
1201 return platform::cas_ptr_strong( pDest, *expected, desired, mo_success, mo_fail );
1203 static bool atomic_compare_exchange_strong( T * volatile * pDest, T ** expected, T * desired ) CDS_NOEXCEPT
1205 return atomic_compare_exchange_strong_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
1207 static bool atomic_compare_exchange_strong( T ** pDest, T ** expected, T * desired ) CDS_NOEXCEPT
1209 return atomic_compare_exchange_strong_explicit( pDest, expected, desired, memory_order_seq_cst, memory_order_relaxed );
1213 template <typename T>
1214 struct atomic_pointer: public atomic_pointer_base<T>
1216 typedef atomic_pointer_base<T> base_class;
1218 static T * atomic_fetch_add_explicit(T * volatile * pDest, ptrdiff_t val, memory_order order) CDS_NOEXCEPT
1220 # ifdef CDS_ATOMIC_fetch_ptr_add_defined
1221 platform::fetch_ptr_add( pDest, val, order );
1223 T * cur = base_class::atomic_load_explicit( pDest, memory_order_relaxed );
1224 do {} while ( !base_class::atomic_compare_exchange_weak_explicit( pDest, &cur, cur + val, order, memory_order_relaxed ));
1228 static T * atomic_fetch_add_explicit(T * * pDest, ptrdiff_t val , memory_order order) CDS_NOEXCEPT
1230 return atomic_fetch_add_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
1232 static T * atomic_fetch_add( T * volatile * pDest, ptrdiff_t val ) CDS_NOEXCEPT
1234 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
1236 static T * atomic_fetch_add( T ** pDest, ptrdiff_t val ) CDS_NOEXCEPT
1238 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
1242 static T * atomic_fetch_sub_explicit(T * volatile * pDest, ptrdiff_t val, memory_order order) CDS_NOEXCEPT
1244 # ifdef CDS_ATOMIC_fetch_ptr_sub_defined
1245 platform::fetch_ptr_sub( pDest, val, order );
1247 T * cur = base_class::atomic_load_explicit( pDest, memory_order_relaxed );
1248 do {} while ( !base_class::atomic_compare_exchange_weak_explicit( pDest, &cur, cur - val, order, memory_order_relaxed ));
1252 static T * atomic_fetch_sub_explicit(T ** pDest, ptrdiff_t val , memory_order order) CDS_NOEXCEPT
1254 return atomic_fetch_sub_explicit( reinterpret_cast<T volatile *>( pDest ), val, order );
1256 static T * atomic_fetch_sub( T volatile * pDest, ptrdiff_t val ) CDS_NOEXCEPT
1258 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
1260 static T * atomic_fetch_sub( T * pDest, ptrdiff_t val ) CDS_NOEXCEPT
1262 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
1267 struct atomic_pointer<void>: public atomic_pointer_base<void>
1269 typedef atomic_pointer_base<void> base_class;
1272 static void * atomic_fetch_add_explicit(void * volatile * pDest, ptrdiff_t val, memory_order order) CDS_NOEXCEPT
1274 void * cur = base_class::atomic_load_explicit( pDest, memory_order_relaxed );
1275 do {} while ( !base_class::atomic_compare_exchange_weak_explicit( pDest, &cur, reinterpret_cast<char *>(cur) + val, order, memory_order_relaxed ));
1278 static void * atomic_fetch_add_explicit(void * * pDest, ptrdiff_t val , memory_order order) CDS_NOEXCEPT
1280 return atomic_fetch_add_explicit( reinterpret_cast<void * volatile *>( pDest ), val, order );
1282 static void * atomic_fetch_add( void * volatile * pDest, ptrdiff_t val ) CDS_NOEXCEPT
1284 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
1286 static void * atomic_fetch_add( void ** pDest, ptrdiff_t val ) CDS_NOEXCEPT
1288 return atomic_fetch_add_explicit( pDest, val, memory_order_seq_cst );
1292 static void * atomic_fetch_sub_explicit(void * volatile * pDest, ptrdiff_t val, memory_order order) CDS_NOEXCEPT
1294 void * cur = base_class::atomic_load_explicit( pDest, memory_order_relaxed );
1295 do {} while ( !base_class::atomic_compare_exchange_weak_explicit( pDest, &cur, reinterpret_cast<char *>(cur) - val, order, memory_order_relaxed ));
1298 static void * atomic_fetch_sub_explicit(void ** pDest, ptrdiff_t val , memory_order order) CDS_NOEXCEPT
1300 return atomic_fetch_sub_explicit( reinterpret_cast<void * volatile *>( pDest ), val, order );
1302 static void * atomic_fetch_sub( void * volatile * pDest, ptrdiff_t val ) CDS_NOEXCEPT
1304 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
1306 static void * atomic_fetch_sub( void ** pDest, ptrdiff_t val ) CDS_NOEXCEPT
1308 return atomic_fetch_sub_explicit( pDest, val, memory_order_seq_cst );
1312 template <typename T>
1313 struct atomic_integral
1316 typename cds::details::aligned_type<T, sizeof(T)>::type volatile m_val;
1318 typedef atomic_integral_ops<T, sizeof(T)> atomic_ops;
1320 typedef T atomic_type;
1322 bool is_lock_free() const volatile CDS_NOEXCEPT
1326 bool is_lock_free() const CDS_NOEXCEPT
1330 void store(T val, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1332 atomic_ops::atomic_store_explicit( &m_val, val, order );
1334 void store(T val, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1336 atomic_ops::atomic_store_explicit( &m_val, val, order );
1339 T load(memory_order order = memory_order_seq_cst) const volatile CDS_NOEXCEPT
1341 return atomic_ops::atomic_load_explicit( &m_val, order );
1343 T load(memory_order order = memory_order_seq_cst) const CDS_NOEXCEPT
1345 return atomic_ops::atomic_load_explicit( &m_val, order );
1348 operator T() const volatile CDS_NOEXCEPT
1352 operator T() const CDS_NOEXCEPT
1357 T exchange(T val, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1359 return atomic_ops::atomic_exchange_explicit( &m_val, val, order );
1361 T exchange(T val, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1363 return atomic_ops::atomic_exchange_explicit( &m_val, val, order );
1366 bool compare_exchange_weak(T& expected, T desired , memory_order success_order, memory_order failure_order) volatile CDS_NOEXCEPT
1368 return atomic_ops::atomic_compare_exchange_weak_explicit( &m_val, &expected, desired, success_order, failure_order );
1370 bool compare_exchange_weak(T& expected, T desired , memory_order success_order, memory_order failure_order) CDS_NOEXCEPT
1372 return atomic_ops::atomic_compare_exchange_weak_explicit( &m_val, &expected, desired, success_order, failure_order );
1374 bool compare_exchange_strong(T& expected, T desired , memory_order success_order, memory_order failure_order) volatile CDS_NOEXCEPT
1376 return atomic_ops::atomic_compare_exchange_strong_explicit( &m_val, &expected, desired, success_order, failure_order );
1378 bool compare_exchange_strong(T& expected, T desired , memory_order success_order, memory_order failure_order) CDS_NOEXCEPT
1380 return atomic_ops::atomic_compare_exchange_strong_explicit( &m_val, &expected, desired, success_order, failure_order );
1382 bool compare_exchange_weak(T& expected, T desired , memory_order success_order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1384 return compare_exchange_weak( expected, desired, success_order, memory_order_relaxed );
1386 bool compare_exchange_weak(T& expected, T desired , memory_order success_order = memory_order_seq_cst) CDS_NOEXCEPT
1388 return compare_exchange_weak( expected, desired, success_order, memory_order_relaxed );
1390 bool compare_exchange_strong(T& expected, T desired , memory_order success_order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1392 return compare_exchange_strong( expected, desired, success_order, memory_order_relaxed );
1394 bool compare_exchange_strong(T& expected, T desired , memory_order success_order = memory_order_seq_cst) CDS_NOEXCEPT
1396 return compare_exchange_strong( expected, desired, success_order, memory_order_relaxed );
1399 T fetch_add(T val, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1401 return atomic_ops::atomic_fetch_add_explicit( &m_val, val, order );
1403 T fetch_add(T val, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1405 return atomic_ops::atomic_fetch_add_explicit( &m_val, val, order );
1407 T fetch_sub(T val, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1409 return atomic_ops::atomic_fetch_sub_explicit( &m_val, val, order );
1411 T fetch_sub(T val, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1413 return atomic_ops::atomic_fetch_sub_explicit( &m_val, val, order );
1415 T fetch_and(T val, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1417 return atomic_ops::atomic_fetch_and_explicit( &m_val, val, order );
1419 T fetch_and(T val, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1421 return atomic_ops::atomic_fetch_and_explicit( &m_val, val, order );
1424 T fetch_or(T val, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1426 return atomic_ops::atomic_fetch_or_explicit( &m_val, val, order );
1428 T fetch_or(T val, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1430 return atomic_ops::atomic_fetch_or_explicit( &m_val, val, order );
1432 T fetch_xor(T val, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1434 return atomic_ops::atomic_fetch_xor_explicit( &m_val, val, order );
1436 T fetch_xor(T val, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1438 return atomic_ops::atomic_fetch_xor_explicit( &m_val, val, order );
1441 atomic_integral() = default;
1442 CDS_CONSTEXPR atomic_integral(T val) CDS_NOEXCEPT
1446 atomic_integral(const atomic_integral&) = delete;
1447 atomic_integral& operator=(const atomic_integral&) = delete;
1448 atomic_integral& operator=(const atomic_integral&) volatile = delete;
1450 T operator=(T val) volatile CDS_NOEXCEPT
1455 T operator=(T val) CDS_NOEXCEPT
1462 T operator++(int) volatile CDS_NOEXCEPT
1464 return fetch_add( 1 );
1466 T operator++(int) CDS_NOEXCEPT
1468 return fetch_add( 1 );
1470 T operator--(int) volatile CDS_NOEXCEPT
1472 return fetch_sub( 1 );
1474 T operator--(int) CDS_NOEXCEPT
1476 return fetch_sub( 1 );
1480 T operator++() volatile CDS_NOEXCEPT
1482 return fetch_add( 1 ) + 1;
1484 T operator++() CDS_NOEXCEPT
1486 return fetch_add( 1 ) + 1;
1488 T operator--() volatile CDS_NOEXCEPT
1490 return fetch_sub( 1 ) - 1;
1492 T operator--() CDS_NOEXCEPT
1494 return fetch_sub( 1 ) - 1;
1498 T operator+=(T val) volatile CDS_NOEXCEPT
1500 return fetch_add( val ) + val;
1502 T operator+=(T val) CDS_NOEXCEPT
1504 return fetch_add( val ) + val;
1506 T operator-=(T val) volatile CDS_NOEXCEPT
1508 return fetch_sub( val ) - val;
1510 T operator-=(T val) CDS_NOEXCEPT
1512 return fetch_sub( val ) - val;
1514 T operator&=(T val) volatile CDS_NOEXCEPT
1516 return fetch_and( val ) & val;
1518 T operator&=(T val) CDS_NOEXCEPT
1520 return fetch_and( val ) & val;
1522 T operator|=(T val) volatile CDS_NOEXCEPT
1524 return fetch_or( val ) | val;
1526 T operator|=(T val) CDS_NOEXCEPT
1528 return fetch_or( val ) | val;
1530 T operator^=(T val) volatile CDS_NOEXCEPT
1532 return fetch_xor( val ) ^ val;
1534 T operator^=(T val) CDS_NOEXCEPT
1536 return fetch_xor( val ) ^ val;
1540 template <typename Type>
1541 struct select_primary_type {
1542 typedef typename details::primary_type<sizeof(Type)>::type type;
1545 struct select_primary_type<bool> {
1549 } // namespace details
1555 typedef details::atomic_generic_ops<T, sizeof(T), typename details::select_primary_type<T>::type > atomic_ops;
1559 bool is_lock_free() const volatile CDS_NOEXCEPT
1563 bool is_lock_free() const CDS_NOEXCEPT
1568 void store(T val, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1570 atomic_ops::atomic_store_explicit( &m_data, val, order );
1572 void store(T val, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1574 atomic_ops::atomic_store_explicit( &m_data, val, order );
1577 T load(memory_order order = memory_order_seq_cst) const volatile CDS_NOEXCEPT
1579 return atomic_ops::atomic_load_explicit( &m_data, order );
1581 T load(memory_order order = memory_order_seq_cst) const CDS_NOEXCEPT
1583 return atomic_ops::atomic_load_explicit( &m_data, order );
1586 operator T() const volatile CDS_NOEXCEPT
1590 operator T() const CDS_NOEXCEPT
1595 T exchange(T val, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1597 return atomic_ops::atomic_exchange_explicit( &m_data, val, order );
1599 T exchange(T val, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1601 return atomic_ops::atomic_exchange_explicit( &m_data, val, order );
1604 bool compare_exchange_weak(T& expected, T desired, memory_order success_order, memory_order failure_order) volatile CDS_NOEXCEPT
1606 return atomic_ops::atomic_compare_exchange_weak_explicit( &m_data, &expected, desired, success_order, failure_order );
1608 bool compare_exchange_weak(T& expected, T desired, memory_order success_order, memory_order failure_order) CDS_NOEXCEPT
1610 return atomic_ops::atomic_compare_exchange_weak_explicit( &m_data, &expected, desired, success_order, failure_order );
1612 bool compare_exchange_strong(T& expected, T desired, memory_order success_order, memory_order failure_order) volatile CDS_NOEXCEPT
1614 return atomic_ops::atomic_compare_exchange_strong_explicit( &m_data, &expected, desired, success_order, failure_order );
1616 bool compare_exchange_strong(T& expected, T desired, memory_order success_order, memory_order failure_order) CDS_NOEXCEPT
1618 return atomic_ops::atomic_compare_exchange_strong_explicit( &m_data, &expected, desired, success_order, failure_order );
1620 bool compare_exchange_weak(T& expected, T desired, memory_order success_order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1622 return compare_exchange_weak( expected, desired, success_order, memory_order_relaxed );
1624 bool compare_exchange_weak(T& expected, T desired, memory_order success_order = memory_order_seq_cst) CDS_NOEXCEPT
1626 return compare_exchange_weak( expected, desired, success_order, memory_order_relaxed );
1628 bool compare_exchange_strong(T& expected, T desired, memory_order success_order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1630 return compare_exchange_strong( expected, desired, success_order, memory_order_relaxed );
1632 bool compare_exchange_strong(T& expected, T desired, memory_order success_order = memory_order_seq_cst) CDS_NOEXCEPT
1634 return compare_exchange_strong( expected, desired, success_order, memory_order_relaxed );
1638 CDS_CONSTEXPR atomic(T val)
1642 atomic(const atomic&) = delete;
1643 atomic& operator=(const atomic&) = delete;
1644 atomic& operator=(const atomic&) volatile = delete;
1646 T operator=(T val) volatile CDS_NOEXCEPT
1651 T operator=(T val) CDS_NOEXCEPT
1658 # define CDS_DECLARE_ATOMIC_INTEGRAL( _type ) \
1660 struct atomic<_type>: public details::atomic_integral<_type> \
1663 typedef details::atomic_integral<_type> base_class ; \
1665 atomic() = default; \
1666 atomic(_type val) CDS_NOEXCEPT : base_class(val) {} \
1667 atomic(const atomic&) = delete; \
1668 atomic& operator=(const atomic&) = delete; \
1669 atomic& operator=(const atomic&) volatile = delete; \
1670 _type operator=(_type val) volatile CDS_NOEXCEPT { return base_class::operator=(val); } \
1671 _type operator=(_type val) CDS_NOEXCEPT { return base_class::operator=(val); } \
1674 CDS_DECLARE_ATOMIC_INTEGRAL(char)
1675 CDS_DECLARE_ATOMIC_INTEGRAL(signed char)
1676 CDS_DECLARE_ATOMIC_INTEGRAL(unsigned char)
1677 CDS_DECLARE_ATOMIC_INTEGRAL(short)
1678 CDS_DECLARE_ATOMIC_INTEGRAL(unsigned short)
1679 CDS_DECLARE_ATOMIC_INTEGRAL(int)
1680 CDS_DECLARE_ATOMIC_INTEGRAL(unsigned int)
1681 CDS_DECLARE_ATOMIC_INTEGRAL(long)
1682 CDS_DECLARE_ATOMIC_INTEGRAL(unsigned long)
1683 CDS_DECLARE_ATOMIC_INTEGRAL(long long)
1684 CDS_DECLARE_ATOMIC_INTEGRAL(unsigned long long)
1685 //#if CDS_COMPILER == CDS_COMPILER_GCC && CDS_COMPILER_VERSION >= 40400
1686 // CDS_DECLARE_ATOMIC_INTEGRAL(char16_t)
1687 // CDS_DECLARE_ATOMIC_INTEGRAL(char32_t)
1689 // CDS_DECLARE_ATOMIC_INTEGRAL(wchar_t)
1691 # undef CDS_DECLARE_ATOMIC_INTEGRAL
1694 template <typename T>
1699 typedef details::atomic_pointer<T> atomic_ops;
1701 bool is_lock_free() const volatile CDS_NOEXCEPT
1705 bool is_lock_free() const CDS_NOEXCEPT
1710 void store(T * val, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1712 atomic_ops::atomic_store_explicit( &m_ptr, val, order );
1714 void store(T * val, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1716 atomic_ops::atomic_store_explicit( &m_ptr, val, order );
1719 T * load(memory_order order = memory_order_seq_cst) const volatile CDS_NOEXCEPT
1721 return atomic_ops::atomic_load_explicit( &m_ptr, order );
1723 T * load(memory_order order = memory_order_seq_cst) const CDS_NOEXCEPT
1725 return atomic_ops::atomic_load_explicit( &m_ptr, order );
1728 operator T *() const volatile CDS_NOEXCEPT
1732 operator T *() const CDS_NOEXCEPT
1737 T * exchange(T * val, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1739 return atomic_ops::atomic_exchange_explicit( &m_ptr, val, order );
1741 T * exchange(T * val, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1743 return atomic_ops::atomic_exchange_explicit( &m_ptr, val, order );
1746 bool compare_exchange_weak(T *& expected, T * desired, memory_order success_order, memory_order failure_order) volatile CDS_NOEXCEPT
1748 return atomic_ops::atomic_compare_exchange_weak_explicit( &m_ptr, &expected, desired, success_order, failure_order );
1750 bool compare_exchange_weak(T *& expected, T * desired, memory_order success_order, memory_order failure_order) CDS_NOEXCEPT
1752 return atomic_ops::atomic_compare_exchange_weak_explicit( &m_ptr, &expected, desired, success_order, failure_order );
1754 bool compare_exchange_strong(T *& expected, T * desired, memory_order success_order, memory_order failure_order) volatile CDS_NOEXCEPT
1756 return atomic_ops::atomic_compare_exchange_strong_explicit( &m_ptr, &expected, desired, success_order, failure_order );
1758 bool compare_exchange_strong(T *& expected, T * desired, memory_order success_order, memory_order failure_order) CDS_NOEXCEPT
1760 return atomic_ops::atomic_compare_exchange_strong_explicit( &m_ptr, &expected, desired, success_order, failure_order );
1762 bool compare_exchange_weak(T *& expected, T * desired, memory_order success_order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1764 return compare_exchange_weak( expected, desired, success_order, memory_order_relaxed );
1766 bool compare_exchange_weak(T *& expected, T * desired, memory_order success_order = memory_order_seq_cst) CDS_NOEXCEPT
1768 return compare_exchange_weak( expected, desired, success_order, memory_order_relaxed );
1770 bool compare_exchange_strong(T *& expected, T * desired, memory_order success_order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1772 return compare_exchange_strong( expected, desired, success_order, memory_order_relaxed );
1774 bool compare_exchange_strong(T *& expected, T * desired, memory_order success_order = memory_order_seq_cst) CDS_NOEXCEPT
1776 return compare_exchange_strong( expected, desired, success_order, memory_order_relaxed );
1779 T * fetch_add(ptrdiff_t offset, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1781 return atomic_ops::atomic_fetch_add_explicit( &m_ptr, offset, order );
1783 T * fetch_add(ptrdiff_t offset, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1785 return atomic_ops::atomic_fetch_add_explicit( &m_ptr, offset, order );
1788 T * fetch_sub(ptrdiff_t offset, memory_order order = memory_order_seq_cst) volatile CDS_NOEXCEPT
1790 return atomic_ops::atomic_fetch_sub_explicit( &m_ptr, offset, order );
1792 T * fetch_sub(ptrdiff_t offset, memory_order order = memory_order_seq_cst) CDS_NOEXCEPT
1794 return atomic_ops::atomic_fetch_sub_explicit( &m_ptr, offset, order );
1798 CDS_CONSTEXPR atomic(T * val) CDS_NOEXCEPT
1802 atomic(const atomic&) = delete;
1803 atomic& operator=(const atomic&) = delete;
1804 atomic& operator=(const atomic&) volatile = delete;
1806 T * operator=(T * val) volatile CDS_NOEXCEPT
1811 T * operator=(T * val) CDS_NOEXCEPT
1819 typedef atomic<bool> atomic_bool;
1820 typedef atomic<char> atomic_char;
1821 typedef atomic<signed char> atomic_schar;
1822 typedef atomic<unsigned char> atomic_uchar;
1823 typedef atomic<short> atomic_short;
1824 typedef atomic<unsigned short> atomic_ushort;
1825 typedef atomic<int> atomic_int;
1826 typedef atomic<unsigned int> atomic_uint;
1827 typedef atomic<long> atomic_long;
1828 typedef atomic<unsigned long> atomic_ulong;
1829 typedef atomic<long long> atomic_llong;
1830 typedef atomic<unsigned long long> atomic_ullong;
1831 #if ( CDS_COMPILER == CDS_COMPILER_GCC && CDS_COMPILER_VERSION >= 40400 ) || CDS_COMPILER == CDS_COMPILER_CLANG
1832 typedef atomic<char16_t> atomic_char16_t;
1833 typedef atomic<char32_t> atomic_char32_t;
1835 typedef atomic<wchar_t> atomic_wchar_t;
1838 typedef atomic<std::int_least8_t> atomic_int_least8_t;
1839 typedef atomic<std::uint_least8_t> atomic_uint_least8_t;
1840 typedef atomic<std::int_least16_t> atomic_int_least16_t;
1841 typedef atomic<std::uint_least16_t> atomic_uint_least16_t;
1842 typedef atomic<std::int_least32_t> atomic_int_least32_t;
1843 typedef atomic<std::uint_least32_t> atomic_uint_least32_t;
1844 typedef atomic<std::int_least64_t> atomic_int_least64_t;
1845 typedef atomic<std::uint_least64_t> atomic_uint_least64_t;
1846 typedef atomic<std::int_fast8_t> atomic_int_fast8_t;
1847 typedef atomic<std::uint_fast8_t> atomic_uint_fast8_t;
1848 typedef atomic<std::int_fast16_t> atomic_int_fast16_t;
1849 typedef atomic<std::uint_fast16_t> atomic_uint_fast16_t;
1850 typedef atomic<std::int_fast32_t> atomic_int_fast32_t;
1851 typedef atomic<std::uint_fast32_t> atomic_uint_fast32_t;
1852 typedef atomic<std::int_fast64_t> atomic_int_fast64_t;
1853 typedef atomic<std::uint_fast64_t> atomic_uint_fast64_t;
1854 typedef atomic<intptr_t> atomic_intptr_t;
1855 typedef atomic<uintptr_t> atomic_uintptr_t;
1856 typedef atomic<size_t> atomic_size_t;
1857 typedef atomic<ptrdiff_t> atomic_ptrdiff_t;
1858 typedef atomic<std::intmax_t> atomic_intmax_t;
1859 typedef atomic<std::uintmax_t> atomic_uintmax_t;
1862 static inline bool atomic_is_lock_free(const volatile atomic<T> * p) CDS_NOEXCEPT
1864 return p->is_lock_free();
1868 static inline bool atomic_is_lock_free(const atomic<T> * p ) CDS_NOEXCEPT
1870 return p->is_lock_free();
1875 static inline void atomic_init(volatile atomic<T> * p, T val) CDS_NOEXCEPT
1881 static inline void atomic_init( atomic<T> * p, T val) CDS_NOEXCEPT
1888 static inline void atomic_store(volatile atomic<T>* p, T val) CDS_NOEXCEPT
1893 static inline void atomic_store(atomic<T>* p, T val) CDS_NOEXCEPT
1899 static inline void atomic_store_explicit(volatile atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
1901 p->store( val, order );
1904 static inline void atomic_store_explicit(atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
1906 p->store( val, order );
1910 static inline T atomic_load(const volatile atomic<T>* p) CDS_NOEXCEPT
1915 static inline T atomic_load(const atomic<T>* p) CDS_NOEXCEPT
1921 static inline T atomic_load_explicit(const volatile atomic<T>* p, memory_order order) CDS_NOEXCEPT
1923 return p->load( order );
1926 static inline T atomic_load_explicit(const atomic<T>* p, memory_order order) CDS_NOEXCEPT
1928 return p->load( order );
1932 static inline T atomic_exchange(volatile atomic<T>* p, T val) CDS_NOEXCEPT
1934 return p->exchange( val );
1937 static inline T atomic_exchange(atomic<T>* p, T val ) CDS_NOEXCEPT
1939 return p->exchange( val );
1943 static inline T atomic_exchange_explicit(volatile atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
1945 return p->exchange( val, order );
1948 static inline T atomic_exchange_explicit(atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
1950 return p->exchange( val, order );
1954 static inline bool atomic_compare_exchange_weak(volatile atomic<T>* p, T* expected, T desired) CDS_NOEXCEPT
1956 return p->compare_exchange_weak( *expected, desired );
1959 static inline bool atomic_compare_exchange_weak(atomic<T>* p, T* expected, T desired) CDS_NOEXCEPT
1961 return p->compare_exchange_weak( *expected, desired );
1965 static inline bool atomic_compare_exchange_strong(volatile atomic<T>* p, T* expected, T desired) CDS_NOEXCEPT
1967 return p->compare_exchange_strong( *expected, desired );
1970 static inline bool atomic_compare_exchange_strong(atomic<T>* p, T* expected, T desired) CDS_NOEXCEPT
1972 return p->compare_exchange_strong( *expected, desired );
1976 static inline bool atomic_compare_exchange_weak_explicit(volatile atomic<T>* p, T* expected, T desired, memory_order success_order, memory_order failure_order) CDS_NOEXCEPT
1978 return p->compare_exchange_weak( *expected, desired, success_order, failure_order );
1981 static inline bool atomic_compare_exchange_weak_explicit(atomic<T>* p, T* expected, T desired, memory_order success_order, memory_order failure_order) CDS_NOEXCEPT
1983 return p->compare_exchange_weak( *expected, desired, success_order, failure_order );
1987 static inline bool atomic_compare_exchange_strong_explicit(volatile atomic<T>* p, T* expected, T desired, memory_order success_order, memory_order failure_order) CDS_NOEXCEPT
1989 return p->compare_exchange_strong( *expected, desired, success_order, failure_order );
1992 static inline bool atomic_compare_exchange_strong_explicit(atomic<T>* p, T* expected, T desired, memory_order success_order, memory_order failure_order) CDS_NOEXCEPT
1994 return p->compare_exchange_strong( *expected, desired, success_order, failure_order );
1998 static inline T atomic_fetch_add(volatile atomic<T>* p, T val) CDS_NOEXCEPT
2000 return p->fetch_add( val );
2003 static inline T atomic_fetch_add(atomic<T>* p, T val) CDS_NOEXCEPT
2005 return p->fetch_add( val );
2008 static inline T * atomic_fetch_add(volatile atomic<T *>* p, ptrdiff_t offset) CDS_NOEXCEPT
2010 return p->fetch_add( offset );
2013 static inline T * atomic_fetch_add(atomic<T *>* p, ptrdiff_t offset) CDS_NOEXCEPT
2015 return p->fetch_add( offset );
2019 static inline T atomic_fetch_add_explicit(volatile atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
2021 return p->fetch_add( val, order );
2024 static inline T atomic_fetch_add_explicit(atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
2026 return p->fetch_add( val, order );
2029 static inline T * atomic_fetch_add_explicit(volatile atomic<T *>* p, ptrdiff_t offset, memory_order order) CDS_NOEXCEPT
2031 return p->fetch_add( offset, order );
2034 static inline T * atomic_fetch_add_explicit(atomic<T *>* p, ptrdiff_t offset, memory_order order) CDS_NOEXCEPT
2036 return p->fetch_add( offset, order );
2040 static inline T atomic_fetch_sub(volatile atomic<T>* p, T val) CDS_NOEXCEPT
2042 return p->fetch_sub( val );
2045 static inline T atomic_fetch_sub(atomic<T>* p, T val) CDS_NOEXCEPT
2047 return p->fetch_sub( val );
2050 static inline T * atomic_fetch_sub(volatile atomic<T *>* p, ptrdiff_t offset) CDS_NOEXCEPT
2052 return p->fetch_sub( offset );
2055 static inline T * atomic_fetch_sub(atomic<T *>* p, ptrdiff_t offset) CDS_NOEXCEPT
2057 return p->fetch_sub( offset );
2061 static inline T atomic_fetch_sub_explicit(volatile atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
2063 return p->fetch_sub( val, order );
2066 static inline T atomic_fetch_sub_explicit(atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
2068 return p->fetch_sub( val, order );
2071 static inline T * atomic_fetch_sub_explicit(volatile atomic<T *>* p, ptrdiff_t offset, memory_order order) CDS_NOEXCEPT
2073 return p->fetch_sub( offset, order );
2076 static inline T * atomic_fetch_sub_explicit(atomic<T *>* p, ptrdiff_t offset, memory_order order) CDS_NOEXCEPT
2078 return p->fetch_sub( offset, order );
2082 static inline T atomic_fetch_and(volatile atomic<T>* p, T val) CDS_NOEXCEPT
2084 return p->fetch_and( val );
2087 static inline T atomic_fetch_and(atomic<T>* p, T val) CDS_NOEXCEPT
2089 return p->fetch_and( val );
2093 static inline T atomic_fetch_and_explicit(volatile atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
2095 return p->fetch_and( val, order );
2098 static inline T atomic_fetch_and_explicit(atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
2100 return p->fetch_and( val, order );
2104 static inline T atomic_fetch_or(volatile atomic<T>* p, T val) CDS_NOEXCEPT
2106 return p->fetch_or( val );
2109 static inline T atomic_fetch_or(atomic<T>* p, T val) CDS_NOEXCEPT
2111 return p->fetch_or( val );
2115 static inline T atomic_fetch_or_explicit(volatile atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
2117 return p->fetch_or( val, order );
2120 static inline T atomic_fetch_or_explicit(atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
2122 return p->fetch_or( val, order );
2126 static inline T atomic_fetch_xor(volatile atomic<T>* p, T val) CDS_NOEXCEPT
2128 return p->fetch_xor( val );
2131 static inline T atomic_fetch_xor(atomic<T>* p, T val) CDS_NOEXCEPT
2133 return p->fetch_xor( val );
2137 static inline T atomic_fetch_xor_explicit(volatile atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
2139 return p->fetch_xor( val, order );
2142 static inline T atomic_fetch_xor_explicit(atomic<T>* p, T val, memory_order order) CDS_NOEXCEPT
2144 return p->fetch_xor( val, order );
2148 typedef struct atomic_flag
2150 void clear( memory_order order = memory_order_seq_cst ) volatile CDS_NOEXCEPT
2152 assert( order != memory_order_acquire
2153 && order != memory_order_acq_rel
2154 && order != memory_order_consume
2156 platform::atomic_flag_clear( &m_Flag, order );
2158 void clear( memory_order order = memory_order_seq_cst ) CDS_NOEXCEPT
2160 assert( order != memory_order_acquire
2161 && order != memory_order_acq_rel
2162 && order != memory_order_consume
2164 platform::atomic_flag_clear( &m_Flag, order );
2167 bool test_and_set( memory_order order = memory_order_seq_cst ) volatile CDS_NOEXCEPT
2169 return platform::atomic_flag_tas( &m_Flag, order );
2171 bool test_and_set( memory_order order = memory_order_seq_cst ) CDS_NOEXCEPT
2173 return platform::atomic_flag_tas( &m_Flag, order );
2176 atomic_flag() = default;
2178 atomic_flag(const atomic_flag&) = delete;
2179 atomic_flag& operator=(const atomic_flag&) = delete;
2180 atomic_flag& operator=(const atomic_flag&) volatile = delete;
2182 platform::atomic_flag_type volatile m_Flag;
2185 static inline bool atomic_flag_test_and_set(volatile atomic_flag* p) CDS_NOEXCEPT
2187 return p->test_and_set();
2189 static inline bool atomic_flag_test_and_set(atomic_flag * p) CDS_NOEXCEPT
2191 return p->test_and_set();
2193 static inline bool atomic_flag_test_and_set_explicit(volatile atomic_flag* p, memory_order order) CDS_NOEXCEPT
2195 return p->test_and_set( order );
2197 static inline bool atomic_flag_test_and_set_explicit(atomic_flag* p, memory_order order) CDS_NOEXCEPT
2199 return p->test_and_set( order );
2201 static inline void atomic_flag_clear(volatile atomic_flag* p) CDS_NOEXCEPT
2205 static inline void atomic_flag_clear(atomic_flag* p) CDS_NOEXCEPT
2209 static inline void atomic_flag_clear_explicit(volatile atomic_flag* p, memory_order order) CDS_NOEXCEPT
2211 return p->clear( order );
2213 static inline void atomic_flag_clear_explicit(atomic_flag* p, memory_order order) CDS_NOEXCEPT
2215 return p->clear( order );
2219 static inline void atomic_thread_fence(memory_order order) CDS_NOEXCEPT
2221 platform::thread_fence( order );
2222 CDS_COMPILER_RW_BARRIER;
2224 static inline void atomic_signal_fence(memory_order order) CDS_NOEXCEPT
2226 platform::signal_fence( order );
2229 }} // namespace cds::cxx11_atomic
2232 #endif // #ifndef CDSLIB_COMPILER_CXX11_ATOMIC_H