2 This file is a part of libcds - Concurrent Data Structures library
4 (C) Copyright Maxim Khizhinsky (libcds.dev@gmail.com) 2006-2017
6 Source code repo: http://github.com/khizmax/libcds/
7 Download: http://sourceforge.net/projects/libcds/files/
9 Redistribution and use in source and binary forms, with or without
10 modification, are permitted provided that the following conditions are met:
12 * Redistributions of source code must retain the above copyright notice, this
13 list of conditions and the following disclaimer.
15 * Redistributions in binary form must reproduce the above copyright notice,
16 this list of conditions and the following disclaimer in the documentation
17 and/or other materials provided with the distribution.
19 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
22 DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
23 FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
25 SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
26 CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
27 OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31 #ifndef CDSLIB_COMPILER_VC_X86_CXX11_ATOMIC_H
32 #define CDSLIB_COMPILER_VC_X86_CXX11_ATOMIC_H
35 #include <emmintrin.h> // for 64bit atomic load/store
36 #include <cds/details/is_aligned.h>
38 #pragma intrinsic( _InterlockedIncrement )
39 #pragma intrinsic( _InterlockedDecrement )
40 #pragma intrinsic( _InterlockedCompareExchange )
41 //#pragma intrinsic( _InterlockedCompareExchangePointer ) // On the x86 architecture, _InterlockedCompareExchangePointer is a macro that calls _InterlockedCompareExchange
42 #pragma intrinsic( _InterlockedCompareExchange16 )
43 #pragma intrinsic( _InterlockedCompareExchange64 )
44 #pragma intrinsic( _InterlockedExchange )
45 //#pragma intrinsic( _InterlockedExchangePointer ) // On the x86 architecture, _InterlockedExchangePointer is a macro that calls _InterlockedExchange
46 #pragma intrinsic( _InterlockedExchangeAdd )
47 #pragma intrinsic( _InterlockedXor )
48 #pragma intrinsic( _InterlockedOr )
49 #pragma intrinsic( _InterlockedAnd )
50 #pragma intrinsic( _interlockedbittestandset )
52 # pragma intrinsic( _InterlockedCompareExchange8 )
53 # pragma intrinsic( _InterlockedExchange8 )
54 # pragma intrinsic( _InterlockedExchange16 )
58 namespace cds { namespace cxx11_atomic {
59 namespace platform { CDS_CXX11_INLINE_NAMESPACE namespace vc { CDS_CXX11_INLINE_NAMESPACE namespace x86 {
61 static inline void fence_before( memory_order order ) CDS_NOEXCEPT
64 case memory_order_relaxed:
65 case memory_order_acquire:
66 case memory_order_consume:
68 case memory_order_release:
69 case memory_order_acq_rel:
70 CDS_COMPILER_RW_BARRIER;
72 case memory_order_seq_cst:
73 CDS_COMPILER_RW_BARRIER;
78 static inline void fence_after( memory_order order ) CDS_NOEXCEPT
81 case memory_order_acquire:
82 case memory_order_acq_rel:
83 CDS_COMPILER_RW_BARRIER;
85 case memory_order_relaxed:
86 case memory_order_consume:
87 case memory_order_release:
89 case memory_order_seq_cst:
90 CDS_COMPILER_RW_BARRIER;
96 static inline void fence_after_load(memory_order order) CDS_NOEXCEPT
99 case memory_order_relaxed:
100 case memory_order_release:
102 case memory_order_acquire:
103 case memory_order_acq_rel:
104 CDS_COMPILER_RW_BARRIER;
106 case memory_order_consume:
108 case memory_order_seq_cst:
115 //-----------------------------------------------------------------------------
117 //-----------------------------------------------------------------------------
118 static inline void thread_fence(memory_order order) CDS_NOEXCEPT
122 case memory_order_relaxed:
123 case memory_order_consume:
125 case memory_order_release:
126 case memory_order_acquire:
127 case memory_order_acq_rel:
128 CDS_COMPILER_RW_BARRIER;
130 case memory_order_seq_cst:
137 static inline void signal_fence(memory_order order) CDS_NOEXCEPT
139 // C++11: 29.8.8: only compiler optimization, no hardware instructions
142 case memory_order_relaxed:
144 case memory_order_consume:
145 case memory_order_release:
146 case memory_order_acquire:
147 case memory_order_acq_rel:
148 case memory_order_seq_cst:
149 CDS_COMPILER_RW_BARRIER;
155 //-----------------------------------------------------------------------------
156 // atomic flag primitives
157 //-----------------------------------------------------------------------------
159 typedef unsigned char atomic_flag_type;
160 static inline bool atomic_flag_tas( atomic_flag_type volatile * pFlag, memory_order /*order*/ ) CDS_NOEXCEPT
162 return _interlockedbittestandset( (long volatile *) pFlag, 0 ) != 0;
165 static inline void atomic_flag_clear( atomic_flag_type volatile * pFlag, memory_order order ) CDS_NOEXCEPT
167 assert( order != memory_order_acquire
168 && order != memory_order_acq_rel
171 fence_before( order );
173 fence_after( order );
177 //-----------------------------------------------------------------------------
179 //-----------------------------------------------------------------------------
182 # pragma warning(push)
183 // Disable warning C4800: 'char' : forcing value to bool 'true' or 'false' (performance warning)
184 # pragma warning( disable: 4800 )
186 template <typename T>
187 static inline bool cas8_strong( T volatile * pDest, T& expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
189 static_assert( sizeof(T) == 1, "Illegal operand size" );
191 # if _MSC_VER >= 1600
193 expected = (T) _InterlockedCompareExchange8( reinterpret_cast<char volatile*>(pDest), (char) desired, (char) expected );
194 return expected == prev;
200 mov al, byte ptr [edx];
202 lock cmpxchg byte ptr [ecx], ah;
203 mov byte ptr [edx], al;
210 # pragma warning(pop)
213 template <typename T>
214 static inline bool cas8_weak( T volatile * pDest, T& expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
216 return cas8_strong( pDest, expected, desired, mo_success, mo_fail );
220 # pragma warning(push)
221 // Disable warning C4800: 'char' : forcing value to bool 'true' or 'false' (performance warning)
222 # pragma warning( disable: 4800 )
224 template <typename T>
225 static inline T exchange8( T volatile * pDest, T v, memory_order order ) CDS_NOEXCEPT
227 static_assert( sizeof(T) == 1, "Illegal operand size" );
229 # if _MSC_VER >= 1600
230 return (T) _InterlockedExchange8( reinterpret_cast<char volatile *>(pDest), (char) v );
235 lock xchg byte ptr [ecx], al;
240 # pragma warning(pop)
243 template <typename T>
244 static inline void store8( T volatile * pDest, T src, memory_order order ) CDS_NOEXCEPT
246 static_assert( sizeof(T) == 1, "Illegal operand size" );
247 assert( order == memory_order_relaxed
248 || order == memory_order_release
249 || order == memory_order_seq_cst
253 if ( order != memory_order_seq_cst ) {
254 fence_before( order );
258 exchange8( pDest, src, order );
262 template <typename T>
263 static inline T load8( T volatile const * pSrc, memory_order order ) CDS_NOEXCEPT
265 static_assert( sizeof(T) == 1, "Illegal operand size" );
266 assert( order == memory_order_relaxed
267 || order == memory_order_consume
268 || order == memory_order_acquire
269 || order == memory_order_seq_cst
274 fence_after_load( order );
278 //-----------------------------------------------------------------------------
280 //-----------------------------------------------------------------------------
282 template <typename T>
283 static inline T exchange16( T volatile * pDest, T v, memory_order /*order*/ ) CDS_NOEXCEPT
285 static_assert( sizeof(T) == 2, "Illegal operand size" );
286 assert( cds::details::is_aligned( pDest, 2 ));
288 # if _MSC_VER >= 1600
289 return (T) _InterlockedExchange16( (short volatile *) pDest, (short) v );
294 lock xchg word ptr [ecx], ax;
299 template <typename T>
300 static inline void store16( T volatile * pDest, T src, memory_order order ) CDS_NOEXCEPT
302 static_assert( sizeof(T) == 2, "Illegal operand size" );
303 assert( order == memory_order_relaxed
304 || order == memory_order_release
305 || order == memory_order_seq_cst
308 assert( cds::details::is_aligned( pDest, 2 ));
310 if ( order != memory_order_seq_cst ) {
311 fence_before( order );
315 exchange16( pDest, src, order );
319 template <typename T>
320 static inline T load16( T volatile const * pSrc, memory_order order ) CDS_NOEXCEPT
322 static_assert( sizeof(T) == 2, "Illegal operand size" );
323 assert( order == memory_order_relaxed
324 || order == memory_order_consume
325 || order == memory_order_acquire
326 || order == memory_order_seq_cst
329 assert( cds::details::is_aligned( pSrc, 2 ));
332 fence_after_load( order );
336 template <typename T>
337 static inline bool cas16_strong( T volatile * pDest, T& expected, T desired, memory_order /*mo_success*/, memory_order /*mo_fail*/ ) CDS_NOEXCEPT
339 static_assert( sizeof(T) == 2, "Illegal operand size" );
340 assert( cds::details::is_aligned( pDest, 2 ));
342 // _InterlockedCompareExchange behave as read-write memory barriers
344 expected = (T) _InterlockedCompareExchange16( (short *) pDest, (short) desired, (short) expected );
345 return expected == prev;
348 template <typename T>
349 static inline bool cas16_weak( T volatile * pDest, T& expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
351 return cas16_strong( pDest, expected, desired, mo_success, mo_fail );
354 //-----------------------------------------------------------------------------
356 //-----------------------------------------------------------------------------
358 template <typename T>
359 static inline T exchange32( T volatile * pDest, T v, memory_order /*order*/ ) CDS_NOEXCEPT
361 static_assert( sizeof(T) == 4, "Illegal operand size" );
362 assert( cds::details::is_aligned( pDest, 4 ));
364 return (T) _InterlockedExchange( (long *) pDest, (long) v );
367 template <typename T>
368 static inline void store32( T volatile * pDest, T src, memory_order order ) CDS_NOEXCEPT
370 static_assert( sizeof(T) == 4, "Illegal operand size" );
371 assert( order == memory_order_relaxed
372 || order == memory_order_release
373 || order == memory_order_seq_cst
376 assert( cds::details::is_aligned( pDest, 4 ));
378 if ( order != memory_order_seq_cst ) {
379 fence_before( order );
383 exchange32( pDest, src, order );
387 template <typename T>
388 static inline T load32( T volatile const * pSrc, memory_order order ) CDS_NOEXCEPT
390 static_assert( sizeof(T) == 4, "Illegal operand size" );
391 assert( order == memory_order_relaxed
392 || order == memory_order_consume
393 || order == memory_order_acquire
394 || order == memory_order_seq_cst
397 assert( cds::details::is_aligned( pSrc, 4 ));
400 fence_after_load( order );
404 template <typename T>
405 static inline bool cas32_strong( T volatile * pDest, T& expected, T desired, memory_order /*mo_success*/, memory_order /*mo_fail*/ ) CDS_NOEXCEPT
407 static_assert( sizeof(T) == 4, "Illegal operand size" );
408 assert( cds::details::is_aligned( pDest, 4 ));
410 // _InterlockedCompareExchange behave as read-write memory barriers
412 expected = (T) _InterlockedCompareExchange( (long *) pDest, (long) desired, (long) expected );
413 return expected == prev;
416 template <typename T>
417 static inline bool cas32_weak( T volatile * pDest, T& expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
419 return cas32_strong( pDest, expected, desired, mo_success, mo_fail );
422 // fetch_xxx may be emulated via cas32
423 // If the platform has special fetch_xxx instruction
424 // then it should define CDS_ATOMIC_fetch32_xxx_defined macro
426 # define CDS_ATOMIC_fetch32_add_defined
427 template <typename T>
428 static inline T fetch32_add( T volatile * pDest, T v, memory_order /*order*/) CDS_NOEXCEPT
430 static_assert( sizeof(T) == 4, "Illegal operand size" );
431 assert( cds::details::is_aligned( pDest, 4 ));
433 // _InterlockedExchangeAdd behave as read-write memory barriers
434 return (T) _InterlockedExchangeAdd( (long *) pDest, (long) v );
437 //-----------------------------------------------------------------------------
439 //-----------------------------------------------------------------------------
441 template <typename T>
442 static inline bool cas64_strong( T volatile * pDest, T& expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
444 static_assert( sizeof(T) == 8, "Illegal operand size" );
445 assert( cds::details::is_aligned( pDest, 8 ));
447 // _InterlockedCompareExchange behave as read-write memory barriers
449 expected = (T) _InterlockedCompareExchange64( (__int64 *) pDest, (__int64) desired, (__int64) expected );
450 return expected == prev;
453 template <typename T>
454 static inline bool cas64_weak( T volatile * pDest, T& expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
456 return cas64_strong( pDest, expected, desired, mo_success, mo_fail );
459 template <typename T>
460 static inline T load64( T volatile const * pSrc, memory_order order ) CDS_NOEXCEPT
462 static_assert( sizeof(T) == 8, "Illegal operand size" );
463 assert( order == memory_order_relaxed
464 || order == memory_order_consume
465 || order == memory_order_acquire
466 || order == memory_order_seq_cst
469 assert( cds::details::is_aligned( pSrc, 8 ));
471 // Atomically loads 64bit value by SSE intrinsics
472 __m128i volatile v = _mm_loadl_epi64( (__m128i const *) pSrc );
473 fence_after_load( order );
474 return (T) v.m128i_i64[0];
478 template <typename T>
479 static inline T exchange64( T volatile * pDest, T v, memory_order order ) CDS_NOEXCEPT
481 static_assert( sizeof(T) == 8, "Illegal operand size" );
483 T cur = load64( pDest, memory_order_relaxed );
485 } while (!cas64_weak( pDest, cur, v, order, memory_order_relaxed ));
489 template <typename T>
490 static inline void store64( T volatile * pDest, T val, memory_order order ) CDS_NOEXCEPT
492 static_assert( sizeof(T) == 8, "Illegal operand size" );
493 assert( order == memory_order_relaxed
494 || order == memory_order_release
495 || order == memory_order_seq_cst
498 assert( cds::details::is_aligned( pDest, 8 ));
500 if ( order != memory_order_seq_cst ) {
502 v.m128i_i64[0] = val;
503 fence_before( order );
504 _mm_storel_epi64( (__m128i *) pDest, v );
507 exchange64( pDest, val, order );
512 //-----------------------------------------------------------------------------
513 // pointer primitives
514 //-----------------------------------------------------------------------------
516 template <typename T>
517 static inline T * exchange_ptr( T * volatile * pDest, T * v, memory_order order ) CDS_NOEXCEPT
519 static_assert( sizeof(T *) == sizeof(void *), "Illegal operand size" );
520 return (T *) _InterlockedExchange( (long volatile *) pDest, (uintptr_t) v );
521 //return (T *) _InterlockedExchangePointer( (void * volatile *) pDest, reinterpret_cast<void *>(v));
524 template <typename T>
525 static inline void store_ptr( T * volatile * pDest, T * src, memory_order order ) CDS_NOEXCEPT
527 static_assert( sizeof(T *) == sizeof(void *), "Illegal operand size" );
528 assert( order == memory_order_relaxed
529 || order == memory_order_release
530 || order == memory_order_seq_cst
534 if ( order != memory_order_seq_cst ) {
535 fence_before( order );
539 exchange_ptr( pDest, src, order );
543 template <typename T>
544 static inline T * load_ptr( T * volatile const * pSrc, memory_order order ) CDS_NOEXCEPT
546 static_assert( sizeof(T *) == sizeof(void *), "Illegal operand size" );
547 assert( order == memory_order_relaxed
548 || order == memory_order_consume
549 || order == memory_order_acquire
550 || order == memory_order_seq_cst
555 fence_after_load( order );
559 template <typename T>
560 static inline bool cas_ptr_strong( T * volatile * pDest, T *& expected, T * desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
562 static_assert( sizeof(T *) == sizeof(void *), "Illegal operand size" );
564 // _InterlockedCompareExchangePointer behave as read-write memory barriers
566 expected = (T *) _InterlockedCompareExchange( (long volatile *) pDest, (uintptr_t) desired, (uintptr_t) prev );
567 return expected == prev;
570 template <typename T>
571 static inline bool cas_ptr_weak( T * volatile * pDest, T *& expected, T * desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
573 return cas_ptr_strong( pDest, expected, desired, mo_success, mo_fail );
575 }} // namespace vc::x86
577 #ifndef CDS_CXX11_INLINE_NAMESPACE_SUPPORT
578 using namespace vc::x86;
580 } // namespace platform
581 }} // namespace cds::cxx11_atomic
584 #endif // #ifndef CDSLIB_COMPILER_VC_X86_CXX11_ATOMIC_H