3 #ifndef __CDS_COMPILER_VC_X86_CXX11_ATOMIC_H
4 #define __CDS_COMPILER_VC_X86_CXX11_ATOMIC_H
7 #include <emmintrin.h> // for 64bit atomic load/store
8 #include <cds/details/is_aligned.h>
10 #pragma intrinsic( _InterlockedIncrement )
11 #pragma intrinsic( _InterlockedDecrement )
12 #pragma intrinsic( _InterlockedCompareExchange )
13 //#pragma intrinsic( _InterlockedCompareExchangePointer ) // On the x86 architecture, _InterlockedCompareExchangePointer is a macro that calls _InterlockedCompareExchange
14 #pragma intrinsic( _InterlockedCompareExchange16 )
15 #pragma intrinsic( _InterlockedCompareExchange64 )
16 #pragma intrinsic( _InterlockedExchange )
17 //#pragma intrinsic( _InterlockedExchangePointer ) // On the x86 architecture, _InterlockedExchangePointer is a macro that calls _InterlockedExchange
18 #pragma intrinsic( _InterlockedExchangeAdd )
19 #pragma intrinsic( _InterlockedXor )
20 #pragma intrinsic( _InterlockedOr )
21 #pragma intrinsic( _InterlockedAnd )
22 #pragma intrinsic( _interlockedbittestandset )
24 # pragma intrinsic( _InterlockedCompareExchange8 )
25 # pragma intrinsic( _InterlockedExchange8 )
26 # pragma intrinsic( _InterlockedExchange16 )
30 namespace cds { namespace cxx11_atomic {
31 namespace platform { CDS_CXX11_INLINE_NAMESPACE namespace vc { CDS_CXX11_INLINE_NAMESPACE namespace x86 {
33 static inline void fence_before( memory_order order ) CDS_NOEXCEPT
36 case memory_order_relaxed:
37 case memory_order_acquire:
38 case memory_order_consume:
40 case memory_order_release:
41 case memory_order_acq_rel:
42 CDS_COMPILER_RW_BARRIER;
44 case memory_order_seq_cst:
45 CDS_COMPILER_RW_BARRIER;
50 static inline void fence_after( memory_order order ) CDS_NOEXCEPT
53 case memory_order_acquire:
54 case memory_order_acq_rel:
55 CDS_COMPILER_RW_BARRIER;
57 case memory_order_relaxed:
58 case memory_order_consume:
59 case memory_order_release:
61 case memory_order_seq_cst:
62 CDS_COMPILER_RW_BARRIER;
68 static inline void fence_after_load(memory_order order) CDS_NOEXCEPT
71 case memory_order_relaxed:
72 case memory_order_release:
74 case memory_order_acquire:
75 case memory_order_acq_rel:
76 CDS_COMPILER_RW_BARRIER;
78 case memory_order_consume:
80 case memory_order_seq_cst:
87 //-----------------------------------------------------------------------------
89 //-----------------------------------------------------------------------------
90 static inline void thread_fence(memory_order order) CDS_NOEXCEPT
94 case memory_order_relaxed:
95 case memory_order_consume:
97 case memory_order_release:
98 case memory_order_acquire:
99 case memory_order_acq_rel:
100 CDS_COMPILER_RW_BARRIER;
102 case memory_order_seq_cst:
109 static inline void signal_fence(memory_order order) CDS_NOEXCEPT
111 // C++11: 29.8.8: only compiler optimization, no hardware instructions
114 case memory_order_relaxed:
116 case memory_order_consume:
117 case memory_order_release:
118 case memory_order_acquire:
119 case memory_order_acq_rel:
120 case memory_order_seq_cst:
121 CDS_COMPILER_RW_BARRIER;
127 //-----------------------------------------------------------------------------
128 // atomic flag primitives
129 //-----------------------------------------------------------------------------
131 typedef unsigned char atomic_flag_type;
132 static inline bool atomic_flag_tas( atomic_flag_type volatile * pFlag, memory_order /*order*/ ) CDS_NOEXCEPT
134 return _interlockedbittestandset( (long volatile *) pFlag, 0 ) != 0;
137 static inline void atomic_flag_clear( atomic_flag_type volatile * pFlag, memory_order order ) CDS_NOEXCEPT
139 assert( order != memory_order_acquire
140 && order != memory_order_acq_rel
143 fence_before( order );
145 fence_after( order );
149 //-----------------------------------------------------------------------------
151 //-----------------------------------------------------------------------------
154 # pragma warning(push)
155 // Disable warning C4800: 'char' : forcing value to bool 'true' or 'false' (performance warning)
156 # pragma warning( disable: 4800 )
158 template <typename T>
159 static inline bool cas8_strong( T volatile * pDest, T& expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
161 static_assert( sizeof(T) == 1, "Illegal operand size" );
163 # if _MSC_VER >= 1600
165 expected = (T) _InterlockedCompareExchange8( reinterpret_cast<char volatile*>(pDest), (char) desired, (char) expected );
166 return expected == prev;
172 mov al, byte ptr [edx];
174 lock cmpxchg byte ptr [ecx], ah;
175 mov byte ptr [edx], al;
182 # pragma warning(pop)
185 template <typename T>
186 static inline bool cas8_weak( T volatile * pDest, T& expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
188 return cas8_strong( pDest, expected, desired, mo_success, mo_fail );
192 # pragma warning(push)
193 // Disable warning C4800: 'char' : forcing value to bool 'true' or 'false' (performance warning)
194 # pragma warning( disable: 4800 )
196 template <typename T>
197 static inline T exchange8( T volatile * pDest, T v, memory_order order ) CDS_NOEXCEPT
199 static_assert( sizeof(T) == 1, "Illegal operand size" );
201 # if _MSC_VER >= 1600
202 return (T) _InterlockedExchange8( reinterpret_cast<char volatile *>(pDest), (char) v );
207 lock xchg byte ptr [ecx], al;
212 # pragma warning(pop)
215 template <typename T>
216 static inline void store8( T volatile * pDest, T src, memory_order order ) CDS_NOEXCEPT
218 static_assert( sizeof(T) == 1, "Illegal operand size" );
219 assert( order == memory_order_relaxed
220 || order == memory_order_release
221 || order == memory_order_seq_cst
225 if ( order != memory_order_seq_cst ) {
226 fence_before( order );
230 exchange8( pDest, src, order );
234 template <typename T>
235 static inline T load8( T volatile const * pSrc, memory_order order ) CDS_NOEXCEPT
237 static_assert( sizeof(T) == 1, "Illegal operand size" );
238 assert( order == memory_order_relaxed
239 || order == memory_order_consume
240 || order == memory_order_acquire
241 || order == memory_order_seq_cst
246 fence_after_load( order );
250 //-----------------------------------------------------------------------------
252 //-----------------------------------------------------------------------------
254 template <typename T>
255 static inline T exchange16( T volatile * pDest, T v, memory_order /*order*/ ) CDS_NOEXCEPT
257 static_assert( sizeof(T) == 2, "Illegal operand size" );
258 assert( cds::details::is_aligned( pDest, 2 ));
260 # if _MSC_VER >= 1600
261 return (T) _InterlockedExchange16( (short volatile *) pDest, (short) v );
266 lock xchg word ptr [ecx], ax;
271 template <typename T>
272 static inline void store16( T volatile * pDest, T src, memory_order order ) CDS_NOEXCEPT
274 static_assert( sizeof(T) == 2, "Illegal operand size" );
275 assert( order == memory_order_relaxed
276 || order == memory_order_release
277 || order == memory_order_seq_cst
280 assert( cds::details::is_aligned( pDest, 2 ));
282 if ( order != memory_order_seq_cst ) {
283 fence_before( order );
287 exchange16( pDest, src, order );
291 template <typename T>
292 static inline T load16( T volatile const * pSrc, memory_order order ) CDS_NOEXCEPT
294 static_assert( sizeof(T) == 2, "Illegal operand size" );
295 assert( order == memory_order_relaxed
296 || order == memory_order_consume
297 || order == memory_order_acquire
298 || order == memory_order_seq_cst
301 assert( cds::details::is_aligned( pSrc, 2 ));
304 fence_after_load( order );
308 template <typename T>
309 static inline bool cas16_strong( T volatile * pDest, T& expected, T desired, memory_order /*mo_success*/, memory_order /*mo_fail*/ ) CDS_NOEXCEPT
311 static_assert( sizeof(T) == 2, "Illegal operand size" );
312 assert( cds::details::is_aligned( pDest, 2 ));
314 // _InterlockedCompareExchange behave as read-write memory barriers
316 expected = (T) _InterlockedCompareExchange16( (short *) pDest, (short) desired, (short) expected );
317 return expected == prev;
320 template <typename T>
321 static inline bool cas16_weak( T volatile * pDest, T& expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
323 return cas16_strong( pDest, expected, desired, mo_success, mo_fail );
326 //-----------------------------------------------------------------------------
328 //-----------------------------------------------------------------------------
330 template <typename T>
331 static inline T exchange32( T volatile * pDest, T v, memory_order /*order*/ ) CDS_NOEXCEPT
333 static_assert( sizeof(T) == 4, "Illegal operand size" );
334 assert( cds::details::is_aligned( pDest, 4 ));
336 return (T) _InterlockedExchange( (long *) pDest, (long) v );
339 template <typename T>
340 static inline void store32( T volatile * pDest, T src, memory_order order ) CDS_NOEXCEPT
342 static_assert( sizeof(T) == 4, "Illegal operand size" );
343 assert( order == memory_order_relaxed
344 || order == memory_order_release
345 || order == memory_order_seq_cst
348 assert( cds::details::is_aligned( pDest, 4 ));
350 if ( order != memory_order_seq_cst ) {
351 fence_before( order );
355 exchange32( pDest, src, order );
359 template <typename T>
360 static inline T load32( T volatile const * pSrc, memory_order order ) CDS_NOEXCEPT
362 static_assert( sizeof(T) == 4, "Illegal operand size" );
363 assert( order == memory_order_relaxed
364 || order == memory_order_consume
365 || order == memory_order_acquire
366 || order == memory_order_seq_cst
369 assert( cds::details::is_aligned( pSrc, 4 ));
372 fence_after_load( order );
376 template <typename T>
377 static inline bool cas32_strong( T volatile * pDest, T& expected, T desired, memory_order /*mo_success*/, memory_order /*mo_fail*/ ) CDS_NOEXCEPT
379 static_assert( sizeof(T) == 4, "Illegal operand size" );
380 assert( cds::details::is_aligned( pDest, 4 ));
382 // _InterlockedCompareExchange behave as read-write memory barriers
384 expected = (T) _InterlockedCompareExchange( (long *) pDest, (long) desired, (long) expected );
385 return expected == prev;
388 template <typename T>
389 static inline bool cas32_weak( T volatile * pDest, T& expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
391 return cas32_strong( pDest, expected, desired, mo_success, mo_fail );
394 // fetch_xxx may be emulated via cas32
395 // If the platform has special fetch_xxx instruction
396 // then it should define CDS_ATOMIC_fetch32_xxx_defined macro
398 # define CDS_ATOMIC_fetch32_add_defined
399 template <typename T>
400 static inline T fetch32_add( T volatile * pDest, T v, memory_order /*order*/) CDS_NOEXCEPT
402 static_assert( sizeof(T) == 4, "Illegal operand size" );
403 assert( cds::details::is_aligned( pDest, 4 ));
405 // _InterlockedExchangeAdd behave as read-write memory barriers
406 return (T) _InterlockedExchangeAdd( (long *) pDest, (long) v );
409 //-----------------------------------------------------------------------------
411 //-----------------------------------------------------------------------------
413 template <typename T>
414 static inline bool cas64_strong( T volatile * pDest, T& expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
416 static_assert( sizeof(T) == 8, "Illegal operand size" );
417 assert( cds::details::is_aligned( pDest, 8 ));
419 // _InterlockedCompareExchange behave as read-write memory barriers
421 expected = (T) _InterlockedCompareExchange64( (__int64 *) pDest, (__int64) desired, (__int64) expected );
422 return expected == prev;
425 template <typename T>
426 static inline bool cas64_weak( T volatile * pDest, T& expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
428 return cas64_strong( pDest, expected, desired, mo_success, mo_fail );
431 template <typename T>
432 static inline T load64( T volatile const * pSrc, memory_order order ) CDS_NOEXCEPT
434 static_assert( sizeof(T) == 8, "Illegal operand size" );
435 assert( order == memory_order_relaxed
436 || order == memory_order_consume
437 || order == memory_order_acquire
438 || order == memory_order_seq_cst
441 assert( cds::details::is_aligned( pSrc, 8 ));
443 // Atomically loads 64bit value by SSE intrinsics
444 __m128i volatile v = _mm_loadl_epi64( (__m128i const *) pSrc );
445 fence_after_load( order );
446 return (T) v.m128i_i64[0];
450 template <typename T>
451 static inline T exchange64( T volatile * pDest, T v, memory_order order ) CDS_NOEXCEPT
453 static_assert( sizeof(T) == 8, "Illegal operand size" );
455 T cur = load64( pDest, memory_order_relaxed );
457 } while (!cas64_weak( pDest, cur, v, order, memory_order_relaxed ));
461 template <typename T>
462 static inline void store64( T volatile * pDest, T val, memory_order order ) CDS_NOEXCEPT
464 static_assert( sizeof(T) == 8, "Illegal operand size" );
465 assert( order == memory_order_relaxed
466 || order == memory_order_release
467 || order == memory_order_seq_cst
470 assert( cds::details::is_aligned( pDest, 8 ));
472 if ( order != memory_order_seq_cst ) {
474 v.m128i_i64[0] = val;
475 fence_before( order );
476 _mm_storel_epi64( (__m128i *) pDest, v );
479 exchange64( pDest, val, order );
484 //-----------------------------------------------------------------------------
485 // pointer primitives
486 //-----------------------------------------------------------------------------
488 template <typename T>
489 static inline T * exchange_ptr( T * volatile * pDest, T * v, memory_order order ) CDS_NOEXCEPT
491 static_assert( sizeof(T *) == sizeof(void *), "Illegal operand size" );
492 return (T *) _InterlockedExchange( (long volatile *) pDest, (uintptr_t) v );
493 //return (T *) _InterlockedExchangePointer( (void * volatile *) pDest, reinterpret_cast<void *>(v) );
496 template <typename T>
497 static inline void store_ptr( T * volatile * pDest, T * src, memory_order order ) CDS_NOEXCEPT
499 static_assert( sizeof(T *) == sizeof(void *), "Illegal operand size" );
500 assert( order == memory_order_relaxed
501 || order == memory_order_release
502 || order == memory_order_seq_cst
506 if ( order != memory_order_seq_cst ) {
507 fence_before( order );
511 exchange_ptr( pDest, src, order );
515 template <typename T>
516 static inline T * load_ptr( T * volatile const * pSrc, memory_order order ) CDS_NOEXCEPT
518 static_assert( sizeof(T *) == sizeof(void *), "Illegal operand size" );
519 assert( order == memory_order_relaxed
520 || order == memory_order_consume
521 || order == memory_order_acquire
522 || order == memory_order_seq_cst
527 fence_after_load( order );
531 template <typename T>
532 static inline bool cas_ptr_strong( T * volatile * pDest, T *& expected, T * desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
534 static_assert( sizeof(T *) == sizeof(void *), "Illegal operand size" );
536 // _InterlockedCompareExchangePointer behave as read-write memory barriers
538 expected = (T *) _InterlockedCompareExchange( (long volatile *) pDest, (uintptr_t) desired, (uintptr_t) prev );
539 return expected == prev;
542 template <typename T>
543 static inline bool cas_ptr_weak( T * volatile * pDest, T *& expected, T * desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
545 return cas_ptr_strong( pDest, expected, desired, mo_success, mo_fail );
547 }} // namespace vc::x86
549 #ifndef CDS_CXX11_INLINE_NAMESPACE_SUPPORT
550 using namespace vc::x86;
552 } // namespace platform
553 }} // namespace cds::cxx11_atomic
556 #endif // #ifndef __CDS_COMPILER_VC_X86_CXX11_ATOMIC_H