2 This file is a part of libcds - Concurrent Data Structures library
4 (C) Copyright Maxim Khizhinsky (libcds.dev@gmail.com) 2006-2017
6 Source code repo: http://github.com/khizmax/libcds/
7 Download: http://sourceforge.net/projects/libcds/files/
9 Redistribution and use in source and binary forms, with or without
10 modification, are permitted provided that the following conditions are met:
12 * Redistributions of source code must retain the above copyright notice, this
13 list of conditions and the following disclaimer.
15 * Redistributions in binary form must reproduce the above copyright notice,
16 this list of conditions and the following disclaimer in the documentation
17 and/or other materials provided with the distribution.
19 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
22 DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
23 FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
25 SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
26 CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
27 OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31 #ifndef CDSLIB_COMPILER_VC_AMD64_CXX11_ATOMIC_H
32 #define CDSLIB_COMPILER_VC_AMD64_CXX11_ATOMIC_H
35 #include <emmintrin.h> // for 128bit atomic load/store
36 #include <cds/details/is_aligned.h>
38 #pragma intrinsic( _InterlockedIncrement )
39 #pragma intrinsic( _InterlockedDecrement )
40 #pragma intrinsic( _InterlockedCompareExchange )
41 #pragma intrinsic( _InterlockedCompareExchangePointer )
42 #pragma intrinsic( _InterlockedCompareExchange16 )
43 #pragma intrinsic( _InterlockedCompareExchange64 )
44 #pragma intrinsic( _InterlockedExchange )
45 #pragma intrinsic( _InterlockedExchange64 )
46 #pragma intrinsic( _InterlockedExchangePointer )
47 #pragma intrinsic( _InterlockedExchangeAdd )
48 #pragma intrinsic( _InterlockedExchangeAdd64 )
49 //#pragma intrinsic( _InterlockedAnd )
50 //#pragma intrinsic( _InterlockedOr )
51 //#pragma intrinsic( _InterlockedXor )
52 //#pragma intrinsic( _InterlockedAnd64 )
53 //#pragma intrinsic( _InterlockedOr64 )
54 //#pragma intrinsic( _InterlockedXor64 )
55 #pragma intrinsic( _interlockedbittestandset )
57 # pragma intrinsic( _InterlockedCompareExchange8 )
58 # pragma intrinsic( _InterlockedExchange8 )
59 # pragma intrinsic( _InterlockedExchange16 )
63 namespace cds { namespace cxx11_atomic {
64 namespace platform { CDS_CXX11_INLINE_NAMESPACE namespace vc { CDS_CXX11_INLINE_NAMESPACE namespace amd64 {
66 static inline void fence_before( memory_order order ) CDS_NOEXCEPT
69 case memory_order_relaxed:
70 case memory_order_acquire:
71 case memory_order_consume:
73 case memory_order_release:
74 case memory_order_acq_rel:
75 CDS_COMPILER_RW_BARRIER;
77 case memory_order_seq_cst:
78 CDS_COMPILER_RW_BARRIER;
83 static inline void fence_after( memory_order order ) CDS_NOEXCEPT
86 case memory_order_acquire:
87 case memory_order_acq_rel:
88 CDS_COMPILER_RW_BARRIER;
90 case memory_order_relaxed:
91 case memory_order_consume:
92 case memory_order_release:
94 case memory_order_seq_cst:
95 CDS_COMPILER_RW_BARRIER;
100 static inline void full_fence()
102 // MS VC does not support inline assembler in C code.
103 // So, we use InterlockedExchange for full fence instead of mfence inst
105 _InterlockedExchange( &t, 0 );
108 static inline void fence_after_load(memory_order order) CDS_NOEXCEPT
111 case memory_order_relaxed:
112 case memory_order_release:
114 case memory_order_acquire:
115 case memory_order_acq_rel:
116 CDS_COMPILER_RW_BARRIER;
118 case memory_order_consume:
120 case memory_order_seq_cst:
127 //-----------------------------------------------------------------------------
129 //-----------------------------------------------------------------------------
130 static inline void thread_fence(memory_order order) CDS_NOEXCEPT
134 case memory_order_relaxed:
135 case memory_order_consume:
137 case memory_order_release:
138 case memory_order_acquire:
139 case memory_order_acq_rel:
140 CDS_COMPILER_RW_BARRIER;
142 case memory_order_seq_cst:
149 static inline void signal_fence(memory_order order) CDS_NOEXCEPT
151 // C++11: 29.8.8: only compiler optimization, no hardware instructions
154 case memory_order_relaxed:
156 case memory_order_consume:
157 case memory_order_release:
158 case memory_order_acquire:
159 case memory_order_acq_rel:
160 case memory_order_seq_cst:
161 CDS_COMPILER_RW_BARRIER;
167 //-----------------------------------------------------------------------------
168 // atomic flag primitives
169 //-----------------------------------------------------------------------------
171 typedef unsigned char atomic_flag_type;
172 static inline bool atomic_flag_tas( atomic_flag_type volatile * pFlag, memory_order /*order*/ ) CDS_NOEXCEPT
174 return _interlockedbittestandset( (long volatile *) pFlag, 0 ) != 0;
177 static inline void atomic_flag_clear( atomic_flag_type volatile * pFlag, memory_order order ) CDS_NOEXCEPT
179 assert( order != memory_order_acquire
180 && order != memory_order_acq_rel
183 fence_before( order );
185 fence_after( order );
188 //-----------------------------------------------------------------------------
190 //-----------------------------------------------------------------------------
193 # pragma warning(push)
194 // Disable warning C4800: 'char' : forcing value to bool 'true' or 'false' (performance warning)
195 # pragma warning( disable: 4800 )
197 template <typename T>
198 static inline bool cas8_strong( T volatile * pDest, T& expected, T desired, memory_order /*mo_success*/, memory_order /*mo_fail*/ ) CDS_NOEXCEPT
200 static_assert( sizeof(T) == 1, "Illegal size of operand" );
202 # if _MSC_VER >= 1600
205 expected = (T) _InterlockedCompareExchange8( (char volatile*) pDest, (char) desired, (char) expected );
206 return expected == prev;
209 unsigned int * pnDest = (unsigned int *)( ((unsigned __int64) pDest) & ~(unsigned __int64(3)));
210 unsigned int nOffset = ((unsigned __int64) pDest) & 3;
211 unsigned int nExpected;
212 unsigned int nDesired;
217 memcpy( reinterpret_cast<T *>(&nExpected) + nOffset, &expected, sizeof(T));
218 memcpy( reinterpret_cast<T *>(&nDesired) + nOffset, &desired, sizeof(T));
220 unsigned int nPrev = (unsigned int) _InterlockedCompareExchange( (long *) pnDest, (long) nDesired, (long) nExpected );
221 if ( nPrev == nExpected )
224 memcpy( &nByte, reinterpret_cast<T *>(&nPrev) + nOffset, sizeof(T));
225 if ( nByte != expected ) {
233 # pragma warning(pop)
236 template <typename T>
237 static inline bool cas8_weak( T volatile * pDest, T& expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
239 return cas8_strong( pDest, expected, desired, mo_success, mo_fail );
243 # pragma warning(push)
244 // Disable warning C4800: 'char' : forcing value to bool 'true' or 'false' (performance warning)
245 # pragma warning( disable: 4800 )
247 template <typename T>
248 static inline T exchange8( T volatile * pDest, T v, memory_order order ) CDS_NOEXCEPT
250 static_assert( sizeof(T) == 1, "Illegal size of operand" );
252 # if _MSC_VER >= 1600
254 return (T) _InterlockedExchange8( (char volatile *) pDest, (char) v );
257 do {} while ( !cas8_strong( pDest, expected, v, order, memory_order_relaxed ));
262 # pragma warning(pop)
265 template <typename T>
266 static inline void store8( T volatile * pDest, T src, memory_order order ) CDS_NOEXCEPT
268 static_assert( sizeof(T) == 1, "Illegal size of operand" );
269 assert( order == memory_order_relaxed
270 || order == memory_order_release
271 || order == memory_order_seq_cst
275 if ( order != memory_order_seq_cst ) {
276 fence_before( order );
280 exchange8( pDest, src, order );
284 template <typename T>
285 static inline T load8( T volatile const * pSrc, memory_order order ) CDS_NOEXCEPT
287 static_assert( sizeof(T) == 1, "Illegal size of operand" );
288 assert( order == memory_order_relaxed
289 || order == memory_order_consume
290 || order == memory_order_acquire
291 || order == memory_order_seq_cst
296 fence_after_load( order );
300 //-----------------------------------------------------------------------------
302 //-----------------------------------------------------------------------------
304 template <typename T>
305 static inline bool cas16_strong( T volatile * pDest, T& expected, T desired, memory_order /*mo_success*/, memory_order /*mo_fail*/ ) CDS_NOEXCEPT
307 static_assert( sizeof(T) == 2, "Illegal size of operand" );
308 assert( cds::details::is_aligned( pDest, 2 ));
310 // _InterlockedCompareExchange behave as read-write memory barriers
312 expected = (T) _InterlockedCompareExchange16( (short *) pDest, (short) desired, (short) expected );
313 return expected == prev;
316 template <typename T>
317 static inline bool cas16_weak( T volatile * pDest, T& expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
319 return cas16_strong( pDest, expected, desired, mo_success, mo_fail );
322 template <typename T>
323 static inline T exchange16( T volatile * pDest, T v, memory_order order ) CDS_NOEXCEPT
325 static_assert( sizeof(T) == 2, "Illegal size of operand" );
326 assert( cds::details::is_aligned( pDest, 2 ));
328 # if _MSC_VER >= 1600
330 return (T) _InterlockedExchange16( (short volatile *) pDest, (short) v );
333 do {} while ( !cas16_strong( pDest, expected, v, order, memory_order_relaxed ));
338 template <typename T>
339 static inline void store16( T volatile * pDest, T src, memory_order order ) CDS_NOEXCEPT
341 static_assert( sizeof(T) == 2, "Illegal size of operand" );
342 assert( order == memory_order_relaxed
343 || order == memory_order_release
344 || order == memory_order_seq_cst
347 assert( cds::details::is_aligned( pDest, 2 ));
349 if ( order != memory_order_seq_cst ) {
350 fence_before( order );
354 exchange16( pDest, src, order );
358 template <typename T>
359 static inline T load16( T volatile const * pSrc, memory_order order ) CDS_NOEXCEPT
361 static_assert( sizeof(T) == 2, "Illegal size of operand" );
362 assert( order == memory_order_relaxed
363 || order == memory_order_consume
364 || order == memory_order_acquire
365 || order == memory_order_seq_cst
368 assert( cds::details::is_aligned( pSrc, 2 ));
371 fence_after_load( order );
375 //-----------------------------------------------------------------------------
377 //-----------------------------------------------------------------------------
379 template <typename T>
380 static inline T exchange32( T volatile * pDest, T v, memory_order /*order*/ ) CDS_NOEXCEPT
382 static_assert( sizeof(T) == 4, "Illegal size of operand" );
383 assert( cds::details::is_aligned( pDest, 4 ));
385 return (T) _InterlockedExchange( (long *) pDest, (long) v );
388 template <typename T>
389 static inline void store32( T volatile * pDest, T src, memory_order order ) CDS_NOEXCEPT
391 static_assert( sizeof(T) == 4, "Illegal size of operand" );
392 assert( order == memory_order_relaxed
393 || order == memory_order_release
394 || order == memory_order_seq_cst
397 assert( cds::details::is_aligned( pDest, 4 ));
399 if ( order != memory_order_seq_cst ) {
400 fence_before( order );
404 exchange32( pDest, src, order );
408 template <typename T>
409 static inline T load32( T volatile const * pSrc, memory_order order ) CDS_NOEXCEPT
411 static_assert( sizeof(T) == 4, "Illegal size of operand" );
412 assert( order == memory_order_relaxed
413 || order == memory_order_consume
414 || order == memory_order_acquire
415 || order == memory_order_seq_cst
418 assert( cds::details::is_aligned( pSrc, 4 ));
421 fence_after_load( order );
425 template <typename T>
426 static inline bool cas32_strong( T volatile * pDest, T& expected, T desired, memory_order /*mo_success*/, memory_order /*mo_fail*/ ) CDS_NOEXCEPT
428 static_assert( sizeof(T) == 4, "Illegal size of operand" );
429 assert( cds::details::is_aligned( pDest, 4 ));
431 // _InterlockedCompareExchange behave as read-write memory barriers
433 expected = (T) _InterlockedCompareExchange( (long *) pDest, (long) desired, (long) expected );
434 return expected == prev;
437 template <typename T>
438 static inline bool cas32_weak( T volatile * pDest, T& expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
440 return cas32_strong( pDest, expected, desired, mo_success, mo_fail );
443 // fetch_xxx may be emulated via cas32
444 // If the platform has special fetch_xxx instruction
445 // then it should define CDS_ATOMIC_fetch32_xxx_defined macro
447 # define CDS_ATOMIC_fetch32_add_defined
448 template <typename T>
449 static inline T fetch32_add( T volatile * pDest, T v, memory_order /*order*/) CDS_NOEXCEPT
451 static_assert( sizeof(T) == 4, "Illegal size of operand" );
452 assert( cds::details::is_aligned( pDest, 4 ));
454 // _InterlockedExchangeAdd behave as read-write memory barriers
455 return (T) _InterlockedExchangeAdd( (long *) pDest, (long) v );
458 //-----------------------------------------------------------------------------
460 //-----------------------------------------------------------------------------
462 template <typename T>
463 static inline bool cas64_strong( T volatile * pDest, T& expected, T desired, memory_order /*mo_success*/, memory_order /*mo_fail*/ ) CDS_NOEXCEPT
465 static_assert( sizeof(T) == 8, "Illegal size of operand" );
466 assert( cds::details::is_aligned( pDest, 8 ));
468 // _InterlockedCompareExchange behave as read-write memory barriers
470 expected = (T) _InterlockedCompareExchange64( (__int64 *) pDest, (__int64) desired, (__int64) expected );
471 return expected == prev;
474 template <typename T>
475 static inline bool cas64_weak( T volatile * pDest, T& expected, T desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
477 return cas64_strong( pDest, expected, desired, mo_success, mo_fail );
480 template <typename T>
481 static inline T load64( T volatile const * pSrc, memory_order order ) CDS_NOEXCEPT
483 static_assert( sizeof(T) == 8, "Illegal size of operand" );
484 assert( order == memory_order_relaxed
485 || order == memory_order_consume
486 || order == memory_order_acquire
487 || order == memory_order_seq_cst
490 assert( cds::details::is_aligned( pSrc, 8 ));
493 fence_after_load( order );
498 template <typename T>
499 static inline T exchange64( T volatile * pDest, T v, memory_order order ) CDS_NOEXCEPT
501 static_assert( sizeof(T) == 8, "Illegal size of operand" );
503 T cur = load64( pDest, memory_order_relaxed );
505 } while (!cas64_weak( pDest, cur, v, order, memory_order_relaxed ));
509 template <typename T>
510 static inline void store64( T volatile * pDest, T val, memory_order order ) CDS_NOEXCEPT
512 static_assert( sizeof(T) == 8, "Illegal size of operand" );
513 assert( order == memory_order_relaxed
514 || order == memory_order_release
515 || order == memory_order_seq_cst
518 assert( cds::details::is_aligned( pDest, 8 ));
520 if ( order != memory_order_seq_cst ) {
521 fence_before( order );
525 exchange64( pDest, val, order );
529 # define CDS_ATOMIC_fetch64_add_defined
530 template <typename T>
531 static inline T fetch64_add( T volatile * pDest, T v, memory_order /*order*/) CDS_NOEXCEPT
533 static_assert( sizeof(T) == 8, "Illegal size of operand" );
534 assert( cds::details::is_aligned( pDest, 8 ));
536 // _InterlockedExchangeAdd64 behave as read-write memory barriers
537 return (T) _InterlockedExchangeAdd64( (__int64 *) pDest, (__int64) v );
540 //-----------------------------------------------------------------------------
541 // pointer primitives
542 //-----------------------------------------------------------------------------
544 template <typename T>
545 static inline T * exchange_ptr( T * volatile * pDest, T * v, memory_order /*order*/ ) CDS_NOEXCEPT
547 static_assert( sizeof(T *) == sizeof(void *), "Illegal size of operand" );
548 return (T *) _InterlockedExchangePointer( (void * volatile *) pDest, reinterpret_cast<void *>(v));
551 template <typename T>
552 static inline void store_ptr( T * volatile * pDest, T * src, memory_order order ) CDS_NOEXCEPT
554 static_assert( sizeof(T *) == sizeof(void *), "Illegal size of operand" );
555 assert( order == memory_order_relaxed
556 || order == memory_order_release
557 || order == memory_order_seq_cst
561 if ( order != memory_order_seq_cst ) {
562 fence_before( order );
566 exchange_ptr( pDest, src, order );
570 template <typename T>
571 static inline T * load_ptr( T * volatile const * pSrc, memory_order order ) CDS_NOEXCEPT
573 static_assert( sizeof(T *) == sizeof(void *), "Illegal size of operand" );
574 assert( order == memory_order_relaxed
575 || order == memory_order_consume
576 || order == memory_order_acquire
577 || order == memory_order_seq_cst
582 fence_after_load( order );
586 template <typename T>
587 static inline bool cas_ptr_strong( T * volatile * pDest, T *& expected, T * desired, memory_order /*mo_success*/, memory_order /*mo_fail*/ ) CDS_NOEXCEPT
589 static_assert( sizeof(T *) == sizeof(void *), "Illegal size of operand" );
591 // _InterlockedCompareExchangePointer behave as read-write memory barriers
593 expected = (T *) _InterlockedCompareExchangePointer( (void * volatile *) pDest, (void *) desired, (void *) expected );
594 return expected == prev;
597 template <typename T>
598 static inline bool cas_ptr_weak( T * volatile * pDest, T *& expected, T * desired, memory_order mo_success, memory_order mo_fail ) CDS_NOEXCEPT
600 return cas_ptr_strong( pDest, expected, desired, mo_success, mo_fail );
603 }} // namespace vc::amd64
605 #ifndef CDS_CXX11_INLINE_NAMESPACE_SUPPORT
606 using namespace vc::amd64;
608 } // namespace platform
609 }} // namespace cds::cxx11_atomic
612 #endif // #ifndef CDSLIB_COMPILER_VC_AMD64_CXX11_ATOMIC_H