5 #include "threads-model.h"
7 memory_order orders[6] = {
8 memory_order_relaxed, memory_order_consume, memory_order_acquire,
9 memory_order_release, memory_order_acq_rel, memory_order_seq_cst
12 /** Performs a read action.*/
13 uint64_t model_read_action(void * obj, memory_order ord) {
14 return model->switch_to_master(new ModelAction(ATOMIC_READ, ord, obj));
17 /** Performs a write action.*/
18 void model_write_action(void * obj, memory_order ord, uint64_t val) {
19 model->switch_to_master(new ModelAction(ATOMIC_WRITE, ord, obj, val));
22 /** Performs an init action. */
23 void model_init_action(void * obj, uint64_t val) {
24 model->switch_to_master(new ModelAction(ATOMIC_INIT, memory_order_relaxed, obj, val));
28 * Performs the read part of a RMW action. The next action must either be the
29 * write part of the RMW action or an explicit close out of the RMW action w/o
32 uint64_t model_rmwr_action(void *obj, memory_order ord) {
33 return model->switch_to_master(new ModelAction(ATOMIC_RMWR, ord, obj));
36 /** Performs the write part of a RMW action. */
37 void model_rmw_action(void *obj, memory_order ord, uint64_t val) {
38 model->switch_to_master(new ModelAction(ATOMIC_RMW, ord, obj, val));
41 /** Closes out a RMW action without doing a write. */
42 void model_rmwc_action(void *obj, memory_order ord) {
43 model->switch_to_master(new ModelAction(ATOMIC_RMWC, ord, obj));
46 /** Issues a fence operation. */
47 void model_fence_action(memory_order ord) {
48 model->switch_to_master(new ModelAction(ATOMIC_FENCE, ord, FENCE_LOCATION));
51 /* --- helper functions --- */
52 uint64_t model_rmwr_action_helper(void *obj, int atomic_index, const char *position) {
53 return model->switch_to_master(
54 new ModelAction(ATOMIC_RMWR, position, orders[atomic_index], obj)
58 void model_rmw_action_helper(void *obj, uint64_t val, int atomic_index, const char * position) {
59 model->switch_to_master(
60 new ModelAction(ATOMIC_RMW, position, orders[atomic_index], obj, val)
65 void model_rmw_action_helper(void *obj, uint64_t val, int atomic_index) {
66 model->switch_to_master(new ModelAction(ATOMIC_RMW, orders[atomic_index], obj, val));
70 void model_rmwc_action_helper(void *obj, int atomic_index, const char *position) {
71 model->switch_to_master(
72 new ModelAction(ATOMIC_RMWC, position, orders[atomic_index], obj)
77 void model_fence_action_helper(int atomic_index) {
78 model->switch_to_master(
79 new ModelAction(ATOMIC_FENCE, orders[atomic_index], FENCE_LOCATION)
85 void cds_atomic_init8(void * obj, uint8_t val, const char * position) {
86 model->switch_to_master(
87 new ModelAction(ATOMIC_INIT, position, memory_order_relaxed, obj, (uint64_t) val)
90 void cds_atomic_init16(void * obj, uint16_t val, const char * position) {
91 model->switch_to_master(
92 new ModelAction(ATOMIC_INIT, position, memory_order_relaxed, obj, (uint64_t) val)
95 void cds_atomic_init32(void * obj, uint32_t val, const char * position) {
96 model->switch_to_master(
97 new ModelAction(ATOMIC_INIT, position, memory_order_relaxed, obj, (uint64_t) val)
100 void cds_atomic_init64(void * obj, uint64_t val, const char * position) {
101 model->switch_to_master(
102 new ModelAction(ATOMIC_INIT, position, memory_order_relaxed, obj, val)
108 uint8_t cds_atomic_load8(void * obj, int atomic_index, const char * position) {
109 return (uint8_t) ( model->switch_to_master(
110 new ModelAction(ATOMIC_READ, position, orders[atomic_index], obj))
113 uint16_t cds_atomic_load16(void * obj, int atomic_index, const char * position) {
114 return (uint16_t) ( model->switch_to_master(
115 new ModelAction(ATOMIC_READ, position, orders[atomic_index], obj))
118 uint32_t cds_atomic_load32(void * obj, int atomic_index, const char * position) {
119 return (uint32_t) ( model->switch_to_master(
120 new ModelAction(ATOMIC_READ, position, orders[atomic_index], obj))
123 uint64_t cds_atomic_load64(void * obj, int atomic_index, const char * position) {
124 return model->switch_to_master(
125 new ModelAction(ATOMIC_READ, position, orders[atomic_index], obj)
130 void cds_atomic_store8(void * obj, uint8_t val, int atomic_index, const char * position) {
131 model->switch_to_master(
132 new ModelAction(ATOMIC_WRITE, position, orders[atomic_index], obj, (uint64_t) val)
135 void cds_atomic_store16(void * obj, uint16_t val, int atomic_index, const char * position) {
136 model->switch_to_master(
137 new ModelAction(ATOMIC_WRITE, position, orders[atomic_index], obj, (uint64_t) val)
140 void cds_atomic_store32(void * obj, uint32_t val, int atomic_index, const char * position) {
141 model->switch_to_master(
142 new ModelAction(ATOMIC_WRITE, position, orders[atomic_index], obj, (uint64_t) val)
145 void cds_atomic_store64(void * obj, uint64_t val, int atomic_index, const char * position) {
146 model->switch_to_master(
147 new ModelAction(ATOMIC_WRITE, position, orders[atomic_index], obj, val)
151 #define _ATOMIC_RMW_(__op__, size, addr, val, atomic_index, position) \
153 uint##size##_t _old = model_rmwr_action_helper(addr, atomic_index, position); \
154 uint##size##_t _copy = _old; \
155 uint##size##_t _val = val; \
157 model_rmw_action_helper(addr, (uint64_t) _copy, atomic_index, position); \
161 // cds atomic exchange
162 uint8_t cds_atomic_exchange8(void* addr, uint8_t val, int atomic_index, const char * position) {
163 _ATOMIC_RMW_( = , 8, addr, val, atomic_index, position);
165 uint16_t cds_atomic_exchange16(void* addr, uint16_t val, int atomic_index, const char * position) {
166 _ATOMIC_RMW_( = , 16, addr, val, atomic_index, position);
168 uint32_t cds_atomic_exchange32(void* addr, uint32_t val, int atomic_index, const char * position) {
169 _ATOMIC_RMW_( = , 32, addr, val, atomic_index, position);
171 uint64_t cds_atomic_exchange64(void* addr, uint64_t val, int atomic_index, const char * position) {
172 _ATOMIC_RMW_( = , 64, addr, val, atomic_index, position);
175 // cds atomic fetch add
176 uint8_t cds_atomic_fetch_add8(void* addr, uint8_t val, int atomic_index, const char * position) {
177 _ATOMIC_RMW_( += , 8, addr, val, atomic_index, position);
179 uint16_t cds_atomic_fetch_add16(void* addr, uint16_t val, int atomic_index, const char * position) {
180 _ATOMIC_RMW_( += , 16, addr, val, atomic_index, position);
182 uint32_t cds_atomic_fetch_add32(void* addr, uint32_t val, int atomic_index, const char * position) {
183 _ATOMIC_RMW_( += , 32, addr, val, atomic_index, position);
185 uint64_t cds_atomic_fetch_add64(void* addr, uint64_t val, int atomic_index, const char * position) {
186 _ATOMIC_RMW_( += , 64, addr, val, atomic_index, position);
189 // cds atomic fetch sub
190 uint8_t cds_atomic_fetch_sub8(void* addr, uint8_t val, int atomic_index, const char * position) {
191 _ATOMIC_RMW_( -= , 8, addr, val, atomic_index, position);
193 uint16_t cds_atomic_fetch_sub16(void* addr, uint16_t val, int atomic_index, const char * position) {
194 _ATOMIC_RMW_( -= , 16, addr, val, atomic_index, position);
196 uint32_t cds_atomic_fetch_sub32(void* addr, uint32_t val, int atomic_index, const char * position) {
197 _ATOMIC_RMW_( -= , 32, addr, val, atomic_index, position);
199 uint64_t cds_atomic_fetch_sub64(void* addr, uint64_t val, int atomic_index, const char * position) {
200 _ATOMIC_RMW_( -= , 64, addr, val, atomic_index, position);
203 // cds atomic fetch and
204 uint8_t cds_atomic_fetch_and8(void* addr, uint8_t val, int atomic_index, const char * position) {
205 _ATOMIC_RMW_( &= , 8, addr, val, atomic_index, position);
207 uint16_t cds_atomic_fetch_and16(void* addr, uint16_t val, int atomic_index, const char * position) {
208 _ATOMIC_RMW_( &= , 16, addr, val, atomic_index, position);
210 uint32_t cds_atomic_fetch_and32(void* addr, uint32_t val, int atomic_index, const char * position) {
211 _ATOMIC_RMW_( &= , 32, addr, val, atomic_index, position);
213 uint64_t cds_atomic_fetch_and64(void* addr, uint64_t val, int atomic_index, const char * position) {
214 _ATOMIC_RMW_( &= , 64, addr, val, atomic_index, position);
217 // cds atomic fetch or
218 uint8_t cds_atomic_fetch_or8(void* addr, uint8_t val, int atomic_index, const char * position) {
219 _ATOMIC_RMW_( |= , 8, addr, val, atomic_index, position);
221 uint16_t cds_atomic_fetch_or16(void* addr, uint16_t val, int atomic_index, const char * position) {
222 _ATOMIC_RMW_( |= , 16, addr, val, atomic_index, position);
224 uint32_t cds_atomic_fetch_or32(void* addr, uint32_t val, int atomic_index, const char * position) {
225 _ATOMIC_RMW_( |= , 32, addr, val, atomic_index, position);
227 uint64_t cds_atomic_fetch_or64(void* addr, uint64_t val, int atomic_index, const char * position) {
228 _ATOMIC_RMW_( |= , 64, addr, val, atomic_index, position);
231 // cds atomic fetch xor
232 uint8_t cds_atomic_fetch_xor8(void* addr, uint8_t val, int atomic_index, const char * position) {
233 _ATOMIC_RMW_( ^= , 8, addr, val, atomic_index, position);
235 uint16_t cds_atomic_fetch_xor16(void* addr, uint16_t val, int atomic_index, const char * position) {
236 _ATOMIC_RMW_( ^= , 16, addr, val, atomic_index, position);
238 uint32_t cds_atomic_fetch_xor32(void* addr, uint32_t val, int atomic_index, const char * position) {
239 _ATOMIC_RMW_( ^= , 32, addr, val, atomic_index, position);
241 uint64_t cds_atomic_fetch_xor64(void* addr, uint64_t val, int atomic_index, const char * position) {
242 _ATOMIC_RMW_( ^= , 64, addr, val, atomic_index, position);
245 // cds atomic compare and exchange
246 // In order to accomodate the LLVM PASS, the return values are not true or false.
248 #define _ATOMIC_CMPSWP_WEAK_ _ATOMIC_CMPSWP_
249 #define _ATOMIC_CMPSWP_(size, addr, expected, desired, atomic_index, position) \
251 uint##size##_t _desired = desired; \
252 uint##size##_t _expected = expected; \
253 uint##size##_t _old = model_rmwr_action_helper(addr, atomic_index, position); \
254 if (_old == _expected) { \
255 model_rmw_action_helper(addr, (uint64_t) _desired, atomic_index, position); return _expected; } \
257 model_rmwc_action_helper(addr, atomic_index, position); _expected = _old; return _old; } \
260 // atomic_compare_exchange version 1: the CmpOperand (corresponds to expected)
261 // extracted from LLVM IR is an integer type.
263 uint8_t cds_atomic_compare_exchange8_v1(void* addr, uint8_t expected, uint8_t desired,
264 int atomic_index_succ, int atomic_index_fail, const char *position )
266 _ATOMIC_CMPSWP_(8, addr, expected, desired,
267 atomic_index_succ, position);
269 uint16_t cds_atomic_compare_exchange16_v1(void* addr, uint16_t expected, uint16_t desired,
270 int atomic_index_succ, int atomic_index_fail, const char *position)
272 _ATOMIC_CMPSWP_(16, addr, expected, desired,
273 atomic_index_succ, position);
275 uint32_t cds_atomic_compare_exchange32_v1(void* addr, uint32_t expected, uint32_t desired,
276 int atomic_index_succ, int atomic_index_fail, const char *position)
278 _ATOMIC_CMPSWP_(32, addr, expected, desired,
279 atomic_index_succ, position);
281 uint64_t cds_atomic_compare_exchange64_v1(void* addr, uint64_t expected, uint64_t desired,
282 int atomic_index_succ, int atomic_index_fail, const char *position)
284 _ATOMIC_CMPSWP_(64, addr, expected, desired,
285 atomic_index_succ, position);
288 // atomic_compare_exchange version 2
289 bool cds_atomic_compare_exchange8_v2(void* addr, uint8_t* expected, uint8_t desired,
290 int atomic_index_succ, int atomic_index_fail, const char *position )
292 uint8_t ret = cds_atomic_compare_exchange8_v1(addr, *expected,
293 desired, atomic_index_succ, atomic_index_fail, position);
294 if (ret == *expected) return true;
297 bool cds_atomic_compare_exchange16_v2(void* addr, uint16_t* expected, uint16_t desired,
298 int atomic_index_succ, int atomic_index_fail, const char *position)
300 uint16_t ret = cds_atomic_compare_exchange16_v1(addr, *expected,
301 desired, atomic_index_succ, atomic_index_fail, position);
302 if (ret == *expected) return true;
305 bool cds_atomic_compare_exchange32_v2(void* addr, uint32_t* expected, uint32_t desired,
306 int atomic_index_succ, int atomic_index_fail, const char *position)
308 uint32_t ret = cds_atomic_compare_exchange32_v1(addr, *expected,
309 desired, atomic_index_succ, atomic_index_fail, position);
310 if (ret == *expected) return true;
313 bool cds_atomic_compare_exchange64_v2(void* addr, uint64_t* expected, uint64_t desired,
314 int atomic_index_succ, int atomic_index_fail, const char *position)
316 uint64_t ret = cds_atomic_compare_exchange64_v1(addr, *expected,
317 desired, atomic_index_succ, atomic_index_fail, position);
318 if (ret == *expected) return true;
323 // cds atomic thread fence
325 void cds_atomic_thread_fence(int atomic_index, const char * position) {
326 model->switch_to_master(
327 new ModelAction(ATOMIC_FENCE, position, orders[atomic_index], FENCE_LOCATION)
332 #define _ATOMIC_CMPSWP_( __a__, __e__, __m__, __x__ ) \
333 ({ volatile __typeof__((__a__)->__f__)* __p__ = & ((__a__)->__f__); \
334 __typeof__(__e__) __q__ = (__e__); \
335 __typeof__(__m__) __v__ = (__m__); \
337 __typeof__((__a__)->__f__) __t__=(__typeof__((__a__)->__f__)) model_rmwr_action((void *)__p__, __x__); \
338 if (__t__ == * __q__ ) { \
339 model_rmw_action((void *)__p__, __x__, (uint64_t) __v__); __r__ = true; } \
340 else { model_rmwc_action((void *)__p__, __x__); *__q__ = __t__; __r__ = false;} \
343 #define _ATOMIC_FENCE_( __x__ ) \
344 ({ model_fence_action(__x__);})
349 #define _ATOMIC_MODIFY_( __a__, __o__, __m__, __x__ ) \
350 ({ volatile __typeof__((__a__)->__f__)* __p__ = & ((__a__)->__f__); \
351 __typeof__((__a__)->__f__) __old__=(__typeof__((__a__)->__f__)) model_rmwr_action((void *)__p__, __x__); \
352 __typeof__(__m__) __v__ = (__m__); \
353 __typeof__((__a__)->__f__) __copy__= __old__; \
354 __copy__ __o__ __v__; \
355 model_rmw_action((void *)__p__, __x__, (uint64_t) __copy__); \
356 __old__ = __old__; Silence clang (-Wunused-value) \