1 /* 2 * Copyright (c) 2008-2013 Apple Inc. All rights reserved. 3 * 4 * @APPLE_APACHE_LICENSE_HEADER_START@ 5 * 6 * Licensed under the Apache License, Version 2.0 (the "License"); 7 * you may not use this file except in compliance with the License. 8 * You may obtain a copy of the License at 9 * 10 * http://www.apache.org/licenses/LICENSE-2.0 11 * 12 * Unless required by applicable law or agreed to in writing, software 13 * distributed under the License is distributed on an "AS IS" BASIS, 14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 * See the License for the specific language governing permissions and 16 * limitations under the License. 17 * 18 * @APPLE_APACHE_LICENSE_HEADER_END@ 19 */ 20 21 /* 22 * IMPORTANT: This header file describes INTERNAL interfaces to libdispatch 23 * which are subject to change in future releases of Mac OS X. Any applications 24 * relying on these interfaces WILL break. 25 */ 26 27 #ifndef __DISPATCH_SHIMS_ATOMIC__ 28 #define __DISPATCH_SHIMS_ATOMIC__ 29 30 // generate error during codegen 31 #define _dispatch_atomic_unimplemented() \ 32 ({ __asm__(".err unimplemented"); }) 33 34 #pragma mark - 35 #pragma mark memory_order 36 37 typedef enum _dispatch_atomic_memory_order 38 { 39 _dispatch_atomic_memory_order_relaxed, 40 _dispatch_atomic_memory_order_consume, 41 _dispatch_atomic_memory_order_acquire, 42 _dispatch_atomic_memory_order_release, 43 _dispatch_atomic_memory_order_acq_rel, 44 _dispatch_atomic_memory_order_seq_cst, 45 } _dispatch_atomic_memory_order; 46 47 #if !DISPATCH_ATOMIC_UP 48 49 #define dispatch_atomic_memory_order_relaxed \ 50 _dispatch_atomic_memory_order_relaxed 51 #define dispatch_atomic_memory_order_acquire \ 52 _dispatch_atomic_memory_order_acquire 53 #define dispatch_atomic_memory_order_release \ 54 _dispatch_atomic_memory_order_release 55 #define dispatch_atomic_memory_order_acq_rel \ 56 _dispatch_atomic_memory_order_acq_rel 57 #define dispatch_atomic_memory_order_seq_cst \ 58 _dispatch_atomic_memory_order_seq_cst 59 60 #else // DISPATCH_ATOMIC_UP 61 62 #define dispatch_atomic_memory_order_relaxed \ 63 _dispatch_atomic_memory_order_relaxed 64 #define dispatch_atomic_memory_order_acquire \ 65 _dispatch_atomic_memory_order_relaxed 66 #define dispatch_atomic_memory_order_release \ 67 _dispatch_atomic_memory_order_relaxed 68 #define dispatch_atomic_memory_order_acq_rel \ 69 _dispatch_atomic_memory_order_relaxed 70 #define dispatch_atomic_memory_order_seq_cst \ 71 _dispatch_atomic_memory_order_relaxed 72 73 #endif // DISPATCH_ATOMIC_UP 74 75 #if __has_extension(c_generic_selections) 76 #define _dispatch_atomic_basetypeof(p) \ 77 typeof(*_Generic((p), \ 78 int*: (int*)(p), \ 79 volatile int*: (int*)(p), \ 80 unsigned int*: (unsigned int*)(p), \ 81 volatile unsigned int*: (unsigned int*)(p), \ 82 long*: (long*)(p), \ 83 volatile long*: (long*)(p), \ 84 unsigned long*: (unsigned long*)(p), \ 85 volatile unsigned long*: (unsigned long*)(p), \ 86 long long*: (long long*)(p), \ 87 volatile long long*: (long long*)(p), \ 88 unsigned long long*: (unsigned long long*)(p), \ 89 volatile unsigned long long*: (unsigned long long*)(p), \ 90 default: (void**)(p))) 91 #endif 92 93 /* clang lies on FreeBSD */ 94 #if !__FreeBSD__ && __has_extension(c_atomic) && __has_extension(c_generic_selections) 95 #pragma mark - 96 #pragma mark c11 97 98 #define _dispatch_atomic_c11_atomic(p) \ 99 _Generic((p), \ 100 int*: (_Atomic(int)*)(p), \ 101 volatile int*: (volatile _Atomic(int)*)(p), \ 102 unsigned int*: (_Atomic(unsigned int)*)(p), \ 103 volatile unsigned int*: (volatile _Atomic(unsigned int)*)(p), \ 104 long*: (_Atomic(long)*)(p), \ 105 volatile long*: (volatile _Atomic(long)*)(p), \ 106 unsigned long*: (_Atomic(unsigned long)*)(p), \ 107 volatile unsigned long*: (volatile _Atomic(unsigned long)*)(p), \ 108 long long*: (_Atomic(long long)*)(p), \ 109 volatile long long*: (volatile _Atomic(long long)*)(p), \ 110 unsigned long long*: (_Atomic(unsigned long long)*)(p), \ 111 volatile unsigned long long*: \ 112 (volatile _Atomic(unsigned long long)*)(p), \ 113 default: (volatile _Atomic(void*)*)(p)) 114 115 #define _dispatch_atomic_barrier(m) \ 116 ({ __c11_atomic_thread_fence(dispatch_atomic_memory_order_##m); }) 117 #define dispatch_atomic_load(p, m) \ 118 ({ _dispatch_atomic_basetypeof(p) _r = \ 119 __c11_atomic_load(_dispatch_atomic_c11_atomic(p), \ 120 dispatch_atomic_memory_order_##m); (typeof(*(p)))_r; }) 121 #define dispatch_atomic_store(p, v, m) \ 122 ({ _dispatch_atomic_basetypeof(p) _v = (v); \ 123 __c11_atomic_store(_dispatch_atomic_c11_atomic(p), _v, \ 124 dispatch_atomic_memory_order_##m); }) 125 #define dispatch_atomic_xchg(p, v, m) \ 126 ({ _dispatch_atomic_basetypeof(p) _v = (v), _r = \ 127 __c11_atomic_exchange(_dispatch_atomic_c11_atomic(p), _v, \ 128 dispatch_atomic_memory_order_##m); (typeof(*(p)))_r; }) 129 #define dispatch_atomic_cmpxchg(p, e, v, m) \ 130 ({ _dispatch_atomic_basetypeof(p) _v = (v), _r = (e); \ 131 __c11_atomic_compare_exchange_strong(_dispatch_atomic_c11_atomic(p), \ 132 &_r, _v, dispatch_atomic_memory_order_##m, \ 133 dispatch_atomic_memory_order_relaxed); }) 134 #define dispatch_atomic_cmpxchgv(p, e, v, g, m) \ 135 ({ _dispatch_atomic_basetypeof(p) _v = (v), _r = (e); _Bool _b = \ 136 __c11_atomic_compare_exchange_strong(_dispatch_atomic_c11_atomic(p), \ 137 &_r, _v, dispatch_atomic_memory_order_##m, \ 138 dispatch_atomic_memory_order_relaxed); *(g) = (typeof(*(p)))_r; _b; }) 139 #define dispatch_atomic_cmpxchgvw(p, e, v, g, m) \ 140 ({ _dispatch_atomic_basetypeof(p) _v = (v), _r = (e); _Bool _b = \ 141 __c11_atomic_compare_exchange_weak(_dispatch_atomic_c11_atomic(p), \ 142 &_r, _v, dispatch_atomic_memory_order_##m, \ 143 dispatch_atomic_memory_order_relaxed); *(g) = (typeof(*(p)))_r; _b; }) 144 #define _dispatch_atomic_c11_op(p, v, m, o, op) \ 145 ({ _dispatch_atomic_basetypeof(p) _v = (v), _r = \ 146 __c11_atomic_fetch_##o(_dispatch_atomic_c11_atomic(p), _v, \ 147 dispatch_atomic_memory_order_##m); (typeof(*(p)))(_r op _v); }) 148 #define _dispatch_atomic_c11_op_orig(p, v, m, o, op) \ 149 ({ _dispatch_atomic_basetypeof(p) _v = (v), _r = \ 150 __c11_atomic_fetch_##o(_dispatch_atomic_c11_atomic(p), _v, \ 151 dispatch_atomic_memory_order_##m); (typeof(*(p)))_r; }) 152 153 #define dispatch_atomic_add(p, v, m) \ 154 _dispatch_atomic_c11_op((p), (v), m, add, +) 155 #define dispatch_atomic_add_orig(p, v, m) \ 156 _dispatch_atomic_c11_op_orig((p), (v), m, add, +) 157 #define dispatch_atomic_sub(p, v, m) \ 158 _dispatch_atomic_c11_op((p), (v), m, sub, -) 159 #define dispatch_atomic_sub_orig(p, v, m) \ 160 _dispatch_atomic_c11_op_orig((p), (v), m, sub, -) 161 #define dispatch_atomic_and(p, v, m) \ 162 _dispatch_atomic_c11_op((p), (v), m, and, &) 163 #define dispatch_atomic_and_orig(p, v, m) \ 164 _dispatch_atomic_c11_op_orig((p), (v), m, and, &) 165 #define dispatch_atomic_or(p, v, m) \ 166 _dispatch_atomic_c11_op((p), (v), m, or, |) 167 #define dispatch_atomic_or_orig(p, v, m) \ 168 _dispatch_atomic_c11_op_orig((p), (v), m, or, |) 169 #define dispatch_atomic_xor(p, v, m) \ 170 _dispatch_atomic_c11_op((p), (v), m, xor, ^) 171 #define dispatch_atomic_xor_orig(p, v, m) \ 172 _dispatch_atomic_c11_op_orig((p), (v), m, xor, ^) 173 174 #elif __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 2) 175 #pragma mark - 176 #pragma mark gnu99 177 178 #define _dispatch_atomic_full_barrier() \ 179 __sync_synchronize() 180 #define _dispatch_atomic_barrier(m) \ 181 ({ switch(dispatch_atomic_memory_order_##m) { \ 182 case _dispatch_atomic_memory_order_relaxed: \ 183 break; \ 184 default: \ 185 _dispatch_atomic_full_barrier(); break; \ 186 } }) 187 // seq_cst: only emulate explicit store(seq_cst) -> load(seq_cst) 188 #define dispatch_atomic_load(p, m) \ 189 ({ typeof(*(p)) _r = *(p); \ 190 switch(dispatch_atomic_memory_order_##m) { \ 191 case _dispatch_atomic_memory_order_seq_cst: \ 192 _dispatch_atomic_barrier(m); /* fallthrough */ \ 193 case _dispatch_atomic_memory_order_relaxed: \ 194 break; \ 195 default: \ 196 _dispatch_atomic_unimplemented(); break; \ 197 } _r; }) 198 #define dispatch_atomic_store(p, v, m) \ 199 ({ switch(dispatch_atomic_memory_order_##m) { \ 200 case _dispatch_atomic_memory_order_release: \ 201 case _dispatch_atomic_memory_order_seq_cst: \ 202 _dispatch_atomic_barrier(m); /* fallthrough */ \ 203 case _dispatch_atomic_memory_order_relaxed: \ 204 *(p) = (v); break; \ 205 default: \ 206 _dispatch_atomic_unimplemented(); break; \ 207 } switch(dispatch_atomic_memory_order_##m) { \ 208 case _dispatch_atomic_memory_order_seq_cst: \ 209 _dispatch_atomic_barrier(m); break; \ 210 default: \ 211 break; \ 212 } }) 213 #if __has_builtin(__sync_swap) 214 #define dispatch_atomic_xchg(p, v, m) \ 215 ((typeof(*(p)))__sync_swap((p), (v))) 216 #else 217 #define dispatch_atomic_xchg(p, v, m) \ 218 ((typeof(*(p)))__sync_lock_test_and_set((p), (v))) 219 #endif 220 #define dispatch_atomic_cmpxchg(p, e, v, m) \ 221 __sync_bool_compare_and_swap((p), (e), (v)) 222 #define dispatch_atomic_cmpxchgv(p, e, v, g, m) \ 223 ({ typeof(*(g)) _e = (e), _r = \ 224 __sync_val_compare_and_swap((p), _e, (v)); \ 225 bool _b = (_e == _r); *(g) = _r; _b; }) 226 #define dispatch_atomic_cmpxchgvw(p, e, v, g, m) \ 227 dispatch_atomic_cmpxchgv((p), (e), (v), (g), m) 228 229 #define dispatch_atomic_add(p, v, m) \ 230 __sync_add_and_fetch((p), (v)) 231 #define dispatch_atomic_add_orig(p, v, m) \ 232 __sync_fetch_and_add((p), (v)) 233 #define dispatch_atomic_sub(p, v, m) \ 234 __sync_sub_and_fetch((p), (v)) 235 #define dispatch_atomic_sub_orig(p, v, m) \ 236 __sync_fetch_and_sub((p), (v)) 237 #define dispatch_atomic_and(p, v, m) \ 238 __sync_and_and_fetch((p), (v)) 239 #define dispatch_atomic_and_orig(p, v, m) \ 240 __sync_fetch_and_and((p), (v)) 241 #define dispatch_atomic_or(p, v, m) \ 242 __sync_or_and_fetch((p), (v)) 243 #define dispatch_atomic_or_orig(p, v, m) \ 244 __sync_fetch_and_or((p), (v)) 245 #define dispatch_atomic_xor(p, v, m) \ 246 __sync_xor_and_fetch((p), (v)) 247 #define dispatch_atomic_xor_orig(p, v, m) \ 248 __sync_fetch_and_xor((p), (v)) 249 250 #if defined(__x86_64__) || defined(__i386__) 251 // GCC emits nothing for __sync_synchronize() on x86_64 & i386 252 #undef _dispatch_atomic_full_barrier 253 #define _dispatch_atomic_full_barrier() \ 254 ({ __asm__ __volatile__( \ 255 "mfence" \ 256 : : : "memory"); }) 257 #undef dispatch_atomic_load 258 #define dispatch_atomic_load(p, m) \ 259 ({ switch(dispatch_atomic_memory_order_##m) { \ 260 case _dispatch_atomic_memory_order_seq_cst: \ 261 case _dispatch_atomic_memory_order_relaxed: \ 262 break; \ 263 default: \ 264 _dispatch_atomic_unimplemented(); break; \ 265 } *(p); }) 266 // xchg is faster than store + mfence 267 #undef dispatch_atomic_store 268 #define dispatch_atomic_store(p, v, m) \ 269 ({ switch(dispatch_atomic_memory_order_##m) { \ 270 case _dispatch_atomic_memory_order_relaxed: \ 271 case _dispatch_atomic_memory_order_release: \ 272 *(p) = (v); break; \ 273 case _dispatch_atomic_memory_order_seq_cst: \ 274 (void)dispatch_atomic_xchg((p), (v), m); break; \ 275 default:\ 276 _dispatch_atomic_unimplemented(); break; \ 277 } }) 278 #endif 279 280 #else 281 #error "Please upgrade to GCC 4.2 or newer." 282 #endif 283 284 #pragma mark - 285 #pragma mark generic 286 287 // assume atomic builtins provide barriers 288 #define dispatch_atomic_barrier(m) 289 // see comment in dispatch_once.c 290 #define dispatch_atomic_maximally_synchronizing_barrier() \ 291 _dispatch_atomic_barrier(seq_cst) 292 293 #define dispatch_atomic_load2o(p, f, m) \ 294 dispatch_atomic_load(&(p)->f, m) 295 #define dispatch_atomic_store2o(p, f, v, m) \ 296 dispatch_atomic_store(&(p)->f, (v), m) 297 #define dispatch_atomic_xchg2o(p, f, v, m) \ 298 dispatch_atomic_xchg(&(p)->f, (v), m) 299 #define dispatch_atomic_cmpxchg2o(p, f, e, v, m) \ 300 dispatch_atomic_cmpxchg(&(p)->f, (e), (v), m) 301 #define dispatch_atomic_cmpxchgv2o(p, f, e, v, g, m) \ 302 dispatch_atomic_cmpxchgv(&(p)->f, (e), (v), (g), m) 303 #define dispatch_atomic_cmpxchgvw2o(p, f, e, v, g, m) \ 304 dispatch_atomic_cmpxchgvw(&(p)->f, (e), (v), (g), m) 305 #define dispatch_atomic_add2o(p, f, v, m) \ 306 dispatch_atomic_add(&(p)->f, (v), m) 307 #define dispatch_atomic_add_orig2o(p, f, v, m) \ 308 dispatch_atomic_add_orig(&(p)->f, (v), m) 309 #define dispatch_atomic_sub2o(p, f, v, m) \ 310 dispatch_atomic_sub(&(p)->f, (v), m) 311 #define dispatch_atomic_sub_orig2o(p, f, v, m) \ 312 dispatch_atomic_sub_orig(&(p)->f, (v), m) 313 #define dispatch_atomic_and2o(p, f, v, m) \ 314 dispatch_atomic_and(&(p)->f, (v), m) 315 #define dispatch_atomic_and_orig2o(p, f, v, m) \ 316 dispatch_atomic_and_orig(&(p)->f, (v), m) 317 #define dispatch_atomic_or2o(p, f, v, m) \ 318 dispatch_atomic_or(&(p)->f, (v), m) 319 #define dispatch_atomic_or_orig2o(p, f, v, m) \ 320 dispatch_atomic_or_orig(&(p)->f, (v), m) 321 #define dispatch_atomic_xor2o(p, f, v, m) \ 322 dispatch_atomic_xor(&(p)->f, (v), m) 323 #define dispatch_atomic_xor_orig2o(p, f, v, m) \ 324 dispatch_atomic_xor_orig(&(p)->f, (v), m) 325 326 #define dispatch_atomic_inc(p, m) \ 327 dispatch_atomic_add((p), 1, m) 328 #define dispatch_atomic_inc_orig(p, m) \ 329 dispatch_atomic_add_orig((p), 1, m) 330 #define dispatch_atomic_inc2o(p, f, m) \ 331 dispatch_atomic_add2o(p, f, 1, m) 332 #define dispatch_atomic_inc_orig2o(p, f, m) \ 333 dispatch_atomic_add_orig2o(p, f, 1, m) 334 #define dispatch_atomic_dec(p, m) \ 335 dispatch_atomic_sub((p), 1, m) 336 #define dispatch_atomic_dec_orig(p, m) \ 337 dispatch_atomic_sub_orig((p), 1, m) 338 #define dispatch_atomic_dec2o(p, f, m) \ 339 dispatch_atomic_sub2o(p, f, 1, m) 340 #define dispatch_atomic_dec_orig2o(p, f, m) \ 341 dispatch_atomic_sub_orig2o(p, f, 1, m) 342 343 #define dispatch_atomic_tsx_xacq_cmpxchgv(p, e, v, g) \ 344 dispatch_atomic_cmpxchgv((p), (e), (v), (g), acquire) 345 #define dispatch_atomic_tsx_xrel_store(p, v) \ 346 dispatch_atomic_store(p, v, release) 347 #define dispatch_atomic_tsx_xacq_cmpxchgv2o(p, f, e, v, g) \ 348 dispatch_atomic_tsx_xacq_cmpxchgv(&(p)->f, (e), (v), (g)) 349 #define dispatch_atomic_tsx_xrel_store2o(p, f, v) \ 350 dispatch_atomic_tsx_xrel_store(&(p)->f, (v)) 351 352 #if defined(__x86_64__) || defined(__i386__) 353 #pragma mark - 354 #pragma mark x86 355 356 #undef dispatch_atomic_maximally_synchronizing_barrier 357 #ifdef __LP64__ 358 #define dispatch_atomic_maximally_synchronizing_barrier() \ 359 ({ unsigned long _clbr; __asm__ __volatile__( \ 360 "cpuid" \ 361 : "=a" (_clbr) : "0" (0) : "rbx", "rcx", "rdx", "cc", "memory"); }) 362 #else 363 #ifdef __llvm__ 364 #define dispatch_atomic_maximally_synchronizing_barrier() \ 365 ({ unsigned long _clbr; __asm__ __volatile__( \ 366 "cpuid" \ 367 : "=a" (_clbr) : "0" (0) : "ebx", "ecx", "edx", "cc", "memory"); }) 368 #else // gcc does not allow inline i386 asm to clobber ebx 369 #define dispatch_atomic_maximally_synchronizing_barrier() \ 370 ({ unsigned long _clbr; __asm__ __volatile__( \ 371 "pushl %%ebx\n\t" \ 372 "cpuid\n\t" \ 373 "popl %%ebx" \ 374 : "=a" (_clbr) : "0" (0) : "ecx", "edx", "cc", "memory"); }) 375 #endif 376 #endif 377 378 379 #endif 380 381 382 #endif // __DISPATCH_SHIMS_ATOMIC__ 383