github.com/moontrade/nogc@v0.1.7/alloc/mimalloc/mimalloc-atomic.h (about) 1 /* ---------------------------------------------------------------------------- 2 Copyright (c) 2018-2021 Microsoft Research, Daan Leijen 3 This is free software; you can redistribute it and/or modify it under the 4 terms of the MIT license. A copy of the license can be found in the file 5 "LICENSE" at the root of this distribution. 6 -----------------------------------------------------------------------------*/ 7 #pragma once 8 #ifndef MIMALLOC_ATOMIC_H 9 #define MIMALLOC_ATOMIC_H 10 11 // -------------------------------------------------------------------------------------------- 12 // Atomics 13 // We need to be portable between C, C++, and MSVC. 14 // We base the primitives on the C/C++ atomics and create a mimimal wrapper for MSVC in C compilation mode. 15 // This is why we try to use only `uintptr_t` and `<type>*` as atomic types. 16 // To gain better insight in the range of used atomics, we use explicitly named memory order operations 17 // instead of passing the memory order as a parameter. 18 // ----------------------------------------------------------------------------------------------- 19 20 #if defined(__cplusplus) 21 // Use C++ atomics 22 #include <atomic> 23 #define _Atomic(tp) std::atomic<tp> 24 #define mi_atomic(name) std::atomic_##name 25 #define mi_memory_order(name) std::memory_order_##name 26 #elif defined(_MSC_VER) 27 // Use MSVC C wrapper for C11 atomics 28 #define _Atomic(tp) tp 29 #define ATOMIC_VAR_INIT(x) x 30 #define mi_atomic(name) mi_atomic_##name 31 #define mi_memory_order(name) mi_memory_order_##name 32 #else 33 // Use C11 atomics 34 #include <stdatomic.h> 35 #define mi_atomic(name) atomic_##name 36 #define mi_memory_order(name) memory_order_##name 37 #endif 38 39 // Various defines for all used memory orders in mimalloc 40 #define mi_atomic_cas_weak(p,expected,desired,mem_success,mem_fail) \ 41 mi_atomic(compare_exchange_weak_explicit)(p,expected,desired,mem_success,mem_fail) 42 43 #define mi_atomic_cas_strong(p,expected,desired,mem_success,mem_fail) \ 44 mi_atomic(compare_exchange_strong_explicit)(p,expected,desired,mem_success,mem_fail) 45 46 #define mi_atomic_load_acquire(p) mi_atomic(load_explicit)(p,mi_memory_order(acquire)) 47 #define mi_atomic_load_relaxed(p) mi_atomic(load_explicit)(p,mi_memory_order(relaxed)) 48 #define mi_atomic_store_release(p,x) mi_atomic(store_explicit)(p,x,mi_memory_order(release)) 49 #define mi_atomic_store_relaxed(p,x) mi_atomic(store_explicit)(p,x,mi_memory_order(relaxed)) 50 #define mi_atomic_exchange_release(p,x) mi_atomic(exchange_explicit)(p,x,mi_memory_order(release)) 51 #define mi_atomic_exchange_acq_rel(p,x) mi_atomic(exchange_explicit)(p,x,mi_memory_order(acq_rel)) 52 #define mi_atomic_cas_weak_release(p,exp,des) mi_atomic_cas_weak(p,exp,des,mi_memory_order(release),mi_memory_order(relaxed)) 53 #define mi_atomic_cas_weak_acq_rel(p,exp,des) mi_atomic_cas_weak(p,exp,des,mi_memory_order(acq_rel),mi_memory_order(acquire)) 54 #define mi_atomic_cas_strong_release(p,exp,des) mi_atomic_cas_strong(p,exp,des,mi_memory_order(release),mi_memory_order(relaxed)) 55 #define mi_atomic_cas_strong_acq_rel(p,exp,des) mi_atomic_cas_strong(p,exp,des,mi_memory_order(acq_rel),mi_memory_order(acquire)) 56 57 #define mi_atomic_add_relaxed(p,x) mi_atomic(fetch_add_explicit)(p,x,mi_memory_order(relaxed)) 58 #define mi_atomic_sub_relaxed(p,x) mi_atomic(fetch_sub_explicit)(p,x,mi_memory_order(relaxed)) 59 #define mi_atomic_add_acq_rel(p,x) mi_atomic(fetch_add_explicit)(p,x,mi_memory_order(acq_rel)) 60 #define mi_atomic_sub_acq_rel(p,x) mi_atomic(fetch_sub_explicit)(p,x,mi_memory_order(acq_rel)) 61 #define mi_atomic_and_acq_rel(p,x) mi_atomic(fetch_and_explicit)(p,x,mi_memory_order(acq_rel)) 62 #define mi_atomic_or_acq_rel(p,x) mi_atomic(fetch_or_explicit)(p,x,mi_memory_order(acq_rel)) 63 64 #define mi_atomic_increment_relaxed(p) mi_atomic_add_relaxed(p,(uintptr_t)1) 65 #define mi_atomic_decrement_relaxed(p) mi_atomic_sub_relaxed(p,(uintptr_t)1) 66 #define mi_atomic_increment_acq_rel(p) mi_atomic_add_acq_rel(p,(uintptr_t)1) 67 #define mi_atomic_decrement_acq_rel(p) mi_atomic_sub_acq_rel(p,(uintptr_t)1) 68 69 static inline void mi_atomic_yield(void); 70 static inline intptr_t mi_atomic_addi(_Atomic(intptr_t)*p, intptr_t add); 71 static inline intptr_t mi_atomic_subi(_Atomic(intptr_t)*p, intptr_t sub); 72 73 74 #if defined(__cplusplus) || !defined(_MSC_VER) 75 76 // In C++/C11 atomics we have polymorphic atomics so can use the typed `ptr` variants (where `tp` is the type of atomic value) 77 // We use these macros so we can provide a typed wrapper in MSVC in C compilation mode as well 78 #define mi_atomic_load_ptr_acquire(tp,p) mi_atomic_load_acquire(p) 79 #define mi_atomic_load_ptr_relaxed(tp,p) mi_atomic_load_relaxed(p) 80 81 // In C++ we need to add casts to help resolve templates if NULL is passed 82 #if defined(__cplusplus) 83 #define mi_atomic_store_ptr_release(tp,p,x) mi_atomic_store_release(p,(tp*)x) 84 #define mi_atomic_store_ptr_relaxed(tp,p,x) mi_atomic_store_relaxed(p,(tp*)x) 85 #define mi_atomic_cas_ptr_weak_release(tp,p,exp,des) mi_atomic_cas_weak_release(p,exp,(tp*)des) 86 #define mi_atomic_cas_ptr_weak_acq_rel(tp,p,exp,des) mi_atomic_cas_weak_acq_rel(p,exp,(tp*)des) 87 #define mi_atomic_cas_ptr_strong_release(tp,p,exp,des) mi_atomic_cas_strong_release(p,exp,(tp*)des) 88 #define mi_atomic_exchange_ptr_release(tp,p,x) mi_atomic_exchange_release(p,(tp*)x) 89 #define mi_atomic_exchange_ptr_acq_rel(tp,p,x) mi_atomic_exchange_acq_rel(p,(tp*)x) 90 #else 91 #define mi_atomic_store_ptr_release(tp,p,x) mi_atomic_store_release(p,x) 92 #define mi_atomic_store_ptr_relaxed(tp,p,x) mi_atomic_store_relaxed(p,x) 93 #define mi_atomic_cas_ptr_weak_release(tp,p,exp,des) mi_atomic_cas_weak_release(p,exp,des) 94 #define mi_atomic_cas_ptr_weak_acq_rel(tp,p,exp,des) mi_atomic_cas_weak_acq_rel(p,exp,des) 95 #define mi_atomic_cas_ptr_strong_release(tp,p,exp,des) mi_atomic_cas_strong_release(p,exp,des) 96 #define mi_atomic_exchange_ptr_release(tp,p,x) mi_atomic_exchange_release(p,x) 97 #define mi_atomic_exchange_ptr_acq_rel(tp,p,x) mi_atomic_exchange_acq_rel(p,x) 98 #endif 99 100 // These are used by the statistics 101 static inline int64_t mi_atomic_addi64_relaxed(volatile int64_t* p, int64_t add) { 102 return mi_atomic(fetch_add_explicit)((_Atomic(int64_t)*)p, add, mi_memory_order(relaxed)); 103 } 104 static inline void mi_atomic_maxi64_relaxed(volatile int64_t* p, int64_t x) { 105 int64_t current = mi_atomic_load_relaxed((_Atomic(int64_t)*)p); 106 while (current < x && !mi_atomic_cas_weak_release((_Atomic(int64_t)*)p, ¤t, x)) { /* nothing */ }; 107 } 108 109 // Used by timers 110 #define mi_atomic_loadi64_acquire(p) mi_atomic(load_explicit)(p,mi_memory_order(acquire)) 111 #define mi_atomic_loadi64_relaxed(p) mi_atomic(load_explicit)(p,mi_memory_order(relaxed)) 112 #define mi_atomic_storei64_release(p,x) mi_atomic(store_explicit)(p,x,mi_memory_order(release)) 113 #define mi_atomic_storei64_relaxed(p,x) mi_atomic(store_explicit)(p,x,mi_memory_order(relaxed)) 114 115 116 117 #elif defined(_MSC_VER) 118 119 // MSVC C compilation wrapper that uses Interlocked operations to model C11 atomics. 120 #define WIN32_LEAN_AND_MEAN 121 #include <windows.h> 122 #include <intrin.h> 123 #ifdef _WIN64 124 typedef LONG64 msc_intptr_t; 125 #define MI_64(f) f##64 126 #else 127 typedef LONG msc_intptr_t; 128 #define MI_64(f) f 129 #endif 130 131 typedef enum mi_memory_order_e { 132 mi_memory_order_relaxed, 133 mi_memory_order_consume, 134 mi_memory_order_acquire, 135 mi_memory_order_release, 136 mi_memory_order_acq_rel, 137 mi_memory_order_seq_cst 138 } mi_memory_order; 139 140 static inline uintptr_t mi_atomic_fetch_add_explicit(_Atomic(uintptr_t)*p, uintptr_t add, mi_memory_order mo) { 141 (void)(mo); 142 return (uintptr_t)MI_64(_InterlockedExchangeAdd)((volatile msc_intptr_t*)p, (msc_intptr_t)add); 143 } 144 static inline uintptr_t mi_atomic_fetch_sub_explicit(_Atomic(uintptr_t)*p, uintptr_t sub, mi_memory_order mo) { 145 (void)(mo); 146 return (uintptr_t)MI_64(_InterlockedExchangeAdd)((volatile msc_intptr_t*)p, -((msc_intptr_t)sub)); 147 } 148 static inline uintptr_t mi_atomic_fetch_and_explicit(_Atomic(uintptr_t)*p, uintptr_t x, mi_memory_order mo) { 149 (void)(mo); 150 return (uintptr_t)MI_64(_InterlockedAnd)((volatile msc_intptr_t*)p, (msc_intptr_t)x); 151 } 152 static inline uintptr_t mi_atomic_fetch_or_explicit(_Atomic(uintptr_t)*p, uintptr_t x, mi_memory_order mo) { 153 (void)(mo); 154 return (uintptr_t)MI_64(_InterlockedOr)((volatile msc_intptr_t*)p, (msc_intptr_t)x); 155 } 156 static inline bool mi_atomic_compare_exchange_strong_explicit(_Atomic(uintptr_t)*p, uintptr_t* expected, uintptr_t desired, mi_memory_order mo1, mi_memory_order mo2) { 157 (void)(mo1); (void)(mo2); 158 uintptr_t read = (uintptr_t)MI_64(_InterlockedCompareExchange)((volatile msc_intptr_t*)p, (msc_intptr_t)desired, (msc_intptr_t)(*expected)); 159 if (read == *expected) { 160 return true; 161 } 162 else { 163 *expected = read; 164 return false; 165 } 166 } 167 static inline bool mi_atomic_compare_exchange_weak_explicit(_Atomic(uintptr_t)*p, uintptr_t* expected, uintptr_t desired, mi_memory_order mo1, mi_memory_order mo2) { 168 return mi_atomic_compare_exchange_strong_explicit(p, expected, desired, mo1, mo2); 169 } 170 static inline uintptr_t mi_atomic_exchange_explicit(_Atomic(uintptr_t)*p, uintptr_t exchange, mi_memory_order mo) { 171 (void)(mo); 172 return (uintptr_t)MI_64(_InterlockedExchange)((volatile msc_intptr_t*)p, (msc_intptr_t)exchange); 173 } 174 static inline void mi_atomic_thread_fence(mi_memory_order mo) { 175 (void)(mo); 176 _Atomic(uintptr_t)x = 0; 177 mi_atomic_exchange_explicit(&x, 1, mo); 178 } 179 static inline uintptr_t mi_atomic_load_explicit(_Atomic(uintptr_t) const* p, mi_memory_order mo) { 180 (void)(mo); 181 #if defined(_M_IX86) || defined(_M_X64) 182 return *p; 183 #else 184 uintptr_t x = *p; 185 if (mo > mi_memory_order_relaxed) { 186 while (!mi_atomic_compare_exchange_weak_explicit(p, &x, x, mo, mi_memory_order_relaxed)) { /* nothing */ }; 187 } 188 return x; 189 #endif 190 } 191 static inline void mi_atomic_store_explicit(_Atomic(uintptr_t)*p, uintptr_t x, mi_memory_order mo) { 192 (void)(mo); 193 #if defined(_M_IX86) || defined(_M_X64) 194 *p = x; 195 #else 196 mi_atomic_exchange_explicit(p, x, mo); 197 #endif 198 } 199 static inline int64_t mi_atomic_loadi64_explicit(_Atomic(int64_t)*p, mi_memory_order mo) { 200 (void)(mo); 201 #if defined(_M_X64) 202 return *p; 203 #else 204 int64_t old = *p; 205 int64_t x = old; 206 while ((old = InterlockedCompareExchange64(p, x, old)) != x) { 207 x = old; 208 } 209 return x; 210 #endif 211 } 212 static inline void mi_atomic_storei64_explicit(_Atomic(int64_t)*p, int64_t x, mi_memory_order mo) { 213 (void)(mo); 214 #if defined(x_M_IX86) || defined(_M_X64) 215 *p = x; 216 #else 217 InterlockedExchange64(p, x); 218 #endif 219 } 220 221 // These are used by the statistics 222 static inline int64_t mi_atomic_addi64_relaxed(volatile _Atomic(int64_t)*p, int64_t add) { 223 #ifdef _WIN64 224 return (int64_t)mi_atomic_addi((int64_t*)p, add); 225 #else 226 int64_t current; 227 int64_t sum; 228 do { 229 current = *p; 230 sum = current + add; 231 } while (_InterlockedCompareExchange64(p, sum, current) != current); 232 return current; 233 #endif 234 } 235 static inline void mi_atomic_maxi64_relaxed(volatile _Atomic(int64_t)*p, int64_t x) { 236 int64_t current; 237 do { 238 current = *p; 239 } while (current < x && _InterlockedCompareExchange64(p, x, current) != current); 240 } 241 242 // The pointer macros cast to `uintptr_t`. 243 #define mi_atomic_load_ptr_acquire(tp,p) (tp*)mi_atomic_load_acquire((_Atomic(uintptr_t)*)(p)) 244 #define mi_atomic_load_ptr_relaxed(tp,p) (tp*)mi_atomic_load_relaxed((_Atomic(uintptr_t)*)(p)) 245 #define mi_atomic_store_ptr_release(tp,p,x) mi_atomic_store_release((_Atomic(uintptr_t)*)(p),(uintptr_t)(x)) 246 #define mi_atomic_store_ptr_relaxed(tp,p,x) mi_atomic_store_relaxed((_Atomic(uintptr_t)*)(p),(uintptr_t)(x)) 247 #define mi_atomic_cas_ptr_weak_release(tp,p,exp,des) mi_atomic_cas_weak_release((_Atomic(uintptr_t)*)(p),(uintptr_t*)exp,(uintptr_t)des) 248 #define mi_atomic_cas_ptr_weak_acq_rel(tp,p,exp,des) mi_atomic_cas_weak_acq_rel((_Atomic(uintptr_t)*)(p),(uintptr_t*)exp,(uintptr_t)des) 249 #define mi_atomic_cas_ptr_strong_release(tp,p,exp,des) mi_atomic_cas_strong_release((_Atomic(uintptr_t)*)(p),(uintptr_t*)exp,(uintptr_t)des) 250 #define mi_atomic_exchange_ptr_release(tp,p,x) (tp*)mi_atomic_exchange_release((_Atomic(uintptr_t)*)(p),(uintptr_t)x) 251 #define mi_atomic_exchange_ptr_acq_rel(tp,p,x) (tp*)mi_atomic_exchange_acq_rel((_Atomic(uintptr_t)*)(p),(uintptr_t)x) 252 253 #define mi_atomic_loadi64_acquire(p) mi_atomic(loadi64_explicit)(p,mi_memory_order(acquire)) 254 #define mi_atomic_loadi64_relaxed(p) mi_atomic(loadi64_explicit)(p,mi_memory_order(relaxed)) 255 #define mi_atomic_storei64_release(p,x) mi_atomic(storei64_explicit)(p,x,mi_memory_order(release)) 256 #define mi_atomic_storei64_relaxed(p,x) mi_atomic(storei64_explicit)(p,x,mi_memory_order(relaxed)) 257 258 259 #endif 260 261 262 // Atomically add a signed value; returns the previous value. 263 static inline intptr_t mi_atomic_addi(_Atomic(intptr_t)*p, intptr_t add) { 264 return (intptr_t)mi_atomic_add_acq_rel((_Atomic(uintptr_t)*)p, (uintptr_t)add); 265 } 266 267 // Atomically subtract a signed value; returns the previous value. 268 static inline intptr_t mi_atomic_subi(_Atomic(intptr_t)*p, intptr_t sub) { 269 return (intptr_t)mi_atomic_addi(p, -sub); 270 } 271 272 // Yield 273 #if defined(__cplusplus) 274 #include <thread> 275 static inline void mi_atomic_yield(void) { 276 std::this_thread::yield(); 277 } 278 #elif defined(_WIN32) 279 #define WIN32_LEAN_AND_MEAN 280 #include <windows.h> 281 static inline void mi_atomic_yield(void) { 282 YieldProcessor(); 283 } 284 #elif defined(__SSE2__) 285 #include <emmintrin.h> 286 static inline void mi_atomic_yield(void) { 287 _mm_pause(); 288 } 289 #elif (defined(__GNUC__) || defined(__clang__)) && \ 290 (defined(__x86_64__) || defined(__i386__) || defined(__arm__) || defined(__armel__) || defined(__ARMEL__) || \ 291 defined(__aarch64__) || defined(__powerpc__) || defined(__ppc__) || defined(__PPC__)) 292 #if defined(__x86_64__) || defined(__i386__) 293 static inline void mi_atomic_yield(void) { 294 __asm__ volatile ("pause" ::: "memory"); 295 } 296 #elif defined(__aarch64__) 297 static inline void mi_atomic_yield(void) { 298 __asm__ volatile("wfe"); 299 } 300 #elif (defined(__arm__) && __ARM_ARCH__ >= 7) 301 static inline void mi_atomic_yield(void) { 302 __asm__ volatile("yield" ::: "memory"); 303 } 304 #elif defined(__powerpc__) || defined(__ppc__) || defined(__PPC__) 305 static inline void mi_atomic_yield(void) { 306 __asm__ __volatile__ ("or 27,27,27" ::: "memory"); 307 } 308 #elif defined(__armel__) || defined(__ARMEL__) 309 static inline void mi_atomic_yield(void) { 310 __asm__ volatile ("nop" ::: "memory"); 311 } 312 #endif 313 #elif defined(__sun) 314 // Fallback for other archs 315 #include <synch.h> 316 static inline void mi_atomic_yield(void) { 317 smt_pause(); 318 } 319 #elif defined(__wasi__) 320 #include <sched.h> 321 static inline void mi_atomic_yield(void) { 322 sched_yield(); 323 } 324 #else 325 #include <unistd.h> 326 static inline void mi_atomic_yield(void) { 327 sleep(0); 328 } 329 #endif 330 331 332 #endif // __MIMALLOC_ATOMIC_H