github.com/pidato/unsafe@v0.1.4/memory/rpmalloc/rpmalloc.h (about) 1 /* rpmalloc.h - Memory allocator - Public Domain - 2016 Mattias Jansson 2 * 3 * This library provides a cross-platform lock free thread caching malloc implementation in C11. 4 * The latest source code is always available at 5 * 6 * https://github.com/mjansson/rpmalloc 7 * 8 * This library is put in the public domain; you can redistribute it and/or modify it without any restrictions. 9 * 10 */ 11 12 #pragma once 13 14 #include <stddef.h> 15 16 #ifdef __cplusplus 17 extern "C" { 18 #endif 19 20 #if defined(__clang__) || defined(__GNUC__) 21 # define RPMALLOC_EXPORT __attribute__((visibility("default"))) 22 # define RPMALLOC_ALLOCATOR 23 # if (defined(__clang_major__) && (__clang_major__ < 4)) || (defined(__GNUC__) && defined(ENABLE_PRELOAD) && ENABLE_PRELOAD) 24 # define RPMALLOC_ATTRIB_MALLOC 25 # define RPMALLOC_ATTRIB_ALLOC_SIZE(size) 26 # define RPMALLOC_ATTRIB_ALLOC_SIZE2(count, size) 27 # else 28 # define RPMALLOC_ATTRIB_MALLOC __attribute__((__malloc__)) 29 # define RPMALLOC_ATTRIB_ALLOC_SIZE(size) __attribute__((alloc_size(size))) 30 # define RPMALLOC_ATTRIB_ALLOC_SIZE2(count, size) __attribute__((alloc_size(count, size))) 31 # endif 32 # define RPMALLOC_CDECL 33 #elif defined(_MSC_VER) 34 # define RPMALLOC_EXPORT 35 # define RPMALLOC_ALLOCATOR __declspec(allocator) __declspec(restrict) 36 # define RPMALLOC_ATTRIB_MALLOC 37 # define RPMALLOC_ATTRIB_ALLOC_SIZE(size) 38 # define RPMALLOC_ATTRIB_ALLOC_SIZE2(count,size) 39 # define RPMALLOC_CDECL __cdecl 40 #else 41 # define RPMALLOC_EXPORT 42 # define RPMALLOC_ALLOCATOR 43 # define RPMALLOC_ATTRIB_MALLOC 44 # define RPMALLOC_ATTRIB_ALLOC_SIZE(size) 45 # define RPMALLOC_ATTRIB_ALLOC_SIZE2(count,size) 46 # define RPMALLOC_CDECL 47 #endif 48 49 //! Define RPMALLOC_CONFIGURABLE to enable configuring sizes. Will introduce 50 // a very small overhead due to some size calculations not being compile time constants 51 #ifndef RPMALLOC_CONFIGURABLE 52 #define RPMALLOC_CONFIGURABLE 0 53 #endif 54 55 //! Define RPMALLOC_FIRST_CLASS_HEAPS to enable heap based API (rpmalloc_heap_* functions). 56 // Will introduce a very small overhead to track fully allocated spans in heaps 57 #ifndef RPMALLOC_FIRST_CLASS_HEAPS 58 #define RPMALLOC_FIRST_CLASS_HEAPS 1 59 #endif 60 61 //! Flag to rpaligned_realloc to not preserve content in reallocation 62 #define RPMALLOC_NO_PRESERVE 1 63 //! Flag to rpaligned_realloc to fail and return null pointer if grow cannot be done in-place, 64 // in which case the original pointer is still valid (just like a call to realloc which failes to allocate 65 // a new block). 66 #define RPMALLOC_GROW_OR_FAIL 2 67 68 typedef struct rpmalloc_global_statistics_t { 69 //! Current amount of virtual memory mapped, all of which might not have been committed (only if ENABLE_STATISTICS=1) 70 size_t mapped; 71 //! Peak amount of virtual memory mapped, all of which might not have been committed (only if ENABLE_STATISTICS=1) 72 size_t mapped_peak; 73 //! Current amount of memory in global caches for small and medium sizes (<32KiB) 74 size_t cached; 75 //! Current amount of memory allocated in huge allocations, i.e larger than LARGE_SIZE_LIMIT which is 2MiB by default (only if ENABLE_STATISTICS=1) 76 size_t huge_alloc; 77 //! Peak amount of memory allocated in huge allocations, i.e larger than LARGE_SIZE_LIMIT which is 2MiB by default (only if ENABLE_STATISTICS=1) 78 size_t huge_alloc_peak; 79 //! Total amount of memory mapped since initialization (only if ENABLE_STATISTICS=1) 80 size_t mapped_total; 81 //! Total amount of memory unmapped since initialization (only if ENABLE_STATISTICS=1) 82 size_t unmapped_total; 83 } rpmalloc_global_statistics_t; 84 85 typedef struct rpmalloc_thread_statistics_t { 86 //! Current number of bytes available in thread size class caches for small and medium sizes (<32KiB) 87 size_t sizecache; 88 //! Current number of bytes available in thread span caches for small and medium sizes (<32KiB) 89 size_t spancache; 90 //! Total number of bytes transitioned from thread cache to global cache (only if ENABLE_STATISTICS=1) 91 size_t thread_to_global; 92 //! Total number of bytes transitioned from global cache to thread cache (only if ENABLE_STATISTICS=1) 93 size_t global_to_thread; 94 //! Per span count statistics (only if ENABLE_STATISTICS=1) 95 struct { 96 //! Currently used number of spans 97 size_t current; 98 //! High water mark of spans used 99 size_t peak; 100 //! Number of spans transitioned to global cache 101 size_t to_global; 102 //! Number of spans transitioned from global cache 103 size_t from_global; 104 //! Number of spans transitioned to thread cache 105 size_t to_cache; 106 //! Number of spans transitioned from thread cache 107 size_t from_cache; 108 //! Number of spans transitioned to reserved state 109 size_t to_reserved; 110 //! Number of spans transitioned from reserved state 111 size_t from_reserved; 112 //! Number of raw memory map calls (not hitting the reserve spans but resulting in actual OS mmap calls) 113 size_t map_calls; 114 } span_use[64]; 115 //! Per size class statistics (only if ENABLE_STATISTICS=1) 116 struct { 117 //! Current number of allocations 118 size_t alloc_current; 119 //! Peak number of allocations 120 size_t alloc_peak; 121 //! Total number of allocations 122 size_t alloc_total; 123 //! Total number of frees 124 size_t free_total; 125 //! Number of spans transitioned to cache 126 size_t spans_to_cache; 127 //! Number of spans transitioned from cache 128 size_t spans_from_cache; 129 //! Number of spans transitioned from reserved state 130 size_t spans_from_reserved; 131 //! Number of raw memory map calls (not hitting the reserve spans but resulting in actual OS mmap calls) 132 size_t map_calls; 133 } size_use[128]; 134 } rpmalloc_thread_statistics_t; 135 136 typedef struct rpmalloc_config_t { 137 //! Map memory pages for the given number of bytes. The returned address MUST be 138 // aligned to the rpmalloc span size, which will always be a power of two. 139 // Optionally the function can store an alignment offset in the offset variable 140 // in case it performs alignment and the returned pointer is offset from the 141 // actual start of the memory region due to this alignment. The alignment offset 142 // will be passed to the memory unmap function. The alignment offset MUST NOT be 143 // larger than 65535 (storable in an uint16_t), if it is you must use natural 144 // alignment to shift it into 16 bits. If you set a memory_map function, you 145 // must also set a memory_unmap function or else the default implementation will 146 // be used for both. This function must be thread safe, it can be called by 147 // multiple threads simultaneously. 148 void* (*memory_map)(size_t size, size_t* offset); 149 //! Unmap the memory pages starting at address and spanning the given number of bytes. 150 // If release is set to non-zero, the unmap is for an entire span range as returned by 151 // a previous call to memory_map and that the entire range should be released. The 152 // release argument holds the size of the entire span range. If release is set to 0, 153 // the unmap is a partial decommit of a subset of the mapped memory range. 154 // If you set a memory_unmap function, you must also set a memory_map function or 155 // else the default implementation will be used for both. This function must be thread 156 // safe, it can be called by multiple threads simultaneously. 157 void (*memory_unmap)(void* address, size_t size, size_t offset, size_t release); 158 //! Called when an assert fails, if asserts are enabled. Will use the standard assert() 159 // if this is not set. 160 void (*error_callback)(const char* message); 161 //! Called when a call to map memory pages fails (out of memory). If this callback is 162 // not set or returns zero the library will return a null pointer in the allocation 163 // call. If this callback returns non-zero the map call will be retried. The argument 164 // passed is the number of bytes that was requested in the map call. Only used if 165 // the default system memory map function is used (memory_map callback is not set). 166 int (*map_fail_callback)(size_t size); 167 //! Size of memory pages. The page size MUST be a power of two. All memory mapping 168 // requests to memory_map will be made with size set to a multiple of the page size. 169 // Used if RPMALLOC_CONFIGURABLE is defined to 1, otherwise system page size is used. 170 size_t page_size; 171 //! Size of a span of memory blocks. MUST be a power of two, and in [4096,262144] 172 // range (unless 0 - set to 0 to use the default span size). Used if RPMALLOC_CONFIGURABLE 173 // is defined to 1. 174 size_t span_size; 175 //! Number of spans to map at each request to map new virtual memory blocks. This can 176 // be used to minimize the system call overhead at the cost of virtual memory address 177 // space. The extra mapped pages will not be written until actually used, so physical 178 // committed memory should not be affected in the default implementation. Will be 179 // aligned to a multiple of spans that match memory page size in case of huge pages. 180 size_t span_map_count; 181 //! Enable use of large/huge pages. If this flag is set to non-zero and page size is 182 // zero, the allocator will try to enable huge pages and auto detect the configuration. 183 // If this is set to non-zero and page_size is also non-zero, the allocator will 184 // assume huge pages have been configured and enabled prior to initializing the 185 // allocator. 186 // For Windows, see https://docs.microsoft.com/en-us/windows/desktop/memory/large-page-support 187 // For Linux, see https://www.kernel.org/doc/Documentation/vm/hugetlbpage.txt 188 int enable_huge_pages; 189 //! Respectively allocated pages and huge allocated pages names for systems 190 // supporting it to be able to distinguish among anonymous regions. 191 const char *page_name; 192 const char *huge_page_name; 193 } rpmalloc_config_t; 194 195 //! Initialize allocator with default configuration 196 RPMALLOC_EXPORT int 197 rpmalloc_initialize(void); 198 199 //! Initialize allocator with given configuration 200 RPMALLOC_EXPORT int 201 rpmalloc_initialize_config(const rpmalloc_config_t* config); 202 203 //! Get allocator configuration 204 RPMALLOC_EXPORT const rpmalloc_config_t* 205 rpmalloc_config(void); 206 207 //! Finalize allocator 208 RPMALLOC_EXPORT void 209 rpmalloc_finalize(void); 210 211 //! Initialize allocator for calling thread 212 RPMALLOC_EXPORT void 213 rpmalloc_thread_initialize(void); 214 215 //! Finalize allocator for calling thread 216 RPMALLOC_EXPORT void 217 rpmalloc_thread_finalize(int release_caches); 218 219 //! Perform deferred deallocations pending for the calling thread heap 220 RPMALLOC_EXPORT void 221 rpmalloc_thread_collect(void); 222 223 //! Query if allocator is initialized for calling thread 224 RPMALLOC_EXPORT int 225 rpmalloc_is_thread_initialized(void); 226 227 //! Get per-thread statistics 228 RPMALLOC_EXPORT void 229 rpmalloc_thread_statistics(rpmalloc_thread_statistics_t* stats); 230 231 //! Get global statistics 232 RPMALLOC_EXPORT void 233 rpmalloc_global_statistics(rpmalloc_global_statistics_t* stats); 234 235 //! Dump all statistics in human readable format to file (should be a FILE*) 236 RPMALLOC_EXPORT void 237 rpmalloc_dump_statistics(void* file); 238 239 //! Allocate a memory block of at least the given size 240 RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void* 241 rpmalloc(size_t size) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE(1); 242 243 //! Free the given memory block 244 RPMALLOC_EXPORT void 245 rpfree(void* ptr); 246 247 //! Allocate a memory block of at least the given size and zero initialize it 248 RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void* 249 rpcalloc(size_t num, size_t size) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE2(1, 2); 250 251 //! Reallocate the given block to at least the given size 252 RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void* 253 rprealloc(void* ptr, size_t size) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE(2); 254 255 //! Reallocate the given block to at least the given size and alignment, 256 // with optional control flags (see RPMALLOC_NO_PRESERVE). 257 // Alignment must be a power of two and a multiple of sizeof(void*), 258 // and should ideally be less than memory page size. A caveat of rpmalloc 259 // internals is that this must also be strictly less than the span size (default 64KiB) 260 RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void* 261 rpaligned_realloc(void* ptr, size_t alignment, size_t size, size_t oldsize, unsigned int flags) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE(3); 262 263 //! Allocate a memory block of at least the given size and alignment. 264 // Alignment must be a power of two and a multiple of sizeof(void*), 265 // and should ideally be less than memory page size. A caveat of rpmalloc 266 // internals is that this must also be strictly less than the span size (default 64KiB) 267 RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void* 268 rpaligned_alloc(size_t alignment, size_t size) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE(2); 269 270 //! Allocate a memory block of at least the given size and alignment, and zero initialize it. 271 // Alignment must be a power of two and a multiple of sizeof(void*), 272 // and should ideally be less than memory page size. A caveat of rpmalloc 273 // internals is that this must also be strictly less than the span size (default 64KiB) 274 RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void* 275 rpaligned_calloc(size_t alignment, size_t num, size_t size) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE2(2, 3); 276 277 //! Allocate a memory block of at least the given size and alignment. 278 // Alignment must be a power of two and a multiple of sizeof(void*), 279 // and should ideally be less than memory page size. A caveat of rpmalloc 280 // internals is that this must also be strictly less than the span size (default 64KiB) 281 RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void* 282 rpmemalign(size_t alignment, size_t size) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE(2); 283 284 //! Allocate a memory block of at least the given size and alignment. 285 // Alignment must be a power of two and a multiple of sizeof(void*), 286 // and should ideally be less than memory page size. A caveat of rpmalloc 287 // internals is that this must also be strictly less than the span size (default 64KiB) 288 RPMALLOC_EXPORT int 289 rpposix_memalign(void** memptr, size_t alignment, size_t size); 290 291 //! Query the usable size of the given memory block (from given pointer to the end of block) 292 RPMALLOC_EXPORT size_t 293 rpmalloc_usable_size(void* ptr); 294 295 //! Dummy empty function for forcing linker symbol inclusion 296 RPMALLOC_EXPORT void 297 rpmalloc_linker_reference(void); 298 299 #if RPMALLOC_FIRST_CLASS_HEAPS 300 301 //! Heap type 302 typedef struct heap_t rpmalloc_heap_t; 303 304 //! Acquire a new heap. Will reuse existing released heaps or allocate memory for a new heap 305 // if none available. Heap API is implemented with the strict assumption that only one single 306 // thread will call heap functions for a given heap at any given time, no functions are thread safe. 307 RPMALLOC_EXPORT rpmalloc_heap_t* 308 rpmalloc_heap_acquire(void); 309 310 //! Release a heap (does NOT free the memory allocated by the heap, use rpmalloc_heap_free_all before destroying the heap). 311 // Releasing a heap will enable it to be reused by other threads. Safe to pass a null pointer. 312 RPMALLOC_EXPORT void 313 rpmalloc_heap_release(rpmalloc_heap_t* heap); 314 315 //! Allocate a memory block of at least the given size using the given heap. 316 RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void* 317 rpmalloc_heap_alloc(rpmalloc_heap_t* heap, size_t size) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE(2); 318 319 //! Allocate a memory block of at least the given size using the given heap. The returned 320 // block will have the requested alignment. Alignment must be a power of two and a multiple of sizeof(void*), 321 // and should ideally be less than memory page size. A caveat of rpmalloc 322 // internals is that this must also be strictly less than the span size (default 64KiB). 323 RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void* 324 rpmalloc_heap_aligned_alloc(rpmalloc_heap_t* heap, size_t alignment, size_t size) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE(3); 325 326 //! Allocate a memory block of at least the given size using the given heap and zero initialize it. 327 RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void* 328 rpmalloc_heap_calloc(rpmalloc_heap_t* heap, size_t num, size_t size) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE2(2, 3); 329 330 //! Allocate a memory block of at least the given size using the given heap and zero initialize it. The returned 331 // block will have the requested alignment. Alignment must either be zero, or a power of two and a multiple of sizeof(void*), 332 // and should ideally be less than memory page size. A caveat of rpmalloc 333 // internals is that this must also be strictly less than the span size (default 64KiB). 334 RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void* 335 rpmalloc_heap_aligned_calloc(rpmalloc_heap_t* heap, size_t alignment, size_t num, size_t size) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE2(2, 3); 336 337 //! Reallocate the given block to at least the given size. The memory block MUST be allocated 338 // by the same heap given to this function. 339 RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void* 340 rpmalloc_heap_realloc(rpmalloc_heap_t* heap, void* ptr, size_t size, unsigned int flags) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE(3); 341 342 //! Reallocate the given block to at least the given size. The memory block MUST be allocated 343 // by the same heap given to this function. The returned block will have the requested alignment. 344 // Alignment must be either zero, or a power of two and a multiple of sizeof(void*), and should ideally be 345 // less than memory page size. A caveat of rpmalloc internals is that this must also be strictly less than 346 // the span size (default 64KiB). 347 RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void* 348 rpmalloc_heap_aligned_realloc(rpmalloc_heap_t* heap, void* ptr, size_t alignment, size_t size, unsigned int flags) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE(4); 349 350 //! Free the given memory block from the given heap. The memory block MUST be allocated 351 // by the same heap given to this function. 352 RPMALLOC_EXPORT void 353 rpmalloc_heap_free(rpmalloc_heap_t* heap, void* ptr); 354 355 //! Free all memory allocated by the heap 356 RPMALLOC_EXPORT void 357 rpmalloc_heap_free_all(rpmalloc_heap_t* heap); 358 359 //! Set the given heap as the current heap for the calling thread. A heap MUST only be current heap 360 // for a single thread, a heap can never be shared between multiple threads. The previous 361 // current heap for the calling thread is released to be reused by other threads. 362 RPMALLOC_EXPORT void 363 rpmalloc_heap_thread_set_current(rpmalloc_heap_t* heap); 364 365 #endif 366 367 #ifdef __cplusplus 368 } 369 #endif