github.com/pidato/unsafe@v0.1.4/memory/rpmalloc/malloc.c (about)

     1  /* malloc.c  -  Memory allocator  -  Public Domain  -  2016 Mattias Jansson
     2   *
     3   * This library provides a cross-platform lock free thread caching malloc implementation in C11.
     4   * The latest source code is always available at
     5   *
     6   * https://github.com/mjansson/rpmalloc
     7   *
     8   * This library is put in the public domain; you can redistribute it and/or modify it without any restrictions.
     9   *
    10   */
    11  
    12  //
    13  // This file provides overrides for the standard library malloc entry points for C and new/delete operators for C++
    14  // It also provides automatic initialization/finalization of process and threads
    15  //
    16  #if defined(__TINYC__)
    17  #include <sys/types.h>
    18  #endif
    19  
    20  #include <stdlib.h>
    21  
    22  #ifndef ARCH_64BIT
    23  #  if defined(__LLP64__) || defined(__LP64__) || defined(_WIN64)
    24  #    define ARCH_64BIT 1
    25  _Static_assert(sizeof(size_t) == 8, "Data type size mismatch");
    26  _Static_assert(sizeof(void*) == 8, "Data type size mismatch");
    27  #  else
    28  #    define ARCH_64BIT 0
    29  _Static_assert(sizeof(size_t) == 4, "Data type size mismatch");
    30  _Static_assert(sizeof(void*) == 4, "Data type size mismatch");
    31  #  endif
    32  #endif
    33  
    34  #if (defined(__GNUC__) || defined(__clang__))
    35  #pragma GCC visibility push(default)
    36  #endif
    37  
    38  #define USE_IMPLEMENT 1
    39  #define USE_INTERPOSE 0
    40  #define USE_ALIAS 0
    41  
    42  #if defined(__APPLE__)
    43  #undef USE_INTERPOSE
    44  #define USE_INTERPOSE 1
    45  
    46  typedef struct interpose_t {
    47  	void* new_func;
    48  	void* orig_func;
    49  } interpose_t;
    50  
    51  #define MAC_INTERPOSE_PAIR(newf, oldf) 	{ (void*)newf, (void*)oldf }
    52  #define MAC_INTERPOSE_SINGLE(newf, oldf) \
    53  __attribute__((used)) static const interpose_t macinterpose##newf##oldf \
    54  __attribute__ ((section("__DATA, __interpose"))) = MAC_INTERPOSE_PAIR(newf, oldf)
    55  
    56  #endif
    57  
    58  #if !defined(_WIN32) && !defined(__APPLE__)
    59  #undef USE_IMPLEMENT
    60  #undef USE_ALIAS
    61  #define USE_IMPLEMENT 0
    62  #define USE_ALIAS 1
    63  #endif
    64  
    65  #ifdef _MSC_VER
    66  #pragma warning (disable : 4100)
    67  #undef malloc
    68  #undef free
    69  #undef calloc
    70  #define RPMALLOC_RESTRICT __declspec(restrict)
    71  #else
    72  #define RPMALLOC_RESTRICT
    73  #endif
    74  
    75  #if ENABLE_OVERRIDE
    76  
    77  typedef struct rp_nothrow_t { int __dummy; } rp_nothrow_t;
    78  
    79  #if USE_IMPLEMENT
    80  
    81  extern inline RPMALLOC_RESTRICT void* RPMALLOC_CDECL malloc(size_t size) { return rpmalloc(size); }
    82  extern inline RPMALLOC_RESTRICT void* RPMALLOC_CDECL calloc(size_t count, size_t size) { return rpcalloc(count, size); }
    83  extern inline RPMALLOC_RESTRICT void* RPMALLOC_CDECL realloc(void* ptr, size_t size) { return rprealloc(ptr, size); }
    84  extern inline void* RPMALLOC_CDECL reallocf(void* ptr, size_t size) { return rprealloc(ptr, size); }
    85  extern inline void* RPMALLOC_CDECL aligned_alloc(size_t alignment, size_t size) { return rpaligned_alloc(alignment, size); }
    86  extern inline void* RPMALLOC_CDECL memalign(size_t alignment, size_t size) { return rpmemalign(alignment, size); }
    87  extern inline int RPMALLOC_CDECL posix_memalign(void** memptr, size_t alignment, size_t size) { return rpposix_memalign(memptr, alignment, size); }
    88  extern inline void RPMALLOC_CDECL free(void* ptr) { rpfree(ptr); }
    89  extern inline void RPMALLOC_CDECL cfree(void* ptr) { rpfree(ptr); }
    90  extern inline size_t RPMALLOC_CDECL malloc_usable_size(void* ptr) { return rpmalloc_usable_size(ptr); }
    91  extern inline size_t RPMALLOC_CDECL malloc_size(void* ptr) { return rpmalloc_usable_size(ptr); }
    92  
    93  #ifdef _WIN32
    94  // For Windows, #include <rpnew.h> in one source file to get the C++ operator overrides implemented in your module
    95  #else
    96  // Overload the C++ operators using the mangled names (https://itanium-cxx-abi.github.io/cxx-abi/abi.html#mangling)
    97  // operators delete and delete[]
    98  #define RPDEFVIS __attribute__((visibility("default")))
    99  extern void _ZdlPv(void* p); void RPDEFVIS _ZdlPv(void* p) { rpfree(p); }
   100  extern void _ZdaPv(void* p); void RPDEFVIS _ZdaPv(void* p) { rpfree(p); }
   101  #if ARCH_64BIT
   102  // 64-bit operators new and new[], normal and aligned
   103  extern void* _Znwm(uint64_t size); void* RPDEFVIS _Znwm(uint64_t size) { return rpmalloc(size); }
   104  extern void* _Znam(uint64_t size); void* RPDEFVIS _Znam(uint64_t size) { return rpmalloc(size); }
   105  extern void* _Znwmm(uint64_t size, uint64_t align); void* RPDEFVIS _Znwmm(uint64_t size, uint64_t align) { return rpaligned_alloc(align, size); }
   106  extern void* _Znamm(uint64_t size, uint64_t align); void* RPDEFVIS _Znamm(uint64_t size, uint64_t align) { return rpaligned_alloc(align, size); }
   107  extern void* _ZnwmSt11align_val_t(uint64_t size, uint64_t align); void* RPDEFVIS _ZnwmSt11align_val_t(uint64_t size, uint64_t align) { return rpaligned_alloc(align, size); }
   108  extern void* _ZnamSt11align_val_t(uint64_t size, uint64_t align); void* RPDEFVIS _ZnamSt11align_val_t(uint64_t size, uint64_t align) { return rpaligned_alloc(align, size); }
   109  extern void* _ZnwmRKSt9nothrow_t(uint64_t size, rp_nothrow_t t); void* RPDEFVIS _ZnwmRKSt9nothrow_t(uint64_t size, rp_nothrow_t t) { (void)sizeof(t); return rpmalloc(size); }
   110  extern void* _ZnamRKSt9nothrow_t(uint64_t size, rp_nothrow_t t); void* RPDEFVIS _ZnamRKSt9nothrow_t(uint64_t size, rp_nothrow_t t) { (void)sizeof(t); return rpmalloc(size); }
   111  extern void* _ZnwmSt11align_val_tRKSt9nothrow_t(uint64_t size, uint64_t align, rp_nothrow_t t); void* RPDEFVIS _ZnwmSt11align_val_tRKSt9nothrow_t(uint64_t size, uint64_t align, rp_nothrow_t t) { (void)sizeof(t); return rpaligned_alloc(align, size); }
   112  extern void* _ZnamSt11align_val_tRKSt9nothrow_t(uint64_t size, uint64_t align, rp_nothrow_t t); void* RPDEFVIS _ZnamSt11align_val_tRKSt9nothrow_t(uint64_t size, uint64_t align, rp_nothrow_t t) { (void)sizeof(t); return rpaligned_alloc(align, size); }
   113  // 64-bit operators sized delete and delete[], normal and aligned
   114  extern void _ZdlPvm(void* p, uint64_t size); void RPDEFVIS _ZdlPvm(void* p, uint64_t size) { rpfree(p); (void)sizeof(size); }
   115  extern void _ZdaPvm(void* p, uint64_t size); void RPDEFVIS _ZdaPvm(void* p, uint64_t size) { rpfree(p); (void)sizeof(size); }
   116  extern void _ZdlPvSt11align_val_t(void* p, uint64_t align); void RPDEFVIS _ZdlPvSt11align_val_t(void* p, uint64_t align) { rpfree(p); (void)sizeof(align); }
   117  extern void _ZdaPvSt11align_val_t(void* p, uint64_t align); void RPDEFVIS _ZdaPvSt11align_val_t(void* p, uint64_t align) { rpfree(p); (void)sizeof(align); }
   118  extern void _ZdlPvmSt11align_val_t(void* p, uint64_t size, uint64_t align); void RPDEFVIS _ZdlPvmSt11align_val_t(void* p, uint64_t size, uint64_t align) { rpfree(p); (void)sizeof(size); (void)sizeof(align); }
   119  extern void _ZdaPvmSt11align_val_t(void* p, uint64_t size, uint64_t align); void RPDEFVIS _ZdaPvmSt11align_val_t(void* p, uint64_t size, uint64_t align) { rpfree(p); (void)sizeof(size); (void)sizeof(align); }
   120  #else
   121  // 32-bit operators new and new[], normal and aligned
   122  extern void* _Znwj(uint32_t size); void* RPDEFVIS _Znwj(uint32_t size) { return rpmalloc(size); }
   123  extern void* _Znaj(uint32_t size); void* RPDEFVIS _Znaj(uint32_t size) { return rpmalloc(size); }
   124  extern void* _Znwjj(uint32_t size, uint32_t align); void* RPDEFVIS _Znwjj(uint32_t size, uint32_t align) { return rpaligned_alloc(align, size); }
   125  extern void* _Znajj(uint32_t size, uint32_t align); void* RPDEFVIS _Znajj(uint32_t size, uint32_t align) { return rpaligned_alloc(align, size); }
   126  extern void* _ZnwjSt11align_val_t(size_t size, size_t align); void* RPDEFVIS _ZnwjSt11align_val_t(size_t size, size_t align) { return rpaligned_alloc(align, size); }
   127  extern void* _ZnajSt11align_val_t(size_t size, size_t align); void* RPDEFVIS _ZnajSt11align_val_t(size_t size, size_t align) { return rpaligned_alloc(align, size); }
   128  extern void* _ZnwjRKSt9nothrow_t(size_t size, rp_nothrow_t t); void* RPDEFVIS _ZnwjRKSt9nothrow_t(size_t size, rp_nothrow_t t) { (void)sizeof(t); return rpmalloc(size); }
   129  extern void* _ZnajRKSt9nothrow_t(size_t size, rp_nothrow_t t); void* RPDEFVIS _ZnajRKSt9nothrow_t(size_t size, rp_nothrow_t t) { (void)sizeof(t); return rpmalloc(size); }
   130  extern void* _ZnwjSt11align_val_tRKSt9nothrow_t(size_t size, size_t align, rp_nothrow_t t); void* RPDEFVIS _ZnwjSt11align_val_tRKSt9nothrow_t(size_t size, size_t align, rp_nothrow_t t) { (void)sizeof(t); return rpaligned_alloc(align, size); }
   131  extern void* _ZnajSt11align_val_tRKSt9nothrow_t(size_t size, size_t align, rp_nothrow_t t); void* RPDEFVIS _ZnajSt11align_val_tRKSt9nothrow_t(size_t size, size_t align, rp_nothrow_t t) { (void)sizeof(t); return rpaligned_alloc(align, size); }
   132  // 32-bit operators sized delete and delete[], normal and aligned
   133  extern void _ZdlPvj(void* p, uint64_t size); void RPDEFVIS _ZdlPvj(void* p, uint64_t size) { rpfree(p); (void)sizeof(size); }
   134  extern void _ZdaPvj(void* p, uint64_t size); void RPDEFVIS _ZdaPvj(void* p, uint64_t size) { rpfree(p); (void)sizeof(size); }
   135  extern void _ZdlPvSt11align_val_t(void* p, uint32_t align); void RPDEFVIS _ZdlPvSt11align_val_t(void* p, uint64_t a) { rpfree(p); (void)sizeof(align); }
   136  extern void _ZdaPvSt11align_val_t(void* p, uint32_t align); void RPDEFVIS _ZdaPvSt11align_val_t(void* p, uint64_t a) { rpfree(p); (void)sizeof(align); }
   137  extern void _ZdlPvjSt11align_val_t(void* p, uint32_t size, uint32_t align); void RPDEFVIS _ZdlPvjSt11align_val_t(void* p, uint64_t size, uint64_t align) { rpfree(p); (void)sizeof(size); (void)sizeof(a); }
   138  extern void _ZdaPvjSt11align_val_t(void* p, uint32_t size, uint32_t align); void RPDEFVIS _ZdaPvjSt11align_val_t(void* p, uint64_t size, uint64_t align) { rpfree(p); (void)sizeof(size); (void)sizeof(a); }
   139  #endif
   140  #endif
   141  #endif
   142  
   143  #if USE_INTERPOSE || USE_ALIAS
   144  
   145  static void* rpmalloc_nothrow(size_t size, rp_nothrow_t t) { (void)sizeof(t); return rpmalloc(size); }
   146  static void* rpaligned_alloc_reverse(size_t size, size_t align) { return rpaligned_alloc(align, size); }
   147  static void* rpaligned_alloc_reverse_nothrow(size_t size, size_t align, rp_nothrow_t t) { (void)sizeof(t); return rpaligned_alloc(align, size); }
   148  static void rpfree_size(void* p, size_t size) { (void)sizeof(size); rpfree(p); }
   149  static void rpfree_aligned(void* p, size_t align) { (void)sizeof(align); rpfree(p); }
   150  static void rpfree_size_aligned(void* p, size_t size, size_t align) { (void)sizeof(size); (void)sizeof(align); rpfree(p); }
   151  
   152  #endif
   153  
   154  #if USE_INTERPOSE
   155  
   156  __attribute__((used)) static const interpose_t macinterpose_malloc[]
   157  __attribute__ ((section("__DATA, __interpose"))) = {
   158  	//new and new[]
   159  	MAC_INTERPOSE_PAIR(rpmalloc, _Znwm),
   160  	MAC_INTERPOSE_PAIR(rpmalloc, _Znam),
   161  	MAC_INTERPOSE_PAIR(rpaligned_alloc_reverse, _Znwmm),
   162  	MAC_INTERPOSE_PAIR(rpaligned_alloc_reverse, _Znamm),
   163  	MAC_INTERPOSE_PAIR(rpmalloc_nothrow, _ZnwmRKSt9nothrow_t),
   164  	MAC_INTERPOSE_PAIR(rpmalloc_nothrow, _ZnamRKSt9nothrow_t),
   165  	MAC_INTERPOSE_PAIR(rpaligned_alloc_reverse, _ZnwmSt11align_val_t),
   166  	MAC_INTERPOSE_PAIR(rpaligned_alloc_reverse, _ZnamSt11align_val_t),
   167  	MAC_INTERPOSE_PAIR(rpaligned_alloc_reverse_nothrow, _ZnwmSt11align_val_tRKSt9nothrow_t),
   168  	MAC_INTERPOSE_PAIR(rpaligned_alloc_reverse_nothrow, _ZnamSt11align_val_tRKSt9nothrow_t),
   169  	//delete and delete[]
   170  	MAC_INTERPOSE_PAIR(rpfree, _ZdlPv),
   171  	MAC_INTERPOSE_PAIR(rpfree, _ZdaPv),
   172  	MAC_INTERPOSE_PAIR(rpfree_size, _ZdlPvm),
   173  	MAC_INTERPOSE_PAIR(rpfree_size, _ZdaPvm),
   174  	MAC_INTERPOSE_PAIR(rpfree_aligned, _ZdlPvSt11align_val_t),
   175  	MAC_INTERPOSE_PAIR(rpfree_aligned, _ZdaPvSt11align_val_t),
   176  	MAC_INTERPOSE_PAIR(rpfree_size_aligned, _ZdlPvmSt11align_val_t),
   177  	MAC_INTERPOSE_PAIR(rpfree_size_aligned, _ZdaPvmSt11align_val_t),
   178  	//libc entry points
   179  	MAC_INTERPOSE_PAIR(rpmalloc, malloc),
   180  	MAC_INTERPOSE_PAIR(rpmalloc, calloc),
   181  	MAC_INTERPOSE_PAIR(rprealloc, realloc),
   182  	MAC_INTERPOSE_PAIR(rprealloc, reallocf),
   183  #if defined(__MAC_10_15) && __MAC_OS_X_VERSION_MIN_REQUIRED >= __MAC_10_15
   184  	MAC_INTERPOSE_PAIR(rpaligned_alloc, aligned_alloc),
   185  #endif
   186  	MAC_INTERPOSE_PAIR(rpmemalign, memalign),
   187  	MAC_INTERPOSE_PAIR(rpposix_memalign, posix_memalign),
   188  	MAC_INTERPOSE_PAIR(rpfree, free),
   189  	MAC_INTERPOSE_PAIR(rpfree, cfree),
   190  	MAC_INTERPOSE_PAIR(rpmalloc_usable_size, malloc_usable_size),
   191  	MAC_INTERPOSE_PAIR(rpmalloc_usable_size, malloc_size)
   192  };
   193  
   194  #endif
   195  
   196  #if USE_ALIAS
   197  
   198  #define RPALIAS(fn) __attribute__((alias(#fn), used, visibility("default")));
   199  
   200  // Alias the C++ operators using the mangled names (https://itanium-cxx-abi.github.io/cxx-abi/abi.html#mangling)
   201  
   202  // operators delete and delete[]
   203  void _ZdlPv(void* p) RPALIAS(rpfree)
   204  void _ZdaPv(void* p) RPALIAS(rpfree)
   205  
   206  #if ARCH_64BIT
   207  // 64-bit operators new and new[], normal and aligned
   208  void* _Znwm(uint64_t size) RPALIAS(rpmalloc)
   209  void* _Znam(uint64_t size) RPALIAS(rpmalloc)
   210  void* _Znwmm(uint64_t size, uint64_t align) RPALIAS(rpaligned_alloc_reverse)
   211  void* _Znamm(uint64_t size, uint64_t align) RPALIAS(rpaligned_alloc_reverse)
   212  void* _ZnwmSt11align_val_t(size_t size, size_t align) RPALIAS(rpaligned_alloc_reverse)
   213  void* _ZnamSt11align_val_t(size_t size, size_t align) RPALIAS(rpaligned_alloc_reverse)
   214  void* _ZnwmRKSt9nothrow_t(size_t size, rp_nothrow_t t) RPALIAS(rpmalloc_nothrow)
   215  void* _ZnamRKSt9nothrow_t(size_t size, rp_nothrow_t t) RPALIAS(rpmalloc_nothrow)
   216  void* _ZnwmSt11align_val_tRKSt9nothrow_t(size_t size, size_t align, rp_nothrow_t t) RPALIAS(rpaligned_alloc_reverse_nothrow)
   217  void* _ZnamSt11align_val_tRKSt9nothrow_t(size_t size, size_t align, rp_nothrow_t t) RPALIAS(rpaligned_alloc_reverse_nothrow)
   218  // 64-bit operators delete and delete[], sized and aligned
   219  void _ZdlPvm(void* p, size_t n) RPALIAS(rpfree_size)
   220  void _ZdaPvm(void* p, size_t n) RPALIAS(rpfree_size)
   221  void _ZdlPvSt11align_val_t(void* p, size_t a) RPALIAS(rpfree_aligned)
   222  void _ZdaPvSt11align_val_t(void* p, size_t a) RPALIAS(rpfree_aligned)
   223  void _ZdlPvmSt11align_val_t(void* p, size_t n, size_t a) RPALIAS(rpfree_size_aligned)
   224  void _ZdaPvmSt11align_val_t(void* p, size_t n, size_t a) RPALIAS(rpfree_size_aligned)
   225  #else
   226  // 32-bit operators new and new[], normal and aligned
   227  void* _Znwj(uint32_t size) RPALIAS(rpmalloc)
   228  void* _Znaj(uint32_t size) RPALIAS(rpmalloc)
   229  void* _Znwjj(uint32_t size, uint32_t align) RPALIAS(rpaligned_alloc_reverse)
   230  void* _Znajj(uint32_t size, uint32_t align) RPALIAS(rpaligned_alloc_reverse)
   231  void* _ZnwjSt11align_val_t(size_t size, size_t align) RPALIAS(rpaligned_alloc_reverse)
   232  void* _ZnajSt11align_val_t(size_t size, size_t align) RPALIAS(rpaligned_alloc_reverse)
   233  void* _ZnwjRKSt9nothrow_t(size_t size, rp_nothrow_t t) RPALIAS(rpmalloc_nothrow)
   234  void* _ZnajRKSt9nothrow_t(size_t size, rp_nothrow_t t) RPALIAS(rpmalloc_nothrow)
   235  void* _ZnwjSt11align_val_tRKSt9nothrow_t(size_t size, size_t align, rp_nothrow_t t) RPALIAS(rpaligned_alloc_reverse_nothrow)
   236  void* _ZnajSt11align_val_tRKSt9nothrow_t(size_t size, size_t align, rp_nothrow_t t) RPALIAS(rpaligned_alloc_reverse_nothrow)
   237  // 32-bit operators delete and delete[], sized and aligned
   238  void _ZdlPvj(void* p, size_t n) RPALIAS(rpfree_size)
   239  void _ZdaPvj(void* p, size_t n) RPALIAS(rpfree_size)
   240  void _ZdlPvSt11align_val_t(void* p, size_t a) RPALIAS(rpfree_aligned)
   241  void _ZdaPvSt11align_val_t(void* p, size_t a) RPALIAS(rpfree_aligned)
   242  void _ZdlPvjSt11align_val_t(void* p, size_t n, size_t a) RPALIAS(rpfree_size_aligned)
   243  void _ZdaPvjSt11align_val_t(void* p, size_t n, size_t a) RPALIAS(rpfree_size_aligned)
   244  #endif
   245  
   246  void* malloc(size_t size) RPALIAS(rpmalloc)
   247  void* calloc(size_t count, size_t size) RPALIAS(rpcalloc)
   248  void* realloc(void* ptr, size_t size) RPALIAS(rprealloc)
   249  void* reallocf(void* ptr, size_t size) RPALIAS(rprealloc)
   250  void* aligned_alloc(size_t alignment, size_t size) RPALIAS(rpaligned_alloc)
   251  void* memalign(size_t alignment, size_t size) RPALIAS(rpmemalign)
   252  int posix_memalign(void** memptr, size_t alignment, size_t size) RPALIAS(rpposix_memalign)
   253  void free(void* ptr) RPALIAS(rpfree)
   254  void cfree(void* ptr) RPALIAS(rpfree)
   255  #if defined(__ANDROID__) || defined(__FreeBSD__)
   256  size_t malloc_usable_size(const void* ptr) RPALIAS(rpmalloc_usable_size)
   257  #else
   258  size_t malloc_usable_size(void* ptr) RPALIAS(rpmalloc_usable_size)
   259  #endif
   260  size_t malloc_size(void* ptr) RPALIAS(rpmalloc_usable_size)
   261  
   262  #endif
   263  
   264  static inline size_t
   265  _rpmalloc_page_size(void) {
   266  	return _memory_page_size;
   267  }
   268  
   269  extern inline void* RPMALLOC_CDECL
   270  reallocarray(void* ptr, size_t count, size_t size) {
   271  	size_t total;
   272  #if ENABLE_VALIDATE_ARGS
   273  #ifdef _MSC_VER
   274  	int err = SizeTMult(count, size, &total);
   275  	if ((err != S_OK) || (total >= MAX_ALLOC_SIZE)) {
   276  		errno = EINVAL;
   277  		return 0;
   278  	}
   279  #else
   280  	int err = __builtin_umull_overflow(count, size, &total);
   281  	if (err || (total >= MAX_ALLOC_SIZE)) {
   282  		errno = EINVAL;
   283  		return 0;
   284  	}
   285  #endif
   286  #else
   287  	total = count * size;
   288  #endif
   289  	return realloc(ptr, total);
   290  }
   291  
   292  extern inline void* RPMALLOC_CDECL
   293  valloc(size_t size) {
   294  	get_thread_heap();
   295  	return rpaligned_alloc(_rpmalloc_page_size(), size);
   296  }
   297  
   298  extern inline void* RPMALLOC_CDECL
   299  pvalloc(size_t size) {
   300  	get_thread_heap();
   301  	const size_t page_size = _rpmalloc_page_size();
   302  	const size_t aligned_size = ((size + page_size - 1) / page_size) * page_size;
   303  #if ENABLE_VALIDATE_ARGS
   304  	if (aligned_size < size) {
   305  		errno = EINVAL;
   306  		return 0;
   307  	}
   308  #endif
   309  	return rpaligned_alloc(_rpmalloc_page_size(), aligned_size);
   310  }
   311  
   312  #endif // ENABLE_OVERRIDE
   313  
   314  #if ENABLE_PRELOAD
   315  
   316  #ifdef _WIN32
   317  
   318  #if defined(BUILD_DYNAMIC_LINK) && BUILD_DYNAMIC_LINK
   319  
   320  extern __declspec(dllexport) BOOL WINAPI
   321  DllMain(HINSTANCE instance, DWORD reason, LPVOID reserved);
   322  
   323  extern __declspec(dllexport) BOOL WINAPI
   324  DllMain(HINSTANCE instance, DWORD reason, LPVOID reserved) {
   325  	(void)sizeof(reserved);
   326  	(void)sizeof(instance);
   327  	if (reason == DLL_PROCESS_ATTACH)
   328  		rpmalloc_initialize();
   329  	else if (reason == DLL_PROCESS_DETACH)
   330  		rpmalloc_finalize();
   331  	else if (reason == DLL_THREAD_ATTACH)
   332  		rpmalloc_thread_initialize();
   333  	else if (reason == DLL_THREAD_DETACH)
   334  		rpmalloc_thread_finalize(1);
   335  	return TRUE;
   336  }
   337  
   338  //end BUILD_DYNAMIC_LINK
   339  #else
   340  
   341  extern void
   342  _global_rpmalloc_init(void) {
   343  	rpmalloc_set_main_thread();
   344  	rpmalloc_initialize();
   345  }
   346  
   347  #if defined(__clang__) || defined(__GNUC__)
   348  
   349  static void __attribute__((constructor))
   350  initializer(void) {
   351  	_global_rpmalloc_init();
   352  }
   353  
   354  #elif defined(_MSC_VER)
   355  
   356  #pragma section(".CRT$XIB",read)
   357  __declspec(allocate(".CRT$XIB")) void (*_rpmalloc_module_init)(void) = _global_rpmalloc_init;
   358  #pragma comment(linker, "/include:_rpmalloc_module_init")
   359  
   360  #endif
   361  
   362  //end !BUILD_DYNAMIC_LINK
   363  #endif
   364  
   365  #else
   366  
   367  #include <pthread.h>
   368  #include <stdlib.h>
   369  #include <stdint.h>
   370  #include <unistd.h>
   371  
   372  extern void
   373  rpmalloc_set_main_thread(void);
   374  
   375  static pthread_key_t destructor_key;
   376  
   377  static void
   378  thread_destructor(void*);
   379  
   380  static void __attribute__((constructor))
   381  initializer(void) {
   382  	rpmalloc_set_main_thread();
   383  	rpmalloc_initialize();
   384  	pthread_key_create(&destructor_key, thread_destructor);
   385  }
   386  
   387  static void __attribute__((destructor))
   388  finalizer(void) {
   389  	rpmalloc_finalize();
   390  }
   391  
   392  typedef struct {
   393  	void* (*real_start)(void*);
   394  	void* real_arg;
   395  } thread_starter_arg;
   396  
   397  static void*
   398  thread_starter(void* argptr) {
   399  	thread_starter_arg* arg = argptr;
   400  	void* (*real_start)(void*) = arg->real_start;
   401  	void* real_arg = arg->real_arg;
   402  	rpmalloc_thread_initialize();
   403  	rpfree(argptr);
   404  	pthread_setspecific(destructor_key, (void*)1);
   405  	return (*real_start)(real_arg);
   406  }
   407  
   408  static void
   409  thread_destructor(void* value) {
   410  	(void)sizeof(value);
   411  	rpmalloc_thread_finalize(1);
   412  }
   413  
   414  #ifdef __APPLE__
   415  
   416  static int
   417  pthread_create_proxy(pthread_t* thread,
   418                       const pthread_attr_t* attr,
   419                       void* (*start_routine)(void*),
   420                       void* arg) {
   421  	rpmalloc_initialize();
   422  	thread_starter_arg* starter_arg = rpmalloc(sizeof(thread_starter_arg));
   423  	starter_arg->real_start = start_routine;
   424  	starter_arg->real_arg = arg;
   425  	return pthread_create(thread, attr, thread_starter, starter_arg);
   426  }
   427  
   428  MAC_INTERPOSE_SINGLE(pthread_create_proxy, pthread_create);
   429  
   430  #else
   431  
   432  #include <dlfcn.h>
   433  
   434  int
   435  pthread_create(pthread_t* thread,
   436                 const pthread_attr_t* attr,
   437                 void* (*start_routine)(void*),
   438                 void* arg) {
   439  #if defined(__linux__) || defined(__FreeBSD__) || defined(__OpenBSD__) || defined(__NetBSD__) || defined(__DragonFly__) || \
   440      defined(__APPLE__) || defined(__HAIKU__)
   441  	char fname[] = "pthread_create";
   442  #else
   443  	char fname[] = "_pthread_create";
   444  #endif
   445  	void* real_pthread_create = dlsym(RTLD_NEXT, fname);
   446  	rpmalloc_thread_initialize();
   447  	thread_starter_arg* starter_arg = rpmalloc(sizeof(thread_starter_arg));
   448  	starter_arg->real_start = start_routine;
   449  	starter_arg->real_arg = arg;
   450  	return (*(int (*)(pthread_t*, const pthread_attr_t*, void* (*)(void*), void*))real_pthread_create)(thread, attr, thread_starter, starter_arg);
   451  }
   452  
   453  #endif
   454  
   455  #endif
   456  
   457  #endif
   458  
   459  #if ENABLE_OVERRIDE
   460  
   461  #if defined(__GLIBC__) && defined(__linux__)
   462  
   463  void* __libc_malloc(size_t size) RPALIAS(rpmalloc)
   464  void* __libc_calloc(size_t count, size_t size) RPALIAS(rpcalloc)
   465  void* __libc_realloc(void* p, size_t size) RPALIAS(rprealloc)
   466  void __libc_free(void* p) RPALIAS(rpfree)
   467  void __libc_cfree(void* p) RPALIAS(rpfree)
   468  void* __libc_memalign(size_t align, size_t size) RPALIAS(rpmemalign)
   469  int __posix_memalign(void** p, size_t align, size_t size) RPALIAS(rpposix_memalign)
   470  
   471  extern void* __libc_valloc(size_t size);
   472  extern void* __libc_pvalloc(size_t size);
   473  
   474  void*
   475  __libc_valloc(size_t size) {
   476  	return valloc(size);
   477  }
   478  
   479  void*
   480  __libc_pvalloc(size_t size) {
   481  	return pvalloc(size);
   482  }
   483  
   484  #endif
   485  
   486  #endif
   487  
   488  #if (defined(__GNUC__) || defined(__clang__))
   489  #pragma GCC visibility pop
   490  #endif