github.com/reiver/go@v0.0.0-20150109200633-1d0c7792f172/src/runtime/malloc1.go (about) 1 // Copyright 2009 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 // See malloc.h for overview. 6 // 7 // TODO(rsc): double-check stats. 8 9 package runtime 10 11 import "unsafe" 12 13 const _MaxArena32 = 2 << 30 14 15 // For use by Go. If it were a C enum it would be made available automatically, 16 // but the value of MaxMem is too large for enum. 17 // XXX - uintptr runtime·maxmem = MaxMem; 18 19 func mlookup(v uintptr, base *uintptr, size *uintptr, sp **mspan) int32 { 20 _g_ := getg() 21 22 _g_.m.mcache.local_nlookup++ 23 if ptrSize == 4 && _g_.m.mcache.local_nlookup >= 1<<30 { 24 // purge cache stats to prevent overflow 25 lock(&mheap_.lock) 26 purgecachedstats(_g_.m.mcache) 27 unlock(&mheap_.lock) 28 } 29 30 s := mHeap_LookupMaybe(&mheap_, unsafe.Pointer(v)) 31 if sp != nil { 32 *sp = s 33 } 34 if s == nil { 35 if base != nil { 36 *base = 0 37 } 38 if size != nil { 39 *size = 0 40 } 41 return 0 42 } 43 44 p := uintptr(s.start) << _PageShift 45 if s.sizeclass == 0 { 46 // Large object. 47 if base != nil { 48 *base = p 49 } 50 if size != nil { 51 *size = s.npages << _PageShift 52 } 53 return 1 54 } 55 56 n := s.elemsize 57 if base != nil { 58 i := (uintptr(v) - uintptr(p)) / n 59 *base = p + i*n 60 } 61 if size != nil { 62 *size = n 63 } 64 65 return 1 66 } 67 68 //go:nosplit 69 func purgecachedstats(c *mcache) { 70 // Protected by either heap or GC lock. 71 h := &mheap_ 72 memstats.heap_alloc += uint64(c.local_cachealloc) 73 c.local_cachealloc = 0 74 memstats.tinyallocs += uint64(c.local_tinyallocs) 75 c.local_tinyallocs = 0 76 memstats.nlookup += uint64(c.local_nlookup) 77 c.local_nlookup = 0 78 h.largefree += uint64(c.local_largefree) 79 c.local_largefree = 0 80 h.nlargefree += uint64(c.local_nlargefree) 81 c.local_nlargefree = 0 82 for i := 0; i < len(c.local_nsmallfree); i++ { 83 h.nsmallfree[i] += uint64(c.local_nsmallfree[i]) 84 c.local_nsmallfree[i] = 0 85 } 86 } 87 88 func mallocinit() { 89 initSizes() 90 91 if class_to_size[_TinySizeClass] != _TinySize { 92 throw("bad TinySizeClass") 93 } 94 95 var p, bitmapSize, spansSize, pSize, limit uintptr 96 var reserved bool 97 98 // limit = runtime.memlimit(); 99 // See https://code.google.com/p/go/issues/detail?id=5049 100 // TODO(rsc): Fix after 1.1. 101 limit = 0 102 103 // Set up the allocation arena, a contiguous area of memory where 104 // allocated data will be found. The arena begins with a bitmap large 105 // enough to hold 4 bits per allocated word. 106 if ptrSize == 8 && (limit == 0 || limit > 1<<30) { 107 // On a 64-bit machine, allocate from a single contiguous reservation. 108 // 128 GB (MaxMem) should be big enough for now. 109 // 110 // The code will work with the reservation at any address, but ask 111 // SysReserve to use 0x0000XXc000000000 if possible (XX=00...7f). 112 // Allocating a 128 GB region takes away 37 bits, and the amd64 113 // doesn't let us choose the top 17 bits, so that leaves the 11 bits 114 // in the middle of 0x00c0 for us to choose. Choosing 0x00c0 means 115 // that the valid memory addresses will begin 0x00c0, 0x00c1, ..., 0x00df. 116 // In little-endian, that's c0 00, c1 00, ..., df 00. None of those are valid 117 // UTF-8 sequences, and they are otherwise as far away from 118 // ff (likely a common byte) as possible. If that fails, we try other 0xXXc0 119 // addresses. An earlier attempt to use 0x11f8 caused out of memory errors 120 // on OS X during thread allocations. 0x00c0 causes conflicts with 121 // AddressSanitizer which reserves all memory up to 0x0100. 122 // These choices are both for debuggability and to reduce the 123 // odds of the conservative garbage collector not collecting memory 124 // because some non-pointer block of memory had a bit pattern 125 // that matched a memory address. 126 // 127 // Actually we reserve 136 GB (because the bitmap ends up being 8 GB) 128 // but it hardly matters: e0 00 is not valid UTF-8 either. 129 // 130 // If this fails we fall back to the 32 bit memory mechanism 131 arenaSize := round(_MaxMem, _PageSize) 132 bitmapSize = arenaSize / (ptrSize * 8 / 4) 133 spansSize = arenaSize / _PageSize * ptrSize 134 spansSize = round(spansSize, _PageSize) 135 for i := 0; i <= 0x7f; i++ { 136 p = uintptr(i)<<40 | uintptrMask&(0x00c0<<32) 137 pSize = bitmapSize + spansSize + arenaSize + _PageSize 138 p = uintptr(sysReserve(unsafe.Pointer(p), pSize, &reserved)) 139 if p != 0 { 140 break 141 } 142 } 143 } 144 145 if p == 0 { 146 // On a 32-bit machine, we can't typically get away 147 // with a giant virtual address space reservation. 148 // Instead we map the memory information bitmap 149 // immediately after the data segment, large enough 150 // to handle another 2GB of mappings (256 MB), 151 // along with a reservation for an initial arena. 152 // When that gets used up, we'll start asking the kernel 153 // for any memory anywhere and hope it's in the 2GB 154 // following the bitmap (presumably the executable begins 155 // near the bottom of memory, so we'll have to use up 156 // most of memory before the kernel resorts to giving out 157 // memory before the beginning of the text segment). 158 // 159 // Alternatively we could reserve 512 MB bitmap, enough 160 // for 4GB of mappings, and then accept any memory the 161 // kernel threw at us, but normally that's a waste of 512 MB 162 // of address space, which is probably too much in a 32-bit world. 163 164 // If we fail to allocate, try again with a smaller arena. 165 // This is necessary on Android L where we share a process 166 // with ART, which reserves virtual memory aggressively. 167 arenaSizes := []uintptr{ 168 512 << 20, 169 256 << 20, 170 } 171 172 for _, arenaSize := range arenaSizes { 173 bitmapSize = _MaxArena32 / (ptrSize * 8 / 4) 174 spansSize = _MaxArena32 / _PageSize * ptrSize 175 if limit > 0 && arenaSize+bitmapSize+spansSize > limit { 176 bitmapSize = (limit / 9) &^ ((1 << _PageShift) - 1) 177 arenaSize = bitmapSize * 8 178 spansSize = arenaSize / _PageSize * ptrSize 179 } 180 spansSize = round(spansSize, _PageSize) 181 182 // SysReserve treats the address we ask for, end, as a hint, 183 // not as an absolute requirement. If we ask for the end 184 // of the data segment but the operating system requires 185 // a little more space before we can start allocating, it will 186 // give out a slightly higher pointer. Except QEMU, which 187 // is buggy, as usual: it won't adjust the pointer upward. 188 // So adjust it upward a little bit ourselves: 1/4 MB to get 189 // away from the running binary image and then round up 190 // to a MB boundary. 191 p = round(uintptr(unsafe.Pointer(&end))+(1<<18), 1<<20) 192 pSize = bitmapSize + spansSize + arenaSize + _PageSize 193 p = uintptr(sysReserve(unsafe.Pointer(p), pSize, &reserved)) 194 if p != 0 { 195 break 196 } 197 } 198 if p == 0 { 199 throw("runtime: cannot reserve arena virtual address space") 200 } 201 } 202 203 // PageSize can be larger than OS definition of page size, 204 // so SysReserve can give us a PageSize-unaligned pointer. 205 // To overcome this we ask for PageSize more and round up the pointer. 206 p1 := round(p, _PageSize) 207 208 mheap_.spans = (**mspan)(unsafe.Pointer(p1)) 209 mheap_.bitmap = p1 + spansSize 210 mheap_.arena_start = p1 + (spansSize + bitmapSize) 211 mheap_.arena_used = mheap_.arena_start 212 mheap_.arena_end = p + pSize 213 mheap_.arena_reserved = reserved 214 215 if mheap_.arena_start&(_PageSize-1) != 0 { 216 println("bad pagesize", hex(p), hex(p1), hex(spansSize), hex(bitmapSize), hex(_PageSize), "start", hex(mheap_.arena_start)) 217 throw("misrounded allocation in mallocinit") 218 } 219 220 // Initialize the rest of the allocator. 221 mHeap_Init(&mheap_, spansSize) 222 _g_ := getg() 223 _g_.m.mcache = allocmcache() 224 } 225 226 func wbshadowinit() { 227 // Initialize write barrier shadow heap if we were asked for it 228 // and we have enough address space (not on 32-bit). 229 if debug.wbshadow == 0 { 230 return 231 } 232 if ptrSize != 8 { 233 print("runtime: GODEBUG=wbshadow=1 disabled on 32-bit system\n") 234 return 235 } 236 237 var reserved bool 238 p1 := sysReserveHigh(mheap_.arena_end-mheap_.arena_start, &reserved) 239 if p1 == nil { 240 throw("cannot map shadow heap") 241 } 242 mheap_.shadow_heap = uintptr(p1) - mheap_.arena_start 243 sysMap(p1, mheap_.arena_used-mheap_.arena_start, reserved, &memstats.other_sys) 244 memmove(p1, unsafe.Pointer(mheap_.arena_start), mheap_.arena_used-mheap_.arena_start) 245 246 mheap_.shadow_reserved = reserved 247 start := ^uintptr(0) 248 end := uintptr(0) 249 if start > uintptr(unsafe.Pointer(&noptrdata)) { 250 start = uintptr(unsafe.Pointer(&noptrdata)) 251 } 252 if start > uintptr(unsafe.Pointer(&data)) { 253 start = uintptr(unsafe.Pointer(&data)) 254 } 255 if start > uintptr(unsafe.Pointer(&noptrbss)) { 256 start = uintptr(unsafe.Pointer(&noptrbss)) 257 } 258 if start > uintptr(unsafe.Pointer(&bss)) { 259 start = uintptr(unsafe.Pointer(&bss)) 260 } 261 if end < uintptr(unsafe.Pointer(&enoptrdata)) { 262 end = uintptr(unsafe.Pointer(&enoptrdata)) 263 } 264 if end < uintptr(unsafe.Pointer(&edata)) { 265 end = uintptr(unsafe.Pointer(&edata)) 266 } 267 if end < uintptr(unsafe.Pointer(&enoptrbss)) { 268 end = uintptr(unsafe.Pointer(&enoptrbss)) 269 } 270 if end < uintptr(unsafe.Pointer(&ebss)) { 271 end = uintptr(unsafe.Pointer(&ebss)) 272 } 273 start &^= _PageSize - 1 274 end = round(end, _PageSize) 275 mheap_.data_start = start 276 mheap_.data_end = end 277 reserved = false 278 p1 = sysReserveHigh(end-start, &reserved) 279 if p1 == nil { 280 throw("cannot map shadow data") 281 } 282 mheap_.shadow_data = uintptr(p1) - start 283 sysMap(p1, end-start, reserved, &memstats.other_sys) 284 memmove(p1, unsafe.Pointer(start), end-start) 285 286 mheap_.shadow_enabled = true 287 } 288 289 // sysReserveHigh reserves space somewhere high in the address space. 290 // sysReserve doesn't actually reserve the full amount requested on 291 // 64-bit systems, because of problems with ulimit. Instead it checks 292 // that it can get the first 64 kB and assumes it can grab the rest as 293 // needed. This doesn't work well with the "let the kernel pick an address" 294 // mode, so don't do that. Pick a high address instead. 295 func sysReserveHigh(n uintptr, reserved *bool) unsafe.Pointer { 296 if ptrSize == 4 { 297 return sysReserve(nil, n, reserved) 298 } 299 300 for i := 0; i <= 0x7f; i++ { 301 p := uintptr(i)<<40 | uintptrMask&(0x00c0<<32) 302 *reserved = false 303 p = uintptr(sysReserve(unsafe.Pointer(p), n, reserved)) 304 if p != 0 { 305 return unsafe.Pointer(p) 306 } 307 } 308 309 return sysReserve(nil, n, reserved) 310 } 311 312 func mHeap_SysAlloc(h *mheap, n uintptr) unsafe.Pointer { 313 if n > uintptr(h.arena_end)-uintptr(h.arena_used) { 314 // We are in 32-bit mode, maybe we didn't use all possible address space yet. 315 // Reserve some more space. 316 p_size := round(n+_PageSize, 256<<20) 317 new_end := h.arena_end + p_size 318 if new_end <= h.arena_start+_MaxArena32 { 319 // TODO: It would be bad if part of the arena 320 // is reserved and part is not. 321 var reserved bool 322 p := uintptr(sysReserve((unsafe.Pointer)(h.arena_end), p_size, &reserved)) 323 if p == h.arena_end { 324 h.arena_end = new_end 325 h.arena_reserved = reserved 326 } else if p+p_size <= h.arena_start+_MaxArena32 { 327 // Keep everything page-aligned. 328 // Our pages are bigger than hardware pages. 329 h.arena_end = p + p_size 330 h.arena_used = p + (-uintptr(p) & (_PageSize - 1)) 331 h.arena_reserved = reserved 332 } else { 333 var stat uint64 334 sysFree((unsafe.Pointer)(p), p_size, &stat) 335 } 336 } 337 } 338 339 if n <= uintptr(h.arena_end)-uintptr(h.arena_used) { 340 // Keep taking from our reservation. 341 p := h.arena_used 342 sysMap((unsafe.Pointer)(p), n, h.arena_reserved, &memstats.heap_sys) 343 h.arena_used += n 344 mHeap_MapBits(h) 345 mHeap_MapSpans(h) 346 if raceenabled { 347 racemapshadow((unsafe.Pointer)(p), n) 348 } 349 if mheap_.shadow_enabled { 350 sysMap(unsafe.Pointer(p+mheap_.shadow_heap), n, h.shadow_reserved, &memstats.other_sys) 351 } 352 353 if uintptr(p)&(_PageSize-1) != 0 { 354 throw("misrounded allocation in MHeap_SysAlloc") 355 } 356 return (unsafe.Pointer)(p) 357 } 358 359 // If using 64-bit, our reservation is all we have. 360 if uintptr(h.arena_end)-uintptr(h.arena_start) >= _MaxArena32 { 361 return nil 362 } 363 364 // On 32-bit, once the reservation is gone we can 365 // try to get memory at a location chosen by the OS 366 // and hope that it is in the range we allocated bitmap for. 367 p_size := round(n, _PageSize) + _PageSize 368 p := uintptr(sysAlloc(p_size, &memstats.heap_sys)) 369 if p == 0 { 370 return nil 371 } 372 373 if p < h.arena_start || uintptr(p)+p_size-uintptr(h.arena_start) >= _MaxArena32 { 374 print("runtime: memory allocated by OS (", p, ") not in usable range [", hex(h.arena_start), ",", hex(h.arena_start+_MaxArena32), ")\n") 375 sysFree((unsafe.Pointer)(p), p_size, &memstats.heap_sys) 376 return nil 377 } 378 379 p_end := p + p_size 380 p += -p & (_PageSize - 1) 381 if uintptr(p)+n > uintptr(h.arena_used) { 382 h.arena_used = p + n 383 if p_end > h.arena_end { 384 h.arena_end = p_end 385 } 386 mHeap_MapBits(h) 387 mHeap_MapSpans(h) 388 if raceenabled { 389 racemapshadow((unsafe.Pointer)(p), n) 390 } 391 } 392 393 if uintptr(p)&(_PageSize-1) != 0 { 394 throw("misrounded allocation in MHeap_SysAlloc") 395 } 396 return (unsafe.Pointer)(p) 397 } 398 399 var end struct{} 400 401 func largeAlloc(size uintptr, flag uint32) *mspan { 402 // print("largeAlloc size=", size, "\n") 403 404 if size+_PageSize < size { 405 throw("out of memory") 406 } 407 npages := size >> _PageShift 408 if size&_PageMask != 0 { 409 npages++ 410 } 411 s := mHeap_Alloc(&mheap_, npages, 0, true, flag&_FlagNoZero == 0) 412 if s == nil { 413 throw("out of memory") 414 } 415 s.limit = uintptr(s.start)<<_PageShift + size 416 v := unsafe.Pointer(uintptr(s.start) << _PageShift) 417 // setup for mark sweep 418 markspan(v, 0, 0, true) 419 return s 420 }