github.com/zxy12/golang_with_comment@v0.0.0-20190701084843-0e6b2aff5ef3/runtime/malloc.go (about) 1 // Copyright 2014 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 // Memory allocator. 6 // 7 // This was originally based on tcmalloc, but has diverged quite a bit. 8 // http://goog-perftools.sourceforge.net/doc/tcmalloc.html 9 10 // The main allocator works in runs of pages. 11 // Small allocation sizes (up to and including 32 kB) are 12 // rounded to one of about 70 size classes, each of which 13 // has its own free set of objects of exactly that size. 14 // Any free page of memory can be split into a set of objects 15 // of one size class, which are then managed using a free bitmap. 16 // 17 // The allocator's data structures are: 18 // 19 // fixalloc: a free-list allocator for fixed-size off-heap objects, 20 // used to manage storage used by the allocator. 21 // mheap: the malloc heap, managed at page (8192-byte) granularity. 22 // mspan: a run of pages managed by the mheap. 23 // mcentral: collects all spans of a given size class. 24 // mcache: a per-P cache of mspans with free space. 25 // mstats: allocation statistics. 26 // 27 // Allocating a small object proceeds up a hierarchy of caches: 28 // 29 // 1. Round the size up to one of the small size classes 30 // and look in the corresponding mspan in this P's mcache. 31 // Scan the mspan's free bitmap to find a free slot. 32 // If there is a free slot, allocate it. 33 // This can all be done without acquiring a lock. 34 // 35 // 2. If the mspan has no free slots, obtain a new mspan 36 // from the mcentral's list of mspans of the required size 37 // class that have free space. 38 // Obtaining a whole span amortizes the cost of locking 39 // the mcentral. 40 // 41 // 3. If the mcentral's mspan list is empty, obtain a run 42 // of pages from the mheap to use for the mspan. 43 // 44 // 4. If the mheap is empty or has no page runs large enough, 45 // allocate a new group of pages (at least 1MB) from the 46 // operating system. Allocating a large run of pages 47 // amortizes the cost of talking to the operating system. 48 // 49 // Sweeping an mspan and freeing objects on it proceeds up a similar 50 // hierarchy: 51 // 52 // 1. If the mspan is being swept in response to allocation, it 53 // is returned to the mcache to satisfy the allocation. 54 // 55 // 2. Otherwise, if the mspan still has allocated objects in it, 56 // it is placed on the mcentral free list for the mspan's size 57 // class. 58 // 59 // 3. Otherwise, if all objects in the mspan are free, the mspan 60 // is now "idle", so it is returned to the mheap and no longer 61 // has a size class. 62 // This may coalesce it with adjacent idle mspans. 63 // 64 // 4. If an mspan remains idle for long enough, return its pages 65 // to the operating system. 66 // 67 // Allocating and freeing a large object uses the mheap 68 // directly, bypassing the mcache and mcentral. 69 // 70 // Free object slots in an mspan are zeroed only if mspan.needzero is 71 // false. If needzero is true, objects are zeroed as they are 72 // allocated. There are various benefits to delaying zeroing this way: 73 // 74 // 1. Stack frame allocation can avoid zeroing altogether. 75 // 76 // 2. It exhibits better temporal locality, since the program is 77 // probably about to write to the memory. 78 // 79 // 3. We don't zero pages that never get reused. 80 81 package runtime 82 83 import ( 84 "runtime/internal/sys" 85 "unsafe" 86 ) 87 88 /* 89 90 // sizeclass 91 // class bytes/obj bytes/span objects waste bytes 92 // 1 8 8192 1024 0 93 // 2 16 8192 512 0 94 // 3 32 8192 256 0 95 // 4 48 8192 170 32 96 // 5 64 8192 128 0 97 // 6 80 8192 102 32 98 // 7 96 8192 85 32 99 // 8 112 8192 73 16 100 // 9 128 8192 64 0 101 // 10 144 8192 56 128 102 // 11 160 8192 51 32 103 // 12 176 8192 46 96 104 // 13 192 8192 42 128 105 // 14 208 8192 39 80 106 // 15 224 8192 36 128 107 // 16 240 8192 34 32 108 // 17 256 8192 32 0 109 // 18 288 8192 28 128 110 // 19 320 8192 25 192 111 // 20 352 8192 23 96 112 // 21 384 8192 21 128 113 // 22 416 8192 19 288 114 // 23 448 8192 18 128 115 // 24 480 8192 17 32 116 // 25 512 8192 16 0 117 // 26 576 8192 14 128 118 // 27 640 8192 12 512 119 // 28 704 8192 11 448 120 // 29 768 8192 10 512 121 // 30 896 8192 9 128 122 // 31 1024 8192 8 0 123 // 32 1152 8192 7 128 124 // 33 1280 8192 6 512 125 // 34 1408 16384 11 896 126 // 35 1536 8192 5 512 127 // 36 1792 16384 9 256 128 // 37 2048 8192 4 0 129 // 38 2304 16384 7 256 130 // 39 2688 8192 3 128 131 // 40 3072 24576 8 0 132 // 41 3200 16384 5 384 133 // 42 3456 24576 7 384 134 // 43 4096 8192 2 0 135 // 44 4864 24576 5 256 136 // 45 5376 16384 3 256 137 // 46 6144 24576 4 0 138 // 47 6528 32768 5 128 139 // 48 6784 40960 6 256 140 // 49 6912 49152 7 768 141 // 50 8192 8192 1 0 142 // 51 9472 57344 6 512 143 // 52 9728 49152 5 512 144 // 53 10240 40960 4 0 145 // 54 10880 32768 3 128 146 // 55 12288 24576 2 0 147 // 56 13568 40960 3 256 148 // 57 14336 57344 4 0 149 // 58 16384 16384 1 0 150 // 59 18432 73728 4 0 151 // 60 19072 57344 3 128 152 // 61 20480 40960 2 0 153 // 62 21760 65536 3 256 154 // 63 24576 24576 1 0 155 // 64 27264 81920 3 128 156 // 65 28672 57344 2 0 157 // 66 32768 32768 1 0 158 159 */ 160 161 const ( 162 debugMalloc = false 163 164 maxTinySize = _TinySize 165 tinySizeClass = _TinySizeClass 166 maxSmallSize = _MaxSmallSize 167 168 pageShift = _PageShift 169 pageSize = _PageSize 170 pageMask = _PageMask 171 // By construction, single page spans of the smallest object class 172 // have the most objects per span. 173 maxObjsPerSpan = pageSize / 8 174 175 mSpanInUse = _MSpanInUse 176 177 concurrentSweep = _ConcurrentSweep 178 179 _PageSize = 1 << _PageShift 180 _PageMask = _PageSize - 1 181 182 // _64bit = 1 on 64-bit systems, 0 on 32-bit systems 183 _64bit = 1 << (^uintptr(0) >> 63) / 2 184 185 // Tiny allocator parameters, see "Tiny allocator" comment in malloc.go. 186 _TinySize = 16 187 _TinySizeClass = int8(2) 188 189 _FixAllocChunk = 16 << 10 // Chunk size for FixAlloc 190 _MaxMHeapList = 1 << (20 - _PageShift) // Maximum page length for fixed-size list in MHeap. 191 _HeapAllocChunk = 1 << 20 // Chunk size for heap growth 192 193 // Per-P, per order stack segment cache size. 194 _StackCacheSize = 32 * 1024 195 196 // Number of orders that get caching. Order 0 is FixedStack 197 // and each successive order is twice as large. 198 // We want to cache 2KB, 4KB, 8KB, and 16KB stacks. Larger stacks 199 // will be allocated directly. 200 // Since FixedStack is different on different systems, we 201 // must vary NumStackOrders to keep the same maximum cached size. 202 // OS | FixedStack | NumStackOrders 203 // -----------------+------------+--------------- 204 // linux/darwin/bsd | 2KB | 4 205 // windows/32 | 4KB | 3 206 // windows/64 | 8KB | 2 207 // plan9 | 4KB | 3 208 _NumStackOrders = 4 - sys.PtrSize/4*sys.GoosWindows - 1*sys.GoosPlan9 209 210 // Number of bits in page to span calculations (4k pages). 211 // On Windows 64-bit we limit the arena to 32GB or 35 bits. 212 // Windows counts memory used by page table into committed memory 213 // of the process, so we can't reserve too much memory. 214 // See https://golang.org/issue/5402 and https://golang.org/issue/5236. 215 // On other 64-bit platforms, we limit the arena to 512GB, or 39 bits. 216 // On 32-bit, we don't bother limiting anything, so we use the full 32-bit address. 217 // The only exception is mips32 which only has access to low 2GB of virtual memory. 218 // On Darwin/arm64, we cannot reserve more than ~5GB of virtual memory, 219 // but as most devices have less than 4GB of physical memory anyway, we 220 // try to be conservative here, and only ask for a 2GB heap. 221 _MHeapMap_TotalBits = (_64bit*sys.GoosWindows)*35 + (_64bit*(1-sys.GoosWindows)*(1-sys.GoosDarwin*sys.GoarchArm64))*39 + sys.GoosDarwin*sys.GoarchArm64*31 + (1-_64bit)*(32-(sys.GoarchMips+sys.GoarchMipsle)) 222 _MHeapMap_Bits = _MHeapMap_TotalBits - _PageShift 223 224 // _MaxMem is the maximum heap arena size minus 1. 225 // 226 // On 32-bit, this is also the maximum heap pointer value, 227 // since the arena starts at address 0. 228 _MaxMem = 1<<_MHeapMap_TotalBits - 1 229 230 // Max number of threads to run garbage collection. 231 // 2, 3, and 4 are all plausible maximums depending 232 // on the hardware details of the machine. The garbage 233 // collector scales well to 32 cpus. 234 _MaxGcproc = 32 235 236 // minLegalPointer is the smallest possible legal pointer. 237 // This is the smallest possible architectural page size, 238 // since we assume that the first page is never mapped. 239 // 240 // This should agree with minZeroPage in the compiler. 241 minLegalPointer uintptr = 4096 242 ) 243 244 // physPageSize is the size in bytes of the OS's physical pages. 245 // Mapping and unmapping operations must be done at multiples of 246 // physPageSize. 247 // 248 // This must be set by the OS init code (typically in osinit) before 249 // mallocinit. 250 var physPageSize uintptr 251 var debugSize bool 252 253 // OS-defined helpers: 254 // 255 // sysAlloc obtains a large chunk of zeroed memory from the 256 // operating system, typically on the order of a hundred kilobytes 257 // or a megabyte. 258 // NOTE: sysAlloc returns OS-aligned memory, but the heap allocator 259 // may use larger alignment, so the caller must be careful to realign the 260 // memory obtained by sysAlloc. 261 // 262 // SysUnused notifies the operating system that the contents 263 // of the memory region are no longer needed and can be reused 264 // for other purposes. 265 // SysUsed notifies the operating system that the contents 266 // of the memory region are needed again. 267 // 268 // SysFree returns it unconditionally; this is only used if 269 // an out-of-memory error has been detected midway through 270 // an allocation. It is okay if SysFree is a no-op. 271 // 272 // SysReserve reserves address space without allocating memory. 273 // If the pointer passed to it is non-nil, the caller wants the 274 // reservation there, but SysReserve can still choose another 275 // location if that one is unavailable. On some systems and in some 276 // cases SysReserve will simply check that the address space is 277 // available and not actually reserve it. If SysReserve returns 278 // non-nil, it sets *reserved to true if the address space is 279 // reserved, false if it has merely been checked. 280 // NOTE: SysReserve returns OS-aligned memory, but the heap allocator 281 // may use larger alignment, so the caller must be careful to realign the 282 // memory obtained by sysAlloc. 283 // 284 // SysMap maps previously reserved address space for use. 285 // The reserved argument is true if the address space was really 286 // reserved, not merely checked. 287 // 288 // SysFault marks a (already sysAlloc'd) region to fault 289 // if accessed. Used only for debugging the runtime. 290 291 func print_const() { 292 /* 293 294 debugMalloc=false 295 maxTinySize=16 296 tinySizeClass=2 297 maxSmallSize=32768 298 pageShift=13 299 pageSize=8192 300 pageMask=8191 301 maxObjsPerSpan=1024 302 mSpanInUse=1 303 concurrentSweep=true 304 _PageSize=8192 305 _PageMask=8191 306 _64bit=1 307 _TinySize=16 308 _TinySizeClass=2 309 _FixAllocChunk=16384 310 _MaxMHeapList=128 311 _HeapAllocChunk=1048576 312 _StackCacheSize=32768 313 _NumStackOrders=4 314 _MHeapMap_TotalBits=39 315 _MaxMem=549755813887 316 _MaxGcproc=32 317 minLegalPointer=4096 318 physPageSize=4096 319 _NumSizeClasses= 67 320 numSpanClasses= 134 321 tinySpanClass= 5 322 */ 323 print("debugMalloc=", debugMalloc, "\n") 324 print("maxTinySize=", maxTinySize, "\n") 325 print("tinySizeClass=", tinySizeClass, "\n") 326 print("maxSmallSize=", maxSmallSize, "\n") 327 print("pageShift=", pageShift, "\n") 328 print("pageSize=", pageSize, "\n") 329 print("pageMask=", pageMask, "\n") 330 print("maxObjsPerSpan=", maxObjsPerSpan, "\n") 331 print("mSpanInUse=", mSpanInUse, "\n") 332 print("concurrentSweep=", concurrentSweep, "\n") 333 print("_PageSize=", _PageSize, "\n") 334 print("_PageMask=", _PageMask, "\n") 335 print("_64bit=", _64bit, "\n") 336 print("_TinySize=", _TinySize, "\n") 337 print("_TinySizeClass=", _TinySizeClass, "\n") 338 print("_FixAllocChunk=", _FixAllocChunk, "\n") 339 print("_MaxMHeapList=", _MaxMHeapList, "\n") 340 print("_HeapAllocChunk=", _HeapAllocChunk, "\n") 341 print("_StackCacheSize=", _StackCacheSize, "\n") 342 print("_NumStackOrders=", _NumStackOrders, "\n") 343 print("_MHeapMap_TotalBits=", _MHeapMap_TotalBits, "\n") 344 print("_MaxMem=", _MaxMem, "\n") 345 print("_MaxGcproc=", _MaxGcproc, "\n") 346 print("minLegalPointer=", minLegalPointer, "\n") 347 print("physPageSize=", physPageSize, "\n") 348 //println("_NumSizeClasses=", _NumSizeClasses) 349 //println("numSpanClasses=", numSpanClasses) 350 //println("tinySpanClass=", tinySpanClass) 351 } 352 353 func mallocinit() { 354 355 //var class_to_size = [_NumSizeClasses]uint16{0, 8, 16, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 288, 320, 352, 384, 416, 448, 480, 512, 576, 640, 704, 768, 896, 1024, 1152, 1280, 1408, 1536, 1792, 2048, 2304, 2688, 3072, 3200, 3456, 4096, 4864, 5376, 6144, 6528, 6784, 6912, 8192, 9472, 9728, 10240, 10880, 12288, 13568, 14336, 16384, 18432, 19072, 20480, 21760, 24576, 27264, 28672, 32768} 356 // _TinySize=16 357 if class_to_size[_TinySizeClass] != _TinySize { 358 throw("bad TinySizeClass") 359 } 360 361 testdefersizes() 362 363 // Copy class sizes out for statistics table. 364 for i := range class_to_size { 365 memstats.by_size[i].size = uint32(class_to_size[i]) 366 } 367 368 // Check physPageSize. 369 if physPageSize == 0 { 370 // The OS init code failed to fetch the physical page size. 371 throw("failed to get system page size") 372 } 373 if physPageSize < minPhysPageSize { // minPhysPageSize=4096 374 print("system page size (", physPageSize, ") is smaller than minimum page size (", minPhysPageSize, ")\n") 375 throw("bad system page size") 376 } 377 if physPageSize&(physPageSize-1) != 0 { 378 print("system page size (", physPageSize, ") must be a power of 2\n") 379 throw("bad system page size") 380 } 381 382 // The auxiliary regions start at p and are laid out in the 383 // following order: spans, bitmap, arena. 384 var p, pSize uintptr 385 var reserved bool 386 387 // The spans array holds one *mspan per _PageSize of arena. 388 var spansSize uintptr = (_MaxMem + 1) / _PageSize * sys.PtrSize 389 spansSize = round(spansSize, _PageSize) 390 // The bitmap holds 2 bits per word of arena. 391 var bitmapSize uintptr = (_MaxMem + 1) / (sys.PtrSize * 8 / 2) 392 bitmapSize = round(bitmapSize, _PageSize) 393 394 print("arena size: ", _MaxMem, "\n") // 512G 395 print("spans size: ", spansSize, "\n") // 512M 396 print("bitmap size: ", bitmapSize, "\n") // 16G 397 //print_const() 398 // Set up the allocation arena, a contiguous area of memory where 399 // allocated data will be found. 400 if sys.PtrSize == 8 { 401 // On a 64-bit machine, allocate from a single contiguous reservation. 402 // 512 GB (MaxMem) should be big enough for now. 403 // 404 // The code will work with the reservation at any address, but ask 405 // SysReserve to use 0x0000XXc000000000 if possible (XX=00...7f). 406 // Allocating a 512 GB region takes away 39 bits, and the amd64 407 // doesn't let us choose the top 17 bits, so that leaves the 9 bits 408 // in the middle of 0x00c0 for us to choose. Choosing 0x00c0 means 409 // that the valid memory addresses will begin 0x00c0, 0x00c1, ..., 0x00df. 410 // In little-endian, that's c0 00, c1 00, ..., df 00. None of those are valid 411 // UTF-8 sequences, and they are otherwise as far away from 412 // ff (likely a common byte) as possible. If that fails, we try other 0xXXc0 413 // addresses. An earlier attempt to use 0x11f8 caused out of memory errors 414 // on OS X during thread allocations. 0x00c0 causes conflicts with 415 // AddressSanitizer which reserves all memory up to 0x0100. 416 // These choices are both for debuggability and to reduce the 417 // odds of a conservative garbage collector (as is still used in gccgo) 418 // not collecting memory because some non-pointer block of memory 419 // had a bit pattern that matched a memory address. 420 // 421 // Actually we reserve 544 GB (because the bitmap ends up being 32 GB) 422 // but it hardly matters: e0 00 is not valid UTF-8 either. 423 // 424 // If this fails we fall back to the 32 bit memory mechanism 425 // 426 // However, on arm64, we ignore all this advice above and slam the 427 // allocation at 0x40 << 32 because when using 4k pages with 3-level 428 // translation buffers, the user address space is limited to 39 bits 429 // On darwin/arm64, the address space is even smaller. 430 arenaSize := round(_MaxMem, _PageSize) 431 pSize = bitmapSize + spansSize + arenaSize + _PageSize 432 for i := 0; i <= 0x7f; i++ { 433 switch { 434 case GOARCH == "arm64" && GOOS == "darwin": 435 p = uintptr(i)<<40 | uintptrMask&(0x0013<<28) 436 case GOARCH == "arm64": 437 p = uintptr(i)<<40 | uintptrMask&(0x0040<<32) 438 default: 439 p = uintptr(i)<<40 | uintptrMask&(0x00c0<<32) 440 } 441 //println("i=", i, ", p=", p) 442 p = uintptr(sysReserve(unsafe.Pointer(p), pSize, &reserved)) 443 if p != 0 { 444 break 445 } 446 } 447 } 448 449 if p == 0 { 450 // On a 32-bit machine, we can't typically get away 451 // with a giant virtual address space reservation. 452 // Instead we map the memory information bitmap 453 // immediately after the data segment, large enough 454 // to handle the entire 4GB address space (256 MB), 455 // along with a reservation for an initial arena. 456 // When that gets used up, we'll start asking the kernel 457 // for any memory anywhere. 458 459 // We want to start the arena low, but if we're linked 460 // against C code, it's possible global constructors 461 // have called malloc and adjusted the process' brk. 462 // Query the brk so we can avoid trying to map the 463 // arena over it (which will cause the kernel to put 464 // the arena somewhere else, likely at a high 465 // address). 466 procBrk := sbrk0() 467 468 // If we fail to allocate, try again with a smaller arena. 469 // This is necessary on Android L where we share a process 470 // with ART, which reserves virtual memory aggressively. 471 // In the worst case, fall back to a 0-sized initial arena, 472 // in the hope that subsequent reservations will succeed. 473 arenaSizes := []uintptr{ 474 512 << 20, 475 256 << 20, 476 128 << 20, 477 0, 478 } 479 480 for _, arenaSize := range arenaSizes { 481 // SysReserve treats the address we ask for, end, as a hint, 482 // not as an absolute requirement. If we ask for the end 483 // of the data segment but the operating system requires 484 // a little more space before we can start allocating, it will 485 // give out a slightly higher pointer. Except QEMU, which 486 // is buggy, as usual: it won't adjust the pointer upward. 487 // So adjust it upward a little bit ourselves: 1/4 MB to get 488 // away from the running binary image and then round up 489 // to a MB boundary. 490 p = round(firstmoduledata.end+(1<<18), 1<<20) 491 pSize = bitmapSize + spansSize + arenaSize + _PageSize 492 if p <= procBrk && procBrk < p+pSize { 493 // Move the start above the brk, 494 // leaving some room for future brk 495 // expansion. 496 p = round(procBrk+(1<<20), 1<<20) 497 } 498 p = uintptr(sysReserve(unsafe.Pointer(p), pSize, &reserved)) 499 if p != 0 { 500 break 501 } 502 } 503 if p == 0 { 504 throw("runtime: cannot reserve arena virtual address space") 505 } 506 } 507 508 // PageSize can be larger than OS definition of page size, 509 // so SysReserve can give us a PageSize-unaligned pointer. 510 // To overcome this we ask for PageSize more and round up the pointer. 511 p1 := round(p, _PageSize) 512 print("p=", p, "\n") //19349504 513 print("p1=", p1, "\n") //19349504 514 pSize -= p1 - p 515 516 spansStart := p1 517 p1 += spansSize 518 mheap_.bitmap = p1 + bitmapSize 519 p1 += bitmapSize 520 if sys.PtrSize == 4 { 521 // Set arena_start such that we can accept memory 522 // reservations located anywhere in the 4GB virtual space. 523 mheap_.arena_start = 0 524 } else { 525 mheap_.arena_start = p1 526 } 527 //sp := (*int)(unsafe.Pointer(p1)) 528 //*sp = 123 529 ////println("p1=", *sp) 530 mheap_.arena_end = p + pSize 531 mheap_.arena_used = p1 532 mheap_.arena_alloc = p1 533 mheap_.arena_reserved = reserved 534 535 print("mheap_.arena_start=", mheap_.arena_start, "\n") 536 537 if mheap_.arena_start&(_PageSize-1) != 0 { 538 //println("bad pagesize", hex(p), hex(p1), hex(spansSize), hex(bitmapSize), hex(_PageSize), "start", hex(mheap_.arena_start)) 539 throw("misrounded allocation in mallocinit") 540 } 541 542 // Initialize the rest of the allocator. 543 mheap_.init(spansStart, spansSize) 544 _g_ := getg() 545 _g_.m.mcache = allocmcache() 546 // 总结一下mallocinit的工作:申请一块虚拟内存,划分spans\bitmap\arena区域的指针指向,把所有结构体中的指针初始化为nil 547 } 548 549 // sysAlloc allocates the next n bytes from` the heap arena. The 550 // returned pointer is always _PageSize aligned and between 551 // h.arena_start and h.arena_end. sysAlloc returns nil on failure. 552 // There is no corresponding free function. 553 func (h *mheap) sysAlloc(n uintptr) unsafe.Pointer { 554 // strandLimit is the maximum number of bytes to strand from 555 // the current arena block. If we would need to strand more 556 // than this, we fall back to sysAlloc'ing just enough for 557 // this allocation. 558 const strandLimit = 16 << 20 559 560 if n > h.arena_end-h.arena_alloc { 561 // If we haven't grown the arena to _MaxMem yet, try 562 // to reserve some more address space. 563 p_size := round(n+_PageSize, 256<<20) 564 new_end := h.arena_end + p_size // Careful: can overflow 565 if h.arena_end <= new_end && new_end-h.arena_start-1 <= _MaxMem { 566 // TODO: It would be bad if part of the arena 567 // is reserved and part is not. 568 var reserved bool 569 p := uintptr(sysReserve(unsafe.Pointer(h.arena_end), p_size, &reserved)) 570 if p == 0 { 571 // TODO: Try smaller reservation 572 // growths in case we're in a crowded 573 // 32-bit address space. 574 goto reservationFailed 575 } 576 // p can be just about anywhere in the address 577 // space, including before arena_end. 578 if p == h.arena_end { 579 // The new block is contiguous with 580 // the current block. Extend the 581 // current arena block. 582 h.arena_end = new_end 583 h.arena_reserved = reserved 584 } else if h.arena_start <= p && p+p_size-h.arena_start-1 <= _MaxMem && h.arena_end-h.arena_alloc < strandLimit { 585 // We were able to reserve more memory 586 // within the arena space, but it's 587 // not contiguous with our previous 588 // reservation. It could be before or 589 // after our current arena_used. 590 // 591 // Keep everything page-aligned. 592 // Our pages are bigger than hardware pages. 593 h.arena_end = p + p_size 594 p = round(p, _PageSize) 595 h.arena_alloc = p 596 h.arena_reserved = reserved 597 } else { 598 // We got a mapping, but either 599 // 600 // 1) It's not in the arena, so we 601 // can't use it. (This should never 602 // happen on 32-bit.) 603 // 604 // 2) We would need to discard too 605 // much of our current arena block to 606 // use it. 607 // 608 // We haven't added this allocation to 609 // the stats, so subtract it from a 610 // fake stat (but avoid underflow). 611 // 612 // We'll fall back to a small sysAlloc. 613 stat := uint64(p_size) 614 sysFree(unsafe.Pointer(p), p_size, &stat) 615 } 616 } 617 } 618 619 if n <= h.arena_end-h.arena_alloc { 620 // Keep taking from our reservation. 621 p := h.arena_alloc 622 sysMap(unsafe.Pointer(p), n, h.arena_reserved, &memstats.heap_sys) 623 h.arena_alloc += n 624 if h.arena_alloc > h.arena_used { 625 h.setArenaUsed(h.arena_alloc, true) 626 } 627 628 if p&(_PageSize-1) != 0 { 629 throw("misrounded allocation in MHeap_SysAlloc") 630 } 631 return unsafe.Pointer(p) 632 } 633 634 reservationFailed: 635 // If using 64-bit, our reservation is all we have. 636 if sys.PtrSize != 4 { 637 return nil 638 } 639 640 // On 32-bit, once the reservation is gone we can 641 // try to get memory at a location chosen by the OS. 642 p_size := round(n, _PageSize) + _PageSize 643 p := uintptr(sysAlloc(p_size, &memstats.heap_sys)) 644 if p == 0 { 645 return nil 646 } 647 648 if p < h.arena_start || p+p_size-h.arena_start > _MaxMem { 649 // This shouldn't be possible because _MaxMem is the 650 // whole address space on 32-bit. 651 top := uint64(h.arena_start) + _MaxMem 652 print("runtime: memory allocated by OS (", hex(p), ") not in usable range [", hex(h.arena_start), ",", hex(top), ")\n") 653 sysFree(unsafe.Pointer(p), p_size, &memstats.heap_sys) 654 return nil 655 } 656 657 p += -p & (_PageSize - 1) 658 if p+n > h.arena_used { 659 h.setArenaUsed(p+n, true) 660 } 661 662 if p&(_PageSize-1) != 0 { 663 throw("misrounded allocation in MHeap_SysAlloc") 664 } 665 return unsafe.Pointer(p) 666 } 667 668 // base address for all 0-byte allocations 669 var zerobase uintptr 670 671 // nextFreeFast returns the next free object if one is quickly available. 672 // Otherwise it returns 0. 673 func nextFreeFast(s *mspan) gclinkptr { 674 theBit := sys.Ctz64(s.allocCache) // Is there a free object in the allocCache? 675 if theBit < 64 { 676 result := s.freeindex + uintptr(theBit) 677 if result < s.nelems { 678 freeidx := result + 1 679 if freeidx%64 == 0 && freeidx != s.nelems { 680 return 0 681 } 682 s.allocCache >>= uint(theBit + 1) 683 s.freeindex = freeidx 684 v := gclinkptr(result*s.elemsize + s.base()) 685 s.allocCount++ 686 return v 687 } 688 } 689 return 0 690 } 691 692 // nextFree returns the next free object from the cached span if one is available. 693 // Otherwise it refills the cache with a span with an available object and 694 // returns that object along with a flag indicating that this was a heavy 695 // weight allocation. If it is a heavy weight allocation the caller must 696 // determine whether a new GC cycle needs to be started or if the GC is active 697 // whether this goroutine needs to assist the GC. 698 func (c *mcache) nextFree(spc spanClass) (v gclinkptr, s *mspan, shouldhelpgc bool) { 699 s = c.alloc[spc] 700 shouldhelpgc = false 701 freeIndex := s.nextFreeIndex() 702 if freeIndex == s.nelems { 703 // The span is full. 704 if uintptr(s.allocCount) != s.nelems { 705 //println("runtime: s.allocCount=", s.allocCount, "s.nelems=", s.nelems) 706 throw("s.allocCount != s.nelems && freeIndex == s.nelems") 707 } 708 systemstack(func() { 709 c.refill(spc) 710 }) 711 shouldhelpgc = true 712 s = c.alloc[spc] 713 714 freeIndex = s.nextFreeIndex() 715 } 716 717 if freeIndex >= s.nelems { 718 throw("freeIndex is not valid") 719 } 720 721 v = gclinkptr(freeIndex*s.elemsize + s.base()) 722 s.allocCount++ 723 if uintptr(s.allocCount) > s.nelems { 724 //println("s.allocCount=", s.allocCount, "s.nelems=", s.nelems) 725 throw("s.allocCount > s.nelems") 726 } 727 return 728 } 729 730 // Allocate an object of size bytes. 731 // Small objects are allocated from the per-P cache's free lists. 732 // Large objects (> 32 kB) are allocated straight from the heap. 733 func mallocgc(size uintptr, typ *_type, needzero bool) unsafe.Pointer { 734 735 if size == 79992 { 736 debugSize = true 737 _ = debugSize 738 } 739 740 if gcphase == _GCmarktermination { 741 throw("mallocgc called with gcphase == _GCmarktermination") 742 } 743 744 if size == 0 { 745 return unsafe.Pointer(&zerobase) 746 } 747 748 if debug.sbrk != 0 { 749 align := uintptr(16) 750 if typ != nil { 751 align = uintptr(typ.align) 752 } 753 return persistentalloc(size, align, &memstats.other_sys) 754 } 755 756 // assistG is the G to charge for this allocation, or nil if 757 // GC is not currently active. 758 var assistG *g 759 if gcBlackenEnabled != 0 { 760 // Charge the current user G for this allocation. 761 assistG = getg() 762 if assistG.m.curg != nil { 763 assistG = assistG.m.curg 764 } 765 // Charge the allocation against the G. We'll account 766 // for internal fragmentation at the end of mallocgc. 767 assistG.gcAssistBytes -= int64(size) 768 769 if assistG.gcAssistBytes < 0 { 770 // This G is in debt. Assist the GC to correct 771 // this before allocating. This must happen 772 // before disabling preemption. 773 gcAssistAlloc(assistG) 774 } 775 } 776 777 // Set mp.mallocing to keep from being preempted by GC. 778 mp := acquirem() 779 if mp.mallocing != 0 { 780 throw("malloc deadlock") 781 } 782 if mp.gsignal == getg() { 783 throw("malloc during signal") 784 } 785 mp.mallocing = 1 786 787 shouldhelpgc := false 788 dataSize := size 789 c := gomcache() 790 var x unsafe.Pointer 791 noscan := typ == nil || typ.kind&kindNoPointers != 0 792 if size <= maxSmallSize { 793 if noscan && size < maxTinySize { 794 // Tiny allocator. 795 // 796 // Tiny allocator combines several tiny allocation requests 797 // into a single memory block. The resulting memory block 798 // is freed when all subobjects are unreachable. The subobjects 799 // must be noscan (don't have pointers), this ensures that 800 // the amount of potentially wasted memory is bounded. 801 // 802 // Size of the memory block used for combining (maxTinySize) is tunable. 803 // Current setting is 16 bytes, which relates to 2x worst case memory 804 // wastage (when all but one subobjects are unreachable). 805 // 8 bytes would result in no wastage at all, but provides less 806 // opportunities for combining. 807 // 32 bytes provides more opportunities for combining, 808 // but can lead to 4x worst case wastage. 809 // The best case winning is 8x regardless of block size. 810 // 811 // Objects obtained from tiny allocator must not be freed explicitly. 812 // So when an object will be freed explicitly, we ensure that 813 // its size >= maxTinySize. 814 // 815 // SetFinalizer has a special case for objects potentially coming 816 // from tiny allocator, it such case it allows to set finalizers 817 // for an inner byte of a memory block. 818 // 819 // The main targets of tiny allocator are small strings and 820 // standalone escaping variables. On a json benchmark 821 // the allocator reduces number of allocations by ~12% and 822 // reduces heap size by ~20%. 823 off := c.tinyoffset 824 ////println("size:", size, "off-start:", off) 825 // Align tiny pointer for required (conservative) alignment. 826 if size&7 == 0 { 827 off = round(off, 8) 828 } else if size&3 == 0 { 829 off = round(off, 4) 830 } else if size&1 == 0 { 831 off = round(off, 2) 832 } 833 ////println("off-round:", off) 834 835 if off+size <= maxTinySize && c.tiny != 0 { 836 // The object fits into existing tiny block. 837 x = unsafe.Pointer(c.tiny + off) 838 c.tinyoffset = off + size 839 c.local_tinyallocs++ 840 mp.mallocing = 0 841 releasem(mp) 842 return x 843 } 844 // Allocate a new maxTinySize block. 845 span := c.alloc[tinySpanClass] 846 v := nextFreeFast(span) 847 if v == 0 { 848 v, _, shouldhelpgc = c.nextFree(tinySpanClass) 849 } 850 x = unsafe.Pointer(v) 851 (*[2]uint64)(x)[0] = 0 852 (*[2]uint64)(x)[1] = 0 853 // See if we need to replace the existing tiny block with the new one 854 // based on amount of remaining free space. 855 if size < c.tinyoffset || c.tiny == 0 { 856 c.tiny = uintptr(x) 857 c.tinyoffset = size 858 } 859 size = maxTinySize 860 } else { 861 var sizeclass uint8 862 if size <= smallSizeMax-8 { 863 sizeclass = size_to_class8[(size+smallSizeDiv-1)/smallSizeDiv] 864 } else { 865 sizeclass = size_to_class128[(size-smallSizeMax+largeSizeDiv-1)/largeSizeDiv] 866 } 867 size = uintptr(class_to_size[sizeclass]) 868 spc := makeSpanClass(sizeclass, noscan) 869 span := c.alloc[spc] 870 v := nextFreeFast(span) 871 if v == 0 { 872 v, span, shouldhelpgc = c.nextFree(spc) 873 } 874 x = unsafe.Pointer(v) 875 if needzero && span.needzero != 0 { 876 memclrNoHeapPointers(unsafe.Pointer(v), size) 877 } 878 } 879 } else { 880 //println("now we need:", size, ",noscan=", noscan) 881 var s *mspan 882 shouldhelpgc = true 883 systemstack(func() { 884 s = largeAlloc(size, needzero, noscan) 885 }) 886 s.freeindex = 1 887 s.allocCount = 1 888 x = unsafe.Pointer(s.base()) 889 size = s.elemsize 890 } 891 892 var scanSize uintptr 893 if !noscan { 894 // If allocating a defer+arg block, now that we've picked a malloc size 895 // large enough to hold everything, cut the "asked for" size down to 896 // just the defer header, so that the GC bitmap will record the arg block 897 // as containing nothing at all (as if it were unused space at the end of 898 // a malloc block caused by size rounding). 899 // The defer arg areas are scanned as part of scanstack. 900 if typ == deferType { 901 dataSize = unsafe.Sizeof(_defer{}) 902 } 903 heapBitsSetType(uintptr(x), size, dataSize, typ) 904 if dataSize > typ.size { 905 // Array allocation. If there are any 906 // pointers, GC has to scan to the last 907 // element. 908 if typ.ptrdata != 0 { 909 scanSize = dataSize - typ.size + typ.ptrdata 910 } 911 } else { 912 scanSize = typ.ptrdata 913 } 914 c.local_scan += scanSize 915 } 916 917 // Ensure that the stores above that initialize x to 918 // type-safe memory and set the heap bits occur before 919 // the caller can make x observable to the garbage 920 // collector. Otherwise, on weakly ordered machines, 921 // the garbage collector could follow a pointer to x, 922 // but see uninitialized memory or stale heap bits. 923 publicationBarrier() 924 925 // Allocate black during GC. 926 // All slots hold nil so no scanning is needed. 927 // This may be racing with GC so do it atomically if there can be 928 // a race marking the bit. 929 if gcphase != _GCoff { 930 gcmarknewobject(uintptr(x), size, scanSize) 931 } 932 933 if raceenabled { 934 racemalloc(x, size) 935 } 936 937 if msanenabled { 938 msanmalloc(x, size) 939 } 940 941 mp.mallocing = 0 942 releasem(mp) 943 944 if debug.allocfreetrace != 0 { 945 tracealloc(x, size, typ) 946 } 947 948 if rate := MemProfileRate; rate > 0 { 949 if size < uintptr(rate) && int32(size) < c.next_sample { 950 c.next_sample -= int32(size) 951 } else { 952 mp := acquirem() 953 profilealloc(mp, x, size) 954 releasem(mp) 955 } 956 } 957 958 if assistG != nil { 959 // Account for internal fragmentation in the assist 960 // debt now that we know it. 961 assistG.gcAssistBytes -= int64(size - dataSize) 962 } 963 964 if shouldhelpgc { 965 if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { 966 gcStart(gcBackgroundMode, t) 967 } 968 } 969 970 return x 971 } 972 973 func largeAlloc(size uintptr, needzero bool, noscan bool) *mspan { 974 975 if size+_PageSize < size { 976 throw("out of memory") 977 } 978 npages := size >> _PageShift 979 if size&_PageMask != 0 { 980 npages++ 981 } 982 //println("largeAlloc size=", size, ",pages=", npages, ",needzero=", needzero) 983 984 // Deduct credit for this span allocation and sweep if 985 // necessary. mHeap_Alloc will also sweep npages, so this only 986 // pays the debt down to npage pages. 987 deductSweepCredit(npages*_PageSize, npages) 988 989 _spanClass := makeSpanClass(0, noscan) 990 //println("spanClass=", _spanClass) 991 s := mheap_.alloc(npages, _spanClass, true, needzero) 992 if s == nil { 993 throw("out of memory") 994 } 995 s.limit = s.base() + size 996 heapBitsForSpan(s.base()).initSpan(s) 997 return s 998 } 999 1000 // implementation of new builtin 1001 // compiler (both frontend and SSA backend) knows the signature 1002 // of this function 1003 func newobject(typ *_type) unsafe.Pointer { 1004 ////println("newobject=", typ.size) 1005 return mallocgc(typ.size, typ, true) 1006 } 1007 1008 //go:linkname reflect_unsafe_New reflect.unsafe_New 1009 func reflect_unsafe_New(typ *_type) unsafe.Pointer { 1010 return newobject(typ) 1011 } 1012 1013 // newarray allocates an array of n elements of type typ. 1014 func newarray(typ *_type, n int) unsafe.Pointer { 1015 ////println("newobject array=", typ.size, "num=", n) 1016 if n < 0 || uintptr(n) > maxSliceCap(typ.size) { 1017 panic(plainError("runtime: allocation size out of range")) 1018 } 1019 return mallocgc(typ.size*uintptr(n), typ, true) 1020 } 1021 1022 //go:linkname reflect_unsafe_NewArray reflect.unsafe_NewArray 1023 func reflect_unsafe_NewArray(typ *_type, n int) unsafe.Pointer { 1024 return newarray(typ, n) 1025 } 1026 1027 func profilealloc(mp *m, x unsafe.Pointer, size uintptr) { 1028 mp.mcache.next_sample = nextSample() 1029 mProf_Malloc(x, size) 1030 } 1031 1032 // nextSample returns the next sampling point for heap profiling. 1033 // It produces a random variable with a geometric distribution and 1034 // mean MemProfileRate. This is done by generating a uniformly 1035 // distributed random number and applying the cumulative distribution 1036 // function for an exponential. 1037 func nextSample() int32 { 1038 if GOOS == "plan9" { 1039 // Plan 9 doesn't support floating point in note handler. 1040 if g := getg(); g == g.m.gsignal { 1041 return nextSampleNoFP() 1042 } 1043 } 1044 1045 period := MemProfileRate 1046 1047 // make nextSample not overflow. Maximum possible step is 1048 // -ln(1/(1<<kRandomBitCount)) * period, approximately 20 * period. 1049 switch { 1050 case period > 0x7000000: 1051 period = 0x7000000 1052 case period == 0: 1053 return 0 1054 } 1055 1056 // Let m be the sample rate, 1057 // the probability distribution function is m*exp(-mx), so the CDF is 1058 // p = 1 - exp(-mx), so 1059 // q = 1 - p == exp(-mx) 1060 // log_e(q) = -mx 1061 // -log_e(q)/m = x 1062 // x = -log_e(q) * period 1063 // x = log_2(q) * (-log_e(2)) * period ; Using log_2 for efficiency 1064 const randomBitCount = 26 1065 q := fastrand()%(1<<randomBitCount) + 1 1066 qlog := fastlog2(float64(q)) - randomBitCount 1067 if qlog > 0 { 1068 qlog = 0 1069 } 1070 const minusLog2 = -0.6931471805599453 // -ln(2) 1071 return int32(qlog*(minusLog2*float64(period))) + 1 1072 } 1073 1074 // nextSampleNoFP is similar to nextSample, but uses older, 1075 // simpler code to avoid floating point. 1076 func nextSampleNoFP() int32 { 1077 // Set first allocation sample size. 1078 rate := MemProfileRate 1079 if rate > 0x3fffffff { // make 2*rate not overflow 1080 rate = 0x3fffffff 1081 } 1082 if rate != 0 { 1083 return int32(fastrand() % uint32(2*rate)) 1084 } 1085 return 0 1086 } 1087 1088 type persistentAlloc struct { 1089 base unsafe.Pointer 1090 off uintptr 1091 } 1092 1093 var globalAlloc struct { 1094 mutex 1095 persistentAlloc 1096 } 1097 1098 // Wrapper around sysAlloc that can allocate small chunks. 1099 // There is no associated free operation. 1100 // Intended for things like function/type/debug-related persistent data. 1101 // If align is 0, uses default align (currently 8). 1102 // The returned memory will be zeroed. 1103 // 1104 // Consider marking persistentalloc'd types go:notinheap. 1105 func persistentalloc(size, align uintptr, sysStat *uint64) unsafe.Pointer { 1106 var p unsafe.Pointer 1107 systemstack(func() { 1108 p = persistentalloc1(size, align, sysStat) 1109 }) 1110 return p 1111 } 1112 1113 // Must run on system stack because stack growth can (re)invoke it. 1114 // See issue 9174. 1115 //go:systemstack 1116 func persistentalloc1(size, align uintptr, sysStat *uint64) unsafe.Pointer { 1117 const ( 1118 chunk = 256 << 10 1119 maxBlock = 64 << 10 // VM reservation granularity is 64K on windows 1120 ) 1121 1122 if size == 0 { 1123 throw("persistentalloc: size == 0") 1124 } 1125 if align != 0 { 1126 if align&(align-1) != 0 { 1127 throw("persistentalloc: align is not a power of 2") 1128 } 1129 if align > _PageSize { 1130 throw("persistentalloc: align is too large") 1131 } 1132 } else { 1133 align = 8 1134 } 1135 1136 if size >= maxBlock { 1137 return sysAlloc(size, sysStat) 1138 } 1139 1140 mp := acquirem() 1141 var persistent *persistentAlloc 1142 if mp != nil && mp.p != 0 { 1143 persistent = &mp.p.ptr().palloc 1144 } else { 1145 lock(&globalAlloc.mutex) 1146 persistent = &globalAlloc.persistentAlloc 1147 } 1148 persistent.off = round(persistent.off, align) 1149 if persistent.off+size > chunk || persistent.base == nil { 1150 persistent.base = sysAlloc(chunk, &memstats.other_sys) 1151 if persistent.base == nil { 1152 if persistent == &globalAlloc.persistentAlloc { 1153 unlock(&globalAlloc.mutex) 1154 } 1155 throw("runtime: cannot allocate memory") 1156 } 1157 persistent.off = 0 1158 } 1159 p := add(persistent.base, persistent.off) 1160 persistent.off += size 1161 releasem(mp) 1162 if persistent == &globalAlloc.persistentAlloc { 1163 unlock(&globalAlloc.mutex) 1164 } 1165 1166 if sysStat != &memstats.other_sys { 1167 mSysStatInc(sysStat, size) 1168 mSysStatDec(&memstats.other_sys, size) 1169 } 1170 return p 1171 } 1172 1173 func stat_mheap(h *mheap) { 1174 //size := (h.arena_used - h.arena_start) 1175 //page := size / _PageSize 1176 ////println("heap:arena_start=", h.arena_start, ",arena_used=", h.arena_used, ",size=", size, page) 1177 ////println("heap:free.len", len(h.free)) 1178 }