github.com/reiver/go@v0.0.0-20150109200633-1d0c7792f172/src/runtime/malloc.go (about) 1 // Copyright 2014 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package runtime 6 7 import "unsafe" 8 9 const ( 10 debugMalloc = false 11 12 flagNoScan = _FlagNoScan 13 flagNoZero = _FlagNoZero 14 15 maxTinySize = _TinySize 16 tinySizeClass = _TinySizeClass 17 maxSmallSize = _MaxSmallSize 18 19 pageShift = _PageShift 20 pageSize = _PageSize 21 pageMask = _PageMask 22 23 bitsPerPointer = _BitsPerPointer 24 bitsMask = _BitsMask 25 pointersPerByte = _PointersPerByte 26 maxGCMask = _MaxGCMask 27 bitsDead = _BitsDead 28 bitsPointer = _BitsPointer 29 bitsScalar = _BitsScalar 30 31 mSpanInUse = _MSpanInUse 32 33 concurrentSweep = _ConcurrentSweep 34 ) 35 36 // Page number (address>>pageShift) 37 type pageID uintptr 38 39 // base address for all 0-byte allocations 40 var zerobase uintptr 41 42 // Determine whether to initiate a GC. 43 // Currently the primitive heuristic we use will start a new 44 // concurrent GC when approximately half the available space 45 // made available by the last GC cycle has been used. 46 // If the GC is already working no need to trigger another one. 47 // This should establish a feedback loop where if the GC does not 48 // have sufficient time to complete then more memory will be 49 // requested from the OS increasing heap size thus allow future 50 // GCs more time to complete. 51 // memstat.heap_alloc and memstat.next_gc reads have benign races 52 // A false negative simple does not start a GC, a false positive 53 // will start a GC needlessly. Neither have correctness issues. 54 func shouldtriggergc() bool { 55 return memstats.heap_alloc+memstats.heap_alloc*3/4 >= memstats.next_gc && atomicloaduint(&bggc.working) == 0 56 } 57 58 // Allocate an object of size bytes. 59 // Small objects are allocated from the per-P cache's free lists. 60 // Large objects (> 32 kB) are allocated straight from the heap. 61 func mallocgc(size uintptr, typ *_type, flags uint32) unsafe.Pointer { 62 shouldhelpgc := false 63 if size == 0 { 64 return unsafe.Pointer(&zerobase) 65 } 66 size0 := size 67 68 if flags&flagNoScan == 0 && typ == nil { 69 throw("malloc missing type") 70 } 71 72 // This function must be atomic wrt GC, but for performance reasons 73 // we don't acquirem/releasem on fast path. The code below does not have 74 // split stack checks, so it can't be preempted by GC. 75 // Functions like roundup/add are inlined. And systemstack/racemalloc are nosplit. 76 // If debugMalloc = true, these assumptions are checked below. 77 if debugMalloc { 78 mp := acquirem() 79 if mp.mallocing != 0 { 80 throw("malloc deadlock") 81 } 82 mp.mallocing = 1 83 if mp.curg != nil { 84 mp.curg.stackguard0 = ^uintptr(0xfff) | 0xbad 85 } 86 } 87 88 c := gomcache() 89 var s *mspan 90 var x unsafe.Pointer 91 if size <= maxSmallSize { 92 if flags&flagNoScan != 0 && size < maxTinySize { 93 // Tiny allocator. 94 // 95 // Tiny allocator combines several tiny allocation requests 96 // into a single memory block. The resulting memory block 97 // is freed when all subobjects are unreachable. The subobjects 98 // must be FlagNoScan (don't have pointers), this ensures that 99 // the amount of potentially wasted memory is bounded. 100 // 101 // Size of the memory block used for combining (maxTinySize) is tunable. 102 // Current setting is 16 bytes, which relates to 2x worst case memory 103 // wastage (when all but one subobjects are unreachable). 104 // 8 bytes would result in no wastage at all, but provides less 105 // opportunities for combining. 106 // 32 bytes provides more opportunities for combining, 107 // but can lead to 4x worst case wastage. 108 // The best case winning is 8x regardless of block size. 109 // 110 // Objects obtained from tiny allocator must not be freed explicitly. 111 // So when an object will be freed explicitly, we ensure that 112 // its size >= maxTinySize. 113 // 114 // SetFinalizer has a special case for objects potentially coming 115 // from tiny allocator, it such case it allows to set finalizers 116 // for an inner byte of a memory block. 117 // 118 // The main targets of tiny allocator are small strings and 119 // standalone escaping variables. On a json benchmark 120 // the allocator reduces number of allocations by ~12% and 121 // reduces heap size by ~20%. 122 tinysize := uintptr(c.tinysize) 123 if size <= tinysize { 124 tiny := unsafe.Pointer(c.tiny) 125 // Align tiny pointer for required (conservative) alignment. 126 if size&7 == 0 { 127 tiny = roundup(tiny, 8) 128 } else if size&3 == 0 { 129 tiny = roundup(tiny, 4) 130 } else if size&1 == 0 { 131 tiny = roundup(tiny, 2) 132 } 133 size1 := size + (uintptr(tiny) - uintptr(unsafe.Pointer(c.tiny))) 134 if size1 <= tinysize { 135 // The object fits into existing tiny block. 136 x = tiny 137 c.tiny = (*byte)(add(x, size)) 138 c.tinysize -= uintptr(size1) 139 c.local_tinyallocs++ 140 if debugMalloc { 141 mp := acquirem() 142 if mp.mallocing == 0 { 143 throw("bad malloc") 144 } 145 mp.mallocing = 0 146 if mp.curg != nil { 147 mp.curg.stackguard0 = mp.curg.stack.lo + _StackGuard 148 } 149 // Note: one releasem for the acquirem just above. 150 // The other for the acquirem at start of malloc. 151 releasem(mp) 152 releasem(mp) 153 } 154 return x 155 } 156 } 157 // Allocate a new maxTinySize block. 158 s = c.alloc[tinySizeClass] 159 v := s.freelist 160 if v.ptr() == nil { 161 systemstack(func() { 162 mCache_Refill(c, tinySizeClass) 163 }) 164 shouldhelpgc = true 165 s = c.alloc[tinySizeClass] 166 v = s.freelist 167 } 168 s.freelist = v.ptr().next 169 s.ref++ 170 //TODO: prefetch v.next 171 x = unsafe.Pointer(v) 172 (*[2]uint64)(x)[0] = 0 173 (*[2]uint64)(x)[1] = 0 174 // See if we need to replace the existing tiny block with the new one 175 // based on amount of remaining free space. 176 if maxTinySize-size > tinysize { 177 c.tiny = (*byte)(add(x, size)) 178 c.tinysize = uintptr(maxTinySize - size) 179 } 180 size = maxTinySize 181 } else { 182 var sizeclass int8 183 if size <= 1024-8 { 184 sizeclass = size_to_class8[(size+7)>>3] 185 } else { 186 sizeclass = size_to_class128[(size-1024+127)>>7] 187 } 188 size = uintptr(class_to_size[sizeclass]) 189 s = c.alloc[sizeclass] 190 v := s.freelist 191 if v.ptr() == nil { 192 systemstack(func() { 193 mCache_Refill(c, int32(sizeclass)) 194 }) 195 shouldhelpgc = true 196 s = c.alloc[sizeclass] 197 v = s.freelist 198 } 199 s.freelist = v.ptr().next 200 s.ref++ 201 //TODO: prefetch 202 x = unsafe.Pointer(v) 203 if flags&flagNoZero == 0 { 204 v.ptr().next = 0 205 if size > 2*ptrSize && ((*[2]uintptr)(x))[1] != 0 { 206 memclr(unsafe.Pointer(v), size) 207 } 208 } 209 } 210 c.local_cachealloc += intptr(size) 211 } else { 212 var s *mspan 213 shouldhelpgc = true 214 systemstack(func() { 215 s = largeAlloc(size, uint32(flags)) 216 }) 217 x = unsafe.Pointer(uintptr(s.start << pageShift)) 218 size = uintptr(s.elemsize) 219 } 220 221 if flags&flagNoScan != 0 { 222 // All objects are pre-marked as noscan. 223 goto marked 224 } 225 226 // If allocating a defer+arg block, now that we've picked a malloc size 227 // large enough to hold everything, cut the "asked for" size down to 228 // just the defer header, so that the GC bitmap will record the arg block 229 // as containing nothing at all (as if it were unused space at the end of 230 // a malloc block caused by size rounding). 231 // The defer arg areas are scanned as part of scanstack. 232 if typ == deferType { 233 size0 = unsafe.Sizeof(_defer{}) 234 } 235 236 // From here till marked label marking the object as allocated 237 // and storing type info in the GC bitmap. 238 { 239 arena_start := uintptr(unsafe.Pointer(mheap_.arena_start)) 240 off := (uintptr(x) - arena_start) / ptrSize 241 xbits := (*uint8)(unsafe.Pointer(arena_start - off/wordsPerBitmapByte - 1)) 242 shift := (off % wordsPerBitmapByte) * gcBits 243 if debugMalloc && ((*xbits>>shift)&(bitMask|bitPtrMask)) != bitBoundary { 244 println("runtime: bits =", (*xbits>>shift)&(bitMask|bitPtrMask)) 245 throw("bad bits in markallocated") 246 } 247 248 var ti, te uintptr 249 var ptrmask *uint8 250 if size == ptrSize { 251 // It's one word and it has pointers, it must be a pointer. 252 *xbits |= (bitsPointer << 2) << shift 253 goto marked 254 } 255 if typ.kind&kindGCProg != 0 { 256 nptr := (uintptr(typ.size) + ptrSize - 1) / ptrSize 257 masksize := nptr 258 if masksize%2 != 0 { 259 masksize *= 2 // repeated 260 } 261 masksize = masksize * pointersPerByte / 8 // 4 bits per word 262 masksize++ // unroll flag in the beginning 263 if masksize > maxGCMask && typ.gc[1] != 0 { 264 // write barriers have not been updated to deal with this case yet. 265 throw("maxGCMask too small for now") 266 // If the mask is too large, unroll the program directly 267 // into the GC bitmap. It's 7 times slower than copying 268 // from the pre-unrolled mask, but saves 1/16 of type size 269 // memory for the mask. 270 systemstack(func() { 271 unrollgcproginplace_m(x, typ, size, size0) 272 }) 273 goto marked 274 } 275 ptrmask = (*uint8)(unsafe.Pointer(uintptr(typ.gc[0]))) 276 // Check whether the program is already unrolled 277 // by checking if the unroll flag byte is set 278 maskword := uintptr(atomicloadp(unsafe.Pointer(ptrmask))) 279 if *(*uint8)(unsafe.Pointer(&maskword)) == 0 { 280 systemstack(func() { 281 unrollgcprog_m(typ) 282 }) 283 } 284 ptrmask = (*uint8)(add(unsafe.Pointer(ptrmask), 1)) // skip the unroll flag byte 285 } else { 286 ptrmask = (*uint8)(unsafe.Pointer(typ.gc[0])) // pointer to unrolled mask 287 } 288 if size == 2*ptrSize { 289 *xbits = *ptrmask | bitBoundary 290 goto marked 291 } 292 te = uintptr(typ.size) / ptrSize 293 // If the type occupies odd number of words, its mask is repeated. 294 if te%2 == 0 { 295 te /= 2 296 } 297 // Copy pointer bitmask into the bitmap. 298 for i := uintptr(0); i < size0; i += 2 * ptrSize { 299 v := *(*uint8)(add(unsafe.Pointer(ptrmask), ti)) 300 ti++ 301 if ti == te { 302 ti = 0 303 } 304 if i == 0 { 305 v |= bitBoundary 306 } 307 if i+ptrSize == size0 { 308 v &^= uint8(bitPtrMask << 4) 309 } 310 311 *xbits = v 312 xbits = (*byte)(add(unsafe.Pointer(xbits), ^uintptr(0))) 313 } 314 if size0%(2*ptrSize) == 0 && size0 < size { 315 // Mark the word after last object's word as bitsDead. 316 *xbits = bitsDead << 2 317 } 318 } 319 marked: 320 321 // GCmarkterminate allocates black 322 // All slots hold nil so no scanning is needed. 323 // This may be racing with GC so do it atomically if there can be 324 // a race marking the bit. 325 if gcphase == _GCmarktermination { 326 systemstack(func() { 327 gcmarknewobject_m(uintptr(x)) 328 }) 329 } 330 331 if mheap_.shadow_enabled { 332 clearshadow(uintptr(x), size) 333 } 334 335 if raceenabled { 336 racemalloc(x, size) 337 } 338 339 if debugMalloc { 340 mp := acquirem() 341 if mp.mallocing == 0 { 342 throw("bad malloc") 343 } 344 mp.mallocing = 0 345 if mp.curg != nil { 346 mp.curg.stackguard0 = mp.curg.stack.lo + _StackGuard 347 } 348 // Note: one releasem for the acquirem just above. 349 // The other for the acquirem at start of malloc. 350 releasem(mp) 351 releasem(mp) 352 } 353 354 if debug.allocfreetrace != 0 { 355 tracealloc(x, size, typ) 356 } 357 358 if rate := MemProfileRate; rate > 0 { 359 if size < uintptr(rate) && int32(size) < c.next_sample { 360 c.next_sample -= int32(size) 361 } else { 362 mp := acquirem() 363 profilealloc(mp, x, size) 364 releasem(mp) 365 } 366 } 367 368 if shouldtriggergc() { 369 gogc(0) 370 } else if shouldhelpgc && atomicloaduint(&bggc.working) == 1 { 371 // bggc.lock not taken since race on bggc.working is benign. 372 // At worse we don't call gchelpwork. 373 // Delay the gchelpwork until the epilogue so that it doesn't 374 // interfere with the inner working of malloc such as 375 // mcache refills that might happen while doing the gchelpwork 376 systemstack(gchelpwork) 377 } 378 379 return x 380 } 381 382 func loadPtrMask(typ *_type) []uint8 { 383 var ptrmask *uint8 384 nptr := (uintptr(typ.size) + ptrSize - 1) / ptrSize 385 if typ.kind&kindGCProg != 0 { 386 masksize := nptr 387 if masksize%2 != 0 { 388 masksize *= 2 // repeated 389 } 390 masksize = masksize * pointersPerByte / 8 // 4 bits per word 391 masksize++ // unroll flag in the beginning 392 if masksize > maxGCMask && typ.gc[1] != 0 { 393 // write barriers have not been updated to deal with this case yet. 394 throw("maxGCMask too small for now") 395 } 396 ptrmask = (*uint8)(unsafe.Pointer(uintptr(typ.gc[0]))) 397 // Check whether the program is already unrolled 398 // by checking if the unroll flag byte is set 399 maskword := uintptr(atomicloadp(unsafe.Pointer(ptrmask))) 400 if *(*uint8)(unsafe.Pointer(&maskword)) == 0 { 401 systemstack(func() { 402 unrollgcprog_m(typ) 403 }) 404 } 405 ptrmask = (*uint8)(add(unsafe.Pointer(ptrmask), 1)) // skip the unroll flag byte 406 } else { 407 ptrmask = (*uint8)(unsafe.Pointer(typ.gc[0])) // pointer to unrolled mask 408 } 409 return (*[1 << 30]byte)(unsafe.Pointer(ptrmask))[:(nptr+1)/2] 410 } 411 412 // implementation of new builtin 413 func newobject(typ *_type) unsafe.Pointer { 414 flags := uint32(0) 415 if typ.kind&kindNoPointers != 0 { 416 flags |= flagNoScan 417 } 418 return mallocgc(uintptr(typ.size), typ, flags) 419 } 420 421 //go:linkname reflect_unsafe_New reflect.unsafe_New 422 func reflect_unsafe_New(typ *_type) unsafe.Pointer { 423 return newobject(typ) 424 } 425 426 // implementation of make builtin for slices 427 func newarray(typ *_type, n uintptr) unsafe.Pointer { 428 flags := uint32(0) 429 if typ.kind&kindNoPointers != 0 { 430 flags |= flagNoScan 431 } 432 if int(n) < 0 || (typ.size > 0 && n > _MaxMem/uintptr(typ.size)) { 433 panic("runtime: allocation size out of range") 434 } 435 return mallocgc(uintptr(typ.size)*n, typ, flags) 436 } 437 438 //go:linkname reflect_unsafe_NewArray reflect.unsafe_NewArray 439 func reflect_unsafe_NewArray(typ *_type, n uintptr) unsafe.Pointer { 440 return newarray(typ, n) 441 } 442 443 // rawmem returns a chunk of pointerless memory. It is 444 // not zeroed. 445 func rawmem(size uintptr) unsafe.Pointer { 446 return mallocgc(size, nil, flagNoScan|flagNoZero) 447 } 448 449 func profilealloc(mp *m, x unsafe.Pointer, size uintptr) { 450 c := mp.mcache 451 rate := MemProfileRate 452 if size < uintptr(rate) { 453 // pick next profile time 454 // If you change this, also change allocmcache. 455 if rate > 0x3fffffff { // make 2*rate not overflow 456 rate = 0x3fffffff 457 } 458 next := int32(fastrand1()) % (2 * int32(rate)) 459 // Subtract the "remainder" of the current allocation. 460 // Otherwise objects that are close in size to sampling rate 461 // will be under-sampled, because we consistently discard this remainder. 462 next -= (int32(size) - c.next_sample) 463 if next < 0 { 464 next = 0 465 } 466 c.next_sample = next 467 } 468 469 mProf_Malloc(x, size) 470 } 471 472 // For now this must be bracketed with a stoptheworld and a starttheworld to ensure 473 // all go routines see the new barrier. 474 func gcinstallmarkwb() { 475 gcphase = _GCmark 476 } 477 478 // force = 0 - start concurrent GC 479 // force = 1 - do STW GC regardless of current heap usage 480 // force = 2 - go STW GC and eager sweep 481 func gogc(force int32) { 482 // The gc is turned off (via enablegc) until the bootstrap has completed. 483 // Also, malloc gets called in the guts of a number of libraries that might be 484 // holding locks. To avoid deadlocks during stoptheworld, don't bother 485 // trying to run gc while holding a lock. The next mallocgc without a lock 486 // will do the gc instead. 487 488 mp := acquirem() 489 if gp := getg(); gp == mp.g0 || mp.locks > 1 || !memstats.enablegc || panicking != 0 || gcpercent < 0 { 490 releasem(mp) 491 return 492 } 493 releasem(mp) 494 mp = nil 495 496 if force == 0 { 497 lock(&bggc.lock) 498 if !bggc.started { 499 bggc.working = 1 500 bggc.started = true 501 go backgroundgc() 502 } else if bggc.working == 0 { 503 bggc.working = 1 504 ready(bggc.g) 505 } 506 unlock(&bggc.lock) 507 } else { 508 gcwork(force) 509 } 510 } 511 512 func gcwork(force int32) { 513 514 semacquire(&worldsema, false) 515 516 // Pick up the remaining unswept/not being swept spans concurrently 517 for gosweepone() != ^uintptr(0) { 518 sweep.nbgsweep++ 519 } 520 521 // Ok, we're doing it! Stop everybody else 522 523 mp := acquirem() 524 mp.gcing = 1 525 releasem(mp) 526 gctimer.count++ 527 if force == 0 { 528 gctimer.cycle.sweepterm = nanotime() 529 } 530 // Pick up the remaining unswept/not being swept spans before we STW 531 for gosweepone() != ^uintptr(0) { 532 sweep.nbgsweep++ 533 } 534 systemstack(stoptheworld) 535 systemstack(finishsweep_m) // finish sweep before we start concurrent scan. 536 if force == 0 { // Do as much work concurrently as possible 537 gcphase = _GCscan 538 systemstack(starttheworld) 539 gctimer.cycle.scan = nanotime() 540 // Do a concurrent heap scan before we stop the world. 541 systemstack(gcscan_m) 542 gctimer.cycle.installmarkwb = nanotime() 543 systemstack(stoptheworld) 544 systemstack(gcinstallmarkwb) 545 systemstack(starttheworld) 546 gctimer.cycle.mark = nanotime() 547 systemstack(gcmark_m) 548 gctimer.cycle.markterm = nanotime() 549 systemstack(stoptheworld) 550 systemstack(gcinstalloffwb_m) 551 } 552 553 startTime := nanotime() 554 if mp != acquirem() { 555 throw("gogc: rescheduled") 556 } 557 558 clearpools() 559 560 // Run gc on the g0 stack. We do this so that the g stack 561 // we're currently running on will no longer change. Cuts 562 // the root set down a bit (g0 stacks are not scanned, and 563 // we don't need to scan gc's internal state). We also 564 // need to switch to g0 so we can shrink the stack. 565 n := 1 566 if debug.gctrace > 1 { 567 n = 2 568 } 569 eagersweep := force >= 2 570 for i := 0; i < n; i++ { 571 if i > 0 { 572 // refresh start time if doing a second GC 573 startTime = nanotime() 574 } 575 // switch to g0, call gc, then switch back 576 systemstack(func() { 577 gc_m(startTime, eagersweep) 578 }) 579 } 580 581 systemstack(func() { 582 gccheckmark_m(startTime, eagersweep) 583 }) 584 585 // all done 586 mp.gcing = 0 587 588 if force == 0 { 589 gctimer.cycle.sweep = nanotime() 590 } 591 592 semrelease(&worldsema) 593 594 if force == 0 { 595 if gctimer.verbose > 1 { 596 GCprinttimes() 597 } else if gctimer.verbose > 0 { 598 calctimes() // ignore result 599 } 600 } 601 602 systemstack(starttheworld) 603 604 releasem(mp) 605 mp = nil 606 607 // now that gc is done, kick off finalizer thread if needed 608 if !concurrentSweep { 609 // give the queued finalizers, if any, a chance to run 610 Gosched() 611 } 612 } 613 614 func GCcheckmarkenable() { 615 systemstack(gccheckmarkenable_m) 616 } 617 618 func GCcheckmarkdisable() { 619 systemstack(gccheckmarkdisable_m) 620 } 621 622 // gctimes records the time in nanoseconds of each phase of the concurrent GC. 623 type gctimes struct { 624 sweepterm int64 // stw 625 scan int64 626 installmarkwb int64 // stw 627 mark int64 628 markterm int64 // stw 629 sweep int64 630 } 631 632 // gcchronograph holds timer information related to GC phases 633 // max records the maximum time spent in each GC phase since GCstarttimes. 634 // total records the total time spent in each GC phase since GCstarttimes. 635 // cycle records the absolute time (as returned by nanoseconds()) that each GC phase last started at. 636 type gcchronograph struct { 637 count int64 638 verbose int64 639 maxpause int64 640 max gctimes 641 total gctimes 642 cycle gctimes 643 } 644 645 var gctimer gcchronograph 646 647 // GCstarttimes initializes the gc times. All previous times are lost. 648 func GCstarttimes(verbose int64) { 649 gctimer = gcchronograph{verbose: verbose} 650 } 651 652 // GCendtimes stops the gc timers. 653 func GCendtimes() { 654 gctimer.verbose = 0 655 } 656 657 // calctimes converts gctimer.cycle into the elapsed times, updates gctimer.total 658 // and updates gctimer.max with the max pause time. 659 func calctimes() gctimes { 660 var times gctimes 661 662 var max = func(a, b int64) int64 { 663 if a > b { 664 return a 665 } 666 return b 667 } 668 669 times.sweepterm = gctimer.cycle.scan - gctimer.cycle.sweepterm 670 gctimer.total.sweepterm += times.sweepterm 671 gctimer.max.sweepterm = max(gctimer.max.sweepterm, times.sweepterm) 672 gctimer.maxpause = max(gctimer.maxpause, gctimer.max.sweepterm) 673 674 times.scan = gctimer.cycle.installmarkwb - gctimer.cycle.scan 675 gctimer.total.scan += times.scan 676 gctimer.max.scan = max(gctimer.max.scan, times.scan) 677 678 times.installmarkwb = gctimer.cycle.mark - gctimer.cycle.installmarkwb 679 gctimer.total.installmarkwb += times.installmarkwb 680 gctimer.max.installmarkwb = max(gctimer.max.installmarkwb, times.installmarkwb) 681 gctimer.maxpause = max(gctimer.maxpause, gctimer.max.installmarkwb) 682 683 times.mark = gctimer.cycle.markterm - gctimer.cycle.mark 684 gctimer.total.mark += times.mark 685 gctimer.max.mark = max(gctimer.max.mark, times.mark) 686 687 times.markterm = gctimer.cycle.sweep - gctimer.cycle.markterm 688 gctimer.total.markterm += times.markterm 689 gctimer.max.markterm = max(gctimer.max.markterm, times.markterm) 690 gctimer.maxpause = max(gctimer.maxpause, gctimer.max.markterm) 691 692 return times 693 } 694 695 // GCprinttimes prints latency information in nanoseconds about various 696 // phases in the GC. The information for each phase includes the maximum pause 697 // and total time since the most recent call to GCstarttimes as well as 698 // the information from the most recent Concurent GC cycle. Calls from the 699 // application to runtime.GC() are ignored. 700 func GCprinttimes() { 701 if gctimer.verbose == 0 { 702 println("GC timers not enabled") 703 return 704 } 705 706 // Explicitly put times on the heap so printPhase can use it. 707 times := new(gctimes) 708 *times = calctimes() 709 cycletime := gctimer.cycle.sweep - gctimer.cycle.sweepterm 710 pause := times.sweepterm + times.installmarkwb + times.markterm 711 gomaxprocs := GOMAXPROCS(-1) 712 713 printlock() 714 print("GC: #", gctimer.count, " ", cycletime, "ns @", gctimer.cycle.sweepterm, " pause=", pause, " maxpause=", gctimer.maxpause, " goroutines=", allglen, " gomaxprocs=", gomaxprocs, "\n") 715 printPhase := func(label string, get func(*gctimes) int64, procs int) { 716 print("GC: ", label, " ", get(times), "ns\tmax=", get(&gctimer.max), "\ttotal=", get(&gctimer.total), "\tprocs=", procs, "\n") 717 } 718 printPhase("sweep term:", func(t *gctimes) int64 { return t.sweepterm }, gomaxprocs) 719 printPhase("scan: ", func(t *gctimes) int64 { return t.scan }, 1) 720 printPhase("install wb:", func(t *gctimes) int64 { return t.installmarkwb }, gomaxprocs) 721 printPhase("mark: ", func(t *gctimes) int64 { return t.mark }, 1) 722 printPhase("mark term: ", func(t *gctimes) int64 { return t.markterm }, gomaxprocs) 723 printunlock() 724 } 725 726 // GC runs a garbage collection. 727 func GC() { 728 gogc(2) 729 } 730 731 // linker-provided 732 var noptrdata struct{} 733 var enoptrdata struct{} 734 var noptrbss struct{} 735 var enoptrbss struct{} 736 737 // SetFinalizer sets the finalizer associated with x to f. 738 // When the garbage collector finds an unreachable block 739 // with an associated finalizer, it clears the association and runs 740 // f(x) in a separate goroutine. This makes x reachable again, but 741 // now without an associated finalizer. Assuming that SetFinalizer 742 // is not called again, the next time the garbage collector sees 743 // that x is unreachable, it will free x. 744 // 745 // SetFinalizer(x, nil) clears any finalizer associated with x. 746 // 747 // The argument x must be a pointer to an object allocated by 748 // calling new or by taking the address of a composite literal. 749 // The argument f must be a function that takes a single argument 750 // to which x's type can be assigned, and can have arbitrary ignored return 751 // values. If either of these is not true, SetFinalizer aborts the 752 // program. 753 // 754 // Finalizers are run in dependency order: if A points at B, both have 755 // finalizers, and they are otherwise unreachable, only the finalizer 756 // for A runs; once A is freed, the finalizer for B can run. 757 // If a cyclic structure includes a block with a finalizer, that 758 // cycle is not guaranteed to be garbage collected and the finalizer 759 // is not guaranteed to run, because there is no ordering that 760 // respects the dependencies. 761 // 762 // The finalizer for x is scheduled to run at some arbitrary time after 763 // x becomes unreachable. 764 // There is no guarantee that finalizers will run before a program exits, 765 // so typically they are useful only for releasing non-memory resources 766 // associated with an object during a long-running program. 767 // For example, an os.File object could use a finalizer to close the 768 // associated operating system file descriptor when a program discards 769 // an os.File without calling Close, but it would be a mistake 770 // to depend on a finalizer to flush an in-memory I/O buffer such as a 771 // bufio.Writer, because the buffer would not be flushed at program exit. 772 // 773 // It is not guaranteed that a finalizer will run if the size of *x is 774 // zero bytes. 775 // 776 // It is not guaranteed that a finalizer will run for objects allocated 777 // in initializers for package-level variables. Such objects may be 778 // linker-allocated, not heap-allocated. 779 // 780 // A single goroutine runs all finalizers for a program, sequentially. 781 // If a finalizer must run for a long time, it should do so by starting 782 // a new goroutine. 783 func SetFinalizer(obj interface{}, finalizer interface{}) { 784 e := (*eface)(unsafe.Pointer(&obj)) 785 etyp := e._type 786 if etyp == nil { 787 throw("runtime.SetFinalizer: first argument is nil") 788 } 789 if etyp.kind&kindMask != kindPtr { 790 throw("runtime.SetFinalizer: first argument is " + *etyp._string + ", not pointer") 791 } 792 ot := (*ptrtype)(unsafe.Pointer(etyp)) 793 if ot.elem == nil { 794 throw("nil elem type!") 795 } 796 797 // find the containing object 798 _, base, _ := findObject(e.data) 799 800 if base == nil { 801 // 0-length objects are okay. 802 if e.data == unsafe.Pointer(&zerobase) { 803 return 804 } 805 806 // Global initializers might be linker-allocated. 807 // var Foo = &Object{} 808 // func main() { 809 // runtime.SetFinalizer(Foo, nil) 810 // } 811 // The relevant segments are: noptrdata, data, bss, noptrbss. 812 // We cannot assume they are in any order or even contiguous, 813 // due to external linking. 814 if uintptr(unsafe.Pointer(&noptrdata)) <= uintptr(e.data) && uintptr(e.data) < uintptr(unsafe.Pointer(&enoptrdata)) || 815 uintptr(unsafe.Pointer(&data)) <= uintptr(e.data) && uintptr(e.data) < uintptr(unsafe.Pointer(&edata)) || 816 uintptr(unsafe.Pointer(&bss)) <= uintptr(e.data) && uintptr(e.data) < uintptr(unsafe.Pointer(&ebss)) || 817 uintptr(unsafe.Pointer(&noptrbss)) <= uintptr(e.data) && uintptr(e.data) < uintptr(unsafe.Pointer(&enoptrbss)) { 818 return 819 } 820 throw("runtime.SetFinalizer: pointer not in allocated block") 821 } 822 823 if e.data != base { 824 // As an implementation detail we allow to set finalizers for an inner byte 825 // of an object if it could come from tiny alloc (see mallocgc for details). 826 if ot.elem == nil || ot.elem.kind&kindNoPointers == 0 || ot.elem.size >= maxTinySize { 827 throw("runtime.SetFinalizer: pointer not at beginning of allocated block") 828 } 829 } 830 831 f := (*eface)(unsafe.Pointer(&finalizer)) 832 ftyp := f._type 833 if ftyp == nil { 834 // switch to system stack and remove finalizer 835 systemstack(func() { 836 removefinalizer(e.data) 837 }) 838 return 839 } 840 841 if ftyp.kind&kindMask != kindFunc { 842 throw("runtime.SetFinalizer: second argument is " + *ftyp._string + ", not a function") 843 } 844 ft := (*functype)(unsafe.Pointer(ftyp)) 845 ins := *(*[]*_type)(unsafe.Pointer(&ft.in)) 846 if ft.dotdotdot || len(ins) != 1 { 847 throw("runtime.SetFinalizer: cannot pass " + *etyp._string + " to finalizer " + *ftyp._string) 848 } 849 fint := ins[0] 850 switch { 851 case fint == etyp: 852 // ok - same type 853 goto okarg 854 case fint.kind&kindMask == kindPtr: 855 if (fint.x == nil || fint.x.name == nil || etyp.x == nil || etyp.x.name == nil) && (*ptrtype)(unsafe.Pointer(fint)).elem == ot.elem { 856 // ok - not same type, but both pointers, 857 // one or the other is unnamed, and same element type, so assignable. 858 goto okarg 859 } 860 case fint.kind&kindMask == kindInterface: 861 ityp := (*interfacetype)(unsafe.Pointer(fint)) 862 if len(ityp.mhdr) == 0 { 863 // ok - satisfies empty interface 864 goto okarg 865 } 866 if assertE2I2(ityp, obj, nil) { 867 goto okarg 868 } 869 } 870 throw("runtime.SetFinalizer: cannot pass " + *etyp._string + " to finalizer " + *ftyp._string) 871 okarg: 872 // compute size needed for return parameters 873 nret := uintptr(0) 874 for _, t := range *(*[]*_type)(unsafe.Pointer(&ft.out)) { 875 nret = round(nret, uintptr(t.align)) + uintptr(t.size) 876 } 877 nret = round(nret, ptrSize) 878 879 // make sure we have a finalizer goroutine 880 createfing() 881 882 systemstack(func() { 883 if !addfinalizer(e.data, (*funcval)(f.data), nret, fint, ot) { 884 throw("runtime.SetFinalizer: finalizer already set") 885 } 886 }) 887 } 888 889 // round n up to a multiple of a. a must be a power of 2. 890 func round(n, a uintptr) uintptr { 891 return (n + a - 1) &^ (a - 1) 892 } 893 894 // Look up pointer v in heap. Return the span containing the object, 895 // the start of the object, and the size of the object. If the object 896 // does not exist, return nil, nil, 0. 897 func findObject(v unsafe.Pointer) (s *mspan, x unsafe.Pointer, n uintptr) { 898 c := gomcache() 899 c.local_nlookup++ 900 if ptrSize == 4 && c.local_nlookup >= 1<<30 { 901 // purge cache stats to prevent overflow 902 lock(&mheap_.lock) 903 purgecachedstats(c) 904 unlock(&mheap_.lock) 905 } 906 907 // find span 908 arena_start := uintptr(unsafe.Pointer(mheap_.arena_start)) 909 arena_used := uintptr(unsafe.Pointer(mheap_.arena_used)) 910 if uintptr(v) < arena_start || uintptr(v) >= arena_used { 911 return 912 } 913 p := uintptr(v) >> pageShift 914 q := p - arena_start>>pageShift 915 s = *(**mspan)(add(unsafe.Pointer(mheap_.spans), q*ptrSize)) 916 if s == nil { 917 return 918 } 919 x = unsafe.Pointer(uintptr(s.start) << pageShift) 920 921 if uintptr(v) < uintptr(x) || uintptr(v) >= uintptr(unsafe.Pointer(s.limit)) || s.state != mSpanInUse { 922 s = nil 923 x = nil 924 return 925 } 926 927 n = uintptr(s.elemsize) 928 if s.sizeclass != 0 { 929 x = add(x, (uintptr(v)-uintptr(x))/n*n) 930 } 931 return 932 } 933 934 var fingCreate uint32 935 936 func createfing() { 937 // start the finalizer goroutine exactly once 938 if fingCreate == 0 && cas(&fingCreate, 0, 1) { 939 go runfinq() 940 } 941 } 942 943 // This is the goroutine that runs all of the finalizers 944 func runfinq() { 945 var ( 946 frame unsafe.Pointer 947 framecap uintptr 948 ) 949 950 for { 951 lock(&finlock) 952 fb := finq 953 finq = nil 954 if fb == nil { 955 gp := getg() 956 fing = gp 957 fingwait = true 958 gp.issystem = true 959 goparkunlock(&finlock, "finalizer wait") 960 gp.issystem = false 961 continue 962 } 963 unlock(&finlock) 964 if raceenabled { 965 racefingo() 966 } 967 for fb != nil { 968 for i := int32(0); i < fb.cnt; i++ { 969 f := (*finalizer)(add(unsafe.Pointer(&fb.fin), uintptr(i)*unsafe.Sizeof(finalizer{}))) 970 971 framesz := unsafe.Sizeof((interface{})(nil)) + uintptr(f.nret) 972 if framecap < framesz { 973 // The frame does not contain pointers interesting for GC, 974 // all not yet finalized objects are stored in finq. 975 // If we do not mark it as FlagNoScan, 976 // the last finalized object is not collected. 977 frame = mallocgc(framesz, nil, flagNoScan) 978 framecap = framesz 979 } 980 981 if f.fint == nil { 982 throw("missing type in runfinq") 983 } 984 switch f.fint.kind & kindMask { 985 case kindPtr: 986 // direct use of pointer 987 *(*unsafe.Pointer)(frame) = f.arg 988 case kindInterface: 989 ityp := (*interfacetype)(unsafe.Pointer(f.fint)) 990 // set up with empty interface 991 (*eface)(frame)._type = &f.ot.typ 992 (*eface)(frame).data = f.arg 993 if len(ityp.mhdr) != 0 { 994 // convert to interface with methods 995 // this conversion is guaranteed to succeed - we checked in SetFinalizer 996 assertE2I(ityp, *(*interface{})(frame), (*fInterface)(frame)) 997 } 998 default: 999 throw("bad kind in runfinq") 1000 } 1001 reflectcall(nil, unsafe.Pointer(f.fn), frame, uint32(framesz), uint32(framesz)) 1002 1003 // drop finalizer queue references to finalized object 1004 f.fn = nil 1005 f.arg = nil 1006 f.ot = nil 1007 } 1008 fb.cnt = 0 1009 next := fb.next 1010 lock(&finlock) 1011 fb.next = finc 1012 finc = fb 1013 unlock(&finlock) 1014 fb = next 1015 } 1016 } 1017 } 1018 1019 var persistent struct { 1020 lock mutex 1021 pos unsafe.Pointer 1022 end unsafe.Pointer 1023 } 1024 1025 // Wrapper around sysAlloc that can allocate small chunks. 1026 // There is no associated free operation. 1027 // Intended for things like function/type/debug-related persistent data. 1028 // If align is 0, uses default align (currently 8). 1029 func persistentalloc(size, align uintptr, stat *uint64) unsafe.Pointer { 1030 const ( 1031 chunk = 256 << 10 1032 maxBlock = 64 << 10 // VM reservation granularity is 64K on windows 1033 ) 1034 1035 if align != 0 { 1036 if align&(align-1) != 0 { 1037 throw("persistentalloc: align is not a power of 2") 1038 } 1039 if align > _PageSize { 1040 throw("persistentalloc: align is too large") 1041 } 1042 } else { 1043 align = 8 1044 } 1045 1046 if size >= maxBlock { 1047 return sysAlloc(size, stat) 1048 } 1049 1050 lock(&persistent.lock) 1051 persistent.pos = roundup(persistent.pos, align) 1052 if uintptr(persistent.pos)+size > uintptr(persistent.end) { 1053 persistent.pos = sysAlloc(chunk, &memstats.other_sys) 1054 if persistent.pos == nil { 1055 unlock(&persistent.lock) 1056 throw("runtime: cannot allocate memory") 1057 } 1058 persistent.end = add(persistent.pos, chunk) 1059 } 1060 p := persistent.pos 1061 persistent.pos = add(persistent.pos, size) 1062 unlock(&persistent.lock) 1063 1064 if stat != &memstats.other_sys { 1065 xadd64(stat, int64(size)) 1066 xadd64(&memstats.other_sys, -int64(size)) 1067 } 1068 return p 1069 }