github.com/MangoDowner/go-gm@v0.0.0-20180818020936-8baa2bd4408c/src/runtime/mheap.go (about) 1 // Copyright 2009 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 // Page heap. 6 // 7 // See malloc.go for overview. 8 9 package runtime 10 11 import ( 12 "runtime/internal/atomic" 13 "runtime/internal/sys" 14 "unsafe" 15 ) 16 17 // minPhysPageSize is a lower-bound on the physical page size. The 18 // true physical page size may be larger than this. In contrast, 19 // sys.PhysPageSize is an upper-bound on the physical page size. 20 const minPhysPageSize = 4096 21 22 // Main malloc heap. 23 // The heap itself is the "free[]" and "large" arrays, 24 // but all the other global data is here too. 25 // 26 // mheap must not be heap-allocated because it contains mSpanLists, 27 // which must not be heap-allocated. 28 // 29 //go:notinheap 30 type mheap struct { 31 lock mutex 32 free [_MaxMHeapList]mSpanList // free lists of given length up to _MaxMHeapList 33 freelarge mTreap // free treap of length >= _MaxMHeapList 34 busy [_MaxMHeapList]mSpanList // busy lists of large spans of given length 35 busylarge mSpanList // busy lists of large spans length >= _MaxMHeapList 36 sweepgen uint32 // sweep generation, see comment in mspan 37 sweepdone uint32 // all spans are swept 38 sweepers uint32 // number of active sweepone calls 39 40 // allspans is a slice of all mspans ever created. Each mspan 41 // appears exactly once. 42 // 43 // The memory for allspans is manually managed and can be 44 // reallocated and move as the heap grows. 45 // 46 // In general, allspans is protected by mheap_.lock, which 47 // prevents concurrent access as well as freeing the backing 48 // store. Accesses during STW might not hold the lock, but 49 // must ensure that allocation cannot happen around the 50 // access (since that may free the backing store). 51 allspans []*mspan // all spans out there 52 53 // spans is a lookup table to map virtual address page IDs to *mspan. 54 // For allocated spans, their pages map to the span itself. 55 // For free spans, only the lowest and highest pages map to the span itself. 56 // Internal pages map to an arbitrary span. 57 // For pages that have never been allocated, spans entries are nil. 58 // 59 // This is backed by a reserved region of the address space so 60 // it can grow without moving. The memory up to len(spans) is 61 // mapped. cap(spans) indicates the total reserved memory. 62 spans []*mspan 63 64 // sweepSpans contains two mspan stacks: one of swept in-use 65 // spans, and one of unswept in-use spans. These two trade 66 // roles on each GC cycle. Since the sweepgen increases by 2 67 // on each cycle, this means the swept spans are in 68 // sweepSpans[sweepgen/2%2] and the unswept spans are in 69 // sweepSpans[1-sweepgen/2%2]. Sweeping pops spans from the 70 // unswept stack and pushes spans that are still in-use on the 71 // swept stack. Likewise, allocating an in-use span pushes it 72 // on the swept stack. 73 sweepSpans [2]gcSweepBuf 74 75 _ uint32 // align uint64 fields on 32-bit for atomics 76 77 // Proportional sweep 78 // 79 // These parameters represent a linear function from heap_live 80 // to page sweep count. The proportional sweep system works to 81 // stay in the black by keeping the current page sweep count 82 // above this line at the current heap_live. 83 // 84 // The line has slope sweepPagesPerByte and passes through a 85 // basis point at (sweepHeapLiveBasis, pagesSweptBasis). At 86 // any given time, the system is at (memstats.heap_live, 87 // pagesSwept) in this space. 88 // 89 // It's important that the line pass through a point we 90 // control rather than simply starting at a (0,0) origin 91 // because that lets us adjust sweep pacing at any time while 92 // accounting for current progress. If we could only adjust 93 // the slope, it would create a discontinuity in debt if any 94 // progress has already been made. 95 pagesInUse uint64 // pages of spans in stats _MSpanInUse; R/W with mheap.lock 96 pagesSwept uint64 // pages swept this cycle; updated atomically 97 pagesSweptBasis uint64 // pagesSwept to use as the origin of the sweep ratio; updated atomically 98 sweepHeapLiveBasis uint64 // value of heap_live to use as the origin of sweep ratio; written with lock, read without 99 sweepPagesPerByte float64 // proportional sweep ratio; written with lock, read without 100 // TODO(austin): pagesInUse should be a uintptr, but the 386 101 // compiler can't 8-byte align fields. 102 103 // Malloc stats. 104 largealloc uint64 // bytes allocated for large objects 105 nlargealloc uint64 // number of large object allocations 106 largefree uint64 // bytes freed for large objects (>maxsmallsize) 107 nlargefree uint64 // number of frees for large objects (>maxsmallsize) 108 nsmallfree [_NumSizeClasses]uint64 // number of frees for small objects (<=maxsmallsize) 109 110 // range of addresses we might see in the heap 111 bitmap uintptr // Points to one byte past the end of the bitmap 112 bitmap_mapped uintptr 113 114 // The arena_* fields indicate the addresses of the Go heap. 115 // 116 // The maximum range of the Go heap is 117 // [arena_start, arena_start+_MaxMem+1). 118 // 119 // The range of the current Go heap is 120 // [arena_start, arena_used). Parts of this range may not be 121 // mapped, but the metadata structures are always mapped for 122 // the full range. 123 arena_start uintptr 124 arena_used uintptr // Set with setArenaUsed. 125 126 // The heap is grown using a linear allocator that allocates 127 // from the block [arena_alloc, arena_end). arena_alloc is 128 // often, but *not always* equal to arena_used. 129 arena_alloc uintptr 130 arena_end uintptr 131 132 // arena_reserved indicates that the memory [arena_alloc, 133 // arena_end) is reserved (e.g., mapped PROT_NONE). If this is 134 // false, we have to be careful not to clobber existing 135 // mappings here. If this is true, then we own the mapping 136 // here and *must* clobber it to use it. 137 arena_reserved bool 138 139 _ uint32 // ensure 64-bit alignment 140 141 // central free lists for small size classes. 142 // the padding makes sure that the MCentrals are 143 // spaced CacheLineSize bytes apart, so that each MCentral.lock 144 // gets its own cache line. 145 // central is indexed by spanClass. 146 central [numSpanClasses]struct { 147 mcentral mcentral 148 pad [sys.CacheLineSize - unsafe.Sizeof(mcentral{})%sys.CacheLineSize]byte 149 } 150 151 spanalloc fixalloc // allocator for span* 152 cachealloc fixalloc // allocator for mcache* 153 treapalloc fixalloc // allocator for treapNodes* used by large objects 154 specialfinalizeralloc fixalloc // allocator for specialfinalizer* 155 specialprofilealloc fixalloc // allocator for specialprofile* 156 speciallock mutex // lock for special record allocators. 157 } 158 159 var mheap_ mheap 160 161 // An MSpan is a run of pages. 162 // 163 // When a MSpan is in the heap free list, state == MSpanFree 164 // and heapmap(s->start) == span, heapmap(s->start+s->npages-1) == span. 165 // 166 // When a MSpan is allocated, state == MSpanInUse or MSpanManual 167 // and heapmap(i) == span for all s->start <= i < s->start+s->npages. 168 169 // Every MSpan is in one doubly-linked list, 170 // either one of the MHeap's free lists or one of the 171 // MCentral's span lists. 172 173 // An MSpan representing actual memory has state _MSpanInUse, 174 // _MSpanManual, or _MSpanFree. Transitions between these states are 175 // constrained as follows: 176 // 177 // * A span may transition from free to in-use or manual during any GC 178 // phase. 179 // 180 // * During sweeping (gcphase == _GCoff), a span may transition from 181 // in-use to free (as a result of sweeping) or manual to free (as a 182 // result of stacks being freed). 183 // 184 // * During GC (gcphase != _GCoff), a span *must not* transition from 185 // manual or in-use to free. Because concurrent GC may read a pointer 186 // and then look up its span, the span state must be monotonic. 187 type mSpanState uint8 188 189 const ( 190 _MSpanDead mSpanState = iota 191 _MSpanInUse // allocated for garbage collected heap 192 _MSpanManual // allocated for manual management (e.g., stack allocator) 193 _MSpanFree 194 ) 195 196 // mSpanStateNames are the names of the span states, indexed by 197 // mSpanState. 198 var mSpanStateNames = []string{ 199 "_MSpanDead", 200 "_MSpanInUse", 201 "_MSpanManual", 202 "_MSpanFree", 203 } 204 205 // mSpanList heads a linked list of spans. 206 // 207 //go:notinheap 208 type mSpanList struct { 209 first *mspan // first span in list, or nil if none 210 last *mspan // last span in list, or nil if none 211 } 212 213 //go:notinheap 214 type mspan struct { 215 next *mspan // next span in list, or nil if none 216 prev *mspan // previous span in list, or nil if none 217 list *mSpanList // For debugging. TODO: Remove. 218 219 startAddr uintptr // address of first byte of span aka s.base() 220 npages uintptr // number of pages in span 221 222 manualFreeList gclinkptr // list of free objects in _MSpanManual spans 223 224 // freeindex is the slot index between 0 and nelems at which to begin scanning 225 // for the next free object in this span. 226 // Each allocation scans allocBits starting at freeindex until it encounters a 0 227 // indicating a free object. freeindex is then adjusted so that subsequent scans begin 228 // just past the newly discovered free object. 229 // 230 // If freeindex == nelem, this span has no free objects. 231 // 232 // allocBits is a bitmap of objects in this span. 233 // If n >= freeindex and allocBits[n/8] & (1<<(n%8)) is 0 234 // then object n is free; 235 // otherwise, object n is allocated. Bits starting at nelem are 236 // undefined and should never be referenced. 237 // 238 // Object n starts at address n*elemsize + (start << pageShift). 239 freeindex uintptr 240 // TODO: Look up nelems from sizeclass and remove this field if it 241 // helps performance. 242 nelems uintptr // number of object in the span. 243 244 // Cache of the allocBits at freeindex. allocCache is shifted 245 // such that the lowest bit corresponds to the bit freeindex. 246 // allocCache holds the complement of allocBits, thus allowing 247 // ctz (count trailing zero) to use it directly. 248 // allocCache may contain bits beyond s.nelems; the caller must ignore 249 // these. 250 allocCache uint64 251 252 // allocBits and gcmarkBits hold pointers to a span's mark and 253 // allocation bits. The pointers are 8 byte aligned. 254 // There are three arenas where this data is held. 255 // free: Dirty arenas that are no longer accessed 256 // and can be reused. 257 // next: Holds information to be used in the next GC cycle. 258 // current: Information being used during this GC cycle. 259 // previous: Information being used during the last GC cycle. 260 // A new GC cycle starts with the call to finishsweep_m. 261 // finishsweep_m moves the previous arena to the free arena, 262 // the current arena to the previous arena, and 263 // the next arena to the current arena. 264 // The next arena is populated as the spans request 265 // memory to hold gcmarkBits for the next GC cycle as well 266 // as allocBits for newly allocated spans. 267 // 268 // The pointer arithmetic is done "by hand" instead of using 269 // arrays to avoid bounds checks along critical performance 270 // paths. 271 // The sweep will free the old allocBits and set allocBits to the 272 // gcmarkBits. The gcmarkBits are replaced with a fresh zeroed 273 // out memory. 274 allocBits *gcBits 275 gcmarkBits *gcBits 276 277 // sweep generation: 278 // if sweepgen == h->sweepgen - 2, the span needs sweeping 279 // if sweepgen == h->sweepgen - 1, the span is currently being swept 280 // if sweepgen == h->sweepgen, the span is swept and ready to use 281 // h->sweepgen is incremented by 2 after every GC 282 283 sweepgen uint32 284 divMul uint16 // for divide by elemsize - divMagic.mul 285 baseMask uint16 // if non-0, elemsize is a power of 2, & this will get object allocation base 286 allocCount uint16 // number of allocated objects 287 spanclass spanClass // size class and noscan (uint8) 288 incache bool // being used by an mcache 289 state mSpanState // mspaninuse etc 290 needzero uint8 // needs to be zeroed before allocation 291 divShift uint8 // for divide by elemsize - divMagic.shift 292 divShift2 uint8 // for divide by elemsize - divMagic.shift2 293 elemsize uintptr // computed from sizeclass or from npages 294 unusedsince int64 // first time spotted by gc in mspanfree state 295 npreleased uintptr // number of pages released to the os 296 limit uintptr // end of data in span 297 speciallock mutex // guards specials list 298 specials *special // linked list of special records sorted by offset. 299 } 300 301 func (s *mspan) base() uintptr { 302 return s.startAddr 303 } 304 305 func (s *mspan) layout() (size, n, total uintptr) { 306 total = s.npages << _PageShift 307 size = s.elemsize 308 if size > 0 { 309 n = total / size 310 } 311 return 312 } 313 314 func recordspan(vh unsafe.Pointer, p unsafe.Pointer) { 315 h := (*mheap)(vh) 316 s := (*mspan)(p) 317 if len(h.allspans) >= cap(h.allspans) { 318 n := 64 * 1024 / sys.PtrSize 319 if n < cap(h.allspans)*3/2 { 320 n = cap(h.allspans) * 3 / 2 321 } 322 var new []*mspan 323 sp := (*slice)(unsafe.Pointer(&new)) 324 sp.array = sysAlloc(uintptr(n)*sys.PtrSize, &memstats.other_sys) 325 if sp.array == nil { 326 throw("runtime: cannot allocate memory") 327 } 328 sp.len = len(h.allspans) 329 sp.cap = n 330 if len(h.allspans) > 0 { 331 copy(new, h.allspans) 332 } 333 oldAllspans := h.allspans 334 h.allspans = new 335 if len(oldAllspans) != 0 { 336 sysFree(unsafe.Pointer(&oldAllspans[0]), uintptr(cap(oldAllspans))*unsafe.Sizeof(oldAllspans[0]), &memstats.other_sys) 337 } 338 } 339 h.allspans = append(h.allspans, s) 340 } 341 342 // A spanClass represents the size class and noscan-ness of a span. 343 // 344 // Each size class has a noscan spanClass and a scan spanClass. The 345 // noscan spanClass contains only noscan objects, which do not contain 346 // pointers and thus do not need to be scanned by the garbage 347 // collector. 348 type spanClass uint8 349 350 const ( 351 numSpanClasses = _NumSizeClasses << 1 352 tinySpanClass = spanClass(tinySizeClass<<1 | 1) 353 ) 354 355 func makeSpanClass(sizeclass uint8, noscan bool) spanClass { 356 return spanClass(sizeclass<<1) | spanClass(bool2int(noscan)) 357 } 358 359 func (sc spanClass) sizeclass() int8 { 360 return int8(sc >> 1) 361 } 362 363 func (sc spanClass) noscan() bool { 364 return sc&1 != 0 365 } 366 367 // inheap reports whether b is a pointer into a (potentially dead) heap object. 368 // It returns false for pointers into _MSpanManual spans. 369 // Non-preemptible because it is used by write barriers. 370 //go:nowritebarrier 371 //go:nosplit 372 func inheap(b uintptr) bool { 373 if b == 0 || b < mheap_.arena_start || b >= mheap_.arena_used { 374 return false 375 } 376 // Not a beginning of a block, consult span table to find the block beginning. 377 s := mheap_.spans[(b-mheap_.arena_start)>>_PageShift] 378 if s == nil || b < s.base() || b >= s.limit || s.state != mSpanInUse { 379 return false 380 } 381 return true 382 } 383 384 // inHeapOrStack is a variant of inheap that returns true for pointers 385 // into any allocated heap span. 386 // 387 //go:nowritebarrier 388 //go:nosplit 389 func inHeapOrStack(b uintptr) bool { 390 if b == 0 || b < mheap_.arena_start || b >= mheap_.arena_used { 391 return false 392 } 393 // Not a beginning of a block, consult span table to find the block beginning. 394 s := mheap_.spans[(b-mheap_.arena_start)>>_PageShift] 395 if s == nil || b < s.base() { 396 return false 397 } 398 switch s.state { 399 case mSpanInUse, _MSpanManual: 400 return b < s.limit 401 default: 402 return false 403 } 404 } 405 406 // TODO: spanOf and spanOfUnchecked are open-coded in a lot of places. 407 // Use the functions instead. 408 409 // spanOf returns the span of p. If p does not point into the heap or 410 // no span contains p, spanOf returns nil. 411 func spanOf(p uintptr) *mspan { 412 if p == 0 || p < mheap_.arena_start || p >= mheap_.arena_used { 413 return nil 414 } 415 return spanOfUnchecked(p) 416 } 417 418 // spanOfUnchecked is equivalent to spanOf, but the caller must ensure 419 // that p points into the heap (that is, mheap_.arena_start <= p < 420 // mheap_.arena_used). 421 func spanOfUnchecked(p uintptr) *mspan { 422 return mheap_.spans[(p-mheap_.arena_start)>>_PageShift] 423 } 424 425 func mlookup(v uintptr, base *uintptr, size *uintptr, sp **mspan) int32 { 426 _g_ := getg() 427 428 _g_.m.mcache.local_nlookup++ 429 if sys.PtrSize == 4 && _g_.m.mcache.local_nlookup >= 1<<30 { 430 // purge cache stats to prevent overflow 431 lock(&mheap_.lock) 432 purgecachedstats(_g_.m.mcache) 433 unlock(&mheap_.lock) 434 } 435 436 s := mheap_.lookupMaybe(unsafe.Pointer(v)) 437 if sp != nil { 438 *sp = s 439 } 440 if s == nil { 441 if base != nil { 442 *base = 0 443 } 444 if size != nil { 445 *size = 0 446 } 447 return 0 448 } 449 450 p := s.base() 451 if s.spanclass.sizeclass() == 0 { 452 // Large object. 453 if base != nil { 454 *base = p 455 } 456 if size != nil { 457 *size = s.npages << _PageShift 458 } 459 return 1 460 } 461 462 n := s.elemsize 463 if base != nil { 464 i := (v - p) / n 465 *base = p + i*n 466 } 467 if size != nil { 468 *size = n 469 } 470 471 return 1 472 } 473 474 // Initialize the heap. 475 func (h *mheap) init(spansStart, spansBytes uintptr) { 476 h.treapalloc.init(unsafe.Sizeof(treapNode{}), nil, nil, &memstats.other_sys) 477 h.spanalloc.init(unsafe.Sizeof(mspan{}), recordspan, unsafe.Pointer(h), &memstats.mspan_sys) 478 h.cachealloc.init(unsafe.Sizeof(mcache{}), nil, nil, &memstats.mcache_sys) 479 h.specialfinalizeralloc.init(unsafe.Sizeof(specialfinalizer{}), nil, nil, &memstats.other_sys) 480 h.specialprofilealloc.init(unsafe.Sizeof(specialprofile{}), nil, nil, &memstats.other_sys) 481 482 // Don't zero mspan allocations. Background sweeping can 483 // inspect a span concurrently with allocating it, so it's 484 // important that the span's sweepgen survive across freeing 485 // and re-allocating a span to prevent background sweeping 486 // from improperly cas'ing it from 0. 487 // 488 // This is safe because mspan contains no heap pointers. 489 h.spanalloc.zero = false 490 491 // h->mapcache needs no init 492 for i := range h.free { 493 h.free[i].init() 494 h.busy[i].init() 495 } 496 497 h.busylarge.init() 498 for i := range h.central { 499 h.central[i].mcentral.init(spanClass(i)) 500 } 501 502 sp := (*slice)(unsafe.Pointer(&h.spans)) 503 sp.array = unsafe.Pointer(spansStart) 504 sp.len = 0 505 sp.cap = int(spansBytes / sys.PtrSize) 506 507 // Map metadata structures. But don't map race detector memory 508 // since we're not actually growing the arena here (and TSAN 509 // gets mad if you map 0 bytes). 510 h.setArenaUsed(h.arena_used, false) 511 } 512 513 // setArenaUsed extends the usable arena to address arena_used and 514 // maps auxiliary VM regions for any newly usable arena space. 515 // 516 // racemap indicates that this memory should be managed by the race 517 // detector. racemap should be true unless this is covering a VM hole. 518 func (h *mheap) setArenaUsed(arena_used uintptr, racemap bool) { 519 // Map auxiliary structures *before* h.arena_used is updated. 520 // Waiting to update arena_used until after the memory has been mapped 521 // avoids faults when other threads try access these regions immediately 522 // after observing the change to arena_used. 523 524 // Map the bitmap. 525 h.mapBits(arena_used) 526 527 // Map spans array. 528 h.mapSpans(arena_used) 529 530 // Tell the race detector about the new heap memory. 531 if racemap && raceenabled { 532 racemapshadow(unsafe.Pointer(h.arena_used), arena_used-h.arena_used) 533 } 534 535 h.arena_used = arena_used 536 } 537 538 // mapSpans makes sure that the spans are mapped 539 // up to the new value of arena_used. 540 // 541 // Don't call this directly. Call mheap.setArenaUsed. 542 func (h *mheap) mapSpans(arena_used uintptr) { 543 // Map spans array, PageSize at a time. 544 n := arena_used 545 n -= h.arena_start 546 n = n / _PageSize * sys.PtrSize 547 n = round(n, physPageSize) 548 need := n / unsafe.Sizeof(h.spans[0]) 549 have := uintptr(len(h.spans)) 550 if have >= need { 551 return 552 } 553 h.spans = h.spans[:need] 554 sysMap(unsafe.Pointer(&h.spans[have]), (need-have)*unsafe.Sizeof(h.spans[0]), h.arena_reserved, &memstats.other_sys) 555 } 556 557 // Sweeps spans in list until reclaims at least npages into heap. 558 // Returns the actual number of pages reclaimed. 559 func (h *mheap) reclaimList(list *mSpanList, npages uintptr) uintptr { 560 n := uintptr(0) 561 sg := mheap_.sweepgen 562 retry: 563 for s := list.first; s != nil; s = s.next { 564 if s.sweepgen == sg-2 && atomic.Cas(&s.sweepgen, sg-2, sg-1) { 565 list.remove(s) 566 // swept spans are at the end of the list 567 list.insertBack(s) // Puts it back on a busy list. s is not in the treap at this point. 568 unlock(&h.lock) 569 snpages := s.npages 570 if s.sweep(false) { 571 n += snpages 572 } 573 lock(&h.lock) 574 if n >= npages { 575 return n 576 } 577 // the span could have been moved elsewhere 578 goto retry 579 } 580 if s.sweepgen == sg-1 { 581 // the span is being sweept by background sweeper, skip 582 continue 583 } 584 // already swept empty span, 585 // all subsequent ones must also be either swept or in process of sweeping 586 break 587 } 588 return n 589 } 590 591 // Sweeps and reclaims at least npage pages into heap. 592 // Called before allocating npage pages. 593 func (h *mheap) reclaim(npage uintptr) { 594 // First try to sweep busy spans with large objects of size >= npage, 595 // this has good chances of reclaiming the necessary space. 596 for i := int(npage); i < len(h.busy); i++ { 597 if h.reclaimList(&h.busy[i], npage) != 0 { 598 return // Bingo! 599 } 600 } 601 602 // Then -- even larger objects. 603 if h.reclaimList(&h.busylarge, npage) != 0 { 604 return // Bingo! 605 } 606 607 // Now try smaller objects. 608 // One such object is not enough, so we need to reclaim several of them. 609 reclaimed := uintptr(0) 610 for i := 0; i < int(npage) && i < len(h.busy); i++ { 611 reclaimed += h.reclaimList(&h.busy[i], npage-reclaimed) 612 if reclaimed >= npage { 613 return 614 } 615 } 616 617 // Now sweep everything that is not yet swept. 618 unlock(&h.lock) 619 for { 620 n := sweepone() 621 if n == ^uintptr(0) { // all spans are swept 622 break 623 } 624 reclaimed += n 625 if reclaimed >= npage { 626 break 627 } 628 } 629 lock(&h.lock) 630 } 631 632 // Allocate a new span of npage pages from the heap for GC'd memory 633 // and record its size class in the HeapMap and HeapMapCache. 634 func (h *mheap) alloc_m(npage uintptr, spanclass spanClass, large bool) *mspan { 635 _g_ := getg() 636 if _g_ != _g_.m.g0 { 637 throw("_mheap_alloc not on g0 stack") 638 } 639 lock(&h.lock) 640 641 // To prevent excessive heap growth, before allocating n pages 642 // we need to sweep and reclaim at least n pages. 643 if h.sweepdone == 0 { 644 // TODO(austin): This tends to sweep a large number of 645 // spans in order to find a few completely free spans 646 // (for example, in the garbage benchmark, this sweeps 647 // ~30x the number of pages its trying to allocate). 648 // If GC kept a bit for whether there were any marks 649 // in a span, we could release these free spans 650 // at the end of GC and eliminate this entirely. 651 if trace.enabled { 652 traceGCSweepStart() 653 } 654 h.reclaim(npage) 655 if trace.enabled { 656 traceGCSweepDone() 657 } 658 } 659 660 // transfer stats from cache to global 661 memstats.heap_scan += uint64(_g_.m.mcache.local_scan) 662 _g_.m.mcache.local_scan = 0 663 memstats.tinyallocs += uint64(_g_.m.mcache.local_tinyallocs) 664 _g_.m.mcache.local_tinyallocs = 0 665 666 s := h.allocSpanLocked(npage, &memstats.heap_inuse) 667 if s != nil { 668 // Record span info, because gc needs to be 669 // able to map interior pointer to containing span. 670 atomic.Store(&s.sweepgen, h.sweepgen) 671 h.sweepSpans[h.sweepgen/2%2].push(s) // Add to swept in-use list. 672 s.state = _MSpanInUse 673 s.allocCount = 0 674 s.spanclass = spanclass 675 if sizeclass := spanclass.sizeclass(); sizeclass == 0 { 676 s.elemsize = s.npages << _PageShift 677 s.divShift = 0 678 s.divMul = 0 679 s.divShift2 = 0 680 s.baseMask = 0 681 } else { 682 s.elemsize = uintptr(class_to_size[sizeclass]) 683 m := &class_to_divmagic[sizeclass] 684 s.divShift = m.shift 685 s.divMul = m.mul 686 s.divShift2 = m.shift2 687 s.baseMask = m.baseMask 688 } 689 690 // update stats, sweep lists 691 h.pagesInUse += uint64(npage) 692 if large { 693 memstats.heap_objects++ 694 mheap_.largealloc += uint64(s.elemsize) 695 mheap_.nlargealloc++ 696 atomic.Xadd64(&memstats.heap_live, int64(npage<<_PageShift)) 697 // Swept spans are at the end of lists. 698 if s.npages < uintptr(len(h.busy)) { 699 h.busy[s.npages].insertBack(s) 700 } else { 701 h.busylarge.insertBack(s) 702 } 703 } 704 } 705 // heap_scan and heap_live were updated. 706 if gcBlackenEnabled != 0 { 707 gcController.revise() 708 } 709 710 if trace.enabled { 711 traceHeapAlloc() 712 } 713 714 // h.spans is accessed concurrently without synchronization 715 // from other threads. Hence, there must be a store/store 716 // barrier here to ensure the writes to h.spans above happen 717 // before the caller can publish a pointer p to an object 718 // allocated from s. As soon as this happens, the garbage 719 // collector running on another processor could read p and 720 // look up s in h.spans. The unlock acts as the barrier to 721 // order these writes. On the read side, the data dependency 722 // between p and the index in h.spans orders the reads. 723 unlock(&h.lock) 724 return s 725 } 726 727 func (h *mheap) alloc(npage uintptr, spanclass spanClass, large bool, needzero bool) *mspan { 728 // Don't do any operations that lock the heap on the G stack. 729 // It might trigger stack growth, and the stack growth code needs 730 // to be able to allocate heap. 731 var s *mspan 732 systemstack(func() { 733 s = h.alloc_m(npage, spanclass, large) 734 }) 735 736 if s != nil { 737 if needzero && s.needzero != 0 { 738 memclrNoHeapPointers(unsafe.Pointer(s.base()), s.npages<<_PageShift) 739 } 740 s.needzero = 0 741 } 742 return s 743 } 744 745 // allocManual allocates a manually-managed span of npage pages. 746 // allocManual returns nil if allocation fails. 747 // 748 // allocManual adds the bytes used to *stat, which should be a 749 // memstats in-use field. Unlike allocations in the GC'd heap, the 750 // allocation does *not* count toward heap_inuse or heap_sys. 751 // 752 // The memory backing the returned span may not be zeroed if 753 // span.needzero is set. 754 // 755 // allocManual must be called on the system stack to prevent stack 756 // growth. Since this is used by the stack allocator, stack growth 757 // during allocManual would self-deadlock. 758 // 759 //go:systemstack 760 func (h *mheap) allocManual(npage uintptr, stat *uint64) *mspan { 761 lock(&h.lock) 762 s := h.allocSpanLocked(npage, stat) 763 if s != nil { 764 s.state = _MSpanManual 765 s.manualFreeList = 0 766 s.allocCount = 0 767 s.spanclass = 0 768 s.nelems = 0 769 s.elemsize = 0 770 s.limit = s.base() + s.npages<<_PageShift 771 // Manually manged memory doesn't count toward heap_sys. 772 memstats.heap_sys -= uint64(s.npages << _PageShift) 773 } 774 775 // This unlock acts as a release barrier. See mheap.alloc_m. 776 unlock(&h.lock) 777 778 return s 779 } 780 781 // Allocates a span of the given size. h must be locked. 782 // The returned span has been removed from the 783 // free list, but its state is still MSpanFree. 784 func (h *mheap) allocSpanLocked(npage uintptr, stat *uint64) *mspan { 785 var list *mSpanList 786 var s *mspan 787 788 // Try in fixed-size lists up to max. 789 for i := int(npage); i < len(h.free); i++ { 790 list = &h.free[i] 791 if !list.isEmpty() { 792 s = list.first 793 list.remove(s) 794 goto HaveSpan 795 } 796 } 797 // Best fit in list of large spans. 798 s = h.allocLarge(npage) // allocLarge removed s from h.freelarge for us 799 if s == nil { 800 if !h.grow(npage) { 801 return nil 802 } 803 s = h.allocLarge(npage) 804 if s == nil { 805 return nil 806 } 807 } 808 809 HaveSpan: 810 // Mark span in use. 811 if s.state != _MSpanFree { 812 throw("MHeap_AllocLocked - MSpan not free") 813 } 814 if s.npages < npage { 815 throw("MHeap_AllocLocked - bad npages") 816 } 817 if s.npreleased > 0 { 818 sysUsed(unsafe.Pointer(s.base()), s.npages<<_PageShift) 819 memstats.heap_released -= uint64(s.npreleased << _PageShift) 820 s.npreleased = 0 821 } 822 823 if s.npages > npage { 824 // Trim extra and put it back in the heap. 825 t := (*mspan)(h.spanalloc.alloc()) 826 t.init(s.base()+npage<<_PageShift, s.npages-npage) 827 s.npages = npage 828 p := (t.base() - h.arena_start) >> _PageShift 829 if p > 0 { 830 h.spans[p-1] = s 831 } 832 h.spans[p] = t 833 h.spans[p+t.npages-1] = t 834 t.needzero = s.needzero 835 s.state = _MSpanManual // prevent coalescing with s 836 t.state = _MSpanManual 837 h.freeSpanLocked(t, false, false, s.unusedsince) 838 s.state = _MSpanFree 839 } 840 s.unusedsince = 0 841 842 p := (s.base() - h.arena_start) >> _PageShift 843 for n := uintptr(0); n < npage; n++ { 844 h.spans[p+n] = s 845 } 846 847 *stat += uint64(npage << _PageShift) 848 memstats.heap_idle -= uint64(npage << _PageShift) 849 850 //println("spanalloc", hex(s.start<<_PageShift)) 851 if s.inList() { 852 throw("still in list") 853 } 854 return s 855 } 856 857 // Large spans have a minimum size of 1MByte. The maximum number of large spans to support 858 // 1TBytes is 1 million, experimentation using random sizes indicates that the depth of 859 // the tree is less that 2x that of a perfectly balanced tree. For 1TByte can be referenced 860 // by a perfectly balanced tree with a a depth of 20. Twice that is an acceptable 40. 861 func (h *mheap) isLargeSpan(npages uintptr) bool { 862 return npages >= uintptr(len(h.free)) 863 } 864 865 // allocLarge allocates a span of at least npage pages from the treap of large spans. 866 // Returns nil if no such span currently exists. 867 func (h *mheap) allocLarge(npage uintptr) *mspan { 868 // Search treap for smallest span with >= npage pages. 869 return h.freelarge.remove(npage) 870 } 871 872 // Try to add at least npage pages of memory to the heap, 873 // returning whether it worked. 874 // 875 // h must be locked. 876 func (h *mheap) grow(npage uintptr) bool { 877 // Ask for a big chunk, to reduce the number of mappings 878 // the operating system needs to track; also amortizes 879 // the overhead of an operating system mapping. 880 // Allocate a multiple of 64kB. 881 npage = round(npage, (64<<10)/_PageSize) 882 ask := npage << _PageShift 883 if ask < _HeapAllocChunk { 884 ask = _HeapAllocChunk 885 } 886 887 v := h.sysAlloc(ask) 888 if v == nil { 889 if ask > npage<<_PageShift { 890 ask = npage << _PageShift 891 v = h.sysAlloc(ask) 892 } 893 if v == nil { 894 print("runtime: out of memory: cannot allocate ", ask, "-byte block (", memstats.heap_sys, " in use)\n") 895 return false 896 } 897 } 898 899 // Create a fake "in use" span and free it, so that the 900 // right coalescing happens. 901 s := (*mspan)(h.spanalloc.alloc()) 902 s.init(uintptr(v), ask>>_PageShift) 903 p := (s.base() - h.arena_start) >> _PageShift 904 for i := p; i < p+s.npages; i++ { 905 h.spans[i] = s 906 } 907 atomic.Store(&s.sweepgen, h.sweepgen) 908 s.state = _MSpanInUse 909 h.pagesInUse += uint64(s.npages) 910 h.freeSpanLocked(s, false, true, 0) 911 return true 912 } 913 914 // Look up the span at the given address. 915 // Address is guaranteed to be in map 916 // and is guaranteed to be start or end of span. 917 func (h *mheap) lookup(v unsafe.Pointer) *mspan { 918 p := uintptr(v) 919 p -= h.arena_start 920 return h.spans[p>>_PageShift] 921 } 922 923 // Look up the span at the given address. 924 // Address is *not* guaranteed to be in map 925 // and may be anywhere in the span. 926 // Map entries for the middle of a span are only 927 // valid for allocated spans. Free spans may have 928 // other garbage in their middles, so we have to 929 // check for that. 930 func (h *mheap) lookupMaybe(v unsafe.Pointer) *mspan { 931 if uintptr(v) < h.arena_start || uintptr(v) >= h.arena_used { 932 return nil 933 } 934 s := h.spans[(uintptr(v)-h.arena_start)>>_PageShift] 935 if s == nil || uintptr(v) < s.base() || uintptr(v) >= uintptr(unsafe.Pointer(s.limit)) || s.state != _MSpanInUse { 936 return nil 937 } 938 return s 939 } 940 941 // Free the span back into the heap. 942 func (h *mheap) freeSpan(s *mspan, acct int32) { 943 systemstack(func() { 944 mp := getg().m 945 lock(&h.lock) 946 memstats.heap_scan += uint64(mp.mcache.local_scan) 947 mp.mcache.local_scan = 0 948 memstats.tinyallocs += uint64(mp.mcache.local_tinyallocs) 949 mp.mcache.local_tinyallocs = 0 950 if msanenabled { 951 // Tell msan that this entire span is no longer in use. 952 base := unsafe.Pointer(s.base()) 953 bytes := s.npages << _PageShift 954 msanfree(base, bytes) 955 } 956 if acct != 0 { 957 memstats.heap_objects-- 958 } 959 if gcBlackenEnabled != 0 { 960 // heap_scan changed. 961 gcController.revise() 962 } 963 h.freeSpanLocked(s, true, true, 0) 964 unlock(&h.lock) 965 }) 966 } 967 968 // freeManual frees a manually-managed span returned by allocManual. 969 // stat must be the same as the stat passed to the allocManual that 970 // allocated s. 971 // 972 // This must only be called when gcphase == _GCoff. See mSpanState for 973 // an explanation. 974 // 975 // freeManual must be called on the system stack to prevent stack 976 // growth, just like allocManual. 977 // 978 //go:systemstack 979 func (h *mheap) freeManual(s *mspan, stat *uint64) { 980 s.needzero = 1 981 lock(&h.lock) 982 *stat -= uint64(s.npages << _PageShift) 983 memstats.heap_sys += uint64(s.npages << _PageShift) 984 h.freeSpanLocked(s, false, true, 0) 985 unlock(&h.lock) 986 } 987 988 // s must be on a busy list (h.busy or h.busylarge) or unlinked. 989 func (h *mheap) freeSpanLocked(s *mspan, acctinuse, acctidle bool, unusedsince int64) { 990 switch s.state { 991 case _MSpanManual: 992 if s.allocCount != 0 { 993 throw("MHeap_FreeSpanLocked - invalid stack free") 994 } 995 case _MSpanInUse: 996 if s.allocCount != 0 || s.sweepgen != h.sweepgen { 997 print("MHeap_FreeSpanLocked - span ", s, " ptr ", hex(s.base()), " allocCount ", s.allocCount, " sweepgen ", s.sweepgen, "/", h.sweepgen, "\n") 998 throw("MHeap_FreeSpanLocked - invalid free") 999 } 1000 h.pagesInUse -= uint64(s.npages) 1001 default: 1002 throw("MHeap_FreeSpanLocked - invalid span state") 1003 } 1004 1005 if acctinuse { 1006 memstats.heap_inuse -= uint64(s.npages << _PageShift) 1007 } 1008 if acctidle { 1009 memstats.heap_idle += uint64(s.npages << _PageShift) 1010 } 1011 s.state = _MSpanFree 1012 if s.inList() { 1013 h.busyList(s.npages).remove(s) 1014 } 1015 1016 // Stamp newly unused spans. The scavenger will use that 1017 // info to potentially give back some pages to the OS. 1018 s.unusedsince = unusedsince 1019 if unusedsince == 0 { 1020 s.unusedsince = nanotime() 1021 } 1022 s.npreleased = 0 1023 1024 // Coalesce with earlier, later spans. 1025 p := (s.base() - h.arena_start) >> _PageShift 1026 if p > 0 { 1027 before := h.spans[p-1] 1028 if before != nil && before.state == _MSpanFree { 1029 // Now adjust s. 1030 s.startAddr = before.startAddr 1031 s.npages += before.npages 1032 s.npreleased = before.npreleased // absorb released pages 1033 s.needzero |= before.needzero 1034 p -= before.npages 1035 h.spans[p] = s 1036 // The size is potentially changing so the treap needs to delete adjacent nodes and 1037 // insert back as a combined node. 1038 if h.isLargeSpan(before.npages) { 1039 // We have a t, it is large so it has to be in the treap so we can remove it. 1040 h.freelarge.removeSpan(before) 1041 } else { 1042 h.freeList(before.npages).remove(before) 1043 } 1044 before.state = _MSpanDead 1045 h.spanalloc.free(unsafe.Pointer(before)) 1046 } 1047 } 1048 1049 // Now check to see if next (greater addresses) span is free and can be coalesced. 1050 if (p + s.npages) < uintptr(len(h.spans)) { 1051 after := h.spans[p+s.npages] 1052 if after != nil && after.state == _MSpanFree { 1053 s.npages += after.npages 1054 s.npreleased += after.npreleased 1055 s.needzero |= after.needzero 1056 h.spans[p+s.npages-1] = s 1057 if h.isLargeSpan(after.npages) { 1058 h.freelarge.removeSpan(after) 1059 } else { 1060 h.freeList(after.npages).remove(after) 1061 } 1062 after.state = _MSpanDead 1063 h.spanalloc.free(unsafe.Pointer(after)) 1064 } 1065 } 1066 1067 // Insert s into appropriate list or treap. 1068 if h.isLargeSpan(s.npages) { 1069 h.freelarge.insert(s) 1070 } else { 1071 h.freeList(s.npages).insert(s) 1072 } 1073 } 1074 1075 func (h *mheap) freeList(npages uintptr) *mSpanList { 1076 return &h.free[npages] 1077 } 1078 1079 func (h *mheap) busyList(npages uintptr) *mSpanList { 1080 if npages < uintptr(len(h.busy)) { 1081 return &h.busy[npages] 1082 } 1083 return &h.busylarge 1084 } 1085 1086 func scavengeTreapNode(t *treapNode, now, limit uint64) uintptr { 1087 s := t.spanKey 1088 var sumreleased uintptr 1089 if (now-uint64(s.unusedsince)) > limit && s.npreleased != s.npages { 1090 start := s.base() 1091 end := start + s.npages<<_PageShift 1092 if physPageSize > _PageSize { 1093 // We can only release pages in 1094 // physPageSize blocks, so round start 1095 // and end in. (Otherwise, madvise 1096 // will round them *out* and release 1097 // more memory than we want.) 1098 start = (start + physPageSize - 1) &^ (physPageSize - 1) 1099 end &^= physPageSize - 1 1100 if end <= start { 1101 // start and end don't span a 1102 // whole physical page. 1103 return sumreleased 1104 } 1105 } 1106 len := end - start 1107 released := len - (s.npreleased << _PageShift) 1108 if physPageSize > _PageSize && released == 0 { 1109 return sumreleased 1110 } 1111 memstats.heap_released += uint64(released) 1112 sumreleased += released 1113 s.npreleased = len >> _PageShift 1114 sysUnused(unsafe.Pointer(start), len) 1115 } 1116 return sumreleased 1117 } 1118 1119 func scavengelist(list *mSpanList, now, limit uint64) uintptr { 1120 if list.isEmpty() { 1121 return 0 1122 } 1123 1124 var sumreleased uintptr 1125 for s := list.first; s != nil; s = s.next { 1126 if (now-uint64(s.unusedsince)) > limit && s.npreleased != s.npages { 1127 start := s.base() 1128 end := start + s.npages<<_PageShift 1129 if physPageSize > _PageSize { 1130 // We can only release pages in 1131 // physPageSize blocks, so round start 1132 // and end in. (Otherwise, madvise 1133 // will round them *out* and release 1134 // more memory than we want.) 1135 start = (start + physPageSize - 1) &^ (physPageSize - 1) 1136 end &^= physPageSize - 1 1137 if end <= start { 1138 // start and end don't span a 1139 // whole physical page. 1140 continue 1141 } 1142 } 1143 len := end - start 1144 1145 released := len - (s.npreleased << _PageShift) 1146 if physPageSize > _PageSize && released == 0 { 1147 continue 1148 } 1149 memstats.heap_released += uint64(released) 1150 sumreleased += released 1151 s.npreleased = len >> _PageShift 1152 sysUnused(unsafe.Pointer(start), len) 1153 } 1154 } 1155 return sumreleased 1156 } 1157 1158 func (h *mheap) scavenge(k int32, now, limit uint64) { 1159 // Disallow malloc or panic while holding the heap lock. We do 1160 // this here because this is an non-mallocgc entry-point to 1161 // the mheap API. 1162 gp := getg() 1163 gp.m.mallocing++ 1164 lock(&h.lock) 1165 var sumreleased uintptr 1166 for i := 0; i < len(h.free); i++ { 1167 sumreleased += scavengelist(&h.free[i], now, limit) 1168 } 1169 sumreleased += scavengetreap(h.freelarge.treap, now, limit) 1170 unlock(&h.lock) 1171 gp.m.mallocing-- 1172 1173 if debug.gctrace > 0 { 1174 if sumreleased > 0 { 1175 print("scvg", k, ": ", sumreleased>>20, " MB released\n") 1176 } 1177 print("scvg", k, ": inuse: ", memstats.heap_inuse>>20, ", idle: ", memstats.heap_idle>>20, ", sys: ", memstats.heap_sys>>20, ", released: ", memstats.heap_released>>20, ", consumed: ", (memstats.heap_sys-memstats.heap_released)>>20, " (MB)\n") 1178 } 1179 } 1180 1181 //go:linkname runtime_debug_freeOSMemory runtime/debug.freeOSMemory 1182 func runtime_debug_freeOSMemory() { 1183 GC() 1184 systemstack(func() { mheap_.scavenge(-1, ^uint64(0), 0) }) 1185 } 1186 1187 // Initialize a new span with the given start and npages. 1188 func (span *mspan) init(base uintptr, npages uintptr) { 1189 // span is *not* zeroed. 1190 span.next = nil 1191 span.prev = nil 1192 span.list = nil 1193 span.startAddr = base 1194 span.npages = npages 1195 span.allocCount = 0 1196 span.spanclass = 0 1197 span.incache = false 1198 span.elemsize = 0 1199 span.state = _MSpanDead 1200 span.unusedsince = 0 1201 span.npreleased = 0 1202 span.speciallock.key = 0 1203 span.specials = nil 1204 span.needzero = 0 1205 span.freeindex = 0 1206 span.allocBits = nil 1207 span.gcmarkBits = nil 1208 } 1209 1210 func (span *mspan) inList() bool { 1211 return span.list != nil 1212 } 1213 1214 // Initialize an empty doubly-linked list. 1215 func (list *mSpanList) init() { 1216 list.first = nil 1217 list.last = nil 1218 } 1219 1220 func (list *mSpanList) remove(span *mspan) { 1221 if span.list != list { 1222 print("runtime: failed MSpanList_Remove span.npages=", span.npages, 1223 " span=", span, " prev=", span.prev, " span.list=", span.list, " list=", list, "\n") 1224 throw("MSpanList_Remove") 1225 } 1226 if list.first == span { 1227 list.first = span.next 1228 } else { 1229 span.prev.next = span.next 1230 } 1231 if list.last == span { 1232 list.last = span.prev 1233 } else { 1234 span.next.prev = span.prev 1235 } 1236 span.next = nil 1237 span.prev = nil 1238 span.list = nil 1239 } 1240 1241 func (list *mSpanList) isEmpty() bool { 1242 return list.first == nil 1243 } 1244 1245 func (list *mSpanList) insert(span *mspan) { 1246 if span.next != nil || span.prev != nil || span.list != nil { 1247 println("runtime: failed MSpanList_Insert", span, span.next, span.prev, span.list) 1248 throw("MSpanList_Insert") 1249 } 1250 span.next = list.first 1251 if list.first != nil { 1252 // The list contains at least one span; link it in. 1253 // The last span in the list doesn't change. 1254 list.first.prev = span 1255 } else { 1256 // The list contains no spans, so this is also the last span. 1257 list.last = span 1258 } 1259 list.first = span 1260 span.list = list 1261 } 1262 1263 func (list *mSpanList) insertBack(span *mspan) { 1264 if span.next != nil || span.prev != nil || span.list != nil { 1265 println("runtime: failed MSpanList_InsertBack", span, span.next, span.prev, span.list) 1266 throw("MSpanList_InsertBack") 1267 } 1268 span.prev = list.last 1269 if list.last != nil { 1270 // The list contains at least one span. 1271 list.last.next = span 1272 } else { 1273 // The list contains no spans, so this is also the first span. 1274 list.first = span 1275 } 1276 list.last = span 1277 span.list = list 1278 } 1279 1280 // takeAll removes all spans from other and inserts them at the front 1281 // of list. 1282 func (list *mSpanList) takeAll(other *mSpanList) { 1283 if other.isEmpty() { 1284 return 1285 } 1286 1287 // Reparent everything in other to list. 1288 for s := other.first; s != nil; s = s.next { 1289 s.list = list 1290 } 1291 1292 // Concatenate the lists. 1293 if list.isEmpty() { 1294 *list = *other 1295 } else { 1296 // Neither list is empty. Put other before list. 1297 other.last.next = list.first 1298 list.first.prev = other.last 1299 list.first = other.first 1300 } 1301 1302 other.first, other.last = nil, nil 1303 } 1304 1305 const ( 1306 _KindSpecialFinalizer = 1 1307 _KindSpecialProfile = 2 1308 // Note: The finalizer special must be first because if we're freeing 1309 // an object, a finalizer special will cause the freeing operation 1310 // to abort, and we want to keep the other special records around 1311 // if that happens. 1312 ) 1313 1314 //go:notinheap 1315 type special struct { 1316 next *special // linked list in span 1317 offset uint16 // span offset of object 1318 kind byte // kind of special 1319 } 1320 1321 // Adds the special record s to the list of special records for 1322 // the object p. All fields of s should be filled in except for 1323 // offset & next, which this routine will fill in. 1324 // Returns true if the special was successfully added, false otherwise. 1325 // (The add will fail only if a record with the same p and s->kind 1326 // already exists.) 1327 func addspecial(p unsafe.Pointer, s *special) bool { 1328 span := mheap_.lookupMaybe(p) 1329 if span == nil { 1330 throw("addspecial on invalid pointer") 1331 } 1332 1333 // Ensure that the span is swept. 1334 // Sweeping accesses the specials list w/o locks, so we have 1335 // to synchronize with it. And it's just much safer. 1336 mp := acquirem() 1337 span.ensureSwept() 1338 1339 offset := uintptr(p) - span.base() 1340 kind := s.kind 1341 1342 lock(&span.speciallock) 1343 1344 // Find splice point, check for existing record. 1345 t := &span.specials 1346 for { 1347 x := *t 1348 if x == nil { 1349 break 1350 } 1351 if offset == uintptr(x.offset) && kind == x.kind { 1352 unlock(&span.speciallock) 1353 releasem(mp) 1354 return false // already exists 1355 } 1356 if offset < uintptr(x.offset) || (offset == uintptr(x.offset) && kind < x.kind) { 1357 break 1358 } 1359 t = &x.next 1360 } 1361 1362 // Splice in record, fill in offset. 1363 s.offset = uint16(offset) 1364 s.next = *t 1365 *t = s 1366 unlock(&span.speciallock) 1367 releasem(mp) 1368 1369 return true 1370 } 1371 1372 // Removes the Special record of the given kind for the object p. 1373 // Returns the record if the record existed, nil otherwise. 1374 // The caller must FixAlloc_Free the result. 1375 func removespecial(p unsafe.Pointer, kind uint8) *special { 1376 span := mheap_.lookupMaybe(p) 1377 if span == nil { 1378 throw("removespecial on invalid pointer") 1379 } 1380 1381 // Ensure that the span is swept. 1382 // Sweeping accesses the specials list w/o locks, so we have 1383 // to synchronize with it. And it's just much safer. 1384 mp := acquirem() 1385 span.ensureSwept() 1386 1387 offset := uintptr(p) - span.base() 1388 1389 lock(&span.speciallock) 1390 t := &span.specials 1391 for { 1392 s := *t 1393 if s == nil { 1394 break 1395 } 1396 // This function is used for finalizers only, so we don't check for 1397 // "interior" specials (p must be exactly equal to s->offset). 1398 if offset == uintptr(s.offset) && kind == s.kind { 1399 *t = s.next 1400 unlock(&span.speciallock) 1401 releasem(mp) 1402 return s 1403 } 1404 t = &s.next 1405 } 1406 unlock(&span.speciallock) 1407 releasem(mp) 1408 return nil 1409 } 1410 1411 // The described object has a finalizer set for it. 1412 // 1413 // specialfinalizer is allocated from non-GC'd memory, so any heap 1414 // pointers must be specially handled. 1415 // 1416 //go:notinheap 1417 type specialfinalizer struct { 1418 special special 1419 fn *funcval // May be a heap pointer. 1420 nret uintptr 1421 fint *_type // May be a heap pointer, but always live. 1422 ot *ptrtype // May be a heap pointer, but always live. 1423 } 1424 1425 // Adds a finalizer to the object p. Returns true if it succeeded. 1426 func addfinalizer(p unsafe.Pointer, f *funcval, nret uintptr, fint *_type, ot *ptrtype) bool { 1427 lock(&mheap_.speciallock) 1428 s := (*specialfinalizer)(mheap_.specialfinalizeralloc.alloc()) 1429 unlock(&mheap_.speciallock) 1430 s.special.kind = _KindSpecialFinalizer 1431 s.fn = f 1432 s.nret = nret 1433 s.fint = fint 1434 s.ot = ot 1435 if addspecial(p, &s.special) { 1436 // This is responsible for maintaining the same 1437 // GC-related invariants as markrootSpans in any 1438 // situation where it's possible that markrootSpans 1439 // has already run but mark termination hasn't yet. 1440 if gcphase != _GCoff { 1441 _, base, _ := findObject(p) 1442 mp := acquirem() 1443 gcw := &mp.p.ptr().gcw 1444 // Mark everything reachable from the object 1445 // so it's retained for the finalizer. 1446 scanobject(uintptr(base), gcw) 1447 // Mark the finalizer itself, since the 1448 // special isn't part of the GC'd heap. 1449 scanblock(uintptr(unsafe.Pointer(&s.fn)), sys.PtrSize, &oneptrmask[0], gcw) 1450 if gcBlackenPromptly { 1451 gcw.dispose() 1452 } 1453 releasem(mp) 1454 } 1455 return true 1456 } 1457 1458 // There was an old finalizer 1459 lock(&mheap_.speciallock) 1460 mheap_.specialfinalizeralloc.free(unsafe.Pointer(s)) 1461 unlock(&mheap_.speciallock) 1462 return false 1463 } 1464 1465 // Removes the finalizer (if any) from the object p. 1466 func removefinalizer(p unsafe.Pointer) { 1467 s := (*specialfinalizer)(unsafe.Pointer(removespecial(p, _KindSpecialFinalizer))) 1468 if s == nil { 1469 return // there wasn't a finalizer to remove 1470 } 1471 lock(&mheap_.speciallock) 1472 mheap_.specialfinalizeralloc.free(unsafe.Pointer(s)) 1473 unlock(&mheap_.speciallock) 1474 } 1475 1476 // The described object is being heap profiled. 1477 // 1478 //go:notinheap 1479 type specialprofile struct { 1480 special special 1481 b *bucket 1482 } 1483 1484 // Set the heap profile bucket associated with addr to b. 1485 func setprofilebucket(p unsafe.Pointer, b *bucket) { 1486 lock(&mheap_.speciallock) 1487 s := (*specialprofile)(mheap_.specialprofilealloc.alloc()) 1488 unlock(&mheap_.speciallock) 1489 s.special.kind = _KindSpecialProfile 1490 s.b = b 1491 if !addspecial(p, &s.special) { 1492 throw("setprofilebucket: profile already set") 1493 } 1494 } 1495 1496 // Do whatever cleanup needs to be done to deallocate s. It has 1497 // already been unlinked from the MSpan specials list. 1498 func freespecial(s *special, p unsafe.Pointer, size uintptr) { 1499 switch s.kind { 1500 case _KindSpecialFinalizer: 1501 sf := (*specialfinalizer)(unsafe.Pointer(s)) 1502 queuefinalizer(p, sf.fn, sf.nret, sf.fint, sf.ot) 1503 lock(&mheap_.speciallock) 1504 mheap_.specialfinalizeralloc.free(unsafe.Pointer(sf)) 1505 unlock(&mheap_.speciallock) 1506 case _KindSpecialProfile: 1507 sp := (*specialprofile)(unsafe.Pointer(s)) 1508 mProf_Free(sp.b, size) 1509 lock(&mheap_.speciallock) 1510 mheap_.specialprofilealloc.free(unsafe.Pointer(sp)) 1511 unlock(&mheap_.speciallock) 1512 default: 1513 throw("bad special kind") 1514 panic("not reached") 1515 } 1516 } 1517 1518 // gcBits is an alloc/mark bitmap. This is always used as *gcBits. 1519 // 1520 //go:notinheap 1521 type gcBits uint8 1522 1523 // bytep returns a pointer to the n'th byte of b. 1524 func (b *gcBits) bytep(n uintptr) *uint8 { 1525 return addb((*uint8)(b), n) 1526 } 1527 1528 // bitp returns a pointer to the byte containing bit n and a mask for 1529 // selecting that bit from *bytep. 1530 func (b *gcBits) bitp(n uintptr) (bytep *uint8, mask uint8) { 1531 return b.bytep(n / 8), 1 << (n % 8) 1532 } 1533 1534 const gcBitsChunkBytes = uintptr(64 << 10) 1535 const gcBitsHeaderBytes = unsafe.Sizeof(gcBitsHeader{}) 1536 1537 type gcBitsHeader struct { 1538 free uintptr // free is the index into bits of the next free byte. 1539 next uintptr // *gcBits triggers recursive type bug. (issue 14620) 1540 } 1541 1542 //go:notinheap 1543 type gcBitsArena struct { 1544 // gcBitsHeader // side step recursive type bug (issue 14620) by including fields by hand. 1545 free uintptr // free is the index into bits of the next free byte; read/write atomically 1546 next *gcBitsArena 1547 bits [gcBitsChunkBytes - gcBitsHeaderBytes]gcBits 1548 } 1549 1550 var gcBitsArenas struct { 1551 lock mutex 1552 free *gcBitsArena 1553 next *gcBitsArena // Read atomically. Write atomically under lock. 1554 current *gcBitsArena 1555 previous *gcBitsArena 1556 } 1557 1558 // tryAlloc allocates from b or returns nil if b does not have enough room. 1559 // This is safe to call concurrently. 1560 func (b *gcBitsArena) tryAlloc(bytes uintptr) *gcBits { 1561 if b == nil || atomic.Loaduintptr(&b.free)+bytes > uintptr(len(b.bits)) { 1562 return nil 1563 } 1564 // Try to allocate from this block. 1565 end := atomic.Xadduintptr(&b.free, bytes) 1566 if end > uintptr(len(b.bits)) { 1567 return nil 1568 } 1569 // There was enough room. 1570 start := end - bytes 1571 return &b.bits[start] 1572 } 1573 1574 // newMarkBits returns a pointer to 8 byte aligned bytes 1575 // to be used for a span's mark bits. 1576 func newMarkBits(nelems uintptr) *gcBits { 1577 blocksNeeded := uintptr((nelems + 63) / 64) 1578 bytesNeeded := blocksNeeded * 8 1579 1580 // Try directly allocating from the current head arena. 1581 head := (*gcBitsArena)(atomic.Loadp(unsafe.Pointer(&gcBitsArenas.next))) 1582 if p := head.tryAlloc(bytesNeeded); p != nil { 1583 return p 1584 } 1585 1586 // There's not enough room in the head arena. We may need to 1587 // allocate a new arena. 1588 lock(&gcBitsArenas.lock) 1589 // Try the head arena again, since it may have changed. Now 1590 // that we hold the lock, the list head can't change, but its 1591 // free position still can. 1592 if p := gcBitsArenas.next.tryAlloc(bytesNeeded); p != nil { 1593 unlock(&gcBitsArenas.lock) 1594 return p 1595 } 1596 1597 // Allocate a new arena. This may temporarily drop the lock. 1598 fresh := newArenaMayUnlock() 1599 // If newArenaMayUnlock dropped the lock, another thread may 1600 // have put a fresh arena on the "next" list. Try allocating 1601 // from next again. 1602 if p := gcBitsArenas.next.tryAlloc(bytesNeeded); p != nil { 1603 // Put fresh back on the free list. 1604 // TODO: Mark it "already zeroed" 1605 fresh.next = gcBitsArenas.free 1606 gcBitsArenas.free = fresh 1607 unlock(&gcBitsArenas.lock) 1608 return p 1609 } 1610 1611 // Allocate from the fresh arena. We haven't linked it in yet, so 1612 // this cannot race and is guaranteed to succeed. 1613 p := fresh.tryAlloc(bytesNeeded) 1614 if p == nil { 1615 throw("markBits overflow") 1616 } 1617 1618 // Add the fresh arena to the "next" list. 1619 fresh.next = gcBitsArenas.next 1620 atomic.StorepNoWB(unsafe.Pointer(&gcBitsArenas.next), unsafe.Pointer(fresh)) 1621 1622 unlock(&gcBitsArenas.lock) 1623 return p 1624 } 1625 1626 // newAllocBits returns a pointer to 8 byte aligned bytes 1627 // to be used for this span's alloc bits. 1628 // newAllocBits is used to provide newly initialized spans 1629 // allocation bits. For spans not being initialized the 1630 // the mark bits are repurposed as allocation bits when 1631 // the span is swept. 1632 func newAllocBits(nelems uintptr) *gcBits { 1633 return newMarkBits(nelems) 1634 } 1635 1636 // nextMarkBitArenaEpoch establishes a new epoch for the arenas 1637 // holding the mark bits. The arenas are named relative to the 1638 // current GC cycle which is demarcated by the call to finishweep_m. 1639 // 1640 // All current spans have been swept. 1641 // During that sweep each span allocated room for its gcmarkBits in 1642 // gcBitsArenas.next block. gcBitsArenas.next becomes the gcBitsArenas.current 1643 // where the GC will mark objects and after each span is swept these bits 1644 // will be used to allocate objects. 1645 // gcBitsArenas.current becomes gcBitsArenas.previous where the span's 1646 // gcAllocBits live until all the spans have been swept during this GC cycle. 1647 // The span's sweep extinguishes all the references to gcBitsArenas.previous 1648 // by pointing gcAllocBits into the gcBitsArenas.current. 1649 // The gcBitsArenas.previous is released to the gcBitsArenas.free list. 1650 func nextMarkBitArenaEpoch() { 1651 lock(&gcBitsArenas.lock) 1652 if gcBitsArenas.previous != nil { 1653 if gcBitsArenas.free == nil { 1654 gcBitsArenas.free = gcBitsArenas.previous 1655 } else { 1656 // Find end of previous arenas. 1657 last := gcBitsArenas.previous 1658 for last = gcBitsArenas.previous; last.next != nil; last = last.next { 1659 } 1660 last.next = gcBitsArenas.free 1661 gcBitsArenas.free = gcBitsArenas.previous 1662 } 1663 } 1664 gcBitsArenas.previous = gcBitsArenas.current 1665 gcBitsArenas.current = gcBitsArenas.next 1666 atomic.StorepNoWB(unsafe.Pointer(&gcBitsArenas.next), nil) // newMarkBits calls newArena when needed 1667 unlock(&gcBitsArenas.lock) 1668 } 1669 1670 // newArenaMayUnlock allocates and zeroes a gcBits arena. 1671 // The caller must hold gcBitsArena.lock. This may temporarily release it. 1672 func newArenaMayUnlock() *gcBitsArena { 1673 var result *gcBitsArena 1674 if gcBitsArenas.free == nil { 1675 unlock(&gcBitsArenas.lock) 1676 result = (*gcBitsArena)(sysAlloc(gcBitsChunkBytes, &memstats.gc_sys)) 1677 if result == nil { 1678 throw("runtime: cannot allocate memory") 1679 } 1680 lock(&gcBitsArenas.lock) 1681 } else { 1682 result = gcBitsArenas.free 1683 gcBitsArenas.free = gcBitsArenas.free.next 1684 memclrNoHeapPointers(unsafe.Pointer(result), gcBitsChunkBytes) 1685 } 1686 result.next = nil 1687 // If result.bits is not 8 byte aligned adjust index so 1688 // that &result.bits[result.free] is 8 byte aligned. 1689 if uintptr(unsafe.Offsetof(gcBitsArena{}.bits))&7 == 0 { 1690 result.free = 0 1691 } else { 1692 result.free = 8 - (uintptr(unsafe.Pointer(&result.bits[0])) & 7) 1693 } 1694 return result 1695 }