github.com/epfl-dcsl/gotee@v0.0.0-20200909122901-014b35f5e5e9/src/runtime/mheap.go (about) 1 // Copyright 2009 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 // Page heap. 6 // 7 // See malloc.go for overview. 8 9 package runtime 10 11 import ( 12 "runtime/internal/atomic" 13 "runtime/internal/sys" 14 "unsafe" 15 ) 16 17 // minPhysPageSize is a lower-bound on the physical page size. The 18 // true physical page size may be larger than this. In contrast, 19 // sys.PhysPageSize is an upper-bound on the physical page size. 20 const minPhysPageSize = 4096 21 22 // Main malloc heap. 23 // The heap itself is the "free[]" and "large" arrays, 24 // but all the other global data is here too. 25 // 26 // mheap must not be heap-allocated because it contains mSpanLists, 27 // which must not be heap-allocated. 28 // 29 //go:notinheap 30 type mheap struct { 31 lock mutex 32 free [_MaxMHeapList]mSpanList // free lists of given length up to _MaxMHeapList 33 freelarge mTreap // free treap of length >= _MaxMHeapList 34 busy [_MaxMHeapList]mSpanList // busy lists of large spans of given length 35 busylarge mSpanList // busy lists of large spans length >= _MaxMHeapList 36 sweepgen uint32 // sweep generation, see comment in mspan 37 sweepdone uint32 // all spans are swept 38 sweepers uint32 // number of active sweepone calls 39 40 // allspans is a slice of all mspans ever created. Each mspan 41 // appears exactly once. 42 // 43 // The memory for allspans is manually managed and can be 44 // reallocated and move as the heap grows. 45 // 46 // In general, allspans is protected by mheap_.lock, which 47 // prevents concurrent access as well as freeing the backing 48 // store. Accesses during STW might not hold the lock, but 49 // must ensure that allocation cannot happen around the 50 // access (since that may free the backing store). 51 allspans []*mspan // all spans out there 52 53 // spans is a lookup table to map virtual address page IDs to *mspan. 54 // For allocated spans, their pages map to the span itself. 55 // For free spans, only the lowest and highest pages map to the span itself. 56 // Internal pages map to an arbitrary span. 57 // For pages that have never been allocated, spans entries are nil. 58 // 59 // Modifications are protected by mheap.lock. Reads can be 60 // performed without locking, but ONLY from indexes that are 61 // known to contain in-use or stack spans. This means there 62 // must not be a safe-point between establishing that an 63 // address is live and looking it up in the spans array. 64 // 65 // This is backed by a reserved region of the address space so 66 // it can grow without moving. The memory up to len(spans) is 67 // mapped. cap(spans) indicates the total reserved memory. 68 spans []*mspan 69 70 // sweepSpans contains two mspan stacks: one of swept in-use 71 // spans, and one of unswept in-use spans. These two trade 72 // roles on each GC cycle. Since the sweepgen increases by 2 73 // on each cycle, this means the swept spans are in 74 // sweepSpans[sweepgen/2%2] and the unswept spans are in 75 // sweepSpans[1-sweepgen/2%2]. Sweeping pops spans from the 76 // unswept stack and pushes spans that are still in-use on the 77 // swept stack. Likewise, allocating an in-use span pushes it 78 // on the swept stack. 79 sweepSpans [2]gcSweepBuf 80 81 _ uint32 // align uint64 fields on 32-bit for atomics 82 83 // Proportional sweep 84 // 85 // These parameters represent a linear function from heap_live 86 // to page sweep count. The proportional sweep system works to 87 // stay in the black by keeping the current page sweep count 88 // above this line at the current heap_live. 89 // 90 // The line has slope sweepPagesPerByte and passes through a 91 // basis point at (sweepHeapLiveBasis, pagesSweptBasis). At 92 // any given time, the system is at (memstats.heap_live, 93 // pagesSwept) in this space. 94 // 95 // It's important that the line pass through a point we 96 // control rather than simply starting at a (0,0) origin 97 // because that lets us adjust sweep pacing at any time while 98 // accounting for current progress. If we could only adjust 99 // the slope, it would create a discontinuity in debt if any 100 // progress has already been made. 101 pagesInUse uint64 // pages of spans in stats _MSpanInUse; R/W with mheap.lock 102 pagesSwept uint64 // pages swept this cycle; updated atomically 103 pagesSweptBasis uint64 // pagesSwept to use as the origin of the sweep ratio; updated atomically 104 sweepHeapLiveBasis uint64 // value of heap_live to use as the origin of sweep ratio; written with lock, read without 105 sweepPagesPerByte float64 // proportional sweep ratio; written with lock, read without 106 // TODO(austin): pagesInUse should be a uintptr, but the 386 107 // compiler can't 8-byte align fields. 108 109 // Malloc stats. 110 largealloc uint64 // bytes allocated for large objects 111 nlargealloc uint64 // number of large object allocations 112 largefree uint64 // bytes freed for large objects (>maxsmallsize) 113 nlargefree uint64 // number of frees for large objects (>maxsmallsize) 114 nsmallfree [_NumSizeClasses]uint64 // number of frees for small objects (<=maxsmallsize) 115 116 // range of addresses we might see in the heap 117 bitmap uintptr // Points to one byte past the end of the bitmap 118 bitmap_mapped uintptr 119 120 // The arena_* fields indicate the addresses of the Go heap. 121 // 122 // The maximum range of the Go heap is 123 // [arena_start, arena_start+_MaxMem+1). 124 // 125 // The range of the current Go heap is 126 // [arena_start, arena_used). Parts of this range may not be 127 // mapped, but the metadata structures are always mapped for 128 // the full range. 129 arena_start uintptr 130 arena_used uintptr // Set with setArenaUsed. 131 132 // The heap is grown using a linear allocator that allocates 133 // from the block [arena_alloc, arena_end). arena_alloc is 134 // often, but *not always* equal to arena_used. 135 arena_alloc uintptr 136 arena_end uintptr 137 138 // arena_reserved indicates that the memory [arena_alloc, 139 // arena_end) is reserved (e.g., mapped PROT_NONE). If this is 140 // false, we have to be careful not to clobber existing 141 // mappings here. If this is true, then we own the mapping 142 // here and *must* clobber it to use it. 143 arena_reserved bool 144 145 _ uint32 // ensure 64-bit alignment 146 147 // central free lists for small size classes. 148 // the padding makes sure that the MCentrals are 149 // spaced CacheLineSize bytes apart, so that each MCentral.lock 150 // gets its own cache line. 151 // central is indexed by spanClass. 152 central [numSpanClasses]struct { 153 mcentral mcentral 154 pad [sys.CacheLineSize - unsafe.Sizeof(mcentral{})%sys.CacheLineSize]byte 155 } 156 157 spanalloc fixalloc // allocator for span* 158 cachealloc fixalloc // allocator for mcache* 159 treapalloc fixalloc // allocator for treapNodes* used by large objects 160 specialfinalizeralloc fixalloc // allocator for specialfinalizer* 161 specialprofilealloc fixalloc // allocator for specialprofile* 162 speciallock mutex // lock for special record allocators. 163 164 unused *specialfinalizer // never set, just here to force the specialfinalizer type into DWARF 165 } 166 167 var mheap_ mheap 168 169 // An MSpan is a run of pages. 170 // 171 // When a MSpan is in the heap free list, state == MSpanFree 172 // and heapmap(s->start) == span, heapmap(s->start+s->npages-1) == span. 173 // 174 // When a MSpan is allocated, state == MSpanInUse or MSpanManual 175 // and heapmap(i) == span for all s->start <= i < s->start+s->npages. 176 177 // Every MSpan is in one doubly-linked list, 178 // either one of the MHeap's free lists or one of the 179 // MCentral's span lists. 180 181 // An MSpan representing actual memory has state _MSpanInUse, 182 // _MSpanManual, or _MSpanFree. Transitions between these states are 183 // constrained as follows: 184 // 185 // * A span may transition from free to in-use or manual during any GC 186 // phase. 187 // 188 // * During sweeping (gcphase == _GCoff), a span may transition from 189 // in-use to free (as a result of sweeping) or manual to free (as a 190 // result of stacks being freed). 191 // 192 // * During GC (gcphase != _GCoff), a span *must not* transition from 193 // manual or in-use to free. Because concurrent GC may read a pointer 194 // and then look up its span, the span state must be monotonic. 195 type mSpanState uint8 196 197 const ( 198 _MSpanDead mSpanState = iota 199 _MSpanInUse // allocated for garbage collected heap 200 _MSpanManual // allocated for manual management (e.g., stack allocator) 201 _MSpanFree 202 ) 203 204 // mSpanStateNames are the names of the span states, indexed by 205 // mSpanState. 206 var mSpanStateNames = []string{ 207 "_MSpanDead", 208 "_MSpanInUse", 209 "_MSpanManual", 210 "_MSpanFree", 211 } 212 213 // mSpanList heads a linked list of spans. 214 // 215 //go:notinheap 216 type mSpanList struct { 217 first *mspan // first span in list, or nil if none 218 last *mspan // last span in list, or nil if none 219 } 220 221 //go:notinheap 222 type mspan struct { 223 next *mspan // next span in list, or nil if none 224 prev *mspan // previous span in list, or nil if none 225 list *mSpanList // For debugging. TODO: Remove. 226 227 startAddr uintptr // address of first byte of span aka s.base() 228 npages uintptr // number of pages in span 229 230 manualFreeList gclinkptr // list of free objects in _MSpanManual spans 231 232 // freeindex is the slot index between 0 and nelems at which to begin scanning 233 // for the next free object in this span. 234 // Each allocation scans allocBits starting at freeindex until it encounters a 0 235 // indicating a free object. freeindex is then adjusted so that subsequent scans begin 236 // just past the newly discovered free object. 237 // 238 // If freeindex == nelem, this span has no free objects. 239 // 240 // allocBits is a bitmap of objects in this span. 241 // If n >= freeindex and allocBits[n/8] & (1<<(n%8)) is 0 242 // then object n is free; 243 // otherwise, object n is allocated. Bits starting at nelem are 244 // undefined and should never be referenced. 245 // 246 // Object n starts at address n*elemsize + (start << pageShift). 247 freeindex uintptr 248 // TODO: Look up nelems from sizeclass and remove this field if it 249 // helps performance. 250 nelems uintptr // number of object in the span. 251 252 // Cache of the allocBits at freeindex. allocCache is shifted 253 // such that the lowest bit corresponds to the bit freeindex. 254 // allocCache holds the complement of allocBits, thus allowing 255 // ctz (count trailing zero) to use it directly. 256 // allocCache may contain bits beyond s.nelems; the caller must ignore 257 // these. 258 allocCache uint64 259 260 // allocBits and gcmarkBits hold pointers to a span's mark and 261 // allocation bits. The pointers are 8 byte aligned. 262 // There are three arenas where this data is held. 263 // free: Dirty arenas that are no longer accessed 264 // and can be reused. 265 // next: Holds information to be used in the next GC cycle. 266 // current: Information being used during this GC cycle. 267 // previous: Information being used during the last GC cycle. 268 // A new GC cycle starts with the call to finishsweep_m. 269 // finishsweep_m moves the previous arena to the free arena, 270 // the current arena to the previous arena, and 271 // the next arena to the current arena. 272 // The next arena is populated as the spans request 273 // memory to hold gcmarkBits for the next GC cycle as well 274 // as allocBits for newly allocated spans. 275 // 276 // The pointer arithmetic is done "by hand" instead of using 277 // arrays to avoid bounds checks along critical performance 278 // paths. 279 // The sweep will free the old allocBits and set allocBits to the 280 // gcmarkBits. The gcmarkBits are replaced with a fresh zeroed 281 // out memory. 282 allocBits *gcBits 283 gcmarkBits *gcBits 284 285 // sweep generation: 286 // if sweepgen == h->sweepgen - 2, the span needs sweeping 287 // if sweepgen == h->sweepgen - 1, the span is currently being swept 288 // if sweepgen == h->sweepgen, the span is swept and ready to use 289 // h->sweepgen is incremented by 2 after every GC 290 291 sweepgen uint32 292 divMul uint16 // for divide by elemsize - divMagic.mul 293 baseMask uint16 // if non-0, elemsize is a power of 2, & this will get object allocation base 294 allocCount uint16 // number of allocated objects 295 spanclass spanClass // size class and noscan (uint8) 296 incache bool // being used by an mcache 297 state mSpanState // mspaninuse etc 298 needzero uint8 // needs to be zeroed before allocation 299 divShift uint8 // for divide by elemsize - divMagic.shift 300 divShift2 uint8 // for divide by elemsize - divMagic.shift2 301 elemsize uintptr // computed from sizeclass or from npages 302 unusedsince int64 // first time spotted by gc in mspanfree state 303 npreleased uintptr // number of pages released to the os 304 limit uintptr // end of data in span 305 speciallock mutex // guards specials list 306 specials *special // linked list of special records sorted by offset. 307 } 308 309 func (s *mspan) base() uintptr { 310 return s.startAddr 311 } 312 313 func (s *mspan) layout() (size, n, total uintptr) { 314 total = s.npages << _PageShift 315 size = s.elemsize 316 if size > 0 { 317 n = total / size 318 } 319 return 320 } 321 322 // recordspan adds a newly allocated span to h.allspans. 323 // 324 // This only happens the first time a span is allocated from 325 // mheap.spanalloc (it is not called when a span is reused). 326 // 327 // Write barriers are disallowed here because it can be called from 328 // gcWork when allocating new workbufs. However, because it's an 329 // indirect call from the fixalloc initializer, the compiler can't see 330 // this. 331 // 332 //go:nowritebarrierrec 333 func recordspan(vh unsafe.Pointer, p unsafe.Pointer) { 334 h := (*mheap)(vh) 335 s := (*mspan)(p) 336 if len(h.allspans) >= cap(h.allspans) { 337 n := 64 * 1024 / sys.PtrSize 338 if n < cap(h.allspans)*3/2 { 339 n = cap(h.allspans) * 3 / 2 340 } 341 var new []*mspan 342 sp := (*slice)(unsafe.Pointer(&new)) 343 sp.array = sysAlloc(uintptr(n)*sys.PtrSize, &memstats.other_sys) 344 if sp.array == nil { 345 throw("runtime: cannot allocate memory") 346 } 347 sp.len = len(h.allspans) 348 sp.cap = n 349 if len(h.allspans) > 0 { 350 copy(new, h.allspans) 351 } 352 oldAllspans := h.allspans 353 *(*notInHeapSlice)(unsafe.Pointer(&h.allspans)) = *(*notInHeapSlice)(unsafe.Pointer(&new)) 354 if len(oldAllspans) != 0 { 355 //TODO aghosn handle that better. 356 sysFree(unsafe.Pointer(&oldAllspans[0]), uintptr(cap(oldAllspans))*unsafe.Sizeof(oldAllspans[0]), &memstats.other_sys) 357 } 358 } 359 h.allspans = h.allspans[:len(h.allspans)+1] 360 h.allspans[len(h.allspans)-1] = s 361 } 362 363 // A spanClass represents the size class and noscan-ness of a span. 364 // 365 // Each size class has a noscan spanClass and a scan spanClass. The 366 // noscan spanClass contains only noscan objects, which do not contain 367 // pointers and thus do not need to be scanned by the garbage 368 // collector. 369 type spanClass uint8 370 371 const ( 372 numSpanClasses = _NumSizeClasses << 1 373 tinySpanClass = spanClass(tinySizeClass<<1 | 1) 374 ) 375 376 func makeSpanClass(sizeclass uint8, noscan bool) spanClass { 377 return spanClass(sizeclass<<1) | spanClass(bool2int(noscan)) 378 } 379 380 func (sc spanClass) sizeclass() int8 { 381 return int8(sc >> 1) 382 } 383 384 func (sc spanClass) noscan() bool { 385 return sc&1 != 0 386 } 387 388 // inheap reports whether b is a pointer into a (potentially dead) heap object. 389 // It returns false for pointers into _MSpanManual spans. 390 // Non-preemptible because it is used by write barriers. 391 //go:nowritebarrier 392 //go:nosplit 393 func inheap(b uintptr) bool { 394 if b == 0 || b < mheap_.arena_start || b >= mheap_.arena_used { 395 return false 396 } 397 // Not a beginning of a block, consult span table to find the block beginning. 398 s := mheap_.spans[(b-mheap_.arena_start)>>_PageShift] 399 if s == nil || b < s.base() || b >= s.limit || s.state != mSpanInUse { 400 return false 401 } 402 return true 403 } 404 405 // inHeapOrStack is a variant of inheap that returns true for pointers 406 // into any allocated heap span. 407 // 408 //go:nowritebarrier 409 //go:nosplit 410 func inHeapOrStack(b uintptr) bool { 411 if b == 0 || b < mheap_.arena_start || b >= mheap_.arena_used { 412 return false 413 } 414 // Not a beginning of a block, consult span table to find the block beginning. 415 s := mheap_.spans[(b-mheap_.arena_start)>>_PageShift] 416 if s == nil || b < s.base() { 417 return false 418 } 419 switch s.state { 420 case mSpanInUse, _MSpanManual: 421 return b < s.limit 422 default: 423 return false 424 } 425 } 426 427 // TODO: spanOf and spanOfUnchecked are open-coded in a lot of places. 428 // Use the functions instead. 429 430 // spanOf returns the span of p. If p does not point into the heap or 431 // no span contains p, spanOf returns nil. 432 func spanOf(p uintptr) *mspan { 433 if p == 0 || p < mheap_.arena_start || p >= mheap_.arena_used { 434 return nil 435 } 436 return spanOfUnchecked(p) 437 } 438 439 // spanOfUnchecked is equivalent to spanOf, but the caller must ensure 440 // that p points into the heap (that is, mheap_.arena_start <= p < 441 // mheap_.arena_used). 442 func spanOfUnchecked(p uintptr) *mspan { 443 return mheap_.spans[(p-mheap_.arena_start)>>_PageShift] 444 } 445 446 func mlookup(v uintptr, base *uintptr, size *uintptr, sp **mspan) int32 { 447 _g_ := getg() 448 449 _g_.m.mcache.local_nlookup++ 450 if sys.PtrSize == 4 && _g_.m.mcache.local_nlookup >= 1<<30 { 451 // purge cache stats to prevent overflow 452 lock(&mheap_.lock) 453 purgecachedstats(_g_.m.mcache) 454 unlock(&mheap_.lock) 455 } 456 457 s := mheap_.lookupMaybe(unsafe.Pointer(v)) 458 if sp != nil { 459 *sp = s 460 } 461 if s == nil { 462 if base != nil { 463 *base = 0 464 } 465 if size != nil { 466 *size = 0 467 } 468 return 0 469 } 470 471 p := s.base() 472 if s.spanclass.sizeclass() == 0 { 473 // Large object. 474 if base != nil { 475 *base = p 476 } 477 if size != nil { 478 *size = s.npages << _PageShift 479 } 480 return 1 481 } 482 483 n := s.elemsize 484 if base != nil { 485 i := (v - p) / n 486 *base = p + i*n 487 } 488 if size != nil { 489 *size = n 490 } 491 492 return 1 493 } 494 495 // Initialize the heap. (TODO(aghosn) here I had a problem with the unlocked) 496 func (h *mheap) init(spansStart, spansBytes uintptr) { 497 h.treapalloc.init(unsafe.Sizeof(treapNode{}), nil, nil, &memstats.other_sys) 498 h.spanalloc.init(unsafe.Sizeof(mspan{}), recordspan, unsafe.Pointer(h), &memstats.mspan_sys) 499 h.cachealloc.init(unsafe.Sizeof(mcache{}), nil, nil, &memstats.mcache_sys) 500 h.specialfinalizeralloc.init(unsafe.Sizeof(specialfinalizer{}), nil, nil, &memstats.other_sys) 501 h.specialprofilealloc.init(unsafe.Sizeof(specialprofile{}), nil, nil, &memstats.other_sys) 502 503 // Don't zero mspan allocations. Background sweeping can 504 // inspect a span concurrently with allocating it, so it's 505 // important that the span's sweepgen survive across freeing 506 // and re-allocating a span to prevent background sweeping 507 // from improperly cas'ing it from 0. 508 // 509 // This is safe because mspan contains no heap pointers. 510 h.spanalloc.zero = false 511 512 // h->mapcache needs no init 513 for i := range h.free { 514 h.free[i].init() 515 h.busy[i].init() 516 } 517 518 h.busylarge.init() 519 for i := range h.central { 520 h.central[i].mcentral.init(spanClass(i)) 521 } 522 523 sp := (*slice)(unsafe.Pointer(&h.spans)) 524 sp.array = unsafe.Pointer(spansStart) 525 sp.len = 0 526 sp.cap = int(spansBytes / sys.PtrSize) 527 528 // Map metadata structures. But don't map race detector memory 529 // since we're not actually growing the arena here (and TSAN 530 // gets mad if you map 0 bytes). 531 h.setArenaUsed(h.arena_used, false) 532 } 533 534 // setArenaUsed extends the usable arena to address arena_used and 535 // maps auxiliary VM regions for any newly usable arena space. 536 // 537 // racemap indicates that this memory should be managed by the race 538 // detector. racemap should be true unless this is covering a VM hole. 539 func (h *mheap) setArenaUsed(arena_used uintptr, racemap bool) { 540 // Map auxiliary structures *before* h.arena_used is updated. 541 // Waiting to update arena_used until after the memory has been mapped 542 // avoids faults when other threads try access these regions immediately 543 // after observing the change to arena_used. 544 545 // Map the bitmap. 546 h.mapBits(arena_used) 547 548 // Map spans array. 549 h.mapSpans(arena_used) 550 551 // Tell the race detector about the new heap memory. 552 if racemap && raceenabled { 553 racemapshadow(unsafe.Pointer(h.arena_used), arena_used-h.arena_used) 554 } 555 556 h.arena_used = arena_used 557 } 558 559 // mapSpans makes sure that the spans are mapped 560 // up to the new value of arena_used. 561 // 562 // Don't call this directly. Call mheap.setArenaUsed. 563 func (h *mheap) mapSpans(arena_used uintptr) { 564 // Map spans array, PageSize at a time. 565 n := arena_used 566 n -= h.arena_start 567 n = n / _PageSize * sys.PtrSize 568 n = round(n, physPageSize) 569 need := n / unsafe.Sizeof(h.spans[0]) 570 have := uintptr(len(h.spans)) 571 if have >= need { 572 return 573 } 574 h.spans = h.spans[:need] 575 sysMap(unsafe.Pointer(&h.spans[have]), (need-have)*unsafe.Sizeof(h.spans[0]), h.arena_reserved, &memstats.other_sys) 576 } 577 578 // Sweeps spans in list until reclaims at least npages into heap. 579 // Returns the actual number of pages reclaimed. 580 func (h *mheap) reclaimList(list *mSpanList, npages uintptr) uintptr { 581 n := uintptr(0) 582 sg := mheap_.sweepgen 583 retry: 584 for s := list.first; s != nil; s = s.next { 585 if s.sweepgen == sg-2 && atomic.Cas(&s.sweepgen, sg-2, sg-1) { 586 list.remove(s) 587 // swept spans are at the end of the list 588 list.insertBack(s) // Puts it back on a busy list. s is not in the treap at this point. 589 unlock(&h.lock) 590 snpages := s.npages 591 if s.sweep(false) { 592 n += snpages 593 } 594 lock(&h.lock) 595 if n >= npages { 596 return n 597 } 598 // the span could have been moved elsewhere 599 goto retry 600 } 601 if s.sweepgen == sg-1 { 602 // the span is being sweept by background sweeper, skip 603 continue 604 } 605 // already swept empty span, 606 // all subsequent ones must also be either swept or in process of sweeping 607 break 608 } 609 return n 610 } 611 612 // Sweeps and reclaims at least npage pages into heap. 613 // Called before allocating npage pages. 614 func (h *mheap) reclaim(npage uintptr) { 615 // First try to sweep busy spans with large objects of size >= npage, 616 // this has good chances of reclaiming the necessary space. 617 for i := int(npage); i < len(h.busy); i++ { 618 if h.reclaimList(&h.busy[i], npage) != 0 { 619 return // Bingo! 620 } 621 } 622 623 // Then -- even larger objects. 624 if h.reclaimList(&h.busylarge, npage) != 0 { 625 return // Bingo! 626 } 627 628 // Now try smaller objects. 629 // One such object is not enough, so we need to reclaim several of them. 630 reclaimed := uintptr(0) 631 for i := 0; i < int(npage) && i < len(h.busy); i++ { 632 reclaimed += h.reclaimList(&h.busy[i], npage-reclaimed) 633 if reclaimed >= npage { 634 return 635 } 636 } 637 638 // Now sweep everything that is not yet swept. 639 unlock(&h.lock) 640 for { 641 n := sweepone() 642 if n == ^uintptr(0) { // all spans are swept 643 break 644 } 645 reclaimed += n 646 if reclaimed >= npage { 647 break 648 } 649 } 650 lock(&h.lock) 651 } 652 653 // Allocate a new span of npage pages from the heap for GC'd memory 654 // and record its size class in the HeapMap and HeapMapCache. 655 func (h *mheap) alloc_m(npage uintptr, spanclass spanClass, large bool) *mspan { 656 _g_ := getg() 657 if _g_ != _g_.m.g0 { 658 throw("_mheap_alloc not on g0 stack") 659 } 660 lock(&h.lock) 661 662 // To prevent excessive heap growth, before allocating n pages 663 // we need to sweep and reclaim at least n pages. 664 if h.sweepdone == 0 { 665 // TODO(austin): This tends to sweep a large number of 666 // spans in order to find a few completely free spans 667 // (for example, in the garbage benchmark, this sweeps 668 // ~30x the number of pages its trying to allocate). 669 // If GC kept a bit for whether there were any marks 670 // in a span, we could release these free spans 671 // at the end of GC and eliminate this entirely. 672 if trace.enabled { 673 traceGCSweepStart() 674 } 675 h.reclaim(npage) 676 if trace.enabled { 677 traceGCSweepDone() 678 } 679 } 680 681 // transfer stats from cache to global 682 memstats.heap_scan += uint64(_g_.m.mcache.local_scan) 683 _g_.m.mcache.local_scan = 0 684 memstats.tinyallocs += uint64(_g_.m.mcache.local_tinyallocs) 685 _g_.m.mcache.local_tinyallocs = 0 686 687 s := h.allocSpanLocked(npage, &memstats.heap_inuse) 688 if s != nil { 689 // Record span info, because gc needs to be 690 // able to map interior pointer to containing span. 691 atomic.Store(&s.sweepgen, h.sweepgen) 692 h.sweepSpans[h.sweepgen/2%2].push(s) // Add to swept in-use list. 693 s.state = _MSpanInUse 694 s.allocCount = 0 695 s.spanclass = spanclass 696 if sizeclass := spanclass.sizeclass(); sizeclass == 0 { 697 s.elemsize = s.npages << _PageShift 698 s.divShift = 0 699 s.divMul = 0 700 s.divShift2 = 0 701 s.baseMask = 0 702 } else { 703 s.elemsize = uintptr(class_to_size[sizeclass]) 704 m := &class_to_divmagic[sizeclass] 705 s.divShift = m.shift 706 s.divMul = m.mul 707 s.divShift2 = m.shift2 708 s.baseMask = m.baseMask 709 } 710 711 // update stats, sweep lists 712 h.pagesInUse += uint64(npage) 713 if large { 714 memstats.heap_objects++ 715 mheap_.largealloc += uint64(s.elemsize) 716 mheap_.nlargealloc++ 717 atomic.Xadd64(&memstats.heap_live, int64(npage<<_PageShift)) 718 // Swept spans are at the end of lists. 719 if s.npages < uintptr(len(h.busy)) { 720 h.busy[s.npages].insertBack(s) 721 } else { 722 h.busylarge.insertBack(s) 723 } 724 } 725 } 726 // heap_scan and heap_live were updated. 727 if gcBlackenEnabled != 0 { 728 gcController.revise() 729 } 730 731 if trace.enabled { 732 traceHeapAlloc() 733 } 734 735 // h.spans is accessed concurrently without synchronization 736 // from other threads. Hence, there must be a store/store 737 // barrier here to ensure the writes to h.spans above happen 738 // before the caller can publish a pointer p to an object 739 // allocated from s. As soon as this happens, the garbage 740 // collector running on another processor could read p and 741 // look up s in h.spans. The unlock acts as the barrier to 742 // order these writes. On the read side, the data dependency 743 // between p and the index in h.spans orders the reads. 744 unlock(&h.lock) 745 return s 746 } 747 748 func (h *mheap) alloc(npage uintptr, spanclass spanClass, large bool, needzero bool) *mspan { 749 // Don't do any operations that lock the heap on the G stack. 750 // It might trigger stack growth, and the stack growth code needs 751 // to be able to allocate heap. 752 var s *mspan 753 systemstack(func() { 754 s = h.alloc_m(npage, spanclass, large) 755 }) 756 757 if s != nil { 758 if needzero && s.needzero != 0 { 759 memclrNoHeapPointers(unsafe.Pointer(s.base()), s.npages<<_PageShift) 760 } 761 s.needzero = 0 762 } 763 return s 764 } 765 766 // allocManual allocates a manually-managed span of npage pages. 767 // allocManual returns nil if allocation fails. 768 // 769 // allocManual adds the bytes used to *stat, which should be a 770 // memstats in-use field. Unlike allocations in the GC'd heap, the 771 // allocation does *not* count toward heap_inuse or heap_sys. 772 // 773 // The memory backing the returned span may not be zeroed if 774 // span.needzero is set. 775 // 776 // allocManual must be called on the system stack to prevent stack 777 // growth. Since this is used by the stack allocator, stack growth 778 // during allocManual would self-deadlock. 779 // 780 //go:systemstack 781 func (h *mheap) allocManual(npage uintptr, stat *uint64) *mspan { 782 lock(&h.lock) 783 s := h.allocSpanLocked(npage, stat) 784 if s != nil { 785 s.state = _MSpanManual 786 s.manualFreeList = 0 787 s.allocCount = 0 788 s.spanclass = 0 789 s.nelems = 0 790 s.elemsize = 0 791 s.limit = s.base() + s.npages<<_PageShift 792 // Manually manged memory doesn't count toward heap_sys. 793 memstats.heap_sys -= uint64(s.npages << _PageShift) 794 } 795 796 // This unlock acts as a release barrier. See mheap.alloc_m. 797 unlock(&h.lock) 798 799 return s 800 } 801 802 // Allocates a span of the given size. h must be locked. 803 // The returned span has been removed from the 804 // free list, but its state is still MSpanFree. 805 func (h *mheap) allocSpanLocked(npage uintptr, stat *uint64) *mspan { 806 var list *mSpanList 807 var s *mspan 808 809 // Try in fixed-size lists up to max. 810 for i := int(npage); i < len(h.free); i++ { 811 list = &h.free[i] 812 if !list.isEmpty() { 813 s = list.first 814 list.remove(s) 815 goto HaveSpan 816 } 817 } 818 // Best fit in list of large spans. 819 s = h.allocLarge(npage) // allocLarge removed s from h.freelarge for us 820 if s == nil { 821 if !h.grow(npage) { 822 return nil 823 } 824 s = h.allocLarge(npage) 825 if s == nil { 826 return nil 827 } 828 } 829 830 HaveSpan: 831 // Mark span in use. 832 if s.state != _MSpanFree { 833 throw("MHeap_AllocLocked - MSpan not free") 834 } 835 if s.npages < npage { 836 throw("MHeap_AllocLocked - bad npages") 837 } 838 if s.npreleased > 0 { 839 sysUsed(unsafe.Pointer(s.base()), s.npages<<_PageShift) 840 memstats.heap_released -= uint64(s.npreleased << _PageShift) 841 s.npreleased = 0 842 } 843 844 if s.npages > npage { 845 // Trim extra and put it back in the heap. 846 t := (*mspan)(h.spanalloc.alloc()) 847 t.init(s.base()+npage<<_PageShift, s.npages-npage) 848 s.npages = npage 849 p := (t.base() - h.arena_start) >> _PageShift 850 if p > 0 { 851 h.spans[p-1] = s 852 } 853 h.spans[p] = t 854 h.spans[p+t.npages-1] = t 855 t.needzero = s.needzero 856 s.state = _MSpanManual // prevent coalescing with s 857 t.state = _MSpanManual 858 h.freeSpanLocked(t, false, false, s.unusedsince) 859 s.state = _MSpanFree 860 } 861 s.unusedsince = 0 862 863 p := (s.base() - h.arena_start) >> _PageShift 864 for n := uintptr(0); n < npage; n++ { 865 h.spans[p+n] = s 866 } 867 868 *stat += uint64(npage << _PageShift) 869 memstats.heap_idle -= uint64(npage << _PageShift) 870 871 //println("spanalloc", hex(s.start<<_PageShift)) 872 if s.inList() { 873 throw("still in list") 874 } 875 return s 876 } 877 878 // Large spans have a minimum size of 1MByte. The maximum number of large spans to support 879 // 1TBytes is 1 million, experimentation using random sizes indicates that the depth of 880 // the tree is less that 2x that of a perfectly balanced tree. For 1TByte can be referenced 881 // by a perfectly balanced tree with a depth of 20. Twice that is an acceptable 40. 882 func (h *mheap) isLargeSpan(npages uintptr) bool { 883 return npages >= uintptr(len(h.free)) 884 } 885 886 // allocLarge allocates a span of at least npage pages from the treap of large spans. 887 // Returns nil if no such span currently exists. 888 func (h *mheap) allocLarge(npage uintptr) *mspan { 889 // Search treap for smallest span with >= npage pages. 890 return h.freelarge.remove(npage) 891 } 892 893 // Try to add at least npage pages of memory to the heap, 894 // returning whether it worked. 895 // 896 // h must be locked. 897 func (h *mheap) grow(npage uintptr) bool { 898 // Ask for a big chunk, to reduce the number of mappings 899 // the operating system needs to track; also amortizes 900 // the overhead of an operating system mapping. 901 // Allocate a multiple of 64kB. 902 npage = round(npage, (64<<10)/_PageSize) 903 ask := npage << _PageShift 904 if ask < _HeapAllocChunk { 905 ask = _HeapAllocChunk 906 } 907 908 v := h.sysAlloc(ask) 909 if v == nil { 910 if ask > npage<<_PageShift { 911 ask = npage << _PageShift 912 v = h.sysAlloc(ask) 913 } 914 if v == nil { 915 print("runtime: out of memory: cannot allocate ", ask, "-byte block (", memstats.heap_sys, " in use)\n") 916 return false 917 } 918 } 919 920 // Create a fake "in use" span and free it, so that the 921 // right coalescing happens. 922 s := (*mspan)(h.spanalloc.alloc()) 923 s.init(uintptr(v), ask>>_PageShift) 924 p := (s.base() - h.arena_start) >> _PageShift 925 for i := p; i < p+s.npages; i++ { 926 h.spans[i] = s 927 } 928 atomic.Store(&s.sweepgen, h.sweepgen) 929 s.state = _MSpanInUse 930 h.pagesInUse += uint64(s.npages) 931 h.freeSpanLocked(s, false, true, 0) 932 return true 933 } 934 935 // Look up the span at the given address. 936 // Address is guaranteed to be in map 937 // and is guaranteed to be start or end of span. 938 func (h *mheap) lookup(v unsafe.Pointer) *mspan { 939 p := uintptr(v) 940 p -= h.arena_start 941 return h.spans[p>>_PageShift] 942 } 943 944 // Look up the span at the given address. 945 // Address is *not* guaranteed to be in map 946 // and may be anywhere in the span. 947 // Map entries for the middle of a span are only 948 // valid for allocated spans. Free spans may have 949 // other garbage in their middles, so we have to 950 // check for that. 951 func (h *mheap) lookupMaybe(v unsafe.Pointer) *mspan { 952 if uintptr(v) < h.arena_start || uintptr(v) >= h.arena_used { 953 return nil 954 } 955 s := h.spans[(uintptr(v)-h.arena_start)>>_PageShift] 956 if s == nil || uintptr(v) < s.base() || uintptr(v) >= uintptr(unsafe.Pointer(s.limit)) || s.state != _MSpanInUse { 957 return nil 958 } 959 return s 960 } 961 962 // Free the span back into the heap. 963 func (h *mheap) freeSpan(s *mspan, acct int32) { 964 systemstack(func() { 965 mp := getg().m 966 lock(&h.lock) 967 memstats.heap_scan += uint64(mp.mcache.local_scan) 968 mp.mcache.local_scan = 0 969 memstats.tinyallocs += uint64(mp.mcache.local_tinyallocs) 970 mp.mcache.local_tinyallocs = 0 971 if msanenabled { 972 // Tell msan that this entire span is no longer in use. 973 base := unsafe.Pointer(s.base()) 974 bytes := s.npages << _PageShift 975 msanfree(base, bytes) 976 } 977 if acct != 0 { 978 memstats.heap_objects-- 979 } 980 if gcBlackenEnabled != 0 { 981 // heap_scan changed. 982 gcController.revise() 983 } 984 h.freeSpanLocked(s, true, true, 0) 985 unlock(&h.lock) 986 }) 987 } 988 989 // freeManual frees a manually-managed span returned by allocManual. 990 // stat must be the same as the stat passed to the allocManual that 991 // allocated s. 992 // 993 // This must only be called when gcphase == _GCoff. See mSpanState for 994 // an explanation. 995 // 996 // freeManual must be called on the system stack to prevent stack 997 // growth, just like allocManual. 998 // 999 //go:systemstack 1000 func (h *mheap) freeManual(s *mspan, stat *uint64) { 1001 s.needzero = 1 1002 lock(&h.lock) 1003 *stat -= uint64(s.npages << _PageShift) 1004 memstats.heap_sys += uint64(s.npages << _PageShift) 1005 h.freeSpanLocked(s, false, true, 0) 1006 unlock(&h.lock) 1007 } 1008 1009 // s must be on a busy list (h.busy or h.busylarge) or unlinked. 1010 func (h *mheap) freeSpanLocked(s *mspan, acctinuse, acctidle bool, unusedsince int64) { 1011 switch s.state { 1012 case _MSpanManual: 1013 if s.allocCount != 0 { 1014 throw("MHeap_FreeSpanLocked - invalid stack free") 1015 } 1016 case _MSpanInUse: 1017 if s.allocCount != 0 || s.sweepgen != h.sweepgen { 1018 print("MHeap_FreeSpanLocked - span ", s, " ptr ", hex(s.base()), " allocCount ", s.allocCount, " sweepgen ", s.sweepgen, "/", h.sweepgen, "\n") 1019 throw("MHeap_FreeSpanLocked - invalid free") 1020 } 1021 h.pagesInUse -= uint64(s.npages) 1022 default: 1023 throw("MHeap_FreeSpanLocked - invalid span state") 1024 } 1025 1026 if acctinuse { 1027 memstats.heap_inuse -= uint64(s.npages << _PageShift) 1028 } 1029 if acctidle { 1030 memstats.heap_idle += uint64(s.npages << _PageShift) 1031 } 1032 s.state = _MSpanFree 1033 if s.inList() { 1034 h.busyList(s.npages).remove(s) 1035 } 1036 1037 // Stamp newly unused spans. The scavenger will use that 1038 // info to potentially give back some pages to the OS. 1039 s.unusedsince = unusedsince 1040 if unusedsince == 0 { 1041 s.unusedsince = nanotime() 1042 } 1043 s.npreleased = 0 1044 1045 // Coalesce with earlier, later spans. 1046 p := (s.base() - h.arena_start) >> _PageShift 1047 if p > 0 { 1048 before := h.spans[p-1] 1049 if before != nil && before.state == _MSpanFree { 1050 // Now adjust s. 1051 s.startAddr = before.startAddr 1052 s.npages += before.npages 1053 s.npreleased = before.npreleased // absorb released pages 1054 s.needzero |= before.needzero 1055 p -= before.npages 1056 h.spans[p] = s 1057 // The size is potentially changing so the treap needs to delete adjacent nodes and 1058 // insert back as a combined node. 1059 if h.isLargeSpan(before.npages) { 1060 // We have a t, it is large so it has to be in the treap so we can remove it. 1061 h.freelarge.removeSpan(before) 1062 } else { 1063 h.freeList(before.npages).remove(before) 1064 } 1065 before.state = _MSpanDead 1066 h.spanalloc.free(unsafe.Pointer(before)) 1067 } 1068 } 1069 1070 // Now check to see if next (greater addresses) span is free and can be coalesced. 1071 if (p + s.npages) < uintptr(len(h.spans)) { 1072 after := h.spans[p+s.npages] 1073 if after != nil && after.state == _MSpanFree { 1074 s.npages += after.npages 1075 s.npreleased += after.npreleased 1076 s.needzero |= after.needzero 1077 h.spans[p+s.npages-1] = s 1078 if h.isLargeSpan(after.npages) { 1079 h.freelarge.removeSpan(after) 1080 } else { 1081 h.freeList(after.npages).remove(after) 1082 } 1083 after.state = _MSpanDead 1084 h.spanalloc.free(unsafe.Pointer(after)) 1085 } 1086 } 1087 1088 // Insert s into appropriate list or treap. 1089 if h.isLargeSpan(s.npages) { 1090 h.freelarge.insert(s) 1091 } else { 1092 h.freeList(s.npages).insert(s) 1093 } 1094 } 1095 1096 func (h *mheap) freeList(npages uintptr) *mSpanList { 1097 return &h.free[npages] 1098 } 1099 1100 func (h *mheap) busyList(npages uintptr) *mSpanList { 1101 if npages < uintptr(len(h.busy)) { 1102 return &h.busy[npages] 1103 } 1104 return &h.busylarge 1105 } 1106 1107 func scavengeTreapNode(t *treapNode, now, limit uint64) uintptr { 1108 s := t.spanKey 1109 var sumreleased uintptr 1110 if (now-uint64(s.unusedsince)) > limit && s.npreleased != s.npages { 1111 start := s.base() 1112 end := start + s.npages<<_PageShift 1113 if physPageSize > _PageSize { 1114 // We can only release pages in 1115 // physPageSize blocks, so round start 1116 // and end in. (Otherwise, madvise 1117 // will round them *out* and release 1118 // more memory than we want.) 1119 start = (start + physPageSize - 1) &^ (physPageSize - 1) 1120 end &^= physPageSize - 1 1121 if end <= start { 1122 // start and end don't span a 1123 // whole physical page. 1124 return sumreleased 1125 } 1126 } 1127 len := end - start 1128 released := len - (s.npreleased << _PageShift) 1129 if physPageSize > _PageSize && released == 0 { 1130 return sumreleased 1131 } 1132 memstats.heap_released += uint64(released) 1133 sumreleased += released 1134 s.npreleased = len >> _PageShift 1135 sysUnused(unsafe.Pointer(start), len) 1136 } 1137 return sumreleased 1138 } 1139 1140 func scavengelist(list *mSpanList, now, limit uint64) uintptr { 1141 if list.isEmpty() { 1142 return 0 1143 } 1144 1145 var sumreleased uintptr 1146 for s := list.first; s != nil; s = s.next { 1147 if (now-uint64(s.unusedsince)) <= limit || s.npreleased == s.npages { 1148 continue 1149 } 1150 start := s.base() 1151 end := start + s.npages<<_PageShift 1152 if physPageSize > _PageSize { 1153 // We can only release pages in 1154 // physPageSize blocks, so round start 1155 // and end in. (Otherwise, madvise 1156 // will round them *out* and release 1157 // more memory than we want.) 1158 start = (start + physPageSize - 1) &^ (physPageSize - 1) 1159 end &^= physPageSize - 1 1160 if end <= start { 1161 // start and end don't span a 1162 // whole physical page. 1163 continue 1164 } 1165 } 1166 len := end - start 1167 1168 released := len - (s.npreleased << _PageShift) 1169 if physPageSize > _PageSize && released == 0 { 1170 continue 1171 } 1172 memstats.heap_released += uint64(released) 1173 sumreleased += released 1174 s.npreleased = len >> _PageShift 1175 sysUnused(unsafe.Pointer(start), len) 1176 } 1177 return sumreleased 1178 } 1179 1180 func (h *mheap) scavenge(k int32, now, limit uint64) { 1181 // Disallow malloc or panic while holding the heap lock. We do 1182 // this here because this is an non-mallocgc entry-point to 1183 // the mheap API. 1184 gp := getg() 1185 gp.m.mallocing++ 1186 lock(&h.lock) 1187 var sumreleased uintptr 1188 for i := 0; i < len(h.free); i++ { 1189 sumreleased += scavengelist(&h.free[i], now, limit) 1190 } 1191 sumreleased += scavengetreap(h.freelarge.treap, now, limit) 1192 unlock(&h.lock) 1193 gp.m.mallocing-- 1194 1195 if debug.gctrace > 0 { 1196 if sumreleased > 0 { 1197 print("scvg", k, ": ", sumreleased>>20, " MB released\n") 1198 } 1199 print("scvg", k, ": inuse: ", memstats.heap_inuse>>20, ", idle: ", memstats.heap_idle>>20, ", sys: ", memstats.heap_sys>>20, ", released: ", memstats.heap_released>>20, ", consumed: ", (memstats.heap_sys-memstats.heap_released)>>20, " (MB)\n") 1200 } 1201 } 1202 1203 //go:linkname runtime_debug_freeOSMemory runtime/debug.freeOSMemory 1204 func runtime_debug_freeOSMemory() { 1205 GC() 1206 systemstack(func() { mheap_.scavenge(-1, ^uint64(0), 0) }) 1207 } 1208 1209 // Initialize a new span with the given start and npages. 1210 func (span *mspan) init(base uintptr, npages uintptr) { 1211 // span is *not* zeroed. 1212 span.next = nil 1213 span.prev = nil 1214 span.list = nil 1215 span.startAddr = base 1216 span.npages = npages 1217 span.allocCount = 0 1218 span.spanclass = 0 1219 span.incache = false 1220 span.elemsize = 0 1221 span.state = _MSpanDead 1222 span.unusedsince = 0 1223 span.npreleased = 0 1224 span.speciallock.key = 0 1225 span.specials = nil 1226 span.needzero = 0 1227 span.freeindex = 0 1228 span.allocBits = nil 1229 span.gcmarkBits = nil 1230 } 1231 1232 func (span *mspan) inList() bool { 1233 return span.list != nil 1234 } 1235 1236 // Initialize an empty doubly-linked list. 1237 func (list *mSpanList) init() { 1238 list.first = nil 1239 list.last = nil 1240 } 1241 1242 func (list *mSpanList) remove(span *mspan) { 1243 if span.list != list { 1244 print("runtime: failed MSpanList_Remove span.npages=", span.npages, 1245 " span=", span, " prev=", span.prev, " span.list=", span.list, " list=", list, "\n") 1246 throw("MSpanList_Remove") 1247 } 1248 if list.first == span { 1249 list.first = span.next 1250 } else { 1251 span.prev.next = span.next 1252 } 1253 if list.last == span { 1254 list.last = span.prev 1255 } else { 1256 span.next.prev = span.prev 1257 } 1258 span.next = nil 1259 span.prev = nil 1260 span.list = nil 1261 } 1262 1263 func (list *mSpanList) isEmpty() bool { 1264 return list.first == nil 1265 } 1266 1267 func (list *mSpanList) insert(span *mspan) { 1268 if span.next != nil || span.prev != nil || span.list != nil { 1269 println("runtime: failed MSpanList_Insert", span, span.next, span.prev, span.list) 1270 throw("MSpanList_Insert") 1271 } 1272 span.next = list.first 1273 if list.first != nil { 1274 // The list contains at least one span; link it in. 1275 // The last span in the list doesn't change. 1276 list.first.prev = span 1277 } else { 1278 // The list contains no spans, so this is also the last span. 1279 list.last = span 1280 } 1281 list.first = span 1282 span.list = list 1283 } 1284 1285 func (list *mSpanList) insertBack(span *mspan) { 1286 if span.next != nil || span.prev != nil || span.list != nil { 1287 println("runtime: failed MSpanList_InsertBack", span, span.next, span.prev, span.list) 1288 throw("MSpanList_InsertBack") 1289 } 1290 span.prev = list.last 1291 if list.last != nil { 1292 // The list contains at least one span. 1293 list.last.next = span 1294 } else { 1295 // The list contains no spans, so this is also the first span. 1296 list.first = span 1297 } 1298 list.last = span 1299 span.list = list 1300 } 1301 1302 // takeAll removes all spans from other and inserts them at the front 1303 // of list. 1304 func (list *mSpanList) takeAll(other *mSpanList) { 1305 if other.isEmpty() { 1306 return 1307 } 1308 1309 // Reparent everything in other to list. 1310 for s := other.first; s != nil; s = s.next { 1311 s.list = list 1312 } 1313 1314 // Concatenate the lists. 1315 if list.isEmpty() { 1316 *list = *other 1317 } else { 1318 // Neither list is empty. Put other before list. 1319 other.last.next = list.first 1320 list.first.prev = other.last 1321 list.first = other.first 1322 } 1323 1324 other.first, other.last = nil, nil 1325 } 1326 1327 const ( 1328 _KindSpecialFinalizer = 1 1329 _KindSpecialProfile = 2 1330 // Note: The finalizer special must be first because if we're freeing 1331 // an object, a finalizer special will cause the freeing operation 1332 // to abort, and we want to keep the other special records around 1333 // if that happens. 1334 ) 1335 1336 //go:notinheap 1337 type special struct { 1338 next *special // linked list in span 1339 offset uint16 // span offset of object 1340 kind byte // kind of special 1341 } 1342 1343 // Adds the special record s to the list of special records for 1344 // the object p. All fields of s should be filled in except for 1345 // offset & next, which this routine will fill in. 1346 // Returns true if the special was successfully added, false otherwise. 1347 // (The add will fail only if a record with the same p and s->kind 1348 // already exists.) 1349 func addspecial(p unsafe.Pointer, s *special) bool { 1350 span := mheap_.lookupMaybe(p) 1351 if span == nil { 1352 throw("addspecial on invalid pointer") 1353 } 1354 1355 // Ensure that the span is swept. 1356 // Sweeping accesses the specials list w/o locks, so we have 1357 // to synchronize with it. And it's just much safer. 1358 mp := acquirem() 1359 span.ensureSwept() 1360 1361 offset := uintptr(p) - span.base() 1362 kind := s.kind 1363 1364 lock(&span.speciallock) 1365 1366 // Find splice point, check for existing record. 1367 t := &span.specials 1368 for { 1369 x := *t 1370 if x == nil { 1371 break 1372 } 1373 if offset == uintptr(x.offset) && kind == x.kind { 1374 unlock(&span.speciallock) 1375 releasem(mp) 1376 return false // already exists 1377 } 1378 if offset < uintptr(x.offset) || (offset == uintptr(x.offset) && kind < x.kind) { 1379 break 1380 } 1381 t = &x.next 1382 } 1383 1384 // Splice in record, fill in offset. 1385 s.offset = uint16(offset) 1386 s.next = *t 1387 *t = s 1388 unlock(&span.speciallock) 1389 releasem(mp) 1390 1391 return true 1392 } 1393 1394 // Removes the Special record of the given kind for the object p. 1395 // Returns the record if the record existed, nil otherwise. 1396 // The caller must FixAlloc_Free the result. 1397 func removespecial(p unsafe.Pointer, kind uint8) *special { 1398 span := mheap_.lookupMaybe(p) 1399 if span == nil { 1400 throw("removespecial on invalid pointer") 1401 } 1402 1403 // Ensure that the span is swept. 1404 // Sweeping accesses the specials list w/o locks, so we have 1405 // to synchronize with it. And it's just much safer. 1406 mp := acquirem() 1407 span.ensureSwept() 1408 1409 offset := uintptr(p) - span.base() 1410 1411 lock(&span.speciallock) 1412 t := &span.specials 1413 for { 1414 s := *t 1415 if s == nil { 1416 break 1417 } 1418 // This function is used for finalizers only, so we don't check for 1419 // "interior" specials (p must be exactly equal to s->offset). 1420 if offset == uintptr(s.offset) && kind == s.kind { 1421 *t = s.next 1422 unlock(&span.speciallock) 1423 releasem(mp) 1424 return s 1425 } 1426 t = &s.next 1427 } 1428 unlock(&span.speciallock) 1429 releasem(mp) 1430 return nil 1431 } 1432 1433 // The described object has a finalizer set for it. 1434 // 1435 // specialfinalizer is allocated from non-GC'd memory, so any heap 1436 // pointers must be specially handled. 1437 // 1438 //go:notinheap 1439 type specialfinalizer struct { 1440 special special 1441 fn *funcval // May be a heap pointer. 1442 nret uintptr 1443 fint *_type // May be a heap pointer, but always live. 1444 ot *ptrtype // May be a heap pointer, but always live. 1445 } 1446 1447 // Adds a finalizer to the object p. Returns true if it succeeded. 1448 func addfinalizer(p unsafe.Pointer, f *funcval, nret uintptr, fint *_type, ot *ptrtype) bool { 1449 lock(&mheap_.speciallock) 1450 s := (*specialfinalizer)(mheap_.specialfinalizeralloc.alloc()) 1451 unlock(&mheap_.speciallock) 1452 s.special.kind = _KindSpecialFinalizer 1453 s.fn = f 1454 s.nret = nret 1455 s.fint = fint 1456 s.ot = ot 1457 if addspecial(p, &s.special) { 1458 // This is responsible for maintaining the same 1459 // GC-related invariants as markrootSpans in any 1460 // situation where it's possible that markrootSpans 1461 // has already run but mark termination hasn't yet. 1462 if gcphase != _GCoff { 1463 _, base, _ := findObject(p) 1464 mp := acquirem() 1465 gcw := &mp.p.ptr().gcw 1466 // Mark everything reachable from the object 1467 // so it's retained for the finalizer. 1468 scanobject(uintptr(base), gcw) 1469 // Mark the finalizer itself, since the 1470 // special isn't part of the GC'd heap. 1471 scanblock(uintptr(unsafe.Pointer(&s.fn)), sys.PtrSize, &oneptrmask[0], gcw) 1472 if gcBlackenPromptly { 1473 gcw.dispose() 1474 } 1475 releasem(mp) 1476 } 1477 return true 1478 } 1479 1480 // There was an old finalizer 1481 lock(&mheap_.speciallock) 1482 mheap_.specialfinalizeralloc.free(unsafe.Pointer(s)) 1483 unlock(&mheap_.speciallock) 1484 return false 1485 } 1486 1487 // Removes the finalizer (if any) from the object p. 1488 func removefinalizer(p unsafe.Pointer) { 1489 s := (*specialfinalizer)(unsafe.Pointer(removespecial(p, _KindSpecialFinalizer))) 1490 if s == nil { 1491 return // there wasn't a finalizer to remove 1492 } 1493 lock(&mheap_.speciallock) 1494 mheap_.specialfinalizeralloc.free(unsafe.Pointer(s)) 1495 unlock(&mheap_.speciallock) 1496 } 1497 1498 // The described object is being heap profiled. 1499 // 1500 //go:notinheap 1501 type specialprofile struct { 1502 special special 1503 b *bucket 1504 } 1505 1506 // Set the heap profile bucket associated with addr to b. 1507 func setprofilebucket(p unsafe.Pointer, b *bucket) { 1508 lock(&mheap_.speciallock) 1509 s := (*specialprofile)(mheap_.specialprofilealloc.alloc()) 1510 unlock(&mheap_.speciallock) 1511 s.special.kind = _KindSpecialProfile 1512 s.b = b 1513 if !addspecial(p, &s.special) { 1514 throw("setprofilebucket: profile already set") 1515 } 1516 } 1517 1518 // Do whatever cleanup needs to be done to deallocate s. It has 1519 // already been unlinked from the MSpan specials list. 1520 func freespecial(s *special, p unsafe.Pointer, size uintptr) { 1521 switch s.kind { 1522 case _KindSpecialFinalizer: 1523 sf := (*specialfinalizer)(unsafe.Pointer(s)) 1524 queuefinalizer(p, sf.fn, sf.nret, sf.fint, sf.ot) 1525 lock(&mheap_.speciallock) 1526 mheap_.specialfinalizeralloc.free(unsafe.Pointer(sf)) 1527 unlock(&mheap_.speciallock) 1528 case _KindSpecialProfile: 1529 sp := (*specialprofile)(unsafe.Pointer(s)) 1530 mProf_Free(sp.b, size) 1531 lock(&mheap_.speciallock) 1532 mheap_.specialprofilealloc.free(unsafe.Pointer(sp)) 1533 unlock(&mheap_.speciallock) 1534 default: 1535 throw("bad special kind") 1536 panic("not reached") 1537 } 1538 } 1539 1540 // gcBits is an alloc/mark bitmap. This is always used as *gcBits. 1541 // 1542 //go:notinheap 1543 type gcBits uint8 1544 1545 // bytep returns a pointer to the n'th byte of b. 1546 func (b *gcBits) bytep(n uintptr) *uint8 { 1547 return addb((*uint8)(b), n) 1548 } 1549 1550 // bitp returns a pointer to the byte containing bit n and a mask for 1551 // selecting that bit from *bytep. 1552 func (b *gcBits) bitp(n uintptr) (bytep *uint8, mask uint8) { 1553 return b.bytep(n / 8), 1 << (n % 8) 1554 } 1555 1556 const gcBitsChunkBytes = uintptr(64 << 10) 1557 const gcBitsHeaderBytes = unsafe.Sizeof(gcBitsHeader{}) 1558 1559 type gcBitsHeader struct { 1560 free uintptr // free is the index into bits of the next free byte. 1561 next uintptr // *gcBits triggers recursive type bug. (issue 14620) 1562 } 1563 1564 //go:notinheap 1565 type gcBitsArena struct { 1566 // gcBitsHeader // side step recursive type bug (issue 14620) by including fields by hand. 1567 free uintptr // free is the index into bits of the next free byte; read/write atomically 1568 next *gcBitsArena 1569 bits [gcBitsChunkBytes - gcBitsHeaderBytes]gcBits 1570 } 1571 1572 var gcBitsArenas struct { 1573 lock mutex 1574 free *gcBitsArena 1575 next *gcBitsArena // Read atomically. Write atomically under lock. 1576 current *gcBitsArena 1577 previous *gcBitsArena 1578 } 1579 1580 // tryAlloc allocates from b or returns nil if b does not have enough room. 1581 // This is safe to call concurrently. 1582 func (b *gcBitsArena) tryAlloc(bytes uintptr) *gcBits { 1583 if b == nil || atomic.Loaduintptr(&b.free)+bytes > uintptr(len(b.bits)) { 1584 return nil 1585 } 1586 // Try to allocate from this block. 1587 end := atomic.Xadduintptr(&b.free, bytes) 1588 if end > uintptr(len(b.bits)) { 1589 return nil 1590 } 1591 // There was enough room. 1592 start := end - bytes 1593 return &b.bits[start] 1594 } 1595 1596 // newMarkBits returns a pointer to 8 byte aligned bytes 1597 // to be used for a span's mark bits. 1598 func newMarkBits(nelems uintptr) *gcBits { 1599 blocksNeeded := uintptr((nelems + 63) / 64) 1600 bytesNeeded := blocksNeeded * 8 1601 1602 // Try directly allocating from the current head arena. 1603 head := (*gcBitsArena)(atomic.Loadp(unsafe.Pointer(&gcBitsArenas.next))) 1604 if p := head.tryAlloc(bytesNeeded); p != nil { 1605 return p 1606 } 1607 1608 // There's not enough room in the head arena. We may need to 1609 // allocate a new arena. 1610 lock(&gcBitsArenas.lock) 1611 // Try the head arena again, since it may have changed. Now 1612 // that we hold the lock, the list head can't change, but its 1613 // free position still can. 1614 if p := gcBitsArenas.next.tryAlloc(bytesNeeded); p != nil { 1615 unlock(&gcBitsArenas.lock) 1616 return p 1617 } 1618 1619 // Allocate a new arena. This may temporarily drop the lock. 1620 fresh := newArenaMayUnlock() 1621 // If newArenaMayUnlock dropped the lock, another thread may 1622 // have put a fresh arena on the "next" list. Try allocating 1623 // from next again. 1624 if p := gcBitsArenas.next.tryAlloc(bytesNeeded); p != nil { 1625 // Put fresh back on the free list. 1626 // TODO: Mark it "already zeroed" 1627 fresh.next = gcBitsArenas.free 1628 gcBitsArenas.free = fresh 1629 unlock(&gcBitsArenas.lock) 1630 return p 1631 } 1632 1633 // Allocate from the fresh arena. We haven't linked it in yet, so 1634 // this cannot race and is guaranteed to succeed. 1635 p := fresh.tryAlloc(bytesNeeded) 1636 if p == nil { 1637 throw("markBits overflow") 1638 } 1639 1640 // Add the fresh arena to the "next" list. 1641 fresh.next = gcBitsArenas.next 1642 atomic.StorepNoWB(unsafe.Pointer(&gcBitsArenas.next), unsafe.Pointer(fresh)) 1643 1644 unlock(&gcBitsArenas.lock) 1645 return p 1646 } 1647 1648 // newAllocBits returns a pointer to 8 byte aligned bytes 1649 // to be used for this span's alloc bits. 1650 // newAllocBits is used to provide newly initialized spans 1651 // allocation bits. For spans not being initialized the 1652 // the mark bits are repurposed as allocation bits when 1653 // the span is swept. 1654 func newAllocBits(nelems uintptr) *gcBits { 1655 return newMarkBits(nelems) 1656 } 1657 1658 // nextMarkBitArenaEpoch establishes a new epoch for the arenas 1659 // holding the mark bits. The arenas are named relative to the 1660 // current GC cycle which is demarcated by the call to finishweep_m. 1661 // 1662 // All current spans have been swept. 1663 // During that sweep each span allocated room for its gcmarkBits in 1664 // gcBitsArenas.next block. gcBitsArenas.next becomes the gcBitsArenas.current 1665 // where the GC will mark objects and after each span is swept these bits 1666 // will be used to allocate objects. 1667 // gcBitsArenas.current becomes gcBitsArenas.previous where the span's 1668 // gcAllocBits live until all the spans have been swept during this GC cycle. 1669 // The span's sweep extinguishes all the references to gcBitsArenas.previous 1670 // by pointing gcAllocBits into the gcBitsArenas.current. 1671 // The gcBitsArenas.previous is released to the gcBitsArenas.free list. 1672 func nextMarkBitArenaEpoch() { 1673 lock(&gcBitsArenas.lock) 1674 if gcBitsArenas.previous != nil { 1675 if gcBitsArenas.free == nil { 1676 gcBitsArenas.free = gcBitsArenas.previous 1677 } else { 1678 // Find end of previous arenas. 1679 last := gcBitsArenas.previous 1680 for last = gcBitsArenas.previous; last.next != nil; last = last.next { 1681 } 1682 last.next = gcBitsArenas.free 1683 gcBitsArenas.free = gcBitsArenas.previous 1684 } 1685 } 1686 gcBitsArenas.previous = gcBitsArenas.current 1687 gcBitsArenas.current = gcBitsArenas.next 1688 atomic.StorepNoWB(unsafe.Pointer(&gcBitsArenas.next), nil) // newMarkBits calls newArena when needed 1689 unlock(&gcBitsArenas.lock) 1690 } 1691 1692 // newArenaMayUnlock allocates and zeroes a gcBits arena. 1693 // The caller must hold gcBitsArena.lock. This may temporarily release it. 1694 func newArenaMayUnlock() *gcBitsArena { 1695 var result *gcBitsArena 1696 if gcBitsArenas.free == nil { 1697 unlock(&gcBitsArenas.lock) 1698 result = (*gcBitsArena)(sysAlloc(gcBitsChunkBytes, &memstats.gc_sys)) 1699 if result == nil { 1700 throw("runtime: cannot allocate memory") 1701 } 1702 lock(&gcBitsArenas.lock) 1703 } else { 1704 result = gcBitsArenas.free 1705 gcBitsArenas.free = gcBitsArenas.free.next 1706 memclrNoHeapPointers(unsafe.Pointer(result), gcBitsChunkBytes) 1707 } 1708 result.next = nil 1709 // If result.bits is not 8 byte aligned adjust index so 1710 // that &result.bits[result.free] is 8 byte aligned. 1711 if uintptr(unsafe.Offsetof(gcBitsArena{}.bits))&7 == 0 { 1712 result.free = 0 1713 } else { 1714 result.free = 8 - (uintptr(unsafe.Pointer(&result.bits[0])) & 7) 1715 } 1716 return result 1717 }