github.com/comwrg/go/src@v0.0.0-20220319063731-c238d0440370/runtime/export_test.go (about) 1 // Copyright 2010 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 // Export guts for testing. 6 7 package runtime 8 9 import ( 10 "runtime/internal/atomic" 11 "runtime/internal/sys" 12 "unsafe" 13 ) 14 15 var Fadd64 = fadd64 16 var Fsub64 = fsub64 17 var Fmul64 = fmul64 18 var Fdiv64 = fdiv64 19 var F64to32 = f64to32 20 var F32to64 = f32to64 21 var Fcmp64 = fcmp64 22 var Fintto64 = fintto64 23 var F64toint = f64toint 24 25 var Entersyscall = entersyscall 26 var Exitsyscall = exitsyscall 27 var LockedOSThread = lockedOSThread 28 var Xadduintptr = atomic.Xadduintptr 29 30 var FuncPC = funcPC 31 32 var Fastlog2 = fastlog2 33 34 var Atoi = atoi 35 var Atoi32 = atoi32 36 37 var Nanotime = nanotime 38 var NetpollBreak = netpollBreak 39 var Usleep = usleep 40 41 var PhysPageSize = physPageSize 42 var PhysHugePageSize = physHugePageSize 43 44 var NetpollGenericInit = netpollGenericInit 45 46 var Memmove = memmove 47 var MemclrNoHeapPointers = memclrNoHeapPointers 48 49 var LockPartialOrder = lockPartialOrder 50 51 type LockRank lockRank 52 53 func (l LockRank) String() string { 54 return lockRank(l).String() 55 } 56 57 const PreemptMSupported = preemptMSupported 58 59 type LFNode struct { 60 Next uint64 61 Pushcnt uintptr 62 } 63 64 func LFStackPush(head *uint64, node *LFNode) { 65 (*lfstack)(head).push((*lfnode)(unsafe.Pointer(node))) 66 } 67 68 func LFStackPop(head *uint64) *LFNode { 69 return (*LFNode)(unsafe.Pointer((*lfstack)(head).pop())) 70 } 71 72 func Netpoll(delta int64) { 73 systemstack(func() { 74 netpoll(delta) 75 }) 76 } 77 78 func GCMask(x interface{}) (ret []byte) { 79 systemstack(func() { 80 ret = getgcmask(x) 81 }) 82 return 83 } 84 85 func RunSchedLocalQueueTest() { 86 _p_ := new(p) 87 gs := make([]g, len(_p_.runq)) 88 for i := 0; i < len(_p_.runq); i++ { 89 if g, _ := runqget(_p_); g != nil { 90 throw("runq is not empty initially") 91 } 92 for j := 0; j < i; j++ { 93 runqput(_p_, &gs[i], false) 94 } 95 for j := 0; j < i; j++ { 96 if g, _ := runqget(_p_); g != &gs[i] { 97 print("bad element at iter ", i, "/", j, "\n") 98 throw("bad element") 99 } 100 } 101 if g, _ := runqget(_p_); g != nil { 102 throw("runq is not empty afterwards") 103 } 104 } 105 } 106 107 func RunSchedLocalQueueStealTest() { 108 p1 := new(p) 109 p2 := new(p) 110 gs := make([]g, len(p1.runq)) 111 for i := 0; i < len(p1.runq); i++ { 112 for j := 0; j < i; j++ { 113 gs[j].sig = 0 114 runqput(p1, &gs[j], false) 115 } 116 gp := runqsteal(p2, p1, true) 117 s := 0 118 if gp != nil { 119 s++ 120 gp.sig++ 121 } 122 for { 123 gp, _ = runqget(p2) 124 if gp == nil { 125 break 126 } 127 s++ 128 gp.sig++ 129 } 130 for { 131 gp, _ = runqget(p1) 132 if gp == nil { 133 break 134 } 135 gp.sig++ 136 } 137 for j := 0; j < i; j++ { 138 if gs[j].sig != 1 { 139 print("bad element ", j, "(", gs[j].sig, ") at iter ", i, "\n") 140 throw("bad element") 141 } 142 } 143 if s != i/2 && s != i/2+1 { 144 print("bad steal ", s, ", want ", i/2, " or ", i/2+1, ", iter ", i, "\n") 145 throw("bad steal") 146 } 147 } 148 } 149 150 // Temporary to enable register ABI bringup. 151 // TODO(register args): convert back to local variables in RunSchedLocalQueueEmptyTest that 152 // get passed to the "go" stmts there. 153 var RunSchedLocalQueueEmptyState struct { 154 done chan bool 155 ready *uint32 156 p *p 157 } 158 159 func RunSchedLocalQueueEmptyTest(iters int) { 160 // Test that runq is not spuriously reported as empty. 161 // Runq emptiness affects scheduling decisions and spurious emptiness 162 // can lead to underutilization (both runnable Gs and idle Ps coexist 163 // for arbitrary long time). 164 done := make(chan bool, 1) 165 RunSchedLocalQueueEmptyState.done = done 166 p := new(p) 167 RunSchedLocalQueueEmptyState.p = p 168 gs := make([]g, 2) 169 ready := new(uint32) 170 RunSchedLocalQueueEmptyState.ready = ready 171 for i := 0; i < iters; i++ { 172 *ready = 0 173 next0 := (i & 1) == 0 174 next1 := (i & 2) == 0 175 runqput(p, &gs[0], next0) 176 go func() { 177 for atomic.Xadd(RunSchedLocalQueueEmptyState.ready, 1); atomic.Load(RunSchedLocalQueueEmptyState.ready) != 2; { 178 } 179 if runqempty(RunSchedLocalQueueEmptyState.p) { 180 //println("next:", next0, next1) 181 throw("queue is empty") 182 } 183 RunSchedLocalQueueEmptyState.done <- true 184 }() 185 for atomic.Xadd(ready, 1); atomic.Load(ready) != 2; { 186 } 187 runqput(p, &gs[1], next1) 188 runqget(p) 189 <-done 190 runqget(p) 191 } 192 } 193 194 var ( 195 StringHash = stringHash 196 BytesHash = bytesHash 197 Int32Hash = int32Hash 198 Int64Hash = int64Hash 199 MemHash = memhash 200 MemHash32 = memhash32 201 MemHash64 = memhash64 202 EfaceHash = efaceHash 203 IfaceHash = ifaceHash 204 ) 205 206 var UseAeshash = &useAeshash 207 208 func MemclrBytes(b []byte) { 209 s := (*slice)(unsafe.Pointer(&b)) 210 memclrNoHeapPointers(s.array, uintptr(s.len)) 211 } 212 213 var HashLoad = &hashLoad 214 215 // entry point for testing 216 func GostringW(w []uint16) (s string) { 217 systemstack(func() { 218 s = gostringw(&w[0]) 219 }) 220 return 221 } 222 223 var Open = open 224 var Close = closefd 225 var Read = read 226 var Write = write 227 228 func Envs() []string { return envs } 229 func SetEnvs(e []string) { envs = e } 230 231 var BigEndian = sys.BigEndian 232 233 // For benchmarking. 234 235 func BenchSetType(n int, x interface{}) { 236 e := *efaceOf(&x) 237 t := e._type 238 var size uintptr 239 var p unsafe.Pointer 240 switch t.kind & kindMask { 241 case kindPtr: 242 t = (*ptrtype)(unsafe.Pointer(t)).elem 243 size = t.size 244 p = e.data 245 case kindSlice: 246 slice := *(*struct { 247 ptr unsafe.Pointer 248 len, cap uintptr 249 })(e.data) 250 t = (*slicetype)(unsafe.Pointer(t)).elem 251 size = t.size * slice.len 252 p = slice.ptr 253 } 254 allocSize := roundupsize(size) 255 systemstack(func() { 256 for i := 0; i < n; i++ { 257 heapBitsSetType(uintptr(p), allocSize, size, t) 258 } 259 }) 260 } 261 262 const PtrSize = sys.PtrSize 263 264 var ForceGCPeriod = &forcegcperiod 265 266 // SetTracebackEnv is like runtime/debug.SetTraceback, but it raises 267 // the "environment" traceback level, so later calls to 268 // debug.SetTraceback (e.g., from testing timeouts) can't lower it. 269 func SetTracebackEnv(level string) { 270 setTraceback(level) 271 traceback_env = traceback_cache 272 } 273 274 var ReadUnaligned32 = readUnaligned32 275 var ReadUnaligned64 = readUnaligned64 276 277 func CountPagesInUse() (pagesInUse, counted uintptr) { 278 stopTheWorld("CountPagesInUse") 279 280 pagesInUse = uintptr(mheap_.pagesInUse) 281 282 for _, s := range mheap_.allspans { 283 if s.state.get() == mSpanInUse { 284 counted += s.npages 285 } 286 } 287 288 startTheWorld() 289 290 return 291 } 292 293 func Fastrand() uint32 { return fastrand() } 294 func Fastrandn(n uint32) uint32 { return fastrandn(n) } 295 296 type ProfBuf profBuf 297 298 func NewProfBuf(hdrsize, bufwords, tags int) *ProfBuf { 299 return (*ProfBuf)(newProfBuf(hdrsize, bufwords, tags)) 300 } 301 302 func (p *ProfBuf) Write(tag *unsafe.Pointer, now int64, hdr []uint64, stk []uintptr) { 303 (*profBuf)(p).write(tag, now, hdr, stk) 304 } 305 306 const ( 307 ProfBufBlocking = profBufBlocking 308 ProfBufNonBlocking = profBufNonBlocking 309 ) 310 311 func (p *ProfBuf) Read(mode profBufReadMode) ([]uint64, []unsafe.Pointer, bool) { 312 return (*profBuf)(p).read(profBufReadMode(mode)) 313 } 314 315 func (p *ProfBuf) Close() { 316 (*profBuf)(p).close() 317 } 318 319 func ReadMetricsSlow(memStats *MemStats, samplesp unsafe.Pointer, len, cap int) { 320 stopTheWorld("ReadMetricsSlow") 321 322 // Initialize the metrics beforehand because this could 323 // allocate and skew the stats. 324 semacquire(&metricsSema) 325 initMetrics() 326 semrelease(&metricsSema) 327 328 systemstack(func() { 329 // Read memstats first. It's going to flush 330 // the mcaches which readMetrics does not do, so 331 // going the other way around may result in 332 // inconsistent statistics. 333 readmemstats_m(memStats) 334 }) 335 336 // Read metrics off the system stack. 337 // 338 // The only part of readMetrics that could allocate 339 // and skew the stats is initMetrics. 340 readMetrics(samplesp, len, cap) 341 342 startTheWorld() 343 } 344 345 // ReadMemStatsSlow returns both the runtime-computed MemStats and 346 // MemStats accumulated by scanning the heap. 347 func ReadMemStatsSlow() (base, slow MemStats) { 348 stopTheWorld("ReadMemStatsSlow") 349 350 // Run on the system stack to avoid stack growth allocation. 351 systemstack(func() { 352 // Make sure stats don't change. 353 getg().m.mallocing++ 354 355 readmemstats_m(&base) 356 357 // Initialize slow from base and zero the fields we're 358 // recomputing. 359 slow = base 360 slow.Alloc = 0 361 slow.TotalAlloc = 0 362 slow.Mallocs = 0 363 slow.Frees = 0 364 slow.HeapReleased = 0 365 var bySize [_NumSizeClasses]struct { 366 Mallocs, Frees uint64 367 } 368 369 // Add up current allocations in spans. 370 for _, s := range mheap_.allspans { 371 if s.state.get() != mSpanInUse { 372 continue 373 } 374 if sizeclass := s.spanclass.sizeclass(); sizeclass == 0 { 375 slow.Mallocs++ 376 slow.Alloc += uint64(s.elemsize) 377 } else { 378 slow.Mallocs += uint64(s.allocCount) 379 slow.Alloc += uint64(s.allocCount) * uint64(s.elemsize) 380 bySize[sizeclass].Mallocs += uint64(s.allocCount) 381 } 382 } 383 384 // Add in frees by just reading the stats for those directly. 385 var m heapStatsDelta 386 memstats.heapStats.unsafeRead(&m) 387 388 // Collect per-sizeclass free stats. 389 var smallFree uint64 390 for i := 0; i < _NumSizeClasses; i++ { 391 slow.Frees += uint64(m.smallFreeCount[i]) 392 bySize[i].Frees += uint64(m.smallFreeCount[i]) 393 bySize[i].Mallocs += uint64(m.smallFreeCount[i]) 394 smallFree += uint64(m.smallFreeCount[i]) * uint64(class_to_size[i]) 395 } 396 slow.Frees += uint64(m.tinyAllocCount) + uint64(m.largeFreeCount) 397 slow.Mallocs += slow.Frees 398 399 slow.TotalAlloc = slow.Alloc + uint64(m.largeFree) + smallFree 400 401 for i := range slow.BySize { 402 slow.BySize[i].Mallocs = bySize[i].Mallocs 403 slow.BySize[i].Frees = bySize[i].Frees 404 } 405 406 for i := mheap_.pages.start; i < mheap_.pages.end; i++ { 407 chunk := mheap_.pages.tryChunkOf(i) 408 if chunk == nil { 409 continue 410 } 411 pg := chunk.scavenged.popcntRange(0, pallocChunkPages) 412 slow.HeapReleased += uint64(pg) * pageSize 413 } 414 for _, p := range allp { 415 pg := sys.OnesCount64(p.pcache.scav) 416 slow.HeapReleased += uint64(pg) * pageSize 417 } 418 419 getg().m.mallocing-- 420 }) 421 422 startTheWorld() 423 return 424 } 425 426 // BlockOnSystemStack switches to the system stack, prints "x\n" to 427 // stderr, and blocks in a stack containing 428 // "runtime.blockOnSystemStackInternal". 429 func BlockOnSystemStack() { 430 systemstack(blockOnSystemStackInternal) 431 } 432 433 func blockOnSystemStackInternal() { 434 print("x\n") 435 lock(&deadlock) 436 lock(&deadlock) 437 } 438 439 type RWMutex struct { 440 rw rwmutex 441 } 442 443 func (rw *RWMutex) RLock() { 444 rw.rw.rlock() 445 } 446 447 func (rw *RWMutex) RUnlock() { 448 rw.rw.runlock() 449 } 450 451 func (rw *RWMutex) Lock() { 452 rw.rw.lock() 453 } 454 455 func (rw *RWMutex) Unlock() { 456 rw.rw.unlock() 457 } 458 459 const RuntimeHmapSize = unsafe.Sizeof(hmap{}) 460 461 func MapBucketsCount(m map[int]int) int { 462 h := *(**hmap)(unsafe.Pointer(&m)) 463 return 1 << h.B 464 } 465 466 func MapBucketsPointerIsNil(m map[int]int) bool { 467 h := *(**hmap)(unsafe.Pointer(&m)) 468 return h.buckets == nil 469 } 470 471 func LockOSCounts() (external, internal uint32) { 472 g := getg() 473 if g.m.lockedExt+g.m.lockedInt == 0 { 474 if g.lockedm != 0 { 475 panic("lockedm on non-locked goroutine") 476 } 477 } else { 478 if g.lockedm == 0 { 479 panic("nil lockedm on locked goroutine") 480 } 481 } 482 return g.m.lockedExt, g.m.lockedInt 483 } 484 485 //go:noinline 486 func TracebackSystemstack(stk []uintptr, i int) int { 487 if i == 0 { 488 pc, sp := getcallerpc(), getcallersp() 489 return gentraceback(pc, sp, 0, getg(), 0, &stk[0], len(stk), nil, nil, _TraceJumpStack) 490 } 491 n := 0 492 systemstack(func() { 493 n = TracebackSystemstack(stk, i-1) 494 }) 495 return n 496 } 497 498 func KeepNArenaHints(n int) { 499 hint := mheap_.arenaHints 500 for i := 1; i < n; i++ { 501 hint = hint.next 502 if hint == nil { 503 return 504 } 505 } 506 hint.next = nil 507 } 508 509 // MapNextArenaHint reserves a page at the next arena growth hint, 510 // preventing the arena from growing there, and returns the range of 511 // addresses that are no longer viable. 512 func MapNextArenaHint() (start, end uintptr) { 513 hint := mheap_.arenaHints 514 addr := hint.addr 515 if hint.down { 516 start, end = addr-heapArenaBytes, addr 517 addr -= physPageSize 518 } else { 519 start, end = addr, addr+heapArenaBytes 520 } 521 sysReserve(unsafe.Pointer(addr), physPageSize) 522 return 523 } 524 525 func GetNextArenaHint() uintptr { 526 return mheap_.arenaHints.addr 527 } 528 529 type G = g 530 531 type Sudog = sudog 532 533 func Getg() *G { 534 return getg() 535 } 536 537 //go:noinline 538 func PanicForTesting(b []byte, i int) byte { 539 return unexportedPanicForTesting(b, i) 540 } 541 542 //go:noinline 543 func unexportedPanicForTesting(b []byte, i int) byte { 544 return b[i] 545 } 546 547 func G0StackOverflow() { 548 systemstack(func() { 549 stackOverflow(nil) 550 }) 551 } 552 553 func stackOverflow(x *byte) { 554 var buf [256]byte 555 stackOverflow(&buf[0]) 556 } 557 558 func MapTombstoneCheck(m map[int]int) { 559 // Make sure emptyOne and emptyRest are distributed correctly. 560 // We should have a series of filled and emptyOne cells, followed by 561 // a series of emptyRest cells. 562 h := *(**hmap)(unsafe.Pointer(&m)) 563 i := interface{}(m) 564 t := *(**maptype)(unsafe.Pointer(&i)) 565 566 for x := 0; x < 1<<h.B; x++ { 567 b0 := (*bmap)(add(h.buckets, uintptr(x)*uintptr(t.bucketsize))) 568 n := 0 569 for b := b0; b != nil; b = b.overflow(t) { 570 for i := 0; i < bucketCnt; i++ { 571 if b.tophash[i] != emptyRest { 572 n++ 573 } 574 } 575 } 576 k := 0 577 for b := b0; b != nil; b = b.overflow(t) { 578 for i := 0; i < bucketCnt; i++ { 579 if k < n && b.tophash[i] == emptyRest { 580 panic("early emptyRest") 581 } 582 if k >= n && b.tophash[i] != emptyRest { 583 panic("late non-emptyRest") 584 } 585 if k == n-1 && b.tophash[i] == emptyOne { 586 panic("last non-emptyRest entry is emptyOne") 587 } 588 k++ 589 } 590 } 591 } 592 } 593 594 func RunGetgThreadSwitchTest() { 595 // Test that getg works correctly with thread switch. 596 // With gccgo, if we generate getg inlined, the backend 597 // may cache the address of the TLS variable, which 598 // will become invalid after a thread switch. This test 599 // checks that the bad caching doesn't happen. 600 601 ch := make(chan int) 602 go func(ch chan int) { 603 ch <- 5 604 LockOSThread() 605 }(ch) 606 607 g1 := getg() 608 609 // Block on a receive. This is likely to get us a thread 610 // switch. If we yield to the sender goroutine, it will 611 // lock the thread, forcing us to resume on a different 612 // thread. 613 <-ch 614 615 g2 := getg() 616 if g1 != g2 { 617 panic("g1 != g2") 618 } 619 620 // Also test getg after some control flow, as the 621 // backend is sensitive to control flow. 622 g3 := getg() 623 if g1 != g3 { 624 panic("g1 != g3") 625 } 626 } 627 628 const ( 629 PageSize = pageSize 630 PallocChunkPages = pallocChunkPages 631 PageAlloc64Bit = pageAlloc64Bit 632 PallocSumBytes = pallocSumBytes 633 ) 634 635 // Expose pallocSum for testing. 636 type PallocSum pallocSum 637 638 func PackPallocSum(start, max, end uint) PallocSum { return PallocSum(packPallocSum(start, max, end)) } 639 func (m PallocSum) Start() uint { return pallocSum(m).start() } 640 func (m PallocSum) Max() uint { return pallocSum(m).max() } 641 func (m PallocSum) End() uint { return pallocSum(m).end() } 642 643 // Expose pallocBits for testing. 644 type PallocBits pallocBits 645 646 func (b *PallocBits) Find(npages uintptr, searchIdx uint) (uint, uint) { 647 return (*pallocBits)(b).find(npages, searchIdx) 648 } 649 func (b *PallocBits) AllocRange(i, n uint) { (*pallocBits)(b).allocRange(i, n) } 650 func (b *PallocBits) Free(i, n uint) { (*pallocBits)(b).free(i, n) } 651 func (b *PallocBits) Summarize() PallocSum { return PallocSum((*pallocBits)(b).summarize()) } 652 func (b *PallocBits) PopcntRange(i, n uint) uint { return (*pageBits)(b).popcntRange(i, n) } 653 654 // SummarizeSlow is a slow but more obviously correct implementation 655 // of (*pallocBits).summarize. Used for testing. 656 func SummarizeSlow(b *PallocBits) PallocSum { 657 var start, max, end uint 658 659 const N = uint(len(b)) * 64 660 for start < N && (*pageBits)(b).get(start) == 0 { 661 start++ 662 } 663 for end < N && (*pageBits)(b).get(N-end-1) == 0 { 664 end++ 665 } 666 run := uint(0) 667 for i := uint(0); i < N; i++ { 668 if (*pageBits)(b).get(i) == 0 { 669 run++ 670 } else { 671 run = 0 672 } 673 if run > max { 674 max = run 675 } 676 } 677 return PackPallocSum(start, max, end) 678 } 679 680 // Expose non-trivial helpers for testing. 681 func FindBitRange64(c uint64, n uint) uint { return findBitRange64(c, n) } 682 683 // Given two PallocBits, returns a set of bit ranges where 684 // they differ. 685 func DiffPallocBits(a, b *PallocBits) []BitRange { 686 ba := (*pageBits)(a) 687 bb := (*pageBits)(b) 688 689 var d []BitRange 690 base, size := uint(0), uint(0) 691 for i := uint(0); i < uint(len(ba))*64; i++ { 692 if ba.get(i) != bb.get(i) { 693 if size == 0 { 694 base = i 695 } 696 size++ 697 } else { 698 if size != 0 { 699 d = append(d, BitRange{base, size}) 700 } 701 size = 0 702 } 703 } 704 if size != 0 { 705 d = append(d, BitRange{base, size}) 706 } 707 return d 708 } 709 710 // StringifyPallocBits gets the bits in the bit range r from b, 711 // and returns a string containing the bits as ASCII 0 and 1 712 // characters. 713 func StringifyPallocBits(b *PallocBits, r BitRange) string { 714 str := "" 715 for j := r.I; j < r.I+r.N; j++ { 716 if (*pageBits)(b).get(j) != 0 { 717 str += "1" 718 } else { 719 str += "0" 720 } 721 } 722 return str 723 } 724 725 // Expose pallocData for testing. 726 type PallocData pallocData 727 728 func (d *PallocData) FindScavengeCandidate(searchIdx uint, min, max uintptr) (uint, uint) { 729 return (*pallocData)(d).findScavengeCandidate(searchIdx, min, max) 730 } 731 func (d *PallocData) AllocRange(i, n uint) { (*pallocData)(d).allocRange(i, n) } 732 func (d *PallocData) ScavengedSetRange(i, n uint) { 733 (*pallocData)(d).scavenged.setRange(i, n) 734 } 735 func (d *PallocData) PallocBits() *PallocBits { 736 return (*PallocBits)(&(*pallocData)(d).pallocBits) 737 } 738 func (d *PallocData) Scavenged() *PallocBits { 739 return (*PallocBits)(&(*pallocData)(d).scavenged) 740 } 741 742 // Expose fillAligned for testing. 743 func FillAligned(x uint64, m uint) uint64 { return fillAligned(x, m) } 744 745 // Expose pageCache for testing. 746 type PageCache pageCache 747 748 const PageCachePages = pageCachePages 749 750 func NewPageCache(base uintptr, cache, scav uint64) PageCache { 751 return PageCache(pageCache{base: base, cache: cache, scav: scav}) 752 } 753 func (c *PageCache) Empty() bool { return (*pageCache)(c).empty() } 754 func (c *PageCache) Base() uintptr { return (*pageCache)(c).base } 755 func (c *PageCache) Cache() uint64 { return (*pageCache)(c).cache } 756 func (c *PageCache) Scav() uint64 { return (*pageCache)(c).scav } 757 func (c *PageCache) Alloc(npages uintptr) (uintptr, uintptr) { 758 return (*pageCache)(c).alloc(npages) 759 } 760 func (c *PageCache) Flush(s *PageAlloc) { 761 cp := (*pageCache)(c) 762 sp := (*pageAlloc)(s) 763 764 systemstack(func() { 765 // None of the tests need any higher-level locking, so we just 766 // take the lock internally. 767 lock(sp.mheapLock) 768 cp.flush(sp) 769 unlock(sp.mheapLock) 770 }) 771 } 772 773 // Expose chunk index type. 774 type ChunkIdx chunkIdx 775 776 // Expose pageAlloc for testing. Note that because pageAlloc is 777 // not in the heap, so is PageAlloc. 778 type PageAlloc pageAlloc 779 780 func (p *PageAlloc) Alloc(npages uintptr) (uintptr, uintptr) { 781 pp := (*pageAlloc)(p) 782 783 var addr, scav uintptr 784 systemstack(func() { 785 // None of the tests need any higher-level locking, so we just 786 // take the lock internally. 787 lock(pp.mheapLock) 788 addr, scav = pp.alloc(npages) 789 unlock(pp.mheapLock) 790 }) 791 return addr, scav 792 } 793 func (p *PageAlloc) AllocToCache() PageCache { 794 pp := (*pageAlloc)(p) 795 796 var c PageCache 797 systemstack(func() { 798 // None of the tests need any higher-level locking, so we just 799 // take the lock internally. 800 lock(pp.mheapLock) 801 c = PageCache(pp.allocToCache()) 802 unlock(pp.mheapLock) 803 }) 804 return c 805 } 806 func (p *PageAlloc) Free(base, npages uintptr) { 807 pp := (*pageAlloc)(p) 808 809 systemstack(func() { 810 // None of the tests need any higher-level locking, so we just 811 // take the lock internally. 812 lock(pp.mheapLock) 813 pp.free(base, npages) 814 unlock(pp.mheapLock) 815 }) 816 } 817 func (p *PageAlloc) Bounds() (ChunkIdx, ChunkIdx) { 818 return ChunkIdx((*pageAlloc)(p).start), ChunkIdx((*pageAlloc)(p).end) 819 } 820 func (p *PageAlloc) Scavenge(nbytes uintptr, mayUnlock bool) (r uintptr) { 821 pp := (*pageAlloc)(p) 822 systemstack(func() { 823 // None of the tests need any higher-level locking, so we just 824 // take the lock internally. 825 lock(pp.mheapLock) 826 r = pp.scavenge(nbytes, mayUnlock) 827 unlock(pp.mheapLock) 828 }) 829 return 830 } 831 func (p *PageAlloc) InUse() []AddrRange { 832 ranges := make([]AddrRange, 0, len(p.inUse.ranges)) 833 for _, r := range p.inUse.ranges { 834 ranges = append(ranges, AddrRange{r}) 835 } 836 return ranges 837 } 838 839 // Returns nil if the PallocData's L2 is missing. 840 func (p *PageAlloc) PallocData(i ChunkIdx) *PallocData { 841 ci := chunkIdx(i) 842 return (*PallocData)((*pageAlloc)(p).tryChunkOf(ci)) 843 } 844 845 // AddrRange is a wrapper around addrRange for testing. 846 type AddrRange struct { 847 addrRange 848 } 849 850 // MakeAddrRange creates a new address range. 851 func MakeAddrRange(base, limit uintptr) AddrRange { 852 return AddrRange{makeAddrRange(base, limit)} 853 } 854 855 // Base returns the virtual base address of the address range. 856 func (a AddrRange) Base() uintptr { 857 return a.addrRange.base.addr() 858 } 859 860 // Base returns the virtual address of the limit of the address range. 861 func (a AddrRange) Limit() uintptr { 862 return a.addrRange.limit.addr() 863 } 864 865 // Equals returns true if the two address ranges are exactly equal. 866 func (a AddrRange) Equals(b AddrRange) bool { 867 return a == b 868 } 869 870 // Size returns the size in bytes of the address range. 871 func (a AddrRange) Size() uintptr { 872 return a.addrRange.size() 873 } 874 875 // AddrRanges is a wrapper around addrRanges for testing. 876 type AddrRanges struct { 877 addrRanges 878 mutable bool 879 } 880 881 // NewAddrRanges creates a new empty addrRanges. 882 // 883 // Note that this initializes addrRanges just like in the 884 // runtime, so its memory is persistentalloc'd. Call this 885 // function sparingly since the memory it allocates is 886 // leaked. 887 // 888 // This AddrRanges is mutable, so we can test methods like 889 // Add. 890 func NewAddrRanges() AddrRanges { 891 r := addrRanges{} 892 r.init(new(sysMemStat)) 893 return AddrRanges{r, true} 894 } 895 896 // MakeAddrRanges creates a new addrRanges populated with 897 // the ranges in a. 898 // 899 // The returned AddrRanges is immutable, so methods like 900 // Add will fail. 901 func MakeAddrRanges(a ...AddrRange) AddrRanges { 902 // Methods that manipulate the backing store of addrRanges.ranges should 903 // not be used on the result from this function (e.g. add) since they may 904 // trigger reallocation. That would normally be fine, except the new 905 // backing store won't come from the heap, but from persistentalloc, so 906 // we'll leak some memory implicitly. 907 ranges := make([]addrRange, 0, len(a)) 908 total := uintptr(0) 909 for _, r := range a { 910 ranges = append(ranges, r.addrRange) 911 total += r.Size() 912 } 913 return AddrRanges{addrRanges{ 914 ranges: ranges, 915 totalBytes: total, 916 sysStat: new(sysMemStat), 917 }, false} 918 } 919 920 // Ranges returns a copy of the ranges described by the 921 // addrRanges. 922 func (a *AddrRanges) Ranges() []AddrRange { 923 result := make([]AddrRange, 0, len(a.addrRanges.ranges)) 924 for _, r := range a.addrRanges.ranges { 925 result = append(result, AddrRange{r}) 926 } 927 return result 928 } 929 930 // FindSucc returns the successor to base. See addrRanges.findSucc 931 // for more details. 932 func (a *AddrRanges) FindSucc(base uintptr) int { 933 return a.findSucc(base) 934 } 935 936 // Add adds a new AddrRange to the AddrRanges. 937 // 938 // The AddrRange must be mutable (i.e. created by NewAddrRanges), 939 // otherwise this method will throw. 940 func (a *AddrRanges) Add(r AddrRange) { 941 if !a.mutable { 942 throw("attempt to mutate immutable AddrRanges") 943 } 944 a.add(r.addrRange) 945 } 946 947 // TotalBytes returns the totalBytes field of the addrRanges. 948 func (a *AddrRanges) TotalBytes() uintptr { 949 return a.addrRanges.totalBytes 950 } 951 952 // BitRange represents a range over a bitmap. 953 type BitRange struct { 954 I, N uint // bit index and length in bits 955 } 956 957 // NewPageAlloc creates a new page allocator for testing and 958 // initializes it with the scav and chunks maps. Each key in these maps 959 // represents a chunk index and each value is a series of bit ranges to 960 // set within each bitmap's chunk. 961 // 962 // The initialization of the pageAlloc preserves the invariant that if a 963 // scavenged bit is set the alloc bit is necessarily unset, so some 964 // of the bits described by scav may be cleared in the final bitmap if 965 // ranges in chunks overlap with them. 966 // 967 // scav is optional, and if nil, the scavenged bitmap will be cleared 968 // (as opposed to all 1s, which it usually is). Furthermore, every 969 // chunk index in scav must appear in chunks; ones that do not are 970 // ignored. 971 func NewPageAlloc(chunks, scav map[ChunkIdx][]BitRange) *PageAlloc { 972 p := new(pageAlloc) 973 974 // We've got an entry, so initialize the pageAlloc. 975 p.init(new(mutex), nil) 976 lockInit(p.mheapLock, lockRankMheap) 977 p.test = true 978 979 for i, init := range chunks { 980 addr := chunkBase(chunkIdx(i)) 981 982 // Mark the chunk's existence in the pageAlloc. 983 systemstack(func() { 984 lock(p.mheapLock) 985 p.grow(addr, pallocChunkBytes) 986 unlock(p.mheapLock) 987 }) 988 989 // Initialize the bitmap and update pageAlloc metadata. 990 chunk := p.chunkOf(chunkIndex(addr)) 991 992 // Clear all the scavenged bits which grow set. 993 chunk.scavenged.clearRange(0, pallocChunkPages) 994 995 // Apply scavenge state if applicable. 996 if scav != nil { 997 if scvg, ok := scav[i]; ok { 998 for _, s := range scvg { 999 // Ignore the case of s.N == 0. setRange doesn't handle 1000 // it and it's a no-op anyway. 1001 if s.N != 0 { 1002 chunk.scavenged.setRange(s.I, s.N) 1003 } 1004 } 1005 } 1006 } 1007 1008 // Apply alloc state. 1009 for _, s := range init { 1010 // Ignore the case of s.N == 0. allocRange doesn't handle 1011 // it and it's a no-op anyway. 1012 if s.N != 0 { 1013 chunk.allocRange(s.I, s.N) 1014 } 1015 } 1016 1017 // Update heap metadata for the allocRange calls above. 1018 systemstack(func() { 1019 lock(p.mheapLock) 1020 p.update(addr, pallocChunkPages, false, false) 1021 unlock(p.mheapLock) 1022 }) 1023 } 1024 1025 systemstack(func() { 1026 lock(p.mheapLock) 1027 p.scavengeStartGen() 1028 unlock(p.mheapLock) 1029 }) 1030 1031 return (*PageAlloc)(p) 1032 } 1033 1034 // FreePageAlloc releases hard OS resources owned by the pageAlloc. Once this 1035 // is called the pageAlloc may no longer be used. The object itself will be 1036 // collected by the garbage collector once it is no longer live. 1037 func FreePageAlloc(pp *PageAlloc) { 1038 p := (*pageAlloc)(pp) 1039 1040 // Free all the mapped space for the summary levels. 1041 if pageAlloc64Bit != 0 { 1042 for l := 0; l < summaryLevels; l++ { 1043 sysFree(unsafe.Pointer(&p.summary[l][0]), uintptr(cap(p.summary[l]))*pallocSumBytes, nil) 1044 } 1045 } else { 1046 resSize := uintptr(0) 1047 for _, s := range p.summary { 1048 resSize += uintptr(cap(s)) * pallocSumBytes 1049 } 1050 sysFree(unsafe.Pointer(&p.summary[0][0]), alignUp(resSize, physPageSize), nil) 1051 } 1052 1053 // Free the mapped space for chunks. 1054 for i := range p.chunks { 1055 if x := p.chunks[i]; x != nil { 1056 p.chunks[i] = nil 1057 // This memory comes from sysAlloc and will always be page-aligned. 1058 sysFree(unsafe.Pointer(x), unsafe.Sizeof(*p.chunks[0]), nil) 1059 } 1060 } 1061 } 1062 1063 // BaseChunkIdx is a convenient chunkIdx value which works on both 1064 // 64 bit and 32 bit platforms, allowing the tests to share code 1065 // between the two. 1066 // 1067 // This should not be higher than 0x100*pallocChunkBytes to support 1068 // mips and mipsle, which only have 31-bit address spaces. 1069 var BaseChunkIdx = func() ChunkIdx { 1070 var prefix uintptr 1071 if pageAlloc64Bit != 0 { 1072 prefix = 0xc000 1073 } else { 1074 prefix = 0x100 1075 } 1076 baseAddr := prefix * pallocChunkBytes 1077 if sys.GoosAix != 0 { 1078 baseAddr += arenaBaseOffset 1079 } 1080 return ChunkIdx(chunkIndex(baseAddr)) 1081 }() 1082 1083 // PageBase returns an address given a chunk index and a page index 1084 // relative to that chunk. 1085 func PageBase(c ChunkIdx, pageIdx uint) uintptr { 1086 return chunkBase(chunkIdx(c)) + uintptr(pageIdx)*pageSize 1087 } 1088 1089 type BitsMismatch struct { 1090 Base uintptr 1091 Got, Want uint64 1092 } 1093 1094 func CheckScavengedBitsCleared(mismatches []BitsMismatch) (n int, ok bool) { 1095 ok = true 1096 1097 // Run on the system stack to avoid stack growth allocation. 1098 systemstack(func() { 1099 getg().m.mallocing++ 1100 1101 // Lock so that we can safely access the bitmap. 1102 lock(&mheap_.lock) 1103 chunkLoop: 1104 for i := mheap_.pages.start; i < mheap_.pages.end; i++ { 1105 chunk := mheap_.pages.tryChunkOf(i) 1106 if chunk == nil { 1107 continue 1108 } 1109 for j := 0; j < pallocChunkPages/64; j++ { 1110 // Run over each 64-bit bitmap section and ensure 1111 // scavenged is being cleared properly on allocation. 1112 // If a used bit and scavenged bit are both set, that's 1113 // an error, and could indicate a larger problem, or 1114 // an accounting problem. 1115 want := chunk.scavenged[j] &^ chunk.pallocBits[j] 1116 got := chunk.scavenged[j] 1117 if want != got { 1118 ok = false 1119 if n >= len(mismatches) { 1120 break chunkLoop 1121 } 1122 mismatches[n] = BitsMismatch{ 1123 Base: chunkBase(i) + uintptr(j)*64*pageSize, 1124 Got: got, 1125 Want: want, 1126 } 1127 n++ 1128 } 1129 } 1130 } 1131 unlock(&mheap_.lock) 1132 1133 getg().m.mallocing-- 1134 }) 1135 return 1136 } 1137 1138 func PageCachePagesLeaked() (leaked uintptr) { 1139 stopTheWorld("PageCachePagesLeaked") 1140 1141 // Walk over destroyed Ps and look for unflushed caches. 1142 deadp := allp[len(allp):cap(allp)] 1143 for _, p := range deadp { 1144 // Since we're going past len(allp) we may see nil Ps. 1145 // Just ignore them. 1146 if p != nil { 1147 leaked += uintptr(sys.OnesCount64(p.pcache.cache)) 1148 } 1149 } 1150 1151 startTheWorld() 1152 return 1153 } 1154 1155 var Semacquire = semacquire 1156 var Semrelease1 = semrelease1 1157 1158 func SemNwait(addr *uint32) uint32 { 1159 root := semroot(addr) 1160 return atomic.Load(&root.nwait) 1161 } 1162 1163 // mspan wrapper for testing. 1164 //go:notinheap 1165 type MSpan mspan 1166 1167 // Allocate an mspan for testing. 1168 func AllocMSpan() *MSpan { 1169 var s *mspan 1170 systemstack(func() { 1171 lock(&mheap_.lock) 1172 s = (*mspan)(mheap_.spanalloc.alloc()) 1173 unlock(&mheap_.lock) 1174 }) 1175 return (*MSpan)(s) 1176 } 1177 1178 // Free an allocated mspan. 1179 func FreeMSpan(s *MSpan) { 1180 systemstack(func() { 1181 lock(&mheap_.lock) 1182 mheap_.spanalloc.free(unsafe.Pointer(s)) 1183 unlock(&mheap_.lock) 1184 }) 1185 } 1186 1187 func MSpanCountAlloc(ms *MSpan, bits []byte) int { 1188 s := (*mspan)(ms) 1189 s.nelems = uintptr(len(bits) * 8) 1190 s.gcmarkBits = (*gcBits)(unsafe.Pointer(&bits[0])) 1191 result := s.countAlloc() 1192 s.gcmarkBits = nil 1193 return result 1194 } 1195 1196 const ( 1197 TimeHistSubBucketBits = timeHistSubBucketBits 1198 TimeHistNumSubBuckets = timeHistNumSubBuckets 1199 TimeHistNumSuperBuckets = timeHistNumSuperBuckets 1200 ) 1201 1202 type TimeHistogram timeHistogram 1203 1204 // Counts returns the counts for the given bucket, subBucket indices. 1205 // Returns true if the bucket was valid, otherwise returns the counts 1206 // for the underflow bucket and false. 1207 func (th *TimeHistogram) Count(bucket, subBucket uint) (uint64, bool) { 1208 t := (*timeHistogram)(th) 1209 i := bucket*TimeHistNumSubBuckets + subBucket 1210 if i >= uint(len(t.counts)) { 1211 return t.underflow, false 1212 } 1213 return t.counts[i], true 1214 } 1215 1216 func (th *TimeHistogram) Record(duration int64) { 1217 (*timeHistogram)(th).record(duration) 1218 } 1219 1220 var TimeHistogramMetricsBuckets = timeHistogramMetricsBuckets 1221 1222 func SetIntArgRegs(a int) int { 1223 lock(&finlock) 1224 old := intArgRegs 1225 if a >= 0 { 1226 intArgRegs = a 1227 } 1228 unlock(&finlock) 1229 return old 1230 } 1231 1232 func FinalizerGAsleep() bool { 1233 lock(&finlock) 1234 result := fingwait 1235 unlock(&finlock) 1236 return result 1237 } 1238 1239 // For GCTestMoveStackOnNextCall, it's important not to introduce an 1240 // extra layer of call, since then there's a return before the "real" 1241 // next call. 1242 var GCTestMoveStackOnNextCall = gcTestMoveStackOnNextCall 1243 1244 // For GCTestIsReachable, it's important that we do this as a call so 1245 // escape analysis can see through it. 1246 func GCTestIsReachable(ptrs ...unsafe.Pointer) (mask uint64) { 1247 return gcTestIsReachable(ptrs...) 1248 } 1249 1250 // For GCTestPointerClass, it's important that we do this as a call so 1251 // escape analysis can see through it. 1252 // 1253 // This is nosplit because gcTestPointerClass is. 1254 // 1255 //go:nosplit 1256 func GCTestPointerClass(p unsafe.Pointer) string { 1257 return gcTestPointerClass(p) 1258 } 1259 1260 const Raceenabled = raceenabled