rsc.io/go@v0.0.0-20150416155037-e040fd465409/src/runtime/proc1.go (about) 1 // Copyright 2009 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package runtime 6 7 import "unsafe" 8 9 var ( 10 m0 m 11 g0 g 12 ) 13 14 // Goroutine scheduler 15 // The scheduler's job is to distribute ready-to-run goroutines over worker threads. 16 // 17 // The main concepts are: 18 // G - goroutine. 19 // M - worker thread, or machine. 20 // P - processor, a resource that is required to execute Go code. 21 // M must have an associated P to execute Go code, however it can be 22 // blocked or in a syscall w/o an associated P. 23 // 24 // Design doc at http://golang.org/s/go11sched. 25 26 const ( 27 // Number of goroutine ids to grab from sched.goidgen to local per-P cache at once. 28 // 16 seems to provide enough amortization, but other than that it's mostly arbitrary number. 29 _GoidCacheBatch = 16 30 ) 31 32 // The bootstrap sequence is: 33 // 34 // call osinit 35 // call schedinit 36 // make & queue new G 37 // call runtime·mstart 38 // 39 // The new G calls runtime·main. 40 func schedinit() { 41 // raceinit must be the first call to race detector. 42 // In particular, it must be done before mallocinit below calls racemapshadow. 43 _g_ := getg() 44 if raceenabled { 45 _g_.racectx = raceinit() 46 } 47 48 sched.maxmcount = 10000 49 50 // Cache the framepointer experiment. This affects stack unwinding. 51 framepointer_enabled = haveexperiment("framepointer") 52 53 tracebackinit() 54 symtabverify() 55 stackinit() 56 mallocinit() 57 mcommoninit(_g_.m) 58 59 goargs() 60 goenvs() 61 parsedebugvars() 62 wbshadowinit() 63 gcinit() 64 65 sched.lastpoll = uint64(nanotime()) 66 procs := 1 67 if n := atoi(gogetenv("GOMAXPROCS")); n > 0 { 68 if n > _MaxGomaxprocs { 69 n = _MaxGomaxprocs 70 } 71 procs = n 72 } 73 if procresize(int32(procs)) != nil { 74 throw("unknown runnable goroutine during bootstrap") 75 } 76 77 if buildVersion == "" { 78 // Condition should never trigger. This code just serves 79 // to ensure runtime·buildVersion is kept in the resulting binary. 80 buildVersion = "unknown" 81 } 82 } 83 84 func dumpgstatus(gp *g) { 85 _g_ := getg() 86 print("runtime: gp: gp=", gp, ", goid=", gp.goid, ", gp->atomicstatus=", readgstatus(gp), "\n") 87 print("runtime: g: g=", _g_, ", goid=", _g_.goid, ", g->atomicstatus=", readgstatus(_g_), "\n") 88 } 89 90 func checkmcount() { 91 // sched lock is held 92 if sched.mcount > sched.maxmcount { 93 print("runtime: program exceeds ", sched.maxmcount, "-thread limit\n") 94 throw("thread exhaustion") 95 } 96 } 97 98 func mcommoninit(mp *m) { 99 _g_ := getg() 100 101 // g0 stack won't make sense for user (and is not necessary unwindable). 102 if _g_ != _g_.m.g0 { 103 callers(1, mp.createstack[:]) 104 } 105 106 mp.fastrand = 0x49f6428a + uint32(mp.id) + uint32(cputicks()) 107 if mp.fastrand == 0 { 108 mp.fastrand = 0x49f6428a 109 } 110 111 lock(&sched.lock) 112 mp.id = sched.mcount 113 sched.mcount++ 114 checkmcount() 115 mpreinit(mp) 116 if mp.gsignal != nil { 117 mp.gsignal.stackguard1 = mp.gsignal.stack.lo + _StackGuard 118 } 119 120 // Add to allm so garbage collector doesn't free g->m 121 // when it is just in a register or thread-local storage. 122 mp.alllink = allm 123 124 // NumCgoCall() iterates over allm w/o schedlock, 125 // so we need to publish it safely. 126 atomicstorep(unsafe.Pointer(&allm), unsafe.Pointer(mp)) 127 unlock(&sched.lock) 128 } 129 130 // Mark gp ready to run. 131 func ready(gp *g, traceskip int) { 132 if trace.enabled { 133 traceGoUnpark(gp, traceskip) 134 } 135 136 status := readgstatus(gp) 137 138 // Mark runnable. 139 _g_ := getg() 140 _g_.m.locks++ // disable preemption because it can be holding p in a local var 141 if status&^_Gscan != _Gwaiting { 142 dumpgstatus(gp) 143 throw("bad g->status in ready") 144 } 145 146 // status is Gwaiting or Gscanwaiting, make Grunnable and put on runq 147 casgstatus(gp, _Gwaiting, _Grunnable) 148 runqput(_g_.m.p, gp) 149 if atomicload(&sched.npidle) != 0 && atomicload(&sched.nmspinning) == 0 { // TODO: fast atomic 150 wakep() 151 } 152 _g_.m.locks-- 153 if _g_.m.locks == 0 && _g_.preempt { // restore the preemption request in case we've cleared it in newstack 154 _g_.stackguard0 = stackPreempt 155 } 156 } 157 158 // readyExecute marks gp ready to run, preempt the current g, and execute gp. 159 // This is used to start concurrent GC promptly when we reach its trigger. 160 func readyExecute(gp *g, traceskip int) { 161 mcall(func(_g_ *g) { 162 if trace.enabled { 163 traceGoUnpark(gp, traceskip) 164 traceGoSched() 165 } 166 167 if _g_.m.locks != 0 { 168 throw("readyExecute: holding locks") 169 } 170 if _g_.m.lockedg != nil { 171 throw("cannot readyExecute from a locked g") 172 } 173 if readgstatus(gp)&^_Gscan != _Gwaiting { 174 dumpgstatus(gp) 175 throw("bad gp.status in readyExecute") 176 } 177 178 // Preempt the current g 179 casgstatus(_g_, _Grunning, _Grunnable) 180 runqput(_g_.m.p, _g_) 181 dropg() 182 183 // Ready gp and switch to it 184 casgstatus(gp, _Gwaiting, _Grunnable) 185 execute(gp) 186 }) 187 } 188 189 func gcprocs() int32 { 190 // Figure out how many CPUs to use during GC. 191 // Limited by gomaxprocs, number of actual CPUs, and MaxGcproc. 192 lock(&sched.lock) 193 n := gomaxprocs 194 if n > ncpu { 195 n = ncpu 196 } 197 if n > _MaxGcproc { 198 n = _MaxGcproc 199 } 200 if n > sched.nmidle+1 { // one M is currently running 201 n = sched.nmidle + 1 202 } 203 unlock(&sched.lock) 204 return n 205 } 206 207 func needaddgcproc() bool { 208 lock(&sched.lock) 209 n := gomaxprocs 210 if n > ncpu { 211 n = ncpu 212 } 213 if n > _MaxGcproc { 214 n = _MaxGcproc 215 } 216 n -= sched.nmidle + 1 // one M is currently running 217 unlock(&sched.lock) 218 return n > 0 219 } 220 221 func helpgc(nproc int32) { 222 _g_ := getg() 223 lock(&sched.lock) 224 pos := 0 225 for n := int32(1); n < nproc; n++ { // one M is currently running 226 if allp[pos].mcache == _g_.m.mcache { 227 pos++ 228 } 229 mp := mget() 230 if mp == nil { 231 throw("gcprocs inconsistency") 232 } 233 mp.helpgc = n 234 mp.p = allp[pos] 235 mp.mcache = allp[pos].mcache 236 pos++ 237 notewakeup(&mp.park) 238 } 239 unlock(&sched.lock) 240 } 241 242 // freezeStopWait is a large value that freezetheworld sets 243 // sched.stopwait to in order to request that all Gs permanently stop. 244 const freezeStopWait = 0x7fffffff 245 246 // Similar to stoptheworld but best-effort and can be called several times. 247 // There is no reverse operation, used during crashing. 248 // This function must not lock any mutexes. 249 func freezetheworld() { 250 if gomaxprocs == 1 { 251 return 252 } 253 // stopwait and preemption requests can be lost 254 // due to races with concurrently executing threads, 255 // so try several times 256 for i := 0; i < 5; i++ { 257 // this should tell the scheduler to not start any new goroutines 258 sched.stopwait = freezeStopWait 259 atomicstore(&sched.gcwaiting, 1) 260 // this should stop running goroutines 261 if !preemptall() { 262 break // no running goroutines 263 } 264 usleep(1000) 265 } 266 // to be sure 267 usleep(1000) 268 preemptall() 269 usleep(1000) 270 } 271 272 func isscanstatus(status uint32) bool { 273 if status == _Gscan { 274 throw("isscanstatus: Bad status Gscan") 275 } 276 return status&_Gscan == _Gscan 277 } 278 279 // All reads and writes of g's status go through readgstatus, casgstatus 280 // castogscanstatus, casfrom_Gscanstatus. 281 //go:nosplit 282 func readgstatus(gp *g) uint32 { 283 return atomicload(&gp.atomicstatus) 284 } 285 286 // The Gscanstatuses are acting like locks and this releases them. 287 // If it proves to be a performance hit we should be able to make these 288 // simple atomic stores but for now we are going to throw if 289 // we see an inconsistent state. 290 func casfrom_Gscanstatus(gp *g, oldval, newval uint32) { 291 success := false 292 293 // Check that transition is valid. 294 switch oldval { 295 default: 296 print("runtime: casfrom_Gscanstatus bad oldval gp=", gp, ", oldval=", hex(oldval), ", newval=", hex(newval), "\n") 297 dumpgstatus(gp) 298 throw("casfrom_Gscanstatus:top gp->status is not in scan state") 299 case _Gscanrunnable, 300 _Gscanwaiting, 301 _Gscanrunning, 302 _Gscansyscall: 303 if newval == oldval&^_Gscan { 304 success = cas(&gp.atomicstatus, oldval, newval) 305 } 306 case _Gscanenqueue: 307 if newval == _Gwaiting { 308 success = cas(&gp.atomicstatus, oldval, newval) 309 } 310 } 311 if !success { 312 print("runtime: casfrom_Gscanstatus failed gp=", gp, ", oldval=", hex(oldval), ", newval=", hex(newval), "\n") 313 dumpgstatus(gp) 314 throw("casfrom_Gscanstatus: gp->status is not in scan state") 315 } 316 if newval == _Grunning { 317 gp.gcscanvalid = false 318 } 319 } 320 321 // This will return false if the gp is not in the expected status and the cas fails. 322 // This acts like a lock acquire while the casfromgstatus acts like a lock release. 323 func castogscanstatus(gp *g, oldval, newval uint32) bool { 324 switch oldval { 325 case _Grunnable, 326 _Gwaiting, 327 _Gsyscall: 328 if newval == oldval|_Gscan { 329 return cas(&gp.atomicstatus, oldval, newval) 330 } 331 case _Grunning: 332 if gp.gcscanvalid { 333 print("runtime: castogscanstatus _Grunning and gp.gcscanvalid is true, newval=", hex(newval), "\n") 334 throw("castogscanstatus") 335 } 336 if newval == _Gscanrunning || newval == _Gscanenqueue { 337 return cas(&gp.atomicstatus, oldval, newval) 338 } 339 } 340 print("runtime: castogscanstatus oldval=", hex(oldval), " newval=", hex(newval), "\n") 341 throw("castogscanstatus") 342 panic("not reached") 343 } 344 345 // If asked to move to or from a Gscanstatus this will throw. Use the castogscanstatus 346 // and casfrom_Gscanstatus instead. 347 // casgstatus will loop if the g->atomicstatus is in a Gscan status until the routine that 348 // put it in the Gscan state is finished. 349 //go:nosplit 350 func casgstatus(gp *g, oldval, newval uint32) { 351 if (oldval&_Gscan != 0) || (newval&_Gscan != 0) || oldval == newval { 352 systemstack(func() { 353 print("runtime: casgstatus: oldval=", hex(oldval), " newval=", hex(newval), "\n") 354 throw("casgstatus: bad incoming values") 355 }) 356 } 357 358 // loop if gp->atomicstatus is in a scan state giving 359 // GC time to finish and change the state to oldval. 360 for !cas(&gp.atomicstatus, oldval, newval) { 361 if oldval == _Gwaiting && gp.atomicstatus == _Grunnable { 362 systemstack(func() { 363 throw("casgstatus: waiting for Gwaiting but is Grunnable") 364 }) 365 } 366 // Help GC if needed. 367 // if gp.preemptscan && !gp.gcworkdone && (oldval == _Grunning || oldval == _Gsyscall) { 368 // gp.preemptscan = false 369 // systemstack(func() { 370 // gcphasework(gp) 371 // }) 372 // } 373 } 374 if newval == _Grunning { 375 gp.gcscanvalid = false 376 } 377 } 378 379 // casgstatus(gp, oldstatus, Gcopystack), assuming oldstatus is Gwaiting or Grunnable. 380 // Returns old status. Cannot call casgstatus directly, because we are racing with an 381 // async wakeup that might come in from netpoll. If we see Gwaiting from the readgstatus, 382 // it might have become Grunnable by the time we get to the cas. If we called casgstatus, 383 // it would loop waiting for the status to go back to Gwaiting, which it never will. 384 //go:nosplit 385 func casgcopystack(gp *g) uint32 { 386 for { 387 oldstatus := readgstatus(gp) &^ _Gscan 388 if oldstatus != _Gwaiting && oldstatus != _Grunnable { 389 throw("copystack: bad status, not Gwaiting or Grunnable") 390 } 391 if cas(&gp.atomicstatus, oldstatus, _Gcopystack) { 392 return oldstatus 393 } 394 } 395 } 396 397 // stopg ensures that gp is stopped at a GC safe point where its stack can be scanned 398 // or in the context of a moving collector the pointers can be flipped from pointing 399 // to old object to pointing to new objects. 400 // If stopg returns true, the caller knows gp is at a GC safe point and will remain there until 401 // the caller calls restartg. 402 // If stopg returns false, the caller is not responsible for calling restartg. This can happen 403 // if another thread, either the gp itself or another GC thread is taking the responsibility 404 // to do the GC work related to this thread. 405 func stopg(gp *g) bool { 406 for { 407 if gp.gcworkdone { 408 return false 409 } 410 411 switch s := readgstatus(gp); s { 412 default: 413 dumpgstatus(gp) 414 throw("stopg: gp->atomicstatus is not valid") 415 416 case _Gdead: 417 return false 418 419 case _Gcopystack: 420 // Loop until a new stack is in place. 421 422 case _Grunnable, 423 _Gsyscall, 424 _Gwaiting: 425 // Claim goroutine by setting scan bit. 426 if !castogscanstatus(gp, s, s|_Gscan) { 427 break 428 } 429 // In scan state, do work. 430 gcphasework(gp) 431 return true 432 433 case _Gscanrunnable, 434 _Gscanwaiting, 435 _Gscansyscall: 436 // Goroutine already claimed by another GC helper. 437 return false 438 439 case _Grunning: 440 // Claim goroutine, so we aren't racing with a status 441 // transition away from Grunning. 442 if !castogscanstatus(gp, _Grunning, _Gscanrunning) { 443 break 444 } 445 446 // Mark gp for preemption. 447 if !gp.gcworkdone { 448 gp.preemptscan = true 449 gp.preempt = true 450 gp.stackguard0 = stackPreempt 451 } 452 453 // Unclaim. 454 casfrom_Gscanstatus(gp, _Gscanrunning, _Grunning) 455 return false 456 } 457 } 458 } 459 460 // The GC requests that this routine be moved from a scanmumble state to a mumble state. 461 func restartg(gp *g) { 462 s := readgstatus(gp) 463 switch s { 464 default: 465 dumpgstatus(gp) 466 throw("restartg: unexpected status") 467 468 case _Gdead: 469 // ok 470 471 case _Gscanrunnable, 472 _Gscanwaiting, 473 _Gscansyscall: 474 casfrom_Gscanstatus(gp, s, s&^_Gscan) 475 476 // Scan is now completed. 477 // Goroutine now needs to be made runnable. 478 // We put it on the global run queue; ready blocks on the global scheduler lock. 479 case _Gscanenqueue: 480 casfrom_Gscanstatus(gp, _Gscanenqueue, _Gwaiting) 481 if gp != getg().m.curg { 482 throw("processing Gscanenqueue on wrong m") 483 } 484 dropg() 485 ready(gp, 0) 486 } 487 } 488 489 func stopscanstart(gp *g) { 490 _g_ := getg() 491 if _g_ == gp { 492 throw("GC not moved to G0") 493 } 494 if stopg(gp) { 495 if !isscanstatus(readgstatus(gp)) { 496 dumpgstatus(gp) 497 throw("GC not in scan state") 498 } 499 restartg(gp) 500 } 501 } 502 503 // Runs on g0 and does the actual work after putting the g back on the run queue. 504 func mquiesce(gpmaster *g) { 505 // enqueue the calling goroutine. 506 restartg(gpmaster) 507 508 activeglen := len(allgs) 509 for i := 0; i < activeglen; i++ { 510 gp := allgs[i] 511 if readgstatus(gp) == _Gdead { 512 gp.gcworkdone = true // noop scan. 513 } else { 514 gp.gcworkdone = false 515 } 516 stopscanstart(gp) 517 } 518 519 // Check that the G's gcwork (such as scanning) has been done. If not do it now. 520 // You can end up doing work here if the page trap on a Grunning Goroutine has 521 // not been sprung or in some race situations. For example a runnable goes dead 522 // and is started up again with a gp->gcworkdone set to false. 523 for i := 0; i < activeglen; i++ { 524 gp := allgs[i] 525 for !gp.gcworkdone { 526 status := readgstatus(gp) 527 if status == _Gdead { 528 //do nothing, scan not needed. 529 gp.gcworkdone = true // scan is a noop 530 break 531 } 532 if status == _Grunning && gp.stackguard0 == uintptr(stackPreempt) && notetsleep(&sched.stopnote, 100*1000) { // nanosecond arg 533 noteclear(&sched.stopnote) 534 } else { 535 stopscanstart(gp) 536 } 537 } 538 } 539 540 for i := 0; i < activeglen; i++ { 541 gp := allgs[i] 542 status := readgstatus(gp) 543 if isscanstatus(status) { 544 print("mstopandscang:bottom: post scan bad status gp=", gp, " has status ", hex(status), "\n") 545 dumpgstatus(gp) 546 } 547 if !gp.gcworkdone && status != _Gdead { 548 print("mstopandscang:bottom: post scan gp=", gp, "->gcworkdone still false\n") 549 dumpgstatus(gp) 550 } 551 } 552 553 schedule() // Never returns. 554 } 555 556 // quiesce moves all the goroutines to a GC safepoint which for now is a at preemption point. 557 // If the global gcphase is GCmark quiesce will ensure that all of the goroutine's stacks 558 // have been scanned before it returns. 559 func quiesce(mastergp *g) { 560 castogscanstatus(mastergp, _Grunning, _Gscanenqueue) 561 // Now move this to the g0 (aka m) stack. 562 // g0 will potentially scan this thread and put mastergp on the runqueue 563 mcall(mquiesce) 564 } 565 566 // Holding worldsema grants an M the right to try to stop the world. 567 // The procedure is: 568 // 569 // semacquire(&worldsema); 570 // m.preemptoff = "reason"; 571 // stoptheworld(); 572 // 573 // ... do stuff ... 574 // 575 // m.preemptoff = ""; 576 // semrelease(&worldsema); 577 // starttheworld(); 578 // 579 var worldsema uint32 = 1 580 581 // This is used by the GC as well as the routines that do stack dumps. In the case 582 // of GC all the routines can be reliably stopped. This is not always the case 583 // when the system is in panic or being exited. 584 func stoptheworld() { 585 _g_ := getg() 586 587 // If we hold a lock, then we won't be able to stop another M 588 // that is blocked trying to acquire the lock. 589 if _g_.m.locks > 0 { 590 throw("stoptheworld: holding locks") 591 } 592 593 lock(&sched.lock) 594 sched.stopwait = gomaxprocs 595 atomicstore(&sched.gcwaiting, 1) 596 preemptall() 597 // stop current P 598 _g_.m.p.status = _Pgcstop // Pgcstop is only diagnostic. 599 sched.stopwait-- 600 // try to retake all P's in Psyscall status 601 for i := 0; i < int(gomaxprocs); i++ { 602 p := allp[i] 603 s := p.status 604 if s == _Psyscall && cas(&p.status, s, _Pgcstop) { 605 if trace.enabled { 606 traceGoSysBlock(p) 607 traceProcStop(p) 608 } 609 p.syscalltick++ 610 sched.stopwait-- 611 } 612 } 613 // stop idle P's 614 for { 615 p := pidleget() 616 if p == nil { 617 break 618 } 619 p.status = _Pgcstop 620 sched.stopwait-- 621 } 622 wait := sched.stopwait > 0 623 unlock(&sched.lock) 624 625 // wait for remaining P's to stop voluntarily 626 if wait { 627 for { 628 // wait for 100us, then try to re-preempt in case of any races 629 if notetsleep(&sched.stopnote, 100*1000) { 630 noteclear(&sched.stopnote) 631 break 632 } 633 preemptall() 634 } 635 } 636 if sched.stopwait != 0 { 637 throw("stoptheworld: not stopped") 638 } 639 for i := 0; i < int(gomaxprocs); i++ { 640 p := allp[i] 641 if p.status != _Pgcstop { 642 throw("stoptheworld: not stopped") 643 } 644 } 645 } 646 647 func mhelpgc() { 648 _g_ := getg() 649 _g_.m.helpgc = -1 650 } 651 652 func starttheworld() { 653 _g_ := getg() 654 655 _g_.m.locks++ // disable preemption because it can be holding p in a local var 656 gp := netpoll(false) // non-blocking 657 injectglist(gp) 658 add := needaddgcproc() 659 lock(&sched.lock) 660 661 procs := gomaxprocs 662 if newprocs != 0 { 663 procs = newprocs 664 newprocs = 0 665 } 666 p1 := procresize(procs) 667 sched.gcwaiting = 0 668 if sched.sysmonwait != 0 { 669 sched.sysmonwait = 0 670 notewakeup(&sched.sysmonnote) 671 } 672 unlock(&sched.lock) 673 674 for p1 != nil { 675 p := p1 676 p1 = p1.link 677 if p.m != nil { 678 mp := p.m 679 p.m = nil 680 if mp.nextp != nil { 681 throw("starttheworld: inconsistent mp->nextp") 682 } 683 mp.nextp = p 684 notewakeup(&mp.park) 685 } else { 686 // Start M to run P. Do not start another M below. 687 newm(nil, p) 688 add = false 689 } 690 } 691 692 // Wakeup an additional proc in case we have excessive runnable goroutines 693 // in local queues or in the global queue. If we don't, the proc will park itself. 694 // If we have lots of excessive work, resetspinning will unpark additional procs as necessary. 695 if atomicload(&sched.npidle) != 0 && atomicload(&sched.nmspinning) == 0 { 696 wakep() 697 } 698 699 if add { 700 // If GC could have used another helper proc, start one now, 701 // in the hope that it will be available next time. 702 // It would have been even better to start it before the collection, 703 // but doing so requires allocating memory, so it's tricky to 704 // coordinate. This lazy approach works out in practice: 705 // we don't mind if the first couple gc rounds don't have quite 706 // the maximum number of procs. 707 newm(mhelpgc, nil) 708 } 709 _g_.m.locks-- 710 if _g_.m.locks == 0 && _g_.preempt { // restore the preemption request in case we've cleared it in newstack 711 _g_.stackguard0 = stackPreempt 712 } 713 } 714 715 // Called to start an M. 716 //go:nosplit 717 func mstart() { 718 _g_ := getg() 719 720 if _g_.stack.lo == 0 { 721 // Initialize stack bounds from system stack. 722 // Cgo may have left stack size in stack.hi. 723 size := _g_.stack.hi 724 if size == 0 { 725 size = 8192 726 } 727 _g_.stack.hi = uintptr(noescape(unsafe.Pointer(&size))) 728 _g_.stack.lo = _g_.stack.hi - size + 1024 729 } 730 // Initialize stack guards so that we can start calling 731 // both Go and C functions with stack growth prologues. 732 _g_.stackguard0 = _g_.stack.lo + _StackGuard 733 _g_.stackguard1 = _g_.stackguard0 734 mstart1() 735 } 736 737 func mstart1() { 738 _g_ := getg() 739 740 if _g_ != _g_.m.g0 { 741 throw("bad runtime·mstart") 742 } 743 744 // Record top of stack for use by mcall. 745 // Once we call schedule we're never coming back, 746 // so other calls can reuse this stack space. 747 gosave(&_g_.m.g0.sched) 748 _g_.m.g0.sched.pc = ^uintptr(0) // make sure it is never used 749 asminit() 750 minit() 751 752 // Install signal handlers; after minit so that minit can 753 // prepare the thread to be able to handle the signals. 754 if _g_.m == &m0 { 755 // Create an extra M for callbacks on threads not created by Go. 756 if iscgo && !cgoHasExtraM { 757 cgoHasExtraM = true 758 newextram() 759 } 760 initsig() 761 } 762 763 if _g_.m.mstartfn != 0 { 764 fn := *(*func())(unsafe.Pointer(&_g_.m.mstartfn)) 765 fn() 766 } 767 768 if _g_.m.helpgc != 0 { 769 _g_.m.helpgc = 0 770 stopm() 771 } else if _g_.m != &m0 { 772 acquirep(_g_.m.nextp) 773 _g_.m.nextp = nil 774 } 775 schedule() 776 } 777 778 // When running with cgo, we call _cgo_thread_start 779 // to start threads for us so that we can play nicely with 780 // foreign code. 781 var cgoThreadStart unsafe.Pointer 782 783 type cgothreadstart struct { 784 g *g 785 tls *uint64 786 fn unsafe.Pointer 787 } 788 789 // Allocate a new m unassociated with any thread. 790 // Can use p for allocation context if needed. 791 func allocm(_p_ *p) *m { 792 _g_ := getg() 793 _g_.m.locks++ // disable GC because it can be called from sysmon 794 if _g_.m.p == nil { 795 acquirep(_p_) // temporarily borrow p for mallocs in this function 796 } 797 mp := new(m) 798 mcommoninit(mp) 799 800 // In case of cgo or Solaris, pthread_create will make us a stack. 801 // Windows and Plan 9 will layout sched stack on OS stack. 802 if iscgo || GOOS == "solaris" || GOOS == "windows" || GOOS == "plan9" { 803 mp.g0 = malg(-1) 804 } else { 805 mp.g0 = malg(8192) 806 } 807 mp.g0.m = mp 808 809 if _p_ == _g_.m.p { 810 releasep() 811 } 812 _g_.m.locks-- 813 if _g_.m.locks == 0 && _g_.preempt { // restore the preemption request in case we've cleared it in newstack 814 _g_.stackguard0 = stackPreempt 815 } 816 817 return mp 818 } 819 820 // needm is called when a cgo callback happens on a 821 // thread without an m (a thread not created by Go). 822 // In this case, needm is expected to find an m to use 823 // and return with m, g initialized correctly. 824 // Since m and g are not set now (likely nil, but see below) 825 // needm is limited in what routines it can call. In particular 826 // it can only call nosplit functions (textflag 7) and cannot 827 // do any scheduling that requires an m. 828 // 829 // In order to avoid needing heavy lifting here, we adopt 830 // the following strategy: there is a stack of available m's 831 // that can be stolen. Using compare-and-swap 832 // to pop from the stack has ABA races, so we simulate 833 // a lock by doing an exchange (via casp) to steal the stack 834 // head and replace the top pointer with MLOCKED (1). 835 // This serves as a simple spin lock that we can use even 836 // without an m. The thread that locks the stack in this way 837 // unlocks the stack by storing a valid stack head pointer. 838 // 839 // In order to make sure that there is always an m structure 840 // available to be stolen, we maintain the invariant that there 841 // is always one more than needed. At the beginning of the 842 // program (if cgo is in use) the list is seeded with a single m. 843 // If needm finds that it has taken the last m off the list, its job 844 // is - once it has installed its own m so that it can do things like 845 // allocate memory - to create a spare m and put it on the list. 846 // 847 // Each of these extra m's also has a g0 and a curg that are 848 // pressed into service as the scheduling stack and current 849 // goroutine for the duration of the cgo callback. 850 // 851 // When the callback is done with the m, it calls dropm to 852 // put the m back on the list. 853 //go:nosplit 854 func needm(x byte) { 855 if iscgo && !cgoHasExtraM { 856 // Can happen if C/C++ code calls Go from a global ctor. 857 // Can not throw, because scheduler is not initialized yet. 858 write(2, unsafe.Pointer(&earlycgocallback[0]), int32(len(earlycgocallback))) 859 exit(1) 860 } 861 862 // Lock extra list, take head, unlock popped list. 863 // nilokay=false is safe here because of the invariant above, 864 // that the extra list always contains or will soon contain 865 // at least one m. 866 mp := lockextra(false) 867 868 // Set needextram when we've just emptied the list, 869 // so that the eventual call into cgocallbackg will 870 // allocate a new m for the extra list. We delay the 871 // allocation until then so that it can be done 872 // after exitsyscall makes sure it is okay to be 873 // running at all (that is, there's no garbage collection 874 // running right now). 875 mp.needextram = mp.schedlink == nil 876 unlockextra(mp.schedlink) 877 878 // Install g (= m->g0) and set the stack bounds 879 // to match the current stack. We don't actually know 880 // how big the stack is, like we don't know how big any 881 // scheduling stack is, but we assume there's at least 32 kB, 882 // which is more than enough for us. 883 setg(mp.g0) 884 _g_ := getg() 885 _g_.stack.hi = uintptr(noescape(unsafe.Pointer(&x))) + 1024 886 _g_.stack.lo = uintptr(noescape(unsafe.Pointer(&x))) - 32*1024 887 _g_.stackguard0 = _g_.stack.lo + _StackGuard 888 889 // Initialize this thread to use the m. 890 asminit() 891 minit() 892 } 893 894 var earlycgocallback = []byte("fatal error: cgo callback before cgo call\n") 895 896 // newextram allocates an m and puts it on the extra list. 897 // It is called with a working local m, so that it can do things 898 // like call schedlock and allocate. 899 func newextram() { 900 // Create extra goroutine locked to extra m. 901 // The goroutine is the context in which the cgo callback will run. 902 // The sched.pc will never be returned to, but setting it to 903 // goexit makes clear to the traceback routines where 904 // the goroutine stack ends. 905 mp := allocm(nil) 906 gp := malg(4096) 907 gp.sched.pc = funcPC(goexit) + _PCQuantum 908 gp.sched.sp = gp.stack.hi 909 gp.sched.sp -= 4 * regSize // extra space in case of reads slightly beyond frame 910 gp.sched.lr = 0 911 gp.sched.g = guintptr(unsafe.Pointer(gp)) 912 gp.syscallpc = gp.sched.pc 913 gp.syscallsp = gp.sched.sp 914 // malg returns status as Gidle, change to Gsyscall before adding to allg 915 // where GC will see it. 916 casgstatus(gp, _Gidle, _Gsyscall) 917 gp.m = mp 918 mp.curg = gp 919 mp.locked = _LockInternal 920 mp.lockedg = gp 921 gp.lockedm = mp 922 gp.goid = int64(xadd64(&sched.goidgen, 1)) 923 if raceenabled { 924 gp.racectx = racegostart(funcPC(newextram)) 925 } 926 // put on allg for garbage collector 927 allgadd(gp) 928 929 // Add m to the extra list. 930 mnext := lockextra(true) 931 mp.schedlink = mnext 932 unlockextra(mp) 933 } 934 935 // dropm is called when a cgo callback has called needm but is now 936 // done with the callback and returning back into the non-Go thread. 937 // It puts the current m back onto the extra list. 938 // 939 // The main expense here is the call to signalstack to release the 940 // m's signal stack, and then the call to needm on the next callback 941 // from this thread. It is tempting to try to save the m for next time, 942 // which would eliminate both these costs, but there might not be 943 // a next time: the current thread (which Go does not control) might exit. 944 // If we saved the m for that thread, there would be an m leak each time 945 // such a thread exited. Instead, we acquire and release an m on each 946 // call. These should typically not be scheduling operations, just a few 947 // atomics, so the cost should be small. 948 // 949 // TODO(rsc): An alternative would be to allocate a dummy pthread per-thread 950 // variable using pthread_key_create. Unlike the pthread keys we already use 951 // on OS X, this dummy key would never be read by Go code. It would exist 952 // only so that we could register at thread-exit-time destructor. 953 // That destructor would put the m back onto the extra list. 954 // This is purely a performance optimization. The current version, 955 // in which dropm happens on each cgo call, is still correct too. 956 // We may have to keep the current version on systems with cgo 957 // but without pthreads, like Windows. 958 func dropm() { 959 // Undo whatever initialization minit did during needm. 960 unminit() 961 962 // Clear m and g, and return m to the extra list. 963 // After the call to setg we can only call nosplit functions 964 // with no pointer manipulation. 965 mp := getg().m 966 mnext := lockextra(true) 967 mp.schedlink = mnext 968 969 setg(nil) 970 unlockextra(mp) 971 } 972 973 var extram uintptr 974 975 // lockextra locks the extra list and returns the list head. 976 // The caller must unlock the list by storing a new list head 977 // to extram. If nilokay is true, then lockextra will 978 // return a nil list head if that's what it finds. If nilokay is false, 979 // lockextra will keep waiting until the list head is no longer nil. 980 //go:nosplit 981 func lockextra(nilokay bool) *m { 982 const locked = 1 983 984 for { 985 old := atomicloaduintptr(&extram) 986 if old == locked { 987 yield := osyield 988 yield() 989 continue 990 } 991 if old == 0 && !nilokay { 992 usleep(1) 993 continue 994 } 995 if casuintptr(&extram, old, locked) { 996 return (*m)(unsafe.Pointer(old)) 997 } 998 yield := osyield 999 yield() 1000 continue 1001 } 1002 } 1003 1004 //go:nosplit 1005 func unlockextra(mp *m) { 1006 atomicstoreuintptr(&extram, uintptr(unsafe.Pointer(mp))) 1007 } 1008 1009 // Create a new m. It will start off with a call to fn, or else the scheduler. 1010 // fn needs to be static and not a heap allocated closure. 1011 // May run with m.p==nil, so write barriers are not allowed. 1012 //go:nowritebarrier 1013 func newm(fn func(), _p_ *p) { 1014 mp := allocm(_p_) 1015 // procresize made _p_ reachable through allp, which doesn't change during GC, so WB can be eliminated 1016 setPNoWriteBarrier(&mp.nextp, _p_) 1017 // Store &fn as a uintptr since it is not heap allocated so the WB can be eliminated 1018 mp.mstartfn = *(*uintptr)(unsafe.Pointer(&fn)) 1019 if iscgo { 1020 var ts cgothreadstart 1021 if _cgo_thread_start == nil { 1022 throw("_cgo_thread_start missing") 1023 } 1024 // mp is reachable via allm and mp.g0 never changes, so WB can be eliminated. 1025 setGNoWriteBarrier(&ts.g, mp.g0) 1026 ts.tls = (*uint64)(unsafe.Pointer(&mp.tls[0])) 1027 ts.fn = unsafe.Pointer(funcPC(mstart)) 1028 asmcgocall(_cgo_thread_start, unsafe.Pointer(&ts)) 1029 return 1030 } 1031 newosproc(mp, unsafe.Pointer(mp.g0.stack.hi)) 1032 } 1033 1034 // Stops execution of the current m until new work is available. 1035 // Returns with acquired P. 1036 func stopm() { 1037 _g_ := getg() 1038 1039 if _g_.m.locks != 0 { 1040 throw("stopm holding locks") 1041 } 1042 if _g_.m.p != nil { 1043 throw("stopm holding p") 1044 } 1045 if _g_.m.spinning { 1046 _g_.m.spinning = false 1047 xadd(&sched.nmspinning, -1) 1048 } 1049 1050 retry: 1051 lock(&sched.lock) 1052 mput(_g_.m) 1053 unlock(&sched.lock) 1054 notesleep(&_g_.m.park) 1055 noteclear(&_g_.m.park) 1056 if _g_.m.helpgc != 0 { 1057 gchelper() 1058 _g_.m.helpgc = 0 1059 _g_.m.mcache = nil 1060 _g_.m.p = nil 1061 goto retry 1062 } 1063 acquirep(_g_.m.nextp) 1064 _g_.m.nextp = nil 1065 } 1066 1067 func mspinning() { 1068 getg().m.spinning = true 1069 } 1070 1071 // Schedules some M to run the p (creates an M if necessary). 1072 // If p==nil, tries to get an idle P, if no idle P's does nothing. 1073 // May run with m.p==nil, so write barriers are not allowed. 1074 //go:nowritebarrier 1075 func startm(_p_ *p, spinning bool) { 1076 lock(&sched.lock) 1077 if _p_ == nil { 1078 _p_ = pidleget() 1079 if _p_ == nil { 1080 unlock(&sched.lock) 1081 if spinning { 1082 xadd(&sched.nmspinning, -1) 1083 } 1084 return 1085 } 1086 } 1087 mp := mget() 1088 unlock(&sched.lock) 1089 if mp == nil { 1090 var fn func() 1091 if spinning { 1092 fn = mspinning 1093 } 1094 newm(fn, _p_) 1095 return 1096 } 1097 if mp.spinning { 1098 throw("startm: m is spinning") 1099 } 1100 if mp.nextp != nil { 1101 throw("startm: m has p") 1102 } 1103 mp.spinning = spinning 1104 // procresize made _p_ reachable through allp, which doesn't change during GC, so WB can be eliminated 1105 setPNoWriteBarrier(&mp.nextp, _p_) 1106 notewakeup(&mp.park) 1107 } 1108 1109 // Hands off P from syscall or locked M. 1110 // Always runs without a P, so write barriers are not allowed. 1111 //go:nowritebarrier 1112 func handoffp(_p_ *p) { 1113 // if it has local work, start it straight away 1114 if _p_.runqhead != _p_.runqtail || sched.runqsize != 0 { 1115 startm(_p_, false) 1116 return 1117 } 1118 // no local work, check that there are no spinning/idle M's, 1119 // otherwise our help is not required 1120 if atomicload(&sched.nmspinning)+atomicload(&sched.npidle) == 0 && cas(&sched.nmspinning, 0, 1) { // TODO: fast atomic 1121 startm(_p_, true) 1122 return 1123 } 1124 lock(&sched.lock) 1125 if sched.gcwaiting != 0 { 1126 _p_.status = _Pgcstop 1127 sched.stopwait-- 1128 if sched.stopwait == 0 { 1129 notewakeup(&sched.stopnote) 1130 } 1131 unlock(&sched.lock) 1132 return 1133 } 1134 if sched.runqsize != 0 { 1135 unlock(&sched.lock) 1136 startm(_p_, false) 1137 return 1138 } 1139 // If this is the last running P and nobody is polling network, 1140 // need to wakeup another M to poll network. 1141 if sched.npidle == uint32(gomaxprocs-1) && atomicload64(&sched.lastpoll) != 0 { 1142 unlock(&sched.lock) 1143 startm(_p_, false) 1144 return 1145 } 1146 pidleput(_p_) 1147 unlock(&sched.lock) 1148 } 1149 1150 // Tries to add one more P to execute G's. 1151 // Called when a G is made runnable (newproc, ready). 1152 func wakep() { 1153 // be conservative about spinning threads 1154 if !cas(&sched.nmspinning, 0, 1) { 1155 return 1156 } 1157 startm(nil, true) 1158 } 1159 1160 // Stops execution of the current m that is locked to a g until the g is runnable again. 1161 // Returns with acquired P. 1162 func stoplockedm() { 1163 _g_ := getg() 1164 1165 if _g_.m.lockedg == nil || _g_.m.lockedg.lockedm != _g_.m { 1166 throw("stoplockedm: inconsistent locking") 1167 } 1168 if _g_.m.p != nil { 1169 // Schedule another M to run this p. 1170 _p_ := releasep() 1171 handoffp(_p_) 1172 } 1173 incidlelocked(1) 1174 // Wait until another thread schedules lockedg again. 1175 notesleep(&_g_.m.park) 1176 noteclear(&_g_.m.park) 1177 status := readgstatus(_g_.m.lockedg) 1178 if status&^_Gscan != _Grunnable { 1179 print("runtime:stoplockedm: g is not Grunnable or Gscanrunnable\n") 1180 dumpgstatus(_g_) 1181 throw("stoplockedm: not runnable") 1182 } 1183 acquirep(_g_.m.nextp) 1184 _g_.m.nextp = nil 1185 } 1186 1187 // Schedules the locked m to run the locked gp. 1188 // May run during STW, so write barriers are not allowed. 1189 //go:nowritebarrier 1190 func startlockedm(gp *g) { 1191 _g_ := getg() 1192 1193 mp := gp.lockedm 1194 if mp == _g_.m { 1195 throw("startlockedm: locked to me") 1196 } 1197 if mp.nextp != nil { 1198 throw("startlockedm: m has p") 1199 } 1200 // directly handoff current P to the locked m 1201 incidlelocked(-1) 1202 _p_ := releasep() 1203 // procresize made _p_ reachable through allp, which doesn't change during GC, so WB can be eliminated 1204 setPNoWriteBarrier(&mp.nextp, _p_) 1205 notewakeup(&mp.park) 1206 stopm() 1207 } 1208 1209 // Stops the current m for stoptheworld. 1210 // Returns when the world is restarted. 1211 func gcstopm() { 1212 _g_ := getg() 1213 1214 if sched.gcwaiting == 0 { 1215 throw("gcstopm: not waiting for gc") 1216 } 1217 if _g_.m.spinning { 1218 _g_.m.spinning = false 1219 xadd(&sched.nmspinning, -1) 1220 } 1221 _p_ := releasep() 1222 lock(&sched.lock) 1223 _p_.status = _Pgcstop 1224 sched.stopwait-- 1225 if sched.stopwait == 0 { 1226 notewakeup(&sched.stopnote) 1227 } 1228 unlock(&sched.lock) 1229 stopm() 1230 } 1231 1232 // Schedules gp to run on the current M. 1233 // Never returns. 1234 func execute(gp *g) { 1235 _g_ := getg() 1236 1237 casgstatus(gp, _Grunnable, _Grunning) 1238 gp.waitsince = 0 1239 gp.preempt = false 1240 gp.stackguard0 = gp.stack.lo + _StackGuard 1241 _g_.m.p.schedtick++ 1242 _g_.m.curg = gp 1243 gp.m = _g_.m 1244 1245 // Check whether the profiler needs to be turned on or off. 1246 hz := sched.profilehz 1247 if _g_.m.profilehz != hz { 1248 resetcpuprofiler(hz) 1249 } 1250 1251 if trace.enabled { 1252 traceGoStart() 1253 } 1254 1255 gogo(&gp.sched) 1256 } 1257 1258 // Finds a runnable goroutine to execute. 1259 // Tries to steal from other P's, get g from global queue, poll network. 1260 func findrunnable() *g { 1261 _g_ := getg() 1262 1263 top: 1264 if sched.gcwaiting != 0 { 1265 gcstopm() 1266 goto top 1267 } 1268 if fingwait && fingwake { 1269 if gp := wakefing(); gp != nil { 1270 ready(gp, 0) 1271 } 1272 } 1273 1274 // local runq 1275 if gp := runqget(_g_.m.p); gp != nil { 1276 return gp 1277 } 1278 1279 // global runq 1280 if sched.runqsize != 0 { 1281 lock(&sched.lock) 1282 gp := globrunqget(_g_.m.p, 0) 1283 unlock(&sched.lock) 1284 if gp != nil { 1285 return gp 1286 } 1287 } 1288 1289 // Poll network. 1290 // This netpoll is only an optimization before we resort to stealing. 1291 // We can safely skip it if there a thread blocked in netpoll already. 1292 // If there is any kind of logical race with that blocked thread 1293 // (e.g. it has already returned from netpoll, but does not set lastpoll yet), 1294 // this thread will do blocking netpoll below anyway. 1295 if netpollinited() && sched.lastpoll != 0 { 1296 if gp := netpoll(false); gp != nil { // non-blocking 1297 // netpoll returns list of goroutines linked by schedlink. 1298 injectglist(gp.schedlink) 1299 casgstatus(gp, _Gwaiting, _Grunnable) 1300 if trace.enabled { 1301 traceGoUnpark(gp, 0) 1302 } 1303 return gp 1304 } 1305 } 1306 1307 // If number of spinning M's >= number of busy P's, block. 1308 // This is necessary to prevent excessive CPU consumption 1309 // when GOMAXPROCS>>1 but the program parallelism is low. 1310 if !_g_.m.spinning && 2*atomicload(&sched.nmspinning) >= uint32(gomaxprocs)-atomicload(&sched.npidle) { // TODO: fast atomic 1311 goto stop 1312 } 1313 if !_g_.m.spinning { 1314 _g_.m.spinning = true 1315 xadd(&sched.nmspinning, 1) 1316 } 1317 // random steal from other P's 1318 for i := 0; i < int(2*gomaxprocs); i++ { 1319 if sched.gcwaiting != 0 { 1320 goto top 1321 } 1322 _p_ := allp[fastrand1()%uint32(gomaxprocs)] 1323 var gp *g 1324 if _p_ == _g_.m.p { 1325 gp = runqget(_p_) 1326 } else { 1327 gp = runqsteal(_g_.m.p, _p_) 1328 } 1329 if gp != nil { 1330 return gp 1331 } 1332 } 1333 stop: 1334 1335 // return P and block 1336 lock(&sched.lock) 1337 if sched.gcwaiting != 0 { 1338 unlock(&sched.lock) 1339 goto top 1340 } 1341 if sched.runqsize != 0 { 1342 gp := globrunqget(_g_.m.p, 0) 1343 unlock(&sched.lock) 1344 return gp 1345 } 1346 _p_ := releasep() 1347 pidleput(_p_) 1348 unlock(&sched.lock) 1349 if _g_.m.spinning { 1350 _g_.m.spinning = false 1351 xadd(&sched.nmspinning, -1) 1352 } 1353 1354 // check all runqueues once again 1355 for i := 0; i < int(gomaxprocs); i++ { 1356 _p_ := allp[i] 1357 if _p_ != nil && _p_.runqhead != _p_.runqtail { 1358 lock(&sched.lock) 1359 _p_ = pidleget() 1360 unlock(&sched.lock) 1361 if _p_ != nil { 1362 acquirep(_p_) 1363 goto top 1364 } 1365 break 1366 } 1367 } 1368 1369 // poll network 1370 if netpollinited() && xchg64(&sched.lastpoll, 0) != 0 { 1371 if _g_.m.p != nil { 1372 throw("findrunnable: netpoll with p") 1373 } 1374 if _g_.m.spinning { 1375 throw("findrunnable: netpoll with spinning") 1376 } 1377 gp := netpoll(true) // block until new work is available 1378 atomicstore64(&sched.lastpoll, uint64(nanotime())) 1379 if gp != nil { 1380 lock(&sched.lock) 1381 _p_ = pidleget() 1382 unlock(&sched.lock) 1383 if _p_ != nil { 1384 acquirep(_p_) 1385 injectglist(gp.schedlink) 1386 casgstatus(gp, _Gwaiting, _Grunnable) 1387 if trace.enabled { 1388 traceGoUnpark(gp, 0) 1389 } 1390 return gp 1391 } 1392 injectglist(gp) 1393 } 1394 } 1395 stopm() 1396 goto top 1397 } 1398 1399 func resetspinning() { 1400 _g_ := getg() 1401 1402 var nmspinning uint32 1403 if _g_.m.spinning { 1404 _g_.m.spinning = false 1405 nmspinning = xadd(&sched.nmspinning, -1) 1406 if nmspinning < 0 { 1407 throw("findrunnable: negative nmspinning") 1408 } 1409 } else { 1410 nmspinning = atomicload(&sched.nmspinning) 1411 } 1412 1413 // M wakeup policy is deliberately somewhat conservative (see nmspinning handling), 1414 // so see if we need to wakeup another P here. 1415 if nmspinning == 0 && atomicload(&sched.npidle) > 0 { 1416 wakep() 1417 } 1418 } 1419 1420 // Injects the list of runnable G's into the scheduler. 1421 // Can run concurrently with GC. 1422 func injectglist(glist *g) { 1423 if glist == nil { 1424 return 1425 } 1426 if trace.enabled { 1427 for gp := glist; gp != nil; gp = gp.schedlink { 1428 traceGoUnpark(gp, 0) 1429 } 1430 } 1431 lock(&sched.lock) 1432 var n int 1433 for n = 0; glist != nil; n++ { 1434 gp := glist 1435 glist = gp.schedlink 1436 casgstatus(gp, _Gwaiting, _Grunnable) 1437 globrunqput(gp) 1438 } 1439 unlock(&sched.lock) 1440 for ; n != 0 && sched.npidle != 0; n-- { 1441 startm(nil, false) 1442 } 1443 } 1444 1445 // One round of scheduler: find a runnable goroutine and execute it. 1446 // Never returns. 1447 func schedule() { 1448 _g_ := getg() 1449 1450 if _g_.m.locks != 0 { 1451 throw("schedule: holding locks") 1452 } 1453 1454 if _g_.m.lockedg != nil { 1455 stoplockedm() 1456 execute(_g_.m.lockedg) // Never returns. 1457 } 1458 1459 top: 1460 if sched.gcwaiting != 0 { 1461 gcstopm() 1462 goto top 1463 } 1464 1465 var gp *g 1466 if trace.enabled || trace.shutdown { 1467 gp = traceReader() 1468 if gp != nil { 1469 casgstatus(gp, _Gwaiting, _Grunnable) 1470 traceGoUnpark(gp, 0) 1471 resetspinning() 1472 } 1473 } 1474 if gp == nil { 1475 // Check the global runnable queue once in a while to ensure fairness. 1476 // Otherwise two goroutines can completely occupy the local runqueue 1477 // by constantly respawning each other. 1478 if _g_.m.p.schedtick%61 == 0 && sched.runqsize > 0 { 1479 lock(&sched.lock) 1480 gp = globrunqget(_g_.m.p, 1) 1481 unlock(&sched.lock) 1482 if gp != nil { 1483 resetspinning() 1484 } 1485 } 1486 } 1487 if gp == nil { 1488 gp = runqget(_g_.m.p) 1489 if gp != nil && _g_.m.spinning { 1490 throw("schedule: spinning with local work") 1491 } 1492 } 1493 if gp == nil { 1494 gp = findrunnable() // blocks until work is available 1495 resetspinning() 1496 } 1497 1498 if gp.lockedm != nil { 1499 // Hands off own p to the locked m, 1500 // then blocks waiting for a new p. 1501 startlockedm(gp) 1502 goto top 1503 } 1504 1505 execute(gp) 1506 } 1507 1508 // dropg removes the association between m and the current goroutine m->curg (gp for short). 1509 // Typically a caller sets gp's status away from Grunning and then 1510 // immediately calls dropg to finish the job. The caller is also responsible 1511 // for arranging that gp will be restarted using ready at an 1512 // appropriate time. After calling dropg and arranging for gp to be 1513 // readied later, the caller can do other work but eventually should 1514 // call schedule to restart the scheduling of goroutines on this m. 1515 func dropg() { 1516 _g_ := getg() 1517 1518 if _g_.m.lockedg == nil { 1519 _g_.m.curg.m = nil 1520 _g_.m.curg = nil 1521 } 1522 } 1523 1524 func parkunlock_c(gp *g, lock unsafe.Pointer) bool { 1525 unlock((*mutex)(lock)) 1526 return true 1527 } 1528 1529 // park continuation on g0. 1530 func park_m(gp *g) { 1531 _g_ := getg() 1532 1533 if trace.enabled { 1534 traceGoPark(_g_.m.waittraceev, _g_.m.waittraceskip, gp) 1535 } 1536 1537 casgstatus(gp, _Grunning, _Gwaiting) 1538 dropg() 1539 1540 if _g_.m.waitunlockf != nil { 1541 fn := *(*func(*g, unsafe.Pointer) bool)(unsafe.Pointer(&_g_.m.waitunlockf)) 1542 ok := fn(gp, _g_.m.waitlock) 1543 _g_.m.waitunlockf = nil 1544 _g_.m.waitlock = nil 1545 if !ok { 1546 if trace.enabled { 1547 traceGoUnpark(gp, 2) 1548 } 1549 casgstatus(gp, _Gwaiting, _Grunnable) 1550 execute(gp) // Schedule it back, never returns. 1551 } 1552 } 1553 schedule() 1554 } 1555 1556 func goschedImpl(gp *g) { 1557 status := readgstatus(gp) 1558 if status&^_Gscan != _Grunning { 1559 dumpgstatus(gp) 1560 throw("bad g status") 1561 } 1562 casgstatus(gp, _Grunning, _Grunnable) 1563 dropg() 1564 lock(&sched.lock) 1565 globrunqput(gp) 1566 unlock(&sched.lock) 1567 1568 schedule() 1569 } 1570 1571 // Gosched continuation on g0. 1572 func gosched_m(gp *g) { 1573 if trace.enabled { 1574 traceGoSched() 1575 } 1576 goschedImpl(gp) 1577 } 1578 1579 func gopreempt_m(gp *g) { 1580 if trace.enabled { 1581 traceGoPreempt() 1582 } 1583 goschedImpl(gp) 1584 } 1585 1586 // Finishes execution of the current goroutine. 1587 func goexit1() { 1588 if raceenabled { 1589 racegoend() 1590 } 1591 if trace.enabled { 1592 traceGoEnd() 1593 } 1594 mcall(goexit0) 1595 } 1596 1597 // goexit continuation on g0. 1598 func goexit0(gp *g) { 1599 _g_ := getg() 1600 1601 casgstatus(gp, _Grunning, _Gdead) 1602 gp.m = nil 1603 gp.lockedm = nil 1604 _g_.m.lockedg = nil 1605 gp.paniconfault = false 1606 gp._defer = nil // should be true already but just in case. 1607 gp._panic = nil // non-nil for Goexit during panic. points at stack-allocated data. 1608 gp.writebuf = nil 1609 gp.waitreason = "" 1610 gp.param = nil 1611 1612 dropg() 1613 1614 if _g_.m.locked&^_LockExternal != 0 { 1615 print("invalid m->locked = ", _g_.m.locked, "\n") 1616 throw("internal lockOSThread error") 1617 } 1618 _g_.m.locked = 0 1619 gfput(_g_.m.p, gp) 1620 schedule() 1621 } 1622 1623 //go:nosplit 1624 //go:nowritebarrier 1625 func save(pc, sp uintptr) { 1626 _g_ := getg() 1627 1628 _g_.sched.pc = pc 1629 _g_.sched.sp = sp 1630 _g_.sched.lr = 0 1631 _g_.sched.ret = 0 1632 _g_.sched.ctxt = nil 1633 _g_.sched.g = guintptr(unsafe.Pointer(_g_)) 1634 } 1635 1636 // The goroutine g is about to enter a system call. 1637 // Record that it's not using the cpu anymore. 1638 // This is called only from the go syscall library and cgocall, 1639 // not from the low-level system calls used by the 1640 // 1641 // Entersyscall cannot split the stack: the gosave must 1642 // make g->sched refer to the caller's stack segment, because 1643 // entersyscall is going to return immediately after. 1644 // 1645 // Nothing entersyscall calls can split the stack either. 1646 // We cannot safely move the stack during an active call to syscall, 1647 // because we do not know which of the uintptr arguments are 1648 // really pointers (back into the stack). 1649 // In practice, this means that we make the fast path run through 1650 // entersyscall doing no-split things, and the slow path has to use systemstack 1651 // to run bigger things on the system stack. 1652 // 1653 // reentersyscall is the entry point used by cgo callbacks, where explicitly 1654 // saved SP and PC are restored. This is needed when exitsyscall will be called 1655 // from a function further up in the call stack than the parent, as g->syscallsp 1656 // must always point to a valid stack frame. entersyscall below is the normal 1657 // entry point for syscalls, which obtains the SP and PC from the caller. 1658 // 1659 // Syscall tracing: 1660 // At the start of a syscall we emit traceGoSysCall to capture the stack trace. 1661 // If the syscall does not block, that is it, we do not emit any other events. 1662 // If the syscall blocks (that is, P is retaken), retaker emits traceGoSysBlock; 1663 // when syscall returns we emit traceGoSysExit and when the goroutine starts running 1664 // (potentially instantly, if exitsyscallfast returns true) we emit traceGoStart. 1665 // To ensure that traceGoSysExit is emitted strictly after traceGoSysBlock, 1666 // we remember current value of syscalltick in m (_g_.m.syscalltick = _g_.m.p.syscalltick), 1667 // whoever emits traceGoSysBlock increments p.syscalltick afterwards; 1668 // and we wait for the increment before emitting traceGoSysExit. 1669 // Note that the increment is done even if tracing is not enabled, 1670 // because tracing can be enabled in the middle of syscall. We don't want the wait to hang. 1671 // 1672 //go:nosplit 1673 func reentersyscall(pc, sp uintptr) { 1674 _g_ := getg() 1675 1676 // Disable preemption because during this function g is in Gsyscall status, 1677 // but can have inconsistent g->sched, do not let GC observe it. 1678 _g_.m.locks++ 1679 1680 if trace.enabled { 1681 systemstack(traceGoSysCall) 1682 } 1683 1684 // Entersyscall must not call any function that might split/grow the stack. 1685 // (See details in comment above.) 1686 // Catch calls that might, by replacing the stack guard with something that 1687 // will trip any stack check and leaving a flag to tell newstack to die. 1688 _g_.stackguard0 = stackPreempt 1689 _g_.throwsplit = true 1690 1691 // Leave SP around for GC and traceback. 1692 save(pc, sp) 1693 _g_.syscallsp = sp 1694 _g_.syscallpc = pc 1695 casgstatus(_g_, _Grunning, _Gsyscall) 1696 if _g_.syscallsp < _g_.stack.lo || _g_.stack.hi < _g_.syscallsp { 1697 systemstack(func() { 1698 print("entersyscall inconsistent ", hex(_g_.syscallsp), " [", hex(_g_.stack.lo), ",", hex(_g_.stack.hi), "]\n") 1699 throw("entersyscall") 1700 }) 1701 } 1702 1703 if atomicload(&sched.sysmonwait) != 0 { // TODO: fast atomic 1704 systemstack(entersyscall_sysmon) 1705 save(pc, sp) 1706 } 1707 1708 _g_.m.syscalltick = _g_.m.p.syscalltick 1709 _g_.m.mcache = nil 1710 _g_.m.p.m = nil 1711 atomicstore(&_g_.m.p.status, _Psyscall) 1712 if sched.gcwaiting != 0 { 1713 systemstack(entersyscall_gcwait) 1714 save(pc, sp) 1715 } 1716 1717 // Goroutines must not split stacks in Gsyscall status (it would corrupt g->sched). 1718 // We set _StackGuard to StackPreempt so that first split stack check calls morestack. 1719 // Morestack detects this case and throws. 1720 _g_.stackguard0 = stackPreempt 1721 _g_.m.locks-- 1722 } 1723 1724 // Standard syscall entry used by the go syscall library and normal cgo calls. 1725 //go:nosplit 1726 func entersyscall(dummy int32) { 1727 reentersyscall(getcallerpc(unsafe.Pointer(&dummy)), getcallersp(unsafe.Pointer(&dummy))) 1728 } 1729 1730 func entersyscall_sysmon() { 1731 lock(&sched.lock) 1732 if atomicload(&sched.sysmonwait) != 0 { 1733 atomicstore(&sched.sysmonwait, 0) 1734 notewakeup(&sched.sysmonnote) 1735 } 1736 unlock(&sched.lock) 1737 } 1738 1739 func entersyscall_gcwait() { 1740 _g_ := getg() 1741 _p_ := _g_.m.p 1742 1743 lock(&sched.lock) 1744 if sched.stopwait > 0 && cas(&_p_.status, _Psyscall, _Pgcstop) { 1745 if trace.enabled { 1746 traceGoSysBlock(_p_) 1747 traceProcStop(_p_) 1748 } 1749 _p_.syscalltick++ 1750 if sched.stopwait--; sched.stopwait == 0 { 1751 notewakeup(&sched.stopnote) 1752 } 1753 } 1754 unlock(&sched.lock) 1755 } 1756 1757 // The same as entersyscall(), but with a hint that the syscall is blocking. 1758 //go:nosplit 1759 func entersyscallblock(dummy int32) { 1760 _g_ := getg() 1761 1762 _g_.m.locks++ // see comment in entersyscall 1763 _g_.throwsplit = true 1764 _g_.stackguard0 = stackPreempt // see comment in entersyscall 1765 _g_.m.syscalltick = _g_.m.p.syscalltick 1766 _g_.m.p.syscalltick++ 1767 1768 // Leave SP around for GC and traceback. 1769 pc := getcallerpc(unsafe.Pointer(&dummy)) 1770 sp := getcallersp(unsafe.Pointer(&dummy)) 1771 save(pc, sp) 1772 _g_.syscallsp = _g_.sched.sp 1773 _g_.syscallpc = _g_.sched.pc 1774 if _g_.syscallsp < _g_.stack.lo || _g_.stack.hi < _g_.syscallsp { 1775 sp1 := sp 1776 sp2 := _g_.sched.sp 1777 sp3 := _g_.syscallsp 1778 systemstack(func() { 1779 print("entersyscallblock inconsistent ", hex(sp1), " ", hex(sp2), " ", hex(sp3), " [", hex(_g_.stack.lo), ",", hex(_g_.stack.hi), "]\n") 1780 throw("entersyscallblock") 1781 }) 1782 } 1783 casgstatus(_g_, _Grunning, _Gsyscall) 1784 if _g_.syscallsp < _g_.stack.lo || _g_.stack.hi < _g_.syscallsp { 1785 systemstack(func() { 1786 print("entersyscallblock inconsistent ", hex(sp), " ", hex(_g_.sched.sp), " ", hex(_g_.syscallsp), " [", hex(_g_.stack.lo), ",", hex(_g_.stack.hi), "]\n") 1787 throw("entersyscallblock") 1788 }) 1789 } 1790 1791 systemstack(entersyscallblock_handoff) 1792 1793 // Resave for traceback during blocked call. 1794 save(getcallerpc(unsafe.Pointer(&dummy)), getcallersp(unsafe.Pointer(&dummy))) 1795 1796 _g_.m.locks-- 1797 } 1798 1799 func entersyscallblock_handoff() { 1800 if trace.enabled { 1801 traceGoSysCall() 1802 traceGoSysBlock(getg().m.p) 1803 } 1804 handoffp(releasep()) 1805 } 1806 1807 // The goroutine g exited its system call. 1808 // Arrange for it to run on a cpu again. 1809 // This is called only from the go syscall library, not 1810 // from the low-level system calls used by the 1811 //go:nosplit 1812 func exitsyscall(dummy int32) { 1813 _g_ := getg() 1814 1815 _g_.m.locks++ // see comment in entersyscall 1816 if getcallersp(unsafe.Pointer(&dummy)) > _g_.syscallsp { 1817 throw("exitsyscall: syscall frame is no longer valid") 1818 } 1819 1820 _g_.waitsince = 0 1821 oldp := _g_.m.p 1822 if exitsyscallfast() { 1823 if _g_.m.mcache == nil { 1824 throw("lost mcache") 1825 } 1826 if trace.enabled { 1827 if oldp != _g_.m.p || _g_.m.syscalltick != _g_.m.p.syscalltick { 1828 systemstack(traceGoStart) 1829 } 1830 } 1831 // There's a cpu for us, so we can run. 1832 _g_.m.p.syscalltick++ 1833 // We need to cas the status and scan before resuming... 1834 casgstatus(_g_, _Gsyscall, _Grunning) 1835 1836 // Garbage collector isn't running (since we are), 1837 // so okay to clear syscallsp. 1838 _g_.syscallsp = 0 1839 _g_.m.locks-- 1840 if _g_.preempt { 1841 // restore the preemption request in case we've cleared it in newstack 1842 _g_.stackguard0 = stackPreempt 1843 } else { 1844 // otherwise restore the real _StackGuard, we've spoiled it in entersyscall/entersyscallblock 1845 _g_.stackguard0 = _g_.stack.lo + _StackGuard 1846 } 1847 _g_.throwsplit = false 1848 return 1849 } 1850 1851 var exitTicks int64 1852 if trace.enabled { 1853 // Wait till traceGoSysBlock event is emited. 1854 // This ensures consistency of the trace (the goroutine is started after it is blocked). 1855 for oldp != nil && oldp.syscalltick == _g_.m.syscalltick { 1856 osyield() 1857 } 1858 // We can't trace syscall exit right now because we don't have a P. 1859 // Tracing code can invoke write barriers that cannot run without a P. 1860 // So instead we remember the syscall exit time and emit the event 1861 // below when we have a P. 1862 exitTicks = cputicks() 1863 } 1864 1865 _g_.m.locks-- 1866 1867 // Call the scheduler. 1868 mcall(exitsyscall0) 1869 1870 // The goroutine must not be re-scheduled up to traceGoSysExit. 1871 // Otherwise we can emit GoStart but not GoSysExit, that would lead 1872 // no an inconsistent trace. 1873 _g_.m.locks++ 1874 1875 if _g_.m.mcache == nil { 1876 throw("lost mcache") 1877 } 1878 1879 // Scheduler returned, so we're allowed to run now. 1880 // Delete the syscallsp information that we left for 1881 // the garbage collector during the system call. 1882 // Must wait until now because until gosched returns 1883 // we don't know for sure that the garbage collector 1884 // is not running. 1885 _g_.syscallsp = 0 1886 _g_.m.p.syscalltick++ 1887 _g_.throwsplit = false 1888 1889 if exitTicks != 0 { 1890 systemstack(func() { 1891 traceGoSysExit(exitTicks) 1892 }) 1893 } 1894 _g_.m.locks-- 1895 } 1896 1897 //go:nosplit 1898 func exitsyscallfast() bool { 1899 _g_ := getg() 1900 1901 // Freezetheworld sets stopwait but does not retake P's. 1902 if sched.stopwait == freezeStopWait { 1903 _g_.m.mcache = nil 1904 _g_.m.p = nil 1905 return false 1906 } 1907 1908 // Try to re-acquire the last P. 1909 if _g_.m.p != nil && _g_.m.p.status == _Psyscall && cas(&_g_.m.p.status, _Psyscall, _Prunning) { 1910 // There's a cpu for us, so we can run. 1911 _g_.m.mcache = _g_.m.p.mcache 1912 _g_.m.p.m = _g_.m 1913 if _g_.m.syscalltick != _g_.m.p.syscalltick { 1914 if trace.enabled { 1915 // The p was retaken and then enter into syscall again (since _g_.m.syscalltick has changed). 1916 // traceGoSysBlock for this syscall was already emitted, 1917 // but here we effectively retake the p from the new syscall running on the same p. 1918 systemstack(func() { 1919 // Denote blocking of the new syscall. 1920 traceGoSysBlock(_g_.m.p) 1921 // Denote completion of the current syscall. 1922 traceGoSysExit(0) 1923 }) 1924 } 1925 _g_.m.p.syscalltick++ 1926 } 1927 return true 1928 } 1929 1930 // Try to get any other idle P. 1931 oldp := _g_.m.p 1932 _g_.m.mcache = nil 1933 _g_.m.p = nil 1934 if sched.pidle != nil { 1935 var ok bool 1936 systemstack(func() { 1937 ok = exitsyscallfast_pidle() 1938 if ok && trace.enabled { 1939 if oldp != nil { 1940 // Wait till traceGoSysBlock event is emited. 1941 // This ensures consistency of the trace (the goroutine is started after it is blocked). 1942 for oldp.syscalltick == _g_.m.syscalltick { 1943 osyield() 1944 } 1945 } 1946 traceGoSysExit(0) 1947 } 1948 }) 1949 if ok { 1950 return true 1951 } 1952 } 1953 return false 1954 } 1955 1956 func exitsyscallfast_pidle() bool { 1957 lock(&sched.lock) 1958 _p_ := pidleget() 1959 if _p_ != nil && atomicload(&sched.sysmonwait) != 0 { 1960 atomicstore(&sched.sysmonwait, 0) 1961 notewakeup(&sched.sysmonnote) 1962 } 1963 unlock(&sched.lock) 1964 if _p_ != nil { 1965 acquirep(_p_) 1966 return true 1967 } 1968 return false 1969 } 1970 1971 // exitsyscall slow path on g0. 1972 // Failed to acquire P, enqueue gp as runnable. 1973 func exitsyscall0(gp *g) { 1974 _g_ := getg() 1975 1976 casgstatus(gp, _Gsyscall, _Grunnable) 1977 dropg() 1978 lock(&sched.lock) 1979 _p_ := pidleget() 1980 if _p_ == nil { 1981 globrunqput(gp) 1982 } else if atomicload(&sched.sysmonwait) != 0 { 1983 atomicstore(&sched.sysmonwait, 0) 1984 notewakeup(&sched.sysmonnote) 1985 } 1986 unlock(&sched.lock) 1987 if _p_ != nil { 1988 acquirep(_p_) 1989 execute(gp) // Never returns. 1990 } 1991 if _g_.m.lockedg != nil { 1992 // Wait until another thread schedules gp and so m again. 1993 stoplockedm() 1994 execute(gp) // Never returns. 1995 } 1996 stopm() 1997 schedule() // Never returns. 1998 } 1999 2000 func beforefork() { 2001 gp := getg().m.curg 2002 2003 // Fork can hang if preempted with signals frequently enough (see issue 5517). 2004 // Ensure that we stay on the same M where we disable profiling. 2005 gp.m.locks++ 2006 if gp.m.profilehz != 0 { 2007 resetcpuprofiler(0) 2008 } 2009 2010 // This function is called before fork in syscall package. 2011 // Code between fork and exec must not allocate memory nor even try to grow stack. 2012 // Here we spoil g->_StackGuard to reliably detect any attempts to grow stack. 2013 // runtime_AfterFork will undo this in parent process, but not in child. 2014 gp.stackguard0 = stackFork 2015 } 2016 2017 // Called from syscall package before fork. 2018 //go:linkname syscall_runtime_BeforeFork syscall.runtime_BeforeFork 2019 //go:nosplit 2020 func syscall_runtime_BeforeFork() { 2021 systemstack(beforefork) 2022 } 2023 2024 func afterfork() { 2025 gp := getg().m.curg 2026 2027 // See the comment in beforefork. 2028 gp.stackguard0 = gp.stack.lo + _StackGuard 2029 2030 hz := sched.profilehz 2031 if hz != 0 { 2032 resetcpuprofiler(hz) 2033 } 2034 gp.m.locks-- 2035 } 2036 2037 // Called from syscall package after fork in parent. 2038 //go:linkname syscall_runtime_AfterFork syscall.runtime_AfterFork 2039 //go:nosplit 2040 func syscall_runtime_AfterFork() { 2041 systemstack(afterfork) 2042 } 2043 2044 // Allocate a new g, with a stack big enough for stacksize bytes. 2045 func malg(stacksize int32) *g { 2046 newg := new(g) 2047 if stacksize >= 0 { 2048 stacksize = round2(_StackSystem + stacksize) 2049 systemstack(func() { 2050 newg.stack = stackalloc(uint32(stacksize)) 2051 }) 2052 newg.stackguard0 = newg.stack.lo + _StackGuard 2053 newg.stackguard1 = ^uintptr(0) 2054 } 2055 return newg 2056 } 2057 2058 // Create a new g running fn with siz bytes of arguments. 2059 // Put it on the queue of g's waiting to run. 2060 // The compiler turns a go statement into a call to this. 2061 // Cannot split the stack because it assumes that the arguments 2062 // are available sequentially after &fn; they would not be 2063 // copied if a stack split occurred. 2064 //go:nosplit 2065 func newproc(siz int32, fn *funcval) { 2066 argp := add(unsafe.Pointer(&fn), ptrSize) 2067 pc := getcallerpc(unsafe.Pointer(&siz)) 2068 systemstack(func() { 2069 newproc1(fn, (*uint8)(argp), siz, 0, pc) 2070 }) 2071 } 2072 2073 // Create a new g running fn with narg bytes of arguments starting 2074 // at argp and returning nret bytes of results. callerpc is the 2075 // address of the go statement that created this. The new g is put 2076 // on the queue of g's waiting to run. 2077 func newproc1(fn *funcval, argp *uint8, narg int32, nret int32, callerpc uintptr) *g { 2078 _g_ := getg() 2079 2080 if fn == nil { 2081 _g_.m.throwing = -1 // do not dump full stacks 2082 throw("go of nil func value") 2083 } 2084 _g_.m.locks++ // disable preemption because it can be holding p in a local var 2085 siz := narg + nret 2086 siz = (siz + 7) &^ 7 2087 2088 // We could allocate a larger initial stack if necessary. 2089 // Not worth it: this is almost always an error. 2090 // 4*sizeof(uintreg): extra space added below 2091 // sizeof(uintreg): caller's LR (arm) or return address (x86, in gostartcall). 2092 if siz >= _StackMin-4*regSize-regSize { 2093 throw("newproc: function arguments too large for new goroutine") 2094 } 2095 2096 _p_ := _g_.m.p 2097 newg := gfget(_p_) 2098 if newg == nil { 2099 newg = malg(_StackMin) 2100 casgstatus(newg, _Gidle, _Gdead) 2101 allgadd(newg) // publishes with a g->status of Gdead so GC scanner doesn't look at uninitialized stack. 2102 } 2103 if newg.stack.hi == 0 { 2104 throw("newproc1: newg missing stack") 2105 } 2106 2107 if readgstatus(newg) != _Gdead { 2108 throw("newproc1: new g is not Gdead") 2109 } 2110 2111 totalSize := 4*regSize + uintptr(siz) // extra space in case of reads slightly beyond frame 2112 if hasLinkRegister { 2113 totalSize += ptrSize 2114 } 2115 totalSize += -totalSize & (spAlign - 1) // align to spAlign 2116 sp := newg.stack.hi - totalSize 2117 spArg := sp 2118 if hasLinkRegister { 2119 // caller's LR 2120 *(*unsafe.Pointer)(unsafe.Pointer(sp)) = nil 2121 spArg += ptrSize 2122 } 2123 memmove(unsafe.Pointer(spArg), unsafe.Pointer(argp), uintptr(narg)) 2124 2125 memclr(unsafe.Pointer(&newg.sched), unsafe.Sizeof(newg.sched)) 2126 newg.sched.sp = sp 2127 newg.sched.pc = funcPC(goexit) + _PCQuantum // +PCQuantum so that previous instruction is in same function 2128 newg.sched.g = guintptr(unsafe.Pointer(newg)) 2129 gostartcallfn(&newg.sched, fn) 2130 newg.gopc = callerpc 2131 newg.startpc = fn.fn 2132 casgstatus(newg, _Gdead, _Grunnable) 2133 2134 if _p_.goidcache == _p_.goidcacheend { 2135 // Sched.goidgen is the last allocated id, 2136 // this batch must be [sched.goidgen+1, sched.goidgen+GoidCacheBatch]. 2137 // At startup sched.goidgen=0, so main goroutine receives goid=1. 2138 _p_.goidcache = xadd64(&sched.goidgen, _GoidCacheBatch) 2139 _p_.goidcache -= _GoidCacheBatch - 1 2140 _p_.goidcacheend = _p_.goidcache + _GoidCacheBatch 2141 } 2142 newg.goid = int64(_p_.goidcache) 2143 _p_.goidcache++ 2144 if raceenabled { 2145 newg.racectx = racegostart(callerpc) 2146 } 2147 if trace.enabled { 2148 traceGoCreate(newg, newg.startpc) 2149 } 2150 runqput(_p_, newg) 2151 2152 if atomicload(&sched.npidle) != 0 && atomicload(&sched.nmspinning) == 0 && unsafe.Pointer(fn.fn) != unsafe.Pointer(funcPC(main)) { // TODO: fast atomic 2153 wakep() 2154 } 2155 _g_.m.locks-- 2156 if _g_.m.locks == 0 && _g_.preempt { // restore the preemption request in case we've cleared it in newstack 2157 _g_.stackguard0 = stackPreempt 2158 } 2159 return newg 2160 } 2161 2162 // Put on gfree list. 2163 // If local list is too long, transfer a batch to the global list. 2164 func gfput(_p_ *p, gp *g) { 2165 if readgstatus(gp) != _Gdead { 2166 throw("gfput: bad status (not Gdead)") 2167 } 2168 2169 stksize := gp.stack.hi - gp.stack.lo 2170 2171 if stksize != _FixedStack { 2172 // non-standard stack size - free it. 2173 stackfree(gp.stack) 2174 gp.stack.lo = 0 2175 gp.stack.hi = 0 2176 gp.stackguard0 = 0 2177 } 2178 2179 gp.schedlink = _p_.gfree 2180 _p_.gfree = gp 2181 _p_.gfreecnt++ 2182 if _p_.gfreecnt >= 64 { 2183 lock(&sched.gflock) 2184 for _p_.gfreecnt >= 32 { 2185 _p_.gfreecnt-- 2186 gp = _p_.gfree 2187 _p_.gfree = gp.schedlink 2188 gp.schedlink = sched.gfree 2189 sched.gfree = gp 2190 sched.ngfree++ 2191 } 2192 unlock(&sched.gflock) 2193 } 2194 } 2195 2196 // Get from gfree list. 2197 // If local list is empty, grab a batch from global list. 2198 func gfget(_p_ *p) *g { 2199 retry: 2200 gp := _p_.gfree 2201 if gp == nil && sched.gfree != nil { 2202 lock(&sched.gflock) 2203 for _p_.gfreecnt < 32 && sched.gfree != nil { 2204 _p_.gfreecnt++ 2205 gp = sched.gfree 2206 sched.gfree = gp.schedlink 2207 sched.ngfree-- 2208 gp.schedlink = _p_.gfree 2209 _p_.gfree = gp 2210 } 2211 unlock(&sched.gflock) 2212 goto retry 2213 } 2214 if gp != nil { 2215 _p_.gfree = gp.schedlink 2216 _p_.gfreecnt-- 2217 if gp.stack.lo == 0 { 2218 // Stack was deallocated in gfput. Allocate a new one. 2219 systemstack(func() { 2220 gp.stack = stackalloc(_FixedStack) 2221 }) 2222 gp.stackguard0 = gp.stack.lo + _StackGuard 2223 } else { 2224 if raceenabled { 2225 racemalloc(unsafe.Pointer(gp.stack.lo), gp.stack.hi-gp.stack.lo) 2226 } 2227 } 2228 } 2229 return gp 2230 } 2231 2232 // Purge all cached G's from gfree list to the global list. 2233 func gfpurge(_p_ *p) { 2234 lock(&sched.gflock) 2235 for _p_.gfreecnt != 0 { 2236 _p_.gfreecnt-- 2237 gp := _p_.gfree 2238 _p_.gfree = gp.schedlink 2239 gp.schedlink = sched.gfree 2240 sched.gfree = gp 2241 sched.ngfree++ 2242 } 2243 unlock(&sched.gflock) 2244 } 2245 2246 // Breakpoint executes a breakpoint trap. 2247 func Breakpoint() { 2248 breakpoint() 2249 } 2250 2251 // dolockOSThread is called by LockOSThread and lockOSThread below 2252 // after they modify m.locked. Do not allow preemption during this call, 2253 // or else the m might be different in this function than in the caller. 2254 //go:nosplit 2255 func dolockOSThread() { 2256 _g_ := getg() 2257 _g_.m.lockedg = _g_ 2258 _g_.lockedm = _g_.m 2259 } 2260 2261 //go:nosplit 2262 2263 // LockOSThread wires the calling goroutine to its current operating system thread. 2264 // Until the calling goroutine exits or calls UnlockOSThread, it will always 2265 // execute in that thread, and no other goroutine can. 2266 func LockOSThread() { 2267 getg().m.locked |= _LockExternal 2268 dolockOSThread() 2269 } 2270 2271 //go:nosplit 2272 func lockOSThread() { 2273 getg().m.locked += _LockInternal 2274 dolockOSThread() 2275 } 2276 2277 // dounlockOSThread is called by UnlockOSThread and unlockOSThread below 2278 // after they update m->locked. Do not allow preemption during this call, 2279 // or else the m might be in different in this function than in the caller. 2280 //go:nosplit 2281 func dounlockOSThread() { 2282 _g_ := getg() 2283 if _g_.m.locked != 0 { 2284 return 2285 } 2286 _g_.m.lockedg = nil 2287 _g_.lockedm = nil 2288 } 2289 2290 //go:nosplit 2291 2292 // UnlockOSThread unwires the calling goroutine from its fixed operating system thread. 2293 // If the calling goroutine has not called LockOSThread, UnlockOSThread is a no-op. 2294 func UnlockOSThread() { 2295 getg().m.locked &^= _LockExternal 2296 dounlockOSThread() 2297 } 2298 2299 //go:nosplit 2300 func unlockOSThread() { 2301 _g_ := getg() 2302 if _g_.m.locked < _LockInternal { 2303 systemstack(badunlockosthread) 2304 } 2305 _g_.m.locked -= _LockInternal 2306 dounlockOSThread() 2307 } 2308 2309 func badunlockosthread() { 2310 throw("runtime: internal error: misuse of lockOSThread/unlockOSThread") 2311 } 2312 2313 func gcount() int32 { 2314 n := int32(allglen) - sched.ngfree 2315 for i := 0; ; i++ { 2316 _p_ := allp[i] 2317 if _p_ == nil { 2318 break 2319 } 2320 n -= _p_.gfreecnt 2321 } 2322 2323 // All these variables can be changed concurrently, so the result can be inconsistent. 2324 // But at least the current goroutine is running. 2325 if n < 1 { 2326 n = 1 2327 } 2328 return n 2329 } 2330 2331 func mcount() int32 { 2332 return sched.mcount 2333 } 2334 2335 var prof struct { 2336 lock uint32 2337 hz int32 2338 } 2339 2340 func _System() { _System() } 2341 func _ExternalCode() { _ExternalCode() } 2342 func _GC() { _GC() } 2343 2344 // Called if we receive a SIGPROF signal. 2345 func sigprof(pc, sp, lr uintptr, gp *g, mp *m) { 2346 if prof.hz == 0 { 2347 return 2348 } 2349 2350 // Profiling runs concurrently with GC, so it must not allocate. 2351 mp.mallocing++ 2352 2353 // Define that a "user g" is a user-created goroutine, and a "system g" 2354 // is one that is m->g0 or m->gsignal. We've only made sure that we 2355 // can unwind user g's, so exclude the system g's. 2356 // 2357 // It is not quite as easy as testing gp == m->curg (the current user g) 2358 // because we might be interrupted for profiling halfway through a 2359 // goroutine switch. The switch involves updating three (or four) values: 2360 // g, PC, SP, and (on arm) LR. The PC must be the last to be updated, 2361 // because once it gets updated the new g is running. 2362 // 2363 // When switching from a user g to a system g, LR is not considered live, 2364 // so the update only affects g, SP, and PC. Since PC must be last, there 2365 // the possible partial transitions in ordinary execution are (1) g alone is updated, 2366 // (2) both g and SP are updated, and (3) SP alone is updated. 2367 // If g is updated, we'll see a system g and not look closer. 2368 // If SP alone is updated, we can detect the partial transition by checking 2369 // whether the SP is within g's stack bounds. (We could also require that SP 2370 // be changed only after g, but the stack bounds check is needed by other 2371 // cases, so there is no need to impose an additional requirement.) 2372 // 2373 // There is one exceptional transition to a system g, not in ordinary execution. 2374 // When a signal arrives, the operating system starts the signal handler running 2375 // with an updated PC and SP. The g is updated last, at the beginning of the 2376 // handler. There are two reasons this is okay. First, until g is updated the 2377 // g and SP do not match, so the stack bounds check detects the partial transition. 2378 // Second, signal handlers currently run with signals disabled, so a profiling 2379 // signal cannot arrive during the handler. 2380 // 2381 // When switching from a system g to a user g, there are three possibilities. 2382 // 2383 // First, it may be that the g switch has no PC update, because the SP 2384 // either corresponds to a user g throughout (as in asmcgocall) 2385 // or because it has been arranged to look like a user g frame 2386 // (as in cgocallback_gofunc). In this case, since the entire 2387 // transition is a g+SP update, a partial transition updating just one of 2388 // those will be detected by the stack bounds check. 2389 // 2390 // Second, when returning from a signal handler, the PC and SP updates 2391 // are performed by the operating system in an atomic update, so the g 2392 // update must be done before them. The stack bounds check detects 2393 // the partial transition here, and (again) signal handlers run with signals 2394 // disabled, so a profiling signal cannot arrive then anyway. 2395 // 2396 // Third, the common case: it may be that the switch updates g, SP, and PC 2397 // separately, as in gogo. 2398 // 2399 // Because gogo is the only instance, we check whether the PC lies 2400 // within that function, and if so, not ask for a traceback. This approach 2401 // requires knowing the size of the gogo function, which we 2402 // record in arch_*.h and check in runtime_test.go. 2403 // 2404 // There is another apparently viable approach, recorded here in case 2405 // the "PC within gogo" check turns out not to be usable. 2406 // It would be possible to delay the update of either g or SP until immediately 2407 // before the PC update instruction. Then, because of the stack bounds check, 2408 // the only problematic interrupt point is just before that PC update instruction, 2409 // and the sigprof handler can detect that instruction and simulate stepping past 2410 // it in order to reach a consistent state. On ARM, the update of g must be made 2411 // in two places (in R10 and also in a TLS slot), so the delayed update would 2412 // need to be the SP update. The sigprof handler must read the instruction at 2413 // the current PC and if it was the known instruction (for example, JMP BX or 2414 // MOV R2, PC), use that other register in place of the PC value. 2415 // The biggest drawback to this solution is that it requires that we can tell 2416 // whether it's safe to read from the memory pointed at by PC. 2417 // In a correct program, we can test PC == nil and otherwise read, 2418 // but if a profiling signal happens at the instant that a program executes 2419 // a bad jump (before the program manages to handle the resulting fault) 2420 // the profiling handler could fault trying to read nonexistent memory. 2421 // 2422 // To recap, there are no constraints on the assembly being used for the 2423 // transition. We simply require that g and SP match and that the PC is not 2424 // in gogo. 2425 traceback := true 2426 gogo := funcPC(gogo) 2427 if gp == nil || gp != mp.curg || 2428 sp < gp.stack.lo || gp.stack.hi < sp || 2429 (gogo <= pc && pc < gogo+_RuntimeGogoBytes) { 2430 traceback = false 2431 } 2432 2433 var stk [maxCPUProfStack]uintptr 2434 n := 0 2435 if traceback { 2436 n = gentraceback(pc, sp, lr, gp, 0, &stk[0], len(stk), nil, nil, _TraceTrap) 2437 } 2438 if !traceback || n <= 0 { 2439 // Normal traceback is impossible or has failed. 2440 // See if it falls into several common cases. 2441 n = 0 2442 if mp.ncgo > 0 && mp.curg != nil && mp.curg.syscallpc != 0 && mp.curg.syscallsp != 0 { 2443 // Cgo, we can't unwind and symbolize arbitrary C code, 2444 // so instead collect Go stack that leads to the cgo call. 2445 // This is especially important on windows, since all syscalls are cgo calls. 2446 n = gentraceback(mp.curg.syscallpc, mp.curg.syscallsp, 0, mp.curg, 0, &stk[0], len(stk), nil, nil, 0) 2447 } 2448 if GOOS == "windows" && n == 0 && mp.libcallg != nil && mp.libcallpc != 0 && mp.libcallsp != 0 { 2449 // Libcall, i.e. runtime syscall on windows. 2450 // Collect Go stack that leads to the call. 2451 n = gentraceback(mp.libcallpc, mp.libcallsp, 0, mp.libcallg, 0, &stk[0], len(stk), nil, nil, 0) 2452 } 2453 if n == 0 { 2454 // If all of the above has failed, account it against abstract "System" or "GC". 2455 n = 2 2456 // "ExternalCode" is better than "etext". 2457 if pc > firstmoduledata.etext { 2458 pc = funcPC(_ExternalCode) + _PCQuantum 2459 } 2460 stk[0] = pc 2461 if mp.preemptoff != "" || mp.helpgc != 0 { 2462 stk[1] = funcPC(_GC) + _PCQuantum 2463 } else { 2464 stk[1] = funcPC(_System) + _PCQuantum 2465 } 2466 } 2467 } 2468 2469 if prof.hz != 0 { 2470 // Simple cas-lock to coordinate with setcpuprofilerate. 2471 for !cas(&prof.lock, 0, 1) { 2472 osyield() 2473 } 2474 if prof.hz != 0 { 2475 cpuprof.add(stk[:n]) 2476 } 2477 atomicstore(&prof.lock, 0) 2478 } 2479 mp.mallocing-- 2480 } 2481 2482 // Arrange to call fn with a traceback hz times a second. 2483 func setcpuprofilerate_m(hz int32) { 2484 // Force sane arguments. 2485 if hz < 0 { 2486 hz = 0 2487 } 2488 2489 // Disable preemption, otherwise we can be rescheduled to another thread 2490 // that has profiling enabled. 2491 _g_ := getg() 2492 _g_.m.locks++ 2493 2494 // Stop profiler on this thread so that it is safe to lock prof. 2495 // if a profiling signal came in while we had prof locked, 2496 // it would deadlock. 2497 resetcpuprofiler(0) 2498 2499 for !cas(&prof.lock, 0, 1) { 2500 osyield() 2501 } 2502 prof.hz = hz 2503 atomicstore(&prof.lock, 0) 2504 2505 lock(&sched.lock) 2506 sched.profilehz = hz 2507 unlock(&sched.lock) 2508 2509 if hz != 0 { 2510 resetcpuprofiler(hz) 2511 } 2512 2513 _g_.m.locks-- 2514 } 2515 2516 // Change number of processors. The world is stopped, sched is locked. 2517 // gcworkbufs are not being modified by either the GC or 2518 // the write barrier code. 2519 // Returns list of Ps with local work, they need to be scheduled by the caller. 2520 func procresize(nprocs int32) *p { 2521 old := gomaxprocs 2522 if old < 0 || old > _MaxGomaxprocs || nprocs <= 0 || nprocs > _MaxGomaxprocs { 2523 throw("procresize: invalid arg") 2524 } 2525 if trace.enabled { 2526 traceGomaxprocs(nprocs) 2527 } 2528 2529 // update statistics 2530 now := nanotime() 2531 if sched.procresizetime != 0 { 2532 sched.totaltime += int64(old) * (now - sched.procresizetime) 2533 } 2534 sched.procresizetime = now 2535 2536 // initialize new P's 2537 for i := int32(0); i < nprocs; i++ { 2538 pp := allp[i] 2539 if pp == nil { 2540 pp = new(p) 2541 pp.id = i 2542 pp.status = _Pgcstop 2543 pp.sudogcache = pp.sudogbuf[:0] 2544 for i := range pp.deferpool { 2545 pp.deferpool[i] = pp.deferpoolbuf[i][:0] 2546 } 2547 atomicstorep(unsafe.Pointer(&allp[i]), unsafe.Pointer(pp)) 2548 } 2549 if pp.mcache == nil { 2550 if old == 0 && i == 0 { 2551 if getg().m.mcache == nil { 2552 throw("missing mcache?") 2553 } 2554 pp.mcache = getg().m.mcache // bootstrap 2555 } else { 2556 pp.mcache = allocmcache() 2557 } 2558 } 2559 } 2560 2561 // free unused P's 2562 for i := nprocs; i < old; i++ { 2563 p := allp[i] 2564 if trace.enabled { 2565 if p == getg().m.p { 2566 // moving to p[0], pretend that we were descheduled 2567 // and then scheduled again to keep the trace sane. 2568 traceGoSched() 2569 traceProcStop(p) 2570 } 2571 } 2572 // move all runable goroutines to the global queue 2573 for p.runqhead != p.runqtail { 2574 // pop from tail of local queue 2575 p.runqtail-- 2576 gp := p.runq[p.runqtail%uint32(len(p.runq))] 2577 // push onto head of global queue 2578 gp.schedlink = sched.runqhead 2579 sched.runqhead = gp 2580 if sched.runqtail == nil { 2581 sched.runqtail = gp 2582 } 2583 sched.runqsize++ 2584 } 2585 for i := range p.sudogbuf { 2586 p.sudogbuf[i] = nil 2587 } 2588 p.sudogcache = p.sudogbuf[:0] 2589 for i := range p.deferpool { 2590 for j := range p.deferpoolbuf[i] { 2591 p.deferpoolbuf[i][j] = nil 2592 } 2593 p.deferpool[i] = p.deferpoolbuf[i][:0] 2594 } 2595 freemcache(p.mcache) 2596 p.mcache = nil 2597 gfpurge(p) 2598 traceProcFree(p) 2599 p.status = _Pdead 2600 // can't free P itself because it can be referenced by an M in syscall 2601 } 2602 2603 _g_ := getg() 2604 if _g_.m.p != nil && _g_.m.p.id < nprocs { 2605 // continue to use the current P 2606 _g_.m.p.status = _Prunning 2607 } else { 2608 // release the current P and acquire allp[0] 2609 if _g_.m.p != nil { 2610 _g_.m.p.m = nil 2611 } 2612 _g_.m.p = nil 2613 _g_.m.mcache = nil 2614 p := allp[0] 2615 p.m = nil 2616 p.status = _Pidle 2617 acquirep(p) 2618 if trace.enabled { 2619 traceGoStart() 2620 } 2621 } 2622 var runnablePs *p 2623 for i := nprocs - 1; i >= 0; i-- { 2624 p := allp[i] 2625 if _g_.m.p == p { 2626 continue 2627 } 2628 p.status = _Pidle 2629 if p.runqhead == p.runqtail { 2630 pidleput(p) 2631 } else { 2632 p.m = mget() 2633 p.link = runnablePs 2634 runnablePs = p 2635 } 2636 } 2637 var int32p *int32 = &gomaxprocs // make compiler check that gomaxprocs is an int32 2638 atomicstore((*uint32)(unsafe.Pointer(int32p)), uint32(nprocs)) 2639 return runnablePs 2640 } 2641 2642 // Associate p and the current m. 2643 // May run during STW, so write barriers are not allowed. 2644 //go:nowritebarrier 2645 func acquirep(_p_ *p) { 2646 _g_ := getg() 2647 2648 if _g_.m.p != nil || _g_.m.mcache != nil { 2649 throw("acquirep: already in go") 2650 } 2651 if _p_.m != nil || _p_.status != _Pidle { 2652 id := int32(0) 2653 if _p_.m != nil { 2654 id = _p_.m.id 2655 } 2656 print("acquirep: p->m=", _p_.m, "(", id, ") p->status=", _p_.status, "\n") 2657 throw("acquirep: invalid p state") 2658 } 2659 // _p_.mcache holds the mcache and _p_ is in allp, so WB can be eliminated 2660 setMcacheNoWriteBarrier(&_g_.m.mcache, _p_.mcache) 2661 // _p_ is in allp so WB can be eliminated 2662 setPNoWriteBarrier(&_g_.m.p, _p_) 2663 // m is in _g_.m and is reachable through allg, so WB can be eliminated 2664 setMNoWriteBarrier(&_p_.m, _g_.m) 2665 _p_.status = _Prunning 2666 2667 if trace.enabled { 2668 traceProcStart() 2669 } 2670 } 2671 2672 // Disassociate p and the current m. 2673 func releasep() *p { 2674 _g_ := getg() 2675 2676 if _g_.m.p == nil || _g_.m.mcache == nil { 2677 throw("releasep: invalid arg") 2678 } 2679 _p_ := _g_.m.p 2680 if _p_.m != _g_.m || _p_.mcache != _g_.m.mcache || _p_.status != _Prunning { 2681 print("releasep: m=", _g_.m, " m->p=", _g_.m.p, " p->m=", _p_.m, " m->mcache=", _g_.m.mcache, " p->mcache=", _p_.mcache, " p->status=", _p_.status, "\n") 2682 throw("releasep: invalid p state") 2683 } 2684 if trace.enabled { 2685 traceProcStop(_g_.m.p) 2686 } 2687 _g_.m.p = nil 2688 _g_.m.mcache = nil 2689 _p_.m = nil 2690 _p_.status = _Pidle 2691 return _p_ 2692 } 2693 2694 func incidlelocked(v int32) { 2695 lock(&sched.lock) 2696 sched.nmidlelocked += v 2697 if v > 0 { 2698 checkdead() 2699 } 2700 unlock(&sched.lock) 2701 } 2702 2703 // Check for deadlock situation. 2704 // The check is based on number of running M's, if 0 -> deadlock. 2705 func checkdead() { 2706 // If we are dying because of a signal caught on an already idle thread, 2707 // freezetheworld will cause all running threads to block. 2708 // And runtime will essentially enter into deadlock state, 2709 // except that there is a thread that will call exit soon. 2710 if panicking > 0 { 2711 return 2712 } 2713 2714 // -1 for sysmon 2715 run := sched.mcount - sched.nmidle - sched.nmidlelocked - 1 2716 if run > 0 { 2717 return 2718 } 2719 if run < 0 { 2720 print("runtime: checkdead: nmidle=", sched.nmidle, " nmidlelocked=", sched.nmidlelocked, " mcount=", sched.mcount, "\n") 2721 throw("checkdead: inconsistent counts") 2722 } 2723 2724 grunning := 0 2725 lock(&allglock) 2726 for i := 0; i < len(allgs); i++ { 2727 gp := allgs[i] 2728 if isSystemGoroutine(gp) { 2729 continue 2730 } 2731 s := readgstatus(gp) 2732 switch s &^ _Gscan { 2733 case _Gwaiting: 2734 grunning++ 2735 case _Grunnable, 2736 _Grunning, 2737 _Gsyscall: 2738 unlock(&allglock) 2739 print("runtime: checkdead: find g ", gp.goid, " in status ", s, "\n") 2740 throw("checkdead: runnable g") 2741 } 2742 } 2743 unlock(&allglock) 2744 if grunning == 0 { // possible if main goroutine calls runtime·Goexit() 2745 throw("no goroutines (main called runtime.Goexit) - deadlock!") 2746 } 2747 2748 // Maybe jump time forward for playground. 2749 gp := timejump() 2750 if gp != nil { 2751 casgstatus(gp, _Gwaiting, _Grunnable) 2752 globrunqput(gp) 2753 _p_ := pidleget() 2754 if _p_ == nil { 2755 throw("checkdead: no p for timer") 2756 } 2757 mp := mget() 2758 if mp == nil { 2759 newm(nil, _p_) 2760 } else { 2761 mp.nextp = _p_ 2762 notewakeup(&mp.park) 2763 } 2764 return 2765 } 2766 2767 getg().m.throwing = -1 // do not dump full stacks 2768 throw("all goroutines are asleep - deadlock!") 2769 } 2770 2771 func sysmon() { 2772 // If we go two minutes without a garbage collection, force one to run. 2773 forcegcperiod := int64(2 * 60 * 1e9) 2774 2775 // If a heap span goes unused for 5 minutes after a garbage collection, 2776 // we hand it back to the operating system. 2777 scavengelimit := int64(5 * 60 * 1e9) 2778 2779 if debug.scavenge > 0 { 2780 // Scavenge-a-lot for testing. 2781 forcegcperiod = 10 * 1e6 2782 scavengelimit = 20 * 1e6 2783 } 2784 2785 lastscavenge := nanotime() 2786 nscavenge := 0 2787 2788 // Make wake-up period small enough for the sampling to be correct. 2789 maxsleep := forcegcperiod / 2 2790 if scavengelimit < forcegcperiod { 2791 maxsleep = scavengelimit / 2 2792 } 2793 2794 lasttrace := int64(0) 2795 idle := 0 // how many cycles in succession we had not wokeup somebody 2796 delay := uint32(0) 2797 for { 2798 if idle == 0 { // start with 20us sleep... 2799 delay = 20 2800 } else if idle > 50 { // start doubling the sleep after 1ms... 2801 delay *= 2 2802 } 2803 if delay > 10*1000 { // up to 10ms 2804 delay = 10 * 1000 2805 } 2806 usleep(delay) 2807 if debug.schedtrace <= 0 && (sched.gcwaiting != 0 || atomicload(&sched.npidle) == uint32(gomaxprocs)) { // TODO: fast atomic 2808 lock(&sched.lock) 2809 if atomicload(&sched.gcwaiting) != 0 || atomicload(&sched.npidle) == uint32(gomaxprocs) { 2810 atomicstore(&sched.sysmonwait, 1) 2811 unlock(&sched.lock) 2812 notetsleep(&sched.sysmonnote, maxsleep) 2813 lock(&sched.lock) 2814 atomicstore(&sched.sysmonwait, 0) 2815 noteclear(&sched.sysmonnote) 2816 idle = 0 2817 delay = 20 2818 } 2819 unlock(&sched.lock) 2820 } 2821 // poll network if not polled for more than 10ms 2822 lastpoll := int64(atomicload64(&sched.lastpoll)) 2823 now := nanotime() 2824 unixnow := unixnanotime() 2825 if lastpoll != 0 && lastpoll+10*1000*1000 < now { 2826 cas64(&sched.lastpoll, uint64(lastpoll), uint64(now)) 2827 gp := netpoll(false) // non-blocking - returns list of goroutines 2828 if gp != nil { 2829 // Need to decrement number of idle locked M's 2830 // (pretending that one more is running) before injectglist. 2831 // Otherwise it can lead to the following situation: 2832 // injectglist grabs all P's but before it starts M's to run the P's, 2833 // another M returns from syscall, finishes running its G, 2834 // observes that there is no work to do and no other running M's 2835 // and reports deadlock. 2836 incidlelocked(-1) 2837 injectglist(gp) 2838 incidlelocked(1) 2839 } 2840 } 2841 // retake P's blocked in syscalls 2842 // and preempt long running G's 2843 if retake(now) != 0 { 2844 idle = 0 2845 } else { 2846 idle++ 2847 } 2848 // check if we need to force a GC 2849 lastgc := int64(atomicload64(&memstats.last_gc)) 2850 if lastgc != 0 && unixnow-lastgc > forcegcperiod && atomicload(&forcegc.idle) != 0 { 2851 lock(&forcegc.lock) 2852 forcegc.idle = 0 2853 forcegc.g.schedlink = nil 2854 injectglist(forcegc.g) 2855 unlock(&forcegc.lock) 2856 } 2857 // scavenge heap once in a while 2858 if lastscavenge+scavengelimit/2 < now { 2859 mHeap_Scavenge(int32(nscavenge), uint64(now), uint64(scavengelimit)) 2860 lastscavenge = now 2861 nscavenge++ 2862 } 2863 if debug.schedtrace > 0 && lasttrace+int64(debug.schedtrace*1000000) <= now { 2864 lasttrace = now 2865 schedtrace(debug.scheddetail > 0) 2866 } 2867 } 2868 } 2869 2870 var pdesc [_MaxGomaxprocs]struct { 2871 schedtick uint32 2872 schedwhen int64 2873 syscalltick uint32 2874 syscallwhen int64 2875 } 2876 2877 // forcePreemptNS is the time slice given to a G before it is 2878 // preempted. 2879 const forcePreemptNS = 10 * 1000 * 1000 // 10ms 2880 2881 func retake(now int64) uint32 { 2882 n := 0 2883 for i := int32(0); i < gomaxprocs; i++ { 2884 _p_ := allp[i] 2885 if _p_ == nil { 2886 continue 2887 } 2888 pd := &pdesc[i] 2889 s := _p_.status 2890 if s == _Psyscall { 2891 // Retake P from syscall if it's there for more than 1 sysmon tick (at least 20us). 2892 t := int64(_p_.syscalltick) 2893 if int64(pd.syscalltick) != t { 2894 pd.syscalltick = uint32(t) 2895 pd.syscallwhen = now 2896 continue 2897 } 2898 // On the one hand we don't want to retake Ps if there is no other work to do, 2899 // but on the other hand we want to retake them eventually 2900 // because they can prevent the sysmon thread from deep sleep. 2901 if _p_.runqhead == _p_.runqtail && atomicload(&sched.nmspinning)+atomicload(&sched.npidle) > 0 && pd.syscallwhen+10*1000*1000 > now { 2902 continue 2903 } 2904 // Need to decrement number of idle locked M's 2905 // (pretending that one more is running) before the CAS. 2906 // Otherwise the M from which we retake can exit the syscall, 2907 // increment nmidle and report deadlock. 2908 incidlelocked(-1) 2909 if cas(&_p_.status, s, _Pidle) { 2910 if trace.enabled { 2911 traceGoSysBlock(_p_) 2912 traceProcStop(_p_) 2913 } 2914 n++ 2915 _p_.syscalltick++ 2916 handoffp(_p_) 2917 } 2918 incidlelocked(1) 2919 } else if s == _Prunning { 2920 // Preempt G if it's running for too long. 2921 t := int64(_p_.schedtick) 2922 if int64(pd.schedtick) != t { 2923 pd.schedtick = uint32(t) 2924 pd.schedwhen = now 2925 continue 2926 } 2927 if pd.schedwhen+forcePreemptNS > now { 2928 continue 2929 } 2930 preemptone(_p_) 2931 } 2932 } 2933 return uint32(n) 2934 } 2935 2936 // Tell all goroutines that they have been preempted and they should stop. 2937 // This function is purely best-effort. It can fail to inform a goroutine if a 2938 // processor just started running it. 2939 // No locks need to be held. 2940 // Returns true if preemption request was issued to at least one goroutine. 2941 func preemptall() bool { 2942 res := false 2943 for i := int32(0); i < gomaxprocs; i++ { 2944 _p_ := allp[i] 2945 if _p_ == nil || _p_.status != _Prunning { 2946 continue 2947 } 2948 if preemptone(_p_) { 2949 res = true 2950 } 2951 } 2952 return res 2953 } 2954 2955 // Tell the goroutine running on processor P to stop. 2956 // This function is purely best-effort. It can incorrectly fail to inform the 2957 // goroutine. It can send inform the wrong goroutine. Even if it informs the 2958 // correct goroutine, that goroutine might ignore the request if it is 2959 // simultaneously executing newstack. 2960 // No lock needs to be held. 2961 // Returns true if preemption request was issued. 2962 // The actual preemption will happen at some point in the future 2963 // and will be indicated by the gp->status no longer being 2964 // Grunning 2965 func preemptone(_p_ *p) bool { 2966 mp := _p_.m 2967 if mp == nil || mp == getg().m { 2968 return false 2969 } 2970 gp := mp.curg 2971 if gp == nil || gp == mp.g0 { 2972 return false 2973 } 2974 2975 gp.preempt = true 2976 2977 // Every call in a go routine checks for stack overflow by 2978 // comparing the current stack pointer to gp->stackguard0. 2979 // Setting gp->stackguard0 to StackPreempt folds 2980 // preemption into the normal stack overflow check. 2981 gp.stackguard0 = stackPreempt 2982 return true 2983 } 2984 2985 var starttime int64 2986 2987 func schedtrace(detailed bool) { 2988 now := nanotime() 2989 if starttime == 0 { 2990 starttime = now 2991 } 2992 2993 lock(&sched.lock) 2994 print("SCHED ", (now-starttime)/1e6, "ms: gomaxprocs=", gomaxprocs, " idleprocs=", sched.npidle, " threads=", sched.mcount, " spinningthreads=", sched.nmspinning, " idlethreads=", sched.nmidle, " runqueue=", sched.runqsize) 2995 if detailed { 2996 print(" gcwaiting=", sched.gcwaiting, " nmidlelocked=", sched.nmidlelocked, " stopwait=", sched.stopwait, " sysmonwait=", sched.sysmonwait, "\n") 2997 } 2998 // We must be careful while reading data from P's, M's and G's. 2999 // Even if we hold schedlock, most data can be changed concurrently. 3000 // E.g. (p->m ? p->m->id : -1) can crash if p->m changes from non-nil to nil. 3001 for i := int32(0); i < gomaxprocs; i++ { 3002 _p_ := allp[i] 3003 if _p_ == nil { 3004 continue 3005 } 3006 mp := _p_.m 3007 h := atomicload(&_p_.runqhead) 3008 t := atomicload(&_p_.runqtail) 3009 if detailed { 3010 id := int32(-1) 3011 if mp != nil { 3012 id = mp.id 3013 } 3014 print(" P", i, ": status=", _p_.status, " schedtick=", _p_.schedtick, " syscalltick=", _p_.syscalltick, " m=", id, " runqsize=", t-h, " gfreecnt=", _p_.gfreecnt, "\n") 3015 } else { 3016 // In non-detailed mode format lengths of per-P run queues as: 3017 // [len1 len2 len3 len4] 3018 print(" ") 3019 if i == 0 { 3020 print("[") 3021 } 3022 print(t - h) 3023 if i == gomaxprocs-1 { 3024 print("]\n") 3025 } 3026 } 3027 } 3028 3029 if !detailed { 3030 unlock(&sched.lock) 3031 return 3032 } 3033 3034 for mp := allm; mp != nil; mp = mp.alllink { 3035 _p_ := mp.p 3036 gp := mp.curg 3037 lockedg := mp.lockedg 3038 id1 := int32(-1) 3039 if _p_ != nil { 3040 id1 = _p_.id 3041 } 3042 id2 := int64(-1) 3043 if gp != nil { 3044 id2 = gp.goid 3045 } 3046 id3 := int64(-1) 3047 if lockedg != nil { 3048 id3 = lockedg.goid 3049 } 3050 print(" M", mp.id, ": p=", id1, " curg=", id2, " mallocing=", mp.mallocing, " throwing=", mp.throwing, " preemptoff=", mp.preemptoff, ""+" locks=", mp.locks, " dying=", mp.dying, " helpgc=", mp.helpgc, " spinning=", mp.spinning, " blocked=", getg().m.blocked, " lockedg=", id3, "\n") 3051 } 3052 3053 lock(&allglock) 3054 for gi := 0; gi < len(allgs); gi++ { 3055 gp := allgs[gi] 3056 mp := gp.m 3057 lockedm := gp.lockedm 3058 id1 := int32(-1) 3059 if mp != nil { 3060 id1 = mp.id 3061 } 3062 id2 := int32(-1) 3063 if lockedm != nil { 3064 id2 = lockedm.id 3065 } 3066 print(" G", gp.goid, ": status=", readgstatus(gp), "(", gp.waitreason, ") m=", id1, " lockedm=", id2, "\n") 3067 } 3068 unlock(&allglock) 3069 unlock(&sched.lock) 3070 } 3071 3072 // Put mp on midle list. 3073 // Sched must be locked. 3074 // May run during STW, so write barriers are not allowed. 3075 //go:nowritebarrier 3076 func mput(mp *m) { 3077 // sched.midle is reachable via allm, so WB can be eliminated. 3078 setMNoWriteBarrier(&mp.schedlink, sched.midle) 3079 // mp is reachable via allm, so WB can be eliminated. 3080 setMNoWriteBarrier(&sched.midle, mp) 3081 sched.nmidle++ 3082 checkdead() 3083 } 3084 3085 // Try to get an m from midle list. 3086 // Sched must be locked. 3087 // May run during STW, so write barriers are not allowed. 3088 //go:nowritebarrier 3089 func mget() *m { 3090 mp := sched.midle 3091 if mp != nil { 3092 // mp.schedlink is reachable via mp, which is on allm, so WB can be eliminated. 3093 setMNoWriteBarrier(&sched.midle, mp.schedlink) 3094 sched.nmidle-- 3095 } 3096 return mp 3097 } 3098 3099 // Put gp on the global runnable queue. 3100 // Sched must be locked. 3101 // May run during STW, so write barriers are not allowed. 3102 //go:nowritebarrier 3103 func globrunqput(gp *g) { 3104 gp.schedlink = nil 3105 if sched.runqtail != nil { 3106 // gp is on allg, so these three WBs can be eliminated. 3107 setGNoWriteBarrier(&sched.runqtail.schedlink, gp) 3108 } else { 3109 setGNoWriteBarrier(&sched.runqhead, gp) 3110 } 3111 setGNoWriteBarrier(&sched.runqtail, gp) 3112 sched.runqsize++ 3113 } 3114 3115 // Put a batch of runnable goroutines on the global runnable queue. 3116 // Sched must be locked. 3117 func globrunqputbatch(ghead *g, gtail *g, n int32) { 3118 gtail.schedlink = nil 3119 if sched.runqtail != nil { 3120 sched.runqtail.schedlink = ghead 3121 } else { 3122 sched.runqhead = ghead 3123 } 3124 sched.runqtail = gtail 3125 sched.runqsize += n 3126 } 3127 3128 // Try get a batch of G's from the global runnable queue. 3129 // Sched must be locked. 3130 func globrunqget(_p_ *p, max int32) *g { 3131 if sched.runqsize == 0 { 3132 return nil 3133 } 3134 3135 n := sched.runqsize/gomaxprocs + 1 3136 if n > sched.runqsize { 3137 n = sched.runqsize 3138 } 3139 if max > 0 && n > max { 3140 n = max 3141 } 3142 if n > int32(len(_p_.runq))/2 { 3143 n = int32(len(_p_.runq)) / 2 3144 } 3145 3146 sched.runqsize -= n 3147 if sched.runqsize == 0 { 3148 sched.runqtail = nil 3149 } 3150 3151 gp := sched.runqhead 3152 sched.runqhead = gp.schedlink 3153 n-- 3154 for ; n > 0; n-- { 3155 gp1 := sched.runqhead 3156 sched.runqhead = gp1.schedlink 3157 runqput(_p_, gp1) 3158 } 3159 return gp 3160 } 3161 3162 // Put p to on _Pidle list. 3163 // Sched must be locked. 3164 // May run during STW, so write barriers are not allowed. 3165 //go:nowritebarrier 3166 func pidleput(_p_ *p) { 3167 // sched.pidle, _p_.link and _p_ are reachable via allp, so WB can be eliminated. 3168 setPNoWriteBarrier(&_p_.link, sched.pidle) 3169 setPNoWriteBarrier(&sched.pidle, _p_) 3170 xadd(&sched.npidle, 1) // TODO: fast atomic 3171 } 3172 3173 // Try get a p from _Pidle list. 3174 // Sched must be locked. 3175 // May run during STW, so write barriers are not allowed. 3176 //go:nowritebarrier 3177 func pidleget() *p { 3178 _p_ := sched.pidle 3179 if _p_ != nil { 3180 // _p_.link is reachable via a _p_ in allp, so WB can be eliminated. 3181 setPNoWriteBarrier(&sched.pidle, _p_.link) 3182 xadd(&sched.npidle, -1) // TODO: fast atomic 3183 } 3184 return _p_ 3185 } 3186 3187 // Try to put g on local runnable queue. 3188 // If it's full, put onto global queue. 3189 // Executed only by the owner P. 3190 func runqput(_p_ *p, gp *g) { 3191 retry: 3192 h := atomicload(&_p_.runqhead) // load-acquire, synchronize with consumers 3193 t := _p_.runqtail 3194 if t-h < uint32(len(_p_.runq)) { 3195 _p_.runq[t%uint32(len(_p_.runq))] = gp 3196 atomicstore(&_p_.runqtail, t+1) // store-release, makes the item available for consumption 3197 return 3198 } 3199 if runqputslow(_p_, gp, h, t) { 3200 return 3201 } 3202 // the queue is not full, now the put above must suceed 3203 goto retry 3204 } 3205 3206 // Put g and a batch of work from local runnable queue on global queue. 3207 // Executed only by the owner P. 3208 func runqputslow(_p_ *p, gp *g, h, t uint32) bool { 3209 var batch [len(_p_.runq)/2 + 1]*g 3210 3211 // First, grab a batch from local queue. 3212 n := t - h 3213 n = n / 2 3214 if n != uint32(len(_p_.runq)/2) { 3215 throw("runqputslow: queue is not full") 3216 } 3217 for i := uint32(0); i < n; i++ { 3218 batch[i] = _p_.runq[(h+i)%uint32(len(_p_.runq))] 3219 } 3220 if !cas(&_p_.runqhead, h, h+n) { // cas-release, commits consume 3221 return false 3222 } 3223 batch[n] = gp 3224 3225 // Link the goroutines. 3226 for i := uint32(0); i < n; i++ { 3227 batch[i].schedlink = batch[i+1] 3228 } 3229 3230 // Now put the batch on global queue. 3231 lock(&sched.lock) 3232 globrunqputbatch(batch[0], batch[n], int32(n+1)) 3233 unlock(&sched.lock) 3234 return true 3235 } 3236 3237 // Get g from local runnable queue. 3238 // Executed only by the owner P. 3239 func runqget(_p_ *p) *g { 3240 for { 3241 h := atomicload(&_p_.runqhead) // load-acquire, synchronize with other consumers 3242 t := _p_.runqtail 3243 if t == h { 3244 return nil 3245 } 3246 gp := _p_.runq[h%uint32(len(_p_.runq))] 3247 if cas(&_p_.runqhead, h, h+1) { // cas-release, commits consume 3248 return gp 3249 } 3250 } 3251 } 3252 3253 // Grabs a batch of goroutines from local runnable queue. 3254 // batch array must be of size len(p->runq)/2. Returns number of grabbed goroutines. 3255 // Can be executed by any P. 3256 func runqgrab(_p_ *p, batch []*g) uint32 { 3257 for { 3258 h := atomicload(&_p_.runqhead) // load-acquire, synchronize with other consumers 3259 t := atomicload(&_p_.runqtail) // load-acquire, synchronize with the producer 3260 n := t - h 3261 n = n - n/2 3262 if n == 0 { 3263 return 0 3264 } 3265 if n > uint32(len(_p_.runq)/2) { // read inconsistent h and t 3266 continue 3267 } 3268 for i := uint32(0); i < n; i++ { 3269 batch[i] = _p_.runq[(h+i)%uint32(len(_p_.runq))] 3270 } 3271 if cas(&_p_.runqhead, h, h+n) { // cas-release, commits consume 3272 return n 3273 } 3274 } 3275 } 3276 3277 // Steal half of elements from local runnable queue of p2 3278 // and put onto local runnable queue of p. 3279 // Returns one of the stolen elements (or nil if failed). 3280 func runqsteal(_p_, p2 *p) *g { 3281 var batch [len(_p_.runq) / 2]*g 3282 3283 n := runqgrab(p2, batch[:]) 3284 if n == 0 { 3285 return nil 3286 } 3287 n-- 3288 gp := batch[n] 3289 if n == 0 { 3290 return gp 3291 } 3292 h := atomicload(&_p_.runqhead) // load-acquire, synchronize with consumers 3293 t := _p_.runqtail 3294 if t-h+n >= uint32(len(_p_.runq)) { 3295 throw("runqsteal: runq overflow") 3296 } 3297 for i := uint32(0); i < n; i++ { 3298 _p_.runq[(t+i)%uint32(len(_p_.runq))] = batch[i] 3299 } 3300 atomicstore(&_p_.runqtail, t+n) // store-release, makes the item available for consumption 3301 return gp 3302 } 3303 3304 func testSchedLocalQueue() { 3305 _p_ := new(p) 3306 gs := make([]g, len(_p_.runq)) 3307 for i := 0; i < len(_p_.runq); i++ { 3308 if runqget(_p_) != nil { 3309 throw("runq is not empty initially") 3310 } 3311 for j := 0; j < i; j++ { 3312 runqput(_p_, &gs[i]) 3313 } 3314 for j := 0; j < i; j++ { 3315 if runqget(_p_) != &gs[i] { 3316 print("bad element at iter ", i, "/", j, "\n") 3317 throw("bad element") 3318 } 3319 } 3320 if runqget(_p_) != nil { 3321 throw("runq is not empty afterwards") 3322 } 3323 } 3324 } 3325 3326 func testSchedLocalQueueSteal() { 3327 p1 := new(p) 3328 p2 := new(p) 3329 gs := make([]g, len(p1.runq)) 3330 for i := 0; i < len(p1.runq); i++ { 3331 for j := 0; j < i; j++ { 3332 gs[j].sig = 0 3333 runqput(p1, &gs[j]) 3334 } 3335 gp := runqsteal(p2, p1) 3336 s := 0 3337 if gp != nil { 3338 s++ 3339 gp.sig++ 3340 } 3341 for { 3342 gp = runqget(p2) 3343 if gp == nil { 3344 break 3345 } 3346 s++ 3347 gp.sig++ 3348 } 3349 for { 3350 gp = runqget(p1) 3351 if gp == nil { 3352 break 3353 } 3354 gp.sig++ 3355 } 3356 for j := 0; j < i; j++ { 3357 if gs[j].sig != 1 { 3358 print("bad element ", j, "(", gs[j].sig, ") at iter ", i, "\n") 3359 throw("bad element") 3360 } 3361 } 3362 if s != i/2 && s != i/2+1 { 3363 print("bad steal ", s, ", want ", i/2, " or ", i/2+1, ", iter ", i, "\n") 3364 throw("bad steal") 3365 } 3366 } 3367 } 3368 3369 func setMaxThreads(in int) (out int) { 3370 lock(&sched.lock) 3371 out = int(sched.maxmcount) 3372 sched.maxmcount = int32(in) 3373 checkmcount() 3374 unlock(&sched.lock) 3375 return 3376 } 3377 3378 func haveexperiment(name string) bool { 3379 x := goexperiment 3380 for x != "" { 3381 xname := "" 3382 i := index(x, ",") 3383 if i < 0 { 3384 xname, x = x, "" 3385 } else { 3386 xname, x = x[:i], x[i+1:] 3387 } 3388 if xname == name { 3389 return true 3390 } 3391 } 3392 return false 3393 } 3394 3395 //go:nosplit 3396 func procPin() int { 3397 _g_ := getg() 3398 mp := _g_.m 3399 3400 mp.locks++ 3401 return int(mp.p.id) 3402 } 3403 3404 //go:nosplit 3405 func procUnpin() { 3406 _g_ := getg() 3407 _g_.m.locks-- 3408 } 3409 3410 //go:linkname sync_runtime_procPin sync.runtime_procPin 3411 //go:nosplit 3412 func sync_runtime_procPin() int { 3413 return procPin() 3414 } 3415 3416 //go:linkname sync_runtime_procUnpin sync.runtime_procUnpin 3417 //go:nosplit 3418 func sync_runtime_procUnpin() { 3419 procUnpin() 3420 } 3421 3422 //go:linkname sync_atomic_runtime_procPin sync/atomic.runtime_procPin 3423 //go:nosplit 3424 func sync_atomic_runtime_procPin() int { 3425 return procPin() 3426 } 3427 3428 //go:linkname sync_atomic_runtime_procUnpin sync/atomic.runtime_procUnpin 3429 //go:nosplit 3430 func sync_atomic_runtime_procUnpin() { 3431 procUnpin() 3432 } 3433 3434 // Active spinning for sync.Mutex. 3435 //go:linkname sync_runtime_canSpin sync.runtime_canSpin 3436 //go:nosplit 3437 func sync_runtime_canSpin(i int) bool { 3438 // sync.Mutex is cooperative, so we are conservative with spinning. 3439 // Spin only few times and only if running on a multicore machine and 3440 // GOMAXPROCS>1 and there is at least one other running P and local runq is empty. 3441 // As opposed to runtime mutex we don't do passive spinning here, 3442 // because there can be work on global runq on on other Ps. 3443 if i >= active_spin || ncpu <= 1 || gomaxprocs <= int32(sched.npidle+sched.nmspinning)+1 { 3444 return false 3445 } 3446 if p := getg().m.p; p.runqhead != p.runqtail { 3447 return false 3448 } 3449 return true 3450 } 3451 3452 //go:linkname sync_runtime_doSpin sync.runtime_doSpin 3453 //go:nosplit 3454 func sync_runtime_doSpin() { 3455 procyield(active_spin_cnt) 3456 }