github.com/mtsmfm/go/src@v0.0.0-20221020090648-44bdcb9f8fde/runtime/runtime2.go (about) 1 // Copyright 2009 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package runtime 6 7 import ( 8 "internal/goarch" 9 "runtime/internal/atomic" 10 "unsafe" 11 ) 12 13 // defined constants 14 const ( 15 // G status 16 // 17 // Beyond indicating the general state of a G, the G status 18 // acts like a lock on the goroutine's stack (and hence its 19 // ability to execute user code). 20 // 21 // If you add to this list, add to the list 22 // of "okay during garbage collection" status 23 // in mgcmark.go too. 24 // 25 // TODO(austin): The _Gscan bit could be much lighter-weight. 26 // For example, we could choose not to run _Gscanrunnable 27 // goroutines found in the run queue, rather than CAS-looping 28 // until they become _Grunnable. And transitions like 29 // _Gscanwaiting -> _Gscanrunnable are actually okay because 30 // they don't affect stack ownership. 31 32 // _Gidle means this goroutine was just allocated and has not 33 // yet been initialized. 34 _Gidle = iota // 0 35 36 // _Grunnable means this goroutine is on a run queue. It is 37 // not currently executing user code. The stack is not owned. 38 _Grunnable // 1 39 40 // _Grunning means this goroutine may execute user code. The 41 // stack is owned by this goroutine. It is not on a run queue. 42 // It is assigned an M and a P (g.m and g.m.p are valid). 43 _Grunning // 2 44 45 // _Gsyscall means this goroutine is executing a system call. 46 // It is not executing user code. The stack is owned by this 47 // goroutine. It is not on a run queue. It is assigned an M. 48 _Gsyscall // 3 49 50 // _Gwaiting means this goroutine is blocked in the runtime. 51 // It is not executing user code. It is not on a run queue, 52 // but should be recorded somewhere (e.g., a channel wait 53 // queue) so it can be ready()d when necessary. The stack is 54 // not owned *except* that a channel operation may read or 55 // write parts of the stack under the appropriate channel 56 // lock. Otherwise, it is not safe to access the stack after a 57 // goroutine enters _Gwaiting (e.g., it may get moved). 58 _Gwaiting // 4 59 60 // _Gmoribund_unused is currently unused, but hardcoded in gdb 61 // scripts. 62 _Gmoribund_unused // 5 63 64 // _Gdead means this goroutine is currently unused. It may be 65 // just exited, on a free list, or just being initialized. It 66 // is not executing user code. It may or may not have a stack 67 // allocated. The G and its stack (if any) are owned by the M 68 // that is exiting the G or that obtained the G from the free 69 // list. 70 _Gdead // 6 71 72 // _Genqueue_unused is currently unused. 73 _Genqueue_unused // 7 74 75 // _Gcopystack means this goroutine's stack is being moved. It 76 // is not executing user code and is not on a run queue. The 77 // stack is owned by the goroutine that put it in _Gcopystack. 78 _Gcopystack // 8 79 80 // _Gpreempted means this goroutine stopped itself for a 81 // suspendG preemption. It is like _Gwaiting, but nothing is 82 // yet responsible for ready()ing it. Some suspendG must CAS 83 // the status to _Gwaiting to take responsibility for 84 // ready()ing this G. 85 _Gpreempted // 9 86 87 // _Gscan combined with one of the above states other than 88 // _Grunning indicates that GC is scanning the stack. The 89 // goroutine is not executing user code and the stack is owned 90 // by the goroutine that set the _Gscan bit. 91 // 92 // _Gscanrunning is different: it is used to briefly block 93 // state transitions while GC signals the G to scan its own 94 // stack. This is otherwise like _Grunning. 95 // 96 // atomicstatus&~Gscan gives the state the goroutine will 97 // return to when the scan completes. 98 _Gscan = 0x1000 99 _Gscanrunnable = _Gscan + _Grunnable // 0x1001 100 _Gscanrunning = _Gscan + _Grunning // 0x1002 101 _Gscansyscall = _Gscan + _Gsyscall // 0x1003 102 _Gscanwaiting = _Gscan + _Gwaiting // 0x1004 103 _Gscanpreempted = _Gscan + _Gpreempted // 0x1009 104 ) 105 106 const ( 107 // P status 108 109 // _Pidle means a P is not being used to run user code or the 110 // scheduler. Typically, it's on the idle P list and available 111 // to the scheduler, but it may just be transitioning between 112 // other states. 113 // 114 // The P is owned by the idle list or by whatever is 115 // transitioning its state. Its run queue is empty. 116 _Pidle = iota 117 118 // _Prunning means a P is owned by an M and is being used to 119 // run user code or the scheduler. Only the M that owns this P 120 // is allowed to change the P's status from _Prunning. The M 121 // may transition the P to _Pidle (if it has no more work to 122 // do), _Psyscall (when entering a syscall), or _Pgcstop (to 123 // halt for the GC). The M may also hand ownership of the P 124 // off directly to another M (e.g., to schedule a locked G). 125 _Prunning 126 127 // _Psyscall means a P is not running user code. It has 128 // affinity to an M in a syscall but is not owned by it and 129 // may be stolen by another M. This is similar to _Pidle but 130 // uses lightweight transitions and maintains M affinity. 131 // 132 // Leaving _Psyscall must be done with a CAS, either to steal 133 // or retake the P. Note that there's an ABA hazard: even if 134 // an M successfully CASes its original P back to _Prunning 135 // after a syscall, it must understand the P may have been 136 // used by another M in the interim. 137 _Psyscall 138 139 // _Pgcstop means a P is halted for STW and owned by the M 140 // that stopped the world. The M that stopped the world 141 // continues to use its P, even in _Pgcstop. Transitioning 142 // from _Prunning to _Pgcstop causes an M to release its P and 143 // park. 144 // 145 // The P retains its run queue and startTheWorld will restart 146 // the scheduler on Ps with non-empty run queues. 147 _Pgcstop 148 149 // _Pdead means a P is no longer used (GOMAXPROCS shrank). We 150 // reuse Ps if GOMAXPROCS increases. A dead P is mostly 151 // stripped of its resources, though a few things remain 152 // (e.g., trace buffers). 153 _Pdead 154 ) 155 156 // Mutual exclusion locks. In the uncontended case, 157 // as fast as spin locks (just a few user-level instructions), 158 // but on the contention path they sleep in the kernel. 159 // A zeroed Mutex is unlocked (no need to initialize each lock). 160 // Initialization is helpful for static lock ranking, but not required. 161 type mutex struct { 162 // Empty struct if lock ranking is disabled, otherwise includes the lock rank 163 lockRankStruct 164 // Futex-based impl treats it as uint32 key, 165 // while sema-based impl as M* waitm. 166 // Used to be a union, but unions break precise GC. 167 key uintptr 168 } 169 170 // sleep and wakeup on one-time events. 171 // before any calls to notesleep or notewakeup, 172 // must call noteclear to initialize the Note. 173 // then, exactly one thread can call notesleep 174 // and exactly one thread can call notewakeup (once). 175 // once notewakeup has been called, the notesleep 176 // will return. future notesleep will return immediately. 177 // subsequent noteclear must be called only after 178 // previous notesleep has returned, e.g. it's disallowed 179 // to call noteclear straight after notewakeup. 180 // 181 // notetsleep is like notesleep but wakes up after 182 // a given number of nanoseconds even if the event 183 // has not yet happened. if a goroutine uses notetsleep to 184 // wake up early, it must wait to call noteclear until it 185 // can be sure that no other goroutine is calling 186 // notewakeup. 187 // 188 // notesleep/notetsleep are generally called on g0, 189 // notetsleepg is similar to notetsleep but is called on user g. 190 type note struct { 191 // Futex-based impl treats it as uint32 key, 192 // while sema-based impl as M* waitm. 193 // Used to be a union, but unions break precise GC. 194 key uintptr 195 } 196 197 type funcval struct { 198 fn uintptr 199 // variable-size, fn-specific data here 200 } 201 202 type iface struct { 203 tab *itab 204 data unsafe.Pointer 205 } 206 207 type eface struct { 208 _type *_type 209 data unsafe.Pointer 210 } 211 212 func efaceOf(ep *any) *eface { 213 return (*eface)(unsafe.Pointer(ep)) 214 } 215 216 // The guintptr, muintptr, and puintptr are all used to bypass write barriers. 217 // It is particularly important to avoid write barriers when the current P has 218 // been released, because the GC thinks the world is stopped, and an 219 // unexpected write barrier would not be synchronized with the GC, 220 // which can lead to a half-executed write barrier that has marked the object 221 // but not queued it. If the GC skips the object and completes before the 222 // queuing can occur, it will incorrectly free the object. 223 // 224 // We tried using special assignment functions invoked only when not 225 // holding a running P, but then some updates to a particular memory 226 // word went through write barriers and some did not. This breaks the 227 // write barrier shadow checking mode, and it is also scary: better to have 228 // a word that is completely ignored by the GC than to have one for which 229 // only a few updates are ignored. 230 // 231 // Gs and Ps are always reachable via true pointers in the 232 // allgs and allp lists or (during allocation before they reach those lists) 233 // from stack variables. 234 // 235 // Ms are always reachable via true pointers either from allm or 236 // freem. Unlike Gs and Ps we do free Ms, so it's important that 237 // nothing ever hold an muintptr across a safe point. 238 239 // A guintptr holds a goroutine pointer, but typed as a uintptr 240 // to bypass write barriers. It is used in the Gobuf goroutine state 241 // and in scheduling lists that are manipulated without a P. 242 // 243 // The Gobuf.g goroutine pointer is almost always updated by assembly code. 244 // In one of the few places it is updated by Go code - func save - it must be 245 // treated as a uintptr to avoid a write barrier being emitted at a bad time. 246 // Instead of figuring out how to emit the write barriers missing in the 247 // assembly manipulation, we change the type of the field to uintptr, 248 // so that it does not require write barriers at all. 249 // 250 // Goroutine structs are published in the allg list and never freed. 251 // That will keep the goroutine structs from being collected. 252 // There is never a time that Gobuf.g's contain the only references 253 // to a goroutine: the publishing of the goroutine in allg comes first. 254 // Goroutine pointers are also kept in non-GC-visible places like TLS, 255 // so I can't see them ever moving. If we did want to start moving data 256 // in the GC, we'd need to allocate the goroutine structs from an 257 // alternate arena. Using guintptr doesn't make that problem any worse. 258 // Note that pollDesc.rg, pollDesc.wg also store g in uintptr form, 259 // so they would need to be updated too if g's start moving. 260 type guintptr uintptr 261 262 //go:nosplit 263 func (gp guintptr) ptr() *g { return (*g)(unsafe.Pointer(gp)) } 264 265 //go:nosplit 266 func (gp *guintptr) set(g *g) { *gp = guintptr(unsafe.Pointer(g)) } 267 268 //go:nosplit 269 func (gp *guintptr) cas(old, new guintptr) bool { 270 return atomic.Casuintptr((*uintptr)(unsafe.Pointer(gp)), uintptr(old), uintptr(new)) 271 } 272 273 // setGNoWB performs *gp = new without a write barrier. 274 // For times when it's impractical to use a guintptr. 275 // 276 //go:nosplit 277 //go:nowritebarrier 278 func setGNoWB(gp **g, new *g) { 279 (*guintptr)(unsafe.Pointer(gp)).set(new) 280 } 281 282 type puintptr uintptr 283 284 //go:nosplit 285 func (pp puintptr) ptr() *p { return (*p)(unsafe.Pointer(pp)) } 286 287 //go:nosplit 288 func (pp *puintptr) set(p *p) { *pp = puintptr(unsafe.Pointer(p)) } 289 290 // muintptr is a *m that is not tracked by the garbage collector. 291 // 292 // Because we do free Ms, there are some additional constrains on 293 // muintptrs: 294 // 295 // 1. Never hold an muintptr locally across a safe point. 296 // 297 // 2. Any muintptr in the heap must be owned by the M itself so it can 298 // ensure it is not in use when the last true *m is released. 299 type muintptr uintptr 300 301 //go:nosplit 302 func (mp muintptr) ptr() *m { return (*m)(unsafe.Pointer(mp)) } 303 304 //go:nosplit 305 func (mp *muintptr) set(m *m) { *mp = muintptr(unsafe.Pointer(m)) } 306 307 // setMNoWB performs *mp = new without a write barrier. 308 // For times when it's impractical to use an muintptr. 309 // 310 //go:nosplit 311 //go:nowritebarrier 312 func setMNoWB(mp **m, new *m) { 313 (*muintptr)(unsafe.Pointer(mp)).set(new) 314 } 315 316 type gobuf struct { 317 // The offsets of sp, pc, and g are known to (hard-coded in) libmach. 318 // 319 // ctxt is unusual with respect to GC: it may be a 320 // heap-allocated funcval, so GC needs to track it, but it 321 // needs to be set and cleared from assembly, where it's 322 // difficult to have write barriers. However, ctxt is really a 323 // saved, live register, and we only ever exchange it between 324 // the real register and the gobuf. Hence, we treat it as a 325 // root during stack scanning, which means assembly that saves 326 // and restores it doesn't need write barriers. It's still 327 // typed as a pointer so that any other writes from Go get 328 // write barriers. 329 sp uintptr 330 pc uintptr 331 g guintptr 332 ctxt unsafe.Pointer 333 ret uintptr 334 lr uintptr 335 bp uintptr // for framepointer-enabled architectures 336 } 337 338 // sudog represents a g in a wait list, such as for sending/receiving 339 // on a channel. 340 // 341 // sudog is necessary because the g ↔ synchronization object relation 342 // is many-to-many. A g can be on many wait lists, so there may be 343 // many sudogs for one g; and many gs may be waiting on the same 344 // synchronization object, so there may be many sudogs for one object. 345 // 346 // sudogs are allocated from a special pool. Use acquireSudog and 347 // releaseSudog to allocate and free them. 348 type sudog struct { 349 // The following fields are protected by the hchan.lock of the 350 // channel this sudog is blocking on. shrinkstack depends on 351 // this for sudogs involved in channel ops. 352 353 g *g 354 355 next *sudog 356 prev *sudog 357 elem unsafe.Pointer // data element (may point to stack) 358 359 // The following fields are never accessed concurrently. 360 // For channels, waitlink is only accessed by g. 361 // For semaphores, all fields (including the ones above) 362 // are only accessed when holding a semaRoot lock. 363 364 acquiretime int64 365 releasetime int64 366 ticket uint32 367 368 // isSelect indicates g is participating in a select, so 369 // g.selectDone must be CAS'd to win the wake-up race. 370 isSelect bool 371 372 // success indicates whether communication over channel c 373 // succeeded. It is true if the goroutine was awoken because a 374 // value was delivered over channel c, and false if awoken 375 // because c was closed. 376 success bool 377 378 parent *sudog // semaRoot binary tree 379 waitlink *sudog // g.waiting list or semaRoot 380 waittail *sudog // semaRoot 381 c *hchan // channel 382 } 383 384 type libcall struct { 385 fn uintptr 386 n uintptr // number of parameters 387 args uintptr // parameters 388 r1 uintptr // return values 389 r2 uintptr 390 err uintptr // error number 391 } 392 393 // Stack describes a Go execution stack. 394 // The bounds of the stack are exactly [lo, hi), 395 // with no implicit data structures on either side. 396 type stack struct { 397 lo uintptr 398 hi uintptr 399 } 400 401 // heldLockInfo gives info on a held lock and the rank of that lock 402 type heldLockInfo struct { 403 lockAddr uintptr 404 rank lockRank 405 } 406 407 type g struct { 408 // Stack parameters. 409 // stack describes the actual stack memory: [stack.lo, stack.hi). 410 // stackguard0 is the stack pointer compared in the Go stack growth prologue. 411 // It is stack.lo+StackGuard normally, but can be StackPreempt to trigger a preemption. 412 // stackguard1 is the stack pointer compared in the C stack growth prologue. 413 // It is stack.lo+StackGuard on g0 and gsignal stacks. 414 // It is ~0 on other goroutine stacks, to trigger a call to morestackc (and crash). 415 stack stack // offset known to runtime/cgo 416 stackguard0 uintptr // offset known to liblink 417 stackguard1 uintptr // offset known to liblink 418 419 _panic *_panic // innermost panic - offset known to liblink 420 _defer *_defer // innermost defer 421 m *m // current m; offset known to arm liblink 422 sched gobuf 423 syscallsp uintptr // if status==Gsyscall, syscallsp = sched.sp to use during gc 424 syscallpc uintptr // if status==Gsyscall, syscallpc = sched.pc to use during gc 425 stktopsp uintptr // expected sp at top of stack, to check in traceback 426 // param is a generic pointer parameter field used to pass 427 // values in particular contexts where other storage for the 428 // parameter would be difficult to find. It is currently used 429 // in three ways: 430 // 1. When a channel operation wakes up a blocked goroutine, it sets param to 431 // point to the sudog of the completed blocking operation. 432 // 2. By gcAssistAlloc1 to signal back to its caller that the goroutine completed 433 // the GC cycle. It is unsafe to do so in any other way, because the goroutine's 434 // stack may have moved in the meantime. 435 // 3. By debugCallWrap to pass parameters to a new goroutine because allocating a 436 // closure in the runtime is forbidden. 437 param unsafe.Pointer 438 atomicstatus atomic.Uint32 439 stackLock uint32 // sigprof/scang lock; TODO: fold in to atomicstatus 440 goid uint64 441 schedlink guintptr 442 waitsince int64 // approx time when the g become blocked 443 waitreason waitReason // if status==Gwaiting 444 445 preempt bool // preemption signal, duplicates stackguard0 = stackpreempt 446 preemptStop bool // transition to _Gpreempted on preemption; otherwise, just deschedule 447 preemptShrink bool // shrink stack at synchronous safe point 448 449 // asyncSafePoint is set if g is stopped at an asynchronous 450 // safe point. This means there are frames on the stack 451 // without precise pointer information. 452 asyncSafePoint bool 453 454 paniconfault bool // panic (instead of crash) on unexpected fault address 455 gcscandone bool // g has scanned stack; protected by _Gscan bit in status 456 throwsplit bool // must not split stack 457 // activeStackChans indicates that there are unlocked channels 458 // pointing into this goroutine's stack. If true, stack 459 // copying needs to acquire channel locks to protect these 460 // areas of the stack. 461 activeStackChans bool 462 // parkingOnChan indicates that the goroutine is about to 463 // park on a chansend or chanrecv. Used to signal an unsafe point 464 // for stack shrinking. 465 parkingOnChan atomic.Bool 466 467 raceignore int8 // ignore race detection events 468 sysblocktraced bool // StartTrace has emitted EvGoInSyscall about this goroutine 469 tracking bool // whether we're tracking this G for sched latency statistics 470 trackingSeq uint8 // used to decide whether to track this G 471 trackingStamp int64 // timestamp of when the G last started being tracked 472 runnableTime int64 // the amount of time spent runnable, cleared when running, only used when tracking 473 sysexitticks int64 // cputicks when syscall has returned (for tracing) 474 traceseq uint64 // trace event sequencer 475 tracelastp puintptr // last P emitted an event for this goroutine 476 lockedm muintptr 477 sig uint32 478 writebuf []byte 479 sigcode0 uintptr 480 sigcode1 uintptr 481 sigpc uintptr 482 gopc uintptr // pc of go statement that created this goroutine 483 ancestors *[]ancestorInfo // ancestor information goroutine(s) that created this goroutine (only used if debug.tracebackancestors) 484 startpc uintptr // pc of goroutine function 485 racectx uintptr 486 waiting *sudog // sudog structures this g is waiting on (that have a valid elem ptr); in lock order 487 cgoCtxt []uintptr // cgo traceback context 488 labels unsafe.Pointer // profiler labels 489 timer *timer // cached timer for time.Sleep 490 selectDone atomic.Uint32 // are we participating in a select and did someone win the race? 491 492 // goroutineProfiled indicates the status of this goroutine's stack for the 493 // current in-progress goroutine profile 494 goroutineProfiled goroutineProfileStateHolder 495 496 // Per-G GC state 497 498 // gcAssistBytes is this G's GC assist credit in terms of 499 // bytes allocated. If this is positive, then the G has credit 500 // to allocate gcAssistBytes bytes without assisting. If this 501 // is negative, then the G must correct this by performing 502 // scan work. We track this in bytes to make it fast to update 503 // and check for debt in the malloc hot path. The assist ratio 504 // determines how this corresponds to scan work debt. 505 gcAssistBytes int64 506 } 507 508 // gTrackingPeriod is the number of transitions out of _Grunning between 509 // latency tracking runs. 510 const gTrackingPeriod = 8 511 512 const ( 513 // tlsSlots is the number of pointer-sized slots reserved for TLS on some platforms, 514 // like Windows. 515 tlsSlots = 6 516 tlsSize = tlsSlots * goarch.PtrSize 517 ) 518 519 // Values for m.freeWait. 520 const ( 521 freeMStack = 0 // M done, free stack and reference. 522 freeMRef = 1 // M done, free reference. 523 freeMWait = 2 // M still in use. 524 ) 525 526 type m struct { 527 g0 *g // goroutine with scheduling stack 528 morebuf gobuf // gobuf arg to morestack 529 divmod uint32 // div/mod denominator for arm - known to liblink 530 _ uint32 // align next field to 8 bytes 531 532 // Fields not known to debuggers. 533 procid uint64 // for debuggers, but offset not hard-coded 534 gsignal *g // signal-handling g 535 goSigStack gsignalStack // Go-allocated signal handling stack 536 sigmask sigset // storage for saved signal mask 537 tls [tlsSlots]uintptr // thread-local storage (for x86 extern register) 538 mstartfn func() 539 curg *g // current running goroutine 540 caughtsig guintptr // goroutine running during fatal signal 541 p puintptr // attached p for executing go code (nil if not executing go code) 542 nextp puintptr 543 oldp puintptr // the p that was attached before executing a syscall 544 id int64 545 mallocing int32 546 throwing throwType 547 preemptoff string // if != "", keep curg running on this m 548 locks int32 549 dying int32 550 profilehz int32 551 spinning bool // m is out of work and is actively looking for work 552 blocked bool // m is blocked on a note 553 newSigstack bool // minit on C thread called sigaltstack 554 printlock int8 555 incgo bool // m is executing a cgo call 556 isextra bool // m is an extra m 557 freeWait atomic.Uint32 // Whether it is safe to free g0 and delete m (one of freeMRef, freeMStack, freeMWait) 558 fastrand uint64 559 needextram bool 560 traceback uint8 561 ncgocall uint64 // number of cgo calls in total 562 ncgo int32 // number of cgo calls currently in progress 563 cgoCallersUse atomic.Uint32 // if non-zero, cgoCallers in use temporarily 564 cgoCallers *cgoCallers // cgo traceback if crashing in cgo call 565 park note 566 alllink *m // on allm 567 schedlink muintptr 568 lockedg guintptr 569 createstack [32]uintptr // stack that created this thread. 570 lockedExt uint32 // tracking for external LockOSThread 571 lockedInt uint32 // tracking for internal lockOSThread 572 nextwaitm muintptr // next m waiting for lock 573 waitunlockf func(*g, unsafe.Pointer) bool 574 waitlock unsafe.Pointer 575 waittraceev byte 576 waittraceskip int 577 startingtrace bool 578 syscalltick uint32 579 freelink *m // on sched.freem 580 581 // these are here because they are too large to be on the stack 582 // of low-level NOSPLIT functions. 583 libcall libcall 584 libcallpc uintptr // for cpu profiler 585 libcallsp uintptr 586 libcallg guintptr 587 syscall libcall // stores syscall parameters on windows 588 589 vdsoSP uintptr // SP for traceback while in VDSO call (0 if not in call) 590 vdsoPC uintptr // PC for traceback while in VDSO call 591 592 // preemptGen counts the number of completed preemption 593 // signals. This is used to detect when a preemption is 594 // requested, but fails. 595 preemptGen atomic.Uint32 596 597 // Whether this is a pending preemption signal on this M. 598 signalPending atomic.Uint32 599 600 dlogPerM 601 602 mOS 603 604 // Up to 10 locks held by this m, maintained by the lock ranking code. 605 locksHeldLen int 606 locksHeld [10]heldLockInfo 607 } 608 609 type p struct { 610 id int32 611 status uint32 // one of pidle/prunning/... 612 link puintptr 613 schedtick uint32 // incremented on every scheduler call 614 syscalltick uint32 // incremented on every system call 615 sysmontick sysmontick // last tick observed by sysmon 616 m muintptr // back-link to associated m (nil if idle) 617 mcache *mcache 618 pcache pageCache 619 raceprocctx uintptr 620 621 deferpool []*_defer // pool of available defer structs (see panic.go) 622 deferpoolbuf [32]*_defer 623 624 // Cache of goroutine ids, amortizes accesses to runtime·sched.goidgen. 625 goidcache uint64 626 goidcacheend uint64 627 628 // Queue of runnable goroutines. Accessed without lock. 629 runqhead uint32 630 runqtail uint32 631 runq [256]guintptr 632 // runnext, if non-nil, is a runnable G that was ready'd by 633 // the current G and should be run next instead of what's in 634 // runq if there's time remaining in the running G's time 635 // slice. It will inherit the time left in the current time 636 // slice. If a set of goroutines is locked in a 637 // communicate-and-wait pattern, this schedules that set as a 638 // unit and eliminates the (potentially large) scheduling 639 // latency that otherwise arises from adding the ready'd 640 // goroutines to the end of the run queue. 641 // 642 // Note that while other P's may atomically CAS this to zero, 643 // only the owner P can CAS it to a valid G. 644 runnext guintptr 645 646 // Available G's (status == Gdead) 647 gFree struct { 648 gList 649 n int32 650 } 651 652 sudogcache []*sudog 653 sudogbuf [128]*sudog 654 655 // Cache of mspan objects from the heap. 656 mspancache struct { 657 // We need an explicit length here because this field is used 658 // in allocation codepaths where write barriers are not allowed, 659 // and eliminating the write barrier/keeping it eliminated from 660 // slice updates is tricky, moreso than just managing the length 661 // ourselves. 662 len int 663 buf [128]*mspan 664 } 665 666 tracebuf traceBufPtr 667 668 // traceSweep indicates the sweep events should be traced. 669 // This is used to defer the sweep start event until a span 670 // has actually been swept. 671 traceSweep bool 672 // traceSwept and traceReclaimed track the number of bytes 673 // swept and reclaimed by sweeping in the current sweep loop. 674 traceSwept, traceReclaimed uintptr 675 676 palloc persistentAlloc // per-P to avoid mutex 677 678 // The when field of the first entry on the timer heap. 679 // This is 0 if the timer heap is empty. 680 timer0When atomic.Int64 681 682 // The earliest known nextwhen field of a timer with 683 // timerModifiedEarlier status. Because the timer may have been 684 // modified again, there need not be any timer with this value. 685 // This is 0 if there are no timerModifiedEarlier timers. 686 timerModifiedEarliest atomic.Int64 687 688 // Per-P GC state 689 gcAssistTime int64 // Nanoseconds in assistAlloc 690 gcFractionalMarkTime int64 // Nanoseconds in fractional mark worker (atomic) 691 692 // limiterEvent tracks events for the GC CPU limiter. 693 limiterEvent limiterEvent 694 695 // gcMarkWorkerMode is the mode for the next mark worker to run in. 696 // That is, this is used to communicate with the worker goroutine 697 // selected for immediate execution by 698 // gcController.findRunnableGCWorker. When scheduling other goroutines, 699 // this field must be set to gcMarkWorkerNotWorker. 700 gcMarkWorkerMode gcMarkWorkerMode 701 // gcMarkWorkerStartTime is the nanotime() at which the most recent 702 // mark worker started. 703 gcMarkWorkerStartTime int64 704 705 // gcw is this P's GC work buffer cache. The work buffer is 706 // filled by write barriers, drained by mutator assists, and 707 // disposed on certain GC state transitions. 708 gcw gcWork 709 710 // wbBuf is this P's GC write barrier buffer. 711 // 712 // TODO: Consider caching this in the running G. 713 wbBuf wbBuf 714 715 runSafePointFn uint32 // if 1, run sched.safePointFn at next safe point 716 717 // statsSeq is a counter indicating whether this P is currently 718 // writing any stats. Its value is even when not, odd when it is. 719 statsSeq atomic.Uint32 720 721 // Lock for timers. We normally access the timers while running 722 // on this P, but the scheduler can also do it from a different P. 723 timersLock mutex 724 725 // Actions to take at some time. This is used to implement the 726 // standard library's time package. 727 // Must hold timersLock to access. 728 timers []*timer 729 730 // Number of timers in P's heap. 731 numTimers atomic.Uint32 732 733 // Number of timerDeleted timers in P's heap. 734 deletedTimers atomic.Uint32 735 736 // Race context used while executing timer functions. 737 timerRaceCtx uintptr 738 739 // maxStackScanDelta accumulates the amount of stack space held by 740 // live goroutines (i.e. those eligible for stack scanning). 741 // Flushed to gcController.maxStackScan once maxStackScanSlack 742 // or -maxStackScanSlack is reached. 743 maxStackScanDelta int64 744 745 // gc-time statistics about current goroutines 746 // Note that this differs from maxStackScan in that this 747 // accumulates the actual stack observed to be used at GC time (hi - sp), 748 // not an instantaneous measure of the total stack size that might need 749 // to be scanned (hi - lo). 750 scannedStackSize uint64 // stack size of goroutines scanned by this P 751 scannedStacks uint64 // number of goroutines scanned by this P 752 753 // preempt is set to indicate that this P should be enter the 754 // scheduler ASAP (regardless of what G is running on it). 755 preempt bool 756 757 // Padding is no longer needed. False sharing is now not a worry because p is large enough 758 // that its size class is an integer multiple of the cache line size (for any of our architectures). 759 } 760 761 type schedt struct { 762 goidgen atomic.Uint64 763 lastpoll atomic.Int64 // time of last network poll, 0 if currently polling 764 pollUntil atomic.Int64 // time to which current poll is sleeping 765 766 lock mutex 767 768 // When increasing nmidle, nmidlelocked, nmsys, or nmfreed, be 769 // sure to call checkdead(). 770 771 midle muintptr // idle m's waiting for work 772 nmidle int32 // number of idle m's waiting for work 773 nmidlelocked int32 // number of locked m's waiting for work 774 mnext int64 // number of m's that have been created and next M ID 775 maxmcount int32 // maximum number of m's allowed (or die) 776 nmsys int32 // number of system m's not counted for deadlock 777 nmfreed int64 // cumulative number of freed m's 778 779 ngsys atomic.Int32 // number of system goroutines 780 781 pidle puintptr // idle p's 782 npidle atomic.Int32 783 nmspinning atomic.Int32 // See "Worker thread parking/unparking" comment in proc.go. 784 needspinning atomic.Uint32 // See "Delicate dance" comment in proc.go. Boolean. Must hold sched.lock to set to 1. 785 786 // Global runnable queue. 787 runq gQueue 788 runqsize int32 789 790 // disable controls selective disabling of the scheduler. 791 // 792 // Use schedEnableUser to control this. 793 // 794 // disable is protected by sched.lock. 795 disable struct { 796 // user disables scheduling of user goroutines. 797 user bool 798 runnable gQueue // pending runnable Gs 799 n int32 // length of runnable 800 } 801 802 // Global cache of dead G's. 803 gFree struct { 804 lock mutex 805 stack gList // Gs with stacks 806 noStack gList // Gs without stacks 807 n int32 808 } 809 810 // Central cache of sudog structs. 811 sudoglock mutex 812 sudogcache *sudog 813 814 // Central pool of available defer structs. 815 deferlock mutex 816 deferpool *_defer 817 818 // freem is the list of m's waiting to be freed when their 819 // m.exited is set. Linked through m.freelink. 820 freem *m 821 822 gcwaiting atomic.Bool // gc is waiting to run 823 stopwait int32 824 stopnote note 825 sysmonwait atomic.Bool 826 sysmonnote note 827 828 // safepointFn should be called on each P at the next GC 829 // safepoint if p.runSafePointFn is set. 830 safePointFn func(*p) 831 safePointWait int32 832 safePointNote note 833 834 profilehz int32 // cpu profiling rate 835 836 procresizetime int64 // nanotime() of last change to gomaxprocs 837 totaltime int64 // ∫gomaxprocs dt up to procresizetime 838 839 // sysmonlock protects sysmon's actions on the runtime. 840 // 841 // Acquire and hold this mutex to block sysmon from interacting 842 // with the rest of the runtime. 843 sysmonlock mutex 844 845 // timeToRun is a distribution of scheduling latencies, defined 846 // as the sum of time a G spends in the _Grunnable state before 847 // it transitions to _Grunning. 848 timeToRun timeHistogram 849 850 // idleTime is the total CPU time Ps have "spent" idle. 851 // 852 // Reset on each GC cycle. 853 idleTime atomic.Int64 854 855 // totalMutexWaitTime is the sum of time goroutines have spent in _Gwaiting 856 // with a waitreason of the form waitReasonSync{RW,}Mutex{R,}Lock. 857 totalMutexWaitTime atomic.Int64 858 } 859 860 // Values for the flags field of a sigTabT. 861 const ( 862 _SigNotify = 1 << iota // let signal.Notify have signal, even if from kernel 863 _SigKill // if signal.Notify doesn't take it, exit quietly 864 _SigThrow // if signal.Notify doesn't take it, exit loudly 865 _SigPanic // if the signal is from the kernel, panic 866 _SigDefault // if the signal isn't explicitly requested, don't monitor it 867 _SigGoExit // cause all runtime procs to exit (only used on Plan 9). 868 _SigSetStack // Don't explicitly install handler, but add SA_ONSTACK to existing libc handler 869 _SigUnblock // always unblock; see blockableSig 870 _SigIgn // _SIG_DFL action is to ignore the signal 871 ) 872 873 // Layout of in-memory per-function information prepared by linker 874 // See https://golang.org/s/go12symtab. 875 // Keep in sync with linker (../cmd/link/internal/ld/pcln.go:/pclntab) 876 // and with package debug/gosym and with symtab.go in package runtime. 877 type _func struct { 878 entryOff uint32 // start pc, as offset from moduledata.text/pcHeader.textStart 879 nameOff int32 // function name, as index into moduledata.funcnametab. 880 881 args int32 // in/out args size 882 deferreturn uint32 // offset of start of a deferreturn call instruction from entry, if any. 883 884 pcsp uint32 885 pcfile uint32 886 pcln uint32 887 npcdata uint32 888 cuOffset uint32 // runtime.cutab offset of this function's CU 889 startLine int32 // line number of start of function (func keyword/TEXT directive) 890 funcID funcID // set for certain special runtime functions 891 flag funcFlag 892 _ [1]byte // pad 893 nfuncdata uint8 // must be last, must end on a uint32-aligned boundary 894 895 // The end of the struct is followed immediately by two variable-length 896 // arrays that reference the pcdata and funcdata locations for this 897 // function. 898 899 // pcdata contains the offset into moduledata.pctab for the start of 900 // that index's table. e.g., 901 // &moduledata.pctab[_func.pcdata[_PCDATA_UnsafePoint]] is the start of 902 // the unsafe point table. 903 // 904 // An offset of 0 indicates that there is no table. 905 // 906 // pcdata [npcdata]uint32 907 908 // funcdata contains the offset past moduledata.gofunc which contains a 909 // pointer to that index's funcdata. e.g., 910 // *(moduledata.gofunc + _func.funcdata[_FUNCDATA_ArgsPointerMaps]) is 911 // the argument pointer map. 912 // 913 // An offset of ^uint32(0) indicates that there is no entry. 914 // 915 // funcdata [nfuncdata]uint32 916 } 917 918 // Pseudo-Func that is returned for PCs that occur in inlined code. 919 // A *Func can be either a *_func or a *funcinl, and they are distinguished 920 // by the first uintptr. 921 type funcinl struct { 922 ones uint32 // set to ^0 to distinguish from _func 923 entry uintptr // entry of the real (the "outermost") frame 924 name string 925 file string 926 line int32 927 startLine int32 928 } 929 930 // layout of Itab known to compilers 931 // allocated in non-garbage-collected memory 932 // Needs to be in sync with 933 // ../cmd/compile/internal/reflectdata/reflect.go:/^func.WriteTabs. 934 type itab struct { 935 inter *interfacetype 936 _type *_type 937 hash uint32 // copy of _type.hash. Used for type switches. 938 _ [4]byte 939 fun [1]uintptr // variable sized. fun[0]==0 means _type does not implement inter. 940 } 941 942 // Lock-free stack node. 943 // Also known to export_test.go. 944 type lfnode struct { 945 next uint64 946 pushcnt uintptr 947 } 948 949 type forcegcstate struct { 950 lock mutex 951 g *g 952 idle atomic.Bool 953 } 954 955 // extendRandom extends the random numbers in r[:n] to the whole slice r. 956 // Treats n<0 as n==0. 957 func extendRandom(r []byte, n int) { 958 if n < 0 { 959 n = 0 960 } 961 for n < len(r) { 962 // Extend random bits using hash function & time seed 963 w := n 964 if w > 16 { 965 w = 16 966 } 967 h := memhash(unsafe.Pointer(&r[n-w]), uintptr(nanotime()), uintptr(w)) 968 for i := 0; i < goarch.PtrSize && n < len(r); i++ { 969 r[n] = byte(h) 970 n++ 971 h >>= 8 972 } 973 } 974 } 975 976 // A _defer holds an entry on the list of deferred calls. 977 // If you add a field here, add code to clear it in deferProcStack. 978 // This struct must match the code in cmd/compile/internal/ssagen/ssa.go:deferstruct 979 // and cmd/compile/internal/ssagen/ssa.go:(*state).call. 980 // Some defers will be allocated on the stack and some on the heap. 981 // All defers are logically part of the stack, so write barriers to 982 // initialize them are not required. All defers must be manually scanned, 983 // and for heap defers, marked. 984 type _defer struct { 985 started bool 986 heap bool 987 // openDefer indicates that this _defer is for a frame with open-coded 988 // defers. We have only one defer record for the entire frame (which may 989 // currently have 0, 1, or more defers active). 990 openDefer bool 991 sp uintptr // sp at time of defer 992 pc uintptr // pc at time of defer 993 fn func() // can be nil for open-coded defers 994 _panic *_panic // panic that is running defer 995 link *_defer // next defer on G; can point to either heap or stack! 996 997 // If openDefer is true, the fields below record values about the stack 998 // frame and associated function that has the open-coded defer(s). sp 999 // above will be the sp for the frame, and pc will be address of the 1000 // deferreturn call in the function. 1001 fd unsafe.Pointer // funcdata for the function associated with the frame 1002 varp uintptr // value of varp for the stack frame 1003 // framepc is the current pc associated with the stack frame. Together, 1004 // with sp above (which is the sp associated with the stack frame), 1005 // framepc/sp can be used as pc/sp pair to continue a stack trace via 1006 // gentraceback(). 1007 framepc uintptr 1008 } 1009 1010 // A _panic holds information about an active panic. 1011 // 1012 // A _panic value must only ever live on the stack. 1013 // 1014 // The argp and link fields are stack pointers, but don't need special 1015 // handling during stack growth: because they are pointer-typed and 1016 // _panic values only live on the stack, regular stack pointer 1017 // adjustment takes care of them. 1018 type _panic struct { 1019 argp unsafe.Pointer // pointer to arguments of deferred call run during panic; cannot move - known to liblink 1020 arg any // argument to panic 1021 link *_panic // link to earlier panic 1022 pc uintptr // where to return to in runtime if this panic is bypassed 1023 sp unsafe.Pointer // where to return to in runtime if this panic is bypassed 1024 recovered bool // whether this panic is over 1025 aborted bool // the panic was aborted 1026 goexit bool 1027 } 1028 1029 // ancestorInfo records details of where a goroutine was started. 1030 type ancestorInfo struct { 1031 pcs []uintptr // pcs from the stack of this goroutine 1032 goid uint64 // goroutine id of this goroutine; original goroutine possibly dead 1033 gopc uintptr // pc of go statement that created this goroutine 1034 } 1035 1036 const ( 1037 _TraceRuntimeFrames = 1 << iota // include frames for internal runtime functions. 1038 _TraceTrap // the initial PC, SP are from a trap, not a return PC from a call 1039 _TraceJumpStack // if traceback is on a systemstack, resume trace at g that called into it 1040 ) 1041 1042 // The maximum number of frames we print for a traceback 1043 const _TracebackMaxFrames = 100 1044 1045 // A waitReason explains why a goroutine has been stopped. 1046 // See gopark. Do not re-use waitReasons, add new ones. 1047 type waitReason uint8 1048 1049 const ( 1050 waitReasonZero waitReason = iota // "" 1051 waitReasonGCAssistMarking // "GC assist marking" 1052 waitReasonIOWait // "IO wait" 1053 waitReasonChanReceiveNilChan // "chan receive (nil chan)" 1054 waitReasonChanSendNilChan // "chan send (nil chan)" 1055 waitReasonDumpingHeap // "dumping heap" 1056 waitReasonGarbageCollection // "garbage collection" 1057 waitReasonGarbageCollectionScan // "garbage collection scan" 1058 waitReasonPanicWait // "panicwait" 1059 waitReasonSelect // "select" 1060 waitReasonSelectNoCases // "select (no cases)" 1061 waitReasonGCAssistWait // "GC assist wait" 1062 waitReasonGCSweepWait // "GC sweep wait" 1063 waitReasonGCScavengeWait // "GC scavenge wait" 1064 waitReasonChanReceive // "chan receive" 1065 waitReasonChanSend // "chan send" 1066 waitReasonFinalizerWait // "finalizer wait" 1067 waitReasonForceGCIdle // "force gc (idle)" 1068 waitReasonSemacquire // "semacquire" 1069 waitReasonSleep // "sleep" 1070 waitReasonSyncCondWait // "sync.Cond.Wait" 1071 waitReasonSyncMutexLock // "sync.Mutex.Lock" 1072 waitReasonSyncRWMutexRLock // "sync.RWMutex.RLock" 1073 waitReasonSyncRWMutexLock // "sync.RWMutex.Lock" 1074 waitReasonTraceReaderBlocked // "trace reader (blocked)" 1075 waitReasonWaitForGCCycle // "wait for GC cycle" 1076 waitReasonGCWorkerIdle // "GC worker (idle)" 1077 waitReasonGCWorkerActive // "GC worker (active)" 1078 waitReasonPreempted // "preempted" 1079 waitReasonDebugCall // "debug call" 1080 waitReasonGCMarkTermination // "GC mark termination" 1081 waitReasonStoppingTheWorld // "stopping the world" 1082 ) 1083 1084 var waitReasonStrings = [...]string{ 1085 waitReasonZero: "", 1086 waitReasonGCAssistMarking: "GC assist marking", 1087 waitReasonIOWait: "IO wait", 1088 waitReasonChanReceiveNilChan: "chan receive (nil chan)", 1089 waitReasonChanSendNilChan: "chan send (nil chan)", 1090 waitReasonDumpingHeap: "dumping heap", 1091 waitReasonGarbageCollection: "garbage collection", 1092 waitReasonGarbageCollectionScan: "garbage collection scan", 1093 waitReasonPanicWait: "panicwait", 1094 waitReasonSelect: "select", 1095 waitReasonSelectNoCases: "select (no cases)", 1096 waitReasonGCAssistWait: "GC assist wait", 1097 waitReasonGCSweepWait: "GC sweep wait", 1098 waitReasonGCScavengeWait: "GC scavenge wait", 1099 waitReasonChanReceive: "chan receive", 1100 waitReasonChanSend: "chan send", 1101 waitReasonFinalizerWait: "finalizer wait", 1102 waitReasonForceGCIdle: "force gc (idle)", 1103 waitReasonSemacquire: "semacquire", 1104 waitReasonSleep: "sleep", 1105 waitReasonSyncCondWait: "sync.Cond.Wait", 1106 waitReasonSyncMutexLock: "sync.Mutex.Lock", 1107 waitReasonSyncRWMutexRLock: "sync.RWMutex.RLock", 1108 waitReasonSyncRWMutexLock: "sync.RWMutex.Lock", 1109 waitReasonTraceReaderBlocked: "trace reader (blocked)", 1110 waitReasonWaitForGCCycle: "wait for GC cycle", 1111 waitReasonGCWorkerIdle: "GC worker (idle)", 1112 waitReasonGCWorkerActive: "GC worker (active)", 1113 waitReasonPreempted: "preempted", 1114 waitReasonDebugCall: "debug call", 1115 waitReasonGCMarkTermination: "GC mark termination", 1116 waitReasonStoppingTheWorld: "stopping the world", 1117 } 1118 1119 func (w waitReason) String() string { 1120 if w < 0 || w >= waitReason(len(waitReasonStrings)) { 1121 return "unknown wait reason" 1122 } 1123 return waitReasonStrings[w] 1124 } 1125 1126 func (w waitReason) isMutexWait() bool { 1127 return w == waitReasonSyncMutexLock || 1128 w == waitReasonSyncRWMutexRLock || 1129 w == waitReasonSyncRWMutexLock 1130 } 1131 1132 var ( 1133 allm *m 1134 gomaxprocs int32 1135 ncpu int32 1136 forcegc forcegcstate 1137 sched schedt 1138 newprocs int32 1139 1140 // allpLock protects P-less reads and size changes of allp, idlepMask, 1141 // and timerpMask, and all writes to allp. 1142 allpLock mutex 1143 // len(allp) == gomaxprocs; may change at safe points, otherwise 1144 // immutable. 1145 allp []*p 1146 // Bitmask of Ps in _Pidle list, one bit per P. Reads and writes must 1147 // be atomic. Length may change at safe points. 1148 // 1149 // Each P must update only its own bit. In order to maintain 1150 // consistency, a P going idle must the idle mask simultaneously with 1151 // updates to the idle P list under the sched.lock, otherwise a racing 1152 // pidleget may clear the mask before pidleput sets the mask, 1153 // corrupting the bitmap. 1154 // 1155 // N.B., procresize takes ownership of all Ps in stopTheWorldWithSema. 1156 idlepMask pMask 1157 // Bitmask of Ps that may have a timer, one bit per P. Reads and writes 1158 // must be atomic. Length may change at safe points. 1159 timerpMask pMask 1160 1161 // Pool of GC parked background workers. Entries are type 1162 // *gcBgMarkWorkerNode. 1163 gcBgMarkWorkerPool lfstack 1164 1165 // Total number of gcBgMarkWorker goroutines. Protected by worldsema. 1166 gcBgMarkWorkerCount int32 1167 1168 // Information about what cpu features are available. 1169 // Packages outside the runtime should not use these 1170 // as they are not an external api. 1171 // Set on startup in asm_{386,amd64}.s 1172 processorVersionInfo uint32 1173 isIntel bool 1174 1175 goarm uint8 // set by cmd/link on arm systems 1176 ) 1177 1178 // Set by the linker so the runtime can determine the buildmode. 1179 var ( 1180 islibrary bool // -buildmode=c-shared 1181 isarchive bool // -buildmode=c-archive 1182 ) 1183 1184 // Must agree with internal/buildcfg.FramePointerEnabled. 1185 const framepointer_enabled = GOARCH == "amd64" || GOARCH == "arm64"