github.com/comwrg/go/src@v0.0.0-20220319063731-c238d0440370/runtime/time.go (about) 1 // Copyright 2009 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 // Time-related runtime and pieces of package time. 6 7 package runtime 8 9 import ( 10 "runtime/internal/atomic" 11 "runtime/internal/sys" 12 "unsafe" 13 ) 14 15 // Package time knows the layout of this structure. 16 // If this struct changes, adjust ../time/sleep.go:/runtimeTimer. 17 type timer struct { 18 // If this timer is on a heap, which P's heap it is on. 19 // puintptr rather than *p to match uintptr in the versions 20 // of this struct defined in other packages. 21 pp puintptr 22 23 // Timer wakes up at when, and then at when+period, ... (period > 0 only) 24 // each time calling f(arg, now) in the timer goroutine, so f must be 25 // a well-behaved function and not block. 26 // 27 // when must be positive on an active timer. 28 when int64 29 period int64 30 f func(interface{}, uintptr) 31 arg interface{} 32 seq uintptr 33 34 // What to set the when field to in timerModifiedXX status. 35 nextwhen int64 36 37 // The status field holds one of the values below. 38 status uint32 39 } 40 41 // Code outside this file has to be careful in using a timer value. 42 // 43 // The pp, status, and nextwhen fields may only be used by code in this file. 44 // 45 // Code that creates a new timer value can set the when, period, f, 46 // arg, and seq fields. 47 // A new timer value may be passed to addtimer (called by time.startTimer). 48 // After doing that no fields may be touched. 49 // 50 // An active timer (one that has been passed to addtimer) may be 51 // passed to deltimer (time.stopTimer), after which it is no longer an 52 // active timer. It is an inactive timer. 53 // In an inactive timer the period, f, arg, and seq fields may be modified, 54 // but not the when field. 55 // It's OK to just drop an inactive timer and let the GC collect it. 56 // It's not OK to pass an inactive timer to addtimer. 57 // Only newly allocated timer values may be passed to addtimer. 58 // 59 // An active timer may be passed to modtimer. No fields may be touched. 60 // It remains an active timer. 61 // 62 // An inactive timer may be passed to resettimer to turn into an 63 // active timer with an updated when field. 64 // It's OK to pass a newly allocated timer value to resettimer. 65 // 66 // Timer operations are addtimer, deltimer, modtimer, resettimer, 67 // cleantimers, adjusttimers, and runtimer. 68 // 69 // We don't permit calling addtimer/deltimer/modtimer/resettimer simultaneously, 70 // but adjusttimers and runtimer can be called at the same time as any of those. 71 // 72 // Active timers live in heaps attached to P, in the timers field. 73 // Inactive timers live there too temporarily, until they are removed. 74 // 75 // addtimer: 76 // timerNoStatus -> timerWaiting 77 // anything else -> panic: invalid value 78 // deltimer: 79 // timerWaiting -> timerModifying -> timerDeleted 80 // timerModifiedEarlier -> timerModifying -> timerDeleted 81 // timerModifiedLater -> timerModifying -> timerDeleted 82 // timerNoStatus -> do nothing 83 // timerDeleted -> do nothing 84 // timerRemoving -> do nothing 85 // timerRemoved -> do nothing 86 // timerRunning -> wait until status changes 87 // timerMoving -> wait until status changes 88 // timerModifying -> wait until status changes 89 // modtimer: 90 // timerWaiting -> timerModifying -> timerModifiedXX 91 // timerModifiedXX -> timerModifying -> timerModifiedYY 92 // timerNoStatus -> timerModifying -> timerWaiting 93 // timerRemoved -> timerModifying -> timerWaiting 94 // timerDeleted -> timerModifying -> timerModifiedXX 95 // timerRunning -> wait until status changes 96 // timerMoving -> wait until status changes 97 // timerRemoving -> wait until status changes 98 // timerModifying -> wait until status changes 99 // cleantimers (looks in P's timer heap): 100 // timerDeleted -> timerRemoving -> timerRemoved 101 // timerModifiedXX -> timerMoving -> timerWaiting 102 // adjusttimers (looks in P's timer heap): 103 // timerDeleted -> timerRemoving -> timerRemoved 104 // timerModifiedXX -> timerMoving -> timerWaiting 105 // runtimer (looks in P's timer heap): 106 // timerNoStatus -> panic: uninitialized timer 107 // timerWaiting -> timerWaiting or 108 // timerWaiting -> timerRunning -> timerNoStatus or 109 // timerWaiting -> timerRunning -> timerWaiting 110 // timerModifying -> wait until status changes 111 // timerModifiedXX -> timerMoving -> timerWaiting 112 // timerDeleted -> timerRemoving -> timerRemoved 113 // timerRunning -> panic: concurrent runtimer calls 114 // timerRemoved -> panic: inconsistent timer heap 115 // timerRemoving -> panic: inconsistent timer heap 116 // timerMoving -> panic: inconsistent timer heap 117 118 // Values for the timer status field. 119 const ( 120 // Timer has no status set yet. 121 timerNoStatus = iota 122 123 // Waiting for timer to fire. 124 // The timer is in some P's heap. 125 timerWaiting 126 127 // Running the timer function. 128 // A timer will only have this status briefly. 129 timerRunning 130 131 // The timer is deleted and should be removed. 132 // It should not be run, but it is still in some P's heap. 133 timerDeleted 134 135 // The timer is being removed. 136 // The timer will only have this status briefly. 137 timerRemoving 138 139 // The timer has been stopped. 140 // It is not in any P's heap. 141 timerRemoved 142 143 // The timer is being modified. 144 // The timer will only have this status briefly. 145 timerModifying 146 147 // The timer has been modified to an earlier time. 148 // The new when value is in the nextwhen field. 149 // The timer is in some P's heap, possibly in the wrong place. 150 timerModifiedEarlier 151 152 // The timer has been modified to the same or a later time. 153 // The new when value is in the nextwhen field. 154 // The timer is in some P's heap, possibly in the wrong place. 155 timerModifiedLater 156 157 // The timer has been modified and is being moved. 158 // The timer will only have this status briefly. 159 timerMoving 160 ) 161 162 // maxWhen is the maximum value for timer's when field. 163 const maxWhen = 1<<63 - 1 164 165 // verifyTimers can be set to true to add debugging checks that the 166 // timer heaps are valid. 167 const verifyTimers = false 168 169 // Package time APIs. 170 // Godoc uses the comments in package time, not these. 171 172 // time.now is implemented in assembly. 173 174 // timeSleep puts the current goroutine to sleep for at least ns nanoseconds. 175 //go:linkname timeSleep time.Sleep 176 func timeSleep(ns int64) { 177 if ns <= 0 { 178 return 179 } 180 181 gp := getg() 182 t := gp.timer 183 if t == nil { 184 t = new(timer) 185 gp.timer = t 186 } 187 t.f = goroutineReady 188 t.arg = gp 189 t.nextwhen = nanotime() + ns 190 if t.nextwhen < 0 { // check for overflow. 191 t.nextwhen = maxWhen 192 } 193 gopark(resetForSleep, unsafe.Pointer(t), waitReasonSleep, traceEvGoSleep, 1) 194 } 195 196 // resetForSleep is called after the goroutine is parked for timeSleep. 197 // We can't call resettimer in timeSleep itself because if this is a short 198 // sleep and there are many goroutines then the P can wind up running the 199 // timer function, goroutineReady, before the goroutine has been parked. 200 func resetForSleep(gp *g, ut unsafe.Pointer) bool { 201 t := (*timer)(ut) 202 resettimer(t, t.nextwhen) 203 return true 204 } 205 206 // startTimer adds t to the timer heap. 207 //go:linkname startTimer time.startTimer 208 func startTimer(t *timer) { 209 if raceenabled { 210 racerelease(unsafe.Pointer(t)) 211 } 212 addtimer(t) 213 } 214 215 // stopTimer stops a timer. 216 // It reports whether t was stopped before being run. 217 //go:linkname stopTimer time.stopTimer 218 func stopTimer(t *timer) bool { 219 return deltimer(t) 220 } 221 222 // resetTimer resets an inactive timer, adding it to the heap. 223 //go:linkname resetTimer time.resetTimer 224 // Reports whether the timer was modified before it was run. 225 func resetTimer(t *timer, when int64) bool { 226 if raceenabled { 227 racerelease(unsafe.Pointer(t)) 228 } 229 return resettimer(t, when) 230 } 231 232 // modTimer modifies an existing timer. 233 //go:linkname modTimer time.modTimer 234 func modTimer(t *timer, when, period int64, f func(interface{}, uintptr), arg interface{}, seq uintptr) { 235 modtimer(t, when, period, f, arg, seq) 236 } 237 238 // Go runtime. 239 240 // Ready the goroutine arg. 241 func goroutineReady(arg interface{}, seq uintptr) { 242 goready(arg.(*g), 0) 243 } 244 245 // addtimer adds a timer to the current P. 246 // This should only be called with a newly created timer. 247 // That avoids the risk of changing the when field of a timer in some P's heap, 248 // which could cause the heap to become unsorted. 249 func addtimer(t *timer) { 250 // when must be positive. A negative value will cause runtimer to 251 // overflow during its delta calculation and never expire other runtime 252 // timers. Zero will cause checkTimers to fail to notice the timer. 253 if t.when <= 0 { 254 throw("timer when must be positive") 255 } 256 if t.period < 0 { 257 throw("timer period must be non-negative") 258 } 259 if t.status != timerNoStatus { 260 throw("addtimer called with initialized timer") 261 } 262 t.status = timerWaiting 263 264 when := t.when 265 266 // Disable preemption while using pp to avoid changing another P's heap. 267 mp := acquirem() 268 269 pp := getg().m.p.ptr() 270 lock(&pp.timersLock) 271 cleantimers(pp) 272 doaddtimer(pp, t) 273 unlock(&pp.timersLock) 274 275 wakeNetPoller(when) 276 277 releasem(mp) 278 } 279 280 // doaddtimer adds t to the current P's heap. 281 // The caller must have locked the timers for pp. 282 func doaddtimer(pp *p, t *timer) { 283 // Timers rely on the network poller, so make sure the poller 284 // has started. 285 if netpollInited == 0 { 286 netpollGenericInit() 287 } 288 289 if t.pp != 0 { 290 throw("doaddtimer: P already set in timer") 291 } 292 t.pp.set(pp) 293 i := len(pp.timers) 294 pp.timers = append(pp.timers, t) 295 siftupTimer(pp.timers, i) 296 if t == pp.timers[0] { 297 atomic.Store64(&pp.timer0When, uint64(t.when)) 298 } 299 atomic.Xadd(&pp.numTimers, 1) 300 } 301 302 // deltimer deletes the timer t. It may be on some other P, so we can't 303 // actually remove it from the timers heap. We can only mark it as deleted. 304 // It will be removed in due course by the P whose heap it is on. 305 // Reports whether the timer was removed before it was run. 306 func deltimer(t *timer) bool { 307 for { 308 switch s := atomic.Load(&t.status); s { 309 case timerWaiting, timerModifiedLater: 310 // Prevent preemption while the timer is in timerModifying. 311 // This could lead to a self-deadlock. See #38070. 312 mp := acquirem() 313 if atomic.Cas(&t.status, s, timerModifying) { 314 // Must fetch t.pp before changing status, 315 // as cleantimers in another goroutine 316 // can clear t.pp of a timerDeleted timer. 317 tpp := t.pp.ptr() 318 if !atomic.Cas(&t.status, timerModifying, timerDeleted) { 319 badTimer() 320 } 321 releasem(mp) 322 atomic.Xadd(&tpp.deletedTimers, 1) 323 // Timer was not yet run. 324 return true 325 } else { 326 releasem(mp) 327 } 328 case timerModifiedEarlier: 329 // Prevent preemption while the timer is in timerModifying. 330 // This could lead to a self-deadlock. See #38070. 331 mp := acquirem() 332 if atomic.Cas(&t.status, s, timerModifying) { 333 // Must fetch t.pp before setting status 334 // to timerDeleted. 335 tpp := t.pp.ptr() 336 if !atomic.Cas(&t.status, timerModifying, timerDeleted) { 337 badTimer() 338 } 339 releasem(mp) 340 atomic.Xadd(&tpp.deletedTimers, 1) 341 // Timer was not yet run. 342 return true 343 } else { 344 releasem(mp) 345 } 346 case timerDeleted, timerRemoving, timerRemoved: 347 // Timer was already run. 348 return false 349 case timerRunning, timerMoving: 350 // The timer is being run or moved, by a different P. 351 // Wait for it to complete. 352 osyield() 353 case timerNoStatus: 354 // Removing timer that was never added or 355 // has already been run. Also see issue 21874. 356 return false 357 case timerModifying: 358 // Simultaneous calls to deltimer and modtimer. 359 // Wait for the other call to complete. 360 osyield() 361 default: 362 badTimer() 363 } 364 } 365 } 366 367 // dodeltimer removes timer i from the current P's heap. 368 // We are locked on the P when this is called. 369 // It returns the smallest changed index in pp.timers. 370 // The caller must have locked the timers for pp. 371 func dodeltimer(pp *p, i int) int { 372 if t := pp.timers[i]; t.pp.ptr() != pp { 373 throw("dodeltimer: wrong P") 374 } else { 375 t.pp = 0 376 } 377 last := len(pp.timers) - 1 378 if i != last { 379 pp.timers[i] = pp.timers[last] 380 } 381 pp.timers[last] = nil 382 pp.timers = pp.timers[:last] 383 smallestChanged := i 384 if i != last { 385 // Moving to i may have moved the last timer to a new parent, 386 // so sift up to preserve the heap guarantee. 387 smallestChanged = siftupTimer(pp.timers, i) 388 siftdownTimer(pp.timers, i) 389 } 390 if i == 0 { 391 updateTimer0When(pp) 392 } 393 atomic.Xadd(&pp.numTimers, -1) 394 return smallestChanged 395 } 396 397 // dodeltimer0 removes timer 0 from the current P's heap. 398 // We are locked on the P when this is called. 399 // It reports whether it saw no problems due to races. 400 // The caller must have locked the timers for pp. 401 func dodeltimer0(pp *p) { 402 if t := pp.timers[0]; t.pp.ptr() != pp { 403 throw("dodeltimer0: wrong P") 404 } else { 405 t.pp = 0 406 } 407 last := len(pp.timers) - 1 408 if last > 0 { 409 pp.timers[0] = pp.timers[last] 410 } 411 pp.timers[last] = nil 412 pp.timers = pp.timers[:last] 413 if last > 0 { 414 siftdownTimer(pp.timers, 0) 415 } 416 updateTimer0When(pp) 417 atomic.Xadd(&pp.numTimers, -1) 418 } 419 420 // modtimer modifies an existing timer. 421 // This is called by the netpoll code or time.Ticker.Reset or time.Timer.Reset. 422 // Reports whether the timer was modified before it was run. 423 func modtimer(t *timer, when, period int64, f func(interface{}, uintptr), arg interface{}, seq uintptr) bool { 424 if when <= 0 { 425 throw("timer when must be positive") 426 } 427 if period < 0 { 428 throw("timer period must be non-negative") 429 } 430 431 status := uint32(timerNoStatus) 432 wasRemoved := false 433 var pending bool 434 var mp *m 435 loop: 436 for { 437 switch status = atomic.Load(&t.status); status { 438 case timerWaiting, timerModifiedEarlier, timerModifiedLater: 439 // Prevent preemption while the timer is in timerModifying. 440 // This could lead to a self-deadlock. See #38070. 441 mp = acquirem() 442 if atomic.Cas(&t.status, status, timerModifying) { 443 pending = true // timer not yet run 444 break loop 445 } 446 releasem(mp) 447 case timerNoStatus, timerRemoved: 448 // Prevent preemption while the timer is in timerModifying. 449 // This could lead to a self-deadlock. See #38070. 450 mp = acquirem() 451 452 // Timer was already run and t is no longer in a heap. 453 // Act like addtimer. 454 if atomic.Cas(&t.status, status, timerModifying) { 455 wasRemoved = true 456 pending = false // timer already run or stopped 457 break loop 458 } 459 releasem(mp) 460 case timerDeleted: 461 // Prevent preemption while the timer is in timerModifying. 462 // This could lead to a self-deadlock. See #38070. 463 mp = acquirem() 464 if atomic.Cas(&t.status, status, timerModifying) { 465 atomic.Xadd(&t.pp.ptr().deletedTimers, -1) 466 pending = false // timer already stopped 467 break loop 468 } 469 releasem(mp) 470 case timerRunning, timerRemoving, timerMoving: 471 // The timer is being run or moved, by a different P. 472 // Wait for it to complete. 473 osyield() 474 case timerModifying: 475 // Multiple simultaneous calls to modtimer. 476 // Wait for the other call to complete. 477 osyield() 478 default: 479 badTimer() 480 } 481 } 482 483 t.period = period 484 t.f = f 485 t.arg = arg 486 t.seq = seq 487 488 if wasRemoved { 489 t.when = when 490 pp := getg().m.p.ptr() 491 lock(&pp.timersLock) 492 doaddtimer(pp, t) 493 unlock(&pp.timersLock) 494 if !atomic.Cas(&t.status, timerModifying, timerWaiting) { 495 badTimer() 496 } 497 releasem(mp) 498 wakeNetPoller(when) 499 } else { 500 // The timer is in some other P's heap, so we can't change 501 // the when field. If we did, the other P's heap would 502 // be out of order. So we put the new when value in the 503 // nextwhen field, and let the other P set the when field 504 // when it is prepared to resort the heap. 505 t.nextwhen = when 506 507 newStatus := uint32(timerModifiedLater) 508 if when < t.when { 509 newStatus = timerModifiedEarlier 510 } 511 512 tpp := t.pp.ptr() 513 514 if newStatus == timerModifiedEarlier { 515 updateTimerModifiedEarliest(tpp, when) 516 } 517 518 // Set the new status of the timer. 519 if !atomic.Cas(&t.status, timerModifying, newStatus) { 520 badTimer() 521 } 522 releasem(mp) 523 524 // If the new status is earlier, wake up the poller. 525 if newStatus == timerModifiedEarlier { 526 wakeNetPoller(when) 527 } 528 } 529 530 return pending 531 } 532 533 // resettimer resets the time when a timer should fire. 534 // If used for an inactive timer, the timer will become active. 535 // This should be called instead of addtimer if the timer value has been, 536 // or may have been, used previously. 537 // Reports whether the timer was modified before it was run. 538 func resettimer(t *timer, when int64) bool { 539 return modtimer(t, when, t.period, t.f, t.arg, t.seq) 540 } 541 542 // cleantimers cleans up the head of the timer queue. This speeds up 543 // programs that create and delete timers; leaving them in the heap 544 // slows down addtimer. Reports whether no timer problems were found. 545 // The caller must have locked the timers for pp. 546 func cleantimers(pp *p) { 547 gp := getg() 548 for { 549 if len(pp.timers) == 0 { 550 return 551 } 552 553 // This loop can theoretically run for a while, and because 554 // it is holding timersLock it cannot be preempted. 555 // If someone is trying to preempt us, just return. 556 // We can clean the timers later. 557 if gp.preemptStop { 558 return 559 } 560 561 t := pp.timers[0] 562 if t.pp.ptr() != pp { 563 throw("cleantimers: bad p") 564 } 565 switch s := atomic.Load(&t.status); s { 566 case timerDeleted: 567 if !atomic.Cas(&t.status, s, timerRemoving) { 568 continue 569 } 570 dodeltimer0(pp) 571 if !atomic.Cas(&t.status, timerRemoving, timerRemoved) { 572 badTimer() 573 } 574 atomic.Xadd(&pp.deletedTimers, -1) 575 case timerModifiedEarlier, timerModifiedLater: 576 if !atomic.Cas(&t.status, s, timerMoving) { 577 continue 578 } 579 // Now we can change the when field. 580 t.when = t.nextwhen 581 // Move t to the right position. 582 dodeltimer0(pp) 583 doaddtimer(pp, t) 584 if !atomic.Cas(&t.status, timerMoving, timerWaiting) { 585 badTimer() 586 } 587 default: 588 // Head of timers does not need adjustment. 589 return 590 } 591 } 592 } 593 594 // moveTimers moves a slice of timers to pp. The slice has been taken 595 // from a different P. 596 // This is currently called when the world is stopped, but the caller 597 // is expected to have locked the timers for pp. 598 func moveTimers(pp *p, timers []*timer) { 599 for _, t := range timers { 600 loop: 601 for { 602 switch s := atomic.Load(&t.status); s { 603 case timerWaiting: 604 if !atomic.Cas(&t.status, s, timerMoving) { 605 continue 606 } 607 t.pp = 0 608 doaddtimer(pp, t) 609 if !atomic.Cas(&t.status, timerMoving, timerWaiting) { 610 badTimer() 611 } 612 break loop 613 case timerModifiedEarlier, timerModifiedLater: 614 if !atomic.Cas(&t.status, s, timerMoving) { 615 continue 616 } 617 t.when = t.nextwhen 618 t.pp = 0 619 doaddtimer(pp, t) 620 if !atomic.Cas(&t.status, timerMoving, timerWaiting) { 621 badTimer() 622 } 623 break loop 624 case timerDeleted: 625 if !atomic.Cas(&t.status, s, timerRemoved) { 626 continue 627 } 628 t.pp = 0 629 // We no longer need this timer in the heap. 630 break loop 631 case timerModifying: 632 // Loop until the modification is complete. 633 osyield() 634 case timerNoStatus, timerRemoved: 635 // We should not see these status values in a timers heap. 636 badTimer() 637 case timerRunning, timerRemoving, timerMoving: 638 // Some other P thinks it owns this timer, 639 // which should not happen. 640 badTimer() 641 default: 642 badTimer() 643 } 644 } 645 } 646 } 647 648 // adjusttimers looks through the timers in the current P's heap for 649 // any timers that have been modified to run earlier, and puts them in 650 // the correct place in the heap. While looking for those timers, 651 // it also moves timers that have been modified to run later, 652 // and removes deleted timers. The caller must have locked the timers for pp. 653 func adjusttimers(pp *p, now int64) { 654 // If we haven't yet reached the time of the first timerModifiedEarlier 655 // timer, don't do anything. This speeds up programs that adjust 656 // a lot of timers back and forth if the timers rarely expire. 657 // We'll postpone looking through all the adjusted timers until 658 // one would actually expire. 659 first := atomic.Load64(&pp.timerModifiedEarliest) 660 if first == 0 || int64(first) > now { 661 if verifyTimers { 662 verifyTimerHeap(pp) 663 } 664 return 665 } 666 667 // We are going to clear all timerModifiedEarlier timers. 668 atomic.Store64(&pp.timerModifiedEarliest, 0) 669 670 var moved []*timer 671 for i := 0; i < len(pp.timers); i++ { 672 t := pp.timers[i] 673 if t.pp.ptr() != pp { 674 throw("adjusttimers: bad p") 675 } 676 switch s := atomic.Load(&t.status); s { 677 case timerDeleted: 678 if atomic.Cas(&t.status, s, timerRemoving) { 679 changed := dodeltimer(pp, i) 680 if !atomic.Cas(&t.status, timerRemoving, timerRemoved) { 681 badTimer() 682 } 683 atomic.Xadd(&pp.deletedTimers, -1) 684 // Go back to the earliest changed heap entry. 685 // "- 1" because the loop will add 1. 686 i = changed - 1 687 } 688 case timerModifiedEarlier, timerModifiedLater: 689 if atomic.Cas(&t.status, s, timerMoving) { 690 // Now we can change the when field. 691 t.when = t.nextwhen 692 // Take t off the heap, and hold onto it. 693 // We don't add it back yet because the 694 // heap manipulation could cause our 695 // loop to skip some other timer. 696 changed := dodeltimer(pp, i) 697 moved = append(moved, t) 698 // Go back to the earliest changed heap entry. 699 // "- 1" because the loop will add 1. 700 i = changed - 1 701 } 702 case timerNoStatus, timerRunning, timerRemoving, timerRemoved, timerMoving: 703 badTimer() 704 case timerWaiting: 705 // OK, nothing to do. 706 case timerModifying: 707 // Check again after modification is complete. 708 osyield() 709 i-- 710 default: 711 badTimer() 712 } 713 } 714 715 if len(moved) > 0 { 716 addAdjustedTimers(pp, moved) 717 } 718 719 if verifyTimers { 720 verifyTimerHeap(pp) 721 } 722 } 723 724 // addAdjustedTimers adds any timers we adjusted in adjusttimers 725 // back to the timer heap. 726 func addAdjustedTimers(pp *p, moved []*timer) { 727 for _, t := range moved { 728 doaddtimer(pp, t) 729 if !atomic.Cas(&t.status, timerMoving, timerWaiting) { 730 badTimer() 731 } 732 } 733 } 734 735 // nobarrierWakeTime looks at P's timers and returns the time when we 736 // should wake up the netpoller. It returns 0 if there are no timers. 737 // This function is invoked when dropping a P, and must run without 738 // any write barriers. 739 //go:nowritebarrierrec 740 func nobarrierWakeTime(pp *p) int64 { 741 next := int64(atomic.Load64(&pp.timer0When)) 742 nextAdj := int64(atomic.Load64(&pp.timerModifiedEarliest)) 743 if next == 0 || (nextAdj != 0 && nextAdj < next) { 744 next = nextAdj 745 } 746 return next 747 } 748 749 // runtimer examines the first timer in timers. If it is ready based on now, 750 // it runs the timer and removes or updates it. 751 // Returns 0 if it ran a timer, -1 if there are no more timers, or the time 752 // when the first timer should run. 753 // The caller must have locked the timers for pp. 754 // If a timer is run, this will temporarily unlock the timers. 755 //go:systemstack 756 func runtimer(pp *p, now int64) int64 { 757 for { 758 t := pp.timers[0] 759 if t.pp.ptr() != pp { 760 throw("runtimer: bad p") 761 } 762 switch s := atomic.Load(&t.status); s { 763 case timerWaiting: 764 if t.when > now { 765 // Not ready to run. 766 return t.when 767 } 768 769 if !atomic.Cas(&t.status, s, timerRunning) { 770 continue 771 } 772 // Note that runOneTimer may temporarily unlock 773 // pp.timersLock. 774 runOneTimer(pp, t, now) 775 return 0 776 777 case timerDeleted: 778 if !atomic.Cas(&t.status, s, timerRemoving) { 779 continue 780 } 781 dodeltimer0(pp) 782 if !atomic.Cas(&t.status, timerRemoving, timerRemoved) { 783 badTimer() 784 } 785 atomic.Xadd(&pp.deletedTimers, -1) 786 if len(pp.timers) == 0 { 787 return -1 788 } 789 790 case timerModifiedEarlier, timerModifiedLater: 791 if !atomic.Cas(&t.status, s, timerMoving) { 792 continue 793 } 794 t.when = t.nextwhen 795 dodeltimer0(pp) 796 doaddtimer(pp, t) 797 if !atomic.Cas(&t.status, timerMoving, timerWaiting) { 798 badTimer() 799 } 800 801 case timerModifying: 802 // Wait for modification to complete. 803 osyield() 804 805 case timerNoStatus, timerRemoved: 806 // Should not see a new or inactive timer on the heap. 807 badTimer() 808 case timerRunning, timerRemoving, timerMoving: 809 // These should only be set when timers are locked, 810 // and we didn't do it. 811 badTimer() 812 default: 813 badTimer() 814 } 815 } 816 } 817 818 // runOneTimer runs a single timer. 819 // The caller must have locked the timers for pp. 820 // This will temporarily unlock the timers while running the timer function. 821 //go:systemstack 822 func runOneTimer(pp *p, t *timer, now int64) { 823 if raceenabled { 824 ppcur := getg().m.p.ptr() 825 if ppcur.timerRaceCtx == 0 { 826 ppcur.timerRaceCtx = racegostart(funcPC(runtimer) + sys.PCQuantum) 827 } 828 raceacquirectx(ppcur.timerRaceCtx, unsafe.Pointer(t)) 829 } 830 831 f := t.f 832 arg := t.arg 833 seq := t.seq 834 835 if t.period > 0 { 836 // Leave in heap but adjust next time to fire. 837 delta := t.when - now 838 t.when += t.period * (1 + -delta/t.period) 839 if t.when < 0 { // check for overflow. 840 t.when = maxWhen 841 } 842 siftdownTimer(pp.timers, 0) 843 if !atomic.Cas(&t.status, timerRunning, timerWaiting) { 844 badTimer() 845 } 846 updateTimer0When(pp) 847 } else { 848 // Remove from heap. 849 dodeltimer0(pp) 850 if !atomic.Cas(&t.status, timerRunning, timerNoStatus) { 851 badTimer() 852 } 853 } 854 855 if raceenabled { 856 // Temporarily use the current P's racectx for g0. 857 gp := getg() 858 if gp.racectx != 0 { 859 throw("runOneTimer: unexpected racectx") 860 } 861 gp.racectx = gp.m.p.ptr().timerRaceCtx 862 } 863 864 unlock(&pp.timersLock) 865 866 f(arg, seq) 867 868 lock(&pp.timersLock) 869 870 if raceenabled { 871 gp := getg() 872 gp.racectx = 0 873 } 874 } 875 876 // clearDeletedTimers removes all deleted timers from the P's timer heap. 877 // This is used to avoid clogging up the heap if the program 878 // starts a lot of long-running timers and then stops them. 879 // For example, this can happen via context.WithTimeout. 880 // 881 // This is the only function that walks through the entire timer heap, 882 // other than moveTimers which only runs when the world is stopped. 883 // 884 // The caller must have locked the timers for pp. 885 func clearDeletedTimers(pp *p) { 886 // We are going to clear all timerModifiedEarlier timers. 887 // Do this now in case new ones show up while we are looping. 888 atomic.Store64(&pp.timerModifiedEarliest, 0) 889 890 cdel := int32(0) 891 to := 0 892 changedHeap := false 893 timers := pp.timers 894 nextTimer: 895 for _, t := range timers { 896 for { 897 switch s := atomic.Load(&t.status); s { 898 case timerWaiting: 899 if changedHeap { 900 timers[to] = t 901 siftupTimer(timers, to) 902 } 903 to++ 904 continue nextTimer 905 case timerModifiedEarlier, timerModifiedLater: 906 if atomic.Cas(&t.status, s, timerMoving) { 907 t.when = t.nextwhen 908 timers[to] = t 909 siftupTimer(timers, to) 910 to++ 911 changedHeap = true 912 if !atomic.Cas(&t.status, timerMoving, timerWaiting) { 913 badTimer() 914 } 915 continue nextTimer 916 } 917 case timerDeleted: 918 if atomic.Cas(&t.status, s, timerRemoving) { 919 t.pp = 0 920 cdel++ 921 if !atomic.Cas(&t.status, timerRemoving, timerRemoved) { 922 badTimer() 923 } 924 changedHeap = true 925 continue nextTimer 926 } 927 case timerModifying: 928 // Loop until modification complete. 929 osyield() 930 case timerNoStatus, timerRemoved: 931 // We should not see these status values in a timer heap. 932 badTimer() 933 case timerRunning, timerRemoving, timerMoving: 934 // Some other P thinks it owns this timer, 935 // which should not happen. 936 badTimer() 937 default: 938 badTimer() 939 } 940 } 941 } 942 943 // Set remaining slots in timers slice to nil, 944 // so that the timer values can be garbage collected. 945 for i := to; i < len(timers); i++ { 946 timers[i] = nil 947 } 948 949 atomic.Xadd(&pp.deletedTimers, -cdel) 950 atomic.Xadd(&pp.numTimers, -cdel) 951 952 timers = timers[:to] 953 pp.timers = timers 954 updateTimer0When(pp) 955 956 if verifyTimers { 957 verifyTimerHeap(pp) 958 } 959 } 960 961 // verifyTimerHeap verifies that the timer heap is in a valid state. 962 // This is only for debugging, and is only called if verifyTimers is true. 963 // The caller must have locked the timers. 964 func verifyTimerHeap(pp *p) { 965 for i, t := range pp.timers { 966 if i == 0 { 967 // First timer has no parent. 968 continue 969 } 970 971 // The heap is 4-ary. See siftupTimer and siftdownTimer. 972 p := (i - 1) / 4 973 if t.when < pp.timers[p].when { 974 print("bad timer heap at ", i, ": ", p, ": ", pp.timers[p].when, ", ", i, ": ", t.when, "\n") 975 throw("bad timer heap") 976 } 977 } 978 if numTimers := int(atomic.Load(&pp.numTimers)); len(pp.timers) != numTimers { 979 println("timer heap len", len(pp.timers), "!= numTimers", numTimers) 980 throw("bad timer heap len") 981 } 982 } 983 984 // updateTimer0When sets the P's timer0When field. 985 // The caller must have locked the timers for pp. 986 func updateTimer0When(pp *p) { 987 if len(pp.timers) == 0 { 988 atomic.Store64(&pp.timer0When, 0) 989 } else { 990 atomic.Store64(&pp.timer0When, uint64(pp.timers[0].when)) 991 } 992 } 993 994 // updateTimerModifiedEarliest updates the recorded nextwhen field of the 995 // earlier timerModifiedEarier value. 996 // The timers for pp will not be locked. 997 func updateTimerModifiedEarliest(pp *p, nextwhen int64) { 998 for { 999 old := atomic.Load64(&pp.timerModifiedEarliest) 1000 if old != 0 && int64(old) < nextwhen { 1001 return 1002 } 1003 if atomic.Cas64(&pp.timerModifiedEarliest, old, uint64(nextwhen)) { 1004 return 1005 } 1006 } 1007 } 1008 1009 // timeSleepUntil returns the time when the next timer should fire, 1010 // and the P that holds the timer heap that that timer is on. 1011 // This is only called by sysmon and checkdead. 1012 func timeSleepUntil() (int64, *p) { 1013 next := int64(maxWhen) 1014 var pret *p 1015 1016 // Prevent allp slice changes. This is like retake. 1017 lock(&allpLock) 1018 for _, pp := range allp { 1019 if pp == nil { 1020 // This can happen if procresize has grown 1021 // allp but not yet created new Ps. 1022 continue 1023 } 1024 1025 w := int64(atomic.Load64(&pp.timer0When)) 1026 if w != 0 && w < next { 1027 next = w 1028 pret = pp 1029 } 1030 1031 w = int64(atomic.Load64(&pp.timerModifiedEarliest)) 1032 if w != 0 && w < next { 1033 next = w 1034 pret = pp 1035 } 1036 } 1037 unlock(&allpLock) 1038 1039 return next, pret 1040 } 1041 1042 // Heap maintenance algorithms. 1043 // These algorithms check for slice index errors manually. 1044 // Slice index error can happen if the program is using racy 1045 // access to timers. We don't want to panic here, because 1046 // it will cause the program to crash with a mysterious 1047 // "panic holding locks" message. Instead, we panic while not 1048 // holding a lock. 1049 1050 // siftupTimer puts the timer at position i in the right place 1051 // in the heap by moving it up toward the top of the heap. 1052 // It returns the smallest changed index. 1053 func siftupTimer(t []*timer, i int) int { 1054 if i >= len(t) { 1055 badTimer() 1056 } 1057 when := t[i].when 1058 if when <= 0 { 1059 badTimer() 1060 } 1061 tmp := t[i] 1062 for i > 0 { 1063 p := (i - 1) / 4 // parent 1064 if when >= t[p].when { 1065 break 1066 } 1067 t[i] = t[p] 1068 i = p 1069 } 1070 if tmp != t[i] { 1071 t[i] = tmp 1072 } 1073 return i 1074 } 1075 1076 // siftdownTimer puts the timer at position i in the right place 1077 // in the heap by moving it down toward the bottom of the heap. 1078 func siftdownTimer(t []*timer, i int) { 1079 n := len(t) 1080 if i >= n { 1081 badTimer() 1082 } 1083 when := t[i].when 1084 if when <= 0 { 1085 badTimer() 1086 } 1087 tmp := t[i] 1088 for { 1089 c := i*4 + 1 // left child 1090 c3 := c + 2 // mid child 1091 if c >= n { 1092 break 1093 } 1094 w := t[c].when 1095 if c+1 < n && t[c+1].when < w { 1096 w = t[c+1].when 1097 c++ 1098 } 1099 if c3 < n { 1100 w3 := t[c3].when 1101 if c3+1 < n && t[c3+1].when < w3 { 1102 w3 = t[c3+1].when 1103 c3++ 1104 } 1105 if w3 < w { 1106 w = w3 1107 c = c3 1108 } 1109 } 1110 if w >= when { 1111 break 1112 } 1113 t[i] = t[c] 1114 i = c 1115 } 1116 if tmp != t[i] { 1117 t[i] = tmp 1118 } 1119 } 1120 1121 // badTimer is called if the timer data structures have been corrupted, 1122 // presumably due to racy use by the program. We panic here rather than 1123 // panicing due to invalid slice access while holding locks. 1124 // See issue #25686. 1125 func badTimer() { 1126 throw("timer data corruption") 1127 }