github.com/sanprasirt/go@v0.0.0-20170607001320-a027466e4b6d/src/runtime/sema.go (about) 1 // Copyright 2009 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 // Semaphore implementation exposed to Go. 6 // Intended use is provide a sleep and wakeup 7 // primitive that can be used in the contended case 8 // of other synchronization primitives. 9 // Thus it targets the same goal as Linux's futex, 10 // but it has much simpler semantics. 11 // 12 // That is, don't think of these as semaphores. 13 // Think of them as a way to implement sleep and wakeup 14 // such that every sleep is paired with a single wakeup, 15 // even if, due to races, the wakeup happens before the sleep. 16 // 17 // See Mullender and Cox, ``Semaphores in Plan 9,'' 18 // http://swtch.com/semaphore.pdf 19 20 package runtime 21 22 import ( 23 "runtime/internal/atomic" 24 "runtime/internal/sys" 25 "unsafe" 26 ) 27 28 // Asynchronous semaphore for sync.Mutex. 29 30 // A semaRoot holds a balanced tree of sudog with distinct addresses (s.elem). 31 // Each of those sudog may in turn point (through s.waitlink) to a list 32 // of other sudogs waiting on the same address. 33 // The operations on the inner lists of sudogs with the same address 34 // are all O(1). The scanning of the top-level semaRoot list is O(log n), 35 // where n is the number of distinct addresses with goroutines blocked 36 // on them that hash to the given semaRoot. 37 // See golang.org/issue/17953 for a program that worked badly 38 // before we introduced the second level of list, and test/locklinear.go 39 // for a test that exercises this. 40 type semaRoot struct { 41 lock mutex 42 treap *sudog // root of balanced tree of unique waiters. 43 nwait uint32 // Number of waiters. Read w/o the lock. 44 } 45 46 // Prime to not correlate with any user patterns. 47 const semTabSize = 251 48 49 var semtable [semTabSize]struct { 50 root semaRoot 51 pad [sys.CacheLineSize - unsafe.Sizeof(semaRoot{})]byte 52 } 53 54 //go:linkname sync_runtime_Semacquire sync.runtime_Semacquire 55 func sync_runtime_Semacquire(addr *uint32) { 56 semacquire1(addr, false, semaBlockProfile) 57 } 58 59 //go:linkname poll_runtime_Semacquire internal/poll.runtime_Semacquire 60 func poll_runtime_Semacquire(addr *uint32) { 61 semacquire1(addr, false, semaBlockProfile) 62 } 63 64 //go:linkname sync_runtime_Semrelease sync.runtime_Semrelease 65 func sync_runtime_Semrelease(addr *uint32, handoff bool) { 66 semrelease1(addr, handoff) 67 } 68 69 //go:linkname sync_runtime_SemacquireMutex sync.runtime_SemacquireMutex 70 func sync_runtime_SemacquireMutex(addr *uint32, lifo bool) { 71 semacquire1(addr, lifo, semaBlockProfile|semaMutexProfile) 72 } 73 74 //go:linkname poll_runtime_Semrelease internal/poll.runtime_Semrelease 75 func poll_runtime_Semrelease(addr *uint32) { 76 semrelease(addr) 77 } 78 79 func readyWithTime(s *sudog, traceskip int) { 80 if s.releasetime != 0 { 81 s.releasetime = cputicks() 82 } 83 goready(s.g, traceskip) 84 } 85 86 type semaProfileFlags int 87 88 const ( 89 semaBlockProfile semaProfileFlags = 1 << iota 90 semaMutexProfile 91 ) 92 93 // Called from runtime. 94 func semacquire(addr *uint32) { 95 semacquire1(addr, false, 0) 96 } 97 98 func semacquire1(addr *uint32, lifo bool, profile semaProfileFlags) { 99 gp := getg() 100 if gp != gp.m.curg { 101 throw("semacquire not on the G stack") 102 } 103 104 // Easy case. 105 if cansemacquire(addr) { 106 return 107 } 108 109 // Harder case: 110 // increment waiter count 111 // try cansemacquire one more time, return if succeeded 112 // enqueue itself as a waiter 113 // sleep 114 // (waiter descriptor is dequeued by signaler) 115 s := acquireSudog() 116 root := semroot(addr) 117 t0 := int64(0) 118 s.releasetime = 0 119 s.acquiretime = 0 120 s.ticket = 0 121 if profile&semaBlockProfile != 0 && blockprofilerate > 0 { 122 t0 = cputicks() 123 s.releasetime = -1 124 } 125 if profile&semaMutexProfile != 0 && mutexprofilerate > 0 { 126 if t0 == 0 { 127 t0 = cputicks() 128 } 129 s.acquiretime = t0 130 } 131 for { 132 lock(&root.lock) 133 // Add ourselves to nwait to disable "easy case" in semrelease. 134 atomic.Xadd(&root.nwait, 1) 135 // Check cansemacquire to avoid missed wakeup. 136 if cansemacquire(addr) { 137 atomic.Xadd(&root.nwait, -1) 138 unlock(&root.lock) 139 break 140 } 141 // Any semrelease after the cansemacquire knows we're waiting 142 // (we set nwait above), so go to sleep. 143 root.queue(addr, s, lifo) 144 goparkunlock(&root.lock, "semacquire", traceEvGoBlockSync, 4) 145 if s.ticket != 0 || cansemacquire(addr) { 146 break 147 } 148 } 149 if s.releasetime > 0 { 150 blockevent(s.releasetime-t0, 3) 151 } 152 releaseSudog(s) 153 } 154 155 func semrelease(addr *uint32) { 156 semrelease1(addr, false) 157 } 158 159 func semrelease1(addr *uint32, handoff bool) { 160 root := semroot(addr) 161 atomic.Xadd(addr, 1) 162 163 // Easy case: no waiters? 164 // This check must happen after the xadd, to avoid a missed wakeup 165 // (see loop in semacquire). 166 if atomic.Load(&root.nwait) == 0 { 167 return 168 } 169 170 // Harder case: search for a waiter and wake it. 171 lock(&root.lock) 172 if atomic.Load(&root.nwait) == 0 { 173 // The count is already consumed by another goroutine, 174 // so no need to wake up another goroutine. 175 unlock(&root.lock) 176 return 177 } 178 s, t0 := root.dequeue(addr) 179 if s != nil { 180 atomic.Xadd(&root.nwait, -1) 181 } 182 unlock(&root.lock) 183 if s != nil { // May be slow, so unlock first 184 acquiretime := s.acquiretime 185 if s.ticket != 0 { 186 throw("corrupted semaphore ticket") 187 } 188 if handoff && cansemacquire(addr) { 189 s.ticket = 1 190 } 191 readyWithTime(s, 5) 192 if acquiretime != 0 { 193 mutexevent(t0-acquiretime, 3) 194 } 195 } 196 } 197 198 func semroot(addr *uint32) *semaRoot { 199 return &semtable[(uintptr(unsafe.Pointer(addr))>>3)%semTabSize].root 200 } 201 202 func cansemacquire(addr *uint32) bool { 203 for { 204 v := atomic.Load(addr) 205 if v == 0 { 206 return false 207 } 208 if atomic.Cas(addr, v, v-1) { 209 return true 210 } 211 } 212 } 213 214 // queue adds s to the blocked goroutines in semaRoot. 215 func (root *semaRoot) queue(addr *uint32, s *sudog, lifo bool) { 216 s.g = getg() 217 s.elem = unsafe.Pointer(addr) 218 s.next = nil 219 s.prev = nil 220 221 var last *sudog 222 pt := &root.treap 223 for t := *pt; t != nil; t = *pt { 224 if t.elem == unsafe.Pointer(addr) { 225 // Already have addr in list. 226 if lifo { 227 // Substitute s in t's place in treap. 228 *pt = s 229 s.ticket = t.ticket 230 s.acquiretime = t.acquiretime 231 s.parent = t.parent 232 s.prev = t.prev 233 s.next = t.next 234 if s.prev != nil { 235 s.prev.parent = s 236 } 237 if s.next != nil { 238 s.next.parent = s 239 } 240 // Add t first in s's wait list. 241 s.waitlink = t 242 s.waittail = t.waittail 243 if s.waittail == nil { 244 s.waittail = t 245 } 246 t.parent = nil 247 t.prev = nil 248 t.next = nil 249 t.waittail = nil 250 } else { 251 // Add s to end of t's wait list. 252 if t.waittail == nil { 253 t.waitlink = s 254 } else { 255 t.waittail.waitlink = s 256 } 257 t.waittail = s 258 s.waitlink = nil 259 } 260 return 261 } 262 last = t 263 if uintptr(unsafe.Pointer(addr)) < uintptr(t.elem) { 264 pt = &t.prev 265 } else { 266 pt = &t.next 267 } 268 } 269 270 // Add s as new leaf in tree of unique addrs. 271 // The balanced tree is a treap using ticket as the random heap priority. 272 // That is, it is a binary tree ordered according to the elem addresses, 273 // but then among the space of possible binary trees respecting those 274 // addresses, it is kept balanced on average by maintaining a heap ordering 275 // on the ticket: s.ticket <= both s.prev.ticket and s.next.ticket. 276 // https://en.wikipedia.org/wiki/Treap 277 // http://faculty.washington.edu/aragon/pubs/rst89.pdf 278 s.ticket = fastrand() 279 s.parent = last 280 *pt = s 281 282 // Rotate up into tree according to ticket (priority). 283 for s.parent != nil && s.parent.ticket > s.ticket { 284 if s.parent.prev == s { 285 root.rotateRight(s.parent) 286 } else { 287 if s.parent.next != s { 288 panic("semaRoot queue") 289 } 290 root.rotateLeft(s.parent) 291 } 292 } 293 } 294 295 // dequeue searches for and finds the first goroutine 296 // in semaRoot blocked on addr. 297 // If the sudog was being profiled, dequeue returns the time 298 // at which it was woken up as now. Otherwise now is 0. 299 func (root *semaRoot) dequeue(addr *uint32) (found *sudog, now int64) { 300 ps := &root.treap 301 s := *ps 302 for ; s != nil; s = *ps { 303 if s.elem == unsafe.Pointer(addr) { 304 goto Found 305 } 306 if uintptr(unsafe.Pointer(addr)) < uintptr(s.elem) { 307 ps = &s.prev 308 } else { 309 ps = &s.next 310 } 311 } 312 return nil, 0 313 314 Found: 315 now = int64(0) 316 if s.acquiretime != 0 { 317 now = cputicks() 318 } 319 if t := s.waitlink; t != nil { 320 // Substitute t, also waiting on addr, for s in root tree of unique addrs. 321 *ps = t 322 t.ticket = s.ticket 323 t.parent = s.parent 324 t.prev = s.prev 325 if t.prev != nil { 326 t.prev.parent = t 327 } 328 t.next = s.next 329 if t.next != nil { 330 t.next.parent = t 331 } 332 if t.waitlink != nil { 333 t.waittail = s.waittail 334 } else { 335 t.waittail = nil 336 } 337 t.acquiretime = now 338 s.waitlink = nil 339 s.waittail = nil 340 } else { 341 // Rotate s down to be leaf of tree for removal, respecting priorities. 342 for s.next != nil || s.prev != nil { 343 if s.next == nil || s.prev != nil && s.prev.ticket < s.next.ticket { 344 root.rotateRight(s) 345 } else { 346 root.rotateLeft(s) 347 } 348 } 349 // Remove s, now a leaf. 350 if s.parent != nil { 351 if s.parent.prev == s { 352 s.parent.prev = nil 353 } else { 354 s.parent.next = nil 355 } 356 } else { 357 root.treap = nil 358 } 359 } 360 s.parent = nil 361 s.elem = nil 362 s.next = nil 363 s.prev = nil 364 s.ticket = 0 365 return s, now 366 } 367 368 // rotateLeft rotates the tree rooted at node x. 369 // turning (x a (y b c)) into (y (x a b) c). 370 func (root *semaRoot) rotateLeft(x *sudog) { 371 // p -> (x a (y b c)) 372 p := x.parent 373 a, y := x.prev, x.next 374 b, c := y.prev, y.next 375 376 y.prev = x 377 x.parent = y 378 y.next = c 379 if c != nil { 380 c.parent = y 381 } 382 x.prev = a 383 if a != nil { 384 a.parent = x 385 } 386 x.next = b 387 if b != nil { 388 b.parent = x 389 } 390 391 y.parent = p 392 if p == nil { 393 root.treap = y 394 } else if p.prev == x { 395 p.prev = y 396 } else { 397 if p.next != x { 398 throw("semaRoot rotateLeft") 399 } 400 p.next = y 401 } 402 } 403 404 // rotateRight rotates the tree rooted at node y. 405 // turning (y (x a b) c) into (x a (y b c)). 406 func (root *semaRoot) rotateRight(y *sudog) { 407 // p -> (y (x a b) c) 408 p := y.parent 409 x, c := y.prev, y.next 410 a, b := x.prev, x.next 411 412 x.prev = a 413 if a != nil { 414 a.parent = x 415 } 416 x.next = y 417 y.parent = x 418 y.prev = b 419 if b != nil { 420 b.parent = y 421 } 422 y.next = c 423 if c != nil { 424 c.parent = y 425 } 426 427 x.parent = p 428 if p == nil { 429 root.treap = x 430 } else if p.prev == y { 431 p.prev = x 432 } else { 433 if p.next != y { 434 throw("semaRoot rotateRight") 435 } 436 p.next = x 437 } 438 } 439 440 // notifyList is a ticket-based notification list used to implement sync.Cond. 441 // 442 // It must be kept in sync with the sync package. 443 type notifyList struct { 444 // wait is the ticket number of the next waiter. It is atomically 445 // incremented outside the lock. 446 wait uint32 447 448 // notify is the ticket number of the next waiter to be notified. It can 449 // be read outside the lock, but is only written to with lock held. 450 // 451 // Both wait & notify can wrap around, and such cases will be correctly 452 // handled as long as their "unwrapped" difference is bounded by 2^31. 453 // For this not to be the case, we'd need to have 2^31+ goroutines 454 // blocked on the same condvar, which is currently not possible. 455 notify uint32 456 457 // List of parked waiters. 458 lock mutex 459 head *sudog 460 tail *sudog 461 } 462 463 // less checks if a < b, considering a & b running counts that may overflow the 464 // 32-bit range, and that their "unwrapped" difference is always less than 2^31. 465 func less(a, b uint32) bool { 466 return int32(a-b) < 0 467 } 468 469 // notifyListAdd adds the caller to a notify list such that it can receive 470 // notifications. The caller must eventually call notifyListWait to wait for 471 // such a notification, passing the returned ticket number. 472 //go:linkname notifyListAdd sync.runtime_notifyListAdd 473 func notifyListAdd(l *notifyList) uint32 { 474 // This may be called concurrently, for example, when called from 475 // sync.Cond.Wait while holding a RWMutex in read mode. 476 return atomic.Xadd(&l.wait, 1) - 1 477 } 478 479 // notifyListWait waits for a notification. If one has been sent since 480 // notifyListAdd was called, it returns immediately. Otherwise, it blocks. 481 //go:linkname notifyListWait sync.runtime_notifyListWait 482 func notifyListWait(l *notifyList, t uint32) { 483 lock(&l.lock) 484 485 // Return right away if this ticket has already been notified. 486 if less(t, l.notify) { 487 unlock(&l.lock) 488 return 489 } 490 491 // Enqueue itself. 492 s := acquireSudog() 493 s.g = getg() 494 s.ticket = t 495 s.releasetime = 0 496 t0 := int64(0) 497 if blockprofilerate > 0 { 498 t0 = cputicks() 499 s.releasetime = -1 500 } 501 if l.tail == nil { 502 l.head = s 503 } else { 504 l.tail.next = s 505 } 506 l.tail = s 507 goparkunlock(&l.lock, "semacquire", traceEvGoBlockCond, 3) 508 if t0 != 0 { 509 blockevent(s.releasetime-t0, 2) 510 } 511 releaseSudog(s) 512 } 513 514 // notifyListNotifyAll notifies all entries in the list. 515 //go:linkname notifyListNotifyAll sync.runtime_notifyListNotifyAll 516 func notifyListNotifyAll(l *notifyList) { 517 // Fast-path: if there are no new waiters since the last notification 518 // we don't need to acquire the lock. 519 if atomic.Load(&l.wait) == atomic.Load(&l.notify) { 520 return 521 } 522 523 // Pull the list out into a local variable, waiters will be readied 524 // outside the lock. 525 lock(&l.lock) 526 s := l.head 527 l.head = nil 528 l.tail = nil 529 530 // Update the next ticket to be notified. We can set it to the current 531 // value of wait because any previous waiters are already in the list 532 // or will notice that they have already been notified when trying to 533 // add themselves to the list. 534 atomic.Store(&l.notify, atomic.Load(&l.wait)) 535 unlock(&l.lock) 536 537 // Go through the local list and ready all waiters. 538 for s != nil { 539 next := s.next 540 s.next = nil 541 readyWithTime(s, 4) 542 s = next 543 } 544 } 545 546 // notifyListNotifyOne notifies one entry in the list. 547 //go:linkname notifyListNotifyOne sync.runtime_notifyListNotifyOne 548 func notifyListNotifyOne(l *notifyList) { 549 // Fast-path: if there are no new waiters since the last notification 550 // we don't need to acquire the lock at all. 551 if atomic.Load(&l.wait) == atomic.Load(&l.notify) { 552 return 553 } 554 555 lock(&l.lock) 556 557 // Re-check under the lock if we need to do anything. 558 t := l.notify 559 if t == atomic.Load(&l.wait) { 560 unlock(&l.lock) 561 return 562 } 563 564 // Update the next notify ticket number. 565 atomic.Store(&l.notify, t+1) 566 567 // Try to find the g that needs to be notified. 568 // If it hasn't made it to the list yet we won't find it, 569 // but it won't park itself once it sees the new notify number. 570 // 571 // This scan looks linear but essentially always stops quickly. 572 // Because g's queue separately from taking numbers, 573 // there may be minor reorderings in the list, but we 574 // expect the g we're looking for to be near the front. 575 // The g has others in front of it on the list only to the 576 // extent that it lost the race, so the iteration will not 577 // be too long. This applies even when the g is missing: 578 // it hasn't yet gotten to sleep and has lost the race to 579 // the (few) other g's that we find on the list. 580 for p, s := (*sudog)(nil), l.head; s != nil; p, s = s, s.next { 581 if s.ticket == t { 582 n := s.next 583 if p != nil { 584 p.next = n 585 } else { 586 l.head = n 587 } 588 if n == nil { 589 l.tail = p 590 } 591 unlock(&l.lock) 592 s.next = nil 593 readyWithTime(s, 4) 594 return 595 } 596 } 597 unlock(&l.lock) 598 } 599 600 //go:linkname notifyListCheck sync.runtime_notifyListCheck 601 func notifyListCheck(sz uintptr) { 602 if sz != unsafe.Sizeof(notifyList{}) { 603 print("runtime: bad notifyList size - sync=", sz, " runtime=", unsafe.Sizeof(notifyList{}), "\n") 604 throw("bad notifyList size") 605 } 606 } 607 608 //go:linkname sync_nanotime sync.runtime_nanotime 609 func sync_nanotime() int64 { 610 return nanotime() 611 }