rsc.io/go@v0.0.0-20150416155037-e040fd465409/src/runtime/chan.go (about) 1 // Copyright 2014 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package runtime 6 7 // This file contains the implementation of Go channels. 8 9 import "unsafe" 10 11 const ( 12 maxAlign = 8 13 hchanSize = unsafe.Sizeof(hchan{}) + uintptr(-int(unsafe.Sizeof(hchan{}))&(maxAlign-1)) 14 debugChan = false 15 ) 16 17 type hchan struct { 18 qcount uint // total data in the queue 19 dataqsiz uint // size of the circular queue 20 buf unsafe.Pointer // points to an array of dataqsiz elements 21 elemsize uint16 22 closed uint32 23 elemtype *_type // element type 24 sendx uint // send index 25 recvx uint // receive index 26 recvq waitq // list of recv waiters 27 sendq waitq // list of send waiters 28 lock mutex 29 } 30 31 type waitq struct { 32 first *sudog 33 last *sudog 34 } 35 36 //go:linkname reflect_makechan reflect.makechan 37 func reflect_makechan(t *chantype, size int64) *hchan { 38 return makechan(t, size) 39 } 40 41 func makechan(t *chantype, size int64) *hchan { 42 elem := t.elem 43 44 // compiler checks this but be safe. 45 if elem.size >= 1<<16 { 46 throw("makechan: invalid channel element type") 47 } 48 if hchanSize%maxAlign != 0 || elem.align > maxAlign { 49 throw("makechan: bad alignment") 50 } 51 if size < 0 || int64(uintptr(size)) != size || (elem.size > 0 && uintptr(size) > (_MaxMem-hchanSize)/uintptr(elem.size)) { 52 panic("makechan: size out of range") 53 } 54 55 var c *hchan 56 if elem.kind&kindNoPointers != 0 || size == 0 { 57 // Allocate memory in one call. 58 // Hchan does not contain pointers interesting for GC in this case: 59 // buf points into the same allocation, elemtype is persistent. 60 // SudoG's are referenced from their owning thread so they can't be collected. 61 // TODO(dvyukov,rlh): Rethink when collector can move allocated objects. 62 c = (*hchan)(mallocgc(hchanSize+uintptr(size)*uintptr(elem.size), nil, flagNoScan)) 63 if size > 0 && elem.size != 0 { 64 c.buf = add(unsafe.Pointer(c), hchanSize) 65 } else { 66 // race detector uses this location for synchronization 67 // Also prevents us from pointing beyond the allocation (see issue 9401). 68 c.buf = unsafe.Pointer(c) 69 } 70 } else { 71 c = new(hchan) 72 c.buf = newarray(elem, uintptr(size)) 73 } 74 c.elemsize = uint16(elem.size) 75 c.elemtype = elem 76 c.dataqsiz = uint(size) 77 78 if debugChan { 79 print("makechan: chan=", c, "; elemsize=", elem.size, "; elemalg=", elem.alg, "; dataqsiz=", size, "\n") 80 } 81 return c 82 } 83 84 // chanbuf(c, i) is pointer to the i'th slot in the buffer. 85 func chanbuf(c *hchan, i uint) unsafe.Pointer { 86 return add(c.buf, uintptr(i)*uintptr(c.elemsize)) 87 } 88 89 // entry point for c <- x from compiled code 90 //go:nosplit 91 func chansend1(t *chantype, c *hchan, elem unsafe.Pointer) { 92 chansend(t, c, elem, true, getcallerpc(unsafe.Pointer(&t))) 93 } 94 95 /* 96 * generic single channel send/recv 97 * If block is not nil, 98 * then the protocol will not 99 * sleep but return if it could 100 * not complete. 101 * 102 * sleep can wake up with g.param == nil 103 * when a channel involved in the sleep has 104 * been closed. it is easiest to loop and re-run 105 * the operation; we'll see that it's now closed. 106 */ 107 func chansend(t *chantype, c *hchan, ep unsafe.Pointer, block bool, callerpc uintptr) bool { 108 if raceenabled { 109 raceReadObjectPC(t.elem, ep, callerpc, funcPC(chansend)) 110 } 111 112 if c == nil { 113 if !block { 114 return false 115 } 116 gopark(nil, nil, "chan send (nil chan)", traceEvGoStop, 2) 117 throw("unreachable") 118 } 119 120 if debugChan { 121 print("chansend: chan=", c, "\n") 122 } 123 124 if raceenabled { 125 racereadpc(unsafe.Pointer(c), callerpc, funcPC(chansend)) 126 } 127 128 // Fast path: check for failed non-blocking operation without acquiring the lock. 129 // 130 // After observing that the channel is not closed, we observe that the channel is 131 // not ready for sending. Each of these observations is a single word-sized read 132 // (first c.closed and second c.recvq.first or c.qcount depending on kind of channel). 133 // Because a closed channel cannot transition from 'ready for sending' to 134 // 'not ready for sending', even if the channel is closed between the two observations, 135 // they imply a moment between the two when the channel was both not yet closed 136 // and not ready for sending. We behave as if we observed the channel at that moment, 137 // and report that the send cannot proceed. 138 // 139 // It is okay if the reads are reordered here: if we observe that the channel is not 140 // ready for sending and then observe that it is not closed, that implies that the 141 // channel wasn't closed during the first observation. 142 if !block && c.closed == 0 && ((c.dataqsiz == 0 && c.recvq.first == nil) || 143 (c.dataqsiz > 0 && c.qcount == c.dataqsiz)) { 144 return false 145 } 146 147 var t0 int64 148 if blockprofilerate > 0 { 149 t0 = cputicks() 150 } 151 152 lock(&c.lock) 153 if c.closed != 0 { 154 unlock(&c.lock) 155 panic("send on closed channel") 156 } 157 158 if c.dataqsiz == 0 { // synchronous channel 159 sg := c.recvq.dequeue() 160 if sg != nil { // found a waiting receiver 161 if raceenabled { 162 racesync(c, sg) 163 } 164 unlock(&c.lock) 165 166 recvg := sg.g 167 if sg.elem != nil { 168 typedmemmove(c.elemtype, unsafe.Pointer(sg.elem), ep) 169 sg.elem = nil 170 } 171 recvg.param = unsafe.Pointer(sg) 172 if sg.releasetime != 0 { 173 sg.releasetime = cputicks() 174 } 175 goready(recvg, 3) 176 return true 177 } 178 179 if !block { 180 unlock(&c.lock) 181 return false 182 } 183 184 // no receiver available: block on this channel. 185 gp := getg() 186 mysg := acquireSudog() 187 mysg.releasetime = 0 188 if t0 != 0 { 189 mysg.releasetime = -1 190 } 191 mysg.elem = ep 192 mysg.waitlink = nil 193 gp.waiting = mysg 194 mysg.g = gp 195 mysg.selectdone = nil 196 gp.param = nil 197 c.sendq.enqueue(mysg) 198 goparkunlock(&c.lock, "chan send", traceEvGoBlockSend, 3) 199 200 // someone woke us up. 201 if mysg != gp.waiting { 202 throw("G waiting list is corrupted!") 203 } 204 gp.waiting = nil 205 if gp.param == nil { 206 if c.closed == 0 { 207 throw("chansend: spurious wakeup") 208 } 209 panic("send on closed channel") 210 } 211 gp.param = nil 212 if mysg.releasetime > 0 { 213 blockevent(int64(mysg.releasetime)-t0, 2) 214 } 215 releaseSudog(mysg) 216 return true 217 } 218 219 // asynchronous channel 220 // wait for some space to write our data 221 var t1 int64 222 for futile := byte(0); c.qcount >= c.dataqsiz; futile = traceFutileWakeup { 223 if !block { 224 unlock(&c.lock) 225 return false 226 } 227 gp := getg() 228 mysg := acquireSudog() 229 mysg.releasetime = 0 230 if t0 != 0 { 231 mysg.releasetime = -1 232 } 233 mysg.g = gp 234 mysg.elem = nil 235 mysg.selectdone = nil 236 c.sendq.enqueue(mysg) 237 goparkunlock(&c.lock, "chan send", traceEvGoBlockSend|futile, 3) 238 239 // someone woke us up - try again 240 if mysg.releasetime > 0 { 241 t1 = mysg.releasetime 242 } 243 releaseSudog(mysg) 244 lock(&c.lock) 245 if c.closed != 0 { 246 unlock(&c.lock) 247 panic("send on closed channel") 248 } 249 } 250 251 // write our data into the channel buffer 252 if raceenabled { 253 raceacquire(chanbuf(c, c.sendx)) 254 racerelease(chanbuf(c, c.sendx)) 255 } 256 typedmemmove(c.elemtype, chanbuf(c, c.sendx), ep) 257 c.sendx++ 258 if c.sendx == c.dataqsiz { 259 c.sendx = 0 260 } 261 c.qcount++ 262 263 // wake up a waiting receiver 264 sg := c.recvq.dequeue() 265 if sg != nil { 266 recvg := sg.g 267 unlock(&c.lock) 268 if sg.releasetime != 0 { 269 sg.releasetime = cputicks() 270 } 271 goready(recvg, 3) 272 } else { 273 unlock(&c.lock) 274 } 275 if t1 > 0 { 276 blockevent(t1-t0, 2) 277 } 278 return true 279 } 280 281 func closechan(c *hchan) { 282 if c == nil { 283 panic("close of nil channel") 284 } 285 286 lock(&c.lock) 287 if c.closed != 0 { 288 unlock(&c.lock) 289 panic("close of closed channel") 290 } 291 292 if raceenabled { 293 callerpc := getcallerpc(unsafe.Pointer(&c)) 294 racewritepc(unsafe.Pointer(c), callerpc, funcPC(closechan)) 295 racerelease(unsafe.Pointer(c)) 296 } 297 298 c.closed = 1 299 300 // release all readers 301 for { 302 sg := c.recvq.dequeue() 303 if sg == nil { 304 break 305 } 306 gp := sg.g 307 sg.elem = nil 308 gp.param = nil 309 if sg.releasetime != 0 { 310 sg.releasetime = cputicks() 311 } 312 goready(gp, 3) 313 } 314 315 // release all writers 316 for { 317 sg := c.sendq.dequeue() 318 if sg == nil { 319 break 320 } 321 gp := sg.g 322 sg.elem = nil 323 gp.param = nil 324 if sg.releasetime != 0 { 325 sg.releasetime = cputicks() 326 } 327 goready(gp, 3) 328 } 329 unlock(&c.lock) 330 } 331 332 // entry points for <- c from compiled code 333 //go:nosplit 334 func chanrecv1(t *chantype, c *hchan, elem unsafe.Pointer) { 335 chanrecv(t, c, elem, true) 336 } 337 338 //go:nosplit 339 func chanrecv2(t *chantype, c *hchan, elem unsafe.Pointer) (received bool) { 340 _, received = chanrecv(t, c, elem, true) 341 return 342 } 343 344 // chanrecv receives on channel c and writes the received data to ep. 345 // ep may be nil, in which case received data is ignored. 346 // If block == false and no elements are available, returns (false, false). 347 // Otherwise, if c is closed, zeros *ep and returns (true, false). 348 // Otherwise, fills in *ep with an element and returns (true, true). 349 func chanrecv(t *chantype, c *hchan, ep unsafe.Pointer, block bool) (selected, received bool) { 350 // raceenabled: don't need to check ep, as it is always on the stack. 351 352 if debugChan { 353 print("chanrecv: chan=", c, "\n") 354 } 355 356 if c == nil { 357 if !block { 358 return 359 } 360 gopark(nil, nil, "chan receive (nil chan)", traceEvGoStop, 2) 361 throw("unreachable") 362 } 363 364 // Fast path: check for failed non-blocking operation without acquiring the lock. 365 // 366 // After observing that the channel is not ready for receiving, we observe that the 367 // channel is not closed. Each of these observations is a single word-sized read 368 // (first c.sendq.first or c.qcount, and second c.closed). 369 // Because a channel cannot be reopened, the later observation of the channel 370 // being not closed implies that it was also not closed at the moment of the 371 // first observation. We behave as if we observed the channel at that moment 372 // and report that the receive cannot proceed. 373 // 374 // The order of operations is important here: reversing the operations can lead to 375 // incorrect behavior when racing with a close. 376 if !block && (c.dataqsiz == 0 && c.sendq.first == nil || 377 c.dataqsiz > 0 && atomicloaduint(&c.qcount) == 0) && 378 atomicload(&c.closed) == 0 { 379 return 380 } 381 382 var t0 int64 383 if blockprofilerate > 0 { 384 t0 = cputicks() 385 } 386 387 lock(&c.lock) 388 if c.dataqsiz == 0 { // synchronous channel 389 if c.closed != 0 { 390 return recvclosed(c, ep) 391 } 392 393 sg := c.sendq.dequeue() 394 if sg != nil { 395 if raceenabled { 396 racesync(c, sg) 397 } 398 unlock(&c.lock) 399 400 if ep != nil { 401 typedmemmove(c.elemtype, ep, sg.elem) 402 } 403 sg.elem = nil 404 gp := sg.g 405 gp.param = unsafe.Pointer(sg) 406 if sg.releasetime != 0 { 407 sg.releasetime = cputicks() 408 } 409 goready(gp, 3) 410 selected = true 411 received = true 412 return 413 } 414 415 if !block { 416 unlock(&c.lock) 417 return 418 } 419 420 // no sender available: block on this channel. 421 gp := getg() 422 mysg := acquireSudog() 423 mysg.releasetime = 0 424 if t0 != 0 { 425 mysg.releasetime = -1 426 } 427 mysg.elem = ep 428 mysg.waitlink = nil 429 gp.waiting = mysg 430 mysg.g = gp 431 mysg.selectdone = nil 432 gp.param = nil 433 c.recvq.enqueue(mysg) 434 goparkunlock(&c.lock, "chan receive", traceEvGoBlockRecv, 3) 435 436 // someone woke us up 437 if mysg != gp.waiting { 438 throw("G waiting list is corrupted!") 439 } 440 gp.waiting = nil 441 if mysg.releasetime > 0 { 442 blockevent(mysg.releasetime-t0, 2) 443 } 444 haveData := gp.param != nil 445 gp.param = nil 446 releaseSudog(mysg) 447 448 if haveData { 449 // a sender sent us some data. It already wrote to ep. 450 selected = true 451 received = true 452 return 453 } 454 455 lock(&c.lock) 456 if c.closed == 0 { 457 throw("chanrecv: spurious wakeup") 458 } 459 return recvclosed(c, ep) 460 } 461 462 // asynchronous channel 463 // wait for some data to appear 464 var t1 int64 465 for futile := byte(0); c.qcount <= 0; futile = traceFutileWakeup { 466 if c.closed != 0 { 467 selected, received = recvclosed(c, ep) 468 if t1 > 0 { 469 blockevent(t1-t0, 2) 470 } 471 return 472 } 473 474 if !block { 475 unlock(&c.lock) 476 return 477 } 478 479 // wait for someone to send an element 480 gp := getg() 481 mysg := acquireSudog() 482 mysg.releasetime = 0 483 if t0 != 0 { 484 mysg.releasetime = -1 485 } 486 mysg.elem = nil 487 mysg.g = gp 488 mysg.selectdone = nil 489 490 c.recvq.enqueue(mysg) 491 goparkunlock(&c.lock, "chan receive", traceEvGoBlockRecv|futile, 3) 492 493 // someone woke us up - try again 494 if mysg.releasetime > 0 { 495 t1 = mysg.releasetime 496 } 497 releaseSudog(mysg) 498 lock(&c.lock) 499 } 500 501 if raceenabled { 502 raceacquire(chanbuf(c, c.recvx)) 503 racerelease(chanbuf(c, c.recvx)) 504 } 505 if ep != nil { 506 typedmemmove(c.elemtype, ep, chanbuf(c, c.recvx)) 507 } 508 memclr(chanbuf(c, c.recvx), uintptr(c.elemsize)) 509 510 c.recvx++ 511 if c.recvx == c.dataqsiz { 512 c.recvx = 0 513 } 514 c.qcount-- 515 516 // ping a sender now that there is space 517 sg := c.sendq.dequeue() 518 if sg != nil { 519 gp := sg.g 520 unlock(&c.lock) 521 if sg.releasetime != 0 { 522 sg.releasetime = cputicks() 523 } 524 goready(gp, 3) 525 } else { 526 unlock(&c.lock) 527 } 528 529 if t1 > 0 { 530 blockevent(t1-t0, 2) 531 } 532 selected = true 533 received = true 534 return 535 } 536 537 // recvclosed is a helper function for chanrecv. Handles cleanup 538 // when the receiver encounters a closed channel. 539 // Caller must hold c.lock, recvclosed will release the lock. 540 func recvclosed(c *hchan, ep unsafe.Pointer) (selected, recevied bool) { 541 if raceenabled { 542 raceacquire(unsafe.Pointer(c)) 543 } 544 unlock(&c.lock) 545 if ep != nil { 546 memclr(ep, uintptr(c.elemsize)) 547 } 548 return true, false 549 } 550 551 // compiler implements 552 // 553 // select { 554 // case c <- v: 555 // ... foo 556 // default: 557 // ... bar 558 // } 559 // 560 // as 561 // 562 // if selectnbsend(c, v) { 563 // ... foo 564 // } else { 565 // ... bar 566 // } 567 // 568 func selectnbsend(t *chantype, c *hchan, elem unsafe.Pointer) (selected bool) { 569 return chansend(t, c, elem, false, getcallerpc(unsafe.Pointer(&t))) 570 } 571 572 // compiler implements 573 // 574 // select { 575 // case v = <-c: 576 // ... foo 577 // default: 578 // ... bar 579 // } 580 // 581 // as 582 // 583 // if selectnbrecv(&v, c) { 584 // ... foo 585 // } else { 586 // ... bar 587 // } 588 // 589 func selectnbrecv(t *chantype, elem unsafe.Pointer, c *hchan) (selected bool) { 590 selected, _ = chanrecv(t, c, elem, false) 591 return 592 } 593 594 // compiler implements 595 // 596 // select { 597 // case v, ok = <-c: 598 // ... foo 599 // default: 600 // ... bar 601 // } 602 // 603 // as 604 // 605 // if c != nil && selectnbrecv2(&v, &ok, c) { 606 // ... foo 607 // } else { 608 // ... bar 609 // } 610 // 611 func selectnbrecv2(t *chantype, elem unsafe.Pointer, received *bool, c *hchan) (selected bool) { 612 // TODO(khr): just return 2 values from this function, now that it is in Go. 613 selected, *received = chanrecv(t, c, elem, false) 614 return 615 } 616 617 //go:linkname reflect_chansend reflect.chansend 618 func reflect_chansend(t *chantype, c *hchan, elem unsafe.Pointer, nb bool) (selected bool) { 619 return chansend(t, c, elem, !nb, getcallerpc(unsafe.Pointer(&t))) 620 } 621 622 //go:linkname reflect_chanrecv reflect.chanrecv 623 func reflect_chanrecv(t *chantype, c *hchan, nb bool, elem unsafe.Pointer) (selected bool, received bool) { 624 return chanrecv(t, c, elem, !nb) 625 } 626 627 //go:linkname reflect_chanlen reflect.chanlen 628 func reflect_chanlen(c *hchan) int { 629 if c == nil { 630 return 0 631 } 632 return int(c.qcount) 633 } 634 635 //go:linkname reflect_chancap reflect.chancap 636 func reflect_chancap(c *hchan) int { 637 if c == nil { 638 return 0 639 } 640 return int(c.dataqsiz) 641 } 642 643 //go:linkname reflect_chanclose reflect.chanclose 644 func reflect_chanclose(c *hchan) { 645 closechan(c) 646 } 647 648 func (q *waitq) enqueue(sgp *sudog) { 649 sgp.next = nil 650 x := q.last 651 if x == nil { 652 sgp.prev = nil 653 q.first = sgp 654 q.last = sgp 655 return 656 } 657 sgp.prev = x 658 x.next = sgp 659 q.last = sgp 660 } 661 662 func (q *waitq) dequeue() *sudog { 663 for { 664 sgp := q.first 665 if sgp == nil { 666 return nil 667 } 668 y := sgp.next 669 if y == nil { 670 q.first = nil 671 q.last = nil 672 } else { 673 y.prev = nil 674 q.first = y 675 sgp.next = nil // mark as removed (see dequeueSudog) 676 } 677 678 // if sgp participates in a select and is already signaled, ignore it 679 if sgp.selectdone != nil { 680 // claim the right to signal 681 if *sgp.selectdone != 0 || !cas(sgp.selectdone, 0, 1) { 682 continue 683 } 684 } 685 686 return sgp 687 } 688 } 689 690 func racesync(c *hchan, sg *sudog) { 691 racerelease(chanbuf(c, 0)) 692 raceacquireg(sg.g, chanbuf(c, 0)) 693 racereleaseg(sg.g, chanbuf(c, 0)) 694 raceacquire(chanbuf(c, 0)) 695 }