github.com/euank/go@v0.0.0-20160829210321-495514729181/src/runtime/chan.go (about) 1 // Copyright 2014 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package runtime 6 7 // This file contains the implementation of Go channels. 8 9 // Invariants: 10 // At least one of c.sendq and c.recvq is empty. 11 // For buffered channels, also: 12 // c.qcount > 0 implies that c.recvq is empty. 13 // c.qcount < c.dataqsiz implies that c.sendq is empty. 14 import ( 15 "runtime/internal/atomic" 16 "unsafe" 17 ) 18 19 const ( 20 maxAlign = 8 21 hchanSize = unsafe.Sizeof(hchan{}) + uintptr(-int(unsafe.Sizeof(hchan{}))&(maxAlign-1)) 22 debugChan = false 23 ) 24 25 type hchan struct { 26 qcount uint // total data in the queue 27 dataqsiz uint // size of the circular queue 28 buf unsafe.Pointer // points to an array of dataqsiz elements 29 elemsize uint16 30 closed uint32 31 elemtype *_type // element type 32 sendx uint // send index 33 recvx uint // receive index 34 recvq waitq // list of recv waiters 35 sendq waitq // list of send waiters 36 37 // lock protects all fields in hchan, as well as several 38 // fields in sudogs blocked on this channel. 39 // 40 // Do not change another G's status while holding this lock 41 // (in particular, do not ready a G), as this can deadlock 42 // with stack shrinking. 43 lock mutex 44 } 45 46 type waitq struct { 47 first *sudog 48 last *sudog 49 } 50 51 //go:linkname reflect_makechan reflect.makechan 52 func reflect_makechan(t *chantype, size int64) *hchan { 53 return makechan(t, size) 54 } 55 56 func makechan(t *chantype, size int64) *hchan { 57 elem := t.elem 58 59 // compiler checks this but be safe. 60 if elem.size >= 1<<16 { 61 throw("makechan: invalid channel element type") 62 } 63 if hchanSize%maxAlign != 0 || elem.align > maxAlign { 64 throw("makechan: bad alignment") 65 } 66 if size < 0 || int64(uintptr(size)) != size || (elem.size > 0 && uintptr(size) > (_MaxMem-hchanSize)/elem.size) { 67 panic(plainError("makechan: size out of range")) 68 } 69 70 var c *hchan 71 if elem.kind&kindNoPointers != 0 || size == 0 { 72 // Allocate memory in one call. 73 // Hchan does not contain pointers interesting for GC in this case: 74 // buf points into the same allocation, elemtype is persistent. 75 // SudoG's are referenced from their owning thread so they can't be collected. 76 // TODO(dvyukov,rlh): Rethink when collector can move allocated objects. 77 c = (*hchan)(mallocgc(hchanSize+uintptr(size)*elem.size, nil, true)) 78 if size > 0 && elem.size != 0 { 79 c.buf = add(unsafe.Pointer(c), hchanSize) 80 } else { 81 // race detector uses this location for synchronization 82 // Also prevents us from pointing beyond the allocation (see issue 9401). 83 c.buf = unsafe.Pointer(c) 84 } 85 } else { 86 c = new(hchan) 87 c.buf = newarray(elem, int(size)) 88 } 89 c.elemsize = uint16(elem.size) 90 c.elemtype = elem 91 c.dataqsiz = uint(size) 92 93 if debugChan { 94 print("makechan: chan=", c, "; elemsize=", elem.size, "; elemalg=", elem.alg, "; dataqsiz=", size, "\n") 95 } 96 return c 97 } 98 99 // chanbuf(c, i) is pointer to the i'th slot in the buffer. 100 func chanbuf(c *hchan, i uint) unsafe.Pointer { 101 return add(c.buf, uintptr(i)*uintptr(c.elemsize)) 102 } 103 104 // entry point for c <- x from compiled code 105 //go:nosplit 106 func chansend1(t *chantype, c *hchan, elem unsafe.Pointer) { 107 chansend(t, c, elem, true, getcallerpc(unsafe.Pointer(&t))) 108 } 109 110 /* 111 * generic single channel send/recv 112 * If block is not nil, 113 * then the protocol will not 114 * sleep but return if it could 115 * not complete. 116 * 117 * sleep can wake up with g.param == nil 118 * when a channel involved in the sleep has 119 * been closed. it is easiest to loop and re-run 120 * the operation; we'll see that it's now closed. 121 */ 122 func chansend(t *chantype, c *hchan, ep unsafe.Pointer, block bool, callerpc uintptr) bool { 123 if raceenabled { 124 raceReadObjectPC(t.elem, ep, callerpc, funcPC(chansend)) 125 } 126 if msanenabled { 127 msanread(ep, t.elem.size) 128 } 129 130 if c == nil { 131 if !block { 132 return false 133 } 134 gopark(nil, nil, "chan send (nil chan)", traceEvGoStop, 2) 135 throw("unreachable") 136 } 137 138 if debugChan { 139 print("chansend: chan=", c, "\n") 140 } 141 142 if raceenabled { 143 racereadpc(unsafe.Pointer(c), callerpc, funcPC(chansend)) 144 } 145 146 // Fast path: check for failed non-blocking operation without acquiring the lock. 147 // 148 // After observing that the channel is not closed, we observe that the channel is 149 // not ready for sending. Each of these observations is a single word-sized read 150 // (first c.closed and second c.recvq.first or c.qcount depending on kind of channel). 151 // Because a closed channel cannot transition from 'ready for sending' to 152 // 'not ready for sending', even if the channel is closed between the two observations, 153 // they imply a moment between the two when the channel was both not yet closed 154 // and not ready for sending. We behave as if we observed the channel at that moment, 155 // and report that the send cannot proceed. 156 // 157 // It is okay if the reads are reordered here: if we observe that the channel is not 158 // ready for sending and then observe that it is not closed, that implies that the 159 // channel wasn't closed during the first observation. 160 if !block && c.closed == 0 && ((c.dataqsiz == 0 && c.recvq.first == nil) || 161 (c.dataqsiz > 0 && c.qcount == c.dataqsiz)) { 162 return false 163 } 164 165 var t0 int64 166 if blockprofilerate > 0 { 167 t0 = cputicks() 168 } 169 170 lock(&c.lock) 171 172 if c.closed != 0 { 173 unlock(&c.lock) 174 panic(plainError("send on closed channel")) 175 } 176 177 if sg := c.recvq.dequeue(); sg != nil { 178 // Found a waiting receiver. We pass the value we want to send 179 // directly to the receiver, bypassing the channel buffer (if any). 180 send(c, sg, ep, func() { unlock(&c.lock) }) 181 return true 182 } 183 184 if c.qcount < c.dataqsiz { 185 // Space is available in the channel buffer. Enqueue the element to send. 186 qp := chanbuf(c, c.sendx) 187 if raceenabled { 188 raceacquire(qp) 189 racerelease(qp) 190 } 191 typedmemmove(c.elemtype, qp, ep) 192 c.sendx++ 193 if c.sendx == c.dataqsiz { 194 c.sendx = 0 195 } 196 c.qcount++ 197 unlock(&c.lock) 198 return true 199 } 200 201 if !block { 202 unlock(&c.lock) 203 return false 204 } 205 206 // Block on the channel. Some receiver will complete our operation for us. 207 gp := getg() 208 mysg := acquireSudog() 209 mysg.releasetime = 0 210 if t0 != 0 { 211 mysg.releasetime = -1 212 } 213 // No stack splits between assigning elem and enqueuing mysg 214 // on gp.waiting where copystack can find it. 215 mysg.elem = ep 216 mysg.waitlink = nil 217 mysg.g = gp 218 mysg.selectdone = nil 219 mysg.c = c 220 gp.waiting = mysg 221 gp.param = nil 222 c.sendq.enqueue(mysg) 223 goparkunlock(&c.lock, "chan send", traceEvGoBlockSend, 3) 224 225 // someone woke us up. 226 if mysg != gp.waiting { 227 throw("G waiting list is corrupted") 228 } 229 gp.waiting = nil 230 if gp.param == nil { 231 if c.closed == 0 { 232 throw("chansend: spurious wakeup") 233 } 234 panic(plainError("send on closed channel")) 235 } 236 gp.param = nil 237 if mysg.releasetime > 0 { 238 blockevent(mysg.releasetime-t0, 2) 239 } 240 mysg.c = nil 241 releaseSudog(mysg) 242 return true 243 } 244 245 // send processes a send operation on an empty channel c. 246 // The value ep sent by the sender is copied to the receiver sg. 247 // The receiver is then woken up to go on its merry way. 248 // Channel c must be empty and locked. send unlocks c with unlockf. 249 // sg must already be dequeued from c. 250 // ep must be non-nil and point to the heap or the caller's stack. 251 func send(c *hchan, sg *sudog, ep unsafe.Pointer, unlockf func()) { 252 if raceenabled { 253 if c.dataqsiz == 0 { 254 racesync(c, sg) 255 } else { 256 // Pretend we go through the buffer, even though 257 // we copy directly. Note that we need to increment 258 // the head/tail locations only when raceenabled. 259 qp := chanbuf(c, c.recvx) 260 raceacquire(qp) 261 racerelease(qp) 262 raceacquireg(sg.g, qp) 263 racereleaseg(sg.g, qp) 264 c.recvx++ 265 if c.recvx == c.dataqsiz { 266 c.recvx = 0 267 } 268 c.sendx = c.recvx // c.sendx = (c.sendx+1) % c.dataqsiz 269 } 270 } 271 if sg.elem != nil { 272 sendDirect(c.elemtype, sg, ep) 273 sg.elem = nil 274 } 275 gp := sg.g 276 unlockf() 277 gp.param = unsafe.Pointer(sg) 278 if sg.releasetime != 0 { 279 sg.releasetime = cputicks() 280 } 281 goready(gp, 4) 282 } 283 284 func sendDirect(t *_type, sg *sudog, src unsafe.Pointer) { 285 // Send on an unbuffered or empty-buffered channel is the only operation 286 // in the entire runtime where one goroutine 287 // writes to the stack of another goroutine. The GC assumes that 288 // stack writes only happen when the goroutine is running and are 289 // only done by that goroutine. Using a write barrier is sufficient to 290 // make up for violating that assumption, but the write barrier has to work. 291 // typedmemmove will call heapBitsBulkBarrier, but the target bytes 292 // are not in the heap, so that will not help. We arrange to call 293 // memmove and typeBitsBulkBarrier instead. 294 295 // Once we read sg.elem out of sg, it will no longer 296 // be updated if the destination's stack gets copied (shrunk). 297 // So make sure that no preemption points can happen between read & use. 298 dst := sg.elem 299 memmove(dst, src, t.size) 300 typeBitsBulkBarrier(t, uintptr(dst), t.size) 301 } 302 303 func closechan(c *hchan) { 304 if c == nil { 305 panic(plainError("close of nil channel")) 306 } 307 308 lock(&c.lock) 309 if c.closed != 0 { 310 unlock(&c.lock) 311 panic(plainError("close of closed channel")) 312 } 313 314 if raceenabled { 315 callerpc := getcallerpc(unsafe.Pointer(&c)) 316 racewritepc(unsafe.Pointer(c), callerpc, funcPC(closechan)) 317 racerelease(unsafe.Pointer(c)) 318 } 319 320 c.closed = 1 321 322 var glist *g 323 324 // release all readers 325 for { 326 sg := c.recvq.dequeue() 327 if sg == nil { 328 break 329 } 330 if sg.elem != nil { 331 memclr(sg.elem, uintptr(c.elemsize)) 332 sg.elem = nil 333 } 334 if sg.releasetime != 0 { 335 sg.releasetime = cputicks() 336 } 337 gp := sg.g 338 gp.param = nil 339 if raceenabled { 340 raceacquireg(gp, unsafe.Pointer(c)) 341 } 342 gp.schedlink.set(glist) 343 glist = gp 344 } 345 346 // release all writers (they will panic) 347 for { 348 sg := c.sendq.dequeue() 349 if sg == nil { 350 break 351 } 352 sg.elem = nil 353 if sg.releasetime != 0 { 354 sg.releasetime = cputicks() 355 } 356 gp := sg.g 357 gp.param = nil 358 if raceenabled { 359 raceacquireg(gp, unsafe.Pointer(c)) 360 } 361 gp.schedlink.set(glist) 362 glist = gp 363 } 364 unlock(&c.lock) 365 366 // Ready all Gs now that we've dropped the channel lock. 367 for glist != nil { 368 gp := glist 369 glist = glist.schedlink.ptr() 370 gp.schedlink = 0 371 goready(gp, 3) 372 } 373 } 374 375 // entry points for <- c from compiled code 376 //go:nosplit 377 func chanrecv1(t *chantype, c *hchan, elem unsafe.Pointer) { 378 chanrecv(t, c, elem, true) 379 } 380 381 //go:nosplit 382 func chanrecv2(t *chantype, c *hchan, elem unsafe.Pointer) (received bool) { 383 _, received = chanrecv(t, c, elem, true) 384 return 385 } 386 387 // chanrecv receives on channel c and writes the received data to ep. 388 // ep may be nil, in which case received data is ignored. 389 // If block == false and no elements are available, returns (false, false). 390 // Otherwise, if c is closed, zeros *ep and returns (true, false). 391 // Otherwise, fills in *ep with an element and returns (true, true). 392 // A non-nil ep must point to the heap or the caller's stack. 393 func chanrecv(t *chantype, c *hchan, ep unsafe.Pointer, block bool) (selected, received bool) { 394 // raceenabled: don't need to check ep, as it is always on the stack 395 // or is new memory allocated by reflect. 396 397 if debugChan { 398 print("chanrecv: chan=", c, "\n") 399 } 400 401 if c == nil { 402 if !block { 403 return 404 } 405 gopark(nil, nil, "chan receive (nil chan)", traceEvGoStop, 2) 406 throw("unreachable") 407 } 408 409 // Fast path: check for failed non-blocking operation without acquiring the lock. 410 // 411 // After observing that the channel is not ready for receiving, we observe that the 412 // channel is not closed. Each of these observations is a single word-sized read 413 // (first c.sendq.first or c.qcount, and second c.closed). 414 // Because a channel cannot be reopened, the later observation of the channel 415 // being not closed implies that it was also not closed at the moment of the 416 // first observation. We behave as if we observed the channel at that moment 417 // and report that the receive cannot proceed. 418 // 419 // The order of operations is important here: reversing the operations can lead to 420 // incorrect behavior when racing with a close. 421 if !block && (c.dataqsiz == 0 && c.sendq.first == nil || 422 c.dataqsiz > 0 && atomic.Loaduint(&c.qcount) == 0) && 423 atomic.Load(&c.closed) == 0 { 424 return 425 } 426 427 var t0 int64 428 if blockprofilerate > 0 { 429 t0 = cputicks() 430 } 431 432 lock(&c.lock) 433 434 if c.closed != 0 && c.qcount == 0 { 435 if raceenabled { 436 raceacquire(unsafe.Pointer(c)) 437 } 438 unlock(&c.lock) 439 if ep != nil { 440 memclr(ep, uintptr(c.elemsize)) 441 } 442 return true, false 443 } 444 445 if sg := c.sendq.dequeue(); sg != nil { 446 // Found a waiting sender. If buffer is size 0, receive value 447 // directly from sender. Otherwise, receive from head of queue 448 // and add sender's value to the tail of the queue (both map to 449 // the same buffer slot because the queue is full). 450 recv(c, sg, ep, func() { unlock(&c.lock) }) 451 return true, true 452 } 453 454 if c.qcount > 0 { 455 // Receive directly from queue 456 qp := chanbuf(c, c.recvx) 457 if raceenabled { 458 raceacquire(qp) 459 racerelease(qp) 460 } 461 if ep != nil { 462 typedmemmove(c.elemtype, ep, qp) 463 } 464 memclr(qp, uintptr(c.elemsize)) 465 c.recvx++ 466 if c.recvx == c.dataqsiz { 467 c.recvx = 0 468 } 469 c.qcount-- 470 unlock(&c.lock) 471 return true, true 472 } 473 474 if !block { 475 unlock(&c.lock) 476 return false, false 477 } 478 479 // no sender available: block on this channel. 480 gp := getg() 481 mysg := acquireSudog() 482 mysg.releasetime = 0 483 if t0 != 0 { 484 mysg.releasetime = -1 485 } 486 // No stack splits between assigning elem and enqueuing mysg 487 // on gp.waiting where copystack can find it. 488 mysg.elem = ep 489 mysg.waitlink = nil 490 gp.waiting = mysg 491 mysg.g = gp 492 mysg.selectdone = nil 493 mysg.c = c 494 gp.param = nil 495 c.recvq.enqueue(mysg) 496 goparkunlock(&c.lock, "chan receive", traceEvGoBlockRecv, 3) 497 498 // someone woke us up 499 if mysg != gp.waiting { 500 throw("G waiting list is corrupted") 501 } 502 gp.waiting = nil 503 if mysg.releasetime > 0 { 504 blockevent(mysg.releasetime-t0, 2) 505 } 506 closed := gp.param == nil 507 gp.param = nil 508 mysg.c = nil 509 releaseSudog(mysg) 510 return true, !closed 511 } 512 513 // recv processes a receive operation on a full channel c. 514 // There are 2 parts: 515 // 1) The value sent by the sender sg is put into the channel 516 // and the sender is woken up to go on its merry way. 517 // 2) The value received by the receiver (the current G) is 518 // written to ep. 519 // For synchronous channels, both values are the same. 520 // For asynchronous channels, the receiver gets its data from 521 // the channel buffer and the sender's data is put in the 522 // channel buffer. 523 // Channel c must be full and locked. recv unlocks c with unlockf. 524 // sg must already be dequeued from c. 525 // A non-nil ep must point to the heap or the caller's stack. 526 func recv(c *hchan, sg *sudog, ep unsafe.Pointer, unlockf func()) { 527 if c.dataqsiz == 0 { 528 if raceenabled { 529 racesync(c, sg) 530 } 531 if ep != nil { 532 // copy data from sender 533 // ep points to our own stack or heap, so nothing 534 // special (ala sendDirect) needed here. 535 typedmemmove(c.elemtype, ep, sg.elem) 536 } 537 } else { 538 // Queue is full. Take the item at the 539 // head of the queue. Make the sender enqueue 540 // its item at the tail of the queue. Since the 541 // queue is full, those are both the same slot. 542 qp := chanbuf(c, c.recvx) 543 if raceenabled { 544 raceacquire(qp) 545 racerelease(qp) 546 raceacquireg(sg.g, qp) 547 racereleaseg(sg.g, qp) 548 } 549 // copy data from queue to receiver 550 if ep != nil { 551 typedmemmove(c.elemtype, ep, qp) 552 } 553 // copy data from sender to queue 554 typedmemmove(c.elemtype, qp, sg.elem) 555 c.recvx++ 556 if c.recvx == c.dataqsiz { 557 c.recvx = 0 558 } 559 c.sendx = c.recvx // c.sendx = (c.sendx+1) % c.dataqsiz 560 } 561 sg.elem = nil 562 gp := sg.g 563 unlockf() 564 gp.param = unsafe.Pointer(sg) 565 if sg.releasetime != 0 { 566 sg.releasetime = cputicks() 567 } 568 goready(gp, 4) 569 } 570 571 // compiler implements 572 // 573 // select { 574 // case c <- v: 575 // ... foo 576 // default: 577 // ... bar 578 // } 579 // 580 // as 581 // 582 // if selectnbsend(c, v) { 583 // ... foo 584 // } else { 585 // ... bar 586 // } 587 // 588 func selectnbsend(t *chantype, c *hchan, elem unsafe.Pointer) (selected bool) { 589 return chansend(t, c, elem, false, getcallerpc(unsafe.Pointer(&t))) 590 } 591 592 // compiler implements 593 // 594 // select { 595 // case v = <-c: 596 // ... foo 597 // default: 598 // ... bar 599 // } 600 // 601 // as 602 // 603 // if selectnbrecv(&v, c) { 604 // ... foo 605 // } else { 606 // ... bar 607 // } 608 // 609 func selectnbrecv(t *chantype, elem unsafe.Pointer, c *hchan) (selected bool) { 610 selected, _ = chanrecv(t, c, elem, false) 611 return 612 } 613 614 // compiler implements 615 // 616 // select { 617 // case v, ok = <-c: 618 // ... foo 619 // default: 620 // ... bar 621 // } 622 // 623 // as 624 // 625 // if c != nil && selectnbrecv2(&v, &ok, c) { 626 // ... foo 627 // } else { 628 // ... bar 629 // } 630 // 631 func selectnbrecv2(t *chantype, elem unsafe.Pointer, received *bool, c *hchan) (selected bool) { 632 // TODO(khr): just return 2 values from this function, now that it is in Go. 633 selected, *received = chanrecv(t, c, elem, false) 634 return 635 } 636 637 //go:linkname reflect_chansend reflect.chansend 638 func reflect_chansend(t *chantype, c *hchan, elem unsafe.Pointer, nb bool) (selected bool) { 639 return chansend(t, c, elem, !nb, getcallerpc(unsafe.Pointer(&t))) 640 } 641 642 //go:linkname reflect_chanrecv reflect.chanrecv 643 func reflect_chanrecv(t *chantype, c *hchan, nb bool, elem unsafe.Pointer) (selected bool, received bool) { 644 return chanrecv(t, c, elem, !nb) 645 } 646 647 //go:linkname reflect_chanlen reflect.chanlen 648 func reflect_chanlen(c *hchan) int { 649 if c == nil { 650 return 0 651 } 652 return int(c.qcount) 653 } 654 655 //go:linkname reflect_chancap reflect.chancap 656 func reflect_chancap(c *hchan) int { 657 if c == nil { 658 return 0 659 } 660 return int(c.dataqsiz) 661 } 662 663 //go:linkname reflect_chanclose reflect.chanclose 664 func reflect_chanclose(c *hchan) { 665 closechan(c) 666 } 667 668 func (q *waitq) enqueue(sgp *sudog) { 669 sgp.next = nil 670 x := q.last 671 if x == nil { 672 sgp.prev = nil 673 q.first = sgp 674 q.last = sgp 675 return 676 } 677 sgp.prev = x 678 x.next = sgp 679 q.last = sgp 680 } 681 682 func (q *waitq) dequeue() *sudog { 683 for { 684 sgp := q.first 685 if sgp == nil { 686 return nil 687 } 688 y := sgp.next 689 if y == nil { 690 q.first = nil 691 q.last = nil 692 } else { 693 y.prev = nil 694 q.first = y 695 sgp.next = nil // mark as removed (see dequeueSudog) 696 } 697 698 // if sgp participates in a select and is already signaled, ignore it 699 if sgp.selectdone != nil { 700 // claim the right to signal 701 if *sgp.selectdone != 0 || !atomic.Cas(sgp.selectdone, 0, 1) { 702 continue 703 } 704 } 705 706 return sgp 707 } 708 } 709 710 func racesync(c *hchan, sg *sudog) { 711 racerelease(chanbuf(c, 0)) 712 raceacquireg(sg.g, chanbuf(c, 0)) 713 racereleaseg(sg.g, chanbuf(c, 0)) 714 raceacquire(chanbuf(c, 0)) 715 }