github.com/MangoDowner/go-gm@v0.0.0-20180818020936-8baa2bd4408c/src/runtime/chan.go (about) 1 // Copyright 2014 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package runtime 6 7 // This file contains the implementation of Go channels. 8 9 // Invariants: 10 // At least one of c.sendq and c.recvq is empty, 11 // except for the case of an unbuffered channel with a single goroutine 12 // blocked on it for both sending and receiving using a select statement, 13 // in which case the length of c.sendq and c.recvq is limited only by the 14 // size of the select statement. 15 // 16 // For buffered channels, also: 17 // c.qcount > 0 implies that c.recvq is empty. 18 // c.qcount < c.dataqsiz implies that c.sendq is empty. 19 20 import ( 21 "runtime/internal/atomic" 22 "unsafe" 23 ) 24 25 const ( 26 maxAlign = 8 27 hchanSize = unsafe.Sizeof(hchan{}) + uintptr(-int(unsafe.Sizeof(hchan{}))&(maxAlign-1)) 28 debugChan = false 29 ) 30 31 type hchan struct { 32 qcount uint // total data in the queue 33 dataqsiz uint // size of the circular queue 34 buf unsafe.Pointer // points to an array of dataqsiz elements 35 elemsize uint16 36 closed uint32 37 elemtype *_type // element type 38 sendx uint // send index 39 recvx uint // receive index 40 recvq waitq // list of recv waiters 41 sendq waitq // list of send waiters 42 43 // lock protects all fields in hchan, as well as several 44 // fields in sudogs blocked on this channel. 45 // 46 // Do not change another G's status while holding this lock 47 // (in particular, do not ready a G), as this can deadlock 48 // with stack shrinking. 49 lock mutex 50 } 51 52 type waitq struct { 53 first *sudog 54 last *sudog 55 } 56 57 //go:linkname reflect_makechan reflect.makechan 58 func reflect_makechan(t *chantype, size int64) *hchan { 59 return makechan(t, size) 60 } 61 62 func makechan(t *chantype, size int64) *hchan { 63 elem := t.elem 64 65 // compiler checks this but be safe. 66 if elem.size >= 1<<16 { 67 throw("makechan: invalid channel element type") 68 } 69 if hchanSize%maxAlign != 0 || elem.align > maxAlign { 70 throw("makechan: bad alignment") 71 } 72 if size < 0 || int64(uintptr(size)) != size || (elem.size > 0 && uintptr(size) > (_MaxMem-hchanSize)/elem.size) { 73 panic(plainError("makechan: size out of range")) 74 } 75 76 var c *hchan 77 if elem.kind&kindNoPointers != 0 || size == 0 { 78 // Allocate memory in one call. 79 // Hchan does not contain pointers interesting for GC in this case: 80 // buf points into the same allocation, elemtype is persistent. 81 // SudoG's are referenced from their owning thread so they can't be collected. 82 // TODO(dvyukov,rlh): Rethink when collector can move allocated objects. 83 c = (*hchan)(mallocgc(hchanSize+uintptr(size)*elem.size, nil, true)) 84 if size > 0 && elem.size != 0 { 85 c.buf = add(unsafe.Pointer(c), hchanSize) 86 } else { 87 // race detector uses this location for synchronization 88 // Also prevents us from pointing beyond the allocation (see issue 9401). 89 c.buf = unsafe.Pointer(c) 90 } 91 } else { 92 c = new(hchan) 93 c.buf = newarray(elem, int(size)) 94 } 95 c.elemsize = uint16(elem.size) 96 c.elemtype = elem 97 c.dataqsiz = uint(size) 98 99 if debugChan { 100 print("makechan: chan=", c, "; elemsize=", elem.size, "; elemalg=", elem.alg, "; dataqsiz=", size, "\n") 101 } 102 return c 103 } 104 105 // chanbuf(c, i) is pointer to the i'th slot in the buffer. 106 func chanbuf(c *hchan, i uint) unsafe.Pointer { 107 return add(c.buf, uintptr(i)*uintptr(c.elemsize)) 108 } 109 110 // entry point for c <- x from compiled code 111 //go:nosplit 112 func chansend1(c *hchan, elem unsafe.Pointer) { 113 chansend(c, elem, true, getcallerpc(unsafe.Pointer(&c))) 114 } 115 116 /* 117 * generic single channel send/recv 118 * If block is not nil, 119 * then the protocol will not 120 * sleep but return if it could 121 * not complete. 122 * 123 * sleep can wake up with g.param == nil 124 * when a channel involved in the sleep has 125 * been closed. it is easiest to loop and re-run 126 * the operation; we'll see that it's now closed. 127 */ 128 func chansend(c *hchan, ep unsafe.Pointer, block bool, callerpc uintptr) bool { 129 if c == nil { 130 if !block { 131 return false 132 } 133 gopark(nil, nil, "chan send (nil chan)", traceEvGoStop, 2) 134 throw("unreachable") 135 } 136 137 if debugChan { 138 print("chansend: chan=", c, "\n") 139 } 140 141 if raceenabled { 142 racereadpc(unsafe.Pointer(c), callerpc, funcPC(chansend)) 143 } 144 145 // Fast path: check for failed non-blocking operation without acquiring the lock. 146 // 147 // After observing that the channel is not closed, we observe that the channel is 148 // not ready for sending. Each of these observations is a single word-sized read 149 // (first c.closed and second c.recvq.first or c.qcount depending on kind of channel). 150 // Because a closed channel cannot transition from 'ready for sending' to 151 // 'not ready for sending', even if the channel is closed between the two observations, 152 // they imply a moment between the two when the channel was both not yet closed 153 // and not ready for sending. We behave as if we observed the channel at that moment, 154 // and report that the send cannot proceed. 155 // 156 // It is okay if the reads are reordered here: if we observe that the channel is not 157 // ready for sending and then observe that it is not closed, that implies that the 158 // channel wasn't closed during the first observation. 159 if !block && c.closed == 0 && ((c.dataqsiz == 0 && c.recvq.first == nil) || 160 (c.dataqsiz > 0 && c.qcount == c.dataqsiz)) { 161 return false 162 } 163 164 var t0 int64 165 if blockprofilerate > 0 { 166 t0 = cputicks() 167 } 168 169 lock(&c.lock) 170 171 if c.closed != 0 { 172 unlock(&c.lock) 173 panic(plainError("send on closed channel")) 174 } 175 176 if sg := c.recvq.dequeue(); sg != nil { 177 // Found a waiting receiver. We pass the value we want to send 178 // directly to the receiver, bypassing the channel buffer (if any). 179 send(c, sg, ep, func() { unlock(&c.lock) }, 3) 180 return true 181 } 182 183 if c.qcount < c.dataqsiz { 184 // Space is available in the channel buffer. Enqueue the element to send. 185 qp := chanbuf(c, c.sendx) 186 if raceenabled { 187 raceacquire(qp) 188 racerelease(qp) 189 } 190 typedmemmove(c.elemtype, qp, ep) 191 c.sendx++ 192 if c.sendx == c.dataqsiz { 193 c.sendx = 0 194 } 195 c.qcount++ 196 unlock(&c.lock) 197 return true 198 } 199 200 if !block { 201 unlock(&c.lock) 202 return false 203 } 204 205 // Block on the channel. Some receiver will complete our operation for us. 206 gp := getg() 207 mysg := acquireSudog() 208 mysg.releasetime = 0 209 if t0 != 0 { 210 mysg.releasetime = -1 211 } 212 // No stack splits between assigning elem and enqueuing mysg 213 // on gp.waiting where copystack can find it. 214 mysg.elem = ep 215 mysg.waitlink = nil 216 mysg.g = gp 217 mysg.selectdone = nil 218 mysg.c = c 219 gp.waiting = mysg 220 gp.param = nil 221 c.sendq.enqueue(mysg) 222 goparkunlock(&c.lock, "chan send", traceEvGoBlockSend, 3) 223 224 // someone woke us up. 225 if mysg != gp.waiting { 226 throw("G waiting list is corrupted") 227 } 228 gp.waiting = nil 229 if gp.param == nil { 230 if c.closed == 0 { 231 throw("chansend: spurious wakeup") 232 } 233 panic(plainError("send on closed channel")) 234 } 235 gp.param = nil 236 if mysg.releasetime > 0 { 237 blockevent(mysg.releasetime-t0, 2) 238 } 239 mysg.c = nil 240 releaseSudog(mysg) 241 return true 242 } 243 244 // send processes a send operation on an empty channel c. 245 // The value ep sent by the sender is copied to the receiver sg. 246 // The receiver is then woken up to go on its merry way. 247 // Channel c must be empty and locked. send unlocks c with unlockf. 248 // sg must already be dequeued from c. 249 // ep must be non-nil and point to the heap or the caller's stack. 250 func send(c *hchan, sg *sudog, ep unsafe.Pointer, unlockf func(), skip int) { 251 if raceenabled { 252 if c.dataqsiz == 0 { 253 racesync(c, sg) 254 } else { 255 // Pretend we go through the buffer, even though 256 // we copy directly. Note that we need to increment 257 // the head/tail locations only when raceenabled. 258 qp := chanbuf(c, c.recvx) 259 raceacquire(qp) 260 racerelease(qp) 261 raceacquireg(sg.g, qp) 262 racereleaseg(sg.g, qp) 263 c.recvx++ 264 if c.recvx == c.dataqsiz { 265 c.recvx = 0 266 } 267 c.sendx = c.recvx // c.sendx = (c.sendx+1) % c.dataqsiz 268 } 269 } 270 if sg.elem != nil { 271 sendDirect(c.elemtype, sg, ep) 272 sg.elem = nil 273 } 274 gp := sg.g 275 unlockf() 276 gp.param = unsafe.Pointer(sg) 277 if sg.releasetime != 0 { 278 sg.releasetime = cputicks() 279 } 280 goready(gp, skip+1) 281 } 282 283 // Sends and receives on unbuffered or empty-buffered channels are the 284 // only operations where one running goroutine writes to the stack of 285 // another running goroutine. The GC assumes that stack writes only 286 // happen when the goroutine is running and are only done by that 287 // goroutine. Using a write barrier is sufficient to make up for 288 // violating that assumption, but the write barrier has to work. 289 // typedmemmove will call bulkBarrierPreWrite, but the target bytes 290 // are not in the heap, so that will not help. We arrange to call 291 // memmove and typeBitsBulkBarrier instead. 292 293 func sendDirect(t *_type, sg *sudog, src unsafe.Pointer) { 294 // src is on our stack, dst is a slot on another stack. 295 296 // Once we read sg.elem out of sg, it will no longer 297 // be updated if the destination's stack gets copied (shrunk). 298 // So make sure that no preemption points can happen between read & use. 299 dst := sg.elem 300 typeBitsBulkBarrier(t, uintptr(dst), uintptr(src), t.size) 301 memmove(dst, src, t.size) 302 } 303 304 func recvDirect(t *_type, sg *sudog, dst unsafe.Pointer) { 305 // dst is on our stack or the heap, src is on another stack. 306 // The channel is locked, so src will not move during this 307 // operation. 308 src := sg.elem 309 typeBitsBulkBarrier(t, uintptr(dst), uintptr(src), t.size) 310 memmove(dst, src, t.size) 311 } 312 313 func closechan(c *hchan) { 314 if c == nil { 315 panic(plainError("close of nil channel")) 316 } 317 318 lock(&c.lock) 319 if c.closed != 0 { 320 unlock(&c.lock) 321 panic(plainError("close of closed channel")) 322 } 323 324 if raceenabled { 325 callerpc := getcallerpc(unsafe.Pointer(&c)) 326 racewritepc(unsafe.Pointer(c), callerpc, funcPC(closechan)) 327 racerelease(unsafe.Pointer(c)) 328 } 329 330 c.closed = 1 331 332 var glist *g 333 334 // release all readers 335 for { 336 sg := c.recvq.dequeue() 337 if sg == nil { 338 break 339 } 340 if sg.elem != nil { 341 typedmemclr(c.elemtype, sg.elem) 342 sg.elem = nil 343 } 344 if sg.releasetime != 0 { 345 sg.releasetime = cputicks() 346 } 347 gp := sg.g 348 gp.param = nil 349 if raceenabled { 350 raceacquireg(gp, unsafe.Pointer(c)) 351 } 352 gp.schedlink.set(glist) 353 glist = gp 354 } 355 356 // release all writers (they will panic) 357 for { 358 sg := c.sendq.dequeue() 359 if sg == nil { 360 break 361 } 362 sg.elem = nil 363 if sg.releasetime != 0 { 364 sg.releasetime = cputicks() 365 } 366 gp := sg.g 367 gp.param = nil 368 if raceenabled { 369 raceacquireg(gp, unsafe.Pointer(c)) 370 } 371 gp.schedlink.set(glist) 372 glist = gp 373 } 374 unlock(&c.lock) 375 376 // Ready all Gs now that we've dropped the channel lock. 377 for glist != nil { 378 gp := glist 379 glist = glist.schedlink.ptr() 380 gp.schedlink = 0 381 goready(gp, 3) 382 } 383 } 384 385 // entry points for <- c from compiled code 386 //go:nosplit 387 func chanrecv1(c *hchan, elem unsafe.Pointer) { 388 chanrecv(c, elem, true) 389 } 390 391 //go:nosplit 392 func chanrecv2(c *hchan, elem unsafe.Pointer) (received bool) { 393 _, received = chanrecv(c, elem, true) 394 return 395 } 396 397 // chanrecv receives on channel c and writes the received data to ep. 398 // ep may be nil, in which case received data is ignored. 399 // If block == false and no elements are available, returns (false, false). 400 // Otherwise, if c is closed, zeros *ep and returns (true, false). 401 // Otherwise, fills in *ep with an element and returns (true, true). 402 // A non-nil ep must point to the heap or the caller's stack. 403 func chanrecv(c *hchan, ep unsafe.Pointer, block bool) (selected, received bool) { 404 // raceenabled: don't need to check ep, as it is always on the stack 405 // or is new memory allocated by reflect. 406 407 if debugChan { 408 print("chanrecv: chan=", c, "\n") 409 } 410 411 if c == nil { 412 if !block { 413 return 414 } 415 gopark(nil, nil, "chan receive (nil chan)", traceEvGoStop, 2) 416 throw("unreachable") 417 } 418 419 // Fast path: check for failed non-blocking operation without acquiring the lock. 420 // 421 // After observing that the channel is not ready for receiving, we observe that the 422 // channel is not closed. Each of these observations is a single word-sized read 423 // (first c.sendq.first or c.qcount, and second c.closed). 424 // Because a channel cannot be reopened, the later observation of the channel 425 // being not closed implies that it was also not closed at the moment of the 426 // first observation. We behave as if we observed the channel at that moment 427 // and report that the receive cannot proceed. 428 // 429 // The order of operations is important here: reversing the operations can lead to 430 // incorrect behavior when racing with a close. 431 if !block && (c.dataqsiz == 0 && c.sendq.first == nil || 432 c.dataqsiz > 0 && atomic.Loaduint(&c.qcount) == 0) && 433 atomic.Load(&c.closed) == 0 { 434 return 435 } 436 437 var t0 int64 438 if blockprofilerate > 0 { 439 t0 = cputicks() 440 } 441 442 lock(&c.lock) 443 444 if c.closed != 0 && c.qcount == 0 { 445 if raceenabled { 446 raceacquire(unsafe.Pointer(c)) 447 } 448 unlock(&c.lock) 449 if ep != nil { 450 typedmemclr(c.elemtype, ep) 451 } 452 return true, false 453 } 454 455 if sg := c.sendq.dequeue(); sg != nil { 456 // Found a waiting sender. If buffer is size 0, receive value 457 // directly from sender. Otherwise, receive from head of queue 458 // and add sender's value to the tail of the queue (both map to 459 // the same buffer slot because the queue is full). 460 recv(c, sg, ep, func() { unlock(&c.lock) }, 3) 461 return true, true 462 } 463 464 if c.qcount > 0 { 465 // Receive directly from queue 466 qp := chanbuf(c, c.recvx) 467 if raceenabled { 468 raceacquire(qp) 469 racerelease(qp) 470 } 471 if ep != nil { 472 typedmemmove(c.elemtype, ep, qp) 473 } 474 typedmemclr(c.elemtype, qp) 475 c.recvx++ 476 if c.recvx == c.dataqsiz { 477 c.recvx = 0 478 } 479 c.qcount-- 480 unlock(&c.lock) 481 return true, true 482 } 483 484 if !block { 485 unlock(&c.lock) 486 return false, false 487 } 488 489 // no sender available: block on this channel. 490 gp := getg() 491 mysg := acquireSudog() 492 mysg.releasetime = 0 493 if t0 != 0 { 494 mysg.releasetime = -1 495 } 496 // No stack splits between assigning elem and enqueuing mysg 497 // on gp.waiting where copystack can find it. 498 mysg.elem = ep 499 mysg.waitlink = nil 500 gp.waiting = mysg 501 mysg.g = gp 502 mysg.selectdone = nil 503 mysg.c = c 504 gp.param = nil 505 c.recvq.enqueue(mysg) 506 goparkunlock(&c.lock, "chan receive", traceEvGoBlockRecv, 3) 507 508 // someone woke us up 509 if mysg != gp.waiting { 510 throw("G waiting list is corrupted") 511 } 512 gp.waiting = nil 513 if mysg.releasetime > 0 { 514 blockevent(mysg.releasetime-t0, 2) 515 } 516 closed := gp.param == nil 517 gp.param = nil 518 mysg.c = nil 519 releaseSudog(mysg) 520 return true, !closed 521 } 522 523 // recv processes a receive operation on a full channel c. 524 // There are 2 parts: 525 // 1) The value sent by the sender sg is put into the channel 526 // and the sender is woken up to go on its merry way. 527 // 2) The value received by the receiver (the current G) is 528 // written to ep. 529 // For synchronous channels, both values are the same. 530 // For asynchronous channels, the receiver gets its data from 531 // the channel buffer and the sender's data is put in the 532 // channel buffer. 533 // Channel c must be full and locked. recv unlocks c with unlockf. 534 // sg must already be dequeued from c. 535 // A non-nil ep must point to the heap or the caller's stack. 536 func recv(c *hchan, sg *sudog, ep unsafe.Pointer, unlockf func(), skip int) { 537 if c.dataqsiz == 0 { 538 if raceenabled { 539 racesync(c, sg) 540 } 541 if ep != nil { 542 // copy data from sender 543 recvDirect(c.elemtype, sg, ep) 544 } 545 } else { 546 // Queue is full. Take the item at the 547 // head of the queue. Make the sender enqueue 548 // its item at the tail of the queue. Since the 549 // queue is full, those are both the same slot. 550 qp := chanbuf(c, c.recvx) 551 if raceenabled { 552 raceacquire(qp) 553 racerelease(qp) 554 raceacquireg(sg.g, qp) 555 racereleaseg(sg.g, qp) 556 } 557 // copy data from queue to receiver 558 if ep != nil { 559 typedmemmove(c.elemtype, ep, qp) 560 } 561 // copy data from sender to queue 562 typedmemmove(c.elemtype, qp, sg.elem) 563 c.recvx++ 564 if c.recvx == c.dataqsiz { 565 c.recvx = 0 566 } 567 c.sendx = c.recvx // c.sendx = (c.sendx+1) % c.dataqsiz 568 } 569 sg.elem = nil 570 gp := sg.g 571 unlockf() 572 gp.param = unsafe.Pointer(sg) 573 if sg.releasetime != 0 { 574 sg.releasetime = cputicks() 575 } 576 goready(gp, skip+1) 577 } 578 579 // compiler implements 580 // 581 // select { 582 // case c <- v: 583 // ... foo 584 // default: 585 // ... bar 586 // } 587 // 588 // as 589 // 590 // if selectnbsend(c, v) { 591 // ... foo 592 // } else { 593 // ... bar 594 // } 595 // 596 func selectnbsend(c *hchan, elem unsafe.Pointer) (selected bool) { 597 return chansend(c, elem, false, getcallerpc(unsafe.Pointer(&c))) 598 } 599 600 // compiler implements 601 // 602 // select { 603 // case v = <-c: 604 // ... foo 605 // default: 606 // ... bar 607 // } 608 // 609 // as 610 // 611 // if selectnbrecv(&v, c) { 612 // ... foo 613 // } else { 614 // ... bar 615 // } 616 // 617 func selectnbrecv(elem unsafe.Pointer, c *hchan) (selected bool) { 618 selected, _ = chanrecv(c, elem, false) 619 return 620 } 621 622 // compiler implements 623 // 624 // select { 625 // case v, ok = <-c: 626 // ... foo 627 // default: 628 // ... bar 629 // } 630 // 631 // as 632 // 633 // if c != nil && selectnbrecv2(&v, &ok, c) { 634 // ... foo 635 // } else { 636 // ... bar 637 // } 638 // 639 func selectnbrecv2(elem unsafe.Pointer, received *bool, c *hchan) (selected bool) { 640 // TODO(khr): just return 2 values from this function, now that it is in Go. 641 selected, *received = chanrecv(c, elem, false) 642 return 643 } 644 645 //go:linkname reflect_chansend reflect.chansend 646 func reflect_chansend(c *hchan, elem unsafe.Pointer, nb bool) (selected bool) { 647 return chansend(c, elem, !nb, getcallerpc(unsafe.Pointer(&c))) 648 } 649 650 //go:linkname reflect_chanrecv reflect.chanrecv 651 func reflect_chanrecv(c *hchan, nb bool, elem unsafe.Pointer) (selected bool, received bool) { 652 return chanrecv(c, elem, !nb) 653 } 654 655 //go:linkname reflect_chanlen reflect.chanlen 656 func reflect_chanlen(c *hchan) int { 657 if c == nil { 658 return 0 659 } 660 return int(c.qcount) 661 } 662 663 //go:linkname reflect_chancap reflect.chancap 664 func reflect_chancap(c *hchan) int { 665 if c == nil { 666 return 0 667 } 668 return int(c.dataqsiz) 669 } 670 671 //go:linkname reflect_chanclose reflect.chanclose 672 func reflect_chanclose(c *hchan) { 673 closechan(c) 674 } 675 676 func (q *waitq) enqueue(sgp *sudog) { 677 sgp.next = nil 678 x := q.last 679 if x == nil { 680 sgp.prev = nil 681 q.first = sgp 682 q.last = sgp 683 return 684 } 685 sgp.prev = x 686 x.next = sgp 687 q.last = sgp 688 } 689 690 func (q *waitq) dequeue() *sudog { 691 for { 692 sgp := q.first 693 if sgp == nil { 694 return nil 695 } 696 y := sgp.next 697 if y == nil { 698 q.first = nil 699 q.last = nil 700 } else { 701 y.prev = nil 702 q.first = y 703 sgp.next = nil // mark as removed (see dequeueSudog) 704 } 705 706 // if sgp participates in a select and is already signaled, ignore it 707 if sgp.selectdone != nil { 708 // claim the right to signal 709 if *sgp.selectdone != 0 || !atomic.Cas(sgp.selectdone, 0, 1) { 710 continue 711 } 712 } 713 714 return sgp 715 } 716 } 717 718 func racesync(c *hchan, sg *sudog) { 719 racerelease(chanbuf(c, 0)) 720 raceacquireg(sg.g, chanbuf(c, 0)) 721 racereleaseg(sg.g, chanbuf(c, 0)) 722 raceacquire(chanbuf(c, 0)) 723 }