github.com/zach-klippenstein/go@v0.0.0-20150108044943-fcfbeb3adf58/src/runtime/netpoll.go (about) 1 // Copyright 2013 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 // +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris windows 6 7 package runtime 8 9 import "unsafe" 10 11 // Integrated network poller (platform-independent part). 12 // A particular implementation (epoll/kqueue) must define the following functions: 13 // func netpollinit() // to initialize the poller 14 // func netpollopen(fd uintptr, pd *pollDesc) int32 // to arm edge-triggered notifications 15 // and associate fd with pd. 16 // An implementation must call the following function to denote that the pd is ready. 17 // func netpollready(gpp **g, pd *pollDesc, mode int32) 18 19 // pollDesc contains 2 binary semaphores, rg and wg, to park reader and writer 20 // goroutines respectively. The semaphore can be in the following states: 21 // pdReady - io readiness notification is pending; 22 // a goroutine consumes the notification by changing the state to nil. 23 // pdWait - a goroutine prepares to park on the semaphore, but not yet parked; 24 // the goroutine commits to park by changing the state to G pointer, 25 // or, alternatively, concurrent io notification changes the state to READY, 26 // or, alternatively, concurrent timeout/close changes the state to nil. 27 // G pointer - the goroutine is blocked on the semaphore; 28 // io notification or timeout/close changes the state to READY or nil respectively 29 // and unparks the goroutine. 30 // nil - nothing of the above. 31 const ( 32 pdReady uintptr = 1 33 pdWait uintptr = 2 34 ) 35 36 const pollBlockSize = 4 * 1024 37 38 // Network poller descriptor. 39 type pollDesc struct { 40 link *pollDesc // in pollcache, protected by pollcache.lock 41 42 // The lock protects pollOpen, pollSetDeadline, pollUnblock and deadlineimpl operations. 43 // This fully covers seq, rt and wt variables. fd is constant throughout the PollDesc lifetime. 44 // pollReset, pollWait, pollWaitCanceled and runtime·netpollready (IO readiness notification) 45 // proceed w/o taking the lock. So closing, rg, rd, wg and wd are manipulated 46 // in a lock-free way by all operations. 47 // NOTE(dvyukov): the following code uses uintptr to store *g (rg/wg), 48 // that will blow up when GC starts moving objects. 49 lock mutex // protectes the following fields 50 fd uintptr 51 closing bool 52 seq uintptr // protects from stale timers and ready notifications 53 rg uintptr // pdReady, pdWait, G waiting for read or nil 54 rt timer // read deadline timer (set if rt.f != nil) 55 rd int64 // read deadline 56 wg uintptr // pdReady, pdWait, G waiting for write or nil 57 wt timer // write deadline timer 58 wd int64 // write deadline 59 user uint32 // user settable cookie 60 } 61 62 type pollCache struct { 63 lock mutex 64 first *pollDesc 65 // PollDesc objects must be type-stable, 66 // because we can get ready notification from epoll/kqueue 67 // after the descriptor is closed/reused. 68 // Stale notifications are detected using seq variable, 69 // seq is incremented when deadlines are changed or descriptor is reused. 70 } 71 72 var pollcache pollCache 73 74 //go:linkname net_runtime_pollServerInit net.runtime_pollServerInit 75 func net_runtime_pollServerInit() { 76 netpollinit() 77 } 78 79 //go:linkname net_runtime_pollOpen net.runtime_pollOpen 80 func net_runtime_pollOpen(fd uintptr) (*pollDesc, int) { 81 pd := pollcache.alloc() 82 lock(&pd.lock) 83 if pd.wg != 0 && pd.wg != pdReady { 84 throw("netpollOpen: blocked write on free descriptor") 85 } 86 if pd.rg != 0 && pd.rg != pdReady { 87 throw("netpollOpen: blocked read on free descriptor") 88 } 89 pd.fd = fd 90 pd.closing = false 91 pd.seq++ 92 pd.rg = 0 93 pd.rd = 0 94 pd.wg = 0 95 pd.wd = 0 96 unlock(&pd.lock) 97 98 var errno int32 99 errno = netpollopen(fd, pd) 100 return pd, int(errno) 101 } 102 103 //go:linkname net_runtime_pollClose net.runtime_pollClose 104 func net_runtime_pollClose(pd *pollDesc) { 105 if !pd.closing { 106 throw("netpollClose: close w/o unblock") 107 } 108 if pd.wg != 0 && pd.wg != pdReady { 109 throw("netpollClose: blocked write on closing descriptor") 110 } 111 if pd.rg != 0 && pd.rg != pdReady { 112 throw("netpollClose: blocked read on closing descriptor") 113 } 114 netpollclose(uintptr(pd.fd)) 115 pollcache.free(pd) 116 } 117 118 func (c *pollCache) free(pd *pollDesc) { 119 lock(&c.lock) 120 pd.link = c.first 121 c.first = pd 122 unlock(&c.lock) 123 } 124 125 //go:linkname net_runtime_pollReset net.runtime_pollReset 126 func net_runtime_pollReset(pd *pollDesc, mode int) int { 127 err := netpollcheckerr(pd, int32(mode)) 128 if err != 0 { 129 return err 130 } 131 if mode == 'r' { 132 pd.rg = 0 133 } else if mode == 'w' { 134 pd.wg = 0 135 } 136 return 0 137 } 138 139 //go:linkname net_runtime_pollWait net.runtime_pollWait 140 func net_runtime_pollWait(pd *pollDesc, mode int) int { 141 err := netpollcheckerr(pd, int32(mode)) 142 if err != 0 { 143 return err 144 } 145 // As for now only Solaris uses level-triggered IO. 146 if GOOS == "solaris" { 147 netpollarm(pd, mode) 148 } 149 for !netpollblock(pd, int32(mode), false) { 150 err = netpollcheckerr(pd, int32(mode)) 151 if err != 0 { 152 return err 153 } 154 // Can happen if timeout has fired and unblocked us, 155 // but before we had a chance to run, timeout has been reset. 156 // Pretend it has not happened and retry. 157 } 158 return 0 159 } 160 161 //go:linkname net_runtime_pollWaitCanceled net.runtime_pollWaitCanceled 162 func net_runtime_pollWaitCanceled(pd *pollDesc, mode int) { 163 // This function is used only on windows after a failed attempt to cancel 164 // a pending async IO operation. Wait for ioready, ignore closing or timeouts. 165 for !netpollblock(pd, int32(mode), true) { 166 } 167 } 168 169 //go:linkname net_runtime_pollSetDeadline net.runtime_pollSetDeadline 170 func net_runtime_pollSetDeadline(pd *pollDesc, d int64, mode int) { 171 lock(&pd.lock) 172 if pd.closing { 173 unlock(&pd.lock) 174 return 175 } 176 pd.seq++ // invalidate current timers 177 // Reset current timers. 178 if pd.rt.f != nil { 179 deltimer(&pd.rt) 180 pd.rt.f = nil 181 } 182 if pd.wt.f != nil { 183 deltimer(&pd.wt) 184 pd.wt.f = nil 185 } 186 // Setup new timers. 187 if d != 0 && d <= nanotime() { 188 d = -1 189 } 190 if mode == 'r' || mode == 'r'+'w' { 191 pd.rd = d 192 } 193 if mode == 'w' || mode == 'r'+'w' { 194 pd.wd = d 195 } 196 if pd.rd > 0 && pd.rd == pd.wd { 197 pd.rt.f = netpollDeadline 198 pd.rt.when = pd.rd 199 // Copy current seq into the timer arg. 200 // Timer func will check the seq against current descriptor seq, 201 // if they differ the descriptor was reused or timers were reset. 202 pd.rt.arg = pd 203 pd.rt.seq = pd.seq 204 addtimer(&pd.rt) 205 } else { 206 if pd.rd > 0 { 207 pd.rt.f = netpollReadDeadline 208 pd.rt.when = pd.rd 209 pd.rt.arg = pd 210 pd.rt.seq = pd.seq 211 addtimer(&pd.rt) 212 } 213 if pd.wd > 0 { 214 pd.wt.f = netpollWriteDeadline 215 pd.wt.when = pd.wd 216 pd.wt.arg = pd 217 pd.wt.seq = pd.seq 218 addtimer(&pd.wt) 219 } 220 } 221 // If we set the new deadline in the past, unblock currently pending IO if any. 222 var rg, wg *g 223 atomicstorep(unsafe.Pointer(&wg), nil) // full memory barrier between stores to rd/wd and load of rg/wg in netpollunblock 224 if pd.rd < 0 { 225 rg = netpollunblock(pd, 'r', false) 226 } 227 if pd.wd < 0 { 228 wg = netpollunblock(pd, 'w', false) 229 } 230 unlock(&pd.lock) 231 if rg != nil { 232 goready(rg) 233 } 234 if wg != nil { 235 goready(wg) 236 } 237 } 238 239 //go:linkname net_runtime_pollUnblock net.runtime_pollUnblock 240 func net_runtime_pollUnblock(pd *pollDesc) { 241 lock(&pd.lock) 242 if pd.closing { 243 throw("netpollUnblock: already closing") 244 } 245 pd.closing = true 246 pd.seq++ 247 var rg, wg *g 248 atomicstorep(unsafe.Pointer(&rg), nil) // full memory barrier between store to closing and read of rg/wg in netpollunblock 249 rg = netpollunblock(pd, 'r', false) 250 wg = netpollunblock(pd, 'w', false) 251 if pd.rt.f != nil { 252 deltimer(&pd.rt) 253 pd.rt.f = nil 254 } 255 if pd.wt.f != nil { 256 deltimer(&pd.wt) 257 pd.wt.f = nil 258 } 259 unlock(&pd.lock) 260 if rg != nil { 261 goready(rg) 262 } 263 if wg != nil { 264 goready(wg) 265 } 266 } 267 268 // make pd ready, newly runnable goroutines (if any) are returned in rg/wg 269 func netpollready(gpp **g, pd *pollDesc, mode int32) { 270 var rg, wg *g 271 if mode == 'r' || mode == 'r'+'w' { 272 rg = netpollunblock(pd, 'r', true) 273 } 274 if mode == 'w' || mode == 'r'+'w' { 275 wg = netpollunblock(pd, 'w', true) 276 } 277 if rg != nil { 278 rg.schedlink = *gpp 279 *gpp = rg 280 } 281 if wg != nil { 282 wg.schedlink = *gpp 283 *gpp = wg 284 } 285 } 286 287 func netpollcheckerr(pd *pollDesc, mode int32) int { 288 if pd.closing { 289 return 1 // errClosing 290 } 291 if (mode == 'r' && pd.rd < 0) || (mode == 'w' && pd.wd < 0) { 292 return 2 // errTimeout 293 } 294 return 0 295 } 296 297 func netpollblockcommit(gp *g, gpp unsafe.Pointer) bool { 298 return casuintptr((*uintptr)(gpp), pdWait, uintptr(unsafe.Pointer(gp))) 299 } 300 301 // returns true if IO is ready, or false if timedout or closed 302 // waitio - wait only for completed IO, ignore errors 303 func netpollblock(pd *pollDesc, mode int32, waitio bool) bool { 304 gpp := &pd.rg 305 if mode == 'w' { 306 gpp = &pd.wg 307 } 308 309 // set the gpp semaphore to WAIT 310 for { 311 old := *gpp 312 if old == pdReady { 313 *gpp = 0 314 return true 315 } 316 if old != 0 { 317 throw("netpollblock: double wait") 318 } 319 if casuintptr(gpp, 0, pdWait) { 320 break 321 } 322 } 323 324 // need to recheck error states after setting gpp to WAIT 325 // this is necessary because runtime_pollUnblock/runtime_pollSetDeadline/deadlineimpl 326 // do the opposite: store to closing/rd/wd, membarrier, load of rg/wg 327 if waitio || netpollcheckerr(pd, mode) == 0 { 328 gopark(netpollblockcommit, unsafe.Pointer(gpp), "IO wait") 329 } 330 // be careful to not lose concurrent READY notification 331 old := xchguintptr(gpp, 0) 332 if old > pdWait { 333 throw("netpollblock: corrupted state") 334 } 335 return old == pdReady 336 } 337 338 func netpollunblock(pd *pollDesc, mode int32, ioready bool) *g { 339 gpp := &pd.rg 340 if mode == 'w' { 341 gpp = &pd.wg 342 } 343 344 for { 345 old := *gpp 346 if old == pdReady { 347 return nil 348 } 349 if old == 0 && !ioready { 350 // Only set READY for ioready. runtime_pollWait 351 // will check for timeout/cancel before waiting. 352 return nil 353 } 354 var new uintptr 355 if ioready { 356 new = pdReady 357 } 358 if casuintptr(gpp, old, new) { 359 if old == pdReady || old == pdWait { 360 old = 0 361 } 362 return (*g)(unsafe.Pointer(old)) 363 } 364 } 365 } 366 367 func netpolldeadlineimpl(pd *pollDesc, seq uintptr, read, write bool) { 368 lock(&pd.lock) 369 // Seq arg is seq when the timer was set. 370 // If it's stale, ignore the timer event. 371 if seq != pd.seq { 372 // The descriptor was reused or timers were reset. 373 unlock(&pd.lock) 374 return 375 } 376 var rg *g 377 if read { 378 if pd.rd <= 0 || pd.rt.f == nil { 379 throw("netpolldeadlineimpl: inconsistent read deadline") 380 } 381 pd.rd = -1 382 atomicstorep(unsafe.Pointer(&pd.rt.f), nil) // full memory barrier between store to rd and load of rg in netpollunblock 383 rg = netpollunblock(pd, 'r', false) 384 } 385 var wg *g 386 if write { 387 if pd.wd <= 0 || pd.wt.f == nil && !read { 388 throw("netpolldeadlineimpl: inconsistent write deadline") 389 } 390 pd.wd = -1 391 atomicstorep(unsafe.Pointer(&pd.wt.f), nil) // full memory barrier between store to wd and load of wg in netpollunblock 392 wg = netpollunblock(pd, 'w', false) 393 } 394 unlock(&pd.lock) 395 if rg != nil { 396 goready(rg) 397 } 398 if wg != nil { 399 goready(wg) 400 } 401 } 402 403 func netpollDeadline(arg interface{}, seq uintptr) { 404 netpolldeadlineimpl(arg.(*pollDesc), seq, true, true) 405 } 406 407 func netpollReadDeadline(arg interface{}, seq uintptr) { 408 netpolldeadlineimpl(arg.(*pollDesc), seq, true, false) 409 } 410 411 func netpollWriteDeadline(arg interface{}, seq uintptr) { 412 netpolldeadlineimpl(arg.(*pollDesc), seq, false, true) 413 } 414 415 func (c *pollCache) alloc() *pollDesc { 416 lock(&c.lock) 417 if c.first == nil { 418 const pdSize = unsafe.Sizeof(pollDesc{}) 419 n := pollBlockSize / pdSize 420 if n == 0 { 421 n = 1 422 } 423 // Must be in non-GC memory because can be referenced 424 // only from epoll/kqueue internals. 425 mem := persistentalloc(n*pdSize, 0, &memstats.other_sys) 426 for i := uintptr(0); i < n; i++ { 427 pd := (*pollDesc)(add(mem, i*pdSize)) 428 pd.link = c.first 429 c.first = pd 430 } 431 } 432 pd := c.first 433 c.first = pd.link 434 unlock(&c.lock) 435 return pd 436 }