github.com/polevpn/netstack@v1.10.9/sleep/sleep_unsafe.go (about) 1 // Copyright 2018 The gVisor Authors. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 // +build go1.11 16 // +build !go1.18 17 18 // Check go:linkname function signatures when updating Go version. 19 20 // Package sleep allows goroutines to efficiently sleep on multiple sources of 21 // notifications (wakers). It offers O(1) complexity, which is different from 22 // multi-channel selects which have O(n) complexity (where n is the number of 23 // channels) and a considerable constant factor. 24 // 25 // It is similar to edge-triggered epoll waits, where the user registers each 26 // object of interest once, and then can repeatedly wait on all of them. 27 // 28 // A Waker object is used to wake a sleeping goroutine (G) up, or prevent it 29 // from going to sleep next. A Sleeper object is used to receive notifications 30 // from wakers, and if no notifications are available, to optionally sleep until 31 // one becomes available. 32 // 33 // A Waker can be associated with at most one Sleeper, but a Sleeper can be 34 // associated with multiple Wakers. A Sleeper has a list of asserted (ready) 35 // wakers; when Fetch() is called repeatedly, elements from this list are 36 // returned until the list becomes empty in which case the goroutine goes to 37 // sleep. When Assert() is called on a Waker, it adds itself to the Sleeper's 38 // asserted list and wakes the G up from its sleep if needed. 39 // 40 // Sleeper objects are expected to be used as follows, with just one goroutine 41 // executing this code: 42 // 43 // // One time set-up. 44 // s := sleep.Sleeper{} 45 // s.AddWaker(&w1, constant1) 46 // s.AddWaker(&w2, constant2) 47 // 48 // // Called repeatedly. 49 // for { 50 // switch id, _ := s.Fetch(true); id { 51 // case constant1: 52 // // Do work triggered by w1 being asserted. 53 // case constant2: 54 // // Do work triggered by w2 being asserted. 55 // } 56 // } 57 // 58 // And Waker objects are expected to call w.Assert() when they want the sleeper 59 // to wake up and perform work. 60 // 61 // The notifications are edge-triggered, which means that if a Waker calls 62 // Assert() several times before the sleeper has the chance to wake up, it will 63 // only be notified once and should perform all pending work (alternatively, it 64 // can also call Assert() on the waker, to ensure that it will wake up again). 65 // 66 // The "unsafeness" here is in the casts to/from unsafe.Pointer, which is safe 67 // when only one type is used for each unsafe.Pointer (which is the case here), 68 // we should just make sure that this remains the case in the future. The usage 69 // of unsafe package could be confined to sharedWaker and sharedSleeper types 70 // that would hold pointers in atomic.Pointers, but the go compiler currently 71 // can't optimize these as well (it won't inline their method calls), which 72 // reduces performance. 73 package sleep 74 75 import ( 76 "sync/atomic" 77 "unsafe" 78 ) 79 80 const ( 81 // preparingG is stored in sleepers to indicate that they're preparing 82 // to sleep. 83 preparingG = 1 84 ) 85 86 var ( 87 // assertedSleeper is a sentinel sleeper. A pointer to it is stored in 88 // wakers that are asserted. 89 assertedSleeper Sleeper 90 ) 91 92 //go:linkname gopark runtime.gopark 93 func gopark(unlockf func(uintptr, *uintptr) bool, wg *uintptr, reason uint8, traceEv byte, traceskip int) 94 95 //go:linkname goready runtime.goready 96 func goready(g uintptr, traceskip int) 97 98 func commitSleep(g uintptr, waitingG *uintptr) bool { 99 for { 100 // Check if the wait was aborted. 101 if atomic.LoadUintptr(waitingG) == 0 { 102 return false 103 } 104 105 // Try to store the G so that wakers know who to wake. 106 if atomic.CompareAndSwapUintptr(waitingG, preparingG, g) { 107 return true 108 } 109 } 110 } 111 112 // Sleeper allows a goroutine to sleep and receive wake up notifications from 113 // Wakers in an efficient way. 114 // 115 // This is similar to edge-triggered epoll in that wakers are added to the 116 // sleeper once and the sleeper can then repeatedly sleep in O(1) time while 117 // waiting on all wakers. 118 // 119 // None of the methods in a Sleeper can be called concurrently. Wakers that have 120 // been added to a sleeper A can only be added to another sleeper after A.Done() 121 // returns. These restrictions allow this to be implemented lock-free. 122 // 123 // This struct is thread-compatible. 124 type Sleeper struct { 125 // sharedList is a "stack" of asserted wakers. They atomically add 126 // themselves to the front of this list as they become asserted. 127 sharedList unsafe.Pointer 128 129 // localList is a list of asserted wakers that is only accessible to the 130 // waiter, and thus doesn't have to be accessed atomically. When 131 // fetching more wakers, the waiter will first go through this list, and 132 // only when it's empty will it atomically fetch wakers from 133 // sharedList. 134 localList *Waker 135 136 // allWakers is a list with all wakers that have been added to this 137 // sleeper. It is used during cleanup to remove associations. 138 allWakers *Waker 139 140 // waitingG holds the G that is sleeping, if any. It is used by wakers 141 // to determine which G, if any, they should wake. 142 waitingG uintptr 143 } 144 145 // AddWaker associates the given waker to the sleeper. id is the value to be 146 // returned when the sleeper is woken by the given waker. 147 func (s *Sleeper) AddWaker(w *Waker, id int) { 148 // Add the waker to the list of all wakers. 149 w.allWakersNext = s.allWakers 150 s.allWakers = w 151 w.id = id 152 153 // Try to associate the waker with the sleeper. If it's already 154 // asserted, we simply enqueue it in the "ready" list. 155 for { 156 p := (*Sleeper)(atomic.LoadPointer(&w.s)) 157 if p == &assertedSleeper { 158 s.enqueueAssertedWaker(w) 159 return 160 } 161 162 if atomic.CompareAndSwapPointer(&w.s, usleeper(p), usleeper(s)) { 163 return 164 } 165 } 166 } 167 168 // nextWaker returns the next waker in the notification list, blocking if 169 // needed. 170 func (s *Sleeper) nextWaker(block bool) *Waker { 171 // Attempt to replenish the local list if it's currently empty. 172 if s.localList == nil { 173 for atomic.LoadPointer(&s.sharedList) == nil { 174 // Fail request if caller requested that we 175 // don't block. 176 if !block { 177 return nil 178 } 179 180 // Indicate to wakers that we're about to sleep, 181 // this allows them to abort the wait by setting 182 // waitingG back to zero (which we'll notice 183 // before committing the sleep). 184 atomic.StoreUintptr(&s.waitingG, preparingG) 185 186 // Check if something was queued while we were 187 // preparing to sleep. We need this interleaving 188 // to avoid missing wake ups. 189 if atomic.LoadPointer(&s.sharedList) != nil { 190 atomic.StoreUintptr(&s.waitingG, 0) 191 break 192 } 193 194 // Try to commit the sleep and report it to the 195 // tracer as a select. 196 // 197 // gopark puts the caller to sleep and calls 198 // commitSleep to decide whether to immediately 199 // wake the caller up or to leave it sleeping. 200 const traceEvGoBlockSelect = 24 201 // See:runtime2.go in the go runtime package for 202 // the values to pass as the waitReason here. 203 const waitReasonSelect = 9 204 gopark(commitSleep, &s.waitingG, waitReasonSelect, traceEvGoBlockSelect, 0) 205 } 206 207 // Pull the shared list out and reverse it in the local 208 // list. Given that wakers push themselves in reverse 209 // order, we fix things here. 210 v := (*Waker)(atomic.SwapPointer(&s.sharedList, nil)) 211 for v != nil { 212 cur := v 213 v = v.next 214 215 cur.next = s.localList 216 s.localList = cur 217 } 218 } 219 220 // Remove the waker in the front of the list. 221 w := s.localList 222 s.localList = w.next 223 224 return w 225 } 226 227 // Fetch fetches the next wake-up notification. If a notification is immediately 228 // available, it is returned right away. Otherwise, the behavior depends on the 229 // value of 'block': if true, the current goroutine blocks until a notification 230 // arrives, then returns it; if false, returns 'ok' as false. 231 // 232 // When 'ok' is true, the value of 'id' corresponds to the id associated with 233 // the waker; when 'ok' is false, 'id' is undefined. 234 // 235 // N.B. This method is *not* thread-safe. Only one goroutine at a time is 236 // allowed to call this method. 237 func (s *Sleeper) Fetch(block bool) (id int, ok bool) { 238 for { 239 w := s.nextWaker(block) 240 if w == nil { 241 return -1, false 242 } 243 244 // Reassociate the waker with the sleeper. If the waker was 245 // still asserted we can return it, otherwise try the next one. 246 old := (*Sleeper)(atomic.SwapPointer(&w.s, usleeper(s))) 247 if old == &assertedSleeper { 248 return w.id, true 249 } 250 } 251 } 252 253 // Done is used to indicate that the caller won't use this Sleeper anymore. It 254 // removes the association with all wakers so that they can be safely reused 255 // by another sleeper after Done() returns. 256 func (s *Sleeper) Done() { 257 // Remove all associations that we can, and build a list of the ones 258 // we could not. An association can be removed right away from waker w 259 // if w.s has a pointer to the sleeper, that is, the waker is not 260 // asserted yet. By atomically switching w.s to nil, we guarantee that 261 // subsequent calls to Assert() on the waker will not result in it being 262 // queued to this sleeper. 263 var pending *Waker 264 w := s.allWakers 265 for w != nil { 266 next := w.allWakersNext 267 for { 268 t := atomic.LoadPointer(&w.s) 269 if t != usleeper(s) { 270 w.allWakersNext = pending 271 pending = w 272 break 273 } 274 275 if atomic.CompareAndSwapPointer(&w.s, t, nil) { 276 break 277 } 278 } 279 w = next 280 } 281 282 // The associations that we could not remove are either asserted, or in 283 // the process of being asserted, or have been asserted and cleared 284 // before being pulled from the sleeper lists. We must wait for them all 285 // to make it to the sleeper lists, so that we know that the wakers 286 // won't do any more work towards waking this sleeper up. 287 for pending != nil { 288 pulled := s.nextWaker(true) 289 290 // Remove the waker we just pulled from the list of associated 291 // wakers. 292 prev := &pending 293 for w := *prev; w != nil; w = *prev { 294 if pulled == w { 295 *prev = w.allWakersNext 296 break 297 } 298 prev = &w.allWakersNext 299 } 300 } 301 s.allWakers = nil 302 } 303 304 // enqueueAssertedWaker enqueues an asserted waker to the "ready" circular list 305 // of wakers that want to notify the sleeper. 306 func (s *Sleeper) enqueueAssertedWaker(w *Waker) { 307 // Add the new waker to the front of the list. 308 for { 309 v := (*Waker)(atomic.LoadPointer(&s.sharedList)) 310 w.next = v 311 if atomic.CompareAndSwapPointer(&s.sharedList, uwaker(v), uwaker(w)) { 312 break 313 } 314 } 315 316 for { 317 // Nothing to do if there isn't a G waiting. 318 g := atomic.LoadUintptr(&s.waitingG) 319 if g == 0 { 320 return 321 } 322 323 // Signal to the sleeper that a waker has been asserted. 324 if atomic.CompareAndSwapUintptr(&s.waitingG, g, 0) { 325 if g != preparingG { 326 // We managed to get a G. Wake it up. 327 goready(g, 0) 328 } 329 } 330 } 331 } 332 333 // Waker represents a source of wake-up notifications to be sent to sleepers. A 334 // waker can be associated with at most one sleeper at a time, and at any given 335 // time is either in asserted or non-asserted state. 336 // 337 // Once asserted, the waker remains so until it is manually cleared or a sleeper 338 // consumes its assertion (i.e., a sleeper wakes up or is prevented from going 339 // to sleep due to the waker). 340 // 341 // This struct is thread-safe, that is, its methods can be called concurrently 342 // by multiple goroutines. 343 type Waker struct { 344 // s is the sleeper that this waker can wake up. Only one sleeper at a 345 // time is allowed. This field can have three classes of values: 346 // nil -- the waker is not asserted: it either is not associated with 347 // a sleeper, or is queued to a sleeper due to being previously 348 // asserted. This is the zero value. 349 // &assertedSleeper -- the waker is asserted. 350 // otherwise -- the waker is not asserted, and is associated with the 351 // given sleeper. Once it transitions to asserted state, the 352 // associated sleeper will be woken. 353 s unsafe.Pointer 354 355 // next is used to form a linked list of asserted wakers in a sleeper. 356 next *Waker 357 358 // allWakersNext is used to form a linked list of all wakers associated 359 // to a given sleeper. 360 allWakersNext *Waker 361 362 // id is the value to be returned to sleepers when they wake up due to 363 // this waker being asserted. 364 id int 365 } 366 367 // Assert moves the waker to an asserted state, if it isn't asserted yet. When 368 // asserted, the waker will cause its matching sleeper to wake up. 369 func (w *Waker) Assert() { 370 // Nothing to do if the waker is already asserted. This check allows us 371 // to complete this case (already asserted) without any interlocked 372 // operations on x86. 373 if atomic.LoadPointer(&w.s) == usleeper(&assertedSleeper) { 374 return 375 } 376 377 // Mark the waker as asserted, and wake up a sleeper if there is one. 378 switch s := (*Sleeper)(atomic.SwapPointer(&w.s, usleeper(&assertedSleeper))); s { 379 case nil: 380 case &assertedSleeper: 381 default: 382 s.enqueueAssertedWaker(w) 383 } 384 } 385 386 // Clear moves the waker to then non-asserted state and returns whether it was 387 // asserted before being cleared. 388 // 389 // N.B. The waker isn't removed from the "ready" list of a sleeper (if it 390 // happens to be in one), but the sleeper will notice that it is not asserted 391 // anymore and won't return it to the caller. 392 func (w *Waker) Clear() bool { 393 // Nothing to do if the waker is not asserted. This check allows us to 394 // complete this case (already not asserted) without any interlocked 395 // operations on x86. 396 if atomic.LoadPointer(&w.s) != usleeper(&assertedSleeper) { 397 return false 398 } 399 400 // Try to store nil in the sleeper, which indicates that the waker is 401 // not asserted. 402 return atomic.CompareAndSwapPointer(&w.s, usleeper(&assertedSleeper), nil) 403 } 404 405 // IsAsserted returns whether the waker is currently asserted (i.e., if it's 406 // currently in a state that would cause its matching sleeper to wake up). 407 func (w *Waker) IsAsserted() bool { 408 return (*Sleeper)(atomic.LoadPointer(&w.s)) == &assertedSleeper 409 } 410 411 func usleeper(s *Sleeper) unsafe.Pointer { 412 return unsafe.Pointer(s) 413 } 414 415 func uwaker(w *Waker) unsafe.Pointer { 416 return unsafe.Pointer(w) 417 }