github.com/pawelgaczynski/giouring@v0.0.0-20230826085535-69588b89acb9/queue.go (about) 1 // MIT License 2 // 3 // Copyright (c) 2023 Paweł Gaczyński 4 // 5 // Permission is hereby granted, free of charge, to any person obtaining a 6 // copy of this software and associated documentation files (the 7 // "Software"), to deal in the Software without restriction, including 8 // without limitation the rights to use, copy, modify, merge, publish, 9 // distribute, sublicense, and/or sell copies of the Software, and to 10 // permit persons to whom the Software is furnished to do so, subject to 11 // the following conditions: 12 // 13 // The above copyright notice and this permission notice shall be included 14 // in all copies or substantial portions of the Software. 15 // 16 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 17 // OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 19 // IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 20 // CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 21 // TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 22 // SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 23 24 package giouring 25 26 import ( 27 "runtime" 28 "sync/atomic" 29 "syscall" 30 "unsafe" 31 32 "golang.org/x/sys/unix" 33 ) 34 35 // liburing: sq_ring_needs_enter 36 func (ring *Ring) sqRingNeedsEnter(submit uint32, flags *uint32) bool { 37 if submit == 0 { 38 return false 39 } 40 41 if (ring.flags & SetupSQPoll) == 0 { 42 return true 43 } 44 45 if atomic.LoadUint32(ring.sqRing.flags)&SQNeedWakeup != 0 { 46 *flags |= EnterSQWakeup 47 48 return true 49 } 50 51 return false 52 } 53 54 // liburing: cq_ring_needs_flush 55 func (ring *Ring) cqRingNeedsFlush() bool { 56 return atomic.LoadUint32(ring.sqRing.flags)&(SQCQOverflow|SQTaskrun) != 0 57 } 58 59 // liburing: cq_ring_needs_enter 60 func (ring *Ring) cqRingNeedsEnter() bool { 61 return (ring.flags&SetupIOPoll) != 0 || ring.cqRingNeedsFlush() 62 } 63 64 // liburing: get_data 65 type getData struct { 66 submit uint32 67 waitNr uint32 68 getFlags uint32 69 sz int 70 hasTS bool 71 arg unsafe.Pointer 72 } 73 74 // liburing: _io_uring_get_cqe 75 func (ring *Ring) privateGetCQE(data *getData) (*CompletionQueueEvent, error) { 76 var cqe *CompletionQueueEvent 77 var looped bool 78 var err error 79 80 for { 81 var needEnter bool 82 var flags uint32 83 var nrAvailable uint32 84 var ret uint 85 var localErr error 86 87 cqe, localErr = internalPeekCQE(ring, &nrAvailable) 88 if localErr != nil { 89 if err == nil { 90 err = localErr 91 } 92 93 break 94 } 95 if cqe == nil && data.waitNr == 0 && data.submit == 0 { 96 if looped || !ring.cqRingNeedsEnter() { 97 if err == nil { 98 err = unix.EAGAIN 99 } 100 101 break 102 } 103 needEnter = true 104 } 105 if data.waitNr > nrAvailable || needEnter { 106 flags = EnterGetEvents | data.getFlags 107 needEnter = true 108 } 109 if ring.sqRingNeedsEnter(data.submit, &flags) { 110 needEnter = true 111 } 112 if !needEnter { 113 break 114 } 115 if looped && data.hasTS { 116 arg := (*GetEventsArg)(data.arg) 117 if cqe == nil && arg.ts != 0 && err == nil { 118 err = unix.ETIME 119 } 120 121 break 122 } 123 if ring.intFlags&IntFlagRegRing != 0 { 124 flags |= EnterRegisteredRing 125 } 126 ret, localErr = ring.Enter2(data.submit, data.waitNr, flags, data.arg, data.sz) 127 if localErr != nil { 128 if err == nil { 129 err = localErr 130 } 131 132 break 133 } 134 data.submit -= uint32(ret) 135 if cqe != nil { 136 break 137 } 138 if !looped { 139 looped = true 140 err = localErr 141 } 142 } 143 144 return cqe, err 145 } 146 147 // liburing: __io_uring_get_cqe 148 func (ring *Ring) internalGetCQE(submit uint32, waitNr uint32, sigmask *unix.Sigset_t) (*CompletionQueueEvent, error) { 149 data := getData{ 150 submit: submit, 151 waitNr: waitNr, 152 getFlags: 0, 153 sz: nSig / szDivider, 154 arg: unsafe.Pointer(sigmask), 155 } 156 157 cqe, err := ring.privateGetCQE(&data) 158 runtime.KeepAlive(data) 159 160 return cqe, err 161 } 162 163 // liburing: io_uring_get_events - https://manpages.debian.org/unstable/liburing-dev/io_uring_get_events.3.en.html 164 func (ring *Ring) GetEvents() (uint, error) { 165 flags := EnterGetEvents 166 167 if ring.intFlags&IntFlagRegRing != 0 { 168 flags |= EnterRegisteredRing 169 } 170 171 return ring.Enter(0, 0, flags, nil) 172 } 173 174 // liburing: io_uring_peek_batch_cqe 175 func (ring *Ring) PeekBatchCQE(cqes []*CompletionQueueEvent) uint32 { 176 var ready uint32 177 var overflowChecked bool 178 var shift int 179 180 if ring.flags&SetupCQE32 != 0 { 181 shift = 1 182 } 183 184 count := uint32(len(cqes)) 185 186 again: 187 ready = ring.CQReady() 188 if ready != 0 { 189 head := *ring.cqRing.head 190 mask := *ring.cqRing.ringMask 191 last := head + count 192 if count > ready { 193 count = ready 194 } 195 for i := 0; head != last; head, i = head+1, i+1 { 196 cqes[i] = (*CompletionQueueEvent)( 197 unsafe.Add( 198 unsafe.Pointer(ring.cqRing.cqes), 199 uintptr((head&mask)<<shift)*unsafe.Sizeof(CompletionQueueEvent{}), 200 ), 201 ) 202 } 203 204 return count 205 } 206 207 if overflowChecked { 208 return 0 209 } 210 211 if ring.cqRingNeedsFlush() { 212 _, _ = ring.GetEvents() 213 overflowChecked = true 214 215 goto again 216 } 217 218 return 0 219 } 220 221 // liburing: __io_uring_flush_sq 222 func (ring *Ring) internalFlushSQ() uint32 { 223 sq := ring.sqRing 224 tail := sq.sqeTail 225 226 if sq.sqeHead != tail { 227 sq.sqeHead = tail 228 atomic.StoreUint32(sq.tail, tail) 229 } 230 231 return tail - atomic.LoadUint32(sq.head) 232 } 233 234 // liburing: io_uring_wait_cqes_new 235 func (ring *Ring) WaitCQEsNew( 236 waitNr uint32, ts *syscall.Timespec, sigmask *unix.Sigset_t, 237 ) (*CompletionQueueEvent, error) { 238 var arg *GetEventsArg 239 var data *getData 240 241 arg = &GetEventsArg{ 242 sigMask: uint64(uintptr(unsafe.Pointer(sigmask))), 243 sigMaskSz: nSig / szDivider, 244 ts: uint64(uintptr(unsafe.Pointer(ts))), 245 } 246 247 data = &getData{ 248 waitNr: waitNr, 249 getFlags: EnterExtArg, 250 sz: int(unsafe.Sizeof(GetEventsArg{})), 251 hasTS: true, 252 arg: unsafe.Pointer(arg), 253 } 254 255 cqe, err := ring.privateGetCQE(data) 256 runtime.KeepAlive(data) 257 258 return cqe, err 259 } 260 261 // liburing: __io_uring_submit_timeout 262 func (ring *Ring) internalSubmitTimeout(waitNr uint32, ts *syscall.Timespec) (uint32, error) { 263 var sqe *SubmissionQueueEntry 264 var err error 265 266 /* 267 * If the SQ ring is full, we may need to submit IO first 268 */ 269 sqe = ring.GetSQE() 270 if sqe == nil { 271 _, err = ring.Submit() 272 if err != nil { 273 return 0, err 274 } 275 sqe = ring.GetSQE() 276 if sqe == nil { 277 return 0, syscall.EAGAIN 278 } 279 } 280 sqe.PrepareTimeout(ts, waitNr, 0) 281 sqe.UserData = liburingUdataTimeout 282 283 return ring.internalFlushSQ(), nil 284 } 285 286 // liburing: io_uring_wait_cqes - https://manpages.debian.org/unstable/liburing-dev/io_uring_wait_cqes.3.en.html 287 func (ring *Ring) WaitCQEs(waitNr uint32, ts *syscall.Timespec, sigmask *unix.Sigset_t) (*CompletionQueueEvent, error) { 288 var toSubmit uint32 289 var err error 290 291 if ts != nil { 292 if ring.features&FeatExtArg != 0 { 293 return ring.WaitCQEsNew(waitNr, ts, sigmask) 294 } 295 toSubmit, err = ring.internalSubmitTimeout(waitNr, ts) 296 if err != nil { 297 return nil, err 298 } 299 } 300 301 return ring.internalGetCQE(toSubmit, waitNr, sigmask) 302 } 303 304 // liburing: io_uring_submit_and_wait_timeout - https://manpages.debian.org/unstable/liburing-dev/io_uring_submit_and_wait_timeout.3.en.html 305 func (ring *Ring) SubmitAndWaitTimeout( 306 waitNr uint32, ts *syscall.Timespec, sigmask *unix.Sigset_t, 307 ) (*CompletionQueueEvent, error) { 308 var toSubmit uint32 309 var err error 310 var cqe *CompletionQueueEvent 311 312 if ts != nil { 313 if ring.features&FeatExtArg != 0 { 314 arg := GetEventsArg{ 315 sigMask: uint64(uintptr(unsafe.Pointer(sigmask))), 316 sigMaskSz: nSig / szDivider, 317 ts: uint64(uintptr(unsafe.Pointer(ts))), 318 } 319 data := getData{ 320 submit: ring.internalFlushSQ(), 321 waitNr: waitNr, 322 getFlags: EnterExtArg, 323 sz: int(unsafe.Sizeof(arg)), 324 hasTS: ts != nil, 325 arg: unsafe.Pointer(&arg), 326 } 327 328 cqe, err = ring.privateGetCQE(&data) 329 runtime.KeepAlive(data) 330 331 return cqe, err 332 } 333 toSubmit, err = ring.internalSubmitTimeout(waitNr, ts) 334 if err != nil { 335 return cqe, err 336 } 337 } else { 338 toSubmit = ring.internalFlushSQ() 339 } 340 341 return ring.internalGetCQE(toSubmit, waitNr, sigmask) 342 } 343 344 // liburing: io_uring_wait_cqe_timeout - https://manpages.debian.org/unstable/liburing-dev/io_uring_wait_cqe_timeout.3.en.html 345 func (ring *Ring) WaitCQETimeout(ts *syscall.Timespec) (*CompletionQueueEvent, error) { 346 return ring.WaitCQEs(1, ts, nil) 347 } 348 349 // liburing: __io_uring_submit 350 func (ring *Ring) internalSubmit(submitted uint32, waitNr uint32, getEvents bool) (uint, error) { 351 cqNeedsEnter := getEvents || waitNr != 0 || ring.cqRingNeedsEnter() 352 353 var flags uint32 354 var ret uint 355 var err error 356 357 flags = 0 358 if ring.sqRingNeedsEnter(submitted, &flags) || cqNeedsEnter { 359 if cqNeedsEnter { 360 flags |= EnterGetEvents 361 } 362 if ring.intFlags&IntFlagRegRing != 0 { 363 flags |= EnterRegisteredRing 364 } 365 366 ret, err = ring.Enter(submitted, waitNr, flags, nil) 367 if err != nil { 368 return 0, err 369 } 370 } else { 371 ret = uint(submitted) 372 } 373 374 return ret, nil 375 } 376 377 // liburing: __io_uring_submit_and_wait 378 func (ring *Ring) internalSubmitAndWait(waitNr uint32) (uint, error) { 379 return ring.internalSubmit(ring.internalFlushSQ(), waitNr, false) 380 } 381 382 // liburing: io_uring_submit - https://manpages.debian.org/unstable/liburing-dev/io_uring_submit.3.en.html 383 func (ring *Ring) Submit() (uint, error) { 384 return ring.internalSubmitAndWait(0) 385 } 386 387 // liburing: io_uring_submit_and_wait - https://manpages.debian.org/unstable/liburing-dev/io_uring_submit_and_wait.3.en.html 388 func (ring *Ring) SubmitAndWait(waitNr uint32) (uint, error) { 389 return ring.internalSubmitAndWait(waitNr) 390 } 391 392 // liburing: io_uring_submit_and_get_events - https://manpages.debian.org/unstable/liburing-dev/io_uring_submit_and_get_events.3.en.html 393 func (ring *Ring) SubmitAndGetEvents() (uint, error) { 394 return ring.internalSubmit(ring.internalFlushSQ(), 0, true) 395 } 396 397 // __io_uring_sqring_wait 398 func (ring *Ring) internalSQRingWait() (uint, error) { 399 flags := EnterSQWait 400 401 if ring.intFlags&IntFlagRegRegRing != 0 { 402 flags |= EnterRegisteredRing 403 } 404 405 return ring.Enter(0, 0, flags, nil) 406 }