github.com/iceber/iouring-go@v0.0.0-20230403020409-002cfd2e2a90/iouring.go (about) 1 //go:build linux 2 // +build linux 3 4 package iouring 5 6 import ( 7 "errors" 8 "log" 9 "os" 10 "runtime" 11 "sync" 12 "syscall" 13 14 iouring_syscall "github.com/iceber/iouring-go/syscall" 15 ) 16 17 // IOURing contains iouring_syscall submission and completion queue. 18 // It's safe for concurrent use by multiple goroutines. 19 type IOURing struct { 20 params *iouring_syscall.IOURingParams 21 fd int 22 23 eventfd int 24 cqeSign chan struct{} 25 26 sq *SubmissionQueue 27 cq *CompletionQueue 28 29 async bool 30 drain bool 31 Flags uint32 32 Features uint32 33 34 submitLock sync.Mutex 35 36 userDataLock sync.RWMutex 37 userDatas map[uint64]*UserData 38 39 fileRegister FileRegister 40 41 fdclosed bool 42 closer chan struct{} 43 closed chan struct{} 44 } 45 46 // New return a IOURing instance by IOURingOptions 47 func New(entries uint, opts ...IOURingOption) (*IOURing, error) { 48 iour := &IOURing{ 49 params: &iouring_syscall.IOURingParams{}, 50 userDatas: make(map[uint64]*UserData), 51 cqeSign: make(chan struct{}, 1), 52 closer: make(chan struct{}), 53 closed: make(chan struct{}), 54 } 55 56 for _, opt := range opts { 57 opt(iour) 58 } 59 60 var err error 61 iour.fd, err = iouring_syscall.IOURingSetup(entries, iour.params) 62 if err != nil { 63 return nil, err 64 } 65 66 if err := mmapIOURing(iour); err != nil { 67 munmapIOURing(iour) 68 return nil, err 69 } 70 71 iour.fileRegister = &fileRegister{ 72 iouringFd: iour.fd, 73 sparseIndexs: make(map[int]int), 74 } 75 iour.Flags = iour.params.Flags 76 iour.Features = iour.params.Features 77 78 if err := iour.registerEventfd(); err != nil { 79 iour.Close() 80 return nil, err 81 } 82 83 if err := registerIOURing(iour); err != nil { 84 iour.Close() 85 return nil, err 86 } 87 88 go iour.run() 89 return iour, nil 90 } 91 92 // Size iouring submission queue size 93 func (iour *IOURing) Size() int { 94 return int(*iour.sq.entries) 95 } 96 97 // Close IOURing 98 func (iour *IOURing) Close() error { 99 iour.submitLock.Lock() 100 defer iour.submitLock.Unlock() 101 102 select { 103 case <-iour.closer: 104 default: 105 close(iour.closer) 106 } 107 108 if iour.eventfd > 0 { 109 if err := removeIOURing(iour); err != nil { 110 return err 111 } 112 syscall.Close(iour.eventfd) 113 iour.eventfd = -1 114 } 115 116 <-iour.closed 117 118 if err := munmapIOURing(iour); err != nil { 119 return err 120 } 121 122 if !iour.fdclosed { 123 if err := syscall.Close(iour.fd); err != nil { 124 return os.NewSyscallError("close", err) 125 } 126 iour.fdclosed = true 127 } 128 129 return nil 130 } 131 132 // IsClosed IOURing is closed 133 func (iour *IOURing) IsClosed() (closed bool) { 134 select { 135 case <-iour.closer: 136 closed = true 137 default: 138 } 139 return 140 } 141 142 func (iour *IOURing) getSQEntry() iouring_syscall.SubmissionQueueEntry { 143 for { 144 sqe := iour.sq.getSQEntry() 145 if sqe != nil { 146 return sqe 147 } 148 runtime.Gosched() 149 } 150 } 151 152 func (iour *IOURing) doRequest(sqe iouring_syscall.SubmissionQueueEntry, request PrepRequest, ch chan<- Result) (*UserData, error) { 153 userData := makeUserData(iour, ch) 154 155 request(sqe, userData) 156 userData.setOpcode(sqe.Opcode()) 157 158 sqe.SetUserData(userData.id) 159 160 userData.request.fd = int(sqe.Fd()) 161 if sqe.Fd() >= 0 { 162 if index, ok := iour.fileRegister.GetFileIndex(int32(sqe.Fd())); ok { 163 sqe.SetFdIndex(int32(index)) 164 } else if iour.Flags&iouring_syscall.IORING_SETUP_SQPOLL != 0 && 165 iour.Features&iouring_syscall.IORING_FEAT_SQPOLL_NONFIXED == 0 { 166 /* 167 Before version 5.10 of the Linux kernel, to successful use SQPoll, the application 168 must register a set of files to be used for IO through iour.RegisterFiles. 169 Failure to do so will result in submitted IO being errored with EBADF 170 171 The presence of this feature can be detected by the IORING_FEAT_SQPOLL_NONFIXED 172 In Version 5.10 and later, it is no longer necessary to register files to use SQPoll 173 */ 174 175 return nil, ErrUnregisteredFile 176 } 177 } 178 179 if iour.async { 180 sqe.SetFlags(iouring_syscall.IOSQE_FLAGS_ASYNC) 181 } 182 if iour.drain { 183 sqe.SetFlags(iouring_syscall.IOSQE_FLAGS_IO_DRAIN) 184 } 185 return userData, nil 186 } 187 188 // SubmitRequest by Request function and io result is notified via channel 189 // return request id, can be used to cancel a request 190 func (iour *IOURing) SubmitRequest(request PrepRequest, ch chan<- Result) (Request, error) { 191 iour.submitLock.Lock() 192 defer iour.submitLock.Unlock() 193 194 if iour.IsClosed() { 195 return nil, ErrIOURingClosed 196 } 197 198 sqe := iour.getSQEntry() 199 userData, err := iour.doRequest(sqe, request, ch) 200 if err != nil { 201 iour.sq.fallback(1) 202 return nil, err 203 } 204 205 iour.userDataLock.Lock() 206 iour.userDatas[userData.id] = userData 207 iour.userDataLock.Unlock() 208 209 if _, err = iour.submit(); err != nil { 210 iour.userDataLock.Lock() 211 delete(iour.userDatas, userData.id) 212 iour.userDataLock.Unlock() 213 return nil, err 214 } 215 216 return userData.request, nil 217 } 218 219 // SubmitRequests by Request functions and io results are notified via channel 220 func (iour *IOURing) SubmitRequests(requests []PrepRequest, ch chan<- Result) (RequestSet, error) { 221 // TODO(iceber): no length limit 222 if len(requests) > int(*iour.sq.entries) { 223 return nil, errors.New("too many requests") 224 } 225 226 iour.submitLock.Lock() 227 defer iour.submitLock.Unlock() 228 229 if iour.IsClosed() { 230 return nil, ErrIOURingClosed 231 } 232 233 var sqeN uint32 234 userDatas := make([]*UserData, 0, len(requests)) 235 for _, request := range requests { 236 sqe := iour.getSQEntry() 237 sqeN++ 238 239 userData, err := iour.doRequest(sqe, request, ch) 240 if err != nil { 241 iour.sq.fallback(sqeN) 242 return nil, err 243 } 244 userDatas = append(userDatas, userData) 245 } 246 247 // must be located before the lock operation to 248 // avoid the compiler's adjustment of the code order. 249 // issue: https://github.com/Iceber/iouring-go/issues/8 250 rset := newRequestSet(userDatas) 251 252 iour.userDataLock.Lock() 253 for _, data := range userDatas { 254 iour.userDatas[data.id] = data 255 } 256 iour.userDataLock.Unlock() 257 258 if _, err := iour.submit(); err != nil { 259 iour.userDataLock.Lock() 260 for _, data := range userDatas { 261 delete(iour.userDatas, data.id) 262 } 263 iour.userDataLock.Unlock() 264 265 return nil, err 266 } 267 268 return rset, nil 269 } 270 271 func (iour *IOURing) needEnter(flags *uint32) bool { 272 if (iour.Flags & iouring_syscall.IORING_SETUP_SQPOLL) == 0 { 273 return true 274 } 275 276 if iour.sq.needWakeup() { 277 *flags |= iouring_syscall.IORING_SQ_NEED_WAKEUP 278 return true 279 } 280 return false 281 } 282 283 func (iour *IOURing) submit() (submitted int, err error) { 284 submitted = iour.sq.flush() 285 286 var flags uint32 287 if !iour.needEnter(&flags) || submitted == 0 { 288 return 289 } 290 291 if (iour.Flags & iouring_syscall.IORING_SETUP_IOPOLL) != 0 { 292 flags |= iouring_syscall.IORING_ENTER_FLAGS_GETEVENTS 293 } 294 295 submitted, err = iouring_syscall.IOURingEnter(iour.fd, uint32(submitted), 0, flags, nil) 296 return 297 } 298 299 /* 300 func (iour *IOURing) submitAndWait(waitCount uint32) (submitted int, err error) { 301 submitted = iour.sq.flush() 302 303 var flags uint32 304 if !iour.needEnter(&flags) && waitCount == 0 { 305 return 306 } 307 308 if waitCount != 0 || (iour.Flags&iouring_syscall.IORING_SETUP_IOPOLL) != 0 { 309 flags |= iouring_syscall.IORING_ENTER_FLAGS_GETEVENTS 310 } 311 312 submitted, err = iouring_syscall.IOURingEnter(iour.fd, uint32(submitted), waitCount, flags, nil) 313 return 314 } 315 */ 316 317 func (iour *IOURing) getCQEvent(wait bool) (cqe iouring_syscall.CompletionQueueEvent, err error) { 318 var tryPeeks int 319 for { 320 if cqe = iour.cq.peek(); cqe != nil { 321 // Copy CQE. 322 cqe = cqe.Clone() 323 iour.cq.advance(1) 324 return 325 } 326 327 if !wait && !iour.sq.cqOverflow() { 328 err = syscall.EAGAIN 329 return 330 } 331 332 if iour.sq.cqOverflow() { 333 _, err = iouring_syscall.IOURingEnter(iour.fd, 0, 0, iouring_syscall.IORING_ENTER_FLAGS_GETEVENTS, nil) 334 if err != nil { 335 return 336 } 337 continue 338 } 339 340 if tryPeeks++; tryPeeks < 3 { 341 runtime.Gosched() 342 continue 343 } 344 345 select { 346 case <-iour.cqeSign: 347 case <-iour.closer: 348 return nil, ErrIOURingClosed 349 } 350 } 351 } 352 353 func (iour *IOURing) run() { 354 for { 355 cqe, err := iour.getCQEvent(true) 356 if cqe == nil || err != nil { 357 if err == ErrIOURingClosed { 358 close(iour.closed) 359 return 360 } 361 log.Println("runComplete error: ", err) 362 continue 363 } 364 365 // log.Println("cqe user data", (cqe.UserData)) 366 367 iour.userDataLock.Lock() 368 userData := iour.userDatas[cqe.UserData()] 369 if userData == nil { 370 iour.userDataLock.Unlock() 371 log.Println("runComplete: notfound user data ", uintptr(cqe.UserData())) 372 continue 373 } 374 delete(iour.userDatas, cqe.UserData()) 375 iour.userDataLock.Unlock() 376 377 userData.request.complate(cqe) 378 379 // ignore link timeout 380 if userData.opcode == iouring_syscall.IORING_OP_LINK_TIMEOUT { 381 continue 382 } 383 384 if userData.resulter != nil { 385 userData.resulter <- userData.request 386 } 387 } 388 } 389 390 // Result submit cancel request 391 func (iour *IOURing) submitCancel(id uint64) (Request, error) { 392 if iour == nil { 393 return nil, ErrRequestCompleted 394 } 395 396 return iour.SubmitRequest(cancelRequest(id), nil) 397 } 398 399 func cancelRequest(id uint64) PrepRequest { 400 return func(sqe iouring_syscall.SubmissionQueueEntry, userData *UserData) { 401 userData.request.resolver = cancelResolver 402 sqe.PrepOperation(iouring_syscall.IORING_OP_ASYNC_CANCEL, -1, id, 0, 0) 403 } 404 }