github.com/sandwichdev/go-internals@v0.0.0-20210605002614-12311ac6b2c5/poll/splice_linux.go (about) 1 // Copyright 2018 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package poll 6 7 import ( 8 "sync/atomic" 9 "syscall" 10 "unsafe" 11 12 "github.com/SandwichDev/go-internals/syscall/unix" 13 ) 14 15 const ( 16 // spliceNonblock makes calls to splice(2) non-blocking. 17 spliceNonblock = 0x2 18 19 // maxSpliceSize is the maximum amount of data Splice asks 20 // the kernel to move in a single call to splice(2). 21 maxSpliceSize = 4 << 20 22 ) 23 24 // Splice transfers at most remain bytes of data from src to dst, using the 25 // splice system call to minimize copies of data from and to userspace. 26 // 27 // Splice creates a temporary pipe, to serve as a buffer for the data transfer. 28 // src and dst must both be stream-oriented sockets. 29 // 30 // If err != nil, sc is the system call which caused the error. 31 func Splice(dst, src *FD, remain int64) (written int64, handled bool, sc string, err error) { 32 prfd, pwfd, sc, err := newTempPipe() 33 if err != nil { 34 return 0, false, sc, err 35 } 36 defer destroyTempPipe(prfd, pwfd) 37 var inPipe, n int 38 for err == nil && remain > 0 { 39 max := maxSpliceSize 40 if int64(max) > remain { 41 max = int(remain) 42 } 43 inPipe, err = spliceDrain(pwfd, src, max) 44 // The operation is considered handled if splice returns no 45 // error, or an error other than EINVAL. An EINVAL means the 46 // kernel does not support splice for the socket type of src. 47 // The failed syscall does not consume any data so it is safe 48 // to fall back to a generic copy. 49 // 50 // spliceDrain should never return EAGAIN, so if err != nil, 51 // Splice cannot continue. 52 // 53 // If inPipe == 0 && err == nil, src is at EOF, and the 54 // transfer is complete. 55 handled = handled || (err != syscall.EINVAL) 56 if err != nil || (inPipe == 0 && err == nil) { 57 break 58 } 59 n, err = splicePump(dst, prfd, inPipe) 60 if n > 0 { 61 written += int64(n) 62 remain -= int64(n) 63 } 64 } 65 if err != nil { 66 return written, handled, "splice", err 67 } 68 return written, true, "", nil 69 } 70 71 // spliceDrain moves data from a socket to a pipe. 72 // 73 // Invariant: when entering spliceDrain, the pipe is empty. It is either in its 74 // initial state, or splicePump has emptied it previously. 75 // 76 // Given this, spliceDrain can reasonably assume that the pipe is ready for 77 // writing, so if splice returns EAGAIN, it must be because the socket is not 78 // ready for reading. 79 // 80 // If spliceDrain returns (0, nil), src is at EOF. 81 func spliceDrain(pipefd int, sock *FD, max int) (int, error) { 82 if err := sock.readLock(); err != nil { 83 return 0, err 84 } 85 defer sock.readUnlock() 86 if err := sock.pd.prepareRead(sock.isFile); err != nil { 87 return 0, err 88 } 89 for { 90 n, err := splice(pipefd, sock.Sysfd, max, spliceNonblock) 91 if err == syscall.EINTR { 92 continue 93 } 94 if err != syscall.EAGAIN { 95 return n, err 96 } 97 if err := sock.pd.waitRead(sock.isFile); err != nil { 98 return n, err 99 } 100 } 101 } 102 103 // splicePump moves all the buffered data from a pipe to a socket. 104 // 105 // Invariant: when entering splicePump, there are exactly inPipe 106 // bytes of data in the pipe, from a previous call to spliceDrain. 107 // 108 // By analogy to the condition from spliceDrain, splicePump 109 // only needs to poll the socket for readiness, if splice returns 110 // EAGAIN. 111 // 112 // If splicePump cannot move all the data in a single call to 113 // splice(2), it loops over the buffered data until it has written 114 // all of it to the socket. This behavior is similar to the Write 115 // step of an io.Copy in userspace. 116 func splicePump(sock *FD, pipefd int, inPipe int) (int, error) { 117 if err := sock.writeLock(); err != nil { 118 return 0, err 119 } 120 defer sock.writeUnlock() 121 if err := sock.pd.prepareWrite(sock.isFile); err != nil { 122 return 0, err 123 } 124 written := 0 125 for inPipe > 0 { 126 n, err := splice(sock.Sysfd, pipefd, inPipe, spliceNonblock) 127 // Here, the condition n == 0 && err == nil should never be 128 // observed, since Splice controls the write side of the pipe. 129 if n > 0 { 130 inPipe -= n 131 written += n 132 continue 133 } 134 if err != syscall.EAGAIN { 135 return written, err 136 } 137 if err := sock.pd.waitWrite(sock.isFile); err != nil { 138 return written, err 139 } 140 } 141 return written, nil 142 } 143 144 // splice wraps the splice system call. Since the current implementation 145 // only uses splice on sockets and pipes, the offset arguments are unused. 146 // splice returns int instead of int64, because callers never ask it to 147 // move more data in a single call than can fit in an int32. 148 func splice(out int, in int, max int, flags int) (int, error) { 149 n, err := syscall.Splice(in, nil, out, nil, max, flags) 150 return int(n), err 151 } 152 153 var disableSplice unsafe.Pointer 154 155 // newTempPipe sets up a temporary pipe for a splice operation. 156 func newTempPipe() (prfd, pwfd int, sc string, err error) { 157 p := (*bool)(atomic.LoadPointer(&disableSplice)) 158 if p != nil && *p { 159 return -1, -1, "splice", syscall.EINVAL 160 } 161 162 var fds [2]int 163 // pipe2 was added in 2.6.27 and our minimum requirement is 2.6.23, so it 164 // might not be implemented. Falling back to pipe is possible, but prior to 165 // 2.6.29 splice returns -EAGAIN instead of 0 when the connection is 166 // closed. 167 const flags = syscall.O_CLOEXEC | syscall.O_NONBLOCK 168 if err := syscall.Pipe2(fds[:], flags); err != nil { 169 return -1, -1, "pipe2", err 170 } 171 172 if p == nil { 173 p = new(bool) 174 defer atomic.StorePointer(&disableSplice, unsafe.Pointer(p)) 175 176 // F_GETPIPE_SZ was added in 2.6.35, which does not have the -EAGAIN bug. 177 if _, _, errno := syscall.Syscall(unix.FcntlSyscall, uintptr(fds[0]), syscall.F_GETPIPE_SZ, 0); errno != 0 { 178 *p = true 179 destroyTempPipe(fds[0], fds[1]) 180 return -1, -1, "fcntl", errno 181 } 182 } 183 184 return fds[0], fds[1], "", nil 185 } 186 187 // destroyTempPipe destroys a temporary pipe. 188 func destroyTempPipe(prfd, pwfd int) error { 189 err := CloseFunc(prfd) 190 err1 := CloseFunc(pwfd) 191 if err == nil { 192 return err1 193 } 194 return err 195 }