github.com/SandwichDev/go-internals@v0.0.0-20210605002614-12311ac6b2c5/poll/splice_linux.go (about)

     1  // Copyright 2018 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package poll
     6  
     7  import (
     8  	"sync/atomic"
     9  	"syscall"
    10  	"unsafe"
    11  
    12  	"github.com/SandwichDev/go-internals/syscall/unix"
    13  )
    14  
    15  const (
    16  	// spliceNonblock makes calls to splice(2) non-blocking.
    17  	spliceNonblock = 0x2
    18  
    19  	// maxSpliceSize is the maximum amount of data Splice asks
    20  	// the kernel to move in a single call to splice(2).
    21  	maxSpliceSize = 4 << 20
    22  )
    23  
    24  // Splice transfers at most remain bytes of data from src to dst, using the
    25  // splice system call to minimize copies of data from and to userspace.
    26  //
    27  // Splice creates a temporary pipe, to serve as a buffer for the data transfer.
    28  // src and dst must both be stream-oriented sockets.
    29  //
    30  // If err != nil, sc is the system call which caused the error.
    31  func Splice(dst, src *FD, remain int64) (written int64, handled bool, sc string, err error) {
    32  	prfd, pwfd, sc, err := newTempPipe()
    33  	if err != nil {
    34  		return 0, false, sc, err
    35  	}
    36  	defer destroyTempPipe(prfd, pwfd)
    37  	var inPipe, n int
    38  	for err == nil && remain > 0 {
    39  		max := maxSpliceSize
    40  		if int64(max) > remain {
    41  			max = int(remain)
    42  		}
    43  		inPipe, err = spliceDrain(pwfd, src, max)
    44  		// The operation is considered handled if splice returns no
    45  		// error, or an error other than EINVAL. An EINVAL means the
    46  		// kernel does not support splice for the socket type of src.
    47  		// The failed syscall does not consume any data so it is safe
    48  		// to fall back to a generic copy.
    49  		//
    50  		// spliceDrain should never return EAGAIN, so if err != nil,
    51  		// Splice cannot continue.
    52  		//
    53  		// If inPipe == 0 && err == nil, src is at EOF, and the
    54  		// transfer is complete.
    55  		handled = handled || (err != syscall.EINVAL)
    56  		if err != nil || (inPipe == 0 && err == nil) {
    57  			break
    58  		}
    59  		n, err = splicePump(dst, prfd, inPipe)
    60  		if n > 0 {
    61  			written += int64(n)
    62  			remain -= int64(n)
    63  		}
    64  	}
    65  	if err != nil {
    66  		return written, handled, "splice", err
    67  	}
    68  	return written, true, "", nil
    69  }
    70  
    71  // spliceDrain moves data from a socket to a pipe.
    72  //
    73  // Invariant: when entering spliceDrain, the pipe is empty. It is either in its
    74  // initial state, or splicePump has emptied it previously.
    75  //
    76  // Given this, spliceDrain can reasonably assume that the pipe is ready for
    77  // writing, so if splice returns EAGAIN, it must be because the socket is not
    78  // ready for reading.
    79  //
    80  // If spliceDrain returns (0, nil), src is at EOF.
    81  func spliceDrain(pipefd int, sock *FD, max int) (int, error) {
    82  	if err := sock.readLock(); err != nil {
    83  		return 0, err
    84  	}
    85  	defer sock.readUnlock()
    86  	if err := sock.pd.prepareRead(sock.isFile); err != nil {
    87  		return 0, err
    88  	}
    89  	for {
    90  		n, err := splice(pipefd, sock.Sysfd, max, spliceNonblock)
    91  		if err == syscall.EINTR {
    92  			continue
    93  		}
    94  		if err != syscall.EAGAIN {
    95  			return n, err
    96  		}
    97  		if err := sock.pd.waitRead(sock.isFile); err != nil {
    98  			return n, err
    99  		}
   100  	}
   101  }
   102  
   103  // splicePump moves all the buffered data from a pipe to a socket.
   104  //
   105  // Invariant: when entering splicePump, there are exactly inPipe
   106  // bytes of data in the pipe, from a previous call to spliceDrain.
   107  //
   108  // By analogy to the condition from spliceDrain, splicePump
   109  // only needs to poll the socket for readiness, if splice returns
   110  // EAGAIN.
   111  //
   112  // If splicePump cannot move all the data in a single call to
   113  // splice(2), it loops over the buffered data until it has written
   114  // all of it to the socket. This behavior is similar to the Write
   115  // step of an io.Copy in userspace.
   116  func splicePump(sock *FD, pipefd int, inPipe int) (int, error) {
   117  	if err := sock.writeLock(); err != nil {
   118  		return 0, err
   119  	}
   120  	defer sock.writeUnlock()
   121  	if err := sock.pd.prepareWrite(sock.isFile); err != nil {
   122  		return 0, err
   123  	}
   124  	written := 0
   125  	for inPipe > 0 {
   126  		n, err := splice(sock.Sysfd, pipefd, inPipe, spliceNonblock)
   127  		// Here, the condition n == 0 && err == nil should never be
   128  		// observed, since Splice controls the write side of the pipe.
   129  		if n > 0 {
   130  			inPipe -= n
   131  			written += n
   132  			continue
   133  		}
   134  		if err != syscall.EAGAIN {
   135  			return written, err
   136  		}
   137  		if err := sock.pd.waitWrite(sock.isFile); err != nil {
   138  			return written, err
   139  		}
   140  	}
   141  	return written, nil
   142  }
   143  
   144  // splice wraps the splice system call. Since the current implementation
   145  // only uses splice on sockets and pipes, the offset arguments are unused.
   146  // splice returns int instead of int64, because callers never ask it to
   147  // move more data in a single call than can fit in an int32.
   148  func splice(out int, in int, max int, flags int) (int, error) {
   149  	n, err := syscall.Splice(in, nil, out, nil, max, flags)
   150  	return int(n), err
   151  }
   152  
   153  var disableSplice unsafe.Pointer
   154  
   155  // newTempPipe sets up a temporary pipe for a splice operation.
   156  func newTempPipe() (prfd, pwfd int, sc string, err error) {
   157  	p := (*bool)(atomic.LoadPointer(&disableSplice))
   158  	if p != nil && *p {
   159  		return -1, -1, "splice", syscall.EINVAL
   160  	}
   161  
   162  	var fds [2]int
   163  	// pipe2 was added in 2.6.27 and our minimum requirement is 2.6.23, so it
   164  	// might not be implemented. Falling back to pipe is possible, but prior to
   165  	// 2.6.29 splice returns -EAGAIN instead of 0 when the connection is
   166  	// closed.
   167  	const flags = syscall.O_CLOEXEC | syscall.O_NONBLOCK
   168  	if err := syscall.Pipe2(fds[:], flags); err != nil {
   169  		return -1, -1, "pipe2", err
   170  	}
   171  
   172  	if p == nil {
   173  		p = new(bool)
   174  		defer atomic.StorePointer(&disableSplice, unsafe.Pointer(p))
   175  
   176  		// F_GETPIPE_SZ was added in 2.6.35, which does not have the -EAGAIN bug.
   177  		if _, _, errno := syscall.Syscall(unix.FcntlSyscall, uintptr(fds[0]), syscall.F_GETPIPE_SZ, 0); errno != 0 {
   178  			*p = true
   179  			destroyTempPipe(fds[0], fds[1])
   180  			return -1, -1, "fcntl", errno
   181  		}
   182  	}
   183  
   184  	return fds[0], fds[1], "", nil
   185  }
   186  
   187  // destroyTempPipe destroys a temporary pipe.
   188  func destroyTempPipe(prfd, pwfd int) error {
   189  	err := CloseFunc(prfd)
   190  	err1 := CloseFunc(pwfd)
   191  	if err == nil {
   192  		return err1
   193  	}
   194  	return err
   195  }