github.com/slava-ustovytski/docker@v1.8.2-rc1/pkg/ioutils/readers.go (about)

     1  package ioutils
     2  
     3  import (
     4  	"bytes"
     5  	"crypto/rand"
     6  	"crypto/sha256"
     7  	"encoding/hex"
     8  	"io"
     9  	"math/big"
    10  	"sync"
    11  	"time"
    12  )
    13  
    14  type readCloserWrapper struct {
    15  	io.Reader
    16  	closer func() error
    17  }
    18  
    19  func (r *readCloserWrapper) Close() error {
    20  	return r.closer()
    21  }
    22  
    23  func NewReadCloserWrapper(r io.Reader, closer func() error) io.ReadCloser {
    24  	return &readCloserWrapper{
    25  		Reader: r,
    26  		closer: closer,
    27  	}
    28  }
    29  
    30  type readerErrWrapper struct {
    31  	reader io.Reader
    32  	closer func()
    33  }
    34  
    35  func (r *readerErrWrapper) Read(p []byte) (int, error) {
    36  	n, err := r.reader.Read(p)
    37  	if err != nil {
    38  		r.closer()
    39  	}
    40  	return n, err
    41  }
    42  
    43  func NewReaderErrWrapper(r io.Reader, closer func()) io.Reader {
    44  	return &readerErrWrapper{
    45  		reader: r,
    46  		closer: closer,
    47  	}
    48  }
    49  
    50  // bufReader allows the underlying reader to continue to produce
    51  // output by pre-emptively reading from the wrapped reader.
    52  // This is achieved by buffering this data in bufReader's
    53  // expanding buffer.
    54  type bufReader struct {
    55  	sync.Mutex
    56  	buf                  *bytes.Buffer
    57  	reader               io.Reader
    58  	err                  error
    59  	wait                 sync.Cond
    60  	drainBuf             []byte
    61  	reuseBuf             []byte
    62  	maxReuse             int64
    63  	resetTimeout         time.Duration
    64  	bufLenResetThreshold int64
    65  	maxReadDataReset     int64
    66  }
    67  
    68  func NewBufReader(r io.Reader) *bufReader {
    69  	var timeout int
    70  	if randVal, err := rand.Int(rand.Reader, big.NewInt(120)); err == nil {
    71  		timeout = int(randVal.Int64()) + 180
    72  	} else {
    73  		timeout = 300
    74  	}
    75  	reader := &bufReader{
    76  		buf:                  &bytes.Buffer{},
    77  		drainBuf:             make([]byte, 1024),
    78  		reuseBuf:             make([]byte, 4096),
    79  		maxReuse:             1000,
    80  		resetTimeout:         time.Second * time.Duration(timeout),
    81  		bufLenResetThreshold: 100 * 1024,
    82  		maxReadDataReset:     10 * 1024 * 1024,
    83  		reader:               r,
    84  	}
    85  	reader.wait.L = &reader.Mutex
    86  	go reader.drain()
    87  	return reader
    88  }
    89  
    90  func NewBufReaderWithDrainbufAndBuffer(r io.Reader, drainBuffer []byte, buffer *bytes.Buffer) *bufReader {
    91  	reader := &bufReader{
    92  		buf:      buffer,
    93  		drainBuf: drainBuffer,
    94  		reader:   r,
    95  	}
    96  	reader.wait.L = &reader.Mutex
    97  	go reader.drain()
    98  	return reader
    99  }
   100  
   101  func (r *bufReader) drain() {
   102  	var (
   103  		duration       time.Duration
   104  		lastReset      time.Time
   105  		now            time.Time
   106  		reset          bool
   107  		bufLen         int64
   108  		dataSinceReset int64
   109  		maxBufLen      int64
   110  		reuseBufLen    int64
   111  		reuseCount     int64
   112  	)
   113  	reuseBufLen = int64(len(r.reuseBuf))
   114  	lastReset = time.Now()
   115  	for {
   116  		n, err := r.reader.Read(r.drainBuf)
   117  		dataSinceReset += int64(n)
   118  		r.Lock()
   119  		bufLen = int64(r.buf.Len())
   120  		if bufLen > maxBufLen {
   121  			maxBufLen = bufLen
   122  		}
   123  
   124  		// Avoid unbounded growth of the buffer over time.
   125  		// This has been discovered to be the only non-intrusive
   126  		// solution to the unbounded growth of the buffer.
   127  		// Alternative solutions such as compression, multiple
   128  		// buffers, channels and other similar pieces of code
   129  		// were reducing throughput, overall Docker performance
   130  		// or simply crashed Docker.
   131  		// This solution releases the buffer when specific
   132  		// conditions are met to avoid the continuous resizing
   133  		// of the buffer for long lived containers.
   134  		//
   135  		// Move data to the front of the buffer if it's
   136  		// smaller than what reuseBuf can store
   137  		if bufLen > 0 && reuseBufLen >= bufLen {
   138  			n, _ := r.buf.Read(r.reuseBuf)
   139  			r.buf.Write(r.reuseBuf[0:n])
   140  			// Take action if the buffer has been reused too many
   141  			// times and if there's data in the buffer.
   142  			// The timeout is also used as means to avoid doing
   143  			// these operations more often or less often than
   144  			// required.
   145  			// The various conditions try to detect heavy activity
   146  			// in the buffer which might be indicators of heavy
   147  			// growth of the buffer.
   148  		} else if reuseCount >= r.maxReuse && bufLen > 0 {
   149  			now = time.Now()
   150  			duration = now.Sub(lastReset)
   151  			timeoutReached := duration >= r.resetTimeout
   152  
   153  			// The timeout has been reached and the
   154  			// buffered data couldn't be moved to the front
   155  			// of the buffer, so the buffer gets reset.
   156  			if timeoutReached && bufLen > reuseBufLen {
   157  				reset = true
   158  			}
   159  			// The amount of buffered data is too high now,
   160  			// reset the buffer.
   161  			if timeoutReached && maxBufLen >= r.bufLenResetThreshold {
   162  				reset = true
   163  			}
   164  			// Reset the buffer if a certain amount of
   165  			// data has gone through the buffer since the
   166  			// last reset.
   167  			if timeoutReached && dataSinceReset >= r.maxReadDataReset {
   168  				reset = true
   169  			}
   170  			// The buffered data is moved to a fresh buffer,
   171  			// swap the old buffer with the new one and
   172  			// reset all counters.
   173  			if reset {
   174  				newbuf := &bytes.Buffer{}
   175  				newbuf.ReadFrom(r.buf)
   176  				r.buf = newbuf
   177  				lastReset = now
   178  				reset = false
   179  				dataSinceReset = 0
   180  				maxBufLen = 0
   181  				reuseCount = 0
   182  			}
   183  		}
   184  		if err != nil {
   185  			r.err = err
   186  		} else {
   187  			r.buf.Write(r.drainBuf[0:n])
   188  		}
   189  		reuseCount++
   190  		r.wait.Signal()
   191  		r.Unlock()
   192  		callSchedulerIfNecessary()
   193  		if err != nil {
   194  			break
   195  		}
   196  	}
   197  }
   198  
   199  func (r *bufReader) Read(p []byte) (n int, err error) {
   200  	r.Lock()
   201  	defer r.Unlock()
   202  	for {
   203  		n, err = r.buf.Read(p)
   204  		if n > 0 {
   205  			return n, err
   206  		}
   207  		if r.err != nil {
   208  			return 0, r.err
   209  		}
   210  		r.wait.Wait()
   211  	}
   212  }
   213  
   214  func (r *bufReader) Close() error {
   215  	closer, ok := r.reader.(io.ReadCloser)
   216  	if !ok {
   217  		return nil
   218  	}
   219  	return closer.Close()
   220  }
   221  
   222  func HashData(src io.Reader) (string, error) {
   223  	h := sha256.New()
   224  	if _, err := io.Copy(h, src); err != nil {
   225  		return "", err
   226  	}
   227  	return "sha256:" + hex.EncodeToString(h.Sum(nil)), nil
   228  }
   229  
   230  type OnEOFReader struct {
   231  	Rc io.ReadCloser
   232  	Fn func()
   233  }
   234  
   235  func (r *OnEOFReader) Read(p []byte) (n int, err error) {
   236  	n, err = r.Rc.Read(p)
   237  	if err == io.EOF {
   238  		r.runFunc()
   239  	}
   240  	return
   241  }
   242  
   243  func (r *OnEOFReader) Close() error {
   244  	err := r.Rc.Close()
   245  	r.runFunc()
   246  	return err
   247  }
   248  
   249  func (r *OnEOFReader) runFunc() {
   250  	if fn := r.Fn; fn != nil {
   251  		fn()
   252  		r.Fn = nil
   253  	}
   254  }