github.com/keltia/go-ipfs@v0.3.8-0.20150909044612-210793031c63/importer/chunk/splitting.go (about)

     1  // package chunk implements streaming block splitters
     2  package chunk
     3  
     4  import (
     5  	"io"
     6  
     7  	"github.com/ipfs/go-ipfs/util"
     8  )
     9  
    10  var log = util.Logger("chunk")
    11  
    12  var DefaultBlockSize int64 = 1024 * 256
    13  
    14  type Splitter interface {
    15  	NextBytes() ([]byte, error)
    16  }
    17  
    18  type SplitterGen func(r io.Reader) Splitter
    19  
    20  func DefaultSplitter(r io.Reader) Splitter {
    21  	return NewSizeSplitter(r, DefaultBlockSize)
    22  }
    23  
    24  func SizeSplitterGen(size int64) SplitterGen {
    25  	return func(r io.Reader) Splitter {
    26  		return NewSizeSplitter(r, size)
    27  	}
    28  }
    29  
    30  func Chan(s Splitter) (<-chan []byte, <-chan error) {
    31  	out := make(chan []byte)
    32  	errs := make(chan error, 1)
    33  	go func() {
    34  		defer close(out)
    35  		defer close(errs)
    36  
    37  		// all-chunks loop (keep creating chunks)
    38  		for {
    39  			b, err := s.NextBytes()
    40  			if err != nil {
    41  				errs <- err
    42  				return
    43  			}
    44  
    45  			out <- b
    46  		}
    47  	}()
    48  	return out, errs
    49  }
    50  
    51  type sizeSplitterv2 struct {
    52  	r    io.Reader
    53  	size int64
    54  	err  error
    55  }
    56  
    57  func NewSizeSplitter(r io.Reader, size int64) Splitter {
    58  	return &sizeSplitterv2{
    59  		r:    r,
    60  		size: size,
    61  	}
    62  }
    63  
    64  func (ss *sizeSplitterv2) NextBytes() ([]byte, error) {
    65  	if ss.err != nil {
    66  		return nil, ss.err
    67  	}
    68  	buf := make([]byte, ss.size)
    69  	n, err := io.ReadFull(ss.r, buf)
    70  	if err == io.ErrUnexpectedEOF {
    71  		ss.err = io.EOF
    72  		err = nil
    73  	}
    74  	if err != nil {
    75  		return nil, err
    76  	}
    77  
    78  	return buf[:n], nil
    79  }