github.com/keltia/go-ipfs@v0.3.8-0.20150909044612-210793031c63/importer/chunk/splitting.go (about) 1 // package chunk implements streaming block splitters 2 package chunk 3 4 import ( 5 "io" 6 7 "github.com/ipfs/go-ipfs/util" 8 ) 9 10 var log = util.Logger("chunk") 11 12 var DefaultBlockSize int64 = 1024 * 256 13 14 type Splitter interface { 15 NextBytes() ([]byte, error) 16 } 17 18 type SplitterGen func(r io.Reader) Splitter 19 20 func DefaultSplitter(r io.Reader) Splitter { 21 return NewSizeSplitter(r, DefaultBlockSize) 22 } 23 24 func SizeSplitterGen(size int64) SplitterGen { 25 return func(r io.Reader) Splitter { 26 return NewSizeSplitter(r, size) 27 } 28 } 29 30 func Chan(s Splitter) (<-chan []byte, <-chan error) { 31 out := make(chan []byte) 32 errs := make(chan error, 1) 33 go func() { 34 defer close(out) 35 defer close(errs) 36 37 // all-chunks loop (keep creating chunks) 38 for { 39 b, err := s.NextBytes() 40 if err != nil { 41 errs <- err 42 return 43 } 44 45 out <- b 46 } 47 }() 48 return out, errs 49 } 50 51 type sizeSplitterv2 struct { 52 r io.Reader 53 size int64 54 err error 55 } 56 57 func NewSizeSplitter(r io.Reader, size int64) Splitter { 58 return &sizeSplitterv2{ 59 r: r, 60 size: size, 61 } 62 } 63 64 func (ss *sizeSplitterv2) NextBytes() ([]byte, error) { 65 if ss.err != nil { 66 return nil, ss.err 67 } 68 buf := make([]byte, ss.size) 69 n, err := io.ReadFull(ss.r, buf) 70 if err == io.ErrUnexpectedEOF { 71 ss.err = io.EOF 72 err = nil 73 } 74 if err != nil { 75 return nil, err 76 } 77 78 return buf[:n], nil 79 }