github.com/hasnat/dolt/go@v0.0.0-20210628190320-9eb5d843fbb7/store/chunks/chunk_serializer.go (about)

     1  // Copyright 2019 Dolthub, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  //
    15  // This file incorporates work covered by the following copyright and
    16  // permission notice:
    17  //
    18  // Copyright 2016 Attic Labs, Inc. All rights reserved.
    19  // Licensed under the Apache License, version 2.0:
    20  // http://www.apache.org/licenses/LICENSE-2.0
    21  
    22  package chunks
    23  
    24  import (
    25  	"bytes"
    26  	"encoding/binary"
    27  	"io"
    28  
    29  	"github.com/dolthub/dolt/go/store/d"
    30  	"github.com/dolthub/dolt/go/store/hash"
    31  )
    32  
    33  /*
    34    Chunk Serialization:
    35      Chunk 0
    36      Chunk 1
    37       ..
    38      Chunk N
    39  
    40    Chunk:
    41      Hash  // 20-byte hash
    42      Len   // 4-byte int
    43      Data  // len(Data) == Len
    44  */
    45  
    46  // Serialize a single Chunk to writer.
    47  func Serialize(chunk Chunk, writer io.Writer) {
    48  	d.PanicIfFalse(chunk.data != nil)
    49  
    50  	h := chunk.Hash()
    51  	n, err := io.Copy(writer, bytes.NewReader(h[:]))
    52  	d.Chk.NoError(err)
    53  	d.PanicIfFalse(int64(hash.ByteLen) == n)
    54  
    55  	// Because of chunking at higher levels, no chunk should never be more than 4GB
    56  	chunkSize := uint32(len(chunk.Data()))
    57  	err = binary.Write(writer, binary.BigEndian, chunkSize)
    58  	d.Chk.NoError(err)
    59  
    60  	n, err = io.Copy(writer, bytes.NewReader(chunk.Data()))
    61  	d.Chk.NoError(err)
    62  	d.PanicIfFalse(uint32(n) == chunkSize)
    63  }
    64  
    65  // Deserialize reads off of |reader| until EOF, sending chunks to
    66  // chunkChan in the order they are read. Objects sent over chunkChan are
    67  // *Chunk.
    68  func Deserialize(reader io.Reader, chunkChan chan<- *Chunk) (err error) {
    69  	for {
    70  		var c Chunk
    71  		c, err = deserializeChunk(reader)
    72  		if err != nil {
    73  			break
    74  		}
    75  		d.Chk.NotEqual(EmptyChunk.Hash(), c.Hash())
    76  		chunkChan <- &c
    77  	}
    78  	if err == io.EOF {
    79  		err = nil
    80  	}
    81  	return
    82  }
    83  
    84  // DeserializeData deserializes a chunk from a byte array
    85  func DeserializeData(data []byte) (Chunk, error) {
    86  	reader := bytes.NewReader(data)
    87  	return deserializeChunk(reader)
    88  }
    89  
    90  func deserializeChunk(reader io.Reader) (Chunk, error) {
    91  	h := hash.Hash{}
    92  	n, err := io.ReadFull(reader, h[:])
    93  	if err != nil {
    94  		return EmptyChunk, err
    95  	}
    96  	d.PanicIfFalse(int(hash.ByteLen) == n)
    97  
    98  	chunkSize := uint32(0)
    99  	if err = binary.Read(reader, binary.BigEndian, &chunkSize); err != nil {
   100  		return EmptyChunk, err
   101  	}
   102  
   103  	data := make([]byte, int(chunkSize))
   104  	if n, err = io.ReadFull(reader, data); err != nil {
   105  		return EmptyChunk, err
   106  	}
   107  	d.PanicIfFalse(int(chunkSize) == n)
   108  	c := NewChunk(data)
   109  	if h != c.Hash() {
   110  		d.Panic("%s != %s", h, c.Hash().String())
   111  	}
   112  	return c, nil
   113  }