github.com/m3db/m3@v1.5.0/src/dbnode/persist/fs/commitlog/chunk_reader.go (about)

     1  // Copyright (c) 2017 Uber Technologies, Inc.
     2  //
     3  // Permission is hereby granted, free of charge, to any person obtaining a copy
     4  // of this software and associated documentation files (the "Software"), to deal
     5  // in the Software without restriction, including without limitation the rights
     6  // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
     7  // copies of the Software, and to permit persons to whom the Software is
     8  // furnished to do so, subject to the following conditions:
     9  //
    10  // The above copyright notice and this permission notice shall be included in
    11  // all copies or substantial portions of the Software.
    12  //
    13  // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    14  // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    15  // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    16  // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    17  // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    18  // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
    19  // THE SOFTWARE.
    20  
    21  package commitlog
    22  
    23  import (
    24  	"bufio"
    25  	"io"
    26  	"os"
    27  
    28  	"github.com/m3db/m3/src/dbnode/digest"
    29  )
    30  
    31  const (
    32  	sizeStart         = 0
    33  	sizeEnd           = chunkHeaderSizeLen
    34  	checksumSizeStart = sizeEnd
    35  	checksumSizeEnd   = checksumSizeStart + chunkHeaderSizeLen
    36  	checksumDataStart = checksumSizeEnd
    37  	checksumDataEnd   = checksumDataStart + chunkHeaderChecksumDataLen
    38  )
    39  
    40  type chunkReader struct {
    41  	fd                 *os.File
    42  	buffer             *bufio.Reader
    43  	chunkData          []byte
    44  	chunkDataRemaining int
    45  	charBuff           []byte
    46  }
    47  
    48  func newChunkReader(bufferLen int) *chunkReader {
    49  	return &chunkReader{
    50  		buffer:    bufio.NewReaderSize(nil, bufferLen),
    51  		chunkData: make([]byte, bufferLen),
    52  		charBuff:  make([]byte, 1),
    53  	}
    54  }
    55  
    56  func (r *chunkReader) reset(fd *os.File) {
    57  	r.fd = fd
    58  	r.buffer.Reset(fd)
    59  	r.chunkDataRemaining = 0
    60  }
    61  
    62  func (r *chunkReader) readHeader() error {
    63  	header, err := r.buffer.Peek(chunkHeaderLen)
    64  	if err != nil {
    65  		return err
    66  	}
    67  
    68  	size := endianness.Uint32(header[sizeStart:sizeEnd])
    69  	checksumSize := digest.
    70  		Buffer(header[checksumSizeStart:checksumSizeEnd]).
    71  		ReadDigest()
    72  	checksumData := digest.
    73  		Buffer(header[checksumDataStart:checksumDataEnd]).
    74  		ReadDigest()
    75  
    76  	// Verify size checksum
    77  	if digest.Checksum(header[sizeStart:sizeEnd]) != checksumSize {
    78  		return errCommitLogReaderChunkSizeChecksumMismatch
    79  	}
    80  
    81  	// Discard the peeked header
    82  	if _, err := r.buffer.Discard(chunkHeaderLen); err != nil {
    83  		return err
    84  	}
    85  
    86  	// Setup a chunk data buffer so that chunk data can be loaded into it.
    87  	chunkDataSize := int(size)
    88  	if chunkDataSize > cap(r.chunkData) {
    89  		// Increase chunkData capacity so that it can fit the new chunkData.
    90  		chunkDataCap := cap(r.chunkData)
    91  		for chunkDataCap < chunkDataSize {
    92  			chunkDataCap *= 2
    93  		}
    94  		r.chunkData = make([]byte, chunkDataSize, chunkDataCap)
    95  	} else {
    96  		// Reuse existing chunk data buffer if possible.
    97  		r.chunkData = r.chunkData[:chunkDataSize]
    98  	}
    99  
   100  	// To validate checksum of chunk data all the chunk data needs to be loaded into memory at once. Chunk data size is // not bounded to the flush size so peeking chunk data in order to compute checksum may result in bufio's buffer
   101  	// full error. To circumnavigate this issue load the chunk data into chunk reader's buffer to compute checksum
   102  	// instead of trying to compute checksum off of fixed size r.buffer by peeking.
   103  	// See https://github.com/m3db/m3/pull/2148 for details.
   104  	_, err = io.ReadFull(r.buffer, r.chunkData)
   105  	if err != nil {
   106  		return err
   107  	}
   108  
   109  	// Verify data checksum
   110  	if digest.Checksum(r.chunkData) != checksumData {
   111  		return errCommitLogReaderChunkSizeChecksumMismatch
   112  	}
   113  
   114  	// Set remaining data to be consumed
   115  	r.chunkDataRemaining = int(size)
   116  
   117  	return nil
   118  }
   119  
   120  func (r *chunkReader) Read(p []byte) (int, error) {
   121  	size := len(p)
   122  	read := 0
   123  	// Check if requesting for size larger than this chunk
   124  	if r.chunkDataRemaining < size {
   125  		// Copy any remaining
   126  		if r.chunkDataRemaining > 0 {
   127  			chunkDataOffset := len(r.chunkData) - r.chunkDataRemaining
   128  			n := copy(p, r.chunkData[chunkDataOffset:])
   129  			r.chunkDataRemaining -= n
   130  			read += n
   131  		}
   132  
   133  		// Read next header
   134  		if err := r.readHeader(); err != nil {
   135  			return read, err
   136  		}
   137  
   138  		// Reset read target
   139  		p = p[read:]
   140  
   141  		// Perform consecutive read(s)
   142  		n, err := r.Read(p)
   143  		read += n
   144  		return read, err
   145  	}
   146  
   147  	chunkDataOffset := len(r.chunkData) - r.chunkDataRemaining
   148  	n := copy(p, r.chunkData[chunkDataOffset:][:len(p)])
   149  	r.chunkDataRemaining -= n
   150  	read += n
   151  	return read, nil
   152  }
   153  
   154  func (r *chunkReader) ReadByte() (c byte, err error) {
   155  	if _, err := r.Read(r.charBuff); err != nil {
   156  		return byte(0), err
   157  	}
   158  	return r.charBuff[0], nil
   159  }