github.com/weaviate/weaviate@v1.24.6/adapters/repos/db/lsmkv/commitlogger_parser.go (about)

     1  //                           _       _
     2  // __      _____  __ ___   ___  __ _| |_ ___
     3  // \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
     4  //  \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
     5  //   \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
     6  //
     7  //  Copyright © 2016 - 2024 Weaviate B.V. All rights reserved.
     8  //
     9  //  CONTACT: hello@weaviate.io
    10  //
    11  
    12  package lsmkv
    13  
    14  import (
    15  	"bytes"
    16  	"encoding/binary"
    17  	"io"
    18  
    19  	"github.com/pkg/errors"
    20  	"github.com/weaviate/weaviate/adapters/repos/db/lsmkv/rwhasher"
    21  )
    22  
    23  type commitloggerParser struct {
    24  	strategy string
    25  
    26  	reader         io.Reader
    27  	checksumReader rwhasher.ReaderHasher
    28  
    29  	bufNode *bytes.Buffer
    30  
    31  	memtable *Memtable
    32  }
    33  
    34  func newCommitLoggerParser(strategy string, reader io.Reader, memtable *Memtable,
    35  ) *commitloggerParser {
    36  	return &commitloggerParser{
    37  		strategy:       strategy,
    38  		reader:         reader,
    39  		checksumReader: rwhasher.NewCRC32Reader(reader),
    40  		bufNode:        bytes.NewBuffer(nil),
    41  		memtable:       memtable,
    42  	}
    43  }
    44  
    45  func (p *commitloggerParser) Do() error {
    46  	switch p.strategy {
    47  	case StrategyReplace:
    48  		return p.doReplace()
    49  	case StrategyMapCollection, StrategySetCollection:
    50  		return p.doCollection()
    51  	case StrategyRoaringSet:
    52  		return p.doRoaringSet()
    53  	default:
    54  		return errors.Errorf("unknown strategy %s on commit log parse", p.strategy)
    55  	}
    56  }
    57  
    58  func (p *commitloggerParser) doRecord() (r io.Reader, err error) {
    59  	var nodeLen uint32
    60  	err = binary.Read(p.checksumReader, binary.LittleEndian, &nodeLen)
    61  	if err != nil {
    62  		return nil, errors.Wrap(err, "read commit node length")
    63  	}
    64  
    65  	p.bufNode.Reset()
    66  
    67  	io.CopyN(p.bufNode, p.checksumReader, int64(nodeLen))
    68  
    69  	// read checksum directly from the reader
    70  	var checksum [4]byte
    71  	_, err = io.ReadFull(p.reader, checksum[:])
    72  	if err != nil {
    73  		return nil, errors.Wrap(err, "read commit checksum")
    74  	}
    75  
    76  	// validate checksum
    77  	if !bytes.Equal(checksum[:], p.checksumReader.Hash()) {
    78  		return nil, errors.Wrap(ErrInvalidChecksum, "read commit entry")
    79  	}
    80  
    81  	return p.bufNode, nil
    82  }