github.com/weaviate/weaviate@v1.24.6/adapters/repos/db/lsmkv/commitlogger_parser_replace.go (about)

     1  //                           _       _
     2  // __      _____  __ ___   ___  __ _| |_ ___
     3  // \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
     4  //  \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
     5  //   \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
     6  //
     7  //  Copyright © 2016 - 2024 Weaviate B.V. All rights reserved.
     8  //
     9  //  CONTACT: hello@weaviate.io
    10  //
    11  
    12  package lsmkv
    13  
    14  import (
    15  	"encoding/binary"
    16  	"fmt"
    17  	"io"
    18  
    19  	"github.com/pkg/errors"
    20  )
    21  
    22  // doReplace parsers all entries into a cache for deduplication first and only
    23  // imports unique entries into the actual memtable as a final step.
    24  func (p *commitloggerParser) doReplace() error {
    25  	nodeCache := make(map[string]segmentReplaceNode)
    26  
    27  	var errWhileParsing error
    28  
    29  	for {
    30  		var commitType CommitType
    31  
    32  		err := binary.Read(p.checksumReader, binary.LittleEndian, &commitType)
    33  		if errors.Is(err, io.EOF) {
    34  			break
    35  		}
    36  		if err != nil {
    37  			errWhileParsing = errors.Wrap(err, "read commit type")
    38  			break
    39  		}
    40  		if !CommitTypeReplace.Is(commitType) {
    41  			return errors.Errorf("found a %s commit on a replace bucket", commitType.String())
    42  		}
    43  
    44  		var version uint8
    45  
    46  		err = binary.Read(p.checksumReader, binary.LittleEndian, &version)
    47  		if err != nil {
    48  			errWhileParsing = errors.Wrap(err, "read commit version")
    49  			break
    50  		}
    51  
    52  		switch version {
    53  		case 0:
    54  			{
    55  				err = p.doReplaceRecordV0(nodeCache)
    56  			}
    57  		case 1:
    58  			{
    59  				err = p.doReplaceRecordV1(nodeCache)
    60  			}
    61  		default:
    62  			{
    63  				return fmt.Errorf("unsupported commit version %d", version)
    64  			}
    65  		}
    66  		if err != nil {
    67  			errWhileParsing = err
    68  			break
    69  		}
    70  	}
    71  
    72  	for _, node := range nodeCache {
    73  		var opts []SecondaryKeyOption
    74  		if p.memtable.secondaryIndices > 0 {
    75  			for i, secKey := range node.secondaryKeys {
    76  				opts = append(opts, WithSecondaryKey(i, secKey))
    77  			}
    78  		}
    79  		if node.tombstone {
    80  			p.memtable.setTombstone(node.primaryKey, opts...)
    81  		} else {
    82  			p.memtable.put(node.primaryKey, node.value, opts...)
    83  		}
    84  	}
    85  
    86  	return errWhileParsing
    87  }
    88  
    89  func (p *commitloggerParser) doReplaceRecordV0(nodeCache map[string]segmentReplaceNode) error {
    90  	return p.parseReplaceNode(p.reader, nodeCache)
    91  }
    92  
    93  func (p *commitloggerParser) doReplaceRecordV1(nodeCache map[string]segmentReplaceNode) error {
    94  	reader, err := p.doRecord()
    95  	if err != nil {
    96  		return err
    97  	}
    98  
    99  	return p.parseReplaceNode(reader, nodeCache)
   100  }
   101  
   102  // parseReplaceNode only parses into the deduplication cache, not into the
   103  // final memtable yet. A second step is required to parse from the cache into
   104  // the actual memtable.
   105  func (p *commitloggerParser) parseReplaceNode(r io.Reader, nodeCache map[string]segmentReplaceNode) error {
   106  	n, err := ParseReplaceNode(r, p.memtable.secondaryIndices)
   107  	if err != nil {
   108  		return err
   109  	}
   110  
   111  	if !n.tombstone {
   112  		nodeCache[string(n.primaryKey)] = n
   113  	} else {
   114  		if existing, ok := nodeCache[string(n.primaryKey)]; ok {
   115  			existing.tombstone = true
   116  			nodeCache[string(n.primaryKey)] = existing
   117  		} else {
   118  			nodeCache[string(n.primaryKey)] = n
   119  		}
   120  	}
   121  
   122  	return nil
   123  }