github.com/weaviate/weaviate@v1.24.6/adapters/repos/db/lsmkv/commitlogger_parser_roaringset.go (about)

     1  //                           _       _
     2  // __      _____  __ ___   ___  __ _| |_ ___
     3  // \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
     4  //  \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
     5  //   \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
     6  //
     7  //  Copyright © 2016 - 2024 Weaviate B.V. All rights reserved.
     8  //
     9  //  CONTACT: hello@weaviate.io
    10  //
    11  
    12  package lsmkv
    13  
    14  import (
    15  	"encoding/binary"
    16  	"fmt"
    17  	"io"
    18  
    19  	"github.com/pkg/errors"
    20  	"github.com/weaviate/weaviate/adapters/repos/db/roaringset"
    21  )
    22  
    23  func (p *commitloggerParser) doRoaringSet() error {
    24  	for {
    25  		var commitType CommitType
    26  
    27  		err := binary.Read(p.checksumReader, binary.LittleEndian, &commitType)
    28  		if errors.Is(err, io.EOF) {
    29  			break
    30  		}
    31  		if err != nil {
    32  			return errors.Wrap(err, "read commit type")
    33  		}
    34  
    35  		if !CommitTypeRoaringSet.Is(commitType) {
    36  			return errors.Errorf("found a %s commit on a roaringset bucket", commitType.String())
    37  		}
    38  
    39  		var version uint8
    40  
    41  		err = binary.Read(p.checksumReader, binary.LittleEndian, &version)
    42  		if err != nil {
    43  			return errors.Wrap(err, "read commit version")
    44  		}
    45  
    46  		switch version {
    47  		case 0:
    48  			{
    49  				err = p.parseRoaringSetNodeV0()
    50  			}
    51  		case 1:
    52  			{
    53  				err = p.parseRoaringSetNodeV1()
    54  			}
    55  		default:
    56  			{
    57  				return fmt.Errorf("unsupported commit version %d", version)
    58  			}
    59  		}
    60  		if err != nil {
    61  			return err
    62  		}
    63  	}
    64  
    65  	return nil
    66  }
    67  
    68  func (p *commitloggerParser) parseRoaringSetNodeV0() error {
    69  	return p.parseRoaringSetNode(p.reader)
    70  }
    71  
    72  func (p *commitloggerParser) parseRoaringSetNodeV1() error {
    73  	reader, err := p.doRecord()
    74  	if err != nil {
    75  		return err
    76  	}
    77  
    78  	return p.parseRoaringSetNode(reader)
    79  }
    80  
    81  func (p *commitloggerParser) parseRoaringSetNode(reader io.Reader) error {
    82  	lenBuf := make([]byte, 8)
    83  	if _, err := io.ReadFull(reader, lenBuf); err != nil {
    84  		return errors.Wrap(err, "read segment len")
    85  	}
    86  	segmentLen := binary.LittleEndian.Uint64(lenBuf)
    87  
    88  	segBuf := make([]byte, segmentLen)
    89  	copy(segBuf, lenBuf)
    90  	if _, err := io.ReadFull(reader, segBuf[8:]); err != nil {
    91  		return errors.Wrap(err, "read segment contents")
    92  	}
    93  
    94  	segment := roaringset.NewSegmentNodeFromBuffer(segBuf)
    95  	key := segment.PrimaryKey()
    96  	if err := p.memtable.roaringSetAddRemoveBitmaps(key, segment.Additions(), segment.Deletions()); err != nil {
    97  		return errors.Wrap(err, "add/remove bitmaps")
    98  	}
    99  
   100  	return nil
   101  }