github.com/weaviate/weaviate@v1.24.6/adapters/repos/db/lsmkv/memtable_flush_roaring_set.go (about)

     1  //                           _       _
     2  // __      _____  __ ___   ___  __ _| |_ ___
     3  // \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
     4  //  \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
     5  //   \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
     6  //
     7  //  Copyright © 2016 - 2024 Weaviate B.V. All rights reserved.
     8  //
     9  //  CONTACT: hello@weaviate.io
    10  //
    11  
    12  package lsmkv
    13  
    14  import (
    15  	"fmt"
    16  	"io"
    17  
    18  	"github.com/weaviate/weaviate/adapters/repos/db/lsmkv/segmentindex"
    19  	"github.com/weaviate/weaviate/adapters/repos/db/roaringset"
    20  )
    21  
    22  func (m *Memtable) flushDataRoaringSet(f io.Writer) ([]segmentindex.Key, error) {
    23  	flat := m.roaringSet.FlattenInOrder()
    24  
    25  	totalDataLength := totalPayloadSizeRoaringSet(flat)
    26  	header := segmentindex.Header{
    27  		IndexStart:       uint64(totalDataLength + segmentindex.HeaderSize),
    28  		Level:            0, // always level zero on a new one
    29  		Version:          0, // always version 0 for now
    30  		SecondaryIndices: 0,
    31  		Strategy:         segmentindex.StrategyRoaringSet,
    32  	}
    33  
    34  	n, err := header.WriteTo(f)
    35  	if err != nil {
    36  		return nil, err
    37  	}
    38  	headerSize := int(n)
    39  	keys := make([]segmentindex.Key, len(flat))
    40  
    41  	totalWritten := headerSize
    42  	for i, node := range flat {
    43  		sn, err := roaringset.NewSegmentNode(node.Key, node.Value.Additions,
    44  			node.Value.Deletions)
    45  		if err != nil {
    46  			return nil, fmt.Errorf("create segment node: %w", err)
    47  		}
    48  
    49  		ki, err := sn.KeyIndexAndWriteTo(f, totalWritten)
    50  		if err != nil {
    51  			return nil, fmt.Errorf("write node %d: %w", i, err)
    52  		}
    53  
    54  		keys[i] = ki
    55  		totalWritten = ki.ValueEnd
    56  	}
    57  
    58  	return keys, nil
    59  }
    60  
    61  func totalPayloadSizeRoaringSet(in []*roaringset.BinarySearchNode) int {
    62  	var sum int
    63  	for _, n := range in {
    64  		sum += 8 // uint64 to segment length
    65  		sum += 8 // uint64 to indicate length of additions bitmap
    66  		sum += len(n.Value.Additions.ToBuffer())
    67  		sum += 8 // uint64 to indicate length of deletions bitmap
    68  		sum += len(n.Value.Deletions.ToBuffer())
    69  		sum += 4 // uint32 to indicate key size
    70  		sum += len(n.Key)
    71  	}
    72  
    73  	return sum
    74  }