github.com/weaviate/weaviate@v1.24.6/adapters/repos/db/lsmkv/segment_replace_strategy.go (about)

     1  //                           _       _
     2  // __      _____  __ ___   ___  __ _| |_ ___
     3  // \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
     4  //  \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
     5  //   \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
     6  //
     7  //  Copyright © 2016 - 2024 Weaviate B.V. All rights reserved.
     8  //
     9  //  CONTACT: hello@weaviate.io
    10  //
    11  
    12  package lsmkv
    13  
    14  import (
    15  	"encoding/binary"
    16  	"errors"
    17  	"fmt"
    18  	"time"
    19  
    20  	"github.com/weaviate/weaviate/adapters/repos/db/lsmkv/segmentindex"
    21  	"github.com/weaviate/weaviate/entities/lsmkv"
    22  )
    23  
    24  func (s *segment) get(key []byte) ([]byte, error) {
    25  	if s.strategy != segmentindex.StrategyReplace {
    26  		return nil, fmt.Errorf("get only possible for strategy %q", StrategyReplace)
    27  	}
    28  
    29  	before := time.Now()
    30  
    31  	if s.useBloomFilter && !s.bloomFilter.Test(key) {
    32  		s.bloomFilterMetrics.trueNegative(before)
    33  		return nil, lsmkv.NotFound
    34  	}
    35  
    36  	node, err := s.index.Get(key)
    37  	if err != nil {
    38  		if errors.Is(err, lsmkv.NotFound) {
    39  			if s.useBloomFilter {
    40  				s.bloomFilterMetrics.falsePositive(before)
    41  			}
    42  			return nil, lsmkv.NotFound
    43  		} else {
    44  			return nil, err
    45  		}
    46  	}
    47  
    48  	defer func() {
    49  		if s.useBloomFilter {
    50  			s.bloomFilterMetrics.truePositive(before)
    51  		}
    52  	}()
    53  
    54  	// We need to copy the data we read from the segment exactly once in this
    55  	// place. This means that future processing can share this memory as much as
    56  	// it wants to, as it can now be considered immutable. If we didn't copy in
    57  	// this place it would only be safe to hold this data while still under the
    58  	// protection of the segmentGroup.maintenanceLock. This lock makes sure that
    59  	// no compaction is started during an ongoing read. However, once read,
    60  	// further processing is no longer protected by lock.
    61  	// If a compaction completes and the old segment is removed, we would be accessing
    62  	// invalid memory without the copy, thus leading to a SEGFAULT.
    63  	// Similar approach was used to fix SEGFAULT in collection strategy
    64  	// https://github.com/weaviate/weaviate/issues/1837
    65  	contentsCopy := make([]byte, node.End-node.Start)
    66  	if err = s.copyNode(contentsCopy, nodeOffset{node.Start, node.End}); err != nil {
    67  		return nil, err
    68  	}
    69  
    70  	return s.replaceStratParseData(contentsCopy)
    71  }
    72  
    73  func (s *segment) getBySecondaryIntoMemory(pos int, key []byte, buffer []byte) ([]byte, error, []byte) {
    74  	if s.strategy != segmentindex.StrategyReplace {
    75  		return nil, fmt.Errorf("get only possible for strategy %q", StrategyReplace), nil
    76  	}
    77  
    78  	if pos > len(s.secondaryIndices) || s.secondaryIndices[pos] == nil {
    79  		return nil, fmt.Errorf("no secondary index at pos %d", pos), nil
    80  	}
    81  
    82  	if s.useBloomFilter && !s.secondaryBloomFilters[pos].Test(key) {
    83  		return nil, lsmkv.NotFound, nil
    84  	}
    85  
    86  	node, err := s.secondaryIndices[pos].Get(key)
    87  	if err != nil {
    88  		return nil, err, nil
    89  	}
    90  
    91  	// We need to copy the data we read from the segment exactly once in this
    92  	// place. This means that future processing can share this memory as much as
    93  	// it wants to, as it can now be considered immutable. If we didn't copy in
    94  	// this place it would only be safe to hold this data while still under the
    95  	// protection of the segmentGroup.maintenanceLock. This lock makes sure that
    96  	// no compaction is started during an ongoing read. However, once read,
    97  	// further processing is no longer protected by lock.
    98  	// If a compaction completes and the old segment is removed, we would be accessing
    99  	// invalid memory without the copy, thus leading to a SEGFAULT.
   100  	// Similar approach was used to fix SEGFAULT in collection strategy
   101  	// https://github.com/weaviate/weaviate/issues/1837
   102  	var contentsCopy []byte
   103  	if uint64(cap(buffer)) >= node.End-node.Start {
   104  		contentsCopy = buffer[:node.End-node.Start]
   105  	} else {
   106  		contentsCopy = make([]byte, node.End-node.Start)
   107  	}
   108  	if err = s.copyNode(contentsCopy, nodeOffset{node.Start, node.End}); err != nil {
   109  		return nil, err, nil
   110  	}
   111  	currContent, err := s.replaceStratParseData(contentsCopy)
   112  	return currContent, err, contentsCopy
   113  }
   114  
   115  func (s *segment) replaceStratParseData(in []byte) ([]byte, error) {
   116  	if len(in) == 0 {
   117  		return nil, lsmkv.NotFound
   118  	}
   119  
   120  	// byte         meaning
   121  	// 0         is tombstone
   122  	// 1-8       data length as Little Endian uint64
   123  	// 9-length  data
   124  
   125  	// check the tombstone byte
   126  	if in[0] == 0x01 {
   127  		return nil, lsmkv.Deleted
   128  	}
   129  
   130  	valueLength := binary.LittleEndian.Uint64(in[1:9])
   131  
   132  	return in[9 : 9+valueLength], nil
   133  }