github.com/weaviate/weaviate@v1.24.6/adapters/repos/db/lsmkv/segmentindex/header.go (about)

     1  //                           _       _
     2  // __      _____  __ ___   ___  __ _| |_ ___
     3  // \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
     4  //  \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
     5  //   \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
     6  //
     7  //  Copyright © 2016 - 2024 Weaviate B.V. All rights reserved.
     8  //
     9  //  CONTACT: hello@weaviate.io
    10  //
    11  
    12  package segmentindex
    13  
    14  import (
    15  	"bufio"
    16  	"bytes"
    17  	"encoding/binary"
    18  	"fmt"
    19  	"io"
    20  )
    21  
    22  // HeaderSize describes the general offset in a segment until the data
    23  // starts, it is composed of 2 bytes for level, 2 bytes for version,
    24  // 2 bytes for secondary index count, 2 bytes for strategy, 8 bytes
    25  // for the pointer to the index part
    26  const HeaderSize = 16
    27  
    28  type Header struct {
    29  	Level            uint16
    30  	Version          uint16
    31  	SecondaryIndices uint16
    32  	Strategy         Strategy
    33  	IndexStart       uint64
    34  }
    35  
    36  func (h *Header) WriteTo(w io.Writer) (int64, error) {
    37  	if err := binary.Write(w, binary.LittleEndian, &h.Level); err != nil {
    38  		return -1, err
    39  	}
    40  	if err := binary.Write(w, binary.LittleEndian, &h.Version); err != nil {
    41  		return -1, err
    42  	}
    43  	if err := binary.Write(w, binary.LittleEndian, &h.SecondaryIndices); err != nil {
    44  		return -1, err
    45  	}
    46  	if err := binary.Write(w, binary.LittleEndian, h.Strategy); err != nil {
    47  		return -1, err
    48  	}
    49  	if err := binary.Write(w, binary.LittleEndian, &h.IndexStart); err != nil {
    50  		return -1, err
    51  	}
    52  
    53  	return int64(HeaderSize), nil
    54  }
    55  
    56  func (h *Header) PrimaryIndex(source []byte) ([]byte, error) {
    57  	if h.SecondaryIndices == 0 {
    58  		return source[h.IndexStart:], nil
    59  	}
    60  
    61  	offsets, err := h.parseSecondaryIndexOffsets(
    62  		source[h.IndexStart:h.secondaryIndexOffsetsEnd()])
    63  	if err != nil {
    64  		return nil, err
    65  	}
    66  
    67  	// the beginning of the first secondary is also the end of the primary
    68  	end := offsets[0]
    69  	return source[h.secondaryIndexOffsetsEnd():end], nil
    70  }
    71  
    72  func (h *Header) secondaryIndexOffsetsEnd() uint64 {
    73  	return h.IndexStart + (uint64(h.SecondaryIndices) * 8)
    74  }
    75  
    76  func (h *Header) parseSecondaryIndexOffsets(source []byte) ([]uint64, error) {
    77  	r := bufio.NewReader(bytes.NewReader(source))
    78  
    79  	offsets := make([]uint64, h.SecondaryIndices)
    80  	if err := binary.Read(r, binary.LittleEndian, &offsets); err != nil {
    81  		return nil, err
    82  	}
    83  
    84  	return offsets, nil
    85  }
    86  
    87  func (h *Header) SecondaryIndex(source []byte, indexID uint16) ([]byte, error) {
    88  	if indexID >= h.SecondaryIndices {
    89  		return nil, fmt.Errorf("retrieve index %d with len %d",
    90  			indexID, h.SecondaryIndices)
    91  	}
    92  
    93  	offsets, err := h.parseSecondaryIndexOffsets(
    94  		source[h.IndexStart:h.secondaryIndexOffsetsEnd()])
    95  	if err != nil {
    96  		return nil, err
    97  	}
    98  
    99  	start := offsets[indexID]
   100  	if indexID == h.SecondaryIndices-1 {
   101  		// this is the last index, return until EOF
   102  		return source[start:], nil
   103  	}
   104  
   105  	end := offsets[indexID+1]
   106  	return source[start:end], nil
   107  }
   108  
   109  func ParseHeader(r io.Reader) (*Header, error) {
   110  	out := &Header{}
   111  
   112  	if err := binary.Read(r, binary.LittleEndian, &out.Level); err != nil {
   113  		return nil, err
   114  	}
   115  
   116  	if err := binary.Read(r, binary.LittleEndian, &out.Version); err != nil {
   117  		return nil, err
   118  	}
   119  
   120  	if err := binary.Read(r, binary.LittleEndian, &out.SecondaryIndices); err != nil {
   121  		return nil, err
   122  	}
   123  
   124  	if out.Version != 0 {
   125  		return nil, fmt.Errorf("unsupported version %d", out.Version)
   126  	}
   127  
   128  	if err := binary.Read(r, binary.LittleEndian, &out.Strategy); err != nil {
   129  		return nil, err
   130  	}
   131  
   132  	if err := binary.Read(r, binary.LittleEndian, &out.IndexStart); err != nil {
   133  		return nil, err
   134  	}
   135  
   136  	return out, nil
   137  }