github.com/weaviate/weaviate@v1.24.6/adapters/repos/db/vector/hnsw/condensor_mmap_reader.go (about)

     1  //                           _       _
     2  // __      _____  __ ___   ___  __ _| |_ ___
     3  // \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
     4  //  \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
     5  //   \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
     6  //
     7  //  Copyright © 2016 - 2024 Weaviate B.V. All rights reserved.
     8  //
     9  //  CONTACT: hello@weaviate.io
    10  //
    11  
    12  package hnsw
    13  
    14  import (
    15  	"bufio"
    16  	"os"
    17  
    18  	"github.com/edsrzf/mmap-go"
    19  	"github.com/pkg/errors"
    20  )
    21  
    22  type MmapCondensorReader struct {
    23  	reader *bufio.Reader
    24  	target []byte
    25  }
    26  
    27  func newMmapCondensorReader() *MmapCondensorReader {
    28  	return &MmapCondensorReader{}
    29  }
    30  
    31  func (r *MmapCondensorReader) Do(source *os.File, index mmapIndex, targetName string) error {
    32  	r.reader = bufio.NewReaderSize(source, 1024*1024)
    33  
    34  	scratchFile, err := os.Create(targetName)
    35  	if err != nil {
    36  		return err
    37  	}
    38  
    39  	size := index.Size()
    40  	if err := scratchFile.Truncate(int64(index.Size())); err != nil {
    41  		return errors.Wrap(err, "truncate scratch file to size")
    42  	}
    43  
    44  	mmapSpace, err := mmap.MapRegion(scratchFile, size, mmap.COPY, 0, 0)
    45  	if err != nil {
    46  		return errors.Wrap(err, "mmap scratch file")
    47  	}
    48  
    49  	r.target = mmapSpace
    50  
    51  	if err := r.loop(); err != nil {
    52  		return err
    53  	}
    54  
    55  	if err := mmapSpace.Unmap(); err != nil {
    56  		return errors.Wrap(err, "munmap scratch file")
    57  	}
    58  
    59  	if err := scratchFile.Close(); err != nil {
    60  		return errors.Wrap(err, "close scratch file")
    61  	}
    62  
    63  	return nil
    64  }
    65  
    66  func (r *MmapCondensorReader) loop() error {
    67  	// TODO: iterate through commit log
    68  	// TODO: get offset for specific part
    69  	// TODO: write into target at correct position
    70  	return nil
    71  }