github.com/weaviate/weaviate@v1.24.6/adapters/repos/db/vector/hnsw/backup.go (about)

     1  //                           _       _
     2  // __      _____  __ ___   ___  __ _| |_ ___
     3  // \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
     4  //  \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
     5  //   \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
     6  //
     7  //  Copyright © 2016 - 2024 Weaviate B.V. All rights reserved.
     8  //
     9  //  CONTACT: hello@weaviate.io
    10  //
    11  
    12  package hnsw
    13  
    14  import (
    15  	"context"
    16  	"fmt"
    17  	"io/fs"
    18  	"os"
    19  	"path/filepath"
    20  )
    21  
    22  // SwitchCommitLogs makes sure that the previously writeable commitlog is
    23  // switched to a new one, thus making the existing file read-only.
    24  func (h *hnsw) SwitchCommitLogs(ctx context.Context) error {
    25  	if err := h.commitLog.SwitchCommitLogs(true); err != nil {
    26  		return fmt.Errorf("switch commitlogs: %w", err)
    27  	}
    28  
    29  	return nil
    30  }
    31  
    32  // ListFiles lists all files that are part of the part of the HNSW
    33  // except the last commit-log which is writable. This operation is typically
    34  // called immediately after calling SwitchCommitlogs which means that the
    35  // latest (writeable) log file is typically empty.
    36  // ListFiles errors if maintenance is not paused, as a stable state
    37  // cannot be guaranteed with maintenance going on in the background.
    38  func (h *hnsw) ListFiles(ctx context.Context, basePath string) ([]string, error) {
    39  	var (
    40  		logRoot = filepath.Join(h.commitLog.RootPath(), fmt.Sprintf("%s.hnsw.commitlog.d", h.commitLog.ID()))
    41  		found   = make(map[string]struct{})
    42  		files   []string
    43  	)
    44  
    45  	err := filepath.WalkDir(logRoot, func(pth string, d fs.DirEntry, err error) error {
    46  		if d.IsDir() {
    47  			return nil
    48  		}
    49  
    50  		st, statErr := os.Stat(pth)
    51  		if statErr != nil {
    52  			return statErr
    53  		}
    54  
    55  		// only list non-empty files
    56  		if st.Size() > 0 {
    57  			rel, relErr := filepath.Rel(basePath, pth)
    58  			if relErr != nil {
    59  				return relErr
    60  			}
    61  			found[rel] = struct{}{}
    62  		}
    63  
    64  		return nil
    65  	})
    66  	if err != nil {
    67  		return nil, fmt.Errorf("failed to list files for hnsw commitlog: %w", err)
    68  	}
    69  
    70  	curr, _, err := getCurrentCommitLogFileName(logRoot)
    71  	if err != nil {
    72  		return nil, fmt.Errorf("current commitlog file name: %w", err)
    73  	}
    74  
    75  	// remove active log from list, as
    76  	// it is not part of the backup
    77  	path, err := filepath.Rel(basePath, filepath.Join(logRoot, curr))
    78  	if err != nil {
    79  		return nil, fmt.Errorf("delete active log: %w", err)
    80  	}
    81  	delete(found, path)
    82  
    83  	files, i := make([]string, len(found)), 0
    84  	for file := range found {
    85  		files[i] = file
    86  		i++
    87  	}
    88  
    89  	return files, nil
    90  }