github.com/weaviate/weaviate@v1.24.6/adapters/repos/db/lsmkv/bucket_backup.go (about)

     1  //                           _       _
     2  // __      _____  __ ___   ___  __ _| |_ ___
     3  // \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
     4  //  \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
     5  //   \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
     6  //
     7  //  Copyright © 2016 - 2024 Weaviate B.V. All rights reserved.
     8  //
     9  //  CONTACT: hello@weaviate.io
    10  //
    11  
    12  package lsmkv
    13  
    14  import (
    15  	"context"
    16  	"io/fs"
    17  	"path"
    18  	"path/filepath"
    19  
    20  	"github.com/pkg/errors"
    21  	"github.com/weaviate/weaviate/entities/storagestate"
    22  )
    23  
    24  // FlushMemtable flushes any active memtable and returns only once the memtable
    25  // has been fully flushed and a stable state on disk has been reached.
    26  //
    27  // This is a preparatory stage for creating backups.
    28  //
    29  // Method should be run only if flushCycle is not running
    30  // (was not started, is stopped, or noop impl is provided)
    31  func (b *Bucket) FlushMemtable() error {
    32  	if b.isReadOnly() {
    33  		return errors.Wrap(storagestate.ErrStatusReadOnly, "flush memtable")
    34  	}
    35  
    36  	// this lock does not currently _need_ to be
    37  	// obtained, as the only other place that
    38  	// grabs this lock is the flush cycle, which
    39  	// has just been stopped above.
    40  	//
    41  	// that being said, we will lock here anyway
    42  	// as flushLock may be added elsewhere in the
    43  	// future
    44  	b.flushLock.Lock()
    45  	if b.active == nil && b.flushing == nil {
    46  		b.flushLock.Unlock()
    47  		return nil
    48  	}
    49  	b.flushLock.Unlock()
    50  
    51  	stat, err := b.active.commitlog.file.Stat()
    52  	if err != nil {
    53  		b.logger.WithField("action", "lsm_wal_stat").
    54  			WithField("path", b.dir).
    55  			WithError(err).
    56  			Fatal("bucket backup memtable flush failed")
    57  	}
    58  
    59  	// attempting a flush&switch on when the active memtable
    60  	// or WAL is empty results in a corrupted backup attempt
    61  	if b.active.Size() > 0 || stat.Size() > 0 {
    62  		if err := b.FlushAndSwitch(); err != nil {
    63  			return err
    64  		}
    65  	}
    66  	return nil
    67  }
    68  
    69  // ListFiles lists all files that currently exist in the Bucket. The files are only
    70  // in a stable state if the memtable is empty, and if compactions are paused. If one
    71  // of those conditions is not given, it errors
    72  func (b *Bucket) ListFiles(ctx context.Context, basePath string) ([]string, error) {
    73  	var (
    74  		bucketRoot = b.disk.dir
    75  		files      []string
    76  	)
    77  
    78  	err := filepath.WalkDir(bucketRoot, func(currPath string, d fs.DirEntry, err error) error {
    79  		if d.IsDir() {
    80  			return nil
    81  		}
    82  		// ignore .wal files because they are not immutable
    83  		if filepath.Ext(currPath) == ".wal" {
    84  			return nil
    85  		}
    86  		files = append(files, path.Join(basePath, path.Base(currPath)))
    87  		return nil
    88  	})
    89  	if err != nil {
    90  		return nil, errors.Errorf("failed to list files for bucket: %s", err)
    91  	}
    92  
    93  	return files, nil
    94  }