github.com/weaviate/weaviate@v1.24.6/adapters/repos/db/shard_backup.go (about)

     1  //                           _       _
     2  // __      _____  __ ___   ___  __ _| |_ ___
     3  // \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
     4  //  \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
     5  //   \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
     6  //
     7  //  Copyright © 2016 - 2024 Weaviate B.V. All rights reserved.
     8  //
     9  //  CONTACT: hello@weaviate.io
    10  //
    11  
    12  package db
    13  
    14  import (
    15  	"context"
    16  	"fmt"
    17  	"os"
    18  	"path/filepath"
    19  
    20  	enterrors "github.com/weaviate/weaviate/entities/errors"
    21  
    22  	"github.com/weaviate/weaviate/entities/backup"
    23  )
    24  
    25  // BeginBackup stops compaction, and flushing memtable and commit log to begin with the backup
    26  func (s *Shard) BeginBackup(ctx context.Context) (err error) {
    27  	defer func() {
    28  		if err != nil {
    29  			err = fmt.Errorf("pause compaction: %w", err)
    30  			if err2 := s.resumeMaintenanceCycles(ctx); err2 != nil {
    31  				err = fmt.Errorf("%w: resume maintenance: %v", err, err2)
    32  			}
    33  		}
    34  	}()
    35  	if err = s.store.PauseCompaction(ctx); err != nil {
    36  		return fmt.Errorf("pause compaction: %w", err)
    37  	}
    38  	if err = s.store.FlushMemtables(ctx); err != nil {
    39  		return fmt.Errorf("flush memtables: %w", err)
    40  	}
    41  	if err = s.cycleCallbacks.vectorCombinedCallbacksCtrl.Deactivate(ctx); err != nil {
    42  		return fmt.Errorf("pause vector maintenance: %w", err)
    43  	}
    44  	if err = s.cycleCallbacks.geoPropsCombinedCallbacksCtrl.Deactivate(ctx); err != nil {
    45  		return fmt.Errorf("pause geo props maintenance: %w", err)
    46  	}
    47  	if s.hasTargetVectors() {
    48  		for targetVector, vectorIndex := range s.vectorIndexes {
    49  			if err = vectorIndex.SwitchCommitLogs(ctx); err != nil {
    50  				return fmt.Errorf("switch commit logs of vector %q: %w", targetVector, err)
    51  			}
    52  		}
    53  	} else {
    54  		if err = s.vectorIndex.SwitchCommitLogs(ctx); err != nil {
    55  			return fmt.Errorf("switch commit logs: %w", err)
    56  		}
    57  	}
    58  	return nil
    59  }
    60  
    61  // ListBackupFiles lists all files used to backup a shard
    62  func (s *Shard) ListBackupFiles(ctx context.Context, ret *backup.ShardDescriptor) error {
    63  	var err error
    64  	if err := s.readBackupMetadata(ret); err != nil {
    65  		return err
    66  	}
    67  
    68  	if ret.Files, err = s.store.ListFiles(ctx, s.index.Config.RootPath); err != nil {
    69  		return err
    70  	}
    71  
    72  	if s.hasTargetVectors() {
    73  		for targetVector, vectorIndex := range s.vectorIndexes {
    74  			files, err := vectorIndex.ListFiles(ctx, s.index.Config.RootPath)
    75  			if err != nil {
    76  				return fmt.Errorf("list files of vector %q: %w", targetVector, err)
    77  			}
    78  			ret.Files = append(ret.Files, files...)
    79  		}
    80  	} else {
    81  		files, err := s.vectorIndex.ListFiles(ctx, s.index.Config.RootPath)
    82  		if err != nil {
    83  			return err
    84  		}
    85  		ret.Files = append(ret.Files, files...)
    86  	}
    87  
    88  	return nil
    89  }
    90  
    91  func (s *Shard) resumeMaintenanceCycles(ctx context.Context) error {
    92  	g := enterrors.NewErrorGroupWrapper(s.index.logger)
    93  
    94  	g.Go(func() error {
    95  		return s.store.ResumeCompaction(ctx)
    96  	})
    97  	g.Go(func() error {
    98  		return s.cycleCallbacks.vectorCombinedCallbacksCtrl.Activate()
    99  	})
   100  	g.Go(func() error {
   101  		return s.cycleCallbacks.geoPropsCombinedCallbacksCtrl.Activate()
   102  	})
   103  
   104  	if err := g.Wait(); err != nil {
   105  		return fmt.Errorf("failed to resume maintenance cycles for shard '%s': %w", s.name, err)
   106  	}
   107  
   108  	return nil
   109  }
   110  
   111  func (s *Shard) readBackupMetadata(d *backup.ShardDescriptor) (err error) {
   112  	d.Name = s.name
   113  	d.Node = s.nodeName()
   114  	fpath := s.counter.FileName()
   115  	if d.DocIDCounter, err = os.ReadFile(fpath); err != nil {
   116  		return fmt.Errorf("read shard doc-id-counter %s: %w", fpath, err)
   117  	}
   118  	d.DocIDCounterPath, err = filepath.Rel(s.index.Config.RootPath, fpath)
   119  	if err != nil {
   120  		return fmt.Errorf("docid counter path: %w", err)
   121  	}
   122  	fpath = s.GetPropertyLengthTracker().FileName()
   123  	if d.PropLengthTracker, err = os.ReadFile(fpath); err != nil {
   124  		return fmt.Errorf("read shard prop-lengths %s: %w", fpath, err)
   125  	}
   126  	d.PropLengthTrackerPath, err = filepath.Rel(s.index.Config.RootPath, fpath)
   127  	if err != nil {
   128  		return fmt.Errorf("proplength tracker path: %w", err)
   129  	}
   130  	fpath = s.versioner.path
   131  	if d.Version, err = os.ReadFile(fpath); err != nil {
   132  		return fmt.Errorf("read shard version %s: %w", fpath, err)
   133  	}
   134  	d.ShardVersionPath, err = filepath.Rel(s.index.Config.RootPath, fpath)
   135  	if err != nil {
   136  		return fmt.Errorf("shard version path: %w", err)
   137  	}
   138  	return nil
   139  }
   140  
   141  func (s *Shard) nodeName() string {
   142  	node, _ := s.index.getSchema.ShardOwner(
   143  		s.index.Config.ClassName.String(), s.name)
   144  	return node
   145  }