github.com/NVIDIA/aistore@v1.3.23-0.20240517131212-7df6609be51d/fs/persistent_md.go (about)

     1  // Package fs provides mountpath and FQN abstractions and methods to resolve/map stored content
     2  /*
     3   * Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.
     4   */
     5  package fs
     6  
     7  import (
     8  	"fmt"
     9  	"os"
    10  	"path/filepath"
    11  
    12  	"github.com/NVIDIA/aistore/cmn"
    13  	"github.com/NVIDIA/aistore/cmn/cos"
    14  	"github.com/NVIDIA/aistore/cmn/debug"
    15  	"github.com/NVIDIA/aistore/cmn/fname"
    16  	"github.com/NVIDIA/aistore/cmn/jsp"
    17  	"github.com/NVIDIA/aistore/cmn/nlog"
    18  	"github.com/NVIDIA/aistore/memsys"
    19  )
    20  
    21  const numMarkers = 1
    22  
    23  // List of AIS metadata files and directories (basenames only)
    24  var mdFilesDirs = []string{
    25  	fname.MarkersDir,
    26  
    27  	fname.Bmd,
    28  	fname.BmdPrevious,
    29  
    30  	fname.Vmd,
    31  }
    32  
    33  func MarkerExists(marker string) bool {
    34  	markerPath := filepath.Join(fname.MarkersDir, marker)
    35  	return CountPersisted(markerPath) > 0
    36  }
    37  
    38  func PersistMarker(marker string) (fatalErr, writeErr error) {
    39  	var (
    40  		cnt             int
    41  		relname         = filepath.Join(fname.MarkersDir, marker)
    42  		availableMpaths = GetAvail()
    43  	)
    44  	if len(availableMpaths) == 0 {
    45  		fatalErr = cmn.ErrNoMountpaths
    46  		return
    47  	}
    48  	for _, mi := range availableMpaths {
    49  		fpath := filepath.Join(mi.Path, relname)
    50  		if err := cos.Stat(fpath); err == nil {
    51  			cnt++
    52  			if cnt > numMarkers {
    53  				if err := cos.RemoveFile(fpath); err != nil {
    54  					if writeErr == nil {
    55  						writeErr = err
    56  					}
    57  					nlog.Errorf("Failed to cleanup %q marker: %v", fpath, err)
    58  				} else {
    59  					cnt--
    60  				}
    61  			}
    62  		} else if cnt < numMarkers {
    63  			if file, err := cos.CreateFile(fpath); err == nil {
    64  				writeErr = err
    65  				file.Close()
    66  				cnt++
    67  			} else {
    68  				nlog.Errorf("Failed to create %q marker: %v", fpath, err)
    69  			}
    70  		}
    71  	}
    72  	if cnt == 0 {
    73  		fatalErr = fmt.Errorf("failed to persist %q marker (%d)", marker, len(availableMpaths))
    74  	}
    75  	return
    76  }
    77  
    78  func RemoveMarker(marker string) (err error) {
    79  	var (
    80  		availableMpaths = GetAvail()
    81  		relname         = filepath.Join(fname.MarkersDir, marker)
    82  	)
    83  	for _, mi := range availableMpaths {
    84  		if er1 := cos.RemoveFile(filepath.Join(mi.Path, relname)); er1 != nil {
    85  			nlog.Errorf("Failed to remove %q marker from %q: %v", relname, mi.Path, er1)
    86  			err = er1
    87  		}
    88  	}
    89  	return
    90  }
    91  
    92  // PersistOnMpaths persists `what` on mountpaths under "mountpath.Path/path" filename.
    93  // It does it on maximum `atMost` mountPaths. If `atMost == 0`, it does it on every mountpath.
    94  // If `backupPath != ""`, it removes files from `backupPath` and moves files from `path` to `backupPath`.
    95  // Returns how many times it has successfully stored a file.
    96  func PersistOnMpaths(fname, backupName string, meta jsp.Opts, atMost int, b []byte, sgl *memsys.SGL) (cnt, availCnt int) {
    97  	var (
    98  		wto             cos.WriterTo2
    99  		bcnt            int
   100  		availableMpaths = GetAvail()
   101  	)
   102  	availCnt = len(availableMpaths)
   103  	debug.Assert(atMost > 0)
   104  	if atMost > availCnt {
   105  		atMost = availCnt
   106  	}
   107  	for _, mi := range availableMpaths {
   108  		if backupName != "" {
   109  			bcnt = mi.backupAtmost(fname, backupName, bcnt, atMost)
   110  		}
   111  		fpath := filepath.Join(mi.Path, fname)
   112  		os.Remove(fpath)
   113  		if cnt >= atMost {
   114  			continue
   115  		}
   116  		if b != nil {
   117  			wto = cos.NewBuffer(b)
   118  		} else if sgl != nil {
   119  			wto = sgl // not reopening - see sgl.WriteTo()
   120  		}
   121  		if err := jsp.SaveMeta(fpath, meta, wto); err != nil {
   122  			nlog.Errorf("Failed to persist %q on %q, err: %v", fname, mi, err)
   123  		} else {
   124  			cnt++
   125  		}
   126  	}
   127  	debug.Func(func() {
   128  		expected := min(atMost, availCnt)
   129  		debug.Assertf(cnt == expected, "expected %q to be persisted on %d mountpaths got %d instead",
   130  			fname, expected, cnt)
   131  	})
   132  	return
   133  }
   134  
   135  func CountPersisted(fname string) (cnt int) {
   136  	available := GetAvail()
   137  	for mpath := range available {
   138  		fpath := filepath.Join(mpath, fname)
   139  		if err := cos.Stat(fpath); err == nil {
   140  			cnt++
   141  		}
   142  	}
   143  	return
   144  }