github.com/NVIDIA/aistore@v1.3.23-0.20240517131212-7df6609be51d/mirror/makencopies.go (about)

     1  // Package mirror provides local mirroring and replica management
     2  /*
     3   * Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.
     4   */
     5  package mirror
     6  
     7  import (
     8  	"fmt"
     9  	"sync"
    10  
    11  	"github.com/NVIDIA/aistore/api/apc"
    12  	"github.com/NVIDIA/aistore/cmn"
    13  	"github.com/NVIDIA/aistore/cmn/cos"
    14  	"github.com/NVIDIA/aistore/cmn/debug"
    15  	"github.com/NVIDIA/aistore/cmn/nlog"
    16  	"github.com/NVIDIA/aistore/core"
    17  	"github.com/NVIDIA/aistore/core/meta"
    18  	"github.com/NVIDIA/aistore/fs"
    19  	"github.com/NVIDIA/aistore/fs/mpather"
    20  	"github.com/NVIDIA/aistore/memsys"
    21  	"github.com/NVIDIA/aistore/xact"
    22  	"github.com/NVIDIA/aistore/xact/xreg"
    23  )
    24  
    25  type (
    26  	mncFactory struct {
    27  		xreg.RenewBase
    28  		xctn *mncXact
    29  		args xreg.MNCArgs
    30  	}
    31  
    32  	// mncXact runs in a background, traverses all local mountpaths, and makes sure
    33  	// the bucket is N-way replicated (where N >= 1).
    34  	mncXact struct {
    35  		p *mncFactory
    36  		xact.BckJog
    37  		_nam, _str string
    38  	}
    39  )
    40  
    41  // interface guard
    42  var (
    43  	_ core.Xact      = (*mncXact)(nil)
    44  	_ xreg.Renewable = (*mncFactory)(nil)
    45  )
    46  
    47  ////////////////
    48  // mncFactory //
    49  ////////////////
    50  
    51  func (*mncFactory) New(args xreg.Args, bck *meta.Bck) xreg.Renewable {
    52  	p := &mncFactory{RenewBase: xreg.RenewBase{Args: args, Bck: bck}, args: *args.Custom.(*xreg.MNCArgs)}
    53  	return p
    54  }
    55  
    56  func (p *mncFactory) Start() error {
    57  	slab, err := core.T.PageMM().GetSlab(memsys.MaxPageSlabSize)
    58  	debug.AssertNoErr(err)
    59  	p.xctn = newMNC(p, slab)
    60  	return nil
    61  }
    62  
    63  func (*mncFactory) Kind() string     { return apc.ActMakeNCopies }
    64  func (p *mncFactory) Get() core.Xact { return p.xctn }
    65  
    66  func (p *mncFactory) WhenPrevIsRunning(prevEntry xreg.Renewable) (wpr xreg.WPR, err error) {
    67  	err = fmt.Errorf("%s is currently running, cannot start a new %q", prevEntry.Get(), p.Str(p.Kind()))
    68  	return
    69  }
    70  
    71  /////////////
    72  // mncXact //
    73  /////////////
    74  
    75  // NOTE: always throttling
    76  func newMNC(p *mncFactory, slab *memsys.Slab) (r *mncXact) {
    77  	debug.Assert(p.args.Tag != "" && p.args.Copies > 0)
    78  	r = &mncXact{p: p}
    79  	mpopts := &mpather.JgroupOpts{
    80  		CTs:      []string{fs.ObjectType},
    81  		VisitObj: r.visitObj,
    82  		Slab:     slab,
    83  		DoLoad:   mpather.LoadUnsafe,
    84  		Throttle: true,
    85  	}
    86  	mpopts.Bck.Copy(p.Bck.Bucket())
    87  	r.BckJog.Init(p.UUID(), apc.ActMakeNCopies, p.Bck, mpopts, cmn.GCO.Get())
    88  
    89  	// name
    90  	s := fmt.Sprintf("-%s-copies-%d", r.p.args.Tag, r.p.args.Copies)
    91  	r._nam = r.Base.Name() + s
    92  	r._str = r.Base.String() + s
    93  	return r
    94  }
    95  
    96  func (r *mncXact) Run(wg *sync.WaitGroup) {
    97  	wg.Done()
    98  	tname := core.T.String()
    99  	if err := fs.ValidateNCopies(tname, r.p.args.Copies); err != nil {
   100  		r.AddErr(err)
   101  		r.Finish()
   102  		return
   103  	}
   104  	r.BckJog.Run()
   105  	nlog.Infoln(r.Name())
   106  	err := r.BckJog.Wait()
   107  	if err != nil {
   108  		r.AddErr(err)
   109  	}
   110  	r.Finish()
   111  }
   112  
   113  func (r *mncXact) visitObj(lom *core.LOM, buf []byte) (err error) {
   114  	var (
   115  		size   int64
   116  		n      = lom.NumCopies()
   117  		copies = r.p.args.Copies
   118  	)
   119  	switch {
   120  	case n == copies:
   121  		return nil
   122  	case n > copies:
   123  		lom.Lock(true)
   124  		size, err = delCopies(lom, copies)
   125  		lom.Unlock(true)
   126  	default:
   127  		lom.Lock(true)
   128  		size, err = addCopies(lom, copies, buf)
   129  		lom.Unlock(true)
   130  	}
   131  
   132  	if err != nil {
   133  		if cos.IsNotExist(err, 0) {
   134  			return nil
   135  		}
   136  		if cos.IsErrOOS(err) {
   137  			r.Abort(err)
   138  		} else {
   139  			cs := fs.Cap()
   140  			if errCap := cs.Err(); errCap != nil {
   141  				r.Abort(fmt.Errorf("errors: [%w] and [%w]", err, errCap))
   142  			} else {
   143  				r.AddErr(err)
   144  			}
   145  		}
   146  		return
   147  	}
   148  
   149  	if cmn.Rom.FastV(5, cos.SmoduleMirror) {
   150  		nlog.Infof("%s: %s, copies %d=>%d, size=%d", r.Base.Name(), lom.Cname(), n, copies, size)
   151  	}
   152  	r.ObjsAdd(1, size)
   153  	if cnt := r.Objs(); cnt%128 == 0 { // TODO: configurable
   154  		cs := fs.Cap()
   155  		if errCap := cs.Err(); errCap != nil {
   156  			r.Abort(err)
   157  		}
   158  	}
   159  	return
   160  }
   161  
   162  func (r *mncXact) String() string { return r._str }
   163  func (r *mncXact) Name() string   { return r._nam }
   164  
   165  func (r *mncXact) Snap() (snap *core.Snap) {
   166  	snap = &core.Snap{}
   167  	r.ToSnap(snap)
   168  
   169  	snap.IdleX = r.IsIdle()
   170  	return
   171  }