github.com/NVIDIA/aistore@v1.3.23-0.20240517131212-7df6609be51d/xact/xs/prune.go (about)

     1  // Package xs is a collection of eXtended actions (xactions), including multi-object
     2  // operations, list-objects, (cluster) rebalance and (target) resilver, ETL, and more.
     3  /*
     4   * Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
     5   */
     6  package xs
     7  
     8  import (
     9  	"context"
    10  	"fmt"
    11  	"net/http"
    12  	"time"
    13  
    14  	"github.com/NVIDIA/aistore/cmn"
    15  	"github.com/NVIDIA/aistore/cmn/cos"
    16  	"github.com/NVIDIA/aistore/cmn/debug"
    17  	"github.com/NVIDIA/aistore/cmn/nlog"
    18  	"github.com/NVIDIA/aistore/cmn/prob"
    19  	"github.com/NVIDIA/aistore/core"
    20  	"github.com/NVIDIA/aistore/core/meta"
    21  	"github.com/NVIDIA/aistore/fs"
    22  	"github.com/NVIDIA/aistore/fs/mpather"
    23  )
    24  
    25  // When synchronizing source => destination:
    26  // remove destination objects that are not present at the source (any longer)
    27  // Limitations (TODO):
    28  // - not supporting apc.ListRange
    29  // - use probabilistic filtering to skip received and locally copied obj-s (see `reb.FilterAdd` et al)
    30  
    31  type prune struct {
    32  	parent         core.Xact
    33  	bckFrom, bckTo *meta.Bck
    34  	smap           *meta.Smap
    35  	prefix         string
    36  	// run
    37  	joggers *mpather.Jgroup
    38  	filter  *prob.Filter
    39  	same    bool
    40  }
    41  
    42  func (rp *prune) init(config *cmn.Config) {
    43  	debug.Assert(rp.bckFrom.IsAIS() || rp.bckFrom.HasVersioningMD(), rp.bckFrom.String())
    44  	rmopts := &mpather.JgroupOpts{
    45  		CTs:      []string{fs.ObjectType},
    46  		VisitObj: rp.do,
    47  		Prefix:   rp.prefix,
    48  		Parallel: 1, // TODO: tune-up
    49  		// DoLoad:  noLoad
    50  	}
    51  	rmopts.Bck.Copy(rp.bckTo.Bucket())
    52  	rp.joggers = mpather.NewJoggerGroup(rmopts, config, "")
    53  	rp.filter = prob.NewDefaultFilter()
    54  	rp.same = rp.bckTo.Equal(rp.bckFrom, true, true)
    55  }
    56  
    57  func (rp *prune) run() {
    58  	rp.joggers.Run()
    59  }
    60  
    61  func (rp *prune) wait() {
    62  	if err := rp.parent.AbortErr(); err != nil {
    63  		rp.joggers.Stop()
    64  		return
    65  	}
    66  
    67  	// wait for: joggers || parent-aborted
    68  	ticker := time.NewTicker(cmn.Rom.MaxKeepalive())
    69  	rp._wait(ticker)
    70  
    71  	// cleanup
    72  	ticker.Stop()
    73  	rp.filter.Reset()
    74  }
    75  
    76  func (rp *prune) _wait(ticker *time.Ticker) {
    77  	for {
    78  		select {
    79  		case <-ticker.C:
    80  			if rp.parent.IsAborted() {
    81  				rp.joggers.Stop()
    82  				return
    83  			}
    84  		case <-rp.joggers.ListenFinished():
    85  			rp.joggers.Stop()
    86  			return
    87  		}
    88  	}
    89  }
    90  
    91  func (rp *prune) do(dst *core.LOM, _ []byte) error {
    92  	debug.Func(func() {
    93  		_, local, err := dst.HrwTarget(rp.smap)
    94  		debug.Assertf(local, "local %t, err: %v", local, err)
    95  	})
    96  	// construct src lom
    97  	var src *core.LOM
    98  	if rp.same {
    99  		src = dst
   100  	} else {
   101  		src = core.AllocLOM(dst.ObjName)
   102  		defer core.FreeLOM(src)
   103  		if src.InitBck(rp.bckFrom.Bucket()) != nil {
   104  			return nil
   105  		}
   106  	}
   107  
   108  	// skip objects already copied by rp.parent (compare w/ reb)
   109  	uname := cos.UnsafeB(src.Uname())
   110  	if rp.filter != nil && rp.filter.Lookup(uname) { // TODO -- FIXME: rm filter nil check once x-tco supports prob. filtering
   111  		rp.filter.Delete(uname)
   112  		return nil
   113  	}
   114  
   115  	// check whether src lom exists
   116  	var (
   117  		err   error
   118  		ecode int
   119  	)
   120  	if src.Bck().IsAIS() {
   121  		tsi, errV := rp.smap.HrwHash2T(src.Digest())
   122  		if errV != nil {
   123  			return fmt.Errorf("prune %s: fatal err: %w", rp.parent.Name(), errV)
   124  		}
   125  		if tsi.ID() == core.T.SID() {
   126  			err = src.Load(false, false)
   127  		} else {
   128  			if present := core.T.HeadObjT2T(src, tsi); !present {
   129  				ecode = http.StatusNotFound
   130  			}
   131  		}
   132  	} else {
   133  		_, ecode, err = core.T.Backend(src.Bck()).HeadObj(context.Background(), src, nil /*origReq*/)
   134  	}
   135  
   136  	if (err == nil && ecode == 0) || !cos.IsNotExist(err, ecode) /*not complaining*/ {
   137  		return nil
   138  	}
   139  
   140  	// source does not exist: try to remove the destination (NOTE best effort)
   141  	if !dst.TryLock(true) {
   142  		return nil
   143  	}
   144  	err = dst.Load(false, true)
   145  	if err == nil {
   146  		err = dst.Remove()
   147  	}
   148  	dst.Unlock(true)
   149  
   150  	if err == nil {
   151  		if cmn.Rom.FastV(5, cos.SmoduleXs) {
   152  			nlog.Infoln(rp.parent.Name(), dst.Cname())
   153  		}
   154  	} else if !cmn.IsErrObjNought(err) && !cmn.IsErrBucketNought(err) {
   155  		rp.parent.AddErr(err, 4, cos.SmoduleXs)
   156  	}
   157  	return nil
   158  }