github.com/NVIDIA/aistore@v1.3.23-0.20240517131212-7df6609be51d/xact/xs/nextpage.go (about)

     1  // Package xs is a collection of eXtended actions (xactions), including multi-object
     2  // operations, list-objects, (cluster) rebalance and (target) resilver, ETL, and more.
     3  /*
     4   * Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.
     5   */
     6  package xs
     7  
     8  // core next-page and next-remote-page methods for object listing
     9  
    10  import (
    11  	"github.com/NVIDIA/aistore/api/apc"
    12  	"github.com/NVIDIA/aistore/cmn"
    13  	"github.com/NVIDIA/aistore/cmn/debug"
    14  	"github.com/NVIDIA/aistore/core"
    15  	"github.com/NVIDIA/aistore/core/meta"
    16  	"github.com/NVIDIA/aistore/fs"
    17  )
    18  
    19  type npgCtx struct {
    20  	bck  *meta.Bck
    21  	wi   walkInfo
    22  	page cmn.LsoRes
    23  	ctx  *core.LsoInvCtx
    24  	idx  int
    25  }
    26  
    27  func newNpgCtx(bck *meta.Bck, msg *apc.LsoMsg, cb lomVisitedCb, ctx *core.LsoInvCtx) (npg *npgCtx) {
    28  	npg = &npgCtx{
    29  		bck: bck,
    30  		wi: walkInfo{
    31  			msg:          msg.Clone(),
    32  			lomVisitedCb: cb,
    33  			wanted:       wanted(msg),
    34  			smap:         core.T.Sowner().Get(),
    35  		},
    36  		ctx: ctx,
    37  	}
    38  	return
    39  }
    40  
    41  // limited usage: bucket summary, multi-obj
    42  func (npg *npgCtx) nextPageA() error {
    43  	npg.page.UUID = npg.wi.msg.UUID
    44  	npg.idx = 0
    45  	opts := &fs.WalkBckOpts{
    46  		WalkOpts: fs.WalkOpts{CTs: []string{fs.ObjectType}, Callback: npg.cb, Sorted: true},
    47  	}
    48  	opts.WalkOpts.Bck.Copy(npg.bck.Bucket())
    49  	opts.ValidateCb = func(fqn string, de fs.DirEntry) error {
    50  		if de.IsDir() {
    51  			return npg.wi.processDir(fqn)
    52  		}
    53  		return nil
    54  	}
    55  	err := fs.WalkBck(opts)
    56  	if err != nil {
    57  		freeLsoEntries(npg.page.Entries)
    58  	} else {
    59  		npg.page.Entries = npg.page.Entries[:npg.idx]
    60  	}
    61  	return err
    62  }
    63  
    64  func (npg *npgCtx) cb(fqn string, de fs.DirEntry) error {
    65  	entry, err := npg.wi.callback(fqn, de)
    66  	if entry == nil && err == nil {
    67  		return nil
    68  	}
    69  	if err != nil {
    70  		return cmn.NewErrAborted(core.T.String()+" ResultSetXact", "query", err)
    71  	}
    72  	if npg.idx < len(npg.page.Entries) {
    73  		*npg.page.Entries[npg.idx] = *entry
    74  	} else {
    75  		debug.Assert(npg.idx == len(npg.page.Entries))
    76  		npg.page.Entries = append(npg.page.Entries, entry)
    77  	}
    78  	npg.idx++
    79  	return nil
    80  }
    81  
    82  // Returns the next page from the remote bucket's "list-objects" result set.
    83  func (npg *npgCtx) nextPageR(nentries cmn.LsoEntries, inclStatusLocalMD bool) (lst *cmn.LsoRes, err error) {
    84  	debug.Assert(!npg.wi.msg.IsFlagSet(apc.LsObjCached))
    85  	lst = &cmn.LsoRes{Entries: nentries}
    86  	if npg.ctx != nil {
    87  		if npg.ctx.Lom == nil {
    88  			_, err = core.T.Backend(npg.bck).GetBucketInv(npg.bck, npg.ctx)
    89  		}
    90  		if err == nil {
    91  			err = core.T.Backend(npg.bck).ListObjectsInv(npg.bck, npg.wi.msg, lst, npg.ctx)
    92  		}
    93  	} else {
    94  		_, err = core.T.Backend(npg.bck).ListObjects(npg.bck, npg.wi.msg, lst)
    95  	}
    96  	if err != nil {
    97  		freeLsoEntries(nentries)
    98  		return nil, err
    99  	}
   100  	debug.Assert(lst.UUID == "" || lst.UUID == npg.wi.msg.UUID)
   101  	lst.UUID = npg.wi.msg.UUID
   102  
   103  	if inclStatusLocalMD {
   104  		err = npg.populate(lst)
   105  	}
   106  	return lst, err
   107  }
   108  
   109  func (npg *npgCtx) populate(lst *cmn.LsoRes) error {
   110  	post := npg.wi.lomVisitedCb
   111  	for _, obj := range lst.Entries {
   112  		if obj.IsDir() {
   113  			// collecting virtual dir-s when apc.LsNoRecursion is on - skipping here
   114  			continue
   115  		}
   116  		si, err := npg.wi.smap.HrwName2T(npg.bck.MakeUname(obj.Name))
   117  		if err != nil {
   118  			return err
   119  		}
   120  		if si.ID() != core.T.SID() {
   121  			continue
   122  		}
   123  		lom := core.AllocLOM(obj.Name)
   124  		if err := lom.InitBck(npg.bck.Bucket()); err != nil {
   125  			core.FreeLOM(lom)
   126  			if cmn.IsErrBucketNought(err) {
   127  				return err
   128  			}
   129  			continue
   130  		}
   131  		if err := lom.Load(true /* cache it*/, false /*locked*/); err != nil {
   132  			core.FreeLOM(lom)
   133  			continue
   134  		}
   135  
   136  		npg.wi.setWanted(obj, lom)
   137  		obj.SetPresent()
   138  
   139  		if post != nil {
   140  			post(lom)
   141  		}
   142  		core.FreeLOM(lom)
   143  	}
   144  	return nil
   145  }