github.com/NVIDIA/aistore@v1.3.23-0.20240517131212-7df6609be51d/ext/etl/dp.go (about)

     1  // Package etl provides utilities to initialize and use transformation pods.
     2  /*
     3   * Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.
     4   */
     5  package etl
     6  
     7  import (
     8  	"time"
     9  
    10  	"github.com/NVIDIA/aistore/api/apc"
    11  	"github.com/NVIDIA/aistore/cmn"
    12  	"github.com/NVIDIA/aistore/cmn/cos"
    13  	"github.com/NVIDIA/aistore/cmn/debug"
    14  	"github.com/NVIDIA/aistore/cmn/nlog"
    15  	"github.com/NVIDIA/aistore/core"
    16  )
    17  
    18  // NOTE: compare with core/ldp.go
    19  
    20  type (
    21  	OfflineDP struct {
    22  		comm           Communicator
    23  		tcbmsg         *apc.TCBMsg
    24  		config         *cmn.Config
    25  		requestTimeout time.Duration
    26  	}
    27  )
    28  
    29  // interface guard
    30  var _ core.DP = (*OfflineDP)(nil)
    31  
    32  func NewOfflineDP(msg *apc.TCBMsg, config *cmn.Config) (*OfflineDP, error) {
    33  	comm, err := GetCommunicator(msg.Transform.Name)
    34  	if err != nil {
    35  		return nil, err
    36  	}
    37  	pr := &OfflineDP{comm: comm, tcbmsg: msg, config: config}
    38  	pr.requestTimeout = time.Duration(msg.Transform.Timeout)
    39  	return pr, nil
    40  }
    41  
    42  // Returns reader resulting from lom ETL transformation.
    43  // TODO -- FIXME: comm.OfflineTransform to support latestVer and sync
    44  func (dp *OfflineDP) Reader(lom *core.LOM, latestVer, sync bool) (cos.ReadOpenCloser, cos.OAH, error) {
    45  	var (
    46  		r      cos.ReadCloseSizer // note: +sizer
    47  		err    error
    48  		action = "read [" + dp.tcbmsg.Transform.Name + "]-transformed " + lom.Cname()
    49  	)
    50  	debug.Assert(!latestVer && !sync, "NIY") // TODO -- FIXME
    51  	call := func() (int, error) {
    52  		r, err = dp.comm.OfflineTransform(lom.Bck(), lom.ObjName, dp.requestTimeout)
    53  		return 0, err
    54  	}
    55  	// TODO: Check if ETL pod is healthy and wait some more if not (yet).
    56  	err = cmn.NetworkCallWithRetry(&cmn.RetryArgs{
    57  		Call:      call,
    58  		Action:    action,
    59  		SoftErr:   5,
    60  		HardErr:   2,
    61  		Sleep:     50 * time.Millisecond,
    62  		BackOff:   true,
    63  		Verbosity: cmn.RetryLogQuiet,
    64  	})
    65  	if cmn.Rom.FastV(5, cos.SmoduleETL) {
    66  		nlog.Infoln(action, err)
    67  	}
    68  	if err != nil {
    69  		return nil, nil, err
    70  	}
    71  	lom.SetAtimeUnix(time.Now().UnixNano())
    72  	oah := &cmn.ObjAttrs{
    73  		Size:  r.Size(),
    74  		Ver:   "",            // transformed object - current version does not apply
    75  		Cksum: cos.NoneCksum, // TODO: checksum
    76  		Atime: lom.AtimeUnix(),
    77  	}
    78  	return cos.NopOpener(r), oah, nil
    79  }