github.com/onflow/flow-go@v0.35.7-crescendo-preview.23-atree-inlining/engine/execution/ingestion/uploader/uploader.go (about)

     1  package uploader
     2  
     3  import (
     4  	"context"
     5  	"time"
     6  
     7  	"github.com/rs/zerolog"
     8  
     9  	"github.com/onflow/flow-go/engine"
    10  	"github.com/onflow/flow-go/engine/execution"
    11  	"github.com/onflow/flow-go/module"
    12  	"github.com/onflow/flow-go/utils/logging"
    13  
    14  	"github.com/sethvargo/go-retry"
    15  )
    16  
    17  type Uploader interface {
    18  	Upload(computationResult *execution.ComputationResult) error
    19  }
    20  
    21  // OnCompleteFunc is the type of function being called at upload completion.
    22  type OnCompleteFunc func(*execution.ComputationResult, error)
    23  
    24  func NewAsyncUploader(uploader Uploader,
    25  	retryInitialTimeout time.Duration,
    26  	maxRetryNumber uint64,
    27  	log zerolog.Logger,
    28  	metrics module.ExecutionMetrics) *AsyncUploader {
    29  	return &AsyncUploader{
    30  		unit:                engine.NewUnit(),
    31  		uploader:            uploader,
    32  		log:                 log.With().Str("component", "block_data_uploader").Logger(),
    33  		metrics:             metrics,
    34  		retryInitialTimeout: retryInitialTimeout,
    35  		maxRetryNumber:      maxRetryNumber,
    36  	}
    37  }
    38  
    39  // AsyncUploader wraps up another Uploader instance and make its upload asynchronous
    40  type AsyncUploader struct {
    41  	module.ReadyDoneAware
    42  	unit                *engine.Unit
    43  	uploader            Uploader
    44  	log                 zerolog.Logger
    45  	metrics             module.ExecutionMetrics
    46  	retryInitialTimeout time.Duration
    47  	maxRetryNumber      uint64
    48  	onComplete          OnCompleteFunc // callback function called after Upload is completed
    49  }
    50  
    51  func (a *AsyncUploader) Ready() <-chan struct{} {
    52  	return a.unit.Ready()
    53  }
    54  
    55  func (a *AsyncUploader) Done() <-chan struct{} {
    56  	return a.unit.Done()
    57  }
    58  
    59  func (a *AsyncUploader) SetOnCompleteCallback(onComplete OnCompleteFunc) {
    60  	a.onComplete = onComplete
    61  }
    62  
    63  func (a *AsyncUploader) Upload(computationResult *execution.ComputationResult) error {
    64  
    65  	backoff := retry.NewFibonacci(a.retryInitialTimeout)
    66  	backoff = retry.WithMaxRetries(a.maxRetryNumber, backoff)
    67  
    68  	a.unit.Launch(func() {
    69  		a.metrics.ExecutionBlockDataUploadStarted()
    70  		start := time.Now()
    71  
    72  		a.log.Debug().Msgf("computation result of block %s is being uploaded",
    73  			computationResult.ExecutableBlock.ID().String())
    74  
    75  		err := retry.Do(a.unit.Ctx(), backoff, func(ctx context.Context) error {
    76  			err := a.uploader.Upload(computationResult)
    77  			if err != nil {
    78  				a.log.Warn().Err(err).Msg("error while uploading block data, retrying")
    79  			}
    80  			return retry.RetryableError(err)
    81  		})
    82  
    83  		if err != nil {
    84  			a.log.Error().Err(err).
    85  				Hex("block_id", logging.Entity(computationResult.ExecutableBlock)).
    86  				Msg("failed to upload block data")
    87  		} else {
    88  			a.log.Debug().Msgf("computation result of block %s was successfully uploaded",
    89  				computationResult.ExecutableBlock.ID().String())
    90  		}
    91  
    92  		a.metrics.ExecutionBlockDataUploadFinished(time.Since(start))
    93  
    94  		if a.onComplete != nil {
    95  			a.onComplete(computationResult, err)
    96  		}
    97  	})
    98  	return nil
    99  }