storj.io/uplink@v1.13.0/private/metaclient/retry.go (about)

     1  // Copyright (C) 2021 Storj Labs, Inc.
     2  // See LICENSE for copying information.
     3  
     4  package metaclient
     5  
     6  import (
     7  	"context"
     8  	"errors"
     9  	"io"
    10  	"net"
    11  	"syscall"
    12  	"time"
    13  
    14  	"storj.io/common/sync2"
    15  )
    16  
    17  // ExponentialBackoff keeps track of how long we should sleep between
    18  // failing attempts.
    19  type ExponentialBackoff struct {
    20  	delay time.Duration
    21  	Max   time.Duration
    22  	Min   time.Duration
    23  }
    24  
    25  func (e *ExponentialBackoff) init() {
    26  	if e.Max == 0 {
    27  		// maximum delay - pulled from net/http.Server.Serve
    28  		e.Max = time.Second
    29  	}
    30  	if e.Min == 0 {
    31  		// minimum delay - pulled from net/http.Server.Serve
    32  		e.Min = 5 * time.Millisecond
    33  	}
    34  }
    35  
    36  // Wait should be called when there is a failure. Each time it is called
    37  // it will sleep an exponentially longer time, up to a max.
    38  func (e *ExponentialBackoff) Wait(ctx context.Context) bool {
    39  	e.init()
    40  	if e.delay == 0 {
    41  		e.delay = e.Min
    42  	} else {
    43  		e.delay *= 2
    44  	}
    45  	if e.delay > e.Max {
    46  		e.delay = e.Max
    47  	}
    48  	return sync2.Sleep(ctx, e.delay)
    49  }
    50  
    51  // Maxed returns true if the wait time has maxed out.
    52  func (e *ExponentialBackoff) Maxed() bool {
    53  	e.init()
    54  	return e.delay == e.Max
    55  }
    56  
    57  // WithRetry attempts to retry a function with exponential backoff. If the retry has occurred
    58  // enough times that the delay is maxed out and the function still returns an error, the error
    59  // is returned.
    60  func WithRetry(ctx context.Context, fn func(ctx context.Context) error) (err error) {
    61  	delay := ExponentialBackoff{
    62  		Min: 100 * time.Millisecond,
    63  		Max: 3 * time.Second,
    64  	}
    65  
    66  	for {
    67  		if err := ctx.Err(); err != nil {
    68  			return err
    69  		}
    70  
    71  		err = fn(ctx)
    72  		if err != nil && needsRetry(err) {
    73  			if !delay.Maxed() {
    74  				if !delay.Wait(ctx) {
    75  					return ctx.Err()
    76  				}
    77  				continue
    78  			}
    79  		}
    80  		return err
    81  	}
    82  }
    83  
    84  func needsRetry(err error) bool {
    85  	if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
    86  		mon.Event("uplink_error_eof")
    87  		// Currently we don't retry with EOF because it's unclear if
    88  		// a query succeeded or failed.
    89  		return false
    90  	}
    91  
    92  	if errors.Is(err, syscall.ECONNRESET) {
    93  		mon.Event("uplink_error_conn_reset_needed_retry")
    94  		return true
    95  	}
    96  	if errors.Is(err, syscall.ECONNREFUSED) {
    97  		mon.Event("uplink_error_conn_refused_needed_retry")
    98  		return true
    99  	}
   100  	var netErr net.Error
   101  	if errors.As(err, &netErr) {
   102  		mon.Event("uplink_net_error_needed_retry")
   103  		return true
   104  	}
   105  
   106  	return false
   107  }