github.com/billybanfield/evergreen@v0.0.0-20170525200750-eeee692790f7/util/retry.go (about) 1 package util 2 3 import ( 4 "math" 5 "math/rand" 6 "time" 7 8 "github.com/jpillora/backoff" 9 "github.com/pkg/errors" 10 ) 11 12 func init() { 13 rand.Seed(time.Now().Unix()) 14 } 15 16 // RetriableError can be returned by any function called with Retry(), 17 // to indicate that it should be retried again after a sleep interval. 18 type RetriableError struct { 19 Failure error 20 } 21 22 func (e RetriableError) Error() string { 23 return e.Failure.Error() 24 } 25 26 // RetriableFunc is any function that takes no parameters and returns only 27 // an error interface. These functions can be used with util.Retry. 28 type RetriableFunc func() error 29 30 func getBackoff(initialSleep time.Duration, numAttempts int) *backoff.Backoff { 31 if initialSleep < 100*time.Millisecond { 32 initialSleep = 100 * time.Millisecond 33 } 34 35 if numAttempts == 0 { 36 numAttempts = 1 37 } 38 39 var factor float64 = 2 40 41 return &backoff.Backoff{ 42 Min: initialSleep, 43 // the maximum value is uncapped. could change this 44 // value so that we didn't avoid very long sleeps in 45 // potential worst cases. 46 Max: time.Duration(float64(initialSleep) * math.Pow(factor, float64(numAttempts))), 47 Factor: factor, 48 Jitter: true, 49 } 50 } 51 52 // Retry provides a mechanism to retry an operation with exponential 53 // backoff (that uses some jitter,) Specify the maximum number of 54 // retry attempts that you want to permit as well as the initial 55 // period that you want to sleep between attempts. 56 // 57 // Retry requires that the starting sleep interval be at least 100 58 // milliseconds, and forces this interval if you attempt to use a 59 // shorter period. 60 // 61 // If you specify 0 attempts, Retry will use an attempt value of one. 62 func Retry(op RetriableFunc, attempts int, sleep time.Duration) (bool, error) { 63 backoff := getBackoff(sleep, attempts) 64 for i := attempts; i >= 0; i-- { 65 err := op() 66 67 if err == nil { 68 //the attempt succeeded, so we return no error 69 return false, nil 70 } 71 72 if _, ok := err.(RetriableError); ok { 73 if i == 0 { 74 // used up all retry attempts, so return the failure. 75 return true, errors.Wrapf(err, "after %d retries, operation failed", attempts) 76 } 77 78 // it's safe to retry this, so sleep for a moment and try again 79 time.Sleep(backoff.Duration()) 80 } else { 81 //function returned err but it can't be retried - fail immediately 82 return false, err 83 } 84 } 85 86 return false, errors.New("unable to complete retry operation") 87 }