github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/pkg/retry/retry_with_opt.go (about)

     1  // Copyright 2021 PingCAP, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  
    14  package retry
    15  
    16  import (
    17  	"context"
    18  	"math"
    19  	"math/rand"
    20  	"strconv"
    21  	"time"
    22  
    23  	"github.com/pingcap/errors"
    24  	cerror "github.com/pingcap/tiflow/pkg/errors"
    25  )
    26  
    27  // Operation is the action need to retry
    28  type Operation func() error
    29  
    30  // Do execute the specified function.
    31  // By default, it retries infinitely until it succeeds or got canceled.
    32  func Do(ctx context.Context, operation Operation, opts ...Option) error {
    33  	retryOption := setOptions(opts...)
    34  	return run(ctx, operation, retryOption)
    35  }
    36  
    37  func setOptions(opts ...Option) *retryOptions {
    38  	retryOption := newRetryOptions()
    39  	for _, opt := range opts {
    40  		opt(retryOption)
    41  	}
    42  	return retryOption
    43  }
    44  
    45  func run(ctx context.Context, op Operation, retryOption *retryOptions) error {
    46  	select {
    47  	case <-ctx.Done():
    48  		return errors.Trace(ctx.Err())
    49  	default:
    50  	}
    51  
    52  	var t *time.Timer
    53  	var start time.Time
    54  	try := uint64(0)
    55  	backOff := time.Duration(0)
    56  	for {
    57  		err := op()
    58  		if err == nil {
    59  			return nil
    60  		}
    61  
    62  		if !retryOption.isRetryable(err) {
    63  			return err
    64  		}
    65  
    66  		try++
    67  		if try >= retryOption.maxTries {
    68  			return cerror.ErrReachMaxTry.
    69  				Wrap(err).GenWithStackByArgs(strconv.Itoa(int(retryOption.maxTries)), err)
    70  		}
    71  		if retryOption.totalRetryDuration > 0 {
    72  			if start.IsZero() {
    73  				start = time.Now()
    74  			} else if time.Since(start) > retryOption.totalRetryDuration {
    75  				return cerror.ErrReachMaxTry.
    76  					Wrap(err).GenWithStackByArgs(retryOption.totalRetryDuration, err)
    77  			}
    78  		}
    79  
    80  		backOff = getBackoffInMs(retryOption.backoffBaseInMs, retryOption.backoffCapInMs, float64(try))
    81  		if t == nil {
    82  			t = time.NewTimer(backOff)
    83  			defer t.Stop()
    84  		} else {
    85  			t.Reset(backOff)
    86  		}
    87  
    88  		select {
    89  		case <-ctx.Done():
    90  			return errors.Trace(ctx.Err())
    91  		case <-t.C:
    92  		}
    93  	}
    94  }
    95  
    96  // getBackoffInMs returns the duration to wait before next try
    97  // See https://www.awsarchitectureblog.com/2015/03/backoff.html
    98  func getBackoffInMs(backoffBaseInMs, backoffCapInMs, try float64) time.Duration {
    99  	temp := int64(math.Min(backoffCapInMs, backoffBaseInMs*math.Exp2(try)) / 2)
   100  	if temp <= 0 {
   101  		temp = 1
   102  	}
   103  	sleep := (temp + rand.Int63n(temp)) * 3
   104  	if sleep <= 0 {
   105  		sleep = math.MaxInt64
   106  	}
   107  	backOff := math.Min(backoffCapInMs, float64(rand.Int63n(sleep))+backoffBaseInMs)
   108  	return time.Duration(backOff) * time.Millisecond
   109  }