github.com/pingcap/br@v5.3.0-alpha.0.20220125034240-ec59c7b6ce30+incompatible/pkg/utils/retry.go (about)

     1  // Copyright 2020 PingCAP, Inc. Licensed under Apache-2.0.
     2  
     3  package utils
     4  
     5  import (
     6  	"context"
     7  	"strings"
     8  	"time"
     9  
    10  	"go.uber.org/multierr"
    11  )
    12  
    13  var retryableServerError = []string{
    14  	"server closed",
    15  	"connection refused",
    16  	"connection reset by peer",
    17  	"channel closed",
    18  	"error trying to connect",
    19  	"connection closed before message completed",
    20  	"body write aborted",
    21  	"error during dispatch",
    22  }
    23  
    24  // RetryableFunc presents a retryable operation.
    25  type RetryableFunc func() error
    26  
    27  // Backoffer implements a backoff policy for retrying operations.
    28  type Backoffer interface {
    29  	// NextBackoff returns a duration to wait before retrying again
    30  	NextBackoff(err error) time.Duration
    31  	// Attempt returns the remain attempt times
    32  	Attempt() int
    33  }
    34  
    35  // WithRetry retries a given operation with a backoff policy.
    36  //
    37  // Returns nil if `retryableFunc` succeeded at least once. Otherwise, returns a
    38  // multierr containing all errors encountered.
    39  func WithRetry(
    40  	ctx context.Context,
    41  	retryableFunc RetryableFunc,
    42  	backoffer Backoffer,
    43  ) error {
    44  	var allErrors error
    45  	for backoffer.Attempt() > 0 {
    46  		err := retryableFunc()
    47  		if err != nil {
    48  			allErrors = multierr.Append(allErrors, err)
    49  			select {
    50  			case <-ctx.Done():
    51  				return allErrors // nolint:wrapcheck
    52  			case <-time.After(backoffer.NextBackoff(err)):
    53  			}
    54  		} else {
    55  			return nil
    56  		}
    57  	}
    58  	return allErrors // nolint:wrapcheck
    59  }
    60  
    61  // MessageIsRetryableStorageError checks whether the message returning from TiKV is retryable ExternalStorageError.
    62  func MessageIsRetryableStorageError(msg string) bool {
    63  	msgLower := strings.ToLower(msg)
    64  	// UNSAFE! TODO: Add a error type for retryable connection error.
    65  	for _, errStr := range retryableServerError {
    66  		if strings.Contains(msgLower, errStr) {
    67  			return true
    68  		}
    69  	}
    70  	return false
    71  }