github.com/pingcap/br@v5.3.0-alpha.0.20220125034240-ec59c7b6ce30+incompatible/pkg/restore/backoff.go (about)

     1  // Copyright 2020 PingCAP, Inc. Licensed under Apache-2.0.
     2  
     3  package restore
     4  
     5  import (
     6  	"time"
     7  
     8  	"github.com/pingcap/errors"
     9  	"github.com/pingcap/log"
    10  	"go.uber.org/zap"
    11  	"google.golang.org/grpc/codes"
    12  	"google.golang.org/grpc/status"
    13  
    14  	berrors "github.com/pingcap/br/pkg/errors"
    15  	"github.com/pingcap/br/pkg/utils"
    16  )
    17  
    18  const (
    19  	importSSTRetryTimes      = 16
    20  	importSSTWaitInterval    = 10 * time.Millisecond
    21  	importSSTMaxWaitInterval = 1 * time.Second
    22  
    23  	downloadSSTRetryTimes      = 8
    24  	downloadSSTWaitInterval    = 1 * time.Second
    25  	downloadSSTMaxWaitInterval = 4 * time.Second
    26  
    27  	resetTSRetryTime       = 16
    28  	resetTSWaitInterval    = 50 * time.Millisecond
    29  	resetTSMaxWaitInterval = 500 * time.Millisecond
    30  )
    31  
    32  type importerBackoffer struct {
    33  	attempt      int
    34  	delayTime    time.Duration
    35  	maxDelayTime time.Duration
    36  }
    37  
    38  // NewBackoffer creates a new controller regulating a truncated exponential backoff.
    39  func NewBackoffer(attempt int, delayTime, maxDelayTime time.Duration) utils.Backoffer {
    40  	return &importerBackoffer{
    41  		attempt:      attempt,
    42  		delayTime:    delayTime,
    43  		maxDelayTime: maxDelayTime,
    44  	}
    45  }
    46  
    47  func newImportSSTBackoffer() utils.Backoffer {
    48  	return NewBackoffer(importSSTRetryTimes, importSSTWaitInterval, importSSTMaxWaitInterval)
    49  }
    50  
    51  func newDownloadSSTBackoffer() utils.Backoffer {
    52  	return NewBackoffer(downloadSSTRetryTimes, downloadSSTWaitInterval, downloadSSTMaxWaitInterval)
    53  }
    54  
    55  func (bo *importerBackoffer) NextBackoff(err error) time.Duration {
    56  	if utils.MessageIsRetryableStorageError(err.Error()) {
    57  		bo.delayTime = 2 * bo.delayTime
    58  		bo.attempt--
    59  	} else {
    60  		switch errors.Cause(err) { // nolint:errorlint
    61  		case berrors.ErrKVEpochNotMatch, berrors.ErrKVDownloadFailed, berrors.ErrKVIngestFailed:
    62  			bo.delayTime = 2 * bo.delayTime
    63  			bo.attempt--
    64  		case berrors.ErrKVRangeIsEmpty, berrors.ErrKVRewriteRuleNotFound:
    65  			// Excepted error, finish the operation
    66  			bo.delayTime = 0
    67  			bo.attempt = 0
    68  		default:
    69  			switch status.Code(err) {
    70  			case codes.Unavailable, codes.Aborted:
    71  				bo.delayTime = 2 * bo.delayTime
    72  				bo.attempt--
    73  			default:
    74  				// Unexcepted error
    75  				bo.delayTime = 0
    76  				bo.attempt = 0
    77  				log.Warn("unexcepted error, stop to retry", zap.Error(err))
    78  			}
    79  		}
    80  	}
    81  	if bo.delayTime > bo.maxDelayTime {
    82  		return bo.maxDelayTime
    83  	}
    84  	return bo.delayTime
    85  }
    86  
    87  func (bo *importerBackoffer) Attempt() int {
    88  	return bo.attempt
    89  }
    90  
    91  type pdReqBackoffer struct {
    92  	attempt      int
    93  	delayTime    time.Duration
    94  	maxDelayTime time.Duration
    95  }
    96  
    97  func newPDReqBackoffer() utils.Backoffer {
    98  	return &pdReqBackoffer{
    99  		attempt:      resetTSRetryTime,
   100  		delayTime:    resetTSWaitInterval,
   101  		maxDelayTime: resetTSMaxWaitInterval,
   102  	}
   103  }
   104  
   105  func (bo *pdReqBackoffer) NextBackoff(err error) time.Duration {
   106  	bo.delayTime = 2 * bo.delayTime
   107  	bo.attempt--
   108  	if bo.delayTime > bo.maxDelayTime {
   109  		return bo.maxDelayTime
   110  	}
   111  	return bo.delayTime
   112  }
   113  
   114  func (bo *pdReqBackoffer) Attempt() int {
   115  	return bo.attempt
   116  }