github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/pkg/retry/error_retry.go (about)

     1  // Copyright 2023 PingCAP, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  
    14  package retry
    15  
    16  import (
    17  	"math"
    18  	"math/rand"
    19  	"time"
    20  
    21  	"github.com/pingcap/log"
    22  	cerror "github.com/pingcap/tiflow/pkg/errors"
    23  	"go.uber.org/zap"
    24  )
    25  
    26  const (
    27  	defaultErrorMaxRetryDuration = 30 * time.Minute
    28  	defaultErrGCInterval         = 10 * time.Minute
    29  	defaultBackoffBaseInS        = 5
    30  	defaultBackoffMaxInS         = 30
    31  )
    32  
    33  // ErrorRetry is used to control the error retry logic.
    34  type ErrorRetry struct {
    35  	// To control the error retry.
    36  	lastInternalError  error
    37  	firstRetryTime     time.Time
    38  	lastErrorRetryTime time.Time
    39  	maxRetryDuration   time.Duration
    40  	errGCInterval      time.Duration
    41  	backoffBase        int64
    42  	backoffMax         int64
    43  }
    44  
    45  // NewDefaultErrorRetry creates a new ErrorRetry with default values.
    46  func NewDefaultErrorRetry() *ErrorRetry {
    47  	return NewErrorRetry(defaultErrorMaxRetryDuration,
    48  		defaultErrGCInterval,
    49  		defaultBackoffBaseInS,
    50  		defaultBackoffMaxInS)
    51  }
    52  
    53  // NewInfiniteErrorRetry creates a new ErrorRetry with infinite duration.
    54  func NewInfiniteErrorRetry() *ErrorRetry {
    55  	return NewErrorRetry(time.Duration(math.MaxInt64),
    56  		defaultErrGCInterval,
    57  		defaultBackoffBaseInS,
    58  		defaultBackoffMaxInS)
    59  }
    60  
    61  // NewErrorRetry creates a new ErrorRetry.
    62  func NewErrorRetry(
    63  	maxRetryDuration time.Duration,
    64  	errGCInterval time.Duration,
    65  	backoffBase int64,
    66  	backoffMax int64,
    67  ) *ErrorRetry {
    68  	return &ErrorRetry{
    69  		maxRetryDuration: maxRetryDuration,
    70  		errGCInterval:    errGCInterval,
    71  		backoffBase:      backoffBase,
    72  		backoffMax:       backoffMax,
    73  	}
    74  }
    75  
    76  // GetRetryBackoff returns the backoff duration for retrying the last error.
    77  // If the retry time is exhausted, it returns the an ChangefeedUnRetryableError.
    78  func (r *ErrorRetry) GetRetryBackoff(err error) (time.Duration, error) {
    79  	// reset firstRetryTime when the last error is too long ago
    80  	// it means the last error is retry success, and the sink is running well for some time
    81  	if r.lastInternalError == nil ||
    82  		time.Since(r.lastErrorRetryTime) >= r.errGCInterval {
    83  		log.Debug("reset firstRetryTime",
    84  			zap.Time("lastErrorRetryTime", r.lastErrorRetryTime),
    85  			zap.Time("now", time.Now()))
    86  		r.firstRetryTime = time.Now()
    87  	}
    88  
    89  	// return an unretryable error if retry time is exhausted
    90  	if time.Since(r.firstRetryTime) >= r.maxRetryDuration {
    91  		log.Debug("error retry exhausted",
    92  			zap.Time("firstRetryTime", r.firstRetryTime),
    93  			zap.Time("lastErrorRetryTime", r.lastErrorRetryTime),
    94  			zap.Time("now", time.Now()))
    95  		return 0, cerror.WrapChangefeedUnretryableErr(err)
    96  	}
    97  
    98  	r.lastInternalError = err
    99  	r.lastErrorRetryTime = time.Now()
   100  
   101  	// interval is in range [defaultBackoffBaseInS, defaultBackoffMaxInS)
   102  	interval := time.Second * time.Duration(
   103  		rand.Int63n(defaultBackoffMaxInS-defaultBackoffBaseInS)+defaultBackoffBaseInS)
   104  	return interval, nil
   105  }