go.temporal.io/server@v1.23.0/common/backoff/retrypolicy.go (about)

     1  // The MIT License
     2  //
     3  // Copyright (c) 2020 Temporal Technologies Inc.  All rights reserved.
     4  //
     5  // Copyright (c) 2020 Uber Technologies, Inc.
     6  //
     7  // Permission is hereby granted, free of charge, to any person obtaining a copy
     8  // of this software and associated documentation files (the "Software"), to deal
     9  // in the Software without restriction, including without limitation the rights
    10  // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
    11  // copies of the Software, and to permit persons to whom the Software is
    12  // furnished to do so, subject to the following conditions:
    13  //
    14  // The above copyright notice and this permission notice shall be included in
    15  // all copies or substantial portions of the Software.
    16  //
    17  // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    18  // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    19  // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    20  // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    21  // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    22  // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
    23  // THE SOFTWARE.
    24  
    25  package backoff
    26  
    27  import (
    28  	"math"
    29  	"math/rand"
    30  	"time"
    31  )
    32  
    33  const (
    34  	// NoInterval represents Maximim interval
    35  	NoInterval                      = 0
    36  	done              time.Duration = -1
    37  	noMaximumAttempts               = 0
    38  
    39  	defaultBackoffCoefficient = 2.0
    40  	defaultMaximumInterval    = 10 * time.Second
    41  	defaultExpirationInterval = time.Minute
    42  	defaultMaximumAttempts    = noMaximumAttempts
    43  
    44  	defaultFirstPhaseMaximumAttempts = 3
    45  )
    46  
    47  var (
    48  	// DisabledRetryPolicy is a retry policy that never retries
    49  	DisabledRetryPolicy RetryPolicy = &disabledRetryPolicyImpl{}
    50  )
    51  
    52  type (
    53  	// RetryPolicy is the API which needs to be implemented by various retry policy implementations
    54  	RetryPolicy interface {
    55  		ComputeNextDelay(elapsedTime time.Duration, numAttempts int) time.Duration
    56  	}
    57  
    58  	// Retrier manages the state of retry operation
    59  	Retrier interface {
    60  		NextBackOff() time.Duration
    61  		Reset()
    62  	}
    63  
    64  	// Clock used by ExponentialRetryPolicy implementation to get the current time.  Mainly used for unit testing
    65  	Clock interface {
    66  		Now() time.Time
    67  	}
    68  
    69  	// ExponentialRetryPolicy provides the implementation for retry policy using a coefficient to compute the next delay.
    70  	// Formula used to compute the next delay is:
    71  	// 	min(initialInterval * pow(backoffCoefficient, currentAttempt), maximumInterval)
    72  	ExponentialRetryPolicy struct {
    73  		initialInterval    time.Duration
    74  		backoffCoefficient float64
    75  		maximumInterval    time.Duration
    76  		expirationInterval time.Duration
    77  		maximumAttempts    int
    78  	}
    79  
    80  	// TwoPhaseRetryPolicy implements a policy that first use one policy to get next delay,
    81  	// and once expired use the second policy for the following retry.
    82  	// It can achieve fast retries in first phase then slowly retires in second phase.
    83  	TwoPhaseRetryPolicy struct {
    84  		firstPolicy  RetryPolicy
    85  		secondPolicy RetryPolicy
    86  	}
    87  
    88  	disabledRetryPolicyImpl struct{}
    89  
    90  	systemClock struct{}
    91  
    92  	retrierImpl struct {
    93  		policy         RetryPolicy
    94  		clock          Clock
    95  		currentAttempt int
    96  		startTime      time.Time
    97  	}
    98  )
    99  
   100  // SystemClock implements Clock interface that uses time.Now().UTC().
   101  var SystemClock = systemClock{}
   102  
   103  // NewExponentialRetryPolicy returns an instance of ExponentialRetryPolicy using the provided initialInterval
   104  func NewExponentialRetryPolicy(initialInterval time.Duration) *ExponentialRetryPolicy {
   105  	p := &ExponentialRetryPolicy{
   106  		initialInterval:    initialInterval,
   107  		backoffCoefficient: defaultBackoffCoefficient,
   108  		maximumInterval:    defaultMaximumInterval,
   109  		expirationInterval: defaultExpirationInterval,
   110  		maximumAttempts:    defaultMaximumAttempts,
   111  	}
   112  
   113  	return p
   114  }
   115  
   116  // NewRetrier is used for creating a new instance of Retrier
   117  func NewRetrier(policy RetryPolicy, clock Clock) Retrier {
   118  	return &retrierImpl{
   119  		policy:         policy,
   120  		clock:          clock,
   121  		startTime:      clock.Now(),
   122  		currentAttempt: 1,
   123  	}
   124  }
   125  
   126  // WithInitialInterval sets the initial interval used by ExponentialRetryPolicy for the very first retry
   127  // All later retries are computed using the following formula:
   128  // initialInterval * math.Pow(backoffCoefficient, currentAttempt)
   129  func (p *ExponentialRetryPolicy) WithInitialInterval(initialInterval time.Duration) *ExponentialRetryPolicy {
   130  	p.initialInterval = initialInterval
   131  	return p
   132  }
   133  
   134  // WithBackoffCoefficient sets the coefficient used by ExponentialRetryPolicy to compute next delay for each retry
   135  // All retries are computed using the following formula:
   136  // initialInterval * math.Pow(backoffCoefficient, currentAttempt)
   137  func (p *ExponentialRetryPolicy) WithBackoffCoefficient(backoffCoefficient float64) *ExponentialRetryPolicy {
   138  	p.backoffCoefficient = backoffCoefficient
   139  	return p
   140  }
   141  
   142  // WithMaximumInterval sets the maximum interval for each retry.
   143  // This does *not* cause the policy to stop retrying when the interval between retries reaches the supplied duration.
   144  // That is what WithExpirationInterval does. Instead, this prevents the interval from exceeding maximumInterval.
   145  func (p *ExponentialRetryPolicy) WithMaximumInterval(maximumInterval time.Duration) *ExponentialRetryPolicy {
   146  	p.maximumInterval = maximumInterval
   147  	return p
   148  }
   149  
   150  // WithExpirationInterval sets the absolute expiration interval for all retries
   151  func (p *ExponentialRetryPolicy) WithExpirationInterval(expirationInterval time.Duration) *ExponentialRetryPolicy {
   152  	p.expirationInterval = expirationInterval
   153  	return p
   154  }
   155  
   156  // WithMaximumAttempts sets the maximum number of retry attempts
   157  func (p *ExponentialRetryPolicy) WithMaximumAttempts(maximumAttempts int) *ExponentialRetryPolicy {
   158  	p.maximumAttempts = maximumAttempts
   159  	return p
   160  }
   161  
   162  // ComputeNextDelay returns the next delay interval.  This is used by Retrier to delay calling the operation again
   163  func (p *ExponentialRetryPolicy) ComputeNextDelay(elapsedTime time.Duration, numAttempts int) time.Duration {
   164  	// Check to see if we ran out of maximum number of attempts
   165  	// NOTE: if maxAttempts is X, return done when numAttempts == X, otherwise there will be attempt X+1
   166  	if p.maximumAttempts != noMaximumAttempts && numAttempts >= p.maximumAttempts {
   167  		return done
   168  	}
   169  
   170  	// Stop retrying after expiration interval is elapsed
   171  	if p.expirationInterval != NoInterval && elapsedTime > p.expirationInterval {
   172  		return done
   173  	}
   174  
   175  	nextInterval := float64(p.initialInterval) * math.Pow(p.backoffCoefficient, float64(numAttempts-1))
   176  	// Disallow retries if initialInterval is negative or nextInterval overflows
   177  	if nextInterval <= 0 {
   178  		return done
   179  	}
   180  	if p.maximumInterval != NoInterval {
   181  		nextInterval = math.Min(nextInterval, float64(p.maximumInterval))
   182  	}
   183  
   184  	if p.expirationInterval != NoInterval {
   185  		remainingTime := float64(math.Max(0, float64(p.expirationInterval-elapsedTime)))
   186  		nextInterval = math.Min(remainingTime, nextInterval)
   187  	}
   188  
   189  	// Bail out if the next interval is smaller than initial retry interval
   190  	nextDuration := time.Duration(nextInterval)
   191  	if nextDuration < p.initialInterval {
   192  		return done
   193  	}
   194  
   195  	// add jitter to avoid global synchronization
   196  	jitterPortion := int(0.2 * nextInterval)
   197  	// Prevent overflow
   198  	if jitterPortion < 1 {
   199  		jitterPortion = 1
   200  	}
   201  	nextInterval = nextInterval*0.8 + float64(rand.Intn(jitterPortion))
   202  
   203  	return time.Duration(nextInterval)
   204  }
   205  
   206  // ComputeNextDelay returns the next delay interval.
   207  func (tp *TwoPhaseRetryPolicy) ComputeNextDelay(elapsedTime time.Duration, numAttempts int) time.Duration {
   208  	nextInterval := tp.firstPolicy.ComputeNextDelay(elapsedTime, numAttempts)
   209  	if nextInterval == done {
   210  		nextInterval = tp.secondPolicy.ComputeNextDelay(elapsedTime, numAttempts-defaultFirstPhaseMaximumAttempts)
   211  	}
   212  	return nextInterval
   213  }
   214  
   215  func (r *disabledRetryPolicyImpl) ComputeNextDelay(_ time.Duration, _ int) time.Duration {
   216  	return done
   217  }
   218  
   219  // Now returns the current time using the system clock
   220  func (t systemClock) Now() time.Time {
   221  	return time.Now().UTC()
   222  }
   223  
   224  // Reset will set the Retrier into initial state
   225  func (r *retrierImpl) Reset() {
   226  	r.startTime = r.clock.Now()
   227  	r.currentAttempt = 1
   228  }
   229  
   230  // NextBackOff returns the next delay interval.  This is used by Retry to delay calling the operation again
   231  func (r *retrierImpl) NextBackOff() time.Duration {
   232  	nextInterval := r.policy.ComputeNextDelay(r.getElapsedTime(), r.currentAttempt)
   233  
   234  	// Now increment the current attempt
   235  	r.currentAttempt++
   236  	return nextInterval
   237  }
   238  
   239  func (r *retrierImpl) getElapsedTime() time.Duration {
   240  	return r.clock.Now().Sub(r.startTime)
   241  }