go.temporal.io/server@v1.23.0/common/backoff/retry.go (about)

     1  // The MIT License
     2  //
     3  // Copyright (c) 2020 Temporal Technologies Inc.  All rights reserved.
     4  //
     5  // Copyright (c) 2020 Uber Technologies, Inc.
     6  //
     7  // Permission is hereby granted, free of charge, to any person obtaining a copy
     8  // of this software and associated documentation files (the "Software"), to deal
     9  // in the Software without restriction, including without limitation the rights
    10  // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
    11  // copies of the Software, and to permit persons to whom the Software is
    12  // furnished to do so, subject to the following conditions:
    13  //
    14  // The above copyright notice and this permission notice shall be included in
    15  // all copies or substantial portions of the Software.
    16  //
    17  // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    18  // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    19  // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    20  // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    21  // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    22  // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
    23  // THE SOFTWARE.
    24  
    25  package backoff
    26  
    27  import (
    28  	"context"
    29  	"sync"
    30  	"time"
    31  
    32  	"go.temporal.io/api/serviceerror"
    33  )
    34  
    35  const (
    36  	throttleRetryInitialInterval    = time.Second
    37  	throttleRetryMaxInterval        = 10 * time.Second
    38  	throttleRetryExpirationInterval = NoInterval
    39  )
    40  
    41  var (
    42  	throttleRetryPolicy = NewExponentialRetryPolicy(throttleRetryInitialInterval).
    43  		WithMaximumInterval(throttleRetryMaxInterval).
    44  		WithExpirationInterval(throttleRetryExpirationInterval)
    45  )
    46  
    47  type (
    48  	// Operation to retry
    49  	Operation func() error
    50  
    51  	// OperationCtx plays the same role as Operation but for context-aware
    52  	// retryable functions.
    53  	OperationCtx func(context.Context) error
    54  
    55  	// IsRetryable handler can be used to exclude certain errors during retry
    56  	IsRetryable func(error) bool
    57  
    58  	// ConcurrentRetrier is used for client-side throttling. It determines whether to
    59  	// throttle outgoing traffic in case downstream backend server rejects
    60  	// requests due to out-of-quota or server busy errors.
    61  	ConcurrentRetrier struct {
    62  		sync.Mutex
    63  		retrier      Retrier // Backoff retrier
    64  		failureCount int64   // Number of consecutive failures seen
    65  	}
    66  )
    67  
    68  // Throttle Sleep if there were failures since the last success call.
    69  func (c *ConcurrentRetrier) Throttle() {
    70  	c.throttleInternal()
    71  }
    72  
    73  func (c *ConcurrentRetrier) throttleInternal() time.Duration {
    74  	next := done
    75  
    76  	// Check if we have failure count.
    77  	failureCount := c.failureCount
    78  	if failureCount > 0 {
    79  		defer c.Unlock()
    80  		c.Lock()
    81  		if c.failureCount > 0 {
    82  			next = c.retrier.NextBackOff()
    83  		}
    84  	}
    85  
    86  	if next != done {
    87  		time.Sleep(next)
    88  	}
    89  
    90  	return next
    91  }
    92  
    93  // Succeeded marks client request succeeded.
    94  func (c *ConcurrentRetrier) Succeeded() {
    95  	defer c.Unlock()
    96  	c.Lock()
    97  	c.failureCount = 0
    98  	c.retrier.Reset()
    99  }
   100  
   101  // Failed marks client request failed because backend is busy.
   102  func (c *ConcurrentRetrier) Failed() {
   103  	defer c.Unlock()
   104  	c.Lock()
   105  	c.failureCount++
   106  }
   107  
   108  // NewConcurrentRetrier returns an instance of concurrent backoff retrier.
   109  func NewConcurrentRetrier(retryPolicy RetryPolicy) *ConcurrentRetrier {
   110  	retrier := NewRetrier(retryPolicy, SystemClock)
   111  	return &ConcurrentRetrier{retrier: retrier}
   112  }
   113  
   114  // ThrottleRetry is a resource aware version of Retry.
   115  // Resource exhausted error will be retried using a different throttle retry policy, instead of the specified one.
   116  func ThrottleRetry(operation Operation, policy RetryPolicy, isRetryable IsRetryable) error {
   117  	ctxOp := func(context.Context) error { return operation() }
   118  	return ThrottleRetryContext(context.Background(), ctxOp, policy, isRetryable)
   119  }
   120  
   121  // ThrottleRetryContext is a context and resource aware version of Retry.
   122  // Context timeout/cancellation errors are never retried, regardless of IsRetryable.
   123  // Resource exhausted error will be retried using a different throttle retry policy, instead of the specified one.
   124  // TODO: allow customizing throttle retry policy and what kind of error are categorized as throttle error.
   125  func ThrottleRetryContext(
   126  	ctx context.Context,
   127  	operation OperationCtx,
   128  	policy RetryPolicy,
   129  	isRetryable IsRetryable,
   130  ) error {
   131  	var err error
   132  	var next time.Duration
   133  
   134  	if isRetryable == nil {
   135  		isRetryable = func(error) bool { return true }
   136  	}
   137  
   138  	deadline, hasDeadline := ctx.Deadline()
   139  
   140  	r := NewRetrier(policy, SystemClock)
   141  	t := NewRetrier(throttleRetryPolicy, SystemClock)
   142  	for ctx.Err() == nil {
   143  		if err = operation(ctx); err == nil {
   144  			return nil
   145  		}
   146  
   147  		if next = r.NextBackOff(); next == done {
   148  			return err
   149  		}
   150  
   151  		if err == ctx.Err() || !isRetryable(err) {
   152  			return err
   153  		}
   154  
   155  		if _, ok := err.(*serviceerror.ResourceExhausted); ok {
   156  			next = max(next, t.NextBackOff())
   157  		}
   158  
   159  		if hasDeadline && SystemClock.Now().Add(next).After(deadline) {
   160  			break
   161  		}
   162  
   163  		timer := time.NewTimer(next)
   164  		select {
   165  		case <-timer.C:
   166  		case <-ctx.Done():
   167  			timer.Stop()
   168  		}
   169  	}
   170  	// always return the last error we got from operation, even if it is not useful
   171  	// this retry utility does not have enough information to do any filtering/mapping
   172  	if err != nil {
   173  		return err
   174  	}
   175  	return ctx.Err()
   176  }
   177  
   178  // IgnoreErrors can be used as IsRetryable handler for Retry function to exclude certain errors from the retry list
   179  func IgnoreErrors(errorsToExclude []error) func(error) bool {
   180  	return func(err error) bool {
   181  		for _, errorToExclude := range errorsToExclude {
   182  			if err == errorToExclude {
   183  				return false
   184  			}
   185  		}
   186  
   187  		return true
   188  	}
   189  }