go.temporal.io/server@v1.23.0/common/backoff/retry.go (about) 1 // The MIT License 2 // 3 // Copyright (c) 2020 Temporal Technologies Inc. All rights reserved. 4 // 5 // Copyright (c) 2020 Uber Technologies, Inc. 6 // 7 // Permission is hereby granted, free of charge, to any person obtaining a copy 8 // of this software and associated documentation files (the "Software"), to deal 9 // in the Software without restriction, including without limitation the rights 10 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 // copies of the Software, and to permit persons to whom the Software is 12 // furnished to do so, subject to the following conditions: 13 // 14 // The above copyright notice and this permission notice shall be included in 15 // all copies or substantial portions of the Software. 16 // 17 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 23 // THE SOFTWARE. 24 25 package backoff 26 27 import ( 28 "context" 29 "sync" 30 "time" 31 32 "go.temporal.io/api/serviceerror" 33 ) 34 35 const ( 36 throttleRetryInitialInterval = time.Second 37 throttleRetryMaxInterval = 10 * time.Second 38 throttleRetryExpirationInterval = NoInterval 39 ) 40 41 var ( 42 throttleRetryPolicy = NewExponentialRetryPolicy(throttleRetryInitialInterval). 43 WithMaximumInterval(throttleRetryMaxInterval). 44 WithExpirationInterval(throttleRetryExpirationInterval) 45 ) 46 47 type ( 48 // Operation to retry 49 Operation func() error 50 51 // OperationCtx plays the same role as Operation but for context-aware 52 // retryable functions. 53 OperationCtx func(context.Context) error 54 55 // IsRetryable handler can be used to exclude certain errors during retry 56 IsRetryable func(error) bool 57 58 // ConcurrentRetrier is used for client-side throttling. It determines whether to 59 // throttle outgoing traffic in case downstream backend server rejects 60 // requests due to out-of-quota or server busy errors. 61 ConcurrentRetrier struct { 62 sync.Mutex 63 retrier Retrier // Backoff retrier 64 failureCount int64 // Number of consecutive failures seen 65 } 66 ) 67 68 // Throttle Sleep if there were failures since the last success call. 69 func (c *ConcurrentRetrier) Throttle() { 70 c.throttleInternal() 71 } 72 73 func (c *ConcurrentRetrier) throttleInternal() time.Duration { 74 next := done 75 76 // Check if we have failure count. 77 failureCount := c.failureCount 78 if failureCount > 0 { 79 defer c.Unlock() 80 c.Lock() 81 if c.failureCount > 0 { 82 next = c.retrier.NextBackOff() 83 } 84 } 85 86 if next != done { 87 time.Sleep(next) 88 } 89 90 return next 91 } 92 93 // Succeeded marks client request succeeded. 94 func (c *ConcurrentRetrier) Succeeded() { 95 defer c.Unlock() 96 c.Lock() 97 c.failureCount = 0 98 c.retrier.Reset() 99 } 100 101 // Failed marks client request failed because backend is busy. 102 func (c *ConcurrentRetrier) Failed() { 103 defer c.Unlock() 104 c.Lock() 105 c.failureCount++ 106 } 107 108 // NewConcurrentRetrier returns an instance of concurrent backoff retrier. 109 func NewConcurrentRetrier(retryPolicy RetryPolicy) *ConcurrentRetrier { 110 retrier := NewRetrier(retryPolicy, SystemClock) 111 return &ConcurrentRetrier{retrier: retrier} 112 } 113 114 // ThrottleRetry is a resource aware version of Retry. 115 // Resource exhausted error will be retried using a different throttle retry policy, instead of the specified one. 116 func ThrottleRetry(operation Operation, policy RetryPolicy, isRetryable IsRetryable) error { 117 ctxOp := func(context.Context) error { return operation() } 118 return ThrottleRetryContext(context.Background(), ctxOp, policy, isRetryable) 119 } 120 121 // ThrottleRetryContext is a context and resource aware version of Retry. 122 // Context timeout/cancellation errors are never retried, regardless of IsRetryable. 123 // Resource exhausted error will be retried using a different throttle retry policy, instead of the specified one. 124 // TODO: allow customizing throttle retry policy and what kind of error are categorized as throttle error. 125 func ThrottleRetryContext( 126 ctx context.Context, 127 operation OperationCtx, 128 policy RetryPolicy, 129 isRetryable IsRetryable, 130 ) error { 131 var err error 132 var next time.Duration 133 134 if isRetryable == nil { 135 isRetryable = func(error) bool { return true } 136 } 137 138 deadline, hasDeadline := ctx.Deadline() 139 140 r := NewRetrier(policy, SystemClock) 141 t := NewRetrier(throttleRetryPolicy, SystemClock) 142 for ctx.Err() == nil { 143 if err = operation(ctx); err == nil { 144 return nil 145 } 146 147 if next = r.NextBackOff(); next == done { 148 return err 149 } 150 151 if err == ctx.Err() || !isRetryable(err) { 152 return err 153 } 154 155 if _, ok := err.(*serviceerror.ResourceExhausted); ok { 156 next = max(next, t.NextBackOff()) 157 } 158 159 if hasDeadline && SystemClock.Now().Add(next).After(deadline) { 160 break 161 } 162 163 timer := time.NewTimer(next) 164 select { 165 case <-timer.C: 166 case <-ctx.Done(): 167 timer.Stop() 168 } 169 } 170 // always return the last error we got from operation, even if it is not useful 171 // this retry utility does not have enough information to do any filtering/mapping 172 if err != nil { 173 return err 174 } 175 return ctx.Err() 176 } 177 178 // IgnoreErrors can be used as IsRetryable handler for Retry function to exclude certain errors from the retry list 179 func IgnoreErrors(errorsToExclude []error) func(error) bool { 180 return func(err error) bool { 181 for _, errorToExclude := range errorsToExclude { 182 if err == errorToExclude { 183 return false 184 } 185 } 186 187 return true 188 } 189 }