github.com/Jeffail/benthos/v3@v3.65.0/lib/output/retry.go (about) 1 package output 2 3 import ( 4 "encoding/json" 5 "errors" 6 "fmt" 7 "sync" 8 "sync/atomic" 9 "time" 10 11 "github.com/Jeffail/benthos/v3/internal/component/output" 12 "github.com/Jeffail/benthos/v3/internal/docs" 13 "github.com/Jeffail/benthos/v3/internal/shutdown" 14 "github.com/Jeffail/benthos/v3/lib/log" 15 "github.com/Jeffail/benthos/v3/lib/metrics" 16 "github.com/Jeffail/benthos/v3/lib/response" 17 "github.com/Jeffail/benthos/v3/lib/types" 18 "github.com/Jeffail/benthos/v3/lib/util/retries" 19 "github.com/cenkalti/backoff/v4" 20 ) 21 22 //------------------------------------------------------------------------------ 23 24 func init() { 25 Constructors[TypeRetry] = TypeSpec{ 26 constructor: fromSimpleConstructor(NewRetry), 27 Summary: ` 28 Attempts to write messages to a child output and if the write fails for any 29 reason the message is retried either until success or, if the retries or max 30 elapsed time fields are non-zero, either is reached.`, 31 Description: ` 32 All messages in Benthos are always retried on an output error, but this would 33 usually involve propagating the error back to the source of the message, whereby 34 it would be reprocessed before reaching the output layer once again. 35 36 This output type is useful whenever we wish to avoid reprocessing a message on 37 the event of a failed send. We might, for example, have a dedupe processor that 38 we want to avoid reapplying to the same message more than once in the pipeline. 39 40 Rather than retrying the same output you may wish to retry the send using a 41 different output target (a dead letter queue). In which case you should instead 42 use the ` + "[`fallback`](/docs/components/outputs/fallback)" + ` output type.`, 43 FieldSpecs: retries.FieldSpecs().Add( 44 docs.FieldCommon("output", "A child output.").HasType(docs.FieldTypeOutput), 45 ), 46 Categories: []Category{ 47 CategoryUtility, 48 }, 49 } 50 } 51 52 //------------------------------------------------------------------------------ 53 54 // RetryConfig contains configuration values for the Retry output type. 55 type RetryConfig struct { 56 Output *Config `json:"output" yaml:"output"` 57 retries.Config `json:",inline" yaml:",inline"` 58 } 59 60 // NewRetryConfig creates a new RetryConfig with default values. 61 func NewRetryConfig() RetryConfig { 62 rConf := retries.NewConfig() 63 rConf.MaxRetries = 0 64 rConf.Backoff.InitialInterval = "100ms" 65 rConf.Backoff.MaxInterval = "1s" 66 rConf.Backoff.MaxElapsedTime = "0s" 67 return RetryConfig{ 68 Output: nil, 69 Config: retries.NewConfig(), 70 } 71 } 72 73 //------------------------------------------------------------------------------ 74 75 type dummyRetryConfig struct { 76 Output interface{} `json:"output" yaml:"output"` 77 retries.Config `json:",inline" yaml:",inline"` 78 } 79 80 // MarshalJSON prints an empty object instead of nil. 81 func (r RetryConfig) MarshalJSON() ([]byte, error) { 82 dummy := dummyRetryConfig{ 83 Output: r.Output, 84 Config: r.Config, 85 } 86 if r.Output == nil { 87 dummy.Output = struct{}{} 88 } 89 return json.Marshal(dummy) 90 } 91 92 // MarshalYAML prints an empty object instead of nil. 93 func (r RetryConfig) MarshalYAML() (interface{}, error) { 94 dummy := dummyRetryConfig{ 95 Output: r.Output, 96 Config: r.Config, 97 } 98 if r.Output == nil { 99 dummy.Output = struct{}{} 100 } 101 return dummy, nil 102 } 103 104 //------------------------------------------------------------------------------ 105 106 // Retry is an output type that continuously writes a message to a child output 107 // until the send is successful. 108 type Retry struct { 109 running int32 110 conf RetryConfig 111 112 wrapped Type 113 backoffCtor func() backoff.BackOff 114 115 stats metrics.Type 116 log log.Modular 117 118 transactionsIn <-chan types.Transaction 119 transactionsOut chan types.Transaction 120 121 closeChan chan struct{} 122 closedChan chan struct{} 123 } 124 125 // NewRetry creates a new Retry input type. 126 func NewRetry( 127 conf Config, 128 mgr types.Manager, 129 log log.Modular, 130 stats metrics.Type, 131 ) (Type, error) { 132 if conf.Retry.Output == nil { 133 return nil, errors.New("cannot create retry output without a child") 134 } 135 136 wrapped, err := New(*conf.Retry.Output, mgr, log, stats) 137 if err != nil { 138 return nil, fmt.Errorf("failed to create output '%v': %v", conf.Retry.Output.Type, err) 139 } 140 141 var boffCtor func() backoff.BackOff 142 if boffCtor, err = conf.Retry.GetCtor(); err != nil { 143 return nil, err 144 } 145 146 return &Retry{ 147 running: 1, 148 conf: conf.Retry, 149 150 log: log, 151 stats: stats, 152 wrapped: wrapped, 153 backoffCtor: boffCtor, 154 transactionsOut: make(chan types.Transaction), 155 156 closeChan: make(chan struct{}), 157 closedChan: make(chan struct{}), 158 }, nil 159 } 160 161 //------------------------------------------------------------------------------ 162 163 func (r *Retry) loop() { 164 // Metrics paths 165 var ( 166 mRunning = r.stats.GetGauge("retry.running") 167 mCount = r.stats.GetCounter("retry.count") 168 mSuccess = r.stats.GetCounter("retry.send.success") 169 mPartsSuccess = r.stats.GetCounter("retry.parts.send.success") 170 mError = r.stats.GetCounter("retry.send.error") 171 mEndOfRetries = r.stats.GetCounter("retry.end_of_retries") 172 ) 173 174 wg := sync.WaitGroup{} 175 176 defer func() { 177 wg.Wait() 178 close(r.transactionsOut) 179 r.wrapped.CloseAsync() 180 _ = r.wrapped.WaitForClose(shutdown.MaximumShutdownWait()) 181 mRunning.Decr(1) 182 close(r.closedChan) 183 }() 184 mRunning.Incr(1) 185 186 errInterruptChan := make(chan struct{}) 187 var errLooped int64 188 189 for atomic.LoadInt32(&r.running) == 1 { 190 // Do not consume another message while pending messages are being 191 // reattempted. 192 for atomic.LoadInt64(&errLooped) > 0 { 193 select { 194 case <-errInterruptChan: 195 case <-time.After(time.Millisecond * 100): 196 // Just incase an interrupt doesn't arrive. 197 case <-r.closeChan: 198 return 199 } 200 } 201 202 var tran types.Transaction 203 var open bool 204 select { 205 case tran, open = <-r.transactionsIn: 206 if !open { 207 return 208 } 209 mCount.Incr(1) 210 case <-r.closeChan: 211 return 212 } 213 214 rChan := make(chan types.Response) 215 select { 216 case r.transactionsOut <- types.NewTransaction(tran.Payload, rChan): 217 case <-r.closeChan: 218 return 219 } 220 221 wg.Add(1) 222 go func(ts types.Transaction, resChan chan types.Response) { 223 var backOff backoff.BackOff 224 var resOut types.Response 225 var inErrLoop bool 226 227 defer func() { 228 wg.Done() 229 if inErrLoop { 230 atomic.AddInt64(&errLooped, -1) 231 232 // We're exiting our error loop, so (attempt to) interrupt the 233 // consumer. 234 select { 235 case errInterruptChan <- struct{}{}: 236 default: 237 } 238 } 239 }() 240 241 for atomic.LoadInt32(&r.running) == 1 { 242 var res types.Response 243 select { 244 case res = <-resChan: 245 case <-r.closeChan: 246 return 247 } 248 249 if res.Error() != nil { 250 if !inErrLoop { 251 inErrLoop = true 252 atomic.AddInt64(&errLooped, 1) 253 } 254 255 mError.Incr(1) 256 257 if backOff == nil { 258 backOff = r.backoffCtor() 259 } 260 261 nextBackoff := backOff.NextBackOff() 262 if nextBackoff == backoff.Stop { 263 mEndOfRetries.Incr(1) 264 r.log.Errorf("Failed to send message: %v\n", res.Error()) 265 resOut = response.NewNoack() 266 break 267 } else { 268 r.log.Warnf("Failed to send message: %v\n", res.Error()) 269 } 270 select { 271 case <-time.After(nextBackoff): 272 case <-r.closeChan: 273 return 274 } 275 276 select { 277 case r.transactionsOut <- types.NewTransaction(ts.Payload, resChan): 278 case <-r.closeChan: 279 return 280 } 281 } else { 282 mSuccess.Incr(1) 283 mPartsSuccess.Incr(int64(ts.Payload.Len())) 284 resOut = response.NewAck() 285 break 286 } 287 } 288 289 select { 290 case ts.ResponseChan <- resOut: 291 case <-r.closeChan: 292 return 293 } 294 }(tran, rChan) 295 } 296 } 297 298 // Consume assigns a messages channel for the output to read. 299 func (r *Retry) Consume(ts <-chan types.Transaction) error { 300 if r.transactionsIn != nil { 301 return types.ErrAlreadyStarted 302 } 303 if err := r.wrapped.Consume(r.transactionsOut); err != nil { 304 return err 305 } 306 r.transactionsIn = ts 307 go r.loop() 308 return nil 309 } 310 311 // Connected returns a boolean indicating whether this output is currently 312 // connected to its target. 313 func (r *Retry) Connected() bool { 314 return r.wrapped.Connected() 315 } 316 317 // MaxInFlight returns the maximum number of in flight messages permitted by the 318 // output. This value can be used to determine a sensible value for parent 319 // outputs, but should not be relied upon as part of dispatcher logic. 320 func (r *Retry) MaxInFlight() (int, bool) { 321 return output.GetMaxInFlight(r.wrapped) 322 } 323 324 // CloseAsync shuts down the Retry input and stops processing requests. 325 func (r *Retry) CloseAsync() { 326 if atomic.CompareAndSwapInt32(&r.running, 1, 0) { 327 close(r.closeChan) 328 } 329 } 330 331 // WaitForClose blocks until the Retry input has closed down. 332 func (r *Retry) WaitForClose(timeout time.Duration) error { 333 select { 334 case <-r.closedChan: 335 case <-time.After(timeout): 336 return types.ErrTimeout 337 } 338 return nil 339 } 340 341 //------------------------------------------------------------------------------