github.com/Jeffail/benthos/v3@v3.65.0/lib/output/drop_on_error.go (about) 1 package output 2 3 import ( 4 "encoding/json" 5 "errors" 6 "fmt" 7 "sync/atomic" 8 "time" 9 10 "github.com/Jeffail/benthos/v3/internal/component/output" 11 "github.com/Jeffail/benthos/v3/internal/docs" 12 "github.com/Jeffail/benthos/v3/internal/shutdown" 13 "github.com/Jeffail/benthos/v3/lib/log" 14 "github.com/Jeffail/benthos/v3/lib/metrics" 15 "github.com/Jeffail/benthos/v3/lib/response" 16 "github.com/Jeffail/benthos/v3/lib/types" 17 ) 18 19 //------------------------------------------------------------------------------ 20 21 func init() { 22 Constructors[TypeDropOnError] = TypeSpec{ 23 constructor: fromSimpleConstructor(NewDropOnError), 24 Status: docs.StatusDeprecated, 25 Summary: ` 26 Attempts to write messages to a child output and if the write fails for any 27 reason the message is dropped instead of being reattempted.`, 28 Description: ` 29 ## Alternatives 30 31 This output has been replaced with the more explicit and configurable ` + "[`drop_on`](/docs/components/outputs/drop_on)" + ` output. 32 33 This output can be combined with a child [` + "`retry`" + `](/docs/components/outputs/retry) 34 output in order to set an explicit number of retry attempts before dropping a 35 message. 36 37 For example, the following configuration attempts to send to a hypothetical 38 output type ` + "`foo`" + ` three times, but if all three attempts fail the 39 message is dropped entirely: 40 41 ` + "```yaml" + ` 42 output: 43 drop_on_error: 44 retry: 45 max_retries: 2 46 output: 47 type: foo 48 ` + "```" + ``, 49 config: docs.FieldComponent().HasType(docs.FieldTypeObject), 50 } 51 } 52 53 //------------------------------------------------------------------------------ 54 55 // DropOnErrorConfig contains configuration values for the DropOnError output 56 // type. 57 type DropOnErrorConfig struct { 58 *Config `yaml:",inline" json:",inline"` 59 } 60 61 // NewDropOnErrorConfig creates a new DropOnErrorConfig with default values. 62 func NewDropOnErrorConfig() DropOnErrorConfig { 63 return DropOnErrorConfig{ 64 Config: nil, 65 } 66 } 67 68 //------------------------------------------------------------------------------ 69 70 // MarshalJSON prints an empty object instead of nil. 71 func (d DropOnErrorConfig) MarshalJSON() ([]byte, error) { 72 if d.Config != nil { 73 return json.Marshal(d.Config) 74 } 75 return json.Marshal(struct{}{}) 76 } 77 78 // MarshalYAML prints an empty object instead of nil. 79 func (d DropOnErrorConfig) MarshalYAML() (interface{}, error) { 80 if d.Config != nil { 81 return *d.Config, nil 82 } 83 return struct{}{}, nil 84 } 85 86 //------------------------------------------------------------------------------ 87 88 // UnmarshalJSON ensures that when parsing child config it is initialised. 89 func (d *DropOnErrorConfig) UnmarshalJSON(bytes []byte) error { 90 if d.Config == nil { 91 nConf := NewConfig() 92 d.Config = &nConf 93 } 94 95 return json.Unmarshal(bytes, d.Config) 96 } 97 98 // UnmarshalYAML ensures that when parsing child config it is initialised. 99 func (d *DropOnErrorConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { 100 if d.Config == nil { 101 nConf := NewConfig() 102 d.Config = &nConf 103 } 104 105 return unmarshal(d.Config) 106 } 107 108 //------------------------------------------------------------------------------ 109 110 // DropOnError is an output type that continuously writes a message to a child output 111 // until the send is successful. 112 type DropOnError struct { 113 running int32 114 115 wrapped Type 116 117 stats metrics.Type 118 log log.Modular 119 120 transactionsIn <-chan types.Transaction 121 transactionsOut chan types.Transaction 122 123 closeChan chan struct{} 124 closedChan chan struct{} 125 } 126 127 // NewDropOnError creates a new DropOnError input type. 128 func NewDropOnError( 129 conf Config, 130 mgr types.Manager, 131 log log.Modular, 132 stats metrics.Type, 133 ) (Type, error) { 134 if conf.DropOnError.Config == nil { 135 return nil, errors.New("cannot create a drop_on_error output without a child") 136 } 137 138 wrapped, err := New(*conf.DropOnError.Config, mgr, log, stats) 139 if err != nil { 140 return nil, fmt.Errorf("failed to create output '%v': %v", conf.DropOnError.Config.Type, err) 141 } 142 143 return &DropOnError{ 144 running: 1, 145 146 log: log, 147 stats: stats, 148 wrapped: wrapped, 149 transactionsOut: make(chan types.Transaction), 150 151 closeChan: make(chan struct{}), 152 closedChan: make(chan struct{}), 153 }, nil 154 } 155 156 //------------------------------------------------------------------------------ 157 158 func (d *DropOnError) loop() { 159 // Metrics paths 160 var ( 161 mDropped = d.stats.GetCounter("drop_on_error.dropped") 162 mDroppedBatch = d.stats.GetCounter("drop_on_error.batch.dropped") 163 ) 164 165 defer func() { 166 close(d.transactionsOut) 167 d.wrapped.CloseAsync() 168 _ = d.wrapped.WaitForClose(shutdown.MaximumShutdownWait()) 169 close(d.closedChan) 170 }() 171 172 resChan := make(chan types.Response) 173 174 for atomic.LoadInt32(&d.running) == 1 { 175 var ts types.Transaction 176 var open bool 177 select { 178 case ts, open = <-d.transactionsIn: 179 if !open { 180 return 181 } 182 case <-d.closeChan: 183 return 184 } 185 186 select { 187 case d.transactionsOut <- types.NewTransaction(ts.Payload, resChan): 188 case <-d.closeChan: 189 return 190 } 191 192 var res types.Response 193 select { 194 case res = <-resChan: 195 case <-d.closeChan: 196 return 197 } 198 199 if res.Error() != nil { 200 mDropped.Incr(int64(ts.Payload.Len())) 201 mDroppedBatch.Incr(1) 202 d.log.Warnf("Message dropped due to: %v\n", res.Error()) 203 } 204 205 select { 206 case ts.ResponseChan <- response.NewAck(): 207 case <-d.closeChan: 208 return 209 } 210 } 211 } 212 213 // Consume assigns a messages channel for the output to read. 214 func (d *DropOnError) Consume(ts <-chan types.Transaction) error { 215 if d.transactionsIn != nil { 216 return types.ErrAlreadyStarted 217 } 218 if err := d.wrapped.Consume(d.transactionsOut); err != nil { 219 return err 220 } 221 d.transactionsIn = ts 222 go d.loop() 223 return nil 224 } 225 226 // Connected returns a boolean indicating whether this output is currently 227 // connected to its target. 228 func (d *DropOnError) Connected() bool { 229 return d.wrapped.Connected() 230 } 231 232 // MaxInFlight returns the maximum number of in flight messages permitted by the 233 // output. This value can be used to determine a sensible value for parent 234 // outputs, but should not be relied upon as part of dispatcher logic. 235 func (d *DropOnError) MaxInFlight() (int, bool) { 236 return output.GetMaxInFlight(d.wrapped) 237 } 238 239 // CloseAsync shuts down the DropOnError input and stops processing requests. 240 func (d *DropOnError) CloseAsync() { 241 if atomic.CompareAndSwapInt32(&d.running, 1, 0) { 242 close(d.closeChan) 243 } 244 } 245 246 // WaitForClose blocks until the DropOnError input has closed down. 247 func (d *DropOnError) WaitForClose(timeout time.Duration) error { 248 select { 249 case <-d.closedChan: 250 case <-time.After(timeout): 251 return types.ErrTimeout 252 } 253 return nil 254 } 255 256 //------------------------------------------------------------------------------