github.com/Jeffail/benthos/v3@v3.65.0/lib/processor/catch.go (about) 1 package processor 2 3 import ( 4 "fmt" 5 "time" 6 7 "github.com/Jeffail/benthos/v3/internal/docs" 8 "github.com/Jeffail/benthos/v3/internal/interop" 9 "github.com/Jeffail/benthos/v3/lib/log" 10 "github.com/Jeffail/benthos/v3/lib/message" 11 "github.com/Jeffail/benthos/v3/lib/metrics" 12 "github.com/Jeffail/benthos/v3/lib/types" 13 ) 14 15 //------------------------------------------------------------------------------ 16 17 func init() { 18 Constructors[TypeCatch] = TypeSpec{ 19 constructor: NewCatch, 20 Categories: []Category{ 21 CategoryComposition, 22 }, 23 Summary: ` 24 Applies a list of child processors _only_ when a previous processing step has 25 failed.`, 26 Description: ` 27 Behaves similarly to the ` + "[`for_each`](/docs/components/processors/for_each)" + ` processor, where a 28 list of child processors are applied to individual messages of a batch. However, 29 processors are only applied to messages that failed a processing step prior to 30 the catch. 31 32 For example, with the following config: 33 34 ` + "```yaml" + ` 35 pipeline: 36 processors: 37 - resource: foo 38 - catch: 39 - resource: bar 40 - resource: baz 41 ` + "```" + ` 42 43 If the processor ` + "`foo`" + ` fails for a particular message, that message 44 will be fed into the processors ` + "`bar` and `baz`" + `. Messages that do not 45 fail for the processor ` + "`foo`" + ` will skip these processors. 46 47 When messages leave the catch block their fail flags are cleared. This processor 48 is useful for when it's possible to recover failed messages, or when special 49 actions (such as logging/metrics) are required before dropping them. 50 51 More information about error handing can be found [here](/docs/configuration/error_handling).`, 52 config: docs.FieldComponent().Array().HasType(docs.FieldTypeProcessor). 53 Linter(func(ctx docs.LintContext, line, col int, value interface{}) []docs.Lint { 54 childProcs, ok := value.([]interface{}) 55 if !ok { 56 return nil 57 } 58 for _, child := range childProcs { 59 childObj, ok := child.(map[string]interface{}) 60 if !ok { 61 continue 62 } 63 if _, exists := childObj["catch"]; exists { 64 // No need to lint as a nested catch will clear errors, 65 // allowing nested try blocks to work as expected. 66 return nil 67 } 68 if _, exists := childObj["try"]; exists { 69 return []docs.Lint{ 70 docs.NewLintError(line, "`catch` block contains a `try` block which will never execute due to errors only being cleared at the end of the `catch`, for more information about nesting `try` within `catch` read: https://www.benthos.dev/docs/components/processors/try#nesting-within-a-catch-block"), 71 } 72 } 73 } 74 return nil 75 }), 76 } 77 } 78 79 //------------------------------------------------------------------------------ 80 81 // CatchConfig is a config struct containing fields for the Catch processor. 82 type CatchConfig []Config 83 84 // NewCatchConfig returns a default CatchConfig. 85 func NewCatchConfig() CatchConfig { 86 return []Config{} 87 } 88 89 //------------------------------------------------------------------------------ 90 91 // Catch is a processor that applies a list of child processors to each message of 92 // a batch individually, where processors are skipped for messages that failed a 93 // previous processor step. 94 type Catch struct { 95 children []types.Processor 96 97 log log.Modular 98 99 mCount metrics.StatCounter 100 mErr metrics.StatCounter 101 mSent metrics.StatCounter 102 mBatchSent metrics.StatCounter 103 } 104 105 // NewCatch returns a Catch processor. 106 func NewCatch( 107 conf Config, mgr types.Manager, log log.Modular, stats metrics.Type, 108 ) (Type, error) { 109 var children []types.Processor 110 for i, pconf := range conf.Catch { 111 pMgr, pLog, pStats := interop.LabelChild(fmt.Sprintf("%v", i), mgr, log, stats) 112 proc, err := New(pconf, pMgr, pLog, pStats) 113 if err != nil { 114 return nil, err 115 } 116 children = append(children, proc) 117 } 118 return &Catch{ 119 children: children, 120 log: log, 121 122 mCount: stats.GetCounter("count"), 123 mErr: stats.GetCounter("error"), 124 mSent: stats.GetCounter("sent"), 125 mBatchSent: stats.GetCounter("batch.sent"), 126 }, nil 127 } 128 129 //------------------------------------------------------------------------------ 130 131 // ProcessMessage applies the processor to a message, either creating >0 132 // resulting messages or a response to be sent back to the message source. 133 func (p *Catch) ProcessMessage(msg types.Message) ([]types.Message, types.Response) { 134 p.mCount.Incr(1) 135 136 resultMsgs := make([]types.Message, msg.Len()) 137 msg.Iter(func(i int, p types.Part) error { 138 tmpMsg := message.New(nil) 139 tmpMsg.SetAll([]types.Part{p}) 140 resultMsgs[i] = tmpMsg 141 return nil 142 }) 143 144 var res types.Response 145 if resultMsgs, res = ExecuteCatchAll(p.children, resultMsgs...); res != nil { 146 return nil, res 147 } 148 149 resMsg := message.New(nil) 150 for _, m := range resultMsgs { 151 m.Iter(func(i int, p types.Part) error { 152 resMsg.Append(p) 153 return nil 154 }) 155 } 156 if resMsg.Len() == 0 { 157 return nil, res 158 } 159 160 resMsg.Iter(func(i int, p types.Part) error { 161 ClearFail(p) 162 return nil 163 }) 164 165 p.mBatchSent.Incr(1) 166 p.mSent.Incr(int64(resMsg.Len())) 167 168 resMsgs := [1]types.Message{resMsg} 169 return resMsgs[:], nil 170 } 171 172 // CloseAsync shuts down the processor and stops processing requests. 173 func (p *Catch) CloseAsync() { 174 for _, c := range p.children { 175 c.CloseAsync() 176 } 177 } 178 179 // WaitForClose blocks until the processor has closed down. 180 func (p *Catch) WaitForClose(timeout time.Duration) error { 181 stopBy := time.Now().Add(timeout) 182 for _, c := range p.children { 183 if err := c.WaitForClose(time.Until(stopBy)); err != nil { 184 return err 185 } 186 } 187 return nil 188 } 189 190 //------------------------------------------------------------------------------