github.com/Jeffail/benthos/v3@v3.65.0/public/service/processor.go (about) 1 package service 2 3 import ( 4 "context" 5 "time" 6 7 "github.com/Jeffail/benthos/v3/internal/component/processor" 8 "github.com/Jeffail/benthos/v3/lib/message" 9 "github.com/Jeffail/benthos/v3/lib/metrics" 10 "github.com/Jeffail/benthos/v3/lib/types" 11 ) 12 13 // Processor is a Benthos processor implementation that works against single 14 // messages. 15 type Processor interface { 16 // Process a message into one or more resulting messages, or return an error 17 // if the message could not be processed. If zero messages are returned and 18 // the error is nil then the message is filtered. 19 // 20 // When an error is returned the input message will continue down the 21 // pipeline but will be marked with the error with *message.SetError, and 22 // metrics and logs will be emitted. The failed message can then be handled 23 // with the patterns outlined in https://www.benthos.dev/docs/configuration/error_handling. 24 // 25 // The Message types returned MUST be derived from the provided message, and 26 // CANNOT be custom implementations of Message. In order to copy the 27 // provided message use the Copy method. 28 Process(context.Context, *Message) (MessageBatch, error) 29 30 Closer 31 } 32 33 //------------------------------------------------------------------------------ 34 35 // BatchProcessor is a Benthos processor implementation that works against 36 // batches of messages, which allows windowed processing. 37 // 38 // Message batches must be created by upstream components (inputs, buffers, etc) 39 // otherwise this processor will simply receive batches containing single 40 // messages. 41 type BatchProcessor interface { 42 // Process a batch of messages into one or more resulting batches, or return 43 // an error if the entire batch could not be processed. If zero messages are 44 // returned and the error is nil then all messages are filtered. 45 // 46 // The provided MessageBatch should NOT be modified, in order to return a 47 // mutated batch a copy of the slice should be created instead. 48 // 49 // When an error is returned all of the input messages will continue down 50 // the pipeline but will be marked with the error with *message.SetError, 51 // and metrics and logs will be emitted. 52 // 53 // In order to add errors to individual messages of the batch for downstream 54 // handling use *message.SetError(err) and return it in the resulting batch 55 // with a nil error. 56 // 57 // The Message types returned MUST be derived from the provided messages, 58 // and CANNOT be custom implementations of Message. In order to copy the 59 // provided messages use the Copy method. 60 ProcessBatch(context.Context, MessageBatch) ([]MessageBatch, error) 61 62 Closer 63 } 64 65 //------------------------------------------------------------------------------ 66 67 // Implements types.Processor for a Processor. 68 type airGapProcessor struct { 69 p Processor 70 } 71 72 func newAirGapProcessor(typeStr string, p Processor, stats metrics.Type) types.Processor { 73 return processor.NewV2ToV1Processor(typeStr, &airGapProcessor{p}, stats) 74 } 75 76 func (a *airGapProcessor) Process(ctx context.Context, msg types.Part) ([]types.Part, error) { 77 msgs, err := a.p.Process(ctx, newMessageFromPart(msg)) 78 if err != nil { 79 return nil, err 80 } 81 parts := make([]types.Part, 0, len(msgs)) 82 for _, msg := range msgs { 83 parts = append(parts, msg.part) 84 } 85 return parts, nil 86 } 87 88 func (a *airGapProcessor) Close(ctx context.Context) error { 89 return a.p.Close(context.Background()) 90 } 91 92 //------------------------------------------------------------------------------ 93 94 // Implements types.Processor for a BatchProcessor. 95 type airGapBatchProcessor struct { 96 p BatchProcessor 97 } 98 99 func newAirGapBatchProcessor(typeStr string, p BatchProcessor, stats metrics.Type) types.Processor { 100 return processor.NewV2BatchedToV1Processor(typeStr, &airGapBatchProcessor{p}, stats) 101 } 102 103 func (a *airGapBatchProcessor) ProcessBatch(ctx context.Context, msgs []types.Part) ([][]types.Part, error) { 104 inputBatch := make([]*Message, len(msgs)) 105 for i, msg := range msgs { 106 inputBatch[i] = newMessageFromPart(msg) 107 } 108 109 outputBatches, err := a.p.ProcessBatch(ctx, inputBatch) 110 if err != nil { 111 return nil, err 112 } 113 114 newBatches := make([][]types.Part, len(outputBatches)) 115 for i, batch := range outputBatches { 116 newBatch := make([]types.Part, len(batch)) 117 for j, msg := range batch { 118 newBatch[j] = msg.part 119 } 120 newBatches[i] = newBatch 121 } 122 123 return newBatches, nil 124 } 125 126 func (a *airGapBatchProcessor) Close(ctx context.Context) error { 127 return a.p.Close(context.Background()) 128 } 129 130 //------------------------------------------------------------------------------ 131 132 // OwnedProcessor provides direct ownership of a processor extracted from a 133 // plugin config. 134 type OwnedProcessor struct { 135 p types.Processor 136 } 137 138 // Process a single message, returns either a batch of zero or more resulting 139 // messages or an error if the message could not be processed. 140 func (o *OwnedProcessor) Process(ctx context.Context, msg *Message) (MessageBatch, error) { 141 outMsg := message.New(nil) 142 143 // TODO: After V4 we can modify the internal message type to remove this 144 // requirement. 145 msg.ensureCopied() 146 outMsg.Append(msg.part) 147 148 iMsgs, res := o.p.ProcessMessage(outMsg) 149 if res != nil && res.Error() != nil { 150 return nil, res.Error() 151 } 152 153 var b MessageBatch 154 for _, iMsg := range iMsgs { 155 _ = iMsg.Iter(func(i int, part types.Part) error { 156 b = append(b, newMessageFromPart(part)) 157 return nil 158 }) 159 } 160 return b, nil 161 } 162 163 // ProcessBatch attempts to process a batch of messages, returns zero or more 164 // batches of resulting messages or an error if the messages could not be 165 // processed. 166 func (o *OwnedProcessor) ProcessBatch(ctx context.Context, batch MessageBatch) ([]MessageBatch, error) { 167 outMsg := message.New(nil) 168 169 for _, msg := range batch { 170 // TODO: After V4 we can modify the internal message type to remove this 171 // requirement. 172 msg.ensureCopied() 173 outMsg.Append(msg.part) 174 } 175 176 iMsgs, res := o.p.ProcessMessage(outMsg) 177 if res != nil && res.Error() != nil { 178 return nil, res.Error() 179 } 180 181 var batches []MessageBatch 182 for _, iMsg := range iMsgs { 183 var b MessageBatch 184 _ = iMsg.Iter(func(i int, part types.Part) error { 185 b = append(b, newMessageFromPart(part)) 186 return nil 187 }) 188 batches = append(batches, b) 189 } 190 return batches, nil 191 } 192 193 // Close the processor, allowing it to clean up resources. It is 194 func (o *OwnedProcessor) Close(ctx context.Context) error { 195 o.p.CloseAsync() 196 for { 197 // Gross but will do for now until we replace these with context params. 198 if err := o.p.WaitForClose(time.Millisecond * 100); err == nil { 199 return nil 200 } 201 select { 202 case <-ctx.Done(): 203 return ctx.Err() 204 default: 205 } 206 } 207 }