github.com/Jeffail/benthos/v3@v3.65.0/public/service/processor.go (about)

     1  package service
     2  
     3  import (
     4  	"context"
     5  	"time"
     6  
     7  	"github.com/Jeffail/benthos/v3/internal/component/processor"
     8  	"github.com/Jeffail/benthos/v3/lib/message"
     9  	"github.com/Jeffail/benthos/v3/lib/metrics"
    10  	"github.com/Jeffail/benthos/v3/lib/types"
    11  )
    12  
    13  // Processor is a Benthos processor implementation that works against single
    14  // messages.
    15  type Processor interface {
    16  	// Process a message into one or more resulting messages, or return an error
    17  	// if the message could not be processed. If zero messages are returned and
    18  	// the error is nil then the message is filtered.
    19  	//
    20  	// When an error is returned the input message will continue down the
    21  	// pipeline but will be marked with the error with *message.SetError, and
    22  	// metrics and logs will be emitted. The failed message can then be handled
    23  	// with the patterns outlined in https://www.benthos.dev/docs/configuration/error_handling.
    24  	//
    25  	// The Message types returned MUST be derived from the provided message, and
    26  	// CANNOT be custom implementations of Message. In order to copy the
    27  	// provided message use the Copy method.
    28  	Process(context.Context, *Message) (MessageBatch, error)
    29  
    30  	Closer
    31  }
    32  
    33  //------------------------------------------------------------------------------
    34  
    35  // BatchProcessor is a Benthos processor implementation that works against
    36  // batches of messages, which allows windowed processing.
    37  //
    38  // Message batches must be created by upstream components (inputs, buffers, etc)
    39  // otherwise this processor will simply receive batches containing single
    40  // messages.
    41  type BatchProcessor interface {
    42  	// Process a batch of messages into one or more resulting batches, or return
    43  	// an error if the entire batch could not be processed. If zero messages are
    44  	// returned and the error is nil then all messages are filtered.
    45  	//
    46  	// The provided MessageBatch should NOT be modified, in order to return a
    47  	// mutated batch a copy of the slice should be created instead.
    48  	//
    49  	// When an error is returned all of the input messages will continue down
    50  	// the pipeline but will be marked with the error with *message.SetError,
    51  	// and metrics and logs will be emitted.
    52  	//
    53  	// In order to add errors to individual messages of the batch for downstream
    54  	// handling use *message.SetError(err) and return it in the resulting batch
    55  	// with a nil error.
    56  	//
    57  	// The Message types returned MUST be derived from the provided messages,
    58  	// and CANNOT be custom implementations of Message. In order to copy the
    59  	// provided messages use the Copy method.
    60  	ProcessBatch(context.Context, MessageBatch) ([]MessageBatch, error)
    61  
    62  	Closer
    63  }
    64  
    65  //------------------------------------------------------------------------------
    66  
    67  // Implements types.Processor for a Processor.
    68  type airGapProcessor struct {
    69  	p Processor
    70  }
    71  
    72  func newAirGapProcessor(typeStr string, p Processor, stats metrics.Type) types.Processor {
    73  	return processor.NewV2ToV1Processor(typeStr, &airGapProcessor{p}, stats)
    74  }
    75  
    76  func (a *airGapProcessor) Process(ctx context.Context, msg types.Part) ([]types.Part, error) {
    77  	msgs, err := a.p.Process(ctx, newMessageFromPart(msg))
    78  	if err != nil {
    79  		return nil, err
    80  	}
    81  	parts := make([]types.Part, 0, len(msgs))
    82  	for _, msg := range msgs {
    83  		parts = append(parts, msg.part)
    84  	}
    85  	return parts, nil
    86  }
    87  
    88  func (a *airGapProcessor) Close(ctx context.Context) error {
    89  	return a.p.Close(context.Background())
    90  }
    91  
    92  //------------------------------------------------------------------------------
    93  
    94  // Implements types.Processor for a BatchProcessor.
    95  type airGapBatchProcessor struct {
    96  	p BatchProcessor
    97  }
    98  
    99  func newAirGapBatchProcessor(typeStr string, p BatchProcessor, stats metrics.Type) types.Processor {
   100  	return processor.NewV2BatchedToV1Processor(typeStr, &airGapBatchProcessor{p}, stats)
   101  }
   102  
   103  func (a *airGapBatchProcessor) ProcessBatch(ctx context.Context, msgs []types.Part) ([][]types.Part, error) {
   104  	inputBatch := make([]*Message, len(msgs))
   105  	for i, msg := range msgs {
   106  		inputBatch[i] = newMessageFromPart(msg)
   107  	}
   108  
   109  	outputBatches, err := a.p.ProcessBatch(ctx, inputBatch)
   110  	if err != nil {
   111  		return nil, err
   112  	}
   113  
   114  	newBatches := make([][]types.Part, len(outputBatches))
   115  	for i, batch := range outputBatches {
   116  		newBatch := make([]types.Part, len(batch))
   117  		for j, msg := range batch {
   118  			newBatch[j] = msg.part
   119  		}
   120  		newBatches[i] = newBatch
   121  	}
   122  
   123  	return newBatches, nil
   124  }
   125  
   126  func (a *airGapBatchProcessor) Close(ctx context.Context) error {
   127  	return a.p.Close(context.Background())
   128  }
   129  
   130  //------------------------------------------------------------------------------
   131  
   132  // OwnedProcessor provides direct ownership of a processor extracted from a
   133  // plugin config.
   134  type OwnedProcessor struct {
   135  	p types.Processor
   136  }
   137  
   138  // Process a single message, returns either a batch of zero or more resulting
   139  // messages or an error if the message could not be processed.
   140  func (o *OwnedProcessor) Process(ctx context.Context, msg *Message) (MessageBatch, error) {
   141  	outMsg := message.New(nil)
   142  
   143  	// TODO: After V4 we can modify the internal message type to remove this
   144  	// requirement.
   145  	msg.ensureCopied()
   146  	outMsg.Append(msg.part)
   147  
   148  	iMsgs, res := o.p.ProcessMessage(outMsg)
   149  	if res != nil && res.Error() != nil {
   150  		return nil, res.Error()
   151  	}
   152  
   153  	var b MessageBatch
   154  	for _, iMsg := range iMsgs {
   155  		_ = iMsg.Iter(func(i int, part types.Part) error {
   156  			b = append(b, newMessageFromPart(part))
   157  			return nil
   158  		})
   159  	}
   160  	return b, nil
   161  }
   162  
   163  // ProcessBatch attempts to process a batch of messages, returns zero or more
   164  // batches of resulting messages or an error if the messages could not be
   165  // processed.
   166  func (o *OwnedProcessor) ProcessBatch(ctx context.Context, batch MessageBatch) ([]MessageBatch, error) {
   167  	outMsg := message.New(nil)
   168  
   169  	for _, msg := range batch {
   170  		// TODO: After V4 we can modify the internal message type to remove this
   171  		// requirement.
   172  		msg.ensureCopied()
   173  		outMsg.Append(msg.part)
   174  	}
   175  
   176  	iMsgs, res := o.p.ProcessMessage(outMsg)
   177  	if res != nil && res.Error() != nil {
   178  		return nil, res.Error()
   179  	}
   180  
   181  	var batches []MessageBatch
   182  	for _, iMsg := range iMsgs {
   183  		var b MessageBatch
   184  		_ = iMsg.Iter(func(i int, part types.Part) error {
   185  			b = append(b, newMessageFromPart(part))
   186  			return nil
   187  		})
   188  		batches = append(batches, b)
   189  	}
   190  	return batches, nil
   191  }
   192  
   193  // Close the processor, allowing it to clean up resources. It is
   194  func (o *OwnedProcessor) Close(ctx context.Context) error {
   195  	o.p.CloseAsync()
   196  	for {
   197  		// Gross but will do for now until we replace these with context params.
   198  		if err := o.p.WaitForClose(time.Millisecond * 100); err == nil {
   199  			return nil
   200  		}
   201  		select {
   202  		case <-ctx.Done():
   203  			return ctx.Err()
   204  		default:
   205  		}
   206  	}
   207  }