github.com/Jeffail/benthos/v3@v3.65.0/lib/processor/bounds_check.go (about)

     1  package processor
     2  
     3  import (
     4  	"errors"
     5  	"time"
     6  
     7  	"github.com/Jeffail/benthos/v3/internal/docs"
     8  	"github.com/Jeffail/benthos/v3/lib/log"
     9  	"github.com/Jeffail/benthos/v3/lib/metrics"
    10  	"github.com/Jeffail/benthos/v3/lib/response"
    11  	"github.com/Jeffail/benthos/v3/lib/types"
    12  )
    13  
    14  //------------------------------------------------------------------------------
    15  
    16  func init() {
    17  	Constructors[TypeBoundsCheck] = TypeSpec{
    18  		constructor: NewBoundsCheck,
    19  		Categories: []Category{
    20  			CategoryUtility,
    21  		},
    22  		Summary: `
    23  Removes messages (and batches) that do not fit within certain size boundaries.`,
    24  		FieldSpecs: docs.FieldSpecs{
    25  			docs.FieldCommon("max_part_size", "The maximum size of a message to allow (in bytes)"),
    26  			docs.FieldCommon("min_part_size", "The minimum size of a message to allow (in bytes)"),
    27  			docs.FieldAdvanced("max_parts", "The maximum size of message batches to allow (in message count)"),
    28  			docs.FieldAdvanced("min_parts", "The minimum size of message batches to allow (in message count)"),
    29  		},
    30  	}
    31  }
    32  
    33  //------------------------------------------------------------------------------
    34  
    35  // BoundsCheckConfig contains configuration fields for the BoundsCheck
    36  // processor.
    37  type BoundsCheckConfig struct {
    38  	MaxParts    int `json:"max_parts" yaml:"max_parts"`
    39  	MinParts    int `json:"min_parts" yaml:"min_parts"`
    40  	MaxPartSize int `json:"max_part_size" yaml:"max_part_size"`
    41  	MinPartSize int `json:"min_part_size" yaml:"min_part_size"`
    42  }
    43  
    44  // NewBoundsCheckConfig returns a BoundsCheckConfig with default values.
    45  func NewBoundsCheckConfig() BoundsCheckConfig {
    46  	return BoundsCheckConfig{
    47  		MaxParts:    100,
    48  		MinParts:    1,
    49  		MaxPartSize: 1 * 1024 * 1024 * 1024, // 1GB
    50  		MinPartSize: 1,
    51  	}
    52  }
    53  
    54  //------------------------------------------------------------------------------
    55  
    56  // BoundsCheck is a processor that checks each message against a set of bounds
    57  // and rejects messages if they aren't within them.
    58  type BoundsCheck struct {
    59  	conf  Config
    60  	log   log.Modular
    61  	stats metrics.Type
    62  
    63  	mCount           metrics.StatCounter
    64  	mDropped         metrics.StatCounter
    65  	mDroppedEmpty    metrics.StatCounter
    66  	mDroppedNumParts metrics.StatCounter
    67  	mDroppedPartSize metrics.StatCounter
    68  	mSent            metrics.StatCounter
    69  	mBatchSent       metrics.StatCounter
    70  }
    71  
    72  // NewBoundsCheck returns a BoundsCheck processor.
    73  func NewBoundsCheck(
    74  	conf Config, mgr types.Manager, log log.Modular, stats metrics.Type,
    75  ) (Type, error) {
    76  	return &BoundsCheck{
    77  		conf:  conf,
    78  		log:   log,
    79  		stats: stats,
    80  
    81  		mCount:           stats.GetCounter("count"),
    82  		mDropped:         stats.GetCounter("dropped"),
    83  		mDroppedEmpty:    stats.GetCounter("dropped_empty"),
    84  		mDroppedNumParts: stats.GetCounter("dropped_num_parts"),
    85  		mDroppedPartSize: stats.GetCounter("dropped_part_size"),
    86  		mSent:            stats.GetCounter("sent"),
    87  		mBatchSent:       stats.GetCounter("batch.sent"),
    88  	}, nil
    89  }
    90  
    91  //------------------------------------------------------------------------------
    92  
    93  // ProcessMessage applies the processor to a message, either creating >0
    94  // resulting messages or a response to be sent back to the message source.
    95  func (m *BoundsCheck) ProcessMessage(msg types.Message) ([]types.Message, types.Response) {
    96  	m.mCount.Incr(1)
    97  
    98  	lParts := msg.Len()
    99  	if lParts < m.conf.BoundsCheck.MinParts {
   100  		m.log.Debugf(
   101  			"Rejecting message due to message parts below minimum (%v): %v\n",
   102  			m.conf.BoundsCheck.MinParts, lParts,
   103  		)
   104  		m.mDropped.Incr(1)
   105  		m.mDroppedEmpty.Incr(1)
   106  		return nil, response.NewAck()
   107  	} else if lParts > m.conf.BoundsCheck.MaxParts {
   108  		m.log.Debugf(
   109  			"Rejecting message due to message parts exceeding limit (%v): %v\n",
   110  			m.conf.BoundsCheck.MaxParts, lParts,
   111  		)
   112  		m.mDropped.Incr(1)
   113  		m.mDroppedNumParts.Incr(1)
   114  		return nil, response.NewAck()
   115  	}
   116  
   117  	var reject bool
   118  	msg.Iter(func(i int, p types.Part) error {
   119  		if size := len(p.Get()); size > m.conf.BoundsCheck.MaxPartSize ||
   120  			size < m.conf.BoundsCheck.MinPartSize {
   121  			m.log.Debugf(
   122  				"Rejecting message due to message part size (%v -> %v): %v\n",
   123  				m.conf.BoundsCheck.MinPartSize,
   124  				m.conf.BoundsCheck.MaxPartSize,
   125  				size,
   126  			)
   127  			reject = true
   128  			return errors.New("exit")
   129  		}
   130  		return nil
   131  	})
   132  
   133  	if reject {
   134  		m.mDropped.Incr(1)
   135  		m.mDroppedPartSize.Incr(1)
   136  		return nil, response.NewAck()
   137  	}
   138  
   139  	m.mBatchSent.Incr(1)
   140  	m.mSent.Incr(int64(msg.Len()))
   141  	msgs := [1]types.Message{msg}
   142  	return msgs[:], nil
   143  }
   144  
   145  // CloseAsync shuts down the processor and stops processing requests.
   146  func (m *BoundsCheck) CloseAsync() {
   147  }
   148  
   149  // WaitForClose blocks until the processor has closed down.
   150  func (m *BoundsCheck) WaitForClose(timeout time.Duration) error {
   151  	return nil
   152  }
   153  
   154  //------------------------------------------------------------------------------