github.com/Jeffail/benthos/v3@v3.65.0/lib/processor/bounds_check.go (about) 1 package processor 2 3 import ( 4 "errors" 5 "time" 6 7 "github.com/Jeffail/benthos/v3/internal/docs" 8 "github.com/Jeffail/benthos/v3/lib/log" 9 "github.com/Jeffail/benthos/v3/lib/metrics" 10 "github.com/Jeffail/benthos/v3/lib/response" 11 "github.com/Jeffail/benthos/v3/lib/types" 12 ) 13 14 //------------------------------------------------------------------------------ 15 16 func init() { 17 Constructors[TypeBoundsCheck] = TypeSpec{ 18 constructor: NewBoundsCheck, 19 Categories: []Category{ 20 CategoryUtility, 21 }, 22 Summary: ` 23 Removes messages (and batches) that do not fit within certain size boundaries.`, 24 FieldSpecs: docs.FieldSpecs{ 25 docs.FieldCommon("max_part_size", "The maximum size of a message to allow (in bytes)"), 26 docs.FieldCommon("min_part_size", "The minimum size of a message to allow (in bytes)"), 27 docs.FieldAdvanced("max_parts", "The maximum size of message batches to allow (in message count)"), 28 docs.FieldAdvanced("min_parts", "The minimum size of message batches to allow (in message count)"), 29 }, 30 } 31 } 32 33 //------------------------------------------------------------------------------ 34 35 // BoundsCheckConfig contains configuration fields for the BoundsCheck 36 // processor. 37 type BoundsCheckConfig struct { 38 MaxParts int `json:"max_parts" yaml:"max_parts"` 39 MinParts int `json:"min_parts" yaml:"min_parts"` 40 MaxPartSize int `json:"max_part_size" yaml:"max_part_size"` 41 MinPartSize int `json:"min_part_size" yaml:"min_part_size"` 42 } 43 44 // NewBoundsCheckConfig returns a BoundsCheckConfig with default values. 45 func NewBoundsCheckConfig() BoundsCheckConfig { 46 return BoundsCheckConfig{ 47 MaxParts: 100, 48 MinParts: 1, 49 MaxPartSize: 1 * 1024 * 1024 * 1024, // 1GB 50 MinPartSize: 1, 51 } 52 } 53 54 //------------------------------------------------------------------------------ 55 56 // BoundsCheck is a processor that checks each message against a set of bounds 57 // and rejects messages if they aren't within them. 58 type BoundsCheck struct { 59 conf Config 60 log log.Modular 61 stats metrics.Type 62 63 mCount metrics.StatCounter 64 mDropped metrics.StatCounter 65 mDroppedEmpty metrics.StatCounter 66 mDroppedNumParts metrics.StatCounter 67 mDroppedPartSize metrics.StatCounter 68 mSent metrics.StatCounter 69 mBatchSent metrics.StatCounter 70 } 71 72 // NewBoundsCheck returns a BoundsCheck processor. 73 func NewBoundsCheck( 74 conf Config, mgr types.Manager, log log.Modular, stats metrics.Type, 75 ) (Type, error) { 76 return &BoundsCheck{ 77 conf: conf, 78 log: log, 79 stats: stats, 80 81 mCount: stats.GetCounter("count"), 82 mDropped: stats.GetCounter("dropped"), 83 mDroppedEmpty: stats.GetCounter("dropped_empty"), 84 mDroppedNumParts: stats.GetCounter("dropped_num_parts"), 85 mDroppedPartSize: stats.GetCounter("dropped_part_size"), 86 mSent: stats.GetCounter("sent"), 87 mBatchSent: stats.GetCounter("batch.sent"), 88 }, nil 89 } 90 91 //------------------------------------------------------------------------------ 92 93 // ProcessMessage applies the processor to a message, either creating >0 94 // resulting messages or a response to be sent back to the message source. 95 func (m *BoundsCheck) ProcessMessage(msg types.Message) ([]types.Message, types.Response) { 96 m.mCount.Incr(1) 97 98 lParts := msg.Len() 99 if lParts < m.conf.BoundsCheck.MinParts { 100 m.log.Debugf( 101 "Rejecting message due to message parts below minimum (%v): %v\n", 102 m.conf.BoundsCheck.MinParts, lParts, 103 ) 104 m.mDropped.Incr(1) 105 m.mDroppedEmpty.Incr(1) 106 return nil, response.NewAck() 107 } else if lParts > m.conf.BoundsCheck.MaxParts { 108 m.log.Debugf( 109 "Rejecting message due to message parts exceeding limit (%v): %v\n", 110 m.conf.BoundsCheck.MaxParts, lParts, 111 ) 112 m.mDropped.Incr(1) 113 m.mDroppedNumParts.Incr(1) 114 return nil, response.NewAck() 115 } 116 117 var reject bool 118 msg.Iter(func(i int, p types.Part) error { 119 if size := len(p.Get()); size > m.conf.BoundsCheck.MaxPartSize || 120 size < m.conf.BoundsCheck.MinPartSize { 121 m.log.Debugf( 122 "Rejecting message due to message part size (%v -> %v): %v\n", 123 m.conf.BoundsCheck.MinPartSize, 124 m.conf.BoundsCheck.MaxPartSize, 125 size, 126 ) 127 reject = true 128 return errors.New("exit") 129 } 130 return nil 131 }) 132 133 if reject { 134 m.mDropped.Incr(1) 135 m.mDroppedPartSize.Incr(1) 136 return nil, response.NewAck() 137 } 138 139 m.mBatchSent.Incr(1) 140 m.mSent.Incr(int64(msg.Len())) 141 msgs := [1]types.Message{msg} 142 return msgs[:], nil 143 } 144 145 // CloseAsync shuts down the processor and stops processing requests. 146 func (m *BoundsCheck) CloseAsync() { 147 } 148 149 // WaitForClose blocks until the processor has closed down. 150 func (m *BoundsCheck) WaitForClose(timeout time.Duration) error { 151 return nil 152 } 153 154 //------------------------------------------------------------------------------