github.com/Jeffail/benthos/v3@v3.65.0/lib/input/broker.go (about) 1 package input 2 3 import ( 4 "encoding/json" 5 "errors" 6 "fmt" 7 "strconv" 8 "strings" 9 10 "github.com/Jeffail/benthos/v3/internal/docs" 11 "github.com/Jeffail/benthos/v3/internal/interop" 12 "github.com/Jeffail/benthos/v3/lib/broker" 13 "github.com/Jeffail/benthos/v3/lib/log" 14 "github.com/Jeffail/benthos/v3/lib/message/batch" 15 "github.com/Jeffail/benthos/v3/lib/metrics" 16 "github.com/Jeffail/benthos/v3/lib/types" 17 "gopkg.in/yaml.v3" 18 ) 19 20 //------------------------------------------------------------------------------ 21 22 var ( 23 // ErrBrokerNoInputs is returned when creating a broker with zero inputs. 24 ErrBrokerNoInputs = errors.New("attempting to create broker input type with no inputs") 25 ) 26 27 //------------------------------------------------------------------------------ 28 29 func init() { 30 Constructors[TypeBroker] = TypeSpec{ 31 constructor: newBrokerHasBatchProcessor, 32 Summary: ` 33 Allows you to combine multiple inputs into a single stream of data, where each input will be read in parallel.`, 34 Description: ` 35 A broker type is configured with its own list of input configurations and a field to specify how many copies of the list of inputs should be created. 36 37 Adding more input types allows you to combine streams from multiple sources into one. For example, reading from both RabbitMQ and Kafka: 38 39 ` + "```yaml" + ` 40 input: 41 broker: 42 copies: 1 43 inputs: 44 - amqp_0_9: 45 url: amqp://guest:guest@localhost:5672/ 46 consumer_tag: benthos-consumer 47 queue: benthos-queue 48 49 # Optional list of input specific processing steps 50 processors: 51 - bloblang: | 52 root.message = this 53 root.meta.link_count = this.links.length() 54 root.user.age = this.user.age.number() 55 56 - kafka: 57 addresses: 58 - localhost:9092 59 client_id: benthos_kafka_input 60 consumer_group: benthos_consumer_group 61 topics: [ benthos_stream:0 ] 62 ` + "```" + ` 63 64 If the number of copies is greater than zero the list will be copied that number 65 of times. For example, if your inputs were of type foo and bar, with 'copies' 66 set to '2', you would end up with two 'foo' inputs and two 'bar' inputs. 67 68 ### Batching 69 70 It's possible to configure a [batch policy](/docs/configuration/batching#batch-policy) 71 with a broker using the ` + "`batching`" + ` fields. When doing this the feeds 72 from all child inputs are combined. Some inputs do not support broker based 73 batching and specify this in their documentation. 74 75 ### Processors 76 77 It is possible to configure [processors](/docs/components/processors/about) at 78 the broker level, where they will be applied to _all_ child inputs, as well as 79 on the individual child inputs. If you have processors at both the broker level 80 _and_ on child inputs then the broker processors will be applied _after_ the 81 child nodes processors.`, 82 Categories: []Category{ 83 CategoryUtility, 84 }, 85 FieldSpecs: docs.FieldSpecs{ 86 docs.FieldAdvanced("copies", "Whatever is specified within `inputs` will be created this many times."), 87 docs.FieldCommon("inputs", "A list of inputs to create.").Array().HasType(docs.FieldTypeInput), 88 batch.FieldSpec(), 89 }, 90 } 91 } 92 93 //------------------------------------------------------------------------------ 94 95 // BrokerConfig contains configuration fields for the Broker input type. 96 type BrokerConfig struct { 97 Copies int `json:"copies" yaml:"copies"` 98 Inputs brokerInputList `json:"inputs" yaml:"inputs"` 99 Batching batch.PolicyConfig `json:"batching" yaml:"batching"` 100 } 101 102 // NewBrokerConfig creates a new BrokerConfig with default values. 103 func NewBrokerConfig() BrokerConfig { 104 return BrokerConfig{ 105 Copies: 1, 106 Inputs: brokerInputList{}, 107 Batching: batch.NewPolicyConfig(), 108 } 109 } 110 111 //------------------------------------------------------------------------------ 112 113 type brokerInputList []Config 114 115 // UnmarshalJSON ensures that when parsing configs that are in a map or slice 116 // the default values are still applied. 117 func (b *brokerInputList) UnmarshalJSON(bytes []byte) error { 118 genericInputs := []interface{}{} 119 if err := json.Unmarshal(bytes, &genericInputs); err != nil { 120 return err 121 } 122 123 inputConfs, err := parseInputConfsWithDefaults(genericInputs) 124 if err != nil { 125 return err 126 } 127 128 *b = inputConfs 129 return nil 130 } 131 132 // UnmarshalYAML ensures that when parsing configs that are in a map or slice 133 // the default values are still applied. 134 func (b *brokerInputList) UnmarshalYAML(unmarshal func(interface{}) error) error { 135 genericInputs := []interface{}{} 136 if err := unmarshal(&genericInputs); err != nil { 137 return err 138 } 139 140 inputConfs, err := parseInputConfsWithDefaults(genericInputs) 141 if err != nil { 142 return err 143 } 144 145 *b = inputConfs 146 return nil 147 } 148 149 //------------------------------------------------------------------------------ 150 151 // parseInputConfsWithDefaults takes a slice of generic input configs and 152 // returns a slice of input configs with default values in place of omitted 153 // values. This is necessary because when unmarshalling config files using 154 // structs you can pre-populate non-reference type struct fields with default 155 // values, but array objects will lose those defaults. 156 // 157 // In order to ensure that omitted values are set to default we initially parse 158 // the array as interface{} types and then individually apply the defaults by 159 // marshalling and unmarshalling. The correct way to do this would be to use 160 // json.RawMessage, but our config files can contain a range of different 161 // formats that we do not know at this stage, therefore we use the more hacky 162 // method as performance is not an issue at this stage. 163 func parseInputConfsWithDefaults(rawInputs []interface{}) ([]Config, error) { 164 inputConfs := []Config{} 165 166 // NOTE: Use yaml here as it supports more types than JSON 167 // (map[interface{}]interface{}). 168 for i, boxedConfig := range rawInputs { 169 newConfigs := make([]Config, 1) 170 label := broker.GetGenericType(boxedConfig) 171 172 // TODO: V4 Remove ditto 173 if i > 0 && strings.Index(label, "ditto") == 0 { 174 broker.RemoveGenericType(boxedConfig) 175 176 // Check if there is a ditto multiplier. 177 if len(label) > 5 && label[5] == '_' { 178 if label[6:] == "0" { 179 // This is a special case where we are expressing that 180 // we want to end up with zero duplicates. 181 newConfigs = nil 182 } else { 183 n, err := strconv.Atoi(label[6:]) 184 if err != nil { 185 return nil, fmt.Errorf("failed to parse ditto multiplier: %v", err) 186 } 187 newConfigs = make([]Config, n) 188 } 189 } else { 190 newConfigs = make([]Config, 1) 191 } 192 193 broker.ComplementGenericConfig(boxedConfig, rawInputs[i-1]) 194 } 195 196 for _, conf := range newConfigs { 197 rawBytes, err := yaml.Marshal(boxedConfig) 198 if err != nil { 199 return nil, err 200 } 201 if err := yaml.Unmarshal(rawBytes, &conf); err != nil { 202 return nil, err 203 } 204 inputConfs = append(inputConfs, conf) 205 } 206 } 207 208 return inputConfs, nil 209 } 210 211 //------------------------------------------------------------------------------ 212 213 // NewBroker creates a new Broker input type. 214 func NewBroker( 215 conf Config, 216 mgr types.Manager, 217 log log.Modular, 218 stats metrics.Type, 219 pipelines ...types.PipelineConstructorFunc, 220 ) (Type, error) { 221 return newBrokerHasBatchProcessor(false, conf, mgr, log, stats, pipelines...) 222 } 223 224 // Deprecated: This is a hack for until the batch processor is removed. 225 // TODO: V4 Remove this. 226 func newBrokerHasBatchProcessor( 227 hasBatchProc bool, 228 conf Config, 229 mgr types.Manager, 230 log log.Modular, 231 stats metrics.Type, 232 pipelines ...types.PipelineConstructorFunc, 233 ) (Type, error) { 234 hasBatchProc, pipelines = appendProcessorsFromConfigBatchAware(hasBatchProc, conf, mgr, log, stats, pipelines...) 235 236 lInputs := len(conf.Broker.Inputs) * conf.Broker.Copies 237 238 if lInputs <= 0 { 239 return nil, ErrBrokerNoInputs 240 } 241 242 var err error 243 var b Type 244 if lInputs == 1 { 245 if b, err = newHasBatchProcessor(hasBatchProc, conf.Broker.Inputs[0], mgr, log, stats, pipelines...); err != nil { 246 return nil, err 247 } 248 } else { 249 inputs := make([]types.Producer, lInputs) 250 251 for j := 0; j < conf.Broker.Copies; j++ { 252 for i, iConf := range conf.Broker.Inputs { 253 iMgr, iLog, iStats := interop.LabelChild(fmt.Sprintf("broker.inputs.%v", i), mgr, log, stats) 254 iStats = metrics.Combine(stats, iStats) 255 inputs[len(conf.Broker.Inputs)*j+i], err = newHasBatchProcessor( 256 hasBatchProc, iConf, iMgr, iLog, iStats, 257 pipelines..., 258 ) 259 if err != nil { 260 return nil, fmt.Errorf("failed to create input '%v' type '%v': %v", i, iConf.Type, err) 261 } 262 } 263 } 264 265 if b, err = broker.NewFanIn(inputs, stats); err != nil { 266 return nil, err 267 } 268 } 269 270 if conf.Broker.Batching.IsNoop() { 271 return b, nil 272 } 273 274 bMgr, bLog, bStats := interop.LabelChild("batching", mgr, log, stats) 275 policy, err := batch.NewPolicy(conf.Broker.Batching, bMgr, bLog, bStats) 276 if err != nil { 277 return nil, fmt.Errorf("failed to construct batch policy: %v", err) 278 } 279 280 return NewBatcher(policy, b, log, stats), nil 281 } 282 283 //------------------------------------------------------------------------------