github.com/Jeffail/benthos/v3@v3.65.0/lib/pipeline/constructor.go (about) 1 package pipeline 2 3 import ( 4 "fmt" 5 6 "github.com/Jeffail/benthos/v3/internal/interop" 7 "github.com/Jeffail/benthos/v3/lib/log" 8 "github.com/Jeffail/benthos/v3/lib/metrics" 9 "github.com/Jeffail/benthos/v3/lib/processor" 10 "github.com/Jeffail/benthos/v3/lib/types" 11 ) 12 13 // Config is a configuration struct for creating parallel processing pipelines. 14 // The number of resuling parallel processing pipelines will match the number of 15 // threads specified. Processors are executed on each message in the order that 16 // they are written. 17 // 18 // In order to fully utilise each processing thread you must either have a 19 // number of parallel inputs that matches or surpasses the number of pipeline 20 // threads, or use a memory buffer. 21 type Config struct { 22 Threads int `json:"threads" yaml:"threads"` 23 Processors []processor.Config `json:"processors" yaml:"processors"` 24 } 25 26 // NewConfig returns a configuration struct fully populated with default values. 27 func NewConfig() Config { 28 return Config{ 29 Threads: 1, 30 Processors: []processor.Config{}, 31 } 32 } 33 34 // SanitiseConfig returns a sanitised version of the Config, meaning sections 35 // that aren't relevant to behaviour are removed. 36 func SanitiseConfig(conf Config) (interface{}, error) { 37 return conf.Sanitised(false) 38 } 39 40 // Sanitised returns a sanitised version of the config, meaning sections that 41 // aren't relevant to behaviour are removed. Also optionally removes deprecated 42 // fields. 43 func (conf Config) Sanitised(removeDeprecated bool) (interface{}, error) { 44 procConfs := make([]interface{}, len(conf.Processors)) 45 for i, pConf := range conf.Processors { 46 var err error 47 if procConfs[i], err = pConf.Sanitised(removeDeprecated); err != nil { 48 return nil, err 49 } 50 } 51 return map[string]interface{}{ 52 "threads": conf.Threads, 53 "processors": procConfs, 54 }, nil 55 } 56 57 //------------------------------------------------------------------------------ 58 59 // New creates an input type based on an input configuration. 60 func New( 61 conf Config, 62 mgr types.Manager, 63 log log.Modular, 64 stats metrics.Type, 65 processorCtors ...types.ProcessorConstructorFunc, 66 ) (Type, error) { 67 processors := make([]types.Processor, len(conf.Processors)+len(processorCtors)) 68 for j, procConf := range conf.Processors { 69 pMgr, pLog, pMetrics := interop.LabelChild(fmt.Sprintf("processor.%v", j), mgr, log, stats) 70 var err error 71 processors[j], err = processor.New(procConf, pMgr, pLog, pMetrics) 72 if err != nil { 73 return nil, fmt.Errorf("failed to create processor '%v': %v", procConf.Type, err) 74 } 75 } 76 for j, procCtor := range processorCtors { 77 var err error 78 processors[j+len(conf.Processors)], err = procCtor() 79 if err != nil { 80 return nil, fmt.Errorf("failed to create processor: %v", err) 81 } 82 } 83 84 if conf.Threads == 1 { 85 return NewProcessor(log, stats, processors...), nil 86 } 87 return newPoolV2(conf.Threads, log, stats, processors...) 88 }