github.com/Jeffail/benthos/v3@v3.65.0/lib/pipeline/constructor.go (about)

     1  package pipeline
     2  
     3  import (
     4  	"fmt"
     5  
     6  	"github.com/Jeffail/benthos/v3/internal/interop"
     7  	"github.com/Jeffail/benthos/v3/lib/log"
     8  	"github.com/Jeffail/benthos/v3/lib/metrics"
     9  	"github.com/Jeffail/benthos/v3/lib/processor"
    10  	"github.com/Jeffail/benthos/v3/lib/types"
    11  )
    12  
    13  // Config is a configuration struct for creating parallel processing pipelines.
    14  // The number of resuling parallel processing pipelines will match the number of
    15  // threads specified. Processors are executed on each message in the order that
    16  // they are written.
    17  //
    18  // In order to fully utilise each processing thread you must either have a
    19  // number of parallel inputs that matches or surpasses the number of pipeline
    20  // threads, or use a memory buffer.
    21  type Config struct {
    22  	Threads    int                `json:"threads" yaml:"threads"`
    23  	Processors []processor.Config `json:"processors" yaml:"processors"`
    24  }
    25  
    26  // NewConfig returns a configuration struct fully populated with default values.
    27  func NewConfig() Config {
    28  	return Config{
    29  		Threads:    1,
    30  		Processors: []processor.Config{},
    31  	}
    32  }
    33  
    34  // SanitiseConfig returns a sanitised version of the Config, meaning sections
    35  // that aren't relevant to behaviour are removed.
    36  func SanitiseConfig(conf Config) (interface{}, error) {
    37  	return conf.Sanitised(false)
    38  }
    39  
    40  // Sanitised returns a sanitised version of the config, meaning sections that
    41  // aren't relevant to behaviour are removed. Also optionally removes deprecated
    42  // fields.
    43  func (conf Config) Sanitised(removeDeprecated bool) (interface{}, error) {
    44  	procConfs := make([]interface{}, len(conf.Processors))
    45  	for i, pConf := range conf.Processors {
    46  		var err error
    47  		if procConfs[i], err = pConf.Sanitised(removeDeprecated); err != nil {
    48  			return nil, err
    49  		}
    50  	}
    51  	return map[string]interface{}{
    52  		"threads":    conf.Threads,
    53  		"processors": procConfs,
    54  	}, nil
    55  }
    56  
    57  //------------------------------------------------------------------------------
    58  
    59  // New creates an input type based on an input configuration.
    60  func New(
    61  	conf Config,
    62  	mgr types.Manager,
    63  	log log.Modular,
    64  	stats metrics.Type,
    65  	processorCtors ...types.ProcessorConstructorFunc,
    66  ) (Type, error) {
    67  	processors := make([]types.Processor, len(conf.Processors)+len(processorCtors))
    68  	for j, procConf := range conf.Processors {
    69  		pMgr, pLog, pMetrics := interop.LabelChild(fmt.Sprintf("processor.%v", j), mgr, log, stats)
    70  		var err error
    71  		processors[j], err = processor.New(procConf, pMgr, pLog, pMetrics)
    72  		if err != nil {
    73  			return nil, fmt.Errorf("failed to create processor '%v': %v", procConf.Type, err)
    74  		}
    75  	}
    76  	for j, procCtor := range processorCtors {
    77  		var err error
    78  		processors[j+len(conf.Processors)], err = procCtor()
    79  		if err != nil {
    80  			return nil, fmt.Errorf("failed to create processor: %v", err)
    81  		}
    82  	}
    83  
    84  	if conf.Threads == 1 {
    85  		return NewProcessor(log, stats, processors...), nil
    86  	}
    87  	return newPoolV2(conf.Threads, log, stats, processors...)
    88  }