github.com/Jeffail/benthos/v3@v3.65.0/lib/buffer/memory.go (about)

     1  package buffer
     2  
     3  import (
     4  	"fmt"
     5  
     6  	"github.com/Jeffail/benthos/v3/internal/docs"
     7  	"github.com/Jeffail/benthos/v3/lib/buffer/parallel"
     8  	"github.com/Jeffail/benthos/v3/lib/log"
     9  	"github.com/Jeffail/benthos/v3/lib/message/batch"
    10  	"github.com/Jeffail/benthos/v3/lib/metrics"
    11  	"github.com/Jeffail/benthos/v3/lib/types"
    12  )
    13  
    14  //------------------------------------------------------------------------------
    15  
    16  func init() {
    17  	Constructors[TypeMemory] = TypeSpec{
    18  		constructor: NewMemory,
    19  		Summary: `
    20  Stores consumed messages in memory and acknowledges them at the input level.
    21  During shutdown Benthos will make a best attempt at flushing all remaining
    22  messages before exiting cleanly.`,
    23  		Description: `
    24  This buffer is appropriate when consuming messages from inputs that do not
    25  gracefully handle back pressure and where delivery guarantees aren't critical.
    26  
    27  This buffer has a configurable limit, where consumption will be stopped with
    28  back pressure upstream if the total size of messages in the buffer reaches this
    29  amount. Since this calculation is only an estimate, and the real size of
    30  messages in RAM is always higher, it is recommended to set the limit
    31  significantly below the amount of RAM available.
    32  
    33  ## Delivery Guarantees
    34  
    35  This buffer intentionally weakens the delivery guarantees of the pipeline and
    36  therefore should never be used in places where data loss is unacceptable.
    37  
    38  ## Batching
    39  
    40  It is possible to batch up messages sent from this buffer using a
    41  [batch policy](/docs/configuration/batching#batch-policy).`,
    42  		FieldSpecs: docs.FieldSpecs{
    43  			docs.FieldCommon("limit", "The maximum buffer size (in bytes) to allow before applying backpressure upstream."),
    44  			docs.FieldCommon("batch_policy", "Optionally configure a policy to flush buffered messages in batches.").WithChildren(
    45  				append(docs.FieldSpecs{
    46  					docs.FieldCommon("enabled", "Whether to batch messages as they are flushed."),
    47  				}, batch.FieldSpec().Children...)...,
    48  			),
    49  		},
    50  	}
    51  }
    52  
    53  //------------------------------------------------------------------------------
    54  
    55  // EnabledBatchPolicyConfig is a batch.PolicyConfig with an enable field.
    56  type EnabledBatchPolicyConfig struct {
    57  	Enabled            bool `json:"enabled" yaml:"enabled"`
    58  	batch.PolicyConfig `json:",inline" yaml:",inline"`
    59  }
    60  
    61  // MemoryConfig is config values for a purely memory based ring buffer type.
    62  type MemoryConfig struct {
    63  	Limit       int                      `json:"limit" yaml:"limit"`
    64  	BatchPolicy EnabledBatchPolicyConfig `json:"batch_policy" yaml:"batch_policy"`
    65  }
    66  
    67  // NewMemoryConfig creates a new MemoryConfig with default values.
    68  func NewMemoryConfig() MemoryConfig {
    69  	return MemoryConfig{
    70  		Limit: 1024 * 1024 * 500, // 500MB
    71  		BatchPolicy: EnabledBatchPolicyConfig{
    72  			Enabled:      false,
    73  			PolicyConfig: batch.NewPolicyConfig(),
    74  		},
    75  	}
    76  }
    77  
    78  //------------------------------------------------------------------------------
    79  
    80  // NewMemory creates a buffer held in memory.
    81  func NewMemory(config Config, mgr types.Manager, log log.Modular, stats metrics.Type) (Type, error) {
    82  	wrap := NewParallelWrapper(config, parallel.NewMemory(config.Memory.Limit), log, stats)
    83  	if !config.Memory.BatchPolicy.Enabled {
    84  		return wrap, nil
    85  	}
    86  	pol, err := batch.NewPolicy(config.Memory.BatchPolicy.PolicyConfig, mgr, log, stats)
    87  	if err != nil {
    88  		return nil, fmt.Errorf("batch policy config error: %v", err)
    89  	}
    90  	return NewParallelBatcher(pol, wrap, log, stats), nil
    91  }
    92  
    93  //------------------------------------------------------------------------------