github.com/observiq/carbon@v0.9.11-0.20200820160507-1b872e368a5e/operator/buffer/buffer.go (about)

     1  package buffer
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  	"time"
     7  
     8  	"github.com/observiq/carbon/entry"
     9  	"github.com/observiq/carbon/errors"
    10  	"github.com/observiq/carbon/operator"
    11  )
    12  
    13  // Buffer is an entity that buffers log entries to an operator
    14  type Buffer interface {
    15  	Flush(context.Context) error
    16  	Add(interface{}, int) error
    17  	AddWait(context.Context, interface{}, int) error
    18  	SetHandler(BundleHandler)
    19  	Process(context.Context, *entry.Entry) error
    20  }
    21  
    22  func NewConfig() Config {
    23  	return Config{
    24  		BufferType:           "memory",
    25  		DelayThreshold:       operator.Duration{Duration: time.Second},
    26  		BundleCountThreshold: 10_000,
    27  		BundleByteThreshold:  4 * 1024 * 1024 * 1024,   // 4MB
    28  		BundleByteLimit:      4 * 1024 * 1024 * 1024,   // 4MB
    29  		BufferedByteLimit:    500 * 1024 * 1024 * 1024, // 500MB
    30  		HandlerLimit:         16,
    31  		Retry:                NewRetryConfig(),
    32  	}
    33  }
    34  
    35  // Config is the configuration of a buffer
    36  type Config struct {
    37  	BufferType           string            `json:"type,omitempty"                   yaml:"type,omitempty"`
    38  	DelayThreshold       operator.Duration `json:"delay_threshold,omitempty"        yaml:"delay_threshold,omitempty"`
    39  	BundleCountThreshold int               `json:"bundle_count_threshold,omitempty" yaml:"buffer_count_threshold,omitempty"`
    40  	BundleByteThreshold  int               `json:"bundle_byte_threshold,omitempty"  yaml:"bundle_byte_threshold,omitempty"`
    41  	BundleByteLimit      int               `json:"bundle_byte_limit,omitempty"      yaml:"bundle_byte_limit,omitempty"`
    42  	BufferedByteLimit    int               `json:"buffered_byte_limit,omitempty"    yaml:"buffered_byte_limit,omitempty"`
    43  	HandlerLimit         int               `json:"handler_limit,omitempty"          yaml:"handler_limit,omitempty"`
    44  	Retry                RetryConfig       `json:"retry,omitempty"                  yaml:"retry,omitempty"`
    45  }
    46  
    47  // Build will build a buffer from the supplied configuration
    48  func (config *Config) Build() (Buffer, error) {
    49  	switch config.BufferType {
    50  	case "memory", "":
    51  		return NewMemoryBuffer(config), nil
    52  	default:
    53  		return nil, errors.NewError(
    54  			fmt.Sprintf("Invalid buffer type %s", config.BufferType),
    55  			"The only supported buffer type is 'memory'",
    56  		)
    57  	}
    58  }
    59  
    60  func NewRetryConfig() RetryConfig {
    61  	return RetryConfig{
    62  		InitialInterval:     operator.Duration{Duration: 500 * time.Millisecond},
    63  		RandomizationFactor: 0.5,
    64  		Multiplier:          1.5,
    65  		MaxInterval:         operator.Duration{Duration: 15 * time.Minute},
    66  	}
    67  }
    68  
    69  // RetryConfig is the configuration of an entity that will retry processing after an error
    70  type RetryConfig struct {
    71  	InitialInterval     operator.Duration `json:"initial_interval,omitempty"     yaml:"initial_interval,omitempty"`
    72  	RandomizationFactor float64           `json:"randomization_factor,omitempty" yaml:"randomization_factor,omitempty"`
    73  	Multiplier          float64           `json:"multiplier,omitempty"           yaml:"multiplier,omitempty"`
    74  	MaxInterval         operator.Duration `json:"max_interval,omitempty"         yaml:"max_interval,omitempty"`
    75  	MaxElapsedTime      operator.Duration `json:"max_elapsed_time,omitempty"     yaml:"max_elapsed_time,omitempty"`
    76  }