github.com/observiq/carbon@v0.9.11-0.20200820160507-1b872e368a5e/operator/buffer/memory_buffer.go (about)

     1  package buffer
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  	"sync/atomic"
     7  	"time"
     8  
     9  	"github.com/cenkalti/backoff/v4"
    10  	"github.com/observiq/carbon/entry"
    11  	"go.uber.org/zap"
    12  	"google.golang.org/api/support/bundler"
    13  )
    14  
    15  // MemoryBuffer is a buffer that holds entries in memory
    16  type MemoryBuffer struct {
    17  	*bundler.Bundler
    18  	config *Config
    19  	cancel context.CancelFunc
    20  }
    21  
    22  // NewMemoryBuffer will return a new memory buffer with the supplied configuration
    23  func NewMemoryBuffer(config *Config) *MemoryBuffer {
    24  	return &MemoryBuffer{config: config}
    25  }
    26  
    27  // BundleHandler is an interface that process multiple entries
    28  type BundleHandler interface {
    29  	ProcessMulti(context.Context, []*entry.Entry) error
    30  	Logger() *zap.SugaredLogger
    31  }
    32  
    33  // SetHandler will set the handler of the memory buffer
    34  func (m *MemoryBuffer) SetHandler(handler BundleHandler) {
    35  	ctx, cancel := context.WithCancel(context.Background())
    36  	currentBundleID := int64(0)
    37  	handleFunc := func(entries interface{}) {
    38  		bundleID := atomic.AddInt64(&currentBundleID, 1)
    39  		b := m.NewExponentialBackOff()
    40  		for {
    41  			err := handler.ProcessMulti(ctx, entries.([]*entry.Entry))
    42  			if err != nil {
    43  				duration := b.NextBackOff()
    44  				if duration == backoff.Stop {
    45  					handler.Logger().Errorw("Failed to flush bundle. Not retrying because we are beyond max backoff", zap.Any("error", err), "bundle_id", bundleID)
    46  					break
    47  				} else {
    48  					handler.Logger().Warnw("Failed to flush bundle", zap.Any("error", err), "backoff_time", duration.String(), "bundle_id", bundleID)
    49  					select {
    50  					case <-ctx.Done():
    51  						handler.Logger().Debugw("Flush retry cancelled by context", "bundle_id", bundleID)
    52  						return
    53  					case <-time.After(duration):
    54  						continue
    55  					}
    56  				}
    57  			}
    58  
    59  			break
    60  		}
    61  	}
    62  
    63  	bd := bundler.NewBundler(&entry.Entry{}, handleFunc)
    64  	bd.DelayThreshold = m.config.DelayThreshold.Raw()
    65  	bd.BundleCountThreshold = m.config.BundleCountThreshold
    66  	bd.BundleByteThreshold = m.config.BundleByteThreshold
    67  	bd.BundleByteLimit = m.config.BundleByteLimit
    68  	bd.BufferedByteLimit = m.config.BufferedByteLimit
    69  	bd.HandlerLimit = m.config.HandlerLimit
    70  
    71  	m.Bundler = bd
    72  	m.cancel = cancel
    73  }
    74  
    75  // Flush will flush the memory buffer
    76  func (m *MemoryBuffer) Flush(ctx context.Context) error {
    77  	finished := make(chan struct{})
    78  	go func() {
    79  		m.Bundler.Flush()
    80  		close(finished)
    81  	}()
    82  
    83  	select {
    84  	case <-finished:
    85  		return nil
    86  	case <-ctx.Done():
    87  		return fmt.Errorf("context cancelled before flush finished")
    88  	}
    89  }
    90  
    91  // Process will add an entry to the current buffer
    92  func (m *MemoryBuffer) Process(ctx context.Context, entry *entry.Entry) error {
    93  	if m.Bundler == nil {
    94  		panic("must call SetHandler before any calls to Process")
    95  	}
    96  
    97  	return m.AddWait(ctx, entry, 100)
    98  }
    99  
   100  // NewExponentialBackOff will return a new exponential backoff for the memory buffer to use
   101  func (m *MemoryBuffer) NewExponentialBackOff() *backoff.ExponentialBackOff {
   102  	b := &backoff.ExponentialBackOff{
   103  		InitialInterval:     m.config.Retry.InitialInterval.Raw(),
   104  		RandomizationFactor: m.config.Retry.RandomizationFactor,
   105  		Multiplier:          m.config.Retry.Multiplier,
   106  		MaxInterval:         m.config.Retry.MaxInterval.Raw(),
   107  		MaxElapsedTime:      m.config.Retry.MaxElapsedTime.Raw(),
   108  		Stop:                backoff.Stop,
   109  		Clock:               backoff.SystemClock,
   110  	}
   111  	b.Reset()
   112  	return b
   113  }