github.com/hernad/nomad@v1.6.112/nomad/stream/event_buffer.go (about)

     1  // Copyright (c) HashiCorp, Inc.
     2  // SPDX-License-Identifier: MPL-2.0
     3  
     4  package stream
     5  
     6  import (
     7  	"context"
     8  	"errors"
     9  	"fmt"
    10  	"sync/atomic"
    11  	"time"
    12  
    13  	"github.com/hernad/nomad/nomad/structs"
    14  )
    15  
    16  // eventBuffer is a single-writer, multiple-reader, fixed length concurrent
    17  // buffer of events that have been published. The buffer is
    18  // the head and tail of an atomically updated single-linked list. Atomic
    19  // accesses are usually to be suspected as premature optimization but this
    20  // specific design has several important features that significantly simplify a
    21  // lot of our PubSub machinery.
    22  //
    23  // eventBuffer is an adaptation of conuls agent/stream/event eventBuffer but
    24  // has been updated to be a max length buffer to work for Nomad's usecase.
    25  //
    26  // The eventBuffer only tracks the most recent set of published events,
    27  // up to the max configured size, older events are dropped from the buffer
    28  // but will only be garbage collected once the slowest reader drops the item.
    29  // Consumers are notified of new events by closing a channel on the previous head
    30  // allowing efficient broadcast to many watchers without having to run multiple
    31  // goroutines or deliver to O(N) separate channels.
    32  //
    33  // Because eventBuffer is a linked list with atomically updated pointers, readers don't
    34  // have to take a lock and can consume at their own pace. Slow readers will eventually
    35  // be forced to reconnect to the lastest head by being notified via a bufferItem's droppedCh.
    36  //
    37  // A new buffer is constructed with a sentinel "empty" bufferItem that has a nil
    38  // Events array. This enables subscribers to start watching for the next update
    39  // immediately.
    40  //
    41  // The zero value eventBuffer is _not_ usable, as it has not been
    42  // initialized with an empty bufferItem so can not be used to wait for the first
    43  // published event. Call newEventBuffer to construct a new buffer.
    44  //
    45  // Calls to Append or purne that mutate the head must be externally
    46  // synchronized. This allows systems that already serialize writes to append
    47  // without lock overhead.
    48  type eventBuffer struct {
    49  	size *int64
    50  
    51  	head atomic.Value
    52  	tail atomic.Value
    53  
    54  	maxSize int64
    55  }
    56  
    57  // newEventBuffer creates an eventBuffer ready for use.
    58  func newEventBuffer(size int64) *eventBuffer {
    59  	zero := int64(0)
    60  	b := &eventBuffer{
    61  		maxSize: size,
    62  		size:    &zero,
    63  	}
    64  
    65  	item := newBufferItem(&structs.Events{Index: 0, Events: nil})
    66  
    67  	b.head.Store(item)
    68  	b.tail.Store(item)
    69  
    70  	return b
    71  }
    72  
    73  // Append a set of events from one raft operation to the buffer and notify
    74  // watchers. After calling append, the caller must not make any further
    75  // mutations to the events as they may have been exposed to subscribers in other
    76  // goroutines. Append only supports a single concurrent caller and must be
    77  // externally synchronized with other Append calls.
    78  func (b *eventBuffer) Append(events *structs.Events) {
    79  	b.appendItem(newBufferItem(events))
    80  }
    81  
    82  func (b *eventBuffer) appendItem(item *bufferItem) {
    83  	// Store the next item to the old tail
    84  	oldTail := b.Tail()
    85  	oldTail.link.next.Store(item)
    86  
    87  	// Update the tail to the new item
    88  	b.tail.Store(item)
    89  
    90  	// Increment the buffer size
    91  	atomic.AddInt64(b.size, 1)
    92  
    93  	// Advance Head until we are under allowable size
    94  	for atomic.LoadInt64(b.size) > b.maxSize {
    95  		b.advanceHead()
    96  	}
    97  
    98  	// notify waiters next event is available
    99  	close(oldTail.link.nextCh)
   100  }
   101  
   102  func newSentinelItem() *bufferItem {
   103  	return newBufferItem(&structs.Events{})
   104  }
   105  
   106  // advanceHead drops the current Head buffer item and notifies readers
   107  // that the item should be discarded by closing droppedCh.
   108  // Slow readers will prevent the old head from being GC'd until they
   109  // discard it.
   110  func (b *eventBuffer) advanceHead() {
   111  	old := b.Head()
   112  
   113  	next := old.link.next.Load()
   114  	// if the next item is nil replace it with a sentinel value
   115  	if next == nil {
   116  		next = newSentinelItem()
   117  	}
   118  
   119  	// notify readers that old is being dropped
   120  	close(old.link.droppedCh)
   121  
   122  	// store the next value to head
   123  	b.head.Store(next)
   124  
   125  	// If the old head is equal to the tail
   126  	// update the tail value as well
   127  	if old == b.Tail() {
   128  		b.tail.Store(next)
   129  	}
   130  
   131  	// In the case of there being a sentinel item or advanceHead being called
   132  	// on a sentinel item, only decrement if there are more than sentinel
   133  	// values
   134  	if atomic.LoadInt64(b.size) > 0 {
   135  		// update the amount of events we have in the buffer
   136  		atomic.AddInt64(b.size, -1)
   137  	}
   138  }
   139  
   140  // Head returns the current head of the buffer. It will always exist but it may
   141  // be a "sentinel" empty item with a nil Events slice to allow consumers to
   142  // watch for the next update. Consumers should always check for empty Events and
   143  // treat them as no-ops. Will panic if eventBuffer was not initialized correctly
   144  // with NewEventBuffer
   145  func (b *eventBuffer) Head() *bufferItem {
   146  	return b.head.Load().(*bufferItem)
   147  }
   148  
   149  // Tail returns the current tail of the buffer. It will always exist but it may
   150  // be a "sentinel" empty item with a Nil Events slice to allow consumers to
   151  // watch for the next update. Consumers should always check for empty Events and
   152  // treat them as no-ops. Will panic if eventBuffer was not initialized correctly
   153  // with NewEventBuffer
   154  func (b *eventBuffer) Tail() *bufferItem {
   155  	return b.tail.Load().(*bufferItem)
   156  }
   157  
   158  // StarStartAtClosest returns the closest bufferItem to a requested starting
   159  // index as well as the offset between the requested index and returned one.
   160  func (b *eventBuffer) StartAtClosest(index uint64) (*bufferItem, int) {
   161  	item := b.Head()
   162  	if index < item.Events.Index {
   163  		return item, int(item.Events.Index) - int(index)
   164  	}
   165  	if item.Events.Index == index {
   166  		return item, 0
   167  	}
   168  
   169  	for {
   170  		prev := item
   171  		item = item.NextNoBlock()
   172  		if item == nil {
   173  			return prev, int(index) - int(prev.Events.Index)
   174  		}
   175  		if index < item.Events.Index {
   176  			return item, int(item.Events.Index) - int(index)
   177  		}
   178  		if index == item.Events.Index {
   179  			return item, 0
   180  		}
   181  	}
   182  }
   183  
   184  // Len returns the current length of the buffer
   185  func (b *eventBuffer) Len() int {
   186  	return int(atomic.LoadInt64(b.size))
   187  }
   188  
   189  // bufferItem represents a set of events published by a single raft operation.
   190  // The first item returned by a newly constructed buffer will have nil Events.
   191  // It is a sentinel value which is used to wait on the next events via Next.
   192  //
   193  // To iterate to the next event, a Next method may be called which may block if
   194  // there is no next element yet.
   195  //
   196  // Holding a pointer to the item keeps all the events published since in memory
   197  // so it's important that subscribers don't hold pointers to buffer items after
   198  // they have been delivered except where it's intentional to maintain a cache or
   199  // trailing store of events for performance reasons.
   200  //
   201  // Subscribers must not mutate the bufferItem or the Events or Encoded payloads
   202  // inside as these are shared between all readers.
   203  type bufferItem struct {
   204  	// Events is the set of events published at one raft index. This may be nil as
   205  	// a sentinel value to allow watching for the first event in a buffer. Callers
   206  	// should check and skip nil Events at any point in the buffer. It will also
   207  	// be nil if the producer appends an Error event because they can't complete
   208  	// the request to populate the buffer. Err will be non-nil in this case.
   209  	Events *structs.Events
   210  
   211  	// Err is non-nil if the producer can't complete their task and terminates the
   212  	// buffer. Subscribers should return the error to clients and cease attempting
   213  	// to read from the buffer.
   214  	Err error
   215  
   216  	// link holds the next pointer and channel. This extra bit of indirection
   217  	// allows us to splice buffers together at arbitrary points without including
   218  	// events in one buffer just for the side-effect of watching for the next set.
   219  	// The link may not be mutated once the event is appended to a buffer.
   220  	link *bufferLink
   221  
   222  	createdAt time.Time
   223  }
   224  
   225  type bufferLink struct {
   226  	// next is an atomically updated pointer to the next event in the buffer. It
   227  	// is written exactly once by the single published and will always be set if
   228  	// ch is closed.
   229  	next atomic.Value
   230  
   231  	// nextCh is closed when the next event is published. It should never be mutated
   232  	// (e.g. set to nil) as that is racey, but is closed once when the next event
   233  	// is published. the next pointer will have been set by the time this is
   234  	// closed.
   235  	nextCh chan struct{}
   236  
   237  	// droppedCh is closed when the event is dropped from the buffer due to
   238  	// sizing constraints.
   239  	droppedCh chan struct{}
   240  }
   241  
   242  // newBufferItem returns a blank buffer item with a link and chan ready to have
   243  // the fields set and be appended to a buffer.
   244  func newBufferItem(events *structs.Events) *bufferItem {
   245  	return &bufferItem{
   246  		link: &bufferLink{
   247  			nextCh:    make(chan struct{}),
   248  			droppedCh: make(chan struct{}),
   249  		},
   250  		Events:    events,
   251  		createdAt: time.Now(),
   252  	}
   253  }
   254  
   255  // Next return the next buffer item in the buffer. It may block until ctx is
   256  // cancelled or until the next item is published.
   257  func (i *bufferItem) Next(ctx context.Context, forceClose <-chan struct{}) (*bufferItem, error) {
   258  	// See if there is already a next value, block if so. Note we don't rely on
   259  	// state change (chan nil) as that's not threadsafe but detecting close is.
   260  	select {
   261  	case <-ctx.Done():
   262  		return nil, ctx.Err()
   263  	case <-forceClose:
   264  		return nil, fmt.Errorf("subscription closed")
   265  	case <-i.link.nextCh:
   266  	}
   267  
   268  	// Check if the reader is too slow and the event buffer as discarded the event
   269  	// This must happen after the above select to prevent a random selection
   270  	// between linkCh and droppedCh
   271  	select {
   272  	case <-i.link.droppedCh:
   273  		return nil, fmt.Errorf("event dropped from buffer")
   274  	default:
   275  	}
   276  
   277  	// If channel closed, there must be a next item to read
   278  	nextRaw := i.link.next.Load()
   279  	if nextRaw == nil {
   280  		// shouldn't be possible
   281  		return nil, errors.New("invalid next item")
   282  	}
   283  	next := nextRaw.(*bufferItem)
   284  	if next.Err != nil {
   285  		return nil, next.Err
   286  	}
   287  	return next, nil
   288  }
   289  
   290  // NextNoBlock returns the next item in the buffer without blocking. If it
   291  // reaches the most recent item it will return nil.
   292  func (i *bufferItem) NextNoBlock() *bufferItem {
   293  	nextRaw := i.link.next.Load()
   294  	if nextRaw == nil {
   295  		return nil
   296  	}
   297  	return nextRaw.(*bufferItem)
   298  }