github.com/hernad/nomad@v1.6.112/nomad/stream/event_buffer.go (about) 1 // Copyright (c) HashiCorp, Inc. 2 // SPDX-License-Identifier: MPL-2.0 3 4 package stream 5 6 import ( 7 "context" 8 "errors" 9 "fmt" 10 "sync/atomic" 11 "time" 12 13 "github.com/hernad/nomad/nomad/structs" 14 ) 15 16 // eventBuffer is a single-writer, multiple-reader, fixed length concurrent 17 // buffer of events that have been published. The buffer is 18 // the head and tail of an atomically updated single-linked list. Atomic 19 // accesses are usually to be suspected as premature optimization but this 20 // specific design has several important features that significantly simplify a 21 // lot of our PubSub machinery. 22 // 23 // eventBuffer is an adaptation of conuls agent/stream/event eventBuffer but 24 // has been updated to be a max length buffer to work for Nomad's usecase. 25 // 26 // The eventBuffer only tracks the most recent set of published events, 27 // up to the max configured size, older events are dropped from the buffer 28 // but will only be garbage collected once the slowest reader drops the item. 29 // Consumers are notified of new events by closing a channel on the previous head 30 // allowing efficient broadcast to many watchers without having to run multiple 31 // goroutines or deliver to O(N) separate channels. 32 // 33 // Because eventBuffer is a linked list with atomically updated pointers, readers don't 34 // have to take a lock and can consume at their own pace. Slow readers will eventually 35 // be forced to reconnect to the lastest head by being notified via a bufferItem's droppedCh. 36 // 37 // A new buffer is constructed with a sentinel "empty" bufferItem that has a nil 38 // Events array. This enables subscribers to start watching for the next update 39 // immediately. 40 // 41 // The zero value eventBuffer is _not_ usable, as it has not been 42 // initialized with an empty bufferItem so can not be used to wait for the first 43 // published event. Call newEventBuffer to construct a new buffer. 44 // 45 // Calls to Append or purne that mutate the head must be externally 46 // synchronized. This allows systems that already serialize writes to append 47 // without lock overhead. 48 type eventBuffer struct { 49 size *int64 50 51 head atomic.Value 52 tail atomic.Value 53 54 maxSize int64 55 } 56 57 // newEventBuffer creates an eventBuffer ready for use. 58 func newEventBuffer(size int64) *eventBuffer { 59 zero := int64(0) 60 b := &eventBuffer{ 61 maxSize: size, 62 size: &zero, 63 } 64 65 item := newBufferItem(&structs.Events{Index: 0, Events: nil}) 66 67 b.head.Store(item) 68 b.tail.Store(item) 69 70 return b 71 } 72 73 // Append a set of events from one raft operation to the buffer and notify 74 // watchers. After calling append, the caller must not make any further 75 // mutations to the events as they may have been exposed to subscribers in other 76 // goroutines. Append only supports a single concurrent caller and must be 77 // externally synchronized with other Append calls. 78 func (b *eventBuffer) Append(events *structs.Events) { 79 b.appendItem(newBufferItem(events)) 80 } 81 82 func (b *eventBuffer) appendItem(item *bufferItem) { 83 // Store the next item to the old tail 84 oldTail := b.Tail() 85 oldTail.link.next.Store(item) 86 87 // Update the tail to the new item 88 b.tail.Store(item) 89 90 // Increment the buffer size 91 atomic.AddInt64(b.size, 1) 92 93 // Advance Head until we are under allowable size 94 for atomic.LoadInt64(b.size) > b.maxSize { 95 b.advanceHead() 96 } 97 98 // notify waiters next event is available 99 close(oldTail.link.nextCh) 100 } 101 102 func newSentinelItem() *bufferItem { 103 return newBufferItem(&structs.Events{}) 104 } 105 106 // advanceHead drops the current Head buffer item and notifies readers 107 // that the item should be discarded by closing droppedCh. 108 // Slow readers will prevent the old head from being GC'd until they 109 // discard it. 110 func (b *eventBuffer) advanceHead() { 111 old := b.Head() 112 113 next := old.link.next.Load() 114 // if the next item is nil replace it with a sentinel value 115 if next == nil { 116 next = newSentinelItem() 117 } 118 119 // notify readers that old is being dropped 120 close(old.link.droppedCh) 121 122 // store the next value to head 123 b.head.Store(next) 124 125 // If the old head is equal to the tail 126 // update the tail value as well 127 if old == b.Tail() { 128 b.tail.Store(next) 129 } 130 131 // In the case of there being a sentinel item or advanceHead being called 132 // on a sentinel item, only decrement if there are more than sentinel 133 // values 134 if atomic.LoadInt64(b.size) > 0 { 135 // update the amount of events we have in the buffer 136 atomic.AddInt64(b.size, -1) 137 } 138 } 139 140 // Head returns the current head of the buffer. It will always exist but it may 141 // be a "sentinel" empty item with a nil Events slice to allow consumers to 142 // watch for the next update. Consumers should always check for empty Events and 143 // treat them as no-ops. Will panic if eventBuffer was not initialized correctly 144 // with NewEventBuffer 145 func (b *eventBuffer) Head() *bufferItem { 146 return b.head.Load().(*bufferItem) 147 } 148 149 // Tail returns the current tail of the buffer. It will always exist but it may 150 // be a "sentinel" empty item with a Nil Events slice to allow consumers to 151 // watch for the next update. Consumers should always check for empty Events and 152 // treat them as no-ops. Will panic if eventBuffer was not initialized correctly 153 // with NewEventBuffer 154 func (b *eventBuffer) Tail() *bufferItem { 155 return b.tail.Load().(*bufferItem) 156 } 157 158 // StarStartAtClosest returns the closest bufferItem to a requested starting 159 // index as well as the offset between the requested index and returned one. 160 func (b *eventBuffer) StartAtClosest(index uint64) (*bufferItem, int) { 161 item := b.Head() 162 if index < item.Events.Index { 163 return item, int(item.Events.Index) - int(index) 164 } 165 if item.Events.Index == index { 166 return item, 0 167 } 168 169 for { 170 prev := item 171 item = item.NextNoBlock() 172 if item == nil { 173 return prev, int(index) - int(prev.Events.Index) 174 } 175 if index < item.Events.Index { 176 return item, int(item.Events.Index) - int(index) 177 } 178 if index == item.Events.Index { 179 return item, 0 180 } 181 } 182 } 183 184 // Len returns the current length of the buffer 185 func (b *eventBuffer) Len() int { 186 return int(atomic.LoadInt64(b.size)) 187 } 188 189 // bufferItem represents a set of events published by a single raft operation. 190 // The first item returned by a newly constructed buffer will have nil Events. 191 // It is a sentinel value which is used to wait on the next events via Next. 192 // 193 // To iterate to the next event, a Next method may be called which may block if 194 // there is no next element yet. 195 // 196 // Holding a pointer to the item keeps all the events published since in memory 197 // so it's important that subscribers don't hold pointers to buffer items after 198 // they have been delivered except where it's intentional to maintain a cache or 199 // trailing store of events for performance reasons. 200 // 201 // Subscribers must not mutate the bufferItem or the Events or Encoded payloads 202 // inside as these are shared between all readers. 203 type bufferItem struct { 204 // Events is the set of events published at one raft index. This may be nil as 205 // a sentinel value to allow watching for the first event in a buffer. Callers 206 // should check and skip nil Events at any point in the buffer. It will also 207 // be nil if the producer appends an Error event because they can't complete 208 // the request to populate the buffer. Err will be non-nil in this case. 209 Events *structs.Events 210 211 // Err is non-nil if the producer can't complete their task and terminates the 212 // buffer. Subscribers should return the error to clients and cease attempting 213 // to read from the buffer. 214 Err error 215 216 // link holds the next pointer and channel. This extra bit of indirection 217 // allows us to splice buffers together at arbitrary points without including 218 // events in one buffer just for the side-effect of watching for the next set. 219 // The link may not be mutated once the event is appended to a buffer. 220 link *bufferLink 221 222 createdAt time.Time 223 } 224 225 type bufferLink struct { 226 // next is an atomically updated pointer to the next event in the buffer. It 227 // is written exactly once by the single published and will always be set if 228 // ch is closed. 229 next atomic.Value 230 231 // nextCh is closed when the next event is published. It should never be mutated 232 // (e.g. set to nil) as that is racey, but is closed once when the next event 233 // is published. the next pointer will have been set by the time this is 234 // closed. 235 nextCh chan struct{} 236 237 // droppedCh is closed when the event is dropped from the buffer due to 238 // sizing constraints. 239 droppedCh chan struct{} 240 } 241 242 // newBufferItem returns a blank buffer item with a link and chan ready to have 243 // the fields set and be appended to a buffer. 244 func newBufferItem(events *structs.Events) *bufferItem { 245 return &bufferItem{ 246 link: &bufferLink{ 247 nextCh: make(chan struct{}), 248 droppedCh: make(chan struct{}), 249 }, 250 Events: events, 251 createdAt: time.Now(), 252 } 253 } 254 255 // Next return the next buffer item in the buffer. It may block until ctx is 256 // cancelled or until the next item is published. 257 func (i *bufferItem) Next(ctx context.Context, forceClose <-chan struct{}) (*bufferItem, error) { 258 // See if there is already a next value, block if so. Note we don't rely on 259 // state change (chan nil) as that's not threadsafe but detecting close is. 260 select { 261 case <-ctx.Done(): 262 return nil, ctx.Err() 263 case <-forceClose: 264 return nil, fmt.Errorf("subscription closed") 265 case <-i.link.nextCh: 266 } 267 268 // Check if the reader is too slow and the event buffer as discarded the event 269 // This must happen after the above select to prevent a random selection 270 // between linkCh and droppedCh 271 select { 272 case <-i.link.droppedCh: 273 return nil, fmt.Errorf("event dropped from buffer") 274 default: 275 } 276 277 // If channel closed, there must be a next item to read 278 nextRaw := i.link.next.Load() 279 if nextRaw == nil { 280 // shouldn't be possible 281 return nil, errors.New("invalid next item") 282 } 283 next := nextRaw.(*bufferItem) 284 if next.Err != nil { 285 return nil, next.Err 286 } 287 return next, nil 288 } 289 290 // NextNoBlock returns the next item in the buffer without blocking. If it 291 // reaches the most recent item it will return nil. 292 func (i *bufferItem) NextNoBlock() *bufferItem { 293 nextRaw := i.link.next.Load() 294 if nextRaw == nil { 295 return nil 296 } 297 return nextRaw.(*bufferItem) 298 }