github.com/Jeffail/benthos/v3@v3.65.0/lib/buffer/single/memory.go (about) 1 package single 2 3 import ( 4 "sync" 5 6 "github.com/Jeffail/benthos/v3/lib/message" 7 "github.com/Jeffail/benthos/v3/lib/types" 8 ) 9 10 //------------------------------------------------------------------------------ 11 12 // MemoryConfig is config values for a purely memory based ring buffer type. 13 type MemoryConfig struct { 14 Limit int `json:"limit" yaml:"limit"` 15 } 16 17 // NewMemoryConfig creates a new MemoryConfig with default values. 18 func NewMemoryConfig() MemoryConfig { 19 return MemoryConfig{ 20 Limit: 1024 * 1024 * 500, // 500MB 21 } 22 } 23 24 // Memory is a purely memory based ring buffer. This buffer blocks when the 25 // buffer is full. 26 type Memory struct { 27 config MemoryConfig 28 29 block []byte 30 readFrom int 31 writtenTo int 32 33 closed bool 34 35 cond *sync.Cond 36 } 37 38 // NewMemory creates a new memory based ring buffer. 39 func NewMemory(config MemoryConfig) *Memory { 40 return &Memory{ 41 config: config, 42 block: make([]byte, config.Limit), 43 readFrom: 0, 44 writtenTo: 0, 45 closed: false, 46 cond: sync.NewCond(&sync.Mutex{}), 47 } 48 } 49 50 //------------------------------------------------------------------------------ 51 52 // backlog reads the current backlog of messages stored. 53 func (m *Memory) backlog() int { 54 if m.writtenTo >= m.readFrom { 55 return m.writtenTo - m.readFrom 56 } 57 return m.config.Limit - m.readFrom + m.writtenTo 58 } 59 60 // readMessageSize reads the size in bytes of a serialised message block 61 // starting at index. 62 func readMessageSize(block []byte, index int) int { 63 if index+3 >= len(block) { 64 return 0 65 } 66 return int(block[0+index])<<24 | 67 int(block[1+index])<<16 | 68 int(block[2+index])<<8 | 69 int(block[3+index]) 70 } 71 72 // writeMessageSize writes the size in bytes of a serialised message block 73 // starting at index. 74 func writeMessageSize(block []byte, index, size int) { 75 block[index+0] = byte(size >> 24) 76 block[index+1] = byte(size >> 16) 77 block[index+2] = byte(size >> 8) 78 block[index+3] = byte(size) 79 } 80 81 //------------------------------------------------------------------------------ 82 83 // CloseOnceEmpty closes the memory buffer once the backlog reaches 0. 84 func (m *Memory) CloseOnceEmpty() { 85 defer func() { 86 m.cond.L.Unlock() 87 m.Close() 88 }() 89 m.cond.L.Lock() 90 91 // Until the backlog is cleared. 92 for m.backlog() > 0 { 93 // Wait for a broadcast from our reader. 94 m.cond.Wait() 95 } 96 } 97 98 // Close unblocks any blocked calls and prevents further writing to the block. 99 func (m *Memory) Close() { 100 m.cond.L.Lock() 101 m.closed = true 102 m.cond.Broadcast() 103 m.cond.L.Unlock() 104 } 105 106 // ShiftMessage removes the last message from the block. Returns the backlog 107 // count. 108 func (m *Memory) ShiftMessage() (int, error) { 109 m.cond.L.Lock() 110 defer func() { 111 m.cond.Broadcast() 112 m.cond.L.Unlock() 113 }() 114 115 msgSize := readMessageSize(m.block, m.readFrom) 116 117 // Messages are written in a contiguous array of bytes, therefore when the 118 // writer reaches the end it will zero the next four bytes (zero size 119 // message) to indicate to the reader that it has looped back to index 0. 120 if msgSize <= 0 { 121 m.readFrom = 0 122 msgSize = readMessageSize(m.block, m.readFrom) 123 } 124 125 // Set new read from position to next message start. 126 m.readFrom = m.readFrom + msgSize + 4 127 128 return m.backlog(), nil 129 } 130 131 // NextMessage reads the next message, this call blocks until there's something 132 // to read. 133 func (m *Memory) NextMessage() (types.Message, error) { 134 m.cond.L.Lock() 135 defer m.cond.L.Unlock() 136 137 index := m.readFrom 138 139 for index == m.writtenTo && !m.closed { 140 m.cond.Wait() 141 } 142 if m.closed { 143 return nil, types.ErrTypeClosed 144 } 145 146 msgSize := readMessageSize(m.block, index) 147 148 // Messages are written in a contiguous array of bytes, therefore when the 149 // writer reaches the end it will zero the next four bytes (zero size 150 // message) to indicate to the reader that it has looped back to index 0. 151 if msgSize <= 0 { 152 index = 0 153 for index == m.writtenTo && !m.closed { 154 m.cond.Wait() 155 } 156 if m.closed { 157 return nil, types.ErrTypeClosed 158 } 159 160 msgSize = readMessageSize(m.block, index) 161 } 162 163 index += 4 164 if index+msgSize > m.config.Limit { 165 return nil, types.ErrBlockCorrupted 166 } 167 168 return message.FromBytes(m.block[index : index+msgSize]) 169 } 170 171 // PushMessage pushes a new message onto the block, returns the backlog count. 172 func (m *Memory) PushMessage(msg types.Message) (int, error) { 173 m.cond.L.Lock() 174 defer func() { 175 m.cond.Broadcast() 176 m.cond.L.Unlock() 177 }() 178 179 block := message.ToBytes(msg) 180 index := m.writtenTo 181 182 if len(block)+4 > m.config.Limit { 183 return 0, types.ErrMessageTooLarge 184 } 185 186 // Block while the reader is catching up. 187 for m.readFrom > index && m.readFrom <= index+len(block)+4 { 188 m.cond.Wait() 189 } 190 if m.closed { 191 return 0, types.ErrTypeClosed 192 } 193 194 // If we can't fit our next message in the remainder of the buffer we will 195 // loop back to index 0. In order to prevent the reader from reading garbage 196 // we set the next message size to 0, which tells the reader to loop back to 197 // index 0. 198 if len(block)+4+index > m.config.Limit { 199 200 // If the reader is currently at 0 then we avoid looping over it. 201 for m.readFrom <= len(block)+4 && !m.closed { 202 m.cond.Wait() 203 } 204 if m.closed { 205 return 0, types.ErrTypeClosed 206 } 207 for i := index; i < m.config.Limit && i < index+4; i++ { 208 m.block[i] = byte(0) 209 } 210 index = 0 211 } 212 213 // Block again if the reader is catching up. 214 for m.readFrom > index && m.readFrom <= index+len(block)+4 && !m.closed { 215 m.cond.Wait() 216 } 217 if m.closed { 218 return 0, types.ErrTypeClosed 219 } 220 221 writeMessageSize(m.block, index, len(block)) 222 copy(m.block[index+4:], block) 223 224 // Move writtenTo index ahead. If writtenTo becomes m.config.Limit we want 225 // it to wrap back to 0 226 m.writtenTo = (index + len(block) + 4) % m.config.Limit 227 228 return m.backlog(), nil 229 } 230 231 //------------------------------------------------------------------------------