github.com/Jeffail/benthos/v3@v3.65.0/lib/buffer/single_wrapper_test.go (about) 1 package buffer 2 3 import ( 4 "testing" 5 "time" 6 7 "github.com/Jeffail/benthos/v3/lib/buffer/single" 8 "github.com/Jeffail/benthos/v3/lib/log" 9 "github.com/Jeffail/benthos/v3/lib/message" 10 "github.com/Jeffail/benthos/v3/lib/metrics" 11 "github.com/Jeffail/benthos/v3/lib/response" 12 "github.com/Jeffail/benthos/v3/lib/types" 13 ) 14 15 //------------------------------------------------------------------------------ 16 17 func TestBasicMemoryBuffer(t *testing.T) { 18 var incr, total uint8 = 100, 50 19 20 tChan := make(chan types.Transaction) 21 resChan := make(chan types.Response) 22 23 conf := NewConfig() 24 b := NewSingleWrapper(conf, single.NewMemory(single.MemoryConfig{ 25 Limit: int(incr+15) * int(total), 26 }), log.Noop(), metrics.Noop()) 27 if err := b.Consume(tChan); err != nil { 28 t.Fatal(err) 29 } 30 31 var i uint8 32 33 // Check correct flow no blocking 34 for ; i < total; i++ { 35 msgBytes := make([][]byte, 1) 36 msgBytes[0] = make([]byte, int(incr)) 37 msgBytes[0][0] = i 38 39 select { 40 // Send to buffer 41 case tChan <- types.NewTransaction(message.New(msgBytes), resChan): 42 case <-time.After(time.Second): 43 t.Errorf("Timed out waiting for unbuffered message %v send", i) 44 return 45 } 46 47 // Instant response from buffer 48 select { 49 case res := <-resChan: 50 if res.Error() != nil { 51 t.Error(res.Error()) 52 } 53 case <-time.After(time.Second): 54 t.Errorf("Timed out waiting for unbuffered message %v response", i) 55 return 56 } 57 58 // Receive on output 59 var outTr types.Transaction 60 select { 61 case outTr = <-b.TransactionChan(): 62 if actual := outTr.Payload.Get(0).Get()[0]; actual != i { 63 t.Errorf("Wrong order receipt of unbuffered message receive: %v != %v", actual, i) 64 } 65 case <-time.After(time.Second): 66 t.Errorf("Timed out waiting for unbuffered message %v read", i) 67 return 68 } 69 70 // Response from output 71 select { 72 case outTr.ResponseChan <- response.NewAck(): 73 case <-time.After(time.Second): 74 t.Errorf("Timed out waiting for unbuffered response send back %v", i) 75 return 76 } 77 } 78 79 for i = 0; i < total; i++ { 80 msgBytes := make([][]byte, 1) 81 msgBytes[0] = make([]byte, int(incr)) 82 msgBytes[0][0] = i 83 84 select { 85 case tChan <- types.NewTransaction(message.New(msgBytes), resChan): 86 case <-time.After(time.Second): 87 t.Errorf("Timed out waiting for buffered message %v send", i) 88 return 89 } 90 select { 91 case res := <-resChan: 92 if res.Error() != nil { 93 t.Error(res.Error()) 94 } 95 case <-time.After(time.Second): 96 t.Errorf("Timed out waiting for buffered message %v response", i) 97 return 98 } 99 } 100 101 // Should have reached limit here 102 msgBytes := make([][]byte, 1) 103 msgBytes[0] = make([]byte, int(incr)) 104 105 select { 106 case tChan <- types.NewTransaction(message.New(msgBytes), resChan): 107 case <-time.After(time.Second): 108 t.Errorf("Timed out waiting for final buffered message send") 109 return 110 } 111 112 // Response should block until buffer is relieved 113 select { 114 case res := <-resChan: 115 if res.Error() != nil { 116 t.Error(res.Error()) 117 } else { 118 t.Errorf("Overflowed response returned before timeout") 119 } 120 return 121 case <-time.After(100 * time.Millisecond): 122 } 123 124 var outTr types.Transaction 125 126 // Extract last message 127 select { 128 case outTr = <-b.TransactionChan(): 129 if actual := outTr.Payload.Get(0).Get()[0]; actual != 0 { 130 t.Errorf("Wrong order receipt of buffered message receive: %v != %v", actual, 0) 131 } 132 outTr.ResponseChan <- response.NewAck() 133 case <-time.After(time.Second): 134 t.Errorf("Timed out waiting for final buffered message read") 135 return 136 } 137 138 // Response from the last attempt should no longer be blocking 139 select { 140 case res := <-resChan: 141 if res.Error() != nil { 142 t.Error(res.Error()) 143 } 144 case <-time.After(100 * time.Millisecond): 145 t.Errorf("Final buffered response blocked") 146 } 147 148 // Extract all other messages 149 for i = 1; i < total; i++ { 150 select { 151 case outTr = <-b.TransactionChan(): 152 if actual := outTr.Payload.Get(0).Get()[0]; actual != i { 153 t.Errorf("Wrong order receipt of buffered message: %v != %v", actual, i) 154 } 155 case <-time.After(time.Second): 156 t.Errorf("Timed out waiting for buffered message %v read", i) 157 return 158 } 159 160 select { 161 case outTr.ResponseChan <- response.NewAck(): 162 case <-time.After(time.Second): 163 t.Errorf("Timed out waiting for buffered response send back %v", i) 164 return 165 } 166 } 167 168 // Get final message 169 select { 170 case outTr = <-b.TransactionChan(): 171 case <-time.After(time.Second): 172 t.Errorf("Timed out waiting for buffered message %v read", i) 173 return 174 } 175 176 select { 177 case outTr.ResponseChan <- response.NewAck(): 178 case <-time.After(time.Second): 179 t.Errorf("Timed out waiting for buffered response send back %v", i) 180 return 181 } 182 183 b.CloseAsync() 184 b.WaitForClose(time.Second) 185 186 close(resChan) 187 close(tChan) 188 } 189 190 func TestBufferClosing(t *testing.T) { 191 var incr, total uint8 = 100, 5 192 193 tChan := make(chan types.Transaction) 194 resChan := make(chan types.Response) 195 196 conf := NewConfig() 197 b := NewSingleWrapper(conf, single.NewMemory(single.MemoryConfig{ 198 Limit: int(incr+15) * int(total), 199 }), log.Noop(), metrics.Noop()) 200 if err := b.Consume(tChan); err != nil { 201 t.Fatal(err) 202 } 203 204 var i uint8 205 206 // Populate buffer with some messages 207 for i = 0; i < total; i++ { 208 msgBytes := make([][]byte, 1) 209 msgBytes[0] = make([]byte, int(incr)) 210 msgBytes[0][0] = i 211 212 select { 213 case tChan <- types.NewTransaction(message.New(msgBytes), resChan): 214 case <-time.After(time.Second): 215 t.Errorf("Timed out waiting for buffered message %v send", i) 216 return 217 } 218 select { 219 case res := <-resChan: 220 if res.Error() != nil { 221 t.Error(res.Error()) 222 } 223 case <-time.After(time.Second): 224 t.Errorf("Timed out waiting for buffered message %v response", i) 225 return 226 } 227 } 228 229 // Close input, this should prompt the stack buffer to CloseOnceEmpty(). 230 close(tChan) 231 232 // Receive all of those messages from the buffer 233 for i = 0; i < total; i++ { 234 select { 235 case val := <-b.TransactionChan(): 236 if actual := val.Payload.Get(0).Get()[0]; actual != i { 237 t.Errorf("Wrong order receipt of buffered message receive: %v != %v", actual, i) 238 } 239 val.ResponseChan <- response.NewAck() 240 case <-time.After(time.Second): 241 t.Errorf("Timed out waiting for final buffered message read") 242 return 243 } 244 } 245 246 // The buffer should now be closed, therefore so should our read channel. 247 select { 248 case _, open := <-b.TransactionChan(): 249 if open { 250 t.Error("Reader channel still open after clearing buffer") 251 } 252 case <-time.After(time.Second): 253 t.Errorf("Timed out waiting for final buffered message read") 254 return 255 } 256 257 // Should already be shut down. 258 b.WaitForClose(time.Second) 259 } 260 261 func BenchmarkSingleMem(b *testing.B) { 262 tChan := make(chan types.Transaction) 263 resChan := make(chan types.Response) 264 265 conf := NewConfig() 266 buffer := NewSingleWrapper(conf, single.NewMemory(single.MemoryConfig{ 267 Limit: 50000000, 268 }), log.Noop(), metrics.Noop()) 269 if err := buffer.Consume(tChan); err != nil { 270 b.Fatal(err) 271 } 272 273 contents := [][]byte{ 274 make([]byte, 1024*1024*1), 275 } 276 277 b.ReportAllocs() 278 b.ResetTimer() 279 for i := 0; i < b.N; i++ { 280 select { 281 case tChan <- types.NewTransaction(message.New(contents), resChan): 282 case <-time.After(time.Second): 283 b.Errorf("Timed out waiting for buffered message %v send", i) 284 return 285 } 286 select { 287 case res := <-resChan: 288 if res.Error() != nil { 289 b.Error(res.Error()) 290 } 291 case <-time.After(time.Second): 292 b.Errorf("Timed out waiting for buffered message %v response", i) 293 return 294 } 295 296 select { 297 case val := <-buffer.TransactionChan(): 298 val.ResponseChan <- response.NewAck() 299 case <-time.After(time.Second): 300 b.Errorf("Timed out waiting for final buffered message read") 301 return 302 } 303 } 304 b.StopTimer() 305 306 buffer.CloseAsync() 307 buffer.WaitForClose(time.Second) 308 } 309 310 func BenchmarkSingleMmap(b *testing.B) { 311 dir := b.TempDir() 312 313 tChan := make(chan types.Transaction) 314 resChan := make(chan types.Response) 315 316 mmapConf := single.NewMmapBufferConfig() 317 mmapConf.CleanUp = true 318 mmapConf.FileSize = 50000000 319 mmapConf.Path = dir 320 321 mmap, err := single.NewMmapBuffer(mmapConf, log.Noop(), metrics.Noop()) 322 if err != nil { 323 b.Fatal(err) 324 } 325 326 conf := NewConfig() 327 buffer := NewSingleWrapper(conf, mmap, log.Noop(), metrics.Noop()) 328 if err := buffer.Consume(tChan); err != nil { 329 b.Error(err) 330 return 331 } 332 333 contents := [][]byte{ 334 make([]byte, 1024*1024*1), 335 } 336 337 b.ReportAllocs() 338 b.ResetTimer() 339 for i := 0; i < b.N; i++ { 340 select { 341 case tChan <- types.NewTransaction(message.New(contents), resChan): 342 case <-time.After(time.Second): 343 b.Errorf("Timed out waiting for buffered message %v send", i) 344 return 345 } 346 select { 347 case res := <-resChan: 348 if res.Error() != nil { 349 b.Error(res.Error()) 350 } 351 case <-time.After(time.Second): 352 b.Errorf("Timed out waiting for buffered message %v response", i) 353 return 354 } 355 356 select { 357 case val := <-buffer.TransactionChan(): 358 val.ResponseChan <- response.NewAck() 359 case <-time.After(time.Second): 360 b.Errorf("Timed out waiting for final buffered message read") 361 return 362 } 363 } 364 b.StopTimer() 365 366 buffer.CloseAsync() 367 buffer.WaitForClose(time.Second) 368 } 369 370 //------------------------------------------------------------------------------