github.com/Jeffail/benthos/v3@v3.65.0/lib/buffer/single/mmap_buffer_test.go (about) 1 package single 2 3 import ( 4 "fmt" 5 "math/rand" 6 "testing" 7 8 "github.com/Jeffail/benthos/v3/lib/log" 9 "github.com/Jeffail/benthos/v3/lib/message" 10 "github.com/Jeffail/benthos/v3/lib/metrics" 11 "github.com/Jeffail/benthos/v3/lib/types" 12 ) 13 14 func TestMmapBufferBasic(t *testing.T) { 15 t.Skip("DEPRECATED") 16 17 dir := t.TempDir() 18 19 n := 100 20 21 conf := NewMmapBufferConfig() 22 conf.FileSize = 100000 23 conf.Path = dir 24 25 block, err := NewMmapBuffer(conf, log.Noop(), metrics.Noop()) 26 if err != nil { 27 t.Error(err) 28 return 29 } 30 defer block.Close() 31 32 for i := 0; i < n; i++ { 33 if _, err := block.PushMessage(message.New( 34 [][]byte{ 35 []byte("hello"), 36 []byte("world"), 37 []byte("12345"), 38 []byte(fmt.Sprintf("test%v", i)), 39 }, 40 )); err != nil { 41 t.Error(err) 42 return 43 } 44 } 45 46 for i := 0; i < n; i++ { 47 m, err := block.NextMessage() 48 if err != nil { 49 t.Error(err) 50 return 51 } 52 if m.Len() != 4 { 53 t.Errorf("Wrong # parts, %v != %v", m.Len(), 4) 54 } else if expected, actual := fmt.Sprintf("test%v", i), string(m.Get(3).Get()); expected != actual { 55 t.Errorf("Wrong order of messages, %v != %v", expected, actual) 56 } 57 if _, err := block.ShiftMessage(); err != nil { 58 t.Error(err) 59 return 60 } 61 } 62 } 63 64 func TestMmapBufferBacklogCounter(t *testing.T) { 65 t.Skip("DEPRECATED") 66 67 dir := t.TempDir() 68 69 conf := NewMmapBufferConfig() 70 conf.FileSize = 100000 71 conf.Path = dir 72 73 block, err := NewMmapBuffer(conf, log.Noop(), metrics.Noop()) 74 if err != nil { 75 t.Error(err) 76 return 77 } 78 defer block.Close() 79 80 if _, err := block.PushMessage(message.New( 81 [][]byte{[]byte("1234")}, // 4 bytes + 4 bytes 82 )); err != nil { 83 t.Error(err) 84 return 85 } 86 87 if expected, actual := 16, block.backlog(); expected != actual { 88 t.Errorf("Wrong backlog count: %v != %v", expected, actual) 89 } 90 91 if _, err := block.PushMessage(message.New( 92 [][]byte{ 93 []byte("1234"), 94 []byte("1234"), 95 }, // ( 4 bytes + 4 bytes ) * 2 96 )); err != nil { 97 t.Error(err) 98 return 99 } 100 101 if expected, actual := 40, block.backlog(); expected != actual { 102 t.Errorf("Wrong backlog count: %v != %v", expected, actual) 103 } 104 105 if _, err := block.ShiftMessage(); err != nil { 106 t.Error(err) 107 return 108 } 109 110 if expected, actual := 24, block.backlog(); expected != actual { 111 t.Errorf("Wrong backlog count: %v != %v", expected, actual) 112 } 113 114 if _, err := block.ShiftMessage(); err != nil { 115 t.Error(err) 116 return 117 } 118 119 if expected, actual := 0, block.backlog(); expected != actual { 120 t.Errorf("Wrong backlog count: %v != %v", expected, actual) 121 } 122 } 123 124 func TestMmapBufferLoopingRandom(t *testing.T) { 125 t.Skip("DEPRECATED") 126 127 dir := t.TempDir() 128 129 conf := NewMmapBufferConfig() 130 conf.FileSize = 8000 131 conf.Path = dir 132 133 block, err := NewMmapBuffer(conf, log.Noop(), metrics.Noop()) 134 if err != nil { 135 t.Error(err) 136 return 137 } 138 defer block.Close() 139 140 n, iter := 50, 5 141 142 for j := 0; j < iter; j++ { 143 for i := 0; i < n; i++ { 144 b := make([]byte, rand.Int()%100) 145 for k := range b { 146 b[k] = '0' 147 } 148 if _, err := block.PushMessage(message.New( 149 [][]byte{ 150 b, 151 []byte(fmt.Sprintf("test%v", i)), 152 }, 153 )); err != nil { 154 t.Error(err) 155 return 156 } 157 } 158 159 for i := 0; i < n; i++ { 160 m, err := block.NextMessage() 161 if err != nil { 162 t.Error(err) 163 return 164 } 165 if m.Len() != 2 { 166 t.Errorf("Wrong # parts, %v != %v", m.Len(), 4) 167 } else if expected, actual := fmt.Sprintf("test%v", i), string(m.Get(1).Get()); expected != actual { 168 t.Errorf("Wrong order of messages, %v != %v", expected, actual) 169 } 170 if _, err := block.ShiftMessage(); err != nil { 171 t.Error(err) 172 return 173 } 174 } 175 } 176 } 177 178 func TestMmapBufferMultiFiles(t *testing.T) { 179 t.Skip("DEPRECATED") 180 181 dir := t.TempDir() 182 183 n := 100 184 185 conf := NewMmapBufferConfig() 186 conf.FileSize = 1000 187 conf.Path = dir 188 189 block, err := NewMmapBuffer(conf, log.Noop(), metrics.Noop()) 190 if err != nil { 191 t.Error(err) 192 return 193 } 194 defer block.Close() 195 196 for i := 0; i < n; i++ { 197 if _, err := block.PushMessage(message.New( 198 [][]byte{ 199 []byte("hello"), 200 []byte("world"), 201 []byte("12345"), 202 []byte(fmt.Sprintf("test%v", i)), 203 }, 204 )); err != nil { 205 t.Error(err) 206 return 207 } 208 } 209 210 for i := 0; i < n; i++ { 211 m, err := block.NextMessage() 212 if err != nil { 213 t.Error(err) 214 return 215 } 216 if m.Len() != 4 { 217 t.Errorf("Wrong # parts, %v != %v", m.Len(), 4) 218 } else if expected, actual := fmt.Sprintf("test%v", i), string(m.Get(3).Get()); expected != actual { 219 t.Errorf("Wrong order of messages, %v != %v", expected, actual) 220 } 221 if _, err := block.ShiftMessage(); err != nil { 222 t.Error(err) 223 return 224 } 225 } 226 } 227 228 func TestMmapBufferRecoverFiles(t *testing.T) { 229 t.Skip("DEPRECATED") 230 231 dir := t.TempDir() 232 233 n := 100 234 235 conf := NewMmapBufferConfig() 236 conf.FileSize = 1000 237 conf.Path = dir 238 239 // Write a load of data 240 block, err := NewMmapBuffer(conf, log.Noop(), metrics.Noop()) 241 if err != nil { 242 t.Error(err) 243 return 244 } 245 246 for i := 0; i < n; i++ { 247 if _, err := block.PushMessage(message.New( 248 [][]byte{ 249 []byte("hello"), 250 []byte("world"), 251 []byte("12345"), 252 []byte(fmt.Sprintf("test%v", i)), 253 }, 254 )); err != nil { 255 t.Error(err) 256 return 257 } 258 } 259 260 // Close down any handlers we have. 261 block.Close() 262 263 // Read the data back 264 block, err = NewMmapBuffer(conf, log.Noop(), metrics.Noop()) 265 if err != nil { 266 t.Error(err) 267 return 268 } 269 270 for i := 0; i < n; i++ { 271 m, err := block.NextMessage() 272 if err != nil { 273 t.Error(err) 274 return 275 } 276 if m.Len() != 4 { 277 t.Errorf("Wrong # parts, %v != %v", m.Len(), 4) 278 } else if expected, actual := fmt.Sprintf("test%v", i), string(m.Get(3).Get()); expected != actual { 279 t.Errorf("Wrong order of messages, %v != %v", expected, actual) 280 } 281 if _, err := block.ShiftMessage(); err != nil { 282 t.Error(err) 283 return 284 } 285 } 286 287 block.Close() 288 } 289 290 func TestMmapBufferRejectLargeMessage(t *testing.T) { 291 t.Skip("DEPRECATED") 292 293 dir := t.TempDir() 294 295 tMsg := message.New(make([][]byte, 1)) 296 tMsg.Get(0).Set([]byte("hello world this message is too long!")) 297 298 conf := NewMmapBufferConfig() 299 conf.FileSize = 10 300 conf.Path = dir 301 302 // Write a load of data 303 block, err := NewMmapBuffer(conf, log.Noop(), metrics.Noop()) 304 if err != nil { 305 t.Error(err) 306 return 307 } 308 _, err = block.PushMessage(tMsg) 309 if exp, actual := types.ErrMessageTooLarge, err; exp != actual { 310 t.Errorf("Unexpected error: %v != %v", exp, actual) 311 } 312 } 313 314 func BenchmarkMmapBufferBasic(b *testing.B) { 315 dir := b.TempDir() 316 317 conf := NewMmapBufferConfig() 318 conf.FileSize = 1000 319 conf.Path = dir 320 321 block, err := NewMmapBuffer(conf, log.Noop(), metrics.Noop()) 322 if err != nil { 323 b.Error(err) 324 return 325 } 326 defer block.Close() 327 328 b.ResetTimer() 329 330 for i := 0; i < b.N; i++ { 331 if _, err := block.PushMessage(message.New( 332 [][]byte{ 333 []byte("hello"), 334 []byte("world"), 335 []byte("12345"), 336 []byte(fmt.Sprintf("test%v", i)), 337 }, 338 )); err != nil { 339 b.Error(err) 340 return 341 } 342 } 343 344 for i := 0; i < b.N; i++ { 345 m, err := block.NextMessage() 346 if err != nil { 347 b.Error(err) 348 return 349 } 350 if m.Len() != 4 { 351 b.Errorf("Wrong # parts, %v != %v", m.Len(), 4) 352 } else if expected, actual := fmt.Sprintf("test%v", i), string(m.Get(3).Get()); expected != actual { 353 b.Errorf("Wrong order of messages, %v != %v", expected, actual) 354 } 355 if _, err := block.ShiftMessage(); err != nil { 356 b.Error(err) 357 return 358 } 359 } 360 }