github.com/Jeffail/benthos/v3@v3.65.0/lib/buffer/single/mmap_buffer_test.go (about)

     1  package single
     2  
     3  import (
     4  	"fmt"
     5  	"math/rand"
     6  	"testing"
     7  
     8  	"github.com/Jeffail/benthos/v3/lib/log"
     9  	"github.com/Jeffail/benthos/v3/lib/message"
    10  	"github.com/Jeffail/benthos/v3/lib/metrics"
    11  	"github.com/Jeffail/benthos/v3/lib/types"
    12  )
    13  
    14  func TestMmapBufferBasic(t *testing.T) {
    15  	t.Skip("DEPRECATED")
    16  
    17  	dir := t.TempDir()
    18  
    19  	n := 100
    20  
    21  	conf := NewMmapBufferConfig()
    22  	conf.FileSize = 100000
    23  	conf.Path = dir
    24  
    25  	block, err := NewMmapBuffer(conf, log.Noop(), metrics.Noop())
    26  	if err != nil {
    27  		t.Error(err)
    28  		return
    29  	}
    30  	defer block.Close()
    31  
    32  	for i := 0; i < n; i++ {
    33  		if _, err := block.PushMessage(message.New(
    34  			[][]byte{
    35  				[]byte("hello"),
    36  				[]byte("world"),
    37  				[]byte("12345"),
    38  				[]byte(fmt.Sprintf("test%v", i)),
    39  			},
    40  		)); err != nil {
    41  			t.Error(err)
    42  			return
    43  		}
    44  	}
    45  
    46  	for i := 0; i < n; i++ {
    47  		m, err := block.NextMessage()
    48  		if err != nil {
    49  			t.Error(err)
    50  			return
    51  		}
    52  		if m.Len() != 4 {
    53  			t.Errorf("Wrong # parts, %v != %v", m.Len(), 4)
    54  		} else if expected, actual := fmt.Sprintf("test%v", i), string(m.Get(3).Get()); expected != actual {
    55  			t.Errorf("Wrong order of messages, %v != %v", expected, actual)
    56  		}
    57  		if _, err := block.ShiftMessage(); err != nil {
    58  			t.Error(err)
    59  			return
    60  		}
    61  	}
    62  }
    63  
    64  func TestMmapBufferBacklogCounter(t *testing.T) {
    65  	t.Skip("DEPRECATED")
    66  
    67  	dir := t.TempDir()
    68  
    69  	conf := NewMmapBufferConfig()
    70  	conf.FileSize = 100000
    71  	conf.Path = dir
    72  
    73  	block, err := NewMmapBuffer(conf, log.Noop(), metrics.Noop())
    74  	if err != nil {
    75  		t.Error(err)
    76  		return
    77  	}
    78  	defer block.Close()
    79  
    80  	if _, err := block.PushMessage(message.New(
    81  		[][]byte{[]byte("1234")}, // 4 bytes + 4 bytes
    82  	)); err != nil {
    83  		t.Error(err)
    84  		return
    85  	}
    86  
    87  	if expected, actual := 16, block.backlog(); expected != actual {
    88  		t.Errorf("Wrong backlog count: %v != %v", expected, actual)
    89  	}
    90  
    91  	if _, err := block.PushMessage(message.New(
    92  		[][]byte{
    93  			[]byte("1234"),
    94  			[]byte("1234"),
    95  		}, // ( 4 bytes + 4 bytes ) * 2
    96  	)); err != nil {
    97  		t.Error(err)
    98  		return
    99  	}
   100  
   101  	if expected, actual := 40, block.backlog(); expected != actual {
   102  		t.Errorf("Wrong backlog count: %v != %v", expected, actual)
   103  	}
   104  
   105  	if _, err := block.ShiftMessage(); err != nil {
   106  		t.Error(err)
   107  		return
   108  	}
   109  
   110  	if expected, actual := 24, block.backlog(); expected != actual {
   111  		t.Errorf("Wrong backlog count: %v != %v", expected, actual)
   112  	}
   113  
   114  	if _, err := block.ShiftMessage(); err != nil {
   115  		t.Error(err)
   116  		return
   117  	}
   118  
   119  	if expected, actual := 0, block.backlog(); expected != actual {
   120  		t.Errorf("Wrong backlog count: %v != %v", expected, actual)
   121  	}
   122  }
   123  
   124  func TestMmapBufferLoopingRandom(t *testing.T) {
   125  	t.Skip("DEPRECATED")
   126  
   127  	dir := t.TempDir()
   128  
   129  	conf := NewMmapBufferConfig()
   130  	conf.FileSize = 8000
   131  	conf.Path = dir
   132  
   133  	block, err := NewMmapBuffer(conf, log.Noop(), metrics.Noop())
   134  	if err != nil {
   135  		t.Error(err)
   136  		return
   137  	}
   138  	defer block.Close()
   139  
   140  	n, iter := 50, 5
   141  
   142  	for j := 0; j < iter; j++ {
   143  		for i := 0; i < n; i++ {
   144  			b := make([]byte, rand.Int()%100)
   145  			for k := range b {
   146  				b[k] = '0'
   147  			}
   148  			if _, err := block.PushMessage(message.New(
   149  				[][]byte{
   150  					b,
   151  					[]byte(fmt.Sprintf("test%v", i)),
   152  				},
   153  			)); err != nil {
   154  				t.Error(err)
   155  				return
   156  			}
   157  		}
   158  
   159  		for i := 0; i < n; i++ {
   160  			m, err := block.NextMessage()
   161  			if err != nil {
   162  				t.Error(err)
   163  				return
   164  			}
   165  			if m.Len() != 2 {
   166  				t.Errorf("Wrong # parts, %v != %v", m.Len(), 4)
   167  			} else if expected, actual := fmt.Sprintf("test%v", i), string(m.Get(1).Get()); expected != actual {
   168  				t.Errorf("Wrong order of messages, %v != %v", expected, actual)
   169  			}
   170  			if _, err := block.ShiftMessage(); err != nil {
   171  				t.Error(err)
   172  				return
   173  			}
   174  		}
   175  	}
   176  }
   177  
   178  func TestMmapBufferMultiFiles(t *testing.T) {
   179  	t.Skip("DEPRECATED")
   180  
   181  	dir := t.TempDir()
   182  
   183  	n := 100
   184  
   185  	conf := NewMmapBufferConfig()
   186  	conf.FileSize = 1000
   187  	conf.Path = dir
   188  
   189  	block, err := NewMmapBuffer(conf, log.Noop(), metrics.Noop())
   190  	if err != nil {
   191  		t.Error(err)
   192  		return
   193  	}
   194  	defer block.Close()
   195  
   196  	for i := 0; i < n; i++ {
   197  		if _, err := block.PushMessage(message.New(
   198  			[][]byte{
   199  				[]byte("hello"),
   200  				[]byte("world"),
   201  				[]byte("12345"),
   202  				[]byte(fmt.Sprintf("test%v", i)),
   203  			},
   204  		)); err != nil {
   205  			t.Error(err)
   206  			return
   207  		}
   208  	}
   209  
   210  	for i := 0; i < n; i++ {
   211  		m, err := block.NextMessage()
   212  		if err != nil {
   213  			t.Error(err)
   214  			return
   215  		}
   216  		if m.Len() != 4 {
   217  			t.Errorf("Wrong # parts, %v != %v", m.Len(), 4)
   218  		} else if expected, actual := fmt.Sprintf("test%v", i), string(m.Get(3).Get()); expected != actual {
   219  			t.Errorf("Wrong order of messages, %v != %v", expected, actual)
   220  		}
   221  		if _, err := block.ShiftMessage(); err != nil {
   222  			t.Error(err)
   223  			return
   224  		}
   225  	}
   226  }
   227  
   228  func TestMmapBufferRecoverFiles(t *testing.T) {
   229  	t.Skip("DEPRECATED")
   230  
   231  	dir := t.TempDir()
   232  
   233  	n := 100
   234  
   235  	conf := NewMmapBufferConfig()
   236  	conf.FileSize = 1000
   237  	conf.Path = dir
   238  
   239  	// Write a load of data
   240  	block, err := NewMmapBuffer(conf, log.Noop(), metrics.Noop())
   241  	if err != nil {
   242  		t.Error(err)
   243  		return
   244  	}
   245  
   246  	for i := 0; i < n; i++ {
   247  		if _, err := block.PushMessage(message.New(
   248  			[][]byte{
   249  				[]byte("hello"),
   250  				[]byte("world"),
   251  				[]byte("12345"),
   252  				[]byte(fmt.Sprintf("test%v", i)),
   253  			},
   254  		)); err != nil {
   255  			t.Error(err)
   256  			return
   257  		}
   258  	}
   259  
   260  	// Close down any handlers we have.
   261  	block.Close()
   262  
   263  	// Read the data back
   264  	block, err = NewMmapBuffer(conf, log.Noop(), metrics.Noop())
   265  	if err != nil {
   266  		t.Error(err)
   267  		return
   268  	}
   269  
   270  	for i := 0; i < n; i++ {
   271  		m, err := block.NextMessage()
   272  		if err != nil {
   273  			t.Error(err)
   274  			return
   275  		}
   276  		if m.Len() != 4 {
   277  			t.Errorf("Wrong # parts, %v != %v", m.Len(), 4)
   278  		} else if expected, actual := fmt.Sprintf("test%v", i), string(m.Get(3).Get()); expected != actual {
   279  			t.Errorf("Wrong order of messages, %v != %v", expected, actual)
   280  		}
   281  		if _, err := block.ShiftMessage(); err != nil {
   282  			t.Error(err)
   283  			return
   284  		}
   285  	}
   286  
   287  	block.Close()
   288  }
   289  
   290  func TestMmapBufferRejectLargeMessage(t *testing.T) {
   291  	t.Skip("DEPRECATED")
   292  
   293  	dir := t.TempDir()
   294  
   295  	tMsg := message.New(make([][]byte, 1))
   296  	tMsg.Get(0).Set([]byte("hello world this message is too long!"))
   297  
   298  	conf := NewMmapBufferConfig()
   299  	conf.FileSize = 10
   300  	conf.Path = dir
   301  
   302  	// Write a load of data
   303  	block, err := NewMmapBuffer(conf, log.Noop(), metrics.Noop())
   304  	if err != nil {
   305  		t.Error(err)
   306  		return
   307  	}
   308  	_, err = block.PushMessage(tMsg)
   309  	if exp, actual := types.ErrMessageTooLarge, err; exp != actual {
   310  		t.Errorf("Unexpected error: %v != %v", exp, actual)
   311  	}
   312  }
   313  
   314  func BenchmarkMmapBufferBasic(b *testing.B) {
   315  	dir := b.TempDir()
   316  
   317  	conf := NewMmapBufferConfig()
   318  	conf.FileSize = 1000
   319  	conf.Path = dir
   320  
   321  	block, err := NewMmapBuffer(conf, log.Noop(), metrics.Noop())
   322  	if err != nil {
   323  		b.Error(err)
   324  		return
   325  	}
   326  	defer block.Close()
   327  
   328  	b.ResetTimer()
   329  
   330  	for i := 0; i < b.N; i++ {
   331  		if _, err := block.PushMessage(message.New(
   332  			[][]byte{
   333  				[]byte("hello"),
   334  				[]byte("world"),
   335  				[]byte("12345"),
   336  				[]byte(fmt.Sprintf("test%v", i)),
   337  			},
   338  		)); err != nil {
   339  			b.Error(err)
   340  			return
   341  		}
   342  	}
   343  
   344  	for i := 0; i < b.N; i++ {
   345  		m, err := block.NextMessage()
   346  		if err != nil {
   347  			b.Error(err)
   348  			return
   349  		}
   350  		if m.Len() != 4 {
   351  			b.Errorf("Wrong # parts, %v != %v", m.Len(), 4)
   352  		} else if expected, actual := fmt.Sprintf("test%v", i), string(m.Get(3).Get()); expected != actual {
   353  			b.Errorf("Wrong order of messages, %v != %v", expected, actual)
   354  		}
   355  		if _, err := block.ShiftMessage(); err != nil {
   356  			b.Error(err)
   357  			return
   358  		}
   359  	}
   360  }