github.com/Jeffail/benthos/v3@v3.65.0/lib/processor/compress_test.go (about)

     1  package processor
     2  
     3  import (
     4  	"bytes"
     5  	"compress/flate"
     6  	"compress/gzip"
     7  	"compress/zlib"
     8  	"reflect"
     9  	"testing"
    10  
    11  	"github.com/Jeffail/benthos/v3/lib/log"
    12  	"github.com/Jeffail/benthos/v3/lib/message"
    13  	"github.com/Jeffail/benthos/v3/lib/metrics"
    14  	"github.com/golang/snappy"
    15  	"github.com/pierrec/lz4/v4"
    16  )
    17  
    18  func TestCompressBadAlgo(t *testing.T) {
    19  	conf := NewConfig()
    20  	conf.Compress.Algorithm = "does not exist"
    21  
    22  	testLog := log.Noop()
    23  
    24  	_, err := NewCompress(conf, nil, testLog, metrics.Noop())
    25  	if err == nil {
    26  		t.Error("Expected error from bad algo")
    27  	}
    28  }
    29  
    30  func TestCompressGZIP(t *testing.T) {
    31  	conf := NewConfig()
    32  	conf.Compress.Algorithm = "gzip"
    33  
    34  	testLog := log.Noop()
    35  
    36  	input := [][]byte{
    37  		[]byte("hello world first part"),
    38  		[]byte("hello world second part"),
    39  		[]byte("third part"),
    40  		[]byte("fourth"),
    41  		[]byte("5"),
    42  	}
    43  
    44  	exp := [][]byte{}
    45  
    46  	for i := range input {
    47  		var buf bytes.Buffer
    48  
    49  		zw := gzip.NewWriter(&buf)
    50  		zw.Write(input[i])
    51  		zw.Close()
    52  
    53  		exp = append(exp, buf.Bytes())
    54  	}
    55  
    56  	if reflect.DeepEqual(input, exp) {
    57  		t.Fatal("Input and exp output are the same")
    58  	}
    59  
    60  	proc, err := NewCompress(conf, nil, testLog, metrics.Noop())
    61  	if err != nil {
    62  		t.Fatal(err)
    63  	}
    64  
    65  	msgs, res := proc.ProcessMessage(message.New(input))
    66  	if len(msgs) != 1 {
    67  		t.Error("Compress failed")
    68  	} else if res != nil {
    69  		t.Errorf("Expected nil response: %v", res)
    70  	}
    71  	if act := message.GetAllBytes(msgs[0]); !reflect.DeepEqual(exp, act) {
    72  		t.Errorf("Unexpected output: %s != %s", act, exp)
    73  	}
    74  }
    75  
    76  func TestCompressZLIB(t *testing.T) {
    77  	conf := NewConfig()
    78  	conf.Compress.Algorithm = "zlib"
    79  
    80  	testLog := log.Noop()
    81  
    82  	input := [][]byte{
    83  		[]byte("hello world first part"),
    84  		[]byte("hello world second part"),
    85  		[]byte("third part"),
    86  		[]byte("fourth"),
    87  		[]byte("5"),
    88  	}
    89  
    90  	exp := [][]byte{}
    91  
    92  	for i := range input {
    93  		var buf bytes.Buffer
    94  
    95  		zw := zlib.NewWriter(&buf)
    96  		zw.Write(input[i])
    97  		zw.Close()
    98  
    99  		exp = append(exp, buf.Bytes())
   100  	}
   101  
   102  	if reflect.DeepEqual(input, exp) {
   103  		t.Fatal("Input and exp output are the same")
   104  	}
   105  
   106  	proc, err := NewCompress(conf, nil, testLog, metrics.Noop())
   107  	if err != nil {
   108  		t.Fatal(err)
   109  	}
   110  
   111  	msgs, res := proc.ProcessMessage(message.New(input))
   112  	if len(msgs) != 1 {
   113  		t.Error("Compress failed")
   114  	} else if res != nil {
   115  		t.Errorf("Expected nil response: %v", res)
   116  	}
   117  	if act := message.GetAllBytes(msgs[0]); !reflect.DeepEqual(exp, act) {
   118  		t.Errorf("Unexpected output: %s != %s", act, exp)
   119  	}
   120  }
   121  
   122  func TestCompressFlate(t *testing.T) {
   123  	conf := NewConfig()
   124  	conf.Compress.Algorithm = "flate"
   125  
   126  	testLog := log.Noop()
   127  
   128  	input := [][]byte{
   129  		[]byte("hello world first part"),
   130  		[]byte("hello world second part"),
   131  		[]byte("third part"),
   132  		[]byte("fourth"),
   133  		[]byte("5"),
   134  	}
   135  
   136  	exp := [][]byte{}
   137  
   138  	for i := range input {
   139  		var buf bytes.Buffer
   140  
   141  		zw, err := flate.NewWriter(&buf, conf.Compress.Level)
   142  		if err != nil {
   143  			t.Fatal(err)
   144  		}
   145  		zw.Write(input[i])
   146  		zw.Close()
   147  
   148  		exp = append(exp, buf.Bytes())
   149  	}
   150  
   151  	if reflect.DeepEqual(input, exp) {
   152  		t.Fatal("Input and exp output are the same")
   153  	}
   154  
   155  	proc, err := NewCompress(conf, nil, testLog, metrics.Noop())
   156  	if err != nil {
   157  		t.Fatal(err)
   158  	}
   159  
   160  	msgs, res := proc.ProcessMessage(message.New(input))
   161  	if len(msgs) != 1 {
   162  		t.Error("Compress failed")
   163  	} else if res != nil {
   164  		t.Errorf("Expected nil response: %v", res)
   165  	}
   166  	if act := message.GetAllBytes(msgs[0]); !reflect.DeepEqual(exp, act) {
   167  		t.Errorf("Unexpected output: %s != %s", act, exp)
   168  	}
   169  }
   170  
   171  func TestCompressSnappy(t *testing.T) {
   172  	conf := NewConfig()
   173  	conf.Compress.Algorithm = "snappy"
   174  
   175  	testLog := log.Noop()
   176  
   177  	input := [][]byte{
   178  		[]byte("hello world first part"),
   179  		[]byte("hello world second part"),
   180  		[]byte("third part"),
   181  		[]byte("fourth"),
   182  		[]byte("5"),
   183  	}
   184  
   185  	exp := [][]byte{}
   186  
   187  	for i := range input {
   188  		output := snappy.Encode(nil, input[i])
   189  		exp = append(exp, output)
   190  	}
   191  
   192  	if reflect.DeepEqual(input, exp) {
   193  		t.Fatal("Input and exp output are the same")
   194  	}
   195  
   196  	proc, err := NewCompress(conf, nil, testLog, metrics.Noop())
   197  	if err != nil {
   198  		t.Fatal(err)
   199  	}
   200  
   201  	msgs, res := proc.ProcessMessage(message.New(input))
   202  	if len(msgs) != 1 {
   203  		t.Error("Compress failed")
   204  	} else if res != nil {
   205  		t.Errorf("Expected nil response: %v", res)
   206  	}
   207  	if act := message.GetAllBytes(msgs[0]); !reflect.DeepEqual(exp, act) {
   208  		t.Errorf("Unexpected output: %s != %s", act, exp)
   209  	}
   210  }
   211  
   212  func TestCompressLZ4(t *testing.T) {
   213  	conf := NewConfig()
   214  	conf.Compress.Algorithm = "lz4"
   215  
   216  	testLog := log.Noop()
   217  
   218  	input := [][]byte{
   219  		[]byte("hello world first part"),
   220  		[]byte("hello world second part"),
   221  		[]byte("third part"),
   222  		[]byte("fourth"),
   223  		[]byte("5"),
   224  	}
   225  
   226  	exp := [][]byte{}
   227  
   228  	for i := range input {
   229  		var buf bytes.Buffer
   230  
   231  		w := lz4.NewWriter(&buf)
   232  		if _, err := w.Write(input[i]); err != nil {
   233  			w.Close()
   234  			t.Fatalf("Failed to compress input: %s", err)
   235  		}
   236  		w.Close()
   237  
   238  		exp = append(exp, buf.Bytes())
   239  	}
   240  
   241  	if reflect.DeepEqual(input, exp) {
   242  		t.Fatal("Input and exp output are the same")
   243  	}
   244  
   245  	proc, err := NewCompress(conf, nil, testLog, metrics.Noop())
   246  	if err != nil {
   247  		t.Fatal(err)
   248  	}
   249  
   250  	msgs, res := proc.ProcessMessage(message.New(input))
   251  	if len(msgs) != 1 {
   252  		t.Error("Compress failed")
   253  	} else if res != nil {
   254  		t.Errorf("Expected nil response: %v", res)
   255  	}
   256  	if act := message.GetAllBytes(msgs[0]); !reflect.DeepEqual(exp, act) {
   257  		t.Errorf("Unexpected output: %s != %s", act, exp)
   258  	}
   259  }
   260  
   261  func TestCompressIndexBounds(t *testing.T) {
   262  	conf := NewConfig()
   263  
   264  	testLog := log.Noop()
   265  
   266  	input := [][]byte{
   267  		[]byte("0"),
   268  		[]byte("1"),
   269  		[]byte("2"),
   270  		[]byte("3"),
   271  		[]byte("4"),
   272  	}
   273  
   274  	compressed := [][]byte{}
   275  
   276  	for i := range input {
   277  		var buf bytes.Buffer
   278  
   279  		zw := gzip.NewWriter(&buf)
   280  		zw.Write(input[i])
   281  		zw.Close()
   282  
   283  		compressed = append(compressed, buf.Bytes())
   284  	}
   285  
   286  	tests := map[int]int{
   287  		-5: 0,
   288  		-4: 1,
   289  		-3: 2,
   290  		-2: 3,
   291  		-1: 4,
   292  		0:  0,
   293  		1:  1,
   294  		2:  2,
   295  		3:  3,
   296  		4:  4,
   297  	}
   298  
   299  	for i, expIndex := range tests {
   300  		conf.Compress.Parts = []int{i}
   301  		proc, err := NewCompress(conf, nil, testLog, metrics.Noop())
   302  		if err != nil {
   303  			t.Fatal(err)
   304  		}
   305  
   306  		msgs, res := proc.ProcessMessage(message.New(input))
   307  		if len(msgs) != 1 {
   308  			t.Errorf("Compress failed on index: %v", i)
   309  		} else if res != nil {
   310  			t.Errorf("Expected nil response: %v", res)
   311  		}
   312  		if exp, act := string(compressed[expIndex]), string(message.GetAllBytes(msgs[0])[expIndex]); exp != act {
   313  			t.Errorf("Unexpected output for index %v: %v != %v", i, act, exp)
   314  		}
   315  		if exp, act := string(compressed[expIndex]), string(message.GetAllBytes(msgs[0])[(expIndex+1)%5]); exp == act {
   316  			t.Errorf("Processor was applied to wrong index %v: %v != %v", (expIndex+1)%5, act, exp)
   317  		}
   318  	}
   319  }
   320  
   321  func TestCompressEmpty(t *testing.T) {
   322  	conf := NewConfig()
   323  	conf.Compress.Parts = []int{0, 1}
   324  
   325  	testLog := log.Noop()
   326  	proc, err := NewCompress(conf, nil, testLog, metrics.Noop())
   327  	if err != nil {
   328  		t.Error(err)
   329  		return
   330  	}
   331  
   332  	msgs, _ := proc.ProcessMessage(message.New([][]byte{}))
   333  	if len(msgs) > 0 {
   334  		t.Error("Expected failure with zero part message")
   335  	}
   336  }