github.com/Jeffail/benthos/v3@v3.65.0/lib/processor/decompress_test.go (about)

     1  package processor
     2  
     3  import (
     4  	"bytes"
     5  	"compress/flate"
     6  	"compress/gzip"
     7  	"compress/zlib"
     8  	"reflect"
     9  	"testing"
    10  
    11  	"github.com/Jeffail/benthos/v3/lib/log"
    12  	"github.com/Jeffail/benthos/v3/lib/message"
    13  	"github.com/Jeffail/benthos/v3/lib/metrics"
    14  	"github.com/golang/snappy"
    15  	"github.com/pierrec/lz4/v4"
    16  )
    17  
    18  func TestDecompressBadAlgo(t *testing.T) {
    19  	conf := NewConfig()
    20  	conf.Decompress.Algorithm = "does not exist"
    21  
    22  	testLog := log.Noop()
    23  
    24  	_, err := NewDecompress(conf, nil, testLog, metrics.Noop())
    25  	if err == nil {
    26  		t.Error("Expected error from bad algo")
    27  	}
    28  }
    29  
    30  func TestDecompressGZIP(t *testing.T) {
    31  	conf := NewConfig()
    32  	conf.Decompress.Algorithm = "gzip"
    33  
    34  	testLog := log.Noop()
    35  
    36  	input := [][]byte{
    37  		[]byte("hello world first part"),
    38  		[]byte("hello world second part"),
    39  		[]byte("third part"),
    40  		[]byte("fourth"),
    41  		[]byte("5"),
    42  	}
    43  
    44  	exp := [][]byte{}
    45  
    46  	for i := range input {
    47  		exp = append(exp, input[i])
    48  
    49  		var buf bytes.Buffer
    50  
    51  		zw := gzip.NewWriter(&buf)
    52  		zw.Write(input[i])
    53  		zw.Close()
    54  
    55  		input[i] = buf.Bytes()
    56  	}
    57  
    58  	if reflect.DeepEqual(input, exp) {
    59  		t.Fatal("Input and exp output are the same")
    60  	}
    61  
    62  	proc, err := NewDecompress(conf, nil, testLog, metrics.Noop())
    63  	if err != nil {
    64  		t.Fatal(err)
    65  	}
    66  
    67  	msgs, res := proc.ProcessMessage(message.New(input))
    68  	if len(msgs) != 1 {
    69  		t.Error("Decompress failed")
    70  	} else if res != nil {
    71  		t.Errorf("Expected nil response: %v", res)
    72  	}
    73  	if act := message.GetAllBytes(msgs[0]); !reflect.DeepEqual(exp, act) {
    74  		t.Errorf("Unexpected output: %s != %s", act, exp)
    75  	}
    76  }
    77  
    78  func TestDecompressSnappy(t *testing.T) {
    79  	conf := NewConfig()
    80  	conf.Decompress.Algorithm = "snappy"
    81  
    82  	input := [][]byte{
    83  		[]byte("hello world first part"),
    84  		[]byte("hello world second part"),
    85  		[]byte("third part"),
    86  		[]byte("fourth"),
    87  		[]byte("5"),
    88  	}
    89  
    90  	exp := [][]byte{}
    91  
    92  	for i := range input {
    93  		exp = append(exp, input[i])
    94  		input[i] = snappy.Encode(nil, input[i])
    95  	}
    96  
    97  	if reflect.DeepEqual(input, exp) {
    98  		t.Fatal("Input and exp output are the same")
    99  	}
   100  
   101  	proc, err := NewDecompress(conf, nil, log.Noop(), metrics.Noop())
   102  	if err != nil {
   103  		t.Fatal(err)
   104  	}
   105  
   106  	msgs, res := proc.ProcessMessage(message.New(input))
   107  	if len(msgs) != 1 {
   108  		t.Error("Decompress failed")
   109  	} else if res != nil {
   110  		t.Errorf("Expected nil response: %v", res)
   111  	}
   112  	if act := message.GetAllBytes(msgs[0]); !reflect.DeepEqual(exp, act) {
   113  		t.Errorf("Unexpected output: %s != %s", act, exp)
   114  	}
   115  }
   116  
   117  func TestDecompressZLIB(t *testing.T) {
   118  	conf := NewConfig()
   119  	conf.Decompress.Algorithm = "zlib"
   120  
   121  	testLog := log.Noop()
   122  
   123  	input := [][]byte{
   124  		[]byte("hello world first part"),
   125  		[]byte("hello world second part"),
   126  		[]byte("third part"),
   127  		[]byte("fourth"),
   128  		[]byte("5"),
   129  	}
   130  
   131  	exp := [][]byte{}
   132  
   133  	for i := range input {
   134  		exp = append(exp, input[i])
   135  
   136  		var buf bytes.Buffer
   137  
   138  		zw := zlib.NewWriter(&buf)
   139  		zw.Write(input[i])
   140  		zw.Close()
   141  
   142  		input[i] = buf.Bytes()
   143  	}
   144  
   145  	if reflect.DeepEqual(input, exp) {
   146  		t.Fatal("Input and exp output are the same")
   147  	}
   148  
   149  	proc, err := NewDecompress(conf, nil, testLog, metrics.Noop())
   150  	if err != nil {
   151  		t.Fatal(err)
   152  	}
   153  
   154  	msgs, res := proc.ProcessMessage(message.New(input))
   155  	if len(msgs) != 1 {
   156  		t.Error("Decompress failed")
   157  	} else if res != nil {
   158  		t.Errorf("Expected nil response: %v", res)
   159  	}
   160  	if act := message.GetAllBytes(msgs[0]); !reflect.DeepEqual(exp, act) {
   161  		t.Errorf("Unexpected output: %s != %s", act, exp)
   162  	}
   163  }
   164  
   165  func TestDecompressFlate(t *testing.T) {
   166  	conf := NewConfig()
   167  	conf.Decompress.Algorithm = "flate"
   168  
   169  	testLog := log.Noop()
   170  
   171  	input := [][]byte{
   172  		[]byte("hello world first part"),
   173  		[]byte("hello world second part"),
   174  		[]byte("third part"),
   175  		[]byte("fourth"),
   176  		[]byte("5"),
   177  	}
   178  
   179  	exp := [][]byte{}
   180  
   181  	for i := range input {
   182  		exp = append(exp, input[i])
   183  
   184  		var buf bytes.Buffer
   185  
   186  		zw, err := flate.NewWriter(&buf, 0)
   187  		if err != nil {
   188  			t.Fatal(err)
   189  		}
   190  		zw.Write(input[i])
   191  		zw.Close()
   192  
   193  		input[i] = buf.Bytes()
   194  	}
   195  
   196  	if reflect.DeepEqual(input, exp) {
   197  		t.Fatal("Input and exp output are the same")
   198  	}
   199  
   200  	proc, err := NewDecompress(conf, nil, testLog, metrics.Noop())
   201  	if err != nil {
   202  		t.Fatal(err)
   203  	}
   204  
   205  	msgs, res := proc.ProcessMessage(message.New(input))
   206  	if len(msgs) != 1 {
   207  		t.Error("Decompress failed")
   208  	} else if res != nil {
   209  		t.Errorf("Expected nil response: %v", res)
   210  	}
   211  	if act := message.GetAllBytes(msgs[0]); !reflect.DeepEqual(exp, act) {
   212  		t.Errorf("Unexpected output: %s != %s", act, exp)
   213  	}
   214  }
   215  
   216  func TestDecompressLZ4(t *testing.T) {
   217  	conf := NewConfig()
   218  	conf.Decompress.Algorithm = "lz4"
   219  
   220  	input := [][]byte{
   221  		[]byte("hello world first part"),
   222  		[]byte("hello world second part"),
   223  		[]byte("third part"),
   224  		[]byte("fourth"),
   225  		[]byte("5"),
   226  	}
   227  
   228  	exp := [][]byte{}
   229  
   230  	for i := range input {
   231  		exp = append(exp, input[i])
   232  
   233  		buf := bytes.Buffer{}
   234  		w := lz4.NewWriter(&buf)
   235  		if _, err := w.Write(input[i]); err != nil {
   236  			w.Close()
   237  			t.Fatalf("Failed to compress input: %s", err)
   238  		}
   239  		w.Close()
   240  
   241  		input[i] = buf.Bytes()
   242  	}
   243  
   244  	if reflect.DeepEqual(input, exp) {
   245  		t.Fatal("Input and exp output are the same")
   246  	}
   247  
   248  	proc, err := NewDecompress(conf, nil, log.Noop(), metrics.Noop())
   249  	if err != nil {
   250  		t.Fatal(err)
   251  	}
   252  
   253  	msgs, res := proc.ProcessMessage(message.New(input))
   254  	if len(msgs) != 1 {
   255  		t.Error("Decompress failed")
   256  	} else if res != nil {
   257  		t.Errorf("Expected nil response: %v", res)
   258  	}
   259  	if act := message.GetAllBytes(msgs[0]); !reflect.DeepEqual(exp, act) {
   260  		t.Errorf("Unexpected output: %s != %s", act, exp)
   261  	}
   262  }
   263  
   264  func TestDecompressIndexBounds(t *testing.T) {
   265  	conf := NewConfig()
   266  
   267  	testLog := log.Noop()
   268  
   269  	input := [][]byte{
   270  		[]byte("0"),
   271  		[]byte("1"),
   272  		[]byte("2"),
   273  		[]byte("3"),
   274  		[]byte("4"),
   275  	}
   276  
   277  	for i := range input {
   278  		var buf bytes.Buffer
   279  
   280  		zw := gzip.NewWriter(&buf)
   281  		zw.Write(input[i])
   282  		zw.Close()
   283  
   284  		input[i] = buf.Bytes()
   285  	}
   286  
   287  	type result struct {
   288  		index int
   289  		value string
   290  	}
   291  
   292  	tests := map[int]result{
   293  		-5: {
   294  			index: 0,
   295  			value: "0",
   296  		},
   297  		-4: {
   298  			index: 1,
   299  			value: "1",
   300  		},
   301  		-3: {
   302  			index: 2,
   303  			value: "2",
   304  		},
   305  		-2: {
   306  			index: 3,
   307  			value: "3",
   308  		},
   309  		-1: {
   310  			index: 4,
   311  			value: "4",
   312  		},
   313  		0: {
   314  			index: 0,
   315  			value: "0",
   316  		},
   317  		1: {
   318  			index: 1,
   319  			value: "1",
   320  		},
   321  		2: {
   322  			index: 2,
   323  			value: "2",
   324  		},
   325  		3: {
   326  			index: 3,
   327  			value: "3",
   328  		},
   329  		4: {
   330  			index: 4,
   331  			value: "4",
   332  		},
   333  	}
   334  
   335  	for i, result := range tests {
   336  		conf.Decompress.Parts = []int{i}
   337  		proc, err := NewDecompress(conf, nil, testLog, metrics.Noop())
   338  		if err != nil {
   339  			t.Fatal(err)
   340  		}
   341  
   342  		msgs, res := proc.ProcessMessage(message.New(input))
   343  		if len(msgs) != 1 {
   344  			t.Errorf("Decompress failed on index: %v", i)
   345  		} else if res != nil {
   346  			t.Errorf("Expected nil response: %v", res)
   347  		}
   348  		if exp, act := result.value, string(message.GetAllBytes(msgs[0])[result.index]); exp != act {
   349  			t.Errorf("Unexpected output for index %v: %v != %v", i, act, exp)
   350  		}
   351  		if exp, act := result.value, string(message.GetAllBytes(msgs[0])[(result.index+1)%5]); exp == act {
   352  			t.Errorf("Processor was applied to wrong index %v: %v != %v", (result.index+1)%5, act, exp)
   353  		}
   354  	}
   355  }