github.com/Jeffail/benthos/v3@v3.65.0/lib/processor/compress_test.go (about) 1 package processor 2 3 import ( 4 "bytes" 5 "compress/flate" 6 "compress/gzip" 7 "compress/zlib" 8 "reflect" 9 "testing" 10 11 "github.com/Jeffail/benthos/v3/lib/log" 12 "github.com/Jeffail/benthos/v3/lib/message" 13 "github.com/Jeffail/benthos/v3/lib/metrics" 14 "github.com/golang/snappy" 15 "github.com/pierrec/lz4/v4" 16 ) 17 18 func TestCompressBadAlgo(t *testing.T) { 19 conf := NewConfig() 20 conf.Compress.Algorithm = "does not exist" 21 22 testLog := log.Noop() 23 24 _, err := NewCompress(conf, nil, testLog, metrics.Noop()) 25 if err == nil { 26 t.Error("Expected error from bad algo") 27 } 28 } 29 30 func TestCompressGZIP(t *testing.T) { 31 conf := NewConfig() 32 conf.Compress.Algorithm = "gzip" 33 34 testLog := log.Noop() 35 36 input := [][]byte{ 37 []byte("hello world first part"), 38 []byte("hello world second part"), 39 []byte("third part"), 40 []byte("fourth"), 41 []byte("5"), 42 } 43 44 exp := [][]byte{} 45 46 for i := range input { 47 var buf bytes.Buffer 48 49 zw := gzip.NewWriter(&buf) 50 zw.Write(input[i]) 51 zw.Close() 52 53 exp = append(exp, buf.Bytes()) 54 } 55 56 if reflect.DeepEqual(input, exp) { 57 t.Fatal("Input and exp output are the same") 58 } 59 60 proc, err := NewCompress(conf, nil, testLog, metrics.Noop()) 61 if err != nil { 62 t.Fatal(err) 63 } 64 65 msgs, res := proc.ProcessMessage(message.New(input)) 66 if len(msgs) != 1 { 67 t.Error("Compress failed") 68 } else if res != nil { 69 t.Errorf("Expected nil response: %v", res) 70 } 71 if act := message.GetAllBytes(msgs[0]); !reflect.DeepEqual(exp, act) { 72 t.Errorf("Unexpected output: %s != %s", act, exp) 73 } 74 } 75 76 func TestCompressZLIB(t *testing.T) { 77 conf := NewConfig() 78 conf.Compress.Algorithm = "zlib" 79 80 testLog := log.Noop() 81 82 input := [][]byte{ 83 []byte("hello world first part"), 84 []byte("hello world second part"), 85 []byte("third part"), 86 []byte("fourth"), 87 []byte("5"), 88 } 89 90 exp := [][]byte{} 91 92 for i := range input { 93 var buf bytes.Buffer 94 95 zw := zlib.NewWriter(&buf) 96 zw.Write(input[i]) 97 zw.Close() 98 99 exp = append(exp, buf.Bytes()) 100 } 101 102 if reflect.DeepEqual(input, exp) { 103 t.Fatal("Input and exp output are the same") 104 } 105 106 proc, err := NewCompress(conf, nil, testLog, metrics.Noop()) 107 if err != nil { 108 t.Fatal(err) 109 } 110 111 msgs, res := proc.ProcessMessage(message.New(input)) 112 if len(msgs) != 1 { 113 t.Error("Compress failed") 114 } else if res != nil { 115 t.Errorf("Expected nil response: %v", res) 116 } 117 if act := message.GetAllBytes(msgs[0]); !reflect.DeepEqual(exp, act) { 118 t.Errorf("Unexpected output: %s != %s", act, exp) 119 } 120 } 121 122 func TestCompressFlate(t *testing.T) { 123 conf := NewConfig() 124 conf.Compress.Algorithm = "flate" 125 126 testLog := log.Noop() 127 128 input := [][]byte{ 129 []byte("hello world first part"), 130 []byte("hello world second part"), 131 []byte("third part"), 132 []byte("fourth"), 133 []byte("5"), 134 } 135 136 exp := [][]byte{} 137 138 for i := range input { 139 var buf bytes.Buffer 140 141 zw, err := flate.NewWriter(&buf, conf.Compress.Level) 142 if err != nil { 143 t.Fatal(err) 144 } 145 zw.Write(input[i]) 146 zw.Close() 147 148 exp = append(exp, buf.Bytes()) 149 } 150 151 if reflect.DeepEqual(input, exp) { 152 t.Fatal("Input and exp output are the same") 153 } 154 155 proc, err := NewCompress(conf, nil, testLog, metrics.Noop()) 156 if err != nil { 157 t.Fatal(err) 158 } 159 160 msgs, res := proc.ProcessMessage(message.New(input)) 161 if len(msgs) != 1 { 162 t.Error("Compress failed") 163 } else if res != nil { 164 t.Errorf("Expected nil response: %v", res) 165 } 166 if act := message.GetAllBytes(msgs[0]); !reflect.DeepEqual(exp, act) { 167 t.Errorf("Unexpected output: %s != %s", act, exp) 168 } 169 } 170 171 func TestCompressSnappy(t *testing.T) { 172 conf := NewConfig() 173 conf.Compress.Algorithm = "snappy" 174 175 testLog := log.Noop() 176 177 input := [][]byte{ 178 []byte("hello world first part"), 179 []byte("hello world second part"), 180 []byte("third part"), 181 []byte("fourth"), 182 []byte("5"), 183 } 184 185 exp := [][]byte{} 186 187 for i := range input { 188 output := snappy.Encode(nil, input[i]) 189 exp = append(exp, output) 190 } 191 192 if reflect.DeepEqual(input, exp) { 193 t.Fatal("Input and exp output are the same") 194 } 195 196 proc, err := NewCompress(conf, nil, testLog, metrics.Noop()) 197 if err != nil { 198 t.Fatal(err) 199 } 200 201 msgs, res := proc.ProcessMessage(message.New(input)) 202 if len(msgs) != 1 { 203 t.Error("Compress failed") 204 } else if res != nil { 205 t.Errorf("Expected nil response: %v", res) 206 } 207 if act := message.GetAllBytes(msgs[0]); !reflect.DeepEqual(exp, act) { 208 t.Errorf("Unexpected output: %s != %s", act, exp) 209 } 210 } 211 212 func TestCompressLZ4(t *testing.T) { 213 conf := NewConfig() 214 conf.Compress.Algorithm = "lz4" 215 216 testLog := log.Noop() 217 218 input := [][]byte{ 219 []byte("hello world first part"), 220 []byte("hello world second part"), 221 []byte("third part"), 222 []byte("fourth"), 223 []byte("5"), 224 } 225 226 exp := [][]byte{} 227 228 for i := range input { 229 var buf bytes.Buffer 230 231 w := lz4.NewWriter(&buf) 232 if _, err := w.Write(input[i]); err != nil { 233 w.Close() 234 t.Fatalf("Failed to compress input: %s", err) 235 } 236 w.Close() 237 238 exp = append(exp, buf.Bytes()) 239 } 240 241 if reflect.DeepEqual(input, exp) { 242 t.Fatal("Input and exp output are the same") 243 } 244 245 proc, err := NewCompress(conf, nil, testLog, metrics.Noop()) 246 if err != nil { 247 t.Fatal(err) 248 } 249 250 msgs, res := proc.ProcessMessage(message.New(input)) 251 if len(msgs) != 1 { 252 t.Error("Compress failed") 253 } else if res != nil { 254 t.Errorf("Expected nil response: %v", res) 255 } 256 if act := message.GetAllBytes(msgs[0]); !reflect.DeepEqual(exp, act) { 257 t.Errorf("Unexpected output: %s != %s", act, exp) 258 } 259 } 260 261 func TestCompressIndexBounds(t *testing.T) { 262 conf := NewConfig() 263 264 testLog := log.Noop() 265 266 input := [][]byte{ 267 []byte("0"), 268 []byte("1"), 269 []byte("2"), 270 []byte("3"), 271 []byte("4"), 272 } 273 274 compressed := [][]byte{} 275 276 for i := range input { 277 var buf bytes.Buffer 278 279 zw := gzip.NewWriter(&buf) 280 zw.Write(input[i]) 281 zw.Close() 282 283 compressed = append(compressed, buf.Bytes()) 284 } 285 286 tests := map[int]int{ 287 -5: 0, 288 -4: 1, 289 -3: 2, 290 -2: 3, 291 -1: 4, 292 0: 0, 293 1: 1, 294 2: 2, 295 3: 3, 296 4: 4, 297 } 298 299 for i, expIndex := range tests { 300 conf.Compress.Parts = []int{i} 301 proc, err := NewCompress(conf, nil, testLog, metrics.Noop()) 302 if err != nil { 303 t.Fatal(err) 304 } 305 306 msgs, res := proc.ProcessMessage(message.New(input)) 307 if len(msgs) != 1 { 308 t.Errorf("Compress failed on index: %v", i) 309 } else if res != nil { 310 t.Errorf("Expected nil response: %v", res) 311 } 312 if exp, act := string(compressed[expIndex]), string(message.GetAllBytes(msgs[0])[expIndex]); exp != act { 313 t.Errorf("Unexpected output for index %v: %v != %v", i, act, exp) 314 } 315 if exp, act := string(compressed[expIndex]), string(message.GetAllBytes(msgs[0])[(expIndex+1)%5]); exp == act { 316 t.Errorf("Processor was applied to wrong index %v: %v != %v", (expIndex+1)%5, act, exp) 317 } 318 } 319 } 320 321 func TestCompressEmpty(t *testing.T) { 322 conf := NewConfig() 323 conf.Compress.Parts = []int{0, 1} 324 325 testLog := log.Noop() 326 proc, err := NewCompress(conf, nil, testLog, metrics.Noop()) 327 if err != nil { 328 t.Error(err) 329 return 330 } 331 332 msgs, _ := proc.ProcessMessage(message.New([][]byte{})) 333 if len(msgs) > 0 { 334 t.Error("Expected failure with zero part message") 335 } 336 }