github.com/Jeffail/benthos/v3@v3.65.0/lib/processor/decompress_test.go (about) 1 package processor 2 3 import ( 4 "bytes" 5 "compress/flate" 6 "compress/gzip" 7 "compress/zlib" 8 "reflect" 9 "testing" 10 11 "github.com/Jeffail/benthos/v3/lib/log" 12 "github.com/Jeffail/benthos/v3/lib/message" 13 "github.com/Jeffail/benthos/v3/lib/metrics" 14 "github.com/golang/snappy" 15 "github.com/pierrec/lz4/v4" 16 ) 17 18 func TestDecompressBadAlgo(t *testing.T) { 19 conf := NewConfig() 20 conf.Decompress.Algorithm = "does not exist" 21 22 testLog := log.Noop() 23 24 _, err := NewDecompress(conf, nil, testLog, metrics.Noop()) 25 if err == nil { 26 t.Error("Expected error from bad algo") 27 } 28 } 29 30 func TestDecompressGZIP(t *testing.T) { 31 conf := NewConfig() 32 conf.Decompress.Algorithm = "gzip" 33 34 testLog := log.Noop() 35 36 input := [][]byte{ 37 []byte("hello world first part"), 38 []byte("hello world second part"), 39 []byte("third part"), 40 []byte("fourth"), 41 []byte("5"), 42 } 43 44 exp := [][]byte{} 45 46 for i := range input { 47 exp = append(exp, input[i]) 48 49 var buf bytes.Buffer 50 51 zw := gzip.NewWriter(&buf) 52 zw.Write(input[i]) 53 zw.Close() 54 55 input[i] = buf.Bytes() 56 } 57 58 if reflect.DeepEqual(input, exp) { 59 t.Fatal("Input and exp output are the same") 60 } 61 62 proc, err := NewDecompress(conf, nil, testLog, metrics.Noop()) 63 if err != nil { 64 t.Fatal(err) 65 } 66 67 msgs, res := proc.ProcessMessage(message.New(input)) 68 if len(msgs) != 1 { 69 t.Error("Decompress failed") 70 } else if res != nil { 71 t.Errorf("Expected nil response: %v", res) 72 } 73 if act := message.GetAllBytes(msgs[0]); !reflect.DeepEqual(exp, act) { 74 t.Errorf("Unexpected output: %s != %s", act, exp) 75 } 76 } 77 78 func TestDecompressSnappy(t *testing.T) { 79 conf := NewConfig() 80 conf.Decompress.Algorithm = "snappy" 81 82 input := [][]byte{ 83 []byte("hello world first part"), 84 []byte("hello world second part"), 85 []byte("third part"), 86 []byte("fourth"), 87 []byte("5"), 88 } 89 90 exp := [][]byte{} 91 92 for i := range input { 93 exp = append(exp, input[i]) 94 input[i] = snappy.Encode(nil, input[i]) 95 } 96 97 if reflect.DeepEqual(input, exp) { 98 t.Fatal("Input and exp output are the same") 99 } 100 101 proc, err := NewDecompress(conf, nil, log.Noop(), metrics.Noop()) 102 if err != nil { 103 t.Fatal(err) 104 } 105 106 msgs, res := proc.ProcessMessage(message.New(input)) 107 if len(msgs) != 1 { 108 t.Error("Decompress failed") 109 } else if res != nil { 110 t.Errorf("Expected nil response: %v", res) 111 } 112 if act := message.GetAllBytes(msgs[0]); !reflect.DeepEqual(exp, act) { 113 t.Errorf("Unexpected output: %s != %s", act, exp) 114 } 115 } 116 117 func TestDecompressZLIB(t *testing.T) { 118 conf := NewConfig() 119 conf.Decompress.Algorithm = "zlib" 120 121 testLog := log.Noop() 122 123 input := [][]byte{ 124 []byte("hello world first part"), 125 []byte("hello world second part"), 126 []byte("third part"), 127 []byte("fourth"), 128 []byte("5"), 129 } 130 131 exp := [][]byte{} 132 133 for i := range input { 134 exp = append(exp, input[i]) 135 136 var buf bytes.Buffer 137 138 zw := zlib.NewWriter(&buf) 139 zw.Write(input[i]) 140 zw.Close() 141 142 input[i] = buf.Bytes() 143 } 144 145 if reflect.DeepEqual(input, exp) { 146 t.Fatal("Input and exp output are the same") 147 } 148 149 proc, err := NewDecompress(conf, nil, testLog, metrics.Noop()) 150 if err != nil { 151 t.Fatal(err) 152 } 153 154 msgs, res := proc.ProcessMessage(message.New(input)) 155 if len(msgs) != 1 { 156 t.Error("Decompress failed") 157 } else if res != nil { 158 t.Errorf("Expected nil response: %v", res) 159 } 160 if act := message.GetAllBytes(msgs[0]); !reflect.DeepEqual(exp, act) { 161 t.Errorf("Unexpected output: %s != %s", act, exp) 162 } 163 } 164 165 func TestDecompressFlate(t *testing.T) { 166 conf := NewConfig() 167 conf.Decompress.Algorithm = "flate" 168 169 testLog := log.Noop() 170 171 input := [][]byte{ 172 []byte("hello world first part"), 173 []byte("hello world second part"), 174 []byte("third part"), 175 []byte("fourth"), 176 []byte("5"), 177 } 178 179 exp := [][]byte{} 180 181 for i := range input { 182 exp = append(exp, input[i]) 183 184 var buf bytes.Buffer 185 186 zw, err := flate.NewWriter(&buf, 0) 187 if err != nil { 188 t.Fatal(err) 189 } 190 zw.Write(input[i]) 191 zw.Close() 192 193 input[i] = buf.Bytes() 194 } 195 196 if reflect.DeepEqual(input, exp) { 197 t.Fatal("Input and exp output are the same") 198 } 199 200 proc, err := NewDecompress(conf, nil, testLog, metrics.Noop()) 201 if err != nil { 202 t.Fatal(err) 203 } 204 205 msgs, res := proc.ProcessMessage(message.New(input)) 206 if len(msgs) != 1 { 207 t.Error("Decompress failed") 208 } else if res != nil { 209 t.Errorf("Expected nil response: %v", res) 210 } 211 if act := message.GetAllBytes(msgs[0]); !reflect.DeepEqual(exp, act) { 212 t.Errorf("Unexpected output: %s != %s", act, exp) 213 } 214 } 215 216 func TestDecompressLZ4(t *testing.T) { 217 conf := NewConfig() 218 conf.Decompress.Algorithm = "lz4" 219 220 input := [][]byte{ 221 []byte("hello world first part"), 222 []byte("hello world second part"), 223 []byte("third part"), 224 []byte("fourth"), 225 []byte("5"), 226 } 227 228 exp := [][]byte{} 229 230 for i := range input { 231 exp = append(exp, input[i]) 232 233 buf := bytes.Buffer{} 234 w := lz4.NewWriter(&buf) 235 if _, err := w.Write(input[i]); err != nil { 236 w.Close() 237 t.Fatalf("Failed to compress input: %s", err) 238 } 239 w.Close() 240 241 input[i] = buf.Bytes() 242 } 243 244 if reflect.DeepEqual(input, exp) { 245 t.Fatal("Input and exp output are the same") 246 } 247 248 proc, err := NewDecompress(conf, nil, log.Noop(), metrics.Noop()) 249 if err != nil { 250 t.Fatal(err) 251 } 252 253 msgs, res := proc.ProcessMessage(message.New(input)) 254 if len(msgs) != 1 { 255 t.Error("Decompress failed") 256 } else if res != nil { 257 t.Errorf("Expected nil response: %v", res) 258 } 259 if act := message.GetAllBytes(msgs[0]); !reflect.DeepEqual(exp, act) { 260 t.Errorf("Unexpected output: %s != %s", act, exp) 261 } 262 } 263 264 func TestDecompressIndexBounds(t *testing.T) { 265 conf := NewConfig() 266 267 testLog := log.Noop() 268 269 input := [][]byte{ 270 []byte("0"), 271 []byte("1"), 272 []byte("2"), 273 []byte("3"), 274 []byte("4"), 275 } 276 277 for i := range input { 278 var buf bytes.Buffer 279 280 zw := gzip.NewWriter(&buf) 281 zw.Write(input[i]) 282 zw.Close() 283 284 input[i] = buf.Bytes() 285 } 286 287 type result struct { 288 index int 289 value string 290 } 291 292 tests := map[int]result{ 293 -5: { 294 index: 0, 295 value: "0", 296 }, 297 -4: { 298 index: 1, 299 value: "1", 300 }, 301 -3: { 302 index: 2, 303 value: "2", 304 }, 305 -2: { 306 index: 3, 307 value: "3", 308 }, 309 -1: { 310 index: 4, 311 value: "4", 312 }, 313 0: { 314 index: 0, 315 value: "0", 316 }, 317 1: { 318 index: 1, 319 value: "1", 320 }, 321 2: { 322 index: 2, 323 value: "2", 324 }, 325 3: { 326 index: 3, 327 value: "3", 328 }, 329 4: { 330 index: 4, 331 value: "4", 332 }, 333 } 334 335 for i, result := range tests { 336 conf.Decompress.Parts = []int{i} 337 proc, err := NewDecompress(conf, nil, testLog, metrics.Noop()) 338 if err != nil { 339 t.Fatal(err) 340 } 341 342 msgs, res := proc.ProcessMessage(message.New(input)) 343 if len(msgs) != 1 { 344 t.Errorf("Decompress failed on index: %v", i) 345 } else if res != nil { 346 t.Errorf("Expected nil response: %v", res) 347 } 348 if exp, act := result.value, string(message.GetAllBytes(msgs[0])[result.index]); exp != act { 349 t.Errorf("Unexpected output for index %v: %v != %v", i, act, exp) 350 } 351 if exp, act := result.value, string(message.GetAllBytes(msgs[0])[(result.index+1)%5]); exp == act { 352 t.Errorf("Processor was applied to wrong index %v: %v != %v", (result.index+1)%5, act, exp) 353 } 354 } 355 }