github.com/insionng/yougam@v0.0.0-20170714101924-2bc18d833463/libraries/klauspost/compress/gzip/gzip_test.go (about) 1 // Copyright 2010 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package gzip 6 7 import ( 8 "bufio" 9 "bytes" 10 oldgz "compress/gzip" 11 "io" 12 "io/ioutil" 13 "math/rand" 14 "testing" 15 "time" 16 ) 17 18 // TestEmpty tests that an empty payload still forms a valid GZIP stream. 19 func TestEmpty(t *testing.T) { 20 buf := new(bytes.Buffer) 21 22 if err := NewWriter(buf).Close(); err != nil { 23 t.Fatalf("Writer.Close: %v", err) 24 } 25 26 r, err := NewReader(buf) 27 if err != nil { 28 t.Fatalf("NewReader: %v", err) 29 } 30 b, err := ioutil.ReadAll(r) 31 if err != nil { 32 t.Fatalf("ReadAll: %v", err) 33 } 34 if len(b) != 0 { 35 t.Fatalf("got %d bytes, want 0", len(b)) 36 } 37 if err := r.Close(); err != nil { 38 t.Fatalf("Reader.Close: %v", err) 39 } 40 } 41 42 // TestRoundTrip tests that gzipping and then gunzipping is the identity 43 // function. 44 func TestRoundTrip(t *testing.T) { 45 buf := new(bytes.Buffer) 46 47 w := NewWriter(buf) 48 w.Comment = "comment" 49 w.Extra = []byte("extra") 50 w.ModTime = time.Unix(1e8, 0) 51 w.Name = "name" 52 if _, err := w.Write([]byte("payload")); err != nil { 53 t.Fatalf("Write: %v", err) 54 } 55 if err := w.Close(); err != nil { 56 t.Fatalf("Writer.Close: %v", err) 57 } 58 59 r, err := NewReader(buf) 60 if err != nil { 61 t.Fatalf("NewReader: %v", err) 62 } 63 b, err := ioutil.ReadAll(r) 64 if err != nil { 65 t.Fatalf("ReadAll: %v", err) 66 } 67 if string(b) != "payload" { 68 t.Fatalf("payload is %q, want %q", string(b), "payload") 69 } 70 if r.Comment != "comment" { 71 t.Fatalf("comment is %q, want %q", r.Comment, "comment") 72 } 73 if string(r.Extra) != "extra" { 74 t.Fatalf("extra is %q, want %q", r.Extra, "extra") 75 } 76 if r.ModTime.Unix() != 1e8 { 77 t.Fatalf("mtime is %d, want %d", r.ModTime.Unix(), uint32(1e8)) 78 } 79 if r.Name != "name" { 80 t.Fatalf("name is %q, want %q", r.Name, "name") 81 } 82 if err := r.Close(); err != nil { 83 t.Fatalf("Reader.Close: %v", err) 84 } 85 } 86 87 // TestLatin1 tests the internal functions for converting to and from Latin-1. 88 func TestLatin1(t *testing.T) { 89 latin1 := []byte{0xc4, 'u', 0xdf, 'e', 'r', 'u', 'n', 'g', 0} 90 utf8 := "Äußerung" 91 z := Reader{r: bufio.NewReader(bytes.NewReader(latin1))} 92 s, err := z.readString() 93 if err != nil { 94 t.Fatalf("readString: %v", err) 95 } 96 if s != utf8 { 97 t.Fatalf("read latin-1: got %q, want %q", s, utf8) 98 } 99 100 buf := bytes.NewBuffer(make([]byte, 0, len(latin1))) 101 c := Writer{w: buf} 102 if err = c.writeString(utf8); err != nil { 103 t.Fatalf("writeString: %v", err) 104 } 105 s = buf.String() 106 if s != string(latin1) { 107 t.Fatalf("write utf-8: got %q, want %q", s, string(latin1)) 108 } 109 } 110 111 // TestLatin1RoundTrip tests that metadata that is representable in Latin-1 112 // survives a round trip. 113 func TestLatin1RoundTrip(t *testing.T) { 114 testCases := []struct { 115 name string 116 ok bool 117 }{ 118 {"", true}, 119 {"ASCII is OK", true}, 120 {"unless it contains a NUL\x00", false}, 121 {"no matter where \x00 occurs", false}, 122 {"\x00\x00\x00", false}, 123 {"Látin-1 also passes (U+00E1)", true}, 124 {"but LĀtin Extended-A (U+0100) does not", false}, 125 {"neither does 日本語", false}, 126 {"invalid UTF-8 also \xffails", false}, 127 {"\x00 as does Látin-1 with NUL", false}, 128 } 129 for _, tc := range testCases { 130 buf := new(bytes.Buffer) 131 132 w := NewWriter(buf) 133 w.Name = tc.name 134 err := w.Close() 135 if (err == nil) != tc.ok { 136 t.Errorf("Writer.Close: name = %q, err = %v", tc.name, err) 137 continue 138 } 139 if !tc.ok { 140 continue 141 } 142 143 r, err := NewReader(buf) 144 if err != nil { 145 t.Errorf("NewReader: %v", err) 146 continue 147 } 148 _, err = ioutil.ReadAll(r) 149 if err != nil { 150 t.Errorf("ReadAll: %v", err) 151 continue 152 } 153 if r.Name != tc.name { 154 t.Errorf("name is %q, want %q", r.Name, tc.name) 155 continue 156 } 157 if err := r.Close(); err != nil { 158 t.Errorf("Reader.Close: %v", err) 159 continue 160 } 161 } 162 } 163 164 func TestWriterFlush(t *testing.T) { 165 buf := new(bytes.Buffer) 166 167 w := NewWriter(buf) 168 w.Comment = "comment" 169 w.Extra = []byte("extra") 170 w.ModTime = time.Unix(1e8, 0) 171 w.Name = "name" 172 173 n0 := buf.Len() 174 if n0 != 0 { 175 t.Fatalf("buffer size = %d before writes; want 0", n0) 176 } 177 178 if err := w.Flush(); err != nil { 179 t.Fatal(err) 180 } 181 182 n1 := buf.Len() 183 if n1 == 0 { 184 t.Fatal("no data after first flush") 185 } 186 187 w.Write([]byte("x")) 188 189 n2 := buf.Len() 190 if n1 != n2 { 191 t.Fatalf("after writing a single byte, size changed from %d to %d; want no change", n1, n2) 192 } 193 194 if err := w.Flush(); err != nil { 195 t.Fatal(err) 196 } 197 198 n3 := buf.Len() 199 if n2 == n3 { 200 t.Fatal("Flush didn't flush any data") 201 } 202 } 203 204 // Multiple gzip files concatenated form a valid gzip file. 205 func TestConcat(t *testing.T) { 206 var buf bytes.Buffer 207 w := NewWriter(&buf) 208 w.Write([]byte("hello ")) 209 w.Close() 210 w = NewWriter(&buf) 211 w.Write([]byte("world\n")) 212 w.Close() 213 214 r, err := NewReader(&buf) 215 data, err := ioutil.ReadAll(r) 216 if string(data) != "hello world\n" || err != nil { 217 t.Fatalf("ReadAll = %q, %v, want %q, nil", data, err, "hello world") 218 } 219 } 220 221 func TestWriterReset(t *testing.T) { 222 buf := new(bytes.Buffer) 223 buf2 := new(bytes.Buffer) 224 z := NewWriter(buf) 225 msg := []byte("hello world") 226 z.Write(msg) 227 z.Close() 228 z.Reset(buf2) 229 z.Write(msg) 230 z.Close() 231 if buf.String() != buf2.String() { 232 t.Errorf("buf2 %q != original buf of %q", buf2.String(), buf.String()) 233 } 234 } 235 236 var testbuf []byte 237 238 func testFile(i, level int, t *testing.T) { 239 dat, _ := ioutil.ReadFile("testdata/test.json") 240 dl := len(dat) 241 if len(testbuf) != i*dl { 242 // Make results predictable 243 testbuf = make([]byte, i*dl) 244 for j := 0; j < i; j++ { 245 copy(testbuf[j*dl:j*dl+dl], dat) 246 } 247 } 248 249 br := bytes.NewBuffer(testbuf) 250 var buf bytes.Buffer 251 w, err := NewWriterLevel(&buf, DefaultCompression) 252 if err != nil { 253 t.Fatal(err) 254 } 255 n, err := io.Copy(w, br) 256 if err != nil { 257 t.Fatal(err) 258 } 259 if int(n) != len(testbuf) { 260 t.Fatal("Short write:", n, "!=", testbuf) 261 } 262 err = w.Close() 263 if err != nil { 264 t.Fatal(err) 265 } 266 r, err := NewReader(&buf) 267 if err != nil { 268 t.Fatal(err.Error()) 269 } 270 decoded, err := ioutil.ReadAll(r) 271 if err != nil { 272 t.Fatal(err.Error()) 273 } 274 if !bytes.Equal(testbuf, decoded) { 275 t.Errorf("decoded content does not match.") 276 } 277 } 278 279 func TestFile1xM2(t *testing.T) { testFile(1, -2, t) } 280 func TestFile1xM1(t *testing.T) { testFile(1, -1, t) } 281 func TestFile1x0(t *testing.T) { testFile(1, 0, t) } 282 func TestFile1x1(t *testing.T) { testFile(1, 1, t) } 283 func TestFile1x2(t *testing.T) { testFile(1, 2, t) } 284 func TestFile1x3(t *testing.T) { testFile(1, 3, t) } 285 func TestFile1x4(t *testing.T) { testFile(1, 4, t) } 286 func TestFile1x5(t *testing.T) { testFile(1, 5, t) } 287 func TestFile1x6(t *testing.T) { testFile(1, 6, t) } 288 func TestFile1x7(t *testing.T) { testFile(1, 7, t) } 289 func TestFile1x8(t *testing.T) { testFile(1, 8, t) } 290 func TestFile1x9(t *testing.T) { testFile(1, 9, t) } 291 func TestFile10(t *testing.T) { testFile(10, DefaultCompression, t) } 292 293 func TestFile50(t *testing.T) { 294 if testing.Short() { 295 t.Skip("skipping during short test") 296 } 297 testFile(50, DefaultCompression, t) 298 } 299 300 func TestFile200(t *testing.T) { 301 if testing.Short() { 302 t.Skip("skipping during short test") 303 } 304 testFile(200, BestSpeed, t) 305 } 306 307 func testBigGzip(i int, t *testing.T) { 308 if len(testbuf) != i { 309 // Make results predictable 310 rand.Seed(1337) 311 testbuf = make([]byte, i) 312 for idx := range testbuf { 313 testbuf[idx] = byte(65 + rand.Intn(20)) 314 } 315 } 316 c := BestCompression 317 if testing.Short() { 318 c = BestSpeed 319 } 320 321 br := bytes.NewBuffer(testbuf) 322 var buf bytes.Buffer 323 w, err := NewWriterLevel(&buf, c) 324 if err != nil { 325 t.Fatal(err) 326 } 327 n, err := io.Copy(w, br) 328 if err != nil { 329 t.Fatal(err) 330 } 331 if int(n) != len(testbuf) { 332 t.Fatal("Short write:", n, "!=", len(testbuf)) 333 } 334 err = w.Close() 335 if err != nil { 336 t.Fatal(err.Error()) 337 } 338 339 r, err := NewReader(&buf) 340 if err != nil { 341 t.Fatal(err.Error()) 342 } 343 decoded, err := ioutil.ReadAll(r) 344 if err != nil { 345 t.Fatal(err.Error()) 346 } 347 if !bytes.Equal(testbuf, decoded) { 348 t.Errorf("decoded content does not match.") 349 } 350 } 351 352 func TestGzip1K(t *testing.T) { testBigGzip(1000, t) } 353 func TestGzip100K(t *testing.T) { testBigGzip(100000, t) } 354 func TestGzip1M(t *testing.T) { 355 if testing.Short() { 356 t.Skip("skipping during short test") 357 } 358 359 testBigGzip(1000000, t) 360 } 361 func TestGzip10M(t *testing.T) { 362 if testing.Short() { 363 t.Skip("skipping during short test") 364 } 365 testBigGzip(10000000, t) 366 } 367 368 // Test if two runs produce identical results. 369 func TestDeterministicLM2(t *testing.T) { testDeterm(-2, t) } 370 371 // Level 0 is not deterministic since it depends on the size of each write. 372 // func TestDeterministicL0(t *testing.T) { testDeterm(0, t) } 373 func TestDeterministicL1(t *testing.T) { testDeterm(1, t) } 374 func TestDeterministicL2(t *testing.T) { testDeterm(2, t) } 375 func TestDeterministicL3(t *testing.T) { testDeterm(3, t) } 376 func TestDeterministicL4(t *testing.T) { testDeterm(4, t) } 377 func TestDeterministicL5(t *testing.T) { testDeterm(5, t) } 378 func TestDeterministicL6(t *testing.T) { testDeterm(6, t) } 379 func TestDeterministicL7(t *testing.T) { testDeterm(7, t) } 380 func TestDeterministicL8(t *testing.T) { testDeterm(8, t) } 381 func TestDeterministicL9(t *testing.T) { testDeterm(9, t) } 382 383 func testDeterm(i int, t *testing.T) { 384 var length = 500000 385 if testing.Short() { 386 length = 100000 387 } 388 rand.Seed(1337) 389 t1 := make([]byte, length) 390 for idx := range t1 { 391 t1[idx] = byte(65 + rand.Intn(8)) 392 } 393 394 br := bytes.NewBuffer(t1) 395 var b1 bytes.Buffer 396 w, err := NewWriterLevel(&b1, i) 397 if err != nil { 398 t.Fatal(err) 399 } 400 _, err = io.Copy(w, br) 401 if err != nil { 402 t.Fatal(err) 403 } 404 w.Flush() 405 w.Close() 406 407 // We recreate the buffer, so we have a goos chance of getting a 408 // different memory address. 409 rand.Seed(1337) 410 t2 := make([]byte, length) 411 for idx := range t2 { 412 t2[idx] = byte(65 + rand.Intn(8)) 413 } 414 415 br2 := bytes.NewBuffer(t2) 416 var b2 bytes.Buffer 417 w2, err := NewWriterLevel(&b2, i) 418 if err != nil { 419 t.Fatal(err) 420 } 421 422 // We write the same data, but with a different size than 423 // the default copy. 424 for { 425 _, err = io.CopyN(w2, br2, 1234) 426 if err == io.EOF { 427 err = nil 428 break 429 } else if err != nil { 430 break 431 } 432 } 433 if err != nil { 434 t.Fatal(err) 435 } 436 w2.Flush() 437 w2.Close() 438 439 b1b := b1.Bytes() 440 b2b := b2.Bytes() 441 442 if bytes.Compare(b1b, b2b) != 0 { 443 t.Fatalf("Level %d did not produce deterministric result, len(a) = %d, len(b) = %d", i, len(b1b), len(b2b)) 444 } 445 } 446 447 func BenchmarkGzipLM2(b *testing.B) { benchmarkGzipN(b, -2) } 448 func BenchmarkGzipL1(b *testing.B) { benchmarkGzipN(b, 1) } 449 func BenchmarkGzipL2(b *testing.B) { benchmarkGzipN(b, 2) } 450 func BenchmarkGzipL3(b *testing.B) { benchmarkGzipN(b, 3) } 451 func BenchmarkGzipL4(b *testing.B) { benchmarkGzipN(b, 4) } 452 func BenchmarkGzipL5(b *testing.B) { benchmarkGzipN(b, 5) } 453 func BenchmarkGzipL6(b *testing.B) { benchmarkGzipN(b, 6) } 454 func BenchmarkGzipL7(b *testing.B) { benchmarkGzipN(b, 7) } 455 func BenchmarkGzipL8(b *testing.B) { benchmarkGzipN(b, 8) } 456 func BenchmarkGzipL9(b *testing.B) { benchmarkGzipN(b, 9) } 457 458 func benchmarkGzipN(b *testing.B, level int) { 459 dat, _ := ioutil.ReadFile("testdata/test.json") 460 dat = append(dat, dat...) 461 dat = append(dat, dat...) 462 dat = append(dat, dat...) 463 dat = append(dat, dat...) 464 dat = append(dat, dat...) 465 b.SetBytes(int64(len(dat))) 466 w, _ := NewWriterLevel(ioutil.Discard, level) 467 b.ResetTimer() 468 for n := 0; n < b.N; n++ { 469 w.Reset(ioutil.Discard) 470 n, err := w.Write(dat) 471 if n != len(dat) { 472 panic("short write") 473 } 474 if err != nil { 475 panic(err) 476 } 477 err = w.Close() 478 if err != nil { 479 panic(err) 480 } 481 } 482 } 483 484 func BenchmarkOldGzipL1(b *testing.B) { benchmarkOldGzipN(b, 1) } 485 func BenchmarkOldGzipL2(b *testing.B) { benchmarkOldGzipN(b, 2) } 486 func BenchmarkOldGzipL3(b *testing.B) { benchmarkOldGzipN(b, 3) } 487 func BenchmarkOldGzipL4(b *testing.B) { benchmarkOldGzipN(b, 4) } 488 func BenchmarkOldGzipL5(b *testing.B) { benchmarkOldGzipN(b, 5) } 489 func BenchmarkOldGzipL6(b *testing.B) { benchmarkOldGzipN(b, 6) } 490 func BenchmarkOldGzipL7(b *testing.B) { benchmarkOldGzipN(b, 7) } 491 func BenchmarkOldGzipL8(b *testing.B) { benchmarkOldGzipN(b, 8) } 492 func BenchmarkOldGzipL9(b *testing.B) { benchmarkOldGzipN(b, 9) } 493 494 func benchmarkOldGzipN(b *testing.B, level int) { 495 dat, _ := ioutil.ReadFile("testdata/test.json") 496 dat = append(dat, dat...) 497 dat = append(dat, dat...) 498 dat = append(dat, dat...) 499 dat = append(dat, dat...) 500 dat = append(dat, dat...) 501 502 b.SetBytes(int64(len(dat))) 503 w, _ := oldgz.NewWriterLevel(ioutil.Discard, level) 504 b.ResetTimer() 505 for n := 0; n < b.N; n++ { 506 w.Reset(ioutil.Discard) 507 n, err := w.Write(dat) 508 if n != len(dat) { 509 panic("short write") 510 } 511 if err != nil { 512 panic(err) 513 } 514 err = w.Close() 515 if err != nil { 516 panic(err) 517 } 518 } 519 }