github.com/letsencrypt/go@v0.0.0-20160714163537-4054769a31f6/src/compress/flate/deflate_test.go (about) 1 // Copyright 2009 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package flate 6 7 import ( 8 "bytes" 9 "fmt" 10 "internal/testenv" 11 "io" 12 "io/ioutil" 13 "reflect" 14 "sync" 15 "testing" 16 ) 17 18 type deflateTest struct { 19 in []byte 20 level int 21 out []byte 22 } 23 24 type deflateInflateTest struct { 25 in []byte 26 } 27 28 type reverseBitsTest struct { 29 in uint16 30 bitCount uint8 31 out uint16 32 } 33 34 var deflateTests = []*deflateTest{ 35 {[]byte{}, 0, []byte{1, 0, 0, 255, 255}}, 36 {[]byte{0x11}, -1, []byte{18, 4, 4, 0, 0, 255, 255}}, 37 {[]byte{0x11}, DefaultCompression, []byte{18, 4, 4, 0, 0, 255, 255}}, 38 {[]byte{0x11}, 4, []byte{18, 4, 4, 0, 0, 255, 255}}, 39 40 {[]byte{0x11}, 0, []byte{0, 1, 0, 254, 255, 17, 1, 0, 0, 255, 255}}, 41 {[]byte{0x11, 0x12}, 0, []byte{0, 2, 0, 253, 255, 17, 18, 1, 0, 0, 255, 255}}, 42 {[]byte{0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11}, 0, 43 []byte{0, 8, 0, 247, 255, 17, 17, 17, 17, 17, 17, 17, 17, 1, 0, 0, 255, 255}, 44 }, 45 {[]byte{}, 2, []byte{1, 0, 0, 255, 255}}, 46 {[]byte{0x11}, 2, []byte{18, 4, 4, 0, 0, 255, 255}}, 47 {[]byte{0x11, 0x12}, 2, []byte{18, 20, 2, 4, 0, 0, 255, 255}}, 48 {[]byte{0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11}, 2, []byte{18, 132, 2, 64, 0, 0, 0, 255, 255}}, 49 {[]byte{}, 9, []byte{1, 0, 0, 255, 255}}, 50 {[]byte{0x11}, 9, []byte{18, 4, 4, 0, 0, 255, 255}}, 51 {[]byte{0x11, 0x12}, 9, []byte{18, 20, 2, 4, 0, 0, 255, 255}}, 52 {[]byte{0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11}, 9, []byte{18, 132, 2, 64, 0, 0, 0, 255, 255}}, 53 } 54 55 var deflateInflateTests = []*deflateInflateTest{ 56 {[]byte{}}, 57 {[]byte{0x11}}, 58 {[]byte{0x11, 0x12}}, 59 {[]byte{0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11}}, 60 {[]byte{0x11, 0x10, 0x13, 0x41, 0x21, 0x21, 0x41, 0x13, 0x87, 0x78, 0x13}}, 61 {largeDataChunk()}, 62 } 63 64 var reverseBitsTests = []*reverseBitsTest{ 65 {1, 1, 1}, 66 {1, 2, 2}, 67 {1, 3, 4}, 68 {1, 4, 8}, 69 {1, 5, 16}, 70 {17, 5, 17}, 71 {257, 9, 257}, 72 {29, 5, 23}, 73 } 74 75 func largeDataChunk() []byte { 76 result := make([]byte, 100000) 77 for i := range result { 78 result[i] = byte(i * i & 0xFF) 79 } 80 return result 81 } 82 83 func TestBulkHash4(t *testing.T) { 84 for _, x := range deflateTests { 85 y := x.out 86 if len(y) < minMatchLength { 87 continue 88 } 89 y = append(y, y...) 90 for j := 4; j < len(y); j++ { 91 y := y[:j] 92 dst := make([]uint32, len(y)-minMatchLength+1) 93 for i := range dst { 94 dst[i] = uint32(i + 100) 95 } 96 bulkHash4(y, dst) 97 for i, got := range dst { 98 want := hash4(y[i:]) 99 if got != want && got == uint32(i)+100 { 100 t.Errorf("Len:%d Index:%d, want 0x%08x but not modified", len(y), i, want) 101 } else if got != want { 102 t.Errorf("Len:%d Index:%d, got 0x%08x want:0x%08x", len(y), i, got, want) 103 } 104 } 105 } 106 } 107 } 108 109 func TestDeflate(t *testing.T) { 110 for _, h := range deflateTests { 111 var buf bytes.Buffer 112 w, err := NewWriter(&buf, h.level) 113 if err != nil { 114 t.Errorf("NewWriter: %v", err) 115 continue 116 } 117 w.Write(h.in) 118 w.Close() 119 if !bytes.Equal(buf.Bytes(), h.out) { 120 t.Errorf("Deflate(%d, %x) = \n%#v, want \n%#v", h.level, h.in, buf.Bytes(), h.out) 121 } 122 } 123 } 124 125 // A sparseReader returns a stream consisting of 0s followed by 1<<16 1s. 126 // This tests missing hash references in a very large input. 127 type sparseReader struct { 128 l int64 129 cur int64 130 } 131 132 func (r *sparseReader) Read(b []byte) (n int, err error) { 133 if r.cur >= r.l { 134 return 0, io.EOF 135 } 136 n = len(b) 137 cur := r.cur + int64(n) 138 if cur > r.l { 139 n -= int(cur - r.l) 140 cur = r.l 141 } 142 for i := range b[0:n] { 143 if r.cur+int64(i) >= r.l-1<<16 { 144 b[i] = 1 145 } else { 146 b[i] = 0 147 } 148 } 149 r.cur = cur 150 return 151 } 152 153 func TestVeryLongSparseChunk(t *testing.T) { 154 if testing.Short() { 155 t.Skip("skipping sparse chunk during short test") 156 } 157 w, err := NewWriter(ioutil.Discard, 1) 158 if err != nil { 159 t.Errorf("NewWriter: %v", err) 160 return 161 } 162 if _, err = io.Copy(w, &sparseReader{l: 23E8}); err != nil { 163 t.Errorf("Compress failed: %v", err) 164 return 165 } 166 } 167 168 type syncBuffer struct { 169 buf bytes.Buffer 170 mu sync.RWMutex 171 closed bool 172 ready chan bool 173 } 174 175 func newSyncBuffer() *syncBuffer { 176 return &syncBuffer{ready: make(chan bool, 1)} 177 } 178 179 func (b *syncBuffer) Read(p []byte) (n int, err error) { 180 for { 181 b.mu.RLock() 182 n, err = b.buf.Read(p) 183 b.mu.RUnlock() 184 if n > 0 || b.closed { 185 return 186 } 187 <-b.ready 188 } 189 } 190 191 func (b *syncBuffer) signal() { 192 select { 193 case b.ready <- true: 194 default: 195 } 196 } 197 198 func (b *syncBuffer) Write(p []byte) (n int, err error) { 199 n, err = b.buf.Write(p) 200 b.signal() 201 return 202 } 203 204 func (b *syncBuffer) WriteMode() { 205 b.mu.Lock() 206 } 207 208 func (b *syncBuffer) ReadMode() { 209 b.mu.Unlock() 210 b.signal() 211 } 212 213 func (b *syncBuffer) Close() error { 214 b.closed = true 215 b.signal() 216 return nil 217 } 218 219 func testSync(t *testing.T, level int, input []byte, name string) { 220 if len(input) == 0 { 221 return 222 } 223 224 t.Logf("--testSync %d, %d, %s", level, len(input), name) 225 buf := newSyncBuffer() 226 buf1 := new(bytes.Buffer) 227 buf.WriteMode() 228 w, err := NewWriter(io.MultiWriter(buf, buf1), level) 229 if err != nil { 230 t.Errorf("NewWriter: %v", err) 231 return 232 } 233 r := NewReader(buf) 234 235 // Write half the input and read back. 236 for i := 0; i < 2; i++ { 237 var lo, hi int 238 if i == 0 { 239 lo, hi = 0, (len(input)+1)/2 240 } else { 241 lo, hi = (len(input)+1)/2, len(input) 242 } 243 t.Logf("#%d: write %d-%d", i, lo, hi) 244 if _, err := w.Write(input[lo:hi]); err != nil { 245 t.Errorf("testSync: write: %v", err) 246 return 247 } 248 if i == 0 { 249 if err := w.Flush(); err != nil { 250 t.Errorf("testSync: flush: %v", err) 251 return 252 } 253 } else { 254 if err := w.Close(); err != nil { 255 t.Errorf("testSync: close: %v", err) 256 } 257 } 258 buf.ReadMode() 259 out := make([]byte, hi-lo+1) 260 m, err := io.ReadAtLeast(r, out, hi-lo) 261 t.Logf("#%d: read %d", i, m) 262 if m != hi-lo || err != nil { 263 t.Errorf("testSync/%d (%d, %d, %s): read %d: %d, %v (%d left)", i, level, len(input), name, hi-lo, m, err, buf.buf.Len()) 264 return 265 } 266 if !bytes.Equal(input[lo:hi], out[:hi-lo]) { 267 t.Errorf("testSync/%d: read wrong bytes: %x vs %x", i, input[lo:hi], out[:hi-lo]) 268 return 269 } 270 // This test originally checked that after reading 271 // the first half of the input, there was nothing left 272 // in the read buffer (buf.buf.Len() != 0) but that is 273 // not necessarily the case: the write Flush may emit 274 // some extra framing bits that are not necessary 275 // to process to obtain the first half of the uncompressed 276 // data. The test ran correctly most of the time, because 277 // the background goroutine had usually read even 278 // those extra bits by now, but it's not a useful thing to 279 // check. 280 buf.WriteMode() 281 } 282 buf.ReadMode() 283 out := make([]byte, 10) 284 if n, err := r.Read(out); n > 0 || err != io.EOF { 285 t.Errorf("testSync (%d, %d, %s): final Read: %d, %v (hex: %x)", level, len(input), name, n, err, out[0:n]) 286 } 287 if buf.buf.Len() != 0 { 288 t.Errorf("testSync (%d, %d, %s): extra data at end", level, len(input), name) 289 } 290 r.Close() 291 292 // stream should work for ordinary reader too 293 r = NewReader(buf1) 294 out, err = ioutil.ReadAll(r) 295 if err != nil { 296 t.Errorf("testSync: read: %s", err) 297 return 298 } 299 r.Close() 300 if !bytes.Equal(input, out) { 301 t.Errorf("testSync: decompress(compress(data)) != data: level=%d input=%s", level, name) 302 } 303 } 304 305 func testToFromWithLevelAndLimit(t *testing.T, level int, input []byte, name string, limit int) { 306 var buffer bytes.Buffer 307 w, err := NewWriter(&buffer, level) 308 if err != nil { 309 t.Errorf("NewWriter: %v", err) 310 return 311 } 312 w.Write(input) 313 w.Close() 314 if limit > 0 && buffer.Len() > limit { 315 t.Errorf("level: %d, len(compress(data)) = %d > limit = %d", level, buffer.Len(), limit) 316 return 317 } 318 if limit > 0 { 319 t.Logf("level: %d, size:%.2f%%, %d b\n", level, float64(buffer.Len()*100)/float64(limit), buffer.Len()) 320 } 321 r := NewReader(&buffer) 322 out, err := ioutil.ReadAll(r) 323 if err != nil { 324 t.Errorf("read: %s", err) 325 return 326 } 327 r.Close() 328 if !bytes.Equal(input, out) { 329 t.Errorf("decompress(compress(data)) != data: level=%d input=%s", level, name) 330 return 331 } 332 testSync(t, level, input, name) 333 } 334 335 func testToFromWithLimit(t *testing.T, input []byte, name string, limit [11]int) { 336 for i := 0; i < 10; i++ { 337 testToFromWithLevelAndLimit(t, i, input, name, limit[i]) 338 } 339 // Test HuffmanCompression 340 testToFromWithLevelAndLimit(t, -2, input, name, limit[10]) 341 } 342 343 func TestDeflateInflate(t *testing.T) { 344 for i, h := range deflateInflateTests { 345 testToFromWithLimit(t, h.in, fmt.Sprintf("#%d", i), [11]int{}) 346 } 347 } 348 349 func TestReverseBits(t *testing.T) { 350 for _, h := range reverseBitsTests { 351 if v := reverseBits(h.in, h.bitCount); v != h.out { 352 t.Errorf("reverseBits(%v,%v) = %v, want %v", 353 h.in, h.bitCount, v, h.out) 354 } 355 } 356 } 357 358 type deflateInflateStringTest struct { 359 filename string 360 label string 361 limit [11]int 362 } 363 364 var deflateInflateStringTests = []deflateInflateStringTest{ 365 { 366 "../testdata/e.txt", 367 "2.718281828...", 368 [...]int{100018, 50650, 50960, 51150, 50930, 50790, 50790, 50790, 50790, 50790, 43683}, 369 }, 370 { 371 "../testdata/Mark.Twain-Tom.Sawyer.txt", 372 "Mark.Twain-Tom.Sawyer", 373 [...]int{407330, 187598, 180361, 172974, 169160, 163476, 160936, 160506, 160295, 160295, 233460}, 374 }, 375 } 376 377 func TestDeflateInflateString(t *testing.T) { 378 if testing.Short() && testenv.Builder() == "" { 379 t.Skip("skipping in short mode") 380 } 381 for _, test := range deflateInflateStringTests { 382 gold, err := ioutil.ReadFile(test.filename) 383 if err != nil { 384 t.Error(err) 385 } 386 testToFromWithLimit(t, gold, test.label, test.limit) 387 if testing.Short() { 388 break 389 } 390 } 391 } 392 393 func TestReaderDict(t *testing.T) { 394 const ( 395 dict = "hello world" 396 text = "hello again world" 397 ) 398 var b bytes.Buffer 399 w, err := NewWriter(&b, 5) 400 if err != nil { 401 t.Fatalf("NewWriter: %v", err) 402 } 403 w.Write([]byte(dict)) 404 w.Flush() 405 b.Reset() 406 w.Write([]byte(text)) 407 w.Close() 408 409 r := NewReaderDict(&b, []byte(dict)) 410 data, err := ioutil.ReadAll(r) 411 if err != nil { 412 t.Fatal(err) 413 } 414 if string(data) != "hello again world" { 415 t.Fatalf("read returned %q want %q", string(data), text) 416 } 417 } 418 419 func TestWriterDict(t *testing.T) { 420 const ( 421 dict = "hello world" 422 text = "hello again world" 423 ) 424 var b bytes.Buffer 425 w, err := NewWriter(&b, 5) 426 if err != nil { 427 t.Fatalf("NewWriter: %v", err) 428 } 429 w.Write([]byte(dict)) 430 w.Flush() 431 b.Reset() 432 w.Write([]byte(text)) 433 w.Close() 434 435 var b1 bytes.Buffer 436 w, _ = NewWriterDict(&b1, 5, []byte(dict)) 437 w.Write([]byte(text)) 438 w.Close() 439 440 if !bytes.Equal(b1.Bytes(), b.Bytes()) { 441 t.Fatalf("writer wrote %q want %q", b1.Bytes(), b.Bytes()) 442 } 443 } 444 445 // See https://golang.org/issue/2508 446 func TestRegression2508(t *testing.T) { 447 if testing.Short() { 448 t.Logf("test disabled with -short") 449 return 450 } 451 w, err := NewWriter(ioutil.Discard, 1) 452 if err != nil { 453 t.Fatalf("NewWriter: %v", err) 454 } 455 buf := make([]byte, 1024) 456 for i := 0; i < 131072; i++ { 457 if _, err := w.Write(buf); err != nil { 458 t.Fatalf("writer failed: %v", err) 459 } 460 } 461 w.Close() 462 } 463 464 func TestWriterReset(t *testing.T) { 465 for level := 0; level <= 9; level++ { 466 if testing.Short() && level > 1 { 467 break 468 } 469 w, err := NewWriter(ioutil.Discard, level) 470 if err != nil { 471 t.Fatalf("NewWriter: %v", err) 472 } 473 buf := []byte("hello world") 474 n := 1024 475 if testing.Short() { 476 n = 10 477 } 478 for i := 0; i < n; i++ { 479 w.Write(buf) 480 } 481 w.Reset(ioutil.Discard) 482 483 wref, err := NewWriter(ioutil.Discard, level) 484 if err != nil { 485 t.Fatalf("NewWriter: %v", err) 486 } 487 488 // DeepEqual doesn't compare functions. 489 w.d.fill, wref.d.fill = nil, nil 490 w.d.step, wref.d.step = nil, nil 491 w.d.bulkHasher, wref.d.bulkHasher = nil, nil 492 // hashMatch is always overwritten when used. 493 copy(w.d.hashMatch[:], wref.d.hashMatch[:]) 494 if len(w.d.tokens) != 0 { 495 t.Errorf("level %d Writer not reset after Reset. %d tokens were present", level, len(w.d.tokens)) 496 } 497 // As long as the length is 0, we don't care about the content. 498 w.d.tokens = wref.d.tokens 499 500 // We don't care if there are values in the window, as long as it is at d.index is 0 501 w.d.window = wref.d.window 502 if !reflect.DeepEqual(w, wref) { 503 t.Errorf("level %d Writer not reset after Reset", level) 504 } 505 } 506 testResetOutput(t, func(w io.Writer) (*Writer, error) { return NewWriter(w, NoCompression) }) 507 testResetOutput(t, func(w io.Writer) (*Writer, error) { return NewWriter(w, DefaultCompression) }) 508 testResetOutput(t, func(w io.Writer) (*Writer, error) { return NewWriter(w, BestCompression) }) 509 dict := []byte("we are the world") 510 testResetOutput(t, func(w io.Writer) (*Writer, error) { return NewWriterDict(w, NoCompression, dict) }) 511 testResetOutput(t, func(w io.Writer) (*Writer, error) { return NewWriterDict(w, DefaultCompression, dict) }) 512 testResetOutput(t, func(w io.Writer) (*Writer, error) { return NewWriterDict(w, BestCompression, dict) }) 513 } 514 515 func testResetOutput(t *testing.T, newWriter func(w io.Writer) (*Writer, error)) { 516 buf := new(bytes.Buffer) 517 w, err := newWriter(buf) 518 if err != nil { 519 t.Fatalf("NewWriter: %v", err) 520 } 521 b := []byte("hello world") 522 for i := 0; i < 1024; i++ { 523 w.Write(b) 524 } 525 w.Close() 526 out1 := buf.Bytes() 527 528 buf2 := new(bytes.Buffer) 529 w.Reset(buf2) 530 for i := 0; i < 1024; i++ { 531 w.Write(b) 532 } 533 w.Close() 534 out2 := buf2.Bytes() 535 536 if len(out1) != len(out2) { 537 t.Errorf("got %d, expected %d bytes", len(out2), len(out1)) 538 return 539 } 540 if !bytes.Equal(out1, out2) { 541 mm := 0 542 for i, b := range out1[:len(out2)] { 543 if b != out2[i] { 544 t.Errorf("mismatch index %d: %#02x, expected %#02x", i, out2[i], b) 545 } 546 mm++ 547 if mm == 10 { 548 t.Fatal("Stopping") 549 } 550 } 551 } 552 t.Logf("got %d bytes", len(out1)) 553 } 554 555 // TestBestSpeed tests that round-tripping through deflate and then inflate 556 // recovers the original input. The Write sizes are near the thresholds in the 557 // compressor.encSpeed method (0, 16, 128), as well as near maxStoreBlockSize 558 // (65535). 559 func TestBestSpeed(t *testing.T) { 560 abc := make([]byte, 128) 561 for i := range abc { 562 abc[i] = byte(i) 563 } 564 abcabc := bytes.Repeat(abc, 131072/len(abc)) 565 var want []byte 566 567 testCases := [][]int{ 568 {65536, 0}, 569 {65536, 1}, 570 {65536, 1, 256}, 571 {65536, 1, 65536}, 572 {65536, 14}, 573 {65536, 15}, 574 {65536, 16}, 575 {65536, 16, 256}, 576 {65536, 16, 65536}, 577 {65536, 127}, 578 {65536, 128}, 579 {65536, 128, 256}, 580 {65536, 128, 65536}, 581 {65536, 129}, 582 {65536, 65536, 256}, 583 {65536, 65536, 65536}, 584 } 585 586 for i, tc := range testCases { 587 for _, firstN := range []int{1, 65534, 65535, 65536, 65537, 131072} { 588 tc[0] = firstN 589 outer: 590 for _, flush := range []bool{false, true} { 591 buf := new(bytes.Buffer) 592 want = want[:0] 593 594 w, err := NewWriter(buf, BestSpeed) 595 if err != nil { 596 t.Errorf("i=%d, firstN=%d, flush=%t: NewWriter: %v", i, firstN, flush, err) 597 continue 598 } 599 for _, n := range tc { 600 want = append(want, abcabc[:n]...) 601 if _, err := w.Write(abcabc[:n]); err != nil { 602 t.Errorf("i=%d, firstN=%d, flush=%t: Write: %v", i, firstN, flush, err) 603 continue outer 604 } 605 if !flush { 606 continue 607 } 608 if err := w.Flush(); err != nil { 609 t.Errorf("i=%d, firstN=%d, flush=%t: Flush: %v", i, firstN, flush, err) 610 continue outer 611 } 612 } 613 if err := w.Close(); err != nil { 614 t.Errorf("i=%d, firstN=%d, flush=%t: Close: %v", i, firstN, flush, err) 615 continue 616 } 617 618 r := NewReader(buf) 619 got, err := ioutil.ReadAll(r) 620 if err != nil { 621 t.Errorf("i=%d, firstN=%d, flush=%t: ReadAll: %v", i, firstN, flush, err) 622 continue 623 } 624 r.Close() 625 626 if !bytes.Equal(got, want) { 627 t.Errorf("i=%d, firstN=%d, flush=%t: corruption during deflate-then-inflate", i, firstN, flush) 628 continue 629 } 630 } 631 } 632 } 633 }