github.com/insionng/yougam@v0.0.0-20170714101924-2bc18d833463/libraries/klauspost/compress/gzip/gzip_test.go (about)

     1  // Copyright 2010 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package gzip
     6  
     7  import (
     8  	"bufio"
     9  	"bytes"
    10  	oldgz "compress/gzip"
    11  	"io"
    12  	"io/ioutil"
    13  	"math/rand"
    14  	"testing"
    15  	"time"
    16  )
    17  
    18  // TestEmpty tests that an empty payload still forms a valid GZIP stream.
    19  func TestEmpty(t *testing.T) {
    20  	buf := new(bytes.Buffer)
    21  
    22  	if err := NewWriter(buf).Close(); err != nil {
    23  		t.Fatalf("Writer.Close: %v", err)
    24  	}
    25  
    26  	r, err := NewReader(buf)
    27  	if err != nil {
    28  		t.Fatalf("NewReader: %v", err)
    29  	}
    30  	b, err := ioutil.ReadAll(r)
    31  	if err != nil {
    32  		t.Fatalf("ReadAll: %v", err)
    33  	}
    34  	if len(b) != 0 {
    35  		t.Fatalf("got %d bytes, want 0", len(b))
    36  	}
    37  	if err := r.Close(); err != nil {
    38  		t.Fatalf("Reader.Close: %v", err)
    39  	}
    40  }
    41  
    42  // TestRoundTrip tests that gzipping and then gunzipping is the identity
    43  // function.
    44  func TestRoundTrip(t *testing.T) {
    45  	buf := new(bytes.Buffer)
    46  
    47  	w := NewWriter(buf)
    48  	w.Comment = "comment"
    49  	w.Extra = []byte("extra")
    50  	w.ModTime = time.Unix(1e8, 0)
    51  	w.Name = "name"
    52  	if _, err := w.Write([]byte("payload")); err != nil {
    53  		t.Fatalf("Write: %v", err)
    54  	}
    55  	if err := w.Close(); err != nil {
    56  		t.Fatalf("Writer.Close: %v", err)
    57  	}
    58  
    59  	r, err := NewReader(buf)
    60  	if err != nil {
    61  		t.Fatalf("NewReader: %v", err)
    62  	}
    63  	b, err := ioutil.ReadAll(r)
    64  	if err != nil {
    65  		t.Fatalf("ReadAll: %v", err)
    66  	}
    67  	if string(b) != "payload" {
    68  		t.Fatalf("payload is %q, want %q", string(b), "payload")
    69  	}
    70  	if r.Comment != "comment" {
    71  		t.Fatalf("comment is %q, want %q", r.Comment, "comment")
    72  	}
    73  	if string(r.Extra) != "extra" {
    74  		t.Fatalf("extra is %q, want %q", r.Extra, "extra")
    75  	}
    76  	if r.ModTime.Unix() != 1e8 {
    77  		t.Fatalf("mtime is %d, want %d", r.ModTime.Unix(), uint32(1e8))
    78  	}
    79  	if r.Name != "name" {
    80  		t.Fatalf("name is %q, want %q", r.Name, "name")
    81  	}
    82  	if err := r.Close(); err != nil {
    83  		t.Fatalf("Reader.Close: %v", err)
    84  	}
    85  }
    86  
    87  // TestLatin1 tests the internal functions for converting to and from Latin-1.
    88  func TestLatin1(t *testing.T) {
    89  	latin1 := []byte{0xc4, 'u', 0xdf, 'e', 'r', 'u', 'n', 'g', 0}
    90  	utf8 := "Äußerung"
    91  	z := Reader{r: bufio.NewReader(bytes.NewReader(latin1))}
    92  	s, err := z.readString()
    93  	if err != nil {
    94  		t.Fatalf("readString: %v", err)
    95  	}
    96  	if s != utf8 {
    97  		t.Fatalf("read latin-1: got %q, want %q", s, utf8)
    98  	}
    99  
   100  	buf := bytes.NewBuffer(make([]byte, 0, len(latin1)))
   101  	c := Writer{w: buf}
   102  	if err = c.writeString(utf8); err != nil {
   103  		t.Fatalf("writeString: %v", err)
   104  	}
   105  	s = buf.String()
   106  	if s != string(latin1) {
   107  		t.Fatalf("write utf-8: got %q, want %q", s, string(latin1))
   108  	}
   109  }
   110  
   111  // TestLatin1RoundTrip tests that metadata that is representable in Latin-1
   112  // survives a round trip.
   113  func TestLatin1RoundTrip(t *testing.T) {
   114  	testCases := []struct {
   115  		name string
   116  		ok   bool
   117  	}{
   118  		{"", true},
   119  		{"ASCII is OK", true},
   120  		{"unless it contains a NUL\x00", false},
   121  		{"no matter where \x00 occurs", false},
   122  		{"\x00\x00\x00", false},
   123  		{"Látin-1 also passes (U+00E1)", true},
   124  		{"but LĀtin Extended-A (U+0100) does not", false},
   125  		{"neither does 日本語", false},
   126  		{"invalid UTF-8 also \xffails", false},
   127  		{"\x00 as does Látin-1 with NUL", false},
   128  	}
   129  	for _, tc := range testCases {
   130  		buf := new(bytes.Buffer)
   131  
   132  		w := NewWriter(buf)
   133  		w.Name = tc.name
   134  		err := w.Close()
   135  		if (err == nil) != tc.ok {
   136  			t.Errorf("Writer.Close: name = %q, err = %v", tc.name, err)
   137  			continue
   138  		}
   139  		if !tc.ok {
   140  			continue
   141  		}
   142  
   143  		r, err := NewReader(buf)
   144  		if err != nil {
   145  			t.Errorf("NewReader: %v", err)
   146  			continue
   147  		}
   148  		_, err = ioutil.ReadAll(r)
   149  		if err != nil {
   150  			t.Errorf("ReadAll: %v", err)
   151  			continue
   152  		}
   153  		if r.Name != tc.name {
   154  			t.Errorf("name is %q, want %q", r.Name, tc.name)
   155  			continue
   156  		}
   157  		if err := r.Close(); err != nil {
   158  			t.Errorf("Reader.Close: %v", err)
   159  			continue
   160  		}
   161  	}
   162  }
   163  
   164  func TestWriterFlush(t *testing.T) {
   165  	buf := new(bytes.Buffer)
   166  
   167  	w := NewWriter(buf)
   168  	w.Comment = "comment"
   169  	w.Extra = []byte("extra")
   170  	w.ModTime = time.Unix(1e8, 0)
   171  	w.Name = "name"
   172  
   173  	n0 := buf.Len()
   174  	if n0 != 0 {
   175  		t.Fatalf("buffer size = %d before writes; want 0", n0)
   176  	}
   177  
   178  	if err := w.Flush(); err != nil {
   179  		t.Fatal(err)
   180  	}
   181  
   182  	n1 := buf.Len()
   183  	if n1 == 0 {
   184  		t.Fatal("no data after first flush")
   185  	}
   186  
   187  	w.Write([]byte("x"))
   188  
   189  	n2 := buf.Len()
   190  	if n1 != n2 {
   191  		t.Fatalf("after writing a single byte, size changed from %d to %d; want no change", n1, n2)
   192  	}
   193  
   194  	if err := w.Flush(); err != nil {
   195  		t.Fatal(err)
   196  	}
   197  
   198  	n3 := buf.Len()
   199  	if n2 == n3 {
   200  		t.Fatal("Flush didn't flush any data")
   201  	}
   202  }
   203  
   204  // Multiple gzip files concatenated form a valid gzip file.
   205  func TestConcat(t *testing.T) {
   206  	var buf bytes.Buffer
   207  	w := NewWriter(&buf)
   208  	w.Write([]byte("hello "))
   209  	w.Close()
   210  	w = NewWriter(&buf)
   211  	w.Write([]byte("world\n"))
   212  	w.Close()
   213  
   214  	r, err := NewReader(&buf)
   215  	data, err := ioutil.ReadAll(r)
   216  	if string(data) != "hello world\n" || err != nil {
   217  		t.Fatalf("ReadAll = %q, %v, want %q, nil", data, err, "hello world")
   218  	}
   219  }
   220  
   221  func TestWriterReset(t *testing.T) {
   222  	buf := new(bytes.Buffer)
   223  	buf2 := new(bytes.Buffer)
   224  	z := NewWriter(buf)
   225  	msg := []byte("hello world")
   226  	z.Write(msg)
   227  	z.Close()
   228  	z.Reset(buf2)
   229  	z.Write(msg)
   230  	z.Close()
   231  	if buf.String() != buf2.String() {
   232  		t.Errorf("buf2 %q != original buf of %q", buf2.String(), buf.String())
   233  	}
   234  }
   235  
   236  var testbuf []byte
   237  
   238  func testFile(i, level int, t *testing.T) {
   239  	dat, _ := ioutil.ReadFile("testdata/test.json")
   240  	dl := len(dat)
   241  	if len(testbuf) != i*dl {
   242  		// Make results predictable
   243  		testbuf = make([]byte, i*dl)
   244  		for j := 0; j < i; j++ {
   245  			copy(testbuf[j*dl:j*dl+dl], dat)
   246  		}
   247  	}
   248  
   249  	br := bytes.NewBuffer(testbuf)
   250  	var buf bytes.Buffer
   251  	w, err := NewWriterLevel(&buf, DefaultCompression)
   252  	if err != nil {
   253  		t.Fatal(err)
   254  	}
   255  	n, err := io.Copy(w, br)
   256  	if err != nil {
   257  		t.Fatal(err)
   258  	}
   259  	if int(n) != len(testbuf) {
   260  		t.Fatal("Short write:", n, "!=", testbuf)
   261  	}
   262  	err = w.Close()
   263  	if err != nil {
   264  		t.Fatal(err)
   265  	}
   266  	r, err := NewReader(&buf)
   267  	if err != nil {
   268  		t.Fatal(err.Error())
   269  	}
   270  	decoded, err := ioutil.ReadAll(r)
   271  	if err != nil {
   272  		t.Fatal(err.Error())
   273  	}
   274  	if !bytes.Equal(testbuf, decoded) {
   275  		t.Errorf("decoded content does not match.")
   276  	}
   277  }
   278  
   279  func TestFile1xM2(t *testing.T) { testFile(1, -2, t) }
   280  func TestFile1xM1(t *testing.T) { testFile(1, -1, t) }
   281  func TestFile1x0(t *testing.T)  { testFile(1, 0, t) }
   282  func TestFile1x1(t *testing.T)  { testFile(1, 1, t) }
   283  func TestFile1x2(t *testing.T)  { testFile(1, 2, t) }
   284  func TestFile1x3(t *testing.T)  { testFile(1, 3, t) }
   285  func TestFile1x4(t *testing.T)  { testFile(1, 4, t) }
   286  func TestFile1x5(t *testing.T)  { testFile(1, 5, t) }
   287  func TestFile1x6(t *testing.T)  { testFile(1, 6, t) }
   288  func TestFile1x7(t *testing.T)  { testFile(1, 7, t) }
   289  func TestFile1x8(t *testing.T)  { testFile(1, 8, t) }
   290  func TestFile1x9(t *testing.T)  { testFile(1, 9, t) }
   291  func TestFile10(t *testing.T)   { testFile(10, DefaultCompression, t) }
   292  
   293  func TestFile50(t *testing.T) {
   294  	if testing.Short() {
   295  		t.Skip("skipping during short test")
   296  	}
   297  	testFile(50, DefaultCompression, t)
   298  }
   299  
   300  func TestFile200(t *testing.T) {
   301  	if testing.Short() {
   302  		t.Skip("skipping during short test")
   303  	}
   304  	testFile(200, BestSpeed, t)
   305  }
   306  
   307  func testBigGzip(i int, t *testing.T) {
   308  	if len(testbuf) != i {
   309  		// Make results predictable
   310  		rand.Seed(1337)
   311  		testbuf = make([]byte, i)
   312  		for idx := range testbuf {
   313  			testbuf[idx] = byte(65 + rand.Intn(20))
   314  		}
   315  	}
   316  	c := BestCompression
   317  	if testing.Short() {
   318  		c = BestSpeed
   319  	}
   320  
   321  	br := bytes.NewBuffer(testbuf)
   322  	var buf bytes.Buffer
   323  	w, err := NewWriterLevel(&buf, c)
   324  	if err != nil {
   325  		t.Fatal(err)
   326  	}
   327  	n, err := io.Copy(w, br)
   328  	if err != nil {
   329  		t.Fatal(err)
   330  	}
   331  	if int(n) != len(testbuf) {
   332  		t.Fatal("Short write:", n, "!=", len(testbuf))
   333  	}
   334  	err = w.Close()
   335  	if err != nil {
   336  		t.Fatal(err.Error())
   337  	}
   338  
   339  	r, err := NewReader(&buf)
   340  	if err != nil {
   341  		t.Fatal(err.Error())
   342  	}
   343  	decoded, err := ioutil.ReadAll(r)
   344  	if err != nil {
   345  		t.Fatal(err.Error())
   346  	}
   347  	if !bytes.Equal(testbuf, decoded) {
   348  		t.Errorf("decoded content does not match.")
   349  	}
   350  }
   351  
   352  func TestGzip1K(t *testing.T)   { testBigGzip(1000, t) }
   353  func TestGzip100K(t *testing.T) { testBigGzip(100000, t) }
   354  func TestGzip1M(t *testing.T) {
   355  	if testing.Short() {
   356  		t.Skip("skipping during short test")
   357  	}
   358  
   359  	testBigGzip(1000000, t)
   360  }
   361  func TestGzip10M(t *testing.T) {
   362  	if testing.Short() {
   363  		t.Skip("skipping during short test")
   364  	}
   365  	testBigGzip(10000000, t)
   366  }
   367  
   368  // Test if two runs produce identical results.
   369  func TestDeterministicLM2(t *testing.T) { testDeterm(-2, t) }
   370  
   371  // Level 0 is not deterministic since it depends on the size of each write.
   372  // func TestDeterministicL0(t *testing.T)  { testDeterm(0, t) }
   373  func TestDeterministicL1(t *testing.T) { testDeterm(1, t) }
   374  func TestDeterministicL2(t *testing.T) { testDeterm(2, t) }
   375  func TestDeterministicL3(t *testing.T) { testDeterm(3, t) }
   376  func TestDeterministicL4(t *testing.T) { testDeterm(4, t) }
   377  func TestDeterministicL5(t *testing.T) { testDeterm(5, t) }
   378  func TestDeterministicL6(t *testing.T) { testDeterm(6, t) }
   379  func TestDeterministicL7(t *testing.T) { testDeterm(7, t) }
   380  func TestDeterministicL8(t *testing.T) { testDeterm(8, t) }
   381  func TestDeterministicL9(t *testing.T) { testDeterm(9, t) }
   382  
   383  func testDeterm(i int, t *testing.T) {
   384  	var length = 500000
   385  	if testing.Short() {
   386  		length = 100000
   387  	}
   388  	rand.Seed(1337)
   389  	t1 := make([]byte, length)
   390  	for idx := range t1 {
   391  		t1[idx] = byte(65 + rand.Intn(8))
   392  	}
   393  
   394  	br := bytes.NewBuffer(t1)
   395  	var b1 bytes.Buffer
   396  	w, err := NewWriterLevel(&b1, i)
   397  	if err != nil {
   398  		t.Fatal(err)
   399  	}
   400  	_, err = io.Copy(w, br)
   401  	if err != nil {
   402  		t.Fatal(err)
   403  	}
   404  	w.Flush()
   405  	w.Close()
   406  
   407  	// We recreate the buffer, so we have a goos chance of getting a
   408  	// different memory address.
   409  	rand.Seed(1337)
   410  	t2 := make([]byte, length)
   411  	for idx := range t2 {
   412  		t2[idx] = byte(65 + rand.Intn(8))
   413  	}
   414  
   415  	br2 := bytes.NewBuffer(t2)
   416  	var b2 bytes.Buffer
   417  	w2, err := NewWriterLevel(&b2, i)
   418  	if err != nil {
   419  		t.Fatal(err)
   420  	}
   421  
   422  	// We write the same data, but with a different size than
   423  	// the default copy.
   424  	for {
   425  		_, err = io.CopyN(w2, br2, 1234)
   426  		if err == io.EOF {
   427  			err = nil
   428  			break
   429  		} else if err != nil {
   430  			break
   431  		}
   432  	}
   433  	if err != nil {
   434  		t.Fatal(err)
   435  	}
   436  	w2.Flush()
   437  	w2.Close()
   438  
   439  	b1b := b1.Bytes()
   440  	b2b := b2.Bytes()
   441  
   442  	if bytes.Compare(b1b, b2b) != 0 {
   443  		t.Fatalf("Level %d did not produce deterministric result, len(a) = %d, len(b) = %d", i, len(b1b), len(b2b))
   444  	}
   445  }
   446  
   447  func BenchmarkGzipLM2(b *testing.B) { benchmarkGzipN(b, -2) }
   448  func BenchmarkGzipL1(b *testing.B)  { benchmarkGzipN(b, 1) }
   449  func BenchmarkGzipL2(b *testing.B)  { benchmarkGzipN(b, 2) }
   450  func BenchmarkGzipL3(b *testing.B)  { benchmarkGzipN(b, 3) }
   451  func BenchmarkGzipL4(b *testing.B)  { benchmarkGzipN(b, 4) }
   452  func BenchmarkGzipL5(b *testing.B)  { benchmarkGzipN(b, 5) }
   453  func BenchmarkGzipL6(b *testing.B)  { benchmarkGzipN(b, 6) }
   454  func BenchmarkGzipL7(b *testing.B)  { benchmarkGzipN(b, 7) }
   455  func BenchmarkGzipL8(b *testing.B)  { benchmarkGzipN(b, 8) }
   456  func BenchmarkGzipL9(b *testing.B)  { benchmarkGzipN(b, 9) }
   457  
   458  func benchmarkGzipN(b *testing.B, level int) {
   459  	dat, _ := ioutil.ReadFile("testdata/test.json")
   460  	dat = append(dat, dat...)
   461  	dat = append(dat, dat...)
   462  	dat = append(dat, dat...)
   463  	dat = append(dat, dat...)
   464  	dat = append(dat, dat...)
   465  	b.SetBytes(int64(len(dat)))
   466  	w, _ := NewWriterLevel(ioutil.Discard, level)
   467  	b.ResetTimer()
   468  	for n := 0; n < b.N; n++ {
   469  		w.Reset(ioutil.Discard)
   470  		n, err := w.Write(dat)
   471  		if n != len(dat) {
   472  			panic("short write")
   473  		}
   474  		if err != nil {
   475  			panic(err)
   476  		}
   477  		err = w.Close()
   478  		if err != nil {
   479  			panic(err)
   480  		}
   481  	}
   482  }
   483  
   484  func BenchmarkOldGzipL1(b *testing.B) { benchmarkOldGzipN(b, 1) }
   485  func BenchmarkOldGzipL2(b *testing.B) { benchmarkOldGzipN(b, 2) }
   486  func BenchmarkOldGzipL3(b *testing.B) { benchmarkOldGzipN(b, 3) }
   487  func BenchmarkOldGzipL4(b *testing.B) { benchmarkOldGzipN(b, 4) }
   488  func BenchmarkOldGzipL5(b *testing.B) { benchmarkOldGzipN(b, 5) }
   489  func BenchmarkOldGzipL6(b *testing.B) { benchmarkOldGzipN(b, 6) }
   490  func BenchmarkOldGzipL7(b *testing.B) { benchmarkOldGzipN(b, 7) }
   491  func BenchmarkOldGzipL8(b *testing.B) { benchmarkOldGzipN(b, 8) }
   492  func BenchmarkOldGzipL9(b *testing.B) { benchmarkOldGzipN(b, 9) }
   493  
   494  func benchmarkOldGzipN(b *testing.B, level int) {
   495  	dat, _ := ioutil.ReadFile("testdata/test.json")
   496  	dat = append(dat, dat...)
   497  	dat = append(dat, dat...)
   498  	dat = append(dat, dat...)
   499  	dat = append(dat, dat...)
   500  	dat = append(dat, dat...)
   501  
   502  	b.SetBytes(int64(len(dat)))
   503  	w, _ := oldgz.NewWriterLevel(ioutil.Discard, level)
   504  	b.ResetTimer()
   505  	for n := 0; n < b.N; n++ {
   506  		w.Reset(ioutil.Discard)
   507  		n, err := w.Write(dat)
   508  		if n != len(dat) {
   509  			panic("short write")
   510  		}
   511  		if err != nil {
   512  			panic(err)
   513  		}
   514  		err = w.Close()
   515  		if err != nil {
   516  			panic(err)
   517  		}
   518  	}
   519  }