github.com/andybalholm/brotli@v1.0.6/brotli_test.go (about)

     1  // Copyright 2016 Google Inc. All Rights Reserved.
     2  //
     3  // Distributed under MIT license.
     4  // See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
     5  
     6  package brotli
     7  
     8  import (
     9  	"bytes"
    10  	"compress/gzip"
    11  	"fmt"
    12  	"io"
    13  	"io/ioutil"
    14  	"math"
    15  	"math/rand"
    16  	"os"
    17  	"testing"
    18  	"time"
    19  )
    20  
    21  func checkCompressedData(compressedData, wantOriginalData []byte) error {
    22  	uncompressed, err := Decode(compressedData)
    23  	if err != nil {
    24  		return fmt.Errorf("brotli decompress failed: %v", err)
    25  	}
    26  	if !bytes.Equal(uncompressed, wantOriginalData) {
    27  		if len(wantOriginalData) != len(uncompressed) {
    28  			return fmt.Errorf(""+
    29  				"Data doesn't uncompress to the original value.\n"+
    30  				"Length of original: %v\n"+
    31  				"Length of uncompressed: %v",
    32  				len(wantOriginalData), len(uncompressed))
    33  		}
    34  		for i := range wantOriginalData {
    35  			if wantOriginalData[i] != uncompressed[i] {
    36  				return fmt.Errorf(""+
    37  					"Data doesn't uncompress to the original value.\n"+
    38  					"Original at %v is %v\n"+
    39  					"Uncompressed at %v is %v",
    40  					i, wantOriginalData[i], i, uncompressed[i])
    41  			}
    42  		}
    43  	}
    44  	return nil
    45  }
    46  
    47  func TestEncoderNoWrite(t *testing.T) {
    48  	out := bytes.Buffer{}
    49  	e := NewWriterOptions(&out, WriterOptions{Quality: 5})
    50  	if err := e.Close(); err != nil {
    51  		t.Errorf("Close()=%v, want nil", err)
    52  	}
    53  	// Check Write after close.
    54  	if _, err := e.Write([]byte("hi")); err == nil {
    55  		t.Errorf("No error after Close() + Write()")
    56  	}
    57  }
    58  
    59  func TestEncoderEmptyWrite(t *testing.T) {
    60  	out := bytes.Buffer{}
    61  	e := NewWriterOptions(&out, WriterOptions{Quality: 5})
    62  	n, err := e.Write([]byte(""))
    63  	if n != 0 || err != nil {
    64  		t.Errorf("Write()=%v,%v, want 0, nil", n, err)
    65  	}
    66  	if err := e.Close(); err != nil {
    67  		t.Errorf("Close()=%v, want nil", err)
    68  	}
    69  }
    70  
    71  func TestWriter(t *testing.T) {
    72  	for level := BestSpeed; level <= BestCompression; level++ {
    73  		// Test basic encoder usage.
    74  		input := []byte("<html><body><H1>Hello world</H1></body></html>")
    75  		out := bytes.Buffer{}
    76  		e := NewWriterOptions(&out, WriterOptions{Quality: level})
    77  		in := bytes.NewReader([]byte(input))
    78  		n, err := io.Copy(e, in)
    79  		if err != nil {
    80  			t.Errorf("Copy Error: %v", err)
    81  		}
    82  		if int(n) != len(input) {
    83  			t.Errorf("Copy() n=%v, want %v", n, len(input))
    84  		}
    85  		if err := e.Close(); err != nil {
    86  			t.Errorf("Close Error after copied %d bytes: %v", n, err)
    87  		}
    88  		if err := checkCompressedData(out.Bytes(), input); err != nil {
    89  			t.Error(err)
    90  		}
    91  
    92  		out2 := bytes.Buffer{}
    93  		e.Reset(&out2)
    94  		n2, err := e.Write(input)
    95  		if err != nil {
    96  			t.Errorf("Write error after Reset: %v", err)
    97  		}
    98  		if n2 != len(input) {
    99  			t.Errorf("Write() after Reset n=%d, want %d", n2, len(input))
   100  		}
   101  		if err := e.Close(); err != nil {
   102  			t.Errorf("Close error after Reset (copied %d) bytes: %v", n2, err)
   103  		}
   104  		if !bytes.Equal(out.Bytes(), out2.Bytes()) {
   105  			t.Error("Compressed data after Reset doesn't equal first time")
   106  		}
   107  	}
   108  }
   109  
   110  func TestIssue22(t *testing.T) {
   111  	f, err := os.Open("testdata/issue22.gz")
   112  	if err != nil {
   113  		t.Fatalf("Error opening test data file: %v", err)
   114  	}
   115  	defer f.Close()
   116  
   117  	zr, err := gzip.NewReader(f)
   118  	if err != nil {
   119  		t.Fatalf("Error creating gzip reader: %v", err)
   120  	}
   121  
   122  	data, err := io.ReadAll(zr)
   123  	if err != nil {
   124  		t.Fatalf("Error reading test data: %v", err)
   125  	}
   126  
   127  	if len(data) != 2851073 {
   128  		t.Fatalf("Wrong length for test data: got %d, want 2851073", len(data))
   129  	}
   130  
   131  	for level := BestSpeed; level <= BestCompression; level++ {
   132  		out := bytes.Buffer{}
   133  		e := NewWriterOptions(&out, WriterOptions{Quality: level})
   134  		n, err := e.Write(data)
   135  		if err != nil {
   136  			t.Errorf("Error compressing data: %v", err)
   137  		}
   138  		if int(n) != len(data) {
   139  			t.Errorf("Write() n=%v, want %v", n, len(data))
   140  		}
   141  		if err := e.Close(); err != nil {
   142  			t.Errorf("Close Error after writing %d bytes: %v", n, err)
   143  		}
   144  		if err := checkCompressedData(out.Bytes(), data); err != nil {
   145  			t.Errorf("Error decompressing data at level %d: %v", level, err)
   146  		}
   147  	}
   148  }
   149  
   150  func TestEncoderStreams(t *testing.T) {
   151  	// Test that output is streamed.
   152  	// Adjust window size to ensure the encoder outputs at least enough bytes
   153  	// to fill the window.
   154  	const lgWin = 16
   155  	windowSize := int(math.Pow(2, lgWin))
   156  	input := make([]byte, 8*windowSize)
   157  	rand.Read(input)
   158  	out := bytes.Buffer{}
   159  	e := NewWriterOptions(&out, WriterOptions{Quality: 11, LGWin: lgWin})
   160  	halfInput := input[:len(input)/2]
   161  	in := bytes.NewReader(halfInput)
   162  
   163  	n, err := io.Copy(e, in)
   164  	if err != nil {
   165  		t.Errorf("Copy Error: %v", err)
   166  	}
   167  
   168  	// We've fed more data than the sliding window size. Check that some
   169  	// compressed data has been output.
   170  	if out.Len() == 0 {
   171  		t.Errorf("Output length is 0 after %d bytes written", n)
   172  	}
   173  	if err := e.Close(); err != nil {
   174  		t.Errorf("Close Error after copied %d bytes: %v", n, err)
   175  	}
   176  	if err := checkCompressedData(out.Bytes(), halfInput); err != nil {
   177  		t.Error(err)
   178  	}
   179  }
   180  
   181  func TestEncoderLargeInput(t *testing.T) {
   182  	for level := BestSpeed; level <= BestCompression; level++ {
   183  		input := make([]byte, 1000000)
   184  		rand.Read(input)
   185  		out := bytes.Buffer{}
   186  		e := NewWriterOptions(&out, WriterOptions{Quality: level})
   187  		in := bytes.NewReader(input)
   188  
   189  		n, err := io.Copy(e, in)
   190  		if err != nil {
   191  			t.Errorf("Copy Error: %v", err)
   192  		}
   193  		if int(n) != len(input) {
   194  			t.Errorf("Copy() n=%v, want %v", n, len(input))
   195  		}
   196  		if err := e.Close(); err != nil {
   197  			t.Errorf("Close Error after copied %d bytes: %v", n, err)
   198  		}
   199  		if err := checkCompressedData(out.Bytes(), input); err != nil {
   200  			t.Error(err)
   201  		}
   202  
   203  		out2 := bytes.Buffer{}
   204  		e.Reset(&out2)
   205  		n2, err := e.Write(input)
   206  		if err != nil {
   207  			t.Errorf("Write error after Reset: %v", err)
   208  		}
   209  		if n2 != len(input) {
   210  			t.Errorf("Write() after Reset n=%d, want %d", n2, len(input))
   211  		}
   212  		if err := e.Close(); err != nil {
   213  			t.Errorf("Close error after Reset (copied %d) bytes: %v", n2, err)
   214  		}
   215  		if !bytes.Equal(out.Bytes(), out2.Bytes()) {
   216  			t.Error("Compressed data after Reset doesn't equal first time")
   217  		}
   218  	}
   219  }
   220  
   221  func TestEncoderFlush(t *testing.T) {
   222  	input := make([]byte, 1000)
   223  	rand.Read(input)
   224  	out := bytes.Buffer{}
   225  	e := NewWriterOptions(&out, WriterOptions{Quality: 5})
   226  	in := bytes.NewReader(input)
   227  	_, err := io.Copy(e, in)
   228  	if err != nil {
   229  		t.Fatalf("Copy Error: %v", err)
   230  	}
   231  	if err := e.Flush(); err != nil {
   232  		t.Fatalf("Flush(): %v", err)
   233  	}
   234  	if out.Len() == 0 {
   235  		t.Fatalf("0 bytes written after Flush()")
   236  	}
   237  	decompressed := make([]byte, 1000)
   238  	reader := NewReader(bytes.NewReader(out.Bytes()))
   239  	n, err := reader.Read(decompressed)
   240  	if n != len(decompressed) || err != nil {
   241  		t.Errorf("Expected <%v, nil>, but <%v, %v>", len(decompressed), n, err)
   242  	}
   243  	if !bytes.Equal(decompressed, input) {
   244  		t.Errorf(""+
   245  			"Decompress after flush: %v\n"+
   246  			"%q\n"+
   247  			"want:\n%q",
   248  			err, decompressed, input)
   249  	}
   250  	if err := e.Close(); err != nil {
   251  		t.Errorf("Close(): %v", err)
   252  	}
   253  }
   254  
   255  type readerWithTimeout struct {
   256  	io.Reader
   257  }
   258  
   259  func (r readerWithTimeout) Read(p []byte) (int, error) {
   260  	type result struct {
   261  		n   int
   262  		err error
   263  	}
   264  	ch := make(chan result)
   265  	go func() {
   266  		n, err := r.Reader.Read(p)
   267  		ch <- result{n, err}
   268  	}()
   269  	select {
   270  	case result := <-ch:
   271  		return result.n, result.err
   272  	case <-time.After(5 * time.Second):
   273  		return 0, fmt.Errorf("read timed out")
   274  	}
   275  }
   276  
   277  func TestDecoderStreaming(t *testing.T) {
   278  	pr, pw := io.Pipe()
   279  	writer := NewWriterOptions(pw, WriterOptions{Quality: 5, LGWin: 20})
   280  	reader := readerWithTimeout{NewReader(pr)}
   281  	defer func() {
   282  		go ioutil.ReadAll(pr) // swallow the "EOF" token from writer.Close
   283  		if err := writer.Close(); err != nil {
   284  			t.Errorf("writer.Close: %v", err)
   285  		}
   286  	}()
   287  
   288  	ch := make(chan []byte)
   289  	errch := make(chan error)
   290  	go func() {
   291  		for {
   292  			segment, ok := <-ch
   293  			if !ok {
   294  				return
   295  			}
   296  			if n, err := writer.Write(segment); err != nil || n != len(segment) {
   297  				errch <- fmt.Errorf("write=%v,%v, want %v,%v", n, err, len(segment), nil)
   298  				return
   299  			}
   300  			if err := writer.Flush(); err != nil {
   301  				errch <- fmt.Errorf("flush: %v", err)
   302  				return
   303  			}
   304  		}
   305  	}()
   306  	defer close(ch)
   307  
   308  	segments := [...][]byte{
   309  		[]byte("first"),
   310  		[]byte("second"),
   311  		[]byte("third"),
   312  	}
   313  	for k, segment := range segments {
   314  		t.Run(fmt.Sprintf("Segment%d", k), func(t *testing.T) {
   315  			select {
   316  			case ch <- segment:
   317  			case err := <-errch:
   318  				t.Fatalf("write: %v", err)
   319  			case <-time.After(5 * time.Second):
   320  				t.Fatalf("timed out")
   321  			}
   322  			wantLen := len(segment)
   323  			got := make([]byte, wantLen)
   324  			if n, err := reader.Read(got); err != nil || n != wantLen || !bytes.Equal(got, segment) {
   325  				t.Fatalf("read[%d]=%q,%v,%v, want %q,%v,%v", k, got, n, err, segment, wantLen, nil)
   326  			}
   327  		})
   328  	}
   329  }
   330  
   331  func TestReader(t *testing.T) {
   332  	content := bytes.Repeat([]byte("hello world!"), 10000)
   333  	encoded, _ := Encode(content, WriterOptions{Quality: 5})
   334  	r := NewReader(bytes.NewReader(encoded))
   335  	var decodedOutput bytes.Buffer
   336  	n, err := io.Copy(&decodedOutput, r)
   337  	if err != nil {
   338  		t.Fatalf("Copy(): n=%v, err=%v", n, err)
   339  	}
   340  	if got := decodedOutput.Bytes(); !bytes.Equal(got, content) {
   341  		t.Errorf(""+
   342  			"Reader output:\n"+
   343  			"%q\n"+
   344  			"want:\n"+
   345  			"<%d bytes>",
   346  			got, len(content))
   347  	}
   348  
   349  	r.Reset(bytes.NewReader(encoded))
   350  	decodedOutput.Reset()
   351  	n, err = io.Copy(&decodedOutput, r)
   352  	if err != nil {
   353  		t.Fatalf("After Reset: Copy(): n=%v, err=%v", n, err)
   354  	}
   355  	if got := decodedOutput.Bytes(); !bytes.Equal(got, content) {
   356  		t.Errorf("After Reset: "+
   357  			"Reader output:\n"+
   358  			"%q\n"+
   359  			"want:\n"+
   360  			"<%d bytes>",
   361  			got, len(content))
   362  	}
   363  
   364  }
   365  
   366  func TestDecode(t *testing.T) {
   367  	content := bytes.Repeat([]byte("hello world!"), 10000)
   368  	encoded, _ := Encode(content, WriterOptions{Quality: 5})
   369  	decoded, err := Decode(encoded)
   370  	if err != nil {
   371  		t.Errorf("Decode: %v", err)
   372  	}
   373  	if !bytes.Equal(decoded, content) {
   374  		t.Errorf(""+
   375  			"Decode content:\n"+
   376  			"%q\n"+
   377  			"want:\n"+
   378  			"<%d bytes>",
   379  			decoded, len(content))
   380  	}
   381  }
   382  
   383  func TestQuality(t *testing.T) {
   384  	content := bytes.Repeat([]byte("hello world!"), 10000)
   385  	for q := 0; q < 12; q++ {
   386  		encoded, _ := Encode(content, WriterOptions{Quality: q})
   387  		decoded, err := Decode(encoded)
   388  		if err != nil {
   389  			t.Errorf("Decode: %v", err)
   390  		}
   391  		if !bytes.Equal(decoded, content) {
   392  			t.Errorf(""+
   393  				"Decode content:\n"+
   394  				"%q\n"+
   395  				"want:\n"+
   396  				"<%d bytes>",
   397  				decoded, len(content))
   398  		}
   399  	}
   400  }
   401  
   402  func TestDecodeFuzz(t *testing.T) {
   403  	// Test that the decoder terminates with corrupted input.
   404  	content := bytes.Repeat([]byte("hello world!"), 100)
   405  	rnd := rand.New(rand.NewSource(0))
   406  	encoded, err := Encode(content, WriterOptions{Quality: 5})
   407  	if err != nil {
   408  		t.Fatalf("Encode(<%d bytes>, _) = _, %s", len(content), err)
   409  	}
   410  	if len(encoded) == 0 {
   411  		t.Fatalf("Encode(<%d bytes>, _) produced empty output", len(content))
   412  	}
   413  	for i := 0; i < 100; i++ {
   414  		enc := append([]byte{}, encoded...)
   415  		for j := 0; j < 5; j++ {
   416  			enc[rnd.Intn(len(enc))] = byte(rnd.Intn(256))
   417  		}
   418  		Decode(enc)
   419  	}
   420  }
   421  
   422  func TestDecodeTrailingData(t *testing.T) {
   423  	content := bytes.Repeat([]byte("hello world!"), 100)
   424  	encoded, _ := Encode(content, WriterOptions{Quality: 5})
   425  	_, err := Decode(append(encoded, 0))
   426  	if err == nil {
   427  		t.Errorf("Expected 'excessive input' error")
   428  	}
   429  }
   430  
   431  func TestEncodeDecode(t *testing.T) {
   432  	for _, test := range []struct {
   433  		data    []byte
   434  		repeats int
   435  	}{
   436  		{nil, 0},
   437  		{[]byte("A"), 1},
   438  		{[]byte("<html><body><H1>Hello world</H1></body></html>"), 10},
   439  		{[]byte("<html><body><H1>Hello world</H1></body></html>"), 1000},
   440  	} {
   441  		t.Logf("case %q x %d", test.data, test.repeats)
   442  		input := bytes.Repeat(test.data, test.repeats)
   443  		encoded, err := Encode(input, WriterOptions{Quality: 5})
   444  		if err != nil {
   445  			t.Errorf("Encode: %v", err)
   446  		}
   447  		// Inputs are compressible, but may be too small to compress.
   448  		if maxSize := len(input)/2 + 20; len(encoded) >= maxSize {
   449  			t.Errorf(""+
   450  				"Encode returned %d bytes, want <%d\n"+
   451  				"Encoded=%q",
   452  				len(encoded), maxSize, encoded)
   453  		}
   454  		decoded, err := Decode(encoded)
   455  		if err != nil {
   456  			t.Errorf("Decode: %v", err)
   457  		}
   458  		if !bytes.Equal(decoded, input) {
   459  			var want string
   460  			if len(input) > 320 {
   461  				want = fmt.Sprintf("<%d bytes>", len(input))
   462  			} else {
   463  				want = fmt.Sprintf("%q", input)
   464  			}
   465  			t.Errorf(""+
   466  				"Decode content:\n"+
   467  				"%q\n"+
   468  				"want:\n"+
   469  				"%s",
   470  				decoded, want)
   471  		}
   472  	}
   473  }
   474  
   475  func TestErrorReset(t *testing.T) {
   476  	compress := func(input []byte) []byte {
   477  		var buf bytes.Buffer
   478  		writer := new(Writer)
   479  		writer.Reset(&buf)
   480  		writer.Write(input)
   481  		writer.Close()
   482  
   483  		return buf.Bytes()
   484  	}
   485  
   486  	corruptReader := func(reader *Reader) {
   487  		buf := bytes.NewBuffer([]byte("trash"))
   488  		reader.Reset(buf)
   489  		_, err := io.ReadAll(reader)
   490  		if err == nil {
   491  			t.Fatalf("successively decompressed invalid input")
   492  		}
   493  	}
   494  
   495  	decompress := func(input []byte, reader *Reader) []byte {
   496  		buf := bytes.NewBuffer(input)
   497  		reader.Reset(buf)
   498  		output, err := io.ReadAll(reader)
   499  		if err != nil {
   500  			t.Fatalf("failed to decompress data %s", err.Error())
   501  		}
   502  
   503  		return output
   504  	}
   505  
   506  	source := []byte("text")
   507  
   508  	compressed := compress(source)
   509  	reader := new(Reader)
   510  	corruptReader(reader)
   511  	decompressed := decompress(compressed, reader)
   512  	if string(source) != string(decompressed) {
   513  		t.Fatalf("decompressed data does not match original state")
   514  	}
   515  }
   516  
   517  // Encode returns content encoded with Brotli.
   518  func Encode(content []byte, options WriterOptions) ([]byte, error) {
   519  	var buf bytes.Buffer
   520  	writer := NewWriterOptions(&buf, options)
   521  	_, err := writer.Write(content)
   522  	if closeErr := writer.Close(); err == nil {
   523  		err = closeErr
   524  	}
   525  	return buf.Bytes(), err
   526  }
   527  
   528  // Decode decodes Brotli encoded data.
   529  func Decode(encodedData []byte) ([]byte, error) {
   530  	r := NewReader(bytes.NewReader(encodedData))
   531  	return ioutil.ReadAll(r)
   532  }
   533  
   534  func BenchmarkEncodeLevels(b *testing.B) {
   535  	opticks, err := ioutil.ReadFile("testdata/Isaac.Newton-Opticks.txt")
   536  	if err != nil {
   537  		b.Fatal(err)
   538  	}
   539  
   540  	for level := BestSpeed; level <= BestCompression; level++ {
   541  		b.Run(fmt.Sprintf("%d", level), func(b *testing.B) {
   542  			b.ReportAllocs()
   543  			b.SetBytes(int64(len(opticks)))
   544  			for i := 0; i < b.N; i++ {
   545  				w := NewWriterLevel(ioutil.Discard, level)
   546  				w.Write(opticks)
   547  				w.Close()
   548  			}
   549  		})
   550  	}
   551  }
   552  
   553  func BenchmarkEncodeLevelsReset(b *testing.B) {
   554  	opticks, err := ioutil.ReadFile("testdata/Isaac.Newton-Opticks.txt")
   555  	if err != nil {
   556  		b.Fatal(err)
   557  	}
   558  
   559  	for level := BestSpeed; level <= BestCompression; level++ {
   560  		buf := new(bytes.Buffer)
   561  		w := NewWriterLevel(buf, level)
   562  		w.Write(opticks)
   563  		w.Close()
   564  		b.Run(fmt.Sprintf("%d", level), func(b *testing.B) {
   565  			b.ReportAllocs()
   566  			b.ReportMetric(float64(len(opticks))/float64(buf.Len()), "ratio")
   567  			b.SetBytes(int64(len(opticks)))
   568  			for i := 0; i < b.N; i++ {
   569  				w.Reset(ioutil.Discard)
   570  				w.Write(opticks)
   571  				w.Close()
   572  			}
   573  		})
   574  	}
   575  }
   576  
   577  func BenchmarkDecodeLevels(b *testing.B) {
   578  	opticks, err := ioutil.ReadFile("testdata/Isaac.Newton-Opticks.txt")
   579  	if err != nil {
   580  		b.Fatal(err)
   581  	}
   582  
   583  	for level := BestSpeed; level <= BestCompression; level++ {
   584  		buf := new(bytes.Buffer)
   585  		w := NewWriterLevel(buf, level)
   586  		w.Write(opticks)
   587  		w.Close()
   588  		compressed := buf.Bytes()
   589  		b.Run(fmt.Sprintf("%d", level), func(b *testing.B) {
   590  			b.ReportAllocs()
   591  			b.SetBytes(int64(len(opticks)))
   592  			for i := 0; i < b.N; i++ {
   593  				io.Copy(ioutil.Discard, NewReader(bytes.NewReader(compressed)))
   594  			}
   595  		})
   596  	}
   597  }