github.com/hoveychen/kafka-go@v0.4.42/compress/snappy/xerial_test.go (about)

     1  package snappy
     2  
     3  import (
     4  	"bytes"
     5  	"crypto/rand"
     6  	"io"
     7  	"testing"
     8  
     9  	goxerialsnappy "github.com/hoveychen/kafka-go/compress/snappy/go-xerial-snappy"
    10  	"github.com/klauspost/compress/snappy"
    11  )
    12  
    13  // Wrap an io.Reader or io.Writer to disable all copy optimizations like
    14  // io.WriterTo or io.ReaderFrom.
    15  // We use this to ensure writes are chunked by io.Copy's internal buffer
    16  // in the tests.
    17  type simpleReader struct{ io.Reader }
    18  type simpleWriter struct{ io.Writer }
    19  
    20  func TestXerialReaderSnappy(t *testing.T) {
    21  	rawData := new(bytes.Buffer)
    22  	rawData.Grow(1024 * 1024)
    23  	io.CopyN(rawData, rand.Reader, 1024*1024)
    24  
    25  	compressedRawData := bytes.NewReader(snappy.Encode(nil, rawData.Bytes()))
    26  
    27  	decompressedData := new(bytes.Buffer)
    28  	io.Copy(decompressedData,
    29  		&xerialReader{reader: compressedRawData, decode: snappy.Decode})
    30  
    31  	b0 := rawData.Bytes()
    32  	b1 := decompressedData.Bytes()
    33  
    34  	if !bytes.Equal(b0, b1) {
    35  		t.Error("data mismatch")
    36  	}
    37  }
    38  
    39  func TestXerialReaderWriter(t *testing.T) {
    40  	rawData := new(bytes.Buffer)
    41  	rawData.Grow(1024 * 1024)
    42  	io.CopyN(rawData, rand.Reader, 1024*1024)
    43  
    44  	framedData := new(bytes.Buffer)
    45  	framedData.Grow(rawData.Len() + 1024)
    46  	w := simpleWriter{&xerialWriter{writer: framedData}}
    47  	r := simpleReader{bytes.NewReader(rawData.Bytes())}
    48  	io.Copy(w, r)
    49  	w.Writer.(*xerialWriter).Flush()
    50  
    51  	unframedData := new(bytes.Buffer)
    52  	unframedData.Grow(rawData.Len())
    53  	io.Copy(unframedData, &xerialReader{reader: framedData})
    54  
    55  	b0 := rawData.Bytes()
    56  	b1 := unframedData.Bytes()
    57  
    58  	if !bytes.Equal(b0, b1) {
    59  		t.Error("data mismatch")
    60  	}
    61  }
    62  
    63  func TestXerialFramedCompression(t *testing.T) {
    64  	rawData := new(bytes.Buffer)
    65  	rawData.Grow(1024 * 1024)
    66  	io.CopyN(rawData, rand.Reader, 1024*1024)
    67  
    68  	framedAndCompressedData := new(bytes.Buffer)
    69  	framedAndCompressedData.Grow(rawData.Len())
    70  	w := simpleWriter{&xerialWriter{writer: framedAndCompressedData, framed: true, encode: snappy.Encode}}
    71  	r := simpleReader{bytes.NewReader(rawData.Bytes())}
    72  	io.Copy(w, r)
    73  	w.Writer.(*xerialWriter).Flush()
    74  
    75  	unframedAndDecompressedData := new(bytes.Buffer)
    76  	unframedAndDecompressedData.Grow(rawData.Len())
    77  	io.Copy(unframedAndDecompressedData,
    78  		simpleReader{&xerialReader{reader: framedAndCompressedData, decode: snappy.Decode}})
    79  
    80  	b0 := rawData.Bytes()
    81  	b1 := unframedAndDecompressedData.Bytes()
    82  
    83  	if !bytes.Equal(b0, b1) {
    84  		t.Error("data mismatch")
    85  	}
    86  }
    87  
    88  func TestXerialFramedCompressionOptimized(t *testing.T) {
    89  	rawData := new(bytes.Buffer)
    90  	rawData.Grow(1024 * 1024)
    91  	io.CopyN(rawData, rand.Reader, 1024*1024)
    92  
    93  	framedAndCompressedData := new(bytes.Buffer)
    94  	framedAndCompressedData.Grow(rawData.Len())
    95  	w := &xerialWriter{writer: framedAndCompressedData, framed: true, encode: snappy.Encode}
    96  	r := simpleReader{bytes.NewReader(rawData.Bytes())}
    97  	io.Copy(w, r)
    98  	w.Flush()
    99  
   100  	unframedAndDecompressedData := new(bytes.Buffer)
   101  	unframedAndDecompressedData.Grow(rawData.Len())
   102  	io.Copy(unframedAndDecompressedData,
   103  		&xerialReader{reader: framedAndCompressedData, decode: snappy.Decode})
   104  
   105  	b0 := rawData.Bytes()
   106  	b1 := unframedAndDecompressedData.Bytes()
   107  
   108  	if !bytes.Equal(b0, b1) {
   109  		t.Error("data mismatch")
   110  	}
   111  }
   112  
   113  func TestXerialReaderAgainstGoXerialSnappy(t *testing.T) {
   114  	rawData := new(bytes.Buffer)
   115  	rawData.Grow(1024 * 1024)
   116  	io.CopyN(rawData, rand.Reader, 1024*1024)
   117  	rawBytes := rawData.Bytes()
   118  
   119  	framedAndCompressedData := []byte{}
   120  	const chunkSize = 999
   121  	for i := 0; i < len(rawBytes); i += chunkSize {
   122  		j := i + chunkSize
   123  		if j > len(rawBytes) {
   124  			j = len(rawBytes)
   125  		}
   126  		framedAndCompressedData = goxerialsnappy.EncodeStream(framedAndCompressedData, rawBytes[i:j])
   127  	}
   128  
   129  	unframedAndDecompressedData := new(bytes.Buffer)
   130  	unframedAndDecompressedData.Grow(rawData.Len())
   131  	io.Copy(unframedAndDecompressedData,
   132  		&xerialReader{reader: bytes.NewReader(framedAndCompressedData), decode: snappy.Decode})
   133  
   134  	b0 := rawBytes
   135  	b1 := unframedAndDecompressedData.Bytes()
   136  
   137  	if !bytes.Equal(b0, b1) {
   138  		t.Error("data mismatch")
   139  	}
   140  }
   141  
   142  func TestXerialWriterAgainstGoXerialSnappy(t *testing.T) {
   143  	rawData := new(bytes.Buffer)
   144  	rawData.Grow(1024 * 1024)
   145  	io.CopyN(rawData, rand.Reader, 1024*1024)
   146  
   147  	framedAndCompressedData := new(bytes.Buffer)
   148  	framedAndCompressedData.Grow(rawData.Len())
   149  	w := &xerialWriter{writer: framedAndCompressedData, framed: true, encode: snappy.Encode}
   150  	r := simpleReader{bytes.NewReader(rawData.Bytes())}
   151  	io.Copy(w, r)
   152  	w.Flush()
   153  
   154  	unframedAndDecompressedData, err := goxerialsnappy.Decode(framedAndCompressedData.Bytes())
   155  	if err != nil {
   156  		t.Error(err)
   157  	}
   158  
   159  	b0 := rawData.Bytes()
   160  	b1 := unframedAndDecompressedData
   161  
   162  	if !bytes.Equal(b0, b1) {
   163  		t.Error("data mismatch")
   164  	}
   165  }