github.com/m3db/m3@v1.5.0/src/x/serialize/decoder_test.go (about)

     1  // Copyright (c) 2018 Uber Technologies, Inc.
     2  //
     3  // Permission is hereby granted, free of charge, to any person obtaining a copy
     4  // of this software and associated documentation files (the "Software"), to deal
     5  // in the Software without restriction, including without limitation the rights
     6  // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
     7  // copies of the Software, and to permit persons to whom the Software is
     8  // furnished to do so, subject to the following conditions:
     9  //
    10  // The above copyright notice and this permission notice shall be included in
    11  // all copies or substantial portions of the Software.
    12  //
    13  // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    14  // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    15  // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    16  // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    17  // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    18  // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
    19  // THE SOFTWARE.
    20  
    21  package serialize
    22  
    23  import (
    24  	"fmt"
    25  	"testing"
    26  
    27  	"github.com/m3db/m3/src/x/checked"
    28  	"github.com/m3db/m3/src/x/ident"
    29  	xtest "github.com/m3db/m3/src/x/test"
    30  
    31  	"github.com/golang/mock/gomock"
    32  	"github.com/stretchr/testify/assert"
    33  	"github.com/stretchr/testify/require"
    34  )
    35  
    36  var (
    37  	testDecodeOpts = NewTagDecoderOptions(TagDecoderOptionsConfig{})
    38  )
    39  
    40  func TestEmptyDecode(t *testing.T) {
    41  	var b []byte
    42  	b = append(b, headerMagicBytes...)
    43  	b = append(b, []byte{0x0, 0x0}...)
    44  
    45  	d := newTagDecoder(testDecodeOpts, nil)
    46  	d.Reset(wrapAsCheckedBytes(b))
    47  	require.False(t, d.Next())
    48  	require.NoError(t, d.Err())
    49  	d.Close()
    50  }
    51  
    52  func TestEmptyTagNameDecode(t *testing.T) {
    53  	var b []byte
    54  	b = append(b, headerMagicBytes...)
    55  	b = append(b, encodeUInt16(1, make([]byte, 2))...) /* num tags */
    56  	b = append(b, encodeUInt16(0, make([]byte, 2))...) /* len empty string */
    57  	b = append(b, encodeUInt16(4, make([]byte, 2))...) /* len defg */
    58  	b = append(b, []byte("defg")...)
    59  
    60  	d := newTagDecoder(testDecodeOpts, nil)
    61  	d.Reset(wrapAsCheckedBytes(b))
    62  	require.False(t, d.Next())
    63  	require.Error(t, d.Err())
    64  	assertFastErr(t, d, "some tag")
    65  }
    66  
    67  func TestEmptyTagValueDecode(t *testing.T) {
    68  	var b []byte
    69  	b = append(b, headerMagicBytes...)
    70  	b = append(b, encodeUInt16(1, make([]byte, 2))...) /* num tags */
    71  	b = append(b, encodeUInt16(1, make([]byte, 2))...) /* len "1" */
    72  	b = append(b, []byte("a")...)                      /* tag name */
    73  	b = append(b, encodeUInt16(0, make([]byte, 2))...) /* len tag value */
    74  
    75  	d := newTagDecoder(testDecodeOpts, nil)
    76  	d.Reset(wrapAsCheckedBytes(b))
    77  	assertNextTag(t, d, "a", "")
    78  	require.False(t, d.Next())
    79  	require.NoError(t, d.Err())
    80  }
    81  
    82  func TestDecodeHeaderMissing(t *testing.T) {
    83  	var b []byte
    84  	b = append(b, []byte{0x0, 0x0}...)
    85  	b = append(b, []byte{0x0, 0x0}...)
    86  
    87  	d := newTestTagDecoder()
    88  	d.Reset(wrapAsCheckedBytes(b))
    89  	require.False(t, d.Next())
    90  	require.Error(t, d.Err())
    91  	assertFastErr(t, d, "some tag")
    92  	d.Close()
    93  }
    94  
    95  func TestDecodeResetsErrState(t *testing.T) {
    96  	var b []byte
    97  	b = append(b, []byte{0x0, 0x0}...)
    98  	b = append(b, []byte{0x0, 0x0}...)
    99  
   100  	d := newTestTagDecoder()
   101  	d.Reset(wrapAsCheckedBytes(b))
   102  	require.False(t, d.Next())
   103  	require.Error(t, d.Err())
   104  
   105  	d.Reset(testTagDecoderBytes())
   106  	require.NoError(t, d.Err())
   107  }
   108  
   109  func TestDecodeSimple(t *testing.T) {
   110  	b := testTagDecoderBytes()
   111  	d := newTestTagDecoder()
   112  	d.Reset(b)
   113  	require.NoError(t, d.Err())
   114  
   115  	assertNextTag(t, d, "abc", "defg")
   116  	assertNextTag(t, d, "x", "bar")
   117  
   118  	require.False(t, d.Next())
   119  	require.NoError(t, d.Err())
   120  
   121  	d.Close()
   122  }
   123  
   124  func TestDecodeAfterRewind(t *testing.T) {
   125  	b := testTagDecoderBytes()
   126  	d := newTestTagDecoder()
   127  	d.Reset(b)
   128  	require.NoError(t, d.Err())
   129  
   130  	count := 10
   131  	printedTags := []byte("abcdefgxbar")
   132  	acBytes := make([]byte, 0, count*len(printedTags))
   133  	exBytes := make([]byte, count*len(printedTags))
   134  	readIter := func(it ident.TagIterator) {
   135  		tag := d.Current()
   136  		acBytes = append(acBytes, tag.Name.Bytes()...)
   137  		acBytes = append(acBytes, tag.Value.Bytes()...)
   138  		assertFastDecode(t, d, tag.Name.String(), tag.Value.String())
   139  	}
   140  
   141  	for i := 0; i < count; i++ {
   142  		require.True(t, d.Next())
   143  		readIter(d)
   144  		require.True(t, d.Next())
   145  		readIter(d)
   146  		require.False(t, d.Next())
   147  		require.NoError(t, d.Err())
   148  		copy(exBytes[i*len(printedTags):], printedTags)
   149  		d.Rewind()
   150  	}
   151  
   152  	assert.Equal(t, exBytes, acBytes)
   153  	assert.Equal(t, string(exBytes), string(acBytes))
   154  
   155  	assert.Equal(t, 1, b.NumRef())
   156  	d.Close()
   157  	assert.Equal(t, 0, b.NumRef())
   158  }
   159  
   160  func TestDecodeTooManyTags(t *testing.T) {
   161  	b := testTagDecoderBytes()
   162  	opts := testDecodeOpts.SetTagSerializationLimits(
   163  		NewTagSerializationLimits().SetMaxNumberTags(1))
   164  	d := newTagDecoder(opts, nil)
   165  	d.Reset(b)
   166  	require.Error(t, d.Err())
   167  }
   168  
   169  func TestDecodeLiteralTooLong(t *testing.T) {
   170  	b := testTagDecoderBytes()
   171  	opts := testDecodeOpts.SetTagSerializationLimits(
   172  		NewTagSerializationLimits().
   173  			SetMaxNumberTags(2).
   174  			SetMaxTagLiteralLength(3))
   175  	d := newTagDecoder(opts, nil)
   176  	d.Reset(b)
   177  	require.NoError(t, d.Err())
   178  	require.False(t, d.Next())
   179  	require.Error(t, d.Err())
   180  }
   181  
   182  func TestDecodeMissingTags(t *testing.T) {
   183  	var b []byte
   184  	b = append(b, headerMagicBytes...)
   185  	b = append(b, encodeUInt16(2, make([]byte, 2))...) /* num tags */
   186  
   187  	d := newTestTagDecoder()
   188  	d.Reset(wrapAsCheckedBytes(b))
   189  	require.NoError(t, d.Err())
   190  	assertFastErr(t, d, "some tag")
   191  
   192  	require.False(t, d.Next())
   193  	require.Error(t, d.Err())
   194  }
   195  
   196  func TestDecodeOwnershipFinalize(t *testing.T) {
   197  	var b []byte
   198  	b = append(b, headerMagicBytes...)
   199  	b = append(b, encodeUInt16(2, make([]byte, 2))...) /* num tags */
   200  
   201  	wrappedBytes := wrapAsCheckedBytes(b)
   202  	require.Equal(t, 0, wrappedBytes.NumRef())
   203  
   204  	d := newTestTagDecoder()
   205  	d.Reset(wrappedBytes)
   206  	require.NoError(t, d.Err())
   207  	require.NotEqual(t, 0, wrappedBytes.NumRef())
   208  
   209  	require.False(t, d.Next())
   210  	require.Error(t, d.Err())
   211  
   212  	d.Close()
   213  	require.Equal(t, 0, wrappedBytes.NumRef())
   214  	wrappedBytes.IncRef()
   215  	require.Nil(t, wrappedBytes.Bytes())
   216  }
   217  
   218  func TestDecodeMissingValue(t *testing.T) {
   219  	var b []byte
   220  	b = append(b, headerMagicBytes...)
   221  	b = append(b, encodeUInt16(2, make([]byte, 2))...) /* num tags */
   222  	b = append(b, encodeUInt16(3, make([]byte, 2))...) /* len abc */
   223  	b = append(b, []byte("abc")...)
   224  
   225  	b = append(b, encodeUInt16(4, make([]byte, 2))...) /* len defg */
   226  	b = append(b, []byte("defg")...)
   227  
   228  	b = append(b, encodeUInt16(1, make([]byte, 2))...) /* len x */
   229  	b = append(b, []byte("x")...)
   230  
   231  	d := newTestTagDecoder()
   232  	d.Reset(wrapAsCheckedBytes(b))
   233  	require.NoError(t, d.Err())
   234  
   235  	assertNextTag(t, d, "abc", "defg")
   236  	require.False(t, d.Next())
   237  	require.Error(t, d.Err())
   238  	assertFastErr(t, d, "x")
   239  }
   240  
   241  func TestDecodeDuplicateLifecycle(t *testing.T) {
   242  	b := testTagDecoderBytes()
   243  	d := newTestTagDecoder()
   244  	d.Reset(b)
   245  	require.NoError(t, d.Err())
   246  
   247  	oldLen := d.Remaining()
   248  	copy := d.Duplicate()
   249  	require.Equal(t, oldLen, copy.Remaining())
   250  
   251  	for copy.Next() {
   252  		tag := copy.Current() // keep looping
   253  		tag.Name.Bytes()      // ensure we can get values too
   254  		tag.Value.Bytes()     // and don't panic
   255  	}
   256  	require.NoError(t, copy.Err())
   257  	copy.Close()
   258  	d.Close()
   259  }
   260  
   261  func TestDecodeDuplicateIteration(t *testing.T) {
   262  	b := testTagDecoderBytes()
   263  	d := newTestTagDecoder()
   264  	d.Reset(b)
   265  	require.NoError(t, d.Err())
   266  	require.True(t, d.Next())
   267  
   268  	oldLen := d.Remaining()
   269  	copy := d.Duplicate()
   270  	require.Equal(t, oldLen, copy.Remaining())
   271  
   272  	for copy.Next() {
   273  		tag := copy.Current() // keep looping
   274  		tag.Name.Bytes()      // ensure we can get values too
   275  		tag.Value.Bytes()     // and don't panic
   276  	}
   277  	require.NoError(t, copy.Err())
   278  	copy.Close()
   279  
   280  	dec := d.(*decoder)
   281  	require.True(t, dec.checkedData.NumRef() >= 3, fmt.Sprintf("%d", dec.checkedData.NumRef()))
   282  	require.NotPanics(t, func() {
   283  		d.Close()
   284  	})
   285  }
   286  
   287  func TestDecodeDuplicateLifecycleMocks(t *testing.T) {
   288  	ctrl := gomock.NewController(xtest.Reporter{t})
   289  	defer ctrl.Finish()
   290  
   291  	rawData := testTagDecoderBytesRaw()
   292  	mockBytes := checked.NewMockBytes(ctrl)
   293  	mockBytes.EXPECT().Bytes().Return(rawData).AnyTimes()
   294  
   295  	mockBytes.EXPECT().IncRef()
   296  	d := newTestTagDecoder()
   297  	d.Reset(mockBytes)
   298  	require.NoError(t, d.Err())
   299  
   300  	mockBytes.EXPECT().IncRef().Times(2)
   301  	require.True(t, d.Next())
   302  	tag := d.Current()
   303  	require.Equal(t, "abc", tag.Name.String())
   304  	require.Equal(t, "defg", tag.Value.String())
   305  
   306  	mockBytes.EXPECT().IncRef().Times(3)
   307  	dupe := d.Duplicate()
   308  	require.NoError(t, dupe.Err())
   309  
   310  	mockBytes.EXPECT().DecRef().Times(2)
   311  	mockBytes.EXPECT().IncRef().Times(2)
   312  	require.True(t, d.Next())
   313  	tag = d.Current()
   314  	require.Equal(t, "x", tag.Name.String())
   315  	require.Equal(t, "bar", tag.Value.String())
   316  
   317  	mockBytes.EXPECT().DecRef().Times(2)
   318  	require.False(t, d.Next())
   319  	require.NoError(t, d.Err())
   320  
   321  	mockBytes.EXPECT().DecRef()
   322  	mockBytes.EXPECT().NumRef().Return(3)
   323  	d.Close()
   324  
   325  	mockBytes.EXPECT().DecRef().Times(2)
   326  	mockBytes.EXPECT().IncRef().Times(2)
   327  	require.True(t, dupe.Next())
   328  	tag = dupe.Current()
   329  	require.Equal(t, "x", tag.Name.String())
   330  	require.Equal(t, "bar", tag.Value.String())
   331  
   332  	mockBytes.EXPECT().DecRef().Times(2)
   333  	require.False(t, dupe.Next())
   334  	require.NoError(t, dupe.Err())
   335  
   336  	mockBytes.EXPECT().DecRef()
   337  	mockBytes.EXPECT().NumRef().Return(0)
   338  	mockBytes.EXPECT().Finalize()
   339  	dupe.Close()
   340  }
   341  
   342  // assert the next tag in the iteration. ensures the decoder and faster decoder match.
   343  func assertNextTag(t *testing.T, d TagDecoder, name, value string) {
   344  	require.True(t, d.Next())
   345  	tag := d.Current()
   346  	require.Equal(t, name, tag.Name.String())
   347  	require.Equal(t, value, tag.Value.String())
   348  	assertFastDecode(t, d, name, value)
   349  }
   350  
   351  // assert the faster decoder returns the provided value for the name.
   352  func assertFastDecode(t *testing.T, d TagDecoder, name, value string) {
   353  	v, found, err := TagValueFromEncodedTagsFast(d.(*decoder).checkedData.Bytes(), []byte(name))
   354  	require.NoError(t, err)
   355  	require.True(t, found)
   356  	require.Equal(t, value, string(v))
   357  }
   358  
   359  // assert the fast decoder returns an error for the provided name.
   360  func assertFastErr(t *testing.T, d TagDecoder, name string) {
   361  	_, _, err := TagValueFromEncodedTagsFast(d.(*decoder).checkedData.Bytes(), []byte(name))
   362  	require.Error(t, err)
   363  }
   364  
   365  func newTestTagDecoder() TagDecoder {
   366  	return newTagDecoder(testDecodeOpts, newTestTagDecoderPool())
   367  }
   368  
   369  func wrapAsCheckedBytes(b []byte) checked.Bytes {
   370  	opts := checked.NewBytesOptions().SetFinalizer(
   371  		checked.BytesFinalizerFn(func(b checked.Bytes) {
   372  			b.IncRef()
   373  			b.Reset(nil)
   374  			b.DecRef()
   375  		}))
   376  	cb := checked.NewBytes(nil, opts)
   377  	cb.IncRef()
   378  	cb.Reset(b)
   379  	cb.DecRef()
   380  	return cb
   381  }
   382  
   383  func testTagDecoderBytesRaw() []byte {
   384  	var b []byte
   385  	b = append(b, headerMagicBytes...)
   386  	b = append(b, encodeUInt16(2, make([]byte, 2))...) /* num tags */
   387  
   388  	b = append(b, encodeUInt16(3, make([]byte, 2))...) /* len abc */
   389  	b = append(b, []byte("abc")...)
   390  
   391  	b = append(b, encodeUInt16(4, make([]byte, 2))...) /* len defg */
   392  	b = append(b, []byte("defg")...)
   393  
   394  	b = append(b, encodeUInt16(1, make([]byte, 2))...) /* len x */
   395  	b = append(b, []byte("x")...)
   396  
   397  	b = append(b, encodeUInt16(3, make([]byte, 2))...) /* len bar */
   398  	b = append(b, []byte("bar")...)
   399  	return b
   400  }
   401  
   402  func testTagDecoderBytes() checked.Bytes {
   403  	return wrapAsCheckedBytes(testTagDecoderBytesRaw())
   404  }