github.com/GuanceCloud/cliutils@v1.1.21/point/encode_test.go (about)

     1  // Unless explicitly stated otherwise all files in this repository are licensed
     2  // under the MIT License.
     3  // This product includes software developed at Guance Cloud (https://www.guance.com/).
     4  // Copyright 2021-present Guance, Inc.
     5  
     6  package point
     7  
     8  import (
     9  	"bytes"
    10  	"compress/gzip"
    11  	"encoding/json"
    12  	"math"
    13  	"strings"
    14  	T "testing"
    15  	"time"
    16  
    17  	"github.com/GuanceCloud/cliutils"
    18  	"github.com/GuanceCloud/cliutils/point/gogopb"
    19  	proto "github.com/gogo/protobuf/proto"
    20  	"github.com/stretchr/testify/assert"
    21  	"github.com/stretchr/testify/require"
    22  )
    23  
    24  // Test if encode change points' payload
    25  func TestIdempotent(t *T.T) {
    26  	cases := []struct {
    27  		name  string
    28  		pts   []*Point
    29  		batch int
    30  	}{
    31  		{
    32  			name:  "ok-32#3",
    33  			pts:   RandPoints(32),
    34  			batch: 3,
    35  		},
    36  
    37  		{
    38  			name:  "ok-32#1",
    39  			pts:   RandPoints(32),
    40  			batch: 1,
    41  		},
    42  
    43  		{
    44  			name:  "ok-32#0",
    45  			pts:   RandPoints(32),
    46  			batch: 0,
    47  		},
    48  
    49  		{
    50  			name:  "nothing",
    51  			pts:   RandPoints(0),
    52  			batch: 0,
    53  		},
    54  	}
    55  
    56  	for _, tc := range cases {
    57  		t.Run(tc.name, func(t *T.T) {
    58  			enc := GetEncoder(WithEncBatchSize(tc.batch))
    59  
    60  			p1, err := enc.Encode(tc.pts)
    61  			assert.NoError(t, err)
    62  			PutEncoder(enc)
    63  
    64  			enc = GetEncoder(WithEncBatchSize(tc.batch))
    65  			p2, err := enc.Encode(tc.pts)
    66  			assert.NoError(t, err)
    67  			PutEncoder(enc)
    68  
    69  			assert.Equal(t, p1, p2)
    70  		})
    71  
    72  		// test encode pb
    73  		t.Run(tc.name+"-pb", func(t *T.T) {
    74  			enc := GetEncoder(WithEncBatchSize(tc.batch), WithEncEncoding(Protobuf))
    75  
    76  			p1, err := enc.Encode(tc.pts)
    77  			assert.NoError(t, err)
    78  			PutEncoder(enc)
    79  
    80  			enc = GetEncoder(WithEncBatchSize(tc.batch), WithEncEncoding(Protobuf))
    81  			p2, err := enc.Encode(tc.pts)
    82  			assert.NoError(t, err)
    83  			PutEncoder(enc)
    84  
    85  			assert.Equal(t, p1, p2)
    86  		})
    87  	}
    88  }
    89  
    90  // TestEncodeEqualty test equality on
    91  //   - multi-part encode: multiple points splited into multiple parts
    92  //   - line-protocol/protobuf: points encode between line-protocol and protobuf are equal
    93  func TestEncodeEqualty(t *T.T) {
    94  	r := NewRander(WithKVSorted(true), WithRandFields(1), WithRandTags(1))
    95  
    96  	nrand := 6
    97  	randBsize := 3
    98  
    99  	var (
   100  		randPts = r.Rand(nrand)
   101  
   102  		simplePts = []*Point{
   103  			NewPointV2(`abc`, NewKVs(map[string]interface{}{"f1": "fv1", "f2": "fv2", "f3": "fv3"}).
   104  				AddTag(`t1`, `tv1`).
   105  				AddTag(`t2`, `tv2`).
   106  				AddTag(`t3`, `tv3`), WithTime(time.Unix(0, 123))),
   107  
   108  			NewPointV2(`def`, NewKVs(map[string]interface{}{"f1": "fv1", "f2": "fv2", "f3": "fv3"}).
   109  				AddTag(`t1`, `tv1`).
   110  				AddTag(`t2`, `tv2`).
   111  				AddTag(`t3`, `tv3`), WithTime(time.Unix(0, 123))),
   112  
   113  			NewPointV2(`xyz`, NewKVs(map[string]interface{}{"f1": "fv1", "f2": "fv2", "f3": "fv3"}).
   114  				AddTag(`t1`, `tv1`).
   115  				AddTag(`t2`, `tv2`).
   116  				AddTag(`t3`, `tv3`), WithTime(time.Unix(0, 123))),
   117  		}
   118  
   119  		__fn = func(n int, data []byte) error {
   120  			t.Logf("batch size: %d, payload: %d", n, len(data))
   121  			return nil
   122  		}
   123  	)
   124  
   125  	cases := []struct {
   126  		name   string
   127  		pts    []*Point
   128  		bsize  int
   129  		fn     EncodeFn
   130  		gz     bool
   131  		expect [][]byte
   132  	}{
   133  		{
   134  			name:  "single-point",
   135  			bsize: 10,
   136  			expect: [][]byte{
   137  				[]byte(`abc,t1=tv1,t2=tv2,t3=tv3 f1="fv1",f2="fv2",f3="fv3" 123`),
   138  			},
   139  
   140  			pts: func() []*Point {
   141  				x, err := NewPoint("abc", map[string]string{
   142  					"t1": "tv1",
   143  					"t2": "tv2",
   144  					"t3": "tv3",
   145  				}, map[string]interface{}{
   146  					"f1": "fv1",
   147  					"f2": "fv2",
   148  					"f3": "fv3",
   149  				}, WithTime(time.Unix(0, 123)), WithKeySorted(true))
   150  
   151  				require.NoError(t, err)
   152  
   153  				t.Logf("pt: %s", x.Pretty())
   154  				return []*Point{x}
   155  			}(),
   156  		},
   157  
   158  		{
   159  			name:  "random-point",
   160  			bsize: randBsize,
   161  			pts:   randPts,
   162  			expect: func() [][]byte {
   163  				enc := GetEncoder(WithEncBatchSize(randBsize))
   164  				defer PutEncoder(enc)
   165  
   166  				x, err := enc.Encode(randPts)
   167  				assert.NoError(t, err)
   168  				return x
   169  			}(),
   170  		},
   171  
   172  		{
   173  			name:  "random-point-with-callback",
   174  			bsize: randBsize,
   175  			pts:   randPts,
   176  			fn:    __fn,
   177  			expect: func() [][]byte {
   178  				enc := GetEncoder(WithEncBatchSize(randBsize))
   179  				defer PutEncoder(enc)
   180  
   181  				bufs, err := enc.Encode(randPts)
   182  				assert.NoError(t, err)
   183  
   184  				if len(randPts)%randBsize == 0 {
   185  					assert.Equal(t, len(randPts)/randBsize, len(bufs))
   186  				} else {
   187  					assert.Equal(t, len(randPts)/randBsize+1, len(bufs), "randPts: %d", len(randPts))
   188  				}
   189  
   190  				for i, buf := range bufs {
   191  					t.Logf("get %dth batch:\n%s", i, buf)
   192  					if i != len(bufs)-1 {
   193  						assert.Equal(t, randBsize, len(bytes.Split(buf, []byte("\n"))))
   194  					}
   195  				}
   196  
   197  				return bufs
   198  			}(),
   199  		},
   200  
   201  		{
   202  			name:  "simple-point-with-callback",
   203  			bsize: 1,
   204  			pts:   simplePts,
   205  			fn:    __fn,
   206  			expect: func() [][]byte {
   207  				enc := GetEncoder(WithEncBatchSize(1))
   208  				defer PutEncoder(enc)
   209  
   210  				bufs, err := enc.Encode(simplePts)
   211  				assert.NoError(t, err)
   212  
   213  				assert.Equal(t, len(simplePts), len(bufs))
   214  
   215  				for i, buf := range bufs {
   216  					t.Logf("get %dth batch:\n%s", i, buf)
   217  					assert.Equal(t, 1, len(bytes.Split(buf, []byte("\n"))))
   218  				}
   219  
   220  				return bufs
   221  			}(),
   222  			// expect: simplePtsExpect,
   223  		},
   224  	}
   225  
   226  	for _, tc := range cases {
   227  		t.Run(tc.name, func(t *T.T) {
   228  			enc := GetEncoder(WithEncBatchSize(tc.bsize), WithEncFn(tc.fn))
   229  			defer PutEncoder(enc)
   230  
   231  			payloads, err := enc.Encode(tc.pts)
   232  			assert.NoError(t, err)
   233  
   234  			assert.Equal(t, len(tc.expect), len(payloads))
   235  
   236  			for idx, ex := range tc.expect {
   237  				assert.Equal(t, len(ex), len(payloads[idx]), "not equal at index: %d, gz: %q, fn: %q", idx, tc.gz, tc.fn)
   238  				assert.Equal(t, ex, payloads[idx], "[%d] expect %s, get %s", idx, string(ex), string(payloads[idx]))
   239  			}
   240  		})
   241  
   242  		t.Run(tc.name+"-pb", func(t *T.T) {
   243  			enc := GetEncoder(WithEncBatchSize(tc.bsize),
   244  				WithEncFn(tc.fn),
   245  				WithEncEncoding(Protobuf))
   246  			defer PutEncoder(enc)
   247  
   248  			assert.Len(t, enc.pbpts.Arr, 0)
   249  
   250  			payloads, err := enc.Encode(tc.pts)
   251  
   252  			assert.NoError(t, err)
   253  
   254  			assert.Equal(t, len(tc.expect), len(payloads))
   255  
   256  			// check PB unmarshal
   257  			for idx := range tc.expect {
   258  				var pbpts PBPoints
   259  				assert.NoError(t, proto.Unmarshal(payloads[idx], &pbpts))
   260  			}
   261  
   262  			// convert PB to line-protocol, check equality
   263  			for idx, p := range payloads {
   264  				lp, err := PB2LP(p)
   265  				assert.NoError(t, err)
   266  				t.Logf("pb -> lp:\n%s", lp)
   267  
   268  				assert.Equal(t, string(tc.expect[idx]), string(lp))
   269  			}
   270  		})
   271  	}
   272  }
   273  
   274  func TestEscapeEncode(t *T.T) {
   275  	t.Run("escaped-lineproto", func(t *T.T) {
   276  		var kvs KVs
   277  		kvs = kvs.Add("f1=2=3=", 3.14, false, false)
   278  		kvs = kvs.Add("f2\tnr", 2, false, false)
   279  		kvs = kvs.Add("f3,", "some-string\nanother-line", false, false)
   280  		kvs = kvs.Add("f4,", false, false, false)
   281  
   282  		kvs = kvs.Add("f\nnext-line,", []byte("hello"), false, false)
   283  		kvs = kvs.Add(`f\other`, []byte("hello"), false, false)
   284  		kvs = kvs.Add("tag=1", "value", true, false)
   285  		kvs = kvs.Add("tag 2", "value", true, false)
   286  		kvs = kvs.Add("tag\t3", "value", true, false)
   287  
   288  		kvs = kvs.Add("tag=1", "value=1", true, false)
   289  		kvs = kvs.Add("tag 2", "value 2", true, false)
   290  		kvs = kvs.Add("tag\t3", "value \t3", true, false)
   291  		kvs = kvs.Add("tag4", "value \n3", true, false)
   292  		kvs = kvs.Add("tag5", `value \`, true, false)         // tag-value got tail \
   293  		kvs = kvs.Add("tag\nnext-line", `value`, true, false) // tag key get \n
   294  
   295  		pt := NewPointV2("some,=abc\"", kvs)
   296  
   297  		lp := pt.LineProto()
   298  		t.Logf("line-protocol: %s", lp)
   299  		t.Logf("pretty: %s", pt.Pretty())
   300  
   301  		dec := GetDecoder(WithDecEncoding(LineProtocol))
   302  		defer PutDecoder(dec)
   303  		pts, err := dec.Decode([]byte(lp))
   304  		assert.NoError(t, err)
   305  		eq, why := pts[0].EqualWithReason(pt)
   306  		assert.Truef(t, eq, "not equal: %s", why)
   307  	})
   308  }
   309  
   310  func TestPBEncode(t *T.T) {
   311  	t.Run(`invalid-utf8-string-field`, func(t *T.T) {
   312  		var kvs KVs
   313  		invalidUTF8 := "a\xffb\xC0\xAFc\xff"
   314  
   315  		t.Logf("invalidUTF8: %s", invalidUTF8) // the printed invalid-utf8 seems equal to `abc'
   316  
   317  		kvs = kvs.Add("invalid-utf8", invalidUTF8, false, false)
   318  
   319  		pt := NewPointV2("p1", kvs)
   320  
   321  		enc := GetEncoder(WithEncEncoding(Protobuf))
   322  		defer PutEncoder(enc)
   323  		arr, err := enc.Encode([]*Point{pt})
   324  		require.NoError(t, err)
   325  		assert.Len(t, arr, 1)
   326  
   327  		require.Lenf(t, pt.pt.Warns, 0, "point: %s", pt.Pretty())
   328  
   329  		// but the real value get from point is "a\xffb\xC0\xAFc\xff"
   330  		assert.Equal(t, invalidUTF8, pt.Get("invalid-utf8"), "abc")
   331  
   332  		t.Logf("pt: %s", pt.Pretty())
   333  	})
   334  
   335  	t.Run(`invalid-utf8-string-field`, func(t *T.T) {
   336  		var kvs KVs
   337  
   338  		invalidUTF8Str := "a\xffb\xC0\xAFc\xff"
   339  
   340  		validUTF8Str := strings.ToValidUTF8(invalidUTF8Str, "0X")
   341  		kvs = kvs.Add("invalid-utf8", validUTF8Str, false, false)
   342  
   343  		pt := NewPointV2("p1", kvs)
   344  
   345  		enc := GetEncoder(WithEncEncoding(Protobuf))
   346  		defer PutEncoder(enc)
   347  		arr, err := enc.Encode([]*Point{pt})
   348  		require.NoError(t, err)
   349  		assert.Len(t, arr, 1)
   350  		t.Logf("pt: %s", pt.Pretty())
   351  	})
   352  
   353  	t.Run(`invalid-utf8-[]byte-field`, func(t *T.T) {
   354  		var kvs KVs
   355  
   356  		invalidUTF8Bytes := []byte("a\xffb\xC0\xAFc\xff")
   357  
   358  		kvs = kvs.Add("invalid-utf8", invalidUTF8Bytes, false, false)
   359  
   360  		pt := NewPointV2("p1", kvs)
   361  
   362  		enc := GetEncoder(WithEncEncoding(Protobuf))
   363  		defer PutEncoder(enc)
   364  		arr, err := enc.Encode([]*Point{pt})
   365  		require.NoError(t, err)
   366  		assert.Len(t, arr, 1)
   367  
   368  		require.Equal(t, invalidUTF8Bytes, pt.Get("invalid-utf8"))
   369  		t.Logf("pt: %s", pt.Pretty())
   370  	})
   371  }
   372  
   373  func TestEncodeWithBytesLimit(t *T.T) {
   374  	t.Run(`bytes-limite`, func(t *T.T) {
   375  		r := NewRander(WithFixedTags(true), WithRandText(3))
   376  		pts := r.Rand(1000)
   377  
   378  		// add anypb data
   379  		for _, pt := range pts {
   380  			pt.MustAdd("s-arr", []string{"s1", "s2"})
   381  			pt.MustAdd("i-arr", []int{1, 2})
   382  			pt.MustAdd("b-arr", []bool{true, false})
   383  			pt.MustAdd("f-arr", []float64{1.414, 3.14})
   384  		}
   385  
   386  		bytesBatchSize := 128 * 1024
   387  		enc := GetEncoder(WithEncBatchBytes(bytesBatchSize), WithEncFn(func(n int, payload []byte) error {
   388  			t.Logf("points: %d, payload: %dbytes", n, len(payload))
   389  			return nil
   390  		}))
   391  		defer PutEncoder(enc)
   392  
   393  		batches, err := enc.Encode(pts)
   394  		assert.NoError(t, err)
   395  		for idx, b := range batches {
   396  			t.Logf("[%d] batch: %d", idx, len(b))
   397  		}
   398  	})
   399  }
   400  
   401  func TestEncodeTags(t *T.T) {
   402  	t.Run("tag-value-begins-with-slash", func(t *T.T) {
   403  		enc := GetEncoder(WithEncEncoding(LineProtocol))
   404  		defer PutEncoder(enc)
   405  
   406  		arr := func() []*Point {
   407  			x, err := NewPoint("abc", map[string]string{
   408  				"service": "/sf-webproxy/api/online_status",
   409  			}, map[string]interface{}{
   410  				"f3": "fv3",
   411  			}, WithTime(time.Unix(0, 123)))
   412  
   413  			require.NoError(t, err)
   414  
   415  			t.Logf("pt: %s", x.Pretty())
   416  			return []*Point{x}
   417  		}()
   418  
   419  		res, err := enc.Encode(arr)
   420  		assert.NoError(t, err)
   421  		t.Logf("%q", res[0])
   422  
   423  		dec := GetDecoder(WithDecEncoding(LineProtocol))
   424  		defer PutDecoder(dec)
   425  
   426  		pts, err := dec.Decode([]byte(`abc,service=/sf-webproxy/api/online_status f3="fv3" 123`))
   427  		assert.NoError(t, err)
   428  		t.Logf("%s", pts[0].LineProto())
   429  	})
   430  }
   431  
   432  func TestEncodeLen(t *T.T) {
   433  	t.Run("encode-len", func(t *T.T) {
   434  		r := NewRander(WithFixedTags(true), WithRandText(3))
   435  		pts := r.Rand(1000)
   436  
   437  		ptsTotalSize := 0
   438  		for _, pt := range pts {
   439  			ptsTotalSize += pt.Size()
   440  		}
   441  
   442  		enc := GetEncoder()
   443  		defer PutEncoder(enc)
   444  
   445  		data1, err := enc.Encode(pts)
   446  		assert.NoError(t, err)
   447  		assert.Equal(t, 1, len(data1), "encoder: %s", enc.String())
   448  
   449  		gzData1 := cliutils.MustGZip(data1[0])
   450  		t.Logf("lp data: %d bytes, gz: %d, pts size: %d", len(data1[0]), len(gzData1), ptsTotalSize)
   451  
   452  		// setup pb
   453  		WithEncEncoding(Protobuf)(enc)
   454  
   455  		data2, err := enc.Encode(pts)
   456  		assert.NoError(t, err)
   457  		assert.Equal(t, 1, len(data2))
   458  
   459  		gzData2 := cliutils.MustGZip(data2[0])
   460  		t.Logf("pb data: %d bytes, gz: %d, pts size: %d", len(data2[0]), len(gzData2), ptsTotalSize)
   461  
   462  		t.Logf("ratio: %f, gz ration: %f",
   463  			100*float64(len(data2[0]))/float64(len(data1[0])),
   464  			100*float64(len(gzData2))/float64(len(gzData1)))
   465  	})
   466  }
   467  
   468  func BenchmarkEncode(b *T.B) {
   469  	r := NewRander(WithFixedTags(true), WithRandText(3))
   470  	pts := r.Rand(1000)
   471  
   472  	buf := make([]byte, 1<<20)
   473  
   474  	b.ResetTimer()
   475  
   476  	b.Run("bench-encode-json", func(b *T.B) {
   477  		for i := 0; i < b.N; i++ {
   478  			enc := GetEncoder(WithEncEncoding(JSON))
   479  			enc.Encode(pts)
   480  			PutEncoder(enc)
   481  		}
   482  	})
   483  
   484  	b.Run("bench-encode-lp", func(b *T.B) {
   485  		for i := 0; i < b.N; i++ {
   486  			enc := GetEncoder()
   487  			enc.Encode(pts)
   488  			PutEncoder(enc)
   489  		}
   490  	})
   491  
   492  	b.Run("bench-encode-pb", func(b *T.B) {
   493  		for i := 0; i < b.N; i++ {
   494  			enc := GetEncoder(WithEncEncoding(Protobuf), WithEncBatchBytes(1<<20))
   495  			enc.Encode(pts)
   496  			PutEncoder(enc)
   497  		}
   498  	})
   499  
   500  	b.Run("v2-encode-pb", func(b *T.B) {
   501  		for i := 0; i < b.N; i++ {
   502  			enc := GetEncoder(WithEncEncoding(Protobuf))
   503  			enc.EncodeV2(pts)
   504  
   505  			for {
   506  				if _, ok := enc.Next(buf); ok {
   507  				} else {
   508  					break
   509  				}
   510  			}
   511  
   512  			PutEncoder(enc)
   513  		}
   514  	})
   515  
   516  	b.Run("v2-encode-lp", func(b *T.B) {
   517  		for i := 0; i < b.N; i++ {
   518  			enc := GetEncoder(WithEncEncoding(LineProtocol))
   519  			enc.EncodeV2(pts)
   520  
   521  			for {
   522  				if _, ok := enc.Next(buf); ok {
   523  				} else {
   524  					break
   525  				}
   526  			}
   527  
   528  			PutEncoder(enc)
   529  		}
   530  	})
   531  }
   532  
   533  func TestGoGoPBDecodePB(t *T.T) {
   534  	r := NewRander(WithFixedTags(true), WithRandText(3))
   535  	pts := r.Rand(3)
   536  
   537  	enc := GetEncoder(WithEncEncoding(Protobuf))
   538  	defer PutEncoder(enc)
   539  
   540  	arr, err := enc.Encode(pts)
   541  	assert.NoError(t, err)
   542  
   543  	var gogopts gogopb.PBPoints
   544  	assert.NoError(t, gogopts.Unmarshal(arr[0]))
   545  
   546  	j, err := json.MarshalIndent(gogopts, "", "  ")
   547  	assert.NoError(t, err)
   548  
   549  	t.Logf("gogopts:\n%s", string(j))
   550  }
   551  
   552  func BenchmarkV2Encode(b *T.B) {
   553  	r := NewRander(WithFixedTags(true), WithRandText(3))
   554  	randPts := r.Rand(10000)
   555  
   556  	buf := make([]byte, 1<<20)
   557  
   558  	b.Logf("start...")
   559  
   560  	b.ResetTimer()
   561  
   562  	b.Run("encode-v1", func(b *T.B) {
   563  		for i := 0; i < b.N; i++ {
   564  			enc := GetEncoder(WithEncEncoding(Protobuf))
   565  			enc.Encode(randPts)
   566  
   567  			assert.NoError(b, enc.LastErr())
   568  			PutEncoder(enc)
   569  		}
   570  	})
   571  
   572  	b.Run("Next", func(b *T.B) {
   573  		for i := 0; i < b.N; i++ {
   574  			enc := GetEncoder(WithEncEncoding(Protobuf))
   575  			enc.EncodeV2(randPts)
   576  
   577  			for {
   578  				if _, ok := enc.Next(buf); ok {
   579  					buf = buf[0:]
   580  				} else {
   581  					break
   582  				}
   583  			}
   584  
   585  			assert.NoError(b, enc.LastErr())
   586  			PutEncoder(enc)
   587  		}
   588  	})
   589  }
   590  
   591  func TestV2Encode(t *T.T) {
   592  	r := NewRander(WithFixedTags(true), WithRandText(3))
   593  	randPts := r.Rand(10000)
   594  
   595  	t.Run("encode-pb", func(t *T.T) {
   596  		enc := GetEncoder(WithEncEncoding(Protobuf))
   597  		enc.EncodeV2(randPts)
   598  		defer PutEncoder(enc)
   599  
   600  		dec := GetDecoder(WithDecEncoding(Protobuf))
   601  		defer PutDecoder(dec)
   602  
   603  		var (
   604  			decodePts []*Point
   605  			round     int
   606  			buf       = make([]byte, 1<<20) // KB
   607  		)
   608  
   609  		for {
   610  			if x, ok := enc.Next(buf); ok {
   611  				decPts, err := dec.Decode(x)
   612  				assert.NoErrorf(t, err, "decode %s failed", x)
   613  
   614  				t.Logf("encoded %d(%d remain) bytes, %d points, encoder: %s",
   615  					len(x), (len(buf) - len(x)), len(decPts), enc.String())
   616  				decodePts = append(decodePts, decPts...)
   617  				round++
   618  				assert.Equal(t, round, enc.parts)
   619  			} else {
   620  				break
   621  			}
   622  		}
   623  
   624  		assert.NoError(t, enc.LastErr())
   625  
   626  		for i, pt := range decodePts {
   627  			assert.Equal(t, randPts[i].Pretty(), pt.Pretty())
   628  		}
   629  	})
   630  
   631  	t.Run("encode-lp", func(t *T.T) {
   632  		enc := GetEncoder(WithEncEncoding(LineProtocol))
   633  		enc.EncodeV2(randPts)
   634  		defer PutEncoder(enc)
   635  
   636  		dec := GetDecoder(WithDecEncoding(LineProtocol))
   637  		defer PutDecoder(dec)
   638  
   639  		var (
   640  			decodePts []*Point
   641  			round     int
   642  			buf       = make([]byte, 1<<20)
   643  		)
   644  
   645  		for {
   646  			if x, ok := enc.Next(buf); ok {
   647  				decPts, err := dec.Decode(x)
   648  				assert.NoErrorf(t, err, "decode %s failed", x)
   649  
   650  				t.Logf("encoded %d(%d remain) bytes, %d points, encoder: %s",
   651  					len(x), (len(buf) - len(x)), len(decPts), enc.String())
   652  
   653  				decodePts = append(decodePts, decPts...)
   654  				round++
   655  				assert.Equal(t, round, enc.parts)
   656  			} else {
   657  				break
   658  			}
   659  		}
   660  
   661  		assert.NoError(t, enc.LastErr())
   662  
   663  		for i, pt := range decodePts {
   664  			ok, why := randPts[i].EqualWithReason(pt)
   665  			require.Truef(t, ok, "reason: %s", why)
   666  		}
   667  	})
   668  
   669  	t.Run("too-small-buffer-lp", func(t *T.T) {
   670  		enc := GetEncoder(WithEncEncoding(LineProtocol))
   671  		enc.EncodeV2(randPts)
   672  		defer PutEncoder(enc)
   673  
   674  		buf := make([]byte, 4) // too small
   675  
   676  		for {
   677  			_, ok := enc.Next(buf)
   678  			require.False(t, ok)
   679  			break
   680  		}
   681  
   682  		assert.Error(t, enc.LastErr())
   683  		t.Logf("go error: %s", enc.LastErr())
   684  	})
   685  
   686  	t.Run("too-small-buffer-pb", func(t *T.T) {
   687  		enc := GetEncoder(WithEncEncoding(Protobuf))
   688  		enc.EncodeV2(randPts)
   689  		defer PutEncoder(enc)
   690  
   691  		buf := make([]byte, 4) // too small
   692  
   693  		for {
   694  			_, ok := enc.Next(buf)
   695  			require.False(t, ok)
   696  			break
   697  		}
   698  
   699  		assert.Error(t, enc.LastErr())
   700  		t.Logf("go error: %s", enc.LastErr())
   701  	})
   702  
   703  	t.Run("with-encode-callback-line-proto", func(t *T.T) {
   704  		fn := func(n int, buf []byte) error {
   705  			assert.Equal(t, 2, n)
   706  			assert.True(t, len(buf) > 0)
   707  
   708  			t.Logf("buf: %q", buf)
   709  			return nil
   710  		}
   711  
   712  		buf := make([]byte, 1<<20)
   713  		randPts := r.Rand(2)
   714  		enc := GetEncoder(WithEncFn(fn), WithEncEncoding(LineProtocol))
   715  		enc.EncodeV2(randPts)
   716  		for {
   717  			if _, ok := enc.Next(buf); !ok {
   718  				break
   719  			}
   720  		}
   721  		PutEncoder(enc)
   722  	})
   723  
   724  	t.Run("with-encode-callback-protobuf", func(t *T.T) {
   725  		fn := func(n int, buf []byte) error {
   726  			assert.Equal(t, 2, n)
   727  			assert.NotNil(t, buf)
   728  
   729  			t.Logf("buf: %q", buf)
   730  			return nil
   731  		}
   732  
   733  		randPts := r.Rand(2)
   734  
   735  		enc := GetEncoder(WithEncFn(fn), WithEncEncoding(Protobuf))
   736  		enc.EncodeV2(randPts)
   737  		buf := make([]byte, 1<<20)
   738  
   739  		for {
   740  			if _, ok := enc.Next(buf); !ok {
   741  				break
   742  			}
   743  		}
   744  		PutEncoder(enc)
   745  	})
   746  }
   747  
   748  func BenchmarkPointsSize(b *T.B) {
   749  	r := NewRander(WithFixedTags(true), WithRandText(3))
   750  	randPts := r.Rand(1)
   751  
   752  	b.ResetTimer()
   753  	b.Run("pt.pt.size", func(b *T.B) {
   754  		for i := 0; i < b.N; i++ {
   755  			randPts[0].pt.Size()
   756  		}
   757  	})
   758  
   759  	b.Run("pt.size", func(b *T.B) {
   760  		for i := 0; i < b.N; i++ {
   761  			randPts[0].Size()
   762  		}
   763  	})
   764  }
   765  
   766  func TestPointsSize(t *T.T) {
   767  	r := NewRander(WithFixedTags(true), WithRandText(3))
   768  	randPts := r.Rand(1)
   769  
   770  	t.Logf("pt.pt.size: %d, pt.size: %d",
   771  		randPts[0].pt.Size(),
   772  		randPts[0].Size())
   773  }
   774  
   775  func TestEncodePayloadSize(t *T.T) {
   776  	r := NewRander(WithFixedTags(true), WithRandText(3))
   777  	randPts := r.Rand(1000)
   778  
   779  	enc := GetEncoder(WithEncEncoding(Protobuf))
   780  	arr, err := enc.Encode(randPts)
   781  	assert.NoError(t, err)
   782  	assert.Len(t, arr, 1)
   783  
   784  	pbPayload := arr[0]
   785  	PutEncoder(enc)
   786  
   787  	enc = GetEncoder(WithEncEncoding(LineProtocol))
   788  	arr, err = enc.Encode(randPts)
   789  	assert.NoError(t, err)
   790  	assert.Len(t, arr, 1)
   791  	lpPayload := arr[0]
   792  	PutEncoder(enc)
   793  
   794  	// gzip compression
   795  	gzSize := func(payload []byte) int {
   796  		var buf bytes.Buffer
   797  		gw := gzip.NewWriter(&buf)
   798  		_, err := gw.Write(payload)
   799  		assert.NoError(t, err)
   800  		assert.NoError(t, gw.Close())
   801  		return buf.Len()
   802  	}
   803  
   804  	t.Logf("pbsize: %d, lpsize: %d, gz pb: %d, gz lp: %d",
   805  		len(pbPayload), len(lpPayload), gzSize(pbPayload), gzSize(lpPayload))
   806  }
   807  
   808  func TestEncodeInfField(t *T.T) {
   809  	var kvs KVs
   810  	kvs = kvs.AddV2("f1", math.Inf(1), true)
   811  	kvs = kvs.AddV2("f2", math.Inf(-1), true)
   812  	kvs = kvs.AddV2("f3", 123, true)
   813  
   814  	pt := NewPointV2("some", kvs)
   815  
   816  	t.Logf("point: %s", pt.Pretty())
   817  
   818  	t.Run("inf-lineproto", func(t *T.T) {
   819  		enc := GetEncoder(WithEncEncoding(LineProtocol))
   820  		defer PutEncoder(enc)
   821  
   822  		enc.EncodeV2([]*Point{pt})
   823  
   824  		buf := make([]byte, 1<<20)
   825  
   826  		for {
   827  			if res, ok := enc.Next(buf); ok {
   828  				t.Logf("res: %s", string(res))
   829  			} else {
   830  				break
   831  			}
   832  		}
   833  
   834  		assert.NoError(t, enc.LastErr())
   835  	})
   836  
   837  	t.Run("inf-protobuf", func(t *T.T) {
   838  		enc := GetEncoder(WithEncEncoding(Protobuf))
   839  		defer PutEncoder(enc)
   840  
   841  		enc.EncodeV2([]*Point{pt})
   842  
   843  		buf := make([]byte, 1<<20)
   844  
   845  		for {
   846  			if res, ok := enc.Next(buf); ok {
   847  				dec := GetDecoder(WithDecEncoding(Protobuf))
   848  				defer PutDecoder(dec)
   849  
   850  				pts, err := dec.Decode(res)
   851  				assert.NoError(t, err)
   852  
   853  				t.Logf("decode point: %s", pts[0].Pretty())
   854  
   855  				assert.Equal(t, uint64(math.MaxUint64), uint64(pts[0].Get("f1").(float64)))
   856  				assert.Equal(t, int64(math.MinInt64), int64(pts[0].Get("f2").(float64)))
   857  				assert.Equal(t, int64(123), pts[0].Get("f3"))
   858  			} else {
   859  				break
   860  			}
   861  		}
   862  
   863  		assert.NoError(t, enc.LastErr())
   864  	})
   865  
   866  	t.Run("inf-pbjson", func(t *T.T) {
   867  		enc := GetEncoder(WithEncEncoding(JSON))
   868  		defer PutEncoder(enc)
   869  
   870  		pt.SetFlag(Ppb)
   871  
   872  		arr, err := enc.Encode([]*Point{pt})
   873  		assert.NoError(t, err)
   874  
   875  		t.Logf("res: %s", string(arr[0]))
   876  	})
   877  }