github.com/GuanceCloud/cliutils@v1.1.21/point/encode_test.go (about) 1 // Unless explicitly stated otherwise all files in this repository are licensed 2 // under the MIT License. 3 // This product includes software developed at Guance Cloud (https://www.guance.com/). 4 // Copyright 2021-present Guance, Inc. 5 6 package point 7 8 import ( 9 "bytes" 10 "compress/gzip" 11 "encoding/json" 12 "math" 13 "strings" 14 T "testing" 15 "time" 16 17 "github.com/GuanceCloud/cliutils" 18 "github.com/GuanceCloud/cliutils/point/gogopb" 19 proto "github.com/gogo/protobuf/proto" 20 "github.com/stretchr/testify/assert" 21 "github.com/stretchr/testify/require" 22 ) 23 24 // Test if encode change points' payload 25 func TestIdempotent(t *T.T) { 26 cases := []struct { 27 name string 28 pts []*Point 29 batch int 30 }{ 31 { 32 name: "ok-32#3", 33 pts: RandPoints(32), 34 batch: 3, 35 }, 36 37 { 38 name: "ok-32#1", 39 pts: RandPoints(32), 40 batch: 1, 41 }, 42 43 { 44 name: "ok-32#0", 45 pts: RandPoints(32), 46 batch: 0, 47 }, 48 49 { 50 name: "nothing", 51 pts: RandPoints(0), 52 batch: 0, 53 }, 54 } 55 56 for _, tc := range cases { 57 t.Run(tc.name, func(t *T.T) { 58 enc := GetEncoder(WithEncBatchSize(tc.batch)) 59 60 p1, err := enc.Encode(tc.pts) 61 assert.NoError(t, err) 62 PutEncoder(enc) 63 64 enc = GetEncoder(WithEncBatchSize(tc.batch)) 65 p2, err := enc.Encode(tc.pts) 66 assert.NoError(t, err) 67 PutEncoder(enc) 68 69 assert.Equal(t, p1, p2) 70 }) 71 72 // test encode pb 73 t.Run(tc.name+"-pb", func(t *T.T) { 74 enc := GetEncoder(WithEncBatchSize(tc.batch), WithEncEncoding(Protobuf)) 75 76 p1, err := enc.Encode(tc.pts) 77 assert.NoError(t, err) 78 PutEncoder(enc) 79 80 enc = GetEncoder(WithEncBatchSize(tc.batch), WithEncEncoding(Protobuf)) 81 p2, err := enc.Encode(tc.pts) 82 assert.NoError(t, err) 83 PutEncoder(enc) 84 85 assert.Equal(t, p1, p2) 86 }) 87 } 88 } 89 90 // TestEncodeEqualty test equality on 91 // - multi-part encode: multiple points splited into multiple parts 92 // - line-protocol/protobuf: points encode between line-protocol and protobuf are equal 93 func TestEncodeEqualty(t *T.T) { 94 r := NewRander(WithKVSorted(true), WithRandFields(1), WithRandTags(1)) 95 96 nrand := 6 97 randBsize := 3 98 99 var ( 100 randPts = r.Rand(nrand) 101 102 simplePts = []*Point{ 103 NewPointV2(`abc`, NewKVs(map[string]interface{}{"f1": "fv1", "f2": "fv2", "f3": "fv3"}). 104 AddTag(`t1`, `tv1`). 105 AddTag(`t2`, `tv2`). 106 AddTag(`t3`, `tv3`), WithTime(time.Unix(0, 123))), 107 108 NewPointV2(`def`, NewKVs(map[string]interface{}{"f1": "fv1", "f2": "fv2", "f3": "fv3"}). 109 AddTag(`t1`, `tv1`). 110 AddTag(`t2`, `tv2`). 111 AddTag(`t3`, `tv3`), WithTime(time.Unix(0, 123))), 112 113 NewPointV2(`xyz`, NewKVs(map[string]interface{}{"f1": "fv1", "f2": "fv2", "f3": "fv3"}). 114 AddTag(`t1`, `tv1`). 115 AddTag(`t2`, `tv2`). 116 AddTag(`t3`, `tv3`), WithTime(time.Unix(0, 123))), 117 } 118 119 __fn = func(n int, data []byte) error { 120 t.Logf("batch size: %d, payload: %d", n, len(data)) 121 return nil 122 } 123 ) 124 125 cases := []struct { 126 name string 127 pts []*Point 128 bsize int 129 fn EncodeFn 130 gz bool 131 expect [][]byte 132 }{ 133 { 134 name: "single-point", 135 bsize: 10, 136 expect: [][]byte{ 137 []byte(`abc,t1=tv1,t2=tv2,t3=tv3 f1="fv1",f2="fv2",f3="fv3" 123`), 138 }, 139 140 pts: func() []*Point { 141 x, err := NewPoint("abc", map[string]string{ 142 "t1": "tv1", 143 "t2": "tv2", 144 "t3": "tv3", 145 }, map[string]interface{}{ 146 "f1": "fv1", 147 "f2": "fv2", 148 "f3": "fv3", 149 }, WithTime(time.Unix(0, 123)), WithKeySorted(true)) 150 151 require.NoError(t, err) 152 153 t.Logf("pt: %s", x.Pretty()) 154 return []*Point{x} 155 }(), 156 }, 157 158 { 159 name: "random-point", 160 bsize: randBsize, 161 pts: randPts, 162 expect: func() [][]byte { 163 enc := GetEncoder(WithEncBatchSize(randBsize)) 164 defer PutEncoder(enc) 165 166 x, err := enc.Encode(randPts) 167 assert.NoError(t, err) 168 return x 169 }(), 170 }, 171 172 { 173 name: "random-point-with-callback", 174 bsize: randBsize, 175 pts: randPts, 176 fn: __fn, 177 expect: func() [][]byte { 178 enc := GetEncoder(WithEncBatchSize(randBsize)) 179 defer PutEncoder(enc) 180 181 bufs, err := enc.Encode(randPts) 182 assert.NoError(t, err) 183 184 if len(randPts)%randBsize == 0 { 185 assert.Equal(t, len(randPts)/randBsize, len(bufs)) 186 } else { 187 assert.Equal(t, len(randPts)/randBsize+1, len(bufs), "randPts: %d", len(randPts)) 188 } 189 190 for i, buf := range bufs { 191 t.Logf("get %dth batch:\n%s", i, buf) 192 if i != len(bufs)-1 { 193 assert.Equal(t, randBsize, len(bytes.Split(buf, []byte("\n")))) 194 } 195 } 196 197 return bufs 198 }(), 199 }, 200 201 { 202 name: "simple-point-with-callback", 203 bsize: 1, 204 pts: simplePts, 205 fn: __fn, 206 expect: func() [][]byte { 207 enc := GetEncoder(WithEncBatchSize(1)) 208 defer PutEncoder(enc) 209 210 bufs, err := enc.Encode(simplePts) 211 assert.NoError(t, err) 212 213 assert.Equal(t, len(simplePts), len(bufs)) 214 215 for i, buf := range bufs { 216 t.Logf("get %dth batch:\n%s", i, buf) 217 assert.Equal(t, 1, len(bytes.Split(buf, []byte("\n")))) 218 } 219 220 return bufs 221 }(), 222 // expect: simplePtsExpect, 223 }, 224 } 225 226 for _, tc := range cases { 227 t.Run(tc.name, func(t *T.T) { 228 enc := GetEncoder(WithEncBatchSize(tc.bsize), WithEncFn(tc.fn)) 229 defer PutEncoder(enc) 230 231 payloads, err := enc.Encode(tc.pts) 232 assert.NoError(t, err) 233 234 assert.Equal(t, len(tc.expect), len(payloads)) 235 236 for idx, ex := range tc.expect { 237 assert.Equal(t, len(ex), len(payloads[idx]), "not equal at index: %d, gz: %q, fn: %q", idx, tc.gz, tc.fn) 238 assert.Equal(t, ex, payloads[idx], "[%d] expect %s, get %s", idx, string(ex), string(payloads[idx])) 239 } 240 }) 241 242 t.Run(tc.name+"-pb", func(t *T.T) { 243 enc := GetEncoder(WithEncBatchSize(tc.bsize), 244 WithEncFn(tc.fn), 245 WithEncEncoding(Protobuf)) 246 defer PutEncoder(enc) 247 248 assert.Len(t, enc.pbpts.Arr, 0) 249 250 payloads, err := enc.Encode(tc.pts) 251 252 assert.NoError(t, err) 253 254 assert.Equal(t, len(tc.expect), len(payloads)) 255 256 // check PB unmarshal 257 for idx := range tc.expect { 258 var pbpts PBPoints 259 assert.NoError(t, proto.Unmarshal(payloads[idx], &pbpts)) 260 } 261 262 // convert PB to line-protocol, check equality 263 for idx, p := range payloads { 264 lp, err := PB2LP(p) 265 assert.NoError(t, err) 266 t.Logf("pb -> lp:\n%s", lp) 267 268 assert.Equal(t, string(tc.expect[idx]), string(lp)) 269 } 270 }) 271 } 272 } 273 274 func TestEscapeEncode(t *T.T) { 275 t.Run("escaped-lineproto", func(t *T.T) { 276 var kvs KVs 277 kvs = kvs.Add("f1=2=3=", 3.14, false, false) 278 kvs = kvs.Add("f2\tnr", 2, false, false) 279 kvs = kvs.Add("f3,", "some-string\nanother-line", false, false) 280 kvs = kvs.Add("f4,", false, false, false) 281 282 kvs = kvs.Add("f\nnext-line,", []byte("hello"), false, false) 283 kvs = kvs.Add(`f\other`, []byte("hello"), false, false) 284 kvs = kvs.Add("tag=1", "value", true, false) 285 kvs = kvs.Add("tag 2", "value", true, false) 286 kvs = kvs.Add("tag\t3", "value", true, false) 287 288 kvs = kvs.Add("tag=1", "value=1", true, false) 289 kvs = kvs.Add("tag 2", "value 2", true, false) 290 kvs = kvs.Add("tag\t3", "value \t3", true, false) 291 kvs = kvs.Add("tag4", "value \n3", true, false) 292 kvs = kvs.Add("tag5", `value \`, true, false) // tag-value got tail \ 293 kvs = kvs.Add("tag\nnext-line", `value`, true, false) // tag key get \n 294 295 pt := NewPointV2("some,=abc\"", kvs) 296 297 lp := pt.LineProto() 298 t.Logf("line-protocol: %s", lp) 299 t.Logf("pretty: %s", pt.Pretty()) 300 301 dec := GetDecoder(WithDecEncoding(LineProtocol)) 302 defer PutDecoder(dec) 303 pts, err := dec.Decode([]byte(lp)) 304 assert.NoError(t, err) 305 eq, why := pts[0].EqualWithReason(pt) 306 assert.Truef(t, eq, "not equal: %s", why) 307 }) 308 } 309 310 func TestPBEncode(t *T.T) { 311 t.Run(`invalid-utf8-string-field`, func(t *T.T) { 312 var kvs KVs 313 invalidUTF8 := "a\xffb\xC0\xAFc\xff" 314 315 t.Logf("invalidUTF8: %s", invalidUTF8) // the printed invalid-utf8 seems equal to `abc' 316 317 kvs = kvs.Add("invalid-utf8", invalidUTF8, false, false) 318 319 pt := NewPointV2("p1", kvs) 320 321 enc := GetEncoder(WithEncEncoding(Protobuf)) 322 defer PutEncoder(enc) 323 arr, err := enc.Encode([]*Point{pt}) 324 require.NoError(t, err) 325 assert.Len(t, arr, 1) 326 327 require.Lenf(t, pt.pt.Warns, 0, "point: %s", pt.Pretty()) 328 329 // but the real value get from point is "a\xffb\xC0\xAFc\xff" 330 assert.Equal(t, invalidUTF8, pt.Get("invalid-utf8"), "abc") 331 332 t.Logf("pt: %s", pt.Pretty()) 333 }) 334 335 t.Run(`invalid-utf8-string-field`, func(t *T.T) { 336 var kvs KVs 337 338 invalidUTF8Str := "a\xffb\xC0\xAFc\xff" 339 340 validUTF8Str := strings.ToValidUTF8(invalidUTF8Str, "0X") 341 kvs = kvs.Add("invalid-utf8", validUTF8Str, false, false) 342 343 pt := NewPointV2("p1", kvs) 344 345 enc := GetEncoder(WithEncEncoding(Protobuf)) 346 defer PutEncoder(enc) 347 arr, err := enc.Encode([]*Point{pt}) 348 require.NoError(t, err) 349 assert.Len(t, arr, 1) 350 t.Logf("pt: %s", pt.Pretty()) 351 }) 352 353 t.Run(`invalid-utf8-[]byte-field`, func(t *T.T) { 354 var kvs KVs 355 356 invalidUTF8Bytes := []byte("a\xffb\xC0\xAFc\xff") 357 358 kvs = kvs.Add("invalid-utf8", invalidUTF8Bytes, false, false) 359 360 pt := NewPointV2("p1", kvs) 361 362 enc := GetEncoder(WithEncEncoding(Protobuf)) 363 defer PutEncoder(enc) 364 arr, err := enc.Encode([]*Point{pt}) 365 require.NoError(t, err) 366 assert.Len(t, arr, 1) 367 368 require.Equal(t, invalidUTF8Bytes, pt.Get("invalid-utf8")) 369 t.Logf("pt: %s", pt.Pretty()) 370 }) 371 } 372 373 func TestEncodeWithBytesLimit(t *T.T) { 374 t.Run(`bytes-limite`, func(t *T.T) { 375 r := NewRander(WithFixedTags(true), WithRandText(3)) 376 pts := r.Rand(1000) 377 378 // add anypb data 379 for _, pt := range pts { 380 pt.MustAdd("s-arr", []string{"s1", "s2"}) 381 pt.MustAdd("i-arr", []int{1, 2}) 382 pt.MustAdd("b-arr", []bool{true, false}) 383 pt.MustAdd("f-arr", []float64{1.414, 3.14}) 384 } 385 386 bytesBatchSize := 128 * 1024 387 enc := GetEncoder(WithEncBatchBytes(bytesBatchSize), WithEncFn(func(n int, payload []byte) error { 388 t.Logf("points: %d, payload: %dbytes", n, len(payload)) 389 return nil 390 })) 391 defer PutEncoder(enc) 392 393 batches, err := enc.Encode(pts) 394 assert.NoError(t, err) 395 for idx, b := range batches { 396 t.Logf("[%d] batch: %d", idx, len(b)) 397 } 398 }) 399 } 400 401 func TestEncodeTags(t *T.T) { 402 t.Run("tag-value-begins-with-slash", func(t *T.T) { 403 enc := GetEncoder(WithEncEncoding(LineProtocol)) 404 defer PutEncoder(enc) 405 406 arr := func() []*Point { 407 x, err := NewPoint("abc", map[string]string{ 408 "service": "/sf-webproxy/api/online_status", 409 }, map[string]interface{}{ 410 "f3": "fv3", 411 }, WithTime(time.Unix(0, 123))) 412 413 require.NoError(t, err) 414 415 t.Logf("pt: %s", x.Pretty()) 416 return []*Point{x} 417 }() 418 419 res, err := enc.Encode(arr) 420 assert.NoError(t, err) 421 t.Logf("%q", res[0]) 422 423 dec := GetDecoder(WithDecEncoding(LineProtocol)) 424 defer PutDecoder(dec) 425 426 pts, err := dec.Decode([]byte(`abc,service=/sf-webproxy/api/online_status f3="fv3" 123`)) 427 assert.NoError(t, err) 428 t.Logf("%s", pts[0].LineProto()) 429 }) 430 } 431 432 func TestEncodeLen(t *T.T) { 433 t.Run("encode-len", func(t *T.T) { 434 r := NewRander(WithFixedTags(true), WithRandText(3)) 435 pts := r.Rand(1000) 436 437 ptsTotalSize := 0 438 for _, pt := range pts { 439 ptsTotalSize += pt.Size() 440 } 441 442 enc := GetEncoder() 443 defer PutEncoder(enc) 444 445 data1, err := enc.Encode(pts) 446 assert.NoError(t, err) 447 assert.Equal(t, 1, len(data1), "encoder: %s", enc.String()) 448 449 gzData1 := cliutils.MustGZip(data1[0]) 450 t.Logf("lp data: %d bytes, gz: %d, pts size: %d", len(data1[0]), len(gzData1), ptsTotalSize) 451 452 // setup pb 453 WithEncEncoding(Protobuf)(enc) 454 455 data2, err := enc.Encode(pts) 456 assert.NoError(t, err) 457 assert.Equal(t, 1, len(data2)) 458 459 gzData2 := cliutils.MustGZip(data2[0]) 460 t.Logf("pb data: %d bytes, gz: %d, pts size: %d", len(data2[0]), len(gzData2), ptsTotalSize) 461 462 t.Logf("ratio: %f, gz ration: %f", 463 100*float64(len(data2[0]))/float64(len(data1[0])), 464 100*float64(len(gzData2))/float64(len(gzData1))) 465 }) 466 } 467 468 func BenchmarkEncode(b *T.B) { 469 r := NewRander(WithFixedTags(true), WithRandText(3)) 470 pts := r.Rand(1000) 471 472 buf := make([]byte, 1<<20) 473 474 b.ResetTimer() 475 476 b.Run("bench-encode-json", func(b *T.B) { 477 for i := 0; i < b.N; i++ { 478 enc := GetEncoder(WithEncEncoding(JSON)) 479 enc.Encode(pts) 480 PutEncoder(enc) 481 } 482 }) 483 484 b.Run("bench-encode-lp", func(b *T.B) { 485 for i := 0; i < b.N; i++ { 486 enc := GetEncoder() 487 enc.Encode(pts) 488 PutEncoder(enc) 489 } 490 }) 491 492 b.Run("bench-encode-pb", func(b *T.B) { 493 for i := 0; i < b.N; i++ { 494 enc := GetEncoder(WithEncEncoding(Protobuf), WithEncBatchBytes(1<<20)) 495 enc.Encode(pts) 496 PutEncoder(enc) 497 } 498 }) 499 500 b.Run("v2-encode-pb", func(b *T.B) { 501 for i := 0; i < b.N; i++ { 502 enc := GetEncoder(WithEncEncoding(Protobuf)) 503 enc.EncodeV2(pts) 504 505 for { 506 if _, ok := enc.Next(buf); ok { 507 } else { 508 break 509 } 510 } 511 512 PutEncoder(enc) 513 } 514 }) 515 516 b.Run("v2-encode-lp", func(b *T.B) { 517 for i := 0; i < b.N; i++ { 518 enc := GetEncoder(WithEncEncoding(LineProtocol)) 519 enc.EncodeV2(pts) 520 521 for { 522 if _, ok := enc.Next(buf); ok { 523 } else { 524 break 525 } 526 } 527 528 PutEncoder(enc) 529 } 530 }) 531 } 532 533 func TestGoGoPBDecodePB(t *T.T) { 534 r := NewRander(WithFixedTags(true), WithRandText(3)) 535 pts := r.Rand(3) 536 537 enc := GetEncoder(WithEncEncoding(Protobuf)) 538 defer PutEncoder(enc) 539 540 arr, err := enc.Encode(pts) 541 assert.NoError(t, err) 542 543 var gogopts gogopb.PBPoints 544 assert.NoError(t, gogopts.Unmarshal(arr[0])) 545 546 j, err := json.MarshalIndent(gogopts, "", " ") 547 assert.NoError(t, err) 548 549 t.Logf("gogopts:\n%s", string(j)) 550 } 551 552 func BenchmarkV2Encode(b *T.B) { 553 r := NewRander(WithFixedTags(true), WithRandText(3)) 554 randPts := r.Rand(10000) 555 556 buf := make([]byte, 1<<20) 557 558 b.Logf("start...") 559 560 b.ResetTimer() 561 562 b.Run("encode-v1", func(b *T.B) { 563 for i := 0; i < b.N; i++ { 564 enc := GetEncoder(WithEncEncoding(Protobuf)) 565 enc.Encode(randPts) 566 567 assert.NoError(b, enc.LastErr()) 568 PutEncoder(enc) 569 } 570 }) 571 572 b.Run("Next", func(b *T.B) { 573 for i := 0; i < b.N; i++ { 574 enc := GetEncoder(WithEncEncoding(Protobuf)) 575 enc.EncodeV2(randPts) 576 577 for { 578 if _, ok := enc.Next(buf); ok { 579 buf = buf[0:] 580 } else { 581 break 582 } 583 } 584 585 assert.NoError(b, enc.LastErr()) 586 PutEncoder(enc) 587 } 588 }) 589 } 590 591 func TestV2Encode(t *T.T) { 592 r := NewRander(WithFixedTags(true), WithRandText(3)) 593 randPts := r.Rand(10000) 594 595 t.Run("encode-pb", func(t *T.T) { 596 enc := GetEncoder(WithEncEncoding(Protobuf)) 597 enc.EncodeV2(randPts) 598 defer PutEncoder(enc) 599 600 dec := GetDecoder(WithDecEncoding(Protobuf)) 601 defer PutDecoder(dec) 602 603 var ( 604 decodePts []*Point 605 round int 606 buf = make([]byte, 1<<20) // KB 607 ) 608 609 for { 610 if x, ok := enc.Next(buf); ok { 611 decPts, err := dec.Decode(x) 612 assert.NoErrorf(t, err, "decode %s failed", x) 613 614 t.Logf("encoded %d(%d remain) bytes, %d points, encoder: %s", 615 len(x), (len(buf) - len(x)), len(decPts), enc.String()) 616 decodePts = append(decodePts, decPts...) 617 round++ 618 assert.Equal(t, round, enc.parts) 619 } else { 620 break 621 } 622 } 623 624 assert.NoError(t, enc.LastErr()) 625 626 for i, pt := range decodePts { 627 assert.Equal(t, randPts[i].Pretty(), pt.Pretty()) 628 } 629 }) 630 631 t.Run("encode-lp", func(t *T.T) { 632 enc := GetEncoder(WithEncEncoding(LineProtocol)) 633 enc.EncodeV2(randPts) 634 defer PutEncoder(enc) 635 636 dec := GetDecoder(WithDecEncoding(LineProtocol)) 637 defer PutDecoder(dec) 638 639 var ( 640 decodePts []*Point 641 round int 642 buf = make([]byte, 1<<20) 643 ) 644 645 for { 646 if x, ok := enc.Next(buf); ok { 647 decPts, err := dec.Decode(x) 648 assert.NoErrorf(t, err, "decode %s failed", x) 649 650 t.Logf("encoded %d(%d remain) bytes, %d points, encoder: %s", 651 len(x), (len(buf) - len(x)), len(decPts), enc.String()) 652 653 decodePts = append(decodePts, decPts...) 654 round++ 655 assert.Equal(t, round, enc.parts) 656 } else { 657 break 658 } 659 } 660 661 assert.NoError(t, enc.LastErr()) 662 663 for i, pt := range decodePts { 664 ok, why := randPts[i].EqualWithReason(pt) 665 require.Truef(t, ok, "reason: %s", why) 666 } 667 }) 668 669 t.Run("too-small-buffer-lp", func(t *T.T) { 670 enc := GetEncoder(WithEncEncoding(LineProtocol)) 671 enc.EncodeV2(randPts) 672 defer PutEncoder(enc) 673 674 buf := make([]byte, 4) // too small 675 676 for { 677 _, ok := enc.Next(buf) 678 require.False(t, ok) 679 break 680 } 681 682 assert.Error(t, enc.LastErr()) 683 t.Logf("go error: %s", enc.LastErr()) 684 }) 685 686 t.Run("too-small-buffer-pb", func(t *T.T) { 687 enc := GetEncoder(WithEncEncoding(Protobuf)) 688 enc.EncodeV2(randPts) 689 defer PutEncoder(enc) 690 691 buf := make([]byte, 4) // too small 692 693 for { 694 _, ok := enc.Next(buf) 695 require.False(t, ok) 696 break 697 } 698 699 assert.Error(t, enc.LastErr()) 700 t.Logf("go error: %s", enc.LastErr()) 701 }) 702 703 t.Run("with-encode-callback-line-proto", func(t *T.T) { 704 fn := func(n int, buf []byte) error { 705 assert.Equal(t, 2, n) 706 assert.True(t, len(buf) > 0) 707 708 t.Logf("buf: %q", buf) 709 return nil 710 } 711 712 buf := make([]byte, 1<<20) 713 randPts := r.Rand(2) 714 enc := GetEncoder(WithEncFn(fn), WithEncEncoding(LineProtocol)) 715 enc.EncodeV2(randPts) 716 for { 717 if _, ok := enc.Next(buf); !ok { 718 break 719 } 720 } 721 PutEncoder(enc) 722 }) 723 724 t.Run("with-encode-callback-protobuf", func(t *T.T) { 725 fn := func(n int, buf []byte) error { 726 assert.Equal(t, 2, n) 727 assert.NotNil(t, buf) 728 729 t.Logf("buf: %q", buf) 730 return nil 731 } 732 733 randPts := r.Rand(2) 734 735 enc := GetEncoder(WithEncFn(fn), WithEncEncoding(Protobuf)) 736 enc.EncodeV2(randPts) 737 buf := make([]byte, 1<<20) 738 739 for { 740 if _, ok := enc.Next(buf); !ok { 741 break 742 } 743 } 744 PutEncoder(enc) 745 }) 746 } 747 748 func BenchmarkPointsSize(b *T.B) { 749 r := NewRander(WithFixedTags(true), WithRandText(3)) 750 randPts := r.Rand(1) 751 752 b.ResetTimer() 753 b.Run("pt.pt.size", func(b *T.B) { 754 for i := 0; i < b.N; i++ { 755 randPts[0].pt.Size() 756 } 757 }) 758 759 b.Run("pt.size", func(b *T.B) { 760 for i := 0; i < b.N; i++ { 761 randPts[0].Size() 762 } 763 }) 764 } 765 766 func TestPointsSize(t *T.T) { 767 r := NewRander(WithFixedTags(true), WithRandText(3)) 768 randPts := r.Rand(1) 769 770 t.Logf("pt.pt.size: %d, pt.size: %d", 771 randPts[0].pt.Size(), 772 randPts[0].Size()) 773 } 774 775 func TestEncodePayloadSize(t *T.T) { 776 r := NewRander(WithFixedTags(true), WithRandText(3)) 777 randPts := r.Rand(1000) 778 779 enc := GetEncoder(WithEncEncoding(Protobuf)) 780 arr, err := enc.Encode(randPts) 781 assert.NoError(t, err) 782 assert.Len(t, arr, 1) 783 784 pbPayload := arr[0] 785 PutEncoder(enc) 786 787 enc = GetEncoder(WithEncEncoding(LineProtocol)) 788 arr, err = enc.Encode(randPts) 789 assert.NoError(t, err) 790 assert.Len(t, arr, 1) 791 lpPayload := arr[0] 792 PutEncoder(enc) 793 794 // gzip compression 795 gzSize := func(payload []byte) int { 796 var buf bytes.Buffer 797 gw := gzip.NewWriter(&buf) 798 _, err := gw.Write(payload) 799 assert.NoError(t, err) 800 assert.NoError(t, gw.Close()) 801 return buf.Len() 802 } 803 804 t.Logf("pbsize: %d, lpsize: %d, gz pb: %d, gz lp: %d", 805 len(pbPayload), len(lpPayload), gzSize(pbPayload), gzSize(lpPayload)) 806 } 807 808 func TestEncodeInfField(t *T.T) { 809 var kvs KVs 810 kvs = kvs.AddV2("f1", math.Inf(1), true) 811 kvs = kvs.AddV2("f2", math.Inf(-1), true) 812 kvs = kvs.AddV2("f3", 123, true) 813 814 pt := NewPointV2("some", kvs) 815 816 t.Logf("point: %s", pt.Pretty()) 817 818 t.Run("inf-lineproto", func(t *T.T) { 819 enc := GetEncoder(WithEncEncoding(LineProtocol)) 820 defer PutEncoder(enc) 821 822 enc.EncodeV2([]*Point{pt}) 823 824 buf := make([]byte, 1<<20) 825 826 for { 827 if res, ok := enc.Next(buf); ok { 828 t.Logf("res: %s", string(res)) 829 } else { 830 break 831 } 832 } 833 834 assert.NoError(t, enc.LastErr()) 835 }) 836 837 t.Run("inf-protobuf", func(t *T.T) { 838 enc := GetEncoder(WithEncEncoding(Protobuf)) 839 defer PutEncoder(enc) 840 841 enc.EncodeV2([]*Point{pt}) 842 843 buf := make([]byte, 1<<20) 844 845 for { 846 if res, ok := enc.Next(buf); ok { 847 dec := GetDecoder(WithDecEncoding(Protobuf)) 848 defer PutDecoder(dec) 849 850 pts, err := dec.Decode(res) 851 assert.NoError(t, err) 852 853 t.Logf("decode point: %s", pts[0].Pretty()) 854 855 assert.Equal(t, uint64(math.MaxUint64), uint64(pts[0].Get("f1").(float64))) 856 assert.Equal(t, int64(math.MinInt64), int64(pts[0].Get("f2").(float64))) 857 assert.Equal(t, int64(123), pts[0].Get("f3")) 858 } else { 859 break 860 } 861 } 862 863 assert.NoError(t, enc.LastErr()) 864 }) 865 866 t.Run("inf-pbjson", func(t *T.T) { 867 enc := GetEncoder(WithEncEncoding(JSON)) 868 defer PutEncoder(enc) 869 870 pt.SetFlag(Ppb) 871 872 arr, err := enc.Encode([]*Point{pt}) 873 assert.NoError(t, err) 874 875 t.Logf("res: %s", string(arr[0])) 876 }) 877 }