github.com/GuanceCloud/cliutils@v1.1.21/point/ptpool_test.go (about) 1 package point 2 3 import ( 4 "fmt" 5 "math" 6 sync "sync" 7 T "testing" 8 "time" 9 10 "github.com/GuanceCloud/cliutils" 11 "github.com/GuanceCloud/cliutils/metrics" 12 gofakeit "github.com/brianvoe/gofakeit/v6" 13 "github.com/stretchr/testify/assert" 14 "github.com/stretchr/testify/require" 15 ) 16 17 var sampleLogs = []string{ 18 `2022-10-27T16:12:54.876+0800 DEBUG ddtrace trace/filters.go:235 keep tid: 971624677789410817 service: compiled-in-example resource: file-not-exists according to PRIORITY_AUTO_KEEP and sampling ratio: 100% `, 19 `2022-10-27T16:12:54.876+0800 DEBUG ddtrace trace/filters.go:235 keep tid: 564726768482716036 service: compiled-in-example resource: ./demo according to PRIORITY_AUTO_KEEP and sampling ratio: 100%`, 20 `2022-10-27T16:12:54.876+0800 DEBUG ddtrace trace/filters.go:102 keep tid: 971624677789410817 service: compiled-in-example resource: file-not-exists according to PRIORITY_AUTO_KEEP.`, 21 `2022-10-27T16:12:54.876+0800 DEBUG ddtrace trace/filters.go:102 keep tid: 564726768482716036 service: compiled-in-example resource: ./demo according to PRIORITY_AUTO_KEEP.`, 22 `2022-10-27T16:12:54.876+0800 DEBUG ddtrace trace/aftergather.go:121 ### send 2 points cost 0ms with error: <nil>`, 23 `2022-10-27T16:12:54.875+0800 DEBUG ddtrace ddtrace/ddtrace_http.go:34 ### received tracing data from path: /v0.4/traces`, 24 `2022-10-27T16:12:54.281+0800 DEBUG filter filter/filter.go:158 filter condition body: {"dataways":null,"filters":{"logging":["{ source = 'datakit' and ( host in [ 'ubt-dev-01' , 'tanb-ubt-dev-test' ] )}"]},"pull_interval":10000000000,"remote_pipelines":null}`, 25 `2022-10-27T16:12:54.184+0800 DEBUG io io/io.go:97 get iodata(1 points) from /v1/write/metric|swap`, 26 `2022-10-27T16:12:54.184+0800 DEBUG filter filter/filter.go:408 update metrics...`, 27 `2022-10-27T16:12:54.184+0800 DEBUG filter filter/filter.go:401 try pull remote filters...`, 28 `2022-10-27T16:12:54.184+0800 DEBUG filter filter/filter.go:262 /v1/write/metric/pts: 1, after: 1`, 29 `2022-10-27T16:12:54.184+0800 DEBUG dataway dataway/send.go:219 send request https://openway.guance.com/v1/datakit/pull?token=tkn_2af4b19d7f5a489fa81f0fff7e63b588&filters=true, proxy: , dwcli: 0x1400049e000, timeout: 30s(30s)`, 30 `2022-10-27T16:12:54.184+0800 DEBUG dataway dataway/cli.go:27 performing request%!(EXTRA string=method, string=GET, string=url, *url.URL=https://openway.guance.com/v1/datakit/pull?token=tkn_2af4b19d7f5a489fa81f0fff7e63b588&filters=true)`, 31 `2022-10-27T16:12:54.183+0800 DEBUG io io/feed.go:91 io feed swap|/v1/write/metric`, 32 `2022-10-27T16:12:54.183+0800 DEBUG filter filter/filter.go:235 no condition filter for metric`, 33 `2022-10-27T16:12:53.688+0800 DEBUG filter filter/filter.go:158 filter condition body: {"dataways":null,"filters":{"logging":["{ source = 'datakit' and ( host in [ 'ubt-dev-01' , 'tanb-ubt-dev-test' ] )}"]},"pull_interval":10000000000,"remote_pipelines":null}`, 34 `2022-10-27T16:12:53.622+0800 DEBUG io io/io.go:97 get iodata(2 points) from /v1/write/tracing|ddtrace`, 35 `2022-10-27T16:12:53.622+0800 DEBUG dataway dataway/send.go:219 send request https://openway.guance.com/v1/datakit/pull?token=tkn_2af4b19d7f5a489fa81f0fff7e63b588&filters=true, proxy: , dwcli: 0x1400049e000, timeout: 30s(30s)`, 36 `2022-10-27T16:12:49.573+0800 DEBUG io io/io.go:265 on tick(10s) to flush dynamicDatawayCategory(0 pts), last flush 9.999510666s ago...`, 37 `2022-10-27T16:12:49.462+0800 DEBUG filter filter/filter.go:158 filter condition body: {"dataways":null,"filters":{"logging":["{ source = 'datakit' and ( host in [ 'ubt-dev-01' , 'tanb-ubt-dev-test' ] )}"]},"pull_interval":10000000000,"remote_pipelines":null}`, 38 `2022-10-27T16:12:49.389+0800 DEBUG filter filter/filter.go:401 try pull remote filters...`, 39 `2022-10-27T16:12:49.389+0800 DEBUG dataway dataway/send.go:219 send request https://openway.guance.com/v1/datakit/pull?token=tkn_2af4b19d7f5a489fa81f0fff7e63b588&filters=true, proxy: , dwcli: 0x1400049e000, timeout: 30s(30s)`, 40 `2022-10-27T16:12:49.389+0800 DEBUG dataway dataway/cli.go:27 performing request%!(EXTRA string=method, string=GET, string=url, *url.URL=https://openway.guance.com/v1/datakit/pull?token=tkn_2af4b19d7f5a489fa81f0fff7e63b588&filters=true)`, 41 `2022-10-27T16:12:49.388+0800 DEBUG filter filter/filter.go:408 update metrics...`, 42 `2022-10-27T16:12:49.388+0800 DEBUG filter filter/filter.go:158 filter condition body: {"dataways":null,"filters":{"logging":["{ source = 'datakit' and ( host in [ 'ubt-dev-01' , 'tanb-ubt-dev-test' ] )}"]},"pull_interval":10000000000,"remote_pipelines":null}`, 43 `2022-10-27T16:12:49.386+0800 DEBUG io io/io.go:97 get iodata(4 points) from /v1/write/tracing|ddtrace`, 44 `2022-10-27T16:12:48.636+0800 DEBUG dataway dataway/send.go:219 send request https://openway.guance.com/v1/datakit/pull?token=tkn_2af4b19d7f5a489fa81f0fff7e63b588&filters=true, proxy: , dwcli: 0x1400049e000, timeout: 30s(30s)`, 45 `2022-10-27T16:12:48.636+0800 DEBUG dataway dataway/cli.go:27 performing request%!(EXTRA string=method, string=GET, string=url, *url.URL=https://openway.guance.com/v1/datakit/pull?token=tkn_2af4b19d7f5a489fa81f0fff7e63b588&filters=true)`, 46 `2022-10-27T16:12:48.444+0800 DEBUG filter filter/filter.go:158 filter condition body: {"dataways":null,"filters":{"logging":["{ source = 'datakit' and ( host in [ 'ubt-dev-01' , 'tanb-ubt-dev-test' ] )}"]},"pull_interval":10000000000,"remote_pipelines":null}`, 47 `2022-10-27T16:12:48.400+0800 DEBUG dataway dataway/send.go:219 send request https://openway.guance.com/v1/datakit/pull?token=tkn_2af4b19d7f5a489fa81f0fff7e63b588&filters=true, proxy: , dwcli: 0x1400049e000, timeout: 30s(30s)`, 48 `2022-10-27T16:12:48.400+0800 DEBUG dataway dataway/cli.go:27 performing request%!(EXTRA string=method, string=GET, string=url, *url.URL=https://openway.guance.com/v1/datakit/pull?token=tkn_2af4b19d7f5a489fa81f0fff7e63b588&filters=true)`, 49 `2022-10-27T16:12:46.815+0800 DEBUG dataway dataway/send.go:219 send request https://openway.guance.com/v1/datakit/pull?token=tkn_2af4b19d7f5a489fa81f0fff7e63b588&filters=true, proxy: , dwcli: 0x1400049e000, timeout: 30s(30s)`, 50 `2022-10-27T16:12:46.815+0800 DEBUG dataway dataway/cli.go:27 performing request%!(EXTRA string=method, string=GET, string=url, *url.URL=https://openway.guance.com/v1/datakit/pull?token=tkn_2af4b19d7f5a489fa81f0fff7e63b588&filters=true)`, 51 `2022-10-27T16:12:46.726+0800 DEBUG filter filter/filter.go:158 filter condition body: {"dataways":null,"filters":{"logging":["{ source = 'datakit' and ( host in [ 'ubt-dev-01' , 'tanb-ubt-dev-test' ] )}"]},"pull_interval":10000000000,"remote_pipelines":null}`, 52 `2022-10-27T16:12:46.703+0800 DEBUG dataway dataway/cli.go:27 performing request%!(EXTRA string=method, string=POST, string=url, *url.URL=https://openway.guance.com/v1/write/tracing?token=tkn_2af4b19d7f5a489fa81f0fff7e63b588)`, 53 `2022-10-27T16:12:46.700+0800 DEBUG dataway dataway/send.go:219 send request https://openway.guance.com/v1/write/tracing?token=tkn_2af4b19d7f5a489fa81f0fff7e63b588, proxy: , dwcli: 0x1400049e000, timeout: 30s(30s)`, 54 `2022-10-27T16:12:46.699+0800 DEBUG sender sender/sender.go:47 sending /v1/write/object(1 pts)...`, 55 `2022-10-27T16:12:46.699+0800 DEBUG sender sender/sender.go:47 sending /v1/write/metric(1 pts)...`, 56 `2022-10-27T16:12:46.699+0800 DEBUG io io/io.go:270 wal try flush failed data on /v1/write/security`, 57 `2022-10-27T16:12:46.699+0800 DEBUG io io/io.go:270 wal try flush failed data on /v1/write/rum`, 58 `2022-10-27T16:12:46.699+0800 DEBUG io io/io.go:270 wal try flush failed data on /v1/write/network`, 59 `2022-10-27T16:12:46.699+0800 DEBUG io io/io.go:270 wal try flush failed data on /v1/write/metric`, 60 `2022-10-27T16:12:46.699+0800 DEBUG io io/io.go:270 wal try flush failed data on /v1/write/logging`, 61 `2022-10-27T16:12:46.699+0800 DEBUG io io/io.go:265 on tick(10s) to flush /v1/write/security(0 pts), last flush 10.000030625s ago...`, 62 `2022-10-27T16:12:46.699+0800 DEBUG io io/io.go:265 on tick(10s) to flush /v1/write/rum(0 pts), last flush 9.999880583s ago...`, 63 `2022-10-27T16:12:46.699+0800 DEBUG io io/io.go:265 on tick(10s) to flush /v1/write/rum(0 pts), last flush 9.999536583s ago...`, 64 `2022-10-27T16:12:46.699+0800 DEBUG io io/io.go:265 on tick(10s) to flush /v1/write/rum(0 pts), last flush 9.999386542s ago...`, 65 `2022-10-27T16:12:46.699+0800 DEBUG io io/io.go:265 on tick(10s) to flush /v1/write/rum(0 pts), last flush 9.999338708s ago...`, 66 `2022-10-27T16:12:46.699+0800 DEBUG io io/io.go:265 on tick(10s) to flush /v1/write/rum(0 pts), last flush 9.998867333s ago...`, 67 `2022-10-27T16:12:46.699+0800 DEBUG io io/io.go:265 on tick(10s) to flush /v1/write/rum(0 pts), last flush 9.998208209s ago...`, 68 `2022-10-27T16:12:46.699+0800 DEBUG io io/io.go:265 on tick(10s) to flush /v1/write/object(1 pts), last flush 9.997395583s ago...`, 69 `2022-10-27T16:12:46.699+0800 DEBUG io io/io.go:265 on tick(10s) to flush /v1/write/network(0 pts), last flush 9.99991425s ago...`, 70 `2022-10-27T16:12:46.699+0800 DEBUG io io/io.go:265 on tick(10s) to flush /v1/write/network(0 pts), last flush 9.999568875s ago...`, 71 `2022-10-27T16:12:46.699+0800 DEBUG io io/io.go:265 on tick(10s) to flush /v1/write/network(0 pts), last flush 9.998325375s ago...`, 72 `2022-10-27T16:12:46.699+0800 DEBUG io io/io.go:265 on tick(10s) to flush /v1/write/network(0 pts), last flush 9.998172s ago...`, 73 `2022-10-27T16:12:46.699+0800 DEBUG io io/io.go:265 on tick(10s) to flush /v1/write/network(0 pts), last flush 9.997431792s ago...`, 74 `2022-10-27T16:12:46.699+0800 DEBUG io io/io.go:265 on tick(10s) to flush /v1/write/metric(1 pts), last flush 9.999472083s ago...`, 75 `2022-10-27T16:12:46.699+0800 DEBUG io io/io.go:265 on tick(10s) to flush /v1/write/metric(0 pts), last flush 9.999964541s ago...`, 76 `2022-10-27T16:12:46.699+0800 DEBUG io io/io.go:265 on tick(10s) to flush /v1/write/metric(0 pts), last flush 9.999953542s ago...`, 77 `2022-10-27T16:12:46.699+0800 DEBUG io io/io.go:265 on tick(10s) to flush /v1/write/metric(0 pts), last flush 9.999944333s ago...`, 78 `2022-10-27T16:12:46.699+0800 DEBUG io io/io.go:265 on tick(10s) to flush /v1/write/metric(0 pts), last flush 9.999897792s ago...`, 79 `2022-10-27T16:12:46.699+0800 DEBUG io io/io.go:265 on tick(10s) to flush /v1/write/metric(0 pts), last flush 9.999869417s ago...`, 80 `2022-10-27T16:12:46.699+0800 DEBUG io io/io.go:265 on tick(10s) to flush /v1/write/metric(0 pts), last flush 9.999858791s ago...`, 81 `2022-10-27T16:12:46.699+0800 DEBUG io io/io.go:265 on tick(10s) to flush /v1/write/metric(0 pts), last flush 9.99767025s ago...`, 82 `2022-10-27T16:12:46.699+0800 DEBUG io io/io.go:265 on tick(10s) to flush /v1/write/logging(0 pts), last flush 9.999887125s ago...`, 83 `2022-10-27T16:12:46.699+0800 DEBUG io io/io.go:265 on tick(10s) to flush /v1/write/logging(0 pts), last flush 9.998371916s ago...`, 84 `2022-10-27T16:12:46.699+0800 DEBUG io io/io.go:265 on tick(10s) to flush /v1/write/logging(0 pts), last flush 9.997611625s ago...`, 85 `2022-10-27T16:12:46.699+0800 DEBUG io io/io.go:265 on tick(10s) to flush /v1/write/logging(0 pts), last flush 9.997412708s ago...`, 86 `2022-10-27T16:12:46.699+0800 DEBUG io io/io.go:265 on tick(10s) to flush /v1/write/logging(0 pts), last flush 10.002298833s ago...`, 87 `2022-10-27T16:12:46.699+0800 DEBUG io io/io.go:265 on tick(10s) to flush /v1/write/logging(0 pts), last flush 10.000082958s ago...`, 88 `2022-10-27T16:12:46.699+0800 DEBUG io io/io.go:265 on tick(10s) to flush /v1/write/logging(0 pts), last flush 10.000006916s ago...`, 89 `2022-10-27T16:12:46.306+0800 DEBUG dataway dataway/send.go:219 send request https://openway.guance.com/v1/datakit/pull?token=tkn_2af4b19d7f5a489fa81f0fff7e63b588&filters=true, proxy: , dwcli: 0x1400049e000, timeout: 30s(30s)`, 90 `2022-10-27T16:12:46.306+0800 DEBUG dataway dataway/cli.go:27 performing request%!(EXTRA string=method, string=GET, string=url, *url.URL=https://openway.guance.com/v1/datakit/pull?token=tkn_2af4b19d7f5a489fa81f0fff7e63b588&filters=true)`, 91 `2022-10-27T16:12:46.305+0800 DEBUG ddtrace trace/filters.go:235 keep tid: 2790747027482021869 service: compiled-in-example resource: ./demo according to PRIORITY_AUTO_KEEP and sampling ratio: 100%`, 92 `2022-10-27T16:12:46.305+0800 DEBUG ddtrace trace/filters.go:235 keep tid: 1965248471827589152 service: compiled-in-example resource: file-not-exists according to PRIORITY_AUTO_KEEP and sampling ratio: 100%`, 93 `2022-10-27T16:12:46.305+0800 DEBUG ddtrace trace/filters.go:102 keep tid: 2790747027482021869 service: compiled-in-example resource: ./demo according to PRIORITY_AUTO_KEEP.`, 94 `2022-10-27T16:12:46.305+0800 DEBUG ddtrace trace/filters.go:102 keep tid: 1965248471827589152 service: compiled-in-example resource: file-not-exists according to PRIORITY_AUTO_KEEP.`, 95 `2022-10-27T16:12:45.481+0800 DEBUG disk disk/utils.go:62 disk---fstype:nullfs ,device:/Applications/网易有道词典.app ,mountpoint:/private/var/folders/71/4pnfjgwn0x3fcy4r3ddxw1fm0000gn/T/AppTranslocation/1A552256-4134-4CAA-A4FF-7D2DEF11A6AC`, 96 `2022-10-27T16:12:45.481+0800 DEBUG disk disk/utils.go:62 disk---fstype:nullfs ,device:/Applications/oss-browser.app ,mountpoint:/private/var/folders/71/4pnfjgwn0x3fcy4r3ddxw1fm0000gn/T/AppTranslocation/97346A30-EA8C-4AC8-991D-3AD64E2479E1`, 97 `2022-10-27T16:12:45.481+0800 DEBUG disk disk/utils.go:62 disk---fstype:nullfs ,device:/Applications/Sublime Text.app ,mountpoint:/private/var/folders/71/4pnfjgwn0x3fcy4r3ddxw1fm0000gn/T/AppTranslocation/0EE2FB5D-6535-47AB-938B-DCB79CE11CE6`, 98 `2022-10-27T16:12:45.481+0800 DEBUG disk disk/utils.go:62 disk---fstype:nullfs ,device:/Applications/Microsoft Remote Desktop.app ,mountpoint:/private/var/folders/71/4pnfjgwn0x3fcy4r3ddxw1fm0000gn/T/AppTranslocation/DD10B11F-2D45-4DFD-B1CB-EF0F2B1FB2F7`, 99 `2022-10-27T16:12:42.051+0800 DEBUG ddtrace trace/filters.go:235 keep tid: 5484031498000114328 service: compiled-in-example resource: ./demo according to PRIORITY_AUTO_KEEP and sampling ratio: 100%`, 100 `2022-10-27T16:12:42.051+0800 DEBUG ddtrace trace/filters.go:235 keep tid: 1409415361793528756 service: compiled-in-example resource: file-not-exists according to PRIORITY_AUTO_KEEP and sampling ratio: 100%`, 101 `2022-10-27T16:12:42.051+0800 DEBUG ddtrace trace/filters.go:102 keep tid: 1409415361793528756 service: compiled-in-example resource: file-not-exists according to PRIORITY_AUTO_KEEP.`, 102 `2022-10-27T16:12:42.051+0800 DEBUG ddtrace trace/aftergather.go:121 ### send 2 points cost 0ms with error: <nil>`, 103 `2022-10-27T16:12:42.051+0800 DEBUG dataway dataway/send.go:219 send request https://openway.guance.com/v1/datakit/pull?token=tkn_2af4b19d7f5a489fa81f0fff7e63b588&filters=true, proxy: , dwcli: 0x1400049e000, timeout: 30s(30s)`, 104 `2022-10-27T16:12:42.051+0800 DEBUG dataway dataway/cli.go:27 performing request%!(EXTRA string=method, string=GET, string=url, *url.URL=https://openway.guance.com/v1/datakit/pull?token=tkn_2af4b19d7f5a489fa81f0fff7e63b588&filters=true)`, 105 `2022-10-27T16:12:42.050+0800 DEBUG ddtrace trace/filters.go:102 keep tid: 5484031498000114328 service: compiled-in-example resource: ./demo according to PRIORITY_AUTO_KEEP.`, 106 `2022-10-27T16:12:42.050+0800 DEBUG ddtrace ddtrace/ddtrace_http.go:34 ### received tracing data from path: /v0.4/traces`, 107 } 108 109 func BenchmarkReservedCapPool(b *T.B) { 110 now := time.Now() 111 112 b.Run("without-pool", func(b *T.B) { 113 for i := 0; i < b.N; i++ { 114 var kvs KVs 115 116 kvs = kvs.Add("f0", 123, false, true) 117 kvs = kvs.Add("f1", 3.14, false, true) 118 kvs = kvs.Add("f2", "hello", false, true) 119 kvs = kvs.Add("f3", []byte("some looooooooooooooooooooooooooooooooooooooooooooooong text"), false, true) 120 kvs = kvs.Add("f4", false, false, false) 121 kvs = kvs.Add("f5", -123, false, false) 122 123 NewPointV2("m1", kvs, WithTime(now), WithPrecheck(false)) 124 } 125 }) 126 127 b.Run("reservedCapPool", func(b *T.B) { 128 pp := NewReservedCapPointPool(1000) 129 SetPointPool(pp) 130 defer func() { 131 SetPointPool(nil) 132 }() 133 134 b.Cleanup(func() { 135 metrics.Unregister(pp) 136 }) 137 138 metrics.MustRegister(pp) 139 140 b.ResetTimer() 141 var kvs KVs 142 for i := 0; i < b.N; i++ { 143 kvs = kvs.Add("f0", 123, false, false) 144 kvs = kvs.Add("f1", 3.14, false, false) 145 kvs = kvs.Add("f2", "hello", false, false) 146 kvs = kvs.Add("f3", []byte("some looooooooooooooooooooooooooooooooooooooooooooooong text"), false, false) 147 kvs = kvs.Add("f4", false, false, false) 148 kvs = kvs.Add("f5", -123, false, false) 149 150 pt := NewPointV2("m1", kvs, WithPrecheck(false)) 151 pp.Put(pt) 152 } 153 154 mfs, err := metrics.Gather() 155 assert.NoError(b, err) 156 157 b.Logf("\n%s", metrics.MetricFamily2Text(mfs)) 158 }) 159 } 160 161 func TestPointPool(t *T.T) { 162 t.Run("reserve-cap-pool-put-kv", func(t *T.T) { 163 pp := NewReservedCapPointPool(1000) 164 SetPointPool(pp) 165 defer ClearPointPool() 166 167 // total add 100 * 4 Field 168 for i := 0; i < 100; i++ { 169 var kvs KVs 170 kvs = kvs.Add(fmt.Sprintf("f%d", i), 123, false, false) 171 kvs = kvs.Add(fmt.Sprintf("f%d", i+1), 123, false, false) 172 kvs = kvs.Add(fmt.Sprintf("f%d", i+2), 123, false, false) 173 kvs = kvs.Add(fmt.Sprintf("f%d", i+3), 123, false, false) 174 175 for _, kv := range kvs { 176 pp.PutKV(kv) 177 } 178 } 179 180 cpp := pp.(*ReservedCapPointPool) 181 182 assert.Equal(t, int64(4), cpp.poolGet()) // pool-get: new object from pool 183 assert.Equal(t, int64(396), cpp.chanGet()) // chan-get: reuse-exist object from channel 184 185 t.Logf("point pool: %s", cpp.String()) 186 }) 187 188 t.Run("point-pool-concurrent", func(t *T.T) { 189 pp := NewReservedCapPointPool(1000) 190 SetPointPool(pp) 191 defer ClearPointPool() 192 193 var wg sync.WaitGroup 194 195 f := func() { 196 defer wg.Done() 197 198 for i := 0; i < 100; i++ { 199 var kvs KVs 200 kvs = kvs.Add(fmt.Sprintf("f-%d", i*100), 123, false, false) 201 kvs = kvs.Add(fmt.Sprintf("f-%d", i*100+1), 123, false, false) 202 kvs = kvs.Add(fmt.Sprintf("f-%d", i*100+2), 123, false, false) 203 kvs = kvs.Add(fmt.Sprintf("f-%d", i*100+3), 123, false, false) 204 205 for _, kv := range kvs { 206 pp.PutKV(kv) 207 } 208 } 209 } 210 211 n := 100 212 wg.Add(n) 213 for i := 0; i < n; i++ { 214 go f() 215 } 216 217 wg.Wait() 218 219 cpp := pp.(*ReservedCapPointPool) 220 221 // all put-back locate in chan, not pool(here chan cap is 1000, too large for only 4 kvs each loop). 222 assert.Equal(t, int64(n*100*4), cpp.chanPut()) 223 224 t.Logf("point pool: %s", cpp.String()) 225 }) 226 } 227 228 func TestReset(t *T.T) { 229 t.Run("reset", func(t *T.T) { 230 var kvs KVs 231 kvs = kvs.Add("f1", 123, false, false) 232 kvs = kvs.Add("f2", false, false, false) 233 234 pt := NewPointV2("" /* go warnning */, kvs, WithTime(time.Now())) 235 pt.Reset() 236 237 assert.True(t, isEmptyPoint(pt)) 238 }) 239 } 240 241 func BenchmarkStringKV(b *T.B) { 242 var ( 243 pp = NewReservedCapPointPool(1000) 244 245 shortString = cliutils.CreateRandomString(32) 246 longString = cliutils.CreateRandomString(1 << 10) // 1KB 247 hugeString = cliutils.CreateRandomString(1 << 20) // 1MB 248 now = time.Now() 249 ) 250 251 SetPointPool(pp) 252 defer ClearPointPool() 253 254 b.Run("string-kv", func(b *T.B) { 255 for i := 0; i < b.N; i++ { 256 var kvs KVs 257 kvs = kvs.Add("short-f1", shortString, false, false) 258 kvs = kvs.Add("long-f2", longString, false, false) 259 kvs = kvs.Add("huge-f3", hugeString, false, false) 260 261 kvs = kvs.Add("local-str-f1", "f1", false, false) 262 kvs = kvs.Add("local-str-f2", "f2", false, false) 263 kvs = kvs.Add("local-str-f3", "f3", false, false) 264 265 pt := NewPointV2("m1", 266 kvs, 267 WithTime(now), 268 WithStrField(true), 269 WithPrecheck(false), 270 ) 271 pp.Put(pt) 272 } 273 }) 274 275 b.Run("non-string-kv", func(b *T.B) { 276 for i := 0; i < b.N; i++ { 277 var kvs KVs 278 kvs = kvs.Add("f-i", 1024, false, false) 279 kvs = kvs.Add("f-b", false, false, false) 280 kvs = kvs.Add("f-f", 3.14, false, false) 281 kvs = kvs.Add("f-u", uint(42), false, false) 282 kvs = kvs.Add("f-max-int", int64(math.MaxInt64), false, false) 283 kvs = kvs.Add("f-max-uint", uint64(math.MaxUint64), false, false) 284 285 pt := NewPointV2("m1", 286 kvs, 287 WithTime(now), 288 WithPrecheck(false), 289 ) 290 pp.Put(pt) 291 } 292 }) 293 } 294 295 func TestPointPoolMetrics(t *T.T) { 296 t.Run("reserved-pool-metrics", func(t *T.T) { 297 pp := NewReservedCapPointPool(100) 298 SetPointPool(pp) 299 defer ClearPointPool() 300 301 t.Cleanup(func() { 302 metrics.Unregister(pp) 303 }) 304 305 metrics.MustRegister(pp) 306 307 // total add 100 * 4 Field 308 for i := 0; i < 100; i++ { 309 func() { 310 var kvs KVs 311 kvs = kvs.Add(fmt.Sprintf("f%d", i), 123, false, false) 312 kvs = kvs.Add(fmt.Sprintf("f%d", i+1), 123, false, false) 313 kvs = kvs.Add(fmt.Sprintf("f%d", i+2), 123, false, false) 314 kvs = kvs.Add(fmt.Sprintf("f%d", i+3), 123, false, false) 315 316 pt := NewPointV2("some", kvs) 317 pp.Put(pt) 318 }() 319 } 320 321 mfs, err := metrics.Gather() 322 assert.NoError(t, err) 323 324 t.Logf("\n%s", metrics.MetricFamily2Text(mfs)) 325 }) 326 } 327 328 func TestReservedCapPointPool(t *T.T) { 329 t.Run(`basic`, func(t *T.T) { 330 pp := NewReservedCapPointPool(100) 331 SetPointPool(pp) 332 defer ClearPointPool() 333 334 var kvs KVs 335 kvs = kvs.Add(fmt.Sprintf("f%d", 0), 123, false, false) 336 kvs = kvs.Add(fmt.Sprintf("f%d", +1), 123, false, false) 337 kvs = kvs.Add(fmt.Sprintf("f%d", +2), 123, false, false) 338 kvs = kvs.Add(fmt.Sprintf("f%d", +3), 123, false, false) 339 340 pt := NewPointV2("some", kvs) 341 t.Logf("pt: %s", pt.Pretty()) 342 343 pp.Put(pt) 344 345 empty := pp.Get() 346 t.Logf("empty pt: %s", empty.Pretty()) 347 }) 348 } 349 350 func TestPoolEscape(t *T.T) { 351 t.Run("escape", func(t *T.T) { 352 // setup point pool 353 pp := NewReservedCapPointPool(32) 354 SetPointPool(pp) 355 metrics.MustRegister(pp) 356 357 enc := GetEncoder(WithEncEncoding(Protobuf)) 358 359 dec := GetDecoder(WithDecEncoding(Protobuf), WithDecEasyproto(false)) 360 361 r := NewRander() 362 pts := r.Rand(100) 363 364 t.Cleanup(func() { 365 PutEncoder(enc) 366 PutDecoder(dec) 367 for _, pt := range pts { 368 pp.Put(pt) 369 } 370 371 ClearPointPool() 372 metrics.Unregister(pp) 373 }) 374 375 enc.EncodeV2(pts) 376 encBuf := make([]byte, 1<<20) 377 for { 378 if buf, ok := enc.Next(encBuf); ok { 379 decPts, err := dec.Decode(buf) 380 assert.NoError(t, err) 381 382 for _, pt := range decPts { 383 require.False(t, pt.HasFlag(Ppooled)) 384 pp.Put(pt) 385 } 386 } else { 387 break 388 } 389 } 390 391 mfs, err := metrics.Gather() 392 assert.NoError(t, err) 393 394 t.Logf("\n%s", metrics.MetricFamily2Text(mfs)) 395 396 mf := metrics.GetMetric(mfs, "pointpool_escaped", 0) 397 assert.Equal(t, 100.0, mf.GetCounter().GetValue()) // decoded 100 points(not easyproto) not from point pool 398 }) 399 400 t.Run("no-escape", func(t *T.T) { 401 // setup point pool 402 pp := NewReservedCapPointPool(32) 403 SetPointPool(pp) 404 metrics.MustRegister(pp) 405 406 enc := GetEncoder(WithEncEncoding(Protobuf)) 407 408 dec := GetDecoder(WithDecEncoding(Protobuf), WithDecEasyproto(true)) 409 410 r := NewRander() 411 pts := r.Rand(100) 412 413 t.Cleanup(func() { 414 PutEncoder(enc) 415 PutDecoder(dec) 416 for _, pt := range pts { 417 pp.Put(pt) 418 } 419 420 ClearPointPool() 421 metrics.Unregister(pp) 422 }) 423 424 enc.EncodeV2(pts) 425 encBuf := make([]byte, 1<<20) 426 for { 427 if buf, ok := enc.Next(encBuf); ok { 428 decPts, err := dec.Decode(buf) 429 assert.NoError(t, err) 430 431 for _, pt := range decPts { 432 pp.Put(pt) 433 } 434 } else { 435 break 436 } 437 } 438 439 mfs, err := metrics.Gather() 440 assert.NoError(t, err) 441 442 t.Logf("\n%s", metrics.MetricFamily2Text(mfs)) 443 444 mf := metrics.GetMetric(mfs, "pointpool_escaped", 0) 445 assert.Equal(t, 0.0, mf.GetCounter().GetValue()) // decoded 100 points(not easyproto) not from point pool 446 }) 447 } 448 449 func TestPoolKVResuable(t *T.T) { 450 type Foo struct { 451 Measurement string 452 453 TS int64 454 455 // tags 456 T1Key string `fake:"{regex:[a-zA-Z0-9_]{64}}"` 457 T2Key string `fake:"{regex:[a-zA-Z0-9_]{64}}"` 458 T3Key string `fake:"{regex:[a-zA-Z0-9_]{64}}"` 459 460 T1 string 461 T2 string 462 T3 string 463 464 SKey, S string `fake:"{regex:[a-zA-Z0-9]{128}}"` 465 466 I8Key string `fake:"{regex:[a-zA-Z0-9_]{64}}"` 467 I16Key string `fake:"{regex:[a-zA-Z0-9_]{64}}"` 468 I32Key string `fake:"{regex:[a-zA-Z0-9_]{64}}"` 469 I64Key string `fake:"{regex:[a-zA-Z0-9_]{64}}"` 470 I64 int64 471 I8 int8 472 I16 int16 473 I32 int32 474 475 U8Key string `fake:"{regex:[a-zA-Z0-9_]{64}}"` 476 U16Key string `fake:"{regex:[a-zA-Z0-9_]{64}}"` 477 U32Key string `fake:"{regex:[a-zA-Z0-9_]{64}}"` 478 U64Key string `fake:"{regex:[a-zA-Z0-9_]{64}}"` 479 480 U8 uint8 481 U16 uint16 482 U32 uint32 483 U64 uint64 484 485 BKey string `fake:"{regex:[a-zA-Z0-9_]{64}}"` 486 DKey string `fake:"{regex:[a-zA-Z0-9_]{64}}"` 487 F64Key string `fake:"{regex:[a-zA-Z0-9_]{64}}"` 488 F32Key string `fake:"{regex:[a-zA-Z0-9_]{64}}"` 489 B bool 490 D []byte 491 F64 float64 492 F32 float32 493 } 494 495 cases := []struct { 496 name string 497 pp PointPool 498 }{ 499 { 500 name: "reserve-cap-pool", 501 pp: NewReservedCapPointPool(1024), 502 }, 503 504 { 505 name: "reserve-0-cap-pool", // regression to v3 506 pp: NewReservedCapPointPool(0), 507 }, 508 } 509 510 for _, tc := range cases { 511 t.Run(tc.name, func(t *T.T) { 512 metrics.MustRegister(tc.pp) 513 514 SetPointPool(tc.pp) 515 t.Cleanup(func() { 516 ClearPointPool() 517 metrics.Unregister(tc.pp) 518 }) 519 520 var f Foo 521 maxPT := (1 << 20) 522 for i := 0; i < maxPT; i++ { 523 assert.NoError(t, gofakeit.Struct(&f)) 524 var kvs KVs 525 kvs = kvs.AddTag("T_"+f.T1Key, f.T1) 526 kvs = kvs.AddTag("T_"+f.T2Key, f.T2) 527 kvs = kvs.AddTag("T_"+f.T3Key, f.T3) 528 529 kvs = kvs.AddV2("S_"+f.SKey, f.S, true) 530 531 kvs = kvs.AddV2("I8_"+f.I8Key, f.I8, true) 532 kvs = kvs.AddV2("I16_"+f.I16Key, f.I16, true) 533 kvs = kvs.AddV2("I32_"+f.I32Key, f.I32, true) 534 kvs = kvs.AddV2("I64_"+f.I64Key, f.I64, true) 535 536 kvs = kvs.AddV2("U8_"+f.U8Key, f.U8, true) 537 kvs = kvs.AddV2("U16_"+f.U16Key, f.U16, true) 538 kvs = kvs.AddV2("U32_"+f.U32Key, f.U32, true) 539 kvs = kvs.AddV2("U64_"+f.U64Key, f.U64, true) 540 541 kvs = kvs.AddV2("F32_"+f.F32Key, f.F32, true) 542 kvs = kvs.AddV2("F64_"+f.F64Key, f.F64, true) 543 544 kvs = kvs.AddV2("B_"+f.BKey, f.B, true) 545 kvs = kvs.AddV2("D_"+f.DKey, f.D, true) 546 547 if f.TS < 0 { 548 f.TS = 0 549 } 550 551 pt := NewPointV2(f.Measurement, kvs, WithTimestamp(f.TS)) 552 553 require.Equal(t, f.T1, pt.Get("T_"+f.T1Key)) 554 require.Equal(t, f.T2, pt.Get("T_"+f.T2Key)) 555 require.Equal(t, f.T3, pt.Get("T_"+f.T3Key)) 556 557 require.Equal(t, f.S, pt.Get("S_"+f.SKey)) 558 559 require.Equal(t, int64(f.I8), pt.Get("I8_"+f.I8Key)) 560 require.Equalf(t, int64(f.I16), pt.Get("I16_"+f.I16Key), "got %s", pt.Pretty()) 561 require.Equal(t, int64(f.I32), pt.Get("I32_"+f.I32Key)) 562 require.Equal(t, f.I64, pt.Get("I64_"+f.I64Key)) 563 564 require.Equal(t, uint64(f.U8), pt.Get("U8_"+f.U8Key)) 565 require.Equal(t, uint64(f.U16), pt.Get("U16_"+f.U16Key)) 566 require.Equal(t, uint64(f.U32), pt.Get("U32_"+f.U32Key)) 567 require.Equal(t, f.U64, pt.Get("U64_"+f.U64Key)) 568 569 require.Equal(t, f.B, pt.Get("B_"+f.BKey), "got %s", pt.Pretty()) 570 require.Equal(t, f.D, pt.Get("D_"+f.DKey)) 571 require.Equalf(t, float64(f.F32), pt.Get("F32_"+f.F32Key), "got %s", pt.Pretty()) 572 require.Equal(t, f.F64, pt.Get("F64_"+f.F64Key)) 573 574 require.Equal(t, f.TS, pt.Time().UnixNano(), "got %s", pt.Pretty()) 575 576 if i == maxPT-1 { 577 t.Logf("point: %s", pt.Pretty()) 578 } 579 580 tc.pp.Put(pt) 581 } 582 583 mfs, err := metrics.Gather() 584 assert.NoError(t, err) 585 586 t.Logf("\n%s", metrics.MetricFamily2Text(mfs)) 587 }) 588 } 589 }