github.com/siglens/siglens@v0.0.0-20240328180423-f7ce9ae441ed/metricsbenchmark_test.go (about) 1 /* 2 Copyright 2023. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package bench 18 19 import ( 20 "fmt" 21 "os" 22 "sync/atomic" 23 "testing" 24 "time" 25 26 "github.com/buger/jsonparser" 27 localstorage "github.com/siglens/siglens/pkg/blob/local" 28 "github.com/siglens/siglens/pkg/config" 29 otsdbquery "github.com/siglens/siglens/pkg/integrations/otsdb/query" 30 otsdbwriter "github.com/siglens/siglens/pkg/integrations/otsdb/writer" 31 "github.com/siglens/siglens/pkg/segment" 32 "github.com/siglens/siglens/pkg/segment/memory/limit" 33 "github.com/siglens/siglens/pkg/segment/query" 34 "github.com/siglens/siglens/pkg/segment/writer" 35 serverutils "github.com/siglens/siglens/pkg/server/utils" 36 "github.com/valyala/fastrand" 37 38 "github.com/siglens/siglens/pkg/segment/writer/metrics" 39 "github.com/siglens/siglens/pkg/segment/writer/metrics/meta" 40 log "github.com/sirupsen/logrus" 41 "github.com/stretchr/testify/assert" 42 ) 43 44 var rawJson = []byte(`[{"metric":"test.metric.0","tags":{"car_type":"Passenger car compact","color":"white","fuel_type":"LPG","group":"group 1","model":"Slk350"},"timestamp":1677604613,"value":595},{"metric":"test.metric.0","tags":{"car_type":"Passenger car compact","color":"green","fuel_type":"LPG","group":"group 1","model":"Xlr"},"timestamp":1677604613,"value":316},{"metric":"test.metric.0","tags":{"car_type":"Van","color":"black","fuel_type":"LPG","group":"group 0","model":"4runner 4wd"},"timestamp":1677604613,"value":316},{"metric":"test.metric.0","tags":{"car_type":"Passenger car medium","color":"gray","fuel_type":"LPG","group":"group 0","model":"Forenza"},"timestamp":1677604613,"value":316},{"metric":"test.metric.0","tags":{"car_type":"Passenger car mini","color":"purple","fuel_type":"Gasoline","group":"group 0","model":"E320 Cdi"},"timestamp":1677604613,"value":15},{"metric":"test.metric.0","tags":{"car_type":"Passenger car light","color":"blue","fuel_type":"Diesel","group":"group 0","model":"Gs 300/gs 430"},"timestamp":1677604613,"value":15},{"metric":"test.metric.0","tags":{"car_type":"Passenger car light","color":"white","fuel_type":"Ethanol","group":"group 0","model":"L-147/148 Murcielago"},"timestamp":1677604613,"value":806},{"metric":"test.metric.0","tags":{"car_type":"Passenger car medium","color":"olive","fuel_type":"CNG","group":"group 0","model":"Colorado Cab Chassis Inc 2wd"},"timestamp":1677604613,"value":549},{"metric":"test.metric.0","tags":{"car_type":"Passenger car compact","color":"silver","fuel_type":"LPG","group":"group 0","model":"Ranger Pickup 2wd"},"timestamp":1677604613,"value":549},{"metric":"test.metric.0","tags":{"car_type":"Van","color":"navy","fuel_type":"CNG","group":"group 0","model":"V70 R Awd"},"timestamp":1677604613,"value":986},{"metric":"test.metric.0","tags":{"car_type":"Pickup truck","color":"fuchsia","fuel_type":"LPG","group":"group 1","model":" 325xi"},"timestamp":1677604613,"value":146},{"metric":"test.metric.0","tags":{"car_type":"Passenger car compact","color":"purple","fuel_type":"Electric","group":"group 0","model":"Uplander Fwd"},"timestamp":1677604613,"value":409},{"metric":"test.metric.0","tags":{"car_type":"Pickup truck","color":"olive","fuel_type":"Electric","group":"group 0","model":"Ridgeline 4wd"},"timestamp":1677604613,"value":409},{"metric":"test.metric.0","tags":{"car_type":"Passenger car medium","color":"purple","fuel_type":"Electric","group":"group 0","model":" 530xi Sport Wagon"},"timestamp":1677604613,"value":409},{"metric":"test.metric.0","tags":{"car_type":"Pickup truck","color":"purple","fuel_type":"CNG","group":"group 0","model":"Gti"},"timestamp":1677604613,"value":409},{"metric":"test.metric.0","tags":{"car_type":"Van","color":"silver","fuel_type":"LPG","group":"group 0","model":"Forenza Wagon"},"timestamp":1677604613,"value":409},{"metric":"test.metric.0","tags":{"car_type":"Passenger car medium","color":"aqua","fuel_type":"Electric","group":"group 1","model":"S60 R Awd"},"timestamp":1677604613,"value":795},{"metric":"test.metric.0","tags":{"car_type":"Passenger car mini","color":"fuchsia","fuel_type":"LPG","group":"group 0","model":"A4 Cabriolet"},"timestamp":1677604613,"value":637},{"metric":"test.metric.0","tags":{"car_type":"Passenger car light","color":"green","fuel_type":"Diesel","group":"group 1","model":"Odyssey 2wd"},"timestamp":1677604613,"value":637},{"metric":"test.metric.0","tags":{"car_type":"Van","color":"aqua","fuel_type":"Electric","group":"group 0","model":"Zephyr"},"timestamp":1677604613,"value":721},{"metric":"test.metric.0","tags":{"car_type":"Sport utility vehicle","color":"green","fuel_type":"Gasoline","group":"group 0","model":"Accent"},"timestamp":1677604613,"value":834},{"metric":"test.metric.0","tags":{"car_type":"Passenger car light","color":"purple","fuel_type":"CNG","group":"group 1","model":" X5"},"timestamp":1677604613,"value":834},{"metric":"test.metric.0","tags":{"car_type":"Passenger car medium","color":"green","fuel_type":"Ethanol","group":"group 0","model":"S4"},"timestamp":1677604613,"value":442},{"metric":"test.metric.0","tags":{"car_type":"Passenger car light","color":"yellow","fuel_type":"LPG","group":"group 0","model":" 750li"},"timestamp":1677604613,"value":442},{"metric":"test.metric.0","tags":{"car_type":"Passenger car heavy","color":"aqua","fuel_type":"Electric","group":"group 0","model":"Ranger Pickup 4wd"},"timestamp":1677604613,"value":5},{"metric":"test.metric.0","tags":{"car_type":"Pickup truck","color":"maroon","fuel_type":"Gasoline","group":"group 0","model":"Sonata"},"timestamp":1677604613,"value":273},{"metric":"test.metric.0","tags":{"car_type":"Passenger car light","color":"yellow","fuel_type":"Ethanol","group":"group 1","model":"V12 Vanquish S"},"timestamp":1677604613,"value":273},{"metric":"test.metric.0","tags":{"car_type":"Sport utility vehicle","color":"fuchsia","fuel_type":"LPG","group":"group 0","model":"Accord Hybrid"},"timestamp":1677604613,"value":493},{"metric":"test.metric.0","tags":{"car_type":"Pickup truck","color":"aqua","fuel_type":"Electric","group":"group 1","model":"Passat"},"timestamp":1677604613,"value":493},{"metric":"test.metric.0","tags":{"car_type":"Passenger car heavy","color":"lime","fuel_type":"Ethanol","group":"group 0","model":"Cts"},"timestamp":1677604613,"value":493},{"metric":"test.metric.0","tags":{"car_type":"Passenger car mini","color":"olive","fuel_type":"CNG","group":"group 1","model":"S40 Fwd"},"timestamp":1677604613,"value":432},{"metric":"test.metric.0","tags":{"car_type":"Van","color":"lime","fuel_type":"LPG","group":"group 1","model":"Rendezvous Fwd"},"timestamp":1677604613,"value":432},{"metric":"test.metric.0","tags":{"car_type":"Passenger car light","color":"gray","fuel_type":"LPG","group":"group 0","model":"Tt Coupe"},"timestamp":1677604613,"value":113},{"metric":"test.metric.0","tags":{"car_type":"Passenger car compact","color":"fuchsia","fuel_type":"LPG","group":"group 0","model":" Mini Cooper Convertible"},"timestamp":1677604613,"value":113},{"metric":"test.metric.0","tags":{"car_type":"Sport utility vehicle","color":"maroon","fuel_type":"CNG","group":"group 1","model":"V50 Awd"},"timestamp":1677604613,"value":113},{"metric":"test.metric.0","tags":{"car_type":"Passenger car light","color":"aqua","fuel_type":"Gasoline","group":"group 1","model":"Dakota Pickup 4wd"},"timestamp":1677604613,"value":169},{"metric":"test.metric.0","tags":{"car_type":"Passenger car medium","color":"purple","fuel_type":"Electric","group":"group 1","model":"Camry Solara"},"timestamp":1677604613,"value":169},{"metric":"test.metric.0","tags":{"car_type":"Passenger car heavy","color":"purple","fuel_type":"Methanol","group":"group 1","model":"S4"},"timestamp":1677604613,"value":609},{"metric":"test.metric.0","tags":{"car_type":"Van","color":"navy","fuel_type":"Electric","group":"group 1","model":"Five Hundred Awd"},"timestamp":1677604613,"value":609},{"metric":"test.metric.0","tags":{"car_type":"Passenger car light","color":"aqua","fuel_type":"Electric","group":"group 1","model":"A8 L"},"timestamp":1677604613,"value":722},{"metric":"test.metric.0","tags":{"car_type":"Sport utility vehicle","color":"lime","fuel_type":"LPG","group":"group 1","model":"Thunderbird"},"timestamp":1677604613,"value":495},{"metric":"test.metric.0","tags":{"car_type":"Passenger car medium","color":"olive","fuel_type":"Diesel","group":"group 1","model":"Grand Cherokee 2wd"},"timestamp":1677604613,"value":495},{"metric":"test.metric.0","tags":{"car_type":"Passenger car compact","color":"purple","fuel_type":"Methanol","group":"group 1","model":"Pathfinder 2wd"},"timestamp":1677604613,"value":184},{"metric":"test.metric.0","tags":{"car_type":"Pickup truck","color":"maroon","fuel_type":"Diesel","group":"group 1","model":"Escalade 2wd"},"timestamp":1677604613,"value":543},{"metric":"test.metric.0","tags":{"car_type":"Passenger car medium","color":"olive","fuel_type":"CNG","group":"group 1","model":"Thunderbird"},"timestamp":1677604613,"value":543},{"metric":"test.metric.0","tags":{"car_type":"Passenger car medium","color":"purple","fuel_type":"Ethanol","group":"group 1","model":"Monte Carlo"},"timestamp":1677604613,"value":633},{"metric":"test.metric.0","tags":{"car_type":"Passenger car medium","color":"yellow","fuel_type":"CNG","group":"group 0","model":"Yaris"},"timestamp":1677604613,"value":633},{"metric":"test.metric.0","tags":{"car_type":"Passenger car medium","color":"olive","fuel_type":"Methanol","group":"group 1","model":"Rx 330 2wd"},"timestamp":1677604613,"value":574},{"metric":"test.metric.0","tags":{"car_type":"Passenger car medium","color":"aqua","fuel_type":"Electric","group":"group 0","model":"Gs 300 4wd"},"timestamp":1677604613,"value":569},{"metric":"test.metric.0","tags":{"car_type":"Passenger car medium","color":"fuchsia","fuel_type":"Gasoline","group":"group 1","model":"F150 Pickup 2wd"},"timestamp":1677604613,"value":903},{"metric":"test.metric.0","tags":{"car_type":"Van","color":"navy","fuel_type":"Diesel","group":"group 1","model":" 325xi"},"timestamp":1677604613,"value":903},{"metric":"test.metric.0","tags":{"car_type":"Sport utility vehicle","color":"maroon","fuel_type":"LPG","group":"group 0","model":"Aveo"},"timestamp":1677604613,"value":903},{"metric":"test.metric.0","tags":{"car_type":"Sport utility vehicle","color":"purple","fuel_type":"Methanol","group":"group 0","model":"Mdx 4wd"},"timestamp":1677604613,"value":395},{"metric":"test.metric.0","tags":{"car_type":"Passenger car medium","color":"purple","fuel_type":"Methanol","group":"group 0","model":"C8 Spyder"},"timestamp":1677604613,"value":703},{"metric":"test.metric.0","tags":{"car_type":"Passenger car heavy","color":"aqua","fuel_type":"CNG","group":"group 0","model":"S4 Cabriolet"},"timestamp":1677604613,"value":703},{"metric":"test.metric.0","tags":{"car_type":"Passenger car heavy","color":"maroon","fuel_type":"Gasoline","group":"group 1","model":" Z4 3.0i"},"timestamp":1677604613,"value":528},{"metric":"test.metric.0","tags":{"car_type":"Passenger car mini","color":"teal","fuel_type":"CNG","group":"group 1","model":"Cr-v 4wd"},"timestamp":1677604613,"value":528},{"metric":"test.metric.0","tags":{"car_type":"Passenger car compact","color":"maroon","fuel_type":"CNG","group":"group 0","model":"F150 Pickup 2wd"},"timestamp":1677604613,"value":626},{"metric":"test.metric.0","tags":{"car_type":"Passenger car light","color":"teal","fuel_type":"Methanol","group":"group 0","model":"Sts Awd"},"timestamp":1677604613,"value":626},{"metric":"test.metric.0","tags":{"car_type":"Van","color":"aqua","fuel_type":"Ethanol","group":"group 0","model":"Wrangler/tj 4wd"},"timestamp":1677604613,"value":626},{"metric":"test.metric.0","tags":{"car_type":"Passenger car medium","color":"purple","fuel_type":"Electric","group":"group 0","model":"Xc 70 Awd"},"timestamp":1677604613,"value":626},{"metric":"test.metric.0","tags":{"car_type":"Passenger car compact","color":"aqua","fuel_type":"Gasoline","group":"group 1","model":"Tucson 4wd"},"timestamp":1677604613,"value":94},{"metric":"test.metric.0","tags":{"car_type":"Passenger car compact","color":"blue","fuel_type":"Diesel","group":"group 1","model":"Colorado Crew Cab 4wd"},"timestamp":1677604613,"value":94},{"metric":"test.metric.0","tags":{"car_type":"Passenger car heavy","color":"navy","fuel_type":"Electric","group":"group 1","model":"Tl"},"timestamp":1677604613,"value":94},{"metric":"test.metric.0","tags":{"car_type":"Sport utility vehicle","color":"silver","fuel_type":"Diesel","group":"group 1","model":"G35"},"timestamp":1677604613,"value":421},{"metric":"test.metric.0","tags":{"car_type":"Sport utility vehicle","color":"silver","fuel_type":"Electric","group":"group 1","model":"C8 Spyder"},"timestamp":1677604613,"value":208},{"metric":"test.metric.0","tags":{"car_type":"Passenger car light","color":"fuchsia","fuel_type":"Diesel","group":"group 1","model":"V70 R Awd"},"timestamp":1677604613,"value":208},{"metric":"test.metric.0","tags":{"car_type":"Passenger car compact","color":"white","fuel_type":"CNG","group":"group 0","model":" M3 Convertible"},"timestamp":1677604613,"value":189},{"metric":"test.metric.0","tags":{"car_type":"Passenger car mini","color":"blue","fuel_type":"Electric","group":"group 0","model":"Clk55 Amg (cabriolet)"},"timestamp":1677604613,"value":189},{"metric":"test.metric.0","tags":{"car_type":"Passenger car light","color":"silver","fuel_type":"LPG","group":"group 1","model":"A4 Avant Quattro"},"timestamp":1677604613,"value":624},{"metric":"test.metric.0","tags":{"car_type":"Passenger car medium","color":"navy","fuel_type":"Diesel","group":"group 1","model":"Titan 4wd"},"timestamp":1677604613,"value":962},{"metric":"test.metric.0","tags":{"car_type":"Passenger car compact","color":"yellow","fuel_type":"Electric","group":"group 0","model":"H3 4wd"},"timestamp":1677604613,"value":962},{"metric":"test.metric.0","tags":{"car_type":"Passenger car heavy","color":"aqua","fuel_type":"Ethanol","group":"group 0","model":"Pathfinder 2wd"},"timestamp":1677604613,"value":962},{"metric":"test.metric.0","tags":{"car_type":"Passenger car medium","color":"olive","fuel_type":"LPG","group":"group 1","model":"Trailblazer Ext 4wd"},"timestamp":1677604613,"value":962},{"metric":"test.metric.0","tags":{"car_type":"Sport utility vehicle","color":"purple","fuel_type":"Diesel","group":"group 0","model":"Solstice"},"timestamp":1677604613,"value":217},{"metric":"test.metric.0","tags":{"car_type":"Van","color":"gray","fuel_type":"Methanol","group":"group 1","model":" Mini Cooper S Convertible"},"timestamp":1677604613,"value":140},{"metric":"test.metric.0","tags":{"car_type":"Passenger car light","color":"white","fuel_type":"LPG","group":"group 0","model":"Sl65 Amg"},"timestamp":1677604613,"value":140},{"metric":"test.metric.0","tags":{"car_type":"Passenger car mini","color":"fuchsia","fuel_type":"Methanol","group":"group 0","model":"Ferrari 612 Scaglietti"},"timestamp":1677604613,"value":974},{"metric":"test.metric.0","tags":{"car_type":"Pickup truck","color":"gray","fuel_type":"LPG","group":"group 1","model":"Coupe Cambiocorsa/gt/g-sport"},"timestamp":1677604613,"value":884},{"metric":"test.metric.0","tags":{"car_type":"Van","color":"fuchsia","fuel_type":"Diesel","group":"group 1","model":"Five Hundred Awd"},"timestamp":1677604613,"value":884},{"metric":"test.metric.0","tags":{"car_type":"Passenger car mini","color":"fuchsia","fuel_type":"Ethanol","group":"group 1","model":"Elise/exige"},"timestamp":1677604613,"value":374},{"metric":"test.metric.0","tags":{"car_type":"Sport utility vehicle","color":"aqua","fuel_type":"Ethanol","group":"group 0","model":"Milan"},"timestamp":1677604613,"value":437},{"metric":"test.metric.0","tags":{"car_type":"Van","color":"blue","fuel_type":"Electric","group":"group 1","model":"Vdp Lwb"},"timestamp":1677604613,"value":931},{"metric":"test.metric.0","tags":{"car_type":"Passenger car heavy","color":"lime","fuel_type":"Methanol","group":"group 0","model":"Crown Victoria Police"},"timestamp":1677604613,"value":931},{"metric":"test.metric.0","tags":{"car_type":"Passenger car mini","color":"aqua","fuel_type":"Electric","group":"group 0","model":"Outlander 2wd"},"timestamp":1677604613,"value":547},{"metric":"test.metric.0","tags":{"car_type":"Passenger car light","color":"maroon","fuel_type":"Gasoline","group":"group 0","model":"E150 Econoline 2wd"},"timestamp":1677604613,"value":547},{"metric":"test.metric.0","tags":{"car_type":"Passenger car light","color":"lime","fuel_type":"Ethanol","group":"group 1","model":"Ls 430"},"timestamp":1677604613,"value":547},{"metric":"test.metric.0","tags":{"car_type":"Sport utility vehicle","color":"lime","fuel_type":"Ethanol","group":"group 1","model":"C15 Silverado Hybrid 2wd"},"timestamp":1677604613,"value":161},{"metric":"test.metric.0","tags":{"car_type":"Passenger car medium","color":"gray","fuel_type":"Electric","group":"group 1","model":"Sportage 4wd"},"timestamp":1677604613,"value":146},{"metric":"test.metric.0","tags":{"car_type":"Passenger car medium","color":"white","fuel_type":"Methanol","group":"group 1","model":"Frontier V6-4wd"},"timestamp":1677604613,"value":146},{"metric":"test.metric.0","tags":{"car_type":"Passenger car heavy","color":"green","fuel_type":"Diesel","group":"group 1","model":"S60 Fwd"},"timestamp":1677604613,"value":146},{"metric":"test.metric.0","tags":{"car_type":"Sport utility vehicle","color":"purple","fuel_type":"Electric","group":"group 1","model":"C350"},"timestamp":1677604613,"value":146},{"metric":"test.metric.0","tags":{"car_type":"Passenger car light","color":"silver","fuel_type":"Diesel","group":"group 0","model":"Mustang"},"timestamp":1677604613,"value":146},{"metric":"test.metric.0","tags":{"car_type":"Passenger car medium","color":"lime","fuel_type":"Gasoline","group":"group 0","model":" 750li"},"timestamp":1677604613,"value":860},{"metric":"test.metric.0","tags":{"car_type":"Sport utility vehicle","color":"black","fuel_type":"CNG","group":"group 1","model":"Maybach 57s"},"timestamp":1677604613,"value":505},{"metric":"test.metric.0","tags":{"car_type":"Passenger car light","color":"maroon","fuel_type":"Gasoline","group":"group 0","model":"Escalade 2wd"},"timestamp":1677604613,"value":505},{"metric":"test.metric.0","tags":{"car_type":"Van","color":"navy","fuel_type":"Ethanol","group":"group 0","model":"F150 Pickup 2wd"},"timestamp":1677604613,"value":693},{"metric":"test.metric.0","tags":{"car_type":"Van","color":"olive","fuel_type":"Electric","group":"group 1","model":"Pathfinder 2wd"},"timestamp":1677604613,"value":49},{"metric":"test.metric.0","tags":{"car_type":"Passenger car heavy","color":"silver","fuel_type":"Electric","group":"group 0","model":"S60 R Awd"},"timestamp":1677604613,"value":49},{"metric":"test.metric.0","tags":{"car_type":"Passenger car light","color":"green","fuel_type":"Methanol","group":"group 0","model":"Scion Tc"},"timestamp":1677604613,"value":49}]`) 45 46 func Benchmark_InsertJson(b *testing.B) { 47 config.InitializeTestingConfig() 48 writer.InitWriterNode() 49 50 sTime := time.Now() 51 totalSuccess := uint64(0) 52 for i := 0; i < 10_000; i++ { 53 success, fail, err := otsdbwriter.HandlePutMetrics(rawJson, uint64(0)) 54 assert.NoError(b, err) 55 assert.Equal(b, success, uint64(100)) 56 assert.Equal(b, fail, uint64(0)) 57 atomic.AddUint64(&totalSuccess, success) 58 if i%1_000 == 0 { 59 log.Infof("Ingested %+v metrics in %+v", totalSuccess, time.Since(sTime)) 60 } 61 } 62 log.Infof("Ingested %+v metrics in %+v", totalSuccess, time.Since(sTime)) 63 64 /* 65 cd pkg/otsdb/writer 66 go test -run=Bench -bench=Benchmark_InsertJson -cpuprofile cpuprofile.out -o rawsearch_cpu 67 go tool pprof ./rawsearch_cpu cpuprofile.out 68 69 (for mem profile) 70 go test -run=Bench -bench=Benchmark_InsertJson -benchmem -memprofile memprofile.out -o rawsearch_mem 71 go tool pprof ./rawsearch_mem memprofile.out 72 73 */ 74 err := os.RemoveAll(config.GetDataPath()) 75 assert.NoError(b, err) 76 } 77 78 type TagsTreeBenchmarkObject struct { 79 metricName []byte 80 tagsHolder metrics.TagsHolder 81 tsid uint64 82 } 83 84 func Benchmark_TagsTree(b *testing.B) { 85 numMetrics := uint32(1) 86 numTags := uint32(10) 87 88 config.InitializeTestingConfig() 89 writer.InitWriterNode() 90 metrics.InitTestingConfig() 91 metrics.InitMetricsSegStore() 92 tTime := int64(0) 93 entryCount := 1_000_000 94 fakeData := make([]TagsTreeBenchmarkObject, entryCount) 95 for i := 0; i < entryCount; i++ { 96 mName := fmt.Sprintf("metric-%d", fastrand.Uint32n(numMetrics)) 97 numRandomTags := fastrand.Uint32n(numTags) 98 tagsholder := metrics.GetTagsHolder() 99 for j := 0; j < int(numRandomTags); j++ { 100 randomTag := fmt.Sprintf("tag-%d", fastrand.Uint32n(numRandomTags)) 101 randomTagValue := fmt.Sprintf("random-string-%d", fastrand.Uint32n(10_000)) 102 randomTagValueType := 1 103 tagsholder.Insert(randomTag, []byte(randomTagValue), jsonparser.ValueType(randomTagValueType)) 104 } 105 tsid, _ := tagsholder.GetTSID([]byte(mName)) 106 fakeData[i] = TagsTreeBenchmarkObject{ 107 metricName: []byte(mName), 108 tagsHolder: *tagsholder, 109 tsid: tsid, 110 } 111 } 112 b.ResetTimer() 113 b.ReportAllocs() 114 for i := 0; i < entryCount; i++ { 115 sTime := time.Now() 116 err := metrics.GetTagsTreeHolder(0, "0").AddTagsForTSID(fakeData[i].metricName, &fakeData[i].tagsHolder, fakeData[i].tsid) 117 if err != nil { 118 log.Errorf("Error adding tags for tsid!") 119 } 120 tTime += time.Since(sTime).Milliseconds() 121 } 122 log.Infof("Average tags tree ingest finished in time %.2fms", float64(tTime)/float64(entryCount)) 123 } 124 125 func Benchmark_MetricsEndToEnd(b *testing.B) { 126 127 /* 128 go test -run=Bench -bench=Benchmark_MetricsEndToEnd -cpuprofile cpuprofile.out -o rawsearch_cpu 129 go tool pprof ./rawsearch_cpu cpuprofile.out 130 131 (for mem profile) 132 go test -run=Bench -bench=Benchmark_MetricsEndToEnd -benchmem -memprofile memprofile.out -o rawsearch_mem 133 go tool pprof ./rawsearch_mem memprofile.out 134 */ 135 136 config.InitializeDefaultConfig() 137 _ = localstorage.InitLocalStorage() 138 limit.InitMemoryLimiter() 139 metrics.InitTestingConfig() 140 err := meta.InitMetricsMeta() 141 if err != nil { 142 b.Fatalf("failed to initialize metrics meta") 143 } 144 145 baseDir := "/Users/ssubramanian/Desktop/SigLens/siglens/data/ingestnodes/Sris-MBP.attlocal.net/" 146 config.SetSmrBaseDirForTestOnly(baseDir) 147 err = query.InitQueryNode(getMyIds, serverutils.ExtractKibanaRequests) 148 if err != nil { 149 b.Fatalf("failed to initialize metrics meta") 150 } 151 startTime := "1678060800" 152 endTime := "1678147140" 153 m := "max:1h-sum:cpu.usage_guest_nice{region=*}" 154 mQRequest, err := otsdbquery.ParseRequest(startTime, endTime, m, 0) 155 assert.NoError(b, err) 156 segment.LogMetricsQuery("metrics query parser", mQRequest, 1) 157 count := 25 158 b.ResetTimer() 159 b.ReportAllocs() 160 tTime := int64(0) 161 for i := 0; i < count; i++ { 162 sTime := time.Now() 163 mQResponse := segment.ExecuteMetricsQuery(&mQRequest.MetricsQuery, &mQRequest.TimeRange, uint64(i)) 164 if mQResponse == nil { 165 b.Fatal("Benchmark_MetricsEndToEnd: Failed to get metrics query response") 166 } 167 mResult, err := mQResponse.GetOTSDBResults(&mQRequest.MetricsQuery) 168 assert.NoError(b, err) 169 log.Errorf("Query %d has %d series in %+v", i, len(mResult), time.Since(sTime)) 170 tTime += time.Since(sTime).Milliseconds() 171 } 172 log.Errorf("After %d iterations avg latency %.2fms", count, float64(tTime)/float64(count)) 173 }