github.com/thanos-io/thanos@v0.32.5/test/e2e/native_histograms_test.go (about) 1 // Copyright (c) The Thanos Authors. 2 // Licensed under the Apache License 2.0. 3 4 package e2e_test 5 6 import ( 7 "context" 8 "fmt" 9 "reflect" 10 "testing" 11 "time" 12 13 "github.com/efficientgo/core/testutil" 14 "github.com/efficientgo/e2e" 15 e2emon "github.com/efficientgo/e2e/monitoring" 16 "github.com/efficientgo/e2e/monitoring/matchers" 17 "github.com/prometheus/common/model" 18 "github.com/prometheus/prometheus/model/histogram" 19 "github.com/prometheus/prometheus/prompb" 20 "github.com/prometheus/prometheus/storage/remote" 21 "github.com/prometheus/prometheus/tsdb/tsdbutil" 22 23 "github.com/thanos-io/thanos/pkg/promclient" 24 "github.com/thanos-io/thanos/pkg/queryfrontend" 25 "github.com/thanos-io/thanos/pkg/receive" 26 "github.com/thanos-io/thanos/test/e2e/e2ethanos" 27 ) 28 29 const testHistogramMetricName = "fake_histogram" 30 31 func TestQueryNativeHistograms(t *testing.T) { 32 e, err := e2e.NewDockerEnvironment("nat-hist-query") 33 testutil.Ok(t, err) 34 t.Cleanup(e2ethanos.CleanScenario(t, e)) 35 36 prom1, sidecar1 := e2ethanos.NewPrometheusWithSidecar(e, "ha1", e2ethanos.DefaultPromConfig("prom-ha", 0, "", "", e2ethanos.LocalPrometheusTarget), "", e2ethanos.DefaultPrometheusImage(), "", "native-histograms", "remote-write-receiver") 37 prom2, sidecar2 := e2ethanos.NewPrometheusWithSidecar(e, "ha2", e2ethanos.DefaultPromConfig("prom-ha", 1, "", "", e2ethanos.LocalPrometheusTarget), "", e2ethanos.DefaultPrometheusImage(), "", "native-histograms", "remote-write-receiver") 38 testutil.Ok(t, e2e.StartAndWaitReady(prom1, sidecar1, prom2, sidecar2)) 39 40 querier := e2ethanos.NewQuerierBuilder(e, "querier", sidecar1.InternalEndpoint("grpc"), sidecar2.InternalEndpoint("grpc")). 41 WithEnabledFeatures([]string{"query-pushdown"}). 42 Init() 43 testutil.Ok(t, e2e.StartAndWaitReady(querier)) 44 45 rawRemoteWriteURL1 := "http://" + prom1.Endpoint("http") + "/api/v1/write" 46 rawRemoteWriteURL2 := "http://" + prom2.Endpoint("http") + "/api/v1/write" 47 48 ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) 49 t.Cleanup(cancel) 50 51 histograms := tsdbutil.GenerateTestHistograms(4) 52 now := time.Now() 53 54 _, err = writeHistograms(ctx, now, testHistogramMetricName, histograms, nil, rawRemoteWriteURL1) 55 testutil.Ok(t, err) 56 _, err = writeHistograms(ctx, now, testHistogramMetricName, histograms, nil, rawRemoteWriteURL2) 57 testutil.Ok(t, err) 58 59 ts := func() time.Time { return now } 60 61 // Make sure we can query histogram from both Prometheus instances. 62 queryAndAssert(t, ctx, prom1.Endpoint("http"), func() string { return testHistogramMetricName }, ts, promclient.QueryOptions{}, expectedHistogramModelVector(testHistogramMetricName, histograms[len(histograms)-1], nil, nil)) 63 queryAndAssert(t, ctx, prom2.Endpoint("http"), func() string { return testHistogramMetricName }, ts, promclient.QueryOptions{}, expectedHistogramModelVector(testHistogramMetricName, histograms[len(histograms)-1], nil, nil)) 64 65 t.Run("query deduplicated histogram", func(t *testing.T) { 66 queryAndAssert(t, ctx, querier.Endpoint("http"), func() string { return testHistogramMetricName }, ts, promclient.QueryOptions{Deduplicate: true}, expectedHistogramModelVector(testHistogramMetricName, histograms[len(histograms)-1], nil, map[string]string{ 67 "prometheus": "prom-ha", 68 })) 69 }) 70 71 t.Run("query histogram using histogram_count fn and deduplication", func(t *testing.T) { 72 queryAndAssert(t, ctx, querier.Endpoint("http"), func() string { return fmt.Sprintf("histogram_count(%v)", testHistogramMetricName) }, ts, promclient.QueryOptions{Deduplicate: true}, model.Vector{ 73 &model.Sample{ 74 Value: 34, 75 Metric: model.Metric{ 76 "foo": "bar", 77 "prometheus": "prom-ha", 78 }, 79 }, 80 }) 81 }) 82 83 t.Run("query histogram using group function for testing pushdown", func(t *testing.T) { 84 queryAndAssert(t, ctx, querier.Endpoint("http"), func() string { return fmt.Sprintf("group(%v)", testHistogramMetricName) }, ts, promclient.QueryOptions{Deduplicate: true}, model.Vector{ 85 &model.Sample{ 86 Value: 1, 87 Metric: model.Metric{}, 88 }, 89 }) 90 }) 91 92 t.Run("query histogram rate and compare to Prometheus result", func(t *testing.T) { 93 query := func() string { return fmt.Sprintf("rate(%v[1m])", testHistogramMetricName) } 94 expected, _ := instantQuery(t, ctx, prom1.Endpoint("http"), query, ts, promclient.QueryOptions{}, 1) 95 expected[0].Metric["prometheus"] = "prom-ha" 96 expected[0].Timestamp = 0 97 queryAndAssert(t, ctx, querier.Endpoint("http"), query, ts, promclient.QueryOptions{Deduplicate: true}, expected) 98 }) 99 } 100 101 func TestWriteNativeHistograms(t *testing.T) { 102 e, err := e2e.NewDockerEnvironment("nat-hist-write") 103 testutil.Ok(t, err) 104 t.Cleanup(e2ethanos.CleanScenario(t, e)) 105 106 ingestor0 := e2ethanos.NewReceiveBuilder(e, "ingestor0").WithIngestionEnabled().WithNativeHistograms().Init() 107 ingestor1 := e2ethanos.NewReceiveBuilder(e, "ingestor1").WithIngestionEnabled().WithNativeHistograms().Init() 108 109 h := receive.HashringConfig{ 110 Endpoints: []receive.Endpoint{ 111 {Address: ingestor0.InternalEndpoint("grpc")}, 112 {Address: ingestor1.InternalEndpoint("grpc")}, 113 }, 114 } 115 116 router0 := e2ethanos.NewReceiveBuilder(e, "router0").WithRouting(2, h).Init() 117 testutil.Ok(t, e2e.StartAndWaitReady(ingestor0, ingestor1, router0)) 118 119 querier := e2ethanos.NewQuerierBuilder(e, "querier", ingestor0.InternalEndpoint("grpc"), ingestor1.InternalEndpoint("grpc")).WithReplicaLabels("receive").Init() 120 testutil.Ok(t, e2e.StartAndWaitReady(querier)) 121 122 rawRemoteWriteURL := "http://" + router0.Endpoint("remote-write") + "/api/v1/receive" 123 124 ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) 125 t.Cleanup(cancel) 126 127 timeNow := time.Now() 128 129 histograms := tsdbutil.GenerateTestHistograms(1) 130 _, err = writeHistograms(ctx, timeNow, testHistogramMetricName, histograms, nil, rawRemoteWriteURL) 131 testutil.Ok(t, err) 132 133 testFloatHistogramMetricName := testHistogramMetricName + "_float" 134 floatHistograms := tsdbutil.GenerateTestFloatHistograms(1) 135 _, err = writeHistograms(ctx, timeNow, testFloatHistogramMetricName, nil, floatHistograms, rawRemoteWriteURL) 136 testutil.Ok(t, err) 137 138 queryAndAssert(t, ctx, querier.Endpoint("http"), func() string { return testHistogramMetricName }, time.Now, promclient.QueryOptions{Deduplicate: true}, expectedHistogramModelVector(testHistogramMetricName, histograms[0], nil, map[string]string{ 139 "tenant_id": "default-tenant", 140 })) 141 142 queryAndAssert(t, ctx, querier.Endpoint("http"), func() string { return testFloatHistogramMetricName }, time.Now, promclient.QueryOptions{Deduplicate: true}, expectedHistogramModelVector(testFloatHistogramMetricName, nil, floatHistograms[0], map[string]string{ 143 "tenant_id": "default-tenant", 144 })) 145 } 146 147 func TestQueryFrontendNativeHistograms(t *testing.T) { 148 e, err := e2e.NewDockerEnvironment("nat-hist-qfe") 149 testutil.Ok(t, err) 150 t.Cleanup(e2ethanos.CleanScenario(t, e)) 151 152 prom1, sidecar1 := e2ethanos.NewPrometheusWithSidecar(e, "ha1", e2ethanos.DefaultPromConfig("prom-ha", 0, "", "", e2ethanos.LocalPrometheusTarget), "", e2ethanos.DefaultPrometheusImage(), "", "native-histograms", "remote-write-receiver") 153 prom2, sidecar2 := e2ethanos.NewPrometheusWithSidecar(e, "ha2", e2ethanos.DefaultPromConfig("prom-ha", 1, "", "", e2ethanos.LocalPrometheusTarget), "", e2ethanos.DefaultPrometheusImage(), "", "native-histograms", "remote-write-receiver") 154 testutil.Ok(t, e2e.StartAndWaitReady(prom1, sidecar1, prom2, sidecar2)) 155 156 querier := e2ethanos.NewQuerierBuilder(e, "querier", sidecar1.InternalEndpoint("grpc"), sidecar2.InternalEndpoint("grpc")).Init() 157 testutil.Ok(t, e2e.StartAndWaitReady(querier)) 158 159 inMemoryCacheConfig := queryfrontend.CacheProviderConfig{ 160 Type: queryfrontend.INMEMORY, 161 Config: queryfrontend.InMemoryResponseCacheConfig{ 162 MaxSizeItems: 1000, 163 Validity: time.Hour, 164 }, 165 } 166 167 queryFrontend := e2ethanos.NewQueryFrontend(e, "query-frontend", "http://"+querier.InternalEndpoint("http"), queryfrontend.Config{}, inMemoryCacheConfig) 168 testutil.Ok(t, e2e.StartAndWaitReady(queryFrontend)) 169 170 rawRemoteWriteURL1 := "http://" + prom1.Endpoint("http") + "/api/v1/write" 171 rawRemoteWriteURL2 := "http://" + prom2.Endpoint("http") + "/api/v1/write" 172 173 ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) 174 t.Cleanup(cancel) 175 176 histograms := tsdbutil.GenerateTestHistograms(4) 177 now := time.Now() 178 _, err = writeHistograms(ctx, now, testHistogramMetricName, histograms, nil, rawRemoteWriteURL1) 179 testutil.Ok(t, err) 180 startTime, err := writeHistograms(ctx, now, testHistogramMetricName, histograms, nil, rawRemoteWriteURL2) 181 testutil.Ok(t, err) 182 183 // Ensure we can get the result from Querier first so that it 184 // doesn't need to retry when we send queries to the frontend later. 185 queryAndAssertSeries(t, ctx, querier.Endpoint("http"), func() string { return testHistogramMetricName }, time.Now, promclient.QueryOptions{Deduplicate: true}, []model.Metric{ 186 { 187 "__name__": testHistogramMetricName, 188 "prometheus": "prom-ha", 189 "foo": "bar", 190 }, 191 }) 192 193 vals, err := querier.SumMetrics( 194 []string{"http_requests_total"}, 195 e2emon.WithLabelMatchers(matchers.MustNewMatcher(matchers.MatchEqual, "handler", "query")), 196 ) 197 198 testutil.Ok(t, err) 199 testutil.Equals(t, 1, len(vals)) 200 queryTimes := vals[0] 201 202 ts := func() time.Time { return now } 203 204 t.Run("query frontend works for instant query", func(t *testing.T) { 205 queryAndAssert(t, ctx, queryFrontend.Endpoint("http"), func() string { return testHistogramMetricName }, ts, promclient.QueryOptions{Deduplicate: true}, expectedHistogramModelVector(testHistogramMetricName, histograms[len(histograms)-1], nil, map[string]string{ 206 "prometheus": "prom-ha", 207 })) 208 209 testutil.Ok(t, queryFrontend.WaitSumMetricsWithOptions( 210 e2emon.Equals(1), 211 []string{"thanos_query_frontend_queries_total"}, 212 e2emon.WithLabelMatchers(matchers.MustNewMatcher(matchers.MatchEqual, "op", "query")), 213 )) 214 215 testutil.Ok(t, querier.WaitSumMetricsWithOptions( 216 e2emon.Equals(queryTimes+1), 217 []string{"http_requests_total"}, 218 e2emon.WithLabelMatchers(matchers.MustNewMatcher(matchers.MatchEqual, "handler", "query")), 219 )) 220 }) 221 222 t.Run("query range query, all but last histogram", func(t *testing.T) { 223 expectedRes := expectedHistogramModelMatrix(testHistogramMetricName, histograms[:len(histograms)-1], nil, startTime, map[string]string{ 224 "prometheus": "prom-ha", 225 }) 226 227 // query all but last sample 228 rangeQuery(t, ctx, queryFrontend.Endpoint("http"), func() string { return testHistogramMetricName }, 229 startTime.UnixMilli(), 230 // Skip last step, so that news samples is not queried and will be queried in next step. 231 now.Add(-30*time.Second).UnixMilli(), 232 30, // Taken from UI. 233 promclient.QueryOptions{ 234 Deduplicate: true, 235 }, func(res model.Matrix) error { 236 if !reflect.DeepEqual(res, expectedRes) { 237 return fmt.Errorf("unexpected results (got %v but expected %v)", res, expectedRes) 238 } 239 return nil 240 }) 241 242 testutil.Ok(t, queryFrontend.WaitSumMetrics(e2emon.Equals(1), "cortex_cache_fetched_keys_total")) 243 testutil.Ok(t, queryFrontend.WaitSumMetrics(e2emon.Equals(0), "cortex_cache_hits_total")) 244 testutil.Ok(t, queryFrontend.WaitSumMetrics(e2emon.Equals(1), "querier_cache_added_new_total")) 245 testutil.Ok(t, queryFrontend.WaitSumMetrics(e2emon.Equals(1), "querier_cache_added_total")) 246 testutil.Ok(t, queryFrontend.WaitSumMetrics(e2emon.Equals(1), "querier_cache_entries")) 247 testutil.Ok(t, queryFrontend.WaitSumMetrics(e2emon.Equals(1), "querier_cache_gets_total")) 248 testutil.Ok(t, queryFrontend.WaitSumMetrics(e2emon.Equals(1), "querier_cache_misses_total")) 249 250 testutil.Ok(t, querier.WaitSumMetricsWithOptions( 251 e2emon.Equals(1), 252 []string{"http_requests_total"}, 253 e2emon.WithLabelMatchers(matchers.MustNewMatcher(matchers.MatchEqual, "handler", "query_range")), 254 )) 255 256 }) 257 258 t.Run("query range, all histograms", func(t *testing.T) { 259 expectedRes := expectedHistogramModelMatrix(testHistogramMetricName, histograms, nil, startTime, map[string]string{ 260 "prometheus": "prom-ha", 261 }) 262 263 rangeQuery(t, ctx, queryFrontend.Endpoint("http"), func() string { return testHistogramMetricName }, 264 startTime.UnixMilli(), 265 now.UnixMilli(), 266 30, // Taken from UI. 267 promclient.QueryOptions{ 268 Deduplicate: true, 269 }, func(res model.Matrix) error { 270 if !reflect.DeepEqual(res, expectedRes) { 271 return fmt.Errorf("unexpected results (got %v but expected %v)", res, expectedRes) 272 } 273 return nil 274 }) 275 276 testutil.Ok(t, queryFrontend.WaitSumMetrics(e2emon.Equals(2), "cortex_cache_fetched_keys_total")) 277 testutil.Ok(t, queryFrontend.WaitSumMetrics(e2emon.Equals(1), "cortex_cache_hits_total")) 278 testutil.Ok(t, queryFrontend.WaitSumMetrics(e2emon.Equals(1), "querier_cache_added_new_total")) 279 testutil.Ok(t, queryFrontend.WaitSumMetrics(e2emon.Equals(2), "querier_cache_added_total")) 280 testutil.Ok(t, queryFrontend.WaitSumMetrics(e2emon.Equals(1), "querier_cache_entries")) 281 testutil.Ok(t, queryFrontend.WaitSumMetrics(e2emon.Equals(2), "querier_cache_gets_total")) 282 testutil.Ok(t, queryFrontend.WaitSumMetrics(e2emon.Equals(1), "querier_cache_misses_total")) 283 284 testutil.Ok(t, querier.WaitSumMetricsWithOptions( 285 e2emon.Equals(2), 286 []string{"http_requests_total"}, 287 e2emon.WithLabelMatchers(matchers.MustNewMatcher(matchers.MatchEqual, "handler", "query_range")), 288 )) 289 }) 290 291 t.Run("query range, all histograms again", func(t *testing.T) { 292 expectedRes := expectedHistogramModelMatrix(testHistogramMetricName, histograms, nil, startTime, map[string]string{ 293 "prometheus": "prom-ha", 294 }) 295 296 rangeQuery(t, ctx, queryFrontend.Endpoint("http"), func() string { return testHistogramMetricName }, 297 startTime.UnixMilli(), 298 now.UnixMilli(), 299 30, // Taken from UI. 300 promclient.QueryOptions{ 301 Deduplicate: true, 302 }, func(res model.Matrix) error { 303 if !reflect.DeepEqual(res, expectedRes) { 304 return fmt.Errorf("unexpected results (got %v but expected %v)", res, expectedRes) 305 } 306 return nil 307 }) 308 309 testutil.Ok(t, queryFrontend.WaitSumMetrics(e2emon.Equals(3), "cortex_cache_fetched_keys_total")) 310 testutil.Ok(t, queryFrontend.WaitSumMetrics(e2emon.Equals(2), "cortex_cache_hits_total")) 311 testutil.Ok(t, queryFrontend.WaitSumMetrics(e2emon.Equals(1), "querier_cache_added_new_total")) 312 testutil.Ok(t, queryFrontend.WaitSumMetrics(e2emon.Equals(3), "querier_cache_added_total")) 313 testutil.Ok(t, queryFrontend.WaitSumMetrics(e2emon.Equals(1), "querier_cache_entries")) 314 testutil.Ok(t, queryFrontend.WaitSumMetrics(e2emon.Equals(3), "querier_cache_gets_total")) 315 testutil.Ok(t, queryFrontend.WaitSumMetrics(e2emon.Equals(1), "querier_cache_misses_total")) 316 317 testutil.Ok(t, querier.WaitSumMetricsWithOptions( 318 e2emon.Equals(3), 319 []string{"http_requests_total"}, 320 e2emon.WithLabelMatchers(matchers.MustNewMatcher(matchers.MatchEqual, "handler", "query_range")), 321 )) 322 323 }) 324 } 325 326 func writeHistograms(ctx context.Context, now time.Time, name string, histograms []*histogram.Histogram, floatHistograms []*histogram.FloatHistogram, rawRemoteWriteURL string) (time.Time, error) { 327 startTime := now.Add(time.Duration(len(histograms)-1) * -30 * time.Second).Truncate(30 * time.Second) 328 prompbHistograms := make([]prompb.Histogram, 0, len(histograms)) 329 330 for i, fh := range floatHistograms { 331 ts := startTime.Add(time.Duration(i) * 30 * time.Second).UnixMilli() 332 prompbHistograms = append(prompbHistograms, remote.FloatHistogramToHistogramProto(ts, fh)) 333 } 334 335 for i, h := range histograms { 336 ts := startTime.Add(time.Duration(i) * 30 * time.Second).UnixMilli() 337 prompbHistograms = append(prompbHistograms, remote.HistogramToHistogramProto(ts, h)) 338 } 339 340 timeSeriespb := prompb.TimeSeries{ 341 Labels: []prompb.Label{ 342 {Name: "__name__", Value: name}, 343 {Name: "foo", Value: "bar"}, 344 }, 345 Histograms: prompbHistograms, 346 } 347 348 return startTime, storeWriteRequest(ctx, rawRemoteWriteURL, &prompb.WriteRequest{ 349 Timeseries: []prompb.TimeSeries{timeSeriespb}, 350 }) 351 } 352 353 func expectedHistogramModelVector(metricName string, histogram *histogram.Histogram, floatHistogram *histogram.FloatHistogram, externalLabels map[string]string) model.Vector { 354 metrics := model.Metric{ 355 "__name__": model.LabelValue(metricName), 356 "foo": "bar", 357 } 358 for labelKey, labelValue := range externalLabels { 359 metrics[model.LabelName(labelKey)] = model.LabelValue(labelValue) 360 } 361 362 var sampleHistogram *model.SampleHistogram 363 364 if histogram != nil { 365 sampleHistogram = histogramToSampleHistogram(histogram) 366 } else { 367 sampleHistogram = floatHistogramToSampleHistogram(floatHistogram) 368 } 369 370 return model.Vector{ 371 &model.Sample{ 372 Metric: metrics, 373 Histogram: sampleHistogram, 374 }, 375 } 376 } 377 378 func expectedHistogramModelMatrix(metricName string, histograms []*histogram.Histogram, floatHistograms []*histogram.FloatHistogram, startTime time.Time, externalLabels map[string]string) model.Matrix { 379 metrics := model.Metric{ 380 "__name__": model.LabelValue(metricName), 381 "foo": "bar", 382 } 383 for labelKey, labelValue := range externalLabels { 384 metrics[model.LabelName(labelKey)] = model.LabelValue(labelValue) 385 } 386 387 sampleHistogramPair := make([]model.SampleHistogramPair, 0, len(histograms)) 388 389 for i, h := range histograms { 390 sampleHistogramPair = append(sampleHistogramPair, model.SampleHistogramPair{ 391 Timestamp: model.Time(startTime.Add(time.Duration(i) * 30 * time.Second).UnixMilli()), 392 Histogram: histogramToSampleHistogram(h), 393 }) 394 } 395 396 for i, fh := range floatHistograms { 397 sampleHistogramPair = append(sampleHistogramPair, model.SampleHistogramPair{ 398 Timestamp: model.Time(startTime.Add(time.Duration(i) * 30 * time.Second).UnixMilli()), 399 Histogram: floatHistogramToSampleHistogram(fh), 400 }) 401 } 402 403 return model.Matrix{ 404 &model.SampleStream{ 405 Metric: metrics, 406 Histograms: sampleHistogramPair, 407 }, 408 } 409 } 410 411 func histogramToSampleHistogram(h *histogram.Histogram) *model.SampleHistogram { 412 var buckets []*model.HistogramBucket 413 414 it := h.NegativeBucketIterator() 415 for it.Next() { 416 buckets = append([]*model.HistogramBucket{bucketToSampleHistogramBucket(it.At())}, buckets...) 417 } 418 419 buckets = append(buckets, bucketToSampleHistogramBucket(h.ZeroBucket())) 420 421 it = h.PositiveBucketIterator() 422 for it.Next() { 423 buckets = append(buckets, bucketToSampleHistogramBucket(it.At())) 424 } 425 426 return &model.SampleHistogram{ 427 Count: model.FloatString(h.Count), 428 Sum: model.FloatString(h.Sum), 429 Buckets: buckets, 430 } 431 } 432 433 func floatHistogramToSampleHistogram(fh *histogram.FloatHistogram) *model.SampleHistogram { 434 var buckets []*model.HistogramBucket 435 436 it := fh.NegativeBucketIterator() 437 for it.Next() { 438 buckets = append([]*model.HistogramBucket{bucketToSampleHistogramBucket(it.At())}, buckets...) 439 } 440 441 buckets = append(buckets, bucketToSampleHistogramBucket(fh.ZeroBucket())) 442 443 it = fh.PositiveBucketIterator() 444 for it.Next() { 445 buckets = append(buckets, bucketToSampleHistogramBucket(it.At())) 446 } 447 448 return &model.SampleHistogram{ 449 Count: model.FloatString(fh.Count), 450 Sum: model.FloatString(fh.Sum), 451 Buckets: buckets, 452 } 453 } 454 455 func bucketToSampleHistogramBucket[BC histogram.BucketCount](bucket histogram.Bucket[BC]) *model.HistogramBucket { 456 return &model.HistogramBucket{ 457 Lower: model.FloatString(bucket.Lower), 458 Upper: model.FloatString(bucket.Upper), 459 Count: model.FloatString(bucket.Count), 460 Boundaries: boundaries(bucket), 461 } 462 } 463 464 func boundaries[BC histogram.BucketCount](bucket histogram.Bucket[BC]) int32 { 465 switch { 466 case bucket.UpperInclusive && !bucket.LowerInclusive: 467 return 0 468 case !bucket.UpperInclusive && bucket.LowerInclusive: 469 return 1 470 case !bucket.UpperInclusive && !bucket.LowerInclusive: 471 return 2 472 default: 473 return 3 474 } 475 }