github.com/thanos-io/thanos@v0.32.5/test/e2e/native_histograms_test.go (about)

     1  // Copyright (c) The Thanos Authors.
     2  // Licensed under the Apache License 2.0.
     3  
     4  package e2e_test
     5  
     6  import (
     7  	"context"
     8  	"fmt"
     9  	"reflect"
    10  	"testing"
    11  	"time"
    12  
    13  	"github.com/efficientgo/core/testutil"
    14  	"github.com/efficientgo/e2e"
    15  	e2emon "github.com/efficientgo/e2e/monitoring"
    16  	"github.com/efficientgo/e2e/monitoring/matchers"
    17  	"github.com/prometheus/common/model"
    18  	"github.com/prometheus/prometheus/model/histogram"
    19  	"github.com/prometheus/prometheus/prompb"
    20  	"github.com/prometheus/prometheus/storage/remote"
    21  	"github.com/prometheus/prometheus/tsdb/tsdbutil"
    22  
    23  	"github.com/thanos-io/thanos/pkg/promclient"
    24  	"github.com/thanos-io/thanos/pkg/queryfrontend"
    25  	"github.com/thanos-io/thanos/pkg/receive"
    26  	"github.com/thanos-io/thanos/test/e2e/e2ethanos"
    27  )
    28  
    29  const testHistogramMetricName = "fake_histogram"
    30  
    31  func TestQueryNativeHistograms(t *testing.T) {
    32  	e, err := e2e.NewDockerEnvironment("nat-hist-query")
    33  	testutil.Ok(t, err)
    34  	t.Cleanup(e2ethanos.CleanScenario(t, e))
    35  
    36  	prom1, sidecar1 := e2ethanos.NewPrometheusWithSidecar(e, "ha1", e2ethanos.DefaultPromConfig("prom-ha", 0, "", "", e2ethanos.LocalPrometheusTarget), "", e2ethanos.DefaultPrometheusImage(), "", "native-histograms", "remote-write-receiver")
    37  	prom2, sidecar2 := e2ethanos.NewPrometheusWithSidecar(e, "ha2", e2ethanos.DefaultPromConfig("prom-ha", 1, "", "", e2ethanos.LocalPrometheusTarget), "", e2ethanos.DefaultPrometheusImage(), "", "native-histograms", "remote-write-receiver")
    38  	testutil.Ok(t, e2e.StartAndWaitReady(prom1, sidecar1, prom2, sidecar2))
    39  
    40  	querier := e2ethanos.NewQuerierBuilder(e, "querier", sidecar1.InternalEndpoint("grpc"), sidecar2.InternalEndpoint("grpc")).
    41  		WithEnabledFeatures([]string{"query-pushdown"}).
    42  		Init()
    43  	testutil.Ok(t, e2e.StartAndWaitReady(querier))
    44  
    45  	rawRemoteWriteURL1 := "http://" + prom1.Endpoint("http") + "/api/v1/write"
    46  	rawRemoteWriteURL2 := "http://" + prom2.Endpoint("http") + "/api/v1/write"
    47  
    48  	ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
    49  	t.Cleanup(cancel)
    50  
    51  	histograms := tsdbutil.GenerateTestHistograms(4)
    52  	now := time.Now()
    53  
    54  	_, err = writeHistograms(ctx, now, testHistogramMetricName, histograms, nil, rawRemoteWriteURL1)
    55  	testutil.Ok(t, err)
    56  	_, err = writeHistograms(ctx, now, testHistogramMetricName, histograms, nil, rawRemoteWriteURL2)
    57  	testutil.Ok(t, err)
    58  
    59  	ts := func() time.Time { return now }
    60  
    61  	// Make sure we can query histogram from both Prometheus instances.
    62  	queryAndAssert(t, ctx, prom1.Endpoint("http"), func() string { return testHistogramMetricName }, ts, promclient.QueryOptions{}, expectedHistogramModelVector(testHistogramMetricName, histograms[len(histograms)-1], nil, nil))
    63  	queryAndAssert(t, ctx, prom2.Endpoint("http"), func() string { return testHistogramMetricName }, ts, promclient.QueryOptions{}, expectedHistogramModelVector(testHistogramMetricName, histograms[len(histograms)-1], nil, nil))
    64  
    65  	t.Run("query deduplicated histogram", func(t *testing.T) {
    66  		queryAndAssert(t, ctx, querier.Endpoint("http"), func() string { return testHistogramMetricName }, ts, promclient.QueryOptions{Deduplicate: true}, expectedHistogramModelVector(testHistogramMetricName, histograms[len(histograms)-1], nil, map[string]string{
    67  			"prometheus": "prom-ha",
    68  		}))
    69  	})
    70  
    71  	t.Run("query histogram using histogram_count fn and deduplication", func(t *testing.T) {
    72  		queryAndAssert(t, ctx, querier.Endpoint("http"), func() string { return fmt.Sprintf("histogram_count(%v)", testHistogramMetricName) }, ts, promclient.QueryOptions{Deduplicate: true}, model.Vector{
    73  			&model.Sample{
    74  				Value: 34,
    75  				Metric: model.Metric{
    76  					"foo":        "bar",
    77  					"prometheus": "prom-ha",
    78  				},
    79  			},
    80  		})
    81  	})
    82  
    83  	t.Run("query histogram using group function for testing pushdown", func(t *testing.T) {
    84  		queryAndAssert(t, ctx, querier.Endpoint("http"), func() string { return fmt.Sprintf("group(%v)", testHistogramMetricName) }, ts, promclient.QueryOptions{Deduplicate: true}, model.Vector{
    85  			&model.Sample{
    86  				Value:  1,
    87  				Metric: model.Metric{},
    88  			},
    89  		})
    90  	})
    91  
    92  	t.Run("query histogram rate and compare to Prometheus result", func(t *testing.T) {
    93  		query := func() string { return fmt.Sprintf("rate(%v[1m])", testHistogramMetricName) }
    94  		expected, _ := instantQuery(t, ctx, prom1.Endpoint("http"), query, ts, promclient.QueryOptions{}, 1)
    95  		expected[0].Metric["prometheus"] = "prom-ha"
    96  		expected[0].Timestamp = 0
    97  		queryAndAssert(t, ctx, querier.Endpoint("http"), query, ts, promclient.QueryOptions{Deduplicate: true}, expected)
    98  	})
    99  }
   100  
   101  func TestWriteNativeHistograms(t *testing.T) {
   102  	e, err := e2e.NewDockerEnvironment("nat-hist-write")
   103  	testutil.Ok(t, err)
   104  	t.Cleanup(e2ethanos.CleanScenario(t, e))
   105  
   106  	ingestor0 := e2ethanos.NewReceiveBuilder(e, "ingestor0").WithIngestionEnabled().WithNativeHistograms().Init()
   107  	ingestor1 := e2ethanos.NewReceiveBuilder(e, "ingestor1").WithIngestionEnabled().WithNativeHistograms().Init()
   108  
   109  	h := receive.HashringConfig{
   110  		Endpoints: []receive.Endpoint{
   111  			{Address: ingestor0.InternalEndpoint("grpc")},
   112  			{Address: ingestor1.InternalEndpoint("grpc")},
   113  		},
   114  	}
   115  
   116  	router0 := e2ethanos.NewReceiveBuilder(e, "router0").WithRouting(2, h).Init()
   117  	testutil.Ok(t, e2e.StartAndWaitReady(ingestor0, ingestor1, router0))
   118  
   119  	querier := e2ethanos.NewQuerierBuilder(e, "querier", ingestor0.InternalEndpoint("grpc"), ingestor1.InternalEndpoint("grpc")).WithReplicaLabels("receive").Init()
   120  	testutil.Ok(t, e2e.StartAndWaitReady(querier))
   121  
   122  	rawRemoteWriteURL := "http://" + router0.Endpoint("remote-write") + "/api/v1/receive"
   123  
   124  	ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
   125  	t.Cleanup(cancel)
   126  
   127  	timeNow := time.Now()
   128  
   129  	histograms := tsdbutil.GenerateTestHistograms(1)
   130  	_, err = writeHistograms(ctx, timeNow, testHistogramMetricName, histograms, nil, rawRemoteWriteURL)
   131  	testutil.Ok(t, err)
   132  
   133  	testFloatHistogramMetricName := testHistogramMetricName + "_float"
   134  	floatHistograms := tsdbutil.GenerateTestFloatHistograms(1)
   135  	_, err = writeHistograms(ctx, timeNow, testFloatHistogramMetricName, nil, floatHistograms, rawRemoteWriteURL)
   136  	testutil.Ok(t, err)
   137  
   138  	queryAndAssert(t, ctx, querier.Endpoint("http"), func() string { return testHistogramMetricName }, time.Now, promclient.QueryOptions{Deduplicate: true}, expectedHistogramModelVector(testHistogramMetricName, histograms[0], nil, map[string]string{
   139  		"tenant_id": "default-tenant",
   140  	}))
   141  
   142  	queryAndAssert(t, ctx, querier.Endpoint("http"), func() string { return testFloatHistogramMetricName }, time.Now, promclient.QueryOptions{Deduplicate: true}, expectedHistogramModelVector(testFloatHistogramMetricName, nil, floatHistograms[0], map[string]string{
   143  		"tenant_id": "default-tenant",
   144  	}))
   145  }
   146  
   147  func TestQueryFrontendNativeHistograms(t *testing.T) {
   148  	e, err := e2e.NewDockerEnvironment("nat-hist-qfe")
   149  	testutil.Ok(t, err)
   150  	t.Cleanup(e2ethanos.CleanScenario(t, e))
   151  
   152  	prom1, sidecar1 := e2ethanos.NewPrometheusWithSidecar(e, "ha1", e2ethanos.DefaultPromConfig("prom-ha", 0, "", "", e2ethanos.LocalPrometheusTarget), "", e2ethanos.DefaultPrometheusImage(), "", "native-histograms", "remote-write-receiver")
   153  	prom2, sidecar2 := e2ethanos.NewPrometheusWithSidecar(e, "ha2", e2ethanos.DefaultPromConfig("prom-ha", 1, "", "", e2ethanos.LocalPrometheusTarget), "", e2ethanos.DefaultPrometheusImage(), "", "native-histograms", "remote-write-receiver")
   154  	testutil.Ok(t, e2e.StartAndWaitReady(prom1, sidecar1, prom2, sidecar2))
   155  
   156  	querier := e2ethanos.NewQuerierBuilder(e, "querier", sidecar1.InternalEndpoint("grpc"), sidecar2.InternalEndpoint("grpc")).Init()
   157  	testutil.Ok(t, e2e.StartAndWaitReady(querier))
   158  
   159  	inMemoryCacheConfig := queryfrontend.CacheProviderConfig{
   160  		Type: queryfrontend.INMEMORY,
   161  		Config: queryfrontend.InMemoryResponseCacheConfig{
   162  			MaxSizeItems: 1000,
   163  			Validity:     time.Hour,
   164  		},
   165  	}
   166  
   167  	queryFrontend := e2ethanos.NewQueryFrontend(e, "query-frontend", "http://"+querier.InternalEndpoint("http"), queryfrontend.Config{}, inMemoryCacheConfig)
   168  	testutil.Ok(t, e2e.StartAndWaitReady(queryFrontend))
   169  
   170  	rawRemoteWriteURL1 := "http://" + prom1.Endpoint("http") + "/api/v1/write"
   171  	rawRemoteWriteURL2 := "http://" + prom2.Endpoint("http") + "/api/v1/write"
   172  
   173  	ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
   174  	t.Cleanup(cancel)
   175  
   176  	histograms := tsdbutil.GenerateTestHistograms(4)
   177  	now := time.Now()
   178  	_, err = writeHistograms(ctx, now, testHistogramMetricName, histograms, nil, rawRemoteWriteURL1)
   179  	testutil.Ok(t, err)
   180  	startTime, err := writeHistograms(ctx, now, testHistogramMetricName, histograms, nil, rawRemoteWriteURL2)
   181  	testutil.Ok(t, err)
   182  
   183  	// Ensure we can get the result from Querier first so that it
   184  	// doesn't need to retry when we send queries to the frontend later.
   185  	queryAndAssertSeries(t, ctx, querier.Endpoint("http"), func() string { return testHistogramMetricName }, time.Now, promclient.QueryOptions{Deduplicate: true}, []model.Metric{
   186  		{
   187  			"__name__":   testHistogramMetricName,
   188  			"prometheus": "prom-ha",
   189  			"foo":        "bar",
   190  		},
   191  	})
   192  
   193  	vals, err := querier.SumMetrics(
   194  		[]string{"http_requests_total"},
   195  		e2emon.WithLabelMatchers(matchers.MustNewMatcher(matchers.MatchEqual, "handler", "query")),
   196  	)
   197  
   198  	testutil.Ok(t, err)
   199  	testutil.Equals(t, 1, len(vals))
   200  	queryTimes := vals[0]
   201  
   202  	ts := func() time.Time { return now }
   203  
   204  	t.Run("query frontend works for instant query", func(t *testing.T) {
   205  		queryAndAssert(t, ctx, queryFrontend.Endpoint("http"), func() string { return testHistogramMetricName }, ts, promclient.QueryOptions{Deduplicate: true}, expectedHistogramModelVector(testHistogramMetricName, histograms[len(histograms)-1], nil, map[string]string{
   206  			"prometheus": "prom-ha",
   207  		}))
   208  
   209  		testutil.Ok(t, queryFrontend.WaitSumMetricsWithOptions(
   210  			e2emon.Equals(1),
   211  			[]string{"thanos_query_frontend_queries_total"},
   212  			e2emon.WithLabelMatchers(matchers.MustNewMatcher(matchers.MatchEqual, "op", "query")),
   213  		))
   214  
   215  		testutil.Ok(t, querier.WaitSumMetricsWithOptions(
   216  			e2emon.Equals(queryTimes+1),
   217  			[]string{"http_requests_total"},
   218  			e2emon.WithLabelMatchers(matchers.MustNewMatcher(matchers.MatchEqual, "handler", "query")),
   219  		))
   220  	})
   221  
   222  	t.Run("query range query, all but last histogram", func(t *testing.T) {
   223  		expectedRes := expectedHistogramModelMatrix(testHistogramMetricName, histograms[:len(histograms)-1], nil, startTime, map[string]string{
   224  			"prometheus": "prom-ha",
   225  		})
   226  
   227  		// query all but last sample
   228  		rangeQuery(t, ctx, queryFrontend.Endpoint("http"), func() string { return testHistogramMetricName },
   229  			startTime.UnixMilli(),
   230  			// Skip last step, so that news samples is not queried and will be queried in next step.
   231  			now.Add(-30*time.Second).UnixMilli(),
   232  			30, // Taken from UI.
   233  			promclient.QueryOptions{
   234  				Deduplicate: true,
   235  			}, func(res model.Matrix) error {
   236  				if !reflect.DeepEqual(res, expectedRes) {
   237  					return fmt.Errorf("unexpected results (got %v but expected %v)", res, expectedRes)
   238  				}
   239  				return nil
   240  			})
   241  
   242  		testutil.Ok(t, queryFrontend.WaitSumMetrics(e2emon.Equals(1), "cortex_cache_fetched_keys_total"))
   243  		testutil.Ok(t, queryFrontend.WaitSumMetrics(e2emon.Equals(0), "cortex_cache_hits_total"))
   244  		testutil.Ok(t, queryFrontend.WaitSumMetrics(e2emon.Equals(1), "querier_cache_added_new_total"))
   245  		testutil.Ok(t, queryFrontend.WaitSumMetrics(e2emon.Equals(1), "querier_cache_added_total"))
   246  		testutil.Ok(t, queryFrontend.WaitSumMetrics(e2emon.Equals(1), "querier_cache_entries"))
   247  		testutil.Ok(t, queryFrontend.WaitSumMetrics(e2emon.Equals(1), "querier_cache_gets_total"))
   248  		testutil.Ok(t, queryFrontend.WaitSumMetrics(e2emon.Equals(1), "querier_cache_misses_total"))
   249  
   250  		testutil.Ok(t, querier.WaitSumMetricsWithOptions(
   251  			e2emon.Equals(1),
   252  			[]string{"http_requests_total"},
   253  			e2emon.WithLabelMatchers(matchers.MustNewMatcher(matchers.MatchEqual, "handler", "query_range")),
   254  		))
   255  
   256  	})
   257  
   258  	t.Run("query range, all histograms", func(t *testing.T) {
   259  		expectedRes := expectedHistogramModelMatrix(testHistogramMetricName, histograms, nil, startTime, map[string]string{
   260  			"prometheus": "prom-ha",
   261  		})
   262  
   263  		rangeQuery(t, ctx, queryFrontend.Endpoint("http"), func() string { return testHistogramMetricName },
   264  			startTime.UnixMilli(),
   265  			now.UnixMilli(),
   266  			30, // Taken from UI.
   267  			promclient.QueryOptions{
   268  				Deduplicate: true,
   269  			}, func(res model.Matrix) error {
   270  				if !reflect.DeepEqual(res, expectedRes) {
   271  					return fmt.Errorf("unexpected results (got %v but expected %v)", res, expectedRes)
   272  				}
   273  				return nil
   274  			})
   275  
   276  		testutil.Ok(t, queryFrontend.WaitSumMetrics(e2emon.Equals(2), "cortex_cache_fetched_keys_total"))
   277  		testutil.Ok(t, queryFrontend.WaitSumMetrics(e2emon.Equals(1), "cortex_cache_hits_total"))
   278  		testutil.Ok(t, queryFrontend.WaitSumMetrics(e2emon.Equals(1), "querier_cache_added_new_total"))
   279  		testutil.Ok(t, queryFrontend.WaitSumMetrics(e2emon.Equals(2), "querier_cache_added_total"))
   280  		testutil.Ok(t, queryFrontend.WaitSumMetrics(e2emon.Equals(1), "querier_cache_entries"))
   281  		testutil.Ok(t, queryFrontend.WaitSumMetrics(e2emon.Equals(2), "querier_cache_gets_total"))
   282  		testutil.Ok(t, queryFrontend.WaitSumMetrics(e2emon.Equals(1), "querier_cache_misses_total"))
   283  
   284  		testutil.Ok(t, querier.WaitSumMetricsWithOptions(
   285  			e2emon.Equals(2),
   286  			[]string{"http_requests_total"},
   287  			e2emon.WithLabelMatchers(matchers.MustNewMatcher(matchers.MatchEqual, "handler", "query_range")),
   288  		))
   289  	})
   290  
   291  	t.Run("query range, all histograms again", func(t *testing.T) {
   292  		expectedRes := expectedHistogramModelMatrix(testHistogramMetricName, histograms, nil, startTime, map[string]string{
   293  			"prometheus": "prom-ha",
   294  		})
   295  
   296  		rangeQuery(t, ctx, queryFrontend.Endpoint("http"), func() string { return testHistogramMetricName },
   297  			startTime.UnixMilli(),
   298  			now.UnixMilli(),
   299  			30, // Taken from UI.
   300  			promclient.QueryOptions{
   301  				Deduplicate: true,
   302  			}, func(res model.Matrix) error {
   303  				if !reflect.DeepEqual(res, expectedRes) {
   304  					return fmt.Errorf("unexpected results (got %v but expected %v)", res, expectedRes)
   305  				}
   306  				return nil
   307  			})
   308  
   309  		testutil.Ok(t, queryFrontend.WaitSumMetrics(e2emon.Equals(3), "cortex_cache_fetched_keys_total"))
   310  		testutil.Ok(t, queryFrontend.WaitSumMetrics(e2emon.Equals(2), "cortex_cache_hits_total"))
   311  		testutil.Ok(t, queryFrontend.WaitSumMetrics(e2emon.Equals(1), "querier_cache_added_new_total"))
   312  		testutil.Ok(t, queryFrontend.WaitSumMetrics(e2emon.Equals(3), "querier_cache_added_total"))
   313  		testutil.Ok(t, queryFrontend.WaitSumMetrics(e2emon.Equals(1), "querier_cache_entries"))
   314  		testutil.Ok(t, queryFrontend.WaitSumMetrics(e2emon.Equals(3), "querier_cache_gets_total"))
   315  		testutil.Ok(t, queryFrontend.WaitSumMetrics(e2emon.Equals(1), "querier_cache_misses_total"))
   316  
   317  		testutil.Ok(t, querier.WaitSumMetricsWithOptions(
   318  			e2emon.Equals(3),
   319  			[]string{"http_requests_total"},
   320  			e2emon.WithLabelMatchers(matchers.MustNewMatcher(matchers.MatchEqual, "handler", "query_range")),
   321  		))
   322  
   323  	})
   324  }
   325  
   326  func writeHistograms(ctx context.Context, now time.Time, name string, histograms []*histogram.Histogram, floatHistograms []*histogram.FloatHistogram, rawRemoteWriteURL string) (time.Time, error) {
   327  	startTime := now.Add(time.Duration(len(histograms)-1) * -30 * time.Second).Truncate(30 * time.Second)
   328  	prompbHistograms := make([]prompb.Histogram, 0, len(histograms))
   329  
   330  	for i, fh := range floatHistograms {
   331  		ts := startTime.Add(time.Duration(i) * 30 * time.Second).UnixMilli()
   332  		prompbHistograms = append(prompbHistograms, remote.FloatHistogramToHistogramProto(ts, fh))
   333  	}
   334  
   335  	for i, h := range histograms {
   336  		ts := startTime.Add(time.Duration(i) * 30 * time.Second).UnixMilli()
   337  		prompbHistograms = append(prompbHistograms, remote.HistogramToHistogramProto(ts, h))
   338  	}
   339  
   340  	timeSeriespb := prompb.TimeSeries{
   341  		Labels: []prompb.Label{
   342  			{Name: "__name__", Value: name},
   343  			{Name: "foo", Value: "bar"},
   344  		},
   345  		Histograms: prompbHistograms,
   346  	}
   347  
   348  	return startTime, storeWriteRequest(ctx, rawRemoteWriteURL, &prompb.WriteRequest{
   349  		Timeseries: []prompb.TimeSeries{timeSeriespb},
   350  	})
   351  }
   352  
   353  func expectedHistogramModelVector(metricName string, histogram *histogram.Histogram, floatHistogram *histogram.FloatHistogram, externalLabels map[string]string) model.Vector {
   354  	metrics := model.Metric{
   355  		"__name__": model.LabelValue(metricName),
   356  		"foo":      "bar",
   357  	}
   358  	for labelKey, labelValue := range externalLabels {
   359  		metrics[model.LabelName(labelKey)] = model.LabelValue(labelValue)
   360  	}
   361  
   362  	var sampleHistogram *model.SampleHistogram
   363  
   364  	if histogram != nil {
   365  		sampleHistogram = histogramToSampleHistogram(histogram)
   366  	} else {
   367  		sampleHistogram = floatHistogramToSampleHistogram(floatHistogram)
   368  	}
   369  
   370  	return model.Vector{
   371  		&model.Sample{
   372  			Metric:    metrics,
   373  			Histogram: sampleHistogram,
   374  		},
   375  	}
   376  }
   377  
   378  func expectedHistogramModelMatrix(metricName string, histograms []*histogram.Histogram, floatHistograms []*histogram.FloatHistogram, startTime time.Time, externalLabels map[string]string) model.Matrix {
   379  	metrics := model.Metric{
   380  		"__name__": model.LabelValue(metricName),
   381  		"foo":      "bar",
   382  	}
   383  	for labelKey, labelValue := range externalLabels {
   384  		metrics[model.LabelName(labelKey)] = model.LabelValue(labelValue)
   385  	}
   386  
   387  	sampleHistogramPair := make([]model.SampleHistogramPair, 0, len(histograms))
   388  
   389  	for i, h := range histograms {
   390  		sampleHistogramPair = append(sampleHistogramPair, model.SampleHistogramPair{
   391  			Timestamp: model.Time(startTime.Add(time.Duration(i) * 30 * time.Second).UnixMilli()),
   392  			Histogram: histogramToSampleHistogram(h),
   393  		})
   394  	}
   395  
   396  	for i, fh := range floatHistograms {
   397  		sampleHistogramPair = append(sampleHistogramPair, model.SampleHistogramPair{
   398  			Timestamp: model.Time(startTime.Add(time.Duration(i) * 30 * time.Second).UnixMilli()),
   399  			Histogram: floatHistogramToSampleHistogram(fh),
   400  		})
   401  	}
   402  
   403  	return model.Matrix{
   404  		&model.SampleStream{
   405  			Metric:     metrics,
   406  			Histograms: sampleHistogramPair,
   407  		},
   408  	}
   409  }
   410  
   411  func histogramToSampleHistogram(h *histogram.Histogram) *model.SampleHistogram {
   412  	var buckets []*model.HistogramBucket
   413  
   414  	it := h.NegativeBucketIterator()
   415  	for it.Next() {
   416  		buckets = append([]*model.HistogramBucket{bucketToSampleHistogramBucket(it.At())}, buckets...)
   417  	}
   418  
   419  	buckets = append(buckets, bucketToSampleHistogramBucket(h.ZeroBucket()))
   420  
   421  	it = h.PositiveBucketIterator()
   422  	for it.Next() {
   423  		buckets = append(buckets, bucketToSampleHistogramBucket(it.At()))
   424  	}
   425  
   426  	return &model.SampleHistogram{
   427  		Count:   model.FloatString(h.Count),
   428  		Sum:     model.FloatString(h.Sum),
   429  		Buckets: buckets,
   430  	}
   431  }
   432  
   433  func floatHistogramToSampleHistogram(fh *histogram.FloatHistogram) *model.SampleHistogram {
   434  	var buckets []*model.HistogramBucket
   435  
   436  	it := fh.NegativeBucketIterator()
   437  	for it.Next() {
   438  		buckets = append([]*model.HistogramBucket{bucketToSampleHistogramBucket(it.At())}, buckets...)
   439  	}
   440  
   441  	buckets = append(buckets, bucketToSampleHistogramBucket(fh.ZeroBucket()))
   442  
   443  	it = fh.PositiveBucketIterator()
   444  	for it.Next() {
   445  		buckets = append(buckets, bucketToSampleHistogramBucket(it.At()))
   446  	}
   447  
   448  	return &model.SampleHistogram{
   449  		Count:   model.FloatString(fh.Count),
   450  		Sum:     model.FloatString(fh.Sum),
   451  		Buckets: buckets,
   452  	}
   453  }
   454  
   455  func bucketToSampleHistogramBucket[BC histogram.BucketCount](bucket histogram.Bucket[BC]) *model.HistogramBucket {
   456  	return &model.HistogramBucket{
   457  		Lower:      model.FloatString(bucket.Lower),
   458  		Upper:      model.FloatString(bucket.Upper),
   459  		Count:      model.FloatString(bucket.Count),
   460  		Boundaries: boundaries(bucket),
   461  	}
   462  }
   463  
   464  func boundaries[BC histogram.BucketCount](bucket histogram.Bucket[BC]) int32 {
   465  	switch {
   466  	case bucket.UpperInclusive && !bucket.LowerInclusive:
   467  		return 0
   468  	case !bucket.UpperInclusive && bucket.LowerInclusive:
   469  		return 1
   470  	case !bucket.UpperInclusive && !bucket.LowerInclusive:
   471  		return 2
   472  	default:
   473  		return 3
   474  	}
   475  }