github.com/thanos-io/thanos@v0.32.5/pkg/store/proxy_heap_test.go (about)

     1  // Copyright (c) The Thanos Authors.
     2  // Licensed under the Apache License 2.0.
     3  
     4  package store
     5  
     6  import (
     7  	"sync"
     8  	"testing"
     9  
    10  	"github.com/efficientgo/core/testutil"
    11  	"github.com/prometheus/prometheus/model/labels"
    12  
    13  	"github.com/thanos-io/thanos/pkg/dedup"
    14  	"github.com/thanos-io/thanos/pkg/errors"
    15  	"github.com/thanos-io/thanos/pkg/store/storepb"
    16  )
    17  
    18  func TestRmLabelsCornerCases(t *testing.T) {
    19  	testutil.Equals(t, rmLabels(labelsFromStrings("aa", "bb"), map[string]struct{}{
    20  		"aa": {},
    21  	}), labels.Labels{})
    22  	testutil.Equals(t, rmLabels(labelsFromStrings(), map[string]struct{}{
    23  		"aa": {},
    24  	}), labels.Labels{})
    25  }
    26  
    27  func TestProxyResponseHeapSort(t *testing.T) {
    28  	for _, tcase := range []struct {
    29  		title string
    30  		input []respSet
    31  		exp   []*storepb.SeriesResponse
    32  	}{
    33  		{
    34  			title: "merge sets with different series and common labels",
    35  			input: []respSet{
    36  				&eagerRespSet{
    37  					wg: &sync.WaitGroup{},
    38  					bufferedResponses: []*storepb.SeriesResponse{
    39  						storeSeriesResponse(t, labelsFromStrings("a", "1", "c", "3")),
    40  						storeSeriesResponse(t, labelsFromStrings("a", "1", "c", "3", "d", "4")),
    41  					},
    42  				},
    43  				&eagerRespSet{
    44  					wg: &sync.WaitGroup{},
    45  					bufferedResponses: []*storepb.SeriesResponse{
    46  						storeSeriesResponse(t, labelsFromStrings("a", "1", "c", "4", "e", "5")),
    47  						storeSeriesResponse(t, labelsFromStrings("a", "1", "d", "4")),
    48  					},
    49  				},
    50  			},
    51  			exp: []*storepb.SeriesResponse{
    52  				storeSeriesResponse(t, labelsFromStrings("a", "1", "c", "3")),
    53  				storeSeriesResponse(t, labelsFromStrings("a", "1", "c", "3", "d", "4")),
    54  				storeSeriesResponse(t, labelsFromStrings("a", "1", "c", "4", "e", "5")),
    55  				storeSeriesResponse(t, labelsFromStrings("a", "1", "d", "4")),
    56  			},
    57  		},
    58  		{
    59  			title: "merge sets with different series and labels",
    60  			input: []respSet{
    61  				&eagerRespSet{
    62  					wg: &sync.WaitGroup{},
    63  					bufferedResponses: []*storepb.SeriesResponse{
    64  						storeSeriesResponse(t, labelsFromStrings("a", "1", "b", "2", "c", "3")),
    65  						storeSeriesResponse(t, labelsFromStrings("b", "2", "c", "3")),
    66  						storeSeriesResponse(t, labelsFromStrings("g", "7", "h", "8", "i", "9")),
    67  					},
    68  				},
    69  				&eagerRespSet{
    70  					wg: &sync.WaitGroup{},
    71  					bufferedResponses: []*storepb.SeriesResponse{
    72  						storeSeriesResponse(t, labelsFromStrings("d", "4", "e", "5")),
    73  						storeSeriesResponse(t, labelsFromStrings("d", "4", "e", "5", "f", "6")),
    74  					},
    75  				},
    76  			},
    77  			exp: []*storepb.SeriesResponse{
    78  				storeSeriesResponse(t, labelsFromStrings("a", "1", "b", "2", "c", "3")),
    79  				storeSeriesResponse(t, labelsFromStrings("b", "2", "c", "3")),
    80  				storeSeriesResponse(t, labelsFromStrings("d", "4", "e", "5")),
    81  				storeSeriesResponse(t, labelsFromStrings("d", "4", "e", "5", "f", "6")),
    82  				storeSeriesResponse(t, labelsFromStrings("g", "7", "h", "8", "i", "9")),
    83  			},
    84  		},
    85  		{
    86  			title: "merge repeated series in stores with different external labels",
    87  			input: []respSet{
    88  				&eagerRespSet{
    89  					wg: &sync.WaitGroup{},
    90  					bufferedResponses: []*storepb.SeriesResponse{
    91  						storeSeriesResponse(t, labelsFromStrings("a", "1", "b", "2", "ext2", "9")),
    92  						storeSeriesResponse(t, labelsFromStrings("a", "1", "b", "2", "ext2", "9")),
    93  					},
    94  					storeLabels: map[string]struct{}{"ext2": {}},
    95  				},
    96  				&eagerRespSet{
    97  					wg: &sync.WaitGroup{},
    98  					bufferedResponses: []*storepb.SeriesResponse{
    99  						storeSeriesResponse(t, labelsFromStrings("a", "1", "b", "2", "ext1", "5", "ext2", "9")),
   100  						storeSeriesResponse(t, labelsFromStrings("a", "1", "b", "2", "ext1", "5", "ext2", "9")),
   101  					},
   102  					storeLabels: map[string]struct{}{"ext1": {}, "ext2": {}},
   103  				},
   104  			},
   105  			exp: []*storepb.SeriesResponse{
   106  				storeSeriesResponse(t, labelsFromStrings("a", "1", "b", "2", "ext1", "5", "ext2", "9")),
   107  				storeSeriesResponse(t, labelsFromStrings("a", "1", "b", "2", "ext1", "5", "ext2", "9")),
   108  				storeSeriesResponse(t, labelsFromStrings("a", "1", "b", "2", "ext2", "9")),
   109  				storeSeriesResponse(t, labelsFromStrings("a", "1", "b", "2", "ext2", "9")),
   110  			},
   111  		},
   112  		{
   113  			title: "merge series with external labels at beginning of series",
   114  			input: []respSet{
   115  				&eagerRespSet{
   116  					wg: &sync.WaitGroup{},
   117  					bufferedResponses: []*storepb.SeriesResponse{
   118  						storeSeriesResponse(t, labelsFromStrings("a", "1", "b", "2", "c", "3")),
   119  						storeSeriesResponse(t, labelsFromStrings("a", "1", "c", "2")),
   120  					},
   121  					storeLabels: map[string]struct{}{"a": {}},
   122  				},
   123  				&eagerRespSet{
   124  					wg: &sync.WaitGroup{},
   125  					bufferedResponses: []*storepb.SeriesResponse{
   126  						storeSeriesResponse(t, labelsFromStrings("a", "1", "b", "1", "c", "3")),
   127  						storeSeriesResponse(t, labelsFromStrings("a", "1", "c", "3")),
   128  					},
   129  					storeLabels: map[string]struct{}{"a": {}},
   130  				},
   131  			},
   132  			exp: []*storepb.SeriesResponse{
   133  				storeSeriesResponse(t, labelsFromStrings("a", "1", "b", "1", "c", "3")),
   134  				storeSeriesResponse(t, labelsFromStrings("a", "1", "b", "2", "c", "3")),
   135  				storeSeriesResponse(t, labelsFromStrings("a", "1", "c", "2")),
   136  				storeSeriesResponse(t, labelsFromStrings("a", "1", "c", "3")),
   137  			},
   138  		},
   139  		{
   140  			title: "merge series in stores with external labels not present in series (e.g. stripped during dedup)",
   141  			input: []respSet{
   142  				&eagerRespSet{
   143  					wg: &sync.WaitGroup{},
   144  					bufferedResponses: []*storepb.SeriesResponse{
   145  						storeSeriesResponse(t, labelsFromStrings("a", "1", "b", "2", "ext2", "9")),
   146  						storeSeriesResponse(t, labelsFromStrings("a", "1", "b", "2", "ext2", "9")),
   147  					},
   148  					storeLabels: map[string]struct{}{"ext2": {}, "replica": {}},
   149  				},
   150  				&eagerRespSet{
   151  					wg: &sync.WaitGroup{},
   152  					bufferedResponses: []*storepb.SeriesResponse{
   153  						storeSeriesResponse(t, labelsFromStrings("a", "1", "b", "2", "ext1", "5", "ext2", "9")),
   154  						storeSeriesResponse(t, labelsFromStrings("a", "1", "b", "2", "ext1", "5", "ext2", "9")),
   155  					},
   156  					storeLabels: map[string]struct{}{"ext1": {}, "ext2": {}, "replica": {}},
   157  				},
   158  			},
   159  			exp: []*storepb.SeriesResponse{
   160  				storeSeriesResponse(t, labelsFromStrings("a", "1", "b", "2", "ext1", "5", "ext2", "9")),
   161  				storeSeriesResponse(t, labelsFromStrings("a", "1", "b", "2", "ext1", "5", "ext2", "9")),
   162  				storeSeriesResponse(t, labelsFromStrings("a", "1", "b", "2", "ext2", "9")),
   163  				storeSeriesResponse(t, labelsFromStrings("a", "1", "b", "2", "ext2", "9")),
   164  			},
   165  		},
   166  		{
   167  			title: "test",
   168  			input: []respSet{
   169  				&eagerRespSet{
   170  					wg: &sync.WaitGroup{},
   171  					bufferedResponses: []*storepb.SeriesResponse{
   172  						storeSeriesResponse(t, labelsFromStrings("cluster", "beam-platform", "instance", "10.70.13.3:15692", "prometheus", "telemetry/observe-prometheus", "receive", "true", "tenant_id", "default-tenant")),
   173  						storeSeriesResponse(t, labelsFromStrings("cluster", "beam-platform", "instance", "10.70.5.3:15692", "prometheus", "telemetry/observe-prometheus", "receive", "true", "tenant_id", "default-tenant")),
   174  						storeSeriesResponse(t, labelsFromStrings("cluster", "beam-platform", "instance", "10.70.6.3:15692", "prometheus", "telemetry/observe-prometheus", "receive", "true", "tenant_id", "default-tenant")),
   175  					},
   176  					storeLabels: map[string]struct{}{"receive": {}, "tenant_id": {}, "thanos_replica": {}},
   177  				},
   178  				&eagerRespSet{
   179  					wg: &sync.WaitGroup{},
   180  					bufferedResponses: []*storepb.SeriesResponse{
   181  						storeSeriesResponse(t, labelsFromStrings("cluster", "beam-platform", "instance", "10.70.13.3:15692", "prometheus", "telemetry/observe-prometheus", "receive", "true", "tenant_id", "default-tenant")),
   182  						storeSeriesResponse(t, labelsFromStrings("cluster", "beam-platform", "instance", "10.70.5.3:15692", "prometheus", "telemetry/observe-prometheus", "receive", "true", "tenant_id", "default-tenant")),
   183  						storeSeriesResponse(t, labelsFromStrings("cluster", "beam-platform", "instance", "10.70.6.3:15692", "prometheus", "telemetry/observe-prometheus", "receive", "true", "tenant_id", "default-tenant")),
   184  					},
   185  					storeLabels: map[string]struct{}{"cluster": {}, "prometheus": {}, "prometheus_replica": {}, "receive": {}, "tenant_id": {}, "thanos_replica": {}, "thanos_ruler_replica": {}},
   186  				},
   187  			},
   188  			exp: []*storepb.SeriesResponse{
   189  				storeSeriesResponse(t, labelsFromStrings("cluster", "beam-platform", "instance", "10.70.13.3:15692", "prometheus", "telemetry/observe-prometheus", "receive", "true", "tenant_id", "default-tenant")),
   190  				storeSeriesResponse(t, labelsFromStrings("cluster", "beam-platform", "instance", "10.70.13.3:15692", "prometheus", "telemetry/observe-prometheus", "receive", "true", "tenant_id", "default-tenant")),
   191  				storeSeriesResponse(t, labelsFromStrings("cluster", "beam-platform", "instance", "10.70.5.3:15692", "prometheus", "telemetry/observe-prometheus", "receive", "true", "tenant_id", "default-tenant")),
   192  				storeSeriesResponse(t, labelsFromStrings("cluster", "beam-platform", "instance", "10.70.5.3:15692", "prometheus", "telemetry/observe-prometheus", "receive", "true", "tenant_id", "default-tenant")),
   193  				storeSeriesResponse(t, labelsFromStrings("cluster", "beam-platform", "instance", "10.70.6.3:15692", "prometheus", "telemetry/observe-prometheus", "receive", "true", "tenant_id", "default-tenant")),
   194  				storeSeriesResponse(t, labelsFromStrings("cluster", "beam-platform", "instance", "10.70.6.3:15692", "prometheus", "telemetry/observe-prometheus", "receive", "true", "tenant_id", "default-tenant")),
   195  			},
   196  		},
   197  	} {
   198  		t.Run(tcase.title, func(t *testing.T) {
   199  			h := NewProxyResponseHeap(tcase.input...)
   200  			if !h.Empty() {
   201  				got := []*storepb.SeriesResponse{h.At()}
   202  				for h.Next() {
   203  					got = append(got, h.At())
   204  				}
   205  				testutil.Equals(t, tcase.exp, got)
   206  			}
   207  		})
   208  	}
   209  }
   210  
   211  func TestSortWithoutLabels(t *testing.T) {
   212  	for _, tcase := range []struct {
   213  		input       []*storepb.SeriesResponse
   214  		exp         []*storepb.SeriesResponse
   215  		dedupLabels map[string]struct{}
   216  	}{
   217  		// Single deduplication label.
   218  		{
   219  			input: []*storepb.SeriesResponse{
   220  				storeSeriesResponse(t, labelsFromStrings("a", "1", "b", "replica-1", "c", "3")),
   221  				storeSeriesResponse(t, labelsFromStrings("a", "1", "b", "replica-1", "c", "3", "d", "4")),
   222  				storeSeriesResponse(t, labelsFromStrings("a", "1", "b", "replica-1", "c", "4")),
   223  				storeSeriesResponse(t, labelsFromStrings("a", "1", "b", "replica-2", "c", "3")),
   224  			},
   225  			exp: []*storepb.SeriesResponse{
   226  				storeSeriesResponse(t, labelsFromStrings("a", "1", "c", "3")),
   227  				storeSeriesResponse(t, labelsFromStrings("a", "1", "c", "3")),
   228  				storeSeriesResponse(t, labelsFromStrings("a", "1", "c", "3", "d", "4")),
   229  				storeSeriesResponse(t, labelsFromStrings("a", "1", "c", "4")),
   230  			},
   231  			dedupLabels: map[string]struct{}{"b": {}},
   232  		},
   233  		// Multi deduplication labels.
   234  		{
   235  			input: []*storepb.SeriesResponse{
   236  				storeSeriesResponse(t, labelsFromStrings("a", "1", "b", "replica-1", "b1", "replica-1", "c", "3")),
   237  				storeSeriesResponse(t, labelsFromStrings("a", "1", "b", "replica-1", "b1", "replica-1", "c", "3", "d", "4")),
   238  				storeSeriesResponse(t, labelsFromStrings("a", "1", "b", "replica-1", "b1", "replica-1", "c", "4")),
   239  				storeSeriesResponse(t, labelsFromStrings("a", "1", "b", "replica-1", "b1", "replica-2", "c", "3")),
   240  				storeSeriesResponse(t, labelsFromStrings("a", "1", "b", "replica-2", "c", "3")),
   241  			},
   242  			exp: []*storepb.SeriesResponse{
   243  				storeSeriesResponse(t, labelsFromStrings("a", "1", "c", "3")),
   244  				storeSeriesResponse(t, labelsFromStrings("a", "1", "c", "3")),
   245  				storeSeriesResponse(t, labelsFromStrings("a", "1", "c", "3")),
   246  				storeSeriesResponse(t, labelsFromStrings("a", "1", "c", "3", "d", "4")),
   247  				storeSeriesResponse(t, labelsFromStrings("a", "1", "c", "4")),
   248  			},
   249  			dedupLabels: map[string]struct{}{"b": {}, "b1": {}},
   250  		},
   251  		// Pushdown label at the end.
   252  		{
   253  			input: []*storepb.SeriesResponse{
   254  				storeSeriesResponse(t, labelsFromStrings("a", "1", "b", "replica-1", "c", "3")),
   255  				storeSeriesResponse(t, labelsFromStrings("a", "1", "b", "replica-1", "c", "3", "d", "4")),
   256  				storeSeriesResponse(t, labelsFromStrings("a", "1", "b", "replica-1", "c", "4", dedup.PushdownMarker.Name, dedup.PushdownMarker.Value)),
   257  				storeSeriesResponse(t, labelsFromStrings("a", "1", "b", "replica-2", "c", "3")),
   258  			},
   259  			exp: []*storepb.SeriesResponse{
   260  				storeSeriesResponse(t, labelsFromStrings("a", "1", "c", "3")),
   261  				storeSeriesResponse(t, labelsFromStrings("a", "1", "c", "3")),
   262  				storeSeriesResponse(t, labelsFromStrings("a", "1", "c", "3", "d", "4")),
   263  				storeSeriesResponse(t, labelsFromStrings("a", "1", "c", "4", dedup.PushdownMarker.Name, dedup.PushdownMarker.Value)),
   264  			},
   265  			dedupLabels: map[string]struct{}{"b": {}},
   266  		},
   267  		// Non series responses mixed.
   268  		{
   269  			input: []*storepb.SeriesResponse{
   270  				storeSeriesResponse(t, labelsFromStrings("a", "1", "b", "replica-1", "c", "3")),
   271  				storeSeriesResponse(t, labelsFromStrings("a", "1", "b", "replica-1", "c", "3", "d", "4")),
   272  				storepb.NewWarnSeriesResponse(errors.Newf("yolo")),
   273  				storeSeriesResponse(t, labelsFromStrings("a", "1", "b", "replica-1", "c", "4")),
   274  				storeSeriesResponse(t, labelsFromStrings("a", "1", "b", "replica-2", "c", "3")),
   275  			},
   276  			exp: []*storepb.SeriesResponse{
   277  				storepb.NewWarnSeriesResponse(errors.Newf("yolo")),
   278  				storeSeriesResponse(t, labelsFromStrings("a", "1", "c", "3")),
   279  				storeSeriesResponse(t, labelsFromStrings("a", "1", "c", "3")),
   280  				storeSeriesResponse(t, labelsFromStrings("a", "1", "c", "3", "d", "4")),
   281  				storeSeriesResponse(t, labelsFromStrings("a", "1", "c", "4")),
   282  			},
   283  			dedupLabels: map[string]struct{}{"b": {}},
   284  		},
   285  		// Longer series.
   286  		{
   287  			input: []*storepb.SeriesResponse{
   288  				storeSeriesResponse(t, labels.FromStrings(
   289  					"__name__", "gitlab_transaction_cache_read_hit_count_total", "action", "widget.json", "controller", "Projects::MergeRequests::ContentController", "env", "gprd", "environment",
   290  					"gprd", "fqdn", "web-08-sv-gprd.c.gitlab-production.internal", "instance", "web-08-sv-gprd.c.gitlab-production.internal:8083", "job", "gitlab-rails", "monitor", "app", "provider",
   291  					"gcp", "region", "us-east", "replica", "01", "shard", "default", "stage", "main", "tier", "sv", "type", "web",
   292  				)),
   293  			},
   294  			exp: []*storepb.SeriesResponse{
   295  				storeSeriesResponse(t, labels.FromStrings(
   296  					// No replica label anymore.
   297  					"__name__", "gitlab_transaction_cache_read_hit_count_total", "action", "widget.json", "controller", "Projects::MergeRequests::ContentController", "env", "gprd", "environment",
   298  					"gprd", "fqdn", "web-08-sv-gprd.c.gitlab-production.internal", "instance", "web-08-sv-gprd.c.gitlab-production.internal:8083", "job", "gitlab-rails", "monitor", "app", "provider",
   299  					"gcp", "region", "us-east", "shard", "default", "stage", "main", "tier", "sv", "type", "web",
   300  				)),
   301  			},
   302  			dedupLabels: map[string]struct{}{"replica": {}},
   303  		},
   304  	} {
   305  		t.Run("", func(t *testing.T) {
   306  			sortWithoutLabels(tcase.input, tcase.dedupLabels)
   307  			testutil.Equals(t, tcase.exp, tcase.input)
   308  		})
   309  	}
   310  }
   311  
   312  // labelsFromStrings is like labels.FromString, but it does not sort the input.
   313  func labelsFromStrings(ss ...string) labels.Labels {
   314  	if len(ss)%2 != 0 {
   315  		panic("invalid number of strings")
   316  	}
   317  	res := make(labels.Labels, 0, len(ss)/2)
   318  	for i := 0; i < len(ss); i += 2 {
   319  		res = append(res, labels.Label{Name: ss[i], Value: ss[i+1]})
   320  	}
   321  
   322  	return res
   323  }
   324  
   325  func BenchmarkSortWithoutLabels(b *testing.B) {
   326  	resps := make([]*storepb.SeriesResponse, 1e4)
   327  	labelsToRemove := map[string]struct{}{
   328  		"a": {}, "b": {},
   329  	}
   330  
   331  	b.ReportAllocs()
   332  	b.ResetTimer()
   333  	for i := 0; i < b.N; i++ {
   334  		b.StopTimer()
   335  		for i := 0; i < 1e4; i++ {
   336  			resps[i] = storeSeriesResponse(b, labels.FromStrings("a", "1", "b", "replica-1", "c", "replica-1", "d", "1"))
   337  		}
   338  		b.StartTimer()
   339  		sortWithoutLabels(resps, labelsToRemove)
   340  	}
   341  }