github.com/thanos-io/thanos@v0.32.5/pkg/receive/receive_test.go (about)

     1  // Copyright (c) The Thanos Authors.
     2  // Licensed under the Apache License 2.0.
     3  
     4  package receive
     5  
     6  import (
     7  	"testing"
     8  	"time"
     9  
    10  	"github.com/go-kit/log"
    11  
    12  	"github.com/stretchr/testify/require"
    13  
    14  	"github.com/prometheus/client_golang/prometheus"
    15  
    16  	"github.com/prometheus/prometheus/model/labels"
    17  	"github.com/prometheus/prometheus/tsdb"
    18  
    19  	"github.com/thanos-io/objstore"
    20  
    21  	"github.com/thanos-io/thanos/pkg/block/metadata"
    22  	"github.com/thanos-io/thanos/pkg/store"
    23  	"github.com/thanos-io/thanos/pkg/store/labelpb"
    24  	"github.com/thanos-io/thanos/pkg/testutil/custom"
    25  )
    26  
    27  func TestMain(m *testing.M) {
    28  	custom.TolerantVerifyLeakMain(m)
    29  }
    30  
    31  func TestAddingExternalLabelsForTenants(t *testing.T) {
    32  	for _, tc := range []struct {
    33  		name                      string
    34  		cfg                       []HashringConfig
    35  		expectedExternalLabelSets []labels.Labels
    36  	}{
    37  		{
    38  			name: "One tenant - No labels",
    39  			cfg: []HashringConfig{
    40  				{
    41  					Endpoints: []Endpoint{{Address: "node1"}},
    42  					Tenants:   []string{"tenant1"},
    43  				},
    44  			},
    45  			expectedExternalLabelSets: []labels.Labels{
    46  				labels.FromStrings("replica", "test", "tenant_id", "tenant1"),
    47  			},
    48  		},
    49  		{
    50  			name: "One tenant - One label",
    51  			cfg: []HashringConfig{
    52  				{
    53  					Endpoints: []Endpoint{{Address: "node1"}},
    54  					Tenants:   []string{"tenant1"},
    55  					ExternalLabels: map[string]string{
    56  						"name1": "value1",
    57  					},
    58  				},
    59  			},
    60  			expectedExternalLabelSets: []labels.Labels{
    61  				labels.FromStrings("name1", "value1", "replica", "test", "tenant_id", "tenant1"),
    62  			},
    63  		},
    64  		{
    65  			name: "One tenant - Multiple labels",
    66  			cfg: []HashringConfig{
    67  				{
    68  					Endpoints: []Endpoint{{Address: "node1"}},
    69  					Tenants:   []string{"tenant1"},
    70  					ExternalLabels: map[string]string{
    71  						"name1": "value1",
    72  						"name2": "value2",
    73  						"name3": "value3",
    74  					},
    75  				},
    76  			},
    77  			expectedExternalLabelSets: []labels.Labels{
    78  				labels.FromStrings("name1", "value1", "name2", "value2", "name3", "value3",
    79  					"replica", "test", "tenant_id", "tenant1"),
    80  			},
    81  		},
    82  		{
    83  			name: "Multiple tenants - No labels",
    84  			cfg: []HashringConfig{
    85  				{
    86  					Endpoints: []Endpoint{{Address: "node1"}},
    87  					Tenants:   []string{"tenant1", "tenant2", "tenant3"},
    88  				},
    89  			},
    90  			expectedExternalLabelSets: []labels.Labels{
    91  				labels.FromStrings("replica", "test", "tenant_id", "tenant1"),
    92  				labels.FromStrings("replica", "test", "tenant_id", "tenant2"),
    93  				labels.FromStrings("replica", "test", "tenant_id", "tenant3"),
    94  			},
    95  		},
    96  		{
    97  			name: "Multiple tenants - One label",
    98  			cfg: []HashringConfig{
    99  				{
   100  					Endpoints: []Endpoint{{Address: "node1"}},
   101  					Tenants:   []string{"tenant1", "tenant2", "tenant3"},
   102  					ExternalLabels: map[string]string{
   103  						"name1": "value1",
   104  					},
   105  				},
   106  			},
   107  			expectedExternalLabelSets: []labels.Labels{
   108  				labels.FromStrings("name1", "value1", "replica", "test", "tenant_id", "tenant1"),
   109  				labels.FromStrings("name1", "value1", "replica", "test", "tenant_id", "tenant2"),
   110  				labels.FromStrings("name1", "value1", "replica", "test", "tenant_id", "tenant3"),
   111  			},
   112  		},
   113  		{
   114  			name: "Multiple tenants - Multiple labels",
   115  			cfg: []HashringConfig{
   116  				{
   117  					Endpoints: []Endpoint{{Address: "node1"}},
   118  					Tenants:   []string{"tenant1", "tenant2", "tenant3"},
   119  					ExternalLabels: map[string]string{
   120  						"name3": "value3",
   121  						"name2": "value2",
   122  						"name1": "value1",
   123  					},
   124  				},
   125  			},
   126  			expectedExternalLabelSets: []labels.Labels{
   127  				labels.FromStrings("name1", "value1", "name2", "value2", "name3", "value3",
   128  					"replica", "test", "tenant_id", "tenant1"),
   129  				labels.FromStrings("name1", "value1", "name2", "value2", "name3", "value3",
   130  					"replica", "test", "tenant_id", "tenant2"),
   131  				labels.FromStrings("name1", "value1", "name2", "value2", "name3", "value3",
   132  					"replica", "test", "tenant_id", "tenant3"),
   133  			},
   134  		},
   135  		{
   136  			name: "Multiple hashrings - No repeated tenants",
   137  			cfg: []HashringConfig{
   138  				{
   139  					Endpoints: []Endpoint{{Address: "node1"}},
   140  					Tenants:   []string{"tenant1", "tenant2", "tenant3"},
   141  					ExternalLabels: map[string]string{
   142  						"name1": "value1",
   143  						"name2": "value2",
   144  						"name3": "value3",
   145  					},
   146  				},
   147  				{
   148  					Endpoints: []Endpoint{{Address: "node2"}},
   149  					Tenants:   []string{"tenant4", "tenant5", "tenant6"},
   150  					ExternalLabels: map[string]string{
   151  						"name6": "value6",
   152  						"name5": "value5",
   153  						"name4": "value4",
   154  					},
   155  				},
   156  			},
   157  			expectedExternalLabelSets: []labels.Labels{
   158  				labels.FromStrings("name1", "value1", "name2", "value2", "name3", "value3",
   159  					"replica", "test", "tenant_id", "tenant1"),
   160  				labels.FromStrings("name1", "value1", "name2", "value2", "name3", "value3",
   161  					"replica", "test", "tenant_id", "tenant2"),
   162  				labels.FromStrings("name1", "value1", "name2", "value2", "name3", "value3",
   163  					"replica", "test", "tenant_id", "tenant3"),
   164  				labels.FromStrings("name4", "value4", "name5", "value5", "name6", "value6",
   165  					"replica", "test", "tenant_id", "tenant4"),
   166  				labels.FromStrings("name4", "value4", "name5", "value5", "name6", "value6",
   167  					"replica", "test", "tenant_id", "tenant5"),
   168  				labels.FromStrings("name4", "value4", "name5", "value5", "name6", "value6",
   169  					"replica", "test", "tenant_id", "tenant6"),
   170  			},
   171  		},
   172  		{
   173  			name: "Multiple hashrings - One repeated tenant",
   174  			cfg: []HashringConfig{
   175  				{
   176  					Endpoints: []Endpoint{{Address: "node1"}},
   177  					Tenants:   []string{"tenant1", "tenant2", "tenant3"},
   178  					ExternalLabels: map[string]string{
   179  						"name3": "value3",
   180  						"name2": "value2",
   181  						"name1": "value1",
   182  					},
   183  				},
   184  				{
   185  					Endpoints: []Endpoint{{Address: "node2"}},
   186  					Tenants:   []string{"tenant4", "tenant5", "tenant1"},
   187  					ExternalLabels: map[string]string{
   188  						"name4": "value4",
   189  						"name5": "value5",
   190  						"name6": "value6",
   191  					},
   192  				},
   193  			},
   194  			expectedExternalLabelSets: []labels.Labels{
   195  				labels.FromStrings("name1", "value1", "name2", "value2", "name3", "value3",
   196  					"replica", "test", "tenant_id", "tenant1"),
   197  				labels.FromStrings("name1", "value1", "name2", "value2", "name3", "value3",
   198  					"replica", "test", "tenant_id", "tenant2"),
   199  				labels.FromStrings("name1", "value1", "name2", "value2", "name3", "value3",
   200  					"replica", "test", "tenant_id", "tenant3"),
   201  				labels.FromStrings("name4", "value4", "name5", "value5", "name6", "value6",
   202  					"replica", "test", "tenant_id", "tenant4"),
   203  				labels.FromStrings("name4", "value4", "name5", "value5", "name6", "value6",
   204  					"replica", "test", "tenant_id", "tenant5"),
   205  			},
   206  		},
   207  	} {
   208  		t.Run(tc.name, func(t *testing.T) {
   209  			m := initializeMultiTSDB(t.TempDir())
   210  
   211  			err := m.SetHashringConfig(tc.cfg)
   212  			require.NoError(t, err)
   213  
   214  			for _, c := range tc.cfg {
   215  				for _, tenantId := range c.Tenants {
   216  					if m.tenants[tenantId] == nil {
   217  						err = appendSample(m, tenantId, time.Now())
   218  						require.NoError(t, err)
   219  					}
   220  				}
   221  			}
   222  
   223  			err = m.Open()
   224  			require.NoError(t, err)
   225  
   226  			storeClients := m.TSDBLocalClients()
   227  			require.Equal(t, len(tc.expectedExternalLabelSets), len(storeClients))
   228  
   229  			setOfExpectedClientLabelSets, setOfActualClientLabelSets := setupSetsOfExpectedAndActualStoreClientLabelSets(
   230  				tc.expectedExternalLabelSets, storeClients)
   231  
   232  			for _, cls := range setOfActualClientLabelSets {
   233  				require.Contains(t, setOfExpectedClientLabelSets, cls)
   234  			}
   235  
   236  			err = m.Flush()
   237  			require.NoError(t, err)
   238  
   239  			err = m.Close()
   240  			require.NoError(t, err)
   241  		})
   242  	}
   243  }
   244  
   245  func TestLabelSetsOfTenantsWhenAddingTenants(t *testing.T) {
   246  	initialConfig := []HashringConfig{
   247  		{
   248  			Endpoints: []Endpoint{{Address: "node1"}},
   249  			Tenants:   []string{"tenant1", "tenant2", "tenant3"},
   250  			ExternalLabels: map[string]string{
   251  				"name1": "value1",
   252  				"name2": "value2",
   253  				"name3": "value3",
   254  			},
   255  		},
   256  	}
   257  	initialExpectedExternalLabelSets := []labels.Labels{
   258  		labels.FromStrings("name1", "value1", "name2", "value2", "name3", "value3",
   259  			"replica", "test", "tenant_id", "tenant1"),
   260  		labels.FromStrings("name1", "value1", "name2", "value2", "name3", "value3",
   261  			"replica", "test", "tenant_id", "tenant2"),
   262  		labels.FromStrings("name1", "value1", "name2", "value2", "name3", "value3",
   263  			"replica", "test", "tenant_id", "tenant3"),
   264  	}
   265  
   266  	changedConfig := []HashringConfig{
   267  		{
   268  			Endpoints: []Endpoint{{Address: "node1"}},
   269  			Tenants:   []string{"tenant1", "tenant2", "tenant3", "tenant4", "tenant5"},
   270  			ExternalLabels: map[string]string{
   271  				"name1": "value1",
   272  				"name2": "value2",
   273  				"name3": "value3",
   274  			},
   275  		},
   276  	}
   277  	changedExpectedExternalLabelSets := []labels.Labels{
   278  		labels.FromStrings("name1", "value1", "name2", "value2", "name3", "value3",
   279  			"replica", "test", "tenant_id", "tenant1"),
   280  		labels.FromStrings("name1", "value1", "name2", "value2", "name3", "value3",
   281  			"replica", "test", "tenant_id", "tenant2"),
   282  		labels.FromStrings("name1", "value1", "name2", "value2", "name3", "value3",
   283  			"replica", "test", "tenant_id", "tenant3"),
   284  		labels.FromStrings("name1", "value1", "name2", "value2", "name3", "value3",
   285  			"replica", "test", "tenant_id", "tenant4"),
   286  		labels.FromStrings("name1", "value1", "name2", "value2", "name3", "value3",
   287  			"replica", "test", "tenant_id", "tenant5"),
   288  	}
   289  
   290  	t.Run("Adding tenants", func(t *testing.T) {
   291  		m := initializeMultiTSDB(t.TempDir())
   292  
   293  		err := m.SetHashringConfig(initialConfig)
   294  		require.NoError(t, err)
   295  
   296  		for _, c := range initialConfig {
   297  			for _, tenantId := range c.Tenants {
   298  				if m.tenants[tenantId] == nil {
   299  					err = appendSample(m, tenantId, time.Now())
   300  					require.NoError(t, err)
   301  				}
   302  			}
   303  		}
   304  
   305  		err = m.Open()
   306  		require.NoError(t, err)
   307  
   308  		initialStoreClients := m.TSDBLocalClients()
   309  		require.Equal(t, len(initialExpectedExternalLabelSets), len(initialStoreClients))
   310  
   311  		initialSetOfExpectedClientLabelSets, initialSetOfActualClientLabelSets := setupSetsOfExpectedAndActualStoreClientLabelSets(
   312  			initialExpectedExternalLabelSets, initialStoreClients)
   313  
   314  		for _, cls := range initialSetOfActualClientLabelSets {
   315  			require.Contains(t, initialSetOfExpectedClientLabelSets, cls)
   316  		}
   317  
   318  		err = m.SetHashringConfig(changedConfig)
   319  		require.NoError(t, err)
   320  
   321  		for _, c := range changedConfig {
   322  			for _, tenantId := range c.Tenants {
   323  				if m.tenants[tenantId] == nil {
   324  					err = appendSample(m, tenantId, time.Now())
   325  					require.NoError(t, err)
   326  				}
   327  			}
   328  		}
   329  
   330  		err = m.Flush()
   331  		require.NoError(t, err)
   332  
   333  		err = m.Open()
   334  		require.NoError(t, err)
   335  
   336  		changedStoreClients := m.TSDBLocalClients()
   337  		require.Equal(t, len(changedExpectedExternalLabelSets), len(changedStoreClients))
   338  
   339  		changedSetOfExpectedClientLabelSets, changedSetOfActualClientLabelSets := setupSetsOfExpectedAndActualStoreClientLabelSets(
   340  			changedExpectedExternalLabelSets, changedStoreClients)
   341  
   342  		for _, cls := range changedSetOfActualClientLabelSets {
   343  			require.Contains(t, changedSetOfExpectedClientLabelSets, cls)
   344  		}
   345  
   346  		err = m.Flush()
   347  		require.NoError(t, err)
   348  
   349  		err = m.Close()
   350  		require.NoError(t, err)
   351  	})
   352  }
   353  
   354  func TestLabelSetsOfTenantsWhenChangingLabels(t *testing.T) {
   355  	initialConfig := []HashringConfig{
   356  		{
   357  			Endpoints: []Endpoint{{Address: "node1"}},
   358  			Tenants:   []string{"tenant1", "tenant2", "tenant3"},
   359  			ExternalLabels: map[string]string{
   360  				"name1": "value1",
   361  				"name2": "value2",
   362  				"name3": "value3",
   363  			},
   364  		},
   365  		{
   366  			Endpoints: []Endpoint{{Address: "node2"}},
   367  			Tenants:   []string{"tenant4", "tenant5", "tenant6"},
   368  			ExternalLabels: map[string]string{
   369  				"name6": "value6",
   370  				"name5": "value5",
   371  				"name4": "value4",
   372  			},
   373  		},
   374  	}
   375  	initialExpectedExternalLabelSets := []labels.Labels{
   376  		labels.FromStrings("name1", "value1", "name2", "value2", "name3", "value3",
   377  			"replica", "test", "tenant_id", "tenant1"),
   378  		labels.FromStrings("name1", "value1", "name2", "value2", "name3", "value3",
   379  			"replica", "test", "tenant_id", "tenant2"),
   380  		labels.FromStrings("name1", "value1", "name2", "value2", "name3", "value3",
   381  			"replica", "test", "tenant_id", "tenant3"),
   382  		labels.FromStrings("name4", "value4", "name5", "value5", "name6", "value6",
   383  			"replica", "test", "tenant_id", "tenant4"),
   384  		labels.FromStrings("name4", "value4", "name5", "value5", "name6", "value6",
   385  			"replica", "test", "tenant_id", "tenant5"),
   386  		labels.FromStrings("name4", "value4", "name5", "value5", "name6", "value6",
   387  			"replica", "test", "tenant_id", "tenant6"),
   388  	}
   389  
   390  	for _, tc := range []struct {
   391  		name                             string
   392  		changedConfig                    []HashringConfig
   393  		changedExpectedExternalLabelSets []labels.Labels
   394  	}{
   395  		{
   396  			name: "Adding labels",
   397  			changedConfig: []HashringConfig{
   398  				{
   399  					Endpoints: []Endpoint{{Address: "node1"}},
   400  					Tenants:   []string{"tenant1", "tenant2", "tenant3"},
   401  					ExternalLabels: map[string]string{
   402  						"name1": "value1",
   403  						"name2": "value2",
   404  						"name3": "value3",
   405  						"name4": "value4",
   406  					},
   407  				},
   408  				{
   409  					Endpoints: []Endpoint{{Address: "node2"}},
   410  					Tenants:   []string{"tenant4", "tenant5", "tenant6"},
   411  					ExternalLabels: map[string]string{
   412  						"name4": "value4",
   413  						"name5": "value5",
   414  						"name6": "value6",
   415  						"name7": "value7",
   416  					},
   417  				},
   418  			},
   419  			changedExpectedExternalLabelSets: []labels.Labels{
   420  				labels.FromStrings("name1", "value1", "name2", "value2", "name3", "value3", "name4", "value4",
   421  					"replica", "test", "tenant_id", "tenant1"),
   422  				labels.FromStrings("name1", "value1", "name2", "value2", "name3", "value3", "name4", "value4",
   423  					"replica", "test", "tenant_id", "tenant2"),
   424  				labels.FromStrings("name1", "value1", "name2", "value2", "name3", "value3", "name4", "value4",
   425  					"replica", "test", "tenant_id", "tenant3"),
   426  				labels.FromStrings("name4", "value4", "name5", "value5", "name6", "value6", "name7", "value7",
   427  					"replica", "test", "tenant_id", "tenant4"),
   428  				labels.FromStrings("name4", "value4", "name5", "value5", "name6", "value6", "name7", "value7",
   429  					"replica", "test", "tenant_id", "tenant5"),
   430  				labels.FromStrings("name4", "value4", "name5", "value5", "name6", "value6", "name7", "value7",
   431  					"replica", "test", "tenant_id", "tenant6"),
   432  			},
   433  		},
   434  		{
   435  			name: "Deleting some labels",
   436  			changedConfig: []HashringConfig{
   437  				{
   438  					Endpoints: []Endpoint{{Address: "node1"}},
   439  					Tenants:   []string{"tenant1", "tenant2", "tenant3"},
   440  					ExternalLabels: map[string]string{
   441  						"name1": "value1",
   442  						"name2": "value2",
   443  					},
   444  				},
   445  				{
   446  					Endpoints: []Endpoint{{Address: "node2"}},
   447  					Tenants:   []string{"tenant4", "tenant5", "tenant6"},
   448  					ExternalLabels: map[string]string{
   449  						"name4": "value4",
   450  						"name5": "value5",
   451  					},
   452  				},
   453  			},
   454  			changedExpectedExternalLabelSets: []labels.Labels{
   455  				labels.FromStrings("name1", "value1", "name2", "value2",
   456  					"replica", "test", "tenant_id", "tenant1"),
   457  				labels.FromStrings("name1", "value1", "name2", "value2",
   458  					"replica", "test", "tenant_id", "tenant2"),
   459  				labels.FromStrings("name1", "value1", "name2", "value2",
   460  					"replica", "test", "tenant_id", "tenant3"),
   461  				labels.FromStrings("name4", "value4", "name5", "value5",
   462  					"replica", "test", "tenant_id", "tenant4"),
   463  				labels.FromStrings("name4", "value4", "name5", "value5",
   464  					"replica", "test", "tenant_id", "tenant5"),
   465  				labels.FromStrings("name4", "value4", "name5", "value5",
   466  					"replica", "test", "tenant_id", "tenant6"),
   467  			},
   468  		},
   469  		{
   470  			name: "Deleting all labels",
   471  			changedConfig: []HashringConfig{
   472  				{
   473  					Endpoints: []Endpoint{{Address: "node1"}},
   474  					Tenants:   []string{"tenant1", "tenant2", "tenant3"},
   475  				},
   476  				{
   477  					Endpoints: []Endpoint{{Address: "node2"}},
   478  					Tenants:   []string{"tenant4", "tenant5", "tenant6"},
   479  				},
   480  			},
   481  			changedExpectedExternalLabelSets: []labels.Labels{
   482  				labels.FromStrings("replica", "test", "tenant_id", "tenant1"),
   483  				labels.FromStrings("replica", "test", "tenant_id", "tenant2"),
   484  				labels.FromStrings("replica", "test", "tenant_id", "tenant3"),
   485  				labels.FromStrings("replica", "test", "tenant_id", "tenant4"),
   486  				labels.FromStrings("replica", "test", "tenant_id", "tenant5"),
   487  				labels.FromStrings("replica", "test", "tenant_id", "tenant6"),
   488  			},
   489  		},
   490  		{
   491  			name: "Changing values of some labels",
   492  			changedConfig: []HashringConfig{
   493  				{
   494  					Endpoints: []Endpoint{{Address: "node1"}},
   495  					Tenants:   []string{"tenant1", "tenant2", "tenant3"},
   496  					ExternalLabels: map[string]string{
   497  						"name1": "value3",
   498  						"name2": "value2",
   499  						"name3": "value3",
   500  					},
   501  				},
   502  				{
   503  					Endpoints: []Endpoint{{Address: "node2"}},
   504  					Tenants:   []string{"tenant4", "tenant5", "tenant6"},
   505  					ExternalLabels: map[string]string{
   506  						"name4": "value6",
   507  						"name5": "value5",
   508  						"name6": "value6",
   509  					},
   510  				},
   511  			},
   512  			changedExpectedExternalLabelSets: []labels.Labels{
   513  				labels.FromStrings("name1", "value3", "name2", "value2", "name3", "value3",
   514  					"replica", "test", "tenant_id", "tenant1"),
   515  				labels.FromStrings("name1", "value3", "name2", "value2", "name3", "value3",
   516  					"replica", "test", "tenant_id", "tenant2"),
   517  				labels.FromStrings("name1", "value3", "name2", "value2", "name3", "value3",
   518  					"replica", "test", "tenant_id", "tenant3"),
   519  				labels.FromStrings("name4", "value6", "name5", "value5", "name6", "value6",
   520  					"replica", "test", "tenant_id", "tenant4"),
   521  				labels.FromStrings("name4", "value6", "name5", "value5", "name6", "value6",
   522  					"replica", "test", "tenant_id", "tenant5"),
   523  				labels.FromStrings("name4", "value6", "name5", "value5", "name6", "value6",
   524  					"replica", "test", "tenant_id", "tenant6"),
   525  			},
   526  		},
   527  	} {
   528  		t.Run(tc.name, func(t *testing.T) {
   529  			m := initializeMultiTSDB(t.TempDir())
   530  
   531  			err := m.SetHashringConfig(initialConfig)
   532  			require.NoError(t, err)
   533  
   534  			for _, c := range initialConfig {
   535  				for _, tenantId := range c.Tenants {
   536  					if m.tenants[tenantId] == nil {
   537  						err = appendSample(m, tenantId, time.Now())
   538  						require.NoError(t, err)
   539  					}
   540  				}
   541  			}
   542  
   543  			err = m.Open()
   544  			require.NoError(t, err)
   545  
   546  			initialStoreClients := m.TSDBLocalClients()
   547  			require.Equal(t, len(initialExpectedExternalLabelSets), len(initialStoreClients))
   548  
   549  			initialSetOfExpectedClientLabelSets, initialSetOfActualClientLabelSets := setupSetsOfExpectedAndActualStoreClientLabelSets(
   550  				initialExpectedExternalLabelSets, initialStoreClients)
   551  
   552  			for _, cls := range initialSetOfActualClientLabelSets {
   553  				require.Contains(t, initialSetOfExpectedClientLabelSets, cls)
   554  			}
   555  
   556  			err = m.SetHashringConfig(tc.changedConfig)
   557  			require.NoError(t, err)
   558  
   559  			err = m.Flush()
   560  			require.NoError(t, err)
   561  
   562  			err = m.Open()
   563  			require.NoError(t, err)
   564  
   565  			changedStoreClients := m.TSDBLocalClients()
   566  			require.Equal(t, len(tc.changedExpectedExternalLabelSets), len(changedStoreClients))
   567  
   568  			changedSetOfExpectedClientLabelSets, changedSetOfActualClientLabelSets := setupSetsOfExpectedAndActualStoreClientLabelSets(
   569  				tc.changedExpectedExternalLabelSets, changedStoreClients)
   570  
   571  			for _, cls := range changedSetOfActualClientLabelSets {
   572  				require.Contains(t, changedSetOfExpectedClientLabelSets, cls)
   573  			}
   574  
   575  			err = m.Flush()
   576  			require.NoError(t, err)
   577  
   578  			err = m.Close()
   579  			require.NoError(t, err)
   580  		})
   581  	}
   582  }
   583  
   584  func TestAddingLabelsWhenTenantAppearsInMultipleHashrings(t *testing.T) {
   585  	initialConfig := []HashringConfig{
   586  		{
   587  			Endpoints: []Endpoint{{Address: "node1"}},
   588  			Tenants:   []string{"tenant1", "tenant2", "tenant3"},
   589  			ExternalLabels: map[string]string{
   590  				"name3": "value3",
   591  				"name2": "value2",
   592  				"name1": "value1",
   593  			},
   594  		},
   595  		{
   596  			Endpoints: []Endpoint{{Address: "node2"}},
   597  			Tenants:   []string{"tenant4", "tenant5", "tenant1"},
   598  			ExternalLabels: map[string]string{
   599  				"name4": "value4",
   600  				"name5": "value5",
   601  				"name6": "value6",
   602  			},
   603  		},
   604  	}
   605  	initialExpectedExternalLabelSets := []labels.Labels{
   606  		labels.FromStrings("name1", "value1", "name2", "value2", "name3", "value3",
   607  			"replica", "test", "tenant_id", "tenant1"),
   608  		labels.FromStrings("name1", "value1", "name2", "value2", "name3", "value3",
   609  			"replica", "test", "tenant_id", "tenant2"),
   610  		labels.FromStrings("name1", "value1", "name2", "value2", "name3", "value3",
   611  			"replica", "test", "tenant_id", "tenant3"),
   612  		labels.FromStrings("name4", "value4", "name5", "value5", "name6", "value6",
   613  			"replica", "test", "tenant_id", "tenant4"),
   614  		labels.FromStrings("name4", "value4", "name5", "value5", "name6", "value6",
   615  			"replica", "test", "tenant_id", "tenant5"),
   616  	}
   617  
   618  	for _, tc := range []struct {
   619  		name                             string
   620  		changedConfig                    []HashringConfig
   621  		changedExpectedExternalLabelSets []labels.Labels
   622  	}{
   623  		{
   624  			name: "Adding labels in first hashring that tenant appears",
   625  			changedConfig: []HashringConfig{
   626  				{
   627  					Endpoints: []Endpoint{{Address: "node1"}},
   628  					Tenants:   []string{"tenant1", "tenant2", "tenant3"},
   629  					ExternalLabels: map[string]string{
   630  						"name1": "value1",
   631  						"name2": "value2",
   632  						"name3": "value3",
   633  						"name4": "value4",
   634  					},
   635  				},
   636  				{
   637  					Endpoints: []Endpoint{{Address: "node2"}},
   638  					Tenants:   []string{"tenant4", "tenant5", "tenant1"},
   639  					ExternalLabels: map[string]string{
   640  						"name4": "value4",
   641  						"name5": "value5",
   642  						"name6": "value6",
   643  					},
   644  				},
   645  			},
   646  			changedExpectedExternalLabelSets: []labels.Labels{
   647  				labels.FromStrings("name1", "value1", "name2", "value2", "name3", "value3", "name4", "value4",
   648  					"replica", "test", "tenant_id", "tenant1"),
   649  				labels.FromStrings("name1", "value1", "name2", "value2", "name3", "value3", "name4", "value4",
   650  					"replica", "test", "tenant_id", "tenant2"),
   651  				labels.FromStrings("name1", "value1", "name2", "value2", "name3", "value3", "name4", "value4",
   652  					"replica", "test", "tenant_id", "tenant3"),
   653  				labels.FromStrings("name4", "value4", "name5", "value5", "name6", "value6",
   654  					"replica", "test", "tenant_id", "tenant4"),
   655  				labels.FromStrings("name4", "value4", "name5", "value5", "name6", "value6",
   656  					"replica", "test", "tenant_id", "tenant5"),
   657  			},
   658  		},
   659  		{
   660  			name: "Adding labels in second hashring that tenant appears",
   661  			changedConfig: []HashringConfig{
   662  				{
   663  					Endpoints: []Endpoint{{Address: "node1"}},
   664  					Tenants:   []string{"tenant1", "tenant2", "tenant3"},
   665  					ExternalLabels: map[string]string{
   666  						"name1": "value1",
   667  						"name2": "value2",
   668  						"name3": "value3",
   669  					},
   670  				},
   671  				{
   672  					Endpoints: []Endpoint{{Address: "node2"}},
   673  					Tenants:   []string{"tenant4", "tenant5", "tenant1"},
   674  					ExternalLabels: map[string]string{
   675  						"name4": "value4",
   676  						"name5": "value5",
   677  						"name6": "value6",
   678  						"name7": "value7",
   679  					},
   680  				},
   681  			},
   682  			changedExpectedExternalLabelSets: []labels.Labels{
   683  				labels.FromStrings("name1", "value1", "name2", "value2", "name3", "value3",
   684  					"replica", "test", "tenant_id", "tenant1"),
   685  				labels.FromStrings("name1", "value1", "name2", "value2", "name3", "value3",
   686  					"replica", "test", "tenant_id", "tenant2"),
   687  				labels.FromStrings("name1", "value1", "name2", "value2", "name3", "value3",
   688  					"replica", "test", "tenant_id", "tenant3"),
   689  				labels.FromStrings("name4", "value4", "name5", "value5", "name6", "value6", "name7", "value7",
   690  					"replica", "test", "tenant_id", "tenant4"),
   691  				labels.FromStrings("name4", "value4", "name5", "value5", "name6", "value6", "name7", "value7",
   692  					"replica", "test", "tenant_id", "tenant5"),
   693  			},
   694  		},
   695  	} {
   696  		t.Run(tc.name, func(t *testing.T) {
   697  			m := initializeMultiTSDB(t.TempDir())
   698  
   699  			err := m.SetHashringConfig(initialConfig)
   700  			require.NoError(t, err)
   701  
   702  			for _, c := range initialConfig {
   703  				for _, tenantId := range c.Tenants {
   704  					if m.tenants[tenantId] == nil {
   705  						err = appendSample(m, tenantId, time.Now())
   706  						require.NoError(t, err)
   707  					}
   708  				}
   709  			}
   710  
   711  			err = m.Open()
   712  			require.NoError(t, err)
   713  
   714  			initialStoreClients := m.TSDBLocalClients()
   715  			require.Equal(t, len(initialExpectedExternalLabelSets), len(initialStoreClients))
   716  
   717  			initialSetOfExpectedClientLabelSets, initialSetOfActualClientLabelSets := setupSetsOfExpectedAndActualStoreClientLabelSets(
   718  				initialExpectedExternalLabelSets, initialStoreClients)
   719  
   720  			for _, cls := range initialSetOfActualClientLabelSets {
   721  				require.Contains(t, initialSetOfExpectedClientLabelSets, cls)
   722  			}
   723  
   724  			err = m.SetHashringConfig(tc.changedConfig)
   725  			require.NoError(t, err)
   726  
   727  			err = m.Flush()
   728  			require.NoError(t, err)
   729  
   730  			err = m.Open()
   731  			require.NoError(t, err)
   732  
   733  			changedStoreClients := m.TSDBLocalClients()
   734  			require.Equal(t, len(tc.changedExpectedExternalLabelSets), len(changedStoreClients))
   735  
   736  			changedSetOfExpectedClientLabelSets, changedSetOfActualClientLabelSets := setupSetsOfExpectedAndActualStoreClientLabelSets(
   737  				tc.changedExpectedExternalLabelSets, changedStoreClients)
   738  
   739  			for _, cls := range changedSetOfActualClientLabelSets {
   740  				require.Contains(t, changedSetOfExpectedClientLabelSets, cls)
   741  			}
   742  
   743  			err = m.Flush()
   744  			require.NoError(t, err)
   745  
   746  			err = m.Close()
   747  			require.NoError(t, err)
   748  		})
   749  	}
   750  }
   751  
   752  func TestReceiverLabelsNotOverwrittenByExternalLabels(t *testing.T) {
   753  	cfg := []HashringConfig{
   754  		{
   755  			Endpoints: []Endpoint{{Address: "node1"}},
   756  			Tenants:   []string{"tenant1"},
   757  			ExternalLabels: map[string]string{
   758  				"replica":   "0",
   759  				"tenant_id": "tenant2",
   760  				"name3":     "value3",
   761  			},
   762  		},
   763  	}
   764  	expectedExternalLabelSets := []labels.Labels{
   765  		labels.FromStrings("name3", "value3", "replica", "test", "tenant_id", "tenant1"),
   766  	}
   767  
   768  	t.Run("Receiver's labels not overwritten by external labels", func(t *testing.T) {
   769  		m := initializeMultiTSDB(t.TempDir())
   770  
   771  		err := m.SetHashringConfig(cfg)
   772  		require.NoError(t, err)
   773  
   774  		for _, c := range cfg {
   775  			for _, tenantId := range c.Tenants {
   776  				if m.tenants[tenantId] == nil {
   777  					err = appendSample(m, tenantId, time.Now())
   778  					require.NoError(t, err)
   779  				}
   780  			}
   781  		}
   782  
   783  		err = m.Open()
   784  		require.NoError(t, err)
   785  
   786  		storeClients := m.TSDBLocalClients()
   787  		require.Equal(t, len(expectedExternalLabelSets), len(storeClients))
   788  
   789  		setOfExpectedClientLabelSets, setOfActualClientLabelSets := setupSetsOfExpectedAndActualStoreClientLabelSets(
   790  			expectedExternalLabelSets, storeClients)
   791  
   792  		for _, cls := range setOfActualClientLabelSets {
   793  			require.Contains(t, setOfExpectedClientLabelSets, cls)
   794  		}
   795  
   796  		err = m.Flush()
   797  		require.NoError(t, err)
   798  
   799  		err = m.Close()
   800  		require.NoError(t, err)
   801  	})
   802  }
   803  
   804  func initializeMultiTSDB(dir string) *MultiTSDB {
   805  	var bucket objstore.Bucket
   806  
   807  	m := NewMultiTSDB(dir, log.NewNopLogger(), prometheus.NewRegistry(),
   808  		&tsdb.Options{
   809  			MinBlockDuration:  (2 * time.Hour).Milliseconds(),
   810  			MaxBlockDuration:  (2 * time.Hour).Milliseconds(),
   811  			RetentionDuration: (6 * time.Hour).Milliseconds(),
   812  		},
   813  		labels.FromStrings("replica", "test"),
   814  		"tenant_id",
   815  		bucket,
   816  		false,
   817  		metadata.NoneFunc,
   818  	)
   819  
   820  	return m
   821  }
   822  
   823  // Set up expected set of label sets of store clients from expected external label sets of all tenants
   824  // and set up actual set of label sets from actual store clients.
   825  func setupSetsOfExpectedAndActualStoreClientLabelSets(
   826  	expectedExternalLabelSets []labels.Labels, actualStoreClients []store.Client) ([][]labels.Labels, [][]labels.Labels) {
   827  	setOfExpectedClientLabelSets := make([][]labels.Labels, len(expectedExternalLabelSets))
   828  	setOfActualClientLabelSets := make([][]labels.Labels, len(actualStoreClients))
   829  
   830  	for i := 0; i < len(actualStoreClients); i++ {
   831  		testStore := store.TSDBStore{}
   832  		testStore.SetExtLset(expectedExternalLabelSets[i])
   833  
   834  		expectedClientLabelSets := labelpb.ZLabelSetsToPromLabelSets(testStore.LabelSet()...)
   835  		setOfExpectedClientLabelSets = append(setOfExpectedClientLabelSets, expectedClientLabelSets)
   836  
   837  		actualClientLabelSets := actualStoreClients[i].LabelSets()
   838  		setOfActualClientLabelSets = append(setOfActualClientLabelSets, actualClientLabelSets)
   839  	}
   840  
   841  	return setOfExpectedClientLabelSets, setOfActualClientLabelSets
   842  }