github.com/matrixorigin/matrixone@v1.2.0/pkg/vm/engine/disttae/route/route_test.go (about)

     1  // Copyright 2021 -2023 Matrix Origin
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //	http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package route
    16  
    17  import (
    18  	"context"
    19  	"sync"
    20  	"testing"
    21  	"time"
    22  
    23  	"github.com/lni/goutils/leaktest"
    24  	"github.com/matrixorigin/matrixone/pkg/clusterservice"
    25  	"github.com/matrixorigin/matrixone/pkg/common/runtime"
    26  	logpb "github.com/matrixorigin/matrixone/pkg/pb/logservice"
    27  	"github.com/matrixorigin/matrixone/pkg/pb/metadata"
    28  	"github.com/stretchr/testify/assert"
    29  )
    30  
    31  type mockHAKeeperClient struct {
    32  	sync.RWMutex
    33  	value logpb.ClusterDetails
    34  	err   error
    35  }
    36  
    37  func (c *mockHAKeeperClient) updateCN(uuid string, labels map[string]metadata.LabelList) {
    38  	c.Lock()
    39  	defer c.Unlock()
    40  	var cs *logpb.CNStore
    41  	for i := range c.value.CNStores {
    42  		if c.value.CNStores[i].UUID == uuid {
    43  			cs = &c.value.CNStores[i]
    44  			break
    45  		}
    46  	}
    47  	if cs != nil {
    48  		cs.Labels = labels
    49  		return
    50  	}
    51  	cs = &logpb.CNStore{
    52  		UUID:      uuid,
    53  		Labels:    labels,
    54  		WorkState: metadata.WorkState_Working,
    55  	}
    56  	c.value.CNStores = append(c.value.CNStores, *cs)
    57  }
    58  
    59  func (c *mockHAKeeperClient) GetClusterDetails(ctx context.Context) (logpb.ClusterDetails, error) {
    60  	c.RLock()
    61  	defer c.RUnlock()
    62  	return c.value, c.err
    63  }
    64  
    65  func runTestWithMOCluster(t *testing.T, fn func(t *testing.T, hc *mockHAKeeperClient, c clusterservice.MOCluster)) {
    66  	defer leaktest.AfterTest(t)()
    67  	rt := runtime.DefaultRuntime()
    68  	runtime.SetupProcessLevelRuntime(rt)
    69  	hc := &mockHAKeeperClient{}
    70  	mc := clusterservice.NewMOCluster(hc, 3*time.Second)
    71  	defer mc.Close()
    72  	rt.SetGlobalVariables(runtime.ClusterService, mc)
    73  
    74  	fn(t, hc, mc)
    75  }
    76  
    77  func TestRouteForSuperTenant_C0_Contain(t *testing.T) {
    78  	runTestWithMOCluster(t, func(t *testing.T, hc *mockHAKeeperClient, c clusterservice.MOCluster) {
    79  		cnLabels1 := map[string]metadata.LabelList{}
    80  		hc.updateCN("cn1", cnLabels1)
    81  
    82  		cnLabels2 := map[string]metadata.LabelList{}
    83  		hc.updateCN("cn2", cnLabels2)
    84  		c.ForceRefresh(true)
    85  
    86  		var cns []*metadata.CNService
    87  
    88  		s := clusterservice.NewSelector().SelectByLabel(map[string]string{
    89  			"account": "sys",
    90  		}, clusterservice.Contain)
    91  		RouteForSuperTenant(s, "dump", nil, func(s *metadata.CNService) {
    92  			cns = append(cns, s)
    93  		})
    94  		assert.Equal(t, 2, len(cns))
    95  	})
    96  }
    97  
    98  func TestRouteForSuperTenant_C0_EQ_Globbing(t *testing.T) {
    99  	runTestWithMOCluster(t, func(t *testing.T, hc *mockHAKeeperClient, c clusterservice.MOCluster) {
   100  		cnLabels1 := map[string]metadata.LabelList{}
   101  		hc.updateCN("cn1", cnLabels1)
   102  
   103  		cnLabels2 := map[string]metadata.LabelList{}
   104  		hc.updateCN("cn2", cnLabels2)
   105  		c.ForceRefresh(true)
   106  
   107  		var cns []*metadata.CNService
   108  
   109  		s := clusterservice.NewSelector().SelectByLabel(map[string]string{
   110  			"account": "sys",
   111  		}, clusterservice.EQ_Globbing)
   112  		RouteForSuperTenant(s, "dump", nil, func(s *metadata.CNService) {
   113  			cns = append(cns, s)
   114  		})
   115  		assert.Equal(t, 2, len(cns))
   116  	})
   117  }
   118  
   119  func TestRouteForSuperTenant_C1_Contain(t *testing.T) {
   120  	runTestWithMOCluster(t, func(t *testing.T, hc *mockHAKeeperClient, c clusterservice.MOCluster) {
   121  		cnLabels1 := map[string]metadata.LabelList{
   122  			"account": {
   123  				Labels: []string{"sys"},
   124  			},
   125  			"k1": {
   126  				Labels: []string{"v1"},
   127  			},
   128  		}
   129  		hc.updateCN("cn1", cnLabels1)
   130  
   131  		cnLabels2 := map[string]metadata.LabelList{
   132  			"k2": {
   133  				Labels: []string{"v2"},
   134  			},
   135  		}
   136  		hc.updateCN("cn2", cnLabels2)
   137  		c.ForceRefresh(true)
   138  
   139  		var cns []*metadata.CNService
   140  
   141  		s := clusterservice.NewSelector().SelectByLabel(map[string]string{
   142  			"account": "sys",
   143  			"k1":      "v1",
   144  		}, clusterservice.Contain)
   145  		RouteForSuperTenant(s, "dump", nil, func(s *metadata.CNService) {
   146  			cns = append(cns, s)
   147  		})
   148  		assert.Equal(t, 1, len(cns))
   149  		assert.Equal(t, "cn1", cns[0].ServiceID)
   150  	})
   151  }
   152  
   153  func TestRouteForSuperTenant_C1_EQ_Globbing(t *testing.T) {
   154  	runTestWithMOCluster(t, func(t *testing.T, hc *mockHAKeeperClient, c clusterservice.MOCluster) {
   155  		cnLabels1 := map[string]metadata.LabelList{
   156  			"account": {
   157  				Labels: []string{"sys"},
   158  			},
   159  			"k1": {
   160  				Labels: []string{"v1"},
   161  			},
   162  		}
   163  		hc.updateCN("cn1", cnLabels1)
   164  
   165  		cnLabels2 := map[string]metadata.LabelList{
   166  			"account": {
   167  				Labels: []string{"sys"},
   168  			},
   169  			"k1": {
   170  				Labels: []string{"*"},
   171  			},
   172  		}
   173  		hc.updateCN("cn2", cnLabels2)
   174  
   175  		cnLabels3 := map[string]metadata.LabelList{
   176  			"k2": {
   177  				Labels: []string{"v2"},
   178  			},
   179  		}
   180  		hc.updateCN("cn3", cnLabels3)
   181  		c.ForceRefresh(true)
   182  
   183  		var cns []*metadata.CNService
   184  
   185  		s := clusterservice.NewSelector().SelectByLabel(map[string]string{
   186  			"account": "sys",
   187  			"k1":      "v1",
   188  		}, clusterservice.EQ_Globbing)
   189  		RouteForSuperTenant(s, "dump", nil, func(s *metadata.CNService) {
   190  			cns = append(cns, s)
   191  		})
   192  		assert.Equal(t, 2, len(cns))
   193  	})
   194  }
   195  
   196  func TestRouteForSuperTenant_C2_Contain(t *testing.T) {
   197  	runTestWithMOCluster(t, func(t *testing.T, hc *mockHAKeeperClient, c clusterservice.MOCluster) {
   198  		cnLabels1 := map[string]metadata.LabelList{
   199  			"k1": {
   200  				Labels: []string{"v1"},
   201  			},
   202  		}
   203  		hc.updateCN("cn1", cnLabels1)
   204  
   205  		cnLabels2 := map[string]metadata.LabelList{
   206  			"k2": {
   207  				Labels: []string{"v2"},
   208  			},
   209  		}
   210  		hc.updateCN("cn2", cnLabels2)
   211  		c.ForceRefresh(true)
   212  
   213  		var cns []*metadata.CNService
   214  
   215  		s := clusterservice.NewSelector().SelectByLabel(map[string]string{
   216  			"account": "sys",
   217  			"k2":      "v2",
   218  		}, clusterservice.Contain)
   219  		RouteForSuperTenant(s, "dump", nil, func(s *metadata.CNService) {
   220  			cns = append(cns, s)
   221  		})
   222  		assert.Equal(t, 1, len(cns))
   223  		assert.Equal(t, "cn2", cns[0].ServiceID)
   224  	})
   225  }
   226  
   227  func TestRouteForSuperTenant_C2_EQ_Globbing(t *testing.T) {
   228  	runTestWithMOCluster(t, func(t *testing.T, hc *mockHAKeeperClient, c clusterservice.MOCluster) {
   229  		cnLabels1 := map[string]metadata.LabelList{
   230  			"k1": {
   231  				Labels: []string{"v1"},
   232  			},
   233  		}
   234  		hc.updateCN("cn1", cnLabels1)
   235  
   236  		cnLabels2 := map[string]metadata.LabelList{
   237  			"k2": {
   238  				Labels: []string{"v2"},
   239  			},
   240  		}
   241  		hc.updateCN("cn2", cnLabels2)
   242  
   243  		cnLabels3 := map[string]metadata.LabelList{
   244  			"k2": {
   245  				Labels: []string{"*"},
   246  			},
   247  		}
   248  		hc.updateCN("cn3", cnLabels3)
   249  		c.ForceRefresh(true)
   250  
   251  		var cns []*metadata.CNService
   252  
   253  		s := clusterservice.NewSelector().SelectByLabel(map[string]string{
   254  			"account": "sys",
   255  			"k2":      "v2",
   256  		}, clusterservice.EQ_Globbing)
   257  		RouteForSuperTenant(s, "dump", nil, func(s *metadata.CNService) {
   258  			cns = append(cns, s)
   259  		})
   260  		assert.Equal(t, 2, len(cns))
   261  	})
   262  }
   263  
   264  func TestRouteForSuperTenant_C3_Contain(t *testing.T) {
   265  	runTestWithMOCluster(t, func(t *testing.T, hc *mockHAKeeperClient, c clusterservice.MOCluster) {
   266  		cnLabels1 := map[string]metadata.LabelList{
   267  			"account": {
   268  				Labels: []string{"sys"},
   269  			},
   270  		}
   271  		hc.updateCN("cn1", cnLabels1)
   272  
   273  		cnLabels2 := map[string]metadata.LabelList{
   274  			"k2": {
   275  				Labels: []string{"v2"},
   276  			},
   277  		}
   278  		hc.updateCN("cn2", cnLabels2)
   279  		c.ForceRefresh(true)
   280  
   281  		var cns []*metadata.CNService
   282  
   283  		s := clusterservice.NewSelector().SelectByLabel(map[string]string{
   284  			"account": "sys",
   285  		}, clusterservice.Contain)
   286  		RouteForSuperTenant(s, "dump", nil, func(s *metadata.CNService) {
   287  			cns = append(cns, s)
   288  		})
   289  		assert.Equal(t, 1, len(cns))
   290  		assert.Equal(t, "cn1", cns[0].ServiceID)
   291  	})
   292  }
   293  
   294  func TestRouteForSuperTenant_C3_EQ_Globbing(t *testing.T) {
   295  	runTestWithMOCluster(t, func(t *testing.T, hc *mockHAKeeperClient, c clusterservice.MOCluster) {
   296  		cnLabels1 := map[string]metadata.LabelList{
   297  			"account": {
   298  				Labels: []string{"sys"},
   299  			},
   300  		}
   301  		hc.updateCN("cn1", cnLabels1)
   302  
   303  		cnLabels2 := map[string]metadata.LabelList{
   304  			"account": {
   305  				Labels: []string{"sy*"},
   306  			},
   307  		}
   308  		hc.updateCN("cn2", cnLabels2)
   309  
   310  		cnLabels3 := map[string]metadata.LabelList{
   311  			"k2": {
   312  				Labels: []string{"v2"},
   313  			},
   314  		}
   315  		hc.updateCN("cn3", cnLabels3)
   316  		c.ForceRefresh(true)
   317  
   318  		var cns []*metadata.CNService
   319  
   320  		s := clusterservice.NewSelector().SelectByLabel(map[string]string{
   321  			"account": "sys",
   322  		}, clusterservice.EQ_Globbing)
   323  		RouteForSuperTenant(s, "dump", nil, func(s *metadata.CNService) {
   324  			cns = append(cns, s)
   325  		})
   326  		assert.Equal(t, 2, len(cns))
   327  	})
   328  }
   329  
   330  func TestRouteForSuperTenant_C4_Contain(t *testing.T) {
   331  	runTestWithMOCluster(t, func(t *testing.T, hc *mockHAKeeperClient, c clusterservice.MOCluster) {
   332  		cnLabels1 := map[string]metadata.LabelList{
   333  			"k1": {
   334  				Labels: []string{"v1"},
   335  			},
   336  		}
   337  		hc.updateCN("cn1", cnLabels1)
   338  
   339  		cnLabels2 := map[string]metadata.LabelList{
   340  			"k2": {
   341  				Labels: []string{"v2"},
   342  			},
   343  		}
   344  		hc.updateCN("cn2", cnLabels2)
   345  		c.ForceRefresh(true)
   346  
   347  		var cns []*metadata.CNService
   348  
   349  		s := clusterservice.NewSelector().SelectByLabel(map[string]string{
   350  			"account": "sys",
   351  		}, clusterservice.Contain)
   352  		RouteForSuperTenant(s, "dump", nil, func(s *metadata.CNService) {
   353  			cns = append(cns, s)
   354  		})
   355  		assert.Equal(t, 2, len(cns))
   356  	})
   357  }
   358  
   359  func TestRouteForSuperTenant_C4_EQ_Globbing(t *testing.T) {
   360  	runTestWithMOCluster(t, func(t *testing.T, hc *mockHAKeeperClient, c clusterservice.MOCluster) {
   361  		cnLabels1 := map[string]metadata.LabelList{
   362  			"k1": {
   363  				Labels: []string{"v1"},
   364  			},
   365  		}
   366  		hc.updateCN("cn1", cnLabels1)
   367  
   368  		cnLabels2 := map[string]metadata.LabelList{
   369  			"k2": {
   370  				Labels: []string{"v2"},
   371  			},
   372  		}
   373  		hc.updateCN("cn2", cnLabels2)
   374  
   375  		cnLabels3 := map[string]metadata.LabelList{
   376  			"k3": {
   377  				Labels: []string{"*"},
   378  			},
   379  		}
   380  		hc.updateCN("cn3", cnLabels3)
   381  		c.ForceRefresh(true)
   382  
   383  		var cns []*metadata.CNService
   384  
   385  		s := clusterservice.NewSelector().SelectByLabel(map[string]string{
   386  			"account": "sys",
   387  		}, clusterservice.EQ_Globbing)
   388  		RouteForSuperTenant(s, "dump", nil, func(s *metadata.CNService) {
   389  			cns = append(cns, s)
   390  		})
   391  		assert.Equal(t, 3, len(cns))
   392  	})
   393  }
   394  
   395  func TestRouteForSuperTenant_C5_Contain(t *testing.T) {
   396  	runTestWithMOCluster(t, func(t *testing.T, hc *mockHAKeeperClient, c clusterservice.MOCluster) {
   397  		cnLabels1 := map[string]metadata.LabelList{
   398  			"k1": {
   399  				Labels: []string{"v1"},
   400  			},
   401  		}
   402  		hc.updateCN("cn1", cnLabels1)
   403  
   404  		cnLabels2 := map[string]metadata.LabelList{}
   405  		hc.updateCN("cn2", cnLabels2)
   406  		c.ForceRefresh(true)
   407  
   408  		var cns []*metadata.CNService
   409  
   410  		s := clusterservice.NewSelector().SelectByLabel(map[string]string{
   411  			"account": "sys",
   412  		}, clusterservice.Contain)
   413  		RouteForSuperTenant(s, "dump", nil, func(s *metadata.CNService) {
   414  			cns = append(cns, s)
   415  		})
   416  		assert.Equal(t, 1, len(cns))
   417  		assert.Equal(t, "cn1", cns[0].ServiceID)
   418  	})
   419  }
   420  
   421  func TestRouteForSuperTenant_C5_EQ_Globbing(t *testing.T) {
   422  	runTestWithMOCluster(t, func(t *testing.T, hc *mockHAKeeperClient, c clusterservice.MOCluster) {
   423  		cnLabels1 := map[string]metadata.LabelList{
   424  			"k1": {
   425  				Labels: []string{"v1"},
   426  			},
   427  		}
   428  		hc.updateCN("cn1", cnLabels1)
   429  
   430  		cnLabels2 := map[string]metadata.LabelList{
   431  			"k2": {
   432  				Labels: []string{"*"},
   433  			},
   434  		}
   435  		hc.updateCN("cn2", cnLabels2)
   436  
   437  		cnLabels3 := map[string]metadata.LabelList{}
   438  		hc.updateCN("cn3", cnLabels3)
   439  		c.ForceRefresh(true)
   440  
   441  		var cns []*metadata.CNService
   442  
   443  		s := clusterservice.NewSelector().SelectByLabel(map[string]string{
   444  			"account": "sys",
   445  		}, clusterservice.EQ_Globbing)
   446  		RouteForSuperTenant(s, "dump", nil, func(s *metadata.CNService) {
   447  			cns = append(cns, s)
   448  		})
   449  		assert.Equal(t, 2, len(cns))
   450  	})
   451  }
   452  
   453  func TestRouteForSuperTenant_C6_Contain(t *testing.T) {
   454  	runTestWithMOCluster(t, func(t *testing.T, hc *mockHAKeeperClient, c clusterservice.MOCluster) {
   455  		cnLabels1 := map[string]metadata.LabelList{
   456  			"account": {
   457  				Labels: []string{"sys"},
   458  			},
   459  		}
   460  		hc.updateCN("cn1", cnLabels1)
   461  
   462  		cnLabels2 := map[string]metadata.LabelList{}
   463  		hc.updateCN("cn2", cnLabels2)
   464  		c.ForceRefresh(true)
   465  
   466  		var cns []*metadata.CNService
   467  
   468  		s := clusterservice.NewSelector().SelectByLabel(map[string]string{
   469  			"account": "sys",
   470  		}, clusterservice.Contain)
   471  		RouteForSuperTenant(s, "dump", nil, func(s *metadata.CNService) {
   472  			cns = append(cns, s)
   473  		})
   474  		assert.Equal(t, 1, len(cns))
   475  		assert.Equal(t, "cn1", cns[0].ServiceID)
   476  	})
   477  }
   478  
   479  func TestRouteForSuperTenant_C6_EQ_Globbing(t *testing.T) {
   480  	runTestWithMOCluster(t, func(t *testing.T, hc *mockHAKeeperClient, c clusterservice.MOCluster) {
   481  		cnLabels1 := map[string]metadata.LabelList{
   482  			"account": {
   483  				Labels: []string{"sys"},
   484  			},
   485  		}
   486  		hc.updateCN("cn1", cnLabels1)
   487  
   488  		cnLabels2 := map[string]metadata.LabelList{
   489  			"account": {
   490  				Labels: []string{"s*"},
   491  			},
   492  		}
   493  		hc.updateCN("cn2", cnLabels2)
   494  
   495  		cnLabels3 := map[string]metadata.LabelList{}
   496  		hc.updateCN("cn3", cnLabels3)
   497  		c.ForceRefresh(true)
   498  
   499  		var cns []*metadata.CNService
   500  
   501  		s := clusterservice.NewSelector().SelectByLabel(map[string]string{
   502  			"account": "sys",
   503  		}, clusterservice.EQ_Globbing)
   504  		RouteForSuperTenant(s, "dump", nil, func(s *metadata.CNService) {
   505  			cns = append(cns, s)
   506  		})
   507  		assert.Equal(t, 2, len(cns))
   508  	})
   509  }
   510  
   511  func TestRouteForSuperTenant_C7_Contain(t *testing.T) {
   512  	runTestWithMOCluster(t, func(t *testing.T, hc *mockHAKeeperClient, c clusterservice.MOCluster) {
   513  		cnLabels1 := map[string]metadata.LabelList{
   514  			"account": {
   515  				Labels: []string{"sys"},
   516  			},
   517  		}
   518  		hc.updateCN("cn1", cnLabels1)
   519  
   520  		cnLabels2 := map[string]metadata.LabelList{
   521  			"k2": {
   522  				Labels: []string{"v2"},
   523  			},
   524  			"k3": {
   525  				Labels: []string{"v3"},
   526  			},
   527  		}
   528  		hc.updateCN("cn2", cnLabels2)
   529  		c.ForceRefresh(true)
   530  
   531  		var cns []*metadata.CNService
   532  
   533  		s := clusterservice.NewSelector().SelectByLabel(map[string]string{
   534  			"account": "sys",
   535  			"k3":      "v3",
   536  		}, clusterservice.Contain)
   537  		RouteForSuperTenant(s, "dump", nil, func(s *metadata.CNService) {
   538  			cns = append(cns, s)
   539  		})
   540  		assert.Equal(t, 1, len(cns))
   541  		assert.Equal(t, "cn2", cns[0].ServiceID)
   542  	})
   543  }
   544  
   545  func TestRouteForSuperTenant_C7_EQ_Globbing(t *testing.T) {
   546  	runTestWithMOCluster(t, func(t *testing.T, hc *mockHAKeeperClient, c clusterservice.MOCluster) {
   547  		cnLabels1 := map[string]metadata.LabelList{
   548  			"account": {
   549  				Labels: []string{"sys"},
   550  			},
   551  		}
   552  		hc.updateCN("cn1", cnLabels1)
   553  
   554  		cnLabels2 := map[string]metadata.LabelList{
   555  			"k2": {
   556  				Labels: []string{"v2"},
   557  			},
   558  			"k3": {
   559  				Labels: []string{"v3"},
   560  			},
   561  		}
   562  		hc.updateCN("cn2", cnLabels2)
   563  
   564  		cnLabels3 := map[string]metadata.LabelList{
   565  			"k2": {
   566  				Labels: []string{"v2"},
   567  			},
   568  			"k3": {
   569  				Labels: []string{"*"},
   570  			},
   571  		}
   572  		hc.updateCN("cn3", cnLabels3)
   573  		c.ForceRefresh(true)
   574  
   575  		var cns []*metadata.CNService
   576  
   577  		s := clusterservice.NewSelector().SelectByLabel(map[string]string{
   578  			"account": "sys",
   579  			"k3":      "v3",
   580  		}, clusterservice.EQ_Globbing)
   581  		RouteForSuperTenant(s, "dump", nil, func(s *metadata.CNService) {
   582  			cns = append(cns, s)
   583  		})
   584  		assert.Equal(t, 3, len(cns))
   585  	})
   586  }
   587  
   588  func TestRouteForSuperTenant_C8_Contain(t *testing.T) {
   589  	runTestWithMOCluster(t, func(t *testing.T, hc *mockHAKeeperClient, c clusterservice.MOCluster) {
   590  		cnLabels1 := map[string]metadata.LabelList{
   591  			"account": {
   592  				Labels: []string{"sys"},
   593  			},
   594  		}
   595  		hc.updateCN("cn1", cnLabels1)
   596  
   597  		cnLabels2 := map[string]metadata.LabelList{
   598  			"k2": {
   599  				Labels: []string{"v2"},
   600  			},
   601  		}
   602  		hc.updateCN("cn2", cnLabels2)
   603  		c.ForceRefresh(true)
   604  
   605  		var cns []*metadata.CNService
   606  
   607  		s := clusterservice.NewSelector().SelectByLabel(map[string]string{
   608  			"account": "sys",
   609  			"k1":      "v1",
   610  		}, clusterservice.Contain)
   611  		RouteForSuperTenant(s, "dump", nil, func(s *metadata.CNService) {
   612  			cns = append(cns, s)
   613  		})
   614  		assert.Equal(t, 2, len(cns))
   615  	})
   616  }
   617  
   618  func TestRouteForSuperTenant_C8_EQ_Globbing(t *testing.T) {
   619  	runTestWithMOCluster(t, func(t *testing.T, hc *mockHAKeeperClient, c clusterservice.MOCluster) {
   620  		cnLabels1 := map[string]metadata.LabelList{
   621  			"account": {
   622  				Labels: []string{"sys"},
   623  			},
   624  		}
   625  		hc.updateCN("cn1", cnLabels1)
   626  
   627  		cnLabels2 := map[string]metadata.LabelList{
   628  			"k2": {
   629  				Labels: []string{"v2"},
   630  			},
   631  		}
   632  		hc.updateCN("cn2", cnLabels2)
   633  
   634  		cnLabels3 := map[string]metadata.LabelList{
   635  			"account": {
   636  				Labels: []string{"sys"},
   637  			},
   638  		}
   639  		hc.updateCN("cn3", cnLabels3)
   640  		c.ForceRefresh(true)
   641  
   642  		var cns []*metadata.CNService
   643  
   644  		s := clusterservice.NewSelector().SelectByLabel(map[string]string{
   645  			"account": "sys",
   646  			"k1":      "v1",
   647  		}, clusterservice.EQ_Globbing)
   648  		RouteForSuperTenant(s, "dump", nil, func(s *metadata.CNService) {
   649  			cns = append(cns, s)
   650  		})
   651  		assert.Equal(t, 3, len(cns))
   652  	})
   653  }
   654  
   655  func TestRouteForCommonTenant_C1_Contain(t *testing.T) {
   656  	runTestWithMOCluster(t, func(t *testing.T, hc *mockHAKeeperClient, c clusterservice.MOCluster) {
   657  		cnLabels1 := map[string]metadata.LabelList{
   658  			"k1": {
   659  				Labels: []string{"v1"},
   660  			},
   661  		}
   662  		hc.updateCN("cn1", cnLabels1)
   663  
   664  		cnLabels2 := map[string]metadata.LabelList{}
   665  		hc.updateCN("cn2", cnLabels2)
   666  		c.ForceRefresh(true)
   667  
   668  		var cns []*metadata.CNService
   669  
   670  		s := clusterservice.NewSelector().SelectByLabel(map[string]string{
   671  			"account": "t1",
   672  		}, clusterservice.Contain)
   673  		RouteForCommonTenant(s, nil, func(s *metadata.CNService) {
   674  			cns = append(cns, s)
   675  		})
   676  		assert.Equal(t, 1, len(cns))
   677  		assert.Equal(t, "cn2", cns[0].ServiceID)
   678  	})
   679  }
   680  
   681  func TestRouteForCommonTenant_C1_EQ_Globbing(t *testing.T) {
   682  	runTestWithMOCluster(t, func(t *testing.T, hc *mockHAKeeperClient, c clusterservice.MOCluster) {
   683  		cnLabels1 := map[string]metadata.LabelList{
   684  			"k1": {
   685  				Labels: []string{"v1"},
   686  			},
   687  		}
   688  		hc.updateCN("cn1", cnLabels1)
   689  
   690  		cnLabels2 := map[string]metadata.LabelList{
   691  			"k1": {
   692  				Labels: []string{"*"},
   693  			},
   694  		}
   695  		hc.updateCN("cn2", cnLabels2)
   696  
   697  		cnLabels3 := map[string]metadata.LabelList{}
   698  		hc.updateCN("cn3", cnLabels3)
   699  		c.ForceRefresh(true)
   700  
   701  		var cns []*metadata.CNService
   702  
   703  		s := clusterservice.NewSelector().SelectByLabel(map[string]string{
   704  			"account": "t1",
   705  		}, clusterservice.Contain)
   706  		RouteForCommonTenant(s, nil, func(s *metadata.CNService) {
   707  			cns = append(cns, s)
   708  		})
   709  		assert.Equal(t, 1, len(cns))
   710  		assert.Equal(t, "cn3", cns[0].ServiceID)
   711  	})
   712  }
   713  
   714  func TestRouteForCommonTenant_C2_Contain(t *testing.T) {
   715  	runTestWithMOCluster(t, func(t *testing.T, hc *mockHAKeeperClient, c clusterservice.MOCluster) {
   716  		cnLabels1 := map[string]metadata.LabelList{
   717  			"account": {
   718  				Labels: []string{"t1"},
   719  			},
   720  			"k1": {
   721  				Labels: []string{"v1"},
   722  			},
   723  		}
   724  		hc.updateCN("cn1", cnLabels1)
   725  
   726  		cnLabels2 := map[string]metadata.LabelList{
   727  			"account": {
   728  				Labels: []string{"t2"},
   729  			},
   730  			"k1": {
   731  				Labels: []string{"v1"},
   732  			},
   733  		}
   734  		hc.updateCN("cn2", cnLabels2)
   735  		c.ForceRefresh(true)
   736  
   737  		var cns []*metadata.CNService
   738  
   739  		s := clusterservice.NewSelector().SelectByLabel(map[string]string{
   740  			"account": "t1",
   741  		}, clusterservice.Contain)
   742  		RouteForCommonTenant(s, nil, func(s *metadata.CNService) {
   743  			cns = append(cns, s)
   744  		})
   745  		assert.Equal(t, 1, len(cns))
   746  		assert.Equal(t, "cn1", cns[0].ServiceID)
   747  	})
   748  }
   749  
   750  func TestRouteForCommonTenant_C2_EQ_Globbing(t *testing.T) {
   751  	runTestWithMOCluster(t, func(t *testing.T, hc *mockHAKeeperClient, c clusterservice.MOCluster) {
   752  		cnLabels1 := map[string]metadata.LabelList{
   753  			"account": {
   754  				Labels: []string{"t1"},
   755  			},
   756  			"k1": {
   757  				Labels: []string{"v1"},
   758  			},
   759  		}
   760  		hc.updateCN("cn1", cnLabels1)
   761  
   762  		cnLabels2 := map[string]metadata.LabelList{
   763  			"account": {
   764  				Labels: []string{"t2"},
   765  			},
   766  			"k1": {
   767  				Labels: []string{"v1"},
   768  			},
   769  		}
   770  		hc.updateCN("cn2", cnLabels2)
   771  		c.ForceRefresh(true)
   772  
   773  		var cns []*metadata.CNService
   774  
   775  		s := clusterservice.NewSelector().SelectByLabel(map[string]string{
   776  			"account": "t1",
   777  		}, clusterservice.EQ_Globbing)
   778  		RouteForCommonTenant(s, nil, func(s *metadata.CNService) {
   779  			cns = append(cns, s)
   780  		})
   781  		assert.Equal(t, 0, len(cns))
   782  	})
   783  }
   784  
   785  func TestRouteForCommonTenant_C3_EQ_Globbing(t *testing.T) {
   786  	runTestWithMOCluster(t, func(t *testing.T, hc *mockHAKeeperClient, c clusterservice.MOCluster) {
   787  		cnLabels1 := map[string]metadata.LabelList{
   788  			"account": {
   789  				Labels: []string{"t1"},
   790  			},
   791  			"k1": {
   792  				Labels: []string{"*"},
   793  			},
   794  		}
   795  		hc.updateCN("cn1", cnLabels1)
   796  
   797  		cnLabels2 := map[string]metadata.LabelList{
   798  			"account": {
   799  				Labels: []string{"t2"},
   800  			},
   801  			"k1": {
   802  				Labels: []string{"v1"},
   803  			},
   804  		}
   805  		hc.updateCN("cn2", cnLabels2)
   806  		c.ForceRefresh(true)
   807  
   808  		var cns []*metadata.CNService
   809  
   810  		s := clusterservice.NewSelector().SelectByLabel(map[string]string{
   811  			"account": "t1",
   812  			"k1":      "v1",
   813  		}, clusterservice.EQ_Globbing)
   814  		RouteForCommonTenant(s, nil, func(s *metadata.CNService) {
   815  			cns = append(cns, s)
   816  		})
   817  		assert.Equal(t, 1, len(cns))
   818  		assert.Equal(t, "cn1", cns[0].ServiceID)
   819  	})
   820  }
   821  
   822  func TestRouteForCommonTenant_C4_EQ_Globbing(t *testing.T) {
   823  	runTestWithMOCluster(t, func(t *testing.T, hc *mockHAKeeperClient, c clusterservice.MOCluster) {
   824  		cnLabels1 := map[string]metadata.LabelList{
   825  			"account": {
   826  				Labels: []string{"t1"},
   827  			},
   828  			"k1": {
   829  				Labels: []string{"*"},
   830  			},
   831  		}
   832  		hc.updateCN("cn1", cnLabels1)
   833  
   834  		cnLabels2 := map[string]metadata.LabelList{
   835  			"account": {
   836  				Labels: []string{"t1"},
   837  			},
   838  			"k1": {
   839  				Labels: []string{"v*"},
   840  			},
   841  		}
   842  		hc.updateCN("cn2", cnLabels2)
   843  
   844  		cnLabels3 := map[string]metadata.LabelList{
   845  			"account": {
   846  				Labels: []string{"t2"},
   847  			},
   848  			"k1": {
   849  				Labels: []string{"v1"},
   850  			},
   851  		}
   852  		hc.updateCN("cn3", cnLabels3)
   853  		c.ForceRefresh(true)
   854  
   855  		var cns []*metadata.CNService
   856  
   857  		s := clusterservice.NewSelector().SelectByLabel(map[string]string{
   858  			"account": "t1",
   859  			"k1":      "v1",
   860  		}, clusterservice.EQ_Globbing)
   861  		RouteForCommonTenant(s, nil, func(s *metadata.CNService) {
   862  			cns = append(cns, s)
   863  		})
   864  		assert.Equal(t, 2, len(cns))
   865  	})
   866  }