istio.io/istio@v0.0.0-20240520182934-d79c90f27776/pilot/pkg/networking/core/loadbalancer/loadbalancer_test.go (about)

     1  // Copyright Istio Authors. All Rights Reserved.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package loadbalancer
    16  
    17  import (
    18  	"reflect"
    19  	"testing"
    20  
    21  	cluster "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
    22  	core "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
    23  	endpoint "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
    24  	. "github.com/onsi/gomega"
    25  	"google.golang.org/protobuf/types/known/durationpb"
    26  	wrappers "google.golang.org/protobuf/types/known/wrapperspb"
    27  
    28  	meshconfig "istio.io/api/mesh/v1alpha1"
    29  	networking "istio.io/api/networking/v1alpha3"
    30  	"istio.io/istio/pilot/pkg/config/memory"
    31  	"istio.io/istio/pilot/pkg/model"
    32  	memregistry "istio.io/istio/pilot/pkg/serviceregistry/memory"
    33  	"istio.io/istio/pkg/config"
    34  	"istio.io/istio/pkg/config/mesh"
    35  	"istio.io/istio/pkg/config/protocol"
    36  	"istio.io/istio/pkg/config/schema/collections"
    37  	"istio.io/istio/pkg/config/schema/gvk"
    38  )
    39  
    40  func TestApplyLocalitySetting(t *testing.T) {
    41  	locality := &core.Locality{
    42  		Region:  "region1",
    43  		Zone:    "zone1",
    44  		SubZone: "subzone1",
    45  	}
    46  
    47  	t.Run("Distribute", func(t *testing.T) {
    48  		tests := []struct {
    49  			name       string
    50  			distribute []*networking.LocalityLoadBalancerSetting_Distribute
    51  			expected   []int
    52  		}{
    53  			{
    54  				name: "distribution between subzones",
    55  				distribute: []*networking.LocalityLoadBalancerSetting_Distribute{
    56  					{
    57  						From: "region1/zone1/subzone1",
    58  						To: map[string]uint32{
    59  							"region1/zone1/subzone1": 80,
    60  							"region1/zone1/subzone2": 15,
    61  							"region1/zone1/subzone3": 5,
    62  						},
    63  					},
    64  				},
    65  				expected: []int{40, 40, 15, 5, 0, 0, 0},
    66  			},
    67  		}
    68  		for _, tt := range tests {
    69  			t.Run(tt.name, func(t *testing.T) {
    70  				env := buildEnvForClustersWithDistribute(tt.distribute)
    71  				cluster := buildFakeCluster()
    72  				ApplyLocalityLoadBalancer(cluster.LoadAssignment, nil, locality, nil, env.Mesh().LocalityLbSetting, true)
    73  				weights := make([]int, 0)
    74  				for _, localityEndpoint := range cluster.LoadAssignment.Endpoints {
    75  					weights = append(weights, int(localityEndpoint.LoadBalancingWeight.GetValue()))
    76  				}
    77  				if !reflect.DeepEqual(weights, tt.expected) {
    78  					t.Errorf("Got weights %v expected %v", weights, tt.expected)
    79  				}
    80  			})
    81  		}
    82  	})
    83  
    84  	t.Run("Failover: all priorities", func(t *testing.T) {
    85  		g := NewWithT(t)
    86  		env := buildEnvForClustersWithFailover()
    87  		cluster := buildFakeCluster()
    88  		ApplyLocalityLoadBalancer(cluster.LoadAssignment, nil, locality, nil, env.Mesh().LocalityLbSetting, true)
    89  		for _, localityEndpoint := range cluster.LoadAssignment.Endpoints {
    90  			if localityEndpoint.Locality.Region == locality.Region {
    91  				if localityEndpoint.Locality.Zone == locality.Zone {
    92  					if localityEndpoint.Locality.SubZone == locality.SubZone {
    93  						g.Expect(localityEndpoint.Priority).To(Equal(uint32(0)))
    94  						continue
    95  					}
    96  					g.Expect(localityEndpoint.Priority).To(Equal(uint32(1)))
    97  					continue
    98  				}
    99  				g.Expect(localityEndpoint.Priority).To(Equal(uint32(2)))
   100  				continue
   101  			}
   102  			if localityEndpoint.Locality.Region == "region2" {
   103  				g.Expect(localityEndpoint.Priority).To(Equal(uint32(3)))
   104  			} else {
   105  				g.Expect(localityEndpoint.Priority).To(Equal(uint32(4)))
   106  			}
   107  		}
   108  	})
   109  
   110  	t.Run("Failover: priorities with gaps", func(t *testing.T) {
   111  		g := NewWithT(t)
   112  		env := buildEnvForClustersWithFailover()
   113  		cluster := buildSmallCluster()
   114  		ApplyLocalityLoadBalancer(cluster.LoadAssignment, nil, locality, nil, env.Mesh().LocalityLbSetting, true)
   115  		for _, localityEndpoint := range cluster.LoadAssignment.Endpoints {
   116  			if localityEndpoint.Locality.Region == locality.Region {
   117  				if localityEndpoint.Locality.Zone == locality.Zone {
   118  					if localityEndpoint.Locality.SubZone == locality.SubZone {
   119  						t.Errorf("Should not exist")
   120  						continue
   121  					}
   122  					g.Expect(localityEndpoint.Priority).To(Equal(uint32(0)))
   123  					continue
   124  				}
   125  				t.Errorf("Should not exist")
   126  				continue
   127  			}
   128  			if localityEndpoint.Locality.Region == "region2" {
   129  				g.Expect(localityEndpoint.Priority).To(Equal(uint32(1)))
   130  			} else {
   131  				t.Errorf("Should not exist")
   132  			}
   133  		}
   134  	})
   135  
   136  	t.Run("Failover: priorities with some nil localities", func(t *testing.T) {
   137  		g := NewWithT(t)
   138  		env := buildEnvForClustersWithFailover()
   139  		cluster := buildSmallClusterWithNilLocalities()
   140  		ApplyLocalityLoadBalancer(cluster.LoadAssignment, nil, locality, nil, env.Mesh().LocalityLbSetting, true)
   141  		for _, localityEndpoint := range cluster.LoadAssignment.Endpoints {
   142  			if localityEndpoint.Locality == nil {
   143  				g.Expect(localityEndpoint.Priority).To(Equal(uint32(2)))
   144  			} else if localityEndpoint.Locality.Region == locality.Region {
   145  				if localityEndpoint.Locality.Zone == locality.Zone {
   146  					if localityEndpoint.Locality.SubZone == locality.SubZone {
   147  						t.Errorf("Should not exist")
   148  						continue
   149  					}
   150  					g.Expect(localityEndpoint.Priority).To(Equal(uint32(0)))
   151  					continue
   152  				}
   153  				t.Errorf("Should not exist")
   154  				continue
   155  			} else if localityEndpoint.Locality.Region == "region2" {
   156  				g.Expect(localityEndpoint.Priority).To(Equal(uint32(1)))
   157  			} else {
   158  				t.Errorf("Should not exist")
   159  			}
   160  		}
   161  	})
   162  
   163  	t.Run("Failover: with locality lb disabled", func(t *testing.T) {
   164  		g := NewWithT(t)
   165  		cluster := buildSmallClusterWithNilLocalities()
   166  		lbsetting := &networking.LocalityLoadBalancerSetting{
   167  			Enabled: &wrappers.BoolValue{Value: false},
   168  		}
   169  		ApplyLocalityLoadBalancer(cluster.LoadAssignment, nil, locality, nil, lbsetting, true)
   170  		for _, localityEndpoint := range cluster.LoadAssignment.Endpoints {
   171  			g.Expect(localityEndpoint.Priority).To(Equal(uint32(0)))
   172  		}
   173  	})
   174  
   175  	t.Run("FailoverPriority", func(t *testing.T) {
   176  		tests := []struct {
   177  			name             string
   178  			failoverPriority []string
   179  			proxyLabels      map[string]string
   180  			expected         []*endpoint.LocalityLbEndpoints
   181  		}{
   182  			{
   183  				name:             "match none label",
   184  				failoverPriority: []string{"topology.istio.io/network", "topology.istio.io/cluster"},
   185  				proxyLabels: map[string]string{
   186  					"topology.istio.io/network": "test",
   187  					"topology.istio.io/cluster": "test",
   188  				},
   189  				expected: []*endpoint.LocalityLbEndpoints{
   190  					{
   191  						Locality: &core.Locality{
   192  							Region:  "region1",
   193  							Zone:    "zone1",
   194  							SubZone: "subzone1",
   195  						},
   196  						LbEndpoints: []*endpoint.LbEndpoint{
   197  							{
   198  								HostIdentifier: buildEndpoint("1.1.1.1"),
   199  								LoadBalancingWeight: &wrappers.UInt32Value{
   200  									Value: 1,
   201  								},
   202  							},
   203  							{
   204  								HostIdentifier: buildEndpoint("2.2.2.2"),
   205  								LoadBalancingWeight: &wrappers.UInt32Value{
   206  									Value: 1,
   207  								},
   208  							},
   209  						},
   210  						LoadBalancingWeight: &wrappers.UInt32Value{
   211  							Value: 2,
   212  						},
   213  						Priority: 0,
   214  					},
   215  					{
   216  						Locality: &core.Locality{
   217  							Region:  "region2",
   218  							Zone:    "zone2",
   219  							SubZone: "subzone2",
   220  						},
   221  						LbEndpoints: []*endpoint.LbEndpoint{
   222  							{
   223  								HostIdentifier: buildEndpoint("3.3.3.3"),
   224  								LoadBalancingWeight: &wrappers.UInt32Value{
   225  									Value: 1,
   226  								},
   227  							},
   228  							{
   229  								HostIdentifier: buildEndpoint("4.4.4.4"),
   230  								LoadBalancingWeight: &wrappers.UInt32Value{
   231  									Value: 1,
   232  								},
   233  							},
   234  						},
   235  						LoadBalancingWeight: &wrappers.UInt32Value{
   236  							Value: 2,
   237  						},
   238  						Priority: 0,
   239  					},
   240  				},
   241  			},
   242  			{
   243  				name:             "match network label",
   244  				failoverPriority: []string{"topology.istio.io/network", "topology.istio.io/cluster"},
   245  				proxyLabels: map[string]string{
   246  					"topology.istio.io/network": "n1",
   247  					"topology.istio.io/cluster": "test",
   248  				},
   249  				expected: []*endpoint.LocalityLbEndpoints{
   250  					{
   251  						Locality: &core.Locality{
   252  							Region:  "region1",
   253  							Zone:    "zone1",
   254  							SubZone: "subzone1",
   255  						},
   256  						LbEndpoints: []*endpoint.LbEndpoint{
   257  							{
   258  								HostIdentifier: buildEndpoint("1.1.1.1"),
   259  								LoadBalancingWeight: &wrappers.UInt32Value{
   260  									Value: 1,
   261  								},
   262  							},
   263  						},
   264  						LoadBalancingWeight: &wrappers.UInt32Value{
   265  							Value: 1,
   266  						},
   267  						Priority: 0,
   268  					},
   269  					{
   270  						Locality: &core.Locality{
   271  							Region:  "region1",
   272  							Zone:    "zone1",
   273  							SubZone: "subzone1",
   274  						},
   275  						LbEndpoints: []*endpoint.LbEndpoint{
   276  							{
   277  								HostIdentifier: buildEndpoint("2.2.2.2"),
   278  								LoadBalancingWeight: &wrappers.UInt32Value{
   279  									Value: 1,
   280  								},
   281  							},
   282  						},
   283  						LoadBalancingWeight: &wrappers.UInt32Value{
   284  							Value: 1,
   285  						},
   286  						Priority: 1,
   287  					},
   288  					{
   289  						Locality: &core.Locality{
   290  							Region:  "region2",
   291  							Zone:    "zone2",
   292  							SubZone: "subzone2",
   293  						},
   294  						LbEndpoints: []*endpoint.LbEndpoint{
   295  							{
   296  								HostIdentifier: buildEndpoint("3.3.3.3"),
   297  								LoadBalancingWeight: &wrappers.UInt32Value{
   298  									Value: 1,
   299  								},
   300  							},
   301  						},
   302  						LoadBalancingWeight: &wrappers.UInt32Value{
   303  							Value: 1,
   304  						},
   305  						Priority: 0,
   306  					},
   307  					{
   308  						Locality: &core.Locality{
   309  							Region:  "region2",
   310  							Zone:    "zone2",
   311  							SubZone: "subzone2",
   312  						},
   313  						LbEndpoints: []*endpoint.LbEndpoint{
   314  							{
   315  								HostIdentifier: buildEndpoint("4.4.4.4"),
   316  								LoadBalancingWeight: &wrappers.UInt32Value{
   317  									Value: 1,
   318  								},
   319  							},
   320  						},
   321  						LoadBalancingWeight: &wrappers.UInt32Value{
   322  							Value: 1,
   323  						},
   324  						Priority: 1,
   325  					},
   326  				},
   327  			},
   328  			{
   329  				name:             "not match the first n label",
   330  				failoverPriority: []string{"topology.istio.io/network", "topology.istio.io/cluster"},
   331  				proxyLabels: map[string]string{
   332  					"topology.istio.io/network": "test",
   333  					"topology.istio.io/cluster": "c1",
   334  				},
   335  				expected: []*endpoint.LocalityLbEndpoints{
   336  					{
   337  						Locality: &core.Locality{
   338  							Region:  "region1",
   339  							Zone:    "zone1",
   340  							SubZone: "subzone1",
   341  						},
   342  						LbEndpoints: []*endpoint.LbEndpoint{
   343  							{
   344  								HostIdentifier: buildEndpoint("1.1.1.1"),
   345  								LoadBalancingWeight: &wrappers.UInt32Value{
   346  									Value: 1,
   347  								},
   348  							},
   349  							{
   350  								HostIdentifier: buildEndpoint("2.2.2.2"),
   351  								LoadBalancingWeight: &wrappers.UInt32Value{
   352  									Value: 1,
   353  								},
   354  							},
   355  						},
   356  						LoadBalancingWeight: &wrappers.UInt32Value{
   357  							Value: 2,
   358  						},
   359  						Priority: 0,
   360  					},
   361  					{
   362  						Locality: &core.Locality{
   363  							Region:  "region2",
   364  							Zone:    "zone2",
   365  							SubZone: "subzone2",
   366  						},
   367  						LbEndpoints: []*endpoint.LbEndpoint{
   368  							{
   369  								HostIdentifier: buildEndpoint("3.3.3.3"),
   370  								LoadBalancingWeight: &wrappers.UInt32Value{
   371  									Value: 1,
   372  								},
   373  							},
   374  							{
   375  								HostIdentifier: buildEndpoint("4.4.4.4"),
   376  								LoadBalancingWeight: &wrappers.UInt32Value{
   377  									Value: 1,
   378  								},
   379  							},
   380  						},
   381  						LoadBalancingWeight: &wrappers.UInt32Value{
   382  							Value: 2,
   383  						},
   384  						Priority: 0,
   385  					},
   386  				},
   387  			},
   388  			{
   389  				name:             "match all labels",
   390  				failoverPriority: []string{"key", "topology.istio.io/network", "topology.istio.io/cluster"},
   391  				proxyLabels: map[string]string{
   392  					"key":                       "value",
   393  					"topology.istio.io/network": "n2",
   394  					"topology.istio.io/cluster": "c2",
   395  				},
   396  				expected: []*endpoint.LocalityLbEndpoints{
   397  					{
   398  						Locality: &core.Locality{
   399  							Region:  "region1",
   400  							Zone:    "zone1",
   401  							SubZone: "subzone1",
   402  						},
   403  						LbEndpoints: []*endpoint.LbEndpoint{
   404  							{
   405  								HostIdentifier: buildEndpoint("2.2.2.2"), // match [key, network, cluster]
   406  								LoadBalancingWeight: &wrappers.UInt32Value{
   407  									Value: 1,
   408  								},
   409  							},
   410  						},
   411  						LoadBalancingWeight: &wrappers.UInt32Value{
   412  							Value: 1,
   413  						},
   414  						Priority: 0,
   415  					},
   416  					{
   417  						Locality: &core.Locality{
   418  							Region:  "region1",
   419  							Zone:    "zone1",
   420  							SubZone: "subzone1",
   421  						},
   422  						LbEndpoints: []*endpoint.LbEndpoint{
   423  							{
   424  								HostIdentifier: buildEndpoint("1.1.1.1"), // match no label
   425  								LoadBalancingWeight: &wrappers.UInt32Value{
   426  									Value: 1,
   427  								},
   428  							},
   429  						},
   430  						LoadBalancingWeight: &wrappers.UInt32Value{
   431  							Value: 1,
   432  						},
   433  						Priority: 3,
   434  					},
   435  					{
   436  						Locality: &core.Locality{
   437  							Region:  "region2",
   438  							Zone:    "zone2",
   439  							SubZone: "subzone2",
   440  						},
   441  						LbEndpoints: []*endpoint.LbEndpoint{
   442  							{
   443  								HostIdentifier: buildEndpoint("4.4.4.4"), // match [key, network]
   444  								LoadBalancingWeight: &wrappers.UInt32Value{
   445  									Value: 1,
   446  								},
   447  							},
   448  						},
   449  						LoadBalancingWeight: &wrappers.UInt32Value{
   450  							Value: 1,
   451  						},
   452  						Priority: 1,
   453  					},
   454  					{
   455  						Locality: &core.Locality{
   456  							Region:  "region2",
   457  							Zone:    "zone2",
   458  							SubZone: "subzone2",
   459  						},
   460  						LbEndpoints: []*endpoint.LbEndpoint{
   461  							{
   462  								HostIdentifier: buildEndpoint("3.3.3.3"), // match [key]
   463  								LoadBalancingWeight: &wrappers.UInt32Value{
   464  									Value: 1,
   465  								},
   466  							},
   467  						},
   468  						LoadBalancingWeight: &wrappers.UInt32Value{
   469  							Value: 1,
   470  						},
   471  						Priority: 2,
   472  					},
   473  				},
   474  			},
   475  			{
   476  				name:             "priority assignment as per the overridden value",
   477  				failoverPriority: []string{"topology.istio.io/network=n1"},
   478  				proxyLabels: map[string]string{
   479  					"topology.istio.io/network": "n2",
   480  					"topology.istio.io/cluster": "test",
   481  				},
   482  				expected: []*endpoint.LocalityLbEndpoints{
   483  					{
   484  						Locality: &core.Locality{
   485  							Region:  "region1",
   486  							Zone:    "zone1",
   487  							SubZone: "subzone1",
   488  						},
   489  						LbEndpoints: []*endpoint.LbEndpoint{
   490  							{
   491  								HostIdentifier: buildEndpoint("1.1.1.1"),
   492  								LoadBalancingWeight: &wrappers.UInt32Value{
   493  									Value: 1,
   494  								},
   495  							},
   496  						},
   497  						LoadBalancingWeight: &wrappers.UInt32Value{
   498  							Value: 1,
   499  						},
   500  						Priority: 0,
   501  					},
   502  					{
   503  						Locality: &core.Locality{
   504  							Region:  "region1",
   505  							Zone:    "zone1",
   506  							SubZone: "subzone1",
   507  						},
   508  						LbEndpoints: []*endpoint.LbEndpoint{
   509  							{
   510  								HostIdentifier: buildEndpoint("2.2.2.2"),
   511  								LoadBalancingWeight: &wrappers.UInt32Value{
   512  									Value: 1,
   513  								},
   514  							},
   515  						},
   516  						LoadBalancingWeight: &wrappers.UInt32Value{
   517  							Value: 1,
   518  						},
   519  						Priority: 1,
   520  					},
   521  					{
   522  						Locality: &core.Locality{
   523  							Region:  "region2",
   524  							Zone:    "zone2",
   525  							SubZone: "subzone2",
   526  						},
   527  						LbEndpoints: []*endpoint.LbEndpoint{
   528  							{
   529  								HostIdentifier: buildEndpoint("3.3.3.3"),
   530  								LoadBalancingWeight: &wrappers.UInt32Value{
   531  									Value: 1,
   532  								},
   533  							},
   534  						},
   535  						LoadBalancingWeight: &wrappers.UInt32Value{
   536  							Value: 1,
   537  						},
   538  						Priority: 0,
   539  					},
   540  					{
   541  						Locality: &core.Locality{
   542  							Region:  "region2",
   543  							Zone:    "zone2",
   544  							SubZone: "subzone2",
   545  						},
   546  						LbEndpoints: []*endpoint.LbEndpoint{
   547  							{
   548  								HostIdentifier: buildEndpoint("4.4.4.4"),
   549  								LoadBalancingWeight: &wrappers.UInt32Value{
   550  									Value: 1,
   551  								},
   552  							},
   553  						},
   554  						LoadBalancingWeight: &wrappers.UInt32Value{
   555  							Value: 1,
   556  						},
   557  						Priority: 1,
   558  					},
   559  				},
   560  			},
   561  			{
   562  				name:             "no endpoints with overridden value",
   563  				failoverPriority: []string{"topology.istio.io/network=n3"},
   564  				proxyLabels: map[string]string{
   565  					"topology.istio.io/network": "n1",
   566  					"topology.istio.io/cluster": "test",
   567  				},
   568  				expected: []*endpoint.LocalityLbEndpoints{
   569  					{
   570  						Locality: &core.Locality{
   571  							Region:  "region1",
   572  							Zone:    "zone1",
   573  							SubZone: "subzone1",
   574  						},
   575  						LbEndpoints: []*endpoint.LbEndpoint{
   576  							{
   577  								HostIdentifier: buildEndpoint("1.1.1.1"),
   578  								LoadBalancingWeight: &wrappers.UInt32Value{
   579  									Value: 1,
   580  								},
   581  							},
   582  							{
   583  								HostIdentifier: buildEndpoint("2.2.2.2"),
   584  								LoadBalancingWeight: &wrappers.UInt32Value{
   585  									Value: 1,
   586  								},
   587  							},
   588  						},
   589  						LoadBalancingWeight: &wrappers.UInt32Value{
   590  							Value: 2,
   591  						},
   592  						Priority: 0,
   593  					},
   594  					{
   595  						Locality: &core.Locality{
   596  							Region:  "region2",
   597  							Zone:    "zone2",
   598  							SubZone: "subzone2",
   599  						},
   600  						LbEndpoints: []*endpoint.LbEndpoint{
   601  							{
   602  								HostIdentifier: buildEndpoint("3.3.3.3"),
   603  								LoadBalancingWeight: &wrappers.UInt32Value{
   604  									Value: 1,
   605  								},
   606  							},
   607  							{
   608  								HostIdentifier: buildEndpoint("4.4.4.4"),
   609  								LoadBalancingWeight: &wrappers.UInt32Value{
   610  									Value: 1,
   611  								},
   612  							},
   613  						},
   614  						LoadBalancingWeight: &wrappers.UInt32Value{
   615  							Value: 2,
   616  						},
   617  						Priority: 0,
   618  					},
   619  				},
   620  			},
   621  		}
   622  
   623  		wrappedEndpoints := buildWrappedLocalityLbEndpoints()
   624  
   625  		for _, tt := range tests {
   626  			t.Run(tt.name, func(t *testing.T) {
   627  				env := buildEnvForClustersWithFailoverPriority(tt.failoverPriority)
   628  				cluster := buildFakeCluster()
   629  				ApplyLocalityLoadBalancer(cluster.LoadAssignment, wrappedEndpoints, locality, tt.proxyLabels, env.Mesh().LocalityLbSetting, true)
   630  
   631  				if len(cluster.LoadAssignment.Endpoints) != len(tt.expected) {
   632  					t.Fatalf("expected endpoints %d but got %d", len(cluster.LoadAssignment.Endpoints), len(tt.expected))
   633  				}
   634  				for i := range cluster.LoadAssignment.Endpoints {
   635  					// TODO Below assertions are not robust to ordering changes in cluster.LoadAssignment.Endpoints[i]
   636  					if cluster.LoadAssignment.Endpoints[i].LoadBalancingWeight.GetValue() != tt.expected[i].LoadBalancingWeight.GetValue() {
   637  						t.Errorf("Got unexpected lb weight %v expected %v", cluster.LoadAssignment.Endpoints[i].LoadBalancingWeight, tt.expected[i].LoadBalancingWeight)
   638  					}
   639  					if cluster.LoadAssignment.Endpoints[i].Priority != tt.expected[i].Priority {
   640  						t.Errorf("Got unexpected priority %v expected %v", cluster.LoadAssignment.Endpoints[i].Priority, tt.expected[i].Priority)
   641  					}
   642  				}
   643  			})
   644  		}
   645  	})
   646  
   647  	t.Run("FailoverPriority with Failover", func(t *testing.T) {
   648  		tests := []struct {
   649  			name             string
   650  			failoverPriority []string
   651  			proxyLabels      map[string]string
   652  			expected         []*endpoint.LocalityLbEndpoints
   653  		}{
   654  			{
   655  				name:             "match all labels",
   656  				failoverPriority: []string{"topology.istio.io/network", "topology.istio.io/cluster"},
   657  				proxyLabels: map[string]string{
   658  					"topology.istio.io/network": "n2",
   659  					"topology.istio.io/cluster": "c2",
   660  				},
   661  				expected: []*endpoint.LocalityLbEndpoints{
   662  					{
   663  						Locality: &core.Locality{
   664  							Region:  "region1",
   665  							Zone:    "zone1",
   666  							SubZone: "subzone1",
   667  						},
   668  						LbEndpoints: []*endpoint.LbEndpoint{
   669  							{
   670  								HostIdentifier: buildEndpoint("1.1.1.1"),
   671  								LoadBalancingWeight: &wrappers.UInt32Value{
   672  									Value: 1,
   673  								},
   674  							},
   675  						},
   676  						LoadBalancingWeight: &wrappers.UInt32Value{
   677  							Value: 1,
   678  						},
   679  						Priority: 3, // does not match failoverPriority but match locality of the client.
   680  					},
   681  					{
   682  						Locality: &core.Locality{
   683  							Region:  "region1",
   684  							Zone:    "zone2",
   685  							SubZone: "subzone2",
   686  						},
   687  						LbEndpoints: []*endpoint.LbEndpoint{
   688  							{
   689  								HostIdentifier: buildEndpoint("2.2.2.2"),
   690  								LoadBalancingWeight: &wrappers.UInt32Value{
   691  									Value: 1,
   692  								},
   693  							},
   694  						},
   695  						LoadBalancingWeight: &wrappers.UInt32Value{
   696  							Value: 1,
   697  						},
   698  						Priority: 0, // highest priority because of label matching and same region.
   699  					},
   700  					{
   701  						Locality: &core.Locality{
   702  							Region:  "region2",
   703  							Zone:    "zone2",
   704  							SubZone: "subzone2",
   705  						},
   706  						LbEndpoints: []*endpoint.LbEndpoint{
   707  							{
   708  								HostIdentifier: buildEndpoint("3.3.3.3"),
   709  								LoadBalancingWeight: &wrappers.UInt32Value{
   710  									Value: 1,
   711  								},
   712  							},
   713  						},
   714  						LoadBalancingWeight: &wrappers.UInt32Value{
   715  							Value: 1,
   716  						},
   717  						Priority: 4, // does not match failoverPriority and locality but mentioned in locality failover settings for the client region.
   718  					},
   719  					{
   720  						Locality: &core.Locality{
   721  							Region:  "region3",
   722  							Zone:    "zone3",
   723  							SubZone: "subzone3",
   724  						},
   725  						LbEndpoints: []*endpoint.LbEndpoint{
   726  							{
   727  								HostIdentifier: buildEndpoint("4.4.4.4"),
   728  								LoadBalancingWeight: &wrappers.UInt32Value{
   729  									Value: 1,
   730  								},
   731  							},
   732  						},
   733  						LoadBalancingWeight: &wrappers.UInt32Value{
   734  							Value: 1,
   735  						},
   736  						Priority: 2, // match the first label of failoverPriority but not the second
   737  					},
   738  					{
   739  						Locality: &core.Locality{
   740  							Region:  "region2",
   741  							Zone:    "zone2",
   742  							SubZone: "subzone2",
   743  						},
   744  						LbEndpoints: []*endpoint.LbEndpoint{
   745  							{
   746  								HostIdentifier: buildEndpoint("5.5.5.5"),
   747  								LoadBalancingWeight: &wrappers.UInt32Value{
   748  									Value: 1,
   749  								},
   750  							},
   751  						},
   752  						LoadBalancingWeight: &wrappers.UInt32Value{
   753  							Value: 1,
   754  						},
   755  						Priority: 1, // matching the same failoverPriority but different locality.
   756  					},
   757  					{
   758  						Locality: &core.Locality{
   759  							Region:  "region3",
   760  							Zone:    "zone3",
   761  							SubZone: "subzone3",
   762  						},
   763  						LbEndpoints: []*endpoint.LbEndpoint{
   764  							{
   765  								HostIdentifier: buildEndpoint("6.6.6.6"),
   766  								LoadBalancingWeight: &wrappers.UInt32Value{
   767  									Value: 1,
   768  								},
   769  							},
   770  						},
   771  						LoadBalancingWeight: &wrappers.UInt32Value{
   772  							Value: 1,
   773  						},
   774  						Priority: 5, // does not match any of the priority constructs so least priority.
   775  					},
   776  				},
   777  			},
   778  		}
   779  		wrappedEndpoints := buildWrappedLocalityLbEndpointsForFailoverPriorityWithFailover()
   780  		for _, tt := range tests {
   781  			t.Run(tt.name, func(t *testing.T) {
   782  				env := buildEnvForClustersWithMixedFailoverPriorityAndLocalityFailover(tt.failoverPriority)
   783  				cluster := buildFakeCluster()
   784  				ApplyLocalityLoadBalancer(cluster.LoadAssignment, wrappedEndpoints, locality, tt.proxyLabels, env.Mesh().LocalityLbSetting, true)
   785  
   786  				if len(cluster.LoadAssignment.Endpoints) != len(tt.expected) {
   787  					t.Fatalf("expected endpoints %d but got %d", len(cluster.LoadAssignment.Endpoints), len(tt.expected))
   788  				}
   789  				for i := range cluster.LoadAssignment.Endpoints {
   790  					// TODO Below assertions are not robust to ordering changes in cluster.LoadAssignment.Endpoints[i]
   791  					if cluster.LoadAssignment.Endpoints[i].LoadBalancingWeight.GetValue() != tt.expected[i].LoadBalancingWeight.GetValue() {
   792  						t.Errorf("Got unexpected lb weight %v expected %v", cluster.LoadAssignment.Endpoints[i].LoadBalancingWeight, tt.expected[i].LoadBalancingWeight)
   793  					}
   794  					if cluster.LoadAssignment.Endpoints[i].Priority != tt.expected[i].Priority {
   795  						t.Errorf("Got unexpected priority %v expected %v", cluster.LoadAssignment.Endpoints[i].Priority, tt.expected[i].Priority)
   796  					}
   797  				}
   798  			})
   799  		}
   800  	})
   801  }
   802  
   803  func TestGetLocalityLbSetting(t *testing.T) {
   804  	// dummy config for test
   805  	failover := []*networking.LocalityLoadBalancerSetting_Failover{nil}
   806  	cases := []struct {
   807  		name     string
   808  		mesh     *networking.LocalityLoadBalancerSetting
   809  		dr       *networking.LocalityLoadBalancerSetting
   810  		expected *networking.LocalityLoadBalancerSetting
   811  	}{
   812  		{
   813  			"all disabled",
   814  			nil,
   815  			nil,
   816  			nil,
   817  		},
   818  		{
   819  			"mesh only",
   820  			&networking.LocalityLoadBalancerSetting{},
   821  			nil,
   822  			&networking.LocalityLoadBalancerSetting{},
   823  		},
   824  		{
   825  			"dr only",
   826  			nil,
   827  			&networking.LocalityLoadBalancerSetting{},
   828  			&networking.LocalityLoadBalancerSetting{},
   829  		},
   830  		{
   831  			"dr only override",
   832  			nil,
   833  			&networking.LocalityLoadBalancerSetting{Enabled: &wrappers.BoolValue{Value: true}},
   834  			&networking.LocalityLoadBalancerSetting{Enabled: &wrappers.BoolValue{Value: true}},
   835  		},
   836  		{
   837  			"both",
   838  			&networking.LocalityLoadBalancerSetting{},
   839  			&networking.LocalityLoadBalancerSetting{Failover: failover},
   840  			&networking.LocalityLoadBalancerSetting{Failover: failover},
   841  		},
   842  		{
   843  			"mesh disabled",
   844  			&networking.LocalityLoadBalancerSetting{Enabled: &wrappers.BoolValue{Value: false}},
   845  			nil,
   846  			nil,
   847  		},
   848  		{
   849  			"dr disabled",
   850  			&networking.LocalityLoadBalancerSetting{Enabled: &wrappers.BoolValue{Value: true}},
   851  			&networking.LocalityLoadBalancerSetting{Enabled: &wrappers.BoolValue{Value: false}},
   852  			nil,
   853  		},
   854  		{
   855  			"dr enabled override mesh disabled",
   856  			&networking.LocalityLoadBalancerSetting{Enabled: &wrappers.BoolValue{Value: false}},
   857  			&networking.LocalityLoadBalancerSetting{Enabled: &wrappers.BoolValue{Value: true}},
   858  			&networking.LocalityLoadBalancerSetting{Enabled: &wrappers.BoolValue{Value: true}},
   859  		},
   860  	}
   861  	for _, tt := range cases {
   862  		t.Run(tt.name, func(t *testing.T) {
   863  			got := GetLocalityLbSetting(tt.mesh, tt.dr)
   864  			if !reflect.DeepEqual(tt.expected, got) {
   865  				t.Fatalf("Expected: %v, got: %v", tt.expected, got)
   866  			}
   867  		})
   868  	}
   869  }
   870  
   871  func buildEnvForClustersWithDistribute(distribute []*networking.LocalityLoadBalancerSetting_Distribute) *model.Environment {
   872  	serviceDiscovery := memregistry.NewServiceDiscovery(&model.Service{
   873  		Hostname:       "test.example.org",
   874  		DefaultAddress: "1.1.1.1",
   875  		Ports: model.PortList{
   876  			&model.Port{
   877  				Name:     "default",
   878  				Port:     8080,
   879  				Protocol: protocol.HTTP,
   880  			},
   881  		},
   882  	})
   883  
   884  	meshConfig := &meshconfig.MeshConfig{
   885  		ConnectTimeout: &durationpb.Duration{
   886  			Seconds: 10,
   887  			Nanos:   1,
   888  		},
   889  		LocalityLbSetting: &networking.LocalityLoadBalancerSetting{
   890  			Distribute: distribute,
   891  		},
   892  	}
   893  
   894  	configStore := memory.Make(collections.Pilot)
   895  
   896  	env := model.NewEnvironment()
   897  	env.ServiceDiscovery = serviceDiscovery
   898  	env.ConfigStore = configStore
   899  	env.Watcher = mesh.NewFixedWatcher(meshConfig)
   900  
   901  	pushContext := model.NewPushContext()
   902  	env.Init()
   903  	_ = pushContext.InitContext(env, nil, nil)
   904  	env.SetPushContext(pushContext)
   905  	pushContext.SetDestinationRulesForTesting([]config.Config{
   906  		{
   907  			Meta: config.Meta{
   908  				GroupVersionKind: gvk.DestinationRule,
   909  				Name:             "acme",
   910  			},
   911  			Spec: &networking.DestinationRule{
   912  				Host: "test.example.org",
   913  				TrafficPolicy: &networking.TrafficPolicy{
   914  					OutlierDetection: &networking.OutlierDetection{},
   915  				},
   916  			},
   917  		},
   918  	})
   919  
   920  	return env
   921  }
   922  
   923  func buildEnvForClustersWithFailover() *model.Environment {
   924  	serviceDiscovery := memregistry.NewServiceDiscovery(&model.Service{
   925  		Hostname:       "test.example.org",
   926  		DefaultAddress: "1.1.1.1",
   927  		Ports: model.PortList{
   928  			&model.Port{
   929  				Name:     "default",
   930  				Port:     8080,
   931  				Protocol: protocol.HTTP,
   932  			},
   933  		},
   934  	})
   935  
   936  	meshConfig := &meshconfig.MeshConfig{
   937  		ConnectTimeout: &durationpb.Duration{
   938  			Seconds: 10,
   939  			Nanos:   1,
   940  		},
   941  		LocalityLbSetting: &networking.LocalityLoadBalancerSetting{
   942  			Failover: []*networking.LocalityLoadBalancerSetting_Failover{
   943  				{
   944  					From: "region1",
   945  					To:   "region2",
   946  				},
   947  			},
   948  		},
   949  	}
   950  
   951  	configStore := memory.Make(collections.Pilot)
   952  
   953  	env := model.NewEnvironment()
   954  	env.ServiceDiscovery = serviceDiscovery
   955  	env.ConfigStore = configStore
   956  	env.Watcher = mesh.NewFixedWatcher(meshConfig)
   957  
   958  	pushContext := model.NewPushContext()
   959  	env.Init()
   960  	_ = pushContext.InitContext(env, nil, nil)
   961  	env.SetPushContext(pushContext)
   962  	pushContext.SetDestinationRulesForTesting([]config.Config{
   963  		{
   964  			Meta: config.Meta{
   965  				GroupVersionKind: gvk.DestinationRule,
   966  				Name:             "acme",
   967  			},
   968  			Spec: &networking.DestinationRule{
   969  				Host: "test.example.org",
   970  				TrafficPolicy: &networking.TrafficPolicy{
   971  					OutlierDetection: &networking.OutlierDetection{},
   972  				},
   973  			},
   974  		},
   975  	})
   976  
   977  	return env
   978  }
   979  
   980  func buildEnvForClustersWithFailoverPriority(failoverPriority []string) *model.Environment {
   981  	serviceDiscovery := memregistry.NewServiceDiscovery(&model.Service{
   982  		Hostname:       "test.example.org",
   983  		DefaultAddress: "1.1.1.1",
   984  		Ports: model.PortList{
   985  			&model.Port{
   986  				Name:     "default",
   987  				Port:     8080,
   988  				Protocol: protocol.HTTP,
   989  			},
   990  		},
   991  	})
   992  
   993  	meshConfig := &meshconfig.MeshConfig{
   994  		ConnectTimeout: &durationpb.Duration{
   995  			Seconds: 10,
   996  			Nanos:   1,
   997  		},
   998  		LocalityLbSetting: &networking.LocalityLoadBalancerSetting{
   999  			FailoverPriority: failoverPriority,
  1000  		},
  1001  	}
  1002  
  1003  	configStore := memory.Make(collections.Pilot)
  1004  
  1005  	env := model.NewEnvironment()
  1006  	env.ServiceDiscovery = serviceDiscovery
  1007  	env.ConfigStore = configStore
  1008  	env.Watcher = mesh.NewFixedWatcher(meshConfig)
  1009  
  1010  	pushContext := model.NewPushContext()
  1011  	env.Init()
  1012  	_ = pushContext.InitContext(env, nil, nil)
  1013  	env.SetPushContext(pushContext)
  1014  	pushContext.SetDestinationRulesForTesting([]config.Config{
  1015  		{
  1016  			Meta: config.Meta{
  1017  				GroupVersionKind: gvk.DestinationRule,
  1018  				Name:             "acme",
  1019  			},
  1020  			Spec: &networking.DestinationRule{
  1021  				Host: "test.example.org",
  1022  				TrafficPolicy: &networking.TrafficPolicy{
  1023  					OutlierDetection: &networking.OutlierDetection{},
  1024  				},
  1025  			},
  1026  		},
  1027  	})
  1028  
  1029  	return env
  1030  }
  1031  
  1032  func buildEnvForClustersWithMixedFailoverPriorityAndLocalityFailover(failoverPriority []string) *model.Environment {
  1033  	serviceDiscovery := memregistry.NewServiceDiscovery(&model.Service{
  1034  		Hostname:       "test.example.org",
  1035  		DefaultAddress: "1.1.1.1",
  1036  		Ports: model.PortList{
  1037  			&model.Port{
  1038  				Name:     "default",
  1039  				Port:     8080,
  1040  				Protocol: protocol.HTTP,
  1041  			},
  1042  		},
  1043  	})
  1044  
  1045  	meshConfig := &meshconfig.MeshConfig{
  1046  		ConnectTimeout: &durationpb.Duration{
  1047  			Seconds: 10,
  1048  			Nanos:   1,
  1049  		},
  1050  		LocalityLbSetting: &networking.LocalityLoadBalancerSetting{
  1051  			FailoverPriority: failoverPriority,
  1052  			Failover: []*networking.LocalityLoadBalancerSetting_Failover{
  1053  				{
  1054  					From: "region1",
  1055  					To:   "region2",
  1056  				},
  1057  			},
  1058  		},
  1059  	}
  1060  
  1061  	configStore := memory.Make(collections.Pilot)
  1062  
  1063  	env := model.NewEnvironment()
  1064  	env.ServiceDiscovery = serviceDiscovery
  1065  	env.ConfigStore = configStore
  1066  	env.Watcher = mesh.NewFixedWatcher(meshConfig)
  1067  
  1068  	pushContext := model.NewPushContext()
  1069  	env.Init()
  1070  	_ = pushContext.InitContext(env, nil, nil)
  1071  	env.SetPushContext(pushContext)
  1072  	pushContext.SetDestinationRulesForTesting([]config.Config{
  1073  		{
  1074  			Meta: config.Meta{
  1075  				GroupVersionKind: gvk.DestinationRule,
  1076  				Name:             "acme",
  1077  			},
  1078  			Spec: &networking.DestinationRule{
  1079  				Host: "test.example.org",
  1080  				TrafficPolicy: &networking.TrafficPolicy{
  1081  					OutlierDetection: &networking.OutlierDetection{},
  1082  				},
  1083  			},
  1084  		},
  1085  	})
  1086  
  1087  	return env
  1088  }
  1089  
  1090  func buildFakeCluster() *cluster.Cluster {
  1091  	return &cluster.Cluster{
  1092  		Name: "outbound|8080||test.example.org",
  1093  		LoadAssignment: &endpoint.ClusterLoadAssignment{
  1094  			ClusterName: "outbound|8080||test.example.org",
  1095  			Endpoints: []*endpoint.LocalityLbEndpoints{
  1096  				{
  1097  					Locality: &core.Locality{
  1098  						Region:  "region1",
  1099  						Zone:    "zone1",
  1100  						SubZone: "subzone1",
  1101  					},
  1102  				},
  1103  				{
  1104  					Locality: &core.Locality{
  1105  						Region:  "region1",
  1106  						Zone:    "zone1",
  1107  						SubZone: "subzone1",
  1108  					},
  1109  				},
  1110  				{
  1111  					Locality: &core.Locality{
  1112  						Region:  "region1",
  1113  						Zone:    "zone1",
  1114  						SubZone: "subzone2",
  1115  					},
  1116  				},
  1117  				{
  1118  					Locality: &core.Locality{
  1119  						Region:  "region1",
  1120  						Zone:    "zone1",
  1121  						SubZone: "subzone3",
  1122  					},
  1123  				},
  1124  				{
  1125  					Locality: &core.Locality{
  1126  						Region:  "region1",
  1127  						Zone:    "zone2",
  1128  						SubZone: "",
  1129  					},
  1130  				},
  1131  				{
  1132  					Locality: &core.Locality{
  1133  						Region:  "region2",
  1134  						Zone:    "",
  1135  						SubZone: "",
  1136  					},
  1137  				},
  1138  				{
  1139  					Locality: &core.Locality{
  1140  						Region:  "region3",
  1141  						Zone:    "",
  1142  						SubZone: "",
  1143  					},
  1144  				},
  1145  			},
  1146  		},
  1147  	}
  1148  }
  1149  
  1150  func buildSmallCluster() *cluster.Cluster {
  1151  	return &cluster.Cluster{
  1152  		Name: "outbound|8080||test.example.org",
  1153  		LoadAssignment: &endpoint.ClusterLoadAssignment{
  1154  			ClusterName: "outbound|8080||test.example.org",
  1155  			Endpoints: []*endpoint.LocalityLbEndpoints{
  1156  				{
  1157  					Locality: &core.Locality{
  1158  						Region:  "region1",
  1159  						Zone:    "zone1",
  1160  						SubZone: "subzone2",
  1161  					},
  1162  				},
  1163  				{
  1164  					Locality: &core.Locality{
  1165  						Region:  "region2",
  1166  						Zone:    "zone1",
  1167  						SubZone: "subzone2",
  1168  					},
  1169  				},
  1170  				{
  1171  					Locality: &core.Locality{
  1172  						Region:  "region2",
  1173  						Zone:    "zone1",
  1174  						SubZone: "subzone2",
  1175  					},
  1176  				},
  1177  			},
  1178  		},
  1179  	}
  1180  }
  1181  
  1182  func buildSmallClusterWithNilLocalities() *cluster.Cluster {
  1183  	return &cluster.Cluster{
  1184  		Name: "outbound|8080||test.example.org",
  1185  		LoadAssignment: &endpoint.ClusterLoadAssignment{
  1186  			ClusterName: "outbound|8080||test.example.org",
  1187  			Endpoints: []*endpoint.LocalityLbEndpoints{
  1188  				{
  1189  					Locality: &core.Locality{
  1190  						Region:  "region1",
  1191  						Zone:    "zone1",
  1192  						SubZone: "subzone2",
  1193  					},
  1194  				},
  1195  				{},
  1196  				{
  1197  					Locality: &core.Locality{
  1198  						Region:  "region2",
  1199  						Zone:    "zone1",
  1200  						SubZone: "subzone2",
  1201  					},
  1202  				},
  1203  			},
  1204  		},
  1205  	}
  1206  }
  1207  
  1208  func buildSmallClusterForFailOverPriority() *cluster.Cluster {
  1209  	return &cluster.Cluster{
  1210  		Name: "outbound|8080||test.example.org",
  1211  		LoadAssignment: &endpoint.ClusterLoadAssignment{
  1212  			ClusterName: "outbound|8080||test.example.org",
  1213  			Endpoints: []*endpoint.LocalityLbEndpoints{
  1214  				{
  1215  					Locality: &core.Locality{
  1216  						Region:  "region1",
  1217  						Zone:    "zone1",
  1218  						SubZone: "subzone1",
  1219  					},
  1220  					LbEndpoints: []*endpoint.LbEndpoint{
  1221  						{
  1222  							HostIdentifier: buildEndpoint("1.1.1.1"),
  1223  							LoadBalancingWeight: &wrappers.UInt32Value{
  1224  								Value: 1,
  1225  							},
  1226  						},
  1227  						{
  1228  							HostIdentifier: buildEndpoint("2.2.2.2"),
  1229  							LoadBalancingWeight: &wrappers.UInt32Value{
  1230  								Value: 1,
  1231  							},
  1232  						},
  1233  					},
  1234  				},
  1235  				{
  1236  					Locality: &core.Locality{
  1237  						Region:  "region2",
  1238  						Zone:    "zone2",
  1239  						SubZone: "subzone2",
  1240  					},
  1241  					LbEndpoints: []*endpoint.LbEndpoint{
  1242  						{
  1243  							HostIdentifier: buildEndpoint("3.3.3.3"),
  1244  							LoadBalancingWeight: &wrappers.UInt32Value{
  1245  								Value: 1,
  1246  							},
  1247  						},
  1248  						{
  1249  							HostIdentifier: buildEndpoint("4.4.4.4"),
  1250  							LoadBalancingWeight: &wrappers.UInt32Value{
  1251  								Value: 1,
  1252  							},
  1253  						},
  1254  					},
  1255  				},
  1256  			},
  1257  		},
  1258  	}
  1259  }
  1260  
  1261  func buildSmallClusterForFailOverPriorityWithFailover() *cluster.Cluster {
  1262  	return &cluster.Cluster{
  1263  		Name: "outbound|8080||test.example.org",
  1264  		LoadAssignment: &endpoint.ClusterLoadAssignment{
  1265  			ClusterName: "outbound|8080||test.example.org",
  1266  			Endpoints: []*endpoint.LocalityLbEndpoints{
  1267  				{
  1268  					Locality: &core.Locality{
  1269  						Region:  "region1",
  1270  						Zone:    "zone1",
  1271  						SubZone: "subzone1",
  1272  					},
  1273  					LbEndpoints: []*endpoint.LbEndpoint{
  1274  						{
  1275  							HostIdentifier: buildEndpoint("1.1.1.1"),
  1276  							LoadBalancingWeight: &wrappers.UInt32Value{
  1277  								Value: 1,
  1278  							},
  1279  						},
  1280  					},
  1281  				},
  1282  				{
  1283  					Locality: &core.Locality{
  1284  						Region:  "region1",
  1285  						Zone:    "zone2",
  1286  						SubZone: "subzone2",
  1287  					},
  1288  					LbEndpoints: []*endpoint.LbEndpoint{
  1289  						{
  1290  							HostIdentifier: buildEndpoint("2.2.2.2"),
  1291  							LoadBalancingWeight: &wrappers.UInt32Value{
  1292  								Value: 1,
  1293  							},
  1294  						},
  1295  					},
  1296  				},
  1297  				{
  1298  					Locality: &core.Locality{
  1299  						Region:  "region2",
  1300  						Zone:    "zone2",
  1301  						SubZone: "subzone2",
  1302  					},
  1303  					LbEndpoints: []*endpoint.LbEndpoint{
  1304  						{
  1305  							HostIdentifier: buildEndpoint("3.3.3.3"),
  1306  							LoadBalancingWeight: &wrappers.UInt32Value{
  1307  								Value: 1,
  1308  							},
  1309  						},
  1310  					},
  1311  				},
  1312  				{
  1313  					Locality: &core.Locality{
  1314  						Region:  "region3",
  1315  						Zone:    "zone3",
  1316  						SubZone: "subzone3",
  1317  					},
  1318  					LbEndpoints: []*endpoint.LbEndpoint{
  1319  						{
  1320  							HostIdentifier: buildEndpoint("4.4.4.4"),
  1321  							LoadBalancingWeight: &wrappers.UInt32Value{
  1322  								Value: 1,
  1323  							},
  1324  						},
  1325  					},
  1326  				},
  1327  				{
  1328  					Locality: &core.Locality{
  1329  						Region:  "region2",
  1330  						Zone:    "zone2",
  1331  						SubZone: "subzone2",
  1332  					},
  1333  					LbEndpoints: []*endpoint.LbEndpoint{
  1334  						{
  1335  							HostIdentifier: buildEndpoint("5.5.5.5"),
  1336  							LoadBalancingWeight: &wrappers.UInt32Value{
  1337  								Value: 1,
  1338  							},
  1339  						},
  1340  					},
  1341  				},
  1342  				{
  1343  					Locality: &core.Locality{
  1344  						Region:  "region3",
  1345  						Zone:    "zone3",
  1346  						SubZone: "subzone3",
  1347  					},
  1348  					LbEndpoints: []*endpoint.LbEndpoint{
  1349  						{
  1350  							HostIdentifier: buildEndpoint("6.6.6.6"),
  1351  							LoadBalancingWeight: &wrappers.UInt32Value{
  1352  								Value: 1,
  1353  							},
  1354  						},
  1355  					},
  1356  				},
  1357  			},
  1358  		},
  1359  	}
  1360  }
  1361  
  1362  func buildEndpoint(ip string) *endpoint.LbEndpoint_Endpoint {
  1363  	return &endpoint.LbEndpoint_Endpoint{
  1364  		Endpoint: &endpoint.Endpoint{
  1365  			Address: &core.Address{
  1366  				Address: &core.Address_SocketAddress{
  1367  					SocketAddress: &core.SocketAddress{
  1368  						Address: ip,
  1369  						PortSpecifier: &core.SocketAddress_PortValue{
  1370  							PortValue: 10001,
  1371  						},
  1372  					},
  1373  				},
  1374  			},
  1375  		},
  1376  	}
  1377  }
  1378  
  1379  func buildWrappedLocalityLbEndpoints() []*WrappedLocalityLbEndpoints {
  1380  	cluster := buildSmallClusterForFailOverPriority()
  1381  	return []*WrappedLocalityLbEndpoints{
  1382  		{
  1383  			IstioEndpoints: []*model.IstioEndpoint{
  1384  				{
  1385  					Labels: map[string]string{
  1386  						"topology.istio.io/network": "n1",
  1387  						"topology.istio.io/cluster": "c1",
  1388  					},
  1389  					Address: "1.1.1.1",
  1390  				},
  1391  				{
  1392  					Labels: map[string]string{
  1393  						"key":                       "value",
  1394  						"topology.istio.io/network": "n2",
  1395  						"topology.istio.io/cluster": "c2",
  1396  					},
  1397  					Address: "2.2.2.2",
  1398  				},
  1399  			},
  1400  			LocalityLbEndpoints: cluster.LoadAssignment.Endpoints[0],
  1401  		},
  1402  		{
  1403  			IstioEndpoints: []*model.IstioEndpoint{
  1404  				{
  1405  					Labels: map[string]string{
  1406  						"key":                       "value",
  1407  						"topology.istio.io/network": "n1",
  1408  						"topology.istio.io/cluster": "c3",
  1409  					},
  1410  					Address: "3.3.3.3",
  1411  				},
  1412  				{
  1413  					Labels: map[string]string{
  1414  						"key":                       "value",
  1415  						"topology.istio.io/network": "n2",
  1416  						"topology.istio.io/cluster": "c4",
  1417  					},
  1418  					Address: "4.4.4.4",
  1419  				},
  1420  			},
  1421  			LocalityLbEndpoints: cluster.LoadAssignment.Endpoints[1],
  1422  		},
  1423  	}
  1424  }
  1425  
  1426  func buildWrappedLocalityLbEndpointsForFailoverPriorityWithFailover() []*WrappedLocalityLbEndpoints {
  1427  	cluster := buildSmallClusterForFailOverPriorityWithFailover()
  1428  	return []*WrappedLocalityLbEndpoints{
  1429  		{
  1430  			IstioEndpoints: []*model.IstioEndpoint{
  1431  				{
  1432  					Labels: map[string]string{
  1433  						"topology.istio.io/network": "n1",
  1434  						"topology.istio.io/cluster": "c1",
  1435  					},
  1436  					Address: "1.1.1.1",
  1437  				},
  1438  			},
  1439  			LocalityLbEndpoints: cluster.LoadAssignment.Endpoints[0],
  1440  		},
  1441  		{
  1442  			IstioEndpoints: []*model.IstioEndpoint{
  1443  				{
  1444  					Labels: map[string]string{
  1445  						"key":                       "value",
  1446  						"topology.istio.io/network": "n2",
  1447  						"topology.istio.io/cluster": "c2",
  1448  					},
  1449  					Address: "2.2.2.2",
  1450  				},
  1451  			},
  1452  			LocalityLbEndpoints: cluster.LoadAssignment.Endpoints[1],
  1453  		},
  1454  		{
  1455  			IstioEndpoints: []*model.IstioEndpoint{
  1456  				{
  1457  					Labels: map[string]string{
  1458  						"key":                       "value",
  1459  						"topology.istio.io/network": "n1",
  1460  						"topology.istio.io/cluster": "c3",
  1461  					},
  1462  					Address: "3.3.3.3",
  1463  				},
  1464  			},
  1465  			LocalityLbEndpoints: cluster.LoadAssignment.Endpoints[2],
  1466  		},
  1467  		{
  1468  			IstioEndpoints: []*model.IstioEndpoint{
  1469  				{
  1470  					Labels: map[string]string{
  1471  						"key":                       "value",
  1472  						"topology.istio.io/network": "n2",
  1473  						"topology.istio.io/cluster": "c4",
  1474  					},
  1475  					Address: "4.4.4.4",
  1476  				},
  1477  			},
  1478  			LocalityLbEndpoints: cluster.LoadAssignment.Endpoints[3],
  1479  		},
  1480  		{
  1481  			IstioEndpoints: []*model.IstioEndpoint{
  1482  				{
  1483  					Labels: map[string]string{
  1484  						"topology.istio.io/network": "n2",
  1485  						"topology.istio.io/cluster": "c2",
  1486  					},
  1487  					Address: "5.5.5.5",
  1488  				},
  1489  			},
  1490  			LocalityLbEndpoints: cluster.LoadAssignment.Endpoints[4],
  1491  		},
  1492  		{
  1493  			IstioEndpoints: []*model.IstioEndpoint{
  1494  				{
  1495  					Labels: map[string]string{
  1496  						"key":                       "value",
  1497  						"topology.istio.io/network": "n1",
  1498  						"topology.istio.io/cluster": "c6",
  1499  					},
  1500  					Address: "6.6.6.6",
  1501  				},
  1502  			},
  1503  			LocalityLbEndpoints: cluster.LoadAssignment.Endpoints[5],
  1504  		},
  1505  	}
  1506  }