github.com/hxx258456/ccgo@v0.0.5-0.20230213014102-48b35f46f66f/grpc/xds/internal/balancer/clusterresolver/eds_impl_test.go (about)

     1  /*
     2   * Copyright 2019 gRPC authors.
     3   *
     4   * Licensed under the Apache License, Version 2.0 (the "License");
     5   * you may not use this file except in compliance with the License.
     6   * You may obtain a copy of the License at
     7   *
     8   *     http://www.apache.org/licenses/LICENSE-2.0
     9   *
    10   * Unless required by applicable law or agreed to in writing, software
    11   * distributed under the License is distributed on an "AS IS" BASIS,
    12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13   * See the License for the specific language governing permissions and
    14   * limitations under the License.
    15   */
    16  
    17  package clusterresolver
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"sort"
    23  	"testing"
    24  	"time"
    25  
    26  	"github.com/google/go-cmp/cmp"
    27  	corepb "github.com/hxx258456/ccgo/go-control-plane/envoy/api/v2/core"
    28  	"github.com/hxx258456/ccgo/grpc/balancer"
    29  	"github.com/hxx258456/ccgo/grpc/balancer/weightedtarget"
    30  	"github.com/hxx258456/ccgo/grpc/connectivity"
    31  	"github.com/hxx258456/ccgo/grpc/internal/balancergroup"
    32  	internalserviceconfig "github.com/hxx258456/ccgo/grpc/internal/serviceconfig"
    33  	"github.com/hxx258456/ccgo/grpc/internal/testutils"
    34  	"github.com/hxx258456/ccgo/grpc/resolver"
    35  	"github.com/hxx258456/ccgo/grpc/xds/internal/balancer/clusterimpl"
    36  	"github.com/hxx258456/ccgo/grpc/xds/internal/balancer/priority"
    37  	xdstestutils "github.com/hxx258456/ccgo/grpc/xds/internal/testutils"
    38  	"github.com/hxx258456/ccgo/grpc/xds/internal/testutils/fakeclient"
    39  	"github.com/hxx258456/ccgo/grpc/xds/internal/xdsclient"
    40  	"github.com/hxx258456/ccgo/grpc/xds/internal/xdsclient/xdsresource"
    41  )
    42  
    43  var (
    44  	testClusterNames  = []string{"test-cluster-1", "test-cluster-2"}
    45  	testSubZones      = []string{"I", "II", "III", "IV"}
    46  	testEndpointAddrs []string
    47  )
    48  
    49  const testBackendAddrsCount = 12
    50  
    51  func init() {
    52  	for i := 0; i < testBackendAddrsCount; i++ {
    53  		testEndpointAddrs = append(testEndpointAddrs, fmt.Sprintf("%d.%d.%d.%d:%d", i, i, i, i, i))
    54  	}
    55  	balancergroup.DefaultSubBalancerCloseTimeout = time.Millisecond
    56  	clusterimpl.NewRandomWRR = testutils.NewTestWRR
    57  	weightedtarget.NewRandomWRR = testutils.NewTestWRR
    58  	balancergroup.DefaultSubBalancerCloseTimeout = time.Millisecond * 100
    59  }
    60  
    61  func setupTestEDS(t *testing.T, initChild *internalserviceconfig.BalancerConfig) (balancer.Balancer, *testutils.TestClientConn, *fakeclient.Client, func()) {
    62  	xdsC := fakeclient.NewClientWithName(testBalancerNameFooBar)
    63  	cc := testutils.NewTestClientConn(t)
    64  	builder := balancer.Get(Name)
    65  	edsb := builder.Build(cc, balancer.BuildOptions{Target: resolver.Target{Endpoint: testEDSServcie}})
    66  	if edsb == nil {
    67  		t.Fatalf("builder.Build(%s) failed and returned nil", Name)
    68  	}
    69  	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
    70  	defer cancel()
    71  	if err := edsb.UpdateClientConnState(balancer.ClientConnState{
    72  		ResolverState: xdsclient.SetClient(resolver.State{}, xdsC),
    73  		BalancerConfig: &LBConfig{
    74  			DiscoveryMechanisms: []DiscoveryMechanism{{
    75  				Cluster: testClusterName,
    76  				Type:    DiscoveryMechanismTypeEDS,
    77  			}},
    78  		},
    79  	}); err != nil {
    80  		edsb.Close()
    81  		xdsC.Close()
    82  		t.Fatal(err)
    83  	}
    84  	if _, err := xdsC.WaitForWatchEDS(ctx); err != nil {
    85  		edsb.Close()
    86  		xdsC.Close()
    87  		t.Fatalf("xdsClient.WatchEndpoints failed with error: %v", err)
    88  	}
    89  	return edsb, cc, xdsC, func() {
    90  		edsb.Close()
    91  		xdsC.Close()
    92  	}
    93  }
    94  
    95  // One locality
    96  //  - add backend
    97  //  - remove backend
    98  //  - replace backend
    99  //  - change drop rate
   100  func (s) TestEDS_OneLocality(t *testing.T) {
   101  	edsb, cc, xdsC, cleanup := setupTestEDS(t, nil)
   102  	defer cleanup()
   103  
   104  	// One locality with one backend.
   105  	clab1 := xdstestutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
   106  	clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil)
   107  	xdsC.InvokeWatchEDSCallback("", parseEDSRespProtoForTesting(clab1.Build()), nil)
   108  
   109  	sc1 := <-cc.NewSubConnCh
   110  	edsb.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
   111  	edsb.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Ready})
   112  
   113  	// Pick with only the first backend.
   114  	if err := testRoundRobinPickerFromCh(cc.NewPickerCh, []balancer.SubConn{sc1}); err != nil {
   115  		t.Fatal(err)
   116  	}
   117  
   118  	// The same locality, add one more backend.
   119  	clab2 := xdstestutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
   120  	clab2.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:2], nil)
   121  	xdsC.InvokeWatchEDSCallback("", parseEDSRespProtoForTesting(clab2.Build()), nil)
   122  
   123  	sc2 := <-cc.NewSubConnCh
   124  	edsb.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
   125  	edsb.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Ready})
   126  
   127  	// Test roundrobin with two subconns.
   128  	if err := testRoundRobinPickerFromCh(cc.NewPickerCh, []balancer.SubConn{sc1, sc2}); err != nil {
   129  		t.Fatal(err)
   130  	}
   131  
   132  	// The same locality, delete first backend.
   133  	clab3 := xdstestutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
   134  	clab3.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[1:2], nil)
   135  	xdsC.InvokeWatchEDSCallback("", parseEDSRespProtoForTesting(clab3.Build()), nil)
   136  
   137  	scToRemove := <-cc.RemoveSubConnCh
   138  	if !cmp.Equal(scToRemove, sc1, cmp.AllowUnexported(testutils.TestSubConn{})) {
   139  		t.Fatalf("RemoveSubConn, want %v, got %v", sc1, scToRemove)
   140  	}
   141  	edsb.UpdateSubConnState(scToRemove, balancer.SubConnState{ConnectivityState: connectivity.Shutdown})
   142  
   143  	// Test pick with only the second subconn.
   144  	if err := testRoundRobinPickerFromCh(cc.NewPickerCh, []balancer.SubConn{sc2}); err != nil {
   145  		t.Fatal(err)
   146  	}
   147  
   148  	// The same locality, replace backend.
   149  	clab4 := xdstestutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
   150  	clab4.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[2:3], nil)
   151  	xdsC.InvokeWatchEDSCallback("", parseEDSRespProtoForTesting(clab4.Build()), nil)
   152  
   153  	sc3 := <-cc.NewSubConnCh
   154  	edsb.UpdateSubConnState(sc3, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
   155  	edsb.UpdateSubConnState(sc3, balancer.SubConnState{ConnectivityState: connectivity.Ready})
   156  	scToRemove = <-cc.RemoveSubConnCh
   157  	if !cmp.Equal(scToRemove, sc2, cmp.AllowUnexported(testutils.TestSubConn{})) {
   158  		t.Fatalf("RemoveSubConn, want %v, got %v", sc2, scToRemove)
   159  	}
   160  	edsb.UpdateSubConnState(scToRemove, balancer.SubConnState{ConnectivityState: connectivity.Shutdown})
   161  
   162  	// Test pick with only the third subconn.
   163  	if err := testRoundRobinPickerFromCh(cc.NewPickerCh, []balancer.SubConn{sc3}); err != nil {
   164  		t.Fatal(err)
   165  	}
   166  
   167  	// The same locality, different drop rate, dropping 50%.
   168  	clab5 := xdstestutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], map[string]uint32{"test-drop": 50})
   169  	clab5.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[2:3], nil)
   170  	xdsC.InvokeWatchEDSCallback("", parseEDSRespProtoForTesting(clab5.Build()), nil)
   171  
   172  	// Picks with drops.
   173  	if err := testPickerFromCh(cc.NewPickerCh, func(p balancer.Picker) error {
   174  		for i := 0; i < 100; i++ {
   175  			_, err := p.Pick(balancer.PickInfo{})
   176  			// TODO: the dropping algorithm needs a design. When the dropping algorithm
   177  			// is fixed, this test also needs fix.
   178  			if i%2 == 0 && err == nil {
   179  				return fmt.Errorf("%d - the even number picks should be drops, got error <nil>", i)
   180  			} else if i%2 != 0 && err != nil {
   181  				return fmt.Errorf("%d - the odd number picks should be non-drops, got error %v", i, err)
   182  			}
   183  		}
   184  		return nil
   185  	}); err != nil {
   186  		t.Fatal(err)
   187  	}
   188  
   189  	// The same locality, remove drops.
   190  	clab6 := xdstestutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
   191  	clab6.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[2:3], nil)
   192  	xdsC.InvokeWatchEDSCallback("", parseEDSRespProtoForTesting(clab6.Build()), nil)
   193  
   194  	// Pick without drops.
   195  	if err := testRoundRobinPickerFromCh(cc.NewPickerCh, []balancer.SubConn{sc3}); err != nil {
   196  		t.Fatal(err)
   197  	}
   198  }
   199  
   200  // 2 locality
   201  //  - start with 2 locality
   202  //  - add locality
   203  //  - remove locality
   204  //  - address change for the <not-the-first> locality
   205  //  - update locality weight
   206  func (s) TestEDS_TwoLocalities(t *testing.T) {
   207  	edsb, cc, xdsC, cleanup := setupTestEDS(t, nil)
   208  	defer cleanup()
   209  
   210  	// Two localities, each with one backend.
   211  	clab1 := xdstestutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
   212  	clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil)
   213  	xdsC.InvokeWatchEDSCallback("", parseEDSRespProtoForTesting(clab1.Build()), nil)
   214  	sc1 := <-cc.NewSubConnCh
   215  	edsb.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
   216  	edsb.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Ready})
   217  
   218  	// Add the second locality later to make sure sc2 belongs to the second
   219  	// locality. Otherwise the test is flaky because of a map is used in EDS to
   220  	// keep localities.
   221  	clab1.AddLocality(testSubZones[1], 1, 0, testEndpointAddrs[1:2], nil)
   222  	xdsC.InvokeWatchEDSCallback("", parseEDSRespProtoForTesting(clab1.Build()), nil)
   223  	sc2 := <-cc.NewSubConnCh
   224  	edsb.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
   225  	edsb.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Ready})
   226  
   227  	// Test roundrobin with two subconns.
   228  	if err := testRoundRobinPickerFromCh(cc.NewPickerCh, []balancer.SubConn{sc1, sc2}); err != nil {
   229  		t.Fatal(err)
   230  	}
   231  
   232  	// Add another locality, with one backend.
   233  	clab2 := xdstestutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
   234  	clab2.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil)
   235  	clab2.AddLocality(testSubZones[1], 1, 0, testEndpointAddrs[1:2], nil)
   236  	clab2.AddLocality(testSubZones[2], 1, 0, testEndpointAddrs[2:3], nil)
   237  	xdsC.InvokeWatchEDSCallback("", parseEDSRespProtoForTesting(clab2.Build()), nil)
   238  
   239  	sc3 := <-cc.NewSubConnCh
   240  	edsb.UpdateSubConnState(sc3, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
   241  	edsb.UpdateSubConnState(sc3, balancer.SubConnState{ConnectivityState: connectivity.Ready})
   242  
   243  	// Test roundrobin with three subconns.
   244  	if err := testRoundRobinPickerFromCh(cc.NewPickerCh, []balancer.SubConn{sc1, sc2, sc3}); err != nil {
   245  		t.Fatal(err)
   246  	}
   247  
   248  	// Remove first locality.
   249  	clab3 := xdstestutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
   250  	clab3.AddLocality(testSubZones[1], 1, 0, testEndpointAddrs[1:2], nil)
   251  	clab3.AddLocality(testSubZones[2], 1, 0, testEndpointAddrs[2:3], nil)
   252  	xdsC.InvokeWatchEDSCallback("", parseEDSRespProtoForTesting(clab3.Build()), nil)
   253  
   254  	scToRemove := <-cc.RemoveSubConnCh
   255  	if !cmp.Equal(scToRemove, sc1, cmp.AllowUnexported(testutils.TestSubConn{})) {
   256  		t.Fatalf("RemoveSubConn, want %v, got %v", sc1, scToRemove)
   257  	}
   258  	edsb.UpdateSubConnState(scToRemove, balancer.SubConnState{ConnectivityState: connectivity.Shutdown})
   259  
   260  	// Test pick with two subconns (without the first one).
   261  	if err := testRoundRobinPickerFromCh(cc.NewPickerCh, []balancer.SubConn{sc2, sc3}); err != nil {
   262  		t.Fatal(err)
   263  	}
   264  
   265  	// Add a backend to the last locality.
   266  	clab4 := xdstestutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
   267  	clab4.AddLocality(testSubZones[1], 1, 0, testEndpointAddrs[1:2], nil)
   268  	clab4.AddLocality(testSubZones[2], 1, 0, testEndpointAddrs[2:4], nil)
   269  	xdsC.InvokeWatchEDSCallback("", parseEDSRespProtoForTesting(clab4.Build()), nil)
   270  
   271  	sc4 := <-cc.NewSubConnCh
   272  	edsb.UpdateSubConnState(sc4, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
   273  	edsb.UpdateSubConnState(sc4, balancer.SubConnState{ConnectivityState: connectivity.Ready})
   274  
   275  	// Test pick with two subconns (without the first one).
   276  	//
   277  	// Locality-1 will be picked twice, and locality-2 will be picked twice.
   278  	// Locality-1 contains only sc2, locality-2 contains sc3 and sc4. So expect
   279  	// two sc2's and sc3, sc4.
   280  	if err := testRoundRobinPickerFromCh(cc.NewPickerCh, []balancer.SubConn{sc2, sc2, sc3, sc4}); err != nil {
   281  		t.Fatal(err)
   282  	}
   283  
   284  	// Change weight of the locality[1].
   285  	clab5 := xdstestutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
   286  	clab5.AddLocality(testSubZones[1], 2, 0, testEndpointAddrs[1:2], nil)
   287  	clab5.AddLocality(testSubZones[2], 1, 0, testEndpointAddrs[2:4], nil)
   288  	xdsC.InvokeWatchEDSCallback("", parseEDSRespProtoForTesting(clab5.Build()), nil)
   289  
   290  	// Test pick with two subconns different locality weight.
   291  	//
   292  	// Locality-1 will be picked four times, and locality-2 will be picked twice
   293  	// (weight 2 and 1). Locality-1 contains only sc2, locality-2 contains sc3 and
   294  	// sc4. So expect four sc2's and sc3, sc4.
   295  	if err := testRoundRobinPickerFromCh(cc.NewPickerCh, []balancer.SubConn{sc2, sc2, sc2, sc2, sc3, sc4}); err != nil {
   296  		t.Fatal(err)
   297  	}
   298  
   299  	// Change weight of the locality[1] to 0, it should never be picked.
   300  	clab6 := xdstestutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
   301  	clab6.AddLocality(testSubZones[1], 0, 0, testEndpointAddrs[1:2], nil)
   302  	clab6.AddLocality(testSubZones[2], 1, 0, testEndpointAddrs[2:4], nil)
   303  	xdsC.InvokeWatchEDSCallback("", parseEDSRespProtoForTesting(clab6.Build()), nil)
   304  
   305  	// Changing weight of locality[1] to 0 caused it to be removed. It's subconn
   306  	// should also be removed.
   307  	//
   308  	// NOTE: this is because we handle locality with weight 0 same as the
   309  	// locality doesn't exist. If this changes in the future, this removeSubConn
   310  	// behavior will also change.
   311  	scToRemove2 := <-cc.RemoveSubConnCh
   312  	if !cmp.Equal(scToRemove2, sc2, cmp.AllowUnexported(testutils.TestSubConn{})) {
   313  		t.Fatalf("RemoveSubConn, want %v, got %v", sc2, scToRemove2)
   314  	}
   315  
   316  	// Test pick with two subconns different locality weight.
   317  	//
   318  	// Locality-1 will be not be picked, and locality-2 will be picked.
   319  	// Locality-2 contains sc3 and sc4. So expect sc3, sc4.
   320  	if err := testRoundRobinPickerFromCh(cc.NewPickerCh, []balancer.SubConn{sc3, sc4}); err != nil {
   321  		t.Fatal(err)
   322  	}
   323  }
   324  
   325  // The EDS balancer gets EDS resp with unhealthy endpoints. Test that only
   326  // healthy ones are used.
   327  func (s) TestEDS_EndpointsHealth(t *testing.T) {
   328  	edsb, cc, xdsC, cleanup := setupTestEDS(t, nil)
   329  	defer cleanup()
   330  
   331  	// Two localities, each 3 backend, one Healthy, one Unhealthy, one Unknown.
   332  	clab1 := xdstestutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
   333  	clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:6], &xdstestutils.AddLocalityOptions{
   334  		Health: []corepb.HealthStatus{
   335  			corepb.HealthStatus_HEALTHY,
   336  			corepb.HealthStatus_UNHEALTHY,
   337  			corepb.HealthStatus_UNKNOWN,
   338  			corepb.HealthStatus_DRAINING,
   339  			corepb.HealthStatus_TIMEOUT,
   340  			corepb.HealthStatus_DEGRADED,
   341  		},
   342  	})
   343  	clab1.AddLocality(testSubZones[1], 1, 0, testEndpointAddrs[6:12], &xdstestutils.AddLocalityOptions{
   344  		Health: []corepb.HealthStatus{
   345  			corepb.HealthStatus_HEALTHY,
   346  			corepb.HealthStatus_UNHEALTHY,
   347  			corepb.HealthStatus_UNKNOWN,
   348  			corepb.HealthStatus_DRAINING,
   349  			corepb.HealthStatus_TIMEOUT,
   350  			corepb.HealthStatus_DEGRADED,
   351  		},
   352  	})
   353  	xdsC.InvokeWatchEDSCallback("", parseEDSRespProtoForTesting(clab1.Build()), nil)
   354  
   355  	var (
   356  		readySCs           []balancer.SubConn
   357  		newSubConnAddrStrs []string
   358  	)
   359  	for i := 0; i < 4; i++ {
   360  		addr := <-cc.NewSubConnAddrsCh
   361  		newSubConnAddrStrs = append(newSubConnAddrStrs, addr[0].Addr)
   362  		sc := <-cc.NewSubConnCh
   363  		edsb.UpdateSubConnState(sc, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
   364  		edsb.UpdateSubConnState(sc, balancer.SubConnState{ConnectivityState: connectivity.Ready})
   365  		readySCs = append(readySCs, sc)
   366  	}
   367  
   368  	wantNewSubConnAddrStrs := []string{
   369  		testEndpointAddrs[0],
   370  		testEndpointAddrs[2],
   371  		testEndpointAddrs[6],
   372  		testEndpointAddrs[8],
   373  	}
   374  	sortStrTrans := cmp.Transformer("Sort", func(in []string) []string {
   375  		out := append([]string(nil), in...) // Copy input to avoid mutating it.
   376  		sort.Strings(out)
   377  		return out
   378  	})
   379  	if !cmp.Equal(newSubConnAddrStrs, wantNewSubConnAddrStrs, sortStrTrans) {
   380  		t.Fatalf("want newSubConn with address %v, got %v", wantNewSubConnAddrStrs, newSubConnAddrStrs)
   381  	}
   382  
   383  	// There should be exactly 4 new SubConns. Check to make sure there's no
   384  	// more subconns being created.
   385  	select {
   386  	case <-cc.NewSubConnCh:
   387  		t.Fatalf("Got unexpected new subconn")
   388  	case <-time.After(time.Microsecond * 100):
   389  	}
   390  
   391  	// Test roundrobin with the subconns.
   392  	if err := testRoundRobinPickerFromCh(cc.NewPickerCh, readySCs); err != nil {
   393  		t.Fatal(err)
   394  	}
   395  }
   396  
   397  // TestEDS_EmptyUpdate covers the cases when eds impl receives an empty update.
   398  //
   399  // It should send an error picker with transient failure to the parent.
   400  func (s) TestEDS_EmptyUpdate(t *testing.T) {
   401  	edsb, cc, xdsC, cleanup := setupTestEDS(t, nil)
   402  	defer cleanup()
   403  
   404  	const cacheTimeout = 100 * time.Microsecond
   405  	oldCacheTimeout := balancergroup.DefaultSubBalancerCloseTimeout
   406  	balancergroup.DefaultSubBalancerCloseTimeout = cacheTimeout
   407  	defer func() { balancergroup.DefaultSubBalancerCloseTimeout = oldCacheTimeout }()
   408  
   409  	// The first update is an empty update.
   410  	xdsC.InvokeWatchEDSCallback("", xdsresource.EndpointsUpdate{}, nil)
   411  	// Pick should fail with transient failure, and all priority removed error.
   412  	if err := testErrPickerFromCh(cc.NewPickerCh, priority.ErrAllPrioritiesRemoved); err != nil {
   413  		t.Fatal(err)
   414  	}
   415  
   416  	// One locality with one backend.
   417  	clab1 := xdstestutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
   418  	clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil)
   419  	xdsC.InvokeWatchEDSCallback("", parseEDSRespProtoForTesting(clab1.Build()), nil)
   420  
   421  	sc1 := <-cc.NewSubConnCh
   422  	edsb.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
   423  	edsb.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Ready})
   424  
   425  	// Pick with only the first backend.
   426  	if err := testRoundRobinPickerFromCh(cc.NewPickerCh, []balancer.SubConn{sc1}); err != nil {
   427  		t.Fatal(err)
   428  	}
   429  
   430  	xdsC.InvokeWatchEDSCallback("", xdsresource.EndpointsUpdate{}, nil)
   431  	// Pick should fail with transient failure, and all priority removed error.
   432  	if err := testErrPickerFromCh(cc.NewPickerCh, priority.ErrAllPrioritiesRemoved); err != nil {
   433  		t.Fatal(err)
   434  	}
   435  
   436  	// Wait for the old SubConn to be removed (which happens when the child
   437  	// policy is closed), so a new update would trigger a new SubConn (we need
   438  	// this new SubConn to tell if the next picker is newly created).
   439  	scToRemove := <-cc.RemoveSubConnCh
   440  	if !cmp.Equal(scToRemove, sc1, cmp.AllowUnexported(testutils.TestSubConn{})) {
   441  		t.Fatalf("RemoveSubConn, want %v, got %v", sc1, scToRemove)
   442  	}
   443  	edsb.UpdateSubConnState(scToRemove, balancer.SubConnState{ConnectivityState: connectivity.Shutdown})
   444  
   445  	// Handle another update with priorities and localities.
   446  	xdsC.InvokeWatchEDSCallback("", parseEDSRespProtoForTesting(clab1.Build()), nil)
   447  
   448  	sc2 := <-cc.NewSubConnCh
   449  	edsb.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
   450  	edsb.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Ready})
   451  
   452  	// Pick with only the first backend.
   453  	if err := testRoundRobinPickerFromCh(cc.NewPickerCh, []balancer.SubConn{sc2}); err != nil {
   454  		t.Fatal(err)
   455  	}
   456  }
   457  
   458  func (s) TestEDS_CircuitBreaking(t *testing.T) {
   459  	edsb, cc, xdsC, cleanup := setupTestEDS(t, nil)
   460  	defer cleanup()
   461  
   462  	var maxRequests uint32 = 50
   463  	if err := edsb.UpdateClientConnState(balancer.ClientConnState{
   464  		BalancerConfig: &LBConfig{
   465  			DiscoveryMechanisms: []DiscoveryMechanism{{
   466  				Cluster:               testClusterName,
   467  				MaxConcurrentRequests: &maxRequests,
   468  				Type:                  DiscoveryMechanismTypeEDS,
   469  			}},
   470  		},
   471  	}); err != nil {
   472  		t.Fatal(err)
   473  	}
   474  
   475  	// One locality with one backend.
   476  	clab1 := xdstestutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil)
   477  	clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil)
   478  	xdsC.InvokeWatchEDSCallback("", parseEDSRespProtoForTesting(clab1.Build()), nil)
   479  	sc1 := <-cc.NewSubConnCh
   480  	edsb.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Connecting})
   481  	edsb.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Ready})
   482  
   483  	// Picks with drops.
   484  	dones := []func(){}
   485  	p := <-cc.NewPickerCh
   486  	for i := 0; i < 100; i++ {
   487  		pr, err := p.Pick(balancer.PickInfo{})
   488  		if i < 50 && err != nil {
   489  			t.Errorf("The first 50%% picks should be non-drops, got error %v", err)
   490  		} else if i > 50 && err == nil {
   491  			t.Errorf("The second 50%% picks should be drops, got error <nil>")
   492  		}
   493  		dones = append(dones, func() {
   494  			if pr.Done != nil {
   495  				pr.Done(balancer.DoneInfo{})
   496  			}
   497  		})
   498  	}
   499  
   500  	for _, done := range dones {
   501  		done()
   502  	}
   503  	dones = []func(){}
   504  
   505  	// Pick without drops.
   506  	for i := 0; i < 50; i++ {
   507  		pr, err := p.Pick(balancer.PickInfo{})
   508  		if err != nil {
   509  			t.Errorf("The third 50%% picks should be non-drops, got error %v", err)
   510  		}
   511  		dones = append(dones, func() {
   512  			if pr.Done != nil {
   513  				pr.Done(balancer.DoneInfo{})
   514  			}
   515  		})
   516  	}
   517  
   518  	// Without this, future tests with the same service name will fail.
   519  	for _, done := range dones {
   520  		done()
   521  	}
   522  
   523  	// Send another update, with only circuit breaking update (and no picker
   524  	// update afterwards). Make sure the new picker uses the new configs.
   525  	var maxRequests2 uint32 = 10
   526  	if err := edsb.UpdateClientConnState(balancer.ClientConnState{
   527  		BalancerConfig: &LBConfig{
   528  			DiscoveryMechanisms: []DiscoveryMechanism{{
   529  				Cluster:               testClusterName,
   530  				MaxConcurrentRequests: &maxRequests2,
   531  				Type:                  DiscoveryMechanismTypeEDS,
   532  			}},
   533  		},
   534  	}); err != nil {
   535  		t.Fatal(err)
   536  	}
   537  
   538  	// Picks with drops.
   539  	dones = []func(){}
   540  	p2 := <-cc.NewPickerCh
   541  	for i := 0; i < 100; i++ {
   542  		pr, err := p2.Pick(balancer.PickInfo{})
   543  		if i < 10 && err != nil {
   544  			t.Errorf("The first 10%% picks should be non-drops, got error %v", err)
   545  		} else if i > 10 && err == nil {
   546  			t.Errorf("The next 90%% picks should be drops, got error <nil>")
   547  		}
   548  		dones = append(dones, func() {
   549  			if pr.Done != nil {
   550  				pr.Done(balancer.DoneInfo{})
   551  			}
   552  		})
   553  	}
   554  
   555  	for _, done := range dones {
   556  		done()
   557  	}
   558  	dones = []func(){}
   559  
   560  	// Pick without drops.
   561  	for i := 0; i < 10; i++ {
   562  		pr, err := p2.Pick(balancer.PickInfo{})
   563  		if err != nil {
   564  			t.Errorf("The next 10%% picks should be non-drops, got error %v", err)
   565  		}
   566  		dones = append(dones, func() {
   567  			if pr.Done != nil {
   568  				pr.Done(balancer.DoneInfo{})
   569  			}
   570  		})
   571  	}
   572  
   573  	// Without this, future tests with the same service name will fail.
   574  	for _, done := range dones {
   575  		done()
   576  	}
   577  }