github.com/hxx258456/ccgo@v0.0.5-0.20230213014102-48b35f46f66f/grpc/xds/internal/balancer/clusterresolver/eds_impl_test.go (about) 1 /* 2 * Copyright 2019 gRPC authors. 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package clusterresolver 18 19 import ( 20 "context" 21 "fmt" 22 "sort" 23 "testing" 24 "time" 25 26 "github.com/google/go-cmp/cmp" 27 corepb "github.com/hxx258456/ccgo/go-control-plane/envoy/api/v2/core" 28 "github.com/hxx258456/ccgo/grpc/balancer" 29 "github.com/hxx258456/ccgo/grpc/balancer/weightedtarget" 30 "github.com/hxx258456/ccgo/grpc/connectivity" 31 "github.com/hxx258456/ccgo/grpc/internal/balancergroup" 32 internalserviceconfig "github.com/hxx258456/ccgo/grpc/internal/serviceconfig" 33 "github.com/hxx258456/ccgo/grpc/internal/testutils" 34 "github.com/hxx258456/ccgo/grpc/resolver" 35 "github.com/hxx258456/ccgo/grpc/xds/internal/balancer/clusterimpl" 36 "github.com/hxx258456/ccgo/grpc/xds/internal/balancer/priority" 37 xdstestutils "github.com/hxx258456/ccgo/grpc/xds/internal/testutils" 38 "github.com/hxx258456/ccgo/grpc/xds/internal/testutils/fakeclient" 39 "github.com/hxx258456/ccgo/grpc/xds/internal/xdsclient" 40 "github.com/hxx258456/ccgo/grpc/xds/internal/xdsclient/xdsresource" 41 ) 42 43 var ( 44 testClusterNames = []string{"test-cluster-1", "test-cluster-2"} 45 testSubZones = []string{"I", "II", "III", "IV"} 46 testEndpointAddrs []string 47 ) 48 49 const testBackendAddrsCount = 12 50 51 func init() { 52 for i := 0; i < testBackendAddrsCount; i++ { 53 testEndpointAddrs = append(testEndpointAddrs, fmt.Sprintf("%d.%d.%d.%d:%d", i, i, i, i, i)) 54 } 55 balancergroup.DefaultSubBalancerCloseTimeout = time.Millisecond 56 clusterimpl.NewRandomWRR = testutils.NewTestWRR 57 weightedtarget.NewRandomWRR = testutils.NewTestWRR 58 balancergroup.DefaultSubBalancerCloseTimeout = time.Millisecond * 100 59 } 60 61 func setupTestEDS(t *testing.T, initChild *internalserviceconfig.BalancerConfig) (balancer.Balancer, *testutils.TestClientConn, *fakeclient.Client, func()) { 62 xdsC := fakeclient.NewClientWithName(testBalancerNameFooBar) 63 cc := testutils.NewTestClientConn(t) 64 builder := balancer.Get(Name) 65 edsb := builder.Build(cc, balancer.BuildOptions{Target: resolver.Target{Endpoint: testEDSServcie}}) 66 if edsb == nil { 67 t.Fatalf("builder.Build(%s) failed and returned nil", Name) 68 } 69 ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) 70 defer cancel() 71 if err := edsb.UpdateClientConnState(balancer.ClientConnState{ 72 ResolverState: xdsclient.SetClient(resolver.State{}, xdsC), 73 BalancerConfig: &LBConfig{ 74 DiscoveryMechanisms: []DiscoveryMechanism{{ 75 Cluster: testClusterName, 76 Type: DiscoveryMechanismTypeEDS, 77 }}, 78 }, 79 }); err != nil { 80 edsb.Close() 81 xdsC.Close() 82 t.Fatal(err) 83 } 84 if _, err := xdsC.WaitForWatchEDS(ctx); err != nil { 85 edsb.Close() 86 xdsC.Close() 87 t.Fatalf("xdsClient.WatchEndpoints failed with error: %v", err) 88 } 89 return edsb, cc, xdsC, func() { 90 edsb.Close() 91 xdsC.Close() 92 } 93 } 94 95 // One locality 96 // - add backend 97 // - remove backend 98 // - replace backend 99 // - change drop rate 100 func (s) TestEDS_OneLocality(t *testing.T) { 101 edsb, cc, xdsC, cleanup := setupTestEDS(t, nil) 102 defer cleanup() 103 104 // One locality with one backend. 105 clab1 := xdstestutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) 106 clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil) 107 xdsC.InvokeWatchEDSCallback("", parseEDSRespProtoForTesting(clab1.Build()), nil) 108 109 sc1 := <-cc.NewSubConnCh 110 edsb.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) 111 edsb.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Ready}) 112 113 // Pick with only the first backend. 114 if err := testRoundRobinPickerFromCh(cc.NewPickerCh, []balancer.SubConn{sc1}); err != nil { 115 t.Fatal(err) 116 } 117 118 // The same locality, add one more backend. 119 clab2 := xdstestutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) 120 clab2.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:2], nil) 121 xdsC.InvokeWatchEDSCallback("", parseEDSRespProtoForTesting(clab2.Build()), nil) 122 123 sc2 := <-cc.NewSubConnCh 124 edsb.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) 125 edsb.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Ready}) 126 127 // Test roundrobin with two subconns. 128 if err := testRoundRobinPickerFromCh(cc.NewPickerCh, []balancer.SubConn{sc1, sc2}); err != nil { 129 t.Fatal(err) 130 } 131 132 // The same locality, delete first backend. 133 clab3 := xdstestutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) 134 clab3.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[1:2], nil) 135 xdsC.InvokeWatchEDSCallback("", parseEDSRespProtoForTesting(clab3.Build()), nil) 136 137 scToRemove := <-cc.RemoveSubConnCh 138 if !cmp.Equal(scToRemove, sc1, cmp.AllowUnexported(testutils.TestSubConn{})) { 139 t.Fatalf("RemoveSubConn, want %v, got %v", sc1, scToRemove) 140 } 141 edsb.UpdateSubConnState(scToRemove, balancer.SubConnState{ConnectivityState: connectivity.Shutdown}) 142 143 // Test pick with only the second subconn. 144 if err := testRoundRobinPickerFromCh(cc.NewPickerCh, []balancer.SubConn{sc2}); err != nil { 145 t.Fatal(err) 146 } 147 148 // The same locality, replace backend. 149 clab4 := xdstestutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) 150 clab4.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[2:3], nil) 151 xdsC.InvokeWatchEDSCallback("", parseEDSRespProtoForTesting(clab4.Build()), nil) 152 153 sc3 := <-cc.NewSubConnCh 154 edsb.UpdateSubConnState(sc3, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) 155 edsb.UpdateSubConnState(sc3, balancer.SubConnState{ConnectivityState: connectivity.Ready}) 156 scToRemove = <-cc.RemoveSubConnCh 157 if !cmp.Equal(scToRemove, sc2, cmp.AllowUnexported(testutils.TestSubConn{})) { 158 t.Fatalf("RemoveSubConn, want %v, got %v", sc2, scToRemove) 159 } 160 edsb.UpdateSubConnState(scToRemove, balancer.SubConnState{ConnectivityState: connectivity.Shutdown}) 161 162 // Test pick with only the third subconn. 163 if err := testRoundRobinPickerFromCh(cc.NewPickerCh, []balancer.SubConn{sc3}); err != nil { 164 t.Fatal(err) 165 } 166 167 // The same locality, different drop rate, dropping 50%. 168 clab5 := xdstestutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], map[string]uint32{"test-drop": 50}) 169 clab5.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[2:3], nil) 170 xdsC.InvokeWatchEDSCallback("", parseEDSRespProtoForTesting(clab5.Build()), nil) 171 172 // Picks with drops. 173 if err := testPickerFromCh(cc.NewPickerCh, func(p balancer.Picker) error { 174 for i := 0; i < 100; i++ { 175 _, err := p.Pick(balancer.PickInfo{}) 176 // TODO: the dropping algorithm needs a design. When the dropping algorithm 177 // is fixed, this test also needs fix. 178 if i%2 == 0 && err == nil { 179 return fmt.Errorf("%d - the even number picks should be drops, got error <nil>", i) 180 } else if i%2 != 0 && err != nil { 181 return fmt.Errorf("%d - the odd number picks should be non-drops, got error %v", i, err) 182 } 183 } 184 return nil 185 }); err != nil { 186 t.Fatal(err) 187 } 188 189 // The same locality, remove drops. 190 clab6 := xdstestutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) 191 clab6.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[2:3], nil) 192 xdsC.InvokeWatchEDSCallback("", parseEDSRespProtoForTesting(clab6.Build()), nil) 193 194 // Pick without drops. 195 if err := testRoundRobinPickerFromCh(cc.NewPickerCh, []balancer.SubConn{sc3}); err != nil { 196 t.Fatal(err) 197 } 198 } 199 200 // 2 locality 201 // - start with 2 locality 202 // - add locality 203 // - remove locality 204 // - address change for the <not-the-first> locality 205 // - update locality weight 206 func (s) TestEDS_TwoLocalities(t *testing.T) { 207 edsb, cc, xdsC, cleanup := setupTestEDS(t, nil) 208 defer cleanup() 209 210 // Two localities, each with one backend. 211 clab1 := xdstestutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) 212 clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil) 213 xdsC.InvokeWatchEDSCallback("", parseEDSRespProtoForTesting(clab1.Build()), nil) 214 sc1 := <-cc.NewSubConnCh 215 edsb.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) 216 edsb.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Ready}) 217 218 // Add the second locality later to make sure sc2 belongs to the second 219 // locality. Otherwise the test is flaky because of a map is used in EDS to 220 // keep localities. 221 clab1.AddLocality(testSubZones[1], 1, 0, testEndpointAddrs[1:2], nil) 222 xdsC.InvokeWatchEDSCallback("", parseEDSRespProtoForTesting(clab1.Build()), nil) 223 sc2 := <-cc.NewSubConnCh 224 edsb.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) 225 edsb.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Ready}) 226 227 // Test roundrobin with two subconns. 228 if err := testRoundRobinPickerFromCh(cc.NewPickerCh, []balancer.SubConn{sc1, sc2}); err != nil { 229 t.Fatal(err) 230 } 231 232 // Add another locality, with one backend. 233 clab2 := xdstestutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) 234 clab2.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil) 235 clab2.AddLocality(testSubZones[1], 1, 0, testEndpointAddrs[1:2], nil) 236 clab2.AddLocality(testSubZones[2], 1, 0, testEndpointAddrs[2:3], nil) 237 xdsC.InvokeWatchEDSCallback("", parseEDSRespProtoForTesting(clab2.Build()), nil) 238 239 sc3 := <-cc.NewSubConnCh 240 edsb.UpdateSubConnState(sc3, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) 241 edsb.UpdateSubConnState(sc3, balancer.SubConnState{ConnectivityState: connectivity.Ready}) 242 243 // Test roundrobin with three subconns. 244 if err := testRoundRobinPickerFromCh(cc.NewPickerCh, []balancer.SubConn{sc1, sc2, sc3}); err != nil { 245 t.Fatal(err) 246 } 247 248 // Remove first locality. 249 clab3 := xdstestutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) 250 clab3.AddLocality(testSubZones[1], 1, 0, testEndpointAddrs[1:2], nil) 251 clab3.AddLocality(testSubZones[2], 1, 0, testEndpointAddrs[2:3], nil) 252 xdsC.InvokeWatchEDSCallback("", parseEDSRespProtoForTesting(clab3.Build()), nil) 253 254 scToRemove := <-cc.RemoveSubConnCh 255 if !cmp.Equal(scToRemove, sc1, cmp.AllowUnexported(testutils.TestSubConn{})) { 256 t.Fatalf("RemoveSubConn, want %v, got %v", sc1, scToRemove) 257 } 258 edsb.UpdateSubConnState(scToRemove, balancer.SubConnState{ConnectivityState: connectivity.Shutdown}) 259 260 // Test pick with two subconns (without the first one). 261 if err := testRoundRobinPickerFromCh(cc.NewPickerCh, []balancer.SubConn{sc2, sc3}); err != nil { 262 t.Fatal(err) 263 } 264 265 // Add a backend to the last locality. 266 clab4 := xdstestutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) 267 clab4.AddLocality(testSubZones[1], 1, 0, testEndpointAddrs[1:2], nil) 268 clab4.AddLocality(testSubZones[2], 1, 0, testEndpointAddrs[2:4], nil) 269 xdsC.InvokeWatchEDSCallback("", parseEDSRespProtoForTesting(clab4.Build()), nil) 270 271 sc4 := <-cc.NewSubConnCh 272 edsb.UpdateSubConnState(sc4, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) 273 edsb.UpdateSubConnState(sc4, balancer.SubConnState{ConnectivityState: connectivity.Ready}) 274 275 // Test pick with two subconns (without the first one). 276 // 277 // Locality-1 will be picked twice, and locality-2 will be picked twice. 278 // Locality-1 contains only sc2, locality-2 contains sc3 and sc4. So expect 279 // two sc2's and sc3, sc4. 280 if err := testRoundRobinPickerFromCh(cc.NewPickerCh, []balancer.SubConn{sc2, sc2, sc3, sc4}); err != nil { 281 t.Fatal(err) 282 } 283 284 // Change weight of the locality[1]. 285 clab5 := xdstestutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) 286 clab5.AddLocality(testSubZones[1], 2, 0, testEndpointAddrs[1:2], nil) 287 clab5.AddLocality(testSubZones[2], 1, 0, testEndpointAddrs[2:4], nil) 288 xdsC.InvokeWatchEDSCallback("", parseEDSRespProtoForTesting(clab5.Build()), nil) 289 290 // Test pick with two subconns different locality weight. 291 // 292 // Locality-1 will be picked four times, and locality-2 will be picked twice 293 // (weight 2 and 1). Locality-1 contains only sc2, locality-2 contains sc3 and 294 // sc4. So expect four sc2's and sc3, sc4. 295 if err := testRoundRobinPickerFromCh(cc.NewPickerCh, []balancer.SubConn{sc2, sc2, sc2, sc2, sc3, sc4}); err != nil { 296 t.Fatal(err) 297 } 298 299 // Change weight of the locality[1] to 0, it should never be picked. 300 clab6 := xdstestutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) 301 clab6.AddLocality(testSubZones[1], 0, 0, testEndpointAddrs[1:2], nil) 302 clab6.AddLocality(testSubZones[2], 1, 0, testEndpointAddrs[2:4], nil) 303 xdsC.InvokeWatchEDSCallback("", parseEDSRespProtoForTesting(clab6.Build()), nil) 304 305 // Changing weight of locality[1] to 0 caused it to be removed. It's subconn 306 // should also be removed. 307 // 308 // NOTE: this is because we handle locality with weight 0 same as the 309 // locality doesn't exist. If this changes in the future, this removeSubConn 310 // behavior will also change. 311 scToRemove2 := <-cc.RemoveSubConnCh 312 if !cmp.Equal(scToRemove2, sc2, cmp.AllowUnexported(testutils.TestSubConn{})) { 313 t.Fatalf("RemoveSubConn, want %v, got %v", sc2, scToRemove2) 314 } 315 316 // Test pick with two subconns different locality weight. 317 // 318 // Locality-1 will be not be picked, and locality-2 will be picked. 319 // Locality-2 contains sc3 and sc4. So expect sc3, sc4. 320 if err := testRoundRobinPickerFromCh(cc.NewPickerCh, []balancer.SubConn{sc3, sc4}); err != nil { 321 t.Fatal(err) 322 } 323 } 324 325 // The EDS balancer gets EDS resp with unhealthy endpoints. Test that only 326 // healthy ones are used. 327 func (s) TestEDS_EndpointsHealth(t *testing.T) { 328 edsb, cc, xdsC, cleanup := setupTestEDS(t, nil) 329 defer cleanup() 330 331 // Two localities, each 3 backend, one Healthy, one Unhealthy, one Unknown. 332 clab1 := xdstestutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) 333 clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:6], &xdstestutils.AddLocalityOptions{ 334 Health: []corepb.HealthStatus{ 335 corepb.HealthStatus_HEALTHY, 336 corepb.HealthStatus_UNHEALTHY, 337 corepb.HealthStatus_UNKNOWN, 338 corepb.HealthStatus_DRAINING, 339 corepb.HealthStatus_TIMEOUT, 340 corepb.HealthStatus_DEGRADED, 341 }, 342 }) 343 clab1.AddLocality(testSubZones[1], 1, 0, testEndpointAddrs[6:12], &xdstestutils.AddLocalityOptions{ 344 Health: []corepb.HealthStatus{ 345 corepb.HealthStatus_HEALTHY, 346 corepb.HealthStatus_UNHEALTHY, 347 corepb.HealthStatus_UNKNOWN, 348 corepb.HealthStatus_DRAINING, 349 corepb.HealthStatus_TIMEOUT, 350 corepb.HealthStatus_DEGRADED, 351 }, 352 }) 353 xdsC.InvokeWatchEDSCallback("", parseEDSRespProtoForTesting(clab1.Build()), nil) 354 355 var ( 356 readySCs []balancer.SubConn 357 newSubConnAddrStrs []string 358 ) 359 for i := 0; i < 4; i++ { 360 addr := <-cc.NewSubConnAddrsCh 361 newSubConnAddrStrs = append(newSubConnAddrStrs, addr[0].Addr) 362 sc := <-cc.NewSubConnCh 363 edsb.UpdateSubConnState(sc, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) 364 edsb.UpdateSubConnState(sc, balancer.SubConnState{ConnectivityState: connectivity.Ready}) 365 readySCs = append(readySCs, sc) 366 } 367 368 wantNewSubConnAddrStrs := []string{ 369 testEndpointAddrs[0], 370 testEndpointAddrs[2], 371 testEndpointAddrs[6], 372 testEndpointAddrs[8], 373 } 374 sortStrTrans := cmp.Transformer("Sort", func(in []string) []string { 375 out := append([]string(nil), in...) // Copy input to avoid mutating it. 376 sort.Strings(out) 377 return out 378 }) 379 if !cmp.Equal(newSubConnAddrStrs, wantNewSubConnAddrStrs, sortStrTrans) { 380 t.Fatalf("want newSubConn with address %v, got %v", wantNewSubConnAddrStrs, newSubConnAddrStrs) 381 } 382 383 // There should be exactly 4 new SubConns. Check to make sure there's no 384 // more subconns being created. 385 select { 386 case <-cc.NewSubConnCh: 387 t.Fatalf("Got unexpected new subconn") 388 case <-time.After(time.Microsecond * 100): 389 } 390 391 // Test roundrobin with the subconns. 392 if err := testRoundRobinPickerFromCh(cc.NewPickerCh, readySCs); err != nil { 393 t.Fatal(err) 394 } 395 } 396 397 // TestEDS_EmptyUpdate covers the cases when eds impl receives an empty update. 398 // 399 // It should send an error picker with transient failure to the parent. 400 func (s) TestEDS_EmptyUpdate(t *testing.T) { 401 edsb, cc, xdsC, cleanup := setupTestEDS(t, nil) 402 defer cleanup() 403 404 const cacheTimeout = 100 * time.Microsecond 405 oldCacheTimeout := balancergroup.DefaultSubBalancerCloseTimeout 406 balancergroup.DefaultSubBalancerCloseTimeout = cacheTimeout 407 defer func() { balancergroup.DefaultSubBalancerCloseTimeout = oldCacheTimeout }() 408 409 // The first update is an empty update. 410 xdsC.InvokeWatchEDSCallback("", xdsresource.EndpointsUpdate{}, nil) 411 // Pick should fail with transient failure, and all priority removed error. 412 if err := testErrPickerFromCh(cc.NewPickerCh, priority.ErrAllPrioritiesRemoved); err != nil { 413 t.Fatal(err) 414 } 415 416 // One locality with one backend. 417 clab1 := xdstestutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) 418 clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil) 419 xdsC.InvokeWatchEDSCallback("", parseEDSRespProtoForTesting(clab1.Build()), nil) 420 421 sc1 := <-cc.NewSubConnCh 422 edsb.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) 423 edsb.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Ready}) 424 425 // Pick with only the first backend. 426 if err := testRoundRobinPickerFromCh(cc.NewPickerCh, []balancer.SubConn{sc1}); err != nil { 427 t.Fatal(err) 428 } 429 430 xdsC.InvokeWatchEDSCallback("", xdsresource.EndpointsUpdate{}, nil) 431 // Pick should fail with transient failure, and all priority removed error. 432 if err := testErrPickerFromCh(cc.NewPickerCh, priority.ErrAllPrioritiesRemoved); err != nil { 433 t.Fatal(err) 434 } 435 436 // Wait for the old SubConn to be removed (which happens when the child 437 // policy is closed), so a new update would trigger a new SubConn (we need 438 // this new SubConn to tell if the next picker is newly created). 439 scToRemove := <-cc.RemoveSubConnCh 440 if !cmp.Equal(scToRemove, sc1, cmp.AllowUnexported(testutils.TestSubConn{})) { 441 t.Fatalf("RemoveSubConn, want %v, got %v", sc1, scToRemove) 442 } 443 edsb.UpdateSubConnState(scToRemove, balancer.SubConnState{ConnectivityState: connectivity.Shutdown}) 444 445 // Handle another update with priorities and localities. 446 xdsC.InvokeWatchEDSCallback("", parseEDSRespProtoForTesting(clab1.Build()), nil) 447 448 sc2 := <-cc.NewSubConnCh 449 edsb.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) 450 edsb.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Ready}) 451 452 // Pick with only the first backend. 453 if err := testRoundRobinPickerFromCh(cc.NewPickerCh, []balancer.SubConn{sc2}); err != nil { 454 t.Fatal(err) 455 } 456 } 457 458 func (s) TestEDS_CircuitBreaking(t *testing.T) { 459 edsb, cc, xdsC, cleanup := setupTestEDS(t, nil) 460 defer cleanup() 461 462 var maxRequests uint32 = 50 463 if err := edsb.UpdateClientConnState(balancer.ClientConnState{ 464 BalancerConfig: &LBConfig{ 465 DiscoveryMechanisms: []DiscoveryMechanism{{ 466 Cluster: testClusterName, 467 MaxConcurrentRequests: &maxRequests, 468 Type: DiscoveryMechanismTypeEDS, 469 }}, 470 }, 471 }); err != nil { 472 t.Fatal(err) 473 } 474 475 // One locality with one backend. 476 clab1 := xdstestutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) 477 clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil) 478 xdsC.InvokeWatchEDSCallback("", parseEDSRespProtoForTesting(clab1.Build()), nil) 479 sc1 := <-cc.NewSubConnCh 480 edsb.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) 481 edsb.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Ready}) 482 483 // Picks with drops. 484 dones := []func(){} 485 p := <-cc.NewPickerCh 486 for i := 0; i < 100; i++ { 487 pr, err := p.Pick(balancer.PickInfo{}) 488 if i < 50 && err != nil { 489 t.Errorf("The first 50%% picks should be non-drops, got error %v", err) 490 } else if i > 50 && err == nil { 491 t.Errorf("The second 50%% picks should be drops, got error <nil>") 492 } 493 dones = append(dones, func() { 494 if pr.Done != nil { 495 pr.Done(balancer.DoneInfo{}) 496 } 497 }) 498 } 499 500 for _, done := range dones { 501 done() 502 } 503 dones = []func(){} 504 505 // Pick without drops. 506 for i := 0; i < 50; i++ { 507 pr, err := p.Pick(balancer.PickInfo{}) 508 if err != nil { 509 t.Errorf("The third 50%% picks should be non-drops, got error %v", err) 510 } 511 dones = append(dones, func() { 512 if pr.Done != nil { 513 pr.Done(balancer.DoneInfo{}) 514 } 515 }) 516 } 517 518 // Without this, future tests with the same service name will fail. 519 for _, done := range dones { 520 done() 521 } 522 523 // Send another update, with only circuit breaking update (and no picker 524 // update afterwards). Make sure the new picker uses the new configs. 525 var maxRequests2 uint32 = 10 526 if err := edsb.UpdateClientConnState(balancer.ClientConnState{ 527 BalancerConfig: &LBConfig{ 528 DiscoveryMechanisms: []DiscoveryMechanism{{ 529 Cluster: testClusterName, 530 MaxConcurrentRequests: &maxRequests2, 531 Type: DiscoveryMechanismTypeEDS, 532 }}, 533 }, 534 }); err != nil { 535 t.Fatal(err) 536 } 537 538 // Picks with drops. 539 dones = []func(){} 540 p2 := <-cc.NewPickerCh 541 for i := 0; i < 100; i++ { 542 pr, err := p2.Pick(balancer.PickInfo{}) 543 if i < 10 && err != nil { 544 t.Errorf("The first 10%% picks should be non-drops, got error %v", err) 545 } else if i > 10 && err == nil { 546 t.Errorf("The next 90%% picks should be drops, got error <nil>") 547 } 548 dones = append(dones, func() { 549 if pr.Done != nil { 550 pr.Done(balancer.DoneInfo{}) 551 } 552 }) 553 } 554 555 for _, done := range dones { 556 done() 557 } 558 dones = []func(){} 559 560 // Pick without drops. 561 for i := 0; i < 10; i++ { 562 pr, err := p2.Pick(balancer.PickInfo{}) 563 if err != nil { 564 t.Errorf("The next 10%% picks should be non-drops, got error %v", err) 565 } 566 dones = append(dones, func() { 567 if pr.Done != nil { 568 pr.Done(balancer.DoneInfo{}) 569 } 570 }) 571 } 572 573 // Without this, future tests with the same service name will fail. 574 for _, done := range dones { 575 done() 576 } 577 }