go.etcd.io/etcd@v3.3.27+incompatible/clientv3/balancer/balancer_test.go (about) 1 // Copyright 2018 The etcd Authors 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package balancer 16 17 import ( 18 "context" 19 "fmt" 20 "strings" 21 "testing" 22 "time" 23 24 "github.com/coreos/etcd/clientv3/balancer/picker" 25 "github.com/coreos/etcd/clientv3/balancer/resolver/endpoint" 26 pb "github.com/coreos/etcd/etcdserver/etcdserverpb" 27 "github.com/coreos/etcd/pkg/mock/mockserver" 28 29 "go.uber.org/zap" 30 "google.golang.org/grpc" 31 "google.golang.org/grpc/codes" 32 "google.golang.org/grpc/peer" 33 "google.golang.org/grpc/status" 34 ) 35 36 // TestRoundRobinBalancedResolvableNoFailover ensures that 37 // requests to a resolvable endpoint can be balanced between 38 // multiple, if any, nodes. And there needs be no failover. 39 func TestRoundRobinBalancedResolvableNoFailover(t *testing.T) { 40 testCases := []struct { 41 name string 42 serverCount int 43 reqN int 44 network string 45 }{ 46 {name: "rrBalanced_1", serverCount: 1, reqN: 5, network: "tcp"}, 47 {name: "rrBalanced_1_unix_sockets", serverCount: 1, reqN: 5, network: "unix"}, 48 {name: "rrBalanced_3", serverCount: 3, reqN: 7, network: "tcp"}, 49 {name: "rrBalanced_5", serverCount: 5, reqN: 10, network: "tcp"}, 50 } 51 52 for _, tc := range testCases { 53 t.Run(tc.name, func(t *testing.T) { 54 ms, err := mockserver.StartMockServersOnNetwork(tc.serverCount, tc.network) 55 if err != nil { 56 t.Fatalf("failed to start mock servers: %v", err) 57 } 58 defer ms.Stop() 59 60 var eps []string 61 for _, svr := range ms.Servers { 62 eps = append(eps, svr.ResolverAddress().Addr) 63 } 64 65 rsv, err := endpoint.NewResolverGroup("nofailover") 66 if err != nil { 67 t.Fatal(err) 68 } 69 defer rsv.Close() 70 rsv.SetEndpoints(eps) 71 72 name := genName() 73 cfg := Config{ 74 Policy: picker.RoundrobinBalanced, 75 Name: name, 76 Logger: zap.NewExample(), 77 } 78 RegisterBuilder(cfg) 79 conn, err := grpc.Dial(fmt.Sprintf("endpoint://nofailover/*"), grpc.WithInsecure(), grpc.WithBalancerName(name)) 80 if err != nil { 81 t.Fatalf("failed to dial mock server: %v", err) 82 } 83 defer conn.Close() 84 cli := pb.NewKVClient(conn) 85 86 reqFunc := func(ctx context.Context) (picked string, err error) { 87 var p peer.Peer 88 _, err = cli.Range(ctx, &pb.RangeRequest{Key: []byte("/x")}, grpc.Peer(&p)) 89 if p.Addr != nil { 90 picked = p.Addr.String() 91 } 92 return picked, err 93 } 94 95 prev, switches := "", 0 96 for i := 0; i < tc.reqN; i++ { 97 picked, err := reqFunc(context.Background()) 98 if err != nil { 99 t.Fatalf("#%d: unexpected failure %v", i, err) 100 } 101 if prev == "" { 102 prev = picked 103 continue 104 } 105 if prev != picked { 106 switches++ 107 } 108 prev = picked 109 } 110 if tc.serverCount > 1 && switches < tc.reqN-3 { // -3 for initial resolutions 111 // TODO: FIX ME 112 t.Skipf("expected balanced loads for %d requests, got switches %d", tc.reqN, switches) 113 } 114 }) 115 } 116 } 117 118 // TestRoundRobinBalancedResolvableFailoverFromServerFail ensures that 119 // loads be rebalanced while one server goes down and comes back. 120 func TestRoundRobinBalancedResolvableFailoverFromServerFail(t *testing.T) { 121 serverCount := 5 122 ms, err := mockserver.StartMockServers(serverCount) 123 if err != nil { 124 t.Fatalf("failed to start mock servers: %s", err) 125 } 126 defer ms.Stop() 127 var eps []string 128 for _, svr := range ms.Servers { 129 eps = append(eps, svr.ResolverAddress().Addr) 130 } 131 132 rsv, err := endpoint.NewResolverGroup("serverfail") 133 if err != nil { 134 t.Fatal(err) 135 } 136 defer rsv.Close() 137 rsv.SetEndpoints(eps) 138 139 name := genName() 140 cfg := Config{ 141 Policy: picker.RoundrobinBalanced, 142 Name: name, 143 Logger: zap.NewExample(), 144 } 145 RegisterBuilder(cfg) 146 conn, err := grpc.Dial(fmt.Sprintf("endpoint://serverfail/mock.server"), grpc.WithInsecure(), grpc.WithBalancerName(name)) 147 if err != nil { 148 t.Fatalf("failed to dial mock server: %s", err) 149 } 150 defer conn.Close() 151 cli := pb.NewKVClient(conn) 152 153 reqFunc := func(ctx context.Context) (picked string, err error) { 154 var p peer.Peer 155 _, err = cli.Range(ctx, &pb.RangeRequest{Key: []byte("/x")}, grpc.Peer(&p)) 156 if p.Addr != nil { 157 picked = p.Addr.String() 158 } 159 return picked, err 160 } 161 162 // stop first server, loads should be redistributed 163 // stopped server should never be picked 164 ms.StopAt(0) 165 available := make(map[string]struct{}) 166 for i := 1; i < serverCount; i++ { 167 available[eps[i]] = struct{}{} 168 } 169 170 reqN := 10 171 prev, switches := "", 0 172 for i := 0; i < reqN; i++ { 173 picked, err := reqFunc(context.Background()) 174 if err != nil && strings.Contains(err.Error(), "transport is closing") { 175 continue 176 } 177 if prev == "" { // first failover 178 if eps[0] == picked { 179 t.Fatalf("expected failover from %q, picked %q", eps[0], picked) 180 } 181 prev = picked 182 continue 183 } 184 if _, ok := available[picked]; !ok { 185 t.Fatalf("picked unavailable address %q (available %v)", picked, available) 186 } 187 if prev != picked { 188 switches++ 189 } 190 prev = picked 191 } 192 if switches < reqN-3 { // -3 for initial resolutions + failover 193 // TODO: FIX ME! 194 t.Skipf("expected balanced loads for %d requests, got switches %d", reqN, switches) 195 } 196 197 // now failed server comes back 198 ms.StartAt(0) 199 200 // enough time for reconnecting to recovered server 201 time.Sleep(time.Second) 202 203 prev, switches = "", 0 204 recoveredAddr, recovered := eps[0], 0 205 available[recoveredAddr] = struct{}{} 206 207 for i := 0; i < 2*reqN; i++ { 208 picked, err := reqFunc(context.Background()) 209 if err != nil { 210 t.Fatalf("#%d: unexpected failure %v", i, err) 211 } 212 if prev == "" { 213 prev = picked 214 continue 215 } 216 if _, ok := available[picked]; !ok { 217 t.Fatalf("#%d: picked unavailable address %q (available %v)", i, picked, available) 218 } 219 if prev != picked { 220 switches++ 221 } 222 if picked == recoveredAddr { 223 recovered++ 224 } 225 prev = picked 226 } 227 if switches < reqN-3 { // -3 for initial resolutions 228 t.Fatalf("expected balanced loads for %d requests, got switches %d", reqN, switches) 229 } 230 if recovered < reqN/serverCount { 231 t.Fatalf("recovered server %q got only %d requests", recoveredAddr, recovered) 232 } 233 } 234 235 // TestRoundRobinBalancedResolvableFailoverFromRequestFail ensures that 236 // loads be rebalanced while some requests are failed. 237 func TestRoundRobinBalancedResolvableFailoverFromRequestFail(t *testing.T) { 238 serverCount := 5 239 ms, err := mockserver.StartMockServers(serverCount) 240 if err != nil { 241 t.Fatalf("failed to start mock servers: %s", err) 242 } 243 defer ms.Stop() 244 var eps []string 245 available := make(map[string]struct{}) 246 for _, svr := range ms.Servers { 247 eps = append(eps, svr.ResolverAddress().Addr) 248 available[svr.Address] = struct{}{} 249 } 250 rsv, err := endpoint.NewResolverGroup("requestfail") 251 if err != nil { 252 t.Fatal(err) 253 } 254 defer rsv.Close() 255 rsv.SetEndpoints(eps) 256 257 name := genName() 258 cfg := Config{ 259 Policy: picker.RoundrobinBalanced, 260 Name: name, 261 Logger: zap.NewExample(), 262 } 263 RegisterBuilder(cfg) 264 conn, err := grpc.Dial(fmt.Sprintf("endpoint://requestfail/mock.server"), grpc.WithInsecure(), grpc.WithBalancerName(name)) 265 if err != nil { 266 t.Fatalf("failed to dial mock server: %s", err) 267 } 268 defer conn.Close() 269 cli := pb.NewKVClient(conn) 270 271 reqFunc := func(ctx context.Context) (picked string, err error) { 272 var p peer.Peer 273 _, err = cli.Range(ctx, &pb.RangeRequest{Key: []byte("/x")}, grpc.Peer(&p)) 274 if p.Addr != nil { 275 picked = p.Addr.String() 276 } 277 return picked, err 278 } 279 280 reqN := 20 281 prev, switches := "", 0 282 for i := 0; i < reqN; i++ { 283 ctx, cancel := context.WithCancel(context.Background()) 284 defer cancel() 285 if i%2 == 0 { 286 cancel() 287 } 288 picked, err := reqFunc(ctx) 289 if i%2 == 0 { 290 if s, ok := status.FromError(err); ok && s.Code() != codes.Canceled || picked != "" { 291 t.Fatalf("#%d: expected %v, got %v", i, context.Canceled, err) 292 } 293 continue 294 } 295 if prev == "" && picked != "" { 296 prev = picked 297 continue 298 } 299 if _, ok := available[picked]; !ok { 300 t.Fatalf("#%d: picked unavailable address %q (available %v)", i, picked, available) 301 } 302 if prev != picked { 303 switches++ 304 } 305 prev = picked 306 } 307 if switches < reqN/2-3 { // -3 for initial resolutions + failover 308 t.Fatalf("expected balanced loads for %d requests, got switches %d", reqN, switches) 309 } 310 }