go.etcd.io/etcd@v3.3.27+incompatible/clientv3/balancer/balancer_test.go (about)

     1  // Copyright 2018 The etcd Authors
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package balancer
    16  
    17  import (
    18  	"context"
    19  	"fmt"
    20  	"strings"
    21  	"testing"
    22  	"time"
    23  
    24  	"github.com/coreos/etcd/clientv3/balancer/picker"
    25  	"github.com/coreos/etcd/clientv3/balancer/resolver/endpoint"
    26  	pb "github.com/coreos/etcd/etcdserver/etcdserverpb"
    27  	"github.com/coreos/etcd/pkg/mock/mockserver"
    28  
    29  	"go.uber.org/zap"
    30  	"google.golang.org/grpc"
    31  	"google.golang.org/grpc/codes"
    32  	"google.golang.org/grpc/peer"
    33  	"google.golang.org/grpc/status"
    34  )
    35  
    36  // TestRoundRobinBalancedResolvableNoFailover ensures that
    37  // requests to a resolvable endpoint can be balanced between
    38  // multiple, if any, nodes. And there needs be no failover.
    39  func TestRoundRobinBalancedResolvableNoFailover(t *testing.T) {
    40  	testCases := []struct {
    41  		name        string
    42  		serverCount int
    43  		reqN        int
    44  		network     string
    45  	}{
    46  		{name: "rrBalanced_1", serverCount: 1, reqN: 5, network: "tcp"},
    47  		{name: "rrBalanced_1_unix_sockets", serverCount: 1, reqN: 5, network: "unix"},
    48  		{name: "rrBalanced_3", serverCount: 3, reqN: 7, network: "tcp"},
    49  		{name: "rrBalanced_5", serverCount: 5, reqN: 10, network: "tcp"},
    50  	}
    51  
    52  	for _, tc := range testCases {
    53  		t.Run(tc.name, func(t *testing.T) {
    54  			ms, err := mockserver.StartMockServersOnNetwork(tc.serverCount, tc.network)
    55  			if err != nil {
    56  				t.Fatalf("failed to start mock servers: %v", err)
    57  			}
    58  			defer ms.Stop()
    59  
    60  			var eps []string
    61  			for _, svr := range ms.Servers {
    62  				eps = append(eps, svr.ResolverAddress().Addr)
    63  			}
    64  
    65  			rsv, err := endpoint.NewResolverGroup("nofailover")
    66  			if err != nil {
    67  				t.Fatal(err)
    68  			}
    69  			defer rsv.Close()
    70  			rsv.SetEndpoints(eps)
    71  
    72  			name := genName()
    73  			cfg := Config{
    74  				Policy: picker.RoundrobinBalanced,
    75  				Name:   name,
    76  				Logger: zap.NewExample(),
    77  			}
    78  			RegisterBuilder(cfg)
    79  			conn, err := grpc.Dial(fmt.Sprintf("endpoint://nofailover/*"), grpc.WithInsecure(), grpc.WithBalancerName(name))
    80  			if err != nil {
    81  				t.Fatalf("failed to dial mock server: %v", err)
    82  			}
    83  			defer conn.Close()
    84  			cli := pb.NewKVClient(conn)
    85  
    86  			reqFunc := func(ctx context.Context) (picked string, err error) {
    87  				var p peer.Peer
    88  				_, err = cli.Range(ctx, &pb.RangeRequest{Key: []byte("/x")}, grpc.Peer(&p))
    89  				if p.Addr != nil {
    90  					picked = p.Addr.String()
    91  				}
    92  				return picked, err
    93  			}
    94  
    95  			prev, switches := "", 0
    96  			for i := 0; i < tc.reqN; i++ {
    97  				picked, err := reqFunc(context.Background())
    98  				if err != nil {
    99  					t.Fatalf("#%d: unexpected failure %v", i, err)
   100  				}
   101  				if prev == "" {
   102  					prev = picked
   103  					continue
   104  				}
   105  				if prev != picked {
   106  					switches++
   107  				}
   108  				prev = picked
   109  			}
   110  			if tc.serverCount > 1 && switches < tc.reqN-3 { // -3 for initial resolutions
   111  				// TODO: FIX ME
   112  				t.Skipf("expected balanced loads for %d requests, got switches %d", tc.reqN, switches)
   113  			}
   114  		})
   115  	}
   116  }
   117  
   118  // TestRoundRobinBalancedResolvableFailoverFromServerFail ensures that
   119  // loads be rebalanced while one server goes down and comes back.
   120  func TestRoundRobinBalancedResolvableFailoverFromServerFail(t *testing.T) {
   121  	serverCount := 5
   122  	ms, err := mockserver.StartMockServers(serverCount)
   123  	if err != nil {
   124  		t.Fatalf("failed to start mock servers: %s", err)
   125  	}
   126  	defer ms.Stop()
   127  	var eps []string
   128  	for _, svr := range ms.Servers {
   129  		eps = append(eps, svr.ResolverAddress().Addr)
   130  	}
   131  
   132  	rsv, err := endpoint.NewResolverGroup("serverfail")
   133  	if err != nil {
   134  		t.Fatal(err)
   135  	}
   136  	defer rsv.Close()
   137  	rsv.SetEndpoints(eps)
   138  
   139  	name := genName()
   140  	cfg := Config{
   141  		Policy: picker.RoundrobinBalanced,
   142  		Name:   name,
   143  		Logger: zap.NewExample(),
   144  	}
   145  	RegisterBuilder(cfg)
   146  	conn, err := grpc.Dial(fmt.Sprintf("endpoint://serverfail/mock.server"), grpc.WithInsecure(), grpc.WithBalancerName(name))
   147  	if err != nil {
   148  		t.Fatalf("failed to dial mock server: %s", err)
   149  	}
   150  	defer conn.Close()
   151  	cli := pb.NewKVClient(conn)
   152  
   153  	reqFunc := func(ctx context.Context) (picked string, err error) {
   154  		var p peer.Peer
   155  		_, err = cli.Range(ctx, &pb.RangeRequest{Key: []byte("/x")}, grpc.Peer(&p))
   156  		if p.Addr != nil {
   157  			picked = p.Addr.String()
   158  		}
   159  		return picked, err
   160  	}
   161  
   162  	// stop first server, loads should be redistributed
   163  	// stopped server should never be picked
   164  	ms.StopAt(0)
   165  	available := make(map[string]struct{})
   166  	for i := 1; i < serverCount; i++ {
   167  		available[eps[i]] = struct{}{}
   168  	}
   169  
   170  	reqN := 10
   171  	prev, switches := "", 0
   172  	for i := 0; i < reqN; i++ {
   173  		picked, err := reqFunc(context.Background())
   174  		if err != nil && strings.Contains(err.Error(), "transport is closing") {
   175  			continue
   176  		}
   177  		if prev == "" { // first failover
   178  			if eps[0] == picked {
   179  				t.Fatalf("expected failover from %q, picked %q", eps[0], picked)
   180  			}
   181  			prev = picked
   182  			continue
   183  		}
   184  		if _, ok := available[picked]; !ok {
   185  			t.Fatalf("picked unavailable address %q (available %v)", picked, available)
   186  		}
   187  		if prev != picked {
   188  			switches++
   189  		}
   190  		prev = picked
   191  	}
   192  	if switches < reqN-3 { // -3 for initial resolutions + failover
   193  		// TODO: FIX ME!
   194  		t.Skipf("expected balanced loads for %d requests, got switches %d", reqN, switches)
   195  	}
   196  
   197  	// now failed server comes back
   198  	ms.StartAt(0)
   199  
   200  	// enough time for reconnecting to recovered server
   201  	time.Sleep(time.Second)
   202  
   203  	prev, switches = "", 0
   204  	recoveredAddr, recovered := eps[0], 0
   205  	available[recoveredAddr] = struct{}{}
   206  
   207  	for i := 0; i < 2*reqN; i++ {
   208  		picked, err := reqFunc(context.Background())
   209  		if err != nil {
   210  			t.Fatalf("#%d: unexpected failure %v", i, err)
   211  		}
   212  		if prev == "" {
   213  			prev = picked
   214  			continue
   215  		}
   216  		if _, ok := available[picked]; !ok {
   217  			t.Fatalf("#%d: picked unavailable address %q (available %v)", i, picked, available)
   218  		}
   219  		if prev != picked {
   220  			switches++
   221  		}
   222  		if picked == recoveredAddr {
   223  			recovered++
   224  		}
   225  		prev = picked
   226  	}
   227  	if switches < reqN-3 { // -3 for initial resolutions
   228  		t.Fatalf("expected balanced loads for %d requests, got switches %d", reqN, switches)
   229  	}
   230  	if recovered < reqN/serverCount {
   231  		t.Fatalf("recovered server %q got only %d requests", recoveredAddr, recovered)
   232  	}
   233  }
   234  
   235  // TestRoundRobinBalancedResolvableFailoverFromRequestFail ensures that
   236  // loads be rebalanced while some requests are failed.
   237  func TestRoundRobinBalancedResolvableFailoverFromRequestFail(t *testing.T) {
   238  	serverCount := 5
   239  	ms, err := mockserver.StartMockServers(serverCount)
   240  	if err != nil {
   241  		t.Fatalf("failed to start mock servers: %s", err)
   242  	}
   243  	defer ms.Stop()
   244  	var eps []string
   245  	available := make(map[string]struct{})
   246  	for _, svr := range ms.Servers {
   247  		eps = append(eps, svr.ResolverAddress().Addr)
   248  		available[svr.Address] = struct{}{}
   249  	}
   250  	rsv, err := endpoint.NewResolverGroup("requestfail")
   251  	if err != nil {
   252  		t.Fatal(err)
   253  	}
   254  	defer rsv.Close()
   255  	rsv.SetEndpoints(eps)
   256  
   257  	name := genName()
   258  	cfg := Config{
   259  		Policy: picker.RoundrobinBalanced,
   260  		Name:   name,
   261  		Logger: zap.NewExample(),
   262  	}
   263  	RegisterBuilder(cfg)
   264  	conn, err := grpc.Dial(fmt.Sprintf("endpoint://requestfail/mock.server"), grpc.WithInsecure(), grpc.WithBalancerName(name))
   265  	if err != nil {
   266  		t.Fatalf("failed to dial mock server: %s", err)
   267  	}
   268  	defer conn.Close()
   269  	cli := pb.NewKVClient(conn)
   270  
   271  	reqFunc := func(ctx context.Context) (picked string, err error) {
   272  		var p peer.Peer
   273  		_, err = cli.Range(ctx, &pb.RangeRequest{Key: []byte("/x")}, grpc.Peer(&p))
   274  		if p.Addr != nil {
   275  			picked = p.Addr.String()
   276  		}
   277  		return picked, err
   278  	}
   279  
   280  	reqN := 20
   281  	prev, switches := "", 0
   282  	for i := 0; i < reqN; i++ {
   283  		ctx, cancel := context.WithCancel(context.Background())
   284  		defer cancel()
   285  		if i%2 == 0 {
   286  			cancel()
   287  		}
   288  		picked, err := reqFunc(ctx)
   289  		if i%2 == 0 {
   290  			if s, ok := status.FromError(err); ok && s.Code() != codes.Canceled || picked != "" {
   291  				t.Fatalf("#%d: expected %v, got %v", i, context.Canceled, err)
   292  			}
   293  			continue
   294  		}
   295  		if prev == "" && picked != "" {
   296  			prev = picked
   297  			continue
   298  		}
   299  		if _, ok := available[picked]; !ok {
   300  			t.Fatalf("#%d: picked unavailable address %q (available %v)", i, picked, available)
   301  		}
   302  		if prev != picked {
   303  			switches++
   304  		}
   305  		prev = picked
   306  	}
   307  	if switches < reqN/2-3 { // -3 for initial resolutions + failover
   308  		t.Fatalf("expected balanced loads for %d requests, got switches %d", reqN, switches)
   309  	}
   310  }