go.etcd.io/etcd@v3.3.27+incompatible/clientv3/integration/lease_test.go (about)

     1  // Copyright 2016 The etcd Authors
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package integration
    16  
    17  import (
    18  	"context"
    19  	"reflect"
    20  	"sort"
    21  	"sync"
    22  	"testing"
    23  	"time"
    24  
    25  	"github.com/coreos/etcd/clientv3"
    26  	"github.com/coreos/etcd/clientv3/concurrency"
    27  	"github.com/coreos/etcd/etcdserver/api/v3rpc/rpctypes"
    28  	"github.com/coreos/etcd/integration"
    29  	"github.com/coreos/etcd/pkg/testutil"
    30  
    31  	"google.golang.org/grpc"
    32  )
    33  
    34  func TestLeaseNotFoundError(t *testing.T) {
    35  	defer testutil.AfterTest(t)
    36  
    37  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
    38  	defer clus.Terminate(t)
    39  
    40  	kv := clus.RandClient()
    41  
    42  	_, err := kv.Put(context.TODO(), "foo", "bar", clientv3.WithLease(clientv3.LeaseID(500)))
    43  	if err != rpctypes.ErrLeaseNotFound {
    44  		t.Fatalf("expected %v, got %v", rpctypes.ErrLeaseNotFound, err)
    45  	}
    46  }
    47  
    48  func TestLeaseGrant(t *testing.T) {
    49  	defer testutil.AfterTest(t)
    50  
    51  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3})
    52  	defer clus.Terminate(t)
    53  
    54  	lapi := clus.RandClient()
    55  
    56  	kv := clus.RandClient()
    57  
    58  	_, merr := lapi.Grant(context.Background(), clientv3.MaxLeaseTTL+1)
    59  	if merr != rpctypes.ErrLeaseTTLTooLarge {
    60  		t.Fatalf("err = %v, want %v", merr, rpctypes.ErrLeaseTTLTooLarge)
    61  	}
    62  
    63  	resp, err := lapi.Grant(context.Background(), 10)
    64  	if err != nil {
    65  		t.Errorf("failed to create lease %v", err)
    66  	}
    67  
    68  	_, err = kv.Put(context.TODO(), "foo", "bar", clientv3.WithLease(resp.ID))
    69  	if err != nil {
    70  		t.Fatalf("failed to create key with lease %v", err)
    71  	}
    72  }
    73  
    74  func TestLeaseRevoke(t *testing.T) {
    75  	defer testutil.AfterTest(t)
    76  
    77  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3})
    78  	defer clus.Terminate(t)
    79  
    80  	lapi := clus.RandClient()
    81  
    82  	kv := clus.RandClient()
    83  
    84  	resp, err := lapi.Grant(context.Background(), 10)
    85  	if err != nil {
    86  		t.Errorf("failed to create lease %v", err)
    87  	}
    88  
    89  	_, err = lapi.Revoke(context.Background(), clientv3.LeaseID(resp.ID))
    90  	if err != nil {
    91  		t.Errorf("failed to revoke lease %v", err)
    92  	}
    93  
    94  	_, err = kv.Put(context.TODO(), "foo", "bar", clientv3.WithLease(resp.ID))
    95  	if err != rpctypes.ErrLeaseNotFound {
    96  		t.Fatalf("err = %v, want %v", err, rpctypes.ErrLeaseNotFound)
    97  	}
    98  }
    99  
   100  func TestLeaseKeepAliveOnce(t *testing.T) {
   101  	defer testutil.AfterTest(t)
   102  
   103  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3})
   104  	defer clus.Terminate(t)
   105  
   106  	lapi := clus.RandClient()
   107  
   108  	resp, err := lapi.Grant(context.Background(), 10)
   109  	if err != nil {
   110  		t.Errorf("failed to create lease %v", err)
   111  	}
   112  
   113  	_, err = lapi.KeepAliveOnce(context.Background(), resp.ID)
   114  	if err != nil {
   115  		t.Errorf("failed to keepalive lease %v", err)
   116  	}
   117  
   118  	_, err = lapi.KeepAliveOnce(context.Background(), clientv3.LeaseID(0))
   119  	if err != rpctypes.ErrLeaseNotFound {
   120  		t.Errorf("expected %v, got %v", rpctypes.ErrLeaseNotFound, err)
   121  	}
   122  }
   123  
   124  func TestLeaseKeepAlive(t *testing.T) {
   125  	defer testutil.AfterTest(t)
   126  
   127  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3})
   128  	defer clus.Terminate(t)
   129  
   130  	lapi := clus.Client(0)
   131  	clus.TakeClient(0)
   132  
   133  	resp, err := lapi.Grant(context.Background(), 10)
   134  	if err != nil {
   135  		t.Errorf("failed to create lease %v", err)
   136  	}
   137  
   138  	rc, kerr := lapi.KeepAlive(context.Background(), resp.ID)
   139  	if kerr != nil {
   140  		t.Errorf("failed to keepalive lease %v", kerr)
   141  	}
   142  
   143  	kresp, ok := <-rc
   144  	if !ok {
   145  		t.Errorf("chan is closed, want not closed")
   146  	}
   147  
   148  	if kresp.ID != resp.ID {
   149  		t.Errorf("ID = %x, want %x", kresp.ID, resp.ID)
   150  	}
   151  
   152  	lapi.Close()
   153  
   154  	_, ok = <-rc
   155  	if ok {
   156  		t.Errorf("chan is not closed, want lease Close() closes chan")
   157  	}
   158  }
   159  
   160  func TestLeaseKeepAliveOneSecond(t *testing.T) {
   161  	defer testutil.AfterTest(t)
   162  
   163  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
   164  	defer clus.Terminate(t)
   165  
   166  	cli := clus.Client(0)
   167  
   168  	resp, err := cli.Grant(context.Background(), 1)
   169  	if err != nil {
   170  		t.Errorf("failed to create lease %v", err)
   171  	}
   172  	rc, kerr := cli.KeepAlive(context.Background(), resp.ID)
   173  	if kerr != nil {
   174  		t.Errorf("failed to keepalive lease %v", kerr)
   175  	}
   176  
   177  	for i := 0; i < 3; i++ {
   178  		if _, ok := <-rc; !ok {
   179  			t.Errorf("chan is closed, want not closed")
   180  		}
   181  	}
   182  }
   183  
   184  // TODO: add a client that can connect to all the members of cluster via unix sock.
   185  // TODO: test handle more complicated failures.
   186  func TestLeaseKeepAliveHandleFailure(t *testing.T) {
   187  	t.Skip("test it when we have a cluster client")
   188  
   189  	defer testutil.AfterTest(t)
   190  
   191  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3})
   192  	defer clus.Terminate(t)
   193  
   194  	// TODO: change this line to get a cluster client
   195  	lapi := clus.RandClient()
   196  
   197  	resp, err := lapi.Grant(context.Background(), 10)
   198  	if err != nil {
   199  		t.Errorf("failed to create lease %v", err)
   200  	}
   201  
   202  	rc, kerr := lapi.KeepAlive(context.Background(), resp.ID)
   203  	if kerr != nil {
   204  		t.Errorf("failed to keepalive lease %v", kerr)
   205  	}
   206  
   207  	kresp := <-rc
   208  	if kresp.ID != resp.ID {
   209  		t.Errorf("ID = %x, want %x", kresp.ID, resp.ID)
   210  	}
   211  
   212  	// restart the connected member.
   213  	clus.Members[0].Stop(t)
   214  
   215  	select {
   216  	case <-rc:
   217  		t.Fatalf("unexpected keepalive")
   218  	case <-time.After(10*time.Second/3 + 1):
   219  	}
   220  
   221  	// recover the member.
   222  	clus.Members[0].Restart(t)
   223  
   224  	kresp = <-rc
   225  	if kresp.ID != resp.ID {
   226  		t.Errorf("ID = %x, want %x", kresp.ID, resp.ID)
   227  	}
   228  
   229  	lapi.Close()
   230  
   231  	_, ok := <-rc
   232  	if ok {
   233  		t.Errorf("chan is not closed, want lease Close() closes chan")
   234  	}
   235  }
   236  
   237  type leaseCh struct {
   238  	lid clientv3.LeaseID
   239  	ch  <-chan *clientv3.LeaseKeepAliveResponse
   240  }
   241  
   242  // TestLeaseKeepAliveNotFound ensures a revoked lease won't halt other leases.
   243  func TestLeaseKeepAliveNotFound(t *testing.T) {
   244  	defer testutil.AfterTest(t)
   245  
   246  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
   247  	defer clus.Terminate(t)
   248  
   249  	cli := clus.RandClient()
   250  	lchs := []leaseCh{}
   251  	for i := 0; i < 3; i++ {
   252  		resp, rerr := cli.Grant(context.TODO(), 5)
   253  		if rerr != nil {
   254  			t.Fatal(rerr)
   255  		}
   256  		kach, kaerr := cli.KeepAlive(context.Background(), resp.ID)
   257  		if kaerr != nil {
   258  			t.Fatal(kaerr)
   259  		}
   260  		lchs = append(lchs, leaseCh{resp.ID, kach})
   261  	}
   262  
   263  	if _, err := cli.Revoke(context.TODO(), lchs[1].lid); err != nil {
   264  		t.Fatal(err)
   265  	}
   266  
   267  	<-lchs[0].ch
   268  	if _, ok := <-lchs[0].ch; !ok {
   269  		t.Fatalf("closed keepalive on wrong lease")
   270  	}
   271  
   272  	timec := time.After(5 * time.Second)
   273  	for range lchs[1].ch {
   274  		select {
   275  		case <-timec:
   276  			t.Fatalf("revoke did not close keep alive")
   277  		default:
   278  		}
   279  	}
   280  }
   281  
   282  func TestLeaseGrantErrConnClosed(t *testing.T) {
   283  	defer testutil.AfterTest(t)
   284  
   285  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
   286  	defer clus.Terminate(t)
   287  
   288  	cli := clus.Client(0)
   289  	clus.TakeClient(0)
   290  
   291  	donec := make(chan struct{})
   292  	go func() {
   293  		defer close(donec)
   294  		_, err := cli.Grant(context.TODO(), 5)
   295  		if err != nil && err != grpc.ErrClientConnClosing && err != context.Canceled {
   296  			// grpc.ErrClientConnClosing if grpc-go balancer calls 'Get' after client.Close.
   297  			// context.Canceled if grpc-go balancer calls 'Get' with an inflight client.Close.
   298  			t.Fatalf("expected %v or %v, got %v", grpc.ErrClientConnClosing, context.Canceled, err)
   299  		}
   300  	}()
   301  
   302  	if err := cli.Close(); err != nil {
   303  		t.Fatal(err)
   304  	}
   305  
   306  	select {
   307  	case <-time.After(integration.RequestWaitTimeout):
   308  		t.Fatal("le.Grant took too long")
   309  	case <-donec:
   310  	}
   311  }
   312  
   313  // TestLeaseKeepAliveFullResponseQueue ensures when response
   314  // queue is full thus dropping keepalive response sends,
   315  // keepalive request is sent with the same rate of TTL / 3.
   316  func TestLeaseKeepAliveFullResponseQueue(t *testing.T) {
   317  	defer testutil.AfterTest(t)
   318  
   319  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
   320  	defer clus.Terminate(t)
   321  
   322  	lapi := clus.Client(0)
   323  
   324  	// expect lease keepalive every 10-second
   325  	lresp, err := lapi.Grant(context.Background(), 30)
   326  	if err != nil {
   327  		t.Fatalf("failed to create lease %v", err)
   328  	}
   329  	id := lresp.ID
   330  
   331  	old := clientv3.LeaseResponseChSize
   332  	defer func() {
   333  		clientv3.LeaseResponseChSize = old
   334  	}()
   335  	clientv3.LeaseResponseChSize = 0
   336  
   337  	// never fetch from response queue, and let it become full
   338  	_, err = lapi.KeepAlive(context.Background(), id)
   339  	if err != nil {
   340  		t.Fatalf("failed to keepalive lease %v", err)
   341  	}
   342  
   343  	// TTL should not be refreshed after 3 seconds
   344  	// expect keepalive to be triggered after TTL/3
   345  	time.Sleep(3 * time.Second)
   346  
   347  	tr, terr := lapi.TimeToLive(context.Background(), id)
   348  	if terr != nil {
   349  		t.Fatalf("failed to get lease information %v", terr)
   350  	}
   351  	if tr.TTL >= 29 {
   352  		t.Errorf("unexpected kept-alive lease TTL %d", tr.TTL)
   353  	}
   354  }
   355  
   356  func TestLeaseGrantNewAfterClose(t *testing.T) {
   357  	defer testutil.AfterTest(t)
   358  
   359  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
   360  	defer clus.Terminate(t)
   361  
   362  	cli := clus.Client(0)
   363  	clus.TakeClient(0)
   364  	if err := cli.Close(); err != nil {
   365  		t.Fatal(err)
   366  	}
   367  
   368  	donec := make(chan struct{})
   369  	go func() {
   370  		if _, err := cli.Grant(context.TODO(), 5); err != context.Canceled && err != grpc.ErrClientConnClosing {
   371  			t.Fatalf("expected %v or %v, got %v", err != context.Canceled, grpc.ErrClientConnClosing, err)
   372  		}
   373  		close(donec)
   374  	}()
   375  	select {
   376  	case <-time.After(integration.RequestWaitTimeout):
   377  		t.Fatal("le.Grant took too long")
   378  	case <-donec:
   379  	}
   380  }
   381  
   382  func TestLeaseRevokeNewAfterClose(t *testing.T) {
   383  	defer testutil.AfterTest(t)
   384  
   385  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
   386  	defer clus.Terminate(t)
   387  
   388  	cli := clus.Client(0)
   389  	resp, err := cli.Grant(context.TODO(), 5)
   390  	if err != nil {
   391  		t.Fatal(err)
   392  	}
   393  	leaseID := resp.ID
   394  
   395  	clus.TakeClient(0)
   396  	if err := cli.Close(); err != nil {
   397  		t.Fatal(err)
   398  	}
   399  
   400  	donec := make(chan struct{})
   401  	go func() {
   402  		if _, err := cli.Revoke(context.TODO(), leaseID); err != context.Canceled && err != grpc.ErrClientConnClosing {
   403  			t.Fatalf("expected %v or %v, got %v", err != context.Canceled, grpc.ErrClientConnClosing, err)
   404  		}
   405  		close(donec)
   406  	}()
   407  	select {
   408  	case <-time.After(integration.RequestWaitTimeout):
   409  		t.Fatal("le.Revoke took too long")
   410  	case <-donec:
   411  	}
   412  }
   413  
   414  // TestLeaseKeepAliveCloseAfterDisconnectRevoke ensures the keep alive channel is closed
   415  // following a disconnection, lease revoke, then reconnect.
   416  func TestLeaseKeepAliveCloseAfterDisconnectRevoke(t *testing.T) {
   417  	defer testutil.AfterTest(t)
   418  
   419  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3})
   420  	defer clus.Terminate(t)
   421  
   422  	cli := clus.Client(0)
   423  
   424  	// setup lease and do a keepalive
   425  	resp, err := cli.Grant(context.Background(), 10)
   426  	if err != nil {
   427  		t.Fatal(err)
   428  	}
   429  	rc, kerr := cli.KeepAlive(context.Background(), resp.ID)
   430  	if kerr != nil {
   431  		t.Fatal(kerr)
   432  	}
   433  	kresp := <-rc
   434  	if kresp.ID != resp.ID {
   435  		t.Fatalf("ID = %x, want %x", kresp.ID, resp.ID)
   436  	}
   437  
   438  	// keep client disconnected
   439  	clus.Members[0].Stop(t)
   440  	time.Sleep(time.Second)
   441  	clus.WaitLeader(t)
   442  
   443  	if _, err := clus.Client(1).Revoke(context.TODO(), resp.ID); err != nil {
   444  		t.Fatal(err)
   445  	}
   446  
   447  	clus.Members[0].Restart(t)
   448  
   449  	// some responses may still be buffered; drain until close
   450  	timer := time.After(time.Duration(kresp.TTL) * time.Second)
   451  	for kresp != nil {
   452  		select {
   453  		case kresp = <-rc:
   454  		case <-timer:
   455  			t.Fatalf("keepalive channel did not close")
   456  		}
   457  	}
   458  }
   459  
   460  // TestLeaseKeepAliveInitTimeout ensures the keep alive channel closes if
   461  // the initial keep alive request never gets a response.
   462  func TestLeaseKeepAliveInitTimeout(t *testing.T) {
   463  	defer testutil.AfterTest(t)
   464  
   465  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
   466  	defer clus.Terminate(t)
   467  
   468  	cli := clus.Client(0)
   469  
   470  	// setup lease and do a keepalive
   471  	resp, err := cli.Grant(context.Background(), 5)
   472  	if err != nil {
   473  		t.Fatal(err)
   474  	}
   475  	// keep client disconnected
   476  	clus.Members[0].Stop(t)
   477  	rc, kerr := cli.KeepAlive(context.Background(), resp.ID)
   478  	if kerr != nil {
   479  		t.Fatal(kerr)
   480  	}
   481  	select {
   482  	case ka, ok := <-rc:
   483  		if ok {
   484  			t.Fatalf("unexpected keepalive %v, expected closed channel", ka)
   485  		}
   486  	case <-time.After(10 * time.Second):
   487  		t.Fatalf("keepalive channel did not close")
   488  	}
   489  
   490  	clus.Members[0].Restart(t)
   491  }
   492  
   493  // TestLeaseKeepAliveInitTimeout ensures the keep alive channel closes if
   494  // a keep alive request after the first never gets a response.
   495  func TestLeaseKeepAliveTTLTimeout(t *testing.T) {
   496  	defer testutil.AfterTest(t)
   497  
   498  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
   499  	defer clus.Terminate(t)
   500  
   501  	cli := clus.Client(0)
   502  
   503  	// setup lease and do a keepalive
   504  	resp, err := cli.Grant(context.Background(), 5)
   505  	if err != nil {
   506  		t.Fatal(err)
   507  	}
   508  	rc, kerr := cli.KeepAlive(context.Background(), resp.ID)
   509  	if kerr != nil {
   510  		t.Fatal(kerr)
   511  	}
   512  	if kresp := <-rc; kresp.ID != resp.ID {
   513  		t.Fatalf("ID = %x, want %x", kresp.ID, resp.ID)
   514  	}
   515  
   516  	// keep client disconnected
   517  	clus.Members[0].Stop(t)
   518  	select {
   519  	case ka, ok := <-rc:
   520  		if ok {
   521  			t.Fatalf("unexpected keepalive %v, expected closed channel", ka)
   522  		}
   523  	case <-time.After(10 * time.Second):
   524  		t.Fatalf("keepalive channel did not close")
   525  	}
   526  
   527  	clus.Members[0].Restart(t)
   528  }
   529  
   530  func TestLeaseTimeToLive(t *testing.T) {
   531  	defer testutil.AfterTest(t)
   532  
   533  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3})
   534  	defer clus.Terminate(t)
   535  
   536  	c := clus.RandClient()
   537  	lapi := c
   538  
   539  	resp, err := lapi.Grant(context.Background(), 10)
   540  	if err != nil {
   541  		t.Errorf("failed to create lease %v", err)
   542  	}
   543  
   544  	kv := clus.RandClient()
   545  	keys := []string{"foo1", "foo2"}
   546  	for i := range keys {
   547  		if _, err = kv.Put(context.TODO(), keys[i], "bar", clientv3.WithLease(resp.ID)); err != nil {
   548  			t.Fatal(err)
   549  		}
   550  	}
   551  
   552  	// linearized read to ensure Puts propagated to server backing lapi
   553  	if _, err := c.Get(context.TODO(), "abc"); err != nil {
   554  		t.Fatal(err)
   555  	}
   556  
   557  	lresp, lerr := lapi.TimeToLive(context.Background(), resp.ID, clientv3.WithAttachedKeys())
   558  	if lerr != nil {
   559  		t.Fatal(lerr)
   560  	}
   561  	if lresp.ID != resp.ID {
   562  		t.Fatalf("leaseID expected %d, got %d", resp.ID, lresp.ID)
   563  	}
   564  	if lresp.GrantedTTL != int64(10) {
   565  		t.Fatalf("GrantedTTL expected %d, got %d", 10, lresp.GrantedTTL)
   566  	}
   567  	if lresp.TTL == 0 || lresp.TTL > lresp.GrantedTTL {
   568  		t.Fatalf("unexpected TTL %d (granted %d)", lresp.TTL, lresp.GrantedTTL)
   569  	}
   570  	ks := make([]string, len(lresp.Keys))
   571  	for i := range lresp.Keys {
   572  		ks[i] = string(lresp.Keys[i])
   573  	}
   574  	sort.Strings(ks)
   575  	if !reflect.DeepEqual(ks, keys) {
   576  		t.Fatalf("keys expected %v, got %v", keys, ks)
   577  	}
   578  
   579  	lresp, lerr = lapi.TimeToLive(context.Background(), resp.ID)
   580  	if lerr != nil {
   581  		t.Fatal(lerr)
   582  	}
   583  	if len(lresp.Keys) != 0 {
   584  		t.Fatalf("unexpected keys %+v", lresp.Keys)
   585  	}
   586  }
   587  
   588  func TestLeaseTimeToLiveLeaseNotFound(t *testing.T) {
   589  	defer testutil.AfterTest(t)
   590  
   591  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
   592  	defer clus.Terminate(t)
   593  
   594  	cli := clus.RandClient()
   595  	resp, err := cli.Grant(context.Background(), 10)
   596  	if err != nil {
   597  		t.Errorf("failed to create lease %v", err)
   598  	}
   599  	_, err = cli.Revoke(context.Background(), resp.ID)
   600  	if err != nil {
   601  		t.Errorf("failed to Revoke lease %v", err)
   602  	}
   603  
   604  	lresp, err := cli.TimeToLive(context.Background(), resp.ID)
   605  	// TimeToLive() should return a response with TTL=-1.
   606  	if err != nil {
   607  		t.Fatalf("expected err to be nil")
   608  	}
   609  	if lresp == nil {
   610  		t.Fatalf("expected lresp not to be nil")
   611  	}
   612  	if lresp.ResponseHeader == nil {
   613  		t.Fatalf("expected ResponseHeader not to be nil")
   614  	}
   615  	if lresp.ID != resp.ID {
   616  		t.Fatalf("expected Lease ID %v, but got %v", resp.ID, lresp.ID)
   617  	}
   618  	if lresp.TTL != -1 {
   619  		t.Fatalf("expected TTL %v, but got %v", lresp.TTL, lresp.TTL)
   620  	}
   621  }
   622  
   623  func TestLeaseLeases(t *testing.T) {
   624  	defer testutil.AfterTest(t)
   625  
   626  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
   627  	defer clus.Terminate(t)
   628  
   629  	cli := clus.RandClient()
   630  
   631  	ids := []clientv3.LeaseID{}
   632  	for i := 0; i < 5; i++ {
   633  		resp, err := cli.Grant(context.Background(), 10)
   634  		if err != nil {
   635  			t.Errorf("failed to create lease %v", err)
   636  		}
   637  		ids = append(ids, resp.ID)
   638  	}
   639  
   640  	resp, err := cli.Leases(context.Background())
   641  	if err != nil {
   642  		t.Fatal(err)
   643  	}
   644  	if len(resp.Leases) != 5 {
   645  		t.Fatalf("len(resp.Leases) expected 5, got %d", len(resp.Leases))
   646  	}
   647  	for i := range resp.Leases {
   648  		if ids[i] != resp.Leases[i].ID {
   649  			t.Fatalf("#%d: lease ID expected %d, got %d", i, ids[i], resp.Leases[i].ID)
   650  		}
   651  	}
   652  }
   653  
   654  // TestLeaseRenewLostQuorum ensures keepalives work after losing quorum
   655  // for a while.
   656  func TestLeaseRenewLostQuorum(t *testing.T) {
   657  	defer testutil.AfterTest(t)
   658  
   659  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3})
   660  	defer clus.Terminate(t)
   661  
   662  	cli := clus.Client(0)
   663  	r, err := cli.Grant(context.TODO(), 4)
   664  	if err != nil {
   665  		t.Fatal(err)
   666  	}
   667  
   668  	kctx, kcancel := context.WithCancel(context.Background())
   669  	defer kcancel()
   670  	ka, err := cli.KeepAlive(kctx, r.ID)
   671  	if err != nil {
   672  		t.Fatal(err)
   673  	}
   674  	// consume first keepalive so next message sends when cluster is down
   675  	<-ka
   676  	lastKa := time.Now()
   677  
   678  	// force keepalive stream message to timeout
   679  	clus.Members[1].Stop(t)
   680  	clus.Members[2].Stop(t)
   681  	// Use TTL-2 since the client closes the keepalive channel if no
   682  	// keepalive arrives before the lease deadline; the client will
   683  	// try to resend a keepalive after TTL/3 seconds, so for a TTL of 4,
   684  	// sleeping for 2s should be sufficient time for issuing a retry.
   685  	// The cluster has two seconds to recover and reply to the keepalive.
   686  	time.Sleep(time.Duration(r.TTL-2) * time.Second)
   687  	clus.Members[1].Restart(t)
   688  	clus.Members[2].Restart(t)
   689  
   690  	if time.Since(lastKa) > time.Duration(r.TTL)*time.Second {
   691  		t.Skip("waited too long for server stop and restart")
   692  	}
   693  
   694  	select {
   695  	case _, ok := <-ka:
   696  		if !ok {
   697  			t.Fatalf("keepalive closed")
   698  		}
   699  	case <-time.After(time.Duration(r.TTL) * time.Second):
   700  		t.Fatalf("timed out waiting for keepalive")
   701  	}
   702  }
   703  
   704  func TestLeaseKeepAliveLoopExit(t *testing.T) {
   705  	defer testutil.AfterTest(t)
   706  
   707  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
   708  	defer clus.Terminate(t)
   709  
   710  	ctx := context.Background()
   711  	cli := clus.Client(0)
   712  	clus.TakeClient(0)
   713  
   714  	resp, err := cli.Grant(ctx, 5)
   715  	if err != nil {
   716  		t.Fatal(err)
   717  	}
   718  	cli.Close()
   719  
   720  	_, err = cli.KeepAlive(ctx, resp.ID)
   721  	if _, ok := err.(clientv3.ErrKeepAliveHalted); !ok {
   722  		t.Fatalf("expected %T, got %v(%T)", clientv3.ErrKeepAliveHalted{}, err, err)
   723  	}
   724  }
   725  
   726  // TestV3LeaseFailureOverlap issues Grant and KeepAlive requests to a cluster
   727  // before, during, and after quorum loss to confirm Grant/KeepAlive tolerates
   728  // transient cluster failure.
   729  func TestV3LeaseFailureOverlap(t *testing.T) {
   730  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 2})
   731  	defer clus.Terminate(t)
   732  
   733  	numReqs := 5
   734  	cli := clus.Client(0)
   735  
   736  	// bring up a session, tear it down
   737  	updown := func(i int) error {
   738  		sess, err := concurrency.NewSession(cli)
   739  		if err != nil {
   740  			return err
   741  		}
   742  		ch := make(chan struct{})
   743  		go func() {
   744  			defer close(ch)
   745  			sess.Close()
   746  		}()
   747  		select {
   748  		case <-ch:
   749  		case <-time.After(time.Minute / 4):
   750  			t.Fatalf("timeout %d", i)
   751  		}
   752  		return nil
   753  	}
   754  
   755  	var wg sync.WaitGroup
   756  	mkReqs := func(n int) {
   757  		wg.Add(numReqs)
   758  		for i := 0; i < numReqs; i++ {
   759  			go func() {
   760  				defer wg.Done()
   761  				err := updown(n)
   762  				if err == nil || err == rpctypes.ErrTimeoutDueToConnectionLost {
   763  					return
   764  				}
   765  				t.Fatal(err)
   766  			}()
   767  		}
   768  	}
   769  
   770  	mkReqs(1)
   771  	clus.Members[1].Stop(t)
   772  	mkReqs(2)
   773  	time.Sleep(time.Second)
   774  	mkReqs(3)
   775  	clus.Members[1].Restart(t)
   776  	mkReqs(4)
   777  	wg.Wait()
   778  }
   779  
   780  // TestLeaseWithRequireLeader checks keep-alive channel close when no leader.
   781  func TestLeaseWithRequireLeader(t *testing.T) {
   782  	defer testutil.AfterTest(t)
   783  
   784  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 2})
   785  	defer clus.Terminate(t)
   786  
   787  	c := clus.Client(0)
   788  	lid1, err1 := c.Grant(context.TODO(), 60)
   789  	if err1 != nil {
   790  		t.Fatal(err1)
   791  	}
   792  	lid2, err2 := c.Grant(context.TODO(), 60)
   793  	if err2 != nil {
   794  		t.Fatal(err2)
   795  	}
   796  	// kaReqLeader close if the leader is lost
   797  	kaReqLeader, kerr1 := c.KeepAlive(clientv3.WithRequireLeader(context.TODO()), lid1.ID)
   798  	if kerr1 != nil {
   799  		t.Fatal(kerr1)
   800  	}
   801  	// kaWait will wait even if the leader is lost
   802  	kaWait, kerr2 := c.KeepAlive(context.TODO(), lid2.ID)
   803  	if kerr2 != nil {
   804  		t.Fatal(kerr2)
   805  	}
   806  
   807  	select {
   808  	case <-kaReqLeader:
   809  	case <-time.After(5 * time.Second):
   810  		t.Fatalf("require leader first keep-alive timed out")
   811  	}
   812  	select {
   813  	case <-kaWait:
   814  	case <-time.After(5 * time.Second):
   815  		t.Fatalf("leader not required first keep-alive timed out")
   816  	}
   817  
   818  	clus.Members[1].Stop(t)
   819  	// kaReqLeader may issue multiple requests while waiting for the first
   820  	// response from proxy server; drain any stray keepalive responses
   821  	time.Sleep(100 * time.Millisecond)
   822  	for len(kaReqLeader) > 0 {
   823  		<-kaReqLeader
   824  	}
   825  
   826  	select {
   827  	case resp, ok := <-kaReqLeader:
   828  		if ok {
   829  			t.Fatalf("expected closed require leader, got response %+v", resp)
   830  		}
   831  	case <-time.After(5 * time.Second):
   832  		t.Fatal("keepalive with require leader took too long to close")
   833  	}
   834  	select {
   835  	case _, ok := <-kaWait:
   836  		if !ok {
   837  			t.Fatalf("got closed channel with no require leader, expected non-closed")
   838  		}
   839  	case <-time.After(10 * time.Millisecond):
   840  		// wait some to detect any closes happening soon after kaReqLeader closing
   841  	}
   842  }