go.etcd.io/etcd@v3.3.27+incompatible/clientv3/integration/lease_test.go (about) 1 // Copyright 2016 The etcd Authors 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package integration 16 17 import ( 18 "context" 19 "reflect" 20 "sort" 21 "sync" 22 "testing" 23 "time" 24 25 "github.com/coreos/etcd/clientv3" 26 "github.com/coreos/etcd/clientv3/concurrency" 27 "github.com/coreos/etcd/etcdserver/api/v3rpc/rpctypes" 28 "github.com/coreos/etcd/integration" 29 "github.com/coreos/etcd/pkg/testutil" 30 31 "google.golang.org/grpc" 32 ) 33 34 func TestLeaseNotFoundError(t *testing.T) { 35 defer testutil.AfterTest(t) 36 37 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}) 38 defer clus.Terminate(t) 39 40 kv := clus.RandClient() 41 42 _, err := kv.Put(context.TODO(), "foo", "bar", clientv3.WithLease(clientv3.LeaseID(500))) 43 if err != rpctypes.ErrLeaseNotFound { 44 t.Fatalf("expected %v, got %v", rpctypes.ErrLeaseNotFound, err) 45 } 46 } 47 48 func TestLeaseGrant(t *testing.T) { 49 defer testutil.AfterTest(t) 50 51 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3}) 52 defer clus.Terminate(t) 53 54 lapi := clus.RandClient() 55 56 kv := clus.RandClient() 57 58 _, merr := lapi.Grant(context.Background(), clientv3.MaxLeaseTTL+1) 59 if merr != rpctypes.ErrLeaseTTLTooLarge { 60 t.Fatalf("err = %v, want %v", merr, rpctypes.ErrLeaseTTLTooLarge) 61 } 62 63 resp, err := lapi.Grant(context.Background(), 10) 64 if err != nil { 65 t.Errorf("failed to create lease %v", err) 66 } 67 68 _, err = kv.Put(context.TODO(), "foo", "bar", clientv3.WithLease(resp.ID)) 69 if err != nil { 70 t.Fatalf("failed to create key with lease %v", err) 71 } 72 } 73 74 func TestLeaseRevoke(t *testing.T) { 75 defer testutil.AfterTest(t) 76 77 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3}) 78 defer clus.Terminate(t) 79 80 lapi := clus.RandClient() 81 82 kv := clus.RandClient() 83 84 resp, err := lapi.Grant(context.Background(), 10) 85 if err != nil { 86 t.Errorf("failed to create lease %v", err) 87 } 88 89 _, err = lapi.Revoke(context.Background(), clientv3.LeaseID(resp.ID)) 90 if err != nil { 91 t.Errorf("failed to revoke lease %v", err) 92 } 93 94 _, err = kv.Put(context.TODO(), "foo", "bar", clientv3.WithLease(resp.ID)) 95 if err != rpctypes.ErrLeaseNotFound { 96 t.Fatalf("err = %v, want %v", err, rpctypes.ErrLeaseNotFound) 97 } 98 } 99 100 func TestLeaseKeepAliveOnce(t *testing.T) { 101 defer testutil.AfterTest(t) 102 103 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3}) 104 defer clus.Terminate(t) 105 106 lapi := clus.RandClient() 107 108 resp, err := lapi.Grant(context.Background(), 10) 109 if err != nil { 110 t.Errorf("failed to create lease %v", err) 111 } 112 113 _, err = lapi.KeepAliveOnce(context.Background(), resp.ID) 114 if err != nil { 115 t.Errorf("failed to keepalive lease %v", err) 116 } 117 118 _, err = lapi.KeepAliveOnce(context.Background(), clientv3.LeaseID(0)) 119 if err != rpctypes.ErrLeaseNotFound { 120 t.Errorf("expected %v, got %v", rpctypes.ErrLeaseNotFound, err) 121 } 122 } 123 124 func TestLeaseKeepAlive(t *testing.T) { 125 defer testutil.AfterTest(t) 126 127 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3}) 128 defer clus.Terminate(t) 129 130 lapi := clus.Client(0) 131 clus.TakeClient(0) 132 133 resp, err := lapi.Grant(context.Background(), 10) 134 if err != nil { 135 t.Errorf("failed to create lease %v", err) 136 } 137 138 rc, kerr := lapi.KeepAlive(context.Background(), resp.ID) 139 if kerr != nil { 140 t.Errorf("failed to keepalive lease %v", kerr) 141 } 142 143 kresp, ok := <-rc 144 if !ok { 145 t.Errorf("chan is closed, want not closed") 146 } 147 148 if kresp.ID != resp.ID { 149 t.Errorf("ID = %x, want %x", kresp.ID, resp.ID) 150 } 151 152 lapi.Close() 153 154 _, ok = <-rc 155 if ok { 156 t.Errorf("chan is not closed, want lease Close() closes chan") 157 } 158 } 159 160 func TestLeaseKeepAliveOneSecond(t *testing.T) { 161 defer testutil.AfterTest(t) 162 163 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}) 164 defer clus.Terminate(t) 165 166 cli := clus.Client(0) 167 168 resp, err := cli.Grant(context.Background(), 1) 169 if err != nil { 170 t.Errorf("failed to create lease %v", err) 171 } 172 rc, kerr := cli.KeepAlive(context.Background(), resp.ID) 173 if kerr != nil { 174 t.Errorf("failed to keepalive lease %v", kerr) 175 } 176 177 for i := 0; i < 3; i++ { 178 if _, ok := <-rc; !ok { 179 t.Errorf("chan is closed, want not closed") 180 } 181 } 182 } 183 184 // TODO: add a client that can connect to all the members of cluster via unix sock. 185 // TODO: test handle more complicated failures. 186 func TestLeaseKeepAliveHandleFailure(t *testing.T) { 187 t.Skip("test it when we have a cluster client") 188 189 defer testutil.AfterTest(t) 190 191 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3}) 192 defer clus.Terminate(t) 193 194 // TODO: change this line to get a cluster client 195 lapi := clus.RandClient() 196 197 resp, err := lapi.Grant(context.Background(), 10) 198 if err != nil { 199 t.Errorf("failed to create lease %v", err) 200 } 201 202 rc, kerr := lapi.KeepAlive(context.Background(), resp.ID) 203 if kerr != nil { 204 t.Errorf("failed to keepalive lease %v", kerr) 205 } 206 207 kresp := <-rc 208 if kresp.ID != resp.ID { 209 t.Errorf("ID = %x, want %x", kresp.ID, resp.ID) 210 } 211 212 // restart the connected member. 213 clus.Members[0].Stop(t) 214 215 select { 216 case <-rc: 217 t.Fatalf("unexpected keepalive") 218 case <-time.After(10*time.Second/3 + 1): 219 } 220 221 // recover the member. 222 clus.Members[0].Restart(t) 223 224 kresp = <-rc 225 if kresp.ID != resp.ID { 226 t.Errorf("ID = %x, want %x", kresp.ID, resp.ID) 227 } 228 229 lapi.Close() 230 231 _, ok := <-rc 232 if ok { 233 t.Errorf("chan is not closed, want lease Close() closes chan") 234 } 235 } 236 237 type leaseCh struct { 238 lid clientv3.LeaseID 239 ch <-chan *clientv3.LeaseKeepAliveResponse 240 } 241 242 // TestLeaseKeepAliveNotFound ensures a revoked lease won't halt other leases. 243 func TestLeaseKeepAliveNotFound(t *testing.T) { 244 defer testutil.AfterTest(t) 245 246 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}) 247 defer clus.Terminate(t) 248 249 cli := clus.RandClient() 250 lchs := []leaseCh{} 251 for i := 0; i < 3; i++ { 252 resp, rerr := cli.Grant(context.TODO(), 5) 253 if rerr != nil { 254 t.Fatal(rerr) 255 } 256 kach, kaerr := cli.KeepAlive(context.Background(), resp.ID) 257 if kaerr != nil { 258 t.Fatal(kaerr) 259 } 260 lchs = append(lchs, leaseCh{resp.ID, kach}) 261 } 262 263 if _, err := cli.Revoke(context.TODO(), lchs[1].lid); err != nil { 264 t.Fatal(err) 265 } 266 267 <-lchs[0].ch 268 if _, ok := <-lchs[0].ch; !ok { 269 t.Fatalf("closed keepalive on wrong lease") 270 } 271 272 timec := time.After(5 * time.Second) 273 for range lchs[1].ch { 274 select { 275 case <-timec: 276 t.Fatalf("revoke did not close keep alive") 277 default: 278 } 279 } 280 } 281 282 func TestLeaseGrantErrConnClosed(t *testing.T) { 283 defer testutil.AfterTest(t) 284 285 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}) 286 defer clus.Terminate(t) 287 288 cli := clus.Client(0) 289 clus.TakeClient(0) 290 291 donec := make(chan struct{}) 292 go func() { 293 defer close(donec) 294 _, err := cli.Grant(context.TODO(), 5) 295 if err != nil && err != grpc.ErrClientConnClosing && err != context.Canceled { 296 // grpc.ErrClientConnClosing if grpc-go balancer calls 'Get' after client.Close. 297 // context.Canceled if grpc-go balancer calls 'Get' with an inflight client.Close. 298 t.Fatalf("expected %v or %v, got %v", grpc.ErrClientConnClosing, context.Canceled, err) 299 } 300 }() 301 302 if err := cli.Close(); err != nil { 303 t.Fatal(err) 304 } 305 306 select { 307 case <-time.After(integration.RequestWaitTimeout): 308 t.Fatal("le.Grant took too long") 309 case <-donec: 310 } 311 } 312 313 // TestLeaseKeepAliveFullResponseQueue ensures when response 314 // queue is full thus dropping keepalive response sends, 315 // keepalive request is sent with the same rate of TTL / 3. 316 func TestLeaseKeepAliveFullResponseQueue(t *testing.T) { 317 defer testutil.AfterTest(t) 318 319 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}) 320 defer clus.Terminate(t) 321 322 lapi := clus.Client(0) 323 324 // expect lease keepalive every 10-second 325 lresp, err := lapi.Grant(context.Background(), 30) 326 if err != nil { 327 t.Fatalf("failed to create lease %v", err) 328 } 329 id := lresp.ID 330 331 old := clientv3.LeaseResponseChSize 332 defer func() { 333 clientv3.LeaseResponseChSize = old 334 }() 335 clientv3.LeaseResponseChSize = 0 336 337 // never fetch from response queue, and let it become full 338 _, err = lapi.KeepAlive(context.Background(), id) 339 if err != nil { 340 t.Fatalf("failed to keepalive lease %v", err) 341 } 342 343 // TTL should not be refreshed after 3 seconds 344 // expect keepalive to be triggered after TTL/3 345 time.Sleep(3 * time.Second) 346 347 tr, terr := lapi.TimeToLive(context.Background(), id) 348 if terr != nil { 349 t.Fatalf("failed to get lease information %v", terr) 350 } 351 if tr.TTL >= 29 { 352 t.Errorf("unexpected kept-alive lease TTL %d", tr.TTL) 353 } 354 } 355 356 func TestLeaseGrantNewAfterClose(t *testing.T) { 357 defer testutil.AfterTest(t) 358 359 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}) 360 defer clus.Terminate(t) 361 362 cli := clus.Client(0) 363 clus.TakeClient(0) 364 if err := cli.Close(); err != nil { 365 t.Fatal(err) 366 } 367 368 donec := make(chan struct{}) 369 go func() { 370 if _, err := cli.Grant(context.TODO(), 5); err != context.Canceled && err != grpc.ErrClientConnClosing { 371 t.Fatalf("expected %v or %v, got %v", err != context.Canceled, grpc.ErrClientConnClosing, err) 372 } 373 close(donec) 374 }() 375 select { 376 case <-time.After(integration.RequestWaitTimeout): 377 t.Fatal("le.Grant took too long") 378 case <-donec: 379 } 380 } 381 382 func TestLeaseRevokeNewAfterClose(t *testing.T) { 383 defer testutil.AfterTest(t) 384 385 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}) 386 defer clus.Terminate(t) 387 388 cli := clus.Client(0) 389 resp, err := cli.Grant(context.TODO(), 5) 390 if err != nil { 391 t.Fatal(err) 392 } 393 leaseID := resp.ID 394 395 clus.TakeClient(0) 396 if err := cli.Close(); err != nil { 397 t.Fatal(err) 398 } 399 400 donec := make(chan struct{}) 401 go func() { 402 if _, err := cli.Revoke(context.TODO(), leaseID); err != context.Canceled && err != grpc.ErrClientConnClosing { 403 t.Fatalf("expected %v or %v, got %v", err != context.Canceled, grpc.ErrClientConnClosing, err) 404 } 405 close(donec) 406 }() 407 select { 408 case <-time.After(integration.RequestWaitTimeout): 409 t.Fatal("le.Revoke took too long") 410 case <-donec: 411 } 412 } 413 414 // TestLeaseKeepAliveCloseAfterDisconnectRevoke ensures the keep alive channel is closed 415 // following a disconnection, lease revoke, then reconnect. 416 func TestLeaseKeepAliveCloseAfterDisconnectRevoke(t *testing.T) { 417 defer testutil.AfterTest(t) 418 419 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3}) 420 defer clus.Terminate(t) 421 422 cli := clus.Client(0) 423 424 // setup lease and do a keepalive 425 resp, err := cli.Grant(context.Background(), 10) 426 if err != nil { 427 t.Fatal(err) 428 } 429 rc, kerr := cli.KeepAlive(context.Background(), resp.ID) 430 if kerr != nil { 431 t.Fatal(kerr) 432 } 433 kresp := <-rc 434 if kresp.ID != resp.ID { 435 t.Fatalf("ID = %x, want %x", kresp.ID, resp.ID) 436 } 437 438 // keep client disconnected 439 clus.Members[0].Stop(t) 440 time.Sleep(time.Second) 441 clus.WaitLeader(t) 442 443 if _, err := clus.Client(1).Revoke(context.TODO(), resp.ID); err != nil { 444 t.Fatal(err) 445 } 446 447 clus.Members[0].Restart(t) 448 449 // some responses may still be buffered; drain until close 450 timer := time.After(time.Duration(kresp.TTL) * time.Second) 451 for kresp != nil { 452 select { 453 case kresp = <-rc: 454 case <-timer: 455 t.Fatalf("keepalive channel did not close") 456 } 457 } 458 } 459 460 // TestLeaseKeepAliveInitTimeout ensures the keep alive channel closes if 461 // the initial keep alive request never gets a response. 462 func TestLeaseKeepAliveInitTimeout(t *testing.T) { 463 defer testutil.AfterTest(t) 464 465 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}) 466 defer clus.Terminate(t) 467 468 cli := clus.Client(0) 469 470 // setup lease and do a keepalive 471 resp, err := cli.Grant(context.Background(), 5) 472 if err != nil { 473 t.Fatal(err) 474 } 475 // keep client disconnected 476 clus.Members[0].Stop(t) 477 rc, kerr := cli.KeepAlive(context.Background(), resp.ID) 478 if kerr != nil { 479 t.Fatal(kerr) 480 } 481 select { 482 case ka, ok := <-rc: 483 if ok { 484 t.Fatalf("unexpected keepalive %v, expected closed channel", ka) 485 } 486 case <-time.After(10 * time.Second): 487 t.Fatalf("keepalive channel did not close") 488 } 489 490 clus.Members[0].Restart(t) 491 } 492 493 // TestLeaseKeepAliveInitTimeout ensures the keep alive channel closes if 494 // a keep alive request after the first never gets a response. 495 func TestLeaseKeepAliveTTLTimeout(t *testing.T) { 496 defer testutil.AfterTest(t) 497 498 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}) 499 defer clus.Terminate(t) 500 501 cli := clus.Client(0) 502 503 // setup lease and do a keepalive 504 resp, err := cli.Grant(context.Background(), 5) 505 if err != nil { 506 t.Fatal(err) 507 } 508 rc, kerr := cli.KeepAlive(context.Background(), resp.ID) 509 if kerr != nil { 510 t.Fatal(kerr) 511 } 512 if kresp := <-rc; kresp.ID != resp.ID { 513 t.Fatalf("ID = %x, want %x", kresp.ID, resp.ID) 514 } 515 516 // keep client disconnected 517 clus.Members[0].Stop(t) 518 select { 519 case ka, ok := <-rc: 520 if ok { 521 t.Fatalf("unexpected keepalive %v, expected closed channel", ka) 522 } 523 case <-time.After(10 * time.Second): 524 t.Fatalf("keepalive channel did not close") 525 } 526 527 clus.Members[0].Restart(t) 528 } 529 530 func TestLeaseTimeToLive(t *testing.T) { 531 defer testutil.AfterTest(t) 532 533 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3}) 534 defer clus.Terminate(t) 535 536 c := clus.RandClient() 537 lapi := c 538 539 resp, err := lapi.Grant(context.Background(), 10) 540 if err != nil { 541 t.Errorf("failed to create lease %v", err) 542 } 543 544 kv := clus.RandClient() 545 keys := []string{"foo1", "foo2"} 546 for i := range keys { 547 if _, err = kv.Put(context.TODO(), keys[i], "bar", clientv3.WithLease(resp.ID)); err != nil { 548 t.Fatal(err) 549 } 550 } 551 552 // linearized read to ensure Puts propagated to server backing lapi 553 if _, err := c.Get(context.TODO(), "abc"); err != nil { 554 t.Fatal(err) 555 } 556 557 lresp, lerr := lapi.TimeToLive(context.Background(), resp.ID, clientv3.WithAttachedKeys()) 558 if lerr != nil { 559 t.Fatal(lerr) 560 } 561 if lresp.ID != resp.ID { 562 t.Fatalf("leaseID expected %d, got %d", resp.ID, lresp.ID) 563 } 564 if lresp.GrantedTTL != int64(10) { 565 t.Fatalf("GrantedTTL expected %d, got %d", 10, lresp.GrantedTTL) 566 } 567 if lresp.TTL == 0 || lresp.TTL > lresp.GrantedTTL { 568 t.Fatalf("unexpected TTL %d (granted %d)", lresp.TTL, lresp.GrantedTTL) 569 } 570 ks := make([]string, len(lresp.Keys)) 571 for i := range lresp.Keys { 572 ks[i] = string(lresp.Keys[i]) 573 } 574 sort.Strings(ks) 575 if !reflect.DeepEqual(ks, keys) { 576 t.Fatalf("keys expected %v, got %v", keys, ks) 577 } 578 579 lresp, lerr = lapi.TimeToLive(context.Background(), resp.ID) 580 if lerr != nil { 581 t.Fatal(lerr) 582 } 583 if len(lresp.Keys) != 0 { 584 t.Fatalf("unexpected keys %+v", lresp.Keys) 585 } 586 } 587 588 func TestLeaseTimeToLiveLeaseNotFound(t *testing.T) { 589 defer testutil.AfterTest(t) 590 591 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}) 592 defer clus.Terminate(t) 593 594 cli := clus.RandClient() 595 resp, err := cli.Grant(context.Background(), 10) 596 if err != nil { 597 t.Errorf("failed to create lease %v", err) 598 } 599 _, err = cli.Revoke(context.Background(), resp.ID) 600 if err != nil { 601 t.Errorf("failed to Revoke lease %v", err) 602 } 603 604 lresp, err := cli.TimeToLive(context.Background(), resp.ID) 605 // TimeToLive() should return a response with TTL=-1. 606 if err != nil { 607 t.Fatalf("expected err to be nil") 608 } 609 if lresp == nil { 610 t.Fatalf("expected lresp not to be nil") 611 } 612 if lresp.ResponseHeader == nil { 613 t.Fatalf("expected ResponseHeader not to be nil") 614 } 615 if lresp.ID != resp.ID { 616 t.Fatalf("expected Lease ID %v, but got %v", resp.ID, lresp.ID) 617 } 618 if lresp.TTL != -1 { 619 t.Fatalf("expected TTL %v, but got %v", lresp.TTL, lresp.TTL) 620 } 621 } 622 623 func TestLeaseLeases(t *testing.T) { 624 defer testutil.AfterTest(t) 625 626 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}) 627 defer clus.Terminate(t) 628 629 cli := clus.RandClient() 630 631 ids := []clientv3.LeaseID{} 632 for i := 0; i < 5; i++ { 633 resp, err := cli.Grant(context.Background(), 10) 634 if err != nil { 635 t.Errorf("failed to create lease %v", err) 636 } 637 ids = append(ids, resp.ID) 638 } 639 640 resp, err := cli.Leases(context.Background()) 641 if err != nil { 642 t.Fatal(err) 643 } 644 if len(resp.Leases) != 5 { 645 t.Fatalf("len(resp.Leases) expected 5, got %d", len(resp.Leases)) 646 } 647 for i := range resp.Leases { 648 if ids[i] != resp.Leases[i].ID { 649 t.Fatalf("#%d: lease ID expected %d, got %d", i, ids[i], resp.Leases[i].ID) 650 } 651 } 652 } 653 654 // TestLeaseRenewLostQuorum ensures keepalives work after losing quorum 655 // for a while. 656 func TestLeaseRenewLostQuorum(t *testing.T) { 657 defer testutil.AfterTest(t) 658 659 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3}) 660 defer clus.Terminate(t) 661 662 cli := clus.Client(0) 663 r, err := cli.Grant(context.TODO(), 4) 664 if err != nil { 665 t.Fatal(err) 666 } 667 668 kctx, kcancel := context.WithCancel(context.Background()) 669 defer kcancel() 670 ka, err := cli.KeepAlive(kctx, r.ID) 671 if err != nil { 672 t.Fatal(err) 673 } 674 // consume first keepalive so next message sends when cluster is down 675 <-ka 676 lastKa := time.Now() 677 678 // force keepalive stream message to timeout 679 clus.Members[1].Stop(t) 680 clus.Members[2].Stop(t) 681 // Use TTL-2 since the client closes the keepalive channel if no 682 // keepalive arrives before the lease deadline; the client will 683 // try to resend a keepalive after TTL/3 seconds, so for a TTL of 4, 684 // sleeping for 2s should be sufficient time for issuing a retry. 685 // The cluster has two seconds to recover and reply to the keepalive. 686 time.Sleep(time.Duration(r.TTL-2) * time.Second) 687 clus.Members[1].Restart(t) 688 clus.Members[2].Restart(t) 689 690 if time.Since(lastKa) > time.Duration(r.TTL)*time.Second { 691 t.Skip("waited too long for server stop and restart") 692 } 693 694 select { 695 case _, ok := <-ka: 696 if !ok { 697 t.Fatalf("keepalive closed") 698 } 699 case <-time.After(time.Duration(r.TTL) * time.Second): 700 t.Fatalf("timed out waiting for keepalive") 701 } 702 } 703 704 func TestLeaseKeepAliveLoopExit(t *testing.T) { 705 defer testutil.AfterTest(t) 706 707 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}) 708 defer clus.Terminate(t) 709 710 ctx := context.Background() 711 cli := clus.Client(0) 712 clus.TakeClient(0) 713 714 resp, err := cli.Grant(ctx, 5) 715 if err != nil { 716 t.Fatal(err) 717 } 718 cli.Close() 719 720 _, err = cli.KeepAlive(ctx, resp.ID) 721 if _, ok := err.(clientv3.ErrKeepAliveHalted); !ok { 722 t.Fatalf("expected %T, got %v(%T)", clientv3.ErrKeepAliveHalted{}, err, err) 723 } 724 } 725 726 // TestV3LeaseFailureOverlap issues Grant and KeepAlive requests to a cluster 727 // before, during, and after quorum loss to confirm Grant/KeepAlive tolerates 728 // transient cluster failure. 729 func TestV3LeaseFailureOverlap(t *testing.T) { 730 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 2}) 731 defer clus.Terminate(t) 732 733 numReqs := 5 734 cli := clus.Client(0) 735 736 // bring up a session, tear it down 737 updown := func(i int) error { 738 sess, err := concurrency.NewSession(cli) 739 if err != nil { 740 return err 741 } 742 ch := make(chan struct{}) 743 go func() { 744 defer close(ch) 745 sess.Close() 746 }() 747 select { 748 case <-ch: 749 case <-time.After(time.Minute / 4): 750 t.Fatalf("timeout %d", i) 751 } 752 return nil 753 } 754 755 var wg sync.WaitGroup 756 mkReqs := func(n int) { 757 wg.Add(numReqs) 758 for i := 0; i < numReqs; i++ { 759 go func() { 760 defer wg.Done() 761 err := updown(n) 762 if err == nil || err == rpctypes.ErrTimeoutDueToConnectionLost { 763 return 764 } 765 t.Fatal(err) 766 }() 767 } 768 } 769 770 mkReqs(1) 771 clus.Members[1].Stop(t) 772 mkReqs(2) 773 time.Sleep(time.Second) 774 mkReqs(3) 775 clus.Members[1].Restart(t) 776 mkReqs(4) 777 wg.Wait() 778 } 779 780 // TestLeaseWithRequireLeader checks keep-alive channel close when no leader. 781 func TestLeaseWithRequireLeader(t *testing.T) { 782 defer testutil.AfterTest(t) 783 784 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 2}) 785 defer clus.Terminate(t) 786 787 c := clus.Client(0) 788 lid1, err1 := c.Grant(context.TODO(), 60) 789 if err1 != nil { 790 t.Fatal(err1) 791 } 792 lid2, err2 := c.Grant(context.TODO(), 60) 793 if err2 != nil { 794 t.Fatal(err2) 795 } 796 // kaReqLeader close if the leader is lost 797 kaReqLeader, kerr1 := c.KeepAlive(clientv3.WithRequireLeader(context.TODO()), lid1.ID) 798 if kerr1 != nil { 799 t.Fatal(kerr1) 800 } 801 // kaWait will wait even if the leader is lost 802 kaWait, kerr2 := c.KeepAlive(context.TODO(), lid2.ID) 803 if kerr2 != nil { 804 t.Fatal(kerr2) 805 } 806 807 select { 808 case <-kaReqLeader: 809 case <-time.After(5 * time.Second): 810 t.Fatalf("require leader first keep-alive timed out") 811 } 812 select { 813 case <-kaWait: 814 case <-time.After(5 * time.Second): 815 t.Fatalf("leader not required first keep-alive timed out") 816 } 817 818 clus.Members[1].Stop(t) 819 // kaReqLeader may issue multiple requests while waiting for the first 820 // response from proxy server; drain any stray keepalive responses 821 time.Sleep(100 * time.Millisecond) 822 for len(kaReqLeader) > 0 { 823 <-kaReqLeader 824 } 825 826 select { 827 case resp, ok := <-kaReqLeader: 828 if ok { 829 t.Fatalf("expected closed require leader, got response %+v", resp) 830 } 831 case <-time.After(5 * time.Second): 832 t.Fatal("keepalive with require leader took too long to close") 833 } 834 select { 835 case _, ok := <-kaWait: 836 if !ok { 837 t.Fatalf("got closed channel with no require leader, expected non-closed") 838 } 839 case <-time.After(10 * time.Millisecond): 840 // wait some to detect any closes happening soon after kaReqLeader closing 841 } 842 }