go.etcd.io/etcd@v3.3.27+incompatible/clientv3/integration/watch_test.go (about) 1 // Copyright 2016 The etcd Authors 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package integration 16 17 import ( 18 "context" 19 "fmt" 20 "math/rand" 21 "reflect" 22 "sort" 23 "testing" 24 "time" 25 26 "github.com/coreos/etcd/clientv3" 27 "github.com/coreos/etcd/etcdserver/api/v3rpc" 28 "github.com/coreos/etcd/etcdserver/api/v3rpc/rpctypes" 29 "github.com/coreos/etcd/integration" 30 mvccpb "github.com/coreos/etcd/mvcc/mvccpb" 31 "github.com/coreos/etcd/pkg/testutil" 32 33 "google.golang.org/grpc/metadata" 34 ) 35 36 type watcherTest func(*testing.T, *watchctx) 37 38 type watchctx struct { 39 clus *integration.ClusterV3 40 w clientv3.Watcher 41 kv clientv3.KV 42 wclientMember int 43 kvMember int 44 ch clientv3.WatchChan 45 } 46 47 func runWatchTest(t *testing.T, f watcherTest) { 48 defer testutil.AfterTest(t) 49 50 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3}) 51 defer clus.Terminate(t) 52 53 wclientMember := rand.Intn(3) 54 w := clus.Client(wclientMember).Watcher 55 // select a different client for KV operations so puts succeed if 56 // a test knocks out the watcher client. 57 kvMember := rand.Intn(3) 58 for kvMember == wclientMember { 59 kvMember = rand.Intn(3) 60 } 61 kv := clus.Client(kvMember).KV 62 63 wctx := &watchctx{clus, w, kv, wclientMember, kvMember, nil} 64 f(t, wctx) 65 } 66 67 // TestWatchMultiWatcher modifies multiple keys and observes the changes. 68 func TestWatchMultiWatcher(t *testing.T) { 69 runWatchTest(t, testWatchMultiWatcher) 70 } 71 72 func testWatchMultiWatcher(t *testing.T, wctx *watchctx) { 73 numKeyUpdates := 4 74 keys := []string{"foo", "bar", "baz"} 75 76 donec := make(chan struct{}) 77 readyc := make(chan struct{}) 78 for _, k := range keys { 79 // key watcher 80 go func(key string) { 81 ch := wctx.w.Watch(context.TODO(), key) 82 if ch == nil { 83 t.Fatalf("expected watcher channel, got nil") 84 } 85 readyc <- struct{}{} 86 for i := 0; i < numKeyUpdates; i++ { 87 resp, ok := <-ch 88 if !ok { 89 t.Fatalf("watcher unexpectedly closed") 90 } 91 v := fmt.Sprintf("%s-%d", key, i) 92 gotv := string(resp.Events[0].Kv.Value) 93 if gotv != v { 94 t.Errorf("#%d: got %s, wanted %s", i, gotv, v) 95 } 96 } 97 donec <- struct{}{} 98 }(k) 99 } 100 // prefix watcher on "b" (bar and baz) 101 go func() { 102 prefixc := wctx.w.Watch(context.TODO(), "b", clientv3.WithPrefix()) 103 if prefixc == nil { 104 t.Fatalf("expected watcher channel, got nil") 105 } 106 readyc <- struct{}{} 107 evs := []*clientv3.Event{} 108 for i := 0; i < numKeyUpdates*2; i++ { 109 resp, ok := <-prefixc 110 if !ok { 111 t.Fatalf("watcher unexpectedly closed") 112 } 113 evs = append(evs, resp.Events...) 114 } 115 116 // check response 117 expected := []string{} 118 bkeys := []string{"bar", "baz"} 119 for _, k := range bkeys { 120 for i := 0; i < numKeyUpdates; i++ { 121 expected = append(expected, fmt.Sprintf("%s-%d", k, i)) 122 } 123 } 124 got := []string{} 125 for _, ev := range evs { 126 got = append(got, string(ev.Kv.Value)) 127 } 128 sort.Strings(got) 129 if !reflect.DeepEqual(expected, got) { 130 t.Errorf("got %v, expected %v", got, expected) 131 } 132 133 // ensure no extra data 134 select { 135 case resp, ok := <-prefixc: 136 if !ok { 137 t.Fatalf("watcher unexpectedly closed") 138 } 139 t.Fatalf("unexpected event %+v", resp) 140 case <-time.After(time.Second): 141 } 142 donec <- struct{}{} 143 }() 144 145 // wait for watcher bring up 146 for i := 0; i < len(keys)+1; i++ { 147 <-readyc 148 } 149 // generate events 150 ctx := context.TODO() 151 for i := 0; i < numKeyUpdates; i++ { 152 for _, k := range keys { 153 v := fmt.Sprintf("%s-%d", k, i) 154 if _, err := wctx.kv.Put(ctx, k, v); err != nil { 155 t.Fatal(err) 156 } 157 } 158 } 159 // wait for watcher shutdown 160 for i := 0; i < len(keys)+1; i++ { 161 <-donec 162 } 163 } 164 165 // TestWatchRange tests watcher creates ranges 166 func TestWatchRange(t *testing.T) { 167 runWatchTest(t, testWatchRange) 168 } 169 170 func testWatchRange(t *testing.T, wctx *watchctx) { 171 if wctx.ch = wctx.w.Watch(context.TODO(), "a", clientv3.WithRange("c")); wctx.ch == nil { 172 t.Fatalf("expected non-nil channel") 173 } 174 putAndWatch(t, wctx, "a", "a") 175 putAndWatch(t, wctx, "b", "b") 176 putAndWatch(t, wctx, "bar", "bar") 177 } 178 179 // TestWatchReconnRequest tests the send failure path when requesting a watcher. 180 func TestWatchReconnRequest(t *testing.T) { 181 runWatchTest(t, testWatchReconnRequest) 182 } 183 184 func testWatchReconnRequest(t *testing.T, wctx *watchctx) { 185 donec, stopc := make(chan struct{}), make(chan struct{}, 1) 186 go func() { 187 timer := time.After(2 * time.Second) 188 defer close(donec) 189 // take down watcher connection 190 for { 191 wctx.clus.Members[wctx.wclientMember].DropConnections() 192 select { 193 case <-timer: 194 // spinning on close may live lock reconnection 195 return 196 case <-stopc: 197 return 198 default: 199 } 200 } 201 }() 202 // should reconnect when requesting watch 203 if wctx.ch = wctx.w.Watch(context.TODO(), "a"); wctx.ch == nil { 204 t.Fatalf("expected non-nil channel") 205 } 206 207 // wait for disconnections to stop 208 stopc <- struct{}{} 209 <-donec 210 211 // spinning on dropping connections may trigger a leader election 212 // due to resource starvation; l-read to ensure the cluster is stable 213 ctx, cancel := context.WithTimeout(context.TODO(), 30*time.Second) 214 if _, err := wctx.kv.Get(ctx, "_"); err != nil { 215 t.Fatal(err) 216 } 217 cancel() 218 219 // ensure watcher works 220 putAndWatch(t, wctx, "a", "a") 221 } 222 223 // TestWatchReconnInit tests watcher resumes correctly if connection lost 224 // before any data was sent. 225 func TestWatchReconnInit(t *testing.T) { 226 runWatchTest(t, testWatchReconnInit) 227 } 228 229 func testWatchReconnInit(t *testing.T, wctx *watchctx) { 230 if wctx.ch = wctx.w.Watch(context.TODO(), "a"); wctx.ch == nil { 231 t.Fatalf("expected non-nil channel") 232 } 233 wctx.clus.Members[wctx.wclientMember].DropConnections() 234 // watcher should recover 235 putAndWatch(t, wctx, "a", "a") 236 } 237 238 // TestWatchReconnRunning tests watcher resumes correctly if connection lost 239 // after data was sent. 240 func TestWatchReconnRunning(t *testing.T) { 241 runWatchTest(t, testWatchReconnRunning) 242 } 243 244 func testWatchReconnRunning(t *testing.T, wctx *watchctx) { 245 if wctx.ch = wctx.w.Watch(context.TODO(), "a"); wctx.ch == nil { 246 t.Fatalf("expected non-nil channel") 247 } 248 putAndWatch(t, wctx, "a", "a") 249 // take down watcher connection 250 wctx.clus.Members[wctx.wclientMember].DropConnections() 251 // watcher should recover 252 putAndWatch(t, wctx, "a", "b") 253 } 254 255 // TestWatchCancelImmediate ensures a closed channel is returned 256 // if the context is cancelled. 257 func TestWatchCancelImmediate(t *testing.T) { 258 runWatchTest(t, testWatchCancelImmediate) 259 } 260 261 func testWatchCancelImmediate(t *testing.T, wctx *watchctx) { 262 ctx, cancel := context.WithCancel(context.Background()) 263 cancel() 264 wch := wctx.w.Watch(ctx, "a") 265 select { 266 case wresp, ok := <-wch: 267 if ok { 268 t.Fatalf("read wch got %v; expected closed channel", wresp) 269 } 270 default: 271 t.Fatalf("closed watcher channel should not block") 272 } 273 } 274 275 // TestWatchCancelInit tests watcher closes correctly after no events. 276 func TestWatchCancelInit(t *testing.T) { 277 runWatchTest(t, testWatchCancelInit) 278 } 279 280 func testWatchCancelInit(t *testing.T, wctx *watchctx) { 281 ctx, cancel := context.WithCancel(context.Background()) 282 if wctx.ch = wctx.w.Watch(ctx, "a"); wctx.ch == nil { 283 t.Fatalf("expected non-nil watcher channel") 284 } 285 cancel() 286 select { 287 case <-time.After(time.Second): 288 t.Fatalf("took too long to cancel") 289 case _, ok := <-wctx.ch: 290 if ok { 291 t.Fatalf("expected watcher channel to close") 292 } 293 } 294 } 295 296 // TestWatchCancelRunning tests watcher closes correctly after events. 297 func TestWatchCancelRunning(t *testing.T) { 298 runWatchTest(t, testWatchCancelRunning) 299 } 300 301 func testWatchCancelRunning(t *testing.T, wctx *watchctx) { 302 ctx, cancel := context.WithCancel(context.Background()) 303 if wctx.ch = wctx.w.Watch(ctx, "a"); wctx.ch == nil { 304 t.Fatalf("expected non-nil watcher channel") 305 } 306 if _, err := wctx.kv.Put(ctx, "a", "a"); err != nil { 307 t.Fatal(err) 308 } 309 cancel() 310 select { 311 case <-time.After(time.Second): 312 t.Fatalf("took too long to cancel") 313 case _, ok := <-wctx.ch: 314 if !ok { 315 // closed before getting put; OK 316 break 317 } 318 // got the PUT; should close next 319 select { 320 case <-time.After(time.Second): 321 t.Fatalf("took too long to close") 322 case v, ok2 := <-wctx.ch: 323 if ok2 { 324 t.Fatalf("expected watcher channel to close, got %v", v) 325 } 326 } 327 } 328 } 329 330 func putAndWatch(t *testing.T, wctx *watchctx, key, val string) { 331 if _, err := wctx.kv.Put(context.TODO(), key, val); err != nil { 332 t.Fatal(err) 333 } 334 select { 335 case <-time.After(5 * time.Second): 336 t.Fatalf("watch timed out") 337 case v, ok := <-wctx.ch: 338 if !ok { 339 t.Fatalf("unexpected watch close") 340 } 341 if string(v.Events[0].Kv.Value) != val { 342 t.Fatalf("bad value got %v, wanted %v", v.Events[0].Kv.Value, val) 343 } 344 } 345 } 346 347 func TestWatchResumeInitRev(t *testing.T) { 348 defer testutil.AfterTest(t) 349 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}) 350 defer clus.Terminate(t) 351 352 cli := clus.Client(0) 353 if _, err := cli.Put(context.TODO(), "b", "2"); err != nil { 354 t.Fatal(err) 355 } 356 if _, err := cli.Put(context.TODO(), "a", "3"); err != nil { 357 t.Fatal(err) 358 } 359 // if resume is broken, it'll pick up this key first instead of a=3 360 if _, err := cli.Put(context.TODO(), "a", "4"); err != nil { 361 t.Fatal(err) 362 } 363 364 wch := clus.Client(0).Watch(context.Background(), "a", clientv3.WithRev(1), clientv3.WithCreatedNotify()) 365 if resp, ok := <-wch; !ok || resp.Header.Revision != 4 { 366 t.Fatalf("got (%v, %v), expected create notification rev=4", resp, ok) 367 } 368 // pause wch 369 clus.Members[0].DropConnections() 370 clus.Members[0].PauseConnections() 371 372 select { 373 case resp, ok := <-wch: 374 t.Skipf("wch should block, got (%+v, %v); drop not fast enough", resp, ok) 375 case <-time.After(100 * time.Millisecond): 376 } 377 378 // resume wch 379 clus.Members[0].UnpauseConnections() 380 381 select { 382 case resp, ok := <-wch: 383 if !ok { 384 t.Fatal("unexpected watch close") 385 } 386 if len(resp.Events) == 0 { 387 t.Fatal("expected event on watch") 388 } 389 if string(resp.Events[0].Kv.Value) != "3" { 390 t.Fatalf("expected value=3, got event %+v", resp.Events[0]) 391 } 392 case <-time.After(5 * time.Second): 393 t.Fatal("watch timed out") 394 } 395 } 396 397 // TestWatchResumeCompacted checks that the watcher gracefully closes in case 398 // that it tries to resume to a revision that's been compacted out of the store. 399 // Since the watcher's server restarts with stale data, the watcher will receive 400 // either a compaction error or all keys by staying in sync before the compaction 401 // is finally applied. 402 func TestWatchResumeCompacted(t *testing.T) { 403 defer testutil.AfterTest(t) 404 405 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3}) 406 defer clus.Terminate(t) 407 408 // create a waiting watcher at rev 1 409 w := clus.Client(0) 410 wch := w.Watch(context.Background(), "foo", clientv3.WithRev(1)) 411 select { 412 case w := <-wch: 413 t.Errorf("unexpected message from wch %v", w) 414 default: 415 } 416 clus.Members[0].Stop(t) 417 418 ticker := time.After(time.Second * 10) 419 for clus.WaitLeader(t) <= 0 { 420 select { 421 case <-ticker: 422 t.Fatalf("failed to wait for new leader") 423 default: 424 time.Sleep(10 * time.Millisecond) 425 } 426 } 427 428 // put some data and compact away 429 numPuts := 5 430 kv := clus.Client(1) 431 for i := 0; i < numPuts; i++ { 432 if _, err := kv.Put(context.TODO(), "foo", "bar"); err != nil { 433 t.Fatal(err) 434 } 435 } 436 if _, err := kv.Compact(context.TODO(), 3); err != nil { 437 t.Fatal(err) 438 } 439 440 clus.Members[0].Restart(t) 441 442 // since watch's server isn't guaranteed to be synced with the cluster when 443 // the watch resumes, there is a window where the watch can stay synced and 444 // read off all events; if the watcher misses the window, it will go out of 445 // sync and get a compaction error. 446 wRev := int64(2) 447 for int(wRev) <= numPuts+1 { 448 var wresp clientv3.WatchResponse 449 var ok bool 450 select { 451 case wresp, ok = <-wch: 452 if !ok { 453 t.Fatalf("expected wresp, but got closed channel") 454 } 455 case <-time.After(5 * time.Second): 456 t.Fatalf("compacted watch timed out") 457 } 458 for _, ev := range wresp.Events { 459 if ev.Kv.ModRevision != wRev { 460 t.Fatalf("expected modRev %v, got %+v", wRev, ev) 461 } 462 wRev++ 463 } 464 if wresp.Err() == nil { 465 continue 466 } 467 if wresp.Err() != rpctypes.ErrCompacted { 468 t.Fatalf("wresp.Err() expected %v, got %+v", rpctypes.ErrCompacted, wresp.Err()) 469 } 470 break 471 } 472 if int(wRev) > numPuts+1 { 473 // got data faster than the compaction 474 return 475 } 476 // received compaction error; ensure the channel closes 477 select { 478 case wresp, ok := <-wch: 479 if ok { 480 t.Fatalf("expected closed channel, but got %v", wresp) 481 } 482 case <-time.After(5 * time.Second): 483 t.Fatalf("timed out waiting for channel close") 484 } 485 } 486 487 // TestWatchCompactRevision ensures the CompactRevision error is given on a 488 // compaction event ahead of a watcher. 489 func TestWatchCompactRevision(t *testing.T) { 490 defer testutil.AfterTest(t) 491 492 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}) 493 defer clus.Terminate(t) 494 495 // set some keys 496 kv := clus.RandClient() 497 for i := 0; i < 5; i++ { 498 if _, err := kv.Put(context.TODO(), "foo", "bar"); err != nil { 499 t.Fatal(err) 500 } 501 } 502 503 w := clus.RandClient() 504 505 if _, err := kv.Compact(context.TODO(), 4); err != nil { 506 t.Fatal(err) 507 } 508 wch := w.Watch(context.Background(), "foo", clientv3.WithRev(2)) 509 510 // get compacted error message 511 wresp, ok := <-wch 512 if !ok { 513 t.Fatalf("expected wresp, but got closed channel") 514 } 515 if wresp.Err() != rpctypes.ErrCompacted { 516 t.Fatalf("wresp.Err() expected %v, but got %v", rpctypes.ErrCompacted, wresp.Err()) 517 } 518 if !wresp.Canceled { 519 t.Fatalf("wresp.Canceled expected true, got %+v", wresp) 520 } 521 522 // ensure the channel is closed 523 if wresp, ok = <-wch; ok { 524 t.Fatalf("expected closed channel, but got %v", wresp) 525 } 526 } 527 528 func TestWatchWithProgressNotify(t *testing.T) { testWatchWithProgressNotify(t, true) } 529 func TestWatchWithProgressNotifyNoEvent(t *testing.T) { testWatchWithProgressNotify(t, false) } 530 531 func testWatchWithProgressNotify(t *testing.T, watchOnPut bool) { 532 defer testutil.AfterTest(t) 533 534 // accelerate report interval so test terminates quickly 535 oldpi := v3rpc.GetProgressReportInterval() 536 // using atomics to avoid race warnings 537 v3rpc.SetProgressReportInterval(3 * time.Second) 538 pi := 3 * time.Second 539 defer func() { v3rpc.SetProgressReportInterval(oldpi) }() 540 541 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3}) 542 defer clus.Terminate(t) 543 544 wc := clus.RandClient() 545 546 opts := []clientv3.OpOption{clientv3.WithProgressNotify()} 547 if watchOnPut { 548 opts = append(opts, clientv3.WithPrefix()) 549 } 550 rch := wc.Watch(context.Background(), "foo", opts...) 551 552 select { 553 case resp := <-rch: // wait for notification 554 if len(resp.Events) != 0 { 555 t.Fatalf("resp.Events expected none, got %+v", resp.Events) 556 } 557 case <-time.After(2 * pi): 558 t.Fatalf("watch response expected in %v, but timed out", pi) 559 } 560 561 kvc := clus.RandClient() 562 if _, err := kvc.Put(context.TODO(), "foox", "bar"); err != nil { 563 t.Fatal(err) 564 } 565 566 select { 567 case resp := <-rch: 568 if resp.Header.Revision != 2 { 569 t.Fatalf("resp.Header.Revision expected 2, got %d", resp.Header.Revision) 570 } 571 if watchOnPut { // wait for put if watch on the put key 572 ev := []*clientv3.Event{{Type: clientv3.EventTypePut, 573 Kv: &mvccpb.KeyValue{Key: []byte("foox"), Value: []byte("bar"), CreateRevision: 2, ModRevision: 2, Version: 1}}} 574 if !reflect.DeepEqual(ev, resp.Events) { 575 t.Fatalf("expected %+v, got %+v", ev, resp.Events) 576 } 577 } else if len(resp.Events) != 0 { // wait for notification otherwise 578 t.Fatalf("expected no events, but got %+v", resp.Events) 579 } 580 case <-time.After(time.Duration(1.5 * float64(pi))): 581 t.Fatalf("watch response expected in %v, but timed out", pi) 582 } 583 } 584 585 func TestWatchRequestProgress(t *testing.T) { 586 testCases := []struct { 587 name string 588 watchers []string 589 }{ 590 {"0-watcher", []string{}}, 591 {"1-watcher", []string{"/"}}, 592 {"2-watcher", []string{"/", "/"}}, 593 } 594 595 for _, c := range testCases { 596 t.Run(c.name, func(t *testing.T) { 597 defer testutil.AfterTest(t) 598 599 watchTimeout := 3 * time.Second 600 601 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3}) 602 defer clus.Terminate(t) 603 604 wc := clus.RandClient() 605 606 var watchChans []clientv3.WatchChan 607 608 for _, prefix := range c.watchers { 609 watchChans = append(watchChans, wc.Watch(context.Background(), prefix, clientv3.WithPrefix())) 610 } 611 612 _, err := wc.Put(context.Background(), "/a", "1") 613 if err != nil { 614 t.Fatal(err) 615 } 616 617 for _, rch := range watchChans { 618 select { 619 case resp := <-rch: // wait for notification 620 if len(resp.Events) != 1 { 621 t.Fatalf("resp.Events expected 1, got %d", len(resp.Events)) 622 } 623 case <-time.After(watchTimeout): 624 t.Fatalf("watch response expected in %v, but timed out", watchTimeout) 625 } 626 } 627 628 // put a value not being watched to increment revision 629 _, err = wc.Put(context.Background(), "x", "1") 630 if err != nil { 631 t.Fatal(err) 632 } 633 634 err = wc.RequestProgress(context.Background()) 635 if err != nil { 636 t.Fatal(err) 637 } 638 639 // verify all watch channels receive a progress notify 640 for _, rch := range watchChans { 641 select { 642 case resp := <-rch: 643 if !resp.IsProgressNotify() { 644 t.Fatalf("expected resp.IsProgressNotify() == true") 645 } 646 if resp.Header.Revision != 3 { 647 t.Fatalf("resp.Header.Revision expected 3, got %d", resp.Header.Revision) 648 } 649 case <-time.After(watchTimeout): 650 t.Fatalf("progress response expected in %v, but timed out", watchTimeout) 651 } 652 } 653 }) 654 } 655 } 656 657 func TestWatchEventType(t *testing.T) { 658 cluster := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}) 659 defer cluster.Terminate(t) 660 661 client := cluster.RandClient() 662 ctx := context.Background() 663 watchChan := client.Watch(ctx, "/", clientv3.WithPrefix()) 664 665 if _, err := client.Put(ctx, "/toDelete", "foo"); err != nil { 666 t.Fatalf("Put failed: %v", err) 667 } 668 if _, err := client.Put(ctx, "/toDelete", "bar"); err != nil { 669 t.Fatalf("Put failed: %v", err) 670 } 671 if _, err := client.Delete(ctx, "/toDelete"); err != nil { 672 t.Fatalf("Delete failed: %v", err) 673 } 674 lcr, err := client.Lease.Grant(ctx, 1) 675 if err != nil { 676 t.Fatalf("lease create failed: %v", err) 677 } 678 if _, err := client.Put(ctx, "/toExpire", "foo", clientv3.WithLease(lcr.ID)); err != nil { 679 t.Fatalf("Put failed: %v", err) 680 } 681 682 tests := []struct { 683 et mvccpb.Event_EventType 684 isCreate bool 685 isModify bool 686 }{{ 687 et: clientv3.EventTypePut, 688 isCreate: true, 689 }, { 690 et: clientv3.EventTypePut, 691 isModify: true, 692 }, { 693 et: clientv3.EventTypeDelete, 694 }, { 695 et: clientv3.EventTypePut, 696 isCreate: true, 697 }, { 698 et: clientv3.EventTypeDelete, 699 }} 700 701 var res []*clientv3.Event 702 703 for { 704 select { 705 case wres := <-watchChan: 706 res = append(res, wres.Events...) 707 case <-time.After(10 * time.Second): 708 t.Fatalf("Should receive %d events and then break out loop", len(tests)) 709 } 710 if len(res) == len(tests) { 711 break 712 } 713 } 714 715 for i, tt := range tests { 716 ev := res[i] 717 if tt.et != ev.Type { 718 t.Errorf("#%d: event type want=%s, get=%s", i, tt.et, ev.Type) 719 } 720 if tt.isCreate && !ev.IsCreate() { 721 t.Errorf("#%d: event should be CreateEvent", i) 722 } 723 if tt.isModify && !ev.IsModify() { 724 t.Errorf("#%d: event should be ModifyEvent", i) 725 } 726 } 727 } 728 729 func TestWatchErrConnClosed(t *testing.T) { 730 defer testutil.AfterTest(t) 731 732 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}) 733 defer clus.Terminate(t) 734 735 cli := clus.Client(0) 736 737 donec := make(chan struct{}) 738 go func() { 739 defer close(donec) 740 ch := cli.Watch(context.TODO(), "foo") 741 742 if wr := <-ch; !isCanceled(wr.Err()) { 743 t.Errorf("expected context canceled, got %v", wr.Err()) 744 } 745 }() 746 747 if err := cli.ActiveConnection().Close(); err != nil { 748 t.Fatal(err) 749 } 750 clus.TakeClient(0) 751 752 select { 753 case <-time.After(integration.RequestWaitTimeout): 754 t.Fatal("wc.Watch took too long") 755 case <-donec: 756 } 757 } 758 759 func TestWatchAfterClose(t *testing.T) { 760 defer testutil.AfterTest(t) 761 762 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}) 763 defer clus.Terminate(t) 764 765 cli := clus.Client(0) 766 clus.TakeClient(0) 767 if err := cli.Close(); err != nil { 768 t.Fatal(err) 769 } 770 771 donec := make(chan struct{}) 772 go func() { 773 cli.Watch(context.TODO(), "foo") 774 if err := cli.Close(); err != nil && err != context.Canceled { 775 t.Errorf("expected %v, got %v", context.Canceled, err) 776 } 777 close(donec) 778 }() 779 select { 780 case <-time.After(integration.RequestWaitTimeout): 781 t.Fatal("wc.Watch took too long") 782 case <-donec: 783 } 784 } 785 786 // TestWatchWithRequireLeader checks the watch channel closes when no leader. 787 func TestWatchWithRequireLeader(t *testing.T) { 788 defer testutil.AfterTest(t) 789 790 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3}) 791 defer clus.Terminate(t) 792 793 // Put a key for the non-require leader watch to read as an event. 794 // The watchers will be on member[0]; put key through member[0] to 795 // ensure that it receives the update so watching after killing quorum 796 // is guaranteed to have the key. 797 liveClient := clus.Client(0) 798 if _, err := liveClient.Put(context.TODO(), "foo", "bar"); err != nil { 799 t.Fatal(err) 800 } 801 802 clus.Members[1].Stop(t) 803 clus.Members[2].Stop(t) 804 clus.Client(1).Close() 805 clus.Client(2).Close() 806 clus.TakeClient(1) 807 clus.TakeClient(2) 808 809 // wait for election timeout, then member[0] will not have a leader. 810 tickDuration := 10 * time.Millisecond 811 // existing streams need three elections before they're torn down; wait until 5 elections cycle 812 // so proxy tests receive a leader loss event on its existing watch before creating a new watch. 813 time.Sleep(time.Duration(5*clus.Members[0].ElectionTicks) * tickDuration) 814 815 chLeader := liveClient.Watch(clientv3.WithRequireLeader(context.TODO()), "foo", clientv3.WithRev(1)) 816 chNoLeader := liveClient.Watch(context.TODO(), "foo", clientv3.WithRev(1)) 817 818 select { 819 case resp, ok := <-chLeader: 820 if !ok { 821 t.Fatalf("expected %v watch channel, got closed channel", rpctypes.ErrNoLeader) 822 } 823 if resp.Err() != rpctypes.ErrNoLeader { 824 t.Fatalf("expected %v watch response error, got %+v", rpctypes.ErrNoLeader, resp) 825 } 826 case <-time.After(integration.RequestWaitTimeout): 827 t.Fatal("watch without leader took too long to close") 828 } 829 830 select { 831 case resp, ok := <-chLeader: 832 if ok { 833 t.Fatalf("expected closed channel, got response %v", resp) 834 } 835 case <-time.After(integration.RequestWaitTimeout): 836 t.Fatal("waited too long for channel to close") 837 } 838 839 if _, ok := <-chNoLeader; !ok { 840 t.Fatalf("expected response, got closed channel") 841 } 842 } 843 844 // TestWatchWithFilter checks that watch filtering works. 845 func TestWatchWithFilter(t *testing.T) { 846 cluster := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}) 847 defer cluster.Terminate(t) 848 849 client := cluster.RandClient() 850 ctx := context.Background() 851 852 wcNoPut := client.Watch(ctx, "a", clientv3.WithFilterPut()) 853 wcNoDel := client.Watch(ctx, "a", clientv3.WithFilterDelete()) 854 855 if _, err := client.Put(ctx, "a", "abc"); err != nil { 856 t.Fatal(err) 857 } 858 if _, err := client.Delete(ctx, "a"); err != nil { 859 t.Fatal(err) 860 } 861 862 npResp := <-wcNoPut 863 if len(npResp.Events) != 1 || npResp.Events[0].Type != clientv3.EventTypeDelete { 864 t.Fatalf("expected delete event, got %+v", npResp.Events) 865 } 866 ndResp := <-wcNoDel 867 if len(ndResp.Events) != 1 || ndResp.Events[0].Type != clientv3.EventTypePut { 868 t.Fatalf("expected put event, got %+v", ndResp.Events) 869 } 870 871 select { 872 case resp := <-wcNoPut: 873 t.Fatalf("unexpected event on filtered put (%+v)", resp) 874 case resp := <-wcNoDel: 875 t.Fatalf("unexpected event on filtered delete (%+v)", resp) 876 case <-time.After(100 * time.Millisecond): 877 } 878 } 879 880 // TestWatchWithCreatedNotification checks that WithCreatedNotify returns a 881 // Created watch response. 882 func TestWatchWithCreatedNotification(t *testing.T) { 883 cluster := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}) 884 defer cluster.Terminate(t) 885 886 client := cluster.RandClient() 887 888 ctx := context.Background() 889 890 createC := client.Watch(ctx, "a", clientv3.WithCreatedNotify()) 891 892 resp := <-createC 893 894 if !resp.Created { 895 t.Fatalf("expected created event, got %v", resp) 896 } 897 } 898 899 // TestWatchWithCreatedNotificationDropConn ensures that 900 // a watcher with created notify does not post duplicate 901 // created events from disconnect. 902 func TestWatchWithCreatedNotificationDropConn(t *testing.T) { 903 cluster := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}) 904 defer cluster.Terminate(t) 905 906 client := cluster.RandClient() 907 908 wch := client.Watch(context.Background(), "a", clientv3.WithCreatedNotify()) 909 910 resp := <-wch 911 912 if !resp.Created { 913 t.Fatalf("expected created event, got %v", resp) 914 } 915 916 cluster.Members[0].DropConnections() 917 918 // check watch channel doesn't post another watch response. 919 select { 920 case wresp := <-wch: 921 t.Fatalf("got unexpected watch response: %+v\n", wresp) 922 case <-time.After(time.Second): 923 // watcher may not reconnect by the time it hits the select, 924 // so it wouldn't have a chance to filter out the second create event 925 } 926 } 927 928 // TestWatchCancelOnServer ensures client watcher cancels propagate back to the server. 929 func TestWatchCancelOnServer(t *testing.T) { 930 cluster := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}) 931 defer cluster.Terminate(t) 932 933 client := cluster.RandClient() 934 numWatches := 10 935 936 // The grpc proxy starts watches to detect leadership after the proxy server 937 // returns as started; to avoid racing on the proxy's internal watches, wait 938 // until require leader watches get create responses to ensure the leadership 939 // watches have started. 940 for { 941 ctx, cancel := context.WithCancel(clientv3.WithRequireLeader(context.TODO())) 942 ww := client.Watch(ctx, "a", clientv3.WithCreatedNotify()) 943 wresp := <-ww 944 cancel() 945 if wresp.Err() == nil { 946 break 947 } 948 } 949 950 cancels := make([]context.CancelFunc, numWatches) 951 for i := 0; i < numWatches; i++ { 952 // force separate streams in client 953 md := metadata.Pairs("some-key", fmt.Sprintf("%d", i)) 954 mctx := metadata.NewOutgoingContext(context.Background(), md) 955 ctx, cancel := context.WithCancel(mctx) 956 cancels[i] = cancel 957 w := client.Watch(ctx, fmt.Sprintf("%d", i), clientv3.WithCreatedNotify()) 958 <-w 959 } 960 961 // get max watches; proxy tests have leadership watches, so total may be >numWatches 962 maxWatches, _ := cluster.Members[0].Metric("etcd_debugging_mvcc_watcher_total") 963 964 // cancel all and wait for cancels to propagate to etcd server 965 for i := 0; i < numWatches; i++ { 966 cancels[i]() 967 } 968 time.Sleep(time.Second) 969 970 minWatches, err := cluster.Members[0].Metric("etcd_debugging_mvcc_watcher_total") 971 if err != nil { 972 t.Fatal(err) 973 } 974 975 maxWatchV, minWatchV := 0, 0 976 n, serr := fmt.Sscanf(maxWatches+" "+minWatches, "%d %d", &maxWatchV, &minWatchV) 977 if n != 2 || serr != nil { 978 t.Fatalf("expected n=2 and err=nil, got n=%d and err=%v", n, serr) 979 } 980 981 if maxWatchV-minWatchV < numWatches { 982 t.Fatalf("expected %d canceled watchers, got %d", numWatches, maxWatchV-minWatchV) 983 } 984 } 985 986 // TestWatchOverlapContextCancel stresses the watcher stream teardown path by 987 // creating/canceling watchers to ensure that new watchers are not taken down 988 // by a torn down watch stream. The sort of race that's being detected: 989 // 1. create w1 using a cancelable ctx with %v as "ctx" 990 // 2. cancel ctx 991 // 3. watcher client begins tearing down watcher grpc stream since no more watchers 992 // 3. start creating watcher w2 using a new "ctx" (not canceled), attaches to old grpc stream 993 // 4. watcher client finishes tearing down stream on "ctx" 994 // 5. w2 comes back canceled 995 func TestWatchOverlapContextCancel(t *testing.T) { 996 f := func(clus *integration.ClusterV3) {} 997 testWatchOverlapContextCancel(t, f) 998 } 999 1000 func TestWatchOverlapDropConnContextCancel(t *testing.T) { 1001 f := func(clus *integration.ClusterV3) { 1002 clus.Members[0].DropConnections() 1003 } 1004 testWatchOverlapContextCancel(t, f) 1005 } 1006 1007 func testWatchOverlapContextCancel(t *testing.T, f func(*integration.ClusterV3)) { 1008 defer testutil.AfterTest(t) 1009 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}) 1010 defer clus.Terminate(t) 1011 1012 n := 100 1013 ctxs, ctxc := make([]context.Context, 5), make([]chan struct{}, 5) 1014 for i := range ctxs { 1015 // make unique stream 1016 md := metadata.Pairs("some-key", fmt.Sprintf("%d", i)) 1017 ctxs[i] = metadata.NewOutgoingContext(context.Background(), md) 1018 // limits the maximum number of outstanding watchers per stream 1019 ctxc[i] = make(chan struct{}, 2) 1020 } 1021 1022 // issue concurrent watches on "abc" with cancel 1023 cli := clus.RandClient() 1024 if _, err := cli.Put(context.TODO(), "abc", "def"); err != nil { 1025 t.Fatal(err) 1026 } 1027 ch := make(chan struct{}, n) 1028 for i := 0; i < n; i++ { 1029 go func() { 1030 defer func() { ch <- struct{}{} }() 1031 idx := rand.Intn(len(ctxs)) 1032 ctx, cancel := context.WithCancel(ctxs[idx]) 1033 ctxc[idx] <- struct{}{} 1034 wch := cli.Watch(ctx, "abc", clientv3.WithRev(1)) 1035 f(clus) 1036 select { 1037 case _, ok := <-wch: 1038 if !ok { 1039 t.Fatalf("unexpected closed channel %p", wch) 1040 } 1041 // may take a second or two to reestablish a watcher because of 1042 // grpc back off policies for disconnects 1043 case <-time.After(5 * time.Second): 1044 t.Errorf("timed out waiting for watch on %p", wch) 1045 } 1046 // randomize how cancel overlaps with watch creation 1047 if rand.Intn(2) == 0 { 1048 <-ctxc[idx] 1049 cancel() 1050 } else { 1051 cancel() 1052 <-ctxc[idx] 1053 } 1054 }() 1055 } 1056 // join on watches 1057 for i := 0; i < n; i++ { 1058 select { 1059 case <-ch: 1060 case <-time.After(5 * time.Second): 1061 t.Fatalf("timed out waiting for completed watch") 1062 } 1063 } 1064 } 1065 1066 // TestWatchCancelAndCloseClient ensures that canceling a watcher then immediately 1067 // closing the client does not return a client closing error. 1068 func TestWatchCancelAndCloseClient(t *testing.T) { 1069 defer testutil.AfterTest(t) 1070 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}) 1071 defer clus.Terminate(t) 1072 cli := clus.Client(0) 1073 ctx, cancel := context.WithCancel(context.Background()) 1074 wch := cli.Watch(ctx, "abc") 1075 donec := make(chan struct{}) 1076 go func() { 1077 defer close(donec) 1078 select { 1079 case wr, ok := <-wch: 1080 if ok { 1081 t.Fatalf("expected closed watch after cancel(), got resp=%+v err=%v", wr, wr.Err()) 1082 } 1083 case <-time.After(5 * time.Second): 1084 t.Fatal("timed out waiting for closed channel") 1085 } 1086 }() 1087 cancel() 1088 if err := cli.Close(); err != nil { 1089 t.Fatal(err) 1090 } 1091 <-donec 1092 clus.TakeClient(0) 1093 } 1094 1095 // TestWatchStressResumeClose establishes a bunch of watchers, disconnects 1096 // to put them in resuming mode, cancels them so some resumes by cancel fail, 1097 // then closes the watcher interface to ensure correct clean up. 1098 func TestWatchStressResumeClose(t *testing.T) { 1099 defer testutil.AfterTest(t) 1100 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}) 1101 defer clus.Terminate(t) 1102 cli := clus.Client(0) 1103 1104 ctx, cancel := context.WithCancel(context.Background()) 1105 // add more watches than can be resumed before the cancel 1106 wchs := make([]clientv3.WatchChan, 2000) 1107 for i := range wchs { 1108 wchs[i] = cli.Watch(ctx, "abc") 1109 } 1110 clus.Members[0].DropConnections() 1111 cancel() 1112 if err := cli.Close(); err != nil { 1113 t.Fatal(err) 1114 } 1115 clus.TakeClient(0) 1116 } 1117 1118 // TestWatchCancelDisconnected ensures canceling a watcher works when 1119 // its grpc stream is disconnected / reconnecting. 1120 func TestWatchCancelDisconnected(t *testing.T) { 1121 defer testutil.AfterTest(t) 1122 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}) 1123 defer clus.Terminate(t) 1124 cli := clus.Client(0) 1125 ctx, cancel := context.WithCancel(context.Background()) 1126 // add more watches than can be resumed before the cancel 1127 wch := cli.Watch(ctx, "abc") 1128 clus.Members[0].Stop(t) 1129 cancel() 1130 select { 1131 case <-wch: 1132 case <-time.After(time.Second): 1133 t.Fatal("took too long to cancel disconnected watcher") 1134 } 1135 } 1136 1137 // TestWatchClose ensures that close does not return error 1138 func TestWatchClose(t *testing.T) { 1139 runWatchTest(t, testWatchClose) 1140 } 1141 1142 func testWatchClose(t *testing.T, wctx *watchctx) { 1143 ctx, cancel := context.WithCancel(context.Background()) 1144 wch := wctx.w.Watch(ctx, "a") 1145 cancel() 1146 if wch == nil { 1147 t.Fatalf("expected watcher channel, got nil") 1148 } 1149 if wctx.w.Close() != nil { 1150 t.Fatalf("watch did not close successfully") 1151 } 1152 wresp, ok := <-wch 1153 if ok { 1154 t.Fatalf("read wch got %v; expected closed channel", wresp) 1155 } 1156 }