github.com/lfch/etcd-io/tests/v3@v3.0.0-20221004140520-eac99acd3e9d/integration/v3_watch_test.go (about) 1 // Copyright 2016 The etcd Authors 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package integration 16 17 import ( 18 "bytes" 19 "context" 20 "fmt" 21 "reflect" 22 "sort" 23 "sync" 24 "testing" 25 "time" 26 27 pb "github.com/lfch/etcd-io/api/v3/etcdserverpb" 28 "github.com/lfch/etcd-io/api/v3/mvccpb" 29 clientv3 "github.com/lfch/etcd-io/client/v3" 30 "github.com/lfch/etcd-io/server/v3/etcdserver/api/v3rpc" 31 "github.com/lfch/etcd-io/tests/v3/framework/integration" 32 ) 33 34 // TestV3WatchFromCurrentRevision tests Watch APIs from current revision. 35 func TestV3WatchFromCurrentRevision(t *testing.T) { 36 integration.BeforeTest(t) 37 tests := []struct { 38 name string 39 40 putKeys []string 41 watchRequest *pb.WatchRequest 42 43 wresps []*pb.WatchResponse 44 }{ 45 { 46 "watch the key, matching", 47 []string{"foo"}, 48 &pb.WatchRequest{RequestUnion: &pb.WatchRequest_CreateRequest{ 49 CreateRequest: &pb.WatchCreateRequest{ 50 Key: []byte("foo")}}}, 51 52 []*pb.WatchResponse{ 53 { 54 Header: &pb.ResponseHeader{Revision: 2}, 55 Created: false, 56 Events: []*mvccpb.Event{ 57 { 58 Type: mvccpb.PUT, 59 Kv: &mvccpb.KeyValue{Key: []byte("foo"), Value: []byte("bar"), CreateRevision: 2, ModRevision: 2, Version: 1}, 60 }, 61 }, 62 }, 63 }, 64 }, 65 { 66 "watch the key, non-matching", 67 []string{"foo"}, 68 &pb.WatchRequest{RequestUnion: &pb.WatchRequest_CreateRequest{ 69 CreateRequest: &pb.WatchCreateRequest{ 70 Key: []byte("helloworld")}}}, 71 72 []*pb.WatchResponse{}, 73 }, 74 { 75 "watch the prefix, matching", 76 []string{"fooLong"}, 77 &pb.WatchRequest{RequestUnion: &pb.WatchRequest_CreateRequest{ 78 CreateRequest: &pb.WatchCreateRequest{ 79 Key: []byte("foo"), 80 RangeEnd: []byte("fop")}}}, 81 82 []*pb.WatchResponse{ 83 { 84 Header: &pb.ResponseHeader{Revision: 2}, 85 Created: false, 86 Events: []*mvccpb.Event{ 87 { 88 Type: mvccpb.PUT, 89 Kv: &mvccpb.KeyValue{Key: []byte("fooLong"), Value: []byte("bar"), CreateRevision: 2, ModRevision: 2, Version: 1}, 90 }, 91 }, 92 }, 93 }, 94 }, 95 { 96 "watch the prefix, non-matching", 97 []string{"foo"}, 98 &pb.WatchRequest{RequestUnion: &pb.WatchRequest_CreateRequest{ 99 CreateRequest: &pb.WatchCreateRequest{ 100 Key: []byte("helloworld"), 101 RangeEnd: []byte("helloworle")}}}, 102 103 []*pb.WatchResponse{}, 104 }, 105 { 106 "watch full range, matching", 107 []string{"fooLong"}, 108 &pb.WatchRequest{RequestUnion: &pb.WatchRequest_CreateRequest{ 109 CreateRequest: &pb.WatchCreateRequest{ 110 Key: []byte(""), 111 RangeEnd: []byte("\x00")}}}, 112 113 []*pb.WatchResponse{ 114 { 115 Header: &pb.ResponseHeader{Revision: 2}, 116 Created: false, 117 Events: []*mvccpb.Event{ 118 { 119 Type: mvccpb.PUT, 120 Kv: &mvccpb.KeyValue{Key: []byte("fooLong"), Value: []byte("bar"), CreateRevision: 2, ModRevision: 2, Version: 1}, 121 }, 122 }, 123 }, 124 }, 125 }, 126 { 127 "multiple puts, one watcher with matching key", 128 []string{"foo", "foo", "foo"}, 129 &pb.WatchRequest{RequestUnion: &pb.WatchRequest_CreateRequest{ 130 CreateRequest: &pb.WatchCreateRequest{ 131 Key: []byte("foo")}}}, 132 133 []*pb.WatchResponse{ 134 { 135 Header: &pb.ResponseHeader{Revision: 2}, 136 Created: false, 137 Events: []*mvccpb.Event{ 138 { 139 Type: mvccpb.PUT, 140 Kv: &mvccpb.KeyValue{Key: []byte("foo"), Value: []byte("bar"), CreateRevision: 2, ModRevision: 2, Version: 1}, 141 }, 142 }, 143 }, 144 { 145 Header: &pb.ResponseHeader{Revision: 3}, 146 Created: false, 147 Events: []*mvccpb.Event{ 148 { 149 Type: mvccpb.PUT, 150 Kv: &mvccpb.KeyValue{Key: []byte("foo"), Value: []byte("bar"), CreateRevision: 2, ModRevision: 3, Version: 2}, 151 }, 152 }, 153 }, 154 { 155 Header: &pb.ResponseHeader{Revision: 4}, 156 Created: false, 157 Events: []*mvccpb.Event{ 158 { 159 Type: mvccpb.PUT, 160 Kv: &mvccpb.KeyValue{Key: []byte("foo"), Value: []byte("bar"), CreateRevision: 2, ModRevision: 4, Version: 3}, 161 }, 162 }, 163 }, 164 }, 165 }, 166 { 167 "multiple puts, one watcher with matching perfix", 168 []string{"foo", "foo", "foo"}, 169 &pb.WatchRequest{RequestUnion: &pb.WatchRequest_CreateRequest{ 170 CreateRequest: &pb.WatchCreateRequest{ 171 Key: []byte("foo"), 172 RangeEnd: []byte("fop")}}}, 173 174 []*pb.WatchResponse{ 175 { 176 Header: &pb.ResponseHeader{Revision: 2}, 177 Created: false, 178 Events: []*mvccpb.Event{ 179 { 180 Type: mvccpb.PUT, 181 Kv: &mvccpb.KeyValue{Key: []byte("foo"), Value: []byte("bar"), CreateRevision: 2, ModRevision: 2, Version: 1}, 182 }, 183 }, 184 }, 185 { 186 Header: &pb.ResponseHeader{Revision: 3}, 187 Created: false, 188 Events: []*mvccpb.Event{ 189 { 190 Type: mvccpb.PUT, 191 Kv: &mvccpb.KeyValue{Key: []byte("foo"), Value: []byte("bar"), CreateRevision: 2, ModRevision: 3, Version: 2}, 192 }, 193 }, 194 }, 195 { 196 Header: &pb.ResponseHeader{Revision: 4}, 197 Created: false, 198 Events: []*mvccpb.Event{ 199 { 200 Type: mvccpb.PUT, 201 Kv: &mvccpb.KeyValue{Key: []byte("foo"), Value: []byte("bar"), CreateRevision: 2, ModRevision: 4, Version: 3}, 202 }, 203 }, 204 }, 205 }, 206 }, 207 } 208 209 for i, tt := range tests { 210 t.Run(tt.name, func(t *testing.T) { 211 clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) 212 defer clus.Terminate(t) 213 214 wAPI := integration.ToGRPC(clus.RandClient()).Watch 215 ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) 216 defer cancel() 217 wStream, err := wAPI.Watch(ctx) 218 if err != nil { 219 t.Fatalf("#%d: wAPI.Watch error: %v", i, err) 220 } 221 222 err = wStream.Send(tt.watchRequest) 223 if err != nil { 224 t.Fatalf("#%d: wStream.Send error: %v", i, err) 225 } 226 227 // ensure watcher request created a new watcher 228 cresp, err := wStream.Recv() 229 if err != nil { 230 t.Fatalf("#%d: wStream.Recv error: %v", i, err) 231 } 232 if !cresp.Created { 233 t.Fatalf("#%d: did not create watchid, got %+v", i, cresp) 234 } 235 if cresp.Canceled { 236 t.Fatalf("#%d: canceled watcher on create %+v", i, cresp) 237 } 238 239 createdWatchId := cresp.WatchId 240 if cresp.Header == nil || cresp.Header.Revision != 1 { 241 t.Fatalf("#%d: header revision got +%v, wanted revison 1", i, cresp) 242 } 243 244 // asynchronously create keys 245 ch := make(chan struct{}, 1) 246 go func() { 247 for _, k := range tt.putKeys { 248 kvc := integration.ToGRPC(clus.RandClient()).KV 249 req := &pb.PutRequest{Key: []byte(k), Value: []byte("bar")} 250 if _, err := kvc.Put(context.TODO(), req); err != nil { 251 t.Errorf("#%d: couldn't put key (%v)", i, err) 252 } 253 } 254 ch <- struct{}{} 255 }() 256 257 // check stream results 258 for j, wresp := range tt.wresps { 259 resp, err := wStream.Recv() 260 if err != nil { 261 t.Errorf("#%d.%d: wStream.Recv error: %v", i, j, err) 262 } 263 264 if resp.Header == nil { 265 t.Fatalf("#%d.%d: unexpected nil resp.Header", i, j) 266 } 267 if resp.Header.Revision != wresp.Header.Revision { 268 t.Errorf("#%d.%d: resp.Header.Revision got = %d, want = %d", i, j, resp.Header.Revision, wresp.Header.Revision) 269 } 270 271 if wresp.Created != resp.Created { 272 t.Errorf("#%d.%d: resp.Created got = %v, want = %v", i, j, resp.Created, wresp.Created) 273 } 274 if resp.WatchId != createdWatchId { 275 t.Errorf("#%d.%d: resp.WatchId got = %d, want = %d", i, j, resp.WatchId, createdWatchId) 276 } 277 278 if !reflect.DeepEqual(resp.Events, wresp.Events) { 279 t.Errorf("#%d.%d: resp.Events got = %+v, want = %+v", i, j, resp.Events, wresp.Events) 280 } 281 } 282 283 rok, nr := waitResponse(wStream, 1*time.Second) 284 if !rok { 285 t.Errorf("unexpected pb.WatchResponse is received %+v", nr) 286 } 287 288 // wait for the client to finish sending the keys before terminating the cluster 289 <-ch 290 }) 291 } 292 } 293 294 // TestV3WatchFutureRevision tests Watch APIs from a future revision. 295 func TestV3WatchFutureRevision(t *testing.T) { 296 integration.BeforeTest(t) 297 298 clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) 299 defer clus.Terminate(t) 300 301 wAPI := integration.ToGRPC(clus.RandClient()).Watch 302 ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) 303 defer cancel() 304 wStream, err := wAPI.Watch(ctx) 305 if err != nil { 306 t.Fatalf("wAPI.Watch error: %v", err) 307 } 308 309 wkey := []byte("foo") 310 wrev := int64(10) 311 req := &pb.WatchRequest{RequestUnion: &pb.WatchRequest_CreateRequest{ 312 CreateRequest: &pb.WatchCreateRequest{Key: wkey, StartRevision: wrev}}} 313 err = wStream.Send(req) 314 if err != nil { 315 t.Fatalf("wStream.Send error: %v", err) 316 } 317 318 // ensure watcher request created a new watcher 319 cresp, err := wStream.Recv() 320 if err != nil { 321 t.Fatalf("wStream.Recv error: %v", err) 322 } 323 if !cresp.Created { 324 t.Fatalf("create %v, want %v", cresp.Created, true) 325 } 326 327 kvc := integration.ToGRPC(clus.RandClient()).KV 328 for { 329 req := &pb.PutRequest{Key: wkey, Value: []byte("bar")} 330 resp, rerr := kvc.Put(context.TODO(), req) 331 if rerr != nil { 332 t.Fatalf("couldn't put key (%v)", rerr) 333 } 334 if resp.Header.Revision == wrev { 335 break 336 } 337 } 338 339 // ensure watcher request created a new watcher 340 cresp, err = wStream.Recv() 341 if err != nil { 342 t.Fatalf("wStream.Recv error: %v", err) 343 } 344 if cresp.Header.Revision != wrev { 345 t.Fatalf("revision = %d, want %d", cresp.Header.Revision, wrev) 346 } 347 if len(cresp.Events) != 1 { 348 t.Fatalf("failed to receive events") 349 } 350 if cresp.Events[0].Kv.ModRevision != wrev { 351 t.Errorf("mod revision = %d, want %d", cresp.Events[0].Kv.ModRevision, wrev) 352 } 353 } 354 355 // TestV3WatchWrongRange tests wrong range does not create watchers. 356 func TestV3WatchWrongRange(t *testing.T) { 357 integration.BeforeTest(t) 358 359 clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) 360 defer clus.Terminate(t) 361 362 wAPI := integration.ToGRPC(clus.RandClient()).Watch 363 ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) 364 defer cancel() 365 wStream, err := wAPI.Watch(ctx) 366 if err != nil { 367 t.Fatalf("wAPI.Watch error: %v", err) 368 } 369 370 tests := []struct { 371 key []byte 372 end []byte 373 canceled bool 374 }{ 375 {[]byte("a"), []byte("a"), true}, // wrong range end 376 {[]byte("b"), []byte("a"), true}, // wrong range end 377 {[]byte("foo"), []byte{0}, false}, // watch request with 'WithFromKey' 378 } 379 for i, tt := range tests { 380 if err := wStream.Send(&pb.WatchRequest{RequestUnion: &pb.WatchRequest_CreateRequest{ 381 CreateRequest: &pb.WatchCreateRequest{Key: tt.key, RangeEnd: tt.end, StartRevision: 1}}}); err != nil { 382 t.Fatalf("#%d: wStream.Send error: %v", i, err) 383 } 384 cresp, err := wStream.Recv() 385 if err != nil { 386 t.Fatalf("#%d: wStream.Recv error: %v", i, err) 387 } 388 if !cresp.Created { 389 t.Fatalf("#%d: create %v, want %v", i, cresp.Created, true) 390 } 391 if cresp.Canceled != tt.canceled { 392 t.Fatalf("#%d: canceled %v, want %v", i, tt.canceled, cresp.Canceled) 393 } 394 if tt.canceled && cresp.WatchId != clientv3.InvalidWatchID { 395 t.Fatalf("#%d: canceled watch ID %d, want %d", i, cresp.WatchId, clientv3.InvalidWatchID) 396 } 397 } 398 } 399 400 // TestV3WatchCancelSynced tests Watch APIs cancellation from synced map. 401 func TestV3WatchCancelSynced(t *testing.T) { 402 integration.BeforeTest(t) 403 testV3WatchCancel(t, 0) 404 } 405 406 // TestV3WatchCancelUnsynced tests Watch APIs cancellation from unsynced map. 407 func TestV3WatchCancelUnsynced(t *testing.T) { 408 integration.BeforeTest(t) 409 testV3WatchCancel(t, 1) 410 } 411 412 func testV3WatchCancel(t *testing.T, startRev int64) { 413 clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) 414 defer clus.Terminate(t) 415 416 ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) 417 defer cancel() 418 wStream, errW := integration.ToGRPC(clus.RandClient()).Watch.Watch(ctx) 419 if errW != nil { 420 t.Fatalf("wAPI.Watch error: %v", errW) 421 } 422 423 wreq := &pb.WatchRequest{RequestUnion: &pb.WatchRequest_CreateRequest{ 424 CreateRequest: &pb.WatchCreateRequest{ 425 Key: []byte("foo"), StartRevision: startRev}}} 426 if err := wStream.Send(wreq); err != nil { 427 t.Fatalf("wStream.Send error: %v", err) 428 } 429 430 wresp, errR := wStream.Recv() 431 if errR != nil { 432 t.Errorf("wStream.Recv error: %v", errR) 433 } 434 if !wresp.Created { 435 t.Errorf("wresp.Created got = %v, want = true", wresp.Created) 436 } 437 438 creq := &pb.WatchRequest{RequestUnion: &pb.WatchRequest_CancelRequest{ 439 CancelRequest: &pb.WatchCancelRequest{ 440 WatchId: wresp.WatchId}}} 441 if err := wStream.Send(creq); err != nil { 442 t.Fatalf("wStream.Send error: %v", err) 443 } 444 445 cresp, err := wStream.Recv() 446 if err != nil { 447 t.Errorf("wStream.Recv error: %v", err) 448 } 449 if !cresp.Canceled { 450 t.Errorf("cresp.Canceled got = %v, want = true", cresp.Canceled) 451 } 452 453 kvc := integration.ToGRPC(clus.RandClient()).KV 454 if _, err := kvc.Put(context.TODO(), &pb.PutRequest{Key: []byte("foo"), Value: []byte("bar")}); err != nil { 455 t.Errorf("couldn't put key (%v)", err) 456 } 457 458 // watch got canceled, so this should block 459 rok, nr := waitResponse(wStream, 1*time.Second) 460 if !rok { 461 t.Errorf("unexpected pb.WatchResponse is received %+v", nr) 462 } 463 } 464 465 // TestV3WatchCurrentPutOverlap ensures current watchers receive all events with 466 // overlapping puts. 467 func TestV3WatchCurrentPutOverlap(t *testing.T) { 468 integration.BeforeTest(t) 469 clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) 470 defer clus.Terminate(t) 471 472 ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) 473 defer cancel() 474 wStream, wErr := integration.ToGRPC(clus.RandClient()).Watch.Watch(ctx) 475 if wErr != nil { 476 t.Fatalf("wAPI.Watch error: %v", wErr) 477 } 478 479 // last mod_revision that will be observed 480 nrRevisions := 32 481 // first revision already allocated as empty revision 482 var wg sync.WaitGroup 483 for i := 1; i < nrRevisions; i++ { 484 wg.Add(1) 485 go func() { 486 defer wg.Done() 487 kvc := integration.ToGRPC(clus.RandClient()).KV 488 req := &pb.PutRequest{Key: []byte("foo"), Value: []byte("bar")} 489 if _, err := kvc.Put(context.TODO(), req); err != nil { 490 t.Errorf("couldn't put key (%v)", err) 491 } 492 }() 493 } 494 495 // maps watcher to current expected revision 496 progress := make(map[int64]int64) 497 498 wreq := &pb.WatchRequest{RequestUnion: &pb.WatchRequest_CreateRequest{ 499 CreateRequest: &pb.WatchCreateRequest{Key: []byte("foo"), RangeEnd: []byte("fop")}}} 500 if err := wStream.Send(wreq); err != nil { 501 t.Fatalf("first watch request failed (%v)", err) 502 } 503 504 more := true 505 progress[-1] = 0 // watcher creation pending 506 for more { 507 resp, err := wStream.Recv() 508 if err != nil { 509 t.Fatalf("wStream.Recv error: %v", err) 510 } 511 512 if resp.Created { 513 // accept events > header revision 514 progress[resp.WatchId] = resp.Header.Revision + 1 515 if resp.Header.Revision == int64(nrRevisions) { 516 // covered all revisions; create no more watchers 517 progress[-1] = int64(nrRevisions) + 1 518 } else if err := wStream.Send(wreq); err != nil { 519 t.Fatalf("watch request failed (%v)", err) 520 } 521 } else if len(resp.Events) == 0 { 522 t.Fatalf("got events %v, want non-empty", resp.Events) 523 } else { 524 wRev, ok := progress[resp.WatchId] 525 if !ok { 526 t.Fatalf("got %+v, but watch id shouldn't exist ", resp) 527 } 528 if resp.Events[0].Kv.ModRevision != wRev { 529 t.Fatalf("got %+v, wanted first revision %d", resp, wRev) 530 } 531 lastRev := resp.Events[len(resp.Events)-1].Kv.ModRevision 532 progress[resp.WatchId] = lastRev + 1 533 } 534 more = false 535 for _, v := range progress { 536 if v <= int64(nrRevisions) { 537 more = true 538 break 539 } 540 } 541 } 542 543 if rok, nr := waitResponse(wStream, time.Second); !rok { 544 t.Errorf("unexpected pb.WatchResponse is received %+v", nr) 545 } 546 547 wg.Wait() 548 } 549 550 // TestV3WatchEmptyKey ensures synced watchers see empty key PUTs as PUT events 551 func TestV3WatchEmptyKey(t *testing.T) { 552 integration.BeforeTest(t) 553 554 clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) 555 defer clus.Terminate(t) 556 557 ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) 558 defer cancel() 559 560 ws, werr := integration.ToGRPC(clus.RandClient()).Watch.Watch(ctx) 561 if werr != nil { 562 t.Fatal(werr) 563 } 564 req := &pb.WatchRequest{RequestUnion: &pb.WatchRequest_CreateRequest{ 565 CreateRequest: &pb.WatchCreateRequest{ 566 Key: []byte("foo")}}} 567 if err := ws.Send(req); err != nil { 568 t.Fatal(err) 569 } 570 if _, err := ws.Recv(); err != nil { 571 t.Fatal(err) 572 } 573 574 // put a key with empty value 575 kvc := integration.ToGRPC(clus.RandClient()).KV 576 preq := &pb.PutRequest{Key: []byte("foo")} 577 if _, err := kvc.Put(context.TODO(), preq); err != nil { 578 t.Fatal(err) 579 } 580 581 // check received PUT 582 resp, rerr := ws.Recv() 583 if rerr != nil { 584 t.Fatal(rerr) 585 } 586 wevs := []*mvccpb.Event{ 587 { 588 Type: mvccpb.PUT, 589 Kv: &mvccpb.KeyValue{Key: []byte("foo"), CreateRevision: 2, ModRevision: 2, Version: 1}, 590 }, 591 } 592 if !reflect.DeepEqual(resp.Events, wevs) { 593 t.Fatalf("got %v, expected %v", resp.Events, wevs) 594 } 595 } 596 597 func TestV3WatchMultipleWatchersSynced(t *testing.T) { 598 integration.BeforeTest(t) 599 testV3WatchMultipleWatchers(t, 0) 600 } 601 602 func TestV3WatchMultipleWatchersUnsynced(t *testing.T) { 603 integration.BeforeTest(t) 604 testV3WatchMultipleWatchers(t, 1) 605 } 606 607 // testV3WatchMultipleWatchers tests multiple watchers on the same key 608 // and one watcher with matching prefix. It first puts the key 609 // that matches all watchers, and another key that matches only 610 // one watcher to test if it receives expected events. 611 func testV3WatchMultipleWatchers(t *testing.T, startRev int64) { 612 clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) 613 defer clus.Terminate(t) 614 615 kvc := integration.ToGRPC(clus.RandClient()).KV 616 617 ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) 618 defer cancel() 619 wStream, errW := integration.ToGRPC(clus.RandClient()).Watch.Watch(ctx) 620 if errW != nil { 621 t.Fatalf("wAPI.Watch error: %v", errW) 622 } 623 624 watchKeyN := 4 625 for i := 0; i < watchKeyN+1; i++ { 626 var wreq *pb.WatchRequest 627 if i < watchKeyN { 628 wreq = &pb.WatchRequest{RequestUnion: &pb.WatchRequest_CreateRequest{ 629 CreateRequest: &pb.WatchCreateRequest{ 630 Key: []byte("foo"), StartRevision: startRev}}} 631 } else { 632 wreq = &pb.WatchRequest{RequestUnion: &pb.WatchRequest_CreateRequest{ 633 CreateRequest: &pb.WatchCreateRequest{ 634 Key: []byte("fo"), RangeEnd: []byte("fp"), StartRevision: startRev}}} 635 } 636 if err := wStream.Send(wreq); err != nil { 637 t.Fatalf("wStream.Send error: %v", err) 638 } 639 } 640 641 ids := make(map[int64]struct{}) 642 for i := 0; i < watchKeyN+1; i++ { 643 wresp, err := wStream.Recv() 644 if err != nil { 645 t.Fatalf("wStream.Recv error: %v", err) 646 } 647 if !wresp.Created { 648 t.Fatalf("wresp.Created got = %v, want = true", wresp.Created) 649 } 650 ids[wresp.WatchId] = struct{}{} 651 } 652 653 if _, err := kvc.Put(context.TODO(), &pb.PutRequest{Key: []byte("foo"), Value: []byte("bar")}); err != nil { 654 t.Fatalf("couldn't put key (%v)", err) 655 } 656 657 for i := 0; i < watchKeyN+1; i++ { 658 wresp, err := wStream.Recv() 659 if err != nil { 660 t.Fatalf("wStream.Recv error: %v", err) 661 } 662 if _, ok := ids[wresp.WatchId]; !ok { 663 t.Errorf("watchId %d is not created!", wresp.WatchId) 664 } else { 665 delete(ids, wresp.WatchId) 666 } 667 if len(wresp.Events) == 0 { 668 t.Errorf("#%d: no events received", i) 669 } 670 for _, ev := range wresp.Events { 671 if string(ev.Kv.Key) != "foo" { 672 t.Errorf("ev.Kv.Key got = %s, want = foo", ev.Kv.Key) 673 } 674 if string(ev.Kv.Value) != "bar" { 675 t.Errorf("ev.Kv.Value got = %s, want = bar", ev.Kv.Value) 676 } 677 } 678 } 679 680 // now put one key that has only one matching watcher 681 if _, err := kvc.Put(context.TODO(), &pb.PutRequest{Key: []byte("fo"), Value: []byte("bar")}); err != nil { 682 t.Fatalf("couldn't put key (%v)", err) 683 } 684 wresp, err := wStream.Recv() 685 if err != nil { 686 t.Errorf("wStream.Recv error: %v", err) 687 } 688 if len(wresp.Events) != 1 { 689 t.Fatalf("len(wresp.Events) got = %d, want = 1", len(wresp.Events)) 690 } 691 if string(wresp.Events[0].Kv.Key) != "fo" { 692 t.Errorf("wresp.Events[0].Kv.Key got = %s, want = fo", wresp.Events[0].Kv.Key) 693 } 694 695 // now Recv should block because there is no more events coming 696 rok, nr := waitResponse(wStream, 1*time.Second) 697 if !rok { 698 t.Errorf("unexpected pb.WatchResponse is received %+v", nr) 699 } 700 } 701 702 func TestV3WatchMultipleEventsTxnSynced(t *testing.T) { 703 integration.BeforeTest(t) 704 testV3WatchMultipleEventsTxn(t, 0) 705 } 706 707 func TestV3WatchMultipleEventsTxnUnsynced(t *testing.T) { 708 integration.BeforeTest(t) 709 testV3WatchMultipleEventsTxn(t, 1) 710 } 711 712 // testV3WatchMultipleEventsTxn tests Watch APIs when it receives multiple events. 713 func testV3WatchMultipleEventsTxn(t *testing.T, startRev int64) { 714 clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) 715 defer clus.Terminate(t) 716 717 ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) 718 defer cancel() 719 wStream, wErr := integration.ToGRPC(clus.RandClient()).Watch.Watch(ctx) 720 if wErr != nil { 721 t.Fatalf("wAPI.Watch error: %v", wErr) 722 } 723 724 wreq := &pb.WatchRequest{RequestUnion: &pb.WatchRequest_CreateRequest{ 725 CreateRequest: &pb.WatchCreateRequest{ 726 Key: []byte("foo"), RangeEnd: []byte("fop"), StartRevision: startRev}}} 727 if err := wStream.Send(wreq); err != nil { 728 t.Fatalf("wStream.Send error: %v", err) 729 } 730 if resp, err := wStream.Recv(); err != nil || !resp.Created { 731 t.Fatalf("create response failed: resp=%v, err=%v", resp, err) 732 } 733 734 kvc := integration.ToGRPC(clus.RandClient()).KV 735 txn := pb.TxnRequest{} 736 for i := 0; i < 3; i++ { 737 ru := &pb.RequestOp{} 738 ru.Request = &pb.RequestOp_RequestPut{ 739 RequestPut: &pb.PutRequest{ 740 Key: []byte(fmt.Sprintf("foo%d", i)), Value: []byte("bar")}} 741 txn.Success = append(txn.Success, ru) 742 } 743 744 tresp, err := kvc.Txn(context.Background(), &txn) 745 if err != nil { 746 t.Fatalf("kvc.Txn error: %v", err) 747 } 748 if !tresp.Succeeded { 749 t.Fatalf("kvc.Txn failed: %+v", tresp) 750 } 751 752 var events []*mvccpb.Event 753 for len(events) < 3 { 754 resp, err := wStream.Recv() 755 if err != nil { 756 t.Errorf("wStream.Recv error: %v", err) 757 } 758 events = append(events, resp.Events...) 759 } 760 sort.Sort(eventsSortByKey(events)) 761 762 wevents := []*mvccpb.Event{ 763 { 764 Type: mvccpb.PUT, 765 Kv: &mvccpb.KeyValue{Key: []byte("foo0"), Value: []byte("bar"), CreateRevision: 2, ModRevision: 2, Version: 1}, 766 }, 767 { 768 Type: mvccpb.PUT, 769 Kv: &mvccpb.KeyValue{Key: []byte("foo1"), Value: []byte("bar"), CreateRevision: 2, ModRevision: 2, Version: 1}, 770 }, 771 { 772 Type: mvccpb.PUT, 773 Kv: &mvccpb.KeyValue{Key: []byte("foo2"), Value: []byte("bar"), CreateRevision: 2, ModRevision: 2, Version: 1}, 774 }, 775 } 776 777 if !reflect.DeepEqual(events, wevents) { 778 t.Errorf("events got = %+v, want = %+v", events, wevents) 779 } 780 781 rok, nr := waitResponse(wStream, 1*time.Second) 782 if !rok { 783 t.Errorf("unexpected pb.WatchResponse is received %+v", nr) 784 } 785 } 786 787 type eventsSortByKey []*mvccpb.Event 788 789 func (evs eventsSortByKey) Len() int { return len(evs) } 790 func (evs eventsSortByKey) Swap(i, j int) { evs[i], evs[j] = evs[j], evs[i] } 791 func (evs eventsSortByKey) Less(i, j int) bool { 792 return bytes.Compare(evs[i].Kv.Key, evs[j].Kv.Key) < 0 793 } 794 795 func TestV3WatchMultipleEventsPutUnsynced(t *testing.T) { 796 integration.BeforeTest(t) 797 clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) 798 defer clus.Terminate(t) 799 800 kvc := integration.ToGRPC(clus.RandClient()).KV 801 802 if _, err := kvc.Put(context.TODO(), &pb.PutRequest{Key: []byte("foo0"), Value: []byte("bar")}); err != nil { 803 t.Fatalf("couldn't put key (%v)", err) 804 } 805 if _, err := kvc.Put(context.TODO(), &pb.PutRequest{Key: []byte("foo1"), Value: []byte("bar")}); err != nil { 806 t.Fatalf("couldn't put key (%v)", err) 807 } 808 809 ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) 810 defer cancel() 811 wStream, wErr := integration.ToGRPC(clus.RandClient()).Watch.Watch(ctx) 812 if wErr != nil { 813 t.Fatalf("wAPI.Watch error: %v", wErr) 814 } 815 816 wreq := &pb.WatchRequest{RequestUnion: &pb.WatchRequest_CreateRequest{ 817 CreateRequest: &pb.WatchCreateRequest{ 818 Key: []byte("foo"), RangeEnd: []byte("fop"), StartRevision: 1}}} 819 if err := wStream.Send(wreq); err != nil { 820 t.Fatalf("wStream.Send error: %v", err) 821 } 822 823 if _, err := kvc.Put(context.TODO(), &pb.PutRequest{Key: []byte("foo0"), Value: []byte("bar")}); err != nil { 824 t.Fatalf("couldn't put key (%v)", err) 825 } 826 if _, err := kvc.Put(context.TODO(), &pb.PutRequest{Key: []byte("foo1"), Value: []byte("bar")}); err != nil { 827 t.Fatalf("couldn't put key (%v)", err) 828 } 829 830 allWevents := []*mvccpb.Event{ 831 { 832 Type: mvccpb.PUT, 833 Kv: &mvccpb.KeyValue{Key: []byte("foo0"), Value: []byte("bar"), CreateRevision: 2, ModRevision: 2, Version: 1}, 834 }, 835 { 836 Type: mvccpb.PUT, 837 Kv: &mvccpb.KeyValue{Key: []byte("foo1"), Value: []byte("bar"), CreateRevision: 3, ModRevision: 3, Version: 1}, 838 }, 839 { 840 Type: mvccpb.PUT, 841 Kv: &mvccpb.KeyValue{Key: []byte("foo0"), Value: []byte("bar"), CreateRevision: 2, ModRevision: 4, Version: 2}, 842 }, 843 { 844 Type: mvccpb.PUT, 845 Kv: &mvccpb.KeyValue{Key: []byte("foo1"), Value: []byte("bar"), CreateRevision: 3, ModRevision: 5, Version: 2}, 846 }, 847 } 848 849 var events []*mvccpb.Event 850 for len(events) < 4 { 851 resp, err := wStream.Recv() 852 if err != nil { 853 t.Errorf("wStream.Recv error: %v", err) 854 } 855 if resp.Created { 856 continue 857 } 858 events = append(events, resp.Events...) 859 // if PUT requests are committed by now, first receive would return 860 // multiple events, but if not, it returns a single event. In SSD, 861 // it should return 4 events at once. 862 } 863 864 if !reflect.DeepEqual(events, allWevents) { 865 t.Errorf("events got = %+v, want = %+v", events, allWevents) 866 } 867 868 rok, nr := waitResponse(wStream, 1*time.Second) 869 if !rok { 870 t.Errorf("unexpected pb.WatchResponse is received %+v", nr) 871 } 872 } 873 874 func TestV3WatchMultipleStreamsSynced(t *testing.T) { 875 integration.BeforeTest(t) 876 testV3WatchMultipleStreams(t, 0) 877 } 878 879 func TestV3WatchMultipleStreamsUnsynced(t *testing.T) { 880 integration.BeforeTest(t) 881 testV3WatchMultipleStreams(t, 1) 882 } 883 884 // testV3WatchMultipleStreams tests multiple watchers on the same key on multiple streams. 885 func testV3WatchMultipleStreams(t *testing.T, startRev int64) { 886 clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) 887 defer clus.Terminate(t) 888 889 wAPI := integration.ToGRPC(clus.RandClient()).Watch 890 kvc := integration.ToGRPC(clus.RandClient()).KV 891 892 streams := make([]pb.Watch_WatchClient, 5) 893 for i := range streams { 894 ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) 895 defer cancel() 896 wStream, errW := wAPI.Watch(ctx) 897 if errW != nil { 898 t.Fatalf("wAPI.Watch error: %v", errW) 899 } 900 wreq := &pb.WatchRequest{RequestUnion: &pb.WatchRequest_CreateRequest{ 901 CreateRequest: &pb.WatchCreateRequest{ 902 Key: []byte("foo"), StartRevision: startRev}}} 903 if err := wStream.Send(wreq); err != nil { 904 t.Fatalf("wStream.Send error: %v", err) 905 } 906 streams[i] = wStream 907 } 908 909 for _, wStream := range streams { 910 wresp, err := wStream.Recv() 911 if err != nil { 912 t.Fatalf("wStream.Recv error: %v", err) 913 } 914 if !wresp.Created { 915 t.Fatalf("wresp.Created got = %v, want = true", wresp.Created) 916 } 917 } 918 919 if _, err := kvc.Put(context.TODO(), &pb.PutRequest{Key: []byte("foo"), Value: []byte("bar")}); err != nil { 920 t.Fatalf("couldn't put key (%v)", err) 921 } 922 923 var wg sync.WaitGroup 924 wg.Add(len(streams)) 925 wevents := []*mvccpb.Event{ 926 { 927 Type: mvccpb.PUT, 928 Kv: &mvccpb.KeyValue{Key: []byte("foo"), Value: []byte("bar"), CreateRevision: 2, ModRevision: 2, Version: 1}, 929 }, 930 } 931 for i := range streams { 932 go func(i int) { 933 defer wg.Done() 934 wStream := streams[i] 935 wresp, err := wStream.Recv() 936 if err != nil { 937 t.Errorf("wStream.Recv error: %v", err) 938 } 939 if wresp.WatchId != 0 { 940 t.Errorf("watchId got = %d, want = 0", wresp.WatchId) 941 } 942 if !reflect.DeepEqual(wresp.Events, wevents) { 943 t.Errorf("wresp.Events got = %+v, want = %+v", wresp.Events, wevents) 944 } 945 // now Recv should block because there is no more events coming 946 rok, nr := waitResponse(wStream, 1*time.Second) 947 if !rok { 948 t.Errorf("unexpected pb.WatchResponse is received %+v", nr) 949 } 950 }(i) 951 } 952 wg.Wait() 953 } 954 955 // waitResponse waits on the given stream for given duration. 956 // If there is no more events, true and a nil response will be 957 // returned closing the WatchClient stream. Or the response will 958 // be returned. 959 func waitResponse(wc pb.Watch_WatchClient, timeout time.Duration) (bool, *pb.WatchResponse) { 960 rCh := make(chan *pb.WatchResponse, 1) 961 donec := make(chan struct{}) 962 defer close(donec) 963 go func() { 964 resp, _ := wc.Recv() 965 select { 966 case rCh <- resp: 967 case <-donec: 968 } 969 }() 970 select { 971 case nr := <-rCh: 972 return false, nr 973 case <-time.After(timeout): 974 } 975 // didn't get response 976 wc.CloseSend() 977 return true, nil 978 } 979 980 func TestWatchWithProgressNotify(t *testing.T) { 981 // accelerate report interval so test terminates quickly 982 oldpi := v3rpc.GetProgressReportInterval() 983 // using atomics to avoid race warnings 984 v3rpc.SetProgressReportInterval(3 * time.Second) 985 testInterval := 3 * time.Second 986 defer func() { v3rpc.SetProgressReportInterval(oldpi) }() 987 988 integration.BeforeTest(t) 989 clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) 990 defer clus.Terminate(t) 991 992 ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) 993 defer cancel() 994 wStream, wErr := integration.ToGRPC(clus.RandClient()).Watch.Watch(ctx) 995 if wErr != nil { 996 t.Fatalf("wAPI.Watch error: %v", wErr) 997 } 998 999 // create two watchers, one with progressNotify set. 1000 wreq := &pb.WatchRequest{RequestUnion: &pb.WatchRequest_CreateRequest{ 1001 CreateRequest: &pb.WatchCreateRequest{Key: []byte("foo"), StartRevision: 1, ProgressNotify: true}}} 1002 if err := wStream.Send(wreq); err != nil { 1003 t.Fatalf("watch request failed (%v)", err) 1004 } 1005 wreq = &pb.WatchRequest{RequestUnion: &pb.WatchRequest_CreateRequest{ 1006 CreateRequest: &pb.WatchCreateRequest{Key: []byte("foo"), StartRevision: 1}}} 1007 if err := wStream.Send(wreq); err != nil { 1008 t.Fatalf("watch request failed (%v)", err) 1009 } 1010 1011 // two creation + one notification 1012 for i := 0; i < 3; i++ { 1013 rok, resp := waitResponse(wStream, testInterval+time.Second) 1014 if resp.Created { 1015 continue 1016 } 1017 1018 if rok { 1019 t.Errorf("failed to receive response from watch stream") 1020 } 1021 if resp.Header.Revision != 1 { 1022 t.Errorf("revision = %d, want 1", resp.Header.Revision) 1023 } 1024 if len(resp.Events) != 0 { 1025 t.Errorf("len(resp.Events) = %d, want 0", len(resp.Events)) 1026 } 1027 } 1028 1029 // no more notification 1030 rok, resp := waitResponse(wStream, time.Second) 1031 if !rok { 1032 t.Errorf("unexpected pb.WatchResponse is received %+v", resp) 1033 } 1034 } 1035 1036 // TestV3WatcMultiOpenhClose opens many watchers concurrently on multiple streams. 1037 func TestV3WatchClose(t *testing.T) { 1038 integration.BeforeTest(t) 1039 clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1, UseBridge: true}) 1040 defer clus.Terminate(t) 1041 1042 c := clus.Client(0) 1043 wapi := integration.ToGRPC(c).Watch 1044 1045 var wg sync.WaitGroup 1046 wg.Add(100) 1047 for i := 0; i < 100; i++ { 1048 go func() { 1049 ctx, cancel := context.WithCancel(context.TODO()) 1050 defer func() { 1051 wg.Done() 1052 cancel() 1053 }() 1054 ws, err := wapi.Watch(ctx) 1055 if err != nil { 1056 return 1057 } 1058 cr := &pb.WatchCreateRequest{Key: []byte("a")} 1059 req := &pb.WatchRequest{ 1060 RequestUnion: &pb.WatchRequest_CreateRequest{ 1061 CreateRequest: cr}} 1062 ws.Send(req) 1063 ws.Recv() 1064 }() 1065 } 1066 1067 clus.Members[0].Bridge().DropConnections() 1068 wg.Wait() 1069 } 1070 1071 // TestV3WatchWithFilter ensures watcher filters out the events correctly. 1072 func TestV3WatchWithFilter(t *testing.T) { 1073 integration.BeforeTest(t) 1074 1075 clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) 1076 defer clus.Terminate(t) 1077 1078 ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) 1079 defer cancel() 1080 1081 ws, werr := integration.ToGRPC(clus.RandClient()).Watch.Watch(ctx) 1082 if werr != nil { 1083 t.Fatal(werr) 1084 } 1085 req := &pb.WatchRequest{RequestUnion: &pb.WatchRequest_CreateRequest{ 1086 CreateRequest: &pb.WatchCreateRequest{ 1087 Key: []byte("foo"), 1088 Filters: []pb.WatchCreateRequest_FilterType{pb.WatchCreateRequest_NOPUT}, 1089 }}} 1090 if err := ws.Send(req); err != nil { 1091 t.Fatal(err) 1092 } 1093 if _, err := ws.Recv(); err != nil { 1094 t.Fatal(err) 1095 } 1096 1097 recv := make(chan *pb.WatchResponse, 1) 1098 go func() { 1099 // check received PUT 1100 resp, rerr := ws.Recv() 1101 if rerr != nil { 1102 t.Error(rerr) 1103 } 1104 recv <- resp 1105 }() 1106 1107 // put a key with empty value 1108 kvc := integration.ToGRPC(clus.RandClient()).KV 1109 preq := &pb.PutRequest{Key: []byte("foo")} 1110 if _, err := kvc.Put(context.TODO(), preq); err != nil { 1111 t.Fatal(err) 1112 } 1113 1114 select { 1115 case <-recv: 1116 t.Fatal("failed to filter out put event") 1117 case <-time.After(100 * time.Millisecond): 1118 } 1119 1120 dreq := &pb.DeleteRangeRequest{Key: []byte("foo")} 1121 if _, err := kvc.DeleteRange(context.TODO(), dreq); err != nil { 1122 t.Fatal(err) 1123 } 1124 1125 select { 1126 case resp := <-recv: 1127 wevs := []*mvccpb.Event{ 1128 { 1129 Type: mvccpb.DELETE, 1130 Kv: &mvccpb.KeyValue{Key: []byte("foo"), ModRevision: 3}, 1131 }, 1132 } 1133 if !reflect.DeepEqual(resp.Events, wevs) { 1134 t.Fatalf("got %v, expected %v", resp.Events, wevs) 1135 } 1136 case <-time.After(100 * time.Millisecond): 1137 t.Fatal("failed to receive delete event") 1138 } 1139 } 1140 1141 func TestV3WatchWithPrevKV(t *testing.T) { 1142 integration.BeforeTest(t) 1143 clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) 1144 defer clus.Terminate(t) 1145 1146 wctx, wcancel := context.WithCancel(context.Background()) 1147 defer wcancel() 1148 1149 tests := []struct { 1150 key string 1151 end string 1152 vals []string 1153 }{{ 1154 key: "foo", 1155 end: "fop", 1156 vals: []string{"bar1", "bar2"}, 1157 }, { 1158 key: "/abc", 1159 end: "/abd", 1160 vals: []string{"first", "second"}, 1161 }} 1162 for i, tt := range tests { 1163 kvc := integration.ToGRPC(clus.RandClient()).KV 1164 if _, err := kvc.Put(context.TODO(), &pb.PutRequest{Key: []byte(tt.key), Value: []byte(tt.vals[0])}); err != nil { 1165 t.Fatal(err) 1166 } 1167 1168 ws, werr := integration.ToGRPC(clus.RandClient()).Watch.Watch(wctx) 1169 if werr != nil { 1170 t.Fatal(werr) 1171 } 1172 1173 req := &pb.WatchRequest{RequestUnion: &pb.WatchRequest_CreateRequest{ 1174 CreateRequest: &pb.WatchCreateRequest{ 1175 Key: []byte(tt.key), 1176 RangeEnd: []byte(tt.end), 1177 PrevKv: true, 1178 }}} 1179 if err := ws.Send(req); err != nil { 1180 t.Fatal(err) 1181 } 1182 if _, err := ws.Recv(); err != nil { 1183 t.Fatal(err) 1184 } 1185 1186 if _, err := kvc.Put(context.TODO(), &pb.PutRequest{Key: []byte(tt.key), Value: []byte(tt.vals[1])}); err != nil { 1187 t.Fatal(err) 1188 } 1189 1190 recv := make(chan *pb.WatchResponse, 1) 1191 go func() { 1192 // check received PUT 1193 resp, rerr := ws.Recv() 1194 if rerr != nil { 1195 t.Error(rerr) 1196 } 1197 recv <- resp 1198 }() 1199 1200 select { 1201 case resp := <-recv: 1202 if tt.vals[1] != string(resp.Events[0].Kv.Value) { 1203 t.Errorf("#%d: unequal value: want=%s, get=%s", i, tt.vals[1], resp.Events[0].Kv.Value) 1204 } 1205 if tt.vals[0] != string(resp.Events[0].PrevKv.Value) { 1206 t.Errorf("#%d: unequal value: want=%s, get=%s", i, tt.vals[0], resp.Events[0].PrevKv.Value) 1207 } 1208 case <-time.After(30 * time.Second): 1209 t.Error("timeout waiting for watch response") 1210 } 1211 } 1212 } 1213 1214 // TestV3WatchCancellation ensures that watch cancellation frees up server resources. 1215 func TestV3WatchCancellation(t *testing.T) { 1216 integration.BeforeTest(t) 1217 1218 clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) 1219 defer clus.Terminate(t) 1220 1221 ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) 1222 defer cancel() 1223 1224 cli := clus.RandClient() 1225 1226 // increment watcher total count and keep a stream open 1227 cli.Watch(ctx, "/foo") 1228 1229 for i := 0; i < 1000; i++ { 1230 ctx, cancel := context.WithCancel(ctx) 1231 cli.Watch(ctx, "/foo") 1232 cancel() 1233 } 1234 1235 // Wait a little for cancellations to take hold 1236 time.Sleep(3 * time.Second) 1237 1238 minWatches, err := clus.Members[0].Metric("etcd_debugging_mvcc_watcher_total") 1239 if err != nil { 1240 t.Fatal(err) 1241 } 1242 1243 var expected string 1244 if integration.ThroughProxy { 1245 // grpc proxy has additional 2 watches open 1246 expected = "3" 1247 } else { 1248 expected = "1" 1249 } 1250 1251 if minWatches != expected { 1252 t.Fatalf("expected %s watch, got %s", expected, minWatches) 1253 } 1254 } 1255 1256 // TestV3WatchCloseCancelRace ensures that watch close doesn't decrement the watcher total too far. 1257 func TestV3WatchCloseCancelRace(t *testing.T) { 1258 integration.BeforeTest(t) 1259 1260 clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) 1261 defer clus.Terminate(t) 1262 1263 ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) 1264 defer cancel() 1265 1266 cli := clus.RandClient() 1267 1268 for i := 0; i < 1000; i++ { 1269 ctx, cancel := context.WithCancel(ctx) 1270 cli.Watch(ctx, "/foo") 1271 cancel() 1272 } 1273 1274 // Wait a little for cancellations to take hold 1275 time.Sleep(3 * time.Second) 1276 1277 minWatches, err := clus.Members[0].Metric("etcd_debugging_mvcc_watcher_total") 1278 if err != nil { 1279 t.Fatal(err) 1280 } 1281 1282 var expected string 1283 if integration.ThroughProxy { 1284 // grpc proxy has additional 2 watches open 1285 expected = "2" 1286 } else { 1287 expected = "0" 1288 } 1289 1290 if minWatches != expected { 1291 t.Fatalf("expected %s watch, got %s", expected, minWatches) 1292 } 1293 }