go.etcd.io/etcd@v3.3.27+incompatible/clientv3/integration/watch_test.go (about)

     1  // Copyright 2016 The etcd Authors
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package integration
    16  
    17  import (
    18  	"context"
    19  	"fmt"
    20  	"math/rand"
    21  	"reflect"
    22  	"sort"
    23  	"testing"
    24  	"time"
    25  
    26  	"github.com/coreos/etcd/clientv3"
    27  	"github.com/coreos/etcd/etcdserver/api/v3rpc"
    28  	"github.com/coreos/etcd/etcdserver/api/v3rpc/rpctypes"
    29  	"github.com/coreos/etcd/integration"
    30  	mvccpb "github.com/coreos/etcd/mvcc/mvccpb"
    31  	"github.com/coreos/etcd/pkg/testutil"
    32  
    33  	"google.golang.org/grpc/metadata"
    34  )
    35  
    36  type watcherTest func(*testing.T, *watchctx)
    37  
    38  type watchctx struct {
    39  	clus          *integration.ClusterV3
    40  	w             clientv3.Watcher
    41  	kv            clientv3.KV
    42  	wclientMember int
    43  	kvMember      int
    44  	ch            clientv3.WatchChan
    45  }
    46  
    47  func runWatchTest(t *testing.T, f watcherTest) {
    48  	defer testutil.AfterTest(t)
    49  
    50  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3})
    51  	defer clus.Terminate(t)
    52  
    53  	wclientMember := rand.Intn(3)
    54  	w := clus.Client(wclientMember).Watcher
    55  	// select a different client for KV operations so puts succeed if
    56  	// a test knocks out the watcher client.
    57  	kvMember := rand.Intn(3)
    58  	for kvMember == wclientMember {
    59  		kvMember = rand.Intn(3)
    60  	}
    61  	kv := clus.Client(kvMember).KV
    62  
    63  	wctx := &watchctx{clus, w, kv, wclientMember, kvMember, nil}
    64  	f(t, wctx)
    65  }
    66  
    67  // TestWatchMultiWatcher modifies multiple keys and observes the changes.
    68  func TestWatchMultiWatcher(t *testing.T) {
    69  	runWatchTest(t, testWatchMultiWatcher)
    70  }
    71  
    72  func testWatchMultiWatcher(t *testing.T, wctx *watchctx) {
    73  	numKeyUpdates := 4
    74  	keys := []string{"foo", "bar", "baz"}
    75  
    76  	donec := make(chan struct{})
    77  	readyc := make(chan struct{})
    78  	for _, k := range keys {
    79  		// key watcher
    80  		go func(key string) {
    81  			ch := wctx.w.Watch(context.TODO(), key)
    82  			if ch == nil {
    83  				t.Fatalf("expected watcher channel, got nil")
    84  			}
    85  			readyc <- struct{}{}
    86  			for i := 0; i < numKeyUpdates; i++ {
    87  				resp, ok := <-ch
    88  				if !ok {
    89  					t.Fatalf("watcher unexpectedly closed")
    90  				}
    91  				v := fmt.Sprintf("%s-%d", key, i)
    92  				gotv := string(resp.Events[0].Kv.Value)
    93  				if gotv != v {
    94  					t.Errorf("#%d: got %s, wanted %s", i, gotv, v)
    95  				}
    96  			}
    97  			donec <- struct{}{}
    98  		}(k)
    99  	}
   100  	// prefix watcher on "b" (bar and baz)
   101  	go func() {
   102  		prefixc := wctx.w.Watch(context.TODO(), "b", clientv3.WithPrefix())
   103  		if prefixc == nil {
   104  			t.Fatalf("expected watcher channel, got nil")
   105  		}
   106  		readyc <- struct{}{}
   107  		evs := []*clientv3.Event{}
   108  		for i := 0; i < numKeyUpdates*2; i++ {
   109  			resp, ok := <-prefixc
   110  			if !ok {
   111  				t.Fatalf("watcher unexpectedly closed")
   112  			}
   113  			evs = append(evs, resp.Events...)
   114  		}
   115  
   116  		// check response
   117  		expected := []string{}
   118  		bkeys := []string{"bar", "baz"}
   119  		for _, k := range bkeys {
   120  			for i := 0; i < numKeyUpdates; i++ {
   121  				expected = append(expected, fmt.Sprintf("%s-%d", k, i))
   122  			}
   123  		}
   124  		got := []string{}
   125  		for _, ev := range evs {
   126  			got = append(got, string(ev.Kv.Value))
   127  		}
   128  		sort.Strings(got)
   129  		if !reflect.DeepEqual(expected, got) {
   130  			t.Errorf("got %v, expected %v", got, expected)
   131  		}
   132  
   133  		// ensure no extra data
   134  		select {
   135  		case resp, ok := <-prefixc:
   136  			if !ok {
   137  				t.Fatalf("watcher unexpectedly closed")
   138  			}
   139  			t.Fatalf("unexpected event %+v", resp)
   140  		case <-time.After(time.Second):
   141  		}
   142  		donec <- struct{}{}
   143  	}()
   144  
   145  	// wait for watcher bring up
   146  	for i := 0; i < len(keys)+1; i++ {
   147  		<-readyc
   148  	}
   149  	// generate events
   150  	ctx := context.TODO()
   151  	for i := 0; i < numKeyUpdates; i++ {
   152  		for _, k := range keys {
   153  			v := fmt.Sprintf("%s-%d", k, i)
   154  			if _, err := wctx.kv.Put(ctx, k, v); err != nil {
   155  				t.Fatal(err)
   156  			}
   157  		}
   158  	}
   159  	// wait for watcher shutdown
   160  	for i := 0; i < len(keys)+1; i++ {
   161  		<-donec
   162  	}
   163  }
   164  
   165  // TestWatchRange tests watcher creates ranges
   166  func TestWatchRange(t *testing.T) {
   167  	runWatchTest(t, testWatchRange)
   168  }
   169  
   170  func testWatchRange(t *testing.T, wctx *watchctx) {
   171  	if wctx.ch = wctx.w.Watch(context.TODO(), "a", clientv3.WithRange("c")); wctx.ch == nil {
   172  		t.Fatalf("expected non-nil channel")
   173  	}
   174  	putAndWatch(t, wctx, "a", "a")
   175  	putAndWatch(t, wctx, "b", "b")
   176  	putAndWatch(t, wctx, "bar", "bar")
   177  }
   178  
   179  // TestWatchReconnRequest tests the send failure path when requesting a watcher.
   180  func TestWatchReconnRequest(t *testing.T) {
   181  	runWatchTest(t, testWatchReconnRequest)
   182  }
   183  
   184  func testWatchReconnRequest(t *testing.T, wctx *watchctx) {
   185  	donec, stopc := make(chan struct{}), make(chan struct{}, 1)
   186  	go func() {
   187  		timer := time.After(2 * time.Second)
   188  		defer close(donec)
   189  		// take down watcher connection
   190  		for {
   191  			wctx.clus.Members[wctx.wclientMember].DropConnections()
   192  			select {
   193  			case <-timer:
   194  				// spinning on close may live lock reconnection
   195  				return
   196  			case <-stopc:
   197  				return
   198  			default:
   199  			}
   200  		}
   201  	}()
   202  	// should reconnect when requesting watch
   203  	if wctx.ch = wctx.w.Watch(context.TODO(), "a"); wctx.ch == nil {
   204  		t.Fatalf("expected non-nil channel")
   205  	}
   206  
   207  	// wait for disconnections to stop
   208  	stopc <- struct{}{}
   209  	<-donec
   210  
   211  	// spinning on dropping connections may trigger a leader election
   212  	// due to resource starvation; l-read to ensure the cluster is stable
   213  	ctx, cancel := context.WithTimeout(context.TODO(), 30*time.Second)
   214  	if _, err := wctx.kv.Get(ctx, "_"); err != nil {
   215  		t.Fatal(err)
   216  	}
   217  	cancel()
   218  
   219  	// ensure watcher works
   220  	putAndWatch(t, wctx, "a", "a")
   221  }
   222  
   223  // TestWatchReconnInit tests watcher resumes correctly if connection lost
   224  // before any data was sent.
   225  func TestWatchReconnInit(t *testing.T) {
   226  	runWatchTest(t, testWatchReconnInit)
   227  }
   228  
   229  func testWatchReconnInit(t *testing.T, wctx *watchctx) {
   230  	if wctx.ch = wctx.w.Watch(context.TODO(), "a"); wctx.ch == nil {
   231  		t.Fatalf("expected non-nil channel")
   232  	}
   233  	wctx.clus.Members[wctx.wclientMember].DropConnections()
   234  	// watcher should recover
   235  	putAndWatch(t, wctx, "a", "a")
   236  }
   237  
   238  // TestWatchReconnRunning tests watcher resumes correctly if connection lost
   239  // after data was sent.
   240  func TestWatchReconnRunning(t *testing.T) {
   241  	runWatchTest(t, testWatchReconnRunning)
   242  }
   243  
   244  func testWatchReconnRunning(t *testing.T, wctx *watchctx) {
   245  	if wctx.ch = wctx.w.Watch(context.TODO(), "a"); wctx.ch == nil {
   246  		t.Fatalf("expected non-nil channel")
   247  	}
   248  	putAndWatch(t, wctx, "a", "a")
   249  	// take down watcher connection
   250  	wctx.clus.Members[wctx.wclientMember].DropConnections()
   251  	// watcher should recover
   252  	putAndWatch(t, wctx, "a", "b")
   253  }
   254  
   255  // TestWatchCancelImmediate ensures a closed channel is returned
   256  // if the context is cancelled.
   257  func TestWatchCancelImmediate(t *testing.T) {
   258  	runWatchTest(t, testWatchCancelImmediate)
   259  }
   260  
   261  func testWatchCancelImmediate(t *testing.T, wctx *watchctx) {
   262  	ctx, cancel := context.WithCancel(context.Background())
   263  	cancel()
   264  	wch := wctx.w.Watch(ctx, "a")
   265  	select {
   266  	case wresp, ok := <-wch:
   267  		if ok {
   268  			t.Fatalf("read wch got %v; expected closed channel", wresp)
   269  		}
   270  	default:
   271  		t.Fatalf("closed watcher channel should not block")
   272  	}
   273  }
   274  
   275  // TestWatchCancelInit tests watcher closes correctly after no events.
   276  func TestWatchCancelInit(t *testing.T) {
   277  	runWatchTest(t, testWatchCancelInit)
   278  }
   279  
   280  func testWatchCancelInit(t *testing.T, wctx *watchctx) {
   281  	ctx, cancel := context.WithCancel(context.Background())
   282  	if wctx.ch = wctx.w.Watch(ctx, "a"); wctx.ch == nil {
   283  		t.Fatalf("expected non-nil watcher channel")
   284  	}
   285  	cancel()
   286  	select {
   287  	case <-time.After(time.Second):
   288  		t.Fatalf("took too long to cancel")
   289  	case _, ok := <-wctx.ch:
   290  		if ok {
   291  			t.Fatalf("expected watcher channel to close")
   292  		}
   293  	}
   294  }
   295  
   296  // TestWatchCancelRunning tests watcher closes correctly after events.
   297  func TestWatchCancelRunning(t *testing.T) {
   298  	runWatchTest(t, testWatchCancelRunning)
   299  }
   300  
   301  func testWatchCancelRunning(t *testing.T, wctx *watchctx) {
   302  	ctx, cancel := context.WithCancel(context.Background())
   303  	if wctx.ch = wctx.w.Watch(ctx, "a"); wctx.ch == nil {
   304  		t.Fatalf("expected non-nil watcher channel")
   305  	}
   306  	if _, err := wctx.kv.Put(ctx, "a", "a"); err != nil {
   307  		t.Fatal(err)
   308  	}
   309  	cancel()
   310  	select {
   311  	case <-time.After(time.Second):
   312  		t.Fatalf("took too long to cancel")
   313  	case _, ok := <-wctx.ch:
   314  		if !ok {
   315  			// closed before getting put; OK
   316  			break
   317  		}
   318  		// got the PUT; should close next
   319  		select {
   320  		case <-time.After(time.Second):
   321  			t.Fatalf("took too long to close")
   322  		case v, ok2 := <-wctx.ch:
   323  			if ok2 {
   324  				t.Fatalf("expected watcher channel to close, got %v", v)
   325  			}
   326  		}
   327  	}
   328  }
   329  
   330  func putAndWatch(t *testing.T, wctx *watchctx, key, val string) {
   331  	if _, err := wctx.kv.Put(context.TODO(), key, val); err != nil {
   332  		t.Fatal(err)
   333  	}
   334  	select {
   335  	case <-time.After(5 * time.Second):
   336  		t.Fatalf("watch timed out")
   337  	case v, ok := <-wctx.ch:
   338  		if !ok {
   339  			t.Fatalf("unexpected watch close")
   340  		}
   341  		if string(v.Events[0].Kv.Value) != val {
   342  			t.Fatalf("bad value got %v, wanted %v", v.Events[0].Kv.Value, val)
   343  		}
   344  	}
   345  }
   346  
   347  func TestWatchResumeInitRev(t *testing.T) {
   348  	defer testutil.AfterTest(t)
   349  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
   350  	defer clus.Terminate(t)
   351  
   352  	cli := clus.Client(0)
   353  	if _, err := cli.Put(context.TODO(), "b", "2"); err != nil {
   354  		t.Fatal(err)
   355  	}
   356  	if _, err := cli.Put(context.TODO(), "a", "3"); err != nil {
   357  		t.Fatal(err)
   358  	}
   359  	// if resume is broken, it'll pick up this key first instead of a=3
   360  	if _, err := cli.Put(context.TODO(), "a", "4"); err != nil {
   361  		t.Fatal(err)
   362  	}
   363  
   364  	wch := clus.Client(0).Watch(context.Background(), "a", clientv3.WithRev(1), clientv3.WithCreatedNotify())
   365  	if resp, ok := <-wch; !ok || resp.Header.Revision != 4 {
   366  		t.Fatalf("got (%v, %v), expected create notification rev=4", resp, ok)
   367  	}
   368  	// pause wch
   369  	clus.Members[0].DropConnections()
   370  	clus.Members[0].PauseConnections()
   371  
   372  	select {
   373  	case resp, ok := <-wch:
   374  		t.Skipf("wch should block, got (%+v, %v); drop not fast enough", resp, ok)
   375  	case <-time.After(100 * time.Millisecond):
   376  	}
   377  
   378  	// resume wch
   379  	clus.Members[0].UnpauseConnections()
   380  
   381  	select {
   382  	case resp, ok := <-wch:
   383  		if !ok {
   384  			t.Fatal("unexpected watch close")
   385  		}
   386  		if len(resp.Events) == 0 {
   387  			t.Fatal("expected event on watch")
   388  		}
   389  		if string(resp.Events[0].Kv.Value) != "3" {
   390  			t.Fatalf("expected value=3, got event %+v", resp.Events[0])
   391  		}
   392  	case <-time.After(5 * time.Second):
   393  		t.Fatal("watch timed out")
   394  	}
   395  }
   396  
   397  // TestWatchResumeCompacted checks that the watcher gracefully closes in case
   398  // that it tries to resume to a revision that's been compacted out of the store.
   399  // Since the watcher's server restarts with stale data, the watcher will receive
   400  // either a compaction error or all keys by staying in sync before the compaction
   401  // is finally applied.
   402  func TestWatchResumeCompacted(t *testing.T) {
   403  	defer testutil.AfterTest(t)
   404  
   405  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3})
   406  	defer clus.Terminate(t)
   407  
   408  	// create a waiting watcher at rev 1
   409  	w := clus.Client(0)
   410  	wch := w.Watch(context.Background(), "foo", clientv3.WithRev(1))
   411  	select {
   412  	case w := <-wch:
   413  		t.Errorf("unexpected message from wch %v", w)
   414  	default:
   415  	}
   416  	clus.Members[0].Stop(t)
   417  
   418  	ticker := time.After(time.Second * 10)
   419  	for clus.WaitLeader(t) <= 0 {
   420  		select {
   421  		case <-ticker:
   422  			t.Fatalf("failed to wait for new leader")
   423  		default:
   424  			time.Sleep(10 * time.Millisecond)
   425  		}
   426  	}
   427  
   428  	// put some data and compact away
   429  	numPuts := 5
   430  	kv := clus.Client(1)
   431  	for i := 0; i < numPuts; i++ {
   432  		if _, err := kv.Put(context.TODO(), "foo", "bar"); err != nil {
   433  			t.Fatal(err)
   434  		}
   435  	}
   436  	if _, err := kv.Compact(context.TODO(), 3); err != nil {
   437  		t.Fatal(err)
   438  	}
   439  
   440  	clus.Members[0].Restart(t)
   441  
   442  	// since watch's server isn't guaranteed to be synced with the cluster when
   443  	// the watch resumes, there is a window where the watch can stay synced and
   444  	// read off all events; if the watcher misses the window, it will go out of
   445  	// sync and get a compaction error.
   446  	wRev := int64(2)
   447  	for int(wRev) <= numPuts+1 {
   448  		var wresp clientv3.WatchResponse
   449  		var ok bool
   450  		select {
   451  		case wresp, ok = <-wch:
   452  			if !ok {
   453  				t.Fatalf("expected wresp, but got closed channel")
   454  			}
   455  		case <-time.After(5 * time.Second):
   456  			t.Fatalf("compacted watch timed out")
   457  		}
   458  		for _, ev := range wresp.Events {
   459  			if ev.Kv.ModRevision != wRev {
   460  				t.Fatalf("expected modRev %v, got %+v", wRev, ev)
   461  			}
   462  			wRev++
   463  		}
   464  		if wresp.Err() == nil {
   465  			continue
   466  		}
   467  		if wresp.Err() != rpctypes.ErrCompacted {
   468  			t.Fatalf("wresp.Err() expected %v, got %+v", rpctypes.ErrCompacted, wresp.Err())
   469  		}
   470  		break
   471  	}
   472  	if int(wRev) > numPuts+1 {
   473  		// got data faster than the compaction
   474  		return
   475  	}
   476  	// received compaction error; ensure the channel closes
   477  	select {
   478  	case wresp, ok := <-wch:
   479  		if ok {
   480  			t.Fatalf("expected closed channel, but got %v", wresp)
   481  		}
   482  	case <-time.After(5 * time.Second):
   483  		t.Fatalf("timed out waiting for channel close")
   484  	}
   485  }
   486  
   487  // TestWatchCompactRevision ensures the CompactRevision error is given on a
   488  // compaction event ahead of a watcher.
   489  func TestWatchCompactRevision(t *testing.T) {
   490  	defer testutil.AfterTest(t)
   491  
   492  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
   493  	defer clus.Terminate(t)
   494  
   495  	// set some keys
   496  	kv := clus.RandClient()
   497  	for i := 0; i < 5; i++ {
   498  		if _, err := kv.Put(context.TODO(), "foo", "bar"); err != nil {
   499  			t.Fatal(err)
   500  		}
   501  	}
   502  
   503  	w := clus.RandClient()
   504  
   505  	if _, err := kv.Compact(context.TODO(), 4); err != nil {
   506  		t.Fatal(err)
   507  	}
   508  	wch := w.Watch(context.Background(), "foo", clientv3.WithRev(2))
   509  
   510  	// get compacted error message
   511  	wresp, ok := <-wch
   512  	if !ok {
   513  		t.Fatalf("expected wresp, but got closed channel")
   514  	}
   515  	if wresp.Err() != rpctypes.ErrCompacted {
   516  		t.Fatalf("wresp.Err() expected %v, but got %v", rpctypes.ErrCompacted, wresp.Err())
   517  	}
   518  	if !wresp.Canceled {
   519  		t.Fatalf("wresp.Canceled expected true, got %+v", wresp)
   520  	}
   521  
   522  	// ensure the channel is closed
   523  	if wresp, ok = <-wch; ok {
   524  		t.Fatalf("expected closed channel, but got %v", wresp)
   525  	}
   526  }
   527  
   528  func TestWatchWithProgressNotify(t *testing.T)        { testWatchWithProgressNotify(t, true) }
   529  func TestWatchWithProgressNotifyNoEvent(t *testing.T) { testWatchWithProgressNotify(t, false) }
   530  
   531  func testWatchWithProgressNotify(t *testing.T, watchOnPut bool) {
   532  	defer testutil.AfterTest(t)
   533  
   534  	// accelerate report interval so test terminates quickly
   535  	oldpi := v3rpc.GetProgressReportInterval()
   536  	// using atomics to avoid race warnings
   537  	v3rpc.SetProgressReportInterval(3 * time.Second)
   538  	pi := 3 * time.Second
   539  	defer func() { v3rpc.SetProgressReportInterval(oldpi) }()
   540  
   541  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3})
   542  	defer clus.Terminate(t)
   543  
   544  	wc := clus.RandClient()
   545  
   546  	opts := []clientv3.OpOption{clientv3.WithProgressNotify()}
   547  	if watchOnPut {
   548  		opts = append(opts, clientv3.WithPrefix())
   549  	}
   550  	rch := wc.Watch(context.Background(), "foo", opts...)
   551  
   552  	select {
   553  	case resp := <-rch: // wait for notification
   554  		if len(resp.Events) != 0 {
   555  			t.Fatalf("resp.Events expected none, got %+v", resp.Events)
   556  		}
   557  	case <-time.After(2 * pi):
   558  		t.Fatalf("watch response expected in %v, but timed out", pi)
   559  	}
   560  
   561  	kvc := clus.RandClient()
   562  	if _, err := kvc.Put(context.TODO(), "foox", "bar"); err != nil {
   563  		t.Fatal(err)
   564  	}
   565  
   566  	select {
   567  	case resp := <-rch:
   568  		if resp.Header.Revision != 2 {
   569  			t.Fatalf("resp.Header.Revision expected 2, got %d", resp.Header.Revision)
   570  		}
   571  		if watchOnPut { // wait for put if watch on the put key
   572  			ev := []*clientv3.Event{{Type: clientv3.EventTypePut,
   573  				Kv: &mvccpb.KeyValue{Key: []byte("foox"), Value: []byte("bar"), CreateRevision: 2, ModRevision: 2, Version: 1}}}
   574  			if !reflect.DeepEqual(ev, resp.Events) {
   575  				t.Fatalf("expected %+v, got %+v", ev, resp.Events)
   576  			}
   577  		} else if len(resp.Events) != 0 { // wait for notification otherwise
   578  			t.Fatalf("expected no events, but got %+v", resp.Events)
   579  		}
   580  	case <-time.After(time.Duration(1.5 * float64(pi))):
   581  		t.Fatalf("watch response expected in %v, but timed out", pi)
   582  	}
   583  }
   584  
   585  func TestWatchRequestProgress(t *testing.T) {
   586  	testCases := []struct {
   587  		name     string
   588  		watchers []string
   589  	}{
   590  		{"0-watcher", []string{}},
   591  		{"1-watcher", []string{"/"}},
   592  		{"2-watcher", []string{"/", "/"}},
   593  	}
   594  
   595  	for _, c := range testCases {
   596  		t.Run(c.name, func(t *testing.T) {
   597  			defer testutil.AfterTest(t)
   598  
   599  			watchTimeout := 3 * time.Second
   600  
   601  			clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3})
   602  			defer clus.Terminate(t)
   603  
   604  			wc := clus.RandClient()
   605  
   606  			var watchChans []clientv3.WatchChan
   607  
   608  			for _, prefix := range c.watchers {
   609  				watchChans = append(watchChans, wc.Watch(context.Background(), prefix, clientv3.WithPrefix()))
   610  			}
   611  
   612  			_, err := wc.Put(context.Background(), "/a", "1")
   613  			if err != nil {
   614  				t.Fatal(err)
   615  			}
   616  
   617  			for _, rch := range watchChans {
   618  				select {
   619  				case resp := <-rch: // wait for notification
   620  					if len(resp.Events) != 1 {
   621  						t.Fatalf("resp.Events expected 1, got %d", len(resp.Events))
   622  					}
   623  				case <-time.After(watchTimeout):
   624  					t.Fatalf("watch response expected in %v, but timed out", watchTimeout)
   625  				}
   626  			}
   627  
   628  			// put a value not being watched to increment revision
   629  			_, err = wc.Put(context.Background(), "x", "1")
   630  			if err != nil {
   631  				t.Fatal(err)
   632  			}
   633  
   634  			err = wc.RequestProgress(context.Background())
   635  			if err != nil {
   636  				t.Fatal(err)
   637  			}
   638  
   639  			// verify all watch channels receive a progress notify
   640  			for _, rch := range watchChans {
   641  				select {
   642  				case resp := <-rch:
   643  					if !resp.IsProgressNotify() {
   644  						t.Fatalf("expected resp.IsProgressNotify() == true")
   645  					}
   646  					if resp.Header.Revision != 3 {
   647  						t.Fatalf("resp.Header.Revision expected 3, got %d", resp.Header.Revision)
   648  					}
   649  				case <-time.After(watchTimeout):
   650  					t.Fatalf("progress response expected in %v, but timed out", watchTimeout)
   651  				}
   652  			}
   653  		})
   654  	}
   655  }
   656  
   657  func TestWatchEventType(t *testing.T) {
   658  	cluster := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
   659  	defer cluster.Terminate(t)
   660  
   661  	client := cluster.RandClient()
   662  	ctx := context.Background()
   663  	watchChan := client.Watch(ctx, "/", clientv3.WithPrefix())
   664  
   665  	if _, err := client.Put(ctx, "/toDelete", "foo"); err != nil {
   666  		t.Fatalf("Put failed: %v", err)
   667  	}
   668  	if _, err := client.Put(ctx, "/toDelete", "bar"); err != nil {
   669  		t.Fatalf("Put failed: %v", err)
   670  	}
   671  	if _, err := client.Delete(ctx, "/toDelete"); err != nil {
   672  		t.Fatalf("Delete failed: %v", err)
   673  	}
   674  	lcr, err := client.Lease.Grant(ctx, 1)
   675  	if err != nil {
   676  		t.Fatalf("lease create failed: %v", err)
   677  	}
   678  	if _, err := client.Put(ctx, "/toExpire", "foo", clientv3.WithLease(lcr.ID)); err != nil {
   679  		t.Fatalf("Put failed: %v", err)
   680  	}
   681  
   682  	tests := []struct {
   683  		et       mvccpb.Event_EventType
   684  		isCreate bool
   685  		isModify bool
   686  	}{{
   687  		et:       clientv3.EventTypePut,
   688  		isCreate: true,
   689  	}, {
   690  		et:       clientv3.EventTypePut,
   691  		isModify: true,
   692  	}, {
   693  		et: clientv3.EventTypeDelete,
   694  	}, {
   695  		et:       clientv3.EventTypePut,
   696  		isCreate: true,
   697  	}, {
   698  		et: clientv3.EventTypeDelete,
   699  	}}
   700  
   701  	var res []*clientv3.Event
   702  
   703  	for {
   704  		select {
   705  		case wres := <-watchChan:
   706  			res = append(res, wres.Events...)
   707  		case <-time.After(10 * time.Second):
   708  			t.Fatalf("Should receive %d events and then break out loop", len(tests))
   709  		}
   710  		if len(res) == len(tests) {
   711  			break
   712  		}
   713  	}
   714  
   715  	for i, tt := range tests {
   716  		ev := res[i]
   717  		if tt.et != ev.Type {
   718  			t.Errorf("#%d: event type want=%s, get=%s", i, tt.et, ev.Type)
   719  		}
   720  		if tt.isCreate && !ev.IsCreate() {
   721  			t.Errorf("#%d: event should be CreateEvent", i)
   722  		}
   723  		if tt.isModify && !ev.IsModify() {
   724  			t.Errorf("#%d: event should be ModifyEvent", i)
   725  		}
   726  	}
   727  }
   728  
   729  func TestWatchErrConnClosed(t *testing.T) {
   730  	defer testutil.AfterTest(t)
   731  
   732  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
   733  	defer clus.Terminate(t)
   734  
   735  	cli := clus.Client(0)
   736  
   737  	donec := make(chan struct{})
   738  	go func() {
   739  		defer close(donec)
   740  		ch := cli.Watch(context.TODO(), "foo")
   741  
   742  		if wr := <-ch; !isCanceled(wr.Err()) {
   743  			t.Errorf("expected context canceled, got %v", wr.Err())
   744  		}
   745  	}()
   746  
   747  	if err := cli.ActiveConnection().Close(); err != nil {
   748  		t.Fatal(err)
   749  	}
   750  	clus.TakeClient(0)
   751  
   752  	select {
   753  	case <-time.After(integration.RequestWaitTimeout):
   754  		t.Fatal("wc.Watch took too long")
   755  	case <-donec:
   756  	}
   757  }
   758  
   759  func TestWatchAfterClose(t *testing.T) {
   760  	defer testutil.AfterTest(t)
   761  
   762  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
   763  	defer clus.Terminate(t)
   764  
   765  	cli := clus.Client(0)
   766  	clus.TakeClient(0)
   767  	if err := cli.Close(); err != nil {
   768  		t.Fatal(err)
   769  	}
   770  
   771  	donec := make(chan struct{})
   772  	go func() {
   773  		cli.Watch(context.TODO(), "foo")
   774  		if err := cli.Close(); err != nil && err != context.Canceled {
   775  			t.Errorf("expected %v, got %v", context.Canceled, err)
   776  		}
   777  		close(donec)
   778  	}()
   779  	select {
   780  	case <-time.After(integration.RequestWaitTimeout):
   781  		t.Fatal("wc.Watch took too long")
   782  	case <-donec:
   783  	}
   784  }
   785  
   786  // TestWatchWithRequireLeader checks the watch channel closes when no leader.
   787  func TestWatchWithRequireLeader(t *testing.T) {
   788  	defer testutil.AfterTest(t)
   789  
   790  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3})
   791  	defer clus.Terminate(t)
   792  
   793  	// Put a key for the non-require leader watch to read as an event.
   794  	// The watchers will be on member[0]; put key through member[0] to
   795  	// ensure that it receives the update so watching after killing quorum
   796  	// is guaranteed to have the key.
   797  	liveClient := clus.Client(0)
   798  	if _, err := liveClient.Put(context.TODO(), "foo", "bar"); err != nil {
   799  		t.Fatal(err)
   800  	}
   801  
   802  	clus.Members[1].Stop(t)
   803  	clus.Members[2].Stop(t)
   804  	clus.Client(1).Close()
   805  	clus.Client(2).Close()
   806  	clus.TakeClient(1)
   807  	clus.TakeClient(2)
   808  
   809  	// wait for election timeout, then member[0] will not have a leader.
   810  	tickDuration := 10 * time.Millisecond
   811  	// existing streams need three elections before they're torn down; wait until 5 elections cycle
   812  	// so proxy tests receive a leader loss event on its existing watch before creating a new watch.
   813  	time.Sleep(time.Duration(5*clus.Members[0].ElectionTicks) * tickDuration)
   814  
   815  	chLeader := liveClient.Watch(clientv3.WithRequireLeader(context.TODO()), "foo", clientv3.WithRev(1))
   816  	chNoLeader := liveClient.Watch(context.TODO(), "foo", clientv3.WithRev(1))
   817  
   818  	select {
   819  	case resp, ok := <-chLeader:
   820  		if !ok {
   821  			t.Fatalf("expected %v watch channel, got closed channel", rpctypes.ErrNoLeader)
   822  		}
   823  		if resp.Err() != rpctypes.ErrNoLeader {
   824  			t.Fatalf("expected %v watch response error, got %+v", rpctypes.ErrNoLeader, resp)
   825  		}
   826  	case <-time.After(integration.RequestWaitTimeout):
   827  		t.Fatal("watch without leader took too long to close")
   828  	}
   829  
   830  	select {
   831  	case resp, ok := <-chLeader:
   832  		if ok {
   833  			t.Fatalf("expected closed channel, got response %v", resp)
   834  		}
   835  	case <-time.After(integration.RequestWaitTimeout):
   836  		t.Fatal("waited too long for channel to close")
   837  	}
   838  
   839  	if _, ok := <-chNoLeader; !ok {
   840  		t.Fatalf("expected response, got closed channel")
   841  	}
   842  }
   843  
   844  // TestWatchWithFilter checks that watch filtering works.
   845  func TestWatchWithFilter(t *testing.T) {
   846  	cluster := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
   847  	defer cluster.Terminate(t)
   848  
   849  	client := cluster.RandClient()
   850  	ctx := context.Background()
   851  
   852  	wcNoPut := client.Watch(ctx, "a", clientv3.WithFilterPut())
   853  	wcNoDel := client.Watch(ctx, "a", clientv3.WithFilterDelete())
   854  
   855  	if _, err := client.Put(ctx, "a", "abc"); err != nil {
   856  		t.Fatal(err)
   857  	}
   858  	if _, err := client.Delete(ctx, "a"); err != nil {
   859  		t.Fatal(err)
   860  	}
   861  
   862  	npResp := <-wcNoPut
   863  	if len(npResp.Events) != 1 || npResp.Events[0].Type != clientv3.EventTypeDelete {
   864  		t.Fatalf("expected delete event, got %+v", npResp.Events)
   865  	}
   866  	ndResp := <-wcNoDel
   867  	if len(ndResp.Events) != 1 || ndResp.Events[0].Type != clientv3.EventTypePut {
   868  		t.Fatalf("expected put event, got %+v", ndResp.Events)
   869  	}
   870  
   871  	select {
   872  	case resp := <-wcNoPut:
   873  		t.Fatalf("unexpected event on filtered put (%+v)", resp)
   874  	case resp := <-wcNoDel:
   875  		t.Fatalf("unexpected event on filtered delete (%+v)", resp)
   876  	case <-time.After(100 * time.Millisecond):
   877  	}
   878  }
   879  
   880  // TestWatchWithCreatedNotification checks that WithCreatedNotify returns a
   881  // Created watch response.
   882  func TestWatchWithCreatedNotification(t *testing.T) {
   883  	cluster := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
   884  	defer cluster.Terminate(t)
   885  
   886  	client := cluster.RandClient()
   887  
   888  	ctx := context.Background()
   889  
   890  	createC := client.Watch(ctx, "a", clientv3.WithCreatedNotify())
   891  
   892  	resp := <-createC
   893  
   894  	if !resp.Created {
   895  		t.Fatalf("expected created event, got %v", resp)
   896  	}
   897  }
   898  
   899  // TestWatchWithCreatedNotificationDropConn ensures that
   900  // a watcher with created notify does not post duplicate
   901  // created events from disconnect.
   902  func TestWatchWithCreatedNotificationDropConn(t *testing.T) {
   903  	cluster := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
   904  	defer cluster.Terminate(t)
   905  
   906  	client := cluster.RandClient()
   907  
   908  	wch := client.Watch(context.Background(), "a", clientv3.WithCreatedNotify())
   909  
   910  	resp := <-wch
   911  
   912  	if !resp.Created {
   913  		t.Fatalf("expected created event, got %v", resp)
   914  	}
   915  
   916  	cluster.Members[0].DropConnections()
   917  
   918  	// check watch channel doesn't post another watch response.
   919  	select {
   920  	case wresp := <-wch:
   921  		t.Fatalf("got unexpected watch response: %+v\n", wresp)
   922  	case <-time.After(time.Second):
   923  		// watcher may not reconnect by the time it hits the select,
   924  		// so it wouldn't have a chance to filter out the second create event
   925  	}
   926  }
   927  
   928  // TestWatchCancelOnServer ensures client watcher cancels propagate back to the server.
   929  func TestWatchCancelOnServer(t *testing.T) {
   930  	cluster := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
   931  	defer cluster.Terminate(t)
   932  
   933  	client := cluster.RandClient()
   934  	numWatches := 10
   935  
   936  	// The grpc proxy starts watches to detect leadership after the proxy server
   937  	// returns as started; to avoid racing on the proxy's internal watches, wait
   938  	// until require leader watches get create responses to ensure the leadership
   939  	// watches have started.
   940  	for {
   941  		ctx, cancel := context.WithCancel(clientv3.WithRequireLeader(context.TODO()))
   942  		ww := client.Watch(ctx, "a", clientv3.WithCreatedNotify())
   943  		wresp := <-ww
   944  		cancel()
   945  		if wresp.Err() == nil {
   946  			break
   947  		}
   948  	}
   949  
   950  	cancels := make([]context.CancelFunc, numWatches)
   951  	for i := 0; i < numWatches; i++ {
   952  		// force separate streams in client
   953  		md := metadata.Pairs("some-key", fmt.Sprintf("%d", i))
   954  		mctx := metadata.NewOutgoingContext(context.Background(), md)
   955  		ctx, cancel := context.WithCancel(mctx)
   956  		cancels[i] = cancel
   957  		w := client.Watch(ctx, fmt.Sprintf("%d", i), clientv3.WithCreatedNotify())
   958  		<-w
   959  	}
   960  
   961  	// get max watches; proxy tests have leadership watches, so total may be >numWatches
   962  	maxWatches, _ := cluster.Members[0].Metric("etcd_debugging_mvcc_watcher_total")
   963  
   964  	// cancel all and wait for cancels to propagate to etcd server
   965  	for i := 0; i < numWatches; i++ {
   966  		cancels[i]()
   967  	}
   968  	time.Sleep(time.Second)
   969  
   970  	minWatches, err := cluster.Members[0].Metric("etcd_debugging_mvcc_watcher_total")
   971  	if err != nil {
   972  		t.Fatal(err)
   973  	}
   974  
   975  	maxWatchV, minWatchV := 0, 0
   976  	n, serr := fmt.Sscanf(maxWatches+" "+minWatches, "%d %d", &maxWatchV, &minWatchV)
   977  	if n != 2 || serr != nil {
   978  		t.Fatalf("expected n=2 and err=nil, got n=%d and err=%v", n, serr)
   979  	}
   980  
   981  	if maxWatchV-minWatchV < numWatches {
   982  		t.Fatalf("expected %d canceled watchers, got %d", numWatches, maxWatchV-minWatchV)
   983  	}
   984  }
   985  
   986  // TestWatchOverlapContextCancel stresses the watcher stream teardown path by
   987  // creating/canceling watchers to ensure that new watchers are not taken down
   988  // by a torn down watch stream. The sort of race that's being detected:
   989  //     1. create w1 using a cancelable ctx with %v as "ctx"
   990  //     2. cancel ctx
   991  //     3. watcher client begins tearing down watcher grpc stream since no more watchers
   992  //     3. start creating watcher w2 using a new "ctx" (not canceled), attaches to old grpc stream
   993  //     4. watcher client finishes tearing down stream on "ctx"
   994  //     5. w2 comes back canceled
   995  func TestWatchOverlapContextCancel(t *testing.T) {
   996  	f := func(clus *integration.ClusterV3) {}
   997  	testWatchOverlapContextCancel(t, f)
   998  }
   999  
  1000  func TestWatchOverlapDropConnContextCancel(t *testing.T) {
  1001  	f := func(clus *integration.ClusterV3) {
  1002  		clus.Members[0].DropConnections()
  1003  	}
  1004  	testWatchOverlapContextCancel(t, f)
  1005  }
  1006  
  1007  func testWatchOverlapContextCancel(t *testing.T, f func(*integration.ClusterV3)) {
  1008  	defer testutil.AfterTest(t)
  1009  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
  1010  	defer clus.Terminate(t)
  1011  
  1012  	n := 100
  1013  	ctxs, ctxc := make([]context.Context, 5), make([]chan struct{}, 5)
  1014  	for i := range ctxs {
  1015  		// make unique stream
  1016  		md := metadata.Pairs("some-key", fmt.Sprintf("%d", i))
  1017  		ctxs[i] = metadata.NewOutgoingContext(context.Background(), md)
  1018  		// limits the maximum number of outstanding watchers per stream
  1019  		ctxc[i] = make(chan struct{}, 2)
  1020  	}
  1021  
  1022  	// issue concurrent watches on "abc" with cancel
  1023  	cli := clus.RandClient()
  1024  	if _, err := cli.Put(context.TODO(), "abc", "def"); err != nil {
  1025  		t.Fatal(err)
  1026  	}
  1027  	ch := make(chan struct{}, n)
  1028  	for i := 0; i < n; i++ {
  1029  		go func() {
  1030  			defer func() { ch <- struct{}{} }()
  1031  			idx := rand.Intn(len(ctxs))
  1032  			ctx, cancel := context.WithCancel(ctxs[idx])
  1033  			ctxc[idx] <- struct{}{}
  1034  			wch := cli.Watch(ctx, "abc", clientv3.WithRev(1))
  1035  			f(clus)
  1036  			select {
  1037  			case _, ok := <-wch:
  1038  				if !ok {
  1039  					t.Fatalf("unexpected closed channel %p", wch)
  1040  				}
  1041  			// may take a second or two to reestablish a watcher because of
  1042  			// grpc back off policies for disconnects
  1043  			case <-time.After(5 * time.Second):
  1044  				t.Errorf("timed out waiting for watch on %p", wch)
  1045  			}
  1046  			// randomize how cancel overlaps with watch creation
  1047  			if rand.Intn(2) == 0 {
  1048  				<-ctxc[idx]
  1049  				cancel()
  1050  			} else {
  1051  				cancel()
  1052  				<-ctxc[idx]
  1053  			}
  1054  		}()
  1055  	}
  1056  	// join on watches
  1057  	for i := 0; i < n; i++ {
  1058  		select {
  1059  		case <-ch:
  1060  		case <-time.After(5 * time.Second):
  1061  			t.Fatalf("timed out waiting for completed watch")
  1062  		}
  1063  	}
  1064  }
  1065  
  1066  // TestWatchCancelAndCloseClient ensures that canceling a watcher then immediately
  1067  // closing the client does not return a client closing error.
  1068  func TestWatchCancelAndCloseClient(t *testing.T) {
  1069  	defer testutil.AfterTest(t)
  1070  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
  1071  	defer clus.Terminate(t)
  1072  	cli := clus.Client(0)
  1073  	ctx, cancel := context.WithCancel(context.Background())
  1074  	wch := cli.Watch(ctx, "abc")
  1075  	donec := make(chan struct{})
  1076  	go func() {
  1077  		defer close(donec)
  1078  		select {
  1079  		case wr, ok := <-wch:
  1080  			if ok {
  1081  				t.Fatalf("expected closed watch after cancel(), got resp=%+v err=%v", wr, wr.Err())
  1082  			}
  1083  		case <-time.After(5 * time.Second):
  1084  			t.Fatal("timed out waiting for closed channel")
  1085  		}
  1086  	}()
  1087  	cancel()
  1088  	if err := cli.Close(); err != nil {
  1089  		t.Fatal(err)
  1090  	}
  1091  	<-donec
  1092  	clus.TakeClient(0)
  1093  }
  1094  
  1095  // TestWatchStressResumeClose establishes a bunch of watchers, disconnects
  1096  // to put them in resuming mode, cancels them so some resumes by cancel fail,
  1097  // then closes the watcher interface to ensure correct clean up.
  1098  func TestWatchStressResumeClose(t *testing.T) {
  1099  	defer testutil.AfterTest(t)
  1100  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
  1101  	defer clus.Terminate(t)
  1102  	cli := clus.Client(0)
  1103  
  1104  	ctx, cancel := context.WithCancel(context.Background())
  1105  	// add more watches than can be resumed before the cancel
  1106  	wchs := make([]clientv3.WatchChan, 2000)
  1107  	for i := range wchs {
  1108  		wchs[i] = cli.Watch(ctx, "abc")
  1109  	}
  1110  	clus.Members[0].DropConnections()
  1111  	cancel()
  1112  	if err := cli.Close(); err != nil {
  1113  		t.Fatal(err)
  1114  	}
  1115  	clus.TakeClient(0)
  1116  }
  1117  
  1118  // TestWatchCancelDisconnected ensures canceling a watcher works when
  1119  // its grpc stream is disconnected / reconnecting.
  1120  func TestWatchCancelDisconnected(t *testing.T) {
  1121  	defer testutil.AfterTest(t)
  1122  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
  1123  	defer clus.Terminate(t)
  1124  	cli := clus.Client(0)
  1125  	ctx, cancel := context.WithCancel(context.Background())
  1126  	// add more watches than can be resumed before the cancel
  1127  	wch := cli.Watch(ctx, "abc")
  1128  	clus.Members[0].Stop(t)
  1129  	cancel()
  1130  	select {
  1131  	case <-wch:
  1132  	case <-time.After(time.Second):
  1133  		t.Fatal("took too long to cancel disconnected watcher")
  1134  	}
  1135  }
  1136  
  1137  // TestWatchClose ensures that close does not return error
  1138  func TestWatchClose(t *testing.T) {
  1139  	runWatchTest(t, testWatchClose)
  1140  }
  1141  
  1142  func testWatchClose(t *testing.T, wctx *watchctx) {
  1143  	ctx, cancel := context.WithCancel(context.Background())
  1144  	wch := wctx.w.Watch(ctx, "a")
  1145  	cancel()
  1146  	if wch == nil {
  1147  		t.Fatalf("expected watcher channel, got nil")
  1148  	}
  1149  	if wctx.w.Close() != nil {
  1150  		t.Fatalf("watch did not close successfully")
  1151  	}
  1152  	wresp, ok := <-wch
  1153  	if ok {
  1154  		t.Fatalf("read wch got %v; expected closed channel", wresp)
  1155  	}
  1156  }