go.etcd.io/etcd@v3.3.27+incompatible/clientv3/integration/leasing_test.go (about)

     1  // Copyright 2017 The etcd Authors
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package integration
    16  
    17  import (
    18  	"context"
    19  	"fmt"
    20  	"math/rand"
    21  	"reflect"
    22  	"sync"
    23  	"testing"
    24  	"time"
    25  
    26  	"github.com/coreos/etcd/clientv3"
    27  	"github.com/coreos/etcd/clientv3/concurrency"
    28  	"github.com/coreos/etcd/clientv3/leasing"
    29  	"github.com/coreos/etcd/integration"
    30  	"github.com/coreos/etcd/pkg/testutil"
    31  )
    32  
    33  func TestLeasingPutGet(t *testing.T) {
    34  	defer testutil.AfterTest(t)
    35  
    36  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3})
    37  	defer clus.Terminate(t)
    38  
    39  	lKV1, closeLKV1, err := leasing.NewKV(clus.Client(0), "foo/")
    40  	testutil.AssertNil(t, err)
    41  	defer closeLKV1()
    42  
    43  	lKV2, closeLKV2, err := leasing.NewKV(clus.Client(1), "foo/")
    44  	testutil.AssertNil(t, err)
    45  	defer closeLKV2()
    46  
    47  	lKV3, closeLKV3, err := leasing.NewKV(clus.Client(2), "foo/")
    48  	testutil.AssertNil(t, err)
    49  	defer closeLKV3()
    50  
    51  	resp, err := lKV1.Get(context.TODO(), "abc")
    52  	if err != nil {
    53  		t.Fatal(err)
    54  	}
    55  	if len(resp.Kvs) != 0 {
    56  		t.Errorf("expected nil, got %q", resp.Kvs[0].Key)
    57  	}
    58  
    59  	if _, err = lKV1.Put(context.TODO(), "abc", "def"); err != nil {
    60  		t.Fatal(err)
    61  	}
    62  	if resp, err = lKV2.Get(context.TODO(), "abc"); err != nil {
    63  		t.Fatal(err)
    64  	}
    65  	if string(resp.Kvs[0].Key) != "abc" {
    66  		t.Errorf("expected key=%q, got key=%q", "abc", resp.Kvs[0].Key)
    67  	}
    68  	if string(resp.Kvs[0].Value) != "def" {
    69  		t.Errorf("expected value=%q, got value=%q", "bar", resp.Kvs[0].Value)
    70  	}
    71  
    72  	if _, err = lKV3.Get(context.TODO(), "abc"); err != nil {
    73  		t.Fatal(err)
    74  	}
    75  	if _, err = lKV2.Put(context.TODO(), "abc", "ghi"); err != nil {
    76  		t.Fatal(err)
    77  	}
    78  
    79  	if resp, err = lKV3.Get(context.TODO(), "abc"); err != nil {
    80  		t.Fatal(err)
    81  	}
    82  	if string(resp.Kvs[0].Key) != "abc" {
    83  		t.Errorf("expected key=%q, got key=%q", "abc", resp.Kvs[0].Key)
    84  	}
    85  
    86  	if string(resp.Kvs[0].Value) != "ghi" {
    87  		t.Errorf("expected value=%q, got value=%q", "bar", resp.Kvs[0].Value)
    88  	}
    89  }
    90  
    91  // TestLeasingInterval checks the leasing KV fetches key intervals.
    92  func TestLeasingInterval(t *testing.T) {
    93  	defer testutil.AfterTest(t)
    94  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
    95  	defer clus.Terminate(t)
    96  
    97  	lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/")
    98  	testutil.AssertNil(t, err)
    99  	defer closeLKV()
   100  
   101  	keys := []string{"abc/a", "abc/b", "abc/a/a"}
   102  	for _, k := range keys {
   103  		if _, err = clus.Client(0).Put(context.TODO(), k, "v"); err != nil {
   104  			t.Fatal(err)
   105  		}
   106  	}
   107  
   108  	resp, err := lkv.Get(context.TODO(), "abc/", clientv3.WithPrefix())
   109  	if err != nil {
   110  		t.Fatal(err)
   111  	}
   112  	if len(resp.Kvs) != 3 {
   113  		t.Fatalf("expected keys %+v, got response keys %+v", keys, resp.Kvs)
   114  	}
   115  
   116  	// load into cache
   117  	if resp, err = lkv.Get(context.TODO(), "abc/a"); err != nil {
   118  		t.Fatal(err)
   119  	}
   120  
   121  	// get when prefix is also a cached key
   122  	if resp, err = lkv.Get(context.TODO(), "abc/a", clientv3.WithPrefix()); err != nil {
   123  		t.Fatal(err)
   124  	}
   125  	if len(resp.Kvs) != 2 {
   126  		t.Fatalf("expected keys %+v, got response keys %+v", keys, resp.Kvs)
   127  	}
   128  }
   129  
   130  // TestLeasingPutInvalidateNew checks the leasing KV updates its cache on a Put to a new key.
   131  func TestLeasingPutInvalidateNew(t *testing.T) {
   132  	defer testutil.AfterTest(t)
   133  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
   134  	defer clus.Terminate(t)
   135  
   136  	lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/")
   137  	testutil.AssertNil(t, err)
   138  	defer closeLKV()
   139  
   140  	if _, err = lkv.Get(context.TODO(), "k"); err != nil {
   141  		t.Fatal(err)
   142  	}
   143  	if _, err = lkv.Put(context.TODO(), "k", "v"); err != nil {
   144  		t.Fatal(err)
   145  	}
   146  
   147  	lkvResp, err := lkv.Get(context.TODO(), "k")
   148  	if err != nil {
   149  		t.Fatal(err)
   150  	}
   151  	cResp, cerr := clus.Client(0).Get(context.TODO(), "k")
   152  	if cerr != nil {
   153  		t.Fatal(cerr)
   154  	}
   155  	if !reflect.DeepEqual(lkvResp, cResp) {
   156  		t.Fatalf(`expected %+v, got response %+v`, cResp, lkvResp)
   157  	}
   158  }
   159  
   160  // TestLeasingPutInvalidateExisting checks the leasing KV updates its cache on a Put to an existing key.
   161  func TestLeasingPutInvalidateExisting(t *testing.T) {
   162  	defer testutil.AfterTest(t)
   163  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
   164  	defer clus.Terminate(t)
   165  
   166  	if _, err := clus.Client(0).Put(context.TODO(), "k", "abc"); err != nil {
   167  		t.Fatal(err)
   168  	}
   169  
   170  	lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/")
   171  	testutil.AssertNil(t, err)
   172  	defer closeLKV()
   173  
   174  	if _, err = lkv.Get(context.TODO(), "k"); err != nil {
   175  		t.Fatal(err)
   176  	}
   177  	if _, err = lkv.Put(context.TODO(), "k", "v"); err != nil {
   178  		t.Fatal(err)
   179  	}
   180  
   181  	lkvResp, err := lkv.Get(context.TODO(), "k")
   182  	if err != nil {
   183  		t.Fatal(err)
   184  	}
   185  	cResp, cerr := clus.Client(0).Get(context.TODO(), "k")
   186  	if cerr != nil {
   187  		t.Fatal(cerr)
   188  	}
   189  	if !reflect.DeepEqual(lkvResp, cResp) {
   190  		t.Fatalf(`expected %+v, got response %+v`, cResp, lkvResp)
   191  	}
   192  }
   193  
   194  // TestLeasingGetNoLeaseTTL checks a key with a TTL is not leased.
   195  func TestLeasingGetNoLeaseTTL(t *testing.T) {
   196  	defer testutil.AfterTest(t)
   197  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
   198  	defer clus.Terminate(t)
   199  
   200  	lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/")
   201  	testutil.AssertNil(t, err)
   202  	defer closeLKV()
   203  
   204  	lresp, err := clus.Client(0).Grant(context.TODO(), 60)
   205  	testutil.AssertNil(t, err)
   206  
   207  	_, err = clus.Client(0).Put(context.TODO(), "k", "v", clientv3.WithLease(lresp.ID))
   208  	testutil.AssertNil(t, err)
   209  
   210  	gresp, err := lkv.Get(context.TODO(), "k")
   211  	testutil.AssertNil(t, err)
   212  	testutil.AssertEqual(t, len(gresp.Kvs), 1)
   213  
   214  	clus.Members[0].Stop(t)
   215  
   216  	ctx, cancel := context.WithTimeout(context.TODO(), time.Second)
   217  	_, err = lkv.Get(ctx, "k")
   218  	cancel()
   219  	testutil.AssertEqual(t, err, ctx.Err())
   220  }
   221  
   222  // TestLeasingGetSerializable checks the leasing KV can make serialized requests
   223  // when the etcd cluster is partitioned.
   224  func TestLeasingGetSerializable(t *testing.T) {
   225  	defer testutil.AfterTest(t)
   226  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 2})
   227  	defer clus.Terminate(t)
   228  
   229  	lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/")
   230  	testutil.AssertNil(t, err)
   231  	defer closeLKV()
   232  
   233  	if _, err = clus.Client(0).Put(context.TODO(), "cached", "abc"); err != nil {
   234  		t.Fatal(err)
   235  	}
   236  	if _, err = lkv.Get(context.TODO(), "cached"); err != nil {
   237  		t.Fatal(err)
   238  	}
   239  
   240  	clus.Members[1].Stop(t)
   241  
   242  	// don't necessarily try to acquire leasing key ownership for new key
   243  	resp, err := lkv.Get(context.TODO(), "uncached", clientv3.WithSerializable())
   244  	if err != nil {
   245  		t.Fatal(err)
   246  	}
   247  	if len(resp.Kvs) != 0 {
   248  		t.Fatalf(`expected no keys, got response %+v`, resp)
   249  	}
   250  
   251  	clus.Members[0].Stop(t)
   252  
   253  	// leasing key ownership should have "cached" locally served
   254  	cachedResp, err := lkv.Get(context.TODO(), "cached", clientv3.WithSerializable())
   255  	if err != nil {
   256  		t.Fatal(err)
   257  	}
   258  	if len(cachedResp.Kvs) != 1 || string(cachedResp.Kvs[0].Value) != "abc" {
   259  		t.Fatalf(`expected "cached"->"abc", got response %+v`, cachedResp)
   260  	}
   261  }
   262  
   263  // TestLeasingPrevKey checks the cache respects WithPrevKV on puts.
   264  func TestLeasingPrevKey(t *testing.T) {
   265  	defer testutil.AfterTest(t)
   266  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 2})
   267  	defer clus.Terminate(t)
   268  
   269  	lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/")
   270  	testutil.AssertNil(t, err)
   271  	defer closeLKV()
   272  
   273  	if _, err = clus.Client(0).Put(context.TODO(), "k", "abc"); err != nil {
   274  		t.Fatal(err)
   275  	}
   276  	// acquire leasing key
   277  	if _, err = lkv.Get(context.TODO(), "k"); err != nil {
   278  		t.Fatal(err)
   279  	}
   280  	resp, err := lkv.Put(context.TODO(), "k", "def", clientv3.WithPrevKV())
   281  	if err != nil {
   282  		t.Fatal(err)
   283  	}
   284  	if resp.PrevKv == nil || string(resp.PrevKv.Value) != "abc" {
   285  		t.Fatalf(`expected PrevKV.Value="abc", got response %+v`, resp)
   286  	}
   287  }
   288  
   289  // TestLeasingRevGet checks the cache respects Get by Revision.
   290  func TestLeasingRevGet(t *testing.T) {
   291  	defer testutil.AfterTest(t)
   292  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
   293  	defer clus.Terminate(t)
   294  
   295  	lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/")
   296  	testutil.AssertNil(t, err)
   297  	defer closeLKV()
   298  
   299  	putResp, err := clus.Client(0).Put(context.TODO(), "k", "abc")
   300  	if err != nil {
   301  		t.Fatal(err)
   302  	}
   303  	if _, err = clus.Client(0).Put(context.TODO(), "k", "def"); err != nil {
   304  		t.Fatal(err)
   305  	}
   306  
   307  	// check historic revision
   308  	getResp, gerr := lkv.Get(context.TODO(), "k", clientv3.WithRev(putResp.Header.Revision))
   309  	if gerr != nil {
   310  		t.Fatal(gerr)
   311  	}
   312  	if len(getResp.Kvs) != 1 || string(getResp.Kvs[0].Value) != "abc" {
   313  		t.Fatalf(`expeted "k"->"abc" at rev=%d, got response %+v`, putResp.Header.Revision, getResp)
   314  	}
   315  	// check current revision
   316  	getResp, gerr = lkv.Get(context.TODO(), "k")
   317  	if gerr != nil {
   318  		t.Fatal(gerr)
   319  	}
   320  	if len(getResp.Kvs) != 1 || string(getResp.Kvs[0].Value) != "def" {
   321  		t.Fatalf(`expeted "k"->"abc" at rev=%d, got response %+v`, putResp.Header.Revision, getResp)
   322  	}
   323  }
   324  
   325  // TestLeasingGetWithOpts checks options that can be served through the cache do not depend on the server.
   326  func TestLeasingGetWithOpts(t *testing.T) {
   327  	defer testutil.AfterTest(t)
   328  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
   329  	defer clus.Terminate(t)
   330  
   331  	lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/")
   332  	testutil.AssertNil(t, err)
   333  	defer closeLKV()
   334  
   335  	if _, err = clus.Client(0).Put(context.TODO(), "k", "abc"); err != nil {
   336  		t.Fatal(err)
   337  	}
   338  	// in cache
   339  	if _, err = lkv.Get(context.TODO(), "k", clientv3.WithKeysOnly()); err != nil {
   340  		t.Fatal(err)
   341  	}
   342  
   343  	clus.Members[0].Stop(t)
   344  
   345  	opts := []clientv3.OpOption{
   346  		clientv3.WithKeysOnly(),
   347  		clientv3.WithLimit(1),
   348  		clientv3.WithMinCreateRev(1),
   349  		clientv3.WithMinModRev(1),
   350  		clientv3.WithSort(clientv3.SortByKey, clientv3.SortAscend),
   351  		clientv3.WithSerializable(),
   352  	}
   353  	for _, opt := range opts {
   354  		if _, err := lkv.Get(context.TODO(), "k", opt); err != nil {
   355  			t.Fatal(err)
   356  		}
   357  	}
   358  
   359  	getOpts := []clientv3.OpOption{}
   360  	for i := 0; i < len(opts); i++ {
   361  		getOpts = append(getOpts, opts[rand.Intn(len(opts))])
   362  	}
   363  	getOpts = getOpts[:rand.Intn(len(opts))]
   364  	if _, err := lkv.Get(context.TODO(), "k", getOpts...); err != nil {
   365  		t.Fatal(err)
   366  	}
   367  }
   368  
   369  // TestLeasingConcurrentPut ensures that a get after concurrent puts returns
   370  // the recently put data.
   371  func TestLeasingConcurrentPut(t *testing.T) {
   372  	defer testutil.AfterTest(t)
   373  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
   374  	defer clus.Terminate(t)
   375  
   376  	lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/")
   377  	testutil.AssertNil(t, err)
   378  	defer closeLKV()
   379  
   380  	// force key into leasing key cache
   381  	if _, err = lkv.Get(context.TODO(), "k"); err != nil {
   382  		t.Fatal(err)
   383  	}
   384  
   385  	// concurrently put through leasing client
   386  	numPuts := 16
   387  	putc := make(chan *clientv3.PutResponse, numPuts)
   388  	for i := 0; i < numPuts; i++ {
   389  		go func() {
   390  			resp, perr := lkv.Put(context.TODO(), "k", "abc")
   391  			if perr != nil {
   392  				t.Fatal(perr)
   393  			}
   394  			putc <- resp
   395  		}()
   396  	}
   397  	// record maximum revision from puts
   398  	maxRev := int64(0)
   399  	for i := 0; i < numPuts; i++ {
   400  		if resp := <-putc; resp.Header.Revision > maxRev {
   401  			maxRev = resp.Header.Revision
   402  		}
   403  	}
   404  
   405  	// confirm Get gives most recently put revisions
   406  	getResp, gerr := lkv.Get(context.TODO(), "k")
   407  	if gerr != nil {
   408  		t.Fatal(err)
   409  	}
   410  	if mr := getResp.Kvs[0].ModRevision; mr != maxRev {
   411  		t.Errorf("expected ModRevision %d, got %d", maxRev, mr)
   412  	}
   413  	if ver := getResp.Kvs[0].Version; ver != int64(numPuts) {
   414  		t.Errorf("expected Version %d, got %d", numPuts, ver)
   415  	}
   416  }
   417  
   418  func TestLeasingDisconnectedGet(t *testing.T) {
   419  	defer testutil.AfterTest(t)
   420  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
   421  	defer clus.Terminate(t)
   422  
   423  	lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/")
   424  	testutil.AssertNil(t, err)
   425  	defer closeLKV()
   426  
   427  	if _, err = clus.Client(0).Put(context.TODO(), "cached", "abc"); err != nil {
   428  		t.Fatal(err)
   429  	}
   430  	// get key so it's cached
   431  	if _, err = lkv.Get(context.TODO(), "cached"); err != nil {
   432  		t.Fatal(err)
   433  	}
   434  
   435  	clus.Members[0].Stop(t)
   436  
   437  	// leasing key ownership should have "cached" locally served
   438  	cachedResp, err := lkv.Get(context.TODO(), "cached")
   439  	if err != nil {
   440  		t.Fatal(err)
   441  	}
   442  	if len(cachedResp.Kvs) != 1 || string(cachedResp.Kvs[0].Value) != "abc" {
   443  		t.Fatalf(`expected "cached"->"abc", got response %+v`, cachedResp)
   444  	}
   445  }
   446  
   447  func TestLeasingDeleteOwner(t *testing.T) {
   448  	defer testutil.AfterTest(t)
   449  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
   450  	defer clus.Terminate(t)
   451  
   452  	lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/")
   453  	testutil.AssertNil(t, err)
   454  	defer closeLKV()
   455  
   456  	if _, err = clus.Client(0).Put(context.TODO(), "k", "abc"); err != nil {
   457  		t.Fatal(err)
   458  	}
   459  
   460  	// get+own / delete / get
   461  	if _, err = lkv.Get(context.TODO(), "k"); err != nil {
   462  		t.Fatal(err)
   463  	}
   464  	if _, err = lkv.Delete(context.TODO(), "k"); err != nil {
   465  		t.Fatal(err)
   466  	}
   467  	resp, err := lkv.Get(context.TODO(), "k")
   468  	if err != nil {
   469  		t.Fatal(err)
   470  	}
   471  
   472  	if len(resp.Kvs) != 0 {
   473  		t.Fatalf(`expected "k" to be deleted, got response %+v`, resp)
   474  	}
   475  	// try to double delete
   476  	if _, err = lkv.Delete(context.TODO(), "k"); err != nil {
   477  		t.Fatal(err)
   478  	}
   479  }
   480  
   481  func TestLeasingDeleteNonOwner(t *testing.T) {
   482  	defer testutil.AfterTest(t)
   483  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
   484  	defer clus.Terminate(t)
   485  
   486  	lkv1, closeLKV1, err := leasing.NewKV(clus.Client(0), "pfx/")
   487  	testutil.AssertNil(t, err)
   488  	defer closeLKV1()
   489  
   490  	lkv2, closeLKV2, err := leasing.NewKV(clus.Client(0), "pfx/")
   491  	testutil.AssertNil(t, err)
   492  	defer closeLKV2()
   493  
   494  	if _, err = clus.Client(0).Put(context.TODO(), "k", "abc"); err != nil {
   495  		t.Fatal(err)
   496  	}
   497  	// acquire ownership
   498  	if _, err = lkv1.Get(context.TODO(), "k"); err != nil {
   499  		t.Fatal(err)
   500  	}
   501  	// delete via non-owner
   502  	if _, err = lkv2.Delete(context.TODO(), "k"); err != nil {
   503  		t.Fatal(err)
   504  	}
   505  
   506  	// key should be removed from lkv1
   507  	resp, err := lkv1.Get(context.TODO(), "k")
   508  	if err != nil {
   509  		t.Fatal(err)
   510  	}
   511  	if len(resp.Kvs) != 0 {
   512  		t.Fatalf(`expected "k" to be deleted, got response %+v`, resp)
   513  	}
   514  }
   515  
   516  func TestLeasingOverwriteResponse(t *testing.T) {
   517  	defer testutil.AfterTest(t)
   518  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
   519  	defer clus.Terminate(t)
   520  
   521  	lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/")
   522  	testutil.AssertNil(t, err)
   523  	defer closeLKV()
   524  
   525  	if _, err = clus.Client(0).Put(context.TODO(), "k", "abc"); err != nil {
   526  		t.Fatal(err)
   527  	}
   528  
   529  	resp, err := lkv.Get(context.TODO(), "k")
   530  	if err != nil {
   531  		t.Fatal(err)
   532  	}
   533  
   534  	resp.Kvs[0].Key[0] = 'z'
   535  	resp.Kvs[0].Value[0] = 'z'
   536  
   537  	resp, err = lkv.Get(context.TODO(), "k")
   538  	if err != nil {
   539  		t.Fatal(err)
   540  	}
   541  
   542  	if string(resp.Kvs[0].Key) != "k" {
   543  		t.Errorf(`expected key "k", got %q`, string(resp.Kvs[0].Key))
   544  	}
   545  	if string(resp.Kvs[0].Value) != "abc" {
   546  		t.Errorf(`expected value "abc", got %q`, string(resp.Kvs[0].Value))
   547  	}
   548  }
   549  
   550  func TestLeasingOwnerPutResponse(t *testing.T) {
   551  	defer testutil.AfterTest(t)
   552  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
   553  	defer clus.Terminate(t)
   554  
   555  	lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/")
   556  	testutil.AssertNil(t, err)
   557  	defer closeLKV()
   558  
   559  	if _, err = clus.Client(0).Put(context.TODO(), "k", "abc"); err != nil {
   560  		t.Fatal(err)
   561  	}
   562  	gresp, gerr := lkv.Get(context.TODO(), "k")
   563  	if gerr != nil {
   564  		t.Fatal(gerr)
   565  	}
   566  	presp, err := lkv.Put(context.TODO(), "k", "def")
   567  	if err != nil {
   568  		t.Fatal(err)
   569  	}
   570  	if presp == nil {
   571  		t.Fatal("expected put response, got nil")
   572  	}
   573  
   574  	clus.Members[0].Stop(t)
   575  
   576  	gresp, gerr = lkv.Get(context.TODO(), "k")
   577  	if gerr != nil {
   578  		t.Fatal(gerr)
   579  	}
   580  	if gresp.Kvs[0].ModRevision != presp.Header.Revision {
   581  		t.Errorf("expected mod revision %d, got %d", presp.Header.Revision, gresp.Kvs[0].ModRevision)
   582  	}
   583  	if gresp.Kvs[0].Version != 2 {
   584  		t.Errorf("expected version 2, got version %d", gresp.Kvs[0].Version)
   585  	}
   586  }
   587  
   588  func TestLeasingTxnOwnerGetRange(t *testing.T) {
   589  	defer testutil.AfterTest(t)
   590  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
   591  	defer clus.Terminate(t)
   592  
   593  	lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/")
   594  	testutil.AssertNil(t, err)
   595  	defer closeLKV()
   596  
   597  	keyCount := rand.Intn(10) + 1
   598  	for i := 0; i < keyCount; i++ {
   599  		k := fmt.Sprintf("k-%d", i)
   600  		if _, err := clus.Client(0).Put(context.TODO(), k, k+k); err != nil {
   601  			t.Fatal(err)
   602  		}
   603  	}
   604  	if _, err := lkv.Get(context.TODO(), "k-"); err != nil {
   605  		t.Fatal(err)
   606  	}
   607  
   608  	tresp, terr := lkv.Txn(context.TODO()).Then(clientv3.OpGet("k-", clientv3.WithPrefix())).Commit()
   609  	if terr != nil {
   610  		t.Fatal(terr)
   611  	}
   612  	if resp := tresp.Responses[0].GetResponseRange(); len(resp.Kvs) != keyCount {
   613  		t.Fatalf("expected %d keys, got response %+v", keyCount, resp.Kvs)
   614  	}
   615  }
   616  
   617  func TestLeasingTxnOwnerGet(t *testing.T) {
   618  	defer testutil.AfterTest(t)
   619  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
   620  	defer clus.Terminate(t)
   621  
   622  	lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/")
   623  	testutil.AssertNil(t, err)
   624  	defer closeLKV()
   625  
   626  	keyCount := rand.Intn(10) + 1
   627  	var ops []clientv3.Op
   628  	presps := make([]*clientv3.PutResponse, keyCount)
   629  	for i := range presps {
   630  		k := fmt.Sprintf("k-%d", i)
   631  		presp, err := clus.Client(0).Put(context.TODO(), k, k+k)
   632  		if err != nil {
   633  			t.Fatal(err)
   634  		}
   635  		presps[i] = presp
   636  
   637  		if _, err = lkv.Get(context.TODO(), k); err != nil {
   638  			t.Fatal(err)
   639  		}
   640  		ops = append(ops, clientv3.OpGet(k))
   641  	}
   642  	ops = ops[:rand.Intn(len(ops))]
   643  
   644  	// served through cache
   645  	clus.Members[0].Stop(t)
   646  
   647  	var thenOps, elseOps []clientv3.Op
   648  	cmps, useThen := randCmps("k-", presps)
   649  
   650  	if useThen {
   651  
   652  		thenOps = ops
   653  		elseOps = []clientv3.Op{clientv3.OpPut("k", "1")}
   654  	} else {
   655  		thenOps = []clientv3.Op{clientv3.OpPut("k", "1")}
   656  		elseOps = ops
   657  	}
   658  
   659  	tresp, terr := lkv.Txn(context.TODO()).
   660  		If(cmps...).
   661  		Then(thenOps...).
   662  		Else(elseOps...).Commit()
   663  
   664  	if terr != nil {
   665  		t.Fatal(terr)
   666  	}
   667  	if tresp.Succeeded != useThen {
   668  		t.Fatalf("expected succeeded=%v, got tresp=%+v", useThen, tresp)
   669  	}
   670  	if len(tresp.Responses) != len(ops) {
   671  		t.Fatalf("expected %d responses, got %d", len(ops), len(tresp.Responses))
   672  	}
   673  	wrev := presps[len(presps)-1].Header.Revision
   674  	if tresp.Header.Revision < wrev {
   675  		t.Fatalf("expected header revision >= %d, got %d", wrev, tresp.Header.Revision)
   676  	}
   677  	for i := range ops {
   678  		k := fmt.Sprintf("k-%d", i)
   679  		rr := tresp.Responses[i].GetResponseRange()
   680  		if rr == nil {
   681  			t.Errorf("expected get response, got %+v", tresp.Responses[i])
   682  		}
   683  		if string(rr.Kvs[0].Key) != k || string(rr.Kvs[0].Value) != k+k {
   684  			t.Errorf(`expected key for %q, got %+v`, k, rr.Kvs)
   685  		}
   686  	}
   687  }
   688  
   689  func TestLeasingTxnOwnerDeleteRange(t *testing.T) {
   690  	defer testutil.AfterTest(t)
   691  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
   692  	defer clus.Terminate(t)
   693  
   694  	lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/")
   695  	testutil.AssertNil(t, err)
   696  	defer closeLKV()
   697  
   698  	keyCount := rand.Intn(10) + 1
   699  	for i := 0; i < keyCount; i++ {
   700  		k := fmt.Sprintf("k-%d", i)
   701  		if _, perr := clus.Client(0).Put(context.TODO(), k, k+k); perr != nil {
   702  			t.Fatal(perr)
   703  		}
   704  	}
   705  
   706  	// cache in lkv
   707  	resp, err := lkv.Get(context.TODO(), "k-", clientv3.WithPrefix())
   708  	if err != nil {
   709  		t.Fatal(err)
   710  	}
   711  	if len(resp.Kvs) != keyCount {
   712  		t.Fatalf("expected %d keys, got %d", keyCount, len(resp.Kvs))
   713  	}
   714  
   715  	if _, terr := lkv.Txn(context.TODO()).Then(clientv3.OpDelete("k-", clientv3.WithPrefix())).Commit(); terr != nil {
   716  		t.Fatal(terr)
   717  	}
   718  
   719  	resp, err = lkv.Get(context.TODO(), "k-", clientv3.WithPrefix())
   720  	if err != nil {
   721  		t.Fatal(err)
   722  	}
   723  	if len(resp.Kvs) != 0 {
   724  		t.Fatalf("expected no keys, got %d", len(resp.Kvs))
   725  	}
   726  }
   727  
   728  func TestLeasingTxnOwnerDelete(t *testing.T) {
   729  	defer testutil.AfterTest(t)
   730  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
   731  	defer clus.Terminate(t)
   732  
   733  	lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/")
   734  	testutil.AssertNil(t, err)
   735  	defer closeLKV()
   736  
   737  	if _, err = clus.Client(0).Put(context.TODO(), "k", "abc"); err != nil {
   738  		t.Fatal(err)
   739  	}
   740  
   741  	// cache in lkv
   742  	if _, gerr := lkv.Get(context.TODO(), "k"); gerr != nil {
   743  		t.Fatal(gerr)
   744  	}
   745  
   746  	if _, terr := lkv.Txn(context.TODO()).Then(clientv3.OpDelete("k")).Commit(); terr != nil {
   747  		t.Fatal(terr)
   748  	}
   749  
   750  	resp, err := lkv.Get(context.TODO(), "k")
   751  	if err != nil {
   752  		t.Fatal(err)
   753  	}
   754  	if len(resp.Kvs) != 0 {
   755  		t.Fatalf("expected no keys, got %d", len(resp.Kvs))
   756  	}
   757  }
   758  
   759  func TestLeasingTxnOwnerIf(t *testing.T) {
   760  	defer testutil.AfterTest(t)
   761  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
   762  	defer clus.Terminate(t)
   763  
   764  	lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/")
   765  	testutil.AssertNil(t, err)
   766  	defer closeLKV()
   767  
   768  	if _, err = clus.Client(0).Put(context.TODO(), "k", "abc"); err != nil {
   769  		t.Fatal(err)
   770  	}
   771  	if _, err = lkv.Get(context.TODO(), "k"); err != nil {
   772  		t.Fatal(err)
   773  	}
   774  
   775  	// served through cache
   776  	clus.Members[0].Stop(t)
   777  
   778  	tests := []struct {
   779  		cmps       []clientv3.Cmp
   780  		wSucceeded bool
   781  		wResponses int
   782  	}{
   783  		// success
   784  		{
   785  			cmps:       []clientv3.Cmp{clientv3.Compare(clientv3.Value("k"), "=", "abc")},
   786  			wSucceeded: true,
   787  			wResponses: 1,
   788  		},
   789  		{
   790  			cmps:       []clientv3.Cmp{clientv3.Compare(clientv3.CreateRevision("k"), "=", 2)},
   791  			wSucceeded: true,
   792  			wResponses: 1,
   793  		},
   794  		{
   795  			cmps:       []clientv3.Cmp{clientv3.Compare(clientv3.ModRevision("k"), "=", 2)},
   796  			wSucceeded: true,
   797  			wResponses: 1,
   798  		},
   799  		{
   800  			cmps:       []clientv3.Cmp{clientv3.Compare(clientv3.Version("k"), "=", 1)},
   801  			wSucceeded: true,
   802  			wResponses: 1,
   803  		},
   804  		// failure
   805  		{
   806  			cmps: []clientv3.Cmp{clientv3.Compare(clientv3.Value("k"), ">", "abc")},
   807  		},
   808  		{
   809  			cmps: []clientv3.Cmp{clientv3.Compare(clientv3.CreateRevision("k"), ">", 2)},
   810  		},
   811  		{
   812  			cmps:       []clientv3.Cmp{clientv3.Compare(clientv3.ModRevision("k"), "=", 2)},
   813  			wSucceeded: true,
   814  			wResponses: 1,
   815  		},
   816  		{
   817  			cmps: []clientv3.Cmp{clientv3.Compare(clientv3.Version("k"), ">", 1)},
   818  		},
   819  		{
   820  			cmps: []clientv3.Cmp{clientv3.Compare(clientv3.Value("k"), "<", "abc")},
   821  		},
   822  		{
   823  			cmps: []clientv3.Cmp{clientv3.Compare(clientv3.CreateRevision("k"), "<", 2)},
   824  		},
   825  		{
   826  			cmps: []clientv3.Cmp{clientv3.Compare(clientv3.ModRevision("k"), "<", 2)},
   827  		},
   828  		{
   829  			cmps: []clientv3.Cmp{clientv3.Compare(clientv3.Version("k"), "<", 1)},
   830  		},
   831  		{
   832  			cmps: []clientv3.Cmp{
   833  				clientv3.Compare(clientv3.Version("k"), "=", 1),
   834  				clientv3.Compare(clientv3.Version("k"), "<", 1),
   835  			},
   836  		},
   837  	}
   838  
   839  	for i, tt := range tests {
   840  		tresp, terr := lkv.Txn(context.TODO()).If(tt.cmps...).Then(clientv3.OpGet("k")).Commit()
   841  		if terr != nil {
   842  			t.Fatal(terr)
   843  		}
   844  		if tresp.Succeeded != tt.wSucceeded {
   845  			t.Errorf("#%d: expected succeeded %v, got %v", i, tt.wSucceeded, tresp.Succeeded)
   846  		}
   847  		if len(tresp.Responses) != tt.wResponses {
   848  			t.Errorf("#%d: expected %d responses, got %d", i, tt.wResponses, len(tresp.Responses))
   849  		}
   850  	}
   851  }
   852  
   853  func TestLeasingTxnCancel(t *testing.T) {
   854  	defer testutil.AfterTest(t)
   855  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3})
   856  	defer clus.Terminate(t)
   857  
   858  	lkv1, closeLKV1, err := leasing.NewKV(clus.Client(0), "pfx/")
   859  	testutil.AssertNil(t, err)
   860  	defer closeLKV1()
   861  
   862  	lkv2, closeLKV2, err := leasing.NewKV(clus.Client(1), "pfx/")
   863  	testutil.AssertNil(t, err)
   864  	defer closeLKV2()
   865  
   866  	// acquire lease but disconnect so no revoke in time
   867  	if _, err = lkv1.Get(context.TODO(), "k"); err != nil {
   868  		t.Fatal(err)
   869  	}
   870  	clus.Members[0].Stop(t)
   871  
   872  	// wait for leader election, if any
   873  	if _, err = clus.Client(1).Get(context.TODO(), "abc"); err != nil {
   874  		t.Fatal(err)
   875  	}
   876  
   877  	ctx, cancel := context.WithCancel(context.TODO())
   878  	go func() {
   879  		time.Sleep(100 * time.Millisecond)
   880  		cancel()
   881  	}()
   882  	if _, err := lkv2.Txn(ctx).Then(clientv3.OpPut("k", "v")).Commit(); err != context.Canceled {
   883  		t.Fatalf("expected %v, got %v", context.Canceled, err)
   884  	}
   885  }
   886  
   887  func TestLeasingTxnNonOwnerPut(t *testing.T) {
   888  	defer testutil.AfterTest(t)
   889  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
   890  	defer clus.Terminate(t)
   891  
   892  	lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/")
   893  	testutil.AssertNil(t, err)
   894  	defer closeLKV()
   895  
   896  	lkv2, closeLKV2, err := leasing.NewKV(clus.Client(0), "pfx/")
   897  	testutil.AssertNil(t, err)
   898  	defer closeLKV2()
   899  
   900  	if _, err = clus.Client(0).Put(context.TODO(), "k", "abc"); err != nil {
   901  		t.Fatal(err)
   902  	}
   903  	if _, err = clus.Client(0).Put(context.TODO(), "k2", "123"); err != nil {
   904  		t.Fatal(err)
   905  	}
   906  	// cache in lkv
   907  	if _, err = lkv.Get(context.TODO(), "k"); err != nil {
   908  		t.Fatal(err)
   909  	}
   910  	if _, err = lkv.Get(context.TODO(), "k2"); err != nil {
   911  		t.Fatal(err)
   912  	}
   913  	// invalidate via lkv2 txn
   914  	opArray := make([]clientv3.Op, 0)
   915  	opArray = append(opArray, clientv3.OpPut("k2", "456"))
   916  	tresp, terr := lkv2.Txn(context.TODO()).Then(
   917  		clientv3.OpTxn(nil, opArray, nil),
   918  		clientv3.OpPut("k", "def"),
   919  		clientv3.OpPut("k3", "999"), // + a key not in any cache
   920  	).Commit()
   921  	if terr != nil {
   922  		t.Fatal(terr)
   923  	}
   924  	if !tresp.Succeeded || len(tresp.Responses) != 3 {
   925  		t.Fatalf("expected txn success, got %+v", tresp)
   926  	}
   927  	// check cache was invalidated
   928  	gresp, gerr := lkv.Get(context.TODO(), "k")
   929  	if gerr != nil {
   930  		t.Fatal(err)
   931  	}
   932  	if len(gresp.Kvs) != 1 || string(gresp.Kvs[0].Value) != "def" {
   933  		t.Errorf(`expected value "def", got %+v`, gresp)
   934  	}
   935  	gresp, gerr = lkv.Get(context.TODO(), "k2")
   936  	if gerr != nil {
   937  		t.Fatal(gerr)
   938  	}
   939  	if len(gresp.Kvs) != 1 || string(gresp.Kvs[0].Value) != "456" {
   940  		t.Errorf(`expected value "def", got %+v`, gresp)
   941  	}
   942  	// check puts were applied and are all in the same revision
   943  	w := clus.Client(0).Watch(
   944  		clus.Client(0).Ctx(),
   945  		"k",
   946  		clientv3.WithRev(tresp.Header.Revision),
   947  		clientv3.WithPrefix())
   948  	wresp := <-w
   949  	c := 0
   950  	evs := []clientv3.Event{}
   951  	for _, ev := range wresp.Events {
   952  		evs = append(evs, *ev)
   953  		if ev.Kv.ModRevision == tresp.Header.Revision {
   954  			c++
   955  		}
   956  	}
   957  	if c != 3 {
   958  		t.Fatalf("expected 3 put events, got %+v", evs)
   959  	}
   960  }
   961  
   962  // TestLeasingTxnRandIfThenOrElse randomly leases keys two separate clients, then
   963  // issues a random If/{Then,Else} transaction on those keys to one client.
   964  func TestLeasingTxnRandIfThenOrElse(t *testing.T) {
   965  	defer testutil.AfterTest(t)
   966  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
   967  	defer clus.Terminate(t)
   968  
   969  	lkv1, closeLKV1, err1 := leasing.NewKV(clus.Client(0), "pfx/")
   970  	testutil.AssertNil(t, err1)
   971  	defer closeLKV1()
   972  
   973  	lkv2, closeLKV2, err2 := leasing.NewKV(clus.Client(0), "pfx/")
   974  	testutil.AssertNil(t, err2)
   975  	defer closeLKV2()
   976  
   977  	keyCount := 16
   978  	dat := make([]*clientv3.PutResponse, keyCount)
   979  	for i := 0; i < keyCount; i++ {
   980  		k, v := fmt.Sprintf("k-%d", i), fmt.Sprintf("%d", i)
   981  		dat[i], err1 = clus.Client(0).Put(context.TODO(), k, v)
   982  		if err1 != nil {
   983  			t.Fatal(err1)
   984  		}
   985  	}
   986  
   987  	// nondeterministically populate leasing caches
   988  	var wg sync.WaitGroup
   989  	getc := make(chan struct{}, keyCount)
   990  	getRandom := func(kv clientv3.KV) {
   991  		defer wg.Done()
   992  		for i := 0; i < keyCount/2; i++ {
   993  			k := fmt.Sprintf("k-%d", rand.Intn(keyCount))
   994  			if _, err := kv.Get(context.TODO(), k); err != nil {
   995  				t.Fatal(err)
   996  			}
   997  			getc <- struct{}{}
   998  		}
   999  	}
  1000  	wg.Add(2)
  1001  	defer wg.Wait()
  1002  	go getRandom(lkv1)
  1003  	go getRandom(lkv2)
  1004  
  1005  	// random list of comparisons, all true
  1006  	cmps, useThen := randCmps("k-", dat)
  1007  	// random list of puts/gets; unique keys
  1008  	ops := []clientv3.Op{}
  1009  	usedIdx := make(map[int]struct{})
  1010  	for i := 0; i < keyCount; i++ {
  1011  		idx := rand.Intn(keyCount)
  1012  		if _, ok := usedIdx[idx]; ok {
  1013  			continue
  1014  		}
  1015  		usedIdx[idx] = struct{}{}
  1016  		k := fmt.Sprintf("k-%d", idx)
  1017  		switch rand.Intn(2) {
  1018  		case 0:
  1019  			ops = append(ops, clientv3.OpGet(k))
  1020  		case 1:
  1021  			ops = append(ops, clientv3.OpPut(k, "a"))
  1022  			// TODO: add delete
  1023  		}
  1024  	}
  1025  	// random lengths
  1026  	ops = ops[:rand.Intn(len(ops))]
  1027  
  1028  	// wait for some gets to populate the leasing caches before committing
  1029  	for i := 0; i < keyCount/2; i++ {
  1030  		<-getc
  1031  	}
  1032  
  1033  	// randomly choose between then and else blocks
  1034  	var thenOps, elseOps []clientv3.Op
  1035  	if useThen {
  1036  		thenOps = ops
  1037  	} else {
  1038  		// force failure
  1039  		elseOps = ops
  1040  	}
  1041  
  1042  	tresp, terr := lkv1.Txn(context.TODO()).If(cmps...).Then(thenOps...).Else(elseOps...).Commit()
  1043  	if terr != nil {
  1044  		t.Fatal(terr)
  1045  	}
  1046  	// cmps always succeed
  1047  	if tresp.Succeeded != useThen {
  1048  		t.Fatalf("expected succeeded=%v, got tresp=%+v", useThen, tresp)
  1049  	}
  1050  	// get should match what was put
  1051  	checkPuts := func(s string, kv clientv3.KV) {
  1052  		for _, op := range ops {
  1053  			if !op.IsPut() {
  1054  				continue
  1055  			}
  1056  			resp, rerr := kv.Get(context.TODO(), string(op.KeyBytes()))
  1057  			if rerr != nil {
  1058  				t.Fatal(rerr)
  1059  			}
  1060  			if len(resp.Kvs) != 1 || string(resp.Kvs[0].Value) != "a" {
  1061  				t.Fatalf(`%s: expected value="a", got %+v`, s, resp.Kvs)
  1062  			}
  1063  		}
  1064  	}
  1065  	checkPuts("client(0)", clus.Client(0))
  1066  	checkPuts("lkv1", lkv1)
  1067  	checkPuts("lkv2", lkv2)
  1068  }
  1069  
  1070  func TestLeasingOwnerPutError(t *testing.T) {
  1071  	defer testutil.AfterTest(t)
  1072  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
  1073  	defer clus.Terminate(t)
  1074  
  1075  	lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/")
  1076  	testutil.AssertNil(t, err)
  1077  	defer closeLKV()
  1078  
  1079  	if _, err = lkv.Get(context.TODO(), "k"); err != nil {
  1080  		t.Fatal(err)
  1081  	}
  1082  
  1083  	clus.Members[0].Stop(t)
  1084  	ctx, cancel := context.WithTimeout(context.TODO(), 100*time.Millisecond)
  1085  	defer cancel()
  1086  	if resp, err := lkv.Put(ctx, "k", "v"); err == nil {
  1087  		t.Fatalf("expected error, got response %+v", resp)
  1088  	}
  1089  }
  1090  
  1091  func TestLeasingOwnerDeleteError(t *testing.T) {
  1092  	defer testutil.AfterTest(t)
  1093  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
  1094  	defer clus.Terminate(t)
  1095  
  1096  	lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/")
  1097  	testutil.AssertNil(t, err)
  1098  	defer closeLKV()
  1099  
  1100  	if _, err = lkv.Get(context.TODO(), "k"); err != nil {
  1101  		t.Fatal(err)
  1102  	}
  1103  
  1104  	clus.Members[0].Stop(t)
  1105  	ctx, cancel := context.WithTimeout(context.TODO(), 100*time.Millisecond)
  1106  	defer cancel()
  1107  	if resp, err := lkv.Delete(ctx, "k"); err == nil {
  1108  		t.Fatalf("expected error, got response %+v", resp)
  1109  	}
  1110  }
  1111  
  1112  func TestLeasingNonOwnerPutError(t *testing.T) {
  1113  	defer testutil.AfterTest(t)
  1114  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
  1115  	defer clus.Terminate(t)
  1116  
  1117  	lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/")
  1118  	testutil.AssertNil(t, err)
  1119  	defer closeLKV()
  1120  
  1121  	clus.Members[0].Stop(t)
  1122  	ctx, cancel := context.WithTimeout(context.TODO(), 100*time.Millisecond)
  1123  	defer cancel()
  1124  	if resp, err := lkv.Put(ctx, "k", "v"); err == nil {
  1125  		t.Fatalf("expected error, got response %+v", resp)
  1126  	}
  1127  }
  1128  
  1129  func TestLeasingOwnerDeletePrefix(t *testing.T) {
  1130  	testLeasingOwnerDelete(t, clientv3.OpDelete("key/", clientv3.WithPrefix()))
  1131  }
  1132  
  1133  func TestLeasingOwnerDeleteFrom(t *testing.T) {
  1134  	testLeasingOwnerDelete(t, clientv3.OpDelete("kd", clientv3.WithFromKey()))
  1135  }
  1136  
  1137  func testLeasingOwnerDelete(t *testing.T, del clientv3.Op) {
  1138  	defer testutil.AfterTest(t)
  1139  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
  1140  	defer clus.Terminate(t)
  1141  
  1142  	lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "0/")
  1143  	testutil.AssertNil(t, err)
  1144  	defer closeLKV()
  1145  
  1146  	for i := 0; i < 8; i++ {
  1147  		if _, err = clus.Client(0).Put(context.TODO(), fmt.Sprintf("key/%d", i), "123"); err != nil {
  1148  			t.Fatal(err)
  1149  		}
  1150  	}
  1151  
  1152  	if _, err = lkv.Get(context.TODO(), "key/1"); err != nil {
  1153  		t.Fatal(err)
  1154  	}
  1155  
  1156  	opResp, delErr := lkv.Do(context.TODO(), del)
  1157  	if delErr != nil {
  1158  		t.Fatal(delErr)
  1159  	}
  1160  	delResp := opResp.Del()
  1161  
  1162  	// confirm keys are invalidated from cache and deleted on etcd
  1163  	for i := 0; i < 8; i++ {
  1164  		resp, err := lkv.Get(context.TODO(), fmt.Sprintf("key/%d", i))
  1165  		if err != nil {
  1166  			t.Fatal(err)
  1167  		}
  1168  		if len(resp.Kvs) != 0 {
  1169  			t.Fatalf("expected no keys on key/%d, got %+v", i, resp)
  1170  		}
  1171  	}
  1172  
  1173  	// confirm keys were deleted atomically
  1174  
  1175  	w := clus.Client(0).Watch(
  1176  		clus.Client(0).Ctx(),
  1177  		"key/",
  1178  		clientv3.WithRev(delResp.Header.Revision),
  1179  		clientv3.WithPrefix())
  1180  
  1181  	if wresp := <-w; len(wresp.Events) != 8 {
  1182  		t.Fatalf("expected %d delete events,got %d", 8, len(wresp.Events))
  1183  	}
  1184  }
  1185  
  1186  func TestLeasingDeleteRangeBounds(t *testing.T) {
  1187  	defer testutil.AfterTest(t)
  1188  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
  1189  	defer clus.Terminate(t)
  1190  
  1191  	delkv, closeDelKV, err := leasing.NewKV(clus.Client(0), "0/")
  1192  	testutil.AssertNil(t, err)
  1193  	defer closeDelKV()
  1194  
  1195  	getkv, closeGetKv, err := leasing.NewKV(clus.Client(0), "0/")
  1196  	testutil.AssertNil(t, err)
  1197  	defer closeGetKv()
  1198  
  1199  	for _, k := range []string{"j", "m"} {
  1200  		if _, err = clus.Client(0).Put(context.TODO(), k, "123"); err != nil {
  1201  			t.Fatal(err)
  1202  		}
  1203  		if _, err = getkv.Get(context.TODO(), k); err != nil {
  1204  			t.Fatal(err)
  1205  		}
  1206  	}
  1207  
  1208  	if _, err = delkv.Delete(context.TODO(), "k", clientv3.WithPrefix()); err != nil {
  1209  		t.Fatal(err)
  1210  	}
  1211  
  1212  	// leases still on server?
  1213  	for _, k := range []string{"j", "m"} {
  1214  		resp, geterr := clus.Client(0).Get(context.TODO(), "0/"+k, clientv3.WithPrefix())
  1215  		if geterr != nil {
  1216  			t.Fatal(geterr)
  1217  		}
  1218  		if len(resp.Kvs) != 1 {
  1219  			t.Fatalf("expected leasing key, got %+v", resp)
  1220  		}
  1221  	}
  1222  
  1223  	// j and m should still have leases registered since not under k*
  1224  	clus.Members[0].Stop(t)
  1225  
  1226  	if _, err = getkv.Get(context.TODO(), "j"); err != nil {
  1227  		t.Fatal(err)
  1228  	}
  1229  	if _, err = getkv.Get(context.TODO(), "m"); err != nil {
  1230  		t.Fatal(err)
  1231  	}
  1232  }
  1233  
  1234  func TestLeasingDeleteRangeContendTxn(t *testing.T) {
  1235  	then := []clientv3.Op{clientv3.OpDelete("key/", clientv3.WithPrefix())}
  1236  	testLeasingDeleteRangeContend(t, clientv3.OpTxn(nil, then, nil))
  1237  }
  1238  
  1239  func TestLeaseDeleteRangeContendDel(t *testing.T) {
  1240  	op := clientv3.OpDelete("key/", clientv3.WithPrefix())
  1241  	testLeasingDeleteRangeContend(t, op)
  1242  }
  1243  
  1244  func testLeasingDeleteRangeContend(t *testing.T, op clientv3.Op) {
  1245  	defer testutil.AfterTest(t)
  1246  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
  1247  	defer clus.Terminate(t)
  1248  
  1249  	delkv, closeDelKV, err := leasing.NewKV(clus.Client(0), "0/")
  1250  	testutil.AssertNil(t, err)
  1251  	defer closeDelKV()
  1252  
  1253  	putkv, closePutKV, err := leasing.NewKV(clus.Client(0), "0/")
  1254  	testutil.AssertNil(t, err)
  1255  	defer closePutKV()
  1256  
  1257  	for i := 0; i < 8; i++ {
  1258  		key := fmt.Sprintf("key/%d", i)
  1259  		if _, err = clus.Client(0).Put(context.TODO(), key, "123"); err != nil {
  1260  			t.Fatal(err)
  1261  		}
  1262  		if _, err = putkv.Get(context.TODO(), key); err != nil {
  1263  			t.Fatal(err)
  1264  		}
  1265  	}
  1266  
  1267  	ctx, cancel := context.WithCancel(context.TODO())
  1268  	donec := make(chan struct{})
  1269  	go func() {
  1270  		defer close(donec)
  1271  		for i := 0; ctx.Err() == nil; i++ {
  1272  			key := fmt.Sprintf("key/%d", i%8)
  1273  			putkv.Put(ctx, key, "123")
  1274  			putkv.Get(ctx, key)
  1275  		}
  1276  	}()
  1277  
  1278  	_, delErr := delkv.Do(context.TODO(), op)
  1279  	cancel()
  1280  	<-donec
  1281  	if delErr != nil {
  1282  		t.Fatal(delErr)
  1283  	}
  1284  
  1285  	// confirm keys on non-deleter match etcd
  1286  	for i := 0; i < 8; i++ {
  1287  		key := fmt.Sprintf("key/%d", i)
  1288  		resp, err := putkv.Get(context.TODO(), key)
  1289  		if err != nil {
  1290  			t.Fatal(err)
  1291  		}
  1292  		servResp, err := clus.Client(0).Get(context.TODO(), key)
  1293  		if err != nil {
  1294  			t.Fatal(err)
  1295  		}
  1296  		if !reflect.DeepEqual(resp.Kvs, servResp.Kvs) {
  1297  			t.Errorf("#%d: expected %+v, got %+v", i, servResp.Kvs, resp.Kvs)
  1298  		}
  1299  	}
  1300  }
  1301  
  1302  func TestLeasingPutGetDeleteConcurrent(t *testing.T) {
  1303  	defer testutil.AfterTest(t)
  1304  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
  1305  	defer clus.Terminate(t)
  1306  
  1307  	lkvs := make([]clientv3.KV, 16)
  1308  	for i := range lkvs {
  1309  		lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/")
  1310  		testutil.AssertNil(t, err)
  1311  		defer closeLKV()
  1312  		lkvs[i] = lkv
  1313  	}
  1314  
  1315  	getdel := func(kv clientv3.KV) {
  1316  		if _, err := kv.Put(context.TODO(), "k", "abc"); err != nil {
  1317  			t.Fatal(err)
  1318  		}
  1319  		time.Sleep(time.Millisecond)
  1320  		if _, err := kv.Get(context.TODO(), "k"); err != nil {
  1321  			t.Fatal(err)
  1322  		}
  1323  		if _, err := kv.Delete(context.TODO(), "k"); err != nil {
  1324  			t.Fatal(err)
  1325  		}
  1326  		time.Sleep(2 * time.Millisecond)
  1327  	}
  1328  
  1329  	var wg sync.WaitGroup
  1330  	wg.Add(16)
  1331  	for i := 0; i < 16; i++ {
  1332  		go func() {
  1333  			defer wg.Done()
  1334  			for _, kv := range lkvs {
  1335  				getdel(kv)
  1336  			}
  1337  		}()
  1338  	}
  1339  	wg.Wait()
  1340  
  1341  	resp, err := lkvs[0].Get(context.TODO(), "k")
  1342  	if err != nil {
  1343  		t.Fatal(err)
  1344  	}
  1345  
  1346  	if len(resp.Kvs) > 0 {
  1347  		t.Fatalf("expected no kvs, got %+v", resp.Kvs)
  1348  	}
  1349  
  1350  	resp, err = clus.Client(0).Get(context.TODO(), "k")
  1351  	if err != nil {
  1352  		t.Fatal(err)
  1353  	}
  1354  	if len(resp.Kvs) > 0 {
  1355  		t.Fatalf("expected no kvs, got %+v", resp.Kvs)
  1356  	}
  1357  }
  1358  
  1359  // TestLeasingReconnectOwnerRevoke checks that revocation works if
  1360  // disconnected when trying to submit revoke txn.
  1361  func TestLeasingReconnectOwnerRevoke(t *testing.T) {
  1362  	defer testutil.AfterTest(t)
  1363  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3})
  1364  	defer clus.Terminate(t)
  1365  
  1366  	lkv1, closeLKV1, err1 := leasing.NewKV(clus.Client(0), "foo/")
  1367  	testutil.AssertNil(t, err1)
  1368  	defer closeLKV1()
  1369  
  1370  	lkv2, closeLKV2, err2 := leasing.NewKV(clus.Client(1), "foo/")
  1371  	testutil.AssertNil(t, err2)
  1372  	defer closeLKV2()
  1373  
  1374  	if _, err := lkv1.Get(context.TODO(), "k"); err != nil {
  1375  		t.Fatal(err)
  1376  	}
  1377  
  1378  	// force leader away from member 0
  1379  	clus.Members[0].Stop(t)
  1380  	clus.WaitLeader(t)
  1381  	clus.Members[0].Restart(t)
  1382  
  1383  	cctx, cancel := context.WithCancel(context.TODO())
  1384  	sdonec, pdonec := make(chan struct{}), make(chan struct{})
  1385  	// make lkv1 connection choppy so Txn fails
  1386  	go func() {
  1387  		defer close(sdonec)
  1388  		for i := 0; i < 10 && cctx.Err() == nil; i++ {
  1389  			clus.Members[0].Stop(t)
  1390  			time.Sleep(10 * time.Millisecond)
  1391  			clus.Members[0].Restart(t)
  1392  		}
  1393  	}()
  1394  	go func() {
  1395  		defer close(pdonec)
  1396  		if _, err := lkv2.Put(cctx, "k", "v"); err != nil {
  1397  			t.Log(err)
  1398  		}
  1399  		resp, err := lkv1.Get(cctx, "k")
  1400  		if err != nil {
  1401  			t.Fatal(err)
  1402  		}
  1403  		if string(resp.Kvs[0].Value) != "v" {
  1404  			t.Fatalf(`expected "v" value, got %+v`, resp)
  1405  		}
  1406  	}()
  1407  	select {
  1408  	case <-pdonec:
  1409  		cancel()
  1410  		<-sdonec
  1411  	case <-time.After(10 * time.Second):
  1412  		cancel()
  1413  		<-sdonec
  1414  		<-pdonec
  1415  		t.Fatal("took to long to revoke and put")
  1416  	}
  1417  }
  1418  
  1419  // TestLeasingReconnectOwnerRevokeCompact checks that revocation works if
  1420  // disconnected and the watch is compacted.
  1421  func TestLeasingReconnectOwnerRevokeCompact(t *testing.T) {
  1422  	defer testutil.AfterTest(t)
  1423  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3})
  1424  	defer clus.Terminate(t)
  1425  
  1426  	lkv1, closeLKV1, err1 := leasing.NewKV(clus.Client(0), "foo/")
  1427  	testutil.AssertNil(t, err1)
  1428  	defer closeLKV1()
  1429  
  1430  	lkv2, closeLKV2, err2 := leasing.NewKV(clus.Client(1), "foo/")
  1431  	testutil.AssertNil(t, err2)
  1432  	defer closeLKV2()
  1433  
  1434  	if _, err := lkv1.Get(context.TODO(), "k"); err != nil {
  1435  		t.Fatal(err)
  1436  	}
  1437  
  1438  	clus.Members[0].Stop(t)
  1439  	clus.WaitLeader(t)
  1440  
  1441  	// put some more revisions for compaction
  1442  	presp, err := clus.Client(1).Put(context.TODO(), "a", "123")
  1443  	if err != nil {
  1444  		t.Fatal(err)
  1445  	}
  1446  	presp, err = clus.Client(1).Put(context.TODO(), "a", "123")
  1447  	if err != nil {
  1448  		t.Fatal(err)
  1449  	}
  1450  	// compact while lkv1 is disconnected
  1451  	rev := presp.Header.Revision
  1452  	if _, err = clus.Client(1).Compact(context.TODO(), rev); err != nil {
  1453  		t.Fatal(err)
  1454  	}
  1455  
  1456  	clus.Members[0].Restart(t)
  1457  
  1458  	cctx, cancel := context.WithTimeout(context.TODO(), 5*time.Second)
  1459  	defer cancel()
  1460  	if _, err = lkv2.Put(cctx, "k", "v"); err != nil {
  1461  		t.Fatal(err)
  1462  	}
  1463  	resp, err := lkv1.Get(cctx, "k")
  1464  	if err != nil {
  1465  		t.Fatal(err)
  1466  	}
  1467  	if string(resp.Kvs[0].Value) != "v" {
  1468  		t.Fatalf(`expected "v" value, got %+v`, resp)
  1469  	}
  1470  }
  1471  
  1472  // TestLeasingReconnectOwnerConsistency checks a write error on an owner will
  1473  // not cause inconsistency between the server and the client.
  1474  func TestLeasingReconnectOwnerConsistency(t *testing.T) {
  1475  	defer testutil.AfterTest(t)
  1476  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
  1477  	defer clus.Terminate(t)
  1478  
  1479  	lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "foo/")
  1480  	defer closeLKV()
  1481  	testutil.AssertNil(t, err)
  1482  
  1483  	if _, err = lkv.Put(context.TODO(), "k", "x"); err != nil {
  1484  		t.Fatal(err)
  1485  	}
  1486  	if _, err = lkv.Put(context.TODO(), "kk", "y"); err != nil {
  1487  		t.Fatal(err)
  1488  	}
  1489  	if _, err = lkv.Get(context.TODO(), "k"); err != nil {
  1490  		t.Fatal(err)
  1491  	}
  1492  
  1493  	for i := 0; i < 10; i++ {
  1494  		v := fmt.Sprintf("%d", i)
  1495  		donec := make(chan struct{})
  1496  		clus.Members[0].DropConnections()
  1497  		go func() {
  1498  			defer close(donec)
  1499  			for i := 0; i < 20; i++ {
  1500  				clus.Members[0].DropConnections()
  1501  				time.Sleep(time.Millisecond)
  1502  			}
  1503  		}()
  1504  		switch rand.Intn(7) {
  1505  		case 0:
  1506  			_, err = lkv.Put(context.TODO(), "k", v)
  1507  		case 1:
  1508  			_, err = lkv.Delete(context.TODO(), "k")
  1509  		case 2:
  1510  			txn := lkv.Txn(context.TODO()).Then(
  1511  				clientv3.OpGet("k"),
  1512  				clientv3.OpDelete("k"),
  1513  			)
  1514  			_, err = txn.Commit()
  1515  		case 3:
  1516  			txn := lkv.Txn(context.TODO()).Then(
  1517  				clientv3.OpGet("k"),
  1518  				clientv3.OpPut("k", v),
  1519  			)
  1520  			_, err = txn.Commit()
  1521  		case 4:
  1522  			_, err = lkv.Do(context.TODO(), clientv3.OpPut("k", v))
  1523  		case 5:
  1524  			_, err = lkv.Do(context.TODO(), clientv3.OpDelete("k"))
  1525  		case 6:
  1526  			_, err = lkv.Delete(context.TODO(), "k", clientv3.WithPrefix())
  1527  		}
  1528  		<-donec
  1529  		if err != nil {
  1530  			// TODO wrap input client to generate errors
  1531  			break
  1532  		}
  1533  	}
  1534  
  1535  	lresp, lerr := lkv.Get(context.TODO(), "k")
  1536  	if lerr != nil {
  1537  		t.Fatal(lerr)
  1538  	}
  1539  	cresp, cerr := clus.Client(0).Get(context.TODO(), "k")
  1540  	if cerr != nil {
  1541  		t.Fatal(cerr)
  1542  	}
  1543  	if !reflect.DeepEqual(lresp.Kvs, cresp.Kvs) {
  1544  		t.Fatalf("expected %+v, got %+v", cresp, lresp)
  1545  	}
  1546  }
  1547  
  1548  func TestLeasingTxnAtomicCache(t *testing.T) {
  1549  	defer testutil.AfterTest(t)
  1550  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
  1551  	defer clus.Terminate(t)
  1552  
  1553  	lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "foo/")
  1554  	testutil.AssertNil(t, err)
  1555  	defer closeLKV()
  1556  
  1557  	puts, gets := make([]clientv3.Op, 16), make([]clientv3.Op, 16)
  1558  	for i := range puts {
  1559  		k := fmt.Sprintf("k-%d", i)
  1560  		puts[i], gets[i] = clientv3.OpPut(k, k), clientv3.OpGet(k)
  1561  	}
  1562  	if _, err = clus.Client(0).Txn(context.TODO()).Then(puts...).Commit(); err != nil {
  1563  		t.Fatal(err)
  1564  	}
  1565  	for i := range gets {
  1566  		if _, err = lkv.Do(context.TODO(), gets[i]); err != nil {
  1567  			t.Fatal(err)
  1568  		}
  1569  	}
  1570  
  1571  	numPutters, numGetters := 16, 16
  1572  
  1573  	var wgPutters, wgGetters sync.WaitGroup
  1574  	wgPutters.Add(numPutters)
  1575  	wgGetters.Add(numGetters)
  1576  
  1577  	f := func() {
  1578  		defer wgPutters.Done()
  1579  		for i := 0; i < 10; i++ {
  1580  			if _, txnerr := lkv.Txn(context.TODO()).Then(puts...).Commit(); err != nil {
  1581  				t.Fatal(txnerr)
  1582  			}
  1583  		}
  1584  	}
  1585  
  1586  	donec := make(chan struct{}, numPutters)
  1587  	g := func() {
  1588  		defer wgGetters.Done()
  1589  		for {
  1590  			select {
  1591  			case <-donec:
  1592  				return
  1593  			default:
  1594  			}
  1595  			tresp, err := lkv.Txn(context.TODO()).Then(gets...).Commit()
  1596  			if err != nil {
  1597  				t.Fatal(err)
  1598  			}
  1599  			revs := make([]int64, len(gets))
  1600  			for i, resp := range tresp.Responses {
  1601  				rr := resp.GetResponseRange()
  1602  				revs[i] = rr.Kvs[0].ModRevision
  1603  			}
  1604  			for i := 1; i < len(revs); i++ {
  1605  				if revs[i] != revs[i-1] {
  1606  					t.Fatalf("expected matching revisions, got %+v", revs)
  1607  				}
  1608  			}
  1609  		}
  1610  	}
  1611  
  1612  	for i := 0; i < numGetters; i++ {
  1613  		go g()
  1614  	}
  1615  	for i := 0; i < numPutters; i++ {
  1616  		go f()
  1617  	}
  1618  
  1619  	wgPutters.Wait()
  1620  	close(donec)
  1621  	wgGetters.Wait()
  1622  }
  1623  
  1624  // TestLeasingReconnectTxn checks that Txn is resilient to disconnects.
  1625  func TestLeasingReconnectTxn(t *testing.T) {
  1626  	defer testutil.AfterTest(t)
  1627  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
  1628  	defer clus.Terminate(t)
  1629  
  1630  	lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "foo/")
  1631  	testutil.AssertNil(t, err)
  1632  	defer closeLKV()
  1633  
  1634  	if _, err = lkv.Get(context.TODO(), "k"); err != nil {
  1635  		t.Fatal(err)
  1636  	}
  1637  
  1638  	donec := make(chan struct{})
  1639  	go func() {
  1640  		defer close(donec)
  1641  		clus.Members[0].DropConnections()
  1642  		for i := 0; i < 10; i++ {
  1643  			clus.Members[0].DropConnections()
  1644  			time.Sleep(time.Millisecond)
  1645  		}
  1646  	}()
  1647  
  1648  	_, lerr := lkv.Txn(context.TODO()).
  1649  		If(clientv3.Compare(clientv3.Version("k"), "=", 0)).
  1650  		Then(clientv3.OpGet("k")).
  1651  		Commit()
  1652  	<-donec
  1653  	if lerr != nil {
  1654  		t.Fatal(lerr)
  1655  	}
  1656  }
  1657  
  1658  // TestLeasingReconnectNonOwnerGet checks a get error on an owner will
  1659  // not cause inconsistency between the server and the client.
  1660  func TestLeasingReconnectNonOwnerGet(t *testing.T) {
  1661  	defer testutil.AfterTest(t)
  1662  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
  1663  	defer clus.Terminate(t)
  1664  
  1665  	lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "foo/")
  1666  	testutil.AssertNil(t, err)
  1667  	defer closeLKV()
  1668  
  1669  	// populate a few keys so some leasing gets have keys
  1670  	for i := 0; i < 4; i++ {
  1671  		k := fmt.Sprintf("k-%d", i*2)
  1672  		if _, err = lkv.Put(context.TODO(), k, k[2:]); err != nil {
  1673  			t.Fatal(err)
  1674  		}
  1675  	}
  1676  
  1677  	n := 0
  1678  	for i := 0; i < 10; i++ {
  1679  		donec := make(chan struct{})
  1680  		clus.Members[0].DropConnections()
  1681  		go func() {
  1682  			defer close(donec)
  1683  			for j := 0; j < 10; j++ {
  1684  				clus.Members[0].DropConnections()
  1685  				time.Sleep(time.Millisecond)
  1686  			}
  1687  		}()
  1688  		_, err = lkv.Get(context.TODO(), fmt.Sprintf("k-%d", i))
  1689  		<-donec
  1690  		n++
  1691  		if err != nil {
  1692  			break
  1693  		}
  1694  	}
  1695  	for i := 0; i < n; i++ {
  1696  		k := fmt.Sprintf("k-%d", i)
  1697  		lresp, lerr := lkv.Get(context.TODO(), k)
  1698  		if lerr != nil {
  1699  			t.Fatal(lerr)
  1700  		}
  1701  		cresp, cerr := clus.Client(0).Get(context.TODO(), k)
  1702  		if cerr != nil {
  1703  			t.Fatal(cerr)
  1704  		}
  1705  		if !reflect.DeepEqual(lresp.Kvs, cresp.Kvs) {
  1706  			t.Fatalf("expected %+v, got %+v", cresp, lresp)
  1707  		}
  1708  	}
  1709  }
  1710  
  1711  func TestLeasingTxnRangeCmp(t *testing.T) {
  1712  	defer testutil.AfterTest(t)
  1713  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
  1714  	defer clus.Terminate(t)
  1715  
  1716  	lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "foo/")
  1717  	testutil.AssertNil(t, err)
  1718  	defer closeLKV()
  1719  
  1720  	if _, err = clus.Client(0).Put(context.TODO(), "k", "a"); err != nil {
  1721  		t.Fatal(err)
  1722  	}
  1723  	// k2 version = 2
  1724  	if _, err = clus.Client(0).Put(context.TODO(), "k2", "a"); err != nil {
  1725  		t.Fatal(err)
  1726  	}
  1727  	if _, err = clus.Client(0).Put(context.TODO(), "k2", "a"); err != nil {
  1728  		t.Fatal(err)
  1729  	}
  1730  
  1731  	// cache k
  1732  	if _, err = lkv.Get(context.TODO(), "k"); err != nil {
  1733  		t.Fatal(err)
  1734  	}
  1735  
  1736  	cmp := clientv3.Compare(clientv3.Version("k").WithPrefix(), "=", 1)
  1737  	tresp, terr := lkv.Txn(context.TODO()).If(cmp).Commit()
  1738  	if terr != nil {
  1739  		t.Fatal(err)
  1740  	}
  1741  	if tresp.Succeeded {
  1742  		t.Fatalf("expected Succeeded=false, got %+v", tresp)
  1743  	}
  1744  }
  1745  
  1746  func TestLeasingDo(t *testing.T) {
  1747  	defer testutil.AfterTest(t)
  1748  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
  1749  	defer clus.Terminate(t)
  1750  
  1751  	lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "foo/")
  1752  	testutil.AssertNil(t, err)
  1753  	defer closeLKV()
  1754  
  1755  	ops := []clientv3.Op{
  1756  		clientv3.OpTxn(nil, nil, nil),
  1757  		clientv3.OpGet("a"),
  1758  		clientv3.OpPut("a/abc", "v"),
  1759  		clientv3.OpDelete("a", clientv3.WithPrefix()),
  1760  		clientv3.OpTxn(nil, nil, nil),
  1761  	}
  1762  	for i, op := range ops {
  1763  		resp, resperr := lkv.Do(context.TODO(), op)
  1764  		if resperr != nil {
  1765  			t.Errorf("#%d: failed (%v)", i, resperr)
  1766  		}
  1767  		switch {
  1768  		case op.IsGet() && resp.Get() == nil:
  1769  			t.Errorf("#%d: get but nil get response", i)
  1770  		case op.IsPut() && resp.Put() == nil:
  1771  			t.Errorf("#%d: put op but nil get response", i)
  1772  		case op.IsDelete() && resp.Del() == nil:
  1773  			t.Errorf("#%d: delete op but nil delete response", i)
  1774  		case op.IsTxn() && resp.Txn() == nil:
  1775  			t.Errorf("#%d: txn op but nil txn response", i)
  1776  		}
  1777  	}
  1778  
  1779  	gresp, err := clus.Client(0).Get(context.TODO(), "a", clientv3.WithPrefix())
  1780  	if err != nil {
  1781  		t.Fatal(err)
  1782  	}
  1783  	if len(gresp.Kvs) != 0 {
  1784  		t.Fatalf("expected no keys, got %+v", gresp.Kvs)
  1785  	}
  1786  }
  1787  
  1788  func TestLeasingTxnOwnerPutBranch(t *testing.T) {
  1789  	defer testutil.AfterTest(t)
  1790  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3})
  1791  	defer clus.Terminate(t)
  1792  
  1793  	lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "foo/")
  1794  	testutil.AssertNil(t, err)
  1795  	defer closeLKV()
  1796  
  1797  	n := 0
  1798  	treeOp := makePutTreeOp("tree", &n, 4)
  1799  	for i := 0; i < n; i++ {
  1800  		k := fmt.Sprintf("tree/%d", i)
  1801  		if _, err = clus.Client(0).Put(context.TODO(), k, "a"); err != nil {
  1802  			t.Fatal(err)
  1803  		}
  1804  		if _, err = lkv.Get(context.TODO(), k); err != nil {
  1805  			t.Fatal(err)
  1806  		}
  1807  	}
  1808  
  1809  	if _, err = lkv.Do(context.TODO(), treeOp); err != nil {
  1810  		t.Fatal(err)
  1811  	}
  1812  
  1813  	// lkv shouldn't need to call out to server for updated leased keys
  1814  	clus.Members[0].Stop(t)
  1815  
  1816  	for i := 0; i < n; i++ {
  1817  		k := fmt.Sprintf("tree/%d", i)
  1818  		lkvResp, err := lkv.Get(context.TODO(), k)
  1819  		if err != nil {
  1820  			t.Fatal(err)
  1821  		}
  1822  		clusResp, err := clus.Client(1).Get(context.TODO(), k)
  1823  		if err != nil {
  1824  			t.Fatal(err)
  1825  		}
  1826  		if !reflect.DeepEqual(clusResp.Kvs, lkvResp.Kvs) {
  1827  			t.Fatalf("expected %+v, got %+v", clusResp.Kvs, lkvResp.Kvs)
  1828  		}
  1829  	}
  1830  }
  1831  
  1832  func makePutTreeOp(pfx string, v *int, depth int) clientv3.Op {
  1833  	key := fmt.Sprintf("%s/%d", pfx, *v)
  1834  	*v = *v + 1
  1835  	if depth == 0 {
  1836  		return clientv3.OpPut(key, "leaf")
  1837  	}
  1838  
  1839  	t, e := makePutTreeOp(pfx, v, depth-1), makePutTreeOp(pfx, v, depth-1)
  1840  	tPut, ePut := clientv3.OpPut(key, "then"), clientv3.OpPut(key, "else")
  1841  
  1842  	cmps := make([]clientv3.Cmp, 1)
  1843  	if rand.Intn(2) == 0 {
  1844  		// follow then path
  1845  		cmps[0] = clientv3.Compare(clientv3.Version("nokey"), "=", 0)
  1846  	} else {
  1847  		// follow else path
  1848  		cmps[0] = clientv3.Compare(clientv3.Version("nokey"), ">", 0)
  1849  	}
  1850  
  1851  	return clientv3.OpTxn(cmps, []clientv3.Op{t, tPut}, []clientv3.Op{e, ePut})
  1852  }
  1853  
  1854  func randCmps(pfx string, dat []*clientv3.PutResponse) (cmps []clientv3.Cmp, then bool) {
  1855  	for i := 0; i < len(dat); i++ {
  1856  		idx := rand.Intn(len(dat))
  1857  		k := fmt.Sprintf("%s%d", pfx, idx)
  1858  		rev := dat[idx].Header.Revision
  1859  		var cmp clientv3.Cmp
  1860  		switch rand.Intn(4) {
  1861  		case 0:
  1862  			cmp = clientv3.Compare(clientv3.CreateRevision(k), ">", rev-1)
  1863  		case 1:
  1864  			cmp = clientv3.Compare(clientv3.Version(k), "=", 1)
  1865  		case 2:
  1866  			cmp = clientv3.Compare(clientv3.CreateRevision(k), "=", rev)
  1867  		case 3:
  1868  			cmp = clientv3.Compare(clientv3.CreateRevision(k), "!=", rev+1)
  1869  
  1870  		}
  1871  		cmps = append(cmps, cmp)
  1872  	}
  1873  	cmps = cmps[:rand.Intn(len(dat))]
  1874  	if rand.Intn(2) == 0 {
  1875  		return cmps, true
  1876  	}
  1877  	i := rand.Intn(len(dat))
  1878  	cmps = append(cmps, clientv3.Compare(clientv3.Version(fmt.Sprintf("k-%d", i)), "=", 0))
  1879  	return cmps, false
  1880  }
  1881  
  1882  func TestLeasingSessionExpire(t *testing.T) {
  1883  	defer testutil.AfterTest(t)
  1884  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3})
  1885  	defer clus.Terminate(t)
  1886  
  1887  	lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "foo/", concurrency.WithTTL(1))
  1888  	testutil.AssertNil(t, err)
  1889  	defer closeLKV()
  1890  
  1891  	lkv2, closeLKV2, err := leasing.NewKV(clus.Client(0), "foo/")
  1892  	testutil.AssertNil(t, err)
  1893  	defer closeLKV2()
  1894  
  1895  	// acquire lease on abc
  1896  	if _, err = lkv.Get(context.TODO(), "abc"); err != nil {
  1897  		t.Fatal(err)
  1898  	}
  1899  
  1900  	// down endpoint lkv uses for keepalives
  1901  	clus.Members[0].Stop(t)
  1902  	if err = waitForLeasingExpire(clus.Client(1), "foo/abc"); err != nil {
  1903  		t.Fatal(err)
  1904  	}
  1905  	waitForExpireAck(t, lkv)
  1906  	clus.Members[0].Restart(t)
  1907  
  1908  	if _, err = lkv2.Put(context.TODO(), "abc", "def"); err != nil {
  1909  		t.Fatal(err)
  1910  	}
  1911  
  1912  	resp, err := lkv.Get(context.TODO(), "abc")
  1913  	if err != nil {
  1914  		t.Fatal(err)
  1915  	}
  1916  	if v := string(resp.Kvs[0].Value); v != "def" {
  1917  		t.Fatalf("expected %q, got %q", "v", v)
  1918  	}
  1919  }
  1920  
  1921  func TestLeasingSessionExpireCancel(t *testing.T) {
  1922  	defer testutil.AfterTest(t)
  1923  	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3})
  1924  	defer clus.Terminate(t)
  1925  
  1926  	tests := []func(context.Context, clientv3.KV) error{
  1927  		func(ctx context.Context, kv clientv3.KV) error {
  1928  			_, err := kv.Get(ctx, "abc")
  1929  			return err
  1930  		},
  1931  		func(ctx context.Context, kv clientv3.KV) error {
  1932  			_, err := kv.Delete(ctx, "abc")
  1933  			return err
  1934  		},
  1935  		func(ctx context.Context, kv clientv3.KV) error {
  1936  			_, err := kv.Put(ctx, "abc", "v")
  1937  			return err
  1938  		},
  1939  		func(ctx context.Context, kv clientv3.KV) error {
  1940  			_, err := kv.Txn(ctx).Then(clientv3.OpGet("abc")).Commit()
  1941  			return err
  1942  		},
  1943  		func(ctx context.Context, kv clientv3.KV) error {
  1944  			_, err := kv.Do(ctx, clientv3.OpPut("abc", "v"))
  1945  			return err
  1946  		},
  1947  		func(ctx context.Context, kv clientv3.KV) error {
  1948  			_, err := kv.Do(ctx, clientv3.OpDelete("abc"))
  1949  			return err
  1950  		},
  1951  		func(ctx context.Context, kv clientv3.KV) error {
  1952  			_, err := kv.Do(ctx, clientv3.OpGet("abc"))
  1953  			return err
  1954  		},
  1955  		func(ctx context.Context, kv clientv3.KV) error {
  1956  			op := clientv3.OpTxn(nil, []clientv3.Op{clientv3.OpGet("abc")}, nil)
  1957  			_, err := kv.Do(ctx, op)
  1958  			return err
  1959  		},
  1960  	}
  1961  	for i := range tests {
  1962  		lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "foo/", concurrency.WithTTL(1))
  1963  		testutil.AssertNil(t, err)
  1964  		defer closeLKV()
  1965  
  1966  		if _, err = lkv.Get(context.TODO(), "abc"); err != nil {
  1967  			t.Fatal(err)
  1968  		}
  1969  
  1970  		// down endpoint lkv uses for keepalives
  1971  		clus.Members[0].Stop(t)
  1972  		if err := waitForLeasingExpire(clus.Client(1), "foo/abc"); err != nil {
  1973  			t.Fatal(err)
  1974  		}
  1975  		waitForExpireAck(t, lkv)
  1976  
  1977  		ctx, cancel := context.WithCancel(context.TODO())
  1978  		errc := make(chan error, 1)
  1979  		go func() { errc <- tests[i](ctx, lkv) }()
  1980  		// some delay to get past for ctx.Err() != nil {} loops
  1981  		time.Sleep(100 * time.Millisecond)
  1982  		cancel()
  1983  
  1984  		select {
  1985  		case err := <-errc:
  1986  			if err != ctx.Err() {
  1987  				t.Errorf("#%d: expected %v, got %v", i, ctx.Err(), err)
  1988  			}
  1989  		case <-time.After(5 * time.Second):
  1990  			t.Errorf("#%d: timed out waiting for cancel", i)
  1991  		}
  1992  		clus.Members[0].Restart(t)
  1993  	}
  1994  }
  1995  
  1996  func waitForLeasingExpire(kv clientv3.KV, lkey string) error {
  1997  	for {
  1998  		time.Sleep(1 * time.Second)
  1999  		resp, err := kv.Get(context.TODO(), lkey, clientv3.WithPrefix())
  2000  		if err != nil {
  2001  			return err
  2002  		}
  2003  		if len(resp.Kvs) == 0 {
  2004  			// server expired the leasing key
  2005  			return nil
  2006  		}
  2007  	}
  2008  }
  2009  
  2010  func waitForExpireAck(t *testing.T, kv clientv3.KV) {
  2011  	// wait for leasing client to acknowledge lost lease
  2012  	for i := 0; i < 10; i++ {
  2013  		ctx, cancel := context.WithTimeout(context.TODO(), time.Second)
  2014  		_, err := kv.Get(ctx, "abc")
  2015  		cancel()
  2016  		if err == ctx.Err() {
  2017  			return
  2018  		}
  2019  		time.Sleep(time.Second)
  2020  	}
  2021  	t.Fatalf("waited too long to acknlowedge lease expiration")
  2022  }