go.etcd.io/etcd@v3.3.27+incompatible/clientv3/integration/leasing_test.go (about) 1 // Copyright 2017 The etcd Authors 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package integration 16 17 import ( 18 "context" 19 "fmt" 20 "math/rand" 21 "reflect" 22 "sync" 23 "testing" 24 "time" 25 26 "github.com/coreos/etcd/clientv3" 27 "github.com/coreos/etcd/clientv3/concurrency" 28 "github.com/coreos/etcd/clientv3/leasing" 29 "github.com/coreos/etcd/integration" 30 "github.com/coreos/etcd/pkg/testutil" 31 ) 32 33 func TestLeasingPutGet(t *testing.T) { 34 defer testutil.AfterTest(t) 35 36 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3}) 37 defer clus.Terminate(t) 38 39 lKV1, closeLKV1, err := leasing.NewKV(clus.Client(0), "foo/") 40 testutil.AssertNil(t, err) 41 defer closeLKV1() 42 43 lKV2, closeLKV2, err := leasing.NewKV(clus.Client(1), "foo/") 44 testutil.AssertNil(t, err) 45 defer closeLKV2() 46 47 lKV3, closeLKV3, err := leasing.NewKV(clus.Client(2), "foo/") 48 testutil.AssertNil(t, err) 49 defer closeLKV3() 50 51 resp, err := lKV1.Get(context.TODO(), "abc") 52 if err != nil { 53 t.Fatal(err) 54 } 55 if len(resp.Kvs) != 0 { 56 t.Errorf("expected nil, got %q", resp.Kvs[0].Key) 57 } 58 59 if _, err = lKV1.Put(context.TODO(), "abc", "def"); err != nil { 60 t.Fatal(err) 61 } 62 if resp, err = lKV2.Get(context.TODO(), "abc"); err != nil { 63 t.Fatal(err) 64 } 65 if string(resp.Kvs[0].Key) != "abc" { 66 t.Errorf("expected key=%q, got key=%q", "abc", resp.Kvs[0].Key) 67 } 68 if string(resp.Kvs[0].Value) != "def" { 69 t.Errorf("expected value=%q, got value=%q", "bar", resp.Kvs[0].Value) 70 } 71 72 if _, err = lKV3.Get(context.TODO(), "abc"); err != nil { 73 t.Fatal(err) 74 } 75 if _, err = lKV2.Put(context.TODO(), "abc", "ghi"); err != nil { 76 t.Fatal(err) 77 } 78 79 if resp, err = lKV3.Get(context.TODO(), "abc"); err != nil { 80 t.Fatal(err) 81 } 82 if string(resp.Kvs[0].Key) != "abc" { 83 t.Errorf("expected key=%q, got key=%q", "abc", resp.Kvs[0].Key) 84 } 85 86 if string(resp.Kvs[0].Value) != "ghi" { 87 t.Errorf("expected value=%q, got value=%q", "bar", resp.Kvs[0].Value) 88 } 89 } 90 91 // TestLeasingInterval checks the leasing KV fetches key intervals. 92 func TestLeasingInterval(t *testing.T) { 93 defer testutil.AfterTest(t) 94 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}) 95 defer clus.Terminate(t) 96 97 lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/") 98 testutil.AssertNil(t, err) 99 defer closeLKV() 100 101 keys := []string{"abc/a", "abc/b", "abc/a/a"} 102 for _, k := range keys { 103 if _, err = clus.Client(0).Put(context.TODO(), k, "v"); err != nil { 104 t.Fatal(err) 105 } 106 } 107 108 resp, err := lkv.Get(context.TODO(), "abc/", clientv3.WithPrefix()) 109 if err != nil { 110 t.Fatal(err) 111 } 112 if len(resp.Kvs) != 3 { 113 t.Fatalf("expected keys %+v, got response keys %+v", keys, resp.Kvs) 114 } 115 116 // load into cache 117 if resp, err = lkv.Get(context.TODO(), "abc/a"); err != nil { 118 t.Fatal(err) 119 } 120 121 // get when prefix is also a cached key 122 if resp, err = lkv.Get(context.TODO(), "abc/a", clientv3.WithPrefix()); err != nil { 123 t.Fatal(err) 124 } 125 if len(resp.Kvs) != 2 { 126 t.Fatalf("expected keys %+v, got response keys %+v", keys, resp.Kvs) 127 } 128 } 129 130 // TestLeasingPutInvalidateNew checks the leasing KV updates its cache on a Put to a new key. 131 func TestLeasingPutInvalidateNew(t *testing.T) { 132 defer testutil.AfterTest(t) 133 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}) 134 defer clus.Terminate(t) 135 136 lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/") 137 testutil.AssertNil(t, err) 138 defer closeLKV() 139 140 if _, err = lkv.Get(context.TODO(), "k"); err != nil { 141 t.Fatal(err) 142 } 143 if _, err = lkv.Put(context.TODO(), "k", "v"); err != nil { 144 t.Fatal(err) 145 } 146 147 lkvResp, err := lkv.Get(context.TODO(), "k") 148 if err != nil { 149 t.Fatal(err) 150 } 151 cResp, cerr := clus.Client(0).Get(context.TODO(), "k") 152 if cerr != nil { 153 t.Fatal(cerr) 154 } 155 if !reflect.DeepEqual(lkvResp, cResp) { 156 t.Fatalf(`expected %+v, got response %+v`, cResp, lkvResp) 157 } 158 } 159 160 // TestLeasingPutInvalidateExisting checks the leasing KV updates its cache on a Put to an existing key. 161 func TestLeasingPutInvalidateExisting(t *testing.T) { 162 defer testutil.AfterTest(t) 163 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}) 164 defer clus.Terminate(t) 165 166 if _, err := clus.Client(0).Put(context.TODO(), "k", "abc"); err != nil { 167 t.Fatal(err) 168 } 169 170 lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/") 171 testutil.AssertNil(t, err) 172 defer closeLKV() 173 174 if _, err = lkv.Get(context.TODO(), "k"); err != nil { 175 t.Fatal(err) 176 } 177 if _, err = lkv.Put(context.TODO(), "k", "v"); err != nil { 178 t.Fatal(err) 179 } 180 181 lkvResp, err := lkv.Get(context.TODO(), "k") 182 if err != nil { 183 t.Fatal(err) 184 } 185 cResp, cerr := clus.Client(0).Get(context.TODO(), "k") 186 if cerr != nil { 187 t.Fatal(cerr) 188 } 189 if !reflect.DeepEqual(lkvResp, cResp) { 190 t.Fatalf(`expected %+v, got response %+v`, cResp, lkvResp) 191 } 192 } 193 194 // TestLeasingGetNoLeaseTTL checks a key with a TTL is not leased. 195 func TestLeasingGetNoLeaseTTL(t *testing.T) { 196 defer testutil.AfterTest(t) 197 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}) 198 defer clus.Terminate(t) 199 200 lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/") 201 testutil.AssertNil(t, err) 202 defer closeLKV() 203 204 lresp, err := clus.Client(0).Grant(context.TODO(), 60) 205 testutil.AssertNil(t, err) 206 207 _, err = clus.Client(0).Put(context.TODO(), "k", "v", clientv3.WithLease(lresp.ID)) 208 testutil.AssertNil(t, err) 209 210 gresp, err := lkv.Get(context.TODO(), "k") 211 testutil.AssertNil(t, err) 212 testutil.AssertEqual(t, len(gresp.Kvs), 1) 213 214 clus.Members[0].Stop(t) 215 216 ctx, cancel := context.WithTimeout(context.TODO(), time.Second) 217 _, err = lkv.Get(ctx, "k") 218 cancel() 219 testutil.AssertEqual(t, err, ctx.Err()) 220 } 221 222 // TestLeasingGetSerializable checks the leasing KV can make serialized requests 223 // when the etcd cluster is partitioned. 224 func TestLeasingGetSerializable(t *testing.T) { 225 defer testutil.AfterTest(t) 226 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 2}) 227 defer clus.Terminate(t) 228 229 lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/") 230 testutil.AssertNil(t, err) 231 defer closeLKV() 232 233 if _, err = clus.Client(0).Put(context.TODO(), "cached", "abc"); err != nil { 234 t.Fatal(err) 235 } 236 if _, err = lkv.Get(context.TODO(), "cached"); err != nil { 237 t.Fatal(err) 238 } 239 240 clus.Members[1].Stop(t) 241 242 // don't necessarily try to acquire leasing key ownership for new key 243 resp, err := lkv.Get(context.TODO(), "uncached", clientv3.WithSerializable()) 244 if err != nil { 245 t.Fatal(err) 246 } 247 if len(resp.Kvs) != 0 { 248 t.Fatalf(`expected no keys, got response %+v`, resp) 249 } 250 251 clus.Members[0].Stop(t) 252 253 // leasing key ownership should have "cached" locally served 254 cachedResp, err := lkv.Get(context.TODO(), "cached", clientv3.WithSerializable()) 255 if err != nil { 256 t.Fatal(err) 257 } 258 if len(cachedResp.Kvs) != 1 || string(cachedResp.Kvs[0].Value) != "abc" { 259 t.Fatalf(`expected "cached"->"abc", got response %+v`, cachedResp) 260 } 261 } 262 263 // TestLeasingPrevKey checks the cache respects WithPrevKV on puts. 264 func TestLeasingPrevKey(t *testing.T) { 265 defer testutil.AfterTest(t) 266 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 2}) 267 defer clus.Terminate(t) 268 269 lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/") 270 testutil.AssertNil(t, err) 271 defer closeLKV() 272 273 if _, err = clus.Client(0).Put(context.TODO(), "k", "abc"); err != nil { 274 t.Fatal(err) 275 } 276 // acquire leasing key 277 if _, err = lkv.Get(context.TODO(), "k"); err != nil { 278 t.Fatal(err) 279 } 280 resp, err := lkv.Put(context.TODO(), "k", "def", clientv3.WithPrevKV()) 281 if err != nil { 282 t.Fatal(err) 283 } 284 if resp.PrevKv == nil || string(resp.PrevKv.Value) != "abc" { 285 t.Fatalf(`expected PrevKV.Value="abc", got response %+v`, resp) 286 } 287 } 288 289 // TestLeasingRevGet checks the cache respects Get by Revision. 290 func TestLeasingRevGet(t *testing.T) { 291 defer testutil.AfterTest(t) 292 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}) 293 defer clus.Terminate(t) 294 295 lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/") 296 testutil.AssertNil(t, err) 297 defer closeLKV() 298 299 putResp, err := clus.Client(0).Put(context.TODO(), "k", "abc") 300 if err != nil { 301 t.Fatal(err) 302 } 303 if _, err = clus.Client(0).Put(context.TODO(), "k", "def"); err != nil { 304 t.Fatal(err) 305 } 306 307 // check historic revision 308 getResp, gerr := lkv.Get(context.TODO(), "k", clientv3.WithRev(putResp.Header.Revision)) 309 if gerr != nil { 310 t.Fatal(gerr) 311 } 312 if len(getResp.Kvs) != 1 || string(getResp.Kvs[0].Value) != "abc" { 313 t.Fatalf(`expeted "k"->"abc" at rev=%d, got response %+v`, putResp.Header.Revision, getResp) 314 } 315 // check current revision 316 getResp, gerr = lkv.Get(context.TODO(), "k") 317 if gerr != nil { 318 t.Fatal(gerr) 319 } 320 if len(getResp.Kvs) != 1 || string(getResp.Kvs[0].Value) != "def" { 321 t.Fatalf(`expeted "k"->"abc" at rev=%d, got response %+v`, putResp.Header.Revision, getResp) 322 } 323 } 324 325 // TestLeasingGetWithOpts checks options that can be served through the cache do not depend on the server. 326 func TestLeasingGetWithOpts(t *testing.T) { 327 defer testutil.AfterTest(t) 328 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}) 329 defer clus.Terminate(t) 330 331 lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/") 332 testutil.AssertNil(t, err) 333 defer closeLKV() 334 335 if _, err = clus.Client(0).Put(context.TODO(), "k", "abc"); err != nil { 336 t.Fatal(err) 337 } 338 // in cache 339 if _, err = lkv.Get(context.TODO(), "k", clientv3.WithKeysOnly()); err != nil { 340 t.Fatal(err) 341 } 342 343 clus.Members[0].Stop(t) 344 345 opts := []clientv3.OpOption{ 346 clientv3.WithKeysOnly(), 347 clientv3.WithLimit(1), 348 clientv3.WithMinCreateRev(1), 349 clientv3.WithMinModRev(1), 350 clientv3.WithSort(clientv3.SortByKey, clientv3.SortAscend), 351 clientv3.WithSerializable(), 352 } 353 for _, opt := range opts { 354 if _, err := lkv.Get(context.TODO(), "k", opt); err != nil { 355 t.Fatal(err) 356 } 357 } 358 359 getOpts := []clientv3.OpOption{} 360 for i := 0; i < len(opts); i++ { 361 getOpts = append(getOpts, opts[rand.Intn(len(opts))]) 362 } 363 getOpts = getOpts[:rand.Intn(len(opts))] 364 if _, err := lkv.Get(context.TODO(), "k", getOpts...); err != nil { 365 t.Fatal(err) 366 } 367 } 368 369 // TestLeasingConcurrentPut ensures that a get after concurrent puts returns 370 // the recently put data. 371 func TestLeasingConcurrentPut(t *testing.T) { 372 defer testutil.AfterTest(t) 373 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}) 374 defer clus.Terminate(t) 375 376 lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/") 377 testutil.AssertNil(t, err) 378 defer closeLKV() 379 380 // force key into leasing key cache 381 if _, err = lkv.Get(context.TODO(), "k"); err != nil { 382 t.Fatal(err) 383 } 384 385 // concurrently put through leasing client 386 numPuts := 16 387 putc := make(chan *clientv3.PutResponse, numPuts) 388 for i := 0; i < numPuts; i++ { 389 go func() { 390 resp, perr := lkv.Put(context.TODO(), "k", "abc") 391 if perr != nil { 392 t.Fatal(perr) 393 } 394 putc <- resp 395 }() 396 } 397 // record maximum revision from puts 398 maxRev := int64(0) 399 for i := 0; i < numPuts; i++ { 400 if resp := <-putc; resp.Header.Revision > maxRev { 401 maxRev = resp.Header.Revision 402 } 403 } 404 405 // confirm Get gives most recently put revisions 406 getResp, gerr := lkv.Get(context.TODO(), "k") 407 if gerr != nil { 408 t.Fatal(err) 409 } 410 if mr := getResp.Kvs[0].ModRevision; mr != maxRev { 411 t.Errorf("expected ModRevision %d, got %d", maxRev, mr) 412 } 413 if ver := getResp.Kvs[0].Version; ver != int64(numPuts) { 414 t.Errorf("expected Version %d, got %d", numPuts, ver) 415 } 416 } 417 418 func TestLeasingDisconnectedGet(t *testing.T) { 419 defer testutil.AfterTest(t) 420 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}) 421 defer clus.Terminate(t) 422 423 lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/") 424 testutil.AssertNil(t, err) 425 defer closeLKV() 426 427 if _, err = clus.Client(0).Put(context.TODO(), "cached", "abc"); err != nil { 428 t.Fatal(err) 429 } 430 // get key so it's cached 431 if _, err = lkv.Get(context.TODO(), "cached"); err != nil { 432 t.Fatal(err) 433 } 434 435 clus.Members[0].Stop(t) 436 437 // leasing key ownership should have "cached" locally served 438 cachedResp, err := lkv.Get(context.TODO(), "cached") 439 if err != nil { 440 t.Fatal(err) 441 } 442 if len(cachedResp.Kvs) != 1 || string(cachedResp.Kvs[0].Value) != "abc" { 443 t.Fatalf(`expected "cached"->"abc", got response %+v`, cachedResp) 444 } 445 } 446 447 func TestLeasingDeleteOwner(t *testing.T) { 448 defer testutil.AfterTest(t) 449 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}) 450 defer clus.Terminate(t) 451 452 lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/") 453 testutil.AssertNil(t, err) 454 defer closeLKV() 455 456 if _, err = clus.Client(0).Put(context.TODO(), "k", "abc"); err != nil { 457 t.Fatal(err) 458 } 459 460 // get+own / delete / get 461 if _, err = lkv.Get(context.TODO(), "k"); err != nil { 462 t.Fatal(err) 463 } 464 if _, err = lkv.Delete(context.TODO(), "k"); err != nil { 465 t.Fatal(err) 466 } 467 resp, err := lkv.Get(context.TODO(), "k") 468 if err != nil { 469 t.Fatal(err) 470 } 471 472 if len(resp.Kvs) != 0 { 473 t.Fatalf(`expected "k" to be deleted, got response %+v`, resp) 474 } 475 // try to double delete 476 if _, err = lkv.Delete(context.TODO(), "k"); err != nil { 477 t.Fatal(err) 478 } 479 } 480 481 func TestLeasingDeleteNonOwner(t *testing.T) { 482 defer testutil.AfterTest(t) 483 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}) 484 defer clus.Terminate(t) 485 486 lkv1, closeLKV1, err := leasing.NewKV(clus.Client(0), "pfx/") 487 testutil.AssertNil(t, err) 488 defer closeLKV1() 489 490 lkv2, closeLKV2, err := leasing.NewKV(clus.Client(0), "pfx/") 491 testutil.AssertNil(t, err) 492 defer closeLKV2() 493 494 if _, err = clus.Client(0).Put(context.TODO(), "k", "abc"); err != nil { 495 t.Fatal(err) 496 } 497 // acquire ownership 498 if _, err = lkv1.Get(context.TODO(), "k"); err != nil { 499 t.Fatal(err) 500 } 501 // delete via non-owner 502 if _, err = lkv2.Delete(context.TODO(), "k"); err != nil { 503 t.Fatal(err) 504 } 505 506 // key should be removed from lkv1 507 resp, err := lkv1.Get(context.TODO(), "k") 508 if err != nil { 509 t.Fatal(err) 510 } 511 if len(resp.Kvs) != 0 { 512 t.Fatalf(`expected "k" to be deleted, got response %+v`, resp) 513 } 514 } 515 516 func TestLeasingOverwriteResponse(t *testing.T) { 517 defer testutil.AfterTest(t) 518 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}) 519 defer clus.Terminate(t) 520 521 lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/") 522 testutil.AssertNil(t, err) 523 defer closeLKV() 524 525 if _, err = clus.Client(0).Put(context.TODO(), "k", "abc"); err != nil { 526 t.Fatal(err) 527 } 528 529 resp, err := lkv.Get(context.TODO(), "k") 530 if err != nil { 531 t.Fatal(err) 532 } 533 534 resp.Kvs[0].Key[0] = 'z' 535 resp.Kvs[0].Value[0] = 'z' 536 537 resp, err = lkv.Get(context.TODO(), "k") 538 if err != nil { 539 t.Fatal(err) 540 } 541 542 if string(resp.Kvs[0].Key) != "k" { 543 t.Errorf(`expected key "k", got %q`, string(resp.Kvs[0].Key)) 544 } 545 if string(resp.Kvs[0].Value) != "abc" { 546 t.Errorf(`expected value "abc", got %q`, string(resp.Kvs[0].Value)) 547 } 548 } 549 550 func TestLeasingOwnerPutResponse(t *testing.T) { 551 defer testutil.AfterTest(t) 552 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}) 553 defer clus.Terminate(t) 554 555 lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/") 556 testutil.AssertNil(t, err) 557 defer closeLKV() 558 559 if _, err = clus.Client(0).Put(context.TODO(), "k", "abc"); err != nil { 560 t.Fatal(err) 561 } 562 gresp, gerr := lkv.Get(context.TODO(), "k") 563 if gerr != nil { 564 t.Fatal(gerr) 565 } 566 presp, err := lkv.Put(context.TODO(), "k", "def") 567 if err != nil { 568 t.Fatal(err) 569 } 570 if presp == nil { 571 t.Fatal("expected put response, got nil") 572 } 573 574 clus.Members[0].Stop(t) 575 576 gresp, gerr = lkv.Get(context.TODO(), "k") 577 if gerr != nil { 578 t.Fatal(gerr) 579 } 580 if gresp.Kvs[0].ModRevision != presp.Header.Revision { 581 t.Errorf("expected mod revision %d, got %d", presp.Header.Revision, gresp.Kvs[0].ModRevision) 582 } 583 if gresp.Kvs[0].Version != 2 { 584 t.Errorf("expected version 2, got version %d", gresp.Kvs[0].Version) 585 } 586 } 587 588 func TestLeasingTxnOwnerGetRange(t *testing.T) { 589 defer testutil.AfterTest(t) 590 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}) 591 defer clus.Terminate(t) 592 593 lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/") 594 testutil.AssertNil(t, err) 595 defer closeLKV() 596 597 keyCount := rand.Intn(10) + 1 598 for i := 0; i < keyCount; i++ { 599 k := fmt.Sprintf("k-%d", i) 600 if _, err := clus.Client(0).Put(context.TODO(), k, k+k); err != nil { 601 t.Fatal(err) 602 } 603 } 604 if _, err := lkv.Get(context.TODO(), "k-"); err != nil { 605 t.Fatal(err) 606 } 607 608 tresp, terr := lkv.Txn(context.TODO()).Then(clientv3.OpGet("k-", clientv3.WithPrefix())).Commit() 609 if terr != nil { 610 t.Fatal(terr) 611 } 612 if resp := tresp.Responses[0].GetResponseRange(); len(resp.Kvs) != keyCount { 613 t.Fatalf("expected %d keys, got response %+v", keyCount, resp.Kvs) 614 } 615 } 616 617 func TestLeasingTxnOwnerGet(t *testing.T) { 618 defer testutil.AfterTest(t) 619 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}) 620 defer clus.Terminate(t) 621 622 lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/") 623 testutil.AssertNil(t, err) 624 defer closeLKV() 625 626 keyCount := rand.Intn(10) + 1 627 var ops []clientv3.Op 628 presps := make([]*clientv3.PutResponse, keyCount) 629 for i := range presps { 630 k := fmt.Sprintf("k-%d", i) 631 presp, err := clus.Client(0).Put(context.TODO(), k, k+k) 632 if err != nil { 633 t.Fatal(err) 634 } 635 presps[i] = presp 636 637 if _, err = lkv.Get(context.TODO(), k); err != nil { 638 t.Fatal(err) 639 } 640 ops = append(ops, clientv3.OpGet(k)) 641 } 642 ops = ops[:rand.Intn(len(ops))] 643 644 // served through cache 645 clus.Members[0].Stop(t) 646 647 var thenOps, elseOps []clientv3.Op 648 cmps, useThen := randCmps("k-", presps) 649 650 if useThen { 651 652 thenOps = ops 653 elseOps = []clientv3.Op{clientv3.OpPut("k", "1")} 654 } else { 655 thenOps = []clientv3.Op{clientv3.OpPut("k", "1")} 656 elseOps = ops 657 } 658 659 tresp, terr := lkv.Txn(context.TODO()). 660 If(cmps...). 661 Then(thenOps...). 662 Else(elseOps...).Commit() 663 664 if terr != nil { 665 t.Fatal(terr) 666 } 667 if tresp.Succeeded != useThen { 668 t.Fatalf("expected succeeded=%v, got tresp=%+v", useThen, tresp) 669 } 670 if len(tresp.Responses) != len(ops) { 671 t.Fatalf("expected %d responses, got %d", len(ops), len(tresp.Responses)) 672 } 673 wrev := presps[len(presps)-1].Header.Revision 674 if tresp.Header.Revision < wrev { 675 t.Fatalf("expected header revision >= %d, got %d", wrev, tresp.Header.Revision) 676 } 677 for i := range ops { 678 k := fmt.Sprintf("k-%d", i) 679 rr := tresp.Responses[i].GetResponseRange() 680 if rr == nil { 681 t.Errorf("expected get response, got %+v", tresp.Responses[i]) 682 } 683 if string(rr.Kvs[0].Key) != k || string(rr.Kvs[0].Value) != k+k { 684 t.Errorf(`expected key for %q, got %+v`, k, rr.Kvs) 685 } 686 } 687 } 688 689 func TestLeasingTxnOwnerDeleteRange(t *testing.T) { 690 defer testutil.AfterTest(t) 691 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}) 692 defer clus.Terminate(t) 693 694 lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/") 695 testutil.AssertNil(t, err) 696 defer closeLKV() 697 698 keyCount := rand.Intn(10) + 1 699 for i := 0; i < keyCount; i++ { 700 k := fmt.Sprintf("k-%d", i) 701 if _, perr := clus.Client(0).Put(context.TODO(), k, k+k); perr != nil { 702 t.Fatal(perr) 703 } 704 } 705 706 // cache in lkv 707 resp, err := lkv.Get(context.TODO(), "k-", clientv3.WithPrefix()) 708 if err != nil { 709 t.Fatal(err) 710 } 711 if len(resp.Kvs) != keyCount { 712 t.Fatalf("expected %d keys, got %d", keyCount, len(resp.Kvs)) 713 } 714 715 if _, terr := lkv.Txn(context.TODO()).Then(clientv3.OpDelete("k-", clientv3.WithPrefix())).Commit(); terr != nil { 716 t.Fatal(terr) 717 } 718 719 resp, err = lkv.Get(context.TODO(), "k-", clientv3.WithPrefix()) 720 if err != nil { 721 t.Fatal(err) 722 } 723 if len(resp.Kvs) != 0 { 724 t.Fatalf("expected no keys, got %d", len(resp.Kvs)) 725 } 726 } 727 728 func TestLeasingTxnOwnerDelete(t *testing.T) { 729 defer testutil.AfterTest(t) 730 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}) 731 defer clus.Terminate(t) 732 733 lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/") 734 testutil.AssertNil(t, err) 735 defer closeLKV() 736 737 if _, err = clus.Client(0).Put(context.TODO(), "k", "abc"); err != nil { 738 t.Fatal(err) 739 } 740 741 // cache in lkv 742 if _, gerr := lkv.Get(context.TODO(), "k"); gerr != nil { 743 t.Fatal(gerr) 744 } 745 746 if _, terr := lkv.Txn(context.TODO()).Then(clientv3.OpDelete("k")).Commit(); terr != nil { 747 t.Fatal(terr) 748 } 749 750 resp, err := lkv.Get(context.TODO(), "k") 751 if err != nil { 752 t.Fatal(err) 753 } 754 if len(resp.Kvs) != 0 { 755 t.Fatalf("expected no keys, got %d", len(resp.Kvs)) 756 } 757 } 758 759 func TestLeasingTxnOwnerIf(t *testing.T) { 760 defer testutil.AfterTest(t) 761 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}) 762 defer clus.Terminate(t) 763 764 lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/") 765 testutil.AssertNil(t, err) 766 defer closeLKV() 767 768 if _, err = clus.Client(0).Put(context.TODO(), "k", "abc"); err != nil { 769 t.Fatal(err) 770 } 771 if _, err = lkv.Get(context.TODO(), "k"); err != nil { 772 t.Fatal(err) 773 } 774 775 // served through cache 776 clus.Members[0].Stop(t) 777 778 tests := []struct { 779 cmps []clientv3.Cmp 780 wSucceeded bool 781 wResponses int 782 }{ 783 // success 784 { 785 cmps: []clientv3.Cmp{clientv3.Compare(clientv3.Value("k"), "=", "abc")}, 786 wSucceeded: true, 787 wResponses: 1, 788 }, 789 { 790 cmps: []clientv3.Cmp{clientv3.Compare(clientv3.CreateRevision("k"), "=", 2)}, 791 wSucceeded: true, 792 wResponses: 1, 793 }, 794 { 795 cmps: []clientv3.Cmp{clientv3.Compare(clientv3.ModRevision("k"), "=", 2)}, 796 wSucceeded: true, 797 wResponses: 1, 798 }, 799 { 800 cmps: []clientv3.Cmp{clientv3.Compare(clientv3.Version("k"), "=", 1)}, 801 wSucceeded: true, 802 wResponses: 1, 803 }, 804 // failure 805 { 806 cmps: []clientv3.Cmp{clientv3.Compare(clientv3.Value("k"), ">", "abc")}, 807 }, 808 { 809 cmps: []clientv3.Cmp{clientv3.Compare(clientv3.CreateRevision("k"), ">", 2)}, 810 }, 811 { 812 cmps: []clientv3.Cmp{clientv3.Compare(clientv3.ModRevision("k"), "=", 2)}, 813 wSucceeded: true, 814 wResponses: 1, 815 }, 816 { 817 cmps: []clientv3.Cmp{clientv3.Compare(clientv3.Version("k"), ">", 1)}, 818 }, 819 { 820 cmps: []clientv3.Cmp{clientv3.Compare(clientv3.Value("k"), "<", "abc")}, 821 }, 822 { 823 cmps: []clientv3.Cmp{clientv3.Compare(clientv3.CreateRevision("k"), "<", 2)}, 824 }, 825 { 826 cmps: []clientv3.Cmp{clientv3.Compare(clientv3.ModRevision("k"), "<", 2)}, 827 }, 828 { 829 cmps: []clientv3.Cmp{clientv3.Compare(clientv3.Version("k"), "<", 1)}, 830 }, 831 { 832 cmps: []clientv3.Cmp{ 833 clientv3.Compare(clientv3.Version("k"), "=", 1), 834 clientv3.Compare(clientv3.Version("k"), "<", 1), 835 }, 836 }, 837 } 838 839 for i, tt := range tests { 840 tresp, terr := lkv.Txn(context.TODO()).If(tt.cmps...).Then(clientv3.OpGet("k")).Commit() 841 if terr != nil { 842 t.Fatal(terr) 843 } 844 if tresp.Succeeded != tt.wSucceeded { 845 t.Errorf("#%d: expected succeeded %v, got %v", i, tt.wSucceeded, tresp.Succeeded) 846 } 847 if len(tresp.Responses) != tt.wResponses { 848 t.Errorf("#%d: expected %d responses, got %d", i, tt.wResponses, len(tresp.Responses)) 849 } 850 } 851 } 852 853 func TestLeasingTxnCancel(t *testing.T) { 854 defer testutil.AfterTest(t) 855 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3}) 856 defer clus.Terminate(t) 857 858 lkv1, closeLKV1, err := leasing.NewKV(clus.Client(0), "pfx/") 859 testutil.AssertNil(t, err) 860 defer closeLKV1() 861 862 lkv2, closeLKV2, err := leasing.NewKV(clus.Client(1), "pfx/") 863 testutil.AssertNil(t, err) 864 defer closeLKV2() 865 866 // acquire lease but disconnect so no revoke in time 867 if _, err = lkv1.Get(context.TODO(), "k"); err != nil { 868 t.Fatal(err) 869 } 870 clus.Members[0].Stop(t) 871 872 // wait for leader election, if any 873 if _, err = clus.Client(1).Get(context.TODO(), "abc"); err != nil { 874 t.Fatal(err) 875 } 876 877 ctx, cancel := context.WithCancel(context.TODO()) 878 go func() { 879 time.Sleep(100 * time.Millisecond) 880 cancel() 881 }() 882 if _, err := lkv2.Txn(ctx).Then(clientv3.OpPut("k", "v")).Commit(); err != context.Canceled { 883 t.Fatalf("expected %v, got %v", context.Canceled, err) 884 } 885 } 886 887 func TestLeasingTxnNonOwnerPut(t *testing.T) { 888 defer testutil.AfterTest(t) 889 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}) 890 defer clus.Terminate(t) 891 892 lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/") 893 testutil.AssertNil(t, err) 894 defer closeLKV() 895 896 lkv2, closeLKV2, err := leasing.NewKV(clus.Client(0), "pfx/") 897 testutil.AssertNil(t, err) 898 defer closeLKV2() 899 900 if _, err = clus.Client(0).Put(context.TODO(), "k", "abc"); err != nil { 901 t.Fatal(err) 902 } 903 if _, err = clus.Client(0).Put(context.TODO(), "k2", "123"); err != nil { 904 t.Fatal(err) 905 } 906 // cache in lkv 907 if _, err = lkv.Get(context.TODO(), "k"); err != nil { 908 t.Fatal(err) 909 } 910 if _, err = lkv.Get(context.TODO(), "k2"); err != nil { 911 t.Fatal(err) 912 } 913 // invalidate via lkv2 txn 914 opArray := make([]clientv3.Op, 0) 915 opArray = append(opArray, clientv3.OpPut("k2", "456")) 916 tresp, terr := lkv2.Txn(context.TODO()).Then( 917 clientv3.OpTxn(nil, opArray, nil), 918 clientv3.OpPut("k", "def"), 919 clientv3.OpPut("k3", "999"), // + a key not in any cache 920 ).Commit() 921 if terr != nil { 922 t.Fatal(terr) 923 } 924 if !tresp.Succeeded || len(tresp.Responses) != 3 { 925 t.Fatalf("expected txn success, got %+v", tresp) 926 } 927 // check cache was invalidated 928 gresp, gerr := lkv.Get(context.TODO(), "k") 929 if gerr != nil { 930 t.Fatal(err) 931 } 932 if len(gresp.Kvs) != 1 || string(gresp.Kvs[0].Value) != "def" { 933 t.Errorf(`expected value "def", got %+v`, gresp) 934 } 935 gresp, gerr = lkv.Get(context.TODO(), "k2") 936 if gerr != nil { 937 t.Fatal(gerr) 938 } 939 if len(gresp.Kvs) != 1 || string(gresp.Kvs[0].Value) != "456" { 940 t.Errorf(`expected value "def", got %+v`, gresp) 941 } 942 // check puts were applied and are all in the same revision 943 w := clus.Client(0).Watch( 944 clus.Client(0).Ctx(), 945 "k", 946 clientv3.WithRev(tresp.Header.Revision), 947 clientv3.WithPrefix()) 948 wresp := <-w 949 c := 0 950 evs := []clientv3.Event{} 951 for _, ev := range wresp.Events { 952 evs = append(evs, *ev) 953 if ev.Kv.ModRevision == tresp.Header.Revision { 954 c++ 955 } 956 } 957 if c != 3 { 958 t.Fatalf("expected 3 put events, got %+v", evs) 959 } 960 } 961 962 // TestLeasingTxnRandIfThenOrElse randomly leases keys two separate clients, then 963 // issues a random If/{Then,Else} transaction on those keys to one client. 964 func TestLeasingTxnRandIfThenOrElse(t *testing.T) { 965 defer testutil.AfterTest(t) 966 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}) 967 defer clus.Terminate(t) 968 969 lkv1, closeLKV1, err1 := leasing.NewKV(clus.Client(0), "pfx/") 970 testutil.AssertNil(t, err1) 971 defer closeLKV1() 972 973 lkv2, closeLKV2, err2 := leasing.NewKV(clus.Client(0), "pfx/") 974 testutil.AssertNil(t, err2) 975 defer closeLKV2() 976 977 keyCount := 16 978 dat := make([]*clientv3.PutResponse, keyCount) 979 for i := 0; i < keyCount; i++ { 980 k, v := fmt.Sprintf("k-%d", i), fmt.Sprintf("%d", i) 981 dat[i], err1 = clus.Client(0).Put(context.TODO(), k, v) 982 if err1 != nil { 983 t.Fatal(err1) 984 } 985 } 986 987 // nondeterministically populate leasing caches 988 var wg sync.WaitGroup 989 getc := make(chan struct{}, keyCount) 990 getRandom := func(kv clientv3.KV) { 991 defer wg.Done() 992 for i := 0; i < keyCount/2; i++ { 993 k := fmt.Sprintf("k-%d", rand.Intn(keyCount)) 994 if _, err := kv.Get(context.TODO(), k); err != nil { 995 t.Fatal(err) 996 } 997 getc <- struct{}{} 998 } 999 } 1000 wg.Add(2) 1001 defer wg.Wait() 1002 go getRandom(lkv1) 1003 go getRandom(lkv2) 1004 1005 // random list of comparisons, all true 1006 cmps, useThen := randCmps("k-", dat) 1007 // random list of puts/gets; unique keys 1008 ops := []clientv3.Op{} 1009 usedIdx := make(map[int]struct{}) 1010 for i := 0; i < keyCount; i++ { 1011 idx := rand.Intn(keyCount) 1012 if _, ok := usedIdx[idx]; ok { 1013 continue 1014 } 1015 usedIdx[idx] = struct{}{} 1016 k := fmt.Sprintf("k-%d", idx) 1017 switch rand.Intn(2) { 1018 case 0: 1019 ops = append(ops, clientv3.OpGet(k)) 1020 case 1: 1021 ops = append(ops, clientv3.OpPut(k, "a")) 1022 // TODO: add delete 1023 } 1024 } 1025 // random lengths 1026 ops = ops[:rand.Intn(len(ops))] 1027 1028 // wait for some gets to populate the leasing caches before committing 1029 for i := 0; i < keyCount/2; i++ { 1030 <-getc 1031 } 1032 1033 // randomly choose between then and else blocks 1034 var thenOps, elseOps []clientv3.Op 1035 if useThen { 1036 thenOps = ops 1037 } else { 1038 // force failure 1039 elseOps = ops 1040 } 1041 1042 tresp, terr := lkv1.Txn(context.TODO()).If(cmps...).Then(thenOps...).Else(elseOps...).Commit() 1043 if terr != nil { 1044 t.Fatal(terr) 1045 } 1046 // cmps always succeed 1047 if tresp.Succeeded != useThen { 1048 t.Fatalf("expected succeeded=%v, got tresp=%+v", useThen, tresp) 1049 } 1050 // get should match what was put 1051 checkPuts := func(s string, kv clientv3.KV) { 1052 for _, op := range ops { 1053 if !op.IsPut() { 1054 continue 1055 } 1056 resp, rerr := kv.Get(context.TODO(), string(op.KeyBytes())) 1057 if rerr != nil { 1058 t.Fatal(rerr) 1059 } 1060 if len(resp.Kvs) != 1 || string(resp.Kvs[0].Value) != "a" { 1061 t.Fatalf(`%s: expected value="a", got %+v`, s, resp.Kvs) 1062 } 1063 } 1064 } 1065 checkPuts("client(0)", clus.Client(0)) 1066 checkPuts("lkv1", lkv1) 1067 checkPuts("lkv2", lkv2) 1068 } 1069 1070 func TestLeasingOwnerPutError(t *testing.T) { 1071 defer testutil.AfterTest(t) 1072 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}) 1073 defer clus.Terminate(t) 1074 1075 lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/") 1076 testutil.AssertNil(t, err) 1077 defer closeLKV() 1078 1079 if _, err = lkv.Get(context.TODO(), "k"); err != nil { 1080 t.Fatal(err) 1081 } 1082 1083 clus.Members[0].Stop(t) 1084 ctx, cancel := context.WithTimeout(context.TODO(), 100*time.Millisecond) 1085 defer cancel() 1086 if resp, err := lkv.Put(ctx, "k", "v"); err == nil { 1087 t.Fatalf("expected error, got response %+v", resp) 1088 } 1089 } 1090 1091 func TestLeasingOwnerDeleteError(t *testing.T) { 1092 defer testutil.AfterTest(t) 1093 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}) 1094 defer clus.Terminate(t) 1095 1096 lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/") 1097 testutil.AssertNil(t, err) 1098 defer closeLKV() 1099 1100 if _, err = lkv.Get(context.TODO(), "k"); err != nil { 1101 t.Fatal(err) 1102 } 1103 1104 clus.Members[0].Stop(t) 1105 ctx, cancel := context.WithTimeout(context.TODO(), 100*time.Millisecond) 1106 defer cancel() 1107 if resp, err := lkv.Delete(ctx, "k"); err == nil { 1108 t.Fatalf("expected error, got response %+v", resp) 1109 } 1110 } 1111 1112 func TestLeasingNonOwnerPutError(t *testing.T) { 1113 defer testutil.AfterTest(t) 1114 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}) 1115 defer clus.Terminate(t) 1116 1117 lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/") 1118 testutil.AssertNil(t, err) 1119 defer closeLKV() 1120 1121 clus.Members[0].Stop(t) 1122 ctx, cancel := context.WithTimeout(context.TODO(), 100*time.Millisecond) 1123 defer cancel() 1124 if resp, err := lkv.Put(ctx, "k", "v"); err == nil { 1125 t.Fatalf("expected error, got response %+v", resp) 1126 } 1127 } 1128 1129 func TestLeasingOwnerDeletePrefix(t *testing.T) { 1130 testLeasingOwnerDelete(t, clientv3.OpDelete("key/", clientv3.WithPrefix())) 1131 } 1132 1133 func TestLeasingOwnerDeleteFrom(t *testing.T) { 1134 testLeasingOwnerDelete(t, clientv3.OpDelete("kd", clientv3.WithFromKey())) 1135 } 1136 1137 func testLeasingOwnerDelete(t *testing.T, del clientv3.Op) { 1138 defer testutil.AfterTest(t) 1139 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}) 1140 defer clus.Terminate(t) 1141 1142 lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "0/") 1143 testutil.AssertNil(t, err) 1144 defer closeLKV() 1145 1146 for i := 0; i < 8; i++ { 1147 if _, err = clus.Client(0).Put(context.TODO(), fmt.Sprintf("key/%d", i), "123"); err != nil { 1148 t.Fatal(err) 1149 } 1150 } 1151 1152 if _, err = lkv.Get(context.TODO(), "key/1"); err != nil { 1153 t.Fatal(err) 1154 } 1155 1156 opResp, delErr := lkv.Do(context.TODO(), del) 1157 if delErr != nil { 1158 t.Fatal(delErr) 1159 } 1160 delResp := opResp.Del() 1161 1162 // confirm keys are invalidated from cache and deleted on etcd 1163 for i := 0; i < 8; i++ { 1164 resp, err := lkv.Get(context.TODO(), fmt.Sprintf("key/%d", i)) 1165 if err != nil { 1166 t.Fatal(err) 1167 } 1168 if len(resp.Kvs) != 0 { 1169 t.Fatalf("expected no keys on key/%d, got %+v", i, resp) 1170 } 1171 } 1172 1173 // confirm keys were deleted atomically 1174 1175 w := clus.Client(0).Watch( 1176 clus.Client(0).Ctx(), 1177 "key/", 1178 clientv3.WithRev(delResp.Header.Revision), 1179 clientv3.WithPrefix()) 1180 1181 if wresp := <-w; len(wresp.Events) != 8 { 1182 t.Fatalf("expected %d delete events,got %d", 8, len(wresp.Events)) 1183 } 1184 } 1185 1186 func TestLeasingDeleteRangeBounds(t *testing.T) { 1187 defer testutil.AfterTest(t) 1188 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}) 1189 defer clus.Terminate(t) 1190 1191 delkv, closeDelKV, err := leasing.NewKV(clus.Client(0), "0/") 1192 testutil.AssertNil(t, err) 1193 defer closeDelKV() 1194 1195 getkv, closeGetKv, err := leasing.NewKV(clus.Client(0), "0/") 1196 testutil.AssertNil(t, err) 1197 defer closeGetKv() 1198 1199 for _, k := range []string{"j", "m"} { 1200 if _, err = clus.Client(0).Put(context.TODO(), k, "123"); err != nil { 1201 t.Fatal(err) 1202 } 1203 if _, err = getkv.Get(context.TODO(), k); err != nil { 1204 t.Fatal(err) 1205 } 1206 } 1207 1208 if _, err = delkv.Delete(context.TODO(), "k", clientv3.WithPrefix()); err != nil { 1209 t.Fatal(err) 1210 } 1211 1212 // leases still on server? 1213 for _, k := range []string{"j", "m"} { 1214 resp, geterr := clus.Client(0).Get(context.TODO(), "0/"+k, clientv3.WithPrefix()) 1215 if geterr != nil { 1216 t.Fatal(geterr) 1217 } 1218 if len(resp.Kvs) != 1 { 1219 t.Fatalf("expected leasing key, got %+v", resp) 1220 } 1221 } 1222 1223 // j and m should still have leases registered since not under k* 1224 clus.Members[0].Stop(t) 1225 1226 if _, err = getkv.Get(context.TODO(), "j"); err != nil { 1227 t.Fatal(err) 1228 } 1229 if _, err = getkv.Get(context.TODO(), "m"); err != nil { 1230 t.Fatal(err) 1231 } 1232 } 1233 1234 func TestLeasingDeleteRangeContendTxn(t *testing.T) { 1235 then := []clientv3.Op{clientv3.OpDelete("key/", clientv3.WithPrefix())} 1236 testLeasingDeleteRangeContend(t, clientv3.OpTxn(nil, then, nil)) 1237 } 1238 1239 func TestLeaseDeleteRangeContendDel(t *testing.T) { 1240 op := clientv3.OpDelete("key/", clientv3.WithPrefix()) 1241 testLeasingDeleteRangeContend(t, op) 1242 } 1243 1244 func testLeasingDeleteRangeContend(t *testing.T, op clientv3.Op) { 1245 defer testutil.AfterTest(t) 1246 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}) 1247 defer clus.Terminate(t) 1248 1249 delkv, closeDelKV, err := leasing.NewKV(clus.Client(0), "0/") 1250 testutil.AssertNil(t, err) 1251 defer closeDelKV() 1252 1253 putkv, closePutKV, err := leasing.NewKV(clus.Client(0), "0/") 1254 testutil.AssertNil(t, err) 1255 defer closePutKV() 1256 1257 for i := 0; i < 8; i++ { 1258 key := fmt.Sprintf("key/%d", i) 1259 if _, err = clus.Client(0).Put(context.TODO(), key, "123"); err != nil { 1260 t.Fatal(err) 1261 } 1262 if _, err = putkv.Get(context.TODO(), key); err != nil { 1263 t.Fatal(err) 1264 } 1265 } 1266 1267 ctx, cancel := context.WithCancel(context.TODO()) 1268 donec := make(chan struct{}) 1269 go func() { 1270 defer close(donec) 1271 for i := 0; ctx.Err() == nil; i++ { 1272 key := fmt.Sprintf("key/%d", i%8) 1273 putkv.Put(ctx, key, "123") 1274 putkv.Get(ctx, key) 1275 } 1276 }() 1277 1278 _, delErr := delkv.Do(context.TODO(), op) 1279 cancel() 1280 <-donec 1281 if delErr != nil { 1282 t.Fatal(delErr) 1283 } 1284 1285 // confirm keys on non-deleter match etcd 1286 for i := 0; i < 8; i++ { 1287 key := fmt.Sprintf("key/%d", i) 1288 resp, err := putkv.Get(context.TODO(), key) 1289 if err != nil { 1290 t.Fatal(err) 1291 } 1292 servResp, err := clus.Client(0).Get(context.TODO(), key) 1293 if err != nil { 1294 t.Fatal(err) 1295 } 1296 if !reflect.DeepEqual(resp.Kvs, servResp.Kvs) { 1297 t.Errorf("#%d: expected %+v, got %+v", i, servResp.Kvs, resp.Kvs) 1298 } 1299 } 1300 } 1301 1302 func TestLeasingPutGetDeleteConcurrent(t *testing.T) { 1303 defer testutil.AfterTest(t) 1304 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}) 1305 defer clus.Terminate(t) 1306 1307 lkvs := make([]clientv3.KV, 16) 1308 for i := range lkvs { 1309 lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "pfx/") 1310 testutil.AssertNil(t, err) 1311 defer closeLKV() 1312 lkvs[i] = lkv 1313 } 1314 1315 getdel := func(kv clientv3.KV) { 1316 if _, err := kv.Put(context.TODO(), "k", "abc"); err != nil { 1317 t.Fatal(err) 1318 } 1319 time.Sleep(time.Millisecond) 1320 if _, err := kv.Get(context.TODO(), "k"); err != nil { 1321 t.Fatal(err) 1322 } 1323 if _, err := kv.Delete(context.TODO(), "k"); err != nil { 1324 t.Fatal(err) 1325 } 1326 time.Sleep(2 * time.Millisecond) 1327 } 1328 1329 var wg sync.WaitGroup 1330 wg.Add(16) 1331 for i := 0; i < 16; i++ { 1332 go func() { 1333 defer wg.Done() 1334 for _, kv := range lkvs { 1335 getdel(kv) 1336 } 1337 }() 1338 } 1339 wg.Wait() 1340 1341 resp, err := lkvs[0].Get(context.TODO(), "k") 1342 if err != nil { 1343 t.Fatal(err) 1344 } 1345 1346 if len(resp.Kvs) > 0 { 1347 t.Fatalf("expected no kvs, got %+v", resp.Kvs) 1348 } 1349 1350 resp, err = clus.Client(0).Get(context.TODO(), "k") 1351 if err != nil { 1352 t.Fatal(err) 1353 } 1354 if len(resp.Kvs) > 0 { 1355 t.Fatalf("expected no kvs, got %+v", resp.Kvs) 1356 } 1357 } 1358 1359 // TestLeasingReconnectOwnerRevoke checks that revocation works if 1360 // disconnected when trying to submit revoke txn. 1361 func TestLeasingReconnectOwnerRevoke(t *testing.T) { 1362 defer testutil.AfterTest(t) 1363 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3}) 1364 defer clus.Terminate(t) 1365 1366 lkv1, closeLKV1, err1 := leasing.NewKV(clus.Client(0), "foo/") 1367 testutil.AssertNil(t, err1) 1368 defer closeLKV1() 1369 1370 lkv2, closeLKV2, err2 := leasing.NewKV(clus.Client(1), "foo/") 1371 testutil.AssertNil(t, err2) 1372 defer closeLKV2() 1373 1374 if _, err := lkv1.Get(context.TODO(), "k"); err != nil { 1375 t.Fatal(err) 1376 } 1377 1378 // force leader away from member 0 1379 clus.Members[0].Stop(t) 1380 clus.WaitLeader(t) 1381 clus.Members[0].Restart(t) 1382 1383 cctx, cancel := context.WithCancel(context.TODO()) 1384 sdonec, pdonec := make(chan struct{}), make(chan struct{}) 1385 // make lkv1 connection choppy so Txn fails 1386 go func() { 1387 defer close(sdonec) 1388 for i := 0; i < 10 && cctx.Err() == nil; i++ { 1389 clus.Members[0].Stop(t) 1390 time.Sleep(10 * time.Millisecond) 1391 clus.Members[0].Restart(t) 1392 } 1393 }() 1394 go func() { 1395 defer close(pdonec) 1396 if _, err := lkv2.Put(cctx, "k", "v"); err != nil { 1397 t.Log(err) 1398 } 1399 resp, err := lkv1.Get(cctx, "k") 1400 if err != nil { 1401 t.Fatal(err) 1402 } 1403 if string(resp.Kvs[0].Value) != "v" { 1404 t.Fatalf(`expected "v" value, got %+v`, resp) 1405 } 1406 }() 1407 select { 1408 case <-pdonec: 1409 cancel() 1410 <-sdonec 1411 case <-time.After(10 * time.Second): 1412 cancel() 1413 <-sdonec 1414 <-pdonec 1415 t.Fatal("took to long to revoke and put") 1416 } 1417 } 1418 1419 // TestLeasingReconnectOwnerRevokeCompact checks that revocation works if 1420 // disconnected and the watch is compacted. 1421 func TestLeasingReconnectOwnerRevokeCompact(t *testing.T) { 1422 defer testutil.AfterTest(t) 1423 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3}) 1424 defer clus.Terminate(t) 1425 1426 lkv1, closeLKV1, err1 := leasing.NewKV(clus.Client(0), "foo/") 1427 testutil.AssertNil(t, err1) 1428 defer closeLKV1() 1429 1430 lkv2, closeLKV2, err2 := leasing.NewKV(clus.Client(1), "foo/") 1431 testutil.AssertNil(t, err2) 1432 defer closeLKV2() 1433 1434 if _, err := lkv1.Get(context.TODO(), "k"); err != nil { 1435 t.Fatal(err) 1436 } 1437 1438 clus.Members[0].Stop(t) 1439 clus.WaitLeader(t) 1440 1441 // put some more revisions for compaction 1442 presp, err := clus.Client(1).Put(context.TODO(), "a", "123") 1443 if err != nil { 1444 t.Fatal(err) 1445 } 1446 presp, err = clus.Client(1).Put(context.TODO(), "a", "123") 1447 if err != nil { 1448 t.Fatal(err) 1449 } 1450 // compact while lkv1 is disconnected 1451 rev := presp.Header.Revision 1452 if _, err = clus.Client(1).Compact(context.TODO(), rev); err != nil { 1453 t.Fatal(err) 1454 } 1455 1456 clus.Members[0].Restart(t) 1457 1458 cctx, cancel := context.WithTimeout(context.TODO(), 5*time.Second) 1459 defer cancel() 1460 if _, err = lkv2.Put(cctx, "k", "v"); err != nil { 1461 t.Fatal(err) 1462 } 1463 resp, err := lkv1.Get(cctx, "k") 1464 if err != nil { 1465 t.Fatal(err) 1466 } 1467 if string(resp.Kvs[0].Value) != "v" { 1468 t.Fatalf(`expected "v" value, got %+v`, resp) 1469 } 1470 } 1471 1472 // TestLeasingReconnectOwnerConsistency checks a write error on an owner will 1473 // not cause inconsistency between the server and the client. 1474 func TestLeasingReconnectOwnerConsistency(t *testing.T) { 1475 defer testutil.AfterTest(t) 1476 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}) 1477 defer clus.Terminate(t) 1478 1479 lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "foo/") 1480 defer closeLKV() 1481 testutil.AssertNil(t, err) 1482 1483 if _, err = lkv.Put(context.TODO(), "k", "x"); err != nil { 1484 t.Fatal(err) 1485 } 1486 if _, err = lkv.Put(context.TODO(), "kk", "y"); err != nil { 1487 t.Fatal(err) 1488 } 1489 if _, err = lkv.Get(context.TODO(), "k"); err != nil { 1490 t.Fatal(err) 1491 } 1492 1493 for i := 0; i < 10; i++ { 1494 v := fmt.Sprintf("%d", i) 1495 donec := make(chan struct{}) 1496 clus.Members[0].DropConnections() 1497 go func() { 1498 defer close(donec) 1499 for i := 0; i < 20; i++ { 1500 clus.Members[0].DropConnections() 1501 time.Sleep(time.Millisecond) 1502 } 1503 }() 1504 switch rand.Intn(7) { 1505 case 0: 1506 _, err = lkv.Put(context.TODO(), "k", v) 1507 case 1: 1508 _, err = lkv.Delete(context.TODO(), "k") 1509 case 2: 1510 txn := lkv.Txn(context.TODO()).Then( 1511 clientv3.OpGet("k"), 1512 clientv3.OpDelete("k"), 1513 ) 1514 _, err = txn.Commit() 1515 case 3: 1516 txn := lkv.Txn(context.TODO()).Then( 1517 clientv3.OpGet("k"), 1518 clientv3.OpPut("k", v), 1519 ) 1520 _, err = txn.Commit() 1521 case 4: 1522 _, err = lkv.Do(context.TODO(), clientv3.OpPut("k", v)) 1523 case 5: 1524 _, err = lkv.Do(context.TODO(), clientv3.OpDelete("k")) 1525 case 6: 1526 _, err = lkv.Delete(context.TODO(), "k", clientv3.WithPrefix()) 1527 } 1528 <-donec 1529 if err != nil { 1530 // TODO wrap input client to generate errors 1531 break 1532 } 1533 } 1534 1535 lresp, lerr := lkv.Get(context.TODO(), "k") 1536 if lerr != nil { 1537 t.Fatal(lerr) 1538 } 1539 cresp, cerr := clus.Client(0).Get(context.TODO(), "k") 1540 if cerr != nil { 1541 t.Fatal(cerr) 1542 } 1543 if !reflect.DeepEqual(lresp.Kvs, cresp.Kvs) { 1544 t.Fatalf("expected %+v, got %+v", cresp, lresp) 1545 } 1546 } 1547 1548 func TestLeasingTxnAtomicCache(t *testing.T) { 1549 defer testutil.AfterTest(t) 1550 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}) 1551 defer clus.Terminate(t) 1552 1553 lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "foo/") 1554 testutil.AssertNil(t, err) 1555 defer closeLKV() 1556 1557 puts, gets := make([]clientv3.Op, 16), make([]clientv3.Op, 16) 1558 for i := range puts { 1559 k := fmt.Sprintf("k-%d", i) 1560 puts[i], gets[i] = clientv3.OpPut(k, k), clientv3.OpGet(k) 1561 } 1562 if _, err = clus.Client(0).Txn(context.TODO()).Then(puts...).Commit(); err != nil { 1563 t.Fatal(err) 1564 } 1565 for i := range gets { 1566 if _, err = lkv.Do(context.TODO(), gets[i]); err != nil { 1567 t.Fatal(err) 1568 } 1569 } 1570 1571 numPutters, numGetters := 16, 16 1572 1573 var wgPutters, wgGetters sync.WaitGroup 1574 wgPutters.Add(numPutters) 1575 wgGetters.Add(numGetters) 1576 1577 f := func() { 1578 defer wgPutters.Done() 1579 for i := 0; i < 10; i++ { 1580 if _, txnerr := lkv.Txn(context.TODO()).Then(puts...).Commit(); err != nil { 1581 t.Fatal(txnerr) 1582 } 1583 } 1584 } 1585 1586 donec := make(chan struct{}, numPutters) 1587 g := func() { 1588 defer wgGetters.Done() 1589 for { 1590 select { 1591 case <-donec: 1592 return 1593 default: 1594 } 1595 tresp, err := lkv.Txn(context.TODO()).Then(gets...).Commit() 1596 if err != nil { 1597 t.Fatal(err) 1598 } 1599 revs := make([]int64, len(gets)) 1600 for i, resp := range tresp.Responses { 1601 rr := resp.GetResponseRange() 1602 revs[i] = rr.Kvs[0].ModRevision 1603 } 1604 for i := 1; i < len(revs); i++ { 1605 if revs[i] != revs[i-1] { 1606 t.Fatalf("expected matching revisions, got %+v", revs) 1607 } 1608 } 1609 } 1610 } 1611 1612 for i := 0; i < numGetters; i++ { 1613 go g() 1614 } 1615 for i := 0; i < numPutters; i++ { 1616 go f() 1617 } 1618 1619 wgPutters.Wait() 1620 close(donec) 1621 wgGetters.Wait() 1622 } 1623 1624 // TestLeasingReconnectTxn checks that Txn is resilient to disconnects. 1625 func TestLeasingReconnectTxn(t *testing.T) { 1626 defer testutil.AfterTest(t) 1627 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}) 1628 defer clus.Terminate(t) 1629 1630 lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "foo/") 1631 testutil.AssertNil(t, err) 1632 defer closeLKV() 1633 1634 if _, err = lkv.Get(context.TODO(), "k"); err != nil { 1635 t.Fatal(err) 1636 } 1637 1638 donec := make(chan struct{}) 1639 go func() { 1640 defer close(donec) 1641 clus.Members[0].DropConnections() 1642 for i := 0; i < 10; i++ { 1643 clus.Members[0].DropConnections() 1644 time.Sleep(time.Millisecond) 1645 } 1646 }() 1647 1648 _, lerr := lkv.Txn(context.TODO()). 1649 If(clientv3.Compare(clientv3.Version("k"), "=", 0)). 1650 Then(clientv3.OpGet("k")). 1651 Commit() 1652 <-donec 1653 if lerr != nil { 1654 t.Fatal(lerr) 1655 } 1656 } 1657 1658 // TestLeasingReconnectNonOwnerGet checks a get error on an owner will 1659 // not cause inconsistency between the server and the client. 1660 func TestLeasingReconnectNonOwnerGet(t *testing.T) { 1661 defer testutil.AfterTest(t) 1662 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}) 1663 defer clus.Terminate(t) 1664 1665 lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "foo/") 1666 testutil.AssertNil(t, err) 1667 defer closeLKV() 1668 1669 // populate a few keys so some leasing gets have keys 1670 for i := 0; i < 4; i++ { 1671 k := fmt.Sprintf("k-%d", i*2) 1672 if _, err = lkv.Put(context.TODO(), k, k[2:]); err != nil { 1673 t.Fatal(err) 1674 } 1675 } 1676 1677 n := 0 1678 for i := 0; i < 10; i++ { 1679 donec := make(chan struct{}) 1680 clus.Members[0].DropConnections() 1681 go func() { 1682 defer close(donec) 1683 for j := 0; j < 10; j++ { 1684 clus.Members[0].DropConnections() 1685 time.Sleep(time.Millisecond) 1686 } 1687 }() 1688 _, err = lkv.Get(context.TODO(), fmt.Sprintf("k-%d", i)) 1689 <-donec 1690 n++ 1691 if err != nil { 1692 break 1693 } 1694 } 1695 for i := 0; i < n; i++ { 1696 k := fmt.Sprintf("k-%d", i) 1697 lresp, lerr := lkv.Get(context.TODO(), k) 1698 if lerr != nil { 1699 t.Fatal(lerr) 1700 } 1701 cresp, cerr := clus.Client(0).Get(context.TODO(), k) 1702 if cerr != nil { 1703 t.Fatal(cerr) 1704 } 1705 if !reflect.DeepEqual(lresp.Kvs, cresp.Kvs) { 1706 t.Fatalf("expected %+v, got %+v", cresp, lresp) 1707 } 1708 } 1709 } 1710 1711 func TestLeasingTxnRangeCmp(t *testing.T) { 1712 defer testutil.AfterTest(t) 1713 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}) 1714 defer clus.Terminate(t) 1715 1716 lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "foo/") 1717 testutil.AssertNil(t, err) 1718 defer closeLKV() 1719 1720 if _, err = clus.Client(0).Put(context.TODO(), "k", "a"); err != nil { 1721 t.Fatal(err) 1722 } 1723 // k2 version = 2 1724 if _, err = clus.Client(0).Put(context.TODO(), "k2", "a"); err != nil { 1725 t.Fatal(err) 1726 } 1727 if _, err = clus.Client(0).Put(context.TODO(), "k2", "a"); err != nil { 1728 t.Fatal(err) 1729 } 1730 1731 // cache k 1732 if _, err = lkv.Get(context.TODO(), "k"); err != nil { 1733 t.Fatal(err) 1734 } 1735 1736 cmp := clientv3.Compare(clientv3.Version("k").WithPrefix(), "=", 1) 1737 tresp, terr := lkv.Txn(context.TODO()).If(cmp).Commit() 1738 if terr != nil { 1739 t.Fatal(err) 1740 } 1741 if tresp.Succeeded { 1742 t.Fatalf("expected Succeeded=false, got %+v", tresp) 1743 } 1744 } 1745 1746 func TestLeasingDo(t *testing.T) { 1747 defer testutil.AfterTest(t) 1748 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1}) 1749 defer clus.Terminate(t) 1750 1751 lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "foo/") 1752 testutil.AssertNil(t, err) 1753 defer closeLKV() 1754 1755 ops := []clientv3.Op{ 1756 clientv3.OpTxn(nil, nil, nil), 1757 clientv3.OpGet("a"), 1758 clientv3.OpPut("a/abc", "v"), 1759 clientv3.OpDelete("a", clientv3.WithPrefix()), 1760 clientv3.OpTxn(nil, nil, nil), 1761 } 1762 for i, op := range ops { 1763 resp, resperr := lkv.Do(context.TODO(), op) 1764 if resperr != nil { 1765 t.Errorf("#%d: failed (%v)", i, resperr) 1766 } 1767 switch { 1768 case op.IsGet() && resp.Get() == nil: 1769 t.Errorf("#%d: get but nil get response", i) 1770 case op.IsPut() && resp.Put() == nil: 1771 t.Errorf("#%d: put op but nil get response", i) 1772 case op.IsDelete() && resp.Del() == nil: 1773 t.Errorf("#%d: delete op but nil delete response", i) 1774 case op.IsTxn() && resp.Txn() == nil: 1775 t.Errorf("#%d: txn op but nil txn response", i) 1776 } 1777 } 1778 1779 gresp, err := clus.Client(0).Get(context.TODO(), "a", clientv3.WithPrefix()) 1780 if err != nil { 1781 t.Fatal(err) 1782 } 1783 if len(gresp.Kvs) != 0 { 1784 t.Fatalf("expected no keys, got %+v", gresp.Kvs) 1785 } 1786 } 1787 1788 func TestLeasingTxnOwnerPutBranch(t *testing.T) { 1789 defer testutil.AfterTest(t) 1790 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3}) 1791 defer clus.Terminate(t) 1792 1793 lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "foo/") 1794 testutil.AssertNil(t, err) 1795 defer closeLKV() 1796 1797 n := 0 1798 treeOp := makePutTreeOp("tree", &n, 4) 1799 for i := 0; i < n; i++ { 1800 k := fmt.Sprintf("tree/%d", i) 1801 if _, err = clus.Client(0).Put(context.TODO(), k, "a"); err != nil { 1802 t.Fatal(err) 1803 } 1804 if _, err = lkv.Get(context.TODO(), k); err != nil { 1805 t.Fatal(err) 1806 } 1807 } 1808 1809 if _, err = lkv.Do(context.TODO(), treeOp); err != nil { 1810 t.Fatal(err) 1811 } 1812 1813 // lkv shouldn't need to call out to server for updated leased keys 1814 clus.Members[0].Stop(t) 1815 1816 for i := 0; i < n; i++ { 1817 k := fmt.Sprintf("tree/%d", i) 1818 lkvResp, err := lkv.Get(context.TODO(), k) 1819 if err != nil { 1820 t.Fatal(err) 1821 } 1822 clusResp, err := clus.Client(1).Get(context.TODO(), k) 1823 if err != nil { 1824 t.Fatal(err) 1825 } 1826 if !reflect.DeepEqual(clusResp.Kvs, lkvResp.Kvs) { 1827 t.Fatalf("expected %+v, got %+v", clusResp.Kvs, lkvResp.Kvs) 1828 } 1829 } 1830 } 1831 1832 func makePutTreeOp(pfx string, v *int, depth int) clientv3.Op { 1833 key := fmt.Sprintf("%s/%d", pfx, *v) 1834 *v = *v + 1 1835 if depth == 0 { 1836 return clientv3.OpPut(key, "leaf") 1837 } 1838 1839 t, e := makePutTreeOp(pfx, v, depth-1), makePutTreeOp(pfx, v, depth-1) 1840 tPut, ePut := clientv3.OpPut(key, "then"), clientv3.OpPut(key, "else") 1841 1842 cmps := make([]clientv3.Cmp, 1) 1843 if rand.Intn(2) == 0 { 1844 // follow then path 1845 cmps[0] = clientv3.Compare(clientv3.Version("nokey"), "=", 0) 1846 } else { 1847 // follow else path 1848 cmps[0] = clientv3.Compare(clientv3.Version("nokey"), ">", 0) 1849 } 1850 1851 return clientv3.OpTxn(cmps, []clientv3.Op{t, tPut}, []clientv3.Op{e, ePut}) 1852 } 1853 1854 func randCmps(pfx string, dat []*clientv3.PutResponse) (cmps []clientv3.Cmp, then bool) { 1855 for i := 0; i < len(dat); i++ { 1856 idx := rand.Intn(len(dat)) 1857 k := fmt.Sprintf("%s%d", pfx, idx) 1858 rev := dat[idx].Header.Revision 1859 var cmp clientv3.Cmp 1860 switch rand.Intn(4) { 1861 case 0: 1862 cmp = clientv3.Compare(clientv3.CreateRevision(k), ">", rev-1) 1863 case 1: 1864 cmp = clientv3.Compare(clientv3.Version(k), "=", 1) 1865 case 2: 1866 cmp = clientv3.Compare(clientv3.CreateRevision(k), "=", rev) 1867 case 3: 1868 cmp = clientv3.Compare(clientv3.CreateRevision(k), "!=", rev+1) 1869 1870 } 1871 cmps = append(cmps, cmp) 1872 } 1873 cmps = cmps[:rand.Intn(len(dat))] 1874 if rand.Intn(2) == 0 { 1875 return cmps, true 1876 } 1877 i := rand.Intn(len(dat)) 1878 cmps = append(cmps, clientv3.Compare(clientv3.Version(fmt.Sprintf("k-%d", i)), "=", 0)) 1879 return cmps, false 1880 } 1881 1882 func TestLeasingSessionExpire(t *testing.T) { 1883 defer testutil.AfterTest(t) 1884 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3}) 1885 defer clus.Terminate(t) 1886 1887 lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "foo/", concurrency.WithTTL(1)) 1888 testutil.AssertNil(t, err) 1889 defer closeLKV() 1890 1891 lkv2, closeLKV2, err := leasing.NewKV(clus.Client(0), "foo/") 1892 testutil.AssertNil(t, err) 1893 defer closeLKV2() 1894 1895 // acquire lease on abc 1896 if _, err = lkv.Get(context.TODO(), "abc"); err != nil { 1897 t.Fatal(err) 1898 } 1899 1900 // down endpoint lkv uses for keepalives 1901 clus.Members[0].Stop(t) 1902 if err = waitForLeasingExpire(clus.Client(1), "foo/abc"); err != nil { 1903 t.Fatal(err) 1904 } 1905 waitForExpireAck(t, lkv) 1906 clus.Members[0].Restart(t) 1907 1908 if _, err = lkv2.Put(context.TODO(), "abc", "def"); err != nil { 1909 t.Fatal(err) 1910 } 1911 1912 resp, err := lkv.Get(context.TODO(), "abc") 1913 if err != nil { 1914 t.Fatal(err) 1915 } 1916 if v := string(resp.Kvs[0].Value); v != "def" { 1917 t.Fatalf("expected %q, got %q", "v", v) 1918 } 1919 } 1920 1921 func TestLeasingSessionExpireCancel(t *testing.T) { 1922 defer testutil.AfterTest(t) 1923 clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3}) 1924 defer clus.Terminate(t) 1925 1926 tests := []func(context.Context, clientv3.KV) error{ 1927 func(ctx context.Context, kv clientv3.KV) error { 1928 _, err := kv.Get(ctx, "abc") 1929 return err 1930 }, 1931 func(ctx context.Context, kv clientv3.KV) error { 1932 _, err := kv.Delete(ctx, "abc") 1933 return err 1934 }, 1935 func(ctx context.Context, kv clientv3.KV) error { 1936 _, err := kv.Put(ctx, "abc", "v") 1937 return err 1938 }, 1939 func(ctx context.Context, kv clientv3.KV) error { 1940 _, err := kv.Txn(ctx).Then(clientv3.OpGet("abc")).Commit() 1941 return err 1942 }, 1943 func(ctx context.Context, kv clientv3.KV) error { 1944 _, err := kv.Do(ctx, clientv3.OpPut("abc", "v")) 1945 return err 1946 }, 1947 func(ctx context.Context, kv clientv3.KV) error { 1948 _, err := kv.Do(ctx, clientv3.OpDelete("abc")) 1949 return err 1950 }, 1951 func(ctx context.Context, kv clientv3.KV) error { 1952 _, err := kv.Do(ctx, clientv3.OpGet("abc")) 1953 return err 1954 }, 1955 func(ctx context.Context, kv clientv3.KV) error { 1956 op := clientv3.OpTxn(nil, []clientv3.Op{clientv3.OpGet("abc")}, nil) 1957 _, err := kv.Do(ctx, op) 1958 return err 1959 }, 1960 } 1961 for i := range tests { 1962 lkv, closeLKV, err := leasing.NewKV(clus.Client(0), "foo/", concurrency.WithTTL(1)) 1963 testutil.AssertNil(t, err) 1964 defer closeLKV() 1965 1966 if _, err = lkv.Get(context.TODO(), "abc"); err != nil { 1967 t.Fatal(err) 1968 } 1969 1970 // down endpoint lkv uses for keepalives 1971 clus.Members[0].Stop(t) 1972 if err := waitForLeasingExpire(clus.Client(1), "foo/abc"); err != nil { 1973 t.Fatal(err) 1974 } 1975 waitForExpireAck(t, lkv) 1976 1977 ctx, cancel := context.WithCancel(context.TODO()) 1978 errc := make(chan error, 1) 1979 go func() { errc <- tests[i](ctx, lkv) }() 1980 // some delay to get past for ctx.Err() != nil {} loops 1981 time.Sleep(100 * time.Millisecond) 1982 cancel() 1983 1984 select { 1985 case err := <-errc: 1986 if err != ctx.Err() { 1987 t.Errorf("#%d: expected %v, got %v", i, ctx.Err(), err) 1988 } 1989 case <-time.After(5 * time.Second): 1990 t.Errorf("#%d: timed out waiting for cancel", i) 1991 } 1992 clus.Members[0].Restart(t) 1993 } 1994 } 1995 1996 func waitForLeasingExpire(kv clientv3.KV, lkey string) error { 1997 for { 1998 time.Sleep(1 * time.Second) 1999 resp, err := kv.Get(context.TODO(), lkey, clientv3.WithPrefix()) 2000 if err != nil { 2001 return err 2002 } 2003 if len(resp.Kvs) == 0 { 2004 // server expired the leasing key 2005 return nil 2006 } 2007 } 2008 } 2009 2010 func waitForExpireAck(t *testing.T, kv clientv3.KV) { 2011 // wait for leasing client to acknowledge lost lease 2012 for i := 0; i < 10; i++ { 2013 ctx, cancel := context.WithTimeout(context.TODO(), time.Second) 2014 _, err := kv.Get(ctx, "abc") 2015 cancel() 2016 if err == ctx.Err() { 2017 return 2018 } 2019 time.Sleep(time.Second) 2020 } 2021 t.Fatalf("waited too long to acknlowedge lease expiration") 2022 }