github.com/clly/consul@v1.4.5/agent/consul/state/kvs_test.go (about) 1 package state 2 3 import ( 4 "reflect" 5 "strings" 6 "testing" 7 "time" 8 9 "github.com/hashicorp/consul/agent/structs" 10 "github.com/hashicorp/go-memdb" 11 ) 12 13 func TestStateStore_GC(t *testing.T) { 14 // Build up a fast GC. 15 ttl := 10 * time.Millisecond 16 gran := 5 * time.Millisecond 17 gc, err := NewTombstoneGC(ttl, gran) 18 if err != nil { 19 t.Fatalf("err: %s", err) 20 } 21 22 // Enable it and attach it to the state store. 23 gc.SetEnabled(true) 24 s, err := NewStateStore(gc) 25 if err != nil { 26 t.Fatalf("err: %s", err) 27 } 28 29 // Create some KV pairs. 30 testSetKey(t, s, 1, "foo", "foo") 31 testSetKey(t, s, 2, "foo/bar", "bar") 32 testSetKey(t, s, 3, "foo/baz", "bar") 33 testSetKey(t, s, 4, "foo/moo", "bar") 34 testSetKey(t, s, 5, "foo/zoo", "bar") 35 36 // Delete a key and make sure the GC sees it. 37 if err := s.KVSDelete(6, "foo/zoo"); err != nil { 38 t.Fatalf("err: %s", err) 39 } 40 select { 41 case idx := <-gc.ExpireCh(): 42 if idx != 6 { 43 t.Fatalf("bad index: %d", idx) 44 } 45 case <-time.After(2 * ttl): 46 t.Fatalf("GC never fired") 47 } 48 49 // Check for the same behavior with a tree delete. 50 if err := s.KVSDeleteTree(7, "foo/moo"); err != nil { 51 t.Fatalf("err: %s", err) 52 } 53 select { 54 case idx := <-gc.ExpireCh(): 55 if idx != 7 { 56 t.Fatalf("bad index: %d", idx) 57 } 58 case <-time.After(2 * ttl): 59 t.Fatalf("GC never fired") 60 } 61 62 // Check for the same behavior with a CAS delete. 63 if ok, err := s.KVSDeleteCAS(8, 3, "foo/baz"); !ok || err != nil { 64 t.Fatalf("err: %s", err) 65 } 66 select { 67 case idx := <-gc.ExpireCh(): 68 if idx != 8 { 69 t.Fatalf("bad index: %d", idx) 70 } 71 case <-time.After(2 * ttl): 72 t.Fatalf("GC never fired") 73 } 74 75 // Finally, try it with an expiring session. 76 testRegisterNode(t, s, 9, "node1") 77 session := &structs.Session{ 78 ID: testUUID(), 79 Node: "node1", 80 Behavior: structs.SessionKeysDelete, 81 } 82 if err := s.SessionCreate(10, session); err != nil { 83 t.Fatalf("err: %s", err) 84 } 85 d := &structs.DirEntry{ 86 Key: "lock", 87 Session: session.ID, 88 } 89 if ok, err := s.KVSLock(11, d); !ok || err != nil { 90 t.Fatalf("err: %v", err) 91 } 92 if err := s.SessionDestroy(12, session.ID); err != nil { 93 t.Fatalf("err: %s", err) 94 } 95 select { 96 case idx := <-gc.ExpireCh(): 97 if idx != 12 { 98 t.Fatalf("bad index: %d", idx) 99 } 100 case <-time.After(2 * ttl): 101 t.Fatalf("GC never fired") 102 } 103 } 104 105 func TestStateStore_ReapTombstones(t *testing.T) { 106 s := testStateStore(t) 107 108 // Create some KV pairs. 109 testSetKey(t, s, 1, "foo", "foo") 110 testSetKey(t, s, 2, "foo/bar", "bar") 111 testSetKey(t, s, 3, "foo/baz", "bar") 112 testSetKey(t, s, 4, "foo/moo", "bar") 113 testSetKey(t, s, 5, "foo/zoo", "bar") 114 115 // Call a delete on some specific keys. 116 if err := s.KVSDelete(6, "foo/baz"); err != nil { 117 t.Fatalf("err: %s", err) 118 } 119 if err := s.KVSDelete(7, "foo/moo"); err != nil { 120 t.Fatalf("err: %s", err) 121 } 122 123 // Pull out the list and check the index, which should come from the 124 // tombstones. 125 idx, _, err := s.KVSList(nil, "foo/") 126 if err != nil { 127 t.Fatalf("err: %s", err) 128 } 129 if idx != 7 { 130 t.Fatalf("bad index: %d", idx) 131 } 132 133 // Reap the tombstones <= 6. 134 if err := s.ReapTombstones(6); err != nil { 135 t.Fatalf("err: %s", err) 136 } 137 138 // Should still be good because 7 is in there. 139 idx, _, err = s.KVSList(nil, "foo/") 140 if err != nil { 141 t.Fatalf("err: %s", err) 142 } 143 if idx != 7 { 144 t.Fatalf("bad index: %d", idx) 145 } 146 147 // Now reap them all. 148 if err := s.ReapTombstones(7); err != nil { 149 t.Fatalf("err: %s", err) 150 } 151 152 // At this point the sub index will slide backwards. 153 idx, _, err = s.KVSList(nil, "foo/") 154 if err != nil { 155 t.Fatalf("err: %s", err) 156 } 157 if idx != 5 { 158 t.Fatalf("bad index: %d", idx) 159 } 160 161 // Make sure the tombstones are actually gone. 162 snap := s.Snapshot() 163 defer snap.Close() 164 stones, err := snap.Tombstones() 165 if err != nil { 166 t.Fatalf("err: %s", err) 167 } 168 if stones.Next() != nil { 169 t.Fatalf("unexpected extra tombstones") 170 } 171 } 172 173 func TestStateStore_KVSSet_KVSGet(t *testing.T) { 174 s := testStateStore(t) 175 176 // Get on an nonexistent key returns nil. 177 ws := memdb.NewWatchSet() 178 idx, result, err := s.KVSGet(ws, "foo") 179 if result != nil || err != nil || idx != 0 { 180 t.Fatalf("expected (0, nil, nil), got : (%#v, %#v, %#v)", idx, result, err) 181 } 182 183 // Write a new K/V entry to the store. 184 entry := &structs.DirEntry{ 185 Key: "foo", 186 Value: []byte("bar"), 187 } 188 if err := s.KVSSet(1, entry); err != nil { 189 t.Fatalf("err: %s", err) 190 } 191 if !watchFired(ws) { 192 t.Fatalf("bad") 193 } 194 195 // Retrieve the K/V entry again. 196 ws = memdb.NewWatchSet() 197 idx, result, err = s.KVSGet(ws, "foo") 198 if err != nil { 199 t.Fatalf("err: %s", err) 200 } 201 if result == nil { 202 t.Fatalf("expected k/v pair, got nothing") 203 } 204 if idx != 1 { 205 t.Fatalf("bad index: %d", idx) 206 } 207 208 // Check that the index was injected into the result. 209 if result.CreateIndex != 1 || result.ModifyIndex != 1 { 210 t.Fatalf("bad index: %d, %d", result.CreateIndex, result.ModifyIndex) 211 } 212 213 // Check that the value matches. 214 if v := string(result.Value); v != "bar" { 215 t.Fatalf("expected 'bar', got: '%s'", v) 216 } 217 218 // Updating the entry works and changes the index. 219 update := &structs.DirEntry{ 220 Key: "foo", 221 Value: []byte("baz"), 222 } 223 if err := s.KVSSet(2, update); err != nil { 224 t.Fatalf("err: %s", err) 225 } 226 if !watchFired(ws) { 227 t.Fatalf("bad") 228 } 229 230 // Fetch the kv pair and check. 231 ws = memdb.NewWatchSet() 232 idx, result, err = s.KVSGet(ws, "foo") 233 if err != nil { 234 t.Fatalf("err: %s", err) 235 } 236 if result.CreateIndex != 1 || result.ModifyIndex != 2 { 237 t.Fatalf("bad index: %d, %d", result.CreateIndex, result.ModifyIndex) 238 } 239 if v := string(result.Value); v != "baz" { 240 t.Fatalf("expected 'baz', got '%s'", v) 241 } 242 if idx != 2 { 243 t.Fatalf("bad index: %d", idx) 244 } 245 246 // Attempt to set the session during an update. 247 update = &structs.DirEntry{ 248 Key: "foo", 249 Value: []byte("zoo"), 250 Session: "nope", 251 } 252 if err := s.KVSSet(3, update); err != nil { 253 t.Fatalf("err: %s", err) 254 } 255 if !watchFired(ws) { 256 t.Fatalf("bad") 257 } 258 259 // Fetch the kv pair and check. 260 ws = memdb.NewWatchSet() 261 idx, result, err = s.KVSGet(ws, "foo") 262 if err != nil { 263 t.Fatalf("err: %s", err) 264 } 265 if result.CreateIndex != 1 || result.ModifyIndex != 3 { 266 t.Fatalf("bad index: %d, %d", result.CreateIndex, result.ModifyIndex) 267 } 268 if v := string(result.Value); v != "zoo" { 269 t.Fatalf("expected 'zoo', got '%s'", v) 270 } 271 if result.Session != "" { 272 t.Fatalf("expected empty session, got '%s", result.Session) 273 } 274 if idx != 3 { 275 t.Fatalf("bad index: %d", idx) 276 } 277 278 // Make a real session and then lock the key to set the session. 279 testRegisterNode(t, s, 4, "node1") 280 session := testUUID() 281 if err := s.SessionCreate(5, &structs.Session{ID: session, Node: "node1"}); err != nil { 282 t.Fatalf("err: %s", err) 283 } 284 update = &structs.DirEntry{ 285 Key: "foo", 286 Value: []byte("locked"), 287 Session: session, 288 } 289 ok, err := s.KVSLock(6, update) 290 if !ok || err != nil { 291 t.Fatalf("didn't get the lock: %v %s", ok, err) 292 } 293 if !watchFired(ws) { 294 t.Fatalf("bad") 295 } 296 297 // Fetch the kv pair and check. 298 ws = memdb.NewWatchSet() 299 idx, result, err = s.KVSGet(ws, "foo") 300 if err != nil { 301 t.Fatalf("err: %s", err) 302 } 303 if result.CreateIndex != 1 || result.ModifyIndex != 6 { 304 t.Fatalf("bad index: %d, %d", result.CreateIndex, result.ModifyIndex) 305 } 306 if v := string(result.Value); v != "locked" { 307 t.Fatalf("expected 'zoo', got '%s'", v) 308 } 309 if result.Session != session { 310 t.Fatalf("expected session, got '%s", result.Session) 311 } 312 if idx != 6 { 313 t.Fatalf("bad index: %d", idx) 314 } 315 316 // Now make an update without the session and make sure it gets applied 317 // and doesn't take away the session (it is allowed to change the value). 318 update = &structs.DirEntry{ 319 Key: "foo", 320 Value: []byte("stoleit"), 321 } 322 if err := s.KVSSet(7, update); err != nil { 323 t.Fatalf("err: %s", err) 324 } 325 if !watchFired(ws) { 326 t.Fatalf("bad") 327 } 328 329 // Fetch the kv pair and check. 330 ws = memdb.NewWatchSet() 331 idx, result, err = s.KVSGet(ws, "foo") 332 if err != nil { 333 t.Fatalf("err: %s", err) 334 } 335 if result.CreateIndex != 1 || result.ModifyIndex != 7 { 336 t.Fatalf("bad index: %d, %d", result.CreateIndex, result.ModifyIndex) 337 } 338 if v := string(result.Value); v != "stoleit" { 339 t.Fatalf("expected 'zoo', got '%s'", v) 340 } 341 if result.Session != session { 342 t.Fatalf("expected session, got '%s", result.Session) 343 } 344 if idx != 7 { 345 t.Fatalf("bad index: %d", idx) 346 } 347 348 // Setting some unrelated key should not fire the watch. 349 testSetKey(t, s, 8, "other", "yup") 350 if watchFired(ws) { 351 t.Fatalf("bad") 352 } 353 354 // Fetch a key that doesn't exist and make sure we get the right 355 // response. 356 idx, result, err = s.KVSGet(nil, "nope") 357 if result != nil || err != nil || idx != 8 { 358 t.Fatalf("expected (8, nil, nil), got : (%#v, %#v, %#v)", idx, result, err) 359 } 360 } 361 362 func TestStateStore_KVSList(t *testing.T) { 363 s := testStateStore(t) 364 365 // Listing an empty KVS returns nothing 366 ws := memdb.NewWatchSet() 367 idx, entries, err := s.KVSList(ws, "") 368 if idx != 0 || entries != nil || err != nil { 369 t.Fatalf("expected (0, nil, nil), got: (%d, %#v, %#v)", idx, entries, err) 370 } 371 372 // Create some KVS entries 373 testSetKey(t, s, 1, "foo", "foo") 374 testSetKey(t, s, 2, "foo/bar", "bar") 375 testSetKey(t, s, 3, "foo/bar/zip", "zip") 376 testSetKey(t, s, 4, "foo/bar/zip/zorp", "zorp") 377 testSetKey(t, s, 5, "foo/bar/baz", "baz") 378 if !watchFired(ws) { 379 t.Fatalf("bad") 380 } 381 382 // List out all of the keys 383 idx, entries, err = s.KVSList(nil, "") 384 if err != nil { 385 t.Fatalf("err: %s", err) 386 } 387 if idx != 5 { 388 t.Fatalf("bad index: %d", idx) 389 } 390 391 // Check that all of the keys were returned 392 if n := len(entries); n != 5 { 393 t.Fatalf("expected 5 kvs entries, got: %d", n) 394 } 395 396 // Try listing with a provided prefix 397 idx, entries, err = s.KVSList(nil, "foo/bar/zip") 398 if err != nil { 399 t.Fatalf("err: %s", err) 400 } 401 if idx != 4 { 402 t.Fatalf("bad index: %d", idx) 403 } 404 405 // Check that only the keys in the prefix were returned 406 if n := len(entries); n != 2 { 407 t.Fatalf("expected 2 kvs entries, got: %d", n) 408 } 409 if entries[0].Key != "foo/bar/zip" || entries[1].Key != "foo/bar/zip/zorp" { 410 t.Fatalf("bad: %#v", entries) 411 } 412 413 // Delete a key and make sure the index comes from the tombstone. 414 ws = memdb.NewWatchSet() 415 idx, _, err = s.KVSList(ws, "foo/bar/baz") 416 if err != nil { 417 t.Fatalf("err: %s", err) 418 } 419 if err := s.KVSDelete(6, "foo/bar/baz"); err != nil { 420 t.Fatalf("err: %s", err) 421 } 422 if !watchFired(ws) { 423 t.Fatalf("bad") 424 } 425 ws = memdb.NewWatchSet() 426 idx, _, err = s.KVSList(ws, "foo/bar/baz") 427 if err != nil { 428 t.Fatalf("err: %s", err) 429 } 430 if idx != 6 { 431 t.Fatalf("bad index: %d", idx) 432 } 433 434 // Set a different key to bump the index. This shouldn't fire the 435 // watch since there's a different prefix. 436 testSetKey(t, s, 7, "some/other/key", "") 437 if watchFired(ws) { 438 t.Fatalf("bad") 439 } 440 441 // Make sure we get the right index from the tombstone. 442 idx, _, err = s.KVSList(nil, "foo/bar/baz") 443 if err != nil { 444 t.Fatalf("err: %s", err) 445 } 446 if idx != 6 { 447 t.Fatalf("bad index: %d", idx) 448 } 449 450 // Now reap the tombstones and make sure we get the latest index 451 // since there are no matching keys. 452 if err := s.ReapTombstones(6); err != nil { 453 t.Fatalf("err: %s", err) 454 } 455 idx, _, err = s.KVSList(nil, "foo/bar/baz") 456 if err != nil { 457 t.Fatalf("err: %s", err) 458 } 459 if idx != 7 { 460 t.Fatalf("bad index: %d", idx) 461 } 462 463 // List all the keys to make sure the index is also correct. 464 idx, _, err = s.KVSList(nil, "") 465 if err != nil { 466 t.Fatalf("err: %s", err) 467 } 468 if idx != 7 { 469 t.Fatalf("bad index: %d", idx) 470 } 471 } 472 473 func TestStateStore_KVSListKeys(t *testing.T) { 474 s := testStateStore(t) 475 476 // Listing keys with no results returns nil. 477 ws := memdb.NewWatchSet() 478 idx, keys, err := s.KVSListKeys(ws, "", "") 479 if idx != 0 || keys != nil || err != nil { 480 t.Fatalf("expected (0, nil, nil), got: (%d, %#v, %#v)", idx, keys, err) 481 } 482 483 // Create some keys. 484 testSetKey(t, s, 1, "foo", "foo") 485 testSetKey(t, s, 2, "foo/bar", "bar") 486 testSetKey(t, s, 3, "foo/bar/baz", "baz") 487 testSetKey(t, s, 4, "foo/bar/zip", "zip") 488 testSetKey(t, s, 5, "foo/bar/zip/zam", "zam") 489 testSetKey(t, s, 6, "foo/bar/zip/zorp", "zorp") 490 testSetKey(t, s, 7, "some/other/prefix", "nack") 491 if !watchFired(ws) { 492 t.Fatalf("bad") 493 } 494 495 // List all the keys. 496 idx, keys, err = s.KVSListKeys(nil, "", "") 497 if err != nil { 498 t.Fatalf("err: %s", err) 499 } 500 if len(keys) != 7 { 501 t.Fatalf("bad keys: %#v", keys) 502 } 503 if idx != 7 { 504 t.Fatalf("bad index: %d", idx) 505 } 506 507 // Query using a prefix and pass a separator. 508 idx, keys, err = s.KVSListKeys(nil, "foo/bar/", "/") 509 if err != nil { 510 t.Fatalf("err: %s", err) 511 } 512 if len(keys) != 3 { 513 t.Fatalf("bad keys: %#v", keys) 514 } 515 if idx != 6 { 516 t.Fatalf("bad index: %d", idx) 517 } 518 519 // Subset of the keys was returned. 520 expect := []string{"foo/bar/baz", "foo/bar/zip", "foo/bar/zip/"} 521 if !reflect.DeepEqual(keys, expect) { 522 t.Fatalf("bad keys: %#v", keys) 523 } 524 525 // Listing keys with no separator returns everything. 526 idx, keys, err = s.KVSListKeys(nil, "foo", "") 527 if err != nil { 528 t.Fatalf("err: %s", err) 529 } 530 if idx != 6 { 531 t.Fatalf("bad index: %d", idx) 532 } 533 expect = []string{"foo", "foo/bar", "foo/bar/baz", "foo/bar/zip", 534 "foo/bar/zip/zam", "foo/bar/zip/zorp"} 535 if !reflect.DeepEqual(keys, expect) { 536 t.Fatalf("bad keys: %#v", keys) 537 } 538 539 // Delete a key and make sure the index comes from the tombstone. 540 ws = memdb.NewWatchSet() 541 idx, _, err = s.KVSListKeys(ws, "foo/bar/baz", "") 542 if err != nil { 543 t.Fatalf("err: %s", err) 544 } 545 if err := s.KVSDelete(8, "foo/bar/baz"); err != nil { 546 t.Fatalf("err: %s", err) 547 } 548 if !watchFired(ws) { 549 t.Fatalf("bad") 550 } 551 ws = memdb.NewWatchSet() 552 idx, _, err = s.KVSListKeys(ws, "foo/bar/baz", "") 553 if err != nil { 554 t.Fatalf("err: %s", err) 555 } 556 if idx != 8 { 557 t.Fatalf("bad index: %d", idx) 558 } 559 560 // Set a different key to bump the index. This shouldn't fire the watch 561 // since there's a different prefix. 562 testSetKey(t, s, 9, "some/other/key", "") 563 if watchFired(ws) { 564 t.Fatalf("bad") 565 } 566 567 // Make sure the index still comes from the tombstone. 568 idx, _, err = s.KVSListKeys(nil, "foo/bar/baz", "") 569 if err != nil { 570 t.Fatalf("err: %s", err) 571 } 572 if idx != 8 { 573 t.Fatalf("bad index: %d", idx) 574 } 575 576 // Now reap the tombstones and make sure we get the latest index 577 // since there are no matching keys. 578 if err := s.ReapTombstones(8); err != nil { 579 t.Fatalf("err: %s", err) 580 } 581 idx, _, err = s.KVSListKeys(nil, "foo/bar/baz", "") 582 if err != nil { 583 t.Fatalf("err: %s", err) 584 } 585 if idx != 9 { 586 t.Fatalf("bad index: %d", idx) 587 } 588 589 // List all the keys to make sure the index is also correct. 590 idx, _, err = s.KVSListKeys(nil, "", "") 591 if err != nil { 592 t.Fatalf("err: %s", err) 593 } 594 if idx != 9 { 595 t.Fatalf("bad index: %d", idx) 596 } 597 } 598 599 func TestStateStore_KVSDelete(t *testing.T) { 600 s := testStateStore(t) 601 602 // Create some KV pairs 603 testSetKey(t, s, 1, "foo", "foo") 604 testSetKey(t, s, 2, "foo/bar", "bar") 605 606 // Call a delete on a specific key 607 if err := s.KVSDelete(3, "foo"); err != nil { 608 t.Fatalf("err: %s", err) 609 } 610 611 // The entry was removed from the state store 612 tx := s.db.Txn(false) 613 defer tx.Abort() 614 e, err := tx.First("kvs", "id", "foo") 615 if err != nil { 616 t.Fatalf("err: %s", err) 617 } 618 if e != nil { 619 t.Fatalf("expected kvs entry to be deleted, got: %#v", e) 620 } 621 622 // Try fetching the other keys to ensure they still exist 623 e, err = tx.First("kvs", "id", "foo/bar") 624 if err != nil { 625 t.Fatalf("err: %s", err) 626 } 627 if e == nil || string(e.(*structs.DirEntry).Value) != "bar" { 628 t.Fatalf("bad kvs entry: %#v", e) 629 } 630 631 // Check that the index table was updated 632 if idx := s.maxIndex("kvs"); idx != 3 { 633 t.Fatalf("bad index: %d", idx) 634 } 635 636 // Check that the tombstone was created and that prevents the index 637 // from sliding backwards. 638 idx, _, err := s.KVSList(nil, "foo") 639 if err != nil { 640 t.Fatalf("err: %s", err) 641 } 642 if idx != 3 { 643 t.Fatalf("bad index: %d", idx) 644 } 645 646 // Now reap the tombstone and watch the index revert to the remaining 647 // foo/bar key's index. 648 if err := s.ReapTombstones(3); err != nil { 649 t.Fatalf("err: %s", err) 650 } 651 idx, _, err = s.KVSList(nil, "foo") 652 if err != nil { 653 t.Fatalf("err: %s", err) 654 } 655 if idx != 2 { 656 t.Fatalf("bad index: %d", idx) 657 } 658 659 // Deleting a nonexistent key should be idempotent and not return an 660 // error 661 if err := s.KVSDelete(4, "foo"); err != nil { 662 t.Fatalf("err: %s", err) 663 } 664 if idx := s.maxIndex("kvs"); idx != 3 { 665 t.Fatalf("bad index: %d", idx) 666 } 667 } 668 669 func TestStateStore_KVSDeleteCAS(t *testing.T) { 670 s := testStateStore(t) 671 672 // Create some KV entries 673 testSetKey(t, s, 1, "foo", "foo") 674 testSetKey(t, s, 2, "bar", "bar") 675 testSetKey(t, s, 3, "baz", "baz") 676 677 // Do a CAS delete with an index lower than the entry 678 ok, err := s.KVSDeleteCAS(4, 1, "bar") 679 if ok || err != nil { 680 t.Fatalf("expected (false, nil), got: (%v, %#v)", ok, err) 681 } 682 683 // Check that the index is untouched and the entry 684 // has not been deleted. 685 idx, e, err := s.KVSGet(nil, "foo") 686 if err != nil { 687 t.Fatalf("err: %s", err) 688 } 689 if e == nil { 690 t.Fatalf("expected a kvs entry, got nil") 691 } 692 if idx != 3 { 693 t.Fatalf("bad index: %d", idx) 694 } 695 696 // Do another CAS delete, this time with the correct index 697 // which should cause the delete to take place. 698 ok, err = s.KVSDeleteCAS(4, 2, "bar") 699 if !ok || err != nil { 700 t.Fatalf("expected (true, nil), got: (%v, %#v)", ok, err) 701 } 702 703 // Entry was deleted and index was updated 704 idx, e, err = s.KVSGet(nil, "bar") 705 if err != nil { 706 t.Fatalf("err: %s", err) 707 } 708 if e != nil { 709 t.Fatalf("entry should be deleted") 710 } 711 if idx != 4 { 712 t.Fatalf("bad index: %d", idx) 713 } 714 715 // Add another key to bump the index. 716 testSetKey(t, s, 5, "some/other/key", "baz") 717 718 // Check that the tombstone was created and that prevents the index 719 // from sliding backwards. 720 idx, _, err = s.KVSList(nil, "bar") 721 if err != nil { 722 t.Fatalf("err: %s", err) 723 } 724 if idx != 4 { 725 t.Fatalf("bad index: %d", idx) 726 } 727 728 // Now reap the tombstone and watch the index move up to the table 729 // index since there are no matching keys. 730 if err := s.ReapTombstones(4); err != nil { 731 t.Fatalf("err: %s", err) 732 } 733 idx, _, err = s.KVSList(nil, "bar") 734 if err != nil { 735 t.Fatalf("err: %s", err) 736 } 737 if idx != 5 { 738 t.Fatalf("bad index: %d", idx) 739 } 740 741 // A delete on a nonexistent key should be idempotent and not return an 742 // error 743 ok, err = s.KVSDeleteCAS(6, 2, "bar") 744 if !ok || err != nil { 745 t.Fatalf("expected (true, nil), got: (%v, %#v)", ok, err) 746 } 747 if idx := s.maxIndex("kvs"); idx != 5 { 748 t.Fatalf("bad index: %d", idx) 749 } 750 } 751 752 func TestStateStore_KVSSetCAS(t *testing.T) { 753 s := testStateStore(t) 754 755 // Doing a CAS with ModifyIndex != 0 and no existing entry 756 // is a no-op. 757 entry := &structs.DirEntry{ 758 Key: "foo", 759 Value: []byte("foo"), 760 RaftIndex: structs.RaftIndex{ 761 CreateIndex: 1, 762 ModifyIndex: 1, 763 }, 764 } 765 ok, err := s.KVSSetCAS(2, entry) 766 if ok || err != nil { 767 t.Fatalf("expected (false, nil), got: (%#v, %#v)", ok, err) 768 } 769 770 // Check that nothing was actually stored 771 tx := s.db.Txn(false) 772 if e, err := tx.First("kvs", "id", "foo"); e != nil || err != nil { 773 t.Fatalf("expected (nil, nil), got: (%#v, %#v)", e, err) 774 } 775 tx.Abort() 776 777 // Index was not updated 778 if idx := s.maxIndex("kvs"); idx != 0 { 779 t.Fatalf("bad index: %d", idx) 780 } 781 782 // Doing a CAS with a ModifyIndex of zero when no entry exists 783 // performs the set and saves into the state store. 784 entry = &structs.DirEntry{ 785 Key: "foo", 786 Value: []byte("foo"), 787 RaftIndex: structs.RaftIndex{ 788 CreateIndex: 0, 789 ModifyIndex: 0, 790 }, 791 } 792 ok, err = s.KVSSetCAS(2, entry) 793 if !ok || err != nil { 794 t.Fatalf("expected (true, nil), got: (%#v, %#v)", ok, err) 795 } 796 797 // Entry was inserted 798 idx, entry, err := s.KVSGet(nil, "foo") 799 if err != nil { 800 t.Fatalf("err: %s", err) 801 } 802 if string(entry.Value) != "foo" || entry.CreateIndex != 2 || entry.ModifyIndex != 2 { 803 t.Fatalf("bad entry: %#v", entry) 804 } 805 if idx != 2 { 806 t.Fatalf("bad index: %d", idx) 807 } 808 809 // Doing a CAS with a ModifyIndex of zero when an entry exists does 810 // not do anything. 811 entry = &structs.DirEntry{ 812 Key: "foo", 813 Value: []byte("foo"), 814 RaftIndex: structs.RaftIndex{ 815 CreateIndex: 0, 816 ModifyIndex: 0, 817 }, 818 } 819 ok, err = s.KVSSetCAS(3, entry) 820 if ok || err != nil { 821 t.Fatalf("expected (false, nil), got: (%#v, %#v)", ok, err) 822 } 823 824 // Doing a CAS with a ModifyIndex which does not match the current 825 // index does not do anything. 826 entry = &structs.DirEntry{ 827 Key: "foo", 828 Value: []byte("bar"), 829 RaftIndex: structs.RaftIndex{ 830 CreateIndex: 3, 831 ModifyIndex: 3, 832 }, 833 } 834 ok, err = s.KVSSetCAS(3, entry) 835 if ok || err != nil { 836 t.Fatalf("expected (false, nil), got: (%#v, %#v)", ok, err) 837 } 838 839 // Entry was not updated in the store 840 idx, entry, err = s.KVSGet(nil, "foo") 841 if err != nil { 842 t.Fatalf("err: %s", err) 843 } 844 if string(entry.Value) != "foo" || entry.CreateIndex != 2 || entry.ModifyIndex != 2 { 845 t.Fatalf("bad entry: %#v", entry) 846 } 847 if idx != 2 { 848 t.Fatalf("bad index: %d", idx) 849 } 850 851 // Doing a CAS with the proper current index should make the 852 // modification. 853 entry = &structs.DirEntry{ 854 Key: "foo", 855 Value: []byte("bar"), 856 RaftIndex: structs.RaftIndex{ 857 CreateIndex: 2, 858 ModifyIndex: 2, 859 }, 860 } 861 ok, err = s.KVSSetCAS(3, entry) 862 if !ok || err != nil { 863 t.Fatalf("expected (true, nil), got: (%#v, %#v)", ok, err) 864 } 865 866 // Entry was updated 867 idx, entry, err = s.KVSGet(nil, "foo") 868 if err != nil { 869 t.Fatalf("err: %s", err) 870 } 871 if string(entry.Value) != "bar" || entry.CreateIndex != 2 || entry.ModifyIndex != 3 { 872 t.Fatalf("bad entry: %#v", entry) 873 } 874 if idx != 3 { 875 t.Fatalf("bad index: %d", idx) 876 } 877 878 // Attempt to update the session during the CAS. 879 entry = &structs.DirEntry{ 880 Key: "foo", 881 Value: []byte("zoo"), 882 Session: "nope", 883 RaftIndex: structs.RaftIndex{ 884 CreateIndex: 2, 885 ModifyIndex: 3, 886 }, 887 } 888 ok, err = s.KVSSetCAS(4, entry) 889 if !ok || err != nil { 890 t.Fatalf("expected (true, nil), got: (%#v, %#v)", ok, err) 891 } 892 893 // Entry was updated, but the session should have been ignored. 894 idx, entry, err = s.KVSGet(nil, "foo") 895 if err != nil { 896 t.Fatalf("err: %s", err) 897 } 898 if string(entry.Value) != "zoo" || entry.CreateIndex != 2 || entry.ModifyIndex != 4 || 899 entry.Session != "" { 900 t.Fatalf("bad entry: %#v", entry) 901 } 902 if idx != 4 { 903 t.Fatalf("bad index: %d", idx) 904 } 905 906 // Now lock it and try the update, which should keep the session. 907 testRegisterNode(t, s, 5, "node1") 908 session := testUUID() 909 if err := s.SessionCreate(6, &structs.Session{ID: session, Node: "node1"}); err != nil { 910 t.Fatalf("err: %s", err) 911 } 912 entry = &structs.DirEntry{ 913 Key: "foo", 914 Value: []byte("locked"), 915 Session: session, 916 RaftIndex: structs.RaftIndex{ 917 CreateIndex: 2, 918 ModifyIndex: 4, 919 }, 920 } 921 ok, err = s.KVSLock(6, entry) 922 if !ok || err != nil { 923 t.Fatalf("didn't get the lock: %v %s", ok, err) 924 } 925 entry = &structs.DirEntry{ 926 Key: "foo", 927 Value: []byte("locked"), 928 RaftIndex: structs.RaftIndex{ 929 CreateIndex: 2, 930 ModifyIndex: 6, 931 }, 932 } 933 ok, err = s.KVSSetCAS(7, entry) 934 if !ok || err != nil { 935 t.Fatalf("expected (true, nil), got: (%#v, %#v)", ok, err) 936 } 937 938 // Entry was updated, and the lock status should have stayed the same. 939 idx, entry, err = s.KVSGet(nil, "foo") 940 if err != nil { 941 t.Fatalf("err: %s", err) 942 } 943 if string(entry.Value) != "locked" || entry.CreateIndex != 2 || entry.ModifyIndex != 7 || 944 entry.Session != session { 945 t.Fatalf("bad entry: %#v", entry) 946 } 947 if idx != 7 { 948 t.Fatalf("bad index: %d", idx) 949 } 950 } 951 952 func TestStateStore_KVSDeleteTree(t *testing.T) { 953 s := testStateStore(t) 954 955 // Create kvs entries in the state store. 956 testSetKey(t, s, 1, "foo/bar", "bar") 957 testSetKey(t, s, 2, "foo/bar/baz", "baz") 958 testSetKey(t, s, 3, "foo/bar/zip", "zip") 959 testSetKey(t, s, 4, "foo/zorp", "zorp") 960 961 // Calling tree deletion which affects nothing does not 962 // modify the table index. 963 if err := s.KVSDeleteTree(9, "bar"); err != nil { 964 t.Fatalf("err: %s", err) 965 } 966 if idx := s.maxIndex("kvs"); idx != 4 { 967 t.Fatalf("bad index: %d", idx) 968 } 969 970 // Call tree deletion with a nested prefix. 971 if err := s.KVSDeleteTree(5, "foo/bar"); err != nil { 972 t.Fatalf("err: %s", err) 973 } 974 975 // Check that all the matching keys were deleted 976 tx := s.db.Txn(false) 977 defer tx.Abort() 978 979 entries, err := tx.Get("kvs", "id") 980 if err != nil { 981 t.Fatalf("err: %s", err) 982 } 983 984 num := 0 985 for entry := entries.Next(); entry != nil; entry = entries.Next() { 986 if entry.(*structs.DirEntry).Key != "foo/zorp" { 987 t.Fatalf("unexpected kvs entry: %#v", entry) 988 } 989 num++ 990 } 991 992 if num != 1 { 993 t.Fatalf("expected 1 key, got: %d", num) 994 } 995 996 // Index should be updated if modifications are made 997 if idx := s.maxIndex("kvs"); idx != 5 { 998 t.Fatalf("bad index: %d", idx) 999 } 1000 1001 // Check that the tombstones ware created and that prevents the index 1002 // from sliding backwards. 1003 idx, _, err := s.KVSList(nil, "foo") 1004 if err != nil { 1005 t.Fatalf("err: %s", err) 1006 } 1007 if idx != 5 { 1008 t.Fatalf("bad index: %d", idx) 1009 } 1010 1011 // Now reap the tombstones and watch the index revert to the remaining 1012 // foo/zorp key's index. 1013 if err := s.ReapTombstones(5); err != nil { 1014 t.Fatalf("err: %s", err) 1015 } 1016 idx, _, err = s.KVSList(nil, "foo") 1017 if err != nil { 1018 t.Fatalf("err: %s", err) 1019 } 1020 if idx != 4 { 1021 t.Fatalf("bad index: %d", idx) 1022 } 1023 } 1024 1025 func TestStateStore_Watches_PrefixDelete(t *testing.T) { 1026 s := testStateStore(t) 1027 1028 // Create some KVS entries 1029 testSetKey(t, s, 1, "foo", "foo") 1030 testSetKey(t, s, 2, "foo/bar", "bar") 1031 testSetKey(t, s, 3, "foo/bar/zip", "zip") 1032 testSetKey(t, s, 4, "foo/bar/zip/zorp", "zorp") 1033 testSetKey(t, s, 5, "foo/bar/zip/zap", "zap") 1034 testSetKey(t, s, 6, "foo/nope", "nope") 1035 1036 ws := memdb.NewWatchSet() 1037 got, _, err := s.KVSList(ws, "foo/bar") 1038 if err != nil { 1039 t.Fatalf("unexpected err: %s", err) 1040 } 1041 var wantIndex uint64 = 5 1042 if got != wantIndex { 1043 t.Fatalf("bad index: %d, expected %d", wantIndex, got) 1044 } 1045 1046 // Delete a key and make sure the index comes from the tombstone. 1047 if err := s.KVSDeleteTree(7, "foo/bar/zip"); err != nil { 1048 t.Fatalf("unexpected err: %s", err) 1049 } 1050 // Make sure watch fires 1051 if !watchFired(ws) { 1052 t.Fatalf("expected watch to fire but it did not") 1053 } 1054 1055 //Verify index matches tombstone 1056 got, _, err = s.KVSList(ws, "foo/bar") 1057 if err != nil { 1058 t.Fatalf("unexpected err: %s", err) 1059 } 1060 wantIndex = 7 1061 if got != wantIndex { 1062 t.Fatalf("bad index: %d, expected %d", got, wantIndex) 1063 } 1064 // Make sure watch fires 1065 if !watchFired(ws) { 1066 t.Fatalf("expected watch to fire but it did not") 1067 } 1068 1069 // Reap tombstone and verify list on the same key reverts its index value 1070 if err := s.ReapTombstones(wantIndex); err != nil { 1071 t.Fatalf("err: %s", err) 1072 } 1073 1074 got, _, err = s.KVSList(nil, "foo/bar") 1075 wantIndex = 2 1076 if err != nil { 1077 t.Fatalf("err: %s", err) 1078 } 1079 if got != wantIndex { 1080 t.Fatalf("bad index: %d, expected %d", got, wantIndex) 1081 } 1082 1083 // Set a different key to bump the index. This shouldn't fire the 1084 // watch since there's a different prefix. 1085 testSetKey(t, s, 8, "some/other/key", "") 1086 1087 // Now ask for the index for a node within the prefix that was deleted 1088 // We expect to get the max index in the tree 1089 wantIndex = 8 1090 ws = memdb.NewWatchSet() 1091 got, _, err = s.KVSList(ws, "foo/bar/baz") 1092 if err != nil { 1093 t.Fatalf("err: %s", err) 1094 } 1095 if watchFired(ws) { 1096 t.Fatalf("Watch should not have fired") 1097 } 1098 if got != wantIndex { 1099 t.Fatalf("bad index: %d, expected %d", got, wantIndex) 1100 } 1101 1102 // List all the keys to make sure the index returned is the max index 1103 got, _, err = s.KVSList(nil, "") 1104 if err != nil { 1105 t.Fatalf("err: %s", err) 1106 } 1107 if got != wantIndex { 1108 t.Fatalf("bad index: %d, expected %d", got, wantIndex) 1109 } 1110 1111 // Delete all the keys, special case where tombstones are not inserted 1112 if err := s.KVSDeleteTree(9, ""); err != nil { 1113 t.Fatalf("unexpected err: %s", err) 1114 } 1115 wantIndex = 9 1116 got, _, err = s.KVSList(nil, "/foo/bar") 1117 if err != nil { 1118 t.Fatalf("err: %s", err) 1119 } 1120 if got != wantIndex { 1121 t.Fatalf("bad index: %d, expected %d", got, wantIndex) 1122 } 1123 1124 } 1125 1126 func TestStateStore_KVSLockDelay(t *testing.T) { 1127 s := testStateStore(t) 1128 1129 // KVSLockDelay is exercised in the lock/unlock and session invalidation 1130 // cases below, so we just do a basic check on a nonexistent key here. 1131 expires := s.KVSLockDelay("/not/there") 1132 if expires.After(time.Now()) { 1133 t.Fatalf("bad: %v", expires) 1134 } 1135 } 1136 1137 func TestStateStore_KVSLock(t *testing.T) { 1138 s := testStateStore(t) 1139 1140 // Lock with no session should fail. 1141 ok, err := s.KVSLock(0, &structs.DirEntry{Key: "foo", Value: []byte("foo")}) 1142 if ok || err == nil || !strings.Contains(err.Error(), "missing session") { 1143 t.Fatalf("didn't detect missing session: %v %s", ok, err) 1144 } 1145 1146 // Now try with a bogus session. 1147 ok, err = s.KVSLock(1, &structs.DirEntry{Key: "foo", Value: []byte("foo"), Session: testUUID()}) 1148 if ok || err == nil || !strings.Contains(err.Error(), "invalid session") { 1149 t.Fatalf("didn't detect invalid session: %v %s", ok, err) 1150 } 1151 1152 // Make a real session. 1153 testRegisterNode(t, s, 2, "node1") 1154 session1 := testUUID() 1155 if err := s.SessionCreate(3, &structs.Session{ID: session1, Node: "node1"}); err != nil { 1156 t.Fatalf("err: %s", err) 1157 } 1158 1159 // Lock and make the key at the same time. 1160 ok, err = s.KVSLock(4, &structs.DirEntry{Key: "foo", Value: []byte("foo"), Session: session1}) 1161 if !ok || err != nil { 1162 t.Fatalf("didn't get the lock: %v %s", ok, err) 1163 } 1164 1165 // Make sure the indexes got set properly. 1166 idx, result, err := s.KVSGet(nil, "foo") 1167 if err != nil { 1168 t.Fatalf("err: %s", err) 1169 } 1170 if result.LockIndex != 1 || result.CreateIndex != 4 || result.ModifyIndex != 4 || 1171 string(result.Value) != "foo" { 1172 t.Fatalf("bad entry: %#v", result) 1173 } 1174 if idx != 4 { 1175 t.Fatalf("bad index: %d", idx) 1176 } 1177 1178 // Re-locking with the same session should update the value and report 1179 // success. 1180 ok, err = s.KVSLock(5, &structs.DirEntry{Key: "foo", Value: []byte("bar"), Session: session1}) 1181 if !ok || err != nil { 1182 t.Fatalf("didn't handle locking an already-locked key: %v %s", ok, err) 1183 } 1184 1185 // Make sure the indexes got set properly, note that the lock index 1186 // won't go up since we didn't lock it again. 1187 idx, result, err = s.KVSGet(nil, "foo") 1188 if err != nil { 1189 t.Fatalf("err: %s", err) 1190 } 1191 if result.LockIndex != 1 || result.CreateIndex != 4 || result.ModifyIndex != 5 || 1192 string(result.Value) != "bar" { 1193 t.Fatalf("bad entry: %#v", result) 1194 } 1195 if idx != 5 { 1196 t.Fatalf("bad index: %d", idx) 1197 } 1198 1199 // Unlock and the re-lock. 1200 ok, err = s.KVSUnlock(6, &structs.DirEntry{Key: "foo", Value: []byte("baz"), Session: session1}) 1201 if !ok || err != nil { 1202 t.Fatalf("didn't handle unlocking a locked key: %v %s", ok, err) 1203 } 1204 ok, err = s.KVSLock(7, &structs.DirEntry{Key: "foo", Value: []byte("zoo"), Session: session1}) 1205 if !ok || err != nil { 1206 t.Fatalf("didn't get the lock: %v %s", ok, err) 1207 } 1208 1209 // Make sure the indexes got set properly. 1210 idx, result, err = s.KVSGet(nil, "foo") 1211 if err != nil { 1212 t.Fatalf("err: %s", err) 1213 } 1214 if result.LockIndex != 2 || result.CreateIndex != 4 || result.ModifyIndex != 7 || 1215 string(result.Value) != "zoo" { 1216 t.Fatalf("bad entry: %#v", result) 1217 } 1218 if idx != 7 { 1219 t.Fatalf("bad index: %d", idx) 1220 } 1221 1222 // Lock an existing key. 1223 testSetKey(t, s, 8, "bar", "bar") 1224 ok, err = s.KVSLock(9, &structs.DirEntry{Key: "bar", Value: []byte("xxx"), Session: session1}) 1225 if !ok || err != nil { 1226 t.Fatalf("didn't get the lock: %v %s", ok, err) 1227 } 1228 1229 // Make sure the indexes got set properly. 1230 idx, result, err = s.KVSGet(nil, "bar") 1231 if err != nil { 1232 t.Fatalf("err: %s", err) 1233 } 1234 if result.LockIndex != 1 || result.CreateIndex != 8 || result.ModifyIndex != 9 || 1235 string(result.Value) != "xxx" { 1236 t.Fatalf("bad entry: %#v", result) 1237 } 1238 if idx != 9 { 1239 t.Fatalf("bad index: %d", idx) 1240 } 1241 1242 // Attempting a re-lock with a different session should also fail. 1243 session2 := testUUID() 1244 if err := s.SessionCreate(10, &structs.Session{ID: session2, Node: "node1"}); err != nil { 1245 t.Fatalf("err: %s", err) 1246 } 1247 1248 // Re-locking should not return an error, but will report that it didn't 1249 // get the lock. 1250 ok, err = s.KVSLock(11, &structs.DirEntry{Key: "bar", Value: []byte("nope"), Session: session2}) 1251 if ok || err != nil { 1252 t.Fatalf("didn't handle locking an already-locked key: %v %s", ok, err) 1253 } 1254 1255 // Make sure the indexes didn't update. 1256 idx, result, err = s.KVSGet(nil, "bar") 1257 if err != nil { 1258 t.Fatalf("err: %s", err) 1259 } 1260 if result.LockIndex != 1 || result.CreateIndex != 8 || result.ModifyIndex != 9 || 1261 string(result.Value) != "xxx" { 1262 t.Fatalf("bad entry: %#v", result) 1263 } 1264 if idx != 9 { 1265 t.Fatalf("bad index: %d", idx) 1266 } 1267 } 1268 1269 func TestStateStore_KVSUnlock(t *testing.T) { 1270 s := testStateStore(t) 1271 1272 // Unlock with no session should fail. 1273 ok, err := s.KVSUnlock(0, &structs.DirEntry{Key: "foo", Value: []byte("bar")}) 1274 if ok || err == nil || !strings.Contains(err.Error(), "missing session") { 1275 t.Fatalf("didn't detect missing session: %v %s", ok, err) 1276 } 1277 1278 // Make a real session. 1279 testRegisterNode(t, s, 1, "node1") 1280 session1 := testUUID() 1281 if err := s.SessionCreate(2, &structs.Session{ID: session1, Node: "node1"}); err != nil { 1282 t.Fatalf("err: %s", err) 1283 } 1284 1285 // Unlock with a real session but no key should not return an error, but 1286 // will report it didn't unlock anything. 1287 ok, err = s.KVSUnlock(3, &structs.DirEntry{Key: "foo", Value: []byte("bar"), Session: session1}) 1288 if ok || err != nil { 1289 t.Fatalf("didn't handle unlocking a missing key: %v %s", ok, err) 1290 } 1291 1292 // Make a key and unlock it, without it being locked. 1293 testSetKey(t, s, 4, "foo", "bar") 1294 ok, err = s.KVSUnlock(5, &structs.DirEntry{Key: "foo", Value: []byte("baz"), Session: session1}) 1295 if ok || err != nil { 1296 t.Fatalf("didn't handle unlocking a non-locked key: %v %s", ok, err) 1297 } 1298 1299 // Make sure the indexes didn't update. 1300 idx, result, err := s.KVSGet(nil, "foo") 1301 if err != nil { 1302 t.Fatalf("err: %s", err) 1303 } 1304 if result.LockIndex != 0 || result.CreateIndex != 4 || result.ModifyIndex != 4 || 1305 string(result.Value) != "bar" { 1306 t.Fatalf("bad entry: %#v", result) 1307 } 1308 if idx != 4 { 1309 t.Fatalf("bad index: %d", idx) 1310 } 1311 1312 // Lock it with the first session. 1313 ok, err = s.KVSLock(6, &structs.DirEntry{Key: "foo", Value: []byte("bar"), Session: session1}) 1314 if !ok || err != nil { 1315 t.Fatalf("didn't get the lock: %v %s", ok, err) 1316 } 1317 1318 // Attempt an unlock with another session. 1319 session2 := testUUID() 1320 if err := s.SessionCreate(7, &structs.Session{ID: session2, Node: "node1"}); err != nil { 1321 t.Fatalf("err: %s", err) 1322 } 1323 ok, err = s.KVSUnlock(8, &structs.DirEntry{Key: "foo", Value: []byte("zoo"), Session: session2}) 1324 if ok || err != nil { 1325 t.Fatalf("didn't handle unlocking with the wrong session: %v %s", ok, err) 1326 } 1327 1328 // Make sure the indexes didn't update. 1329 idx, result, err = s.KVSGet(nil, "foo") 1330 if err != nil { 1331 t.Fatalf("err: %s", err) 1332 } 1333 if result.LockIndex != 1 || result.CreateIndex != 4 || result.ModifyIndex != 6 || 1334 string(result.Value) != "bar" { 1335 t.Fatalf("bad entry: %#v", result) 1336 } 1337 if idx != 6 { 1338 t.Fatalf("bad index: %d", idx) 1339 } 1340 1341 // Now do the unlock with the correct session. 1342 ok, err = s.KVSUnlock(9, &structs.DirEntry{Key: "foo", Value: []byte("zoo"), Session: session1}) 1343 if !ok || err != nil { 1344 t.Fatalf("didn't handle unlocking with the correct session: %v %s", ok, err) 1345 } 1346 1347 // Make sure the indexes got set properly. 1348 idx, result, err = s.KVSGet(nil, "foo") 1349 if err != nil { 1350 t.Fatalf("err: %s", err) 1351 } 1352 if result.LockIndex != 1 || result.CreateIndex != 4 || result.ModifyIndex != 9 || 1353 string(result.Value) != "zoo" { 1354 t.Fatalf("bad entry: %#v", result) 1355 } 1356 if idx != 9 { 1357 t.Fatalf("bad index: %d", idx) 1358 } 1359 1360 // Unlocking again should fail and not change anything. 1361 ok, err = s.KVSUnlock(10, &structs.DirEntry{Key: "foo", Value: []byte("nope"), Session: session1}) 1362 if ok || err != nil { 1363 t.Fatalf("didn't handle unlocking with the previous session: %v %s", ok, err) 1364 } 1365 1366 // Make sure the indexes didn't update. 1367 idx, result, err = s.KVSGet(nil, "foo") 1368 if err != nil { 1369 t.Fatalf("err: %s", err) 1370 } 1371 if result.LockIndex != 1 || result.CreateIndex != 4 || result.ModifyIndex != 9 || 1372 string(result.Value) != "zoo" { 1373 t.Fatalf("bad entry: %#v", result) 1374 } 1375 if idx != 9 { 1376 t.Fatalf("bad index: %d", idx) 1377 } 1378 } 1379 1380 func TestStateStore_KVS_Snapshot_Restore(t *testing.T) { 1381 s := testStateStore(t) 1382 1383 // Build up some entries to seed. 1384 entries := structs.DirEntries{ 1385 &structs.DirEntry{ 1386 Key: "aaa", 1387 Flags: 23, 1388 Value: []byte("hello"), 1389 }, 1390 &structs.DirEntry{ 1391 Key: "bar/a", 1392 Value: []byte("one"), 1393 }, 1394 &structs.DirEntry{ 1395 Key: "bar/b", 1396 Value: []byte("two"), 1397 }, 1398 &structs.DirEntry{ 1399 Key: "bar/c", 1400 Value: []byte("three"), 1401 }, 1402 } 1403 for i, entry := range entries { 1404 if err := s.KVSSet(uint64(i+1), entry); err != nil { 1405 t.Fatalf("err: %s", err) 1406 } 1407 } 1408 1409 // Make a node and session so we can test a locked key. 1410 testRegisterNode(t, s, 5, "node1") 1411 session := testUUID() 1412 if err := s.SessionCreate(6, &structs.Session{ID: session, Node: "node1"}); err != nil { 1413 t.Fatalf("err: %s", err) 1414 } 1415 entries[3].Session = session 1416 if ok, err := s.KVSLock(7, entries[3]); !ok || err != nil { 1417 t.Fatalf("didn't get the lock: %v %s", ok, err) 1418 } 1419 1420 // This is required for the compare later. 1421 entries[3].LockIndex = 1 1422 1423 // Snapshot the keys. 1424 snap := s.Snapshot() 1425 defer snap.Close() 1426 1427 // Alter the real state store. 1428 if err := s.KVSSet(8, &structs.DirEntry{Key: "aaa", Value: []byte("nope")}); err != nil { 1429 t.Fatalf("err: %s", err) 1430 } 1431 1432 // Verify the snapshot. 1433 if idx := snap.LastIndex(); idx != 7 { 1434 t.Fatalf("bad index: %d", idx) 1435 } 1436 iter, err := snap.KVs() 1437 if err != nil { 1438 t.Fatalf("err: %s", err) 1439 } 1440 var dump structs.DirEntries 1441 for entry := iter.Next(); entry != nil; entry = iter.Next() { 1442 dump = append(dump, entry.(*structs.DirEntry)) 1443 } 1444 if !reflect.DeepEqual(dump, entries) { 1445 t.Fatalf("bad: %#v", dump) 1446 } 1447 1448 // Restore the values into a new state store. 1449 func() { 1450 s := testStateStore(t) 1451 restore := s.Restore() 1452 for _, entry := range dump { 1453 if err := restore.KVS(entry); err != nil { 1454 t.Fatalf("err: %s", err) 1455 } 1456 } 1457 restore.Commit() 1458 1459 // Read the restored keys back out and verify they match. 1460 idx, res, err := s.KVSList(nil, "") 1461 if err != nil { 1462 t.Fatalf("err: %s", err) 1463 } 1464 if idx != 7 { 1465 t.Fatalf("bad index: %d", idx) 1466 } 1467 if !reflect.DeepEqual(res, entries) { 1468 t.Fatalf("bad: %#v", res) 1469 } 1470 1471 // Check that the index was updated. 1472 if idx := s.maxIndex("kvs"); idx != 7 { 1473 t.Fatalf("bad index: %d", idx) 1474 } 1475 }() 1476 } 1477 1478 func TestStateStore_Tombstone_Snapshot_Restore(t *testing.T) { 1479 s := testStateStore(t) 1480 1481 // Insert a key and then delete it to create a tombstone. 1482 testSetKey(t, s, 1, "foo/bar", "bar") 1483 testSetKey(t, s, 2, "foo/bar/baz", "bar") 1484 testSetKey(t, s, 3, "foo/bar/zoo", "bar") 1485 if err := s.KVSDelete(4, "foo/bar"); err != nil { 1486 t.Fatalf("err: %s", err) 1487 } 1488 1489 // Snapshot the Tombstones. 1490 snap := s.Snapshot() 1491 defer snap.Close() 1492 1493 // Alter the real state store. 1494 if err := s.ReapTombstones(4); err != nil { 1495 t.Fatalf("err: %s", err) 1496 } 1497 idx, _, err := s.KVSList(nil, "foo/bar") 1498 if err != nil { 1499 t.Fatalf("err: %s", err) 1500 } 1501 if idx != 3 { 1502 t.Fatalf("bad index: %d", idx) 1503 } 1504 1505 // Verify the snapshot. 1506 stones, err := snap.Tombstones() 1507 if err != nil { 1508 t.Fatalf("err: %s", err) 1509 } 1510 var dump []*Tombstone 1511 for stone := stones.Next(); stone != nil; stone = stones.Next() { 1512 dump = append(dump, stone.(*Tombstone)) 1513 } 1514 if len(dump) != 1 { 1515 t.Fatalf("bad %#v", dump) 1516 } 1517 stone := dump[0] 1518 if stone.Key != "foo/bar" || stone.Index != 4 { 1519 t.Fatalf("bad: %#v", stone) 1520 } 1521 1522 // Restore the values into a new state store. 1523 func() { 1524 s := testStateStore(t) 1525 restore := s.Restore() 1526 for _, stone := range dump { 1527 if err := restore.Tombstone(stone); err != nil { 1528 t.Fatalf("err: %s", err) 1529 } 1530 } 1531 restore.Commit() 1532 1533 // See if the stone works properly in a list query. 1534 idx, _, err := s.KVSList(nil, "foo/bar") 1535 if err != nil { 1536 t.Fatalf("err: %s", err) 1537 } 1538 if idx != 4 { 1539 t.Fatalf("bad index: %d", idx) 1540 } 1541 1542 // Make sure it reaps correctly. We should still get a 4 for 1543 // the index here because it will be using the last index from 1544 // the tombstone table. 1545 if err := s.ReapTombstones(4); err != nil { 1546 t.Fatalf("err: %s", err) 1547 } 1548 idx, _, err = s.KVSList(nil, "foo/bar") 1549 if err != nil { 1550 t.Fatalf("err: %s", err) 1551 } 1552 if idx != 4 { 1553 t.Fatalf("bad index: %d", idx) 1554 } 1555 1556 // But make sure the tombstone is actually gone. 1557 snap := s.Snapshot() 1558 defer snap.Close() 1559 stones, err := snap.Tombstones() 1560 if err != nil { 1561 t.Fatalf("err: %s", err) 1562 } 1563 if stones.Next() != nil { 1564 t.Fatalf("unexpected extra tombstones") 1565 } 1566 }() 1567 }