github.com/fibonacci-chain/fbc@v0.0.0-20231124064014-c7636198c1e9/libs/iavl/tree_test.go (about) 1 // nolint:errcheck 2 package iavl 3 4 import ( 5 "bytes" 6 "flag" 7 "fmt" 8 "os" 9 "runtime" 10 "strconv" 11 "testing" 12 13 cmn "github.com/fibonacci-chain/fbc/libs/iavl/common" 14 "github.com/fibonacci-chain/fbc/libs/tendermint/libs/rand" 15 db "github.com/fibonacci-chain/fbc/libs/tm-db" 16 "github.com/stretchr/testify/assert" 17 "github.com/stretchr/testify/require" 18 ) 19 20 var testLevelDB bool 21 var testFuzzIterations int 22 var random *cmn.Rand 23 24 func SetupTest() { 25 random = cmn.NewRand() 26 random.Seed(0) // for determinism 27 flag.BoolVar(&testLevelDB, "test.leveldb", false, "test leveldb backend") 28 flag.IntVar(&testFuzzIterations, "test.fuzz-iterations", 100000, "number of fuzz testing iterations") 29 flag.Parse() 30 } 31 32 func getTestDB() (db.DB, func()) { 33 if testLevelDB { 34 d, err := db.NewGoLevelDB("test", ".") 35 if err != nil { 36 panic(err) 37 } 38 return d, func() { 39 d.Close() 40 os.RemoveAll("./test.db") 41 } 42 } 43 return db.NewMemDB(), func() {} 44 } 45 46 func TestVersionedRandomTree(t *testing.T) { 47 require := require.New(t) 48 SetupTest() 49 d, closeDB := getTestDB() 50 defer closeDB() 51 52 tree, err := NewMutableTree(d, 100) 53 require.NoError(err) 54 versions := 50 55 keysPerVersion := 30 56 57 // Create a tree of size 1000 with 100 versions. 58 for i := 1; i <= versions; i++ { 59 for j := 0; j < keysPerVersion; j++ { 60 k := []byte(cmn.RandStr(8)) 61 v := []byte(cmn.RandStr(8)) 62 tree.Set(k, v) 63 } 64 tree.SaveVersion(false) 65 } 66 require.Equal(versions, len(tree.ndb.roots()), "wrong number of roots") 67 require.Equal(versions*keysPerVersion, len(tree.ndb.leafNodes()), "wrong number of nodes") 68 69 // Before deleting old versions, we should have equal or more nodes in the 70 // db than in the current tree version. 71 require.True(len(tree.ndb.nodes()) >= tree.nodeSize()) 72 73 // Ensure it returns all versions in sorted order 74 available := tree.AvailableVersions() 75 assert.Equal(t, versions, len(available)) 76 assert.Equal(t, 1, available[0]) 77 assert.Equal(t, versions, available[len(available)-1]) 78 79 for i := 1; i < versions; i++ { 80 tree.DeleteVersion(int64(i)) 81 } 82 83 require.Len(tree.versions.Clone(), 1, "tree must have one version left") 84 tr, err := tree.GetImmutable(int64(versions)) 85 require.NoError(err, "GetImmutable should not error for version %d", versions) 86 require.Equal(tr.root, tree.root) 87 88 // we should only have one available version now 89 available = tree.AvailableVersions() 90 assert.Equal(t, 1, len(available)) 91 assert.Equal(t, versions, available[0]) 92 93 // After cleaning up all previous versions, we should have as many nodes 94 // in the db as in the current tree version. 95 require.Len(tree.ndb.leafNodes(), int(tree.Size())) 96 97 require.Equal(tree.nodeSize(), len(tree.ndb.nodes())) 98 } 99 100 func TestVersionedRandomTreeSmallKeys(t *testing.T) { 101 require := require.New(t) 102 d, closeDB := getTestDB() 103 defer closeDB() 104 105 tree, err := NewMutableTree(db.NewPrefixDB(d, []byte(randstr(32))), 100) 106 require.NoError(err) 107 singleVersionTree, err := getTestTree(0) 108 require.NoError(err) 109 versions := 20 110 keysPerVersion := 50 111 112 for i := 1; i <= versions; i++ { 113 for j := 0; j < keysPerVersion; j++ { 114 // Keys of size one are likely to be overwritten. 115 k := []byte(cmn.RandStr(1)) 116 v := []byte(cmn.RandStr(8)) 117 tree.Set(k, v) 118 singleVersionTree.Set(k, v) 119 } 120 tree.SaveVersion(false) 121 } 122 singleVersionTree.SaveVersion(false) 123 124 for i := 1; i < versions; i++ { 125 tree.DeleteVersion(int64(i)) 126 } 127 128 // After cleaning up all previous versions, we should have as many nodes 129 // in the db as in the current tree version. The simple tree must be equal 130 // too. 131 require.Len(tree.ndb.leafNodes(), int(tree.Size())) 132 require.Len(tree.ndb.nodes(), tree.nodeSize()) 133 require.Len(tree.ndb.nodes(), singleVersionTree.nodeSize()) 134 135 // Try getting random keys. 136 for i := 0; i < keysPerVersion; i++ { 137 val := tree.Get([]byte(cmn.RandStr(1))) 138 require.NotNil(val) 139 require.NotEmpty(val) 140 } 141 } 142 143 func TestVersionedRandomTreeSmallKeysRandomDeletes(t *testing.T) { 144 require := require.New(t) 145 d, closeDB := getTestDB() 146 defer closeDB() 147 148 tree, err := NewMutableTree(db.NewPrefixDB(d, []byte(randstr(32))), 100) 149 require.NoError(err) 150 singleVersionTree, err := getTestTree(0) 151 require.NoError(err) 152 versions := 30 153 keysPerVersion := 50 154 155 for i := 1; i <= versions; i++ { 156 for j := 0; j < keysPerVersion; j++ { 157 // Keys of size one are likely to be overwritten. 158 k := []byte(cmn.RandStr(1)) 159 v := []byte(cmn.RandStr(8)) 160 tree.Set(k, v) 161 singleVersionTree.Set(k, v) 162 } 163 tree.SaveVersion(false) 164 } 165 singleVersionTree.SaveVersion(false) 166 167 for _, i := range cmn.RandPerm(versions - 1) { 168 tree.DeleteVersion(int64(i + 1)) 169 } 170 171 // After cleaning up all previous versions, we should have as many nodes 172 // in the db as in the current tree version. The simple tree must be equal 173 // too. 174 require.Len(tree.ndb.leafNodes(), int(tree.Size())) 175 require.Len(tree.ndb.nodes(), tree.nodeSize()) 176 require.Len(tree.ndb.nodes(), singleVersionTree.nodeSize()) 177 178 // Try getting random keys. 179 for i := 0; i < keysPerVersion; i++ { 180 val := tree.Get([]byte(cmn.RandStr(1))) 181 require.NotNil(val) 182 require.NotEmpty(val) 183 } 184 } 185 186 func TestVersionedTreeSpecial1(t *testing.T) { 187 tree, err := getTestTree(100) 188 require.NoError(t, err) 189 190 tree.Set([]byte("C"), []byte("so43QQFN")) 191 tree.SaveVersion(false) 192 193 tree.Set([]byte("A"), []byte("ut7sTTAO")) 194 tree.SaveVersion(false) 195 196 tree.Set([]byte("X"), []byte("AoWWC1kN")) 197 tree.SaveVersion(false) 198 199 tree.Set([]byte("T"), []byte("MhkWjkVy")) 200 tree.SaveVersion(false) 201 202 tree.DeleteVersion(1) 203 tree.DeleteVersion(2) 204 tree.DeleteVersion(3) 205 206 require.Equal(t, tree.nodeSize(), len(tree.ndb.nodes())) 207 } 208 209 func TestVersionedRandomTreeSpecial2(t *testing.T) { 210 require := require.New(t) 211 tree, err := getTestTree(100) 212 require.NoError(err) 213 214 tree.Set([]byte("OFMe2Yvm"), []byte("ez2OtQtE")) 215 tree.Set([]byte("WEN4iN7Y"), []byte("kQNyUalI")) 216 tree.SaveVersion(false) 217 218 tree.Set([]byte("1yY3pXHr"), []byte("udYznpII")) 219 tree.Set([]byte("7OSHNE7k"), []byte("ff181M2d")) 220 tree.SaveVersion(false) 221 222 tree.DeleteVersion(1) 223 require.Len(tree.ndb.nodes(), tree.nodeSize()) 224 } 225 226 func TestVersionedEmptyTree(t *testing.T) { 227 require := require.New(t) 228 d, closeDB := getTestDB() 229 defer closeDB() 230 231 tree, err := NewMutableTree(d, 0) 232 require.NoError(err) 233 234 hash, v, _, err := tree.SaveVersion(false) 235 require.Nil(hash) 236 require.EqualValues(1, v) 237 require.NoError(err) 238 239 hash, v, _, err = tree.SaveVersion(false) 240 require.Nil(hash) 241 require.EqualValues(2, v) 242 require.NoError(err) 243 244 hash, v, _, err = tree.SaveVersion(false) 245 require.Nil(hash) 246 require.EqualValues(3, v) 247 require.NoError(err) 248 249 hash, v, _, err = tree.SaveVersion(false) 250 require.Nil(hash) 251 require.EqualValues(4, v) 252 require.NoError(err) 253 254 require.EqualValues(4, tree.Version()) 255 256 require.True(tree.VersionExists(1)) 257 require.True(tree.VersionExists(3)) 258 259 require.NoError(tree.DeleteVersion(1)) 260 require.NoError(tree.DeleteVersion(3)) 261 262 require.False(tree.VersionExists(1)) 263 require.False(tree.VersionExists(3)) 264 265 tree.Set([]byte("k"), []byte("v")) 266 require.EqualValues(5, tree.root.version) 267 268 // Now reload the tree. 269 270 tree, err = NewMutableTree(d, 0) 271 require.NoError(err) 272 tree.Load() 273 274 require.False(tree.VersionExists(1)) 275 require.True(tree.VersionExists(2)) 276 require.False(tree.VersionExists(3)) 277 278 t2, err := tree.GetImmutable(2) 279 require.NoError(err, "GetImmutable should not fail for version 2") 280 281 require.Empty(t2.root) 282 } 283 284 func TestVersionedTree(t *testing.T) { 285 require := require.New(t) 286 d, closeDB := getTestDB() 287 defer closeDB() 288 289 tree, err := NewMutableTree(d, 0) 290 require.NoError(err) 291 292 // We start with empty database 293 require.Equal(0, tree.ndb.size()) 294 require.True(tree.IsEmpty()) 295 require.False(tree.IsFastCacheEnabled()) 296 297 // version 0 298 299 tree.Set([]byte("key1"), []byte("val0")) 300 tree.Set([]byte("key2"), []byte("val0")) 301 302 // Still zero keys, since we haven't written them. 303 require.Len(tree.ndb.leafNodes(), 0) 304 require.False(tree.IsEmpty()) 305 306 // Now let's write the keys to storage. 307 hash1, v, _, err := tree.SaveVersion(false) 308 require.NoError(err) 309 require.False(tree.IsEmpty()) 310 require.EqualValues(1, v) 311 312 // -----1----- 313 // key1 = val0 version=1 314 // key2 = val0 version=1 315 // key2 (root) version=1 316 // ----------- 317 318 nodes1 := tree.ndb.leafNodes() 319 require.Len(nodes1, 2, "db should have a size of 2") 320 321 // version 1 322 323 tree.Set([]byte("key1"), []byte("val1")) 324 tree.Set([]byte("key2"), []byte("val1")) 325 tree.Set([]byte("key3"), []byte("val1")) 326 require.Len(tree.ndb.leafNodes(), len(nodes1)) 327 328 hash2, v2, _, err := tree.SaveVersion(false) 329 require.NoError(err) 330 require.False(bytes.Equal(hash1, hash2)) 331 require.EqualValues(v+1, v2) 332 333 // Recreate a new tree and load it, to make sure it works in this 334 // scenario. 335 tree, err = NewMutableTree(d, 100) 336 require.NoError(err) 337 _, err = tree.Load() 338 require.NoError(err) 339 340 require.Len(tree.versions.Clone(), 2, "wrong number of versions") 341 require.EqualValues(v2, tree.Version()) 342 343 // -----1----- 344 // key1 = val0 <orphaned> 345 // key2 = val0 <orphaned> 346 // -----2----- 347 // key1 = val1 348 // key2 = val1 349 // key3 = val1 350 // ----------- 351 352 nodes2 := tree.ndb.leafNodes() 353 require.Len(nodes2, 5, "db should have grown in size") 354 require.Len(tree.ndb.orphans(), 3, "db should have three orphans") 355 356 // Create three more orphans. 357 tree.Remove([]byte("key1")) // orphans both leaf node and inner node containing "key1" and "key2" 358 tree.Set([]byte("key2"), []byte("val2")) 359 360 hash3, v3, _, _ := tree.SaveVersion(false) 361 require.EqualValues(3, v3) 362 363 // -----1----- 364 // key1 = val0 <orphaned> (replaced) 365 // key2 = val0 <orphaned> (replaced) 366 // -----2----- 367 // key1 = val1 <orphaned> (removed) 368 // key2 = val1 <orphaned> (replaced) 369 // key3 = val1 370 // -----3----- 371 // key2 = val2 372 // ----------- 373 374 nodes3 := tree.ndb.leafNodes() 375 require.Len(nodes3, 6, "wrong number of nodes") 376 require.Len(tree.ndb.orphans(), 7, "wrong number of orphans") 377 378 hash4, _, _, _ := tree.SaveVersion(false) 379 require.EqualValues(hash3, hash4) 380 require.NotNil(hash4) 381 382 tree, err = NewMutableTree(d, 100) 383 require.NoError(err) 384 _, err = tree.Load() 385 require.NoError(err) 386 387 // ------------ 388 // DB UNCHANGED 389 // ------------ 390 391 nodes4 := tree.ndb.leafNodes() 392 require.Len(nodes4, len(nodes3), "db should not have changed in size") 393 394 tree.Set([]byte("key1"), []byte("val0")) 395 396 // "key2" 397 _, val := tree.GetVersioned([]byte("key2"), 0) 398 require.Nil(val) 399 400 _, val = tree.GetVersioned([]byte("key2"), 1) 401 require.Equal("val0", string(val)) 402 403 _, val = tree.GetVersioned([]byte("key2"), 2) 404 require.Equal("val1", string(val)) 405 406 _, val = tree.GetWithIndex([]byte("key2")) 407 require.Equal("val2", string(val)) 408 409 // "key1" 410 _, val = tree.GetVersioned([]byte("key1"), 1) 411 require.Equal("val0", string(val)) 412 413 _, val = tree.GetVersioned([]byte("key1"), 2) 414 require.Equal("val1", string(val)) 415 416 _, val = tree.GetVersioned([]byte("key1"), 3) 417 require.Nil(val) 418 419 _, val = tree.GetVersioned([]byte("key1"), 4) 420 require.Nil(val) 421 422 _, val = tree.GetWithIndex([]byte("key1")) 423 require.Equal("val0", string(val)) 424 425 // "key3" 426 _, val = tree.GetVersioned([]byte("key3"), 0) 427 require.Nil(val) 428 429 _, val = tree.GetVersioned([]byte("key3"), 2) 430 require.Equal("val1", string(val)) 431 432 _, val = tree.GetVersioned([]byte("key3"), 3) 433 require.Equal("val1", string(val)) 434 435 // Delete a version. After this the keys in that version should not be found. 436 437 tree.DeleteVersion(2) 438 439 // -----1----- 440 // key1 = val0 441 // key2 = val0 442 // -----2----- 443 // key3 = val1 444 // -----3----- 445 // key2 = val2 446 // ----------- 447 448 nodes5 := tree.ndb.leafNodes() 449 require.True(len(nodes5) < len(nodes4), "db should have shrunk after delete %d !< %d", len(nodes5), len(nodes4)) 450 451 _, val = tree.GetVersioned([]byte("key2"), 2) 452 require.Nil(val) 453 454 _, val = tree.GetVersioned([]byte("key3"), 2) 455 require.Nil(val) 456 457 // But they should still exist in the latest version. 458 459 val = tree.Get([]byte("key2")) 460 require.Equal("val2", string(val)) 461 462 val = tree.Get([]byte("key3")) 463 require.Equal("val1", string(val)) 464 465 // Version 1 should still be available. 466 467 _, val = tree.GetVersioned([]byte("key1"), 1) 468 require.Equal("val0", string(val)) 469 470 _, val = tree.GetVersioned([]byte("key2"), 1) 471 require.Equal("val0", string(val)) 472 } 473 474 func TestVersionedTreeVersionDeletingEfficiency(t *testing.T) { 475 d, closeDB := getTestDB() 476 defer closeDB() 477 478 tree, err := NewMutableTree(d, 0) 479 require.NoError(t, err) 480 481 tree.Set([]byte("key0"), []byte("val0")) 482 tree.Set([]byte("key1"), []byte("val0")) 483 tree.Set([]byte("key2"), []byte("val0")) 484 tree.SaveVersion(false) 485 486 require.Len(t, tree.ndb.leafNodes(), 3) 487 488 tree.Set([]byte("key1"), []byte("val1")) 489 tree.Set([]byte("key2"), []byte("val1")) 490 tree.Set([]byte("key3"), []byte("val1")) 491 tree.SaveVersion(false) 492 493 require.Len(t, tree.ndb.leafNodes(), 6) 494 495 tree.Set([]byte("key0"), []byte("val2")) 496 tree.Remove([]byte("key1")) 497 tree.Set([]byte("key2"), []byte("val2")) 498 tree.SaveVersion(false) 499 500 require.Len(t, tree.ndb.leafNodes(), 8) 501 502 tree.DeleteVersion(2) 503 504 require.Len(t, tree.ndb.leafNodes(), 6) 505 506 tree.DeleteVersion(1) 507 508 require.Len(t, tree.ndb.leafNodes(), 3) 509 510 tree2, err := getTestTree(0) 511 require.NoError(t, err) 512 tree2.Set([]byte("key0"), []byte("val2")) 513 tree2.Set([]byte("key2"), []byte("val2")) 514 tree2.Set([]byte("key3"), []byte("val1")) 515 tree2.SaveVersion(false) 516 517 require.Equal(t, tree2.nodeSize(), tree.nodeSize()) 518 } 519 520 func TestVersionedTreeOrphanDeleting(t *testing.T) { 521 tree, err := getTestTree(0) 522 require.NoError(t, err) 523 524 tree.Set([]byte("key0"), []byte("val0")) 525 tree.Set([]byte("key1"), []byte("val0")) 526 tree.Set([]byte("key2"), []byte("val0")) 527 tree.SaveVersion(false) 528 529 tree.Set([]byte("key1"), []byte("val1")) 530 tree.Set([]byte("key2"), []byte("val1")) 531 tree.Set([]byte("key3"), []byte("val1")) 532 tree.SaveVersion(false) 533 534 tree.Set([]byte("key0"), []byte("val2")) 535 tree.Remove([]byte("key1")) 536 tree.Set([]byte("key2"), []byte("val2")) 537 tree.SaveVersion(false) 538 539 tree.DeleteVersion(2) 540 541 val := tree.Get([]byte("key0")) 542 require.Equal(t, val, []byte("val2")) 543 544 val = tree.Get([]byte("key1")) 545 require.Nil(t, val) 546 547 val = tree.Get([]byte("key2")) 548 require.Equal(t, val, []byte("val2")) 549 550 val = tree.Get([]byte("key3")) 551 require.Equal(t, val, []byte("val1")) 552 553 tree.DeleteVersion(1) 554 555 require.Len(t, tree.ndb.leafNodes(), 3) 556 } 557 558 func TestVersionedTreeSpecialCase(t *testing.T) { 559 require := require.New(t) 560 d, closeDB := getTestDB() 561 defer closeDB() 562 563 tree, err := NewMutableTree(db.NewPrefixDB(d, []byte(randstr(32))), 0) 564 require.NoError(err) 565 566 tree.Set([]byte("key1"), []byte("val0")) 567 tree.Set([]byte("key2"), []byte("val0")) 568 tree.SaveVersion(false) 569 570 tree.Set([]byte("key1"), []byte("val1")) 571 tree.Set([]byte("key2"), []byte("val1")) 572 tree.SaveVersion(false) 573 574 tree.Set([]byte("key2"), []byte("val2")) 575 tree.SaveVersion(false) 576 577 tree.DeleteVersion(2) 578 579 _, val := tree.GetVersioned([]byte("key2"), 1) 580 require.Equal("val0", string(val)) 581 } 582 583 func TestVersionedTreeSpecialCase2(t *testing.T) { 584 require := require.New(t) 585 586 d := db.NewMemDB() 587 tree, err := NewMutableTree(d, 100) 588 require.NoError(err) 589 590 tree.Set([]byte("key1"), []byte("val0")) 591 tree.Set([]byte("key2"), []byte("val0")) 592 tree.SaveVersion(false) 593 594 tree.Set([]byte("key1"), []byte("val1")) 595 tree.Set([]byte("key2"), []byte("val1")) 596 tree.SaveVersion(false) 597 598 tree.Set([]byte("key2"), []byte("val2")) 599 tree.SaveVersion(false) 600 601 tree, err = NewMutableTree(d, 100) 602 require.NoError(err) 603 _, err = tree.Load() 604 require.NoError(err) 605 606 require.NoError(tree.DeleteVersion(2)) 607 608 _, val := tree.GetVersioned([]byte("key2"), 1) 609 require.Equal("val0", string(val)) 610 } 611 612 func TestVersionedTreeSpecialCase3(t *testing.T) { 613 require := require.New(t) 614 tree, err := getTestTree(0) 615 require.NoError(err) 616 617 tree.Set([]byte("m"), []byte("liWT0U6G")) 618 tree.Set([]byte("G"), []byte("7PxRXwUA")) 619 tree.SaveVersion(false) 620 621 tree.Set([]byte("7"), []byte("XRLXgf8C")) 622 tree.SaveVersion(false) 623 624 tree.Set([]byte("r"), []byte("bBEmIXBU")) 625 tree.SaveVersion(false) 626 627 tree.Set([]byte("i"), []byte("kkIS35te")) 628 tree.SaveVersion(false) 629 630 tree.Set([]byte("k"), []byte("CpEnpzKJ")) 631 tree.SaveVersion(false) 632 633 tree.DeleteVersion(1) 634 tree.DeleteVersion(2) 635 tree.DeleteVersion(3) 636 tree.DeleteVersion(4) 637 638 require.Equal(tree.nodeSize(), len(tree.ndb.nodes())) 639 } 640 641 func TestVersionedTreeSaveAndLoad(t *testing.T) { 642 require := require.New(t) 643 d := db.NewMemDB() 644 tree, err := NewMutableTree(d, 0) 645 require.NoError(err) 646 647 // Loading with an empty root is a no-op. 648 tree.Load() 649 650 tree.Set([]byte("C"), []byte("so43QQFN")) 651 tree.SaveVersion(false) 652 653 tree.Set([]byte("A"), []byte("ut7sTTAO")) 654 tree.SaveVersion(false) 655 656 tree.Set([]byte("X"), []byte("AoWWC1kN")) 657 tree.SaveVersion(false) 658 659 tree.SaveVersion(false) 660 tree.SaveVersion(false) 661 tree.SaveVersion(false) 662 663 preHash := tree.Hash() 664 require.NotNil(preHash) 665 666 require.Equal(int64(6), tree.Version()) 667 668 // Reload the tree, to test that roots and orphans are properly loaded. 669 ntree, err := NewMutableTree(d, 0) 670 require.NoError(err) 671 ntree.Load() 672 673 require.False(ntree.IsEmpty()) 674 require.Equal(int64(6), ntree.Version()) 675 676 postHash := ntree.Hash() 677 require.Equal(preHash, postHash) 678 679 ntree.Set([]byte("T"), []byte("MhkWjkVy")) 680 ntree.SaveVersion(false) 681 682 ntree.DeleteVersion(6) 683 ntree.DeleteVersion(5) 684 ntree.DeleteVersion(1) 685 ntree.DeleteVersion(2) 686 ntree.DeleteVersion(4) 687 ntree.DeleteVersion(3) 688 689 require.False(ntree.IsEmpty()) 690 require.Equal(int64(4), ntree.Size()) 691 require.Len(ntree.ndb.nodes(), ntree.nodeSize()) 692 } 693 694 func TestVersionedTreeErrors(t *testing.T) { 695 require := require.New(t) 696 tree, err := getTestTree(100) 697 require.NoError(err) 698 699 // Can't delete non-existent versions. 700 require.Error(tree.DeleteVersion(1)) 701 require.Error(tree.DeleteVersion(99)) 702 703 tree.Set([]byte("key"), []byte("val")) 704 705 // Saving with content is ok. 706 _, _, _, err = tree.SaveVersion(false) 707 require.NoError(err) 708 709 // Can't delete current version. 710 require.Error(tree.DeleteVersion(1)) 711 712 // Trying to get a key from a version which doesn't exist. 713 _, val := tree.GetVersioned([]byte("key"), 404) 714 require.Nil(val) 715 716 // Same thing with proof. We get an error because a proof couldn't be 717 // constructed. 718 val, proof, err := tree.GetVersionedWithProof([]byte("key"), 404) 719 require.Nil(val) 720 require.Empty(proof) 721 require.Error(err) 722 } 723 724 func TestVersionedCheckpoints(t *testing.T) { 725 require := require.New(t) 726 d, closeDB := getTestDB() 727 defer closeDB() 728 729 tree, err := NewMutableTree(d, 100) 730 require.NoError(err) 731 versions := 50 732 keysPerVersion := 10 733 versionsPerCheckpoint := 5 734 keys := map[int64]([][]byte){} 735 736 for i := 1; i <= versions; i++ { 737 for j := 0; j < keysPerVersion; j++ { 738 k := []byte(cmn.RandStr(1)) 739 v := []byte(cmn.RandStr(8)) 740 keys[int64(i)] = append(keys[int64(i)], k) 741 tree.Set(k, v) 742 } 743 tree.SaveVersion(false) 744 } 745 746 for i := 1; i <= versions; i++ { 747 if i%versionsPerCheckpoint != 0 { 748 tree.DeleteVersion(int64(i)) 749 } 750 } 751 752 // Make sure all keys exist at least once. 753 for _, ks := range keys { 754 for _, k := range ks { 755 _, val := tree.GetWithIndex(k) 756 require.NotEmpty(val) 757 } 758 } 759 760 // Make sure all keys from deleted versions aren't present. 761 for i := 1; i <= versions; i++ { 762 if i%versionsPerCheckpoint != 0 { 763 for _, k := range keys[int64(i)] { 764 _, val := tree.GetVersioned(k, int64(i)) 765 require.Nil(val) 766 } 767 } 768 } 769 770 // Make sure all keys exist at all checkpoints. 771 for i := 1; i <= versions; i++ { 772 for _, k := range keys[int64(i)] { 773 if i%versionsPerCheckpoint == 0 { 774 _, val := tree.GetVersioned(k, int64(i)) 775 require.NotEmpty(val) 776 } 777 } 778 } 779 } 780 781 func TestVersionedCheckpointsSpecialCase(t *testing.T) { 782 require := require.New(t) 783 tree, err := getTestTree(0) 784 require.NoError(err) 785 key := []byte("k") 786 787 tree.Set(key, []byte("val1")) 788 789 tree.SaveVersion(false) 790 // ... 791 tree.SaveVersion(false) 792 // ... 793 tree.SaveVersion(false) 794 // ... 795 // This orphans "k" at version 1. 796 tree.Set(key, []byte("val2")) 797 tree.SaveVersion(false) 798 799 // When version 1 is deleted, the orphans should move to the next 800 // checkpoint, which is version 10. 801 tree.DeleteVersion(1) 802 803 _, val := tree.GetVersioned(key, 2) 804 require.NotEmpty(val) 805 require.Equal([]byte("val1"), val) 806 } 807 808 func TestVersionedCheckpointsSpecialCase2(t *testing.T) { 809 tree, err := getTestTree(0) 810 require.NoError(t, err) 811 812 tree.Set([]byte("U"), []byte("XamDUtiJ")) 813 tree.Set([]byte("A"), []byte("UkZBuYIU")) 814 tree.Set([]byte("H"), []byte("7a9En4uw")) 815 tree.Set([]byte("V"), []byte("5HXU3pSI")) 816 tree.SaveVersion(false) 817 818 tree.Set([]byte("U"), []byte("Replaced")) 819 tree.Set([]byte("A"), []byte("Replaced")) 820 tree.SaveVersion(false) 821 822 tree.Set([]byte("X"), []byte("New")) 823 tree.SaveVersion(false) 824 825 tree.DeleteVersion(1) 826 tree.DeleteVersion(2) 827 } 828 829 func TestVersionedCheckpointsSpecialCase3(t *testing.T) { 830 tree, err := getTestTree(0) 831 require.NoError(t, err) 832 833 tree.Set([]byte("n"), []byte("2wUCUs8q")) 834 tree.Set([]byte("l"), []byte("WQ7mvMbc")) 835 tree.SaveVersion(false) 836 837 tree.Set([]byte("N"), []byte("ved29IqU")) 838 tree.Set([]byte("v"), []byte("01jquVXU")) 839 tree.SaveVersion(false) 840 841 tree.Set([]byte("l"), []byte("bhIpltPM")) 842 tree.Set([]byte("B"), []byte("rj97IKZh")) 843 tree.SaveVersion(false) 844 845 tree.DeleteVersion(2) 846 847 tree.GetVersioned([]byte("m"), 1) 848 } 849 850 func TestVersionedCheckpointsSpecialCase4(t *testing.T) { 851 tree, err := NewMutableTree(db.NewMemDB(), 0) 852 require.NoError(t, err) 853 854 tree.Set([]byte("U"), []byte("XamDUtiJ")) 855 tree.Set([]byte("A"), []byte("UkZBuYIU")) 856 tree.Set([]byte("H"), []byte("7a9En4uw")) 857 tree.Set([]byte("V"), []byte("5HXU3pSI")) 858 tree.SaveVersion(false) 859 860 tree.Remove([]byte("U")) 861 tree.Remove([]byte("A")) 862 tree.SaveVersion(false) 863 864 tree.Set([]byte("X"), []byte("New")) 865 tree.SaveVersion(false) 866 867 _, val := tree.GetVersioned([]byte("A"), 2) 868 require.Nil(t, val) 869 870 _, val = tree.GetVersioned([]byte("A"), 1) 871 require.NotEmpty(t, val) 872 873 tree.DeleteVersion(1) 874 tree.DeleteVersion(2) 875 876 _, val = tree.GetVersioned([]byte("A"), 2) 877 require.Nil(t, val) 878 879 _, val = tree.GetVersioned([]byte("A"), 1) 880 require.Nil(t, val) 881 } 882 883 func TestVersionedCheckpointsSpecialCase5(t *testing.T) { 884 tree, err := getTestTree(0) 885 require.NoError(t, err) 886 887 tree.Set([]byte("R"), []byte("ygZlIzeW")) 888 tree.SaveVersion(false) 889 890 tree.Set([]byte("j"), []byte("ZgmCWyo2")) 891 tree.SaveVersion(false) 892 893 tree.Set([]byte("R"), []byte("vQDaoz6Z")) 894 tree.SaveVersion(false) 895 896 tree.DeleteVersion(1) 897 898 tree.GetVersioned([]byte("R"), 2) 899 } 900 901 func TestVersionedCheckpointsSpecialCase6(t *testing.T) { 902 tree, err := getTestTree(0) 903 require.NoError(t, err) 904 905 tree.Set([]byte("Y"), []byte("MW79JQeV")) 906 tree.Set([]byte("7"), []byte("Kp0ToUJB")) 907 tree.Set([]byte("Z"), []byte("I26B1jPG")) 908 tree.Set([]byte("6"), []byte("ZG0iXq3h")) 909 tree.Set([]byte("2"), []byte("WOR27LdW")) 910 tree.Set([]byte("4"), []byte("MKMvc6cn")) 911 tree.SaveVersion(false) 912 913 tree.Set([]byte("1"), []byte("208dOu40")) 914 tree.Set([]byte("G"), []byte("7isI9OQH")) 915 tree.Set([]byte("8"), []byte("zMC1YwpH")) 916 tree.SaveVersion(false) 917 918 tree.Set([]byte("7"), []byte("bn62vWbq")) 919 tree.Set([]byte("5"), []byte("wZuLGDkZ")) 920 tree.SaveVersion(false) 921 922 tree.DeleteVersion(1) 923 tree.DeleteVersion(2) 924 925 tree.GetVersioned([]byte("Y"), 1) 926 tree.GetVersioned([]byte("7"), 1) 927 tree.GetVersioned([]byte("Z"), 1) 928 tree.GetVersioned([]byte("6"), 1) 929 tree.GetVersioned([]byte("s"), 1) 930 tree.GetVersioned([]byte("2"), 1) 931 tree.GetVersioned([]byte("4"), 1) 932 } 933 934 func TestVersionedCheckpointsSpecialCase7(t *testing.T) { 935 tree, err := getTestTree(100) 936 require.NoError(t, err) 937 938 tree.Set([]byte("n"), []byte("OtqD3nyn")) 939 tree.Set([]byte("W"), []byte("kMdhJjF5")) 940 tree.Set([]byte("A"), []byte("BM3BnrIb")) 941 tree.Set([]byte("I"), []byte("QvtCH970")) 942 tree.Set([]byte("L"), []byte("txKgOTqD")) 943 tree.Set([]byte("Y"), []byte("NAl7PC5L")) 944 tree.SaveVersion(false) 945 946 tree.Set([]byte("7"), []byte("qWcEAlyX")) 947 tree.SaveVersion(false) 948 949 tree.Set([]byte("M"), []byte("HdQwzA64")) 950 tree.Set([]byte("3"), []byte("2Naa77fo")) 951 tree.Set([]byte("A"), []byte("SRuwKOTm")) 952 tree.Set([]byte("I"), []byte("oMX4aAOy")) 953 tree.Set([]byte("4"), []byte("dKfvbEOc")) 954 tree.SaveVersion(false) 955 956 tree.Set([]byte("D"), []byte("3U4QbXCC")) 957 tree.Set([]byte("B"), []byte("FxExhiDq")) 958 tree.SaveVersion(false) 959 960 tree.Set([]byte("A"), []byte("tWQgbFCY")) 961 tree.SaveVersion(false) 962 963 tree.DeleteVersion(4) 964 965 tree.GetVersioned([]byte("A"), 3) 966 } 967 968 func TestVersionedTreeEfficiency(t *testing.T) { 969 require := require.New(t) 970 971 tree, err := NewMutableTree(db.NewMemDB(), 0) 972 require.NoError(err) 973 versions := 20 974 keysPerVersion := 100 975 keysAddedPerVersion := map[int]int{} 976 977 keysAdded := 0 978 for i := 1; i <= versions; i++ { 979 for j := 0; j < keysPerVersion; j++ { 980 // Keys of size one are likely to be overwritten. 981 tree.Set([]byte(cmn.RandStr(1)), []byte(cmn.RandStr(8))) 982 } 983 sizeBefore := len(tree.ndb.nodes()) 984 tree.SaveVersion(false) 985 sizeAfter := len(tree.ndb.nodes()) 986 change := sizeAfter - sizeBefore 987 keysAddedPerVersion[i] = change 988 keysAdded += change 989 } 990 991 keysDeleted := 0 992 for i := 1; i < versions; i++ { 993 if tree.VersionExists(int64(i)) { 994 sizeBefore := len(tree.ndb.nodes()) 995 tree.DeleteVersion(int64(i)) 996 sizeAfter := len(tree.ndb.nodes()) 997 998 change := sizeBefore - sizeAfter 999 keysDeleted += change 1000 1001 require.InDelta(change, keysAddedPerVersion[i], float64(keysPerVersion)/5) 1002 } 1003 } 1004 require.Equal(keysAdded-tree.nodeSize(), keysDeleted) 1005 } 1006 1007 func TestVersionedTreeProofs(t *testing.T) { 1008 require := require.New(t) 1009 1010 tree, err := getTestTree(0) 1011 require.NoError(err) 1012 1013 tree.Set([]byte("k1"), []byte("v1")) 1014 tree.Set([]byte("k2"), []byte("v1")) 1015 tree.Set([]byte("k3"), []byte("v1")) 1016 _, _, _, err = tree.SaveVersion(false) 1017 require.NoError(err) 1018 1019 // fmt.Println("TREE VERSION 1") 1020 // printNode(tree.ndb, tree.root, 0) 1021 // fmt.Println("TREE VERSION 1 END") 1022 1023 root1 := tree.Hash() 1024 1025 tree.Set([]byte("k2"), []byte("v2")) 1026 tree.Set([]byte("k4"), []byte("v2")) 1027 _, _, _, err = tree.SaveVersion(false) 1028 require.NoError(err) 1029 1030 // fmt.Println("TREE VERSION 2") 1031 // printNode(tree.ndb, tree.root, 0) 1032 // fmt.Println("TREE VERSION END") 1033 1034 root2 := tree.Hash() 1035 require.NotEqual(root1, root2) 1036 1037 tree.Remove([]byte("k2")) 1038 _, _, _, err = tree.SaveVersion(false) 1039 require.NoError(err) 1040 1041 // fmt.Println("TREE VERSION 3") 1042 // printNode(tree.ndb, tree.root, 0) 1043 // fmt.Println("TREE VERSION END") 1044 1045 root3 := tree.Hash() 1046 require.NotEqual(root2, root3) 1047 1048 val, proof, err := tree.GetVersionedWithProof([]byte("k2"), 1) 1049 require.NoError(err) 1050 require.EqualValues(val, []byte("v1")) 1051 require.NoError(proof.Verify(root1), proof.String()) 1052 require.NoError(proof.VerifyItem([]byte("k2"), val)) 1053 1054 val, proof, err = tree.GetVersionedWithProof([]byte("k4"), 1) 1055 require.NoError(err) 1056 require.Nil(val) 1057 require.NoError(proof.Verify(root1)) 1058 require.NoError(proof.VerifyAbsence([]byte("k4"))) 1059 1060 val, proof, err = tree.GetVersionedWithProof([]byte("k2"), 2) 1061 require.NoError(err) 1062 require.EqualValues(val, []byte("v2")) 1063 require.NoError(proof.Verify(root2), proof.String()) 1064 require.NoError(proof.VerifyItem([]byte("k2"), val)) 1065 1066 val, proof, err = tree.GetVersionedWithProof([]byte("k1"), 2) 1067 require.NoError(err) 1068 require.EqualValues(val, []byte("v1")) 1069 require.NoError(proof.Verify(root2)) 1070 require.NoError(proof.VerifyItem([]byte("k1"), val)) 1071 1072 val, proof, err = tree.GetVersionedWithProof([]byte("k2"), 3) 1073 1074 require.NoError(err) 1075 require.Nil(val) 1076 require.NoError(proof.Verify(root3)) 1077 require.NoError(proof.VerifyAbsence([]byte("k2"))) 1078 require.Error(proof.Verify(root1)) 1079 require.Error(proof.Verify(root2)) 1080 } 1081 1082 func TestOrphans(t *testing.T) { 1083 // If you create a sequence of saved versions 1084 // Then randomly delete versions other than the first and last until only those two remain 1085 // Any remaining orphan nodes should either have fromVersion == firstVersion || toVersion == lastVersion 1086 require := require.New(t) 1087 tree, err := NewMutableTree(db.NewMemDB(), 100) 1088 require.NoError(err) 1089 1090 NUMVERSIONS := 100 1091 NUMUPDATES := 100 1092 1093 for i := 0; i < NUMVERSIONS; i++ { 1094 for j := 1; j < NUMUPDATES; j++ { 1095 tree.Set(randBytes(2), randBytes(2)) 1096 } 1097 _, _, _, err := tree.SaveVersion(false) 1098 require.NoError(err, "SaveVersion should not error") 1099 } 1100 1101 idx := cmn.RandPerm(NUMVERSIONS - 2) 1102 for _, v := range idx { 1103 err := tree.DeleteVersion(int64(v + 1)) 1104 require.NoError(err, "DeleteVersion should not error") 1105 } 1106 1107 tree.ndb.traverseOrphans(func(k, v []byte) { 1108 var fromVersion, toVersion int64 1109 orphanKeyFormat.Scan(k, &toVersion, &fromVersion) 1110 require.True(fromVersion == int64(1) || toVersion == int64(99), fmt.Sprintf(`Unexpected orphan key exists: %v with fromVersion = %d and toVersion = %d.\n 1111 Any orphan remaining in db should have either fromVersion == 1 or toVersion == 99. Since Version 1 and 99 are only versions in db`, k, fromVersion, toVersion)) 1112 }) 1113 } 1114 1115 func TestVersionedTreeHash(t *testing.T) { 1116 require := require.New(t) 1117 tree, err := getTestTree(0) 1118 require.NoError(err) 1119 1120 require.Nil(tree.Hash()) 1121 tree.Set([]byte("I"), []byte("D")) 1122 require.Nil(tree.Hash()) 1123 1124 hash1, _, _, _ := tree.SaveVersion(false) 1125 1126 tree.Set([]byte("I"), []byte("F")) 1127 require.EqualValues(hash1, tree.Hash()) 1128 1129 hash2, _, _, _ := tree.SaveVersion(false) 1130 1131 val, proof, err := tree.GetVersionedWithProof([]byte("I"), 2) 1132 require.NoError(err) 1133 require.EqualValues(val, []byte("F")) 1134 require.NoError(proof.Verify(hash2)) 1135 require.NoError(proof.VerifyItem([]byte("I"), val)) 1136 } 1137 1138 func TestNilValueSemantics(t *testing.T) { 1139 require := require.New(t) 1140 tree, err := getTestTree(0) 1141 require.NoError(err) 1142 1143 require.Panics(func() { 1144 tree.Set([]byte("k"), nil) 1145 }) 1146 } 1147 1148 func TestCopyValueSemantics(t *testing.T) { 1149 require := require.New(t) 1150 1151 tree, err := getTestTree(0) 1152 require.NoError(err) 1153 1154 val := []byte("v1") 1155 1156 tree.Set([]byte("k"), val) 1157 v := tree.Get([]byte("k")) 1158 require.Equal([]byte("v1"), v) 1159 1160 val[1] = '2' 1161 1162 val = tree.Get([]byte("k")) 1163 require.Equal([]byte("v2"), val) 1164 } 1165 1166 func TestRollback(t *testing.T) { 1167 require := require.New(t) 1168 1169 tree, err := getTestTree(0) 1170 require.NoError(err) 1171 1172 tree.Set([]byte("k"), []byte("v")) 1173 tree.SaveVersion(false) 1174 1175 tree.Set([]byte("r"), []byte("v")) 1176 tree.Set([]byte("s"), []byte("v")) 1177 1178 tree.Rollback() 1179 1180 tree.Set([]byte("t"), []byte("v")) 1181 1182 tree.SaveVersion(false) 1183 1184 require.Equal(int64(2), tree.Size()) 1185 1186 val := tree.Get([]byte("r")) 1187 require.Nil(val) 1188 1189 val = tree.Get([]byte("s")) 1190 require.Nil(val) 1191 1192 val = tree.Get([]byte("t")) 1193 require.Equal([]byte("v"), val) 1194 } 1195 1196 func TestLazyLoadVersion(t *testing.T) { 1197 tree, err := getTestTree(0) 1198 require.NoError(t, err) 1199 maxVersions := 10 1200 1201 version, err := tree.LazyLoadVersion(0) 1202 require.NoError(t, err, "unexpected error") 1203 require.Equal(t, version, int64(0), "expected latest version to be zero") 1204 1205 for i := 0; i < maxVersions; i++ { 1206 tree.Set([]byte(fmt.Sprintf("key_%d", i+1)), []byte(fmt.Sprintf("value_%d", i+1))) 1207 1208 _, _, _, err = tree.SaveVersion(false) 1209 require.NoError(t, err, "SaveVersion should not fail") 1210 } 1211 1212 // require the ability to lazy load the latest version 1213 version, err = tree.LazyLoadVersion(int64(maxVersions)) 1214 require.NoError(t, err, "unexpected error when lazy loading version") 1215 require.Equal(t, version, int64(maxVersions)) 1216 1217 value := tree.Get([]byte(fmt.Sprintf("key_%d", maxVersions))) 1218 require.Equal(t, value, []byte(fmt.Sprintf("value_%d", maxVersions)), "unexpected value") 1219 1220 // require the ability to lazy load an older version 1221 version, err = tree.LazyLoadVersion(int64(maxVersions - 1)) 1222 require.NoError(t, err, "unexpected error when lazy loading version") 1223 require.Equal(t, version, int64(maxVersions-1)) 1224 1225 value = tree.Get([]byte(fmt.Sprintf("key_%d", maxVersions-1))) 1226 require.Equal(t, value, []byte(fmt.Sprintf("value_%d", maxVersions-1)), "unexpected value") 1227 1228 // require the inability to lazy load a non-valid version 1229 version, err = tree.LazyLoadVersion(int64(maxVersions + 1)) 1230 require.Error(t, err, "expected error when lazy loading version") 1231 require.Equal(t, version, int64(maxVersions)) 1232 } 1233 1234 func TestOverwrite(t *testing.T) { 1235 require := require.New(t) 1236 1237 mdb := db.NewMemDB() 1238 tree, err := NewMutableTree(mdb, 0) 1239 require.NoError(err) 1240 1241 // Set one kv pair and save version 1 1242 tree.Set([]byte("key1"), []byte("value1")) 1243 _, _, _, err = tree.SaveVersion(false) 1244 require.NoError(err, "SaveVersion should not fail") 1245 1246 // Set another kv pair and save version 2 1247 tree.Set([]byte("key2"), []byte("value2")) 1248 _, _, _, err = tree.SaveVersion(false) 1249 require.NoError(err, "SaveVersion should not fail") 1250 1251 // Reload tree at version 1 1252 tree, err = NewMutableTree(mdb, 0) 1253 require.NoError(err) 1254 _, err = tree.LoadVersion(int64(1)) 1255 require.NoError(err, "LoadVersion should not fail") 1256 1257 // Attempt to put a different kv pair into the tree and save 1258 tree.Set([]byte("key2"), []byte("different value 2")) 1259 _, _, _, err = tree.SaveVersion(false) 1260 require.Error(err, "SaveVersion should fail because of changed value") 1261 1262 // Replay the original transition from version 1 to version 2 and attempt to save 1263 tree.Set([]byte("key2"), []byte("value2")) 1264 _, _, _, err = tree.SaveVersion(false) 1265 require.NoError(err, "SaveVersion should not fail, overwrite was idempotent") 1266 } 1267 1268 func TestOverwriteEmpty(t *testing.T) { 1269 // version 1 and version 2 are empty trees, version3 has value 1270 buildTreeVersion3 := func() (*MutableTree, int64, error) { 1271 mdb := db.NewMemDB() 1272 tree, err := NewMutableTree(mdb, 0) 1273 if err != nil { 1274 return nil, 0, err 1275 } 1276 1277 // Save empty version 1 1278 _, version, _, err := tree.SaveVersion(false) 1279 if err != nil { 1280 return tree, version, err 1281 } 1282 1283 // Save empty version 2 1284 _, version, _, err = tree.SaveVersion(false) 1285 if err != nil { 1286 return tree, version, err 1287 } 1288 1289 // Save a key in version 3 1290 tree.Set([]byte("key"), []byte("value")) 1291 _, version, _, err = tree.SaveVersion(false) 1292 1293 return tree, version, err 1294 } 1295 1296 testCases := []struct { 1297 name string 1298 saveVersion func() (int64, error) 1299 wantVersion int64 1300 wantErr assert.ErrorAssertionFunc 1301 }{ 1302 { 1303 "over write old empty tree with new key should fail", 1304 func() (int64, error) { 1305 tree, version, err := buildTreeVersion3() 1306 1307 // Load version 1 and attempt to save a different key 1308 version, err = tree.LoadVersion(1) 1309 if err != nil { 1310 return version, err 1311 } 1312 tree.Set([]byte("foo"), []byte("bar")) 1313 _, version, _, err = tree.SaveVersion(false) 1314 return version, err 1315 }, 1316 2, 1317 assert.Error, 1318 }, 1319 { 1320 "over write old empty tree without new key should work", 1321 func() (int64, error) { 1322 tree, version, err := buildTreeVersion3() 1323 1324 // Load version 1 and attempt to save a different key 1325 version, err = tree.LoadVersion(1) 1326 if err != nil { 1327 return version, err 1328 } 1329 tree.Set([]byte("foo"), []byte("bar")) 1330 1331 // However, deleting the key and saving an empty version should work, 1332 // since it's the same as the existing version. 1333 tree.Remove([]byte("foo")) 1334 _, version, _, err = tree.SaveVersion(false) 1335 return version, err 1336 }, 1337 2, 1338 assert.NoError, 1339 }, 1340 } 1341 1342 for _, tc := range testCases { 1343 t.Run(tc.name, func(t *testing.T) { 1344 version, err := tc.saveVersion() 1345 1346 if !tc.wantErr(t, err) { 1347 return 1348 } 1349 assert.Equal(t, tc.wantVersion, version) 1350 }) 1351 } 1352 } 1353 1354 func TestLoadVersionForOverwriting(t *testing.T) { 1355 require := require.New(t) 1356 1357 mdb := db.NewMemDB() 1358 tree, err := NewMutableTree(mdb, 0) 1359 require.NoError(err) 1360 1361 maxLength := 100 1362 for count := 1; count <= maxLength; count++ { 1363 countStr := strconv.Itoa(count) 1364 // Set one kv pair and save version 1365 tree.Set([]byte("key"+countStr), []byte("value"+countStr)) 1366 _, _, _, err = tree.SaveVersion(false) 1367 require.NoError(err, "SaveVersion should not fail") 1368 } 1369 1370 tree, err = NewMutableTree(mdb, 0) 1371 require.NoError(err) 1372 targetVersion, _ := tree.LoadVersionForOverwriting(int64(maxLength * 2)) 1373 require.Equal(targetVersion, int64(maxLength), "targetVersion shouldn't larger than the actual tree latest version") 1374 1375 tree, err = NewMutableTree(mdb, 0) 1376 require.NoError(err) 1377 _, err = tree.LoadVersionForOverwriting(int64(maxLength / 2)) 1378 require.NoError(err, "LoadVersion should not fail") 1379 1380 for version := 1; version <= maxLength/2; version++ { 1381 exist := tree.VersionExists(int64(version)) 1382 require.True(exist, "versions no more than 50 should exist") 1383 } 1384 1385 for version := (maxLength / 2) + 1; version <= maxLength; version++ { 1386 exist := tree.VersionExists(int64(version)) 1387 require.False(exist, "versions more than 50 should have been deleted") 1388 } 1389 1390 tree.Set([]byte("key49"), []byte("value49 different")) 1391 _, _, _, err = tree.SaveVersion(false) 1392 require.NoError(err, "SaveVersion should not fail, overwrite was allowed") 1393 1394 tree.Set([]byte("key50"), []byte("value50 different")) 1395 _, _, _, err = tree.SaveVersion(false) 1396 require.NoError(err, "SaveVersion should not fail, overwrite was allowed") 1397 1398 // Reload tree at version 50, the latest tree version is 52 1399 tree, err = NewMutableTree(mdb, 0) 1400 require.NoError(err) 1401 _, err = tree.LoadVersion(int64(maxLength / 2)) 1402 require.NoError(err, "LoadVersion should not fail") 1403 1404 tree.Set([]byte("key49"), []byte("value49 different")) 1405 _, _, _, err = tree.SaveVersion(false) 1406 require.NoError(err, "SaveVersion should not fail, write the same value") 1407 1408 tree.Set([]byte("key50"), []byte("value50 different different")) 1409 _, _, _, err = tree.SaveVersion(false) 1410 require.Error(err, "SaveVersion should fail, overwrite was not allowed") 1411 1412 tree.Set([]byte("key50"), []byte("value50 different")) 1413 _, _, _, err = tree.SaveVersion(false) 1414 require.NoError(err, "SaveVersion should not fail, write the same value") 1415 1416 // The tree version now is 52 which is equal to latest version. 1417 // Now any key value can be written into the tree 1418 tree.Set([]byte("key any value"), []byte("value any value")) 1419 _, _, _, err = tree.SaveVersion(false) 1420 require.NoError(err, "SaveVersion should not fail.") 1421 } 1422 1423 func TestDeleteVersionsCompare(t *testing.T) { 1424 require := require.New(t) 1425 1426 var databaseSizeDeleteVersionsRange, databaseSizeDeleteVersion, databaseSizeDeleteVersions string 1427 1428 const maxLength = 100 1429 const fromLength = 5 1430 { 1431 mdb := db.NewMemDB() 1432 tree, err := NewMutableTree(mdb, 0) 1433 require.NoError(err) 1434 1435 versions := make([]int64, 0, maxLength) 1436 for count := 1; count <= maxLength; count++ { 1437 versions = append(versions, int64(count)) 1438 countStr := strconv.Itoa(count) 1439 // Set kv pair and save version 1440 tree.Set([]byte("aaa"), []byte("bbb")) 1441 tree.Set([]byte("key"+countStr), []byte("value"+countStr)) 1442 _, _, _, err = tree.SaveVersion(false) 1443 require.NoError(err, "SaveVersion should not fail") 1444 } 1445 1446 tree, err = NewMutableTree(mdb, 0) 1447 require.NoError(err) 1448 targetVersion, err := tree.LoadVersion(int64(maxLength)) 1449 require.NoError(err) 1450 require.Equal(targetVersion, int64(maxLength), "targetVersion shouldn't larger than the actual tree latest version") 1451 1452 err = tree.DeleteVersionsRange(versions[fromLength], versions[int64(maxLength/2)]) 1453 require.NoError(err, "DeleteVersionsRange should not fail") 1454 1455 databaseSizeDeleteVersionsRange = mdb.Stats()["database.size"] 1456 } 1457 { 1458 mdb := db.NewMemDB() 1459 tree, err := NewMutableTree(mdb, 0) 1460 require.NoError(err) 1461 1462 versions := make([]int64, 0, maxLength) 1463 for count := 1; count <= maxLength; count++ { 1464 versions = append(versions, int64(count)) 1465 countStr := strconv.Itoa(count) 1466 // Set kv pair and save version 1467 tree.Set([]byte("aaa"), []byte("bbb")) 1468 tree.Set([]byte("key"+countStr), []byte("value"+countStr)) 1469 _, _, _, err = tree.SaveVersion(false) 1470 require.NoError(err, "SaveVersion should not fail") 1471 } 1472 1473 tree, err = NewMutableTree(mdb, 0) 1474 require.NoError(err) 1475 targetVersion, err := tree.LoadVersion(int64(maxLength)) 1476 require.NoError(err) 1477 require.Equal(targetVersion, int64(maxLength), "targetVersion shouldn't larger than the actual tree latest version") 1478 1479 for _, version := range versions[fromLength:int64(maxLength/2)] { 1480 err = tree.DeleteVersion(version) 1481 require.NoError(err, "DeleteVersion should not fail for %v", version) 1482 } 1483 1484 databaseSizeDeleteVersion = mdb.Stats()["database.size"] 1485 } 1486 { 1487 mdb := db.NewMemDB() 1488 tree, err := NewMutableTree(mdb, 0) 1489 require.NoError(err) 1490 1491 versions := make([]int64, 0, maxLength) 1492 for count := 1; count <= maxLength; count++ { 1493 versions = append(versions, int64(count)) 1494 countStr := strconv.Itoa(count) 1495 // Set kv pair and save version 1496 tree.Set([]byte("aaa"), []byte("bbb")) 1497 tree.Set([]byte("key"+countStr), []byte("value"+countStr)) 1498 _, _, _, err = tree.SaveVersion(false) 1499 require.NoError(err, "SaveVersion should not fail") 1500 } 1501 1502 tree, err = NewMutableTree(mdb, 0) 1503 require.NoError(err) 1504 targetVersion, err := tree.LoadVersion(int64(maxLength)) 1505 require.NoError(err) 1506 require.Equal(targetVersion, int64(maxLength), "targetVersion shouldn't larger than the actual tree latest version") 1507 1508 err = tree.DeleteVersions(versions[fromLength:int64(maxLength/2)]...) 1509 require.NoError(err, "DeleteVersions should not fail") 1510 1511 databaseSizeDeleteVersions = mdb.Stats()["database.size"] 1512 } 1513 1514 require.Equal(databaseSizeDeleteVersion, databaseSizeDeleteVersionsRange) 1515 require.Equal(databaseSizeDeleteVersion, databaseSizeDeleteVersions) 1516 } 1517 1518 // BENCHMARKS 1519 1520 func BenchmarkTreeLoadAndDelete(b *testing.B) { 1521 numVersions := 5000 1522 numKeysPerVersion := 10 1523 1524 d, err := db.NewGoLevelDB("bench", ".") 1525 if err != nil { 1526 panic(err) 1527 } 1528 defer d.Close() 1529 defer os.RemoveAll("./bench.db") 1530 1531 tree, err := NewMutableTree(d, 0) 1532 require.NoError(b, err) 1533 for v := 1; v < numVersions; v++ { 1534 for i := 0; i < numKeysPerVersion; i++ { 1535 tree.Set([]byte(cmn.RandStr(16)), cmn.RandBytes(32)) 1536 } 1537 tree.SaveVersion(false) 1538 } 1539 1540 b.Run("LoadAndDelete", func(b *testing.B) { 1541 for n := 0; n < b.N; n++ { 1542 b.StopTimer() 1543 tree, err = NewMutableTree(d, 0) 1544 require.NoError(b, err) 1545 runtime.GC() 1546 b.StartTimer() 1547 1548 // Load the tree from disk. 1549 tree.Load() 1550 1551 // Delete about 10% of the versions randomly. 1552 // The trade-off is usually between load efficiency and delete 1553 // efficiency, which is why we do both in this benchmark. 1554 // If we can load quickly into a data-structure that allows for 1555 // efficient deletes, we are golden. 1556 for v := 0; v < numVersions/10; v++ { 1557 version := (cmn.RandInt() % numVersions) + 1 1558 tree.DeleteVersion(int64(version)) 1559 } 1560 } 1561 }) 1562 } 1563 1564 func TestLoadVersionForOverwritingCase2(t *testing.T) { 1565 require := require.New(t) 1566 1567 tree, _ := NewMutableTreeWithOpts(db.NewMemDB(), 0, nil) 1568 1569 for i := byte(0); i < 20; i++ { 1570 tree.Set([]byte{i}, []byte{i}) 1571 } 1572 1573 _, _, _, err := tree.SaveVersion(false) 1574 require.NoError(err, "SaveVersion should not fail") 1575 1576 for i := byte(0); i < 20; i++ { 1577 tree.Set([]byte{i}, []byte{i + 1}) 1578 } 1579 1580 _, _, _, err = tree.SaveVersion(false) 1581 require.NoError(err, "SaveVersion should not fail with the same key") 1582 1583 for i := byte(0); i < 20; i++ { 1584 tree.Set([]byte{i}, []byte{i + 2}) 1585 } 1586 tree.SaveVersion(false) 1587 1588 removedNodes := []*Node{} 1589 1590 for _, n := range tree.ndb.nodes() { 1591 if n.version > 1 { 1592 removedNodes = append(removedNodes, n) 1593 } 1594 } 1595 1596 _, err = tree.LoadVersionForOverwriting(1) 1597 require.NoError(err, "LoadVersionForOverwriting should not fail") 1598 1599 for i := byte(0); i < 20; i++ { 1600 _, v := tree.GetWithIndex([]byte{i}) 1601 require.Equal([]byte{i}, v) 1602 } 1603 1604 for _, n := range removedNodes { 1605 has, _ := tree.ndb.Has(n.hash) 1606 require.False(has, "LoadVersionForOverwriting should remove useless nodes") 1607 } 1608 1609 tree.Set([]byte{0x2}, []byte{0x3}) 1610 1611 _, _, _, err = tree.SaveVersion(false) 1612 require.NoError(err, "SaveVersion should not fail") 1613 1614 err = tree.DeleteVersion(1) 1615 require.NoError(err, "DeleteVersion should not fail") 1616 1617 tree.Set([]byte{0x1}, []byte{0x3}) 1618 1619 _, _, _, err = tree.SaveVersion(false) 1620 require.NoError(err, "SaveVersion should not fail") 1621 } 1622 1623 func TestLoadVersionForOverwritingCase3(t *testing.T) { 1624 require := require.New(t) 1625 1626 tree, err := NewMutableTreeWithOpts(db.NewMemDB(), 0, nil) 1627 require.NoError(err) 1628 1629 for i := byte(0); i < 20; i++ { 1630 tree.Set([]byte{i}, []byte{i}) 1631 } 1632 _, _, _, err = tree.SaveVersion(false) 1633 require.NoError(err) 1634 1635 for i := byte(0); i < 20; i++ { 1636 tree.Set([]byte{i}, []byte{i + 1}) 1637 } 1638 _, _, _, err = tree.SaveVersion(false) 1639 require.NoError(err) 1640 1641 removedNodes := []*Node{} 1642 1643 for _, n := range tree.ndb.nodes() { 1644 if n.version > 1 { 1645 removedNodes = append(removedNodes, n) 1646 } 1647 } 1648 1649 for i := byte(0); i < 20; i++ { 1650 tree.Remove([]byte{i}) 1651 } 1652 _, _, _, err = tree.SaveVersion(false) 1653 require.NoError(err) 1654 1655 _, err = tree.LoadVersionForOverwriting(1) 1656 require.NoError(err) 1657 for _, n := range removedNodes { 1658 has, err := tree.ndb.Has(n.hash) 1659 require.NoError(err) 1660 require.False(has, "LoadVersionForOverwriting should remove useless nodes") 1661 } 1662 1663 for i := byte(0); i < 20; i++ { 1664 _, v := tree.GetWithIndex([]byte{i}) 1665 require.Equal([]byte{i}, v) 1666 } 1667 } 1668 1669 func TestIterate_ImmutableTree_Version1(t *testing.T) { 1670 tree, mirror := getRandomizedTreeAndMirror(t) 1671 1672 _, _, _, err := tree.SaveVersion(false) 1673 require.NoError(t, err) 1674 1675 immutableTree, err := tree.GetImmutable(1) 1676 require.NoError(t, err) 1677 1678 assertImmutableMirrorIterate(t, immutableTree, mirror) 1679 } 1680 1681 func TestIterate_ImmutableTree_Version2(t *testing.T) { 1682 tree, mirror := getRandomizedTreeAndMirror(t) 1683 1684 _, _, _, err := tree.SaveVersion(false) 1685 require.NoError(t, err) 1686 1687 randomizeTreeAndMirror(t, tree, mirror) 1688 1689 _, _, _, err = tree.SaveVersion(false) 1690 require.NoError(t, err) 1691 1692 immutableTree, err := tree.GetImmutable(2) 1693 require.NoError(t, err) 1694 1695 assertImmutableMirrorIterate(t, immutableTree, mirror) 1696 } 1697 1698 func TestGetByIndex_ImmutableTree(t *testing.T) { 1699 tree, mirror := getRandomizedTreeAndMirror(t) 1700 mirrorKeys := getSortedMirrorKeys(mirror) 1701 1702 _, _, _, err := tree.SaveVersion(false) 1703 require.NoError(t, err) 1704 1705 immutableTree, err := tree.GetImmutable(1) 1706 require.NoError(t, err) 1707 1708 require.True(t, immutableTree.IsFastCacheEnabled()) 1709 1710 for index, expectedKey := range mirrorKeys { 1711 expectedValue := mirror[expectedKey] 1712 1713 actualKey, actualValue := immutableTree.GetByIndex(int64(index)) 1714 1715 require.Equal(t, expectedKey, string(actualKey)) 1716 require.Equal(t, expectedValue, string(actualValue)) 1717 } 1718 } 1719 1720 func TestGetWithIndex_ImmutableTree(t *testing.T) { 1721 tree, mirror := getRandomizedTreeAndMirror(t) 1722 mirrorKeys := getSortedMirrorKeys(mirror) 1723 1724 _, _, _, err := tree.SaveVersion(false) 1725 require.NoError(t, err) 1726 1727 immutableTree, err := tree.GetImmutable(1) 1728 require.NoError(t, err) 1729 1730 require.True(t, immutableTree.IsFastCacheEnabled()) 1731 1732 for expectedIndex, key := range mirrorKeys { 1733 expectedValue := mirror[key] 1734 1735 actualIndex, actualValue := immutableTree.GetWithIndex([]byte(key)) 1736 1737 require.Equal(t, expectedValue, string(actualValue)) 1738 require.Equal(t, int64(expectedIndex), actualIndex) 1739 } 1740 } 1741 1742 func Benchmark_GetWithIndex(b *testing.B) { 1743 db := db.NewDB("test", db.MemDBBackend, "") 1744 1745 const numKeyVals = 100000 1746 1747 t, err := NewMutableTree(db, numKeyVals) 1748 require.NoError(b, err) 1749 1750 keys := make([][]byte, 0, numKeyVals) 1751 1752 for i := 0; i < numKeyVals; i++ { 1753 key := randBytes(10) 1754 keys = append(keys, key) 1755 t.Set(key, randBytes(10)) 1756 } 1757 _, _, _, err = t.SaveVersion(false) 1758 require.NoError(b, err) 1759 1760 b.ReportAllocs() 1761 runtime.GC() 1762 1763 b.Run("fast", func(sub *testing.B) { 1764 require.True(b, t.IsFastCacheEnabled()) 1765 b.ResetTimer() 1766 for i := 0; i < sub.N; i++ { 1767 randKey := rand.Intn(numKeyVals) 1768 t.GetWithIndex(keys[randKey]) 1769 } 1770 }) 1771 1772 b.Run("regular", func(sub *testing.B) { 1773 // get non-latest version to force regular storage 1774 _, latestVersion, _, err := t.SaveVersion(false) 1775 require.NoError(b, err) 1776 1777 itree, err := t.GetImmutable(latestVersion - 1) 1778 require.NoError(b, err) 1779 1780 require.False(b, itree.IsFastCacheEnabled()) 1781 b.ResetTimer() 1782 for i := 0; i < sub.N; i++ { 1783 randKey := rand.Intn(numKeyVals) 1784 itree.GetWithIndex(keys[randKey]) 1785 } 1786 }) 1787 } 1788 1789 func Benchmark_GetByIndex(b *testing.B) { 1790 db := db.NewDB("test", db.MemDBBackend, "") 1791 1792 const numKeyVals = 100000 1793 1794 t, err := NewMutableTree(db, numKeyVals) 1795 require.NoError(b, err) 1796 1797 for i := 0; i < numKeyVals; i++ { 1798 key := randBytes(10) 1799 t.Set(key, randBytes(10)) 1800 } 1801 _, _, _, err = t.SaveVersion(false) 1802 require.NoError(b, err) 1803 1804 b.ReportAllocs() 1805 runtime.GC() 1806 1807 b.Run("fast", func(sub *testing.B) { 1808 require.True(b, t.IsFastCacheEnabled()) 1809 b.ResetTimer() 1810 for i := 0; i < sub.N; i++ { 1811 randIdx := rand.Intn(numKeyVals) 1812 t.GetByIndex(int64(randIdx)) 1813 } 1814 }) 1815 1816 b.Run("regular", func(sub *testing.B) { 1817 // get non-latest version to force regular storage 1818 _, latestVersion, _, err := t.SaveVersion(false) 1819 1820 itree, err := t.GetImmutable(latestVersion - 1) 1821 require.NoError(b, err) 1822 1823 require.False(b, itree.IsFastCacheEnabled()) 1824 b.ResetTimer() 1825 for i := 0; i < sub.N; i++ { 1826 randIdx := rand.Intn(numKeyVals) 1827 itree.GetByIndex(int64(randIdx)) 1828 } 1829 }) 1830 }