github.com/linapex/ethereum-go-chinese@v0.0.0-20190316121929-f8b7a73c3fa1/swarm/shed/index_test.go (about) 1 2 //<developer> 3 // <name>linapex 曹一峰</name> 4 // <email>linapex@163.com</email> 5 // <wx>superexc</wx> 6 // <qqgroup>128148617</qqgroup> 7 // <url>https://jsq.ink</url> 8 // <role>pku engineer</role> 9 // <date>2019-03-16 19:16:44</date> 10 //</624450117969121280> 11 12 13 package shed 14 15 import ( 16 "bytes" 17 "encoding/binary" 18 "fmt" 19 "sort" 20 "testing" 21 "time" 22 23 "github.com/syndtr/goleveldb/leveldb" 24 ) 25 26 //用于此文件中测试的索引函数。 27 var retrievalIndexFuncs = IndexFuncs{ 28 EncodeKey: func(fields Item) (key []byte, err error) { 29 return fields.Address, nil 30 }, 31 DecodeKey: func(key []byte) (e Item, err error) { 32 e.Address = key 33 return e, nil 34 }, 35 EncodeValue: func(fields Item) (value []byte, err error) { 36 b := make([]byte, 8) 37 binary.BigEndian.PutUint64(b, uint64(fields.StoreTimestamp)) 38 value = append(b, fields.Data...) 39 return value, nil 40 }, 41 DecodeValue: func(keyItem Item, value []byte) (e Item, err error) { 42 e.StoreTimestamp = int64(binary.BigEndian.Uint64(value[:8])) 43 e.Data = value[8:] 44 return e, nil 45 }, 46 } 47 48 //testindex验证索引实现的put、get和delete函数。 49 func TestIndex(t *testing.T) { 50 db, cleanupFunc := newTestDB(t) 51 defer cleanupFunc() 52 53 index, err := db.NewIndex("retrieval", retrievalIndexFuncs) 54 if err != nil { 55 t.Fatal(err) 56 } 57 58 t.Run("put", func(t *testing.T) { 59 want := Item{ 60 Address: []byte("put-hash"), 61 Data: []byte("DATA"), 62 StoreTimestamp: time.Now().UTC().UnixNano(), 63 } 64 65 err := index.Put(want) 66 if err != nil { 67 t.Fatal(err) 68 } 69 got, err := index.Get(Item{ 70 Address: want.Address, 71 }) 72 if err != nil { 73 t.Fatal(err) 74 } 75 checkItem(t, got, want) 76 77 t.Run("overwrite", func(t *testing.T) { 78 want := Item{ 79 Address: []byte("put-hash"), 80 Data: []byte("New DATA"), 81 StoreTimestamp: time.Now().UTC().UnixNano(), 82 } 83 84 err = index.Put(want) 85 if err != nil { 86 t.Fatal(err) 87 } 88 got, err := index.Get(Item{ 89 Address: want.Address, 90 }) 91 if err != nil { 92 t.Fatal(err) 93 } 94 checkItem(t, got, want) 95 }) 96 }) 97 98 t.Run("put in batch", func(t *testing.T) { 99 want := Item{ 100 Address: []byte("put-in-batch-hash"), 101 Data: []byte("DATA"), 102 StoreTimestamp: time.Now().UTC().UnixNano(), 103 } 104 105 batch := new(leveldb.Batch) 106 index.PutInBatch(batch, want) 107 err := db.WriteBatch(batch) 108 if err != nil { 109 t.Fatal(err) 110 } 111 got, err := index.Get(Item{ 112 Address: want.Address, 113 }) 114 if err != nil { 115 t.Fatal(err) 116 } 117 checkItem(t, got, want) 118 119 t.Run("overwrite", func(t *testing.T) { 120 want := Item{ 121 Address: []byte("put-in-batch-hash"), 122 Data: []byte("New DATA"), 123 StoreTimestamp: time.Now().UTC().UnixNano(), 124 } 125 126 batch := new(leveldb.Batch) 127 index.PutInBatch(batch, want) 128 db.WriteBatch(batch) 129 if err != nil { 130 t.Fatal(err) 131 } 132 got, err := index.Get(Item{ 133 Address: want.Address, 134 }) 135 if err != nil { 136 t.Fatal(err) 137 } 138 checkItem(t, got, want) 139 }) 140 }) 141 142 t.Run("put in batch twice", func(t *testing.T) { 143 //确保具有相同db键的最后一项 144 //实际已保存 145 batch := new(leveldb.Batch) 146 address := []byte("put-in-batch-twice-hash") 147 148 //放第一个项目 149 index.PutInBatch(batch, Item{ 150 Address: address, 151 Data: []byte("DATA"), 152 StoreTimestamp: time.Now().UTC().UnixNano(), 153 }) 154 155 want := Item{ 156 Address: address, 157 Data: []byte("New DATA"), 158 StoreTimestamp: time.Now().UTC().UnixNano(), 159 } 160 //然后放入将产生相同密钥的项 161 //但数据库中的值不同 162 index.PutInBatch(batch, want) 163 db.WriteBatch(batch) 164 if err != nil { 165 t.Fatal(err) 166 } 167 got, err := index.Get(Item{ 168 Address: address, 169 }) 170 if err != nil { 171 t.Fatal(err) 172 } 173 checkItem(t, got, want) 174 }) 175 176 t.Run("delete", func(t *testing.T) { 177 want := Item{ 178 Address: []byte("delete-hash"), 179 Data: []byte("DATA"), 180 StoreTimestamp: time.Now().UTC().UnixNano(), 181 } 182 183 err := index.Put(want) 184 if err != nil { 185 t.Fatal(err) 186 } 187 got, err := index.Get(Item{ 188 Address: want.Address, 189 }) 190 if err != nil { 191 t.Fatal(err) 192 } 193 checkItem(t, got, want) 194 195 err = index.Delete(Item{ 196 Address: want.Address, 197 }) 198 if err != nil { 199 t.Fatal(err) 200 } 201 202 wantErr := leveldb.ErrNotFound 203 got, err = index.Get(Item{ 204 Address: want.Address, 205 }) 206 if err != wantErr { 207 t.Fatalf("got error %v, want %v", err, wantErr) 208 } 209 }) 210 211 t.Run("delete in batch", func(t *testing.T) { 212 want := Item{ 213 Address: []byte("delete-in-batch-hash"), 214 Data: []byte("DATA"), 215 StoreTimestamp: time.Now().UTC().UnixNano(), 216 } 217 218 err := index.Put(want) 219 if err != nil { 220 t.Fatal(err) 221 } 222 got, err := index.Get(Item{ 223 Address: want.Address, 224 }) 225 if err != nil { 226 t.Fatal(err) 227 } 228 checkItem(t, got, want) 229 230 batch := new(leveldb.Batch) 231 index.DeleteInBatch(batch, Item{ 232 Address: want.Address, 233 }) 234 err = db.WriteBatch(batch) 235 if err != nil { 236 t.Fatal(err) 237 } 238 239 wantErr := leveldb.ErrNotFound 240 got, err = index.Get(Item{ 241 Address: want.Address, 242 }) 243 if err != wantErr { 244 t.Fatalf("got error %v, want %v", err, wantErr) 245 } 246 }) 247 } 248 249 //TestIndex迭代验证索引迭代 250 //正确性功能。 251 func TestIndex_Iterate(t *testing.T) { 252 db, cleanupFunc := newTestDB(t) 253 defer cleanupFunc() 254 255 index, err := db.NewIndex("retrieval", retrievalIndexFuncs) 256 if err != nil { 257 t.Fatal(err) 258 } 259 260 items := []Item{ 261 { 262 Address: []byte("iterate-hash-01"), 263 Data: []byte("data80"), 264 }, 265 { 266 Address: []byte("iterate-hash-03"), 267 Data: []byte("data22"), 268 }, 269 { 270 Address: []byte("iterate-hash-05"), 271 Data: []byte("data41"), 272 }, 273 { 274 Address: []byte("iterate-hash-02"), 275 Data: []byte("data84"), 276 }, 277 { 278 Address: []byte("iterate-hash-06"), 279 Data: []byte("data1"), 280 }, 281 } 282 batch := new(leveldb.Batch) 283 for _, i := range items { 284 index.PutInBatch(batch, i) 285 } 286 err = db.WriteBatch(batch) 287 if err != nil { 288 t.Fatal(err) 289 } 290 item04 := Item{ 291 Address: []byte("iterate-hash-04"), 292 Data: []byte("data0"), 293 } 294 err = index.Put(item04) 295 if err != nil { 296 t.Fatal(err) 297 } 298 items = append(items, item04) 299 300 sort.SliceStable(items, func(i, j int) bool { 301 return bytes.Compare(items[i].Address, items[j].Address) < 0 302 }) 303 304 t.Run("all", func(t *testing.T) { 305 var i int 306 err := index.Iterate(func(item Item) (stop bool, err error) { 307 if i > len(items)-1 { 308 return true, fmt.Errorf("got unexpected index item: %#v", item) 309 } 310 want := items[i] 311 checkItem(t, item, want) 312 i++ 313 return false, nil 314 }, nil) 315 if err != nil { 316 t.Fatal(err) 317 } 318 }) 319 320 t.Run("start from", func(t *testing.T) { 321 startIndex := 2 322 i := startIndex 323 err := index.Iterate(func(item Item) (stop bool, err error) { 324 if i > len(items)-1 { 325 return true, fmt.Errorf("got unexpected index item: %#v", item) 326 } 327 want := items[i] 328 checkItem(t, item, want) 329 i++ 330 return false, nil 331 }, &IterateOptions{ 332 StartFrom: &items[startIndex], 333 }) 334 if err != nil { 335 t.Fatal(err) 336 } 337 }) 338 339 t.Run("skip start from", func(t *testing.T) { 340 startIndex := 2 341 i := startIndex + 1 342 err := index.Iterate(func(item Item) (stop bool, err error) { 343 if i > len(items)-1 { 344 return true, fmt.Errorf("got unexpected index item: %#v", item) 345 } 346 want := items[i] 347 checkItem(t, item, want) 348 i++ 349 return false, nil 350 }, &IterateOptions{ 351 StartFrom: &items[startIndex], 352 SkipStartFromItem: true, 353 }) 354 if err != nil { 355 t.Fatal(err) 356 } 357 }) 358 359 t.Run("stop", func(t *testing.T) { 360 var i int 361 stopIndex := 3 362 var count int 363 err := index.Iterate(func(item Item) (stop bool, err error) { 364 if i > len(items)-1 { 365 return true, fmt.Errorf("got unexpected index item: %#v", item) 366 } 367 want := items[i] 368 checkItem(t, item, want) 369 count++ 370 if i == stopIndex { 371 return true, nil 372 } 373 i++ 374 return false, nil 375 }, nil) 376 if err != nil { 377 t.Fatal(err) 378 } 379 wantItemsCount := stopIndex + 1 380 if count != wantItemsCount { 381 t.Errorf("got %v items, expected %v", count, wantItemsCount) 382 } 383 }) 384 385 t.Run("no overflow", func(t *testing.T) { 386 secondIndex, err := db.NewIndex("second-index", retrievalIndexFuncs) 387 if err != nil { 388 t.Fatal(err) 389 } 390 391 secondItem := Item{ 392 Address: []byte("iterate-hash-10"), 393 Data: []byte("data-second"), 394 } 395 err = secondIndex.Put(secondItem) 396 if err != nil { 397 t.Fatal(err) 398 } 399 400 var i int 401 err = index.Iterate(func(item Item) (stop bool, err error) { 402 if i > len(items)-1 { 403 return true, fmt.Errorf("got unexpected index item: %#v", item) 404 } 405 want := items[i] 406 checkItem(t, item, want) 407 i++ 408 return false, nil 409 }, nil) 410 if err != nil { 411 t.Fatal(err) 412 } 413 414 i = 0 415 err = secondIndex.Iterate(func(item Item) (stop bool, err error) { 416 if i > 1 { 417 return true, fmt.Errorf("got unexpected index item: %#v", item) 418 } 419 checkItem(t, item, secondItem) 420 i++ 421 return false, nil 422 }, nil) 423 if err != nil { 424 t.Fatal(err) 425 } 426 }) 427 } 428 429 //testindex_iterate_withprefix验证索引迭代 430 //功能正确。 431 func TestIndex_Iterate_withPrefix(t *testing.T) { 432 db, cleanupFunc := newTestDB(t) 433 defer cleanupFunc() 434 435 index, err := db.NewIndex("retrieval", retrievalIndexFuncs) 436 if err != nil { 437 t.Fatal(err) 438 } 439 440 allItems := []Item{ 441 {Address: []byte("want-hash-00"), Data: []byte("data80")}, 442 {Address: []byte("skip-hash-01"), Data: []byte("data81")}, 443 {Address: []byte("skip-hash-02"), Data: []byte("data82")}, 444 {Address: []byte("skip-hash-03"), Data: []byte("data83")}, 445 {Address: []byte("want-hash-04"), Data: []byte("data84")}, 446 {Address: []byte("want-hash-05"), Data: []byte("data85")}, 447 {Address: []byte("want-hash-06"), Data: []byte("data86")}, 448 {Address: []byte("want-hash-07"), Data: []byte("data87")}, 449 {Address: []byte("want-hash-08"), Data: []byte("data88")}, 450 {Address: []byte("want-hash-09"), Data: []byte("data89")}, 451 {Address: []byte("skip-hash-10"), Data: []byte("data90")}, 452 } 453 batch := new(leveldb.Batch) 454 for _, i := range allItems { 455 index.PutInBatch(batch, i) 456 } 457 err = db.WriteBatch(batch) 458 if err != nil { 459 t.Fatal(err) 460 } 461 462 prefix := []byte("want") 463 464 items := make([]Item, 0) 465 for _, item := range allItems { 466 if bytes.HasPrefix(item.Address, prefix) { 467 items = append(items, item) 468 } 469 } 470 sort.SliceStable(items, func(i, j int) bool { 471 return bytes.Compare(items[i].Address, items[j].Address) < 0 472 }) 473 474 t.Run("with prefix", func(t *testing.T) { 475 var i int 476 err := index.Iterate(func(item Item) (stop bool, err error) { 477 if i > len(items)-1 { 478 return true, fmt.Errorf("got unexpected index item: %#v", item) 479 } 480 want := items[i] 481 checkItem(t, item, want) 482 i++ 483 return false, nil 484 }, &IterateOptions{ 485 Prefix: prefix, 486 }) 487 if err != nil { 488 t.Fatal(err) 489 } 490 if i != len(items) { 491 t.Errorf("got %v items, want %v", i, len(items)) 492 } 493 }) 494 495 t.Run("with prefix and start from", func(t *testing.T) { 496 startIndex := 2 497 var count int 498 i := startIndex 499 err := index.Iterate(func(item Item) (stop bool, err error) { 500 if i > len(items)-1 { 501 return true, fmt.Errorf("got unexpected index item: %#v", item) 502 } 503 want := items[i] 504 checkItem(t, item, want) 505 i++ 506 count++ 507 return false, nil 508 }, &IterateOptions{ 509 StartFrom: &items[startIndex], 510 Prefix: prefix, 511 }) 512 if err != nil { 513 t.Fatal(err) 514 } 515 wantCount := len(items) - startIndex 516 if count != wantCount { 517 t.Errorf("got %v items, want %v", count, wantCount) 518 } 519 }) 520 521 t.Run("with prefix and skip start from", func(t *testing.T) { 522 startIndex := 2 523 var count int 524 i := startIndex + 1 525 err := index.Iterate(func(item Item) (stop bool, err error) { 526 if i > len(items)-1 { 527 return true, fmt.Errorf("got unexpected index item: %#v", item) 528 } 529 want := items[i] 530 checkItem(t, item, want) 531 i++ 532 count++ 533 return false, nil 534 }, &IterateOptions{ 535 StartFrom: &items[startIndex], 536 SkipStartFromItem: true, 537 Prefix: prefix, 538 }) 539 if err != nil { 540 t.Fatal(err) 541 } 542 wantCount := len(items) - startIndex - 1 543 if count != wantCount { 544 t.Errorf("got %v items, want %v", count, wantCount) 545 } 546 }) 547 548 t.Run("stop", func(t *testing.T) { 549 var i int 550 stopIndex := 3 551 var count int 552 err := index.Iterate(func(item Item) (stop bool, err error) { 553 if i > len(items)-1 { 554 return true, fmt.Errorf("got unexpected index item: %#v", item) 555 } 556 want := items[i] 557 checkItem(t, item, want) 558 count++ 559 if i == stopIndex { 560 return true, nil 561 } 562 i++ 563 return false, nil 564 }, &IterateOptions{ 565 Prefix: prefix, 566 }) 567 if err != nil { 568 t.Fatal(err) 569 } 570 wantItemsCount := stopIndex + 1 571 if count != wantItemsCount { 572 t.Errorf("got %v items, expected %v", count, wantItemsCount) 573 } 574 }) 575 576 t.Run("no overflow", func(t *testing.T) { 577 secondIndex, err := db.NewIndex("second-index", retrievalIndexFuncs) 578 if err != nil { 579 t.Fatal(err) 580 } 581 582 secondItem := Item{ 583 Address: []byte("iterate-hash-10"), 584 Data: []byte("data-second"), 585 } 586 err = secondIndex.Put(secondItem) 587 if err != nil { 588 t.Fatal(err) 589 } 590 591 var i int 592 err = index.Iterate(func(item Item) (stop bool, err error) { 593 if i > len(items)-1 { 594 return true, fmt.Errorf("got unexpected index item: %#v", item) 595 } 596 want := items[i] 597 checkItem(t, item, want) 598 i++ 599 return false, nil 600 }, &IterateOptions{ 601 Prefix: prefix, 602 }) 603 if err != nil { 604 t.Fatal(err) 605 } 606 if i != len(items) { 607 t.Errorf("got %v items, want %v", i, len(items)) 608 } 609 }) 610 } 611 612 //如果index.count和index.countFrom 613 //返回正确的项目数。 614 func TestIndex_count(t *testing.T) { 615 db, cleanupFunc := newTestDB(t) 616 defer cleanupFunc() 617 618 index, err := db.NewIndex("retrieval", retrievalIndexFuncs) 619 if err != nil { 620 t.Fatal(err) 621 } 622 623 items := []Item{ 624 { 625 Address: []byte("iterate-hash-01"), 626 Data: []byte("data80"), 627 }, 628 { 629 Address: []byte("iterate-hash-02"), 630 Data: []byte("data84"), 631 }, 632 { 633 Address: []byte("iterate-hash-03"), 634 Data: []byte("data22"), 635 }, 636 { 637 Address: []byte("iterate-hash-04"), 638 Data: []byte("data41"), 639 }, 640 { 641 Address: []byte("iterate-hash-05"), 642 Data: []byte("data1"), 643 }, 644 } 645 batch := new(leveldb.Batch) 646 for _, i := range items { 647 index.PutInBatch(batch, i) 648 } 649 err = db.WriteBatch(batch) 650 if err != nil { 651 t.Fatal(err) 652 } 653 654 t.Run("Count", func(t *testing.T) { 655 got, err := index.Count() 656 if err != nil { 657 t.Fatal(err) 658 } 659 660 want := len(items) 661 if got != want { 662 t.Errorf("got %v items count, want %v", got, want) 663 } 664 }) 665 666 t.Run("CountFrom", func(t *testing.T) { 667 got, err := index.CountFrom(Item{ 668 Address: items[1].Address, 669 }) 670 if err != nil { 671 t.Fatal(err) 672 } 673 674 want := len(items) - 1 675 if got != want { 676 t.Errorf("got %v items count, want %v", got, want) 677 } 678 }) 679 680 //用其他项更新索引 681 t.Run("add item", func(t *testing.T) { 682 item04 := Item{ 683 Address: []byte("iterate-hash-06"), 684 Data: []byte("data0"), 685 } 686 err = index.Put(item04) 687 if err != nil { 688 t.Fatal(err) 689 } 690 691 count := len(items) + 1 692 693 t.Run("Count", func(t *testing.T) { 694 got, err := index.Count() 695 if err != nil { 696 t.Fatal(err) 697 } 698 699 want := count 700 if got != want { 701 t.Errorf("got %v items count, want %v", got, want) 702 } 703 }) 704 705 t.Run("CountFrom", func(t *testing.T) { 706 got, err := index.CountFrom(Item{ 707 Address: items[1].Address, 708 }) 709 if err != nil { 710 t.Fatal(err) 711 } 712 713 want := count - 1 714 if got != want { 715 t.Errorf("got %v items count, want %v", got, want) 716 } 717 }) 718 }) 719 720 //删除一些项目 721 t.Run("delete items", func(t *testing.T) { 722 deleteCount := 3 723 724 for _, item := range items[:deleteCount] { 725 err := index.Delete(item) 726 if err != nil { 727 t.Fatal(err) 728 } 729 } 730 731 count := len(items) + 1 - deleteCount 732 733 t.Run("Count", func(t *testing.T) { 734 got, err := index.Count() 735 if err != nil { 736 t.Fatal(err) 737 } 738 739 want := count 740 if got != want { 741 t.Errorf("got %v items count, want %v", got, want) 742 } 743 }) 744 745 t.Run("CountFrom", func(t *testing.T) { 746 got, err := index.CountFrom(Item{ 747 Address: items[deleteCount+1].Address, 748 }) 749 if err != nil { 750 t.Fatal(err) 751 } 752 753 want := count - 1 754 if got != want { 755 t.Errorf("got %v items count, want %v", got, want) 756 } 757 }) 758 }) 759 } 760 761 //checkitem是一个测试助手函数,用于比较两个索引项是否相同。 762 func checkItem(t *testing.T, got, want Item) { 763 t.Helper() 764 765 if !bytes.Equal(got.Address, want.Address) { 766 t.Errorf("got hash %q, expected %q", string(got.Address), string(want.Address)) 767 } 768 if !bytes.Equal(got.Data, want.Data) { 769 t.Errorf("got data %q, expected %q", string(got.Data), string(want.Data)) 770 } 771 if got.StoreTimestamp != want.StoreTimestamp { 772 t.Errorf("got store timestamp %v, expected %v", got.StoreTimestamp, want.StoreTimestamp) 773 } 774 if got.AccessTimestamp != want.AccessTimestamp { 775 t.Errorf("got access timestamp %v, expected %v", got.AccessTimestamp, want.AccessTimestamp) 776 } 777 } 778