github.com/mrwater98/go-ethereum@v1.9.7/trie/iterator_test.go (about)

     1  // Copyright 2014 The go-ethereum Authors
     2  // This file is part of the go-ethereum library.
     3  //
     4  // The go-ethereum library is free software: you can redistribute it and/or modify
     5  // it under the terms of the GNU Lesser General Public License as published by
     6  // the Free Software Foundation, either version 3 of the License, or
     7  // (at your option) any later version.
     8  //
     9  // The go-ethereum library is distributed in the hope that it will be useful,
    10  // but WITHOUT ANY WARRANTY; without even the implied warranty of
    11  // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    12  // GNU Lesser General Public License for more details.
    13  //
    14  // You should have received a copy of the GNU Lesser General Public License
    15  // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
    16  
    17  package trie
    18  
    19  import (
    20  	"bytes"
    21  	"fmt"
    22  	"math/rand"
    23  	"testing"
    24  
    25  	"github.com/ethereum/go-ethereum/common"
    26  	"github.com/ethereum/go-ethereum/ethdb/memorydb"
    27  )
    28  
    29  func TestIterator(t *testing.T) {
    30  	trie := newEmpty()
    31  	vals := []struct{ k, v string }{
    32  		{"do", "verb"},
    33  		{"ether", "wookiedoo"},
    34  		{"horse", "stallion"},
    35  		{"shaman", "horse"},
    36  		{"doge", "coin"},
    37  		{"dog", "puppy"},
    38  		{"somethingveryoddindeedthis is", "myothernodedata"},
    39  	}
    40  	all := make(map[string]string)
    41  	for _, val := range vals {
    42  		all[val.k] = val.v
    43  		trie.Update([]byte(val.k), []byte(val.v))
    44  	}
    45  	trie.Commit(nil)
    46  
    47  	found := make(map[string]string)
    48  	it := NewIterator(trie.NodeIterator(nil))
    49  	for it.Next() {
    50  		found[string(it.Key)] = string(it.Value)
    51  	}
    52  
    53  	for k, v := range all {
    54  		if found[k] != v {
    55  			t.Errorf("iterator value mismatch for %s: got %q want %q", k, found[k], v)
    56  		}
    57  	}
    58  }
    59  
    60  type kv struct {
    61  	k, v []byte
    62  	t    bool
    63  }
    64  
    65  func TestIteratorLargeData(t *testing.T) {
    66  	trie := newEmpty()
    67  	vals := make(map[string]*kv)
    68  
    69  	for i := byte(0); i < 255; i++ {
    70  		value := &kv{common.LeftPadBytes([]byte{i}, 32), []byte{i}, false}
    71  		value2 := &kv{common.LeftPadBytes([]byte{10, i}, 32), []byte{i}, false}
    72  		trie.Update(value.k, value.v)
    73  		trie.Update(value2.k, value2.v)
    74  		vals[string(value.k)] = value
    75  		vals[string(value2.k)] = value2
    76  	}
    77  
    78  	it := NewIterator(trie.NodeIterator(nil))
    79  	for it.Next() {
    80  		vals[string(it.Key)].t = true
    81  	}
    82  
    83  	var untouched []*kv
    84  	for _, value := range vals {
    85  		if !value.t {
    86  			untouched = append(untouched, value)
    87  		}
    88  	}
    89  
    90  	if len(untouched) > 0 {
    91  		t.Errorf("Missed %d nodes", len(untouched))
    92  		for _, value := range untouched {
    93  			t.Error(value)
    94  		}
    95  	}
    96  }
    97  
    98  // Tests that the node iterator indeed walks over the entire database contents.
    99  func TestNodeIteratorCoverage(t *testing.T) {
   100  	// Create some arbitrary test trie to iterate
   101  	db, trie, _ := makeTestTrie()
   102  
   103  	// Gather all the node hashes found by the iterator
   104  	hashes := make(map[common.Hash]struct{})
   105  	for it := trie.NodeIterator(nil); it.Next(true); {
   106  		if it.Hash() != (common.Hash{}) {
   107  			hashes[it.Hash()] = struct{}{}
   108  		}
   109  	}
   110  	// Cross check the hashes and the database itself
   111  	for hash := range hashes {
   112  		if _, err := db.Node(hash); err != nil {
   113  			t.Errorf("failed to retrieve reported node %x: %v", hash, err)
   114  		}
   115  	}
   116  	for hash, obj := range db.dirties {
   117  		if obj != nil && hash != (common.Hash{}) {
   118  			if _, ok := hashes[hash]; !ok {
   119  				t.Errorf("state entry not reported %x", hash)
   120  			}
   121  		}
   122  	}
   123  	it := db.diskdb.NewIterator()
   124  	for it.Next() {
   125  		key := it.Key()
   126  		if _, ok := hashes[common.BytesToHash(key)]; !ok {
   127  			t.Errorf("state entry not reported %x", key)
   128  		}
   129  	}
   130  	it.Release()
   131  }
   132  
   133  type kvs struct{ k, v string }
   134  
   135  var testdata1 = []kvs{
   136  	{"barb", "ba"},
   137  	{"bard", "bc"},
   138  	{"bars", "bb"},
   139  	{"bar", "b"},
   140  	{"fab", "z"},
   141  	{"food", "ab"},
   142  	{"foos", "aa"},
   143  	{"foo", "a"},
   144  }
   145  
   146  var testdata2 = []kvs{
   147  	{"aardvark", "c"},
   148  	{"bar", "b"},
   149  	{"barb", "bd"},
   150  	{"bars", "be"},
   151  	{"fab", "z"},
   152  	{"foo", "a"},
   153  	{"foos", "aa"},
   154  	{"food", "ab"},
   155  	{"jars", "d"},
   156  }
   157  
   158  func TestIteratorSeek(t *testing.T) {
   159  	trie := newEmpty()
   160  	for _, val := range testdata1 {
   161  		trie.Update([]byte(val.k), []byte(val.v))
   162  	}
   163  
   164  	// Seek to the middle.
   165  	it := NewIterator(trie.NodeIterator([]byte("fab")))
   166  	if err := checkIteratorOrder(testdata1[4:], it); err != nil {
   167  		t.Fatal(err)
   168  	}
   169  
   170  	// Seek to a non-existent key.
   171  	it = NewIterator(trie.NodeIterator([]byte("barc")))
   172  	if err := checkIteratorOrder(testdata1[1:], it); err != nil {
   173  		t.Fatal(err)
   174  	}
   175  
   176  	// Seek beyond the end.
   177  	it = NewIterator(trie.NodeIterator([]byte("z")))
   178  	if err := checkIteratorOrder(nil, it); err != nil {
   179  		t.Fatal(err)
   180  	}
   181  }
   182  
   183  func checkIteratorOrder(want []kvs, it *Iterator) error {
   184  	for it.Next() {
   185  		if len(want) == 0 {
   186  			return fmt.Errorf("didn't expect any more values, got key %q", it.Key)
   187  		}
   188  		if !bytes.Equal(it.Key, []byte(want[0].k)) {
   189  			return fmt.Errorf("wrong key: got %q, want %q", it.Key, want[0].k)
   190  		}
   191  		want = want[1:]
   192  	}
   193  	if len(want) > 0 {
   194  		return fmt.Errorf("iterator ended early, want key %q", want[0])
   195  	}
   196  	return nil
   197  }
   198  
   199  func TestDifferenceIterator(t *testing.T) {
   200  	triea := newEmpty()
   201  	for _, val := range testdata1 {
   202  		triea.Update([]byte(val.k), []byte(val.v))
   203  	}
   204  	triea.Commit(nil)
   205  
   206  	trieb := newEmpty()
   207  	for _, val := range testdata2 {
   208  		trieb.Update([]byte(val.k), []byte(val.v))
   209  	}
   210  	trieb.Commit(nil)
   211  
   212  	found := make(map[string]string)
   213  	di, _ := NewDifferenceIterator(triea.NodeIterator(nil), trieb.NodeIterator(nil))
   214  	it := NewIterator(di)
   215  	for it.Next() {
   216  		found[string(it.Key)] = string(it.Value)
   217  	}
   218  
   219  	all := []struct{ k, v string }{
   220  		{"aardvark", "c"},
   221  		{"barb", "bd"},
   222  		{"bars", "be"},
   223  		{"jars", "d"},
   224  	}
   225  	for _, item := range all {
   226  		if found[item.k] != item.v {
   227  			t.Errorf("iterator value mismatch for %s: got %v want %v", item.k, found[item.k], item.v)
   228  		}
   229  	}
   230  	if len(found) != len(all) {
   231  		t.Errorf("iterator count mismatch: got %d values, want %d", len(found), len(all))
   232  	}
   233  }
   234  
   235  func TestUnionIterator(t *testing.T) {
   236  	triea := newEmpty()
   237  	for _, val := range testdata1 {
   238  		triea.Update([]byte(val.k), []byte(val.v))
   239  	}
   240  	triea.Commit(nil)
   241  
   242  	trieb := newEmpty()
   243  	for _, val := range testdata2 {
   244  		trieb.Update([]byte(val.k), []byte(val.v))
   245  	}
   246  	trieb.Commit(nil)
   247  
   248  	di, _ := NewUnionIterator([]NodeIterator{triea.NodeIterator(nil), trieb.NodeIterator(nil)})
   249  	it := NewIterator(di)
   250  
   251  	all := []struct{ k, v string }{
   252  		{"aardvark", "c"},
   253  		{"barb", "ba"},
   254  		{"barb", "bd"},
   255  		{"bard", "bc"},
   256  		{"bars", "bb"},
   257  		{"bars", "be"},
   258  		{"bar", "b"},
   259  		{"fab", "z"},
   260  		{"food", "ab"},
   261  		{"foos", "aa"},
   262  		{"foo", "a"},
   263  		{"jars", "d"},
   264  	}
   265  
   266  	for i, kv := range all {
   267  		if !it.Next() {
   268  			t.Errorf("Iterator ends prematurely at element %d", i)
   269  		}
   270  		if kv.k != string(it.Key) {
   271  			t.Errorf("iterator value mismatch for element %d: got key %s want %s", i, it.Key, kv.k)
   272  		}
   273  		if kv.v != string(it.Value) {
   274  			t.Errorf("iterator value mismatch for element %d: got value %s want %s", i, it.Value, kv.v)
   275  		}
   276  	}
   277  	if it.Next() {
   278  		t.Errorf("Iterator returned extra values.")
   279  	}
   280  }
   281  
   282  func TestIteratorNoDups(t *testing.T) {
   283  	var tr Trie
   284  	for _, val := range testdata1 {
   285  		tr.Update([]byte(val.k), []byte(val.v))
   286  	}
   287  	checkIteratorNoDups(t, tr.NodeIterator(nil), nil)
   288  }
   289  
   290  // This test checks that nodeIterator.Next can be retried after inserting missing trie nodes.
   291  func TestIteratorContinueAfterErrorDisk(t *testing.T)    { testIteratorContinueAfterError(t, false) }
   292  func TestIteratorContinueAfterErrorMemonly(t *testing.T) { testIteratorContinueAfterError(t, true) }
   293  
   294  func testIteratorContinueAfterError(t *testing.T, memonly bool) {
   295  	diskdb := memorydb.New()
   296  	triedb := NewDatabase(diskdb)
   297  
   298  	tr, _ := New(common.Hash{}, triedb)
   299  	for _, val := range testdata1 {
   300  		tr.Update([]byte(val.k), []byte(val.v))
   301  	}
   302  	tr.Commit(nil)
   303  	if !memonly {
   304  		triedb.Commit(tr.Hash(), true)
   305  	}
   306  	wantNodeCount := checkIteratorNoDups(t, tr.NodeIterator(nil), nil)
   307  
   308  	var (
   309  		diskKeys [][]byte
   310  		memKeys  []common.Hash
   311  	)
   312  	if memonly {
   313  		memKeys = triedb.Nodes()
   314  	} else {
   315  		it := diskdb.NewIterator()
   316  		for it.Next() {
   317  			diskKeys = append(diskKeys, it.Key())
   318  		}
   319  		it.Release()
   320  	}
   321  	for i := 0; i < 20; i++ {
   322  		// Create trie that will load all nodes from DB.
   323  		tr, _ := New(tr.Hash(), triedb)
   324  
   325  		// Remove a random node from the database. It can't be the root node
   326  		// because that one is already loaded.
   327  		var (
   328  			rkey common.Hash
   329  			rval []byte
   330  			robj *cachedNode
   331  		)
   332  		for {
   333  			if memonly {
   334  				rkey = memKeys[rand.Intn(len(memKeys))]
   335  			} else {
   336  				copy(rkey[:], diskKeys[rand.Intn(len(diskKeys))])
   337  			}
   338  			if rkey != tr.Hash() {
   339  				break
   340  			}
   341  		}
   342  		if memonly {
   343  			robj = triedb.dirties[rkey]
   344  			delete(triedb.dirties, rkey)
   345  		} else {
   346  			rval, _ = diskdb.Get(rkey[:])
   347  			diskdb.Delete(rkey[:])
   348  		}
   349  		// Iterate until the error is hit.
   350  		seen := make(map[string]bool)
   351  		it := tr.NodeIterator(nil)
   352  		checkIteratorNoDups(t, it, seen)
   353  		missing, ok := it.Error().(*MissingNodeError)
   354  		if !ok || missing.NodeHash != rkey {
   355  			t.Fatal("didn't hit missing node, got", it.Error())
   356  		}
   357  
   358  		// Add the node back and continue iteration.
   359  		if memonly {
   360  			triedb.dirties[rkey] = robj
   361  		} else {
   362  			diskdb.Put(rkey[:], rval)
   363  		}
   364  		checkIteratorNoDups(t, it, seen)
   365  		if it.Error() != nil {
   366  			t.Fatal("unexpected error", it.Error())
   367  		}
   368  		if len(seen) != wantNodeCount {
   369  			t.Fatal("wrong node iteration count, got", len(seen), "want", wantNodeCount)
   370  		}
   371  	}
   372  }
   373  
   374  // Similar to the test above, this one checks that failure to create nodeIterator at a
   375  // certain key prefix behaves correctly when Next is called. The expectation is that Next
   376  // should retry seeking before returning true for the first time.
   377  func TestIteratorContinueAfterSeekErrorDisk(t *testing.T) {
   378  	testIteratorContinueAfterSeekError(t, false)
   379  }
   380  func TestIteratorContinueAfterSeekErrorMemonly(t *testing.T) {
   381  	testIteratorContinueAfterSeekError(t, true)
   382  }
   383  
   384  func testIteratorContinueAfterSeekError(t *testing.T, memonly bool) {
   385  	// Commit test trie to db, then remove the node containing "bars".
   386  	diskdb := memorydb.New()
   387  	triedb := NewDatabase(diskdb)
   388  
   389  	ctr, _ := New(common.Hash{}, triedb)
   390  	for _, val := range testdata1 {
   391  		ctr.Update([]byte(val.k), []byte(val.v))
   392  	}
   393  	root, _ := ctr.Commit(nil)
   394  	if !memonly {
   395  		triedb.Commit(root, true)
   396  	}
   397  	barNodeHash := common.HexToHash("05041990364eb72fcb1127652ce40d8bab765f2bfe53225b1170d276cc101c2e")
   398  	var (
   399  		barNodeBlob []byte
   400  		barNodeObj  *cachedNode
   401  	)
   402  	if memonly {
   403  		barNodeObj = triedb.dirties[barNodeHash]
   404  		delete(triedb.dirties, barNodeHash)
   405  	} else {
   406  		barNodeBlob, _ = diskdb.Get(barNodeHash[:])
   407  		diskdb.Delete(barNodeHash[:])
   408  	}
   409  	// Create a new iterator that seeks to "bars". Seeking can't proceed because
   410  	// the node is missing.
   411  	tr, _ := New(root, triedb)
   412  	it := tr.NodeIterator([]byte("bars"))
   413  	missing, ok := it.Error().(*MissingNodeError)
   414  	if !ok {
   415  		t.Fatal("want MissingNodeError, got", it.Error())
   416  	} else if missing.NodeHash != barNodeHash {
   417  		t.Fatal("wrong node missing")
   418  	}
   419  	// Reinsert the missing node.
   420  	if memonly {
   421  		triedb.dirties[barNodeHash] = barNodeObj
   422  	} else {
   423  		diskdb.Put(barNodeHash[:], barNodeBlob)
   424  	}
   425  	// Check that iteration produces the right set of values.
   426  	if err := checkIteratorOrder(testdata1[2:], NewIterator(it)); err != nil {
   427  		t.Fatal(err)
   428  	}
   429  }
   430  
   431  func checkIteratorNoDups(t *testing.T, it NodeIterator, seen map[string]bool) int {
   432  	if seen == nil {
   433  		seen = make(map[string]bool)
   434  	}
   435  	for it.Next(true) {
   436  		if seen[string(it.Path())] {
   437  			t.Fatalf("iterator visited node path %x twice", it.Path())
   438  		}
   439  		seen[string(it.Path())] = true
   440  	}
   441  	return len(seen)
   442  }