github.com/nspcc-dev/neo-go@v0.105.2-0.20240517133400-6be757af3eba/pkg/core/storage/memcached_store_test.go (about)

     1  package storage
     2  
     3  import (
     4  	"bytes"
     5  	"fmt"
     6  	"sort"
     7  	"testing"
     8  
     9  	"github.com/nspcc-dev/neo-go/internal/random"
    10  	"github.com/stretchr/testify/assert"
    11  	"github.com/stretchr/testify/require"
    12  )
    13  
    14  func TestMemCachedPutGetDelete(t *testing.T) {
    15  	ps := NewMemoryStore()
    16  	s := NewMemCachedStore(ps)
    17  	key := []byte("foo")
    18  	value := []byte("bar")
    19  
    20  	s.Put(key, value)
    21  
    22  	result, err := s.Get(key)
    23  	assert.Nil(t, err)
    24  	require.Equal(t, value, result)
    25  
    26  	s.Delete(key)
    27  
    28  	_, err = s.Get(key)
    29  	assert.NotNil(t, err)
    30  	assert.Equal(t, err, ErrKeyNotFound)
    31  
    32  	// Double delete.
    33  	s.Delete(key)
    34  
    35  	_, err = s.Get(key)
    36  	assert.NotNil(t, err)
    37  	assert.Equal(t, err, ErrKeyNotFound)
    38  
    39  	// Nonexistent.
    40  	key = []byte("sparse")
    41  	s.Delete(key)
    42  
    43  	_, err = s.Get(key)
    44  	assert.NotNil(t, err)
    45  	assert.Equal(t, err, ErrKeyNotFound)
    46  }
    47  
    48  func testMemCachedStorePersist(t *testing.T, ps Store) {
    49  	// cached Store
    50  	ts := NewMemCachedStore(ps)
    51  	// persisting nothing should do nothing
    52  	c, err := ts.Persist()
    53  	assert.Equal(t, nil, err)
    54  	assert.Equal(t, 0, c)
    55  	// persisting one key should result in one key in ps and nothing in ts
    56  	ts.Put([]byte("key"), []byte("value"))
    57  	checkBatch(t, ts, []KeyValueExists{{KeyValue: KeyValue{Key: []byte("key"), Value: []byte("value")}}}, nil)
    58  	c, err = ts.Persist()
    59  	checkBatch(t, ts, nil, nil)
    60  	assert.Equal(t, nil, err)
    61  	assert.Equal(t, 1, c)
    62  	v, err := ps.Get([]byte("key"))
    63  	assert.Equal(t, nil, err)
    64  	assert.Equal(t, []byte("value"), v)
    65  	v, err = ts.MemoryStore.Get([]byte("key"))
    66  	assert.Equal(t, ErrKeyNotFound, err)
    67  	assert.Equal(t, []byte(nil), v)
    68  	// now we overwrite the previous `key` contents and also add `key2`,
    69  	ts.Put([]byte("key"), []byte("newvalue"))
    70  	ts.Put([]byte("key2"), []byte("value2"))
    71  	// this is to check that now key is written into the ps before we do
    72  	// persist
    73  	v, err = ps.Get([]byte("key2"))
    74  	assert.Equal(t, ErrKeyNotFound, err)
    75  	assert.Equal(t, []byte(nil), v)
    76  	checkBatch(t, ts, []KeyValueExists{
    77  		{KeyValue: KeyValue{Key: []byte("key"), Value: []byte("newvalue")}, Exists: true},
    78  		{KeyValue: KeyValue{Key: []byte("key2"), Value: []byte("value2")}},
    79  	}, nil)
    80  	// two keys should be persisted (one overwritten and one new) and
    81  	// available in the ps
    82  	c, err = ts.Persist()
    83  	checkBatch(t, ts, nil, nil)
    84  	assert.Equal(t, nil, err)
    85  	assert.Equal(t, 2, c)
    86  	v, err = ts.MemoryStore.Get([]byte("key"))
    87  	assert.Equal(t, ErrKeyNotFound, err)
    88  	assert.Equal(t, []byte(nil), v)
    89  	v, err = ts.MemoryStore.Get([]byte("key2"))
    90  	assert.Equal(t, ErrKeyNotFound, err)
    91  	assert.Equal(t, []byte(nil), v)
    92  	v, err = ps.Get([]byte("key"))
    93  	assert.Equal(t, nil, err)
    94  	assert.Equal(t, []byte("newvalue"), v)
    95  	v, err = ps.Get([]byte("key2"))
    96  	assert.Equal(t, nil, err)
    97  	assert.Equal(t, []byte("value2"), v)
    98  	checkBatch(t, ts, nil, nil)
    99  	// we've persisted some values, make sure successive persist is a no-op
   100  	c, err = ts.Persist()
   101  	assert.Equal(t, nil, err)
   102  	assert.Equal(t, 0, c)
   103  	// test persisting deletions
   104  	ts.Delete([]byte("key"))
   105  	checkBatch(t, ts, nil, []KeyValueExists{{KeyValue: KeyValue{Key: []byte("key")}, Exists: true}})
   106  	c, err = ts.Persist()
   107  	checkBatch(t, ts, nil, nil)
   108  	assert.Equal(t, nil, err)
   109  	assert.Equal(t, 1, c)
   110  	v, err = ps.Get([]byte("key"))
   111  	assert.Equal(t, ErrKeyNotFound, err)
   112  	assert.Equal(t, []byte(nil), v)
   113  	v, err = ps.Get([]byte("key2"))
   114  	assert.Equal(t, nil, err)
   115  	assert.Equal(t, []byte("value2"), v)
   116  }
   117  
   118  func checkBatch(t *testing.T, ts *MemCachedStore, put []KeyValueExists, del []KeyValueExists) {
   119  	b := ts.GetBatch()
   120  	assert.Equal(t, len(put), len(b.Put), "wrong number of put elements in a batch")
   121  	assert.Equal(t, len(del), len(b.Deleted), "wrong number of deleted elements in a batch")
   122  
   123  	for i := range put {
   124  		assert.Contains(t, b.Put, put[i])
   125  	}
   126  
   127  	for i := range del {
   128  		assert.Contains(t, b.Deleted, del[i])
   129  	}
   130  }
   131  
   132  func TestMemCachedPersist(t *testing.T) {
   133  	t.Run("MemoryStore", func(t *testing.T) {
   134  		ps := NewMemoryStore()
   135  		testMemCachedStorePersist(t, ps)
   136  	})
   137  	t.Run("MemoryCachedStore", func(t *testing.T) {
   138  		ps1 := NewMemoryStore()
   139  		ps2 := NewMemCachedStore(ps1)
   140  		testMemCachedStorePersist(t, ps2)
   141  	})
   142  	t.Run("BoltDBStore", func(t *testing.T) {
   143  		ps := newBoltStoreForTesting(t)
   144  		t.Cleanup(func() {
   145  			err := ps.Close()
   146  			require.NoError(t, err)
   147  		})
   148  		testMemCachedStorePersist(t, ps)
   149  	})
   150  }
   151  
   152  func TestCachedGetFromPersistent(t *testing.T) {
   153  	key := []byte("key")
   154  	value := []byte("value")
   155  	ps := NewMemoryStore()
   156  	ts := NewMemCachedStore(ps)
   157  
   158  	assert.NoError(t, ps.PutChangeSet(map[string][]byte{string(key): value}, nil))
   159  	val, err := ts.Get(key)
   160  	assert.Nil(t, err)
   161  	assert.Equal(t, value, val)
   162  	ts.Delete(key)
   163  	val, err = ts.Get(key)
   164  	assert.Equal(t, err, ErrKeyNotFound)
   165  	assert.Nil(t, val)
   166  }
   167  
   168  func TestCachedSeek(t *testing.T) {
   169  	var (
   170  		// Given this prefix...
   171  		goodPrefix = []byte{'f'}
   172  		// these pairs should be found...
   173  		lowerKVs = []KeyValue{
   174  			{[]byte("foo"), []byte("bar")},
   175  			{[]byte("faa"), []byte("bra")},
   176  		}
   177  		// and these should be not.
   178  		deletedKVs = []KeyValue{
   179  			{[]byte("fee"), []byte("pow")},
   180  			{[]byte("fii"), []byte("qaz")},
   181  		}
   182  		// and these should be not.
   183  		updatedKVs = []KeyValue{
   184  			{[]byte("fuu"), []byte("wop")},
   185  			{[]byte("fyy"), []byte("zaq")},
   186  		}
   187  		ps = NewMemoryStore()
   188  		ts = NewMemCachedStore(ps)
   189  	)
   190  	for _, v := range lowerKVs {
   191  		require.NoError(t, ps.PutChangeSet(map[string][]byte{string(v.Key): v.Value}, nil))
   192  	}
   193  	for _, v := range deletedKVs {
   194  		require.NoError(t, ps.PutChangeSet(map[string][]byte{string(v.Key): v.Value}, nil))
   195  		ts.Delete(v.Key)
   196  	}
   197  	for _, v := range updatedKVs {
   198  		require.NoError(t, ps.PutChangeSet(map[string][]byte{string(v.Key): v.Value}, nil))
   199  		ts.Put(v.Key, v.Value)
   200  	}
   201  	foundKVs := make(map[string][]byte)
   202  	ts.Seek(SeekRange{Prefix: goodPrefix}, func(k, v []byte) bool {
   203  		foundKVs[string(k)] = v
   204  		return true
   205  	})
   206  	assert.Equal(t, len(foundKVs), len(lowerKVs)+len(updatedKVs))
   207  	for _, kv := range lowerKVs {
   208  		assert.Equal(t, kv.Value, foundKVs[string(kv.Key)])
   209  	}
   210  	for _, kv := range deletedKVs {
   211  		_, ok := foundKVs[string(kv.Key)]
   212  		assert.Equal(t, false, ok)
   213  	}
   214  	for _, kv := range updatedKVs {
   215  		assert.Equal(t, kv.Value, foundKVs[string(kv.Key)])
   216  	}
   217  }
   218  
   219  func benchmarkCachedSeek(t *testing.B, ps Store, psElementsCount, tsElementsCount int) {
   220  	var (
   221  		searchPrefix      = []byte{1}
   222  		badPrefix         = []byte{2}
   223  		lowerPrefixGood   = append(searchPrefix, 1)
   224  		lowerPrefixBad    = append(badPrefix, 1)
   225  		deletedPrefixGood = append(searchPrefix, 2)
   226  		deletedPrefixBad  = append(badPrefix, 2)
   227  		updatedPrefixGood = append(searchPrefix, 3)
   228  		updatedPrefixBad  = append(badPrefix, 3)
   229  
   230  		ts = NewMemCachedStore(ps)
   231  	)
   232  	for i := 0; i < psElementsCount; i++ {
   233  		// lower KVs with matching prefix that should be found
   234  		ts.Put(append(lowerPrefixGood, random.Bytes(10)...), []byte("value"))
   235  		// lower KVs with non-matching prefix that shouldn't be found
   236  		ts.Put(append(lowerPrefixBad, random.Bytes(10)...), []byte("value"))
   237  
   238  		// deleted KVs with matching prefix that shouldn't be found
   239  		key := append(deletedPrefixGood, random.Bytes(10)...)
   240  		ts.Put(key, []byte("deleted"))
   241  		if i < tsElementsCount {
   242  			ts.Delete(key)
   243  		}
   244  		// deleted KVs with non-matching prefix that shouldn't be found
   245  		key = append(deletedPrefixBad, random.Bytes(10)...)
   246  		ts.Put(key, []byte("deleted"))
   247  		if i < tsElementsCount {
   248  			ts.Delete(key)
   249  		}
   250  
   251  		// updated KVs with matching prefix that should be found
   252  		key = append(updatedPrefixGood, random.Bytes(10)...)
   253  		ts.Put(key, []byte("stub"))
   254  		if i < tsElementsCount {
   255  			ts.Put(key, []byte("updated"))
   256  		}
   257  		// updated KVs with non-matching prefix that shouldn't be found
   258  		key = append(updatedPrefixBad, random.Bytes(10)...)
   259  		ts.Put(key, []byte("stub"))
   260  		if i < tsElementsCount {
   261  			ts.Put(key, []byte("updated"))
   262  		}
   263  	}
   264  	_, err := ts.PersistSync()
   265  	require.NoError(t, err)
   266  
   267  	t.ReportAllocs()
   268  	t.ResetTimer()
   269  	for n := 0; n < t.N; n++ {
   270  		ts.Seek(SeekRange{Prefix: searchPrefix}, func(k, v []byte) bool { return true })
   271  	}
   272  	t.StopTimer()
   273  }
   274  
   275  func BenchmarkCachedSeek(t *testing.B) {
   276  	var stores = map[string]func(testing.TB) Store{
   277  		"MemPS": func(t testing.TB) Store {
   278  			return NewMemoryStore()
   279  		},
   280  		"BoltPS":  newBoltStoreForTesting,
   281  		"LevelPS": newLevelDBForTesting,
   282  	}
   283  	for psName, newPS := range stores {
   284  		for psCount := 100; psCount <= 10000; psCount *= 10 {
   285  			for tsCount := 10; tsCount <= psCount; tsCount *= 10 {
   286  				t.Run(fmt.Sprintf("%s_%dTSItems_%dPSItems", psName, tsCount, psCount), func(t *testing.B) {
   287  					ps := newPS(t)
   288  					benchmarkCachedSeek(t, ps, psCount, tsCount)
   289  					ps.Close()
   290  				})
   291  			}
   292  		}
   293  	}
   294  }
   295  
   296  func newMemCachedStoreForTesting(t testing.TB) Store {
   297  	return NewMemCachedStore(NewMemoryStore())
   298  }
   299  
   300  type BadStore struct {
   301  	onPutBatch func()
   302  }
   303  
   304  func (b *BadStore) Delete(k []byte) error {
   305  	return nil
   306  }
   307  func (b *BadStore) Get([]byte) ([]byte, error) {
   308  	return nil, ErrKeyNotFound
   309  }
   310  func (b *BadStore) Put(k, v []byte) error {
   311  	return nil
   312  }
   313  func (b *BadStore) PutChangeSet(_ map[string][]byte, _ map[string][]byte) error {
   314  	b.onPutBatch()
   315  	return ErrKeyNotFound
   316  }
   317  func (b *BadStore) Seek(rng SeekRange, f func(k, v []byte) bool) {
   318  }
   319  func (b *BadStore) SeekGC(rng SeekRange, keep func(k, v []byte) bool) error {
   320  	return nil
   321  }
   322  func (b *BadStore) Close() error {
   323  	return nil
   324  }
   325  
   326  func TestMemCachedPersistFailing(t *testing.T) {
   327  	var (
   328  		bs BadStore
   329  		t1 = []byte("t1")
   330  		t2 = []byte("t2")
   331  		b1 = []byte("b1")
   332  	)
   333  	// cached Store
   334  	ts := NewMemCachedStore(&bs)
   335  	// Set a pair of keys.
   336  	ts.Put(t1, t1)
   337  	ts.Put(t2, t2)
   338  	// This will be called during Persist().
   339  	bs.onPutBatch = func() {
   340  		// Drop one, add one.
   341  		ts.Put(b1, b1)
   342  		ts.Delete(t1)
   343  	}
   344  	_, err := ts.Persist()
   345  	require.Error(t, err)
   346  	// PutBatch() failed in Persist, but we still should have proper state.
   347  	_, err = ts.Get(t1)
   348  	require.Error(t, err)
   349  	res, err := ts.Get(t2)
   350  	require.NoError(t, err)
   351  	require.Equal(t, t2, res)
   352  	res, err = ts.Get(b1)
   353  	require.NoError(t, err)
   354  	require.Equal(t, b1, res)
   355  }
   356  
   357  func TestPrivateMemCachedPersistFailing(t *testing.T) {
   358  	var (
   359  		bs BadStore
   360  		t1 = []byte("t1")
   361  		t2 = []byte("t2")
   362  	)
   363  	// cached Store
   364  	ts := NewPrivateMemCachedStore(&bs)
   365  	// Set a pair of keys.
   366  	ts.Put(t1, t1)
   367  	ts.Put(t2, t2)
   368  	// This will be called during Persist().
   369  	bs.onPutBatch = func() {}
   370  
   371  	_, err := ts.Persist()
   372  	require.Error(t, err)
   373  	// PutBatch() failed in Persist, but we still should have proper state.
   374  	res, err := ts.Get(t1)
   375  	require.NoError(t, err)
   376  	require.Equal(t, t1, res)
   377  	res, err = ts.Get(t2)
   378  	require.NoError(t, err)
   379  	require.Equal(t, t2, res)
   380  }
   381  
   382  func TestCachedSeekSorting(t *testing.T) {
   383  	var (
   384  		// Given this prefix...
   385  		goodPrefix = []byte{1}
   386  		// these pairs should be found...
   387  		lowerKVs = []KeyValue{
   388  			{[]byte{1, 2, 3}, []byte("bra")},
   389  			{[]byte{1, 2, 5}, []byte("bar")},
   390  			{[]byte{1, 3, 3}, []byte("bra")},
   391  			{[]byte{1, 3, 5}, []byte("bra")},
   392  		}
   393  		// and these should be not.
   394  		deletedKVs = []KeyValue{
   395  			{[]byte{1, 7, 3}, []byte("pow")},
   396  			{[]byte{1, 7, 4}, []byte("qaz")},
   397  		}
   398  		// and these should be not.
   399  		updatedKVs = []KeyValue{
   400  			{[]byte{1, 2, 4}, []byte("zaq")},
   401  			{[]byte{1, 2, 6}, []byte("zaq")},
   402  			{[]byte{1, 3, 2}, []byte("wop")},
   403  			{[]byte{1, 3, 4}, []byte("zaq")},
   404  		}
   405  	)
   406  	for _, newCached := range []func(Store) *MemCachedStore{NewMemCachedStore, NewPrivateMemCachedStore} {
   407  		ps := NewMemoryStore()
   408  		ts := newCached(ps)
   409  		for _, v := range lowerKVs {
   410  			require.NoError(t, ps.PutChangeSet(map[string][]byte{string(v.Key): v.Value}, nil))
   411  		}
   412  		for _, v := range deletedKVs {
   413  			require.NoError(t, ps.PutChangeSet(map[string][]byte{string(v.Key): v.Value}, nil))
   414  			ts.Delete(v.Key)
   415  		}
   416  		for _, v := range updatedKVs {
   417  			require.NoError(t, ps.PutChangeSet(map[string][]byte{string(v.Key): v.Value}, nil))
   418  			ts.Put(v.Key, v.Value)
   419  		}
   420  		var foundKVs []KeyValue
   421  		ts.Seek(SeekRange{Prefix: goodPrefix}, func(k, v []byte) bool {
   422  			foundKVs = append(foundKVs, KeyValue{Key: bytes.Clone(k), Value: bytes.Clone(v)})
   423  			return true
   424  		})
   425  		assert.Equal(t, len(foundKVs), len(lowerKVs)+len(updatedKVs))
   426  		expected := append(lowerKVs, updatedKVs...)
   427  		sort.Slice(expected, func(i, j int) bool {
   428  			return bytes.Compare(expected[i].Key, expected[j].Key) < 0
   429  		})
   430  		require.Equal(t, expected, foundKVs)
   431  	}
   432  }