github.com/nspcc-dev/neo-go@v0.105.2-0.20240517133400-6be757af3eba/pkg/core/storage/memcached_store_test.go (about) 1 package storage 2 3 import ( 4 "bytes" 5 "fmt" 6 "sort" 7 "testing" 8 9 "github.com/nspcc-dev/neo-go/internal/random" 10 "github.com/stretchr/testify/assert" 11 "github.com/stretchr/testify/require" 12 ) 13 14 func TestMemCachedPutGetDelete(t *testing.T) { 15 ps := NewMemoryStore() 16 s := NewMemCachedStore(ps) 17 key := []byte("foo") 18 value := []byte("bar") 19 20 s.Put(key, value) 21 22 result, err := s.Get(key) 23 assert.Nil(t, err) 24 require.Equal(t, value, result) 25 26 s.Delete(key) 27 28 _, err = s.Get(key) 29 assert.NotNil(t, err) 30 assert.Equal(t, err, ErrKeyNotFound) 31 32 // Double delete. 33 s.Delete(key) 34 35 _, err = s.Get(key) 36 assert.NotNil(t, err) 37 assert.Equal(t, err, ErrKeyNotFound) 38 39 // Nonexistent. 40 key = []byte("sparse") 41 s.Delete(key) 42 43 _, err = s.Get(key) 44 assert.NotNil(t, err) 45 assert.Equal(t, err, ErrKeyNotFound) 46 } 47 48 func testMemCachedStorePersist(t *testing.T, ps Store) { 49 // cached Store 50 ts := NewMemCachedStore(ps) 51 // persisting nothing should do nothing 52 c, err := ts.Persist() 53 assert.Equal(t, nil, err) 54 assert.Equal(t, 0, c) 55 // persisting one key should result in one key in ps and nothing in ts 56 ts.Put([]byte("key"), []byte("value")) 57 checkBatch(t, ts, []KeyValueExists{{KeyValue: KeyValue{Key: []byte("key"), Value: []byte("value")}}}, nil) 58 c, err = ts.Persist() 59 checkBatch(t, ts, nil, nil) 60 assert.Equal(t, nil, err) 61 assert.Equal(t, 1, c) 62 v, err := ps.Get([]byte("key")) 63 assert.Equal(t, nil, err) 64 assert.Equal(t, []byte("value"), v) 65 v, err = ts.MemoryStore.Get([]byte("key")) 66 assert.Equal(t, ErrKeyNotFound, err) 67 assert.Equal(t, []byte(nil), v) 68 // now we overwrite the previous `key` contents and also add `key2`, 69 ts.Put([]byte("key"), []byte("newvalue")) 70 ts.Put([]byte("key2"), []byte("value2")) 71 // this is to check that now key is written into the ps before we do 72 // persist 73 v, err = ps.Get([]byte("key2")) 74 assert.Equal(t, ErrKeyNotFound, err) 75 assert.Equal(t, []byte(nil), v) 76 checkBatch(t, ts, []KeyValueExists{ 77 {KeyValue: KeyValue{Key: []byte("key"), Value: []byte("newvalue")}, Exists: true}, 78 {KeyValue: KeyValue{Key: []byte("key2"), Value: []byte("value2")}}, 79 }, nil) 80 // two keys should be persisted (one overwritten and one new) and 81 // available in the ps 82 c, err = ts.Persist() 83 checkBatch(t, ts, nil, nil) 84 assert.Equal(t, nil, err) 85 assert.Equal(t, 2, c) 86 v, err = ts.MemoryStore.Get([]byte("key")) 87 assert.Equal(t, ErrKeyNotFound, err) 88 assert.Equal(t, []byte(nil), v) 89 v, err = ts.MemoryStore.Get([]byte("key2")) 90 assert.Equal(t, ErrKeyNotFound, err) 91 assert.Equal(t, []byte(nil), v) 92 v, err = ps.Get([]byte("key")) 93 assert.Equal(t, nil, err) 94 assert.Equal(t, []byte("newvalue"), v) 95 v, err = ps.Get([]byte("key2")) 96 assert.Equal(t, nil, err) 97 assert.Equal(t, []byte("value2"), v) 98 checkBatch(t, ts, nil, nil) 99 // we've persisted some values, make sure successive persist is a no-op 100 c, err = ts.Persist() 101 assert.Equal(t, nil, err) 102 assert.Equal(t, 0, c) 103 // test persisting deletions 104 ts.Delete([]byte("key")) 105 checkBatch(t, ts, nil, []KeyValueExists{{KeyValue: KeyValue{Key: []byte("key")}, Exists: true}}) 106 c, err = ts.Persist() 107 checkBatch(t, ts, nil, nil) 108 assert.Equal(t, nil, err) 109 assert.Equal(t, 1, c) 110 v, err = ps.Get([]byte("key")) 111 assert.Equal(t, ErrKeyNotFound, err) 112 assert.Equal(t, []byte(nil), v) 113 v, err = ps.Get([]byte("key2")) 114 assert.Equal(t, nil, err) 115 assert.Equal(t, []byte("value2"), v) 116 } 117 118 func checkBatch(t *testing.T, ts *MemCachedStore, put []KeyValueExists, del []KeyValueExists) { 119 b := ts.GetBatch() 120 assert.Equal(t, len(put), len(b.Put), "wrong number of put elements in a batch") 121 assert.Equal(t, len(del), len(b.Deleted), "wrong number of deleted elements in a batch") 122 123 for i := range put { 124 assert.Contains(t, b.Put, put[i]) 125 } 126 127 for i := range del { 128 assert.Contains(t, b.Deleted, del[i]) 129 } 130 } 131 132 func TestMemCachedPersist(t *testing.T) { 133 t.Run("MemoryStore", func(t *testing.T) { 134 ps := NewMemoryStore() 135 testMemCachedStorePersist(t, ps) 136 }) 137 t.Run("MemoryCachedStore", func(t *testing.T) { 138 ps1 := NewMemoryStore() 139 ps2 := NewMemCachedStore(ps1) 140 testMemCachedStorePersist(t, ps2) 141 }) 142 t.Run("BoltDBStore", func(t *testing.T) { 143 ps := newBoltStoreForTesting(t) 144 t.Cleanup(func() { 145 err := ps.Close() 146 require.NoError(t, err) 147 }) 148 testMemCachedStorePersist(t, ps) 149 }) 150 } 151 152 func TestCachedGetFromPersistent(t *testing.T) { 153 key := []byte("key") 154 value := []byte("value") 155 ps := NewMemoryStore() 156 ts := NewMemCachedStore(ps) 157 158 assert.NoError(t, ps.PutChangeSet(map[string][]byte{string(key): value}, nil)) 159 val, err := ts.Get(key) 160 assert.Nil(t, err) 161 assert.Equal(t, value, val) 162 ts.Delete(key) 163 val, err = ts.Get(key) 164 assert.Equal(t, err, ErrKeyNotFound) 165 assert.Nil(t, val) 166 } 167 168 func TestCachedSeek(t *testing.T) { 169 var ( 170 // Given this prefix... 171 goodPrefix = []byte{'f'} 172 // these pairs should be found... 173 lowerKVs = []KeyValue{ 174 {[]byte("foo"), []byte("bar")}, 175 {[]byte("faa"), []byte("bra")}, 176 } 177 // and these should be not. 178 deletedKVs = []KeyValue{ 179 {[]byte("fee"), []byte("pow")}, 180 {[]byte("fii"), []byte("qaz")}, 181 } 182 // and these should be not. 183 updatedKVs = []KeyValue{ 184 {[]byte("fuu"), []byte("wop")}, 185 {[]byte("fyy"), []byte("zaq")}, 186 } 187 ps = NewMemoryStore() 188 ts = NewMemCachedStore(ps) 189 ) 190 for _, v := range lowerKVs { 191 require.NoError(t, ps.PutChangeSet(map[string][]byte{string(v.Key): v.Value}, nil)) 192 } 193 for _, v := range deletedKVs { 194 require.NoError(t, ps.PutChangeSet(map[string][]byte{string(v.Key): v.Value}, nil)) 195 ts.Delete(v.Key) 196 } 197 for _, v := range updatedKVs { 198 require.NoError(t, ps.PutChangeSet(map[string][]byte{string(v.Key): v.Value}, nil)) 199 ts.Put(v.Key, v.Value) 200 } 201 foundKVs := make(map[string][]byte) 202 ts.Seek(SeekRange{Prefix: goodPrefix}, func(k, v []byte) bool { 203 foundKVs[string(k)] = v 204 return true 205 }) 206 assert.Equal(t, len(foundKVs), len(lowerKVs)+len(updatedKVs)) 207 for _, kv := range lowerKVs { 208 assert.Equal(t, kv.Value, foundKVs[string(kv.Key)]) 209 } 210 for _, kv := range deletedKVs { 211 _, ok := foundKVs[string(kv.Key)] 212 assert.Equal(t, false, ok) 213 } 214 for _, kv := range updatedKVs { 215 assert.Equal(t, kv.Value, foundKVs[string(kv.Key)]) 216 } 217 } 218 219 func benchmarkCachedSeek(t *testing.B, ps Store, psElementsCount, tsElementsCount int) { 220 var ( 221 searchPrefix = []byte{1} 222 badPrefix = []byte{2} 223 lowerPrefixGood = append(searchPrefix, 1) 224 lowerPrefixBad = append(badPrefix, 1) 225 deletedPrefixGood = append(searchPrefix, 2) 226 deletedPrefixBad = append(badPrefix, 2) 227 updatedPrefixGood = append(searchPrefix, 3) 228 updatedPrefixBad = append(badPrefix, 3) 229 230 ts = NewMemCachedStore(ps) 231 ) 232 for i := 0; i < psElementsCount; i++ { 233 // lower KVs with matching prefix that should be found 234 ts.Put(append(lowerPrefixGood, random.Bytes(10)...), []byte("value")) 235 // lower KVs with non-matching prefix that shouldn't be found 236 ts.Put(append(lowerPrefixBad, random.Bytes(10)...), []byte("value")) 237 238 // deleted KVs with matching prefix that shouldn't be found 239 key := append(deletedPrefixGood, random.Bytes(10)...) 240 ts.Put(key, []byte("deleted")) 241 if i < tsElementsCount { 242 ts.Delete(key) 243 } 244 // deleted KVs with non-matching prefix that shouldn't be found 245 key = append(deletedPrefixBad, random.Bytes(10)...) 246 ts.Put(key, []byte("deleted")) 247 if i < tsElementsCount { 248 ts.Delete(key) 249 } 250 251 // updated KVs with matching prefix that should be found 252 key = append(updatedPrefixGood, random.Bytes(10)...) 253 ts.Put(key, []byte("stub")) 254 if i < tsElementsCount { 255 ts.Put(key, []byte("updated")) 256 } 257 // updated KVs with non-matching prefix that shouldn't be found 258 key = append(updatedPrefixBad, random.Bytes(10)...) 259 ts.Put(key, []byte("stub")) 260 if i < tsElementsCount { 261 ts.Put(key, []byte("updated")) 262 } 263 } 264 _, err := ts.PersistSync() 265 require.NoError(t, err) 266 267 t.ReportAllocs() 268 t.ResetTimer() 269 for n := 0; n < t.N; n++ { 270 ts.Seek(SeekRange{Prefix: searchPrefix}, func(k, v []byte) bool { return true }) 271 } 272 t.StopTimer() 273 } 274 275 func BenchmarkCachedSeek(t *testing.B) { 276 var stores = map[string]func(testing.TB) Store{ 277 "MemPS": func(t testing.TB) Store { 278 return NewMemoryStore() 279 }, 280 "BoltPS": newBoltStoreForTesting, 281 "LevelPS": newLevelDBForTesting, 282 } 283 for psName, newPS := range stores { 284 for psCount := 100; psCount <= 10000; psCount *= 10 { 285 for tsCount := 10; tsCount <= psCount; tsCount *= 10 { 286 t.Run(fmt.Sprintf("%s_%dTSItems_%dPSItems", psName, tsCount, psCount), func(t *testing.B) { 287 ps := newPS(t) 288 benchmarkCachedSeek(t, ps, psCount, tsCount) 289 ps.Close() 290 }) 291 } 292 } 293 } 294 } 295 296 func newMemCachedStoreForTesting(t testing.TB) Store { 297 return NewMemCachedStore(NewMemoryStore()) 298 } 299 300 type BadStore struct { 301 onPutBatch func() 302 } 303 304 func (b *BadStore) Delete(k []byte) error { 305 return nil 306 } 307 func (b *BadStore) Get([]byte) ([]byte, error) { 308 return nil, ErrKeyNotFound 309 } 310 func (b *BadStore) Put(k, v []byte) error { 311 return nil 312 } 313 func (b *BadStore) PutChangeSet(_ map[string][]byte, _ map[string][]byte) error { 314 b.onPutBatch() 315 return ErrKeyNotFound 316 } 317 func (b *BadStore) Seek(rng SeekRange, f func(k, v []byte) bool) { 318 } 319 func (b *BadStore) SeekGC(rng SeekRange, keep func(k, v []byte) bool) error { 320 return nil 321 } 322 func (b *BadStore) Close() error { 323 return nil 324 } 325 326 func TestMemCachedPersistFailing(t *testing.T) { 327 var ( 328 bs BadStore 329 t1 = []byte("t1") 330 t2 = []byte("t2") 331 b1 = []byte("b1") 332 ) 333 // cached Store 334 ts := NewMemCachedStore(&bs) 335 // Set a pair of keys. 336 ts.Put(t1, t1) 337 ts.Put(t2, t2) 338 // This will be called during Persist(). 339 bs.onPutBatch = func() { 340 // Drop one, add one. 341 ts.Put(b1, b1) 342 ts.Delete(t1) 343 } 344 _, err := ts.Persist() 345 require.Error(t, err) 346 // PutBatch() failed in Persist, but we still should have proper state. 347 _, err = ts.Get(t1) 348 require.Error(t, err) 349 res, err := ts.Get(t2) 350 require.NoError(t, err) 351 require.Equal(t, t2, res) 352 res, err = ts.Get(b1) 353 require.NoError(t, err) 354 require.Equal(t, b1, res) 355 } 356 357 func TestPrivateMemCachedPersistFailing(t *testing.T) { 358 var ( 359 bs BadStore 360 t1 = []byte("t1") 361 t2 = []byte("t2") 362 ) 363 // cached Store 364 ts := NewPrivateMemCachedStore(&bs) 365 // Set a pair of keys. 366 ts.Put(t1, t1) 367 ts.Put(t2, t2) 368 // This will be called during Persist(). 369 bs.onPutBatch = func() {} 370 371 _, err := ts.Persist() 372 require.Error(t, err) 373 // PutBatch() failed in Persist, but we still should have proper state. 374 res, err := ts.Get(t1) 375 require.NoError(t, err) 376 require.Equal(t, t1, res) 377 res, err = ts.Get(t2) 378 require.NoError(t, err) 379 require.Equal(t, t2, res) 380 } 381 382 func TestCachedSeekSorting(t *testing.T) { 383 var ( 384 // Given this prefix... 385 goodPrefix = []byte{1} 386 // these pairs should be found... 387 lowerKVs = []KeyValue{ 388 {[]byte{1, 2, 3}, []byte("bra")}, 389 {[]byte{1, 2, 5}, []byte("bar")}, 390 {[]byte{1, 3, 3}, []byte("bra")}, 391 {[]byte{1, 3, 5}, []byte("bra")}, 392 } 393 // and these should be not. 394 deletedKVs = []KeyValue{ 395 {[]byte{1, 7, 3}, []byte("pow")}, 396 {[]byte{1, 7, 4}, []byte("qaz")}, 397 } 398 // and these should be not. 399 updatedKVs = []KeyValue{ 400 {[]byte{1, 2, 4}, []byte("zaq")}, 401 {[]byte{1, 2, 6}, []byte("zaq")}, 402 {[]byte{1, 3, 2}, []byte("wop")}, 403 {[]byte{1, 3, 4}, []byte("zaq")}, 404 } 405 ) 406 for _, newCached := range []func(Store) *MemCachedStore{NewMemCachedStore, NewPrivateMemCachedStore} { 407 ps := NewMemoryStore() 408 ts := newCached(ps) 409 for _, v := range lowerKVs { 410 require.NoError(t, ps.PutChangeSet(map[string][]byte{string(v.Key): v.Value}, nil)) 411 } 412 for _, v := range deletedKVs { 413 require.NoError(t, ps.PutChangeSet(map[string][]byte{string(v.Key): v.Value}, nil)) 414 ts.Delete(v.Key) 415 } 416 for _, v := range updatedKVs { 417 require.NoError(t, ps.PutChangeSet(map[string][]byte{string(v.Key): v.Value}, nil)) 418 ts.Put(v.Key, v.Value) 419 } 420 var foundKVs []KeyValue 421 ts.Seek(SeekRange{Prefix: goodPrefix}, func(k, v []byte) bool { 422 foundKVs = append(foundKVs, KeyValue{Key: bytes.Clone(k), Value: bytes.Clone(v)}) 423 return true 424 }) 425 assert.Equal(t, len(foundKVs), len(lowerKVs)+len(updatedKVs)) 426 expected := append(lowerKVs, updatedKVs...) 427 sort.Slice(expected, func(i, j int) bool { 428 return bytes.Compare(expected[i].Key, expected[j].Key) < 0 429 }) 430 require.Equal(t, expected, foundKVs) 431 } 432 }