github.com/tacshi/go-ethereum@v0.0.0-20230616113857-84a434e20921/core/state/snapshot/snapshot_test.go (about) 1 // Copyright 2017 The go-ethereum Authors 2 // This file is part of the go-ethereum library. 3 // 4 // The go-ethereum library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The go-ethereum library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 16 17 package snapshot 18 19 import ( 20 crand "crypto/rand" 21 "encoding/binary" 22 "fmt" 23 "math/big" 24 "math/rand" 25 "testing" 26 "time" 27 28 "github.com/VictoriaMetrics/fastcache" 29 "github.com/tacshi/go-ethereum/common" 30 "github.com/tacshi/go-ethereum/core/rawdb" 31 "github.com/tacshi/go-ethereum/core/types" 32 "github.com/tacshi/go-ethereum/rlp" 33 ) 34 35 // randomHash generates a random blob of data and returns it as a hash. 36 func randomHash() common.Hash { 37 var hash common.Hash 38 if n, err := crand.Read(hash[:]); n != common.HashLength || err != nil { 39 panic(err) 40 } 41 return hash 42 } 43 44 // randomAccount generates a random account and returns it RLP encoded. 45 func randomAccount() []byte { 46 root := randomHash() 47 a := Account{ 48 Balance: big.NewInt(rand.Int63()), 49 Nonce: rand.Uint64(), 50 Root: root[:], 51 CodeHash: types.EmptyCodeHash[:], 52 } 53 data, _ := rlp.EncodeToBytes(a) 54 return data 55 } 56 57 // randomAccountSet generates a set of random accounts with the given strings as 58 // the account address hashes. 59 func randomAccountSet(hashes ...string) map[common.Hash][]byte { 60 accounts := make(map[common.Hash][]byte) 61 for _, hash := range hashes { 62 accounts[common.HexToHash(hash)] = randomAccount() 63 } 64 return accounts 65 } 66 67 // randomStorageSet generates a set of random slots with the given strings as 68 // the slot addresses. 69 func randomStorageSet(accounts []string, hashes [][]string, nilStorage [][]string) map[common.Hash]map[common.Hash][]byte { 70 storages := make(map[common.Hash]map[common.Hash][]byte) 71 for index, account := range accounts { 72 storages[common.HexToHash(account)] = make(map[common.Hash][]byte) 73 74 if index < len(hashes) { 75 hashes := hashes[index] 76 for _, hash := range hashes { 77 storages[common.HexToHash(account)][common.HexToHash(hash)] = randomHash().Bytes() 78 } 79 } 80 if index < len(nilStorage) { 81 nils := nilStorage[index] 82 for _, hash := range nils { 83 storages[common.HexToHash(account)][common.HexToHash(hash)] = nil 84 } 85 } 86 } 87 return storages 88 } 89 90 // Tests that if a disk layer becomes stale, no active external references will 91 // be returned with junk data. This version of the test flattens every diff layer 92 // to check internal corner case around the bottom-most memory accumulator. 93 func TestDiskLayerExternalInvalidationFullFlatten(t *testing.T) { 94 // Create an empty base layer and a snapshot tree out of it 95 base := &diskLayer{ 96 diskdb: rawdb.NewMemoryDatabase(), 97 root: common.HexToHash("0x01"), 98 cache: fastcache.New(1024 * 500), 99 } 100 snaps := &Tree{ 101 layers: map[common.Hash]snapshot{ 102 base.root: base, 103 }, 104 } 105 // Retrieve a reference to the base and commit a diff on top 106 ref := snaps.Snapshot(base.root) 107 108 accounts := map[common.Hash][]byte{ 109 common.HexToHash("0xa1"): randomAccount(), 110 } 111 if err := snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, accounts, nil); err != nil { 112 t.Fatalf("failed to create a diff layer: %v", err) 113 } 114 if n := len(snaps.layers); n != 2 { 115 t.Errorf("pre-cap layer count mismatch: have %d, want %d", n, 2) 116 } 117 // Commit the diff layer onto the disk and ensure it's persisted 118 if err := snaps.Cap(common.HexToHash("0x02"), 0); err != nil { 119 t.Fatalf("failed to merge diff layer onto disk: %v", err) 120 } 121 // Since the base layer was modified, ensure that data retrieval on the external reference fail 122 if acc, err := ref.Account(common.HexToHash("0x01")); err != ErrSnapshotStale { 123 t.Errorf("stale reference returned account: %#x (err: %v)", acc, err) 124 } 125 if slot, err := ref.Storage(common.HexToHash("0xa1"), common.HexToHash("0xb1")); err != ErrSnapshotStale { 126 t.Errorf("stale reference returned storage slot: %#x (err: %v)", slot, err) 127 } 128 if n := len(snaps.layers); n != 1 { 129 t.Errorf("post-cap layer count mismatch: have %d, want %d", n, 1) 130 fmt.Println(snaps.layers) 131 } 132 } 133 134 // Tests that if a disk layer becomes stale, no active external references will 135 // be returned with junk data. This version of the test retains the bottom diff 136 // layer to check the usual mode of operation where the accumulator is retained. 137 func TestDiskLayerExternalInvalidationPartialFlatten(t *testing.T) { 138 // Create an empty base layer and a snapshot tree out of it 139 base := &diskLayer{ 140 diskdb: rawdb.NewMemoryDatabase(), 141 root: common.HexToHash("0x01"), 142 cache: fastcache.New(1024 * 500), 143 } 144 snaps := &Tree{ 145 layers: map[common.Hash]snapshot{ 146 base.root: base, 147 }, 148 } 149 // Retrieve a reference to the base and commit two diffs on top 150 ref := snaps.Snapshot(base.root) 151 152 accounts := map[common.Hash][]byte{ 153 common.HexToHash("0xa1"): randomAccount(), 154 } 155 if err := snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, accounts, nil); err != nil { 156 t.Fatalf("failed to create a diff layer: %v", err) 157 } 158 if err := snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, accounts, nil); err != nil { 159 t.Fatalf("failed to create a diff layer: %v", err) 160 } 161 if n := len(snaps.layers); n != 3 { 162 t.Errorf("pre-cap layer count mismatch: have %d, want %d", n, 3) 163 } 164 // Commit the diff layer onto the disk and ensure it's persisted 165 defer func(memcap uint64) { aggregatorMemoryLimit = memcap }(aggregatorMemoryLimit) 166 aggregatorMemoryLimit = 0 167 168 if err := snaps.Cap(common.HexToHash("0x03"), 1); err != nil { 169 t.Fatalf("failed to merge accumulator onto disk: %v", err) 170 } 171 // Since the base layer was modified, ensure that data retrievals on the external reference fail 172 if acc, err := ref.Account(common.HexToHash("0x01")); err != ErrSnapshotStale { 173 t.Errorf("stale reference returned account: %#x (err: %v)", acc, err) 174 } 175 if slot, err := ref.Storage(common.HexToHash("0xa1"), common.HexToHash("0xb1")); err != ErrSnapshotStale { 176 t.Errorf("stale reference returned storage slot: %#x (err: %v)", slot, err) 177 } 178 if n := len(snaps.layers); n != 2 { 179 t.Errorf("post-cap layer count mismatch: have %d, want %d", n, 2) 180 fmt.Println(snaps.layers) 181 } 182 } 183 184 // Tests that if a diff layer becomes stale, no active external references will 185 // be returned with junk data. This version of the test retains the bottom diff 186 // layer to check the usual mode of operation where the accumulator is retained. 187 func TestDiffLayerExternalInvalidationPartialFlatten(t *testing.T) { 188 // Create an empty base layer and a snapshot tree out of it 189 base := &diskLayer{ 190 diskdb: rawdb.NewMemoryDatabase(), 191 root: common.HexToHash("0x01"), 192 cache: fastcache.New(1024 * 500), 193 } 194 snaps := &Tree{ 195 layers: map[common.Hash]snapshot{ 196 base.root: base, 197 }, 198 } 199 // Commit three diffs on top and retrieve a reference to the bottommost 200 accounts := map[common.Hash][]byte{ 201 common.HexToHash("0xa1"): randomAccount(), 202 } 203 if err := snaps.Update(common.HexToHash("0x02"), common.HexToHash("0x01"), nil, accounts, nil); err != nil { 204 t.Fatalf("failed to create a diff layer: %v", err) 205 } 206 if err := snaps.Update(common.HexToHash("0x03"), common.HexToHash("0x02"), nil, accounts, nil); err != nil { 207 t.Fatalf("failed to create a diff layer: %v", err) 208 } 209 if err := snaps.Update(common.HexToHash("0x04"), common.HexToHash("0x03"), nil, accounts, nil); err != nil { 210 t.Fatalf("failed to create a diff layer: %v", err) 211 } 212 if n := len(snaps.layers); n != 4 { 213 t.Errorf("pre-cap layer count mismatch: have %d, want %d", n, 4) 214 } 215 ref := snaps.Snapshot(common.HexToHash("0x02")) 216 217 // Doing a Cap operation with many allowed layers should be a no-op 218 exp := len(snaps.layers) 219 if err := snaps.Cap(common.HexToHash("0x04"), 2000); err != nil { 220 t.Fatalf("failed to flatten diff layer into accumulator: %v", err) 221 } 222 if got := len(snaps.layers); got != exp { 223 t.Errorf("layers modified, got %d exp %d", got, exp) 224 } 225 // Flatten the diff layer into the bottom accumulator 226 if err := snaps.Cap(common.HexToHash("0x04"), 1); err != nil { 227 t.Fatalf("failed to flatten diff layer into accumulator: %v", err) 228 } 229 // Since the accumulator diff layer was modified, ensure that data retrievals on the external reference fail 230 if acc, err := ref.Account(common.HexToHash("0x01")); err != ErrSnapshotStale { 231 t.Errorf("stale reference returned account: %#x (err: %v)", acc, err) 232 } 233 if slot, err := ref.Storage(common.HexToHash("0xa1"), common.HexToHash("0xb1")); err != ErrSnapshotStale { 234 t.Errorf("stale reference returned storage slot: %#x (err: %v)", slot, err) 235 } 236 if n := len(snaps.layers); n != 3 { 237 t.Errorf("post-cap layer count mismatch: have %d, want %d", n, 3) 238 fmt.Println(snaps.layers) 239 } 240 } 241 242 // TestPostCapBasicDataAccess tests some functionality regarding capping/flattening. 243 func TestPostCapBasicDataAccess(t *testing.T) { 244 t.Skip("Arbitrum: Test fails due to our modifications to Tree.Cap function.") 245 // setAccount is a helper to construct a random account entry and assign it to 246 // an account slot in a snapshot 247 setAccount := func(accKey string) map[common.Hash][]byte { 248 return map[common.Hash][]byte{ 249 common.HexToHash(accKey): randomAccount(), 250 } 251 } 252 // Create a starting base layer and a snapshot tree out of it 253 base := &diskLayer{ 254 diskdb: rawdb.NewMemoryDatabase(), 255 root: common.HexToHash("0x01"), 256 cache: fastcache.New(1024 * 500), 257 } 258 snaps := &Tree{ 259 layers: map[common.Hash]snapshot{ 260 base.root: base, 261 }, 262 } 263 // The lowest difflayer 264 snaps.Update(common.HexToHash("0xa1"), common.HexToHash("0x01"), nil, setAccount("0xa1"), nil) 265 snaps.Update(common.HexToHash("0xa2"), common.HexToHash("0xa1"), nil, setAccount("0xa2"), nil) 266 snaps.Update(common.HexToHash("0xb2"), common.HexToHash("0xa1"), nil, setAccount("0xb2"), nil) 267 268 snaps.Update(common.HexToHash("0xa3"), common.HexToHash("0xa2"), nil, setAccount("0xa3"), nil) 269 snaps.Update(common.HexToHash("0xb3"), common.HexToHash("0xb2"), nil, setAccount("0xb3"), nil) 270 271 // checkExist verifies if an account exists in a snapshot 272 checkExist := func(layer *diffLayer, key string) error { 273 if data, _ := layer.Account(common.HexToHash(key)); data == nil { 274 return fmt.Errorf("expected %x to exist, got nil", common.HexToHash(key)) 275 } 276 return nil 277 } 278 // shouldErr checks that an account access errors as expected 279 shouldErr := func(layer *diffLayer, key string) error { 280 if data, err := layer.Account(common.HexToHash(key)); err == nil { 281 return fmt.Errorf("expected error, got data %x", data) 282 } 283 return nil 284 } 285 // check basics 286 snap := snaps.Snapshot(common.HexToHash("0xb3")).(*diffLayer) 287 288 if err := checkExist(snap, "0xa1"); err != nil { 289 t.Error(err) 290 } 291 if err := checkExist(snap, "0xb2"); err != nil { 292 t.Error(err) 293 } 294 if err := checkExist(snap, "0xb3"); err != nil { 295 t.Error(err) 296 } 297 // Cap to a bad root should fail 298 if err := snaps.Cap(common.HexToHash("0x1337"), 0); err == nil { 299 t.Errorf("expected error, got none") 300 } 301 // Now, merge the a-chain 302 snaps.Cap(common.HexToHash("0xa3"), 0) 303 304 // At this point, a2 got merged into a1. Thus, a1 is now modified, and as a1 is 305 // the parent of b2, b2 should no longer be able to iterate into parent. 306 307 // These should still be accessible 308 if err := checkExist(snap, "0xb2"); err != nil { 309 t.Error(err) 310 } 311 if err := checkExist(snap, "0xb3"); err != nil { 312 t.Error(err) 313 } 314 // But these would need iteration into the modified parent 315 if err := shouldErr(snap, "0xa1"); err != nil { 316 t.Error(err) 317 } 318 if err := shouldErr(snap, "0xa2"); err != nil { 319 t.Error(err) 320 } 321 if err := shouldErr(snap, "0xa3"); err != nil { 322 t.Error(err) 323 } 324 // Now, merge it again, just for fun. It should now error, since a3 325 // is a disk layer 326 if err := snaps.Cap(common.HexToHash("0xa3"), 0); err == nil { 327 t.Error("expected error capping the disk layer, got none") 328 } 329 } 330 331 // TestSnaphots tests the functionality for retrieving the snapshot 332 // with given head root and the desired depth. 333 func TestSnaphots(t *testing.T) { 334 // setAccount is a helper to construct a random account entry and assign it to 335 // an account slot in a snapshot 336 setAccount := func(accKey string) map[common.Hash][]byte { 337 return map[common.Hash][]byte{ 338 common.HexToHash(accKey): randomAccount(), 339 } 340 } 341 makeRoot := func(height uint64) common.Hash { 342 var buffer [8]byte 343 binary.BigEndian.PutUint64(buffer[:], height) 344 return common.BytesToHash(buffer[:]) 345 } 346 // Create a starting base layer and a snapshot tree out of it 347 base := &diskLayer{ 348 diskdb: rawdb.NewMemoryDatabase(), 349 root: makeRoot(1), 350 cache: fastcache.New(1024 * 500), 351 } 352 snaps := &Tree{ 353 layers: map[common.Hash]snapshot{ 354 base.root: base, 355 }, 356 } 357 // Construct the snapshots with 129 layers, flattening whatever's above that 358 var ( 359 last = common.HexToHash("0x01") 360 head common.Hash 361 ) 362 for i := 0; i < 129; i++ { 363 head = makeRoot(uint64(i + 2)) 364 snaps.Update(head, last, nil, setAccount(fmt.Sprintf("%d", i+2)), nil) 365 last = head 366 snaps.Cap(head, 128) // 130 layers (128 diffs + 1 accumulator + 1 disk) 367 } 368 var cases = []struct { 369 headRoot common.Hash 370 limit int 371 nodisk bool 372 expected int 373 expectBottom common.Hash 374 }{ 375 {head, 0, false, 0, common.Hash{}}, 376 {head, 64, false, 64, makeRoot(129 + 2 - 64)}, 377 {head, 128, false, 128, makeRoot(3)}, // Normal diff layers, no accumulator 378 {head, 129, true, 129, makeRoot(2)}, // All diff layers, including accumulator 379 {head, 130, false, 130, makeRoot(1)}, // All diff layers + disk layer 380 } 381 for i, c := range cases { 382 layers := snaps.Snapshots(c.headRoot, c.limit, c.nodisk) 383 if len(layers) != c.expected { 384 t.Errorf("non-overflow test %d: returned snapshot layers are mismatched, want %v, got %v", i, c.expected, len(layers)) 385 } 386 if len(layers) == 0 { 387 continue 388 } 389 bottommost := layers[len(layers)-1] 390 if bottommost.Root() != c.expectBottom { 391 t.Errorf("non-overflow test %d: snapshot mismatch, want %v, get %v", i, c.expectBottom, bottommost.Root()) 392 } 393 } 394 // Above we've tested the normal capping, which leaves the accumulator live. 395 // Test that if the bottommost accumulator diff layer overflows the allowed 396 // memory limit, the snapshot tree gets capped to one less layer. 397 // Commit the diff layer onto the disk and ensure it's persisted 398 defer func(memcap uint64) { aggregatorMemoryLimit = memcap }(aggregatorMemoryLimit) 399 aggregatorMemoryLimit = 0 400 401 snaps.Cap(head, 128) // 129 (128 diffs + 1 overflown accumulator + 1 disk) 402 403 cases = []struct { 404 headRoot common.Hash 405 limit int 406 nodisk bool 407 expected int 408 expectBottom common.Hash 409 }{ 410 {head, 0, false, 0, common.Hash{}}, 411 {head, 64, false, 64, makeRoot(129 + 2 - 64)}, 412 {head, 128, false, 128, makeRoot(3)}, // All diff layers, accumulator was flattened 413 {head, 129, true, 128, makeRoot(3)}, // All diff layers, accumulator was flattened 414 {head, 130, false, 129, makeRoot(2)}, // All diff layers + disk layer 415 } 416 for i, c := range cases { 417 layers := snaps.Snapshots(c.headRoot, c.limit, c.nodisk) 418 if len(layers) != c.expected { 419 t.Errorf("overflow test %d: returned snapshot layers are mismatched, want %v, got %v", i, c.expected, len(layers)) 420 } 421 if len(layers) == 0 { 422 continue 423 } 424 bottommost := layers[len(layers)-1] 425 if bottommost.Root() != c.expectBottom { 426 t.Errorf("overflow test %d: snapshot mismatch, want %v, get %v", i, c.expectBottom, bottommost.Root()) 427 } 428 } 429 } 430 431 // TestReadStateDuringFlattening tests the scenario that, during the 432 // bottom diff layers are merging which tags these as stale, the read 433 // happens via a pre-created top snapshot layer which tries to access 434 // the state in these stale layers. Ensure this read can retrieve the 435 // right state back(block until the flattening is finished) instead of 436 // an unexpected error(snapshot layer is stale). 437 func TestReadStateDuringFlattening(t *testing.T) { 438 // setAccount is a helper to construct a random account entry and assign it to 439 // an account slot in a snapshot 440 setAccount := func(accKey string) map[common.Hash][]byte { 441 return map[common.Hash][]byte{ 442 common.HexToHash(accKey): randomAccount(), 443 } 444 } 445 // Create a starting base layer and a snapshot tree out of it 446 base := &diskLayer{ 447 diskdb: rawdb.NewMemoryDatabase(), 448 root: common.HexToHash("0x01"), 449 cache: fastcache.New(1024 * 500), 450 } 451 snaps := &Tree{ 452 layers: map[common.Hash]snapshot{ 453 base.root: base, 454 }, 455 } 456 // 4 layers in total, 3 diff layers and 1 disk layers 457 snaps.Update(common.HexToHash("0xa1"), common.HexToHash("0x01"), nil, setAccount("0xa1"), nil) 458 snaps.Update(common.HexToHash("0xa2"), common.HexToHash("0xa1"), nil, setAccount("0xa2"), nil) 459 snaps.Update(common.HexToHash("0xa3"), common.HexToHash("0xa2"), nil, setAccount("0xa3"), nil) 460 461 // Obtain the topmost snapshot handler for state accessing 462 snap := snaps.Snapshot(common.HexToHash("0xa3")) 463 464 // Register the testing hook to access the state after flattening 465 var result = make(chan *Account) 466 snaps.onFlatten = func() { 467 // Spin up a thread to read the account from the pre-created 468 // snapshot handler. It's expected to be blocked. 469 go func() { 470 account, _ := snap.Account(common.HexToHash("0xa1")) 471 result <- account 472 }() 473 select { 474 case res := <-result: 475 t.Fatalf("Unexpected return %v", res) 476 case <-time.NewTimer(time.Millisecond * 300).C: 477 } 478 } 479 // Cap the snap tree, which will mark the bottom-most layer as stale. 480 snaps.Cap(common.HexToHash("0xa3"), 1) 481 select { 482 case account := <-result: 483 if account == nil { 484 t.Fatal("Failed to retrieve account") 485 } 486 case <-time.NewTimer(time.Millisecond * 300).C: 487 t.Fatal("Unexpected blocker") 488 } 489 }