github.com/MetalBlockchain/subnet-evm@v0.4.9/core/state/snapshot/generate_test.go (about) 1 // (c) 2019-2020, Ava Labs, Inc. 2 // 3 // This file is a derived work, based on the go-ethereum library whose original 4 // notices appear below. 5 // 6 // It is distributed under a license compatible with the licensing terms of the 7 // original code from which it is derived. 8 // 9 // Much love to the original authors for their work. 10 // ********** 11 // Copyright 2019 The go-ethereum Authors 12 // This file is part of the go-ethereum library. 13 // 14 // The go-ethereum library is free software: you can redistribute it and/or modify 15 // it under the terms of the GNU Lesser General Public License as published by 16 // the Free Software Foundation, either version 3 of the License, or 17 // (at your option) any later version. 18 // 19 // The go-ethereum library is distributed in the hope that it will be useful, 20 // but WITHOUT ANY WARRANTY; without even the implied warranty of 21 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 22 // GNU Lesser General Public License for more details. 23 // 24 // You should have received a copy of the GNU Lesser General Public License 25 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 26 27 package snapshot 28 29 import ( 30 "fmt" 31 "math/big" 32 "os" 33 "testing" 34 "time" 35 36 "github.com/MetalBlockchain/subnet-evm/core/rawdb" 37 "github.com/MetalBlockchain/subnet-evm/ethdb" 38 "github.com/MetalBlockchain/subnet-evm/trie" 39 "github.com/ethereum/go-ethereum/common" 40 "github.com/ethereum/go-ethereum/log" 41 "github.com/ethereum/go-ethereum/rlp" 42 "golang.org/x/crypto/sha3" 43 ) 44 45 var testBlockHash = common.HexToHash("0xdeadbeef") 46 47 func hashData(input []byte) common.Hash { 48 var hasher = sha3.NewLegacyKeccak256() 49 var hash common.Hash 50 hasher.Reset() 51 hasher.Write(input) 52 hasher.Sum(hash[:0]) 53 return hash 54 } 55 56 // Tests that snapshot generation from an empty database. 57 func TestGeneration(t *testing.T) { 58 // We can't use statedb to make a test trie (circular dependency), so make 59 // a fake one manually. We're going with a small account trie of 3 accounts, 60 var helper = newHelper() 61 stRoot := helper.makeStorageTrie(common.Hash{}, []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, false) 62 63 helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) 64 helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) 65 helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: emptyCode.Bytes()}) 66 67 helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) 68 helper.makeStorageTrie(hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) 69 70 root, snap := helper.CommitAndGenerate() // two of which also has the same 3-slot storage trie attached. 71 72 if have, want := root, common.HexToHash("0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd"); have != want { 73 t.Fatalf("have %#x want %#x", have, want) 74 } 75 select { 76 case <-snap.genPending: 77 // Snapshot generation succeeded 78 79 case <-time.After(250 * time.Millisecond): 80 t.Errorf("Snapshot generation failed") 81 } 82 checkSnapRoot(t, snap, root) 83 84 // Signal abortion to the generator and wait for it to tear down 85 stop := make(chan struct{}) 86 snap.genAbort <- stop 87 <-stop 88 } 89 90 // Tests that snapshot generation with existent flat state. 91 func TestGenerateExistentState(t *testing.T) { 92 // We can't use statedb to make a test trie (circular dependency), so make 93 // a fake one manually. We're going with a small account trie of 3 accounts, 94 // two of which also has the same 3-slot storage trie attached. 95 var helper = newHelper() 96 97 stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) 98 helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) 99 helper.addSnapAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) 100 helper.addSnapStorage("acc-1", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) 101 102 helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) 103 helper.addSnapAccount("acc-2", &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) 104 105 stRoot = helper.makeStorageTrie(hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) 106 helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: emptyCode.Bytes()}) 107 helper.addSnapAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: emptyCode.Bytes()}) 108 helper.addSnapStorage("acc-3", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) 109 110 root, snap := helper.CommitAndGenerate() 111 select { 112 case <-snap.genPending: 113 // Snapshot generation succeeded 114 115 case <-time.After(250 * time.Millisecond): 116 t.Errorf("Snapshot generation failed") 117 } 118 checkSnapRoot(t, snap, root) 119 120 // Signal abortion to the generator and wait for it to tear down 121 stop := make(chan struct{}) 122 snap.genAbort <- stop 123 <-stop 124 } 125 126 func checkSnapRoot(t *testing.T, snap *diskLayer, trieRoot common.Hash) { 127 t.Helper() 128 accIt := snap.AccountIterator(common.Hash{}) 129 defer accIt.Release() 130 snapRoot, err := generateTrieRoot(nil, accIt, common.Hash{}, stackTrieGenerate, 131 func(db ethdb.KeyValueWriter, accountHash, codeHash common.Hash, stat *generateStats) (common.Hash, error) { 132 storageIt, _ := snap.StorageIterator(accountHash, common.Hash{}) 133 defer storageIt.Release() 134 135 hash, err := generateTrieRoot(nil, storageIt, accountHash, stackTrieGenerate, nil, stat, false) 136 if err != nil { 137 return common.Hash{}, err 138 } 139 return hash, nil 140 }, newGenerateStats(), true) 141 if err != nil { 142 t.Fatal(err) 143 } 144 if snapRoot != trieRoot { 145 t.Fatalf("snaproot: %#x != trieroot #%x", snapRoot, trieRoot) 146 } 147 if err := CheckDanglingStorage(snap.diskdb); err != nil { 148 t.Fatalf("Detected dangling storages: %v", err) 149 } 150 } 151 152 type testHelper struct { 153 diskdb ethdb.Database 154 triedb *trie.Database 155 accTrie *trie.StateTrie 156 nodes *trie.MergedNodeSet 157 } 158 159 func newHelper() *testHelper { 160 diskdb := rawdb.NewMemoryDatabase() 161 triedb := trie.NewDatabase(diskdb) 162 accTrie, _ := trie.NewStateTrie(common.Hash{}, common.Hash{}, triedb) 163 return &testHelper{ 164 diskdb: diskdb, 165 triedb: triedb, 166 accTrie: accTrie, 167 nodes: trie.NewMergedNodeSet(), 168 } 169 } 170 171 func (t *testHelper) addTrieAccount(acckey string, acc *Account) { 172 val, _ := rlp.EncodeToBytes(acc) 173 t.accTrie.Update([]byte(acckey), val) 174 } 175 176 func (t *testHelper) addSnapAccount(acckey string, acc *Account) { 177 val, _ := rlp.EncodeToBytes(acc) 178 key := hashData([]byte(acckey)) 179 rawdb.WriteAccountSnapshot(t.diskdb, key, val) 180 } 181 182 func (t *testHelper) addAccount(acckey string, acc *Account) { 183 t.addTrieAccount(acckey, acc) 184 t.addSnapAccount(acckey, acc) 185 } 186 187 func (t *testHelper) addSnapStorage(accKey string, keys []string, vals []string) { 188 accHash := hashData([]byte(accKey)) 189 for i, key := range keys { 190 rawdb.WriteStorageSnapshot(t.diskdb, accHash, hashData([]byte(key)), []byte(vals[i])) 191 } 192 } 193 194 func (t *testHelper) makeStorageTrie(owner common.Hash, keys []string, vals []string, commit bool) []byte { 195 stTrie, _ := trie.NewStateTrie(owner, common.Hash{}, t.triedb) 196 for i, k := range keys { 197 stTrie.Update([]byte(k), []byte(vals[i])) 198 } 199 if !commit { 200 return stTrie.Hash().Bytes() 201 } 202 root, nodes, _ := stTrie.Commit(false) 203 if nodes != nil { 204 t.nodes.Merge(nodes) 205 } 206 return root.Bytes() 207 } 208 209 func (t *testHelper) Commit() common.Hash { 210 root, nodes, _ := t.accTrie.Commit(true) 211 if nodes != nil { 212 t.nodes.Merge(nodes) 213 } 214 t.triedb.Update(t.nodes) 215 t.triedb.Commit(root, false, nil) 216 return root 217 } 218 219 func (t *testHelper) CommitAndGenerate() (common.Hash, *diskLayer) { 220 root := t.Commit() 221 snap := generateSnapshot(t.diskdb, t.triedb, 16, testBlockHash, root, nil) 222 return root, snap 223 } 224 225 // Tests that snapshot generation with existent flat state, where the flat state 226 // contains some errors: 227 // - the contract with empty storage root but has storage entries in the disk 228 // - the contract with non empty storage root but empty storage slots 229 // - the contract(non-empty storage) misses some storage slots 230 // - miss in the beginning 231 // - miss in the middle 232 // - miss in the end 233 // - the contract(non-empty storage) has wrong storage slots 234 // - wrong slots in the beginning 235 // - wrong slots in the middle 236 // - wrong slots in the end 237 // - the contract(non-empty storage) has extra storage slots 238 // - extra slots in the beginning 239 // - extra slots in the middle 240 // - extra slots in the end 241 func TestGenerateExistentStateWithWrongStorage(t *testing.T) { 242 helper := newHelper() 243 244 // Account one, empty root but non-empty database 245 helper.addAccount("acc-1", &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) 246 helper.addSnapStorage("acc-1", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) 247 248 // Account two, non empty root but empty database 249 stRoot := helper.makeStorageTrie(hashData([]byte("acc-2")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) 250 helper.addAccount("acc-2", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) 251 252 // Miss slots 253 { 254 // Account three, non empty root but misses slots in the beginning 255 helper.makeStorageTrie(hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) 256 helper.addAccount("acc-3", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) 257 helper.addSnapStorage("acc-3", []string{"key-2", "key-3"}, []string{"val-2", "val-3"}) 258 259 // Account four, non empty root but misses slots in the middle 260 helper.makeStorageTrie(hashData([]byte("acc-4")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) 261 helper.addAccount("acc-4", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) 262 helper.addSnapStorage("acc-4", []string{"key-1", "key-3"}, []string{"val-1", "val-3"}) 263 264 // Account five, non empty root but misses slots in the end 265 helper.makeStorageTrie(hashData([]byte("acc-5")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) 266 helper.addAccount("acc-5", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) 267 helper.addSnapStorage("acc-5", []string{"key-1", "key-2"}, []string{"val-1", "val-2"}) 268 } 269 270 // Wrong storage slots 271 { 272 // Account six, non empty root but wrong slots in the beginning 273 helper.makeStorageTrie(hashData([]byte("acc-6")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) 274 helper.addAccount("acc-6", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) 275 helper.addSnapStorage("acc-6", []string{"key-1", "key-2", "key-3"}, []string{"badval-1", "val-2", "val-3"}) 276 277 // Account seven, non empty root but wrong slots in the middle 278 helper.makeStorageTrie(hashData([]byte("acc-7")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) 279 helper.addAccount("acc-7", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) 280 helper.addSnapStorage("acc-7", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "badval-2", "val-3"}) 281 282 // Account eight, non empty root but wrong slots in the end 283 helper.makeStorageTrie(hashData([]byte("acc-8")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) 284 helper.addAccount("acc-8", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) 285 helper.addSnapStorage("acc-8", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "badval-3"}) 286 287 // Account 9, non empty root but rotated slots 288 helper.makeStorageTrie(hashData([]byte("acc-9")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) 289 helper.addAccount("acc-9", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) 290 helper.addSnapStorage("acc-9", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-3", "val-2"}) 291 } 292 293 // Extra storage slots 294 { 295 // Account 10, non empty root but extra slots in the beginning 296 helper.makeStorageTrie(hashData([]byte("acc-10")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) 297 helper.addAccount("acc-10", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) 298 helper.addSnapStorage("acc-10", []string{"key-0", "key-1", "key-2", "key-3"}, []string{"val-0", "val-1", "val-2", "val-3"}) 299 300 // Account 11, non empty root but extra slots in the middle 301 helper.makeStorageTrie(hashData([]byte("acc-11")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) 302 helper.addAccount("acc-11", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) 303 helper.addSnapStorage("acc-11", []string{"key-1", "key-2", "key-2-1", "key-3"}, []string{"val-1", "val-2", "val-2-1", "val-3"}) 304 305 // Account 12, non empty root but extra slots in the end 306 helper.makeStorageTrie(hashData([]byte("acc-12")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) 307 helper.addAccount("acc-12", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) 308 helper.addSnapStorage("acc-12", []string{"key-1", "key-2", "key-3", "key-4"}, []string{"val-1", "val-2", "val-3", "val-4"}) 309 } 310 311 root, snap := helper.CommitAndGenerate() 312 t.Logf("Root: %#x\n", root) // Root = 0x8746cce9fd9c658b2cfd639878ed6584b7a2b3e73bb40f607fcfa156002429a0 313 314 select { 315 case <-snap.genPending: 316 // Snapshot generation succeeded 317 318 case <-time.After(250 * time.Millisecond): 319 t.Errorf("Snapshot generation failed") 320 } 321 checkSnapRoot(t, snap, root) 322 // Signal abortion to the generator and wait for it to tear down 323 stop := make(chan struct{}) 324 snap.genAbort <- stop 325 <-stop 326 } 327 328 // Tests that snapshot generation with existent flat state, where the flat state 329 // contains some errors: 330 // - miss accounts 331 // - wrong accounts 332 // - extra accounts 333 func TestGenerateExistentStateWithWrongAccounts(t *testing.T) { 334 helper := newHelper() 335 336 helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) 337 helper.makeStorageTrie(hashData([]byte("acc-2")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) 338 helper.makeStorageTrie(hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) 339 helper.makeStorageTrie(hashData([]byte("acc-4")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) 340 stRoot := helper.makeStorageTrie(hashData([]byte("acc-6")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) 341 342 // Trie accounts [acc-1, acc-2, acc-3, acc-4, acc-6] 343 // Extra accounts [acc-0, acc-5, acc-7] 344 345 // Missing accounts, only in the trie 346 { 347 helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) // Beginning 348 helper.addTrieAccount("acc-4", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) // Middle 349 helper.addTrieAccount("acc-6", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) // End 350 } 351 352 // Wrong accounts 353 { 354 helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) 355 helper.addSnapAccount("acc-2", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: common.Hex2Bytes("0x1234")}) 356 357 helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) 358 helper.addSnapAccount("acc-3", &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) 359 } 360 361 // Extra accounts, only in the snap 362 { 363 helper.addSnapAccount("acc-0", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyRoot.Bytes()}) // before the beginning 364 helper.addSnapAccount("acc-5", &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: common.Hex2Bytes("0x1234")}) // Middle 365 helper.addSnapAccount("acc-7", &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyRoot.Bytes()}) // after the end 366 } 367 368 root, snap := helper.CommitAndGenerate() 369 t.Logf("Root: %#x\n", root) // Root = 0x825891472281463511e7ebcc7f109e4f9200c20fa384754e11fd605cd98464e8 370 371 select { 372 case <-snap.genPending: 373 // Snapshot generation succeeded 374 375 case <-time.After(250 * time.Millisecond): 376 t.Errorf("Snapshot generation failed") 377 } 378 checkSnapRoot(t, snap, root) 379 380 // Signal abortion to the generator and wait for it to tear down 381 stop := make(chan struct{}) 382 snap.genAbort <- stop 383 <-stop 384 } 385 386 // Tests that snapshot generation errors out correctly in case of a missing trie 387 // node in the account trie. 388 func TestGenerateCorruptAccountTrie(t *testing.T) { 389 // We can't use statedb to make a test trie (circular dependency), so make 390 // a fake one manually. We're going with a small account trie of 3 accounts, 391 // without any storage slots to keep the test smaller. 392 helper := newHelper() 393 394 helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) // 0xc7a30f39aff471c95d8a837497ad0e49b65be475cc0953540f80cfcdbdcd9074 395 helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 396 helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) // 0x19ead688e907b0fab07176120dceec244a72aff2f0aa51e8b827584e378772f4 397 398 root := helper.Commit() // Root: 0xa04693ea110a31037fb5ee814308a6f1d76bdab0b11676bdf4541d2de55ba978 399 400 // Delete an account trie leaf and ensure the generator chokes 401 helper.triedb.Commit(root, false, nil) 402 helper.diskdb.Delete(common.HexToHash("0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7").Bytes()) 403 404 snap := generateSnapshot(helper.diskdb, helper.triedb, 16, testBlockHash, root, nil) 405 select { 406 case <-snap.genPending: 407 // Snapshot generation succeeded 408 t.Errorf("Snapshot generated against corrupt account trie") 409 410 case <-time.After(250 * time.Millisecond): 411 // Not generated fast enough, hopefully blocked inside on missing trie node fail 412 } 413 // Signal abortion to the generator and wait for it to tear down 414 stop := make(chan struct{}) 415 snap.genAbort <- stop 416 <-stop 417 } 418 419 // Tests that snapshot generation errors out correctly in case of a missing root 420 // trie node for a storage trie. It's similar to internal corruption but it is 421 // handled differently inside the generator. 422 func TestGenerateMissingStorageTrie(t *testing.T) { 423 // We can't use statedb to make a test trie (circular dependency), so make 424 // a fake one manually. We're going with a small account trie of 3 accounts, 425 // two of which also has the same 3-slot storage trie attached. 426 helper := newHelper() 427 428 stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) // 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67 429 helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e 430 helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 431 stRoot = helper.makeStorageTrie(hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) 432 helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: emptyCode.Bytes()}) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2 433 root := helper.Commit() 434 435 // Delete a storage trie root and ensure the generator chokes 436 helper.diskdb.Delete(stRoot) // We can only corrupt the disk database, so flush the tries out 437 438 snap := generateSnapshot(helper.diskdb, helper.triedb, 16, testBlockHash, root, nil) 439 select { 440 case <-snap.genPending: 441 // Snapshot generation succeeded 442 t.Errorf("Snapshot generated against corrupt storage trie") 443 444 case <-time.After(250 * time.Millisecond): 445 // Not generated fast enough, hopefully blocked inside on missing trie node fail 446 } 447 // Signal abortion to the generator and wait for it to tear down 448 stop := make(chan struct{}) 449 snap.genAbort <- stop 450 <-stop 451 } 452 453 // Tests that snapshot generation errors out correctly in case of a missing trie 454 // node in a storage trie. 455 func TestGenerateCorruptStorageTrie(t *testing.T) { 456 // We can't use statedb to make a test trie (circular dependency), so make 457 // a fake one manually. We're going with a small account trie of 3 accounts, 458 // two of which also has the same 3-slot storage trie attached. 459 helper := newHelper() 460 461 stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) // 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67 462 helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) // 0x1314700b81afc49f94db3623ef1df38f3ed18b73a1b7ea2f6c095118cf6118a0 463 helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) // 0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371 464 stRoot = helper.makeStorageTrie(hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) // 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67 465 helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: emptyCode.Bytes()}) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2 466 root := helper.Commit() 467 468 // Delete a storage trie leaf and ensure the generator chokes 469 helper.diskdb.Delete(common.HexToHash("0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371").Bytes()) 470 471 snap := generateSnapshot(helper.diskdb, helper.triedb, 16, testBlockHash, root, nil) 472 select { 473 case <-snap.genPending: 474 // Snapshot generation succeeded 475 t.Errorf("Snapshot generated against corrupt storage trie") 476 477 case <-time.After(250 * time.Millisecond): 478 // Not generated fast enough, hopefully blocked inside on missing trie node fail 479 } 480 // Signal abortion to the generator and wait for it to tear down 481 stop := make(chan struct{}) 482 snap.genAbort <- stop 483 <-stop 484 } 485 486 // Tests that snapshot generation when an extra account with storage exists in the snap state. 487 func TestGenerateWithExtraAccounts(t *testing.T) { 488 helper := newHelper() 489 { 490 // Account one in the trie 491 stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), 492 []string{"key-1", "key-2", "key-3", "key-4", "key-5"}, 493 []string{"val-1", "val-2", "val-3", "val-4", "val-5"}, 494 true, 495 ) 496 acc := &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()} 497 val, _ := rlp.EncodeToBytes(acc) 498 helper.accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e 499 500 // Identical in the snap 501 key := hashData([]byte("acc-1")) 502 rawdb.WriteAccountSnapshot(helper.triedb.DiskDB(), key, val) 503 rawdb.WriteStorageSnapshot(helper.triedb.DiskDB(), key, hashData([]byte("key-1")), []byte("val-1")) 504 rawdb.WriteStorageSnapshot(helper.triedb.DiskDB(), key, hashData([]byte("key-2")), []byte("val-2")) 505 rawdb.WriteStorageSnapshot(helper.triedb.DiskDB(), key, hashData([]byte("key-3")), []byte("val-3")) 506 rawdb.WriteStorageSnapshot(helper.triedb.DiskDB(), key, hashData([]byte("key-4")), []byte("val-4")) 507 rawdb.WriteStorageSnapshot(helper.triedb.DiskDB(), key, hashData([]byte("key-5")), []byte("val-5")) 508 } 509 { 510 // Account two exists only in the snapshot 511 stRoot := helper.makeStorageTrie(hashData([]byte("acc-2")), 512 []string{"key-1", "key-2", "key-3", "key-4", "key-5"}, 513 []string{"val-1", "val-2", "val-3", "val-4", "val-5"}, 514 true, 515 ) 516 acc := &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()} 517 val, _ := rlp.EncodeToBytes(acc) 518 key := hashData([]byte("acc-2")) 519 rawdb.WriteAccountSnapshot(helper.triedb.DiskDB(), key, val) 520 rawdb.WriteStorageSnapshot(helper.triedb.DiskDB(), key, hashData([]byte("b-key-1")), []byte("b-val-1")) 521 rawdb.WriteStorageSnapshot(helper.triedb.DiskDB(), key, hashData([]byte("b-key-2")), []byte("b-val-2")) 522 rawdb.WriteStorageSnapshot(helper.triedb.DiskDB(), key, hashData([]byte("b-key-3")), []byte("b-val-3")) 523 } 524 root := helper.Commit() 525 526 // To verify the test: If we now inspect the snap db, there should exist extraneous storage items 527 if data := rawdb.ReadStorageSnapshot(helper.triedb.DiskDB(), hashData([]byte("acc-2")), hashData([]byte("b-key-1"))); data == nil { 528 t.Fatalf("expected snap storage to exist") 529 } 530 snap := generateSnapshot(helper.diskdb, helper.triedb, 16, testBlockHash, root, nil) 531 select { 532 case <-snap.genPending: 533 // Snapshot generation succeeded 534 535 case <-time.After(250 * time.Millisecond): 536 t.Errorf("Snapshot generation failed") 537 } 538 checkSnapRoot(t, snap, root) 539 540 // Signal abortion to the generator and wait for it to tear down 541 stop := make(chan struct{}) 542 snap.genAbort <- stop 543 <-stop 544 // If we now inspect the snap db, there should exist no extraneous storage items 545 if data := rawdb.ReadStorageSnapshot(helper.triedb.DiskDB(), hashData([]byte("acc-2")), hashData([]byte("b-key-1"))); data != nil { 546 t.Fatalf("expected slot to be removed, got %v", string(data)) 547 } 548 } 549 550 func enableLogging() { 551 log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))) 552 } 553 554 // Tests that snapshot generation when an extra account with storage exists in the snap state. 555 func TestGenerateWithManyExtraAccounts(t *testing.T) { 556 if false { 557 enableLogging() 558 } 559 helper := newHelper() 560 { 561 // Account one in the trie 562 stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), 563 []string{"key-1", "key-2", "key-3"}, 564 []string{"val-1", "val-2", "val-3"}, 565 true, 566 ) 567 acc := &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()} 568 val, _ := rlp.EncodeToBytes(acc) 569 helper.accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e 570 571 // Identical in the snap 572 key := hashData([]byte("acc-1")) 573 rawdb.WriteAccountSnapshot(helper.diskdb, key, val) 574 rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("key-1")), []byte("val-1")) 575 rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("key-2")), []byte("val-2")) 576 rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("key-3")), []byte("val-3")) 577 } 578 { 579 // 100 accounts exist only in snapshot 580 for i := 0; i < 1000; i++ { 581 //acc := &Account{Balance: big.NewInt(int64(i)), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()} 582 acc := &Account{Balance: big.NewInt(int64(i)), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()} 583 val, _ := rlp.EncodeToBytes(acc) 584 key := hashData([]byte(fmt.Sprintf("acc-%d", i))) 585 rawdb.WriteAccountSnapshot(helper.diskdb, key, val) 586 } 587 } 588 root, snap := helper.CommitAndGenerate() 589 select { 590 case <-snap.genPending: 591 // Snapshot generation succeeded 592 593 case <-time.After(5 * time.Second): 594 t.Errorf("Snapshot generation failed") 595 } 596 checkSnapRoot(t, snap, root) 597 // Signal abortion to the generator and wait for it to tear down 598 stop := make(chan struct{}) 599 snap.genAbort <- stop 600 <-stop 601 } 602 603 // Tests this case 604 // maxAccountRange 3 605 // snapshot-accounts: 01, 02, 03, 04, 05, 06, 07 606 // trie-accounts: 03, 07 607 // 608 // We iterate three snapshot storage slots (max = 3) from the database. They are 0x01, 0x02, 0x03. 609 // The trie has a lot of deletions. 610 // So in trie, we iterate 2 entries 0x03, 0x07. We create the 0x07 in the database and abort the procedure, because the trie is exhausted. 611 // But in the database, we still have the stale storage slots 0x04, 0x05. They are not iterated yet, but the procedure is finished. 612 func TestGenerateWithExtraBeforeAndAfter(t *testing.T) { 613 if false { 614 enableLogging() 615 } 616 helper := newHelper() 617 { 618 acc := &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()} 619 val, _ := rlp.EncodeToBytes(acc) 620 helper.accTrie.Update(common.HexToHash("0x03").Bytes(), val) 621 helper.accTrie.Update(common.HexToHash("0x07").Bytes(), val) 622 623 rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x01"), val) 624 rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x02"), val) 625 rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x03"), val) 626 rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x04"), val) 627 rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x05"), val) 628 rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x06"), val) 629 rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x07"), val) 630 } 631 root, snap := helper.CommitAndGenerate() 632 select { 633 case <-snap.genPending: 634 // Snapshot generation succeeded 635 636 case <-time.After(250 * time.Millisecond): 637 t.Errorf("Snapshot generation failed") 638 } 639 checkSnapRoot(t, snap, root) 640 // Signal abortion to the generator and wait for it to tear down 641 stop := make(chan struct{}) 642 snap.genAbort <- stop 643 <-stop 644 } 645 646 // TestGenerateWithMalformedSnapdata tests what happes if we have some junk 647 // in the snapshot database, which cannot be parsed back to an account 648 func TestGenerateWithMalformedSnapdata(t *testing.T) { 649 if false { 650 enableLogging() 651 } 652 helper := newHelper() 653 { 654 acc := &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()} 655 val, _ := rlp.EncodeToBytes(acc) 656 helper.accTrie.Update(common.HexToHash("0x03").Bytes(), val) 657 658 junk := make([]byte, 100) 659 copy(junk, []byte{0xde, 0xad}) 660 rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x02"), junk) 661 rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x03"), junk) 662 rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x04"), junk) 663 rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x05"), junk) 664 } 665 root, snap := helper.CommitAndGenerate() 666 select { 667 case <-snap.genPending: 668 // Snapshot generation succeeded 669 670 case <-time.After(250 * time.Millisecond): 671 t.Errorf("Snapshot generation failed") 672 } 673 checkSnapRoot(t, snap, root) 674 // Signal abortion to the generator and wait for it to tear down 675 stop := make(chan struct{}) 676 snap.genAbort <- stop 677 <-stop 678 // If we now inspect the snap db, there should exist no extraneous storage items 679 if data := rawdb.ReadStorageSnapshot(helper.diskdb, hashData([]byte("acc-2")), hashData([]byte("b-key-1"))); data != nil { 680 t.Fatalf("expected slot to be removed, got %v", string(data)) 681 } 682 } 683 684 func TestGenerateFromEmptySnap(t *testing.T) { 685 //enableLogging() 686 helper := newHelper() 687 // Add 1K accounts to the trie 688 for i := 0; i < 400; i++ { 689 stRoot := helper.makeStorageTrie(hashData([]byte(fmt.Sprintf("acc-%d", i))), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) 690 helper.addTrieAccount(fmt.Sprintf("acc-%d", i), 691 &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) 692 } 693 root, snap := helper.CommitAndGenerate() 694 t.Logf("Root: %#x\n", root) // Root: 0x6f7af6d2e1a1bf2b84a3beb3f8b64388465fbc1e274ca5d5d3fc787ca78f59e4 695 696 select { 697 case <-snap.genPending: 698 // Snapshot generation succeeded 699 700 case <-time.After(1 * time.Second): 701 t.Errorf("Snapshot generation failed") 702 } 703 checkSnapRoot(t, snap, root) 704 // Signal abortion to the generator and wait for it to tear down 705 stop := make(chan struct{}) 706 snap.genAbort <- stop 707 <-stop 708 } 709 710 // Tests that snapshot generation with existent flat state, where the flat state 711 // storage is correct, but incomplete. 712 // The incomplete part is on the second range 713 // snap: [ 0x01, 0x02, 0x03, 0x04] , [ 0x05, 0x06, 0x07, {missing}] (with storageCheck = 4) 714 // trie: 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08 715 // This hits a case where the snap verification passes, but there are more elements in the trie 716 // which we must also add. 717 func TestGenerateWithIncompleteStorage(t *testing.T) { 718 helper := newHelper() 719 stKeys := []string{"1", "2", "3", "4", "5", "6", "7", "8"} 720 stVals := []string{"v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8"} 721 // We add 8 accounts, each one is missing exactly one of the storage slots. This means 722 // we don't have to order the keys and figure out exactly which hash-key winds up 723 // on the sensitive spots at the boundaries 724 for i := 0; i < 8; i++ { 725 accKey := fmt.Sprintf("acc-%d", i) 726 stRoot := helper.makeStorageTrie(hashData([]byte(accKey)), stKeys, stVals, true) 727 helper.addAccount(accKey, &Account{Balance: big.NewInt(int64(i)), Root: stRoot, CodeHash: emptyCode.Bytes()}) 728 var moddedKeys []string 729 var moddedVals []string 730 for ii := 0; ii < 8; ii++ { 731 if ii != i { 732 moddedKeys = append(moddedKeys, stKeys[ii]) 733 moddedVals = append(moddedVals, stVals[ii]) 734 } 735 } 736 helper.addSnapStorage(accKey, moddedKeys, moddedVals) 737 } 738 root, snap := helper.CommitAndGenerate() 739 t.Logf("Root: %#x\n", root) // Root: 0xca73f6f05ba4ca3024ef340ef3dfca8fdabc1b677ff13f5a9571fd49c16e67ff 740 741 select { 742 case <-snap.genPending: 743 // Snapshot generation succeeded 744 745 case <-time.After(250 * time.Millisecond): 746 t.Errorf("Snapshot generation failed") 747 } 748 checkSnapRoot(t, snap, root) 749 // Signal abortion to the generator and wait for it to tear down 750 stop := make(chan struct{}) 751 snap.genAbort <- stop 752 <-stop 753 } 754 755 func incKey(key []byte) []byte { 756 for i := len(key) - 1; i >= 0; i-- { 757 key[i]++ 758 if key[i] != 0x0 { 759 break 760 } 761 } 762 return key 763 } 764 765 func decKey(key []byte) []byte { 766 for i := len(key) - 1; i >= 0; i-- { 767 key[i]-- 768 if key[i] != 0xff { 769 break 770 } 771 } 772 return key 773 } 774 775 func populateDangling(disk ethdb.KeyValueStore) { 776 populate := func(accountHash common.Hash, keys []string, vals []string) { 777 for i, key := range keys { 778 rawdb.WriteStorageSnapshot(disk, accountHash, hashData([]byte(key)), []byte(vals[i])) 779 } 780 } 781 // Dangling storages of the "first" account 782 populate(common.Hash{}, []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) 783 784 // Dangling storages of the "last" account 785 populate(common.HexToHash("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) 786 787 // Dangling storages around the account 1 788 hash := decKey(hashData([]byte("acc-1")).Bytes()) 789 populate(common.BytesToHash(hash), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) 790 hash = incKey(hashData([]byte("acc-1")).Bytes()) 791 populate(common.BytesToHash(hash), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) 792 793 // Dangling storages around the account 2 794 hash = decKey(hashData([]byte("acc-2")).Bytes()) 795 populate(common.BytesToHash(hash), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) 796 hash = incKey(hashData([]byte("acc-2")).Bytes()) 797 populate(common.BytesToHash(hash), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) 798 799 // Dangling storages around the account 3 800 hash = decKey(hashData([]byte("acc-3")).Bytes()) 801 populate(common.BytesToHash(hash), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) 802 hash = incKey(hashData([]byte("acc-3")).Bytes()) 803 populate(common.BytesToHash(hash), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) 804 805 // Dangling storages of the random account 806 populate(randomHash(), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) 807 populate(randomHash(), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) 808 populate(randomHash(), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) 809 } 810 811 // Tests that snapshot generation with dangling storages. Dangling storage means 812 // the storage data is existent while the corresponding account data is missing. 813 // 814 // This test will populate some dangling storages to see if they can be cleaned up. 815 func TestGenerateCompleteSnapshotWithDanglingStorage(t *testing.T) { 816 var helper = newHelper() 817 818 stRoot := helper.makeStorageTrie(hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) 819 helper.addAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) 820 helper.addAccount("acc-2", &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) 821 822 helper.makeStorageTrie(hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) 823 helper.addAccount("acc-3", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) 824 825 helper.addSnapStorage("acc-1", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) 826 helper.addSnapStorage("acc-3", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) 827 828 populateDangling(helper.diskdb) 829 830 root, snap := helper.CommitAndGenerate() 831 select { 832 case <-snap.genPending: 833 // Snapshot generation succeeded 834 835 case <-time.After(3 * time.Second): 836 t.Errorf("Snapshot generation failed") 837 } 838 checkSnapRoot(t, snap, root) 839 840 // Signal abortion to the generator and wait for it to tear down 841 stop := make(chan struct{}) 842 snap.genAbort <- stop 843 <-stop 844 } 845 846 // Tests that snapshot generation with dangling storages. Dangling storage means 847 // the storage data is existent while the corresponding account data is missing. 848 // 849 // This test will populate some dangling storages to see if they can be cleaned up. 850 func TestGenerateBrokenSnapshotWithDanglingStorage(t *testing.T) { 851 var helper = newHelper() 852 853 stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) 854 helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) 855 helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) 856 857 helper.makeStorageTrie(hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) 858 helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: emptyCode.Bytes()}) 859 860 populateDangling(helper.diskdb) 861 862 root, snap := helper.CommitAndGenerate() 863 select { 864 case <-snap.genPending: 865 // Snapshot generation succeeded 866 867 case <-time.After(3 * time.Second): 868 t.Errorf("Snapshot generation failed") 869 } 870 checkSnapRoot(t, snap, root) 871 872 // Signal abortion to the generator and wait for it to tear down 873 stop := make(chan struct{}) 874 snap.genAbort <- stop 875 <-stop 876 }