github.com/nspcc-dev/neo-go@v0.105.2-0.20240517133400-6be757af3eba/pkg/core/statesync/neotest_test.go (about) 1 package statesync_test 2 3 import ( 4 "bytes" 5 "testing" 6 7 "github.com/nspcc-dev/neo-go/internal/basicchain" 8 "github.com/nspcc-dev/neo-go/pkg/config" 9 "github.com/nspcc-dev/neo-go/pkg/core/block" 10 "github.com/nspcc-dev/neo-go/pkg/core/mpt" 11 "github.com/nspcc-dev/neo-go/pkg/core/storage" 12 "github.com/nspcc-dev/neo-go/pkg/neotest" 13 "github.com/nspcc-dev/neo-go/pkg/neotest/chain" 14 "github.com/nspcc-dev/neo-go/pkg/util" 15 "github.com/stretchr/testify/require" 16 ) 17 18 func TestStateSyncModule_Init(t *testing.T) { 19 const ( 20 stateSyncInterval = 2 21 maxTraceable = 3 22 ) 23 spoutCfg := func(c *config.Blockchain) { 24 c.StateRootInHeader = true 25 c.P2PStateExchangeExtensions = true 26 c.StateSyncInterval = stateSyncInterval 27 c.MaxTraceableBlocks = maxTraceable 28 } 29 bcSpout, validators, committee := chain.NewMultiWithCustomConfig(t, spoutCfg) 30 e := neotest.NewExecutor(t, bcSpout, validators, committee) 31 for i := 0; i <= 2*stateSyncInterval+int(maxTraceable)+1; i++ { 32 e.AddNewBlock(t) 33 } 34 35 boltCfg := func(c *config.Blockchain) { 36 spoutCfg(c) 37 c.Ledger.KeepOnlyLatestState = true 38 c.Ledger.RemoveUntraceableBlocks = true 39 } 40 41 t.Run("inactive: spout chain is too low to start state sync process", func(t *testing.T) { 42 bcBolt, _, _ := chain.NewMultiWithCustomConfig(t, boltCfg) 43 module := bcBolt.GetStateSyncModule() 44 require.NoError(t, module.Init(uint32(2*stateSyncInterval-1))) 45 require.False(t, module.IsActive()) 46 }) 47 48 t.Run("inactive: bolt chain height is close enough to spout chain height", func(t *testing.T) { 49 bcBolt, _, _ := chain.NewMultiWithCustomConfig(t, boltCfg) 50 for i := uint32(1); i < bcSpout.BlockHeight()-stateSyncInterval; i++ { 51 b, err := bcSpout.GetBlock(bcSpout.GetHeaderHash(i)) 52 require.NoError(t, err) 53 require.NoError(t, bcBolt.AddBlock(b)) 54 } 55 module := bcBolt.GetStateSyncModule() 56 require.NoError(t, module.Init(bcSpout.BlockHeight())) 57 require.False(t, module.IsActive()) 58 }) 59 60 t.Run("error: bolt chain is too low to start state sync process", func(t *testing.T) { 61 bcBolt, validatorsBolt, committeeBolt := chain.NewMultiWithCustomConfig(t, boltCfg) 62 eBolt := neotest.NewExecutor(t, bcBolt, validatorsBolt, committeeBolt) 63 eBolt.AddNewBlock(t) 64 65 module := bcBolt.GetStateSyncModule() 66 require.Error(t, module.Init(uint32(3*stateSyncInterval))) 67 }) 68 69 t.Run("initialized: no previous state sync point", func(t *testing.T) { 70 bcBolt, _, _ := chain.NewMultiWithCustomConfig(t, boltCfg) 71 72 module := bcBolt.GetStateSyncModule() 73 require.NoError(t, module.Init(bcSpout.BlockHeight())) 74 require.True(t, module.IsActive()) 75 require.True(t, module.IsInitialized()) 76 require.True(t, module.NeedHeaders()) 77 require.False(t, module.NeedMPTNodes()) 78 }) 79 80 t.Run("error: outdated state sync point in the storage", func(t *testing.T) { 81 bcBolt, _, _ := chain.NewMultiWithCustomConfig(t, boltCfg) 82 module := bcBolt.GetStateSyncModule() 83 require.NoError(t, module.Init(bcSpout.BlockHeight())) 84 85 module = bcBolt.GetStateSyncModule() 86 require.Error(t, module.Init(bcSpout.BlockHeight()+2*uint32(stateSyncInterval))) 87 }) 88 89 t.Run("initialized: valid previous state sync point in the storage", func(t *testing.T) { 90 bcBolt, _, _ := chain.NewMultiWithCustomConfig(t, boltCfg) 91 module := bcBolt.GetStateSyncModule() 92 require.NoError(t, module.Init(bcSpout.BlockHeight())) 93 94 module = bcBolt.GetStateSyncModule() 95 require.NoError(t, module.Init(bcSpout.BlockHeight())) 96 require.True(t, module.IsActive()) 97 require.True(t, module.IsInitialized()) 98 require.True(t, module.NeedHeaders()) 99 require.False(t, module.NeedMPTNodes()) 100 }) 101 102 t.Run("initialization from headers/blocks/mpt synced stages", func(t *testing.T) { 103 bcBolt, validatorsBolt, committeeBolt := chain.NewMultiWithCustomConfig(t, boltCfg) 104 eBolt := neotest.NewExecutor(t, bcBolt, validatorsBolt, committeeBolt) 105 module := bcBolt.GetStateSyncModule() 106 require.NoError(t, module.Init(bcSpout.BlockHeight())) 107 108 // firstly, fetch all headers to create proper DB state (where headers are in sync) 109 stateSyncPoint := (bcSpout.BlockHeight() / stateSyncInterval) * stateSyncInterval 110 var expectedHeader *block.Header 111 for i := uint32(1); i <= bcSpout.HeaderHeight(); i++ { 112 header, err := bcSpout.GetHeader(bcSpout.GetHeaderHash(i)) 113 require.NoError(t, err) 114 require.NoError(t, module.AddHeaders(header)) 115 if i == stateSyncPoint+1 { 116 expectedHeader = header 117 } 118 } 119 require.True(t, module.IsActive()) 120 require.True(t, module.IsInitialized()) 121 require.False(t, module.NeedHeaders()) 122 require.True(t, module.NeedMPTNodes()) 123 124 // then create new statesync module with the same DB and check that state is proper 125 // (headers are in sync) 126 module = bcBolt.GetStateSyncModule() 127 require.NoError(t, module.Init(bcSpout.BlockHeight())) 128 require.True(t, module.IsActive()) 129 require.True(t, module.IsInitialized()) 130 require.False(t, module.NeedHeaders()) 131 require.True(t, module.NeedMPTNodes()) 132 unknownNodes := module.GetUnknownMPTNodesBatch(2) 133 require.Equal(t, 1, len(unknownNodes)) 134 require.Equal(t, expectedHeader.PrevStateRoot, unknownNodes[0]) 135 136 // add several blocks to create DB state where blocks are not in sync yet, but it's not a genesis. 137 for i := stateSyncPoint - maxTraceable + 1; i <= stateSyncPoint-stateSyncInterval-1; i++ { 138 block, err := bcSpout.GetBlock(bcSpout.GetHeaderHash(i)) 139 require.NoError(t, err) 140 require.NoError(t, module.AddBlock(block)) 141 } 142 require.True(t, module.IsActive()) 143 require.True(t, module.IsInitialized()) 144 require.False(t, module.NeedHeaders()) 145 require.True(t, module.NeedMPTNodes()) 146 require.Equal(t, uint32(stateSyncPoint-stateSyncInterval-1), module.BlockHeight()) 147 148 // then create new statesync module with the same DB and check that state is proper 149 // (blocks are not in sync yet) 150 module = bcBolt.GetStateSyncModule() 151 require.NoError(t, module.Init(bcSpout.BlockHeight())) 152 require.True(t, module.IsActive()) 153 require.True(t, module.IsInitialized()) 154 require.False(t, module.NeedHeaders()) 155 require.True(t, module.NeedMPTNodes()) 156 unknownNodes = module.GetUnknownMPTNodesBatch(2) 157 require.Equal(t, 1, len(unknownNodes)) 158 require.Equal(t, expectedHeader.PrevStateRoot, unknownNodes[0]) 159 require.Equal(t, uint32(stateSyncPoint-stateSyncInterval-1), module.BlockHeight()) 160 161 // add rest of blocks to create DB state where blocks are in sync 162 for i := stateSyncPoint - stateSyncInterval; i <= stateSyncPoint; i++ { 163 block, err := bcSpout.GetBlock(bcSpout.GetHeaderHash(i)) 164 require.NoError(t, err) 165 require.NoError(t, module.AddBlock(block)) 166 } 167 require.True(t, module.IsActive()) 168 require.True(t, module.IsInitialized()) 169 require.False(t, module.NeedHeaders()) 170 require.True(t, module.NeedMPTNodes()) 171 lastBlock, err := bcBolt.GetBlock(expectedHeader.PrevHash) 172 require.NoError(t, err) 173 require.Equal(t, uint32(stateSyncPoint), lastBlock.Index) 174 require.Equal(t, uint32(stateSyncPoint), module.BlockHeight()) 175 176 // then create new statesync module with the same DB and check that state is proper 177 // (headers and blocks are in sync) 178 module = bcBolt.GetStateSyncModule() 179 require.NoError(t, module.Init(bcSpout.BlockHeight())) 180 require.True(t, module.IsActive()) 181 require.True(t, module.IsInitialized()) 182 require.False(t, module.NeedHeaders()) 183 require.True(t, module.NeedMPTNodes()) 184 unknownNodes = module.GetUnknownMPTNodesBatch(2) 185 require.Equal(t, 1, len(unknownNodes)) 186 require.Equal(t, expectedHeader.PrevStateRoot, unknownNodes[0]) 187 require.Equal(t, uint32(stateSyncPoint), module.BlockHeight()) 188 189 // add a few MPT nodes to create DB state where some of MPT nodes are missing 190 count := 5 191 for { 192 unknownHashes := module.GetUnknownMPTNodesBatch(1) // restore nodes one-by-one 193 if len(unknownHashes) == 0 { 194 break 195 } 196 err := bcSpout.GetStateSyncModule().Traverse(unknownHashes[0], func(node mpt.Node, nodeBytes []byte) bool { 197 require.NoError(t, module.AddMPTNodes([][]byte{nodeBytes})) 198 return true // add nodes one-by-one 199 }) 200 require.NoError(t, err) 201 count-- 202 if count < 0 { 203 break 204 } 205 } 206 207 // then create new statesync module with the same DB and check that state is proper 208 // (headers and blocks are in sync, mpt is not yet synced) 209 module = bcBolt.GetStateSyncModule() 210 require.NoError(t, module.Init(bcSpout.BlockHeight())) 211 require.True(t, module.IsActive()) 212 require.True(t, module.IsInitialized()) 213 require.False(t, module.NeedHeaders()) 214 require.True(t, module.NeedMPTNodes()) 215 unknownNodes = module.GetUnknownMPTNodesBatch(100) 216 require.True(t, len(unknownNodes) > 0) 217 require.NotContains(t, unknownNodes, expectedHeader.PrevStateRoot) 218 require.Equal(t, uint32(stateSyncPoint), module.BlockHeight()) 219 220 // add the rest of MPT nodes and jump to state 221 alreadyRequested := make(map[util.Uint256]struct{}) 222 for { 223 unknownHashes := module.GetUnknownMPTNodesBatch(1) // restore nodes one-by-one 224 if len(unknownHashes) == 0 { 225 break 226 } 227 if _, ok := alreadyRequested[unknownHashes[0]]; ok { 228 t.Fatal("bug: node was requested twice") 229 } 230 alreadyRequested[unknownHashes[0]] = struct{}{} 231 var callbackCalled bool 232 err := bcSpout.GetStateSyncModule().Traverse(unknownHashes[0], func(node mpt.Node, nodeBytes []byte) bool { 233 require.NoError(t, module.AddMPTNodes([][]byte{bytes.Clone(nodeBytes)})) 234 callbackCalled = true 235 return true // add nodes one-by-one 236 }) 237 require.NoError(t, err) 238 require.True(t, callbackCalled) 239 } 240 241 // check that module is inactive and statejump is completed 242 require.False(t, module.IsActive()) 243 require.False(t, module.NeedHeaders()) 244 require.False(t, module.NeedMPTNodes()) 245 unknownNodes = module.GetUnknownMPTNodesBatch(1) 246 require.True(t, len(unknownNodes) == 0) 247 require.Equal(t, uint32(stateSyncPoint), module.BlockHeight()) 248 require.Equal(t, uint32(stateSyncPoint), bcBolt.BlockHeight()) 249 250 // create new module from completed state: the module should recognise that state sync is completed 251 module = bcBolt.GetStateSyncModule() 252 require.NoError(t, module.Init(bcSpout.BlockHeight())) 253 require.False(t, module.IsActive()) 254 require.False(t, module.NeedHeaders()) 255 require.False(t, module.NeedMPTNodes()) 256 unknownNodes = module.GetUnknownMPTNodesBatch(1) 257 require.True(t, len(unknownNodes) == 0) 258 require.Equal(t, uint32(stateSyncPoint), module.BlockHeight()) 259 require.Equal(t, uint32(stateSyncPoint), bcBolt.BlockHeight()) 260 261 // add one more block to the restored chain and start new module: the module should recognise state sync is completed 262 // and regular blocks processing was started 263 eBolt.AddNewBlock(t) 264 module = bcBolt.GetStateSyncModule() 265 require.NoError(t, module.Init(bcSpout.BlockHeight())) 266 require.False(t, module.IsActive()) 267 require.False(t, module.NeedHeaders()) 268 require.False(t, module.NeedMPTNodes()) 269 unknownNodes = module.GetUnknownMPTNodesBatch(1) 270 require.True(t, len(unknownNodes) == 0) 271 require.Equal(t, uint32(stateSyncPoint)+1, module.BlockHeight()) 272 require.Equal(t, uint32(stateSyncPoint)+1, bcBolt.BlockHeight()) 273 }) 274 } 275 276 func TestStateSyncModule_RestoreBasicChain(t *testing.T) { 277 check := func(t *testing.T, spoutEnableGC bool) { 278 const ( 279 stateSyncInterval = 4 280 maxTraceable = 6 281 stateSyncPoint = 24 282 ) 283 spoutCfg := func(c *config.Blockchain) { 284 c.Ledger.KeepOnlyLatestState = spoutEnableGC 285 c.Ledger.RemoveUntraceableBlocks = spoutEnableGC 286 c.StateRootInHeader = true 287 c.P2PStateExchangeExtensions = true 288 c.StateSyncInterval = stateSyncInterval 289 c.MaxTraceableBlocks = maxTraceable 290 c.P2PSigExtensions = true // `basicchain.Init` assumes Notary is enabled. 291 } 292 bcSpoutStore := storage.NewMemoryStore() 293 bcSpout, validators, committee := chain.NewMultiWithCustomConfigAndStore(t, spoutCfg, bcSpoutStore, false) 294 go bcSpout.Run() // Will close it manually at the end. 295 e := neotest.NewExecutor(t, bcSpout, validators, committee) 296 basicchain.Init(t, "../../../", e) 297 298 // make spout chain higher than latest state sync point (add several blocks up to stateSyncPoint+2) 299 e.AddNewBlock(t) 300 e.AddNewBlock(t) // This block is stateSyncPoint-th block. 301 e.AddNewBlock(t) 302 require.Equal(t, stateSyncPoint+2, int(bcSpout.BlockHeight())) 303 304 boltCfg := func(c *config.Blockchain) { 305 spoutCfg(c) 306 c.Ledger.KeepOnlyLatestState = true 307 c.Ledger.RemoveUntraceableBlocks = true 308 } 309 bcBoltStore := storage.NewMemoryStore() 310 bcBolt, _, _ := chain.NewMultiWithCustomConfigAndStore(t, boltCfg, bcBoltStore, false) 311 go bcBolt.Run() // Will close it manually at the end. 312 module := bcBolt.GetStateSyncModule() 313 314 t.Run("error: add headers before initialisation", func(t *testing.T) { 315 h, err := bcSpout.GetHeader(bcSpout.GetHeaderHash(1)) 316 require.NoError(t, err) 317 require.Error(t, module.AddHeaders(h)) 318 }) 319 t.Run("no error: add blocks before initialisation", func(t *testing.T) { 320 b, err := bcSpout.GetBlock(bcSpout.GetHeaderHash(bcSpout.BlockHeight())) 321 require.NoError(t, err) 322 require.NoError(t, module.AddBlock(b)) 323 }) 324 t.Run("error: add MPT nodes without initialisation", func(t *testing.T) { 325 require.Error(t, module.AddMPTNodes([][]byte{})) 326 }) 327 328 require.NoError(t, module.Init(bcSpout.BlockHeight())) 329 require.True(t, module.IsActive()) 330 require.True(t, module.IsInitialized()) 331 require.True(t, module.NeedHeaders()) 332 require.False(t, module.NeedMPTNodes()) 333 334 // add headers to module 335 headers := make([]*block.Header, 0, bcSpout.HeaderHeight()) 336 for i := uint32(1); i <= bcSpout.HeaderHeight(); i++ { 337 h, err := bcSpout.GetHeader(bcSpout.GetHeaderHash(i)) 338 require.NoError(t, err) 339 headers = append(headers, h) 340 } 341 require.NoError(t, module.AddHeaders(headers...)) 342 require.True(t, module.IsActive()) 343 require.True(t, module.IsInitialized()) 344 require.False(t, module.NeedHeaders()) 345 require.True(t, module.NeedMPTNodes()) 346 require.Equal(t, bcSpout.HeaderHeight(), bcBolt.HeaderHeight()) 347 348 // add blocks 349 t.Run("error: unexpected block index", func(t *testing.T) { 350 b, err := bcSpout.GetBlock(bcSpout.GetHeaderHash(stateSyncPoint - maxTraceable)) 351 require.NoError(t, err) 352 require.Error(t, module.AddBlock(b)) 353 }) 354 t.Run("error: missing state root in block header", func(t *testing.T) { 355 b := &block.Block{ 356 Header: block.Header{ 357 Index: uint32(stateSyncPoint) - maxTraceable + 1, 358 StateRootEnabled: false, 359 }, 360 } 361 require.Error(t, module.AddBlock(b)) 362 }) 363 t.Run("error: invalid block merkle root", func(t *testing.T) { 364 b := &block.Block{ 365 Header: block.Header{ 366 Index: uint32(stateSyncPoint) - maxTraceable + 1, 367 StateRootEnabled: true, 368 MerkleRoot: util.Uint256{1, 2, 3}, 369 }, 370 } 371 require.Error(t, module.AddBlock(b)) 372 }) 373 374 for i := uint32(stateSyncPoint - maxTraceable + 1); i <= stateSyncPoint; i++ { 375 b, err := bcSpout.GetBlock(bcSpout.GetHeaderHash(i)) 376 require.NoError(t, err) 377 require.NoError(t, module.AddBlock(b)) 378 } 379 require.True(t, module.IsActive()) 380 require.True(t, module.IsInitialized()) 381 require.False(t, module.NeedHeaders()) 382 require.True(t, module.NeedMPTNodes()) 383 require.Equal(t, uint32(stateSyncPoint), module.BlockHeight()) 384 385 // add MPT nodes in batches 386 h, err := bcSpout.GetHeader(bcSpout.GetHeaderHash(stateSyncPoint + 1)) 387 require.NoError(t, err) 388 unknownHashes := module.GetUnknownMPTNodesBatch(100) 389 require.Equal(t, 1, len(unknownHashes)) 390 require.Equal(t, h.PrevStateRoot, unknownHashes[0]) 391 nodesMap := make(map[util.Uint256][]byte) 392 393 sm := bcSpout.GetStateModule() 394 sroo, err := sm.GetStateRoot(uint32(stateSyncPoint)) 395 require.NoError(t, err) 396 require.Equal(t, sroo.Root, h.PrevStateRoot) 397 err = bcSpout.GetStateSyncModule().Traverse(h.PrevStateRoot, func(n mpt.Node, nodeBytes []byte) bool { 398 nodesMap[n.Hash()] = nodeBytes 399 return false 400 }) 401 require.NoError(t, err) 402 for { 403 need := module.GetUnknownMPTNodesBatch(10) 404 if len(need) == 0 { 405 break 406 } 407 add := make([][]byte, len(need)) 408 for i, h := range need { 409 nodeBytes, ok := nodesMap[h] 410 if !ok { 411 t.Fatal("unknown or restored node requested") 412 } 413 add[i] = nodeBytes 414 delete(nodesMap, h) 415 } 416 require.NoError(t, module.AddMPTNodes(add)) 417 } 418 require.False(t, module.IsActive()) 419 require.False(t, module.NeedHeaders()) 420 require.False(t, module.NeedMPTNodes()) 421 unknownNodes := module.GetUnknownMPTNodesBatch(1) 422 require.True(t, len(unknownNodes) == 0) 423 require.Equal(t, uint32(stateSyncPoint), module.BlockHeight()) 424 require.Equal(t, uint32(stateSyncPoint), bcBolt.BlockHeight()) 425 426 // add missing blocks to bcBolt: should be ok, because state is synced 427 for i := uint32(stateSyncPoint + 1); i <= bcSpout.BlockHeight(); i++ { 428 b, err := bcSpout.GetBlock(bcSpout.GetHeaderHash(i)) 429 require.NoError(t, err) 430 require.NoError(t, bcBolt.AddBlock(b)) 431 } 432 require.Equal(t, bcSpout.BlockHeight(), bcBolt.BlockHeight()) 433 434 // compare storage states 435 fetchStorage := func(ps storage.Store, storagePrefix byte) []storage.KeyValue { 436 var kv []storage.KeyValue 437 ps.Seek(storage.SeekRange{Prefix: []byte{storagePrefix}}, func(k, v []byte) bool { 438 key := bytes.Clone(k) 439 value := bytes.Clone(v) 440 if key[0] == byte(storage.STTempStorage) { 441 key[0] = byte(storage.STStorage) 442 } 443 kv = append(kv, storage.KeyValue{ 444 Key: key, 445 Value: value, 446 }) 447 return true 448 }) 449 return kv 450 } 451 // Both blockchains are running, so we need to wait until recent changes will be persisted 452 // to the underlying backend store. Close blockchains to ensure persist was completed. 453 bcSpout.Close() 454 bcBolt.Close() 455 expected := fetchStorage(bcSpoutStore, byte(storage.STStorage)) 456 actual := fetchStorage(bcBoltStore, byte(storage.STTempStorage)) 457 require.ElementsMatch(t, expected, actual) 458 459 // no temp items should be left 460 var haveItems bool 461 bcBoltStore.Seek(storage.SeekRange{Prefix: []byte{byte(storage.STStorage)}}, func(_, _ []byte) bool { 462 haveItems = true 463 return false 464 }) 465 require.False(t, haveItems) 466 } 467 t.Run("source node is archive", func(t *testing.T) { 468 check(t, false) 469 }) 470 t.Run("source node is light with GC", func(t *testing.T) { 471 check(t, true) 472 }) 473 }