github.com/vipernet-xyz/tm@v0.34.24/statesync/syncer_test.go (about) 1 package statesync 2 3 import ( 4 "errors" 5 "testing" 6 "time" 7 8 "github.com/stretchr/testify/assert" 9 "github.com/stretchr/testify/mock" 10 "github.com/stretchr/testify/require" 11 12 abci "github.com/vipernet-xyz/tm/abci/types" 13 "github.com/vipernet-xyz/tm/config" 14 "github.com/vipernet-xyz/tm/libs/log" 15 tmsync "github.com/vipernet-xyz/tm/libs/sync" 16 "github.com/vipernet-xyz/tm/p2p" 17 p2pmocks "github.com/vipernet-xyz/tm/p2p/mocks" 18 tmstate "github.com/vipernet-xyz/tm/proto/tendermint/state" 19 ssproto "github.com/vipernet-xyz/tm/proto/tendermint/statesync" 20 tmversion "github.com/vipernet-xyz/tm/proto/tendermint/version" 21 "github.com/vipernet-xyz/tm/proxy" 22 proxymocks "github.com/vipernet-xyz/tm/proxy/mocks" 23 sm "github.com/vipernet-xyz/tm/state" 24 "github.com/vipernet-xyz/tm/statesync/mocks" 25 "github.com/vipernet-xyz/tm/types" 26 "github.com/vipernet-xyz/tm/version" 27 ) 28 29 const testAppVersion = 9 30 31 // Sets up a basic syncer that can be used to test OfferSnapshot requests 32 func setupOfferSyncer(t *testing.T) (*syncer, *proxymocks.AppConnSnapshot) { 33 connQuery := &proxymocks.AppConnQuery{} 34 connSnapshot := &proxymocks.AppConnSnapshot{} 35 stateProvider := &mocks.StateProvider{} 36 stateProvider.On("AppHash", mock.Anything, mock.Anything).Return([]byte("app_hash"), nil) 37 cfg := config.DefaultStateSyncConfig() 38 syncer := newSyncer(*cfg, log.NewNopLogger(), connSnapshot, connQuery, stateProvider, "") 39 40 return syncer, connSnapshot 41 } 42 43 // Sets up a simple peer mock with an ID 44 func simplePeer(id string) *p2pmocks.Peer { 45 peer := &p2pmocks.Peer{} 46 peer.On("ID").Return(p2p.ID(id)) 47 return peer 48 } 49 50 func TestSyncer_SyncAny(t *testing.T) { 51 state := sm.State{ 52 ChainID: "chain", 53 Version: tmstate.Version{ 54 Consensus: tmversion.Consensus{ 55 Block: version.BlockProtocol, 56 App: testAppVersion, 57 }, 58 Software: version.TMCoreSemVer, 59 }, 60 61 LastBlockHeight: 1, 62 LastBlockID: types.BlockID{Hash: []byte("blockhash")}, 63 LastBlockTime: time.Now(), 64 LastResultsHash: []byte("last_results_hash"), 65 AppHash: []byte("app_hash"), 66 67 LastValidators: &types.ValidatorSet{Proposer: &types.Validator{Address: []byte("val1")}}, 68 Validators: &types.ValidatorSet{Proposer: &types.Validator{Address: []byte("val2")}}, 69 NextValidators: &types.ValidatorSet{Proposer: &types.Validator{Address: []byte("val3")}}, 70 71 ConsensusParams: *types.DefaultConsensusParams(), 72 LastHeightConsensusParamsChanged: 1, 73 } 74 commit := &types.Commit{BlockID: types.BlockID{Hash: []byte("blockhash")}} 75 76 chunks := []*chunk{ 77 {Height: 1, Format: 1, Index: 0, Chunk: []byte{1, 1, 0}}, 78 {Height: 1, Format: 1, Index: 1, Chunk: []byte{1, 1, 1}}, 79 {Height: 1, Format: 1, Index: 2, Chunk: []byte{1, 1, 2}}, 80 } 81 s := &snapshot{Height: 1, Format: 1, Chunks: 3, Hash: []byte{1, 2, 3}} 82 83 stateProvider := &mocks.StateProvider{} 84 stateProvider.On("AppHash", mock.Anything, uint64(1)).Return(state.AppHash, nil) 85 stateProvider.On("AppHash", mock.Anything, uint64(2)).Return([]byte("app_hash_2"), nil) 86 stateProvider.On("Commit", mock.Anything, uint64(1)).Return(commit, nil) 87 stateProvider.On("State", mock.Anything, uint64(1)).Return(state, nil) 88 connSnapshot := &proxymocks.AppConnSnapshot{} 89 connQuery := &proxymocks.AppConnQuery{} 90 91 cfg := config.DefaultStateSyncConfig() 92 syncer := newSyncer(*cfg, log.NewNopLogger(), connSnapshot, connQuery, stateProvider, "") 93 94 // Adding a chunk should error when no sync is in progress 95 _, err := syncer.AddChunk(&chunk{Height: 1, Format: 1, Index: 0, Chunk: []byte{1}}) 96 require.Error(t, err) 97 98 // Adding a couple of peers should trigger snapshot discovery messages 99 peerA := &p2pmocks.Peer{} 100 peerA.On("ID").Return(p2p.ID("a")) 101 peerA.On("SendEnvelope", mock.MatchedBy(func(i interface{}) bool { 102 e, ok := i.(p2p.Envelope) 103 if !ok { 104 return false 105 } 106 req, ok := e.Message.(*ssproto.SnapshotsRequest) 107 return ok && e.ChannelID == SnapshotChannel && req != nil 108 })).Return(true) 109 syncer.AddPeer(peerA) 110 peerA.AssertExpectations(t) 111 112 peerB := &p2pmocks.Peer{} 113 peerB.On("ID").Return(p2p.ID("b")) 114 peerB.On("SendEnvelope", mock.MatchedBy(func(i interface{}) bool { 115 e, ok := i.(p2p.Envelope) 116 if !ok { 117 return false 118 } 119 req, ok := e.Message.(*ssproto.SnapshotsRequest) 120 return ok && e.ChannelID == SnapshotChannel && req != nil 121 })).Return(true) 122 syncer.AddPeer(peerB) 123 peerB.AssertExpectations(t) 124 125 // Both peers report back with snapshots. One of them also returns a snapshot we don't want, in 126 // format 2, which will be rejected by the ABCI application. 127 new, err := syncer.AddSnapshot(peerA, s) 128 require.NoError(t, err) 129 assert.True(t, new) 130 131 new, err = syncer.AddSnapshot(peerB, s) 132 require.NoError(t, err) 133 assert.False(t, new) 134 135 new, err = syncer.AddSnapshot(peerB, &snapshot{Height: 2, Format: 2, Chunks: 3, Hash: []byte{1}}) 136 require.NoError(t, err) 137 assert.True(t, new) 138 139 // We start a sync, with peers sending back chunks when requested. We first reject the snapshot 140 // with height 2 format 2, and accept the snapshot at height 1. 141 connSnapshot.On("OfferSnapshotSync", abci.RequestOfferSnapshot{ 142 Snapshot: &abci.Snapshot{ 143 Height: 2, 144 Format: 2, 145 Chunks: 3, 146 Hash: []byte{1}, 147 }, 148 AppHash: []byte("app_hash_2"), 149 }).Return(&abci.ResponseOfferSnapshot{Result: abci.ResponseOfferSnapshot_REJECT_FORMAT}, nil) 150 connSnapshot.On("OfferSnapshotSync", abci.RequestOfferSnapshot{ 151 Snapshot: &abci.Snapshot{ 152 Height: s.Height, 153 Format: s.Format, 154 Chunks: s.Chunks, 155 Hash: s.Hash, 156 Metadata: s.Metadata, 157 }, 158 AppHash: []byte("app_hash"), 159 }).Times(2).Return(&abci.ResponseOfferSnapshot{Result: abci.ResponseOfferSnapshot_ACCEPT}, nil) 160 161 chunkRequests := make(map[uint32]int) 162 chunkRequestsMtx := tmsync.Mutex{} 163 onChunkRequest := func(args mock.Arguments) { 164 e, ok := args[0].(p2p.Envelope) 165 require.True(t, ok) 166 msg := e.Message.(*ssproto.ChunkRequest) 167 require.EqualValues(t, 1, msg.Height) 168 require.EqualValues(t, 1, msg.Format) 169 require.LessOrEqual(t, msg.Index, uint32(len(chunks))) 170 171 added, err := syncer.AddChunk(chunks[msg.Index]) 172 require.NoError(t, err) 173 assert.True(t, added) 174 175 chunkRequestsMtx.Lock() 176 chunkRequests[msg.Index]++ 177 chunkRequestsMtx.Unlock() 178 } 179 peerA.On("SendEnvelope", mock.MatchedBy(func(i interface{}) bool { 180 e, ok := i.(p2p.Envelope) 181 return ok && e.ChannelID == ChunkChannel 182 })).Maybe().Run(onChunkRequest).Return(true) 183 peerB.On("SendEnvelope", mock.MatchedBy(func(i interface{}) bool { 184 e, ok := i.(p2p.Envelope) 185 return ok && e.ChannelID == ChunkChannel 186 })).Maybe().Run(onChunkRequest).Return(true) 187 188 // The first time we're applying chunk 2 we tell it to retry the snapshot and discard chunk 1, 189 // which should cause it to keep the existing chunk 0 and 2, and restart restoration from 190 // beginning. We also wait for a little while, to exercise the retry logic in fetchChunks(). 191 connSnapshot.On("ApplySnapshotChunkSync", abci.RequestApplySnapshotChunk{ 192 Index: 2, Chunk: []byte{1, 1, 2}, 193 }).Once().Run(func(args mock.Arguments) { time.Sleep(2 * time.Second) }).Return( 194 &abci.ResponseApplySnapshotChunk{ 195 Result: abci.ResponseApplySnapshotChunk_RETRY_SNAPSHOT, 196 RefetchChunks: []uint32{1}, 197 }, nil) 198 199 connSnapshot.On("ApplySnapshotChunkSync", abci.RequestApplySnapshotChunk{ 200 Index: 0, Chunk: []byte{1, 1, 0}, 201 }).Times(2).Return(&abci.ResponseApplySnapshotChunk{Result: abci.ResponseApplySnapshotChunk_ACCEPT}, nil) 202 connSnapshot.On("ApplySnapshotChunkSync", abci.RequestApplySnapshotChunk{ 203 Index: 1, Chunk: []byte{1, 1, 1}, 204 }).Times(2).Return(&abci.ResponseApplySnapshotChunk{Result: abci.ResponseApplySnapshotChunk_ACCEPT}, nil) 205 connSnapshot.On("ApplySnapshotChunkSync", abci.RequestApplySnapshotChunk{ 206 Index: 2, Chunk: []byte{1, 1, 2}, 207 }).Once().Return(&abci.ResponseApplySnapshotChunk{Result: abci.ResponseApplySnapshotChunk_ACCEPT}, nil) 208 connQuery.On("InfoSync", proxy.RequestInfo).Return(&abci.ResponseInfo{ 209 AppVersion: testAppVersion, 210 LastBlockHeight: 1, 211 LastBlockAppHash: []byte("app_hash"), 212 }, nil) 213 214 newState, lastCommit, err := syncer.SyncAny(0, func() {}) 215 require.NoError(t, err) 216 217 time.Sleep(50 * time.Millisecond) // wait for peers to receive requests 218 219 chunkRequestsMtx.Lock() 220 assert.Equal(t, map[uint32]int{0: 1, 1: 2, 2: 1}, chunkRequests) 221 chunkRequestsMtx.Unlock() 222 223 expectState := state 224 225 assert.Equal(t, expectState, newState) 226 assert.Equal(t, commit, lastCommit) 227 228 connSnapshot.AssertExpectations(t) 229 connQuery.AssertExpectations(t) 230 peerA.AssertExpectations(t) 231 peerB.AssertExpectations(t) 232 } 233 234 func TestSyncer_SyncAny_noSnapshots(t *testing.T) { 235 syncer, _ := setupOfferSyncer(t) 236 _, _, err := syncer.SyncAny(0, func() {}) 237 assert.Equal(t, errNoSnapshots, err) 238 } 239 240 func TestSyncer_SyncAny_abort(t *testing.T) { 241 syncer, connSnapshot := setupOfferSyncer(t) 242 243 s := &snapshot{Height: 1, Format: 1, Chunks: 3, Hash: []byte{1, 2, 3}} 244 _, err := syncer.AddSnapshot(simplePeer("id"), s) 245 require.NoError(t, err) 246 connSnapshot.On("OfferSnapshotSync", abci.RequestOfferSnapshot{ 247 Snapshot: toABCI(s), AppHash: []byte("app_hash"), 248 }).Once().Return(&abci.ResponseOfferSnapshot{Result: abci.ResponseOfferSnapshot_ABORT}, nil) 249 250 _, _, err = syncer.SyncAny(0, func() {}) 251 assert.Equal(t, errAbort, err) 252 connSnapshot.AssertExpectations(t) 253 } 254 255 func TestSyncer_SyncAny_reject(t *testing.T) { 256 syncer, connSnapshot := setupOfferSyncer(t) 257 258 // s22 is tried first, then s12, then s11, then errNoSnapshots 259 s22 := &snapshot{Height: 2, Format: 2, Chunks: 3, Hash: []byte{1, 2, 3}} 260 s12 := &snapshot{Height: 1, Format: 2, Chunks: 3, Hash: []byte{1, 2, 3}} 261 s11 := &snapshot{Height: 1, Format: 1, Chunks: 3, Hash: []byte{1, 2, 3}} 262 _, err := syncer.AddSnapshot(simplePeer("id"), s22) 263 require.NoError(t, err) 264 _, err = syncer.AddSnapshot(simplePeer("id"), s12) 265 require.NoError(t, err) 266 _, err = syncer.AddSnapshot(simplePeer("id"), s11) 267 require.NoError(t, err) 268 269 connSnapshot.On("OfferSnapshotSync", abci.RequestOfferSnapshot{ 270 Snapshot: toABCI(s22), AppHash: []byte("app_hash"), 271 }).Once().Return(&abci.ResponseOfferSnapshot{Result: abci.ResponseOfferSnapshot_REJECT}, nil) 272 273 connSnapshot.On("OfferSnapshotSync", abci.RequestOfferSnapshot{ 274 Snapshot: toABCI(s12), AppHash: []byte("app_hash"), 275 }).Once().Return(&abci.ResponseOfferSnapshot{Result: abci.ResponseOfferSnapshot_REJECT}, nil) 276 277 connSnapshot.On("OfferSnapshotSync", abci.RequestOfferSnapshot{ 278 Snapshot: toABCI(s11), AppHash: []byte("app_hash"), 279 }).Once().Return(&abci.ResponseOfferSnapshot{Result: abci.ResponseOfferSnapshot_REJECT}, nil) 280 281 _, _, err = syncer.SyncAny(0, func() {}) 282 assert.Equal(t, errNoSnapshots, err) 283 connSnapshot.AssertExpectations(t) 284 } 285 286 func TestSyncer_SyncAny_reject_format(t *testing.T) { 287 syncer, connSnapshot := setupOfferSyncer(t) 288 289 // s22 is tried first, which reject s22 and s12, then s11 will abort. 290 s22 := &snapshot{Height: 2, Format: 2, Chunks: 3, Hash: []byte{1, 2, 3}} 291 s12 := &snapshot{Height: 1, Format: 2, Chunks: 3, Hash: []byte{1, 2, 3}} 292 s11 := &snapshot{Height: 1, Format: 1, Chunks: 3, Hash: []byte{1, 2, 3}} 293 _, err := syncer.AddSnapshot(simplePeer("id"), s22) 294 require.NoError(t, err) 295 _, err = syncer.AddSnapshot(simplePeer("id"), s12) 296 require.NoError(t, err) 297 _, err = syncer.AddSnapshot(simplePeer("id"), s11) 298 require.NoError(t, err) 299 300 connSnapshot.On("OfferSnapshotSync", abci.RequestOfferSnapshot{ 301 Snapshot: toABCI(s22), AppHash: []byte("app_hash"), 302 }).Once().Return(&abci.ResponseOfferSnapshot{Result: abci.ResponseOfferSnapshot_REJECT_FORMAT}, nil) 303 304 connSnapshot.On("OfferSnapshotSync", abci.RequestOfferSnapshot{ 305 Snapshot: toABCI(s11), AppHash: []byte("app_hash"), 306 }).Once().Return(&abci.ResponseOfferSnapshot{Result: abci.ResponseOfferSnapshot_ABORT}, nil) 307 308 _, _, err = syncer.SyncAny(0, func() {}) 309 assert.Equal(t, errAbort, err) 310 connSnapshot.AssertExpectations(t) 311 } 312 313 func TestSyncer_SyncAny_reject_sender(t *testing.T) { 314 syncer, connSnapshot := setupOfferSyncer(t) 315 316 peerA := simplePeer("a") 317 peerB := simplePeer("b") 318 peerC := simplePeer("c") 319 320 // sbc will be offered first, which will be rejected with reject_sender, causing all snapshots 321 // submitted by both b and c (i.e. sb, sc, sbc) to be rejected. Finally, sa will reject and 322 // errNoSnapshots is returned. 323 sa := &snapshot{Height: 1, Format: 1, Chunks: 3, Hash: []byte{1, 2, 3}} 324 sb := &snapshot{Height: 2, Format: 1, Chunks: 3, Hash: []byte{1, 2, 3}} 325 sc := &snapshot{Height: 3, Format: 1, Chunks: 3, Hash: []byte{1, 2, 3}} 326 sbc := &snapshot{Height: 4, Format: 1, Chunks: 3, Hash: []byte{1, 2, 3}} 327 _, err := syncer.AddSnapshot(peerA, sa) 328 require.NoError(t, err) 329 _, err = syncer.AddSnapshot(peerB, sb) 330 require.NoError(t, err) 331 _, err = syncer.AddSnapshot(peerC, sc) 332 require.NoError(t, err) 333 _, err = syncer.AddSnapshot(peerB, sbc) 334 require.NoError(t, err) 335 _, err = syncer.AddSnapshot(peerC, sbc) 336 require.NoError(t, err) 337 338 connSnapshot.On("OfferSnapshotSync", abci.RequestOfferSnapshot{ 339 Snapshot: toABCI(sbc), AppHash: []byte("app_hash"), 340 }).Once().Return(&abci.ResponseOfferSnapshot{Result: abci.ResponseOfferSnapshot_REJECT_SENDER}, nil) 341 342 connSnapshot.On("OfferSnapshotSync", abci.RequestOfferSnapshot{ 343 Snapshot: toABCI(sa), AppHash: []byte("app_hash"), 344 }).Once().Return(&abci.ResponseOfferSnapshot{Result: abci.ResponseOfferSnapshot_REJECT}, nil) 345 346 _, _, err = syncer.SyncAny(0, func() {}) 347 assert.Equal(t, errNoSnapshots, err) 348 connSnapshot.AssertExpectations(t) 349 } 350 351 func TestSyncer_SyncAny_abciError(t *testing.T) { 352 syncer, connSnapshot := setupOfferSyncer(t) 353 354 errBoom := errors.New("boom") 355 s := &snapshot{Height: 1, Format: 1, Chunks: 3, Hash: []byte{1, 2, 3}} 356 _, err := syncer.AddSnapshot(simplePeer("id"), s) 357 require.NoError(t, err) 358 connSnapshot.On("OfferSnapshotSync", abci.RequestOfferSnapshot{ 359 Snapshot: toABCI(s), AppHash: []byte("app_hash"), 360 }).Once().Return(nil, errBoom) 361 362 _, _, err = syncer.SyncAny(0, func() {}) 363 assert.True(t, errors.Is(err, errBoom)) 364 connSnapshot.AssertExpectations(t) 365 } 366 367 func TestSyncer_offerSnapshot(t *testing.T) { 368 unknownErr := errors.New("unknown error") 369 boom := errors.New("boom") 370 371 testcases := map[string]struct { 372 result abci.ResponseOfferSnapshot_Result 373 err error 374 expectErr error 375 }{ 376 "accept": {abci.ResponseOfferSnapshot_ACCEPT, nil, nil}, 377 "abort": {abci.ResponseOfferSnapshot_ABORT, nil, errAbort}, 378 "reject": {abci.ResponseOfferSnapshot_REJECT, nil, errRejectSnapshot}, 379 "reject_format": {abci.ResponseOfferSnapshot_REJECT_FORMAT, nil, errRejectFormat}, 380 "reject_sender": {abci.ResponseOfferSnapshot_REJECT_SENDER, nil, errRejectSender}, 381 "unknown": {abci.ResponseOfferSnapshot_UNKNOWN, nil, unknownErr}, 382 "error": {0, boom, boom}, 383 "unknown non-zero": {9, nil, unknownErr}, 384 } 385 for name, tc := range testcases { 386 tc := tc 387 t.Run(name, func(t *testing.T) { 388 syncer, connSnapshot := setupOfferSyncer(t) 389 s := &snapshot{Height: 1, Format: 1, Chunks: 3, Hash: []byte{1, 2, 3}, trustedAppHash: []byte("app_hash")} 390 connSnapshot.On("OfferSnapshotSync", abci.RequestOfferSnapshot{ 391 Snapshot: toABCI(s), 392 AppHash: []byte("app_hash"), 393 }).Return(&abci.ResponseOfferSnapshot{Result: tc.result}, tc.err) 394 err := syncer.offerSnapshot(s) 395 if tc.expectErr == unknownErr { 396 require.Error(t, err) 397 } else { 398 unwrapped := errors.Unwrap(err) 399 if unwrapped != nil { 400 err = unwrapped 401 } 402 assert.Equal(t, tc.expectErr, err) 403 } 404 }) 405 } 406 } 407 408 func TestSyncer_applyChunks_Results(t *testing.T) { 409 unknownErr := errors.New("unknown error") 410 boom := errors.New("boom") 411 412 testcases := map[string]struct { 413 result abci.ResponseApplySnapshotChunk_Result 414 err error 415 expectErr error 416 }{ 417 "accept": {abci.ResponseApplySnapshotChunk_ACCEPT, nil, nil}, 418 "abort": {abci.ResponseApplySnapshotChunk_ABORT, nil, errAbort}, 419 "retry": {abci.ResponseApplySnapshotChunk_RETRY, nil, nil}, 420 "retry_snapshot": {abci.ResponseApplySnapshotChunk_RETRY_SNAPSHOT, nil, errRetrySnapshot}, 421 "reject_snapshot": {abci.ResponseApplySnapshotChunk_REJECT_SNAPSHOT, nil, errRejectSnapshot}, 422 "unknown": {abci.ResponseApplySnapshotChunk_UNKNOWN, nil, unknownErr}, 423 "error": {0, boom, boom}, 424 "unknown non-zero": {9, nil, unknownErr}, 425 } 426 for name, tc := range testcases { 427 tc := tc 428 t.Run(name, func(t *testing.T) { 429 connQuery := &proxymocks.AppConnQuery{} 430 connSnapshot := &proxymocks.AppConnSnapshot{} 431 stateProvider := &mocks.StateProvider{} 432 stateProvider.On("AppHash", mock.Anything, mock.Anything).Return([]byte("app_hash"), nil) 433 434 cfg := config.DefaultStateSyncConfig() 435 syncer := newSyncer(*cfg, log.NewNopLogger(), connSnapshot, connQuery, stateProvider, "") 436 437 body := []byte{1, 2, 3} 438 chunks, err := newChunkQueue(&snapshot{Height: 1, Format: 1, Chunks: 1}, "") 439 require.NoError(t, err) 440 _, err = chunks.Add(&chunk{Height: 1, Format: 1, Index: 0, Chunk: body}) 441 require.NoError(t, err) 442 443 connSnapshot.On("ApplySnapshotChunkSync", abci.RequestApplySnapshotChunk{ 444 Index: 0, Chunk: body, 445 }).Once().Return(&abci.ResponseApplySnapshotChunk{Result: tc.result}, tc.err) 446 if tc.result == abci.ResponseApplySnapshotChunk_RETRY { 447 connSnapshot.On("ApplySnapshotChunkSync", abci.RequestApplySnapshotChunk{ 448 Index: 0, Chunk: body, 449 }).Once().Return(&abci.ResponseApplySnapshotChunk{ 450 Result: abci.ResponseApplySnapshotChunk_ACCEPT}, nil) 451 } 452 453 err = syncer.applyChunks(chunks) 454 if tc.expectErr == unknownErr { 455 require.Error(t, err) 456 } else { 457 unwrapped := errors.Unwrap(err) 458 if unwrapped != nil { 459 err = unwrapped 460 } 461 assert.Equal(t, tc.expectErr, err) 462 } 463 connSnapshot.AssertExpectations(t) 464 }) 465 } 466 } 467 468 func TestSyncer_applyChunks_RefetchChunks(t *testing.T) { 469 // Discarding chunks via refetch_chunks should work the same for all results 470 testcases := map[string]struct { 471 result abci.ResponseApplySnapshotChunk_Result 472 }{ 473 "accept": {abci.ResponseApplySnapshotChunk_ACCEPT}, 474 "abort": {abci.ResponseApplySnapshotChunk_ABORT}, 475 "retry": {abci.ResponseApplySnapshotChunk_RETRY}, 476 "retry_snapshot": {abci.ResponseApplySnapshotChunk_RETRY_SNAPSHOT}, 477 "reject_snapshot": {abci.ResponseApplySnapshotChunk_REJECT_SNAPSHOT}, 478 } 479 for name, tc := range testcases { 480 tc := tc 481 t.Run(name, func(t *testing.T) { 482 connQuery := &proxymocks.AppConnQuery{} 483 connSnapshot := &proxymocks.AppConnSnapshot{} 484 stateProvider := &mocks.StateProvider{} 485 stateProvider.On("AppHash", mock.Anything, mock.Anything).Return([]byte("app_hash"), nil) 486 487 cfg := config.DefaultStateSyncConfig() 488 syncer := newSyncer(*cfg, log.NewNopLogger(), connSnapshot, connQuery, stateProvider, "") 489 490 chunks, err := newChunkQueue(&snapshot{Height: 1, Format: 1, Chunks: 3}, "") 491 require.NoError(t, err) 492 added, err := chunks.Add(&chunk{Height: 1, Format: 1, Index: 0, Chunk: []byte{0}}) 493 require.True(t, added) 494 require.NoError(t, err) 495 added, err = chunks.Add(&chunk{Height: 1, Format: 1, Index: 1, Chunk: []byte{1}}) 496 require.True(t, added) 497 require.NoError(t, err) 498 added, err = chunks.Add(&chunk{Height: 1, Format: 1, Index: 2, Chunk: []byte{2}}) 499 require.True(t, added) 500 require.NoError(t, err) 501 502 // The first two chunks are accepted, before the last one asks for 1 to be refetched 503 connSnapshot.On("ApplySnapshotChunkSync", abci.RequestApplySnapshotChunk{ 504 Index: 0, Chunk: []byte{0}, 505 }).Once().Return(&abci.ResponseApplySnapshotChunk{Result: abci.ResponseApplySnapshotChunk_ACCEPT}, nil) 506 connSnapshot.On("ApplySnapshotChunkSync", abci.RequestApplySnapshotChunk{ 507 Index: 1, Chunk: []byte{1}, 508 }).Once().Return(&abci.ResponseApplySnapshotChunk{Result: abci.ResponseApplySnapshotChunk_ACCEPT}, nil) 509 connSnapshot.On("ApplySnapshotChunkSync", abci.RequestApplySnapshotChunk{ 510 Index: 2, Chunk: []byte{2}, 511 }).Once().Return(&abci.ResponseApplySnapshotChunk{ 512 Result: tc.result, 513 RefetchChunks: []uint32{1}, 514 }, nil) 515 516 // Since removing the chunk will cause Next() to block, we spawn a goroutine, then 517 // check the queue contents, and finally close the queue to end the goroutine. 518 // We don't really care about the result of applyChunks, since it has separate test. 519 go func() { 520 syncer.applyChunks(chunks) //nolint:errcheck // purposefully ignore error 521 }() 522 523 time.Sleep(50 * time.Millisecond) 524 assert.True(t, chunks.Has(0)) 525 assert.False(t, chunks.Has(1)) 526 assert.True(t, chunks.Has(2)) 527 err = chunks.Close() 528 require.NoError(t, err) 529 }) 530 } 531 } 532 533 func TestSyncer_applyChunks_RejectSenders(t *testing.T) { 534 // Banning chunks senders via ban_chunk_senders should work the same for all results 535 testcases := map[string]struct { 536 result abci.ResponseApplySnapshotChunk_Result 537 }{ 538 "accept": {abci.ResponseApplySnapshotChunk_ACCEPT}, 539 "abort": {abci.ResponseApplySnapshotChunk_ABORT}, 540 "retry": {abci.ResponseApplySnapshotChunk_RETRY}, 541 "retry_snapshot": {abci.ResponseApplySnapshotChunk_RETRY_SNAPSHOT}, 542 "reject_snapshot": {abci.ResponseApplySnapshotChunk_REJECT_SNAPSHOT}, 543 } 544 for name, tc := range testcases { 545 tc := tc 546 t.Run(name, func(t *testing.T) { 547 connQuery := &proxymocks.AppConnQuery{} 548 connSnapshot := &proxymocks.AppConnSnapshot{} 549 stateProvider := &mocks.StateProvider{} 550 stateProvider.On("AppHash", mock.Anything, mock.Anything).Return([]byte("app_hash"), nil) 551 552 cfg := config.DefaultStateSyncConfig() 553 syncer := newSyncer(*cfg, log.NewNopLogger(), connSnapshot, connQuery, stateProvider, "") 554 555 // Set up three peers across two snapshots, and ask for one of them to be banned. 556 // It should be banned from all snapshots. 557 peerA := simplePeer("a") 558 peerB := simplePeer("b") 559 peerC := simplePeer("c") 560 561 s1 := &snapshot{Height: 1, Format: 1, Chunks: 3} 562 s2 := &snapshot{Height: 2, Format: 1, Chunks: 3} 563 _, err := syncer.AddSnapshot(peerA, s1) 564 require.NoError(t, err) 565 _, err = syncer.AddSnapshot(peerA, s2) 566 require.NoError(t, err) 567 _, err = syncer.AddSnapshot(peerB, s1) 568 require.NoError(t, err) 569 _, err = syncer.AddSnapshot(peerB, s2) 570 require.NoError(t, err) 571 _, err = syncer.AddSnapshot(peerC, s1) 572 require.NoError(t, err) 573 _, err = syncer.AddSnapshot(peerC, s2) 574 require.NoError(t, err) 575 576 chunks, err := newChunkQueue(s1, "") 577 require.NoError(t, err) 578 added, err := chunks.Add(&chunk{Height: 1, Format: 1, Index: 0, Chunk: []byte{0}, Sender: peerA.ID()}) 579 require.True(t, added) 580 require.NoError(t, err) 581 added, err = chunks.Add(&chunk{Height: 1, Format: 1, Index: 1, Chunk: []byte{1}, Sender: peerB.ID()}) 582 require.True(t, added) 583 require.NoError(t, err) 584 added, err = chunks.Add(&chunk{Height: 1, Format: 1, Index: 2, Chunk: []byte{2}, Sender: peerC.ID()}) 585 require.True(t, added) 586 require.NoError(t, err) 587 588 // The first two chunks are accepted, before the last one asks for b sender to be rejected 589 connSnapshot.On("ApplySnapshotChunkSync", abci.RequestApplySnapshotChunk{ 590 Index: 0, Chunk: []byte{0}, Sender: "a", 591 }).Once().Return(&abci.ResponseApplySnapshotChunk{Result: abci.ResponseApplySnapshotChunk_ACCEPT}, nil) 592 connSnapshot.On("ApplySnapshotChunkSync", abci.RequestApplySnapshotChunk{ 593 Index: 1, Chunk: []byte{1}, Sender: "b", 594 }).Once().Return(&abci.ResponseApplySnapshotChunk{Result: abci.ResponseApplySnapshotChunk_ACCEPT}, nil) 595 connSnapshot.On("ApplySnapshotChunkSync", abci.RequestApplySnapshotChunk{ 596 Index: 2, Chunk: []byte{2}, Sender: "c", 597 }).Once().Return(&abci.ResponseApplySnapshotChunk{ 598 Result: tc.result, 599 RejectSenders: []string{string(peerB.ID())}, 600 }, nil) 601 602 // On retry, the last chunk will be tried again, so we just accept it then. 603 if tc.result == abci.ResponseApplySnapshotChunk_RETRY { 604 connSnapshot.On("ApplySnapshotChunkSync", abci.RequestApplySnapshotChunk{ 605 Index: 2, Chunk: []byte{2}, Sender: "c", 606 }).Once().Return(&abci.ResponseApplySnapshotChunk{Result: abci.ResponseApplySnapshotChunk_ACCEPT}, nil) 607 } 608 609 // We don't really care about the result of applyChunks, since it has separate test. 610 // However, it will block on e.g. retry result, so we spawn a goroutine that will 611 // be shut down when the chunk queue closes. 612 go func() { 613 syncer.applyChunks(chunks) //nolint:errcheck // purposefully ignore error 614 }() 615 616 time.Sleep(50 * time.Millisecond) 617 618 s1peers := syncer.snapshots.GetPeers(s1) 619 assert.Len(t, s1peers, 2) 620 assert.EqualValues(t, "a", s1peers[0].ID()) 621 assert.EqualValues(t, "c", s1peers[1].ID()) 622 623 syncer.snapshots.GetPeers(s1) 624 assert.Len(t, s1peers, 2) 625 assert.EqualValues(t, "a", s1peers[0].ID()) 626 assert.EqualValues(t, "c", s1peers[1].ID()) 627 628 err = chunks.Close() 629 require.NoError(t, err) 630 }) 631 } 632 } 633 634 func TestSyncer_verifyApp(t *testing.T) { 635 boom := errors.New("boom") 636 const appVersion = 9 637 appVersionMismatchErr := errors.New("app version mismatch. Expected: 9, got: 2") 638 s := &snapshot{Height: 3, Format: 1, Chunks: 5, Hash: []byte{1, 2, 3}, trustedAppHash: []byte("app_hash")} 639 640 testcases := map[string]struct { 641 response *abci.ResponseInfo 642 err error 643 expectErr error 644 }{ 645 "verified": {&abci.ResponseInfo{ 646 LastBlockHeight: 3, 647 LastBlockAppHash: []byte("app_hash"), 648 AppVersion: appVersion, 649 }, nil, nil}, 650 "invalid app version": {&abci.ResponseInfo{ 651 LastBlockHeight: 3, 652 LastBlockAppHash: []byte("app_hash"), 653 AppVersion: 2, 654 }, nil, appVersionMismatchErr}, 655 "invalid height": {&abci.ResponseInfo{ 656 LastBlockHeight: 5, 657 LastBlockAppHash: []byte("app_hash"), 658 AppVersion: appVersion, 659 }, nil, errVerifyFailed}, 660 "invalid hash": {&abci.ResponseInfo{ 661 LastBlockHeight: 3, 662 LastBlockAppHash: []byte("xxx"), 663 AppVersion: appVersion, 664 }, nil, errVerifyFailed}, 665 "error": {nil, boom, boom}, 666 } 667 for name, tc := range testcases { 668 tc := tc 669 t.Run(name, func(t *testing.T) { 670 connQuery := &proxymocks.AppConnQuery{} 671 connSnapshot := &proxymocks.AppConnSnapshot{} 672 stateProvider := &mocks.StateProvider{} 673 674 cfg := config.DefaultStateSyncConfig() 675 syncer := newSyncer(*cfg, log.NewNopLogger(), connSnapshot, connQuery, stateProvider, "") 676 677 connQuery.On("InfoSync", proxy.RequestInfo).Return(tc.response, tc.err) 678 err := syncer.verifyApp(s, appVersion) 679 unwrapped := errors.Unwrap(err) 680 if unwrapped != nil { 681 err = unwrapped 682 } 683 require.Equal(t, tc.expectErr, err) 684 }) 685 } 686 } 687 688 func toABCI(s *snapshot) *abci.Snapshot { 689 return &abci.Snapshot{ 690 Height: s.Height, 691 Format: s.Format, 692 Chunks: s.Chunks, 693 Hash: s.Hash, 694 Metadata: s.Metadata, 695 } 696 }