github.com/DFWallet/tendermint-cosmos@v0.0.2/statesync/syncer_test.go (about) 1 package statesync 2 3 import ( 4 "errors" 5 "testing" 6 "time" 7 8 "github.com/stretchr/testify/assert" 9 "github.com/stretchr/testify/mock" 10 "github.com/stretchr/testify/require" 11 12 abci "github.com/DFWallet/tendermint-cosmos/abci/types" 13 "github.com/DFWallet/tendermint-cosmos/config" 14 "github.com/DFWallet/tendermint-cosmos/libs/log" 15 tmsync "github.com/DFWallet/tendermint-cosmos/libs/sync" 16 "github.com/DFWallet/tendermint-cosmos/p2p" 17 p2pmocks "github.com/DFWallet/tendermint-cosmos/p2p/mocks" 18 tmstate "github.com/DFWallet/tendermint-cosmos/proto/tendermint/state" 19 ssproto "github.com/DFWallet/tendermint-cosmos/proto/tendermint/statesync" 20 tmversion "github.com/DFWallet/tendermint-cosmos/proto/tendermint/version" 21 "github.com/DFWallet/tendermint-cosmos/proxy" 22 proxymocks "github.com/DFWallet/tendermint-cosmos/proxy/mocks" 23 sm "github.com/DFWallet/tendermint-cosmos/state" 24 "github.com/DFWallet/tendermint-cosmos/statesync/mocks" 25 "github.com/DFWallet/tendermint-cosmos/types" 26 "github.com/DFWallet/tendermint-cosmos/version" 27 ) 28 29 // Sets up a basic syncer that can be used to test OfferSnapshot requests 30 func setupOfferSyncer(t *testing.T) (*syncer, *proxymocks.AppConnSnapshot) { 31 connQuery := &proxymocks.AppConnQuery{} 32 connSnapshot := &proxymocks.AppConnSnapshot{} 33 stateProvider := &mocks.StateProvider{} 34 stateProvider.On("AppHash", mock.Anything, mock.Anything).Return([]byte("app_hash"), nil) 35 cfg := config.DefaultStateSyncConfig() 36 syncer := newSyncer(*cfg, log.NewNopLogger(), connSnapshot, connQuery, stateProvider, "") 37 38 return syncer, connSnapshot 39 } 40 41 // Sets up a simple peer mock with an ID 42 func simplePeer(id string) *p2pmocks.Peer { 43 peer := &p2pmocks.Peer{} 44 peer.On("ID").Return(p2p.ID(id)) 45 return peer 46 } 47 48 func TestSyncer_SyncAny(t *testing.T) { 49 state := sm.State{ 50 ChainID: "chain", 51 Version: tmstate.Version{ 52 Consensus: tmversion.Consensus{ 53 Block: version.BlockProtocol, 54 App: 0, 55 }, 56 Software: version.TMCoreSemVer, 57 }, 58 59 LastBlockHeight: 1, 60 LastBlockID: types.BlockID{Hash: []byte("blockhash")}, 61 LastBlockTime: time.Now(), 62 LastResultsHash: []byte("last_results_hash"), 63 AppHash: []byte("app_hash"), 64 65 LastValidators: &types.ValidatorSet{Proposer: &types.Validator{Address: []byte("val1")}}, 66 Validators: &types.ValidatorSet{Proposer: &types.Validator{Address: []byte("val2")}}, 67 NextValidators: &types.ValidatorSet{Proposer: &types.Validator{Address: []byte("val3")}}, 68 69 ConsensusParams: *types.DefaultConsensusParams(), 70 LastHeightConsensusParamsChanged: 1, 71 } 72 commit := &types.Commit{BlockID: types.BlockID{Hash: []byte("blockhash")}} 73 74 chunks := []*chunk{ 75 {Height: 1, Format: 1, Index: 0, Chunk: []byte{1, 1, 0}}, 76 {Height: 1, Format: 1, Index: 1, Chunk: []byte{1, 1, 1}}, 77 {Height: 1, Format: 1, Index: 2, Chunk: []byte{1, 1, 2}}, 78 } 79 s := &snapshot{Height: 1, Format: 1, Chunks: 3, Hash: []byte{1, 2, 3}} 80 81 stateProvider := &mocks.StateProvider{} 82 stateProvider.On("AppHash", mock.Anything, uint64(1)).Return(state.AppHash, nil) 83 stateProvider.On("AppHash", mock.Anything, uint64(2)).Return([]byte("app_hash_2"), nil) 84 stateProvider.On("Commit", mock.Anything, uint64(1)).Return(commit, nil) 85 stateProvider.On("State", mock.Anything, uint64(1)).Return(state, nil) 86 connSnapshot := &proxymocks.AppConnSnapshot{} 87 connQuery := &proxymocks.AppConnQuery{} 88 89 cfg := config.DefaultStateSyncConfig() 90 syncer := newSyncer(*cfg, log.NewNopLogger(), connSnapshot, connQuery, stateProvider, "") 91 92 // Adding a chunk should error when no sync is in progress 93 _, err := syncer.AddChunk(&chunk{Height: 1, Format: 1, Index: 0, Chunk: []byte{1}}) 94 require.Error(t, err) 95 96 // Adding a couple of peers should trigger snapshot discovery messages 97 peerA := &p2pmocks.Peer{} 98 peerA.On("ID").Return(p2p.ID("a")) 99 peerA.On("Send", SnapshotChannel, mustEncodeMsg(&ssproto.SnapshotsRequest{})).Return(true) 100 syncer.AddPeer(peerA) 101 peerA.AssertExpectations(t) 102 103 peerB := &p2pmocks.Peer{} 104 peerB.On("ID").Return(p2p.ID("b")) 105 peerB.On("Send", SnapshotChannel, mustEncodeMsg(&ssproto.SnapshotsRequest{})).Return(true) 106 syncer.AddPeer(peerB) 107 peerB.AssertExpectations(t) 108 109 // Both peers report back with snapshots. One of them also returns a snapshot we don't want, in 110 // format 2, which will be rejected by the ABCI application. 111 new, err := syncer.AddSnapshot(peerA, s) 112 require.NoError(t, err) 113 assert.True(t, new) 114 115 new, err = syncer.AddSnapshot(peerB, s) 116 require.NoError(t, err) 117 assert.False(t, new) 118 119 new, err = syncer.AddSnapshot(peerB, &snapshot{Height: 2, Format: 2, Chunks: 3, Hash: []byte{1}}) 120 require.NoError(t, err) 121 assert.True(t, new) 122 123 // We start a sync, with peers sending back chunks when requested. We first reject the snapshot 124 // with height 2 format 2, and accept the snapshot at height 1. 125 connSnapshot.On("OfferSnapshotSync", abci.RequestOfferSnapshot{ 126 Snapshot: &abci.Snapshot{ 127 Height: 2, 128 Format: 2, 129 Chunks: 3, 130 Hash: []byte{1}, 131 }, 132 AppHash: []byte("app_hash_2"), 133 }).Return(&abci.ResponseOfferSnapshot{Result: abci.ResponseOfferSnapshot_REJECT_FORMAT}, nil) 134 connSnapshot.On("OfferSnapshotSync", abci.RequestOfferSnapshot{ 135 Snapshot: &abci.Snapshot{ 136 Height: s.Height, 137 Format: s.Format, 138 Chunks: s.Chunks, 139 Hash: s.Hash, 140 Metadata: s.Metadata, 141 }, 142 AppHash: []byte("app_hash"), 143 }).Times(2).Return(&abci.ResponseOfferSnapshot{Result: abci.ResponseOfferSnapshot_ACCEPT}, nil) 144 145 chunkRequests := make(map[uint32]int) 146 chunkRequestsMtx := tmsync.Mutex{} 147 onChunkRequest := func(args mock.Arguments) { 148 pb, err := decodeMsg(args[1].([]byte)) 149 require.NoError(t, err) 150 msg := pb.(*ssproto.ChunkRequest) 151 require.EqualValues(t, 1, msg.Height) 152 require.EqualValues(t, 1, msg.Format) 153 require.LessOrEqual(t, msg.Index, uint32(len(chunks))) 154 155 added, err := syncer.AddChunk(chunks[msg.Index]) 156 require.NoError(t, err) 157 assert.True(t, added) 158 159 chunkRequestsMtx.Lock() 160 chunkRequests[msg.Index]++ 161 chunkRequestsMtx.Unlock() 162 } 163 peerA.On("Send", ChunkChannel, mock.Anything).Maybe().Run(onChunkRequest).Return(true) 164 peerB.On("Send", ChunkChannel, mock.Anything).Maybe().Run(onChunkRequest).Return(true) 165 166 // The first time we're applying chunk 2 we tell it to retry the snapshot and discard chunk 1, 167 // which should cause it to keep the existing chunk 0 and 2, and restart restoration from 168 // beginning. We also wait for a little while, to exercise the retry logic in fetchChunks(). 169 connSnapshot.On("ApplySnapshotChunkSync", abci.RequestApplySnapshotChunk{ 170 Index: 2, Chunk: []byte{1, 1, 2}, 171 }).Once().Run(func(args mock.Arguments) { time.Sleep(2 * time.Second) }).Return( 172 &abci.ResponseApplySnapshotChunk{ 173 Result: abci.ResponseApplySnapshotChunk_RETRY_SNAPSHOT, 174 RefetchChunks: []uint32{1}, 175 }, nil) 176 177 connSnapshot.On("ApplySnapshotChunkSync", abci.RequestApplySnapshotChunk{ 178 Index: 0, Chunk: []byte{1, 1, 0}, 179 }).Times(2).Return(&abci.ResponseApplySnapshotChunk{Result: abci.ResponseApplySnapshotChunk_ACCEPT}, nil) 180 connSnapshot.On("ApplySnapshotChunkSync", abci.RequestApplySnapshotChunk{ 181 Index: 1, Chunk: []byte{1, 1, 1}, 182 }).Times(2).Return(&abci.ResponseApplySnapshotChunk{Result: abci.ResponseApplySnapshotChunk_ACCEPT}, nil) 183 connSnapshot.On("ApplySnapshotChunkSync", abci.RequestApplySnapshotChunk{ 184 Index: 2, Chunk: []byte{1, 1, 2}, 185 }).Once().Return(&abci.ResponseApplySnapshotChunk{Result: abci.ResponseApplySnapshotChunk_ACCEPT}, nil) 186 connQuery.On("InfoSync", proxy.RequestInfo).Return(&abci.ResponseInfo{ 187 AppVersion: 9, 188 LastBlockHeight: 1, 189 LastBlockAppHash: []byte("app_hash"), 190 }, nil) 191 192 newState, lastCommit, err := syncer.SyncAny(0, func() {}) 193 require.NoError(t, err) 194 195 time.Sleep(50 * time.Millisecond) // wait for peers to receive requests 196 197 chunkRequestsMtx.Lock() 198 assert.Equal(t, map[uint32]int{0: 1, 1: 2, 2: 1}, chunkRequests) 199 chunkRequestsMtx.Unlock() 200 201 // The syncer should have updated the state app version from the ABCI info response. 202 expectState := state 203 expectState.Version.Consensus.App = 9 204 205 assert.Equal(t, expectState, newState) 206 assert.Equal(t, commit, lastCommit) 207 208 connSnapshot.AssertExpectations(t) 209 connQuery.AssertExpectations(t) 210 peerA.AssertExpectations(t) 211 peerB.AssertExpectations(t) 212 } 213 214 func TestSyncer_SyncAny_noSnapshots(t *testing.T) { 215 syncer, _ := setupOfferSyncer(t) 216 _, _, err := syncer.SyncAny(0, func() {}) 217 assert.Equal(t, errNoSnapshots, err) 218 } 219 220 func TestSyncer_SyncAny_abort(t *testing.T) { 221 syncer, connSnapshot := setupOfferSyncer(t) 222 223 s := &snapshot{Height: 1, Format: 1, Chunks: 3, Hash: []byte{1, 2, 3}} 224 _, err := syncer.AddSnapshot(simplePeer("id"), s) 225 require.NoError(t, err) 226 connSnapshot.On("OfferSnapshotSync", abci.RequestOfferSnapshot{ 227 Snapshot: toABCI(s), AppHash: []byte("app_hash"), 228 }).Once().Return(&abci.ResponseOfferSnapshot{Result: abci.ResponseOfferSnapshot_ABORT}, nil) 229 230 _, _, err = syncer.SyncAny(0, func() {}) 231 assert.Equal(t, errAbort, err) 232 connSnapshot.AssertExpectations(t) 233 } 234 235 func TestSyncer_SyncAny_reject(t *testing.T) { 236 syncer, connSnapshot := setupOfferSyncer(t) 237 238 // s22 is tried first, then s12, then s11, then errNoSnapshots 239 s22 := &snapshot{Height: 2, Format: 2, Chunks: 3, Hash: []byte{1, 2, 3}} 240 s12 := &snapshot{Height: 1, Format: 2, Chunks: 3, Hash: []byte{1, 2, 3}} 241 s11 := &snapshot{Height: 1, Format: 1, Chunks: 3, Hash: []byte{1, 2, 3}} 242 _, err := syncer.AddSnapshot(simplePeer("id"), s22) 243 require.NoError(t, err) 244 _, err = syncer.AddSnapshot(simplePeer("id"), s12) 245 require.NoError(t, err) 246 _, err = syncer.AddSnapshot(simplePeer("id"), s11) 247 require.NoError(t, err) 248 249 connSnapshot.On("OfferSnapshotSync", abci.RequestOfferSnapshot{ 250 Snapshot: toABCI(s22), AppHash: []byte("app_hash"), 251 }).Once().Return(&abci.ResponseOfferSnapshot{Result: abci.ResponseOfferSnapshot_REJECT}, nil) 252 253 connSnapshot.On("OfferSnapshotSync", abci.RequestOfferSnapshot{ 254 Snapshot: toABCI(s12), AppHash: []byte("app_hash"), 255 }).Once().Return(&abci.ResponseOfferSnapshot{Result: abci.ResponseOfferSnapshot_REJECT}, nil) 256 257 connSnapshot.On("OfferSnapshotSync", abci.RequestOfferSnapshot{ 258 Snapshot: toABCI(s11), AppHash: []byte("app_hash"), 259 }).Once().Return(&abci.ResponseOfferSnapshot{Result: abci.ResponseOfferSnapshot_REJECT}, nil) 260 261 _, _, err = syncer.SyncAny(0, func() {}) 262 assert.Equal(t, errNoSnapshots, err) 263 connSnapshot.AssertExpectations(t) 264 } 265 266 func TestSyncer_SyncAny_reject_format(t *testing.T) { 267 syncer, connSnapshot := setupOfferSyncer(t) 268 269 // s22 is tried first, which reject s22 and s12, then s11 will abort. 270 s22 := &snapshot{Height: 2, Format: 2, Chunks: 3, Hash: []byte{1, 2, 3}} 271 s12 := &snapshot{Height: 1, Format: 2, Chunks: 3, Hash: []byte{1, 2, 3}} 272 s11 := &snapshot{Height: 1, Format: 1, Chunks: 3, Hash: []byte{1, 2, 3}} 273 _, err := syncer.AddSnapshot(simplePeer("id"), s22) 274 require.NoError(t, err) 275 _, err = syncer.AddSnapshot(simplePeer("id"), s12) 276 require.NoError(t, err) 277 _, err = syncer.AddSnapshot(simplePeer("id"), s11) 278 require.NoError(t, err) 279 280 connSnapshot.On("OfferSnapshotSync", abci.RequestOfferSnapshot{ 281 Snapshot: toABCI(s22), AppHash: []byte("app_hash"), 282 }).Once().Return(&abci.ResponseOfferSnapshot{Result: abci.ResponseOfferSnapshot_REJECT_FORMAT}, nil) 283 284 connSnapshot.On("OfferSnapshotSync", abci.RequestOfferSnapshot{ 285 Snapshot: toABCI(s11), AppHash: []byte("app_hash"), 286 }).Once().Return(&abci.ResponseOfferSnapshot{Result: abci.ResponseOfferSnapshot_ABORT}, nil) 287 288 _, _, err = syncer.SyncAny(0, func() {}) 289 assert.Equal(t, errAbort, err) 290 connSnapshot.AssertExpectations(t) 291 } 292 293 func TestSyncer_SyncAny_reject_sender(t *testing.T) { 294 syncer, connSnapshot := setupOfferSyncer(t) 295 296 peerA := simplePeer("a") 297 peerB := simplePeer("b") 298 peerC := simplePeer("c") 299 300 // sbc will be offered first, which will be rejected with reject_sender, causing all snapshots 301 // submitted by both b and c (i.e. sb, sc, sbc) to be rejected. Finally, sa will reject and 302 // errNoSnapshots is returned. 303 sa := &snapshot{Height: 1, Format: 1, Chunks: 3, Hash: []byte{1, 2, 3}} 304 sb := &snapshot{Height: 2, Format: 1, Chunks: 3, Hash: []byte{1, 2, 3}} 305 sc := &snapshot{Height: 3, Format: 1, Chunks: 3, Hash: []byte{1, 2, 3}} 306 sbc := &snapshot{Height: 4, Format: 1, Chunks: 3, Hash: []byte{1, 2, 3}} 307 _, err := syncer.AddSnapshot(peerA, sa) 308 require.NoError(t, err) 309 _, err = syncer.AddSnapshot(peerB, sb) 310 require.NoError(t, err) 311 _, err = syncer.AddSnapshot(peerC, sc) 312 require.NoError(t, err) 313 _, err = syncer.AddSnapshot(peerB, sbc) 314 require.NoError(t, err) 315 _, err = syncer.AddSnapshot(peerC, sbc) 316 require.NoError(t, err) 317 318 connSnapshot.On("OfferSnapshotSync", abci.RequestOfferSnapshot{ 319 Snapshot: toABCI(sbc), AppHash: []byte("app_hash"), 320 }).Once().Return(&abci.ResponseOfferSnapshot{Result: abci.ResponseOfferSnapshot_REJECT_SENDER}, nil) 321 322 connSnapshot.On("OfferSnapshotSync", abci.RequestOfferSnapshot{ 323 Snapshot: toABCI(sa), AppHash: []byte("app_hash"), 324 }).Once().Return(&abci.ResponseOfferSnapshot{Result: abci.ResponseOfferSnapshot_REJECT}, nil) 325 326 _, _, err = syncer.SyncAny(0, func() {}) 327 assert.Equal(t, errNoSnapshots, err) 328 connSnapshot.AssertExpectations(t) 329 } 330 331 func TestSyncer_SyncAny_abciError(t *testing.T) { 332 syncer, connSnapshot := setupOfferSyncer(t) 333 334 errBoom := errors.New("boom") 335 s := &snapshot{Height: 1, Format: 1, Chunks: 3, Hash: []byte{1, 2, 3}} 336 _, err := syncer.AddSnapshot(simplePeer("id"), s) 337 require.NoError(t, err) 338 connSnapshot.On("OfferSnapshotSync", abci.RequestOfferSnapshot{ 339 Snapshot: toABCI(s), AppHash: []byte("app_hash"), 340 }).Once().Return(nil, errBoom) 341 342 _, _, err = syncer.SyncAny(0, func() {}) 343 assert.True(t, errors.Is(err, errBoom)) 344 connSnapshot.AssertExpectations(t) 345 } 346 347 func TestSyncer_offerSnapshot(t *testing.T) { 348 unknownErr := errors.New("unknown error") 349 boom := errors.New("boom") 350 351 testcases := map[string]struct { 352 result abci.ResponseOfferSnapshot_Result 353 err error 354 expectErr error 355 }{ 356 "accept": {abci.ResponseOfferSnapshot_ACCEPT, nil, nil}, 357 "abort": {abci.ResponseOfferSnapshot_ABORT, nil, errAbort}, 358 "reject": {abci.ResponseOfferSnapshot_REJECT, nil, errRejectSnapshot}, 359 "reject_format": {abci.ResponseOfferSnapshot_REJECT_FORMAT, nil, errRejectFormat}, 360 "reject_sender": {abci.ResponseOfferSnapshot_REJECT_SENDER, nil, errRejectSender}, 361 "unknown": {abci.ResponseOfferSnapshot_UNKNOWN, nil, unknownErr}, 362 "error": {0, boom, boom}, 363 "unknown non-zero": {9, nil, unknownErr}, 364 } 365 for name, tc := range testcases { 366 tc := tc 367 t.Run(name, func(t *testing.T) { 368 syncer, connSnapshot := setupOfferSyncer(t) 369 s := &snapshot{Height: 1, Format: 1, Chunks: 3, Hash: []byte{1, 2, 3}, trustedAppHash: []byte("app_hash")} 370 connSnapshot.On("OfferSnapshotSync", abci.RequestOfferSnapshot{ 371 Snapshot: toABCI(s), 372 AppHash: []byte("app_hash"), 373 }).Return(&abci.ResponseOfferSnapshot{Result: tc.result}, tc.err) 374 err := syncer.offerSnapshot(s) 375 if tc.expectErr == unknownErr { 376 require.Error(t, err) 377 } else { 378 unwrapped := errors.Unwrap(err) 379 if unwrapped != nil { 380 err = unwrapped 381 } 382 assert.Equal(t, tc.expectErr, err) 383 } 384 }) 385 } 386 } 387 388 func TestSyncer_applyChunks_Results(t *testing.T) { 389 unknownErr := errors.New("unknown error") 390 boom := errors.New("boom") 391 392 testcases := map[string]struct { 393 result abci.ResponseApplySnapshotChunk_Result 394 err error 395 expectErr error 396 }{ 397 "accept": {abci.ResponseApplySnapshotChunk_ACCEPT, nil, nil}, 398 "abort": {abci.ResponseApplySnapshotChunk_ABORT, nil, errAbort}, 399 "retry": {abci.ResponseApplySnapshotChunk_RETRY, nil, nil}, 400 "retry_snapshot": {abci.ResponseApplySnapshotChunk_RETRY_SNAPSHOT, nil, errRetrySnapshot}, 401 "reject_snapshot": {abci.ResponseApplySnapshotChunk_REJECT_SNAPSHOT, nil, errRejectSnapshot}, 402 "unknown": {abci.ResponseApplySnapshotChunk_UNKNOWN, nil, unknownErr}, 403 "error": {0, boom, boom}, 404 "unknown non-zero": {9, nil, unknownErr}, 405 } 406 for name, tc := range testcases { 407 tc := tc 408 t.Run(name, func(t *testing.T) { 409 connQuery := &proxymocks.AppConnQuery{} 410 connSnapshot := &proxymocks.AppConnSnapshot{} 411 stateProvider := &mocks.StateProvider{} 412 stateProvider.On("AppHash", mock.Anything, mock.Anything).Return([]byte("app_hash"), nil) 413 414 cfg := config.DefaultStateSyncConfig() 415 syncer := newSyncer(*cfg, log.NewNopLogger(), connSnapshot, connQuery, stateProvider, "") 416 417 body := []byte{1, 2, 3} 418 chunks, err := newChunkQueue(&snapshot{Height: 1, Format: 1, Chunks: 1}, "") 419 require.NoError(t, err) 420 _, err = chunks.Add(&chunk{Height: 1, Format: 1, Index: 0, Chunk: body}) 421 require.NoError(t, err) 422 423 connSnapshot.On("ApplySnapshotChunkSync", abci.RequestApplySnapshotChunk{ 424 Index: 0, Chunk: body, 425 }).Once().Return(&abci.ResponseApplySnapshotChunk{Result: tc.result}, tc.err) 426 if tc.result == abci.ResponseApplySnapshotChunk_RETRY { 427 connSnapshot.On("ApplySnapshotChunkSync", abci.RequestApplySnapshotChunk{ 428 Index: 0, Chunk: body, 429 }).Once().Return(&abci.ResponseApplySnapshotChunk{ 430 Result: abci.ResponseApplySnapshotChunk_ACCEPT}, nil) 431 } 432 433 err = syncer.applyChunks(chunks) 434 if tc.expectErr == unknownErr { 435 require.Error(t, err) 436 } else { 437 unwrapped := errors.Unwrap(err) 438 if unwrapped != nil { 439 err = unwrapped 440 } 441 assert.Equal(t, tc.expectErr, err) 442 } 443 connSnapshot.AssertExpectations(t) 444 }) 445 } 446 } 447 448 func TestSyncer_applyChunks_RefetchChunks(t *testing.T) { 449 // Discarding chunks via refetch_chunks should work the same for all results 450 testcases := map[string]struct { 451 result abci.ResponseApplySnapshotChunk_Result 452 }{ 453 "accept": {abci.ResponseApplySnapshotChunk_ACCEPT}, 454 "abort": {abci.ResponseApplySnapshotChunk_ABORT}, 455 "retry": {abci.ResponseApplySnapshotChunk_RETRY}, 456 "retry_snapshot": {abci.ResponseApplySnapshotChunk_RETRY_SNAPSHOT}, 457 "reject_snapshot": {abci.ResponseApplySnapshotChunk_REJECT_SNAPSHOT}, 458 } 459 for name, tc := range testcases { 460 tc := tc 461 t.Run(name, func(t *testing.T) { 462 connQuery := &proxymocks.AppConnQuery{} 463 connSnapshot := &proxymocks.AppConnSnapshot{} 464 stateProvider := &mocks.StateProvider{} 465 stateProvider.On("AppHash", mock.Anything, mock.Anything).Return([]byte("app_hash"), nil) 466 467 cfg := config.DefaultStateSyncConfig() 468 syncer := newSyncer(*cfg, log.NewNopLogger(), connSnapshot, connQuery, stateProvider, "") 469 470 chunks, err := newChunkQueue(&snapshot{Height: 1, Format: 1, Chunks: 3}, "") 471 require.NoError(t, err) 472 added, err := chunks.Add(&chunk{Height: 1, Format: 1, Index: 0, Chunk: []byte{0}}) 473 require.True(t, added) 474 require.NoError(t, err) 475 added, err = chunks.Add(&chunk{Height: 1, Format: 1, Index: 1, Chunk: []byte{1}}) 476 require.True(t, added) 477 require.NoError(t, err) 478 added, err = chunks.Add(&chunk{Height: 1, Format: 1, Index: 2, Chunk: []byte{2}}) 479 require.True(t, added) 480 require.NoError(t, err) 481 482 // The first two chunks are accepted, before the last one asks for 1 to be refetched 483 connSnapshot.On("ApplySnapshotChunkSync", abci.RequestApplySnapshotChunk{ 484 Index: 0, Chunk: []byte{0}, 485 }).Once().Return(&abci.ResponseApplySnapshotChunk{Result: abci.ResponseApplySnapshotChunk_ACCEPT}, nil) 486 connSnapshot.On("ApplySnapshotChunkSync", abci.RequestApplySnapshotChunk{ 487 Index: 1, Chunk: []byte{1}, 488 }).Once().Return(&abci.ResponseApplySnapshotChunk{Result: abci.ResponseApplySnapshotChunk_ACCEPT}, nil) 489 connSnapshot.On("ApplySnapshotChunkSync", abci.RequestApplySnapshotChunk{ 490 Index: 2, Chunk: []byte{2}, 491 }).Once().Return(&abci.ResponseApplySnapshotChunk{ 492 Result: tc.result, 493 RefetchChunks: []uint32{1}, 494 }, nil) 495 496 // Since removing the chunk will cause Next() to block, we spawn a goroutine, then 497 // check the queue contents, and finally close the queue to end the goroutine. 498 // We don't really care about the result of applyChunks, since it has separate test. 499 go func() { 500 syncer.applyChunks(chunks) //nolint:errcheck // purposefully ignore error 501 }() 502 503 time.Sleep(50 * time.Millisecond) 504 assert.True(t, chunks.Has(0)) 505 assert.False(t, chunks.Has(1)) 506 assert.True(t, chunks.Has(2)) 507 err = chunks.Close() 508 require.NoError(t, err) 509 }) 510 } 511 } 512 513 func TestSyncer_applyChunks_RejectSenders(t *testing.T) { 514 // Banning chunks senders via ban_chunk_senders should work the same for all results 515 testcases := map[string]struct { 516 result abci.ResponseApplySnapshotChunk_Result 517 }{ 518 "accept": {abci.ResponseApplySnapshotChunk_ACCEPT}, 519 "abort": {abci.ResponseApplySnapshotChunk_ABORT}, 520 "retry": {abci.ResponseApplySnapshotChunk_RETRY}, 521 "retry_snapshot": {abci.ResponseApplySnapshotChunk_RETRY_SNAPSHOT}, 522 "reject_snapshot": {abci.ResponseApplySnapshotChunk_REJECT_SNAPSHOT}, 523 } 524 for name, tc := range testcases { 525 tc := tc 526 t.Run(name, func(t *testing.T) { 527 connQuery := &proxymocks.AppConnQuery{} 528 connSnapshot := &proxymocks.AppConnSnapshot{} 529 stateProvider := &mocks.StateProvider{} 530 stateProvider.On("AppHash", mock.Anything, mock.Anything).Return([]byte("app_hash"), nil) 531 532 cfg := config.DefaultStateSyncConfig() 533 syncer := newSyncer(*cfg, log.NewNopLogger(), connSnapshot, connQuery, stateProvider, "") 534 535 // Set up three peers across two snapshots, and ask for one of them to be banned. 536 // It should be banned from all snapshots. 537 peerA := simplePeer("a") 538 peerB := simplePeer("b") 539 peerC := simplePeer("c") 540 541 s1 := &snapshot{Height: 1, Format: 1, Chunks: 3} 542 s2 := &snapshot{Height: 2, Format: 1, Chunks: 3} 543 _, err := syncer.AddSnapshot(peerA, s1) 544 require.NoError(t, err) 545 _, err = syncer.AddSnapshot(peerA, s2) 546 require.NoError(t, err) 547 _, err = syncer.AddSnapshot(peerB, s1) 548 require.NoError(t, err) 549 _, err = syncer.AddSnapshot(peerB, s2) 550 require.NoError(t, err) 551 _, err = syncer.AddSnapshot(peerC, s1) 552 require.NoError(t, err) 553 _, err = syncer.AddSnapshot(peerC, s2) 554 require.NoError(t, err) 555 556 chunks, err := newChunkQueue(s1, "") 557 require.NoError(t, err) 558 added, err := chunks.Add(&chunk{Height: 1, Format: 1, Index: 0, Chunk: []byte{0}, Sender: peerA.ID()}) 559 require.True(t, added) 560 require.NoError(t, err) 561 added, err = chunks.Add(&chunk{Height: 1, Format: 1, Index: 1, Chunk: []byte{1}, Sender: peerB.ID()}) 562 require.True(t, added) 563 require.NoError(t, err) 564 added, err = chunks.Add(&chunk{Height: 1, Format: 1, Index: 2, Chunk: []byte{2}, Sender: peerC.ID()}) 565 require.True(t, added) 566 require.NoError(t, err) 567 568 // The first two chunks are accepted, before the last one asks for b sender to be rejected 569 connSnapshot.On("ApplySnapshotChunkSync", abci.RequestApplySnapshotChunk{ 570 Index: 0, Chunk: []byte{0}, Sender: "a", 571 }).Once().Return(&abci.ResponseApplySnapshotChunk{Result: abci.ResponseApplySnapshotChunk_ACCEPT}, nil) 572 connSnapshot.On("ApplySnapshotChunkSync", abci.RequestApplySnapshotChunk{ 573 Index: 1, Chunk: []byte{1}, Sender: "b", 574 }).Once().Return(&abci.ResponseApplySnapshotChunk{Result: abci.ResponseApplySnapshotChunk_ACCEPT}, nil) 575 connSnapshot.On("ApplySnapshotChunkSync", abci.RequestApplySnapshotChunk{ 576 Index: 2, Chunk: []byte{2}, Sender: "c", 577 }).Once().Return(&abci.ResponseApplySnapshotChunk{ 578 Result: tc.result, 579 RejectSenders: []string{string(peerB.ID())}, 580 }, nil) 581 582 // On retry, the last chunk will be tried again, so we just accept it then. 583 if tc.result == abci.ResponseApplySnapshotChunk_RETRY { 584 connSnapshot.On("ApplySnapshotChunkSync", abci.RequestApplySnapshotChunk{ 585 Index: 2, Chunk: []byte{2}, Sender: "c", 586 }).Once().Return(&abci.ResponseApplySnapshotChunk{Result: abci.ResponseApplySnapshotChunk_ACCEPT}, nil) 587 } 588 589 // We don't really care about the result of applyChunks, since it has separate test. 590 // However, it will block on e.g. retry result, so we spawn a goroutine that will 591 // be shut down when the chunk queue closes. 592 go func() { 593 syncer.applyChunks(chunks) //nolint:errcheck // purposefully ignore error 594 }() 595 596 time.Sleep(50 * time.Millisecond) 597 598 s1peers := syncer.snapshots.GetPeers(s1) 599 assert.Len(t, s1peers, 2) 600 assert.EqualValues(t, "a", s1peers[0].ID()) 601 assert.EqualValues(t, "c", s1peers[1].ID()) 602 603 syncer.snapshots.GetPeers(s1) 604 assert.Len(t, s1peers, 2) 605 assert.EqualValues(t, "a", s1peers[0].ID()) 606 assert.EqualValues(t, "c", s1peers[1].ID()) 607 608 err = chunks.Close() 609 require.NoError(t, err) 610 }) 611 } 612 } 613 614 func TestSyncer_verifyApp(t *testing.T) { 615 boom := errors.New("boom") 616 s := &snapshot{Height: 3, Format: 1, Chunks: 5, Hash: []byte{1, 2, 3}, trustedAppHash: []byte("app_hash")} 617 618 testcases := map[string]struct { 619 response *abci.ResponseInfo 620 err error 621 expectErr error 622 }{ 623 "verified": {&abci.ResponseInfo{ 624 LastBlockHeight: 3, 625 LastBlockAppHash: []byte("app_hash"), 626 AppVersion: 9, 627 }, nil, nil}, 628 "invalid height": {&abci.ResponseInfo{ 629 LastBlockHeight: 5, 630 LastBlockAppHash: []byte("app_hash"), 631 AppVersion: 9, 632 }, nil, errVerifyFailed}, 633 "invalid hash": {&abci.ResponseInfo{ 634 LastBlockHeight: 3, 635 LastBlockAppHash: []byte("xxx"), 636 AppVersion: 9, 637 }, nil, errVerifyFailed}, 638 "error": {nil, boom, boom}, 639 } 640 for name, tc := range testcases { 641 tc := tc 642 t.Run(name, func(t *testing.T) { 643 connQuery := &proxymocks.AppConnQuery{} 644 connSnapshot := &proxymocks.AppConnSnapshot{} 645 stateProvider := &mocks.StateProvider{} 646 647 cfg := config.DefaultStateSyncConfig() 648 syncer := newSyncer(*cfg, log.NewNopLogger(), connSnapshot, connQuery, stateProvider, "") 649 650 connQuery.On("InfoSync", proxy.RequestInfo).Return(tc.response, tc.err) 651 version, err := syncer.verifyApp(s) 652 unwrapped := errors.Unwrap(err) 653 if unwrapped != nil { 654 err = unwrapped 655 } 656 assert.Equal(t, tc.expectErr, err) 657 if err == nil { 658 assert.Equal(t, tc.response.AppVersion, version) 659 } 660 }) 661 } 662 } 663 664 func toABCI(s *snapshot) *abci.Snapshot { 665 return &abci.Snapshot{ 666 Height: s.Height, 667 Format: s.Format, 668 Chunks: s.Chunks, 669 Hash: s.Hash, 670 Metadata: s.Metadata, 671 } 672 }