github.com/KYVENetwork/cometbft/v38@v38.0.3/evidence/reactor_test.go (about) 1 package evidence_test 2 3 import ( 4 "encoding/hex" 5 "fmt" 6 "sync" 7 "testing" 8 "time" 9 10 "github.com/fortytw2/leaktest" 11 "github.com/go-kit/log/term" 12 "github.com/stretchr/testify/assert" 13 "github.com/stretchr/testify/mock" 14 "github.com/stretchr/testify/require" 15 16 dbm "github.com/cometbft/cometbft-db" 17 18 cfg "github.com/KYVENetwork/cometbft/v38/config" 19 "github.com/KYVENetwork/cometbft/v38/crypto" 20 "github.com/KYVENetwork/cometbft/v38/crypto/tmhash" 21 "github.com/KYVENetwork/cometbft/v38/evidence" 22 "github.com/KYVENetwork/cometbft/v38/evidence/mocks" 23 "github.com/KYVENetwork/cometbft/v38/libs/log" 24 "github.com/KYVENetwork/cometbft/v38/p2p" 25 p2pmocks "github.com/KYVENetwork/cometbft/v38/p2p/mocks" 26 cmtproto "github.com/KYVENetwork/cometbft/v38/proto/cometbft/v38/types" 27 sm "github.com/KYVENetwork/cometbft/v38/state" 28 "github.com/KYVENetwork/cometbft/v38/types" 29 ) 30 31 var ( 32 numEvidence = 10 33 timeout = 120 * time.Second // ridiculously high because CircleCI is slow 34 ) 35 36 // We have N evidence reactors connected to one another. The first reactor 37 // receives a number of evidence at varying heights. We test that all 38 // other reactors receive the evidence and add it to their own respective 39 // evidence pools. 40 func TestReactorBroadcastEvidence(t *testing.T) { 41 config := cfg.TestConfig() 42 N := 7 43 44 // create statedb for everyone 45 stateDBs := make([]sm.Store, N) 46 val := types.NewMockPV() 47 // we need validators saved for heights at least as high as we have evidence for 48 height := int64(numEvidence) + 10 49 for i := 0; i < N; i++ { 50 stateDBs[i] = initializeValidatorState(val, height) 51 } 52 53 // make reactors from statedb 54 reactors, pools := makeAndConnectReactorsAndPools(config, stateDBs) 55 56 // set the peer height on each reactor 57 for _, r := range reactors { 58 for _, peer := range r.Switch.Peers().List() { 59 ps := peerState{height} 60 peer.Set(types.PeerStateKey, ps) 61 } 62 } 63 64 // send a bunch of valid evidence to the first reactor's evpool 65 // and wait for them all to be received in the others 66 evList := sendEvidence(t, pools[0], val, numEvidence) 67 waitForEvidence(t, evList, pools) 68 } 69 70 // We have two evidence reactors connected to one another but are at different heights. 71 // Reactor 1 which is ahead receives a number of evidence. It should only send the evidence 72 // that is below the height of the peer to that peer. 73 func TestReactorSelectiveBroadcast(t *testing.T) { 74 config := cfg.TestConfig() 75 76 val := types.NewMockPV() 77 height1 := int64(numEvidence) + 10 78 height2 := int64(numEvidence) / 2 79 80 // DB1 is ahead of DB2 81 stateDB1 := initializeValidatorState(val, height1) 82 stateDB2 := initializeValidatorState(val, height2) 83 84 // make reactors from statedb 85 reactors, pools := makeAndConnectReactorsAndPools(config, []sm.Store{stateDB1, stateDB2}) 86 87 // set the peer height on each reactor 88 for _, r := range reactors { 89 for _, peer := range r.Switch.Peers().List() { 90 ps := peerState{height1} 91 peer.Set(types.PeerStateKey, ps) 92 } 93 } 94 95 // update the first reactor peer's height to be very small 96 peer := reactors[0].Switch.Peers().List()[0] 97 ps := peerState{height2} 98 peer.Set(types.PeerStateKey, ps) 99 100 // send a bunch of valid evidence to the first reactor's evpool 101 evList := sendEvidence(t, pools[0], val, numEvidence) 102 103 // only ones less than the peers height should make it through 104 waitForEvidence(t, evList[:numEvidence/2-1], []*evidence.Pool{pools[1]}) 105 106 // peers should still be connected 107 peers := reactors[1].Switch.Peers().List() 108 assert.Equal(t, 1, len(peers)) 109 } 110 111 // This tests aims to ensure that reactors don't send evidence that they have committed or that ar 112 // not ready for the peer through three scenarios. 113 // First, committed evidence to a newly connected peer 114 // Second, evidence to a peer that is behind 115 // Third, evidence that was pending and became committed just before the peer caught up 116 func TestReactorsGossipNoCommittedEvidence(t *testing.T) { 117 config := cfg.TestConfig() 118 119 val := types.NewMockPV() 120 var height int64 = 10 121 122 // DB1 is ahead of DB2 123 stateDB1 := initializeValidatorState(val, height-1) 124 stateDB2 := initializeValidatorState(val, height-2) 125 state, err := stateDB1.Load() 126 require.NoError(t, err) 127 state.LastBlockHeight++ 128 129 // make reactors from statedb 130 reactors, pools := makeAndConnectReactorsAndPools(config, []sm.Store{stateDB1, stateDB2}) 131 132 evList := sendEvidence(t, pools[0], val, 2) 133 pools[0].Update(state, evList) 134 require.EqualValues(t, uint32(0), pools[0].Size()) 135 136 time.Sleep(100 * time.Millisecond) 137 138 peer := reactors[0].Switch.Peers().List()[0] 139 ps := peerState{height - 2} 140 peer.Set(types.PeerStateKey, ps) 141 142 peer = reactors[1].Switch.Peers().List()[0] 143 ps = peerState{height} 144 peer.Set(types.PeerStateKey, ps) 145 146 // wait to see that no evidence comes through 147 time.Sleep(300 * time.Millisecond) 148 149 // the second pool should not have received any evidence because it has already been committed 150 assert.Equal(t, uint32(0), pools[1].Size(), "second reactor should not have received evidence") 151 152 // the first reactor receives three more evidence 153 evList = make([]types.Evidence, 3) 154 for i := 0; i < 3; i++ { 155 ev, err := types.NewMockDuplicateVoteEvidenceWithValidator(height-3+int64(i), 156 time.Date(2019, 1, 1, 0, 0, 0, 0, time.UTC), val, state.ChainID) 157 require.NoError(t, err) 158 err = pools[0].AddEvidence(ev) 159 require.NoError(t, err) 160 evList[i] = ev 161 } 162 163 // wait to see that only one evidence is sent 164 time.Sleep(300 * time.Millisecond) 165 166 // the second pool should only have received the first evidence because it is behind 167 peerEv, _ := pools[1].PendingEvidence(10000) 168 assert.EqualValues(t, []types.Evidence{evList[0]}, peerEv) 169 170 // the last evidence is committed and the second reactor catches up in state to the first 171 // reactor. We therefore expect that the second reactor only receives one more evidence, the 172 // one that is still pending and not the evidence that has already been committed. 173 state.LastBlockHeight++ 174 pools[0].Update(state, []types.Evidence{evList[2]}) 175 // the first reactor should have the two remaining pending evidence 176 require.EqualValues(t, uint32(2), pools[0].Size()) 177 178 // now update the state of the second reactor 179 pools[1].Update(state, types.EvidenceList{}) 180 peer = reactors[0].Switch.Peers().List()[0] 181 ps = peerState{height} 182 peer.Set(types.PeerStateKey, ps) 183 184 // wait to see that only two evidence is sent 185 time.Sleep(300 * time.Millisecond) 186 187 peerEv, _ = pools[1].PendingEvidence(1000) 188 assert.EqualValues(t, []types.Evidence{evList[0], evList[1]}, peerEv) 189 } 190 191 func TestReactorBroadcastEvidenceMemoryLeak(t *testing.T) { 192 evidenceTime := time.Date(2019, 1, 1, 0, 0, 0, 0, time.UTC) 193 evidenceDB := dbm.NewMemDB() 194 blockStore := &mocks.BlockStore{} 195 blockStore.On("LoadBlockMeta", mock.AnythingOfType("int64")).Return( 196 &types.BlockMeta{Header: types.Header{Time: evidenceTime}}, 197 ) 198 val := types.NewMockPV() 199 stateStore := initializeValidatorState(val, 1) 200 pool, err := evidence.NewPool(evidenceDB, stateStore, blockStore) 201 require.NoError(t, err) 202 203 p := &p2pmocks.Peer{} 204 205 p.On("IsRunning").Once().Return(true) 206 p.On("IsRunning").Return(false) 207 // check that we are not leaking any go-routines 208 // i.e. broadcastEvidenceRoutine finishes when peer is stopped 209 defer leaktest.CheckTimeout(t, 10*time.Second)() 210 211 p.On("Send", mock.MatchedBy(func(i interface{}) bool { 212 e, ok := i.(p2p.Envelope) 213 return ok && e.ChannelID == evidence.EvidenceChannel 214 })).Return(false) 215 quitChan := make(<-chan struct{}) 216 p.On("Quit").Return(quitChan) 217 ps := peerState{2} 218 p.On("Get", types.PeerStateKey).Return(ps) 219 p.On("ID").Return("ABC") 220 p.On("String").Return("mock") 221 222 r := evidence.NewReactor(pool) 223 r.SetLogger(log.TestingLogger()) 224 r.AddPeer(p) 225 226 _ = sendEvidence(t, pool, val, 2) 227 } 228 229 // evidenceLogger is a TestingLogger which uses a different 230 // color for each validator ("validator" key must exist). 231 func evidenceLogger() log.Logger { 232 return log.TestingLoggerWithColorFn(func(keyvals ...interface{}) term.FgBgColor { 233 for i := 0; i < len(keyvals)-1; i += 2 { 234 if keyvals[i] == "validator" { 235 return term.FgBgColor{Fg: term.Color(uint8(keyvals[i+1].(int) + 1))} 236 } 237 } 238 return term.FgBgColor{} 239 }) 240 } 241 242 // connect N evidence reactors through N switches 243 func makeAndConnectReactorsAndPools(config *cfg.Config, stateStores []sm.Store) ([]*evidence.Reactor, 244 []*evidence.Pool, 245 ) { 246 N := len(stateStores) 247 248 reactors := make([]*evidence.Reactor, N) 249 pools := make([]*evidence.Pool, N) 250 logger := evidenceLogger() 251 evidenceTime := time.Date(2019, 1, 1, 0, 0, 0, 0, time.UTC) 252 253 for i := 0; i < N; i++ { 254 evidenceDB := dbm.NewMemDB() 255 blockStore := &mocks.BlockStore{} 256 blockStore.On("LoadBlockMeta", mock.AnythingOfType("int64")).Return( 257 &types.BlockMeta{Header: types.Header{Time: evidenceTime}}, 258 ) 259 pool, err := evidence.NewPool(evidenceDB, stateStores[i], blockStore) 260 if err != nil { 261 panic(err) 262 } 263 pools[i] = pool 264 reactors[i] = evidence.NewReactor(pool) 265 reactors[i].SetLogger(logger.With("validator", i)) 266 } 267 268 p2p.MakeConnectedSwitches(config.P2P, N, func(i int, s *p2p.Switch) *p2p.Switch { 269 s.AddReactor("EVIDENCE", reactors[i]) 270 return s 271 }, p2p.Connect2Switches) 272 273 return reactors, pools 274 } 275 276 // wait for all evidence on all reactors 277 func waitForEvidence(t *testing.T, evs types.EvidenceList, pools []*evidence.Pool) { 278 // wait for the evidence in all evpools 279 wg := new(sync.WaitGroup) 280 for i := 0; i < len(pools); i++ { 281 wg.Add(1) 282 go _waitForEvidence(t, wg, evs, i, pools) 283 } 284 285 done := make(chan struct{}) 286 go func() { 287 wg.Wait() 288 close(done) 289 }() 290 291 timer := time.After(timeout) 292 select { 293 case <-timer: 294 t.Fatal("Timed out waiting for evidence") 295 case <-done: 296 } 297 } 298 299 // wait for all evidence on a single evpool 300 func _waitForEvidence( 301 t *testing.T, 302 wg *sync.WaitGroup, 303 evs types.EvidenceList, 304 poolIdx int, 305 pools []*evidence.Pool, 306 ) { 307 evpool := pools[poolIdx] 308 var evList []types.Evidence 309 currentPoolSize := 0 310 for currentPoolSize != len(evs) { 311 evList, _ = evpool.PendingEvidence(int64(len(evs) * 500)) // each evidence should not be more than 500 bytes 312 currentPoolSize = len(evList) 313 time.Sleep(time.Millisecond * 100) 314 } 315 316 // put the reaped evidence in a map so we can quickly check we got everything 317 evMap := make(map[string]types.Evidence) 318 for _, e := range evList { 319 evMap[string(e.Hash())] = e 320 } 321 for i, expectedEv := range evs { 322 gotEv := evMap[string(expectedEv.Hash())] 323 assert.Equal(t, expectedEv, gotEv, 324 fmt.Sprintf("evidence at index %d on pool %d don't match: %v vs %v", 325 i, poolIdx, expectedEv, gotEv)) 326 } 327 328 wg.Done() 329 } 330 331 func sendEvidence(t *testing.T, evpool *evidence.Pool, val types.PrivValidator, n int) types.EvidenceList { 332 evList := make([]types.Evidence, n) 333 for i := 0; i < n; i++ { 334 ev, err := types.NewMockDuplicateVoteEvidenceWithValidator(int64(i+1), 335 time.Date(2019, 1, 1, 0, 0, 0, 0, time.UTC), val, evidenceChainID) 336 require.NoError(t, err) 337 err = evpool.AddEvidence(ev) 338 require.NoError(t, err) 339 evList[i] = ev 340 } 341 return evList 342 } 343 344 type peerState struct { 345 height int64 346 } 347 348 func (ps peerState) GetHeight() int64 { 349 return ps.height 350 } 351 352 func exampleVote(t byte) *types.Vote { 353 stamp, err := time.Parse(types.TimeFormat, "2017-12-25T03:00:01.234Z") 354 if err != nil { 355 panic(err) 356 } 357 358 return &types.Vote{ 359 Type: cmtproto.SignedMsgType(t), 360 Height: 3, 361 Round: 2, 362 Timestamp: stamp, 363 BlockID: types.BlockID{ 364 Hash: tmhash.Sum([]byte("blockID_hash")), 365 PartSetHeader: types.PartSetHeader{ 366 Total: 1000000, 367 Hash: tmhash.Sum([]byte("blockID_part_set_header_hash")), 368 }, 369 }, 370 ValidatorAddress: crypto.AddressHash([]byte("validator_address")), 371 ValidatorIndex: 56789, 372 } 373 } 374 375 //nolint:lll //ignore line length for tests 376 func TestEvidenceVectors(t *testing.T) { 377 val := &types.Validator{ 378 Address: crypto.AddressHash([]byte("validator_address")), 379 VotingPower: 10, 380 } 381 382 valSet := types.NewValidatorSet([]*types.Validator{val}) 383 384 dupl, err := types.NewDuplicateVoteEvidence( 385 exampleVote(1), 386 exampleVote(2), 387 defaultEvidenceTime, 388 valSet, 389 ) 390 require.NoError(t, err) 391 392 testCases := []struct { 393 testName string 394 evidenceList []types.Evidence 395 expBytes string 396 }{ 397 {"DuplicateVoteEvidence", []types.Evidence{dupl}, "0a85020a82020a79080210031802224a0a208b01023386c371778ecb6368573e539afc3cc860ec3a2f614e54fe5652f4fc80122608c0843d122072db3d959635dff1bb567bedaa70573392c5159666a3f8caf11e413aac52207a2a0b08b1d381d20510809dca6f32146af1f4111082efb388211bc72c55bcd61e9ac3d538d5bb031279080110031802224a0a208b01023386c371778ecb6368573e539afc3cc860ec3a2f614e54fe5652f4fc80122608c0843d122072db3d959635dff1bb567bedaa70573392c5159666a3f8caf11e413aac52207a2a0b08b1d381d20510809dca6f32146af1f4111082efb388211bc72c55bcd61e9ac3d538d5bb03180a200a2a060880dbaae105"}, 398 } 399 400 for _, tc := range testCases { 401 tc := tc 402 403 evi := make([]cmtproto.Evidence, len(tc.evidenceList)) 404 for i := 0; i < len(tc.evidenceList); i++ { 405 ev, err := types.EvidenceToProto(tc.evidenceList[i]) 406 require.NoError(t, err, tc.testName) 407 evi[i] = *ev 408 } 409 410 epl := cmtproto.EvidenceList{ 411 Evidence: evi, 412 } 413 414 bz, err := epl.Marshal() 415 require.NoError(t, err, tc.testName) 416 417 require.Equal(t, tc.expBytes, hex.EncodeToString(bz), tc.testName) 418 419 } 420 }