github.com/vipernet-xyz/tendermint-core@v0.32.0/blockchain/v0/reactor.go (about) 1 package v0 2 3 import ( 4 "errors" 5 "fmt" 6 "reflect" 7 "time" 8 9 amino "github.com/tendermint/go-amino" 10 11 "github.com/tendermint/tendermint/libs/log" 12 "github.com/tendermint/tendermint/p2p" 13 sm "github.com/tendermint/tendermint/state" 14 "github.com/tendermint/tendermint/store" 15 "github.com/tendermint/tendermint/types" 16 ) 17 18 const ( 19 // BlockchainChannel is a channel for blocks and status updates (`BlockStore` height) 20 BlockchainChannel = byte(0x40) 21 22 trySyncIntervalMS = 10 23 24 // stop syncing when last block's time is 25 // within this much of the system time. 26 // stopSyncingDurationMinutes = 10 27 28 // ask for best height every 10s 29 statusUpdateIntervalSeconds = 10 30 // check if we should switch to consensus reactor 31 switchToConsensusIntervalSeconds = 1 32 33 // NOTE: keep up to date with bcBlockResponseMessage 34 bcBlockResponseMessagePrefixSize = 4 35 bcBlockResponseMessageFieldKeySize = 1 36 maxMsgSize = types.MaxBlockSizeBytes + 37 bcBlockResponseMessagePrefixSize + 38 bcBlockResponseMessageFieldKeySize 39 ) 40 41 type consensusReactor interface { 42 // for when we switch from blockchain reactor and fast sync to 43 // the consensus machine 44 SwitchToConsensus(sm.State, uint64) 45 } 46 47 type peerError struct { 48 err error 49 peerID p2p.ID 50 } 51 52 func (e peerError) Error() string { 53 return fmt.Sprintf("error with peer %v: %s", e.peerID, e.err.Error()) 54 } 55 56 // BlockchainReactor handles long-term catchup syncing. 57 type BlockchainReactor struct { 58 p2p.BaseReactor 59 60 // immutable 61 initialState sm.State 62 63 blockExec *sm.BlockExecutor 64 store *store.BlockStore 65 pool *BlockPool 66 fastSync bool 67 68 requestsCh <-chan BlockRequest 69 errorsCh <-chan peerError 70 } 71 72 // NewBlockchainReactor returns new reactor instance. 73 func NewBlockchainReactor(state sm.State, blockExec *sm.BlockExecutor, store *store.BlockStore, 74 fastSync bool) *BlockchainReactor { 75 76 if state.LastBlockHeight != store.Height() { 77 panic(fmt.Sprintf("state (%v) and store (%v) height mismatch", state.LastBlockHeight, 78 store.Height())) 79 } 80 81 requestsCh := make(chan BlockRequest, maxTotalRequesters) 82 83 const capacity = 1000 // must be bigger than peers count 84 errorsCh := make(chan peerError, capacity) // so we don't block in #Receive#pool.AddBlock 85 86 pool := NewBlockPool( 87 store.Height()+1, 88 requestsCh, 89 errorsCh, 90 ) 91 92 bcR := &BlockchainReactor{ 93 initialState: state, 94 blockExec: blockExec, 95 store: store, 96 pool: pool, 97 fastSync: fastSync, 98 requestsCh: requestsCh, 99 errorsCh: errorsCh, 100 } 101 bcR.BaseReactor = *p2p.NewBaseReactor("BlockchainReactor", bcR) 102 return bcR 103 } 104 105 // SetLogger implements service.Service by setting the logger on reactor and pool. 106 func (bcR *BlockchainReactor) SetLogger(l log.Logger) { 107 bcR.BaseService.Logger = l 108 bcR.pool.Logger = l 109 } 110 111 // OnStart implements service.Service. 112 func (bcR *BlockchainReactor) OnStart() error { 113 if bcR.fastSync { 114 err := bcR.pool.Start() 115 if err != nil { 116 return err 117 } 118 go bcR.poolRoutine() 119 } 120 return nil 121 } 122 123 // OnStop implements service.Service. 124 func (bcR *BlockchainReactor) OnStop() { 125 bcR.pool.Stop() 126 } 127 128 // GetChannels implements Reactor 129 func (bcR *BlockchainReactor) GetChannels() []*p2p.ChannelDescriptor { 130 return []*p2p.ChannelDescriptor{ 131 { 132 ID: BlockchainChannel, 133 Priority: 10, 134 SendQueueCapacity: 1000, 135 RecvBufferCapacity: 50 * 4096, 136 RecvMessageCapacity: maxMsgSize, 137 }, 138 } 139 } 140 141 // AddPeer implements Reactor by sending our state to peer. 142 func (bcR *BlockchainReactor) AddPeer(peer p2p.Peer) { 143 msgBytes := cdc.MustMarshalBinaryBare(&bcStatusResponseMessage{ 144 Height: bcR.store.Height(), 145 Base: bcR.store.Base(), 146 }) 147 peer.Send(BlockchainChannel, msgBytes) 148 // it's OK if send fails. will try later in poolRoutine 149 150 // peer is added to the pool once we receive the first 151 // bcStatusResponseMessage from the peer and call pool.SetPeerRange 152 } 153 154 // RemovePeer implements Reactor by removing peer from the pool. 155 func (bcR *BlockchainReactor) RemovePeer(peer p2p.Peer, reason interface{}) { 156 bcR.pool.RemovePeer(peer.ID()) 157 } 158 159 // respondToPeer loads a block and sends it to the requesting peer, 160 // if we have it. Otherwise, we'll respond saying we don't have it. 161 func (bcR *BlockchainReactor) respondToPeer(msg *bcBlockRequestMessage, 162 src p2p.Peer) (queued bool) { 163 164 block := bcR.store.LoadBlock(msg.Height) 165 if block != nil { 166 msgBytes := cdc.MustMarshalBinaryBare(&bcBlockResponseMessage{Block: block}) 167 return src.TrySend(BlockchainChannel, msgBytes) 168 } 169 170 bcR.Logger.Info("Peer asking for a block we don't have", "src", src, "height", msg.Height) 171 172 msgBytes := cdc.MustMarshalBinaryBare(&bcNoBlockResponseMessage{Height: msg.Height}) 173 return src.TrySend(BlockchainChannel, msgBytes) 174 } 175 176 // Receive implements Reactor by handling 4 types of messages (look below). 177 func (bcR *BlockchainReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte) { 178 msg, err := decodeMsg(msgBytes) 179 if err != nil { 180 bcR.Logger.Error("Error decoding message", "src", src, "chId", chID, "msg", msg, "err", err, "bytes", msgBytes) 181 bcR.Switch.StopPeerForError(src, err) 182 return 183 } 184 185 if err = msg.ValidateBasic(); err != nil { 186 bcR.Logger.Error("Peer sent us invalid msg", "peer", src, "msg", msg, "err", err) 187 bcR.Switch.StopPeerForError(src, err) 188 return 189 } 190 191 bcR.Logger.Debug("Receive", "src", src, "chID", chID, "msg", msg) 192 193 switch msg := msg.(type) { 194 case *bcBlockRequestMessage: 195 bcR.respondToPeer(msg, src) 196 case *bcBlockResponseMessage: 197 bcR.pool.AddBlock(src.ID(), msg.Block, len(msgBytes)) 198 case *bcStatusRequestMessage: 199 // Send peer our state. 200 src.TrySend(BlockchainChannel, cdc.MustMarshalBinaryBare(&bcStatusResponseMessage{ 201 Height: bcR.store.Height(), 202 Base: bcR.store.Base(), 203 })) 204 case *bcStatusResponseMessage: 205 // Got a peer status. Unverified. 206 bcR.pool.SetPeerRange(src.ID(), msg.Base, msg.Height) 207 case *bcNoBlockResponseMessage: 208 bcR.Logger.Debug("Peer does not have requested block", "peer", src, "height", msg.Height) 209 default: 210 bcR.Logger.Error(fmt.Sprintf("Unknown message type %v", reflect.TypeOf(msg))) 211 } 212 } 213 214 // Handle messages from the poolReactor telling the reactor what to do. 215 // NOTE: Don't sleep in the FOR_LOOP or otherwise slow it down! 216 func (bcR *BlockchainReactor) poolRoutine() { 217 218 trySyncTicker := time.NewTicker(trySyncIntervalMS * time.Millisecond) 219 statusUpdateTicker := time.NewTicker(statusUpdateIntervalSeconds * time.Second) 220 switchToConsensusTicker := time.NewTicker(switchToConsensusIntervalSeconds * time.Second) 221 222 blocksSynced := uint64(0) 223 224 chainID := bcR.initialState.ChainID 225 state := bcR.initialState 226 227 lastHundred := time.Now() 228 lastRate := 0.0 229 230 didProcessCh := make(chan struct{}, 1) 231 232 go func() { 233 for { 234 select { 235 case <-bcR.Quit(): 236 return 237 case <-bcR.pool.Quit(): 238 return 239 case request := <-bcR.requestsCh: 240 peer := bcR.Switch.Peers().Get(request.PeerID) 241 if peer == nil { 242 continue 243 } 244 msgBytes := cdc.MustMarshalBinaryBare(&bcBlockRequestMessage{request.Height}) 245 queued := peer.TrySend(BlockchainChannel, msgBytes) 246 if !queued { 247 bcR.Logger.Debug("Send queue is full, drop block request", "peer", peer.ID(), "height", request.Height) 248 } 249 case err := <-bcR.errorsCh: 250 peer := bcR.Switch.Peers().Get(err.peerID) 251 if peer != nil { 252 bcR.Switch.StopPeerForError(peer, err) 253 } 254 255 case <-statusUpdateTicker.C: 256 // ask for status updates 257 go bcR.BroadcastStatusRequest() // nolint: errcheck 258 259 } 260 } 261 }() 262 263 FOR_LOOP: 264 for { 265 select { 266 case <-switchToConsensusTicker.C: 267 height, numPending, lenRequesters := bcR.pool.GetStatus() 268 outbound, inbound, _ := bcR.Switch.NumPeers() 269 bcR.Logger.Debug("Consensus ticker", "numPending", numPending, "total", lenRequesters, 270 "outbound", outbound, "inbound", inbound) 271 if bcR.pool.IsCaughtUp() { 272 bcR.Logger.Info("Time to switch to consensus reactor!", "height", height) 273 bcR.pool.Stop() 274 conR, ok := bcR.Switch.Reactor("CONSENSUS").(consensusReactor) 275 if ok { 276 conR.SwitchToConsensus(state, blocksSynced) 277 } 278 // else { 279 // should only happen during testing 280 // } 281 282 break FOR_LOOP 283 } 284 285 case <-trySyncTicker.C: // chan time 286 select { 287 case didProcessCh <- struct{}{}: 288 default: 289 } 290 291 case <-didProcessCh: 292 // NOTE: It is a subtle mistake to process more than a single block 293 // at a time (e.g. 10) here, because we only TrySend 1 request per 294 // loop. The ratio mismatch can result in starving of blocks, a 295 // sudden burst of requests and responses, and repeat. 296 // Consequently, it is better to split these routines rather than 297 // coupling them as it's written here. TODO uncouple from request 298 // routine. 299 300 // See if there are any blocks to sync. 301 first, second := bcR.pool.PeekTwoBlocks() 302 //bcR.Logger.Info("TrySync peeked", "first", first, "second", second) 303 if first == nil || second == nil { 304 // We need both to sync the first block. 305 continue FOR_LOOP 306 } else { 307 // Try again quickly next loop. 308 didProcessCh <- struct{}{} 309 } 310 311 firstParts := first.MakePartSet(types.BlockPartSizeBytes) 312 firstPartsHeader := firstParts.Header() 313 firstID := types.BlockID{Hash: first.Hash(), PartsHeader: firstPartsHeader} 314 // Finally, verify the first block using the second's commit 315 // NOTE: we can probably make this more efficient, but note that calling 316 // first.Hash() doesn't verify the tx contents, so MakePartSet() is 317 // currently necessary. 318 319 err := state.Validators.VerifyCommitLight( 320 chainID, firstID, first.Height, second.LastCommit) 321 if err != nil { 322 bcR.Logger.Error("Error in validation", "err", err) 323 peerID := bcR.pool.RedoRequest(first.Height) 324 peer := bcR.Switch.Peers().Get(peerID) 325 if peer != nil { 326 // NOTE: we've already removed the peer's request, but we 327 // still need to clean up the rest. 328 bcR.Switch.StopPeerForError(peer, fmt.Errorf("blockchainReactor validation error: %v", err)) 329 } 330 peerID2 := bcR.pool.RedoRequest(second.Height) 331 peer2 := bcR.Switch.Peers().Get(peerID2) 332 if peer2 != nil && peer2 != peer { 333 // NOTE: we've already removed the peer's request, but we 334 // still need to clean up the rest. 335 bcR.Switch.StopPeerForError(peer2, fmt.Errorf("blockchainReactor validation error: %v", err)) 336 } 337 continue FOR_LOOP 338 } else { 339 bcR.pool.PopRequest() 340 341 // TODO: batch saves so we dont persist to disk every block 342 bcR.store.SaveBlock(first, firstParts, second.LastCommit) 343 344 // TODO: same thing for app - but we would need a way to 345 // get the hash without persisting the state 346 var err error 347 state, _, err = bcR.blockExec.ApplyBlock(state, firstID, first) 348 if err != nil { 349 // TODO This is bad, are we zombie? 350 panic(fmt.Sprintf("Failed to process committed block (%d:%X): %v", first.Height, first.Hash(), err)) 351 } 352 blocksSynced++ 353 354 if blocksSynced%100 == 0 { 355 lastRate = 0.9*lastRate + 0.1*(100/time.Since(lastHundred).Seconds()) 356 bcR.Logger.Info("Fast Sync Rate", "height", bcR.pool.height, 357 "max_peer_height", bcR.pool.MaxPeerHeight(), "blocks/s", lastRate) 358 lastHundred = time.Now() 359 } 360 } 361 continue FOR_LOOP 362 363 case <-bcR.Quit(): 364 break FOR_LOOP 365 } 366 } 367 } 368 369 // BroadcastStatusRequest broadcasts `BlockStore` base and height. 370 func (bcR *BlockchainReactor) BroadcastStatusRequest() error { 371 msgBytes := cdc.MustMarshalBinaryBare(&bcStatusRequestMessage{ 372 Base: bcR.store.Base(), 373 Height: bcR.store.Height(), 374 }) 375 bcR.Switch.Broadcast(BlockchainChannel, msgBytes) 376 return nil 377 } 378 379 //----------------------------------------------------------------------------- 380 // Messages 381 382 // BlockchainMessage is a generic message for this reactor. 383 type BlockchainMessage interface { 384 ValidateBasic() error 385 } 386 387 // RegisterBlockchainMessages registers the fast sync messages for amino encoding. 388 func RegisterBlockchainMessages(cdc *amino.Codec) { 389 cdc.RegisterInterface((*BlockchainMessage)(nil), nil) 390 cdc.RegisterConcrete(&bcBlockRequestMessage{}, "tendermint/blockchain/BlockRequest", nil) 391 cdc.RegisterConcrete(&bcBlockResponseMessage{}, "tendermint/blockchain/BlockResponse", nil) 392 cdc.RegisterConcrete(&bcNoBlockResponseMessage{}, "tendermint/blockchain/NoBlockResponse", nil) 393 cdc.RegisterConcrete(&bcStatusResponseMessage{}, "tendermint/blockchain/StatusResponse", nil) 394 cdc.RegisterConcrete(&bcStatusRequestMessage{}, "tendermint/blockchain/StatusRequest", nil) 395 } 396 397 func decodeMsg(bz []byte) (msg BlockchainMessage, err error) { 398 if len(bz) > maxMsgSize { 399 return msg, fmt.Errorf("msg exceeds max size (%d > %d)", len(bz), maxMsgSize) 400 } 401 err = cdc.UnmarshalBinaryBare(bz, &msg) 402 return 403 } 404 405 //------------------------------------- 406 407 type bcBlockRequestMessage struct { 408 Height int64 409 } 410 411 // ValidateBasic performs basic validation. 412 func (m *bcBlockRequestMessage) ValidateBasic() error { 413 if m.Height < 0 { 414 return errors.New("negative Height") 415 } 416 return nil 417 } 418 419 func (m *bcBlockRequestMessage) String() string { 420 return fmt.Sprintf("[bcBlockRequestMessage %v]", m.Height) 421 } 422 423 type bcNoBlockResponseMessage struct { 424 Height int64 425 } 426 427 // ValidateBasic performs basic validation. 428 func (m *bcNoBlockResponseMessage) ValidateBasic() error { 429 if m.Height < 0 { 430 return errors.New("negative Height") 431 } 432 return nil 433 } 434 435 func (m *bcNoBlockResponseMessage) String() string { 436 return fmt.Sprintf("[bcNoBlockResponseMessage %d]", m.Height) 437 } 438 439 //------------------------------------- 440 441 type bcBlockResponseMessage struct { 442 Block *types.Block 443 } 444 445 // ValidateBasic performs basic validation. 446 func (m *bcBlockResponseMessage) ValidateBasic() error { 447 return m.Block.ValidateBasic() 448 } 449 450 func (m *bcBlockResponseMessage) String() string { 451 return fmt.Sprintf("[bcBlockResponseMessage %v]", m.Block.Height) 452 } 453 454 //------------------------------------- 455 456 type bcStatusRequestMessage struct { 457 Height int64 458 Base int64 459 } 460 461 // ValidateBasic performs basic validation. 462 func (m *bcStatusRequestMessage) ValidateBasic() error { 463 if m.Base < 0 { 464 return errors.New("negative Base") 465 } 466 if m.Height < 0 { 467 return errors.New("negative Height") 468 } 469 if m.Base > m.Height { 470 return fmt.Errorf("base %v cannot be greater than height %v", m.Base, m.Height) 471 } 472 return nil 473 } 474 475 func (m *bcStatusRequestMessage) String() string { 476 return fmt.Sprintf("[bcStatusRequestMessage %v:%v]", m.Base, m.Height) 477 } 478 479 //------------------------------------- 480 481 type bcStatusResponseMessage struct { 482 Height int64 483 Base int64 484 } 485 486 // ValidateBasic performs basic validation. 487 func (m *bcStatusResponseMessage) ValidateBasic() error { 488 if m.Base < 0 { 489 return errors.New("negative Base") 490 } 491 if m.Height < 0 { 492 return errors.New("negative Height") 493 } 494 if m.Base > m.Height { 495 return fmt.Errorf("base %v cannot be greater than height %v", m.Base, m.Height) 496 } 497 return nil 498 } 499 500 func (m *bcStatusResponseMessage) String() string { 501 return fmt.Sprintf("[bcStatusResponseMessage %v:%v]", m.Base, m.Height) 502 }