github.com/evdatsion/aphelion-dpos-bft@v0.32.1/blockchain/reactor.go (about) 1 package blockchain 2 3 import ( 4 "errors" 5 "fmt" 6 "reflect" 7 "time" 8 9 amino "github.com/evdatsion/go-amino" 10 11 "github.com/evdatsion/aphelion-dpos-bft/libs/log" 12 "github.com/evdatsion/aphelion-dpos-bft/p2p" 13 sm "github.com/evdatsion/aphelion-dpos-bft/state" 14 "github.com/evdatsion/aphelion-dpos-bft/types" 15 ) 16 17 const ( 18 // BlockchainChannel is a channel for blocks and status updates (`BlockStore` height) 19 BlockchainChannel = byte(0x40) 20 21 trySyncIntervalMS = 10 22 23 // stop syncing when last block's time is 24 // within this much of the system time. 25 // stopSyncingDurationMinutes = 10 26 27 // ask for best height every 10s 28 statusUpdateIntervalSeconds = 10 29 // check if we should switch to consensus reactor 30 switchToConsensusIntervalSeconds = 1 31 32 // NOTE: keep up to date with bcBlockResponseMessage 33 bcBlockResponseMessagePrefixSize = 4 34 bcBlockResponseMessageFieldKeySize = 1 35 maxMsgSize = types.MaxBlockSizeBytes + 36 bcBlockResponseMessagePrefixSize + 37 bcBlockResponseMessageFieldKeySize 38 ) 39 40 type consensusReactor interface { 41 // for when we switch from blockchain reactor and fast sync to 42 // the consensus machine 43 SwitchToConsensus(sm.State, int) 44 } 45 46 type peerError struct { 47 err error 48 peerID p2p.ID 49 } 50 51 func (e peerError) Error() string { 52 return fmt.Sprintf("error with peer %v: %s", e.peerID, e.err.Error()) 53 } 54 55 // BlockchainReactor handles long-term catchup syncing. 56 type BlockchainReactor struct { 57 p2p.BaseReactor 58 59 // immutable 60 initialState sm.State 61 62 blockExec *sm.BlockExecutor 63 store *BlockStore 64 pool *BlockPool 65 fastSync bool 66 67 requestsCh <-chan BlockRequest 68 errorsCh <-chan peerError 69 } 70 71 // NewBlockchainReactor returns new reactor instance. 72 func NewBlockchainReactor(state sm.State, blockExec *sm.BlockExecutor, store *BlockStore, 73 fastSync bool) *BlockchainReactor { 74 75 if state.LastBlockHeight != store.Height() { 76 panic(fmt.Sprintf("state (%v) and store (%v) height mismatch", state.LastBlockHeight, 77 store.Height())) 78 } 79 80 requestsCh := make(chan BlockRequest, maxTotalRequesters) 81 82 const capacity = 1000 // must be bigger than peers count 83 errorsCh := make(chan peerError, capacity) // so we don't block in #Receive#pool.AddBlock 84 85 pool := NewBlockPool( 86 store.Height()+1, 87 requestsCh, 88 errorsCh, 89 ) 90 91 bcR := &BlockchainReactor{ 92 initialState: state, 93 blockExec: blockExec, 94 store: store, 95 pool: pool, 96 fastSync: fastSync, 97 requestsCh: requestsCh, 98 errorsCh: errorsCh, 99 } 100 bcR.BaseReactor = *p2p.NewBaseReactor("BlockchainReactor", bcR) 101 return bcR 102 } 103 104 // SetLogger implements cmn.Service by setting the logger on reactor and pool. 105 func (bcR *BlockchainReactor) SetLogger(l log.Logger) { 106 bcR.BaseService.Logger = l 107 bcR.pool.Logger = l 108 } 109 110 // OnStart implements cmn.Service. 111 func (bcR *BlockchainReactor) OnStart() error { 112 if bcR.fastSync { 113 err := bcR.pool.Start() 114 if err != nil { 115 return err 116 } 117 go bcR.poolRoutine() 118 } 119 return nil 120 } 121 122 // OnStop implements cmn.Service. 123 func (bcR *BlockchainReactor) OnStop() { 124 bcR.pool.Stop() 125 } 126 127 // GetChannels implements Reactor 128 func (bcR *BlockchainReactor) GetChannels() []*p2p.ChannelDescriptor { 129 return []*p2p.ChannelDescriptor{ 130 { 131 ID: BlockchainChannel, 132 Priority: 10, 133 SendQueueCapacity: 1000, 134 RecvBufferCapacity: 50 * 4096, 135 RecvMessageCapacity: maxMsgSize, 136 }, 137 } 138 } 139 140 // AddPeer implements Reactor by sending our state to peer. 141 func (bcR *BlockchainReactor) AddPeer(peer p2p.Peer) { 142 msgBytes := cdc.MustMarshalBinaryBare(&bcStatusResponseMessage{bcR.store.Height()}) 143 if !peer.Send(BlockchainChannel, msgBytes) { 144 // doing nothing, will try later in `poolRoutine` 145 } 146 // peer is added to the pool once we receive the first 147 // bcStatusResponseMessage from the peer and call pool.SetPeerHeight 148 } 149 150 // RemovePeer implements Reactor by removing peer from the pool. 151 func (bcR *BlockchainReactor) RemovePeer(peer p2p.Peer, reason interface{}) { 152 bcR.pool.RemovePeer(peer.ID()) 153 } 154 155 // respondToPeer loads a block and sends it to the requesting peer, 156 // if we have it. Otherwise, we'll respond saying we don't have it. 157 // According to the Tendermint spec, if all nodes are honest, 158 // no node should be requesting for a block that's non-existent. 159 func (bcR *BlockchainReactor) respondToPeer(msg *bcBlockRequestMessage, 160 src p2p.Peer) (queued bool) { 161 162 block := bcR.store.LoadBlock(msg.Height) 163 if block != nil { 164 msgBytes := cdc.MustMarshalBinaryBare(&bcBlockResponseMessage{Block: block}) 165 return src.TrySend(BlockchainChannel, msgBytes) 166 } 167 168 bcR.Logger.Info("Peer asking for a block we don't have", "src", src, "height", msg.Height) 169 170 msgBytes := cdc.MustMarshalBinaryBare(&bcNoBlockResponseMessage{Height: msg.Height}) 171 return src.TrySend(BlockchainChannel, msgBytes) 172 } 173 174 // Receive implements Reactor by handling 4 types of messages (look below). 175 func (bcR *BlockchainReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte) { 176 msg, err := decodeMsg(msgBytes) 177 if err != nil { 178 bcR.Logger.Error("Error decoding message", "src", src, "chId", chID, "msg", msg, "err", err, "bytes", msgBytes) 179 bcR.Switch.StopPeerForError(src, err) 180 return 181 } 182 183 if err = msg.ValidateBasic(); err != nil { 184 bcR.Logger.Error("Peer sent us invalid msg", "peer", src, "msg", msg, "err", err) 185 bcR.Switch.StopPeerForError(src, err) 186 return 187 } 188 189 bcR.Logger.Debug("Receive", "src", src, "chID", chID, "msg", msg) 190 191 switch msg := msg.(type) { 192 case *bcBlockRequestMessage: 193 if queued := bcR.respondToPeer(msg, src); !queued { 194 // Unfortunately not queued since the queue is full. 195 } 196 case *bcBlockResponseMessage: 197 bcR.pool.AddBlock(src.ID(), msg.Block, len(msgBytes)) 198 case *bcStatusRequestMessage: 199 // Send peer our state. 200 msgBytes := cdc.MustMarshalBinaryBare(&bcStatusResponseMessage{bcR.store.Height()}) 201 queued := src.TrySend(BlockchainChannel, msgBytes) 202 if !queued { 203 // sorry 204 } 205 case *bcStatusResponseMessage: 206 // Got a peer status. Unverified. 207 bcR.pool.SetPeerHeight(src.ID(), msg.Height) 208 default: 209 bcR.Logger.Error(fmt.Sprintf("Unknown message type %v", reflect.TypeOf(msg))) 210 } 211 } 212 213 // Handle messages from the poolReactor telling the reactor what to do. 214 // NOTE: Don't sleep in the FOR_LOOP or otherwise slow it down! 215 func (bcR *BlockchainReactor) poolRoutine() { 216 217 trySyncTicker := time.NewTicker(trySyncIntervalMS * time.Millisecond) 218 statusUpdateTicker := time.NewTicker(statusUpdateIntervalSeconds * time.Second) 219 switchToConsensusTicker := time.NewTicker(switchToConsensusIntervalSeconds * time.Second) 220 221 blocksSynced := 0 222 223 chainID := bcR.initialState.ChainID 224 state := bcR.initialState 225 226 lastHundred := time.Now() 227 lastRate := 0.0 228 229 didProcessCh := make(chan struct{}, 1) 230 231 go func() { 232 for { 233 select { 234 case <-bcR.Quit(): 235 return 236 case <-bcR.pool.Quit(): 237 return 238 case request := <-bcR.requestsCh: 239 peer := bcR.Switch.Peers().Get(request.PeerID) 240 if peer == nil { 241 continue 242 } 243 msgBytes := cdc.MustMarshalBinaryBare(&bcBlockRequestMessage{request.Height}) 244 queued := peer.TrySend(BlockchainChannel, msgBytes) 245 if !queued { 246 bcR.Logger.Debug("Send queue is full, drop block request", "peer", peer.ID(), "height", request.Height) 247 } 248 case err := <-bcR.errorsCh: 249 peer := bcR.Switch.Peers().Get(err.peerID) 250 if peer != nil { 251 bcR.Switch.StopPeerForError(peer, err) 252 } 253 254 case <-statusUpdateTicker.C: 255 // ask for status updates 256 go bcR.BroadcastStatusRequest() // nolint: errcheck 257 258 } 259 } 260 }() 261 262 FOR_LOOP: 263 for { 264 select { 265 case <-switchToConsensusTicker.C: 266 height, numPending, lenRequesters := bcR.pool.GetStatus() 267 outbound, inbound, _ := bcR.Switch.NumPeers() 268 bcR.Logger.Debug("Consensus ticker", "numPending", numPending, "total", lenRequesters, 269 "outbound", outbound, "inbound", inbound) 270 if bcR.pool.IsCaughtUp() { 271 bcR.Logger.Info("Time to switch to consensus reactor!", "height", height) 272 bcR.pool.Stop() 273 conR, ok := bcR.Switch.Reactor("CONSENSUS").(consensusReactor) 274 if ok { 275 conR.SwitchToConsensus(state, blocksSynced) 276 } else { 277 // should only happen during testing 278 } 279 280 break FOR_LOOP 281 } 282 283 case <-trySyncTicker.C: // chan time 284 select { 285 case didProcessCh <- struct{}{}: 286 default: 287 } 288 289 case <-didProcessCh: 290 // NOTE: It is a subtle mistake to process more than a single block 291 // at a time (e.g. 10) here, because we only TrySend 1 request per 292 // loop. The ratio mismatch can result in starving of blocks, a 293 // sudden burst of requests and responses, and repeat. 294 // Consequently, it is better to split these routines rather than 295 // coupling them as it's written here. TODO uncouple from request 296 // routine. 297 298 // See if there are any blocks to sync. 299 first, second := bcR.pool.PeekTwoBlocks() 300 //bcR.Logger.Info("TrySync peeked", "first", first, "second", second) 301 if first == nil || second == nil { 302 // We need both to sync the first block. 303 continue FOR_LOOP 304 } else { 305 // Try again quickly next loop. 306 didProcessCh <- struct{}{} 307 } 308 309 firstParts := first.MakePartSet(types.BlockPartSizeBytes) 310 firstPartsHeader := firstParts.Header() 311 firstID := types.BlockID{Hash: first.Hash(), PartsHeader: firstPartsHeader} 312 // Finally, verify the first block using the second's commit 313 // NOTE: we can probably make this more efficient, but note that calling 314 // first.Hash() doesn't verify the tx contents, so MakePartSet() is 315 // currently necessary. 316 err := state.Validators.VerifyCommit( 317 chainID, firstID, first.Height, second.LastCommit) 318 if err != nil { 319 bcR.Logger.Error("Error in validation", "err", err) 320 peerID := bcR.pool.RedoRequest(first.Height) 321 peer := bcR.Switch.Peers().Get(peerID) 322 if peer != nil { 323 // NOTE: we've already removed the peer's request, but we 324 // still need to clean up the rest. 325 bcR.Switch.StopPeerForError(peer, fmt.Errorf("BlockchainReactor validation error: %v", err)) 326 } 327 peerID2 := bcR.pool.RedoRequest(second.Height) 328 peer2 := bcR.Switch.Peers().Get(peerID2) 329 if peer2 != nil && peer2 != peer { 330 // NOTE: we've already removed the peer's request, but we 331 // still need to clean up the rest. 332 bcR.Switch.StopPeerForError(peer2, fmt.Errorf("BlockchainReactor validation error: %v", err)) 333 } 334 continue FOR_LOOP 335 } else { 336 bcR.pool.PopRequest() 337 338 // TODO: batch saves so we dont persist to disk every block 339 bcR.store.SaveBlock(first, firstParts, second.LastCommit) 340 341 // TODO: same thing for app - but we would need a way to 342 // get the hash without persisting the state 343 var err error 344 state, err = bcR.blockExec.ApplyBlock(state, firstID, first) 345 if err != nil { 346 // TODO This is bad, are we zombie? 347 panic(fmt.Sprintf("Failed to process committed block (%d:%X): %v", first.Height, first.Hash(), err)) 348 } 349 blocksSynced++ 350 351 if blocksSynced%100 == 0 { 352 lastRate = 0.9*lastRate + 0.1*(100/time.Since(lastHundred).Seconds()) 353 bcR.Logger.Info("Fast Sync Rate", "height", bcR.pool.height, 354 "max_peer_height", bcR.pool.MaxPeerHeight(), "blocks/s", lastRate) 355 lastHundred = time.Now() 356 } 357 } 358 continue FOR_LOOP 359 360 case <-bcR.Quit(): 361 break FOR_LOOP 362 } 363 } 364 } 365 366 // BroadcastStatusRequest broadcasts `BlockStore` height. 367 func (bcR *BlockchainReactor) BroadcastStatusRequest() error { 368 msgBytes := cdc.MustMarshalBinaryBare(&bcStatusRequestMessage{bcR.store.Height()}) 369 bcR.Switch.Broadcast(BlockchainChannel, msgBytes) 370 return nil 371 } 372 373 //----------------------------------------------------------------------------- 374 // Messages 375 376 // BlockchainMessage is a generic message for this reactor. 377 type BlockchainMessage interface { 378 ValidateBasic() error 379 } 380 381 func RegisterBlockchainMessages(cdc *amino.Codec) { 382 cdc.RegisterInterface((*BlockchainMessage)(nil), nil) 383 cdc.RegisterConcrete(&bcBlockRequestMessage{}, "tendermint/blockchain/BlockRequest", nil) 384 cdc.RegisterConcrete(&bcBlockResponseMessage{}, "tendermint/blockchain/BlockResponse", nil) 385 cdc.RegisterConcrete(&bcNoBlockResponseMessage{}, "tendermint/blockchain/NoBlockResponse", nil) 386 cdc.RegisterConcrete(&bcStatusResponseMessage{}, "tendermint/blockchain/StatusResponse", nil) 387 cdc.RegisterConcrete(&bcStatusRequestMessage{}, "tendermint/blockchain/StatusRequest", nil) 388 } 389 390 func decodeMsg(bz []byte) (msg BlockchainMessage, err error) { 391 if len(bz) > maxMsgSize { 392 return msg, fmt.Errorf("Msg exceeds max size (%d > %d)", len(bz), maxMsgSize) 393 } 394 err = cdc.UnmarshalBinaryBare(bz, &msg) 395 return 396 } 397 398 //------------------------------------- 399 400 type bcBlockRequestMessage struct { 401 Height int64 402 } 403 404 // ValidateBasic performs basic validation. 405 func (m *bcBlockRequestMessage) ValidateBasic() error { 406 if m.Height < 0 { 407 return errors.New("Negative Height") 408 } 409 return nil 410 } 411 412 func (m *bcBlockRequestMessage) String() string { 413 return fmt.Sprintf("[bcBlockRequestMessage %v]", m.Height) 414 } 415 416 type bcNoBlockResponseMessage struct { 417 Height int64 418 } 419 420 // ValidateBasic performs basic validation. 421 func (m *bcNoBlockResponseMessage) ValidateBasic() error { 422 if m.Height < 0 { 423 return errors.New("Negative Height") 424 } 425 return nil 426 } 427 428 func (brm *bcNoBlockResponseMessage) String() string { 429 return fmt.Sprintf("[bcNoBlockResponseMessage %d]", brm.Height) 430 } 431 432 //------------------------------------- 433 434 type bcBlockResponseMessage struct { 435 Block *types.Block 436 } 437 438 // ValidateBasic performs basic validation. 439 func (m *bcBlockResponseMessage) ValidateBasic() error { 440 return m.Block.ValidateBasic() 441 } 442 443 func (m *bcBlockResponseMessage) String() string { 444 return fmt.Sprintf("[bcBlockResponseMessage %v]", m.Block.Height) 445 } 446 447 //------------------------------------- 448 449 type bcStatusRequestMessage struct { 450 Height int64 451 } 452 453 // ValidateBasic performs basic validation. 454 func (m *bcStatusRequestMessage) ValidateBasic() error { 455 if m.Height < 0 { 456 return errors.New("Negative Height") 457 } 458 return nil 459 } 460 461 func (m *bcStatusRequestMessage) String() string { 462 return fmt.Sprintf("[bcStatusRequestMessage %v]", m.Height) 463 } 464 465 //------------------------------------- 466 467 type bcStatusResponseMessage struct { 468 Height int64 469 } 470 471 // ValidateBasic performs basic validation. 472 func (m *bcStatusResponseMessage) ValidateBasic() error { 473 if m.Height < 0 { 474 return errors.New("Negative Height") 475 } 476 return nil 477 } 478 479 func (m *bcStatusResponseMessage) String() string { 480 return fmt.Sprintf("[bcStatusResponseMessage %v]", m.Height) 481 }