github.com/Team-Kujira/tendermint@v0.34.24-indexer/blockchain/v1/reactor.go (about) 1 package v1 2 3 import ( 4 "fmt" 5 "time" 6 7 "github.com/gogo/protobuf/proto" 8 9 "github.com/tendermint/tendermint/behaviour" 10 bc "github.com/tendermint/tendermint/blockchain" 11 "github.com/tendermint/tendermint/libs/log" 12 "github.com/tendermint/tendermint/p2p" 13 bcproto "github.com/tendermint/tendermint/proto/tendermint/blockchain" 14 sm "github.com/tendermint/tendermint/state" 15 "github.com/tendermint/tendermint/store" 16 "github.com/tendermint/tendermint/types" 17 ) 18 19 const ( 20 // BlockchainChannel is a channel for blocks and status updates (`BlockStore` height) 21 BlockchainChannel = byte(0x40) 22 trySyncIntervalMS = 10 23 trySendIntervalMS = 10 24 25 // ask for best height every 10s 26 statusUpdateIntervalSeconds = 10 27 ) 28 29 var ( 30 // Maximum number of requests that can be pending per peer, i.e. for which requests have been sent but blocks 31 // have not been received. 32 maxRequestsPerPeer = 20 33 // Maximum number of block requests for the reactor, pending or for which blocks have been received. 34 maxNumRequests = 64 35 ) 36 37 type consensusReactor interface { 38 // for when we switch from blockchain reactor and fast sync to 39 // the consensus machine 40 SwitchToConsensus(state sm.State, skipWAL bool) 41 } 42 43 // BlockchainReactor handles long-term catchup syncing. 44 type BlockchainReactor struct { 45 p2p.BaseReactor 46 47 initialState sm.State // immutable 48 state sm.State 49 50 blockExec *sm.BlockExecutor 51 store *store.BlockStore 52 53 fastSync bool 54 stateSynced bool 55 56 fsm *BcReactorFSM 57 blocksSynced uint64 58 59 // Receive goroutine forwards messages to this channel to be processed in the context of the poolRoutine. 60 messagesForFSMCh chan bcReactorMessage 61 62 // Switch goroutine may send RemovePeer to the blockchain reactor. This is an error message that is relayed 63 // to this channel to be processed in the context of the poolRoutine. 64 errorsForFSMCh chan bcReactorMessage 65 66 // This channel is used by the FSM and indirectly the block pool to report errors to the blockchain reactor and 67 // the switch. 68 eventsFromFSMCh chan bcFsmMessage 69 70 swReporter *behaviour.SwitchReporter 71 } 72 73 // NewBlockchainReactor returns new reactor instance. 74 func NewBlockchainReactor(state sm.State, blockExec *sm.BlockExecutor, store *store.BlockStore, 75 fastSync bool) *BlockchainReactor { 76 77 if state.LastBlockHeight != store.Height() { 78 panic(fmt.Sprintf("state (%v) and store (%v) height mismatch", state.LastBlockHeight, 79 store.Height())) 80 } 81 82 const capacity = 1000 83 eventsFromFSMCh := make(chan bcFsmMessage, capacity) 84 messagesForFSMCh := make(chan bcReactorMessage, capacity) 85 errorsForFSMCh := make(chan bcReactorMessage, capacity) 86 87 startHeight := store.Height() + 1 88 if startHeight == 1 { 89 startHeight = state.InitialHeight 90 } 91 bcR := &BlockchainReactor{ 92 initialState: state, 93 state: state, 94 blockExec: blockExec, 95 fastSync: fastSync, 96 store: store, 97 messagesForFSMCh: messagesForFSMCh, 98 eventsFromFSMCh: eventsFromFSMCh, 99 errorsForFSMCh: errorsForFSMCh, 100 } 101 fsm := NewFSM(startHeight, bcR) 102 bcR.fsm = fsm 103 bcR.BaseReactor = *p2p.NewBaseReactor("BlockchainReactor", bcR) 104 // bcR.swReporter = behaviour.NewSwitchReporter(bcR.BaseReactor.Switch) 105 106 return bcR 107 } 108 109 // bcReactorMessage is used by the reactor to send messages to the FSM. 110 type bcReactorMessage struct { 111 event bReactorEvent 112 data bReactorEventData 113 } 114 115 type bFsmEvent uint 116 117 const ( 118 // message type events 119 peerErrorEv = iota + 1 120 syncFinishedEv 121 ) 122 123 type bFsmEventData struct { 124 peerID p2p.ID 125 err error 126 } 127 128 // bcFsmMessage is used by the FSM to send messages to the reactor 129 type bcFsmMessage struct { 130 event bFsmEvent 131 data bFsmEventData 132 } 133 134 // SetLogger implements service.Service by setting the logger on reactor and pool. 135 func (bcR *BlockchainReactor) SetLogger(l log.Logger) { 136 bcR.BaseService.Logger = l 137 bcR.fsm.SetLogger(l) 138 } 139 140 // OnStart implements service.Service. 141 func (bcR *BlockchainReactor) OnStart() error { 142 bcR.swReporter = behaviour.NewSwitchReporter(bcR.BaseReactor.Switch) 143 if bcR.fastSync { 144 go bcR.poolRoutine() 145 } 146 return nil 147 } 148 149 // OnStop implements service.Service. 150 func (bcR *BlockchainReactor) OnStop() { 151 _ = bcR.Stop() 152 } 153 154 // SwitchToFastSync is called by the state sync reactor when switching to fast sync. 155 func (bcR *BlockchainReactor) SwitchToFastSync(state sm.State) error { 156 bcR.fastSync = true 157 bcR.initialState = state 158 bcR.state = state 159 bcR.stateSynced = true 160 161 bcR.fsm = NewFSM(state.LastBlockHeight+1, bcR) 162 bcR.fsm.SetLogger(bcR.Logger) 163 go bcR.poolRoutine() 164 return nil 165 } 166 167 // GetChannels implements Reactor 168 func (bcR *BlockchainReactor) GetChannels() []*p2p.ChannelDescriptor { 169 return []*p2p.ChannelDescriptor{ 170 { 171 ID: BlockchainChannel, 172 Priority: 10, 173 SendQueueCapacity: 2000, 174 RecvBufferCapacity: 50 * 4096, 175 RecvMessageCapacity: bc.MaxMsgSize, 176 MessageType: &bcproto.Message{}, 177 }, 178 } 179 } 180 181 // AddPeer implements Reactor by sending our state to peer. 182 func (bcR *BlockchainReactor) AddPeer(peer p2p.Peer) { 183 p2p.SendEnvelopeShim(peer, p2p.Envelope{ //nolint: staticcheck 184 ChannelID: BlockchainChannel, 185 Message: &bcproto.StatusResponse{ 186 Base: bcR.store.Base(), 187 Height: bcR.store.Height(), 188 }, 189 }, bcR.Logger) 190 // it's OK if send fails. will try later in poolRoutine 191 192 // peer is added to the pool once we receive the first 193 // bcStatusResponseMessage from the peer and call pool.updatePeer() 194 } 195 196 // sendBlockToPeer loads a block and sends it to the requesting peer. 197 // If the block doesn't exist a bcNoBlockResponseMessage is sent. 198 // If all nodes are honest, no node should be requesting for a block that doesn't exist. 199 func (bcR *BlockchainReactor) sendBlockToPeer(msg *bcproto.BlockRequest, 200 src p2p.Peer) (queued bool) { 201 202 block := bcR.store.LoadBlock(msg.Height) 203 if block != nil { 204 pbbi, err := block.ToProto() 205 if err != nil { 206 bcR.Logger.Error("Could not send block message to peer", "err", err) 207 return false 208 } 209 return p2p.TrySendEnvelopeShim(src, p2p.Envelope{ //nolint: staticcheck 210 ChannelID: BlockchainChannel, 211 Message: &bcproto.BlockResponse{Block: pbbi}, 212 }, bcR.Logger) 213 } 214 215 bcR.Logger.Info("peer asking for a block we don't have", "src", src, "height", msg.Height) 216 217 return p2p.TrySendEnvelopeShim(src, p2p.Envelope{ //nolint: staticcheck 218 ChannelID: BlockchainChannel, 219 Message: &bcproto.NoBlockResponse{Height: msg.Height}, 220 }, bcR.Logger) 221 } 222 223 func (bcR *BlockchainReactor) sendStatusResponseToPeer(msg *bcproto.StatusRequest, src p2p.Peer) (queued bool) { 224 return p2p.TrySendEnvelopeShim(src, p2p.Envelope{ //nolint: staticcheck 225 ChannelID: BlockchainChannel, 226 Message: &bcproto.StatusResponse{ 227 Base: bcR.store.Base(), 228 Height: bcR.store.Height(), 229 }, 230 }, bcR.Logger) 231 } 232 233 // RemovePeer implements Reactor by removing peer from the pool. 234 func (bcR *BlockchainReactor) RemovePeer(peer p2p.Peer, reason interface{}) { 235 msgData := bcReactorMessage{ 236 event: peerRemoveEv, 237 data: bReactorEventData{ 238 peerID: peer.ID(), 239 err: errSwitchRemovesPeer, 240 }, 241 } 242 bcR.errorsForFSMCh <- msgData 243 } 244 245 // Receive implements Reactor by handling 4 types of messages (look below). 246 func (bcR *BlockchainReactor) ReceiveEnvelope(e p2p.Envelope) { 247 if err := bc.ValidateMsg(e.Message); err != nil { 248 bcR.Logger.Error("peer sent us invalid msg", "peer", e.Src, "msg", e.Message, "err", err) 249 _ = bcR.swReporter.Report(behaviour.BadMessage(e.Src.ID(), err.Error())) 250 return 251 } 252 253 bcR.Logger.Debug("Receive", "src", e.Src, "chID", e.ChannelID, "msg", e.Message) 254 255 switch msg := e.Message.(type) { 256 case *bcproto.BlockRequest: 257 if queued := bcR.sendBlockToPeer(msg, e.Src); !queued { 258 // Unfortunately not queued since the queue is full. 259 bcR.Logger.Error("Could not send block message to peer", "src", e.Src, "height", msg.Height) 260 } 261 262 case *bcproto.StatusRequest: 263 // Send peer our state. 264 if queued := bcR.sendStatusResponseToPeer(msg, e.Src); !queued { 265 // Unfortunately not queued since the queue is full. 266 bcR.Logger.Error("Could not send status message to peer", "src", e.Src) 267 } 268 269 case *bcproto.BlockResponse: 270 bi, err := types.BlockFromProto(msg.Block) 271 if err != nil { 272 bcR.Logger.Error("error transition block from protobuf", "err", err) 273 return 274 } 275 msgForFSM := bcReactorMessage{ 276 event: blockResponseEv, 277 data: bReactorEventData{ 278 peerID: e.Src.ID(), 279 height: bi.Height, 280 block: bi, 281 length: msg.Size(), 282 }, 283 } 284 bcR.Logger.Info("Received", "src", e.Src, "height", bi.Height) 285 bcR.messagesForFSMCh <- msgForFSM 286 case *bcproto.NoBlockResponse: 287 msgForFSM := bcReactorMessage{ 288 event: noBlockResponseEv, 289 data: bReactorEventData{ 290 peerID: e.Src.ID(), 291 height: msg.Height, 292 }, 293 } 294 bcR.Logger.Debug("Peer does not have requested block", "peer", e.Src, "height", msg.Height) 295 bcR.messagesForFSMCh <- msgForFSM 296 297 case *bcproto.StatusResponse: 298 // Got a peer status. Unverified. 299 msgForFSM := bcReactorMessage{ 300 event: statusResponseEv, 301 data: bReactorEventData{ 302 peerID: e.Src.ID(), 303 height: msg.Height, 304 length: msg.Size(), 305 }, 306 } 307 bcR.messagesForFSMCh <- msgForFSM 308 309 default: 310 bcR.Logger.Error(fmt.Sprintf("unknown message type %T", msg)) 311 } 312 } 313 314 func (bcR *BlockchainReactor) Receive(chID byte, peer p2p.Peer, msgBytes []byte) { 315 msg := &bcproto.Message{} 316 err := proto.Unmarshal(msgBytes, msg) 317 if err != nil { 318 panic(err) 319 } 320 uw, err := msg.Unwrap() 321 if err != nil { 322 panic(err) 323 } 324 bcR.ReceiveEnvelope(p2p.Envelope{ 325 ChannelID: chID, 326 Src: peer, 327 Message: uw, 328 }) 329 } 330 331 // processBlocksRoutine processes blocks until signlaed to stop over the stopProcessing channel 332 func (bcR *BlockchainReactor) processBlocksRoutine(stopProcessing chan struct{}) { 333 334 processReceivedBlockTicker := time.NewTicker(trySyncIntervalMS * time.Millisecond) 335 doProcessBlockCh := make(chan struct{}, 1) 336 337 lastHundred := time.Now() 338 lastRate := 0.0 339 340 ForLoop: 341 for { 342 select { 343 case <-stopProcessing: 344 bcR.Logger.Info("finishing block execution") 345 break ForLoop 346 case <-processReceivedBlockTicker.C: // try to execute blocks 347 select { 348 case doProcessBlockCh <- struct{}{}: 349 default: 350 } 351 case <-doProcessBlockCh: 352 for { 353 err := bcR.processBlock() 354 if err == errMissingBlock { 355 break 356 } 357 // Notify FSM of block processing result. 358 msgForFSM := bcReactorMessage{ 359 event: processedBlockEv, 360 data: bReactorEventData{ 361 err: err, 362 }, 363 } 364 _ = bcR.fsm.Handle(&msgForFSM) 365 366 if err != nil { 367 break 368 } 369 370 bcR.blocksSynced++ 371 if bcR.blocksSynced%100 == 0 { 372 lastRate = 0.9*lastRate + 0.1*(100/time.Since(lastHundred).Seconds()) 373 height, maxPeerHeight := bcR.fsm.Status() 374 bcR.Logger.Info("Fast Sync Rate", "height", height, 375 "max_peer_height", maxPeerHeight, "blocks/s", lastRate) 376 lastHundred = time.Now() 377 } 378 } 379 } 380 } 381 } 382 383 // poolRoutine receives and handles messages from the Receive() routine and from the FSM. 384 func (bcR *BlockchainReactor) poolRoutine() { 385 386 bcR.fsm.Start() 387 388 sendBlockRequestTicker := time.NewTicker(trySendIntervalMS * time.Millisecond) 389 statusUpdateTicker := time.NewTicker(statusUpdateIntervalSeconds * time.Second) 390 391 stopProcessing := make(chan struct{}, 1) 392 go bcR.processBlocksRoutine(stopProcessing) 393 394 ForLoop: 395 for { 396 select { 397 398 case <-sendBlockRequestTicker.C: 399 if !bcR.fsm.NeedsBlocks() { 400 continue 401 } 402 _ = bcR.fsm.Handle(&bcReactorMessage{ 403 event: makeRequestsEv, 404 data: bReactorEventData{ 405 maxNumRequests: maxNumRequests}}) 406 407 case <-statusUpdateTicker.C: 408 // Ask for status updates. 409 go bcR.sendStatusRequest() 410 411 case msg := <-bcR.messagesForFSMCh: 412 // Sent from the Receive() routine when status (statusResponseEv) and 413 // block (blockResponseEv) response events are received 414 _ = bcR.fsm.Handle(&msg) 415 416 case msg := <-bcR.errorsForFSMCh: 417 // Sent from the switch.RemovePeer() routine (RemovePeerEv) and 418 // FSM state timer expiry routine (stateTimeoutEv). 419 _ = bcR.fsm.Handle(&msg) 420 421 case msg := <-bcR.eventsFromFSMCh: 422 switch msg.event { 423 case syncFinishedEv: 424 stopProcessing <- struct{}{} 425 // Sent from the FSM when it enters finished state. 426 break ForLoop 427 case peerErrorEv: 428 // Sent from the FSM when it detects peer error 429 bcR.reportPeerErrorToSwitch(msg.data.err, msg.data.peerID) 430 if msg.data.err == errNoPeerResponse { 431 // Sent from the peer timeout handler routine 432 _ = bcR.fsm.Handle(&bcReactorMessage{ 433 event: peerRemoveEv, 434 data: bReactorEventData{ 435 peerID: msg.data.peerID, 436 err: msg.data.err, 437 }, 438 }) 439 } 440 // else { 441 // For slow peers, or errors due to blocks received from wrong peer 442 // the FSM had already removed the peers 443 // } 444 default: 445 bcR.Logger.Error("Event from FSM not supported", "type", msg.event) 446 } 447 448 case <-bcR.Quit(): 449 break ForLoop 450 } 451 } 452 } 453 454 func (bcR *BlockchainReactor) reportPeerErrorToSwitch(err error, peerID p2p.ID) { 455 peer := bcR.Switch.Peers().Get(peerID) 456 if peer != nil { 457 _ = bcR.swReporter.Report(behaviour.BadMessage(peerID, err.Error())) 458 } 459 } 460 461 func (bcR *BlockchainReactor) processBlock() error { 462 463 first, second, err := bcR.fsm.FirstTwoBlocks() 464 if err != nil { 465 // We need both to sync the first block. 466 return err 467 } 468 469 chainID := bcR.initialState.ChainID 470 471 firstParts := first.MakePartSet(types.BlockPartSizeBytes) 472 firstPartSetHeader := firstParts.Header() 473 firstID := types.BlockID{Hash: first.Hash(), PartSetHeader: firstPartSetHeader} 474 // Finally, verify the first block using the second's commit 475 // NOTE: we can probably make this more efficient, but note that calling 476 // first.Hash() doesn't verify the tx contents, so MakePartSet() is 477 // currently necessary. 478 err = bcR.state.Validators.VerifyCommitLight(chainID, firstID, first.Height, second.LastCommit) 479 if err != nil { 480 bcR.Logger.Error("error during commit verification", "err", err, 481 "first", first.Height, "second", second.Height) 482 return errBlockVerificationFailure 483 } 484 485 bcR.store.SaveBlock(first, firstParts, second.LastCommit) 486 487 bcR.state, _, err = bcR.blockExec.ApplyBlock(bcR.state, firstID, first) 488 if err != nil { 489 panic(fmt.Sprintf("failed to process committed block (%d:%X): %v", first.Height, first.Hash(), err)) 490 } 491 492 return nil 493 } 494 495 // Implements bcRNotifier 496 // sendStatusRequest broadcasts `BlockStore` height. 497 func (bcR *BlockchainReactor) sendStatusRequest() { 498 bcR.Switch.BroadcastEnvelope(p2p.Envelope{ 499 ChannelID: BlockchainChannel, 500 Message: &bcproto.StatusRequest{}, 501 }) 502 } 503 504 // Implements bcRNotifier 505 // BlockRequest sends `BlockRequest` height. 506 func (bcR *BlockchainReactor) sendBlockRequest(peerID p2p.ID, height int64) error { 507 peer := bcR.Switch.Peers().Get(peerID) 508 if peer == nil { 509 return errNilPeerForBlockRequest 510 } 511 512 queued := p2p.TrySendEnvelopeShim(peer, p2p.Envelope{ //nolint: staticcheck 513 ChannelID: BlockchainChannel, 514 Message: &bcproto.BlockRequest{Height: height}, 515 }, bcR.Logger) 516 if !queued { 517 return errSendQueueFull 518 } 519 return nil 520 } 521 522 // Implements bcRNotifier 523 func (bcR *BlockchainReactor) switchToConsensus() { 524 conR, ok := bcR.Switch.Reactor("CONSENSUS").(consensusReactor) 525 if ok { 526 conR.SwitchToConsensus(bcR.state, bcR.blocksSynced > 0 || bcR.stateSynced) 527 bcR.eventsFromFSMCh <- bcFsmMessage{event: syncFinishedEv} 528 } 529 // else { 530 // Should only happen during testing. 531 // } 532 } 533 534 // Implements bcRNotifier 535 // Called by FSM and pool: 536 // - pool calls when it detects slow peer or when peer times out 537 // - FSM calls when: 538 // - adding a block (addBlock) fails 539 // - reactor processing of a block reports failure and FSM sends back the peers of first and second blocks 540 func (bcR *BlockchainReactor) sendPeerError(err error, peerID p2p.ID) { 541 bcR.Logger.Info("sendPeerError:", "peer", peerID, "error", err) 542 msgData := bcFsmMessage{ 543 event: peerErrorEv, 544 data: bFsmEventData{ 545 peerID: peerID, 546 err: err, 547 }, 548 } 549 bcR.eventsFromFSMCh <- msgData 550 } 551 552 // Implements bcRNotifier 553 func (bcR *BlockchainReactor) resetStateTimer(name string, timer **time.Timer, timeout time.Duration) { 554 if timer == nil { 555 panic("nil timer pointer parameter") 556 } 557 if *timer == nil { 558 *timer = time.AfterFunc(timeout, func() { 559 msg := bcReactorMessage{ 560 event: stateTimeoutEv, 561 data: bReactorEventData{ 562 stateName: name, 563 }, 564 } 565 bcR.errorsForFSMCh <- msg 566 }) 567 } else { 568 (*timer).Reset(timeout) 569 } 570 }