github.com/vipernet-xyz/tendermint-core@v0.32.0/blockchain/v1/reactor.go (about) 1 package v1 2 3 import ( 4 "errors" 5 "fmt" 6 "reflect" 7 "sync/atomic" 8 "time" 9 10 amino "github.com/tendermint/go-amino" 11 12 "github.com/tendermint/tendermint/behaviour" 13 "github.com/tendermint/tendermint/libs/log" 14 "github.com/tendermint/tendermint/p2p" 15 sm "github.com/tendermint/tendermint/state" 16 "github.com/tendermint/tendermint/store" 17 "github.com/tendermint/tendermint/types" 18 ) 19 20 const ( 21 // BlockchainChannel is a channel for blocks and status updates (`BlockStore` height) 22 BlockchainChannel = byte(0x40) 23 trySyncIntervalMS = 10 24 trySendIntervalMS = 10 25 26 // ask for best height every 10s 27 statusUpdateIntervalSeconds = 10 28 29 // NOTE: keep up to date with bcBlockResponseMessage 30 bcBlockResponseMessagePrefixSize = 4 31 bcBlockResponseMessageFieldKeySize = 1 32 maxMsgSize = types.MaxBlockSizeBytes + 33 bcBlockResponseMessagePrefixSize + 34 bcBlockResponseMessageFieldKeySize 35 ) 36 37 var ( 38 // Maximum number of requests that can be pending per peer, i.e. for which requests have been sent but blocks 39 // have not been received. 40 maxRequestsPerPeer = 20 41 // Maximum number of block requests for the reactor, pending or for which blocks have been received. 42 maxNumRequests = 64 43 44 counter int64 45 ) 46 47 type consensusReactor interface { 48 // for when we switch from blockchain reactor and fast sync to 49 // the consensus machine 50 SwitchToConsensus(sm.State, uint64) 51 } 52 53 // BlockchainReactor handles long-term catchup syncing. 54 type BlockchainReactor struct { 55 p2p.BaseReactor 56 57 initialState sm.State // immutable 58 state sm.State 59 60 blockExec *sm.BlockExecutor 61 store *store.BlockStore 62 63 fastSync bool 64 65 fsm *BcReactorFSM 66 blocksSynced uint64 67 68 // Receive goroutine forwards messages to this channel to be processed in the context of the poolRoutine. 69 messagesForFSMCh chan bcReactorMessage 70 71 // Switch goroutine may send RemovePeer to the blockchain reactor. This is an error message that is relayed 72 // to this channel to be processed in the context of the poolRoutine. 73 errorsForFSMCh chan bcReactorMessage 74 75 // This channel is used by the FSM and indirectly the block pool to report errors to the blockchain reactor and 76 // the switch. 77 eventsFromFSMCh chan bcFsmMessage 78 79 swReporter *behaviour.SwitchReporter 80 81 syncEnded int32 82 } 83 84 // NewBlockchainReactor returns new reactor instance. 85 func NewBlockchainReactor(state sm.State, blockExec *sm.BlockExecutor, store *store.BlockStore, 86 fastSync bool) *BlockchainReactor { 87 88 if state.LastBlockHeight != store.Height() { 89 panic(fmt.Sprintf("state (%v) and store (%v) height mismatch", state.LastBlockHeight, 90 store.Height())) 91 } 92 93 const capacity = 1000 94 eventsFromFSMCh := make(chan bcFsmMessage, capacity) 95 messagesForFSMCh := make(chan bcReactorMessage, capacity) 96 errorsForFSMCh := make(chan bcReactorMessage, capacity) 97 98 startHeight := store.Height() + 1 99 bcR := &BlockchainReactor{ 100 initialState: state, 101 state: state, 102 blockExec: blockExec, 103 fastSync: fastSync, 104 store: store, 105 messagesForFSMCh: messagesForFSMCh, 106 eventsFromFSMCh: eventsFromFSMCh, 107 errorsForFSMCh: errorsForFSMCh, 108 } 109 fsm := NewFSM(startHeight, bcR) 110 bcR.fsm = fsm 111 bcR.BaseReactor = *p2p.NewBaseReactor("BlockchainReactor", bcR) 112 //bcR.swReporter = behaviour.NewSwitchReporter(bcR.BaseReactor.Switch) 113 114 return bcR 115 } 116 117 // bcReactorMessage is used by the reactor to send messages to the FSM. 118 type bcReactorMessage struct { 119 event bReactorEvent 120 data bReactorEventData 121 } 122 123 type bFsmEvent uint 124 125 const ( 126 // message type events 127 peerErrorEv = iota + 1 128 syncFinishedEv 129 ) 130 131 type bFsmEventData struct { 132 peerID p2p.ID 133 err error 134 } 135 136 // bcFsmMessage is used by the FSM to send messages to the reactor 137 type bcFsmMessage struct { 138 event bFsmEvent 139 data bFsmEventData 140 } 141 142 // SetLogger implements service.Service by setting the logger on reactor and pool. 143 func (bcR *BlockchainReactor) SetLogger(l log.Logger) { 144 bcR.BaseService.Logger = l 145 bcR.fsm.SetLogger(l) 146 } 147 148 // OnStart implements service.Service. 149 func (bcR *BlockchainReactor) OnStart() error { 150 bcR.swReporter = behaviour.NewSwitchReporter(bcR.BaseReactor.Switch) 151 if bcR.fastSync { 152 go bcR.poolRoutine() 153 } else { 154 bcR.setSyncEnded() 155 } 156 return nil 157 } 158 159 // OnStop implements service.Service. 160 func (bcR *BlockchainReactor) OnStop() { 161 _ = bcR.Stop() 162 } 163 164 func (bcR *BlockchainReactor) isSyncEnded() bool { 165 return atomic.LoadInt32(&(bcR.syncEnded)) != 0 166 } 167 168 func (bcR *BlockchainReactor) setSyncEnded() { 169 atomic.StoreInt32(&(bcR.syncEnded), 1) 170 } 171 172 // GetChannels implements Reactor 173 func (bcR *BlockchainReactor) GetChannels() []*p2p.ChannelDescriptor { 174 return []*p2p.ChannelDescriptor{ 175 { 176 ID: BlockchainChannel, 177 Priority: 10, 178 SendQueueCapacity: 2000, 179 RecvBufferCapacity: 50 * 4096, 180 RecvMessageCapacity: maxMsgSize, 181 }, 182 } 183 } 184 185 // AddPeer implements Reactor by sending our state to peer. 186 func (bcR *BlockchainReactor) AddPeer(peer p2p.Peer) { 187 msgBytes := cdc.MustMarshalBinaryBare(&bcStatusResponseMessage{ 188 Base: bcR.store.Base(), 189 Height: bcR.store.Height(), 190 }) 191 peer.Send(BlockchainChannel, msgBytes) 192 // it's OK if send fails. will try later in poolRoutine 193 194 // peer is added to the pool once we receive the first 195 // bcStatusResponseMessage from the peer and call pool.updatePeer() 196 } 197 198 // sendBlockToPeer loads a block and sends it to the requesting peer. 199 // If the block doesn't exist a bcNoBlockResponseMessage is sent. 200 // If all nodes are honest, no node should be requesting for a block that doesn't exist. 201 func (bcR *BlockchainReactor) sendBlockToPeer(msg *bcBlockRequestMessage, 202 src p2p.Peer) (queued bool) { 203 204 block := bcR.store.LoadBlock(msg.Height) 205 if block != nil { 206 msgBytes := cdc.MustMarshalBinaryBare(&bcBlockResponseMessage{Block: block}) 207 return src.TrySend(BlockchainChannel, msgBytes) 208 } 209 210 bcR.Logger.Info("peer asking for a block we don't have", "src", src, "height", msg.Height) 211 212 msgBytes := cdc.MustMarshalBinaryBare(&bcNoBlockResponseMessage{Height: msg.Height}) 213 return src.TrySend(BlockchainChannel, msgBytes) 214 } 215 216 func (bcR *BlockchainReactor) sendStatusResponseToPeer(msg *bcStatusRequestMessage, src p2p.Peer) (queued bool) { 217 msgBytes := cdc.MustMarshalBinaryBare(&bcStatusResponseMessage{ 218 Base: bcR.store.Base(), 219 Height: bcR.store.Height(), 220 }) 221 return src.TrySend(BlockchainChannel, msgBytes) 222 } 223 224 // RemovePeer implements Reactor by removing peer from the pool. 225 func (bcR *BlockchainReactor) RemovePeer(peer p2p.Peer, reason interface{}) { 226 if bcR.isSyncEnded() { 227 return 228 } 229 msgData := bcReactorMessage{ 230 event: peerRemoveEv, 231 data: bReactorEventData{ 232 peerID: peer.ID(), 233 err: errSwitchRemovesPeer, 234 }, 235 } 236 atomic.AddInt64(&counter, 1) 237 //log2.Println("Size of the ErrorQueue ", len(bcR.errorsForFSMCh)) 238 //log2.Println("Added peer: Size of the RemovePeerCounter ", counter) 239 bcR.errorsForFSMCh <- msgData 240 } 241 242 // Receive implements Reactor by handling 4 types of messages (look below). 243 func (bcR *BlockchainReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte) { 244 msg, err := decodeMsg(msgBytes) 245 if err != nil { 246 bcR.Logger.Error("error decoding message", 247 "src", src, "chId", chID, "msg", msg, "err", err, "bytes", msgBytes) 248 _ = bcR.swReporter.Report(behaviour.BadMessage(src.ID(), err.Error())) 249 return 250 } 251 252 if err = msg.ValidateBasic(); err != nil { 253 bcR.Logger.Error("peer sent us invalid msg", "peer", src, "msg", msg, "err", err) 254 _ = bcR.swReporter.Report(behaviour.BadMessage(src.ID(), err.Error())) 255 return 256 } 257 258 bcR.Logger.Debug("Receive", "src", src, "chID", chID, "msg", msg) 259 260 switch msg := msg.(type) { 261 case *bcBlockRequestMessage: 262 if queued := bcR.sendBlockToPeer(msg, src); !queued { 263 // Unfortunately not queued since the queue is full. 264 bcR.Logger.Error("Could not send block message to peer", "src", src, "height", msg.Height) 265 } 266 267 case *bcStatusRequestMessage: 268 // Send peer our state. 269 if queued := bcR.sendStatusResponseToPeer(msg, src); !queued { 270 // Unfortunately not queued since the queue is full. 271 bcR.Logger.Error("Could not send status message to peer", "src", src) 272 } 273 274 case *bcBlockResponseMessage: 275 if bcR.isSyncEnded() { 276 return 277 } 278 279 msgForFSM := bcReactorMessage{ 280 event: blockResponseEv, 281 data: bReactorEventData{ 282 peerID: src.ID(), 283 height: msg.Block.Height, 284 block: msg.Block, 285 length: len(msgBytes), 286 }, 287 } 288 bcR.Logger.Info("Received", "src", src, "height", msg.Block.Height) 289 bcR.messagesForFSMCh <- msgForFSM 290 291 case *bcStatusResponseMessage: 292 if bcR.isSyncEnded() { 293 return 294 } 295 // Got a peer status. Unverified. 296 msgForFSM := bcReactorMessage{ 297 event: statusResponseEv, 298 data: bReactorEventData{ 299 peerID: src.ID(), 300 height: msg.Height, 301 length: len(msgBytes), 302 }, 303 } 304 bcR.messagesForFSMCh <- msgForFSM 305 306 default: 307 bcR.Logger.Error(fmt.Sprintf("unknown message type %v", reflect.TypeOf(msg))) 308 } 309 } 310 311 // processBlocksRoutine processes blocks until signlaed to stop over the stopProcessing channel 312 func (bcR *BlockchainReactor) processBlocksRoutine(stopProcessing chan struct{}) { 313 314 processReceivedBlockTicker := time.NewTicker(trySyncIntervalMS * time.Millisecond) 315 doProcessBlockCh := make(chan struct{}, 1) 316 317 lastHundred := time.Now() 318 lastRate := 0.0 319 320 ForLoop: 321 for { 322 select { 323 case <-bcR.Quit(): 324 break ForLoop 325 case <-stopProcessing: 326 bcR.Logger.Info("Finishing block execution") 327 break ForLoop 328 case <-processReceivedBlockTicker.C: // try to execute blocks 329 select { 330 case doProcessBlockCh <- struct{}{}: 331 default: 332 } 333 case <-doProcessBlockCh: 334 for { 335 err := bcR.processBlock() 336 if err == errMissingBlock { 337 break 338 } 339 // Notify FSM of block processing result. 340 msgForFSM := bcReactorMessage{ 341 event: processedBlockEv, 342 data: bReactorEventData{ 343 err: err, 344 }, 345 } 346 _ = bcR.fsm.Handle(&msgForFSM) 347 348 if err != nil { 349 break 350 } 351 352 bcR.blocksSynced++ 353 if bcR.blocksSynced%100 == 0 { 354 lastRate = 0.9*lastRate + 0.1*(100/time.Since(lastHundred).Seconds()) 355 height, maxPeerHeight := bcR.fsm.Status() 356 bcR.Logger.Info("Fast Sync Rate", "height", height, 357 "max_peer_height", maxPeerHeight, "blocks/s", lastRate) 358 lastHundred = time.Now() 359 } 360 } 361 } 362 } 363 } 364 365 // poolRoutine receives and handles messages from the Receive() routine and from the FSM. 366 func (bcR *BlockchainReactor) poolRoutine() { 367 368 bcR.fsm.Start() 369 370 sendBlockRequestTicker := time.NewTicker(trySendIntervalMS * time.Millisecond) 371 statusUpdateTicker := time.NewTicker(statusUpdateIntervalSeconds * time.Second) 372 373 stopProcessing := make(chan struct{}, 1) 374 go bcR.processBlocksRoutine(stopProcessing) 375 376 ForLoop: 377 for { 378 select { 379 380 case <-sendBlockRequestTicker.C: 381 if !bcR.fsm.NeedsBlocks() { 382 continue 383 } 384 _ = bcR.fsm.Handle(&bcReactorMessage{ 385 event: makeRequestsEv, 386 data: bReactorEventData{ 387 maxNumRequests: maxNumRequests}}) 388 389 case <-statusUpdateTicker.C: 390 // Ask for status updates. 391 go bcR.sendStatusRequest() 392 393 case msg := <-bcR.messagesForFSMCh: 394 // Sent from the Receive() routine when status (statusResponseEv) and 395 // block (blockResponseEv) response events are received 396 _ = bcR.fsm.Handle(&msg) 397 398 case msg := <-bcR.errorsForFSMCh: 399 if msg.event == peerRemoveEv { 400 atomic.AddInt64(&counter, -1) 401 //log2.Println("Removed peer. Size of the RemovePeerCounter ", counter) 402 } 403 // Sent from the switch.RemovePeer() routine (RemovePeerEv) and 404 // FSM state timer expiry routine (stateTimeoutEv). 405 _ = bcR.fsm.Handle(&msg) 406 407 case msg := <-bcR.eventsFromFSMCh: 408 switch msg.event { 409 case syncFinishedEv: 410 stopProcessing <- struct{}{} 411 bcR.setSyncEnded() 412 // Sent from the FSM when it enters finished state. 413 break ForLoop 414 case peerErrorEv: 415 // Sent from the FSM when it detects peer error 416 bcR.reportPeerErrorToSwitch(msg.data.err, msg.data.peerID) 417 if msg.data.err == errNoPeerResponse { 418 // Sent from the peer timeout handler routine 419 _ = bcR.fsm.Handle(&bcReactorMessage{ 420 event: peerRemoveEv, 421 data: bReactorEventData{ 422 peerID: msg.data.peerID, 423 err: msg.data.err, 424 }, 425 }) 426 } 427 // else { 428 // For slow peers, or errors due to blocks received from wrong peer 429 // the FSM had already removed the peers 430 // } 431 default: 432 bcR.Logger.Error("Event from FSM not supported", "type", msg.event) 433 } 434 435 case <-bcR.Quit(): 436 break ForLoop 437 } 438 } 439 } 440 441 func (bcR *BlockchainReactor) reportPeerErrorToSwitch(err error, peerID p2p.ID) { 442 peer := bcR.Switch.Peers().Get(peerID) 443 if peer != nil { 444 _ = bcR.swReporter.Report(behaviour.BadMessage(peerID, err.Error())) 445 } 446 } 447 448 func (bcR *BlockchainReactor) processBlock() error { 449 450 first, second, err := bcR.fsm.FirstTwoBlocks() 451 if err != nil { 452 // We need both to sync the first block. 453 return err 454 } 455 456 chainID := bcR.initialState.ChainID 457 458 firstParts := first.MakePartSet(types.BlockPartSizeBytes) 459 firstPartsHeader := firstParts.Header() 460 firstID := types.BlockID{Hash: first.Hash(), PartsHeader: firstPartsHeader} 461 // Finally, verify the first block using the second's commit 462 // NOTE: we can probably make this more efficient, but note that calling 463 // first.Hash() doesn't verify the tx contents, so MakePartSet() is 464 // currently necessary. 465 err = bcR.state.Validators.VerifyCommit(chainID, firstID, first.Height, second.LastCommit) 466 if err != nil { 467 bcR.Logger.Error("error during commit verification", "err", err, 468 "first", first.Height, "second", second.Height) 469 return errBlockVerificationFailure 470 } 471 472 bcR.store.SaveBlock(first, firstParts, second.LastCommit) 473 474 bcR.state, _, err = bcR.blockExec.ApplyBlock(bcR.state, firstID, first) 475 if err != nil { 476 panic(fmt.Sprintf("failed to process committed block (%d:%X): %v", first.Height, first.Hash(), err)) 477 } 478 479 return nil 480 } 481 482 // Implements bcRNotifier 483 // sendStatusRequest broadcasts `BlockStore` height. 484 func (bcR *BlockchainReactor) sendStatusRequest() { 485 msgBytes := cdc.MustMarshalBinaryBare(&bcStatusRequestMessage{ 486 Base: bcR.store.Base(), 487 Height: bcR.store.Height(), 488 }) 489 bcR.Switch.Broadcast(BlockchainChannel, msgBytes) 490 } 491 492 // Implements bcRNotifier 493 // BlockRequest sends `BlockRequest` height. 494 func (bcR *BlockchainReactor) sendBlockRequest(peerID p2p.ID, height int64) error { 495 peer := bcR.Switch.Peers().Get(peerID) 496 if peer == nil { 497 return errNilPeerForBlockRequest 498 } 499 500 msgBytes := cdc.MustMarshalBinaryBare(&bcBlockRequestMessage{height}) 501 queued := peer.TrySend(BlockchainChannel, msgBytes) 502 if !queued { 503 return errSendQueueFull 504 } 505 return nil 506 } 507 508 // Implements bcRNotifier 509 func (bcR *BlockchainReactor) switchToConsensus() { 510 conR, ok := bcR.Switch.Reactor("CONSENSUS").(consensusReactor) 511 if ok { 512 conR.SwitchToConsensus(bcR.state, bcR.blocksSynced) 513 bcR.eventsFromFSMCh <- bcFsmMessage{event: syncFinishedEv} 514 } 515 // else { 516 // Should only happen during testing. 517 // } 518 } 519 520 // Implements bcRNotifier 521 // Called by FSM and pool: 522 // - pool calls when it detects slow peer or when peer times out 523 // - FSM calls when: 524 // - adding a block (addBlock) fails 525 // - reactor processing of a block reports failure and FSM sends back the peers of first and second blocks 526 func (bcR *BlockchainReactor) sendPeerError(err error, peerID p2p.ID) { 527 bcR.Logger.Debug("sendPeerError:", "peer", peerID, "error", err) 528 msgData := bcFsmMessage{ 529 event: peerErrorEv, 530 data: bFsmEventData{ 531 peerID: peerID, 532 err: err, 533 }, 534 } 535 bcR.eventsFromFSMCh <- msgData 536 } 537 538 // Implements bcRNotifier 539 func (bcR *BlockchainReactor) resetStateTimer(name string, timer **time.Timer, timeout time.Duration) { 540 if timer == nil { 541 panic("nil timer pointer parameter") 542 } 543 if *timer == nil { 544 *timer = time.AfterFunc(timeout, func() { 545 msg := bcReactorMessage{ 546 event: stateTimeoutEv, 547 data: bReactorEventData{ 548 stateName: name, 549 }, 550 } 551 bcR.errorsForFSMCh <- msg 552 }) 553 } else { 554 (*timer).Reset(timeout) 555 } 556 } 557 558 //----------------------------------------------------------------------------- 559 // Messages 560 561 // BlockchainMessage is a generic message for this reactor. 562 type BlockchainMessage interface { 563 ValidateBasic() error 564 } 565 566 // RegisterBlockchainMessages registers the fast sync messages for amino encoding. 567 func RegisterBlockchainMessages(cdc *amino.Codec) { 568 cdc.RegisterInterface((*BlockchainMessage)(nil), nil) 569 cdc.RegisterConcrete(&bcBlockRequestMessage{}, "tendermint/blockchain/BlockRequest", nil) 570 cdc.RegisterConcrete(&bcBlockResponseMessage{}, "tendermint/blockchain/BlockResponse", nil) 571 cdc.RegisterConcrete(&bcNoBlockResponseMessage{}, "tendermint/blockchain/NoBlockResponse", nil) 572 cdc.RegisterConcrete(&bcStatusResponseMessage{}, "tendermint/blockchain/StatusResponse", nil) 573 cdc.RegisterConcrete(&bcStatusRequestMessage{}, "tendermint/blockchain/StatusRequest", nil) 574 } 575 576 func decodeMsg(bz []byte) (msg BlockchainMessage, err error) { 577 if len(bz) > maxMsgSize { 578 return msg, fmt.Errorf("msg exceeds max size (%d > %d)", len(bz), maxMsgSize) 579 } 580 err = cdc.UnmarshalBinaryBare(bz, &msg) 581 return 582 } 583 584 //------------------------------------- 585 586 type bcBlockRequestMessage struct { 587 Height int64 588 } 589 590 // ValidateBasic performs basic validation. 591 func (m *bcBlockRequestMessage) ValidateBasic() error { 592 if m.Height < 0 { 593 return errors.New("negative Height") 594 } 595 return nil 596 } 597 598 func (m *bcBlockRequestMessage) String() string { 599 return fmt.Sprintf("[bcBlockRequestMessage %v]", m.Height) 600 } 601 602 type bcNoBlockResponseMessage struct { 603 Height int64 604 } 605 606 // ValidateBasic performs basic validation. 607 func (m *bcNoBlockResponseMessage) ValidateBasic() error { 608 if m.Height < 0 { 609 return errors.New("negative Height") 610 } 611 return nil 612 } 613 614 func (m *bcNoBlockResponseMessage) String() string { 615 return fmt.Sprintf("[bcNoBlockResponseMessage %d]", m.Height) 616 } 617 618 //------------------------------------- 619 620 type bcBlockResponseMessage struct { 621 Block *types.Block 622 } 623 624 // ValidateBasic performs basic validation. 625 func (m *bcBlockResponseMessage) ValidateBasic() error { 626 return m.Block.ValidateBasic() 627 } 628 629 func (m *bcBlockResponseMessage) String() string { 630 return fmt.Sprintf("[bcBlockResponseMessage %v]", m.Block.Height) 631 } 632 633 //------------------------------------- 634 635 type bcStatusRequestMessage struct { 636 Height int64 637 Base int64 638 } 639 640 // ValidateBasic performs basic validation. 641 func (m *bcStatusRequestMessage) ValidateBasic() error { 642 if m.Height < 0 { 643 return errors.New("negative Height") 644 } 645 if m.Base < 0 { 646 return errors.New("negative Base") 647 } 648 if m.Base > m.Height { 649 return fmt.Errorf("base %v cannot be greater than height %v", m.Base, m.Height) 650 } 651 return nil 652 } 653 654 func (m *bcStatusRequestMessage) String() string { 655 return fmt.Sprintf("[bcStatusRequestMessage %v:%v]", m.Base, m.Height) 656 } 657 658 //------------------------------------- 659 660 type bcStatusResponseMessage struct { 661 Height int64 662 Base int64 663 } 664 665 // ValidateBasic performs basic validation. 666 func (m *bcStatusResponseMessage) ValidateBasic() error { 667 if m.Height < 0 { 668 return errors.New("negative Height") 669 } 670 if m.Base < 0 { 671 return errors.New("negative Base") 672 } 673 if m.Base > m.Height { 674 return fmt.Errorf("base %v cannot be greater than height %v", m.Base, m.Height) 675 } 676 return nil 677 } 678 679 func (m *bcStatusResponseMessage) String() string { 680 return fmt.Sprintf("[bcStatusResponseMessage %v:%v]", m.Base, m.Height) 681 }