github.com/okex/exchain@v1.8.0/libs/tendermint/blockchain/v1/reactor.go (about) 1 package v1 2 3 import ( 4 "errors" 5 "fmt" 6 "reflect" 7 "time" 8 9 amino "github.com/tendermint/go-amino" 10 11 "github.com/okex/exchain/libs/tendermint/behaviour" 12 "github.com/okex/exchain/libs/tendermint/libs/log" 13 "github.com/okex/exchain/libs/tendermint/p2p" 14 sm "github.com/okex/exchain/libs/tendermint/state" 15 "github.com/okex/exchain/libs/tendermint/store" 16 "github.com/okex/exchain/libs/tendermint/types" 17 ) 18 19 const ( 20 // BlockchainChannel is a channel for blocks and status updates (`BlockStore` height) 21 BlockchainChannel = byte(0x40) 22 trySyncIntervalMS = 10 23 trySendIntervalMS = 10 24 25 // ask for best height every 10s 26 statusUpdateIntervalSeconds = 10 27 28 // NOTE: keep up to date with bcBlockResponseMessage 29 bcBlockResponseMessagePrefixSize = 4 30 bcBlockResponseMessageFieldKeySize = 1 31 maxMsgSize = types.MaxBlockSizeBytes + 32 bcBlockResponseMessagePrefixSize + 33 bcBlockResponseMessageFieldKeySize 34 ) 35 36 var ( 37 // Maximum number of requests that can be pending per peer, i.e. for which requests have been sent but blocks 38 // have not been received. 39 maxRequestsPerPeer = 20 40 // Maximum number of block requests for the reactor, pending or for which blocks have been received. 41 maxNumRequests = 64 42 ) 43 44 type consensusReactor interface { 45 // for when we switch from blockchain reactor and fast sync to 46 // the consensus machine 47 SwitchToConsensus(sm.State, uint64) 48 } 49 50 // BlockchainReactor handles long-term catchup syncing. 51 type BlockchainReactor struct { 52 p2p.BaseReactor 53 54 initialState sm.State // immutable 55 state sm.State 56 57 blockExec *sm.BlockExecutor 58 store *store.BlockStore 59 60 fastSync bool 61 62 fsm *BcReactorFSM 63 blocksSynced uint64 64 65 // Receive goroutine forwards messages to this channel to be processed in the context of the poolRoutine. 66 messagesForFSMCh chan bcReactorMessage 67 68 // Switch goroutine may send RemovePeer to the blockchain reactor. This is an error message that is relayed 69 // to this channel to be processed in the context of the poolRoutine. 70 errorsForFSMCh chan bcReactorMessage 71 72 // This channel is used by the FSM and indirectly the block pool to report errors to the blockchain reactor and 73 // the switch. 74 eventsFromFSMCh chan bcFsmMessage 75 76 swReporter *behaviour.SwitchReporter 77 } 78 79 // NewBlockchainReactor returns new reactor instance. 80 func NewBlockchainReactor(state sm.State, blockExec *sm.BlockExecutor, store *store.BlockStore, 81 fastSync bool) *BlockchainReactor { 82 83 if state.LastBlockHeight != store.Height() { 84 panic(fmt.Sprintf("state (%v) and store (%v) height mismatch", state.LastBlockHeight, 85 store.Height())) 86 } 87 88 const capacity = 1000 89 eventsFromFSMCh := make(chan bcFsmMessage, capacity) 90 messagesForFSMCh := make(chan bcReactorMessage, capacity) 91 errorsForFSMCh := make(chan bcReactorMessage, capacity) 92 93 startHeight := store.Height() + 1 94 bcR := &BlockchainReactor{ 95 initialState: state, 96 state: state, 97 blockExec: blockExec, 98 fastSync: fastSync, 99 store: store, 100 messagesForFSMCh: messagesForFSMCh, 101 eventsFromFSMCh: eventsFromFSMCh, 102 errorsForFSMCh: errorsForFSMCh, 103 } 104 fsm := NewFSM(startHeight, bcR) 105 bcR.fsm = fsm 106 bcR.BaseReactor = *p2p.NewBaseReactor("BlockchainReactor", bcR) 107 //bcR.swReporter = behaviour.NewSwitchReporter(bcR.BaseReactor.Switch) 108 109 return bcR 110 } 111 112 // bcReactorMessage is used by the reactor to send messages to the FSM. 113 type bcReactorMessage struct { 114 event bReactorEvent 115 data bReactorEventData 116 } 117 118 type bFsmEvent uint 119 120 const ( 121 // message type events 122 peerErrorEv = iota + 1 123 syncFinishedEv 124 ) 125 126 type bFsmEventData struct { 127 peerID p2p.ID 128 err error 129 } 130 131 // bcFsmMessage is used by the FSM to send messages to the reactor 132 type bcFsmMessage struct { 133 event bFsmEvent 134 data bFsmEventData 135 } 136 137 // SetLogger implements service.Service by setting the logger on reactor and pool. 138 func (bcR *BlockchainReactor) SetLogger(l log.Logger) { 139 bcR.BaseService.Logger = l 140 bcR.fsm.SetLogger(l) 141 } 142 143 // OnStart implements service.Service. 144 func (bcR *BlockchainReactor) OnStart() error { 145 bcR.swReporter = behaviour.NewSwitchReporter(bcR.BaseReactor.Switch) 146 if bcR.fastSync { 147 go bcR.poolRoutine() 148 } 149 return nil 150 } 151 152 // OnStop implements service.Service. 153 func (bcR *BlockchainReactor) OnStop() { 154 _ = bcR.Stop() 155 } 156 157 // GetChannels implements Reactor 158 func (bcR *BlockchainReactor) GetChannels() []*p2p.ChannelDescriptor { 159 return []*p2p.ChannelDescriptor{ 160 { 161 ID: BlockchainChannel, 162 Priority: 10, 163 SendQueueCapacity: 2000, 164 RecvBufferCapacity: 50 * 4096, 165 RecvMessageCapacity: maxMsgSize, 166 }, 167 } 168 } 169 170 // AddPeer implements Reactor by sending our state to peer. 171 func (bcR *BlockchainReactor) AddPeer(peer p2p.Peer) { 172 msgBytes := cdc.MustMarshalBinaryBare(&bcStatusResponseMessage{ 173 Base: bcR.store.Base(), 174 Height: bcR.store.Height(), 175 }) 176 peer.Send(BlockchainChannel, msgBytes) 177 // it's OK if send fails. will try later in poolRoutine 178 179 // peer is added to the pool once we receive the first 180 // bcStatusResponseMessage from the peer and call pool.updatePeer() 181 } 182 183 // sendBlockToPeer loads a block and sends it to the requesting peer. 184 // If the block doesn't exist a bcNoBlockResponseMessage is sent. 185 // If all nodes are honest, no node should be requesting for a block that doesn't exist. 186 func (bcR *BlockchainReactor) sendBlockToPeer(msg *bcBlockRequestMessage, 187 src p2p.Peer) (queued bool) { 188 189 block := bcR.store.LoadBlock(msg.Height) 190 if block != nil { 191 msgBytes := cdc.MustMarshalBinaryBare(&bcBlockResponseMessage{Block: block}) 192 return src.TrySend(BlockchainChannel, msgBytes) 193 } 194 195 bcR.Logger.Info("peer asking for a block we don't have", "src", src, "height", msg.Height) 196 197 msgBytes := cdc.MustMarshalBinaryBare(&bcNoBlockResponseMessage{Height: msg.Height}) 198 return src.TrySend(BlockchainChannel, msgBytes) 199 } 200 201 func (bcR *BlockchainReactor) sendStatusResponseToPeer(msg *bcStatusRequestMessage, src p2p.Peer) (queued bool) { 202 msgBytes := cdc.MustMarshalBinaryBare(&bcStatusResponseMessage{ 203 Base: bcR.store.Base(), 204 Height: bcR.store.Height(), 205 }) 206 return src.TrySend(BlockchainChannel, msgBytes) 207 } 208 209 // RemovePeer implements Reactor by removing peer from the pool. 210 func (bcR *BlockchainReactor) RemovePeer(peer p2p.Peer, reason interface{}) { 211 msgData := bcReactorMessage{ 212 event: peerRemoveEv, 213 data: bReactorEventData{ 214 peerID: peer.ID(), 215 err: errSwitchRemovesPeer, 216 }, 217 } 218 bcR.errorsForFSMCh <- msgData 219 } 220 221 // Receive implements Reactor by handling 4 types of messages (look below). 222 func (bcR *BlockchainReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte) { 223 msg, err := decodeMsg(msgBytes) 224 if err != nil { 225 bcR.Logger.Error("error decoding message", 226 "src", src, "chId", chID, "msg", msg, "err", err, "bytes", msgBytes) 227 _ = bcR.swReporter.Report(behaviour.BadMessage(src.ID(), err.Error())) 228 return 229 } 230 231 if err = msg.ValidateBasic(); err != nil { 232 bcR.Logger.Error("peer sent us invalid msg", "peer", src, "msg", msg, "err", err) 233 _ = bcR.swReporter.Report(behaviour.BadMessage(src.ID(), err.Error())) 234 return 235 } 236 237 bcR.Logger.Debug("Receive", "src", src, "chID", chID, "msg", msg) 238 239 switch msg := msg.(type) { 240 case *bcBlockRequestMessage: 241 if queued := bcR.sendBlockToPeer(msg, src); !queued { 242 // Unfortunately not queued since the queue is full. 243 bcR.Logger.Error("Could not send block message to peer", "src", src, "height", msg.Height) 244 } 245 246 case *bcStatusRequestMessage: 247 // Send peer our state. 248 if queued := bcR.sendStatusResponseToPeer(msg, src); !queued { 249 // Unfortunately not queued since the queue is full. 250 bcR.Logger.Error("Could not send status message to peer", "src", src) 251 } 252 253 case *bcBlockResponseMessage: 254 msgForFSM := bcReactorMessage{ 255 event: blockResponseEv, 256 data: bReactorEventData{ 257 peerID: src.ID(), 258 height: msg.Block.Height, 259 block: msg.Block, 260 length: len(msgBytes), 261 }, 262 } 263 bcR.Logger.Info("Received", "src", src, "height", msg.Block.Height) 264 bcR.messagesForFSMCh <- msgForFSM 265 266 case *bcStatusResponseMessage: 267 // Got a peer status. Unverified. 268 msgForFSM := bcReactorMessage{ 269 event: statusResponseEv, 270 data: bReactorEventData{ 271 peerID: src.ID(), 272 height: msg.Height, 273 length: len(msgBytes), 274 }, 275 } 276 bcR.messagesForFSMCh <- msgForFSM 277 278 default: 279 bcR.Logger.Error(fmt.Sprintf("unknown message type %v", reflect.TypeOf(msg))) 280 } 281 } 282 283 // processBlocksRoutine processes blocks until signlaed to stop over the stopProcessing channel 284 func (bcR *BlockchainReactor) processBlocksRoutine(stopProcessing chan struct{}) { 285 286 processReceivedBlockTicker := time.NewTicker(trySyncIntervalMS * time.Millisecond) 287 doProcessBlockCh := make(chan struct{}, 1) 288 289 lastHundred := time.Now() 290 lastRate := 0.0 291 292 ForLoop: 293 for { 294 select { 295 case <-stopProcessing: 296 bcR.Logger.Info("finishing block execution") 297 break ForLoop 298 case <-processReceivedBlockTicker.C: // try to execute blocks 299 select { 300 case doProcessBlockCh <- struct{}{}: 301 default: 302 } 303 case <-doProcessBlockCh: 304 for { 305 err := bcR.processBlock() 306 if err == errMissingBlock { 307 break 308 } 309 // Notify FSM of block processing result. 310 msgForFSM := bcReactorMessage{ 311 event: processedBlockEv, 312 data: bReactorEventData{ 313 err: err, 314 }, 315 } 316 _ = bcR.fsm.Handle(&msgForFSM) 317 318 if err != nil { 319 break 320 } 321 322 bcR.blocksSynced++ 323 if bcR.blocksSynced%100 == 0 { 324 lastRate = 0.9*lastRate + 0.1*(100/time.Since(lastHundred).Seconds()) 325 height, maxPeerHeight := bcR.fsm.Status() 326 bcR.Logger.Info("Fast Sync Rate", "height", height, 327 "max_peer_height", maxPeerHeight, "blocks/s", lastRate) 328 lastHundred = time.Now() 329 } 330 } 331 } 332 } 333 } 334 335 // poolRoutine receives and handles messages from the Receive() routine and from the FSM. 336 func (bcR *BlockchainReactor) poolRoutine() { 337 338 bcR.fsm.Start() 339 340 sendBlockRequestTicker := time.NewTicker(trySendIntervalMS * time.Millisecond) 341 statusUpdateTicker := time.NewTicker(statusUpdateIntervalSeconds * time.Second) 342 343 stopProcessing := make(chan struct{}, 1) 344 go bcR.processBlocksRoutine(stopProcessing) 345 346 ForLoop: 347 for { 348 select { 349 350 case <-sendBlockRequestTicker.C: 351 if !bcR.fsm.NeedsBlocks() { 352 continue 353 } 354 _ = bcR.fsm.Handle(&bcReactorMessage{ 355 event: makeRequestsEv, 356 data: bReactorEventData{ 357 maxNumRequests: maxNumRequests}}) 358 359 case <-statusUpdateTicker.C: 360 // Ask for status updates. 361 go bcR.sendStatusRequest() 362 363 case msg := <-bcR.messagesForFSMCh: 364 // Sent from the Receive() routine when status (statusResponseEv) and 365 // block (blockResponseEv) response events are received 366 _ = bcR.fsm.Handle(&msg) 367 368 case msg := <-bcR.errorsForFSMCh: 369 // Sent from the switch.RemovePeer() routine (RemovePeerEv) and 370 // FSM state timer expiry routine (stateTimeoutEv). 371 _ = bcR.fsm.Handle(&msg) 372 373 case msg := <-bcR.eventsFromFSMCh: 374 switch msg.event { 375 case syncFinishedEv: 376 stopProcessing <- struct{}{} 377 // Sent from the FSM when it enters finished state. 378 break ForLoop 379 case peerErrorEv: 380 // Sent from the FSM when it detects peer error 381 bcR.reportPeerErrorToSwitch(msg.data.err, msg.data.peerID) 382 if msg.data.err == errNoPeerResponse { 383 // Sent from the peer timeout handler routine 384 _ = bcR.fsm.Handle(&bcReactorMessage{ 385 event: peerRemoveEv, 386 data: bReactorEventData{ 387 peerID: msg.data.peerID, 388 err: msg.data.err, 389 }, 390 }) 391 } 392 // else { 393 // For slow peers, or errors due to blocks received from wrong peer 394 // the FSM had already removed the peers 395 // } 396 default: 397 bcR.Logger.Error("Event from FSM not supported", "type", msg.event) 398 } 399 400 case <-bcR.Quit(): 401 break ForLoop 402 } 403 } 404 } 405 406 func (bcR *BlockchainReactor) reportPeerErrorToSwitch(err error, peerID p2p.ID) { 407 peer := bcR.Switch.Peers().Get(peerID) 408 if peer != nil { 409 _ = bcR.swReporter.Report(behaviour.BadMessage(peerID, err.Error())) 410 } 411 } 412 413 func (bcR *BlockchainReactor) processBlock() error { 414 415 first, second, err := bcR.fsm.FirstTwoBlocks() 416 if err != nil { 417 // We need both to sync the first block. 418 return err 419 } 420 421 chainID := bcR.initialState.ChainID 422 423 firstParts := first.MakePartSet(types.BlockPartSizeBytes) 424 firstPartsHeader := firstParts.Header() 425 firstID := types.BlockID{Hash: first.Hash(), PartsHeader: firstPartsHeader} 426 // Finally, verify the first block using the second's commit 427 // NOTE: we can probably make this more efficient, but note that calling 428 // first.Hash() doesn't verify the tx contents, so MakePartSet() is 429 // currently necessary. 430 err = bcR.state.Validators.VerifyCommitLight(chainID, firstID, first.Height, second.LastCommit) 431 if err != nil { 432 bcR.Logger.Error("error during commit verification", "err", err, 433 "first", first.Height, "second", second.Height) 434 return errBlockVerificationFailure 435 } 436 437 bcR.store.SaveBlock(first, firstParts, second.LastCommit) 438 439 bcR.state, _, err = bcR.blockExec.ApplyBlock(bcR.state, firstID, first) 440 if err != nil { 441 panic(fmt.Sprintf("failed to process committed block (%d:%X): %v", first.Height, first.Hash(), err)) 442 } 443 444 return nil 445 } 446 447 // Implements bcRNotifier 448 // sendStatusRequest broadcasts `BlockStore` height. 449 func (bcR *BlockchainReactor) sendStatusRequest() { 450 msgBytes := cdc.MustMarshalBinaryBare(&bcStatusRequestMessage{ 451 Base: bcR.store.Base(), 452 Height: bcR.store.Height(), 453 }) 454 bcR.Switch.Broadcast(BlockchainChannel, msgBytes) 455 } 456 457 // Implements bcRNotifier 458 // BlockRequest sends `BlockRequest` height. 459 func (bcR *BlockchainReactor) sendBlockRequest(peerID p2p.ID, height int64) error { 460 peer := bcR.Switch.Peers().Get(peerID) 461 if peer == nil { 462 return errNilPeerForBlockRequest 463 } 464 465 msgBytes := cdc.MustMarshalBinaryBare(&bcBlockRequestMessage{height}) 466 queued := peer.TrySend(BlockchainChannel, msgBytes) 467 if !queued { 468 return errSendQueueFull 469 } 470 return nil 471 } 472 473 // Implements bcRNotifier 474 func (bcR *BlockchainReactor) switchToConsensus() { 475 conR, ok := bcR.Switch.Reactor("CONSENSUS").(consensusReactor) 476 if ok { 477 conR.SwitchToConsensus(bcR.state, bcR.blocksSynced) 478 bcR.eventsFromFSMCh <- bcFsmMessage{event: syncFinishedEv} 479 } 480 // else { 481 // Should only happen during testing. 482 // } 483 } 484 485 // Implements bcRNotifier 486 // Called by FSM and pool: 487 // - pool calls when it detects slow peer or when peer times out 488 // - FSM calls when: 489 // - adding a block (addBlock) fails 490 // - reactor processing of a block reports failure and FSM sends back the peers of first and second blocks 491 func (bcR *BlockchainReactor) sendPeerError(err error, peerID p2p.ID) { 492 bcR.Logger.Info("sendPeerError:", "peer", peerID, "error", err) 493 msgData := bcFsmMessage{ 494 event: peerErrorEv, 495 data: bFsmEventData{ 496 peerID: peerID, 497 err: err, 498 }, 499 } 500 bcR.eventsFromFSMCh <- msgData 501 } 502 503 // Implements bcRNotifier 504 func (bcR *BlockchainReactor) resetStateTimer(name string, timer **time.Timer, timeout time.Duration) { 505 if timer == nil { 506 panic("nil timer pointer parameter") 507 } 508 if *timer == nil { 509 *timer = time.AfterFunc(timeout, func() { 510 msg := bcReactorMessage{ 511 event: stateTimeoutEv, 512 data: bReactorEventData{ 513 stateName: name, 514 }, 515 } 516 bcR.errorsForFSMCh <- msg 517 }) 518 } else { 519 (*timer).Reset(timeout) 520 } 521 } 522 523 //----------------------------------------------------------------------------- 524 // Messages 525 526 // BlockchainMessage is a generic message for this reactor. 527 type BlockchainMessage interface { 528 ValidateBasic() error 529 } 530 531 // RegisterBlockchainMessages registers the fast sync messages for amino encoding. 532 func RegisterBlockchainMessages(cdc *amino.Codec) { 533 cdc.RegisterInterface((*BlockchainMessage)(nil), nil) 534 cdc.RegisterConcrete(&bcBlockRequestMessage{}, "tendermint/blockchain/BlockRequest", nil) 535 cdc.RegisterConcrete(&bcBlockResponseMessage{}, "tendermint/blockchain/BlockResponse", nil) 536 cdc.RegisterConcrete(&bcNoBlockResponseMessage{}, "tendermint/blockchain/NoBlockResponse", nil) 537 cdc.RegisterConcrete(&bcStatusResponseMessage{}, "tendermint/blockchain/StatusResponse", nil) 538 cdc.RegisterConcrete(&bcStatusRequestMessage{}, "tendermint/blockchain/StatusRequest", nil) 539 } 540 541 func decodeMsg(bz []byte) (msg BlockchainMessage, err error) { 542 if len(bz) > maxMsgSize { 543 return msg, fmt.Errorf("msg exceeds max size (%d > %d)", len(bz), maxMsgSize) 544 } 545 err = cdc.UnmarshalBinaryBare(bz, &msg) 546 return 547 } 548 549 //------------------------------------- 550 551 type bcBlockRequestMessage struct { 552 Height int64 553 } 554 555 // ValidateBasic performs basic validation. 556 func (m *bcBlockRequestMessage) ValidateBasic() error { 557 if m.Height < 0 { 558 return errors.New("negative Height") 559 } 560 return nil 561 } 562 563 func (m *bcBlockRequestMessage) String() string { 564 return fmt.Sprintf("[bcBlockRequestMessage %v]", m.Height) 565 } 566 567 type bcNoBlockResponseMessage struct { 568 Height int64 569 } 570 571 // ValidateBasic performs basic validation. 572 func (m *bcNoBlockResponseMessage) ValidateBasic() error { 573 if m.Height < 0 { 574 return errors.New("negative Height") 575 } 576 return nil 577 } 578 579 func (m *bcNoBlockResponseMessage) String() string { 580 return fmt.Sprintf("[bcNoBlockResponseMessage %d]", m.Height) 581 } 582 583 //------------------------------------- 584 585 type bcBlockResponseMessage struct { 586 Block *types.Block 587 } 588 589 // ValidateBasic performs basic validation. 590 func (m *bcBlockResponseMessage) ValidateBasic() error { 591 return m.Block.ValidateBasic() 592 } 593 594 func (m *bcBlockResponseMessage) String() string { 595 return fmt.Sprintf("[bcBlockResponseMessage %v]", m.Block.Height) 596 } 597 598 //------------------------------------- 599 600 type bcStatusRequestMessage struct { 601 Height int64 602 Base int64 603 } 604 605 // ValidateBasic performs basic validation. 606 func (m *bcStatusRequestMessage) ValidateBasic() error { 607 if m.Height < 0 { 608 return errors.New("negative Height") 609 } 610 if m.Base < 0 { 611 return errors.New("negative Base") 612 } 613 if m.Base > m.Height { 614 return fmt.Errorf("base %v cannot be greater than height %v", m.Base, m.Height) 615 } 616 return nil 617 } 618 619 func (m *bcStatusRequestMessage) String() string { 620 return fmt.Sprintf("[bcStatusRequestMessage %v:%v]", m.Base, m.Height) 621 } 622 623 //------------------------------------- 624 625 type bcStatusResponseMessage struct { 626 Height int64 627 Base int64 628 } 629 630 // ValidateBasic performs basic validation. 631 func (m *bcStatusResponseMessage) ValidateBasic() error { 632 if m.Height < 0 { 633 return errors.New("negative Height") 634 } 635 if m.Base < 0 { 636 return errors.New("negative Base") 637 } 638 if m.Base > m.Height { 639 return fmt.Errorf("base %v cannot be greater than height %v", m.Base, m.Height) 640 } 641 return nil 642 } 643 644 func (m *bcStatusResponseMessage) String() string { 645 return fmt.Sprintf("[bcStatusResponseMessage %v:%v]", m.Base, m.Height) 646 }