github.com/evdatsion/aphelion-dpos-bft@v0.32.1/blockchain/reactor.go (about)

     1  package blockchain
     2  
     3  import (
     4  	"errors"
     5  	"fmt"
     6  	"reflect"
     7  	"time"
     8  
     9  	amino "github.com/evdatsion/go-amino"
    10  
    11  	"github.com/evdatsion/aphelion-dpos-bft/libs/log"
    12  	"github.com/evdatsion/aphelion-dpos-bft/p2p"
    13  	sm "github.com/evdatsion/aphelion-dpos-bft/state"
    14  	"github.com/evdatsion/aphelion-dpos-bft/types"
    15  )
    16  
    17  const (
    18  	// BlockchainChannel is a channel for blocks and status updates (`BlockStore` height)
    19  	BlockchainChannel = byte(0x40)
    20  
    21  	trySyncIntervalMS = 10
    22  
    23  	// stop syncing when last block's time is
    24  	// within this much of the system time.
    25  	// stopSyncingDurationMinutes = 10
    26  
    27  	// ask for best height every 10s
    28  	statusUpdateIntervalSeconds = 10
    29  	// check if we should switch to consensus reactor
    30  	switchToConsensusIntervalSeconds = 1
    31  
    32  	// NOTE: keep up to date with bcBlockResponseMessage
    33  	bcBlockResponseMessagePrefixSize   = 4
    34  	bcBlockResponseMessageFieldKeySize = 1
    35  	maxMsgSize                         = types.MaxBlockSizeBytes +
    36  		bcBlockResponseMessagePrefixSize +
    37  		bcBlockResponseMessageFieldKeySize
    38  )
    39  
    40  type consensusReactor interface {
    41  	// for when we switch from blockchain reactor and fast sync to
    42  	// the consensus machine
    43  	SwitchToConsensus(sm.State, int)
    44  }
    45  
    46  type peerError struct {
    47  	err    error
    48  	peerID p2p.ID
    49  }
    50  
    51  func (e peerError) Error() string {
    52  	return fmt.Sprintf("error with peer %v: %s", e.peerID, e.err.Error())
    53  }
    54  
    55  // BlockchainReactor handles long-term catchup syncing.
    56  type BlockchainReactor struct {
    57  	p2p.BaseReactor
    58  
    59  	// immutable
    60  	initialState sm.State
    61  
    62  	blockExec *sm.BlockExecutor
    63  	store     *BlockStore
    64  	pool      *BlockPool
    65  	fastSync  bool
    66  
    67  	requestsCh <-chan BlockRequest
    68  	errorsCh   <-chan peerError
    69  }
    70  
    71  // NewBlockchainReactor returns new reactor instance.
    72  func NewBlockchainReactor(state sm.State, blockExec *sm.BlockExecutor, store *BlockStore,
    73  	fastSync bool) *BlockchainReactor {
    74  
    75  	if state.LastBlockHeight != store.Height() {
    76  		panic(fmt.Sprintf("state (%v) and store (%v) height mismatch", state.LastBlockHeight,
    77  			store.Height()))
    78  	}
    79  
    80  	requestsCh := make(chan BlockRequest, maxTotalRequesters)
    81  
    82  	const capacity = 1000                      // must be bigger than peers count
    83  	errorsCh := make(chan peerError, capacity) // so we don't block in #Receive#pool.AddBlock
    84  
    85  	pool := NewBlockPool(
    86  		store.Height()+1,
    87  		requestsCh,
    88  		errorsCh,
    89  	)
    90  
    91  	bcR := &BlockchainReactor{
    92  		initialState: state,
    93  		blockExec:    blockExec,
    94  		store:        store,
    95  		pool:         pool,
    96  		fastSync:     fastSync,
    97  		requestsCh:   requestsCh,
    98  		errorsCh:     errorsCh,
    99  	}
   100  	bcR.BaseReactor = *p2p.NewBaseReactor("BlockchainReactor", bcR)
   101  	return bcR
   102  }
   103  
   104  // SetLogger implements cmn.Service by setting the logger on reactor and pool.
   105  func (bcR *BlockchainReactor) SetLogger(l log.Logger) {
   106  	bcR.BaseService.Logger = l
   107  	bcR.pool.Logger = l
   108  }
   109  
   110  // OnStart implements cmn.Service.
   111  func (bcR *BlockchainReactor) OnStart() error {
   112  	if bcR.fastSync {
   113  		err := bcR.pool.Start()
   114  		if err != nil {
   115  			return err
   116  		}
   117  		go bcR.poolRoutine()
   118  	}
   119  	return nil
   120  }
   121  
   122  // OnStop implements cmn.Service.
   123  func (bcR *BlockchainReactor) OnStop() {
   124  	bcR.pool.Stop()
   125  }
   126  
   127  // GetChannels implements Reactor
   128  func (bcR *BlockchainReactor) GetChannels() []*p2p.ChannelDescriptor {
   129  	return []*p2p.ChannelDescriptor{
   130  		{
   131  			ID:                  BlockchainChannel,
   132  			Priority:            10,
   133  			SendQueueCapacity:   1000,
   134  			RecvBufferCapacity:  50 * 4096,
   135  			RecvMessageCapacity: maxMsgSize,
   136  		},
   137  	}
   138  }
   139  
   140  // AddPeer implements Reactor by sending our state to peer.
   141  func (bcR *BlockchainReactor) AddPeer(peer p2p.Peer) {
   142  	msgBytes := cdc.MustMarshalBinaryBare(&bcStatusResponseMessage{bcR.store.Height()})
   143  	if !peer.Send(BlockchainChannel, msgBytes) {
   144  		// doing nothing, will try later in `poolRoutine`
   145  	}
   146  	// peer is added to the pool once we receive the first
   147  	// bcStatusResponseMessage from the peer and call pool.SetPeerHeight
   148  }
   149  
   150  // RemovePeer implements Reactor by removing peer from the pool.
   151  func (bcR *BlockchainReactor) RemovePeer(peer p2p.Peer, reason interface{}) {
   152  	bcR.pool.RemovePeer(peer.ID())
   153  }
   154  
   155  // respondToPeer loads a block and sends it to the requesting peer,
   156  // if we have it. Otherwise, we'll respond saying we don't have it.
   157  // According to the Tendermint spec, if all nodes are honest,
   158  // no node should be requesting for a block that's non-existent.
   159  func (bcR *BlockchainReactor) respondToPeer(msg *bcBlockRequestMessage,
   160  	src p2p.Peer) (queued bool) {
   161  
   162  	block := bcR.store.LoadBlock(msg.Height)
   163  	if block != nil {
   164  		msgBytes := cdc.MustMarshalBinaryBare(&bcBlockResponseMessage{Block: block})
   165  		return src.TrySend(BlockchainChannel, msgBytes)
   166  	}
   167  
   168  	bcR.Logger.Info("Peer asking for a block we don't have", "src", src, "height", msg.Height)
   169  
   170  	msgBytes := cdc.MustMarshalBinaryBare(&bcNoBlockResponseMessage{Height: msg.Height})
   171  	return src.TrySend(BlockchainChannel, msgBytes)
   172  }
   173  
   174  // Receive implements Reactor by handling 4 types of messages (look below).
   175  func (bcR *BlockchainReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte) {
   176  	msg, err := decodeMsg(msgBytes)
   177  	if err != nil {
   178  		bcR.Logger.Error("Error decoding message", "src", src, "chId", chID, "msg", msg, "err", err, "bytes", msgBytes)
   179  		bcR.Switch.StopPeerForError(src, err)
   180  		return
   181  	}
   182  
   183  	if err = msg.ValidateBasic(); err != nil {
   184  		bcR.Logger.Error("Peer sent us invalid msg", "peer", src, "msg", msg, "err", err)
   185  		bcR.Switch.StopPeerForError(src, err)
   186  		return
   187  	}
   188  
   189  	bcR.Logger.Debug("Receive", "src", src, "chID", chID, "msg", msg)
   190  
   191  	switch msg := msg.(type) {
   192  	case *bcBlockRequestMessage:
   193  		if queued := bcR.respondToPeer(msg, src); !queued {
   194  			// Unfortunately not queued since the queue is full.
   195  		}
   196  	case *bcBlockResponseMessage:
   197  		bcR.pool.AddBlock(src.ID(), msg.Block, len(msgBytes))
   198  	case *bcStatusRequestMessage:
   199  		// Send peer our state.
   200  		msgBytes := cdc.MustMarshalBinaryBare(&bcStatusResponseMessage{bcR.store.Height()})
   201  		queued := src.TrySend(BlockchainChannel, msgBytes)
   202  		if !queued {
   203  			// sorry
   204  		}
   205  	case *bcStatusResponseMessage:
   206  		// Got a peer status. Unverified.
   207  		bcR.pool.SetPeerHeight(src.ID(), msg.Height)
   208  	default:
   209  		bcR.Logger.Error(fmt.Sprintf("Unknown message type %v", reflect.TypeOf(msg)))
   210  	}
   211  }
   212  
   213  // Handle messages from the poolReactor telling the reactor what to do.
   214  // NOTE: Don't sleep in the FOR_LOOP or otherwise slow it down!
   215  func (bcR *BlockchainReactor) poolRoutine() {
   216  
   217  	trySyncTicker := time.NewTicker(trySyncIntervalMS * time.Millisecond)
   218  	statusUpdateTicker := time.NewTicker(statusUpdateIntervalSeconds * time.Second)
   219  	switchToConsensusTicker := time.NewTicker(switchToConsensusIntervalSeconds * time.Second)
   220  
   221  	blocksSynced := 0
   222  
   223  	chainID := bcR.initialState.ChainID
   224  	state := bcR.initialState
   225  
   226  	lastHundred := time.Now()
   227  	lastRate := 0.0
   228  
   229  	didProcessCh := make(chan struct{}, 1)
   230  
   231  	go func() {
   232  		for {
   233  			select {
   234  			case <-bcR.Quit():
   235  				return
   236  			case <-bcR.pool.Quit():
   237  				return
   238  			case request := <-bcR.requestsCh:
   239  				peer := bcR.Switch.Peers().Get(request.PeerID)
   240  				if peer == nil {
   241  					continue
   242  				}
   243  				msgBytes := cdc.MustMarshalBinaryBare(&bcBlockRequestMessage{request.Height})
   244  				queued := peer.TrySend(BlockchainChannel, msgBytes)
   245  				if !queued {
   246  					bcR.Logger.Debug("Send queue is full, drop block request", "peer", peer.ID(), "height", request.Height)
   247  				}
   248  			case err := <-bcR.errorsCh:
   249  				peer := bcR.Switch.Peers().Get(err.peerID)
   250  				if peer != nil {
   251  					bcR.Switch.StopPeerForError(peer, err)
   252  				}
   253  
   254  			case <-statusUpdateTicker.C:
   255  				// ask for status updates
   256  				go bcR.BroadcastStatusRequest() // nolint: errcheck
   257  
   258  			}
   259  		}
   260  	}()
   261  
   262  FOR_LOOP:
   263  	for {
   264  		select {
   265  		case <-switchToConsensusTicker.C:
   266  			height, numPending, lenRequesters := bcR.pool.GetStatus()
   267  			outbound, inbound, _ := bcR.Switch.NumPeers()
   268  			bcR.Logger.Debug("Consensus ticker", "numPending", numPending, "total", lenRequesters,
   269  				"outbound", outbound, "inbound", inbound)
   270  			if bcR.pool.IsCaughtUp() {
   271  				bcR.Logger.Info("Time to switch to consensus reactor!", "height", height)
   272  				bcR.pool.Stop()
   273  				conR, ok := bcR.Switch.Reactor("CONSENSUS").(consensusReactor)
   274  				if ok {
   275  					conR.SwitchToConsensus(state, blocksSynced)
   276  				} else {
   277  					// should only happen during testing
   278  				}
   279  
   280  				break FOR_LOOP
   281  			}
   282  
   283  		case <-trySyncTicker.C: // chan time
   284  			select {
   285  			case didProcessCh <- struct{}{}:
   286  			default:
   287  			}
   288  
   289  		case <-didProcessCh:
   290  			// NOTE: It is a subtle mistake to process more than a single block
   291  			// at a time (e.g. 10) here, because we only TrySend 1 request per
   292  			// loop.  The ratio mismatch can result in starving of blocks, a
   293  			// sudden burst of requests and responses, and repeat.
   294  			// Consequently, it is better to split these routines rather than
   295  			// coupling them as it's written here.  TODO uncouple from request
   296  			// routine.
   297  
   298  			// See if there are any blocks to sync.
   299  			first, second := bcR.pool.PeekTwoBlocks()
   300  			//bcR.Logger.Info("TrySync peeked", "first", first, "second", second)
   301  			if first == nil || second == nil {
   302  				// We need both to sync the first block.
   303  				continue FOR_LOOP
   304  			} else {
   305  				// Try again quickly next loop.
   306  				didProcessCh <- struct{}{}
   307  			}
   308  
   309  			firstParts := first.MakePartSet(types.BlockPartSizeBytes)
   310  			firstPartsHeader := firstParts.Header()
   311  			firstID := types.BlockID{Hash: first.Hash(), PartsHeader: firstPartsHeader}
   312  			// Finally, verify the first block using the second's commit
   313  			// NOTE: we can probably make this more efficient, but note that calling
   314  			// first.Hash() doesn't verify the tx contents, so MakePartSet() is
   315  			// currently necessary.
   316  			err := state.Validators.VerifyCommit(
   317  				chainID, firstID, first.Height, second.LastCommit)
   318  			if err != nil {
   319  				bcR.Logger.Error("Error in validation", "err", err)
   320  				peerID := bcR.pool.RedoRequest(first.Height)
   321  				peer := bcR.Switch.Peers().Get(peerID)
   322  				if peer != nil {
   323  					// NOTE: we've already removed the peer's request, but we
   324  					// still need to clean up the rest.
   325  					bcR.Switch.StopPeerForError(peer, fmt.Errorf("BlockchainReactor validation error: %v", err))
   326  				}
   327  				peerID2 := bcR.pool.RedoRequest(second.Height)
   328  				peer2 := bcR.Switch.Peers().Get(peerID2)
   329  				if peer2 != nil && peer2 != peer {
   330  					// NOTE: we've already removed the peer's request, but we
   331  					// still need to clean up the rest.
   332  					bcR.Switch.StopPeerForError(peer2, fmt.Errorf("BlockchainReactor validation error: %v", err))
   333  				}
   334  				continue FOR_LOOP
   335  			} else {
   336  				bcR.pool.PopRequest()
   337  
   338  				// TODO: batch saves so we dont persist to disk every block
   339  				bcR.store.SaveBlock(first, firstParts, second.LastCommit)
   340  
   341  				// TODO: same thing for app - but we would need a way to
   342  				// get the hash without persisting the state
   343  				var err error
   344  				state, err = bcR.blockExec.ApplyBlock(state, firstID, first)
   345  				if err != nil {
   346  					// TODO This is bad, are we zombie?
   347  					panic(fmt.Sprintf("Failed to process committed block (%d:%X): %v", first.Height, first.Hash(), err))
   348  				}
   349  				blocksSynced++
   350  
   351  				if blocksSynced%100 == 0 {
   352  					lastRate = 0.9*lastRate + 0.1*(100/time.Since(lastHundred).Seconds())
   353  					bcR.Logger.Info("Fast Sync Rate", "height", bcR.pool.height,
   354  						"max_peer_height", bcR.pool.MaxPeerHeight(), "blocks/s", lastRate)
   355  					lastHundred = time.Now()
   356  				}
   357  			}
   358  			continue FOR_LOOP
   359  
   360  		case <-bcR.Quit():
   361  			break FOR_LOOP
   362  		}
   363  	}
   364  }
   365  
   366  // BroadcastStatusRequest broadcasts `BlockStore` height.
   367  func (bcR *BlockchainReactor) BroadcastStatusRequest() error {
   368  	msgBytes := cdc.MustMarshalBinaryBare(&bcStatusRequestMessage{bcR.store.Height()})
   369  	bcR.Switch.Broadcast(BlockchainChannel, msgBytes)
   370  	return nil
   371  }
   372  
   373  //-----------------------------------------------------------------------------
   374  // Messages
   375  
   376  // BlockchainMessage is a generic message for this reactor.
   377  type BlockchainMessage interface {
   378  	ValidateBasic() error
   379  }
   380  
   381  func RegisterBlockchainMessages(cdc *amino.Codec) {
   382  	cdc.RegisterInterface((*BlockchainMessage)(nil), nil)
   383  	cdc.RegisterConcrete(&bcBlockRequestMessage{}, "tendermint/blockchain/BlockRequest", nil)
   384  	cdc.RegisterConcrete(&bcBlockResponseMessage{}, "tendermint/blockchain/BlockResponse", nil)
   385  	cdc.RegisterConcrete(&bcNoBlockResponseMessage{}, "tendermint/blockchain/NoBlockResponse", nil)
   386  	cdc.RegisterConcrete(&bcStatusResponseMessage{}, "tendermint/blockchain/StatusResponse", nil)
   387  	cdc.RegisterConcrete(&bcStatusRequestMessage{}, "tendermint/blockchain/StatusRequest", nil)
   388  }
   389  
   390  func decodeMsg(bz []byte) (msg BlockchainMessage, err error) {
   391  	if len(bz) > maxMsgSize {
   392  		return msg, fmt.Errorf("Msg exceeds max size (%d > %d)", len(bz), maxMsgSize)
   393  	}
   394  	err = cdc.UnmarshalBinaryBare(bz, &msg)
   395  	return
   396  }
   397  
   398  //-------------------------------------
   399  
   400  type bcBlockRequestMessage struct {
   401  	Height int64
   402  }
   403  
   404  // ValidateBasic performs basic validation.
   405  func (m *bcBlockRequestMessage) ValidateBasic() error {
   406  	if m.Height < 0 {
   407  		return errors.New("Negative Height")
   408  	}
   409  	return nil
   410  }
   411  
   412  func (m *bcBlockRequestMessage) String() string {
   413  	return fmt.Sprintf("[bcBlockRequestMessage %v]", m.Height)
   414  }
   415  
   416  type bcNoBlockResponseMessage struct {
   417  	Height int64
   418  }
   419  
   420  // ValidateBasic performs basic validation.
   421  func (m *bcNoBlockResponseMessage) ValidateBasic() error {
   422  	if m.Height < 0 {
   423  		return errors.New("Negative Height")
   424  	}
   425  	return nil
   426  }
   427  
   428  func (brm *bcNoBlockResponseMessage) String() string {
   429  	return fmt.Sprintf("[bcNoBlockResponseMessage %d]", brm.Height)
   430  }
   431  
   432  //-------------------------------------
   433  
   434  type bcBlockResponseMessage struct {
   435  	Block *types.Block
   436  }
   437  
   438  // ValidateBasic performs basic validation.
   439  func (m *bcBlockResponseMessage) ValidateBasic() error {
   440  	return m.Block.ValidateBasic()
   441  }
   442  
   443  func (m *bcBlockResponseMessage) String() string {
   444  	return fmt.Sprintf("[bcBlockResponseMessage %v]", m.Block.Height)
   445  }
   446  
   447  //-------------------------------------
   448  
   449  type bcStatusRequestMessage struct {
   450  	Height int64
   451  }
   452  
   453  // ValidateBasic performs basic validation.
   454  func (m *bcStatusRequestMessage) ValidateBasic() error {
   455  	if m.Height < 0 {
   456  		return errors.New("Negative Height")
   457  	}
   458  	return nil
   459  }
   460  
   461  func (m *bcStatusRequestMessage) String() string {
   462  	return fmt.Sprintf("[bcStatusRequestMessage %v]", m.Height)
   463  }
   464  
   465  //-------------------------------------
   466  
   467  type bcStatusResponseMessage struct {
   468  	Height int64
   469  }
   470  
   471  // ValidateBasic performs basic validation.
   472  func (m *bcStatusResponseMessage) ValidateBasic() error {
   473  	if m.Height < 0 {
   474  		return errors.New("Negative Height")
   475  	}
   476  	return nil
   477  }
   478  
   479  func (m *bcStatusResponseMessage) String() string {
   480  	return fmt.Sprintf("[bcStatusResponseMessage %v]", m.Height)
   481  }