github.com/turingchain2020/turingchain@v1.1.21/blockchain/blocksyn.go (about)

     1  // Copyright Turing Corp. 2018 All Rights Reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package blockchain
     6  
     7  import (
     8  	"bytes"
     9  	"math/big"
    10  	"sort"
    11  	"sync"
    12  	"sync/atomic"
    13  	"time"
    14  
    15  	"github.com/turingchain2020/turingchain/common"
    16  	"github.com/turingchain2020/turingchain/types"
    17  )
    18  
    19  //var
    20  var (
    21  	BackBlockNum            int64 = 128                  //节点高度不增加时向后取blocks的个数
    22  	BackwardBlockNum        int64 = 16                   //本节点高度不增加时并且落后peer的高度数
    23  	checkHeightNoIncSeconds int64 = 5 * 60               //高度不增长时的检测周期目前暂定5分钟
    24  	checkBlockHashSeconds   int64 = 1 * 60               //1分钟检测一次tip hash和peer 对应高度的hash是否一致
    25  	fetchPeerListSeconds    int64 = 5                    //5 秒获取一个peerlist
    26  	MaxRollBlockNum         int64 = 10000                //最大回退block数量
    27  	ReduceHeight                  = MaxRollBlockNum      // 距离最大高度的可精简高度
    28  	SafetyReduceHeight            = ReduceHeight * 3 / 2 // 安全精简高度
    29  	//TODO
    30  	batchsyncblocknum int64 = 5000 //同步阶段,如果自己高度小于最大高度5000个时,saveblock到db时批量处理不刷盘
    31  
    32  	synlog = chainlog.New("submodule", "syn")
    33  )
    34  
    35  //PeerInfo blockchain模块需要保存的peerinfo
    36  type PeerInfo struct {
    37  	Name       string
    38  	ParentHash []byte
    39  	Height     int64
    40  	Hash       []byte
    41  }
    42  
    43  //PeerInfoList 节点列表
    44  type PeerInfoList []*PeerInfo
    45  
    46  //Len 长度
    47  func (list PeerInfoList) Len() int {
    48  	return len(list)
    49  }
    50  
    51  //Less 小于
    52  func (list PeerInfoList) Less(i, j int) bool {
    53  	if list[i].Height < list[j].Height {
    54  		return true
    55  	} else if list[i].Height > list[j].Height {
    56  		return false
    57  	} else {
    58  		return list[i].Name < list[j].Name
    59  	}
    60  }
    61  
    62  //Swap 交换
    63  func (list PeerInfoList) Swap(i, j int) {
    64  	temp := list[i]
    65  	list[i] = list[j]
    66  	list[j] = temp
    67  }
    68  
    69  //FaultPeerInfo 可疑故障节点信息
    70  type FaultPeerInfo struct {
    71  	Peer        *PeerInfo
    72  	FaultHeight int64
    73  	FaultHash   []byte
    74  	ErrInfo     error
    75  	ReqFlag     bool
    76  }
    77  
    78  //BestPeerInfo 用于记录最优链的信息
    79  type BestPeerInfo struct {
    80  	Peer        *PeerInfo
    81  	Height      int64
    82  	Hash        []byte
    83  	Td          *big.Int
    84  	ReqFlag     bool
    85  	IsBestChain bool
    86  }
    87  
    88  //BlockOnChain ...
    89  //记录最新区块上链的时间,长时间没有更新需要做对应的超时处理
    90  //主要是处理联盟链区块高度相差一个区块
    91  //整个网络长时间不出块时需要主动去获取最新的区块
    92  //BlockOnChain struct
    93  type BlockOnChain struct {
    94  	sync.RWMutex
    95  	Height      int64
    96  	OnChainTime int64
    97  }
    98  
    99  //initOnChainTimeout 初始化
   100  func (chain *BlockChain) initOnChainTimeout() {
   101  	chain.blockOnChain.Lock()
   102  	defer chain.blockOnChain.Unlock()
   103  
   104  	chain.blockOnChain.Height = -1
   105  	chain.blockOnChain.OnChainTime = types.Now().Unix()
   106  }
   107  
   108  //OnChainTimeout 最新区块长时间没有更新并超过设置的超时时间
   109  func (chain *BlockChain) OnChainTimeout(height int64) bool {
   110  	chain.blockOnChain.Lock()
   111  	defer chain.blockOnChain.Unlock()
   112  
   113  	if chain.onChainTimeout == 0 {
   114  		return false
   115  	}
   116  
   117  	curTime := types.Now().Unix()
   118  	if chain.blockOnChain.Height != height {
   119  		chain.blockOnChain.Height = height
   120  		chain.blockOnChain.OnChainTime = curTime
   121  		return false
   122  	}
   123  	if curTime-chain.blockOnChain.OnChainTime > chain.onChainTimeout {
   124  		synlog.Debug("OnChainTimeout", "curTime", curTime, "blockOnChain", chain.blockOnChain)
   125  		return true
   126  	}
   127  	return false
   128  }
   129  
   130  //SynRoutine 同步事务
   131  func (chain *BlockChain) SynRoutine() {
   132  	//获取peerlist的定时器,默认1分钟
   133  	fetchPeerListTicker := time.NewTicker(time.Duration(fetchPeerListSeconds) * time.Second)
   134  
   135  	//向peer请求同步block的定时器,默认2s
   136  	blockSynTicker := time.NewTicker(chain.blockSynInterVal * time.Second)
   137  
   138  	//5分钟检测一次bestchain主链高度是否有增长,如果没有增长可能是目前主链在侧链上,
   139  	//需要从最高peer向后同步指定的headers用来获取分叉点,再后从指定peer获取分叉点以后的blocks
   140  	checkHeightNoIncreaseTicker := time.NewTicker(time.Duration(checkHeightNoIncSeconds) * time.Second)
   141  
   142  	//目前暂定1分钟检测一次本bestchain的tiphash和最高peer的对应高度的blockshash是否一致。
   143  	//如果不一致可能两个节点在各自的链上挖矿,需要从peer的对应高度向后获取指定数量的headers寻找分叉点
   144  	//考虑叉后的第一个block没有广播到本节点,导致接下来广播过来的blocks都是孤儿节点,无法进行主侧链总难度对比
   145  	checkBlockHashTicker := time.NewTicker(time.Duration(checkBlockHashSeconds) * time.Second)
   146  
   147  	//5分钟检测一次系统时间,不同步提示告警
   148  	checkClockDriftTicker := time.NewTicker(300 * time.Second)
   149  	if chain.cfg.DisableClockDriftCheck {
   150  		checkBlockHashTicker.Stop()
   151  		checkClockDriftTicker.C = nil
   152  	}
   153  	//3分钟尝试检测一次故障peer是否已经恢复
   154  	recoveryFaultPeerTicker := time.NewTicker(180 * time.Second)
   155  
   156  	//2分钟尝试检测一次最优链,确保本节点在最优链
   157  	checkBestChainTicker := time.NewTicker(120 * time.Second)
   158  
   159  	//30s尝试从peer节点请求ChunkRecord
   160  	chunkRecordSynTicker := time.NewTicker(30 * time.Second)
   161  
   162  	//节点下载模式
   163  	go chain.DownLoadBlocks()
   164  
   165  	for {
   166  		select {
   167  		case <-chain.quit:
   168  			//synlog.Info("quit SynRoutine!")
   169  			return
   170  		case <-blockSynTicker.C:
   171  			//synlog.Info("blockSynTicker")
   172  			if chain.GetDownloadSyncStatus() == normalDownLoadMode {
   173  				go chain.SynBlocksFromPeers()
   174  			}
   175  
   176  		case <-fetchPeerListTicker.C:
   177  			//synlog.Info("blockUpdateTicker")
   178  			chain.tickerwg.Add(1)
   179  			go chain.FetchPeerList()
   180  
   181  		case <-checkHeightNoIncreaseTicker.C:
   182  			//synlog.Info("CheckHeightNoIncrease")
   183  			chain.tickerwg.Add(1)
   184  			go chain.CheckHeightNoIncrease()
   185  
   186  		case <-checkBlockHashTicker.C:
   187  			//synlog.Info("checkBlockHashTicker")
   188  			chain.tickerwg.Add(1)
   189  			go chain.CheckTipBlockHash()
   190  
   191  			//定时检查系统时间,如果系统时间有问题,那么会有一个报警
   192  		case <-checkClockDriftTicker.C:
   193  			// ntp可能存在一直没有回应的情况导致go线程不退出,暂时不在WaitGroup中处理
   194  			go chain.checkClockDrift()
   195  
   196  			//定时检查故障peer,如果执行出错高度的blockhash值有变化,说明故障peer已经纠正
   197  		case <-recoveryFaultPeerTicker.C:
   198  			chain.tickerwg.Add(1)
   199  			go chain.RecoveryFaultPeer()
   200  
   201  			//定时检查peerlist中的节点是否在同一条链上,获取同一高度的blockhash来做对比
   202  		case <-checkBestChainTicker.C:
   203  			chain.tickerwg.Add(1)
   204  			go chain.CheckBestChain(false)
   205  
   206  			//定时检查ChunkRecord的同步情况
   207  		case <-chunkRecordSynTicker.C:
   208  			if !chain.cfg.DisableShard {
   209  				go chain.ChunkRecordSync()
   210  			}
   211  		}
   212  	}
   213  }
   214  
   215  /*
   216  FetchBlock 函数功能:
   217  通过向P2P模块送 EventFetchBlock(types.RequestGetBlock),向其他节点主动请求区块,
   218  P2P区块收到这个消息后,会向blockchain 模块回复, EventReply。
   219  其他节点如果有这个范围的区块,P2P模块收到其他节点发来的数据,
   220  会发送送EventAddBlocks(types.Blocks) 给 blockchain 模块,
   221  blockchain 模块回复 EventReply
   222  syncOrfork:true fork分叉处理,不需要处理请求block的个数
   223            :fasle 区块同步处理,一次请求128个block
   224  */
   225  func (chain *BlockChain) FetchBlock(start int64, end int64, pid []string, syncOrfork bool) (err error) {
   226  	if chain.client == nil {
   227  		synlog.Error("FetchBlock chain client not bind message queue.")
   228  		return types.ErrClientNotBindQueue
   229  	}
   230  
   231  	synlog.Debug("FetchBlock input", "StartHeight", start, "EndHeight", end, "pid", pid)
   232  	blockcount := end - start
   233  	if blockcount < 0 {
   234  		return types.ErrStartBigThanEnd
   235  	}
   236  	var requestblock types.ReqBlocks
   237  	requestblock.Start = start
   238  	requestblock.IsDetail = false
   239  	requestblock.Pid = pid
   240  
   241  	//同步block一次请求128个
   242  	if blockcount >= chain.MaxFetchBlockNum {
   243  		requestblock.End = start + chain.MaxFetchBlockNum - 1
   244  	} else {
   245  		requestblock.End = end
   246  	}
   247  	var cb func()
   248  	var timeoutcb func(height int64)
   249  	if syncOrfork {
   250  		//还有区块需要请求,挂接钩子回调函数
   251  		if requestblock.End < chain.downLoadInfo.EndHeight {
   252  			cb = func() {
   253  				chain.ReqDownLoadBlocks()
   254  			}
   255  			timeoutcb = func(height int64) {
   256  				chain.DownLoadTimeOutProc(height)
   257  			}
   258  			chain.UpdateDownLoadStartHeight(requestblock.End + 1)
   259  			//快速下载时需要及时更新bestpeer,防止下载侧链的block
   260  			if chain.GetDownloadSyncStatus() == fastDownLoadMode {
   261  				chain.UpdateDownLoadPids()
   262  			}
   263  		} else { // 所有DownLoad block已请求结束,恢复DownLoadInfo为默认值
   264  			chain.DefaultDownLoadInfo()
   265  		}
   266  		err = chain.downLoadTask.Start(requestblock.Start, requestblock.End, cb, timeoutcb)
   267  		if err != nil {
   268  			return err
   269  		}
   270  	} else {
   271  		if chain.GetPeerMaxBlkHeight()-requestblock.End > BackBlockNum {
   272  			cb = func() {
   273  				chain.SynBlocksFromPeers()
   274  			}
   275  		}
   276  		err = chain.syncTask.Start(requestblock.Start, requestblock.End, cb, timeoutcb)
   277  		if err != nil {
   278  			return err
   279  		}
   280  	}
   281  
   282  	synlog.Info("FetchBlock", "Start", requestblock.Start, "End", requestblock.End)
   283  	msg := chain.client.NewMessage("p2p", types.EventFetchBlocks, &requestblock)
   284  	Err := chain.client.Send(msg, true)
   285  	if Err != nil {
   286  		synlog.Error("FetchBlock", "client.Send err:", Err)
   287  		return err
   288  	}
   289  	resp, err := chain.client.Wait(msg)
   290  	if err != nil {
   291  		synlog.Error("FetchBlock", "client.Wait err:", err)
   292  		return err
   293  	}
   294  	return resp.Err()
   295  }
   296  
   297  //FetchPeerList 从p2p模块获取peerlist,用于获取active链上最新的高度。
   298  //如果没有收到广播block就主动向p2p模块发送请求
   299  func (chain *BlockChain) FetchPeerList() {
   300  	defer chain.tickerwg.Done()
   301  	err := chain.fetchPeerList()
   302  	if err != nil {
   303  		synlog.Error("FetchPeerList.", "err", err)
   304  	}
   305  }
   306  
   307  func (chain *BlockChain) fetchPeerList() error {
   308  	if chain.client == nil {
   309  		synlog.Error("fetchPeerList chain client not bind message queue.")
   310  		return nil
   311  	}
   312  	msg := chain.client.NewMessage("p2p", types.EventPeerInfo, nil)
   313  	Err := chain.client.SendTimeout(msg, true, 30*time.Second)
   314  	if Err != nil {
   315  		synlog.Error("fetchPeerList", "client.Send err:", Err)
   316  		return Err
   317  	}
   318  	resp, err := chain.client.WaitTimeout(msg, 60*time.Second)
   319  	if err != nil {
   320  		synlog.Error("fetchPeerList", "client.Wait err:", err)
   321  		return err
   322  	}
   323  
   324  	peerlist := resp.GetData().(*types.PeerList)
   325  	if peerlist == nil {
   326  		synlog.Error("fetchPeerList", "peerlist", "is nil")
   327  		return types.ErrNoPeer
   328  	}
   329  	curheigt := chain.GetBlockHeight()
   330  
   331  	var peerInfoList PeerInfoList
   332  	for _, peer := range peerlist.GetPeers() {
   333  		//过滤掉自己和小于自己5个高度的节点
   334  		if peer == nil || peer.Self || curheigt > peer.Header.Height+5 {
   335  			continue
   336  		}
   337  		var peerInfo PeerInfo
   338  		peerInfo.Name = peer.Name
   339  		peerInfo.ParentHash = peer.Header.ParentHash
   340  		peerInfo.Height = peer.Header.Height
   341  		peerInfo.Hash = peer.Header.Hash
   342  		peerInfoList = append(peerInfoList, &peerInfo)
   343  	}
   344  	//peerlist中没有比自己节点高的就不做处理直接返回
   345  	if len(peerInfoList) == 0 {
   346  		return nil
   347  	}
   348  	//按照height给peer排序从小到大
   349  	sort.Sort(peerInfoList)
   350  
   351  	subInfoList := peerInfoList
   352  
   353  	chain.peerMaxBlklock.Lock()
   354  	chain.peerList = subInfoList
   355  	chain.peerMaxBlklock.Unlock()
   356  
   357  	//获取到peerlist之后,需要判断是否已经发起了最优链的检测。如果没有就触发一次最优链的检测
   358  	if atomic.LoadInt32(&chain.firstcheckbestchain) == 0 {
   359  		synlog.Info("fetchPeerList trigger first CheckBestChain")
   360  		chain.CheckBestChain(true)
   361  	}
   362  	return nil
   363  }
   364  
   365  //GetRcvLastCastBlkHeight 存储广播的block最新高度
   366  func (chain *BlockChain) GetRcvLastCastBlkHeight() int64 {
   367  	chain.castlock.Lock()
   368  	defer chain.castlock.Unlock()
   369  	return chain.rcvLastBlockHeight
   370  }
   371  
   372  //UpdateRcvCastBlkHeight 更新广播的block最新高度
   373  func (chain *BlockChain) UpdateRcvCastBlkHeight(height int64) {
   374  	chain.castlock.Lock()
   375  	defer chain.castlock.Unlock()
   376  	chain.rcvLastBlockHeight = height
   377  }
   378  
   379  //GetsynBlkHeight 存储已经同步到db的block高度
   380  func (chain *BlockChain) GetsynBlkHeight() int64 {
   381  	chain.synBlocklock.Lock()
   382  	defer chain.synBlocklock.Unlock()
   383  	return chain.synBlockHeight
   384  }
   385  
   386  //UpdatesynBlkHeight 更新已经同步到db的block高度
   387  func (chain *BlockChain) UpdatesynBlkHeight(height int64) {
   388  	chain.synBlocklock.Lock()
   389  	defer chain.synBlocklock.Unlock()
   390  	chain.synBlockHeight = height
   391  }
   392  
   393  //GetPeerMaxBlkHeight 获取peerlist中合法的最新block高度
   394  func (chain *BlockChain) GetPeerMaxBlkHeight() int64 {
   395  	chain.peerMaxBlklock.Lock()
   396  	defer chain.peerMaxBlklock.Unlock()
   397  
   398  	//获取peerlist中最高的高度,peerlist是已经按照高度排序了的。
   399  	if chain.peerList != nil {
   400  		peerlen := len(chain.peerList)
   401  		for i := peerlen - 1; i >= 0; i-- {
   402  			if chain.peerList[i] != nil {
   403  				ok := chain.IsFaultPeer(chain.peerList[i].Name)
   404  				if !ok {
   405  					return chain.peerList[i].Height
   406  				}
   407  			}
   408  		}
   409  		//没有合法的peer,此时本节点可能在侧链上,返回peerlist中最高的peer尝试矫正
   410  		maxpeer := chain.peerList[peerlen-1]
   411  		if maxpeer != nil {
   412  			synlog.Debug("GetPeerMaxBlkHeight all peers are faultpeer maybe self on Side chain", "pid", maxpeer.Name, "Height", maxpeer.Height, "Hash", common.ToHex(maxpeer.Hash))
   413  			return maxpeer.Height
   414  		}
   415  	}
   416  	return -1
   417  }
   418  
   419  //GetPeerInfo 通过peerid获取peerinfo
   420  func (chain *BlockChain) GetPeerInfo(pid string) *PeerInfo {
   421  	chain.peerMaxBlklock.Lock()
   422  	defer chain.peerMaxBlklock.Unlock()
   423  
   424  	//获取peerinfo
   425  	if chain.peerList != nil {
   426  		for _, peer := range chain.peerList {
   427  			if pid == peer.Name {
   428  				return peer
   429  			}
   430  		}
   431  	}
   432  	return nil
   433  }
   434  
   435  //GetMaxPeerInfo 获取peerlist中最高节点的peerinfo
   436  func (chain *BlockChain) GetMaxPeerInfo() *PeerInfo {
   437  	chain.peerMaxBlklock.Lock()
   438  	defer chain.peerMaxBlklock.Unlock()
   439  
   440  	//获取peerlist中高度最高的peer,peerlist是已经按照高度排序了的。
   441  	if chain.peerList != nil {
   442  		peerlen := len(chain.peerList)
   443  		for i := peerlen - 1; i >= 0; i-- {
   444  			if chain.peerList[i] != nil {
   445  				ok := chain.IsFaultPeer(chain.peerList[i].Name)
   446  				if !ok {
   447  					return chain.peerList[i]
   448  				}
   449  			}
   450  		}
   451  		//没有合法的peer,此时本节点可能在侧链上,返回peerlist中最高的peer尝试矫正
   452  		maxpeer := chain.peerList[peerlen-1]
   453  		if maxpeer != nil {
   454  			synlog.Debug("GetMaxPeerInfo all peers are faultpeer maybe self on Side chain", "pid", maxpeer.Name, "Height", maxpeer.Height, "Hash", common.ToHex(maxpeer.Hash))
   455  			return maxpeer
   456  		}
   457  	}
   458  	return nil
   459  }
   460  
   461  //GetPeers 获取所有peers
   462  func (chain *BlockChain) GetPeers() PeerInfoList {
   463  	chain.peerMaxBlklock.Lock()
   464  	defer chain.peerMaxBlklock.Unlock()
   465  
   466  	//获取peerinfo
   467  	var peers PeerInfoList
   468  
   469  	if chain.peerList != nil {
   470  		peers = append(peers, chain.peerList...)
   471  	}
   472  	return peers
   473  }
   474  
   475  //GetPeersMap 获取peers的map列表方便查找
   476  func (chain *BlockChain) GetPeersMap() map[string]bool {
   477  	chain.peerMaxBlklock.Lock()
   478  	defer chain.peerMaxBlklock.Unlock()
   479  	peersmap := make(map[string]bool)
   480  
   481  	if chain.peerList != nil {
   482  		for _, peer := range chain.peerList {
   483  			peersmap[peer.Name] = true
   484  		}
   485  	}
   486  	return peersmap
   487  }
   488  
   489  //IsFaultPeer 判断指定pid是否在故障faultPeerList中
   490  func (chain *BlockChain) IsFaultPeer(pid string) bool {
   491  	chain.faultpeerlock.Lock()
   492  	defer chain.faultpeerlock.Unlock()
   493  
   494  	return chain.faultPeerList[pid] != nil
   495  }
   496  
   497  //IsErrExecBlock 判断此block是否被记录在本节点执行错误。
   498  func (chain *BlockChain) IsErrExecBlock(height int64, hash []byte) (bool, error) {
   499  	chain.faultpeerlock.Lock()
   500  	defer chain.faultpeerlock.Unlock()
   501  
   502  	//循环遍历故障peerlist,尝试检测故障peer是否已经恢复
   503  	for _, faultpeer := range chain.faultPeerList {
   504  		if faultpeer.FaultHeight == height && bytes.Equal(hash, faultpeer.FaultHash) {
   505  			return true, faultpeer.ErrInfo
   506  		}
   507  	}
   508  	return false, nil
   509  }
   510  
   511  //GetFaultPeer 获取指定pid是否在故障faultPeerList中
   512  func (chain *BlockChain) GetFaultPeer(pid string) *FaultPeerInfo {
   513  	chain.faultpeerlock.Lock()
   514  	defer chain.faultpeerlock.Unlock()
   515  
   516  	return chain.faultPeerList[pid]
   517  }
   518  
   519  //RecoveryFaultPeer 尝试恢复故障peer节点,定时从出错的peer获取出错block的头信息。
   520  //看对应的block是否有更新。有更新就说明故障peer节点已经恢复ok
   521  func (chain *BlockChain) RecoveryFaultPeer() {
   522  	chain.faultpeerlock.Lock()
   523  	defer chain.faultpeerlock.Unlock()
   524  
   525  	defer chain.tickerwg.Done()
   526  
   527  	//循环遍历故障peerlist,尝试检测故障peer是否已经恢复
   528  	for pid, faultpeer := range chain.faultPeerList {
   529  		//需要考虑回退的情况,可能本地节点已经回退,恢复以前的故障节点,获取故障高度的区块的hash是否在本地已经校验通过。校验通过就直接说明故障已经恢复
   530  		//获取本节点指定高度的blockhash做判断,hash相同就确认故障已经恢复
   531  		blockhash, err := chain.blockStore.GetBlockHashByHeight(faultpeer.FaultHeight)
   532  		if err == nil {
   533  			if bytes.Equal(faultpeer.FaultHash, blockhash) {
   534  				synlog.Debug("RecoveryFaultPeer ", "Height", faultpeer.FaultHeight, "FaultHash", common.ToHex(faultpeer.FaultHash), "pid", pid)
   535  				delete(chain.faultPeerList, pid)
   536  				continue
   537  			}
   538  		}
   539  
   540  		err = chain.FetchBlockHeaders(faultpeer.FaultHeight, faultpeer.FaultHeight, pid)
   541  		if err == nil {
   542  			chain.faultPeerList[pid].ReqFlag = true
   543  		}
   544  		synlog.Debug("RecoveryFaultPeer", "pid", faultpeer.Peer.Name, "FaultHeight", faultpeer.FaultHeight, "FaultHash", common.ToHex(faultpeer.FaultHash), "Err", faultpeer.ErrInfo)
   545  	}
   546  }
   547  
   548  //AddFaultPeer 添加故障节点到故障FaultPeerList中
   549  func (chain *BlockChain) AddFaultPeer(faultpeer *FaultPeerInfo) {
   550  	chain.faultpeerlock.Lock()
   551  	defer chain.faultpeerlock.Unlock()
   552  
   553  	//此节点已经存在故障peerlist中打印信息
   554  	faultnode := chain.faultPeerList[faultpeer.Peer.Name]
   555  	if faultnode != nil {
   556  		synlog.Debug("AddFaultPeer old", "pid", faultnode.Peer.Name, "FaultHeight", faultnode.FaultHeight, "FaultHash", common.ToHex(faultnode.FaultHash), "Err", faultnode.ErrInfo)
   557  	}
   558  	chain.faultPeerList[faultpeer.Peer.Name] = faultpeer
   559  	synlog.Debug("AddFaultPeer new", "pid", faultpeer.Peer.Name, "FaultHeight", faultpeer.FaultHeight, "FaultHash", common.ToHex(faultpeer.FaultHash), "Err", faultpeer.ErrInfo)
   560  }
   561  
   562  //RemoveFaultPeer 此pid对应的故障已经修复,将此pid从故障列表中移除
   563  func (chain *BlockChain) RemoveFaultPeer(pid string) {
   564  	chain.faultpeerlock.Lock()
   565  	defer chain.faultpeerlock.Unlock()
   566  	synlog.Debug("RemoveFaultPeer", "pid", pid)
   567  
   568  	delete(chain.faultPeerList, pid)
   569  }
   570  
   571  //UpdateFaultPeer 更新此故障peer的请求标志位
   572  func (chain *BlockChain) UpdateFaultPeer(pid string, reqFlag bool) {
   573  	chain.faultpeerlock.Lock()
   574  	defer chain.faultpeerlock.Unlock()
   575  
   576  	faultpeer := chain.faultPeerList[pid]
   577  	if faultpeer != nil {
   578  		faultpeer.ReqFlag = reqFlag
   579  	}
   580  }
   581  
   582  //RecordFaultPeer 当blcok执行出错时,记录出错block高度,hash值,以及出错信息和对应的peerid
   583  func (chain *BlockChain) RecordFaultPeer(pid string, height int64, hash []byte, err error) {
   584  
   585  	var faultnode FaultPeerInfo
   586  
   587  	//通过pid获取peerinfo
   588  	peerinfo := chain.GetPeerInfo(pid)
   589  	if peerinfo == nil {
   590  		synlog.Error("RecordFaultPeerNode GetPeerInfo is nil", "pid", pid)
   591  		return
   592  	}
   593  	faultnode.Peer = peerinfo
   594  	faultnode.FaultHeight = height
   595  	faultnode.FaultHash = hash
   596  	faultnode.ErrInfo = err
   597  	faultnode.ReqFlag = false
   598  	chain.AddFaultPeer(&faultnode)
   599  }
   600  
   601  //SynBlocksFromPeers blockSynSeconds时间检测一次本节点的height是否有增长,没有增长就需要通过对端peerlist获取最新高度,发起同步
   602  func (chain *BlockChain) SynBlocksFromPeers() {
   603  
   604  	curheight := chain.GetBlockHeight()
   605  	RcvLastCastBlkHeight := chain.GetRcvLastCastBlkHeight()
   606  	peerMaxBlkHeight := chain.GetPeerMaxBlkHeight()
   607  
   608  	// 节点同步阶段自己高度小于最大高度batchsyncblocknum时存储block到db批量处理时不刷盘
   609  	if peerMaxBlkHeight > curheight+batchsyncblocknum && !chain.cfgBatchSync {
   610  		atomic.CompareAndSwapInt32(&chain.isbatchsync, 1, 0)
   611  	} else if peerMaxBlkHeight >= 0 {
   612  		atomic.CompareAndSwapInt32(&chain.isbatchsync, 0, 1)
   613  	}
   614  
   615  	//如果任务正常,那么不重复启动任务
   616  	if chain.syncTask.InProgress() {
   617  		synlog.Info("chain syncTask InProgress")
   618  		return
   619  	}
   620  	//如果此时系统正在处理回滚,不启动同步的任务。
   621  	//等分叉回滚处理结束之后再启动同步任务继续同步
   622  	if chain.downLoadTask.InProgress() {
   623  		synlog.Info("chain downLoadTask InProgress")
   624  		return
   625  	}
   626  	//获取peers的最新高度.处理没有收到广播block的情况
   627  	//落后超过2个区块时主动同步区块,落后一个区块时需要判断是否超时
   628  	backWardThanTwo := curheight+1 < peerMaxBlkHeight
   629  	backWardOne := curheight+1 == peerMaxBlkHeight && chain.OnChainTimeout(curheight)
   630  
   631  	if backWardThanTwo || backWardOne {
   632  		synlog.Info("SynBlocksFromPeers", "curheight", curheight, "LastCastBlkHeight", RcvLastCastBlkHeight, "peerMaxBlkHeight", peerMaxBlkHeight)
   633  		pids := chain.GetBestChainPids()
   634  		if pids != nil {
   635  			recvChunk := chain.GetCurRecvChunkNum()
   636  			curShouldChunk, _, _ := chain.CalcChunkInfo(curheight + 1)
   637  			// TODO 后期可修改为同步节点不使用FetchChunkBlock,即让对端节点去查找具体的chunk,这里不做区分
   638  			if !chain.cfg.DisableShard && chain.cfg.EnableFetchP2pstore &&
   639  				curheight+MaxRollBlockNum < peerMaxBlkHeight && recvChunk >= curShouldChunk {
   640  				err := chain.FetchChunkBlock(curheight+1, peerMaxBlkHeight, pids, false)
   641  				if err != nil {
   642  					synlog.Error("SynBlocksFromPeers FetchChunkBlock", "err", err)
   643  				}
   644  			} else {
   645  				err := chain.FetchBlock(curheight+1, peerMaxBlkHeight, pids, false)
   646  				if err != nil {
   647  					synlog.Error("SynBlocksFromPeers FetchBlock", "err", err)
   648  				}
   649  			}
   650  		} else {
   651  			synlog.Info("SynBlocksFromPeers GetBestChainPids is nil")
   652  		}
   653  	}
   654  }
   655  
   656  //CheckHeightNoIncrease 在规定时间本链的高度没有增长,但peerlist中最新高度远远高于本节点高度,
   657  //可能当前链是在分支链上,需从指定最长链的peer向后请求指定数量的blockheader
   658  //请求bestchain.Height -BackBlockNum -- bestchain.Height的header
   659  //需要考虑收不到分叉之后的第一个广播block,这样就会导致后面的广播block都在孤儿节点中了。
   660  func (chain *BlockChain) CheckHeightNoIncrease() {
   661  	defer chain.tickerwg.Done()
   662  
   663  	//获取当前主链的最新高度
   664  	tipheight := chain.bestChain.Height()
   665  	laststorheight := chain.blockStore.Height()
   666  
   667  	if tipheight != laststorheight {
   668  		synlog.Error("CheckHeightNoIncrease", "tipheight", tipheight, "laststorheight", laststorheight)
   669  		return
   670  	}
   671  	//获取上个检测周期时的检测高度
   672  	checkheight := chain.GetsynBlkHeight()
   673  
   674  	//bestchain的tip高度在变化,更新最新的检测高度即可,高度可能在增长或者回退
   675  	if tipheight != checkheight {
   676  		chain.UpdatesynBlkHeight(tipheight)
   677  		return
   678  	}
   679  	//一个检测周期发现本节点bestchain的tip高度没有变化。
   680  	//远远落后于高度的peer节点并且最高peer节点不是最优链,本节点可能在侧链上,
   681  	//需要从最新的peer上向后取BackBlockNum个headers
   682  	maxpeer := chain.GetMaxPeerInfo()
   683  	if maxpeer == nil {
   684  		synlog.Error("CheckHeightNoIncrease GetMaxPeerInfo is nil")
   685  		return
   686  	}
   687  	peermaxheight := maxpeer.Height
   688  	pid := maxpeer.Name
   689  	var err error
   690  	if peermaxheight > tipheight && (peermaxheight-tipheight) > BackwardBlockNum && !chain.isBestChainPeer(pid) {
   691  		//从指定peer向后请求BackBlockNum个blockheaders
   692  		synlog.Debug("CheckHeightNoIncrease", "tipheight", tipheight, "pid", pid)
   693  		if tipheight > BackBlockNum {
   694  			err = chain.FetchBlockHeaders(tipheight-BackBlockNum, tipheight, pid)
   695  		} else {
   696  			err = chain.FetchBlockHeaders(0, tipheight, pid)
   697  		}
   698  		if err != nil {
   699  			synlog.Error("CheckHeightNoIncrease FetchBlockHeaders", "err", err)
   700  		}
   701  	}
   702  }
   703  
   704  //FetchBlockHeaders 从指定pid获取start到end之间的headers
   705  func (chain *BlockChain) FetchBlockHeaders(start int64, end int64, pid string) (err error) {
   706  	if chain.client == nil {
   707  		synlog.Error("FetchBlockHeaders chain client not bind message queue.")
   708  		return types.ErrClientNotBindQueue
   709  	}
   710  
   711  	chainlog.Debug("FetchBlockHeaders", "StartHeight", start, "EndHeight", end, "pid", pid)
   712  
   713  	var requestblock types.ReqBlocks
   714  	requestblock.Start = start
   715  	requestblock.End = end
   716  	requestblock.IsDetail = false
   717  	requestblock.Pid = []string{pid}
   718  
   719  	msg := chain.client.NewMessage("p2p", types.EventFetchBlockHeaders, &requestblock)
   720  	Err := chain.client.Send(msg, true)
   721  	if Err != nil {
   722  		synlog.Error("FetchBlockHeaders", "client.Send err:", Err)
   723  		return err
   724  	}
   725  	resp, err := chain.client.Wait(msg)
   726  	if err != nil {
   727  		synlog.Error("FetchBlockHeaders", "client.Wait err:", err)
   728  		return err
   729  	}
   730  	return resp.Err()
   731  }
   732  
   733  //ProcBlockHeader 一个block header消息的处理,分tiphash的校验,故障peer的故障block是否恢复的校验
   734  func (chain *BlockChain) ProcBlockHeader(headers *types.Headers, peerid string) error {
   735  
   736  	//判断是否是用于检测故障peer而请求的block header
   737  	faultPeer := chain.GetFaultPeer(peerid)
   738  	if faultPeer != nil && faultPeer.ReqFlag && faultPeer.FaultHeight == headers.Items[0].Height {
   739  		//同一高度的block hash有更新,表示故障peer的故障已经恢复,将此peer从故障peerlist中移除
   740  		if !bytes.Equal(headers.Items[0].Hash, faultPeer.FaultHash) {
   741  			chain.RemoveFaultPeer(peerid)
   742  		} else {
   743  			chain.UpdateFaultPeer(peerid, false)
   744  		}
   745  		return nil
   746  	}
   747  
   748  	//检测最优链的处理
   749  	bestchainPeer := chain.GetBestChainPeer(peerid)
   750  	if bestchainPeer != nil && bestchainPeer.ReqFlag && headers.Items[0].Height == bestchainPeer.Height {
   751  		chain.CheckBestChainProc(headers, peerid)
   752  		return nil
   753  	}
   754  
   755  	// 用于tiphash对比而请求的block header
   756  	height := headers.Items[0].Height
   757  	//获取height高度在本节点的headers信息
   758  	header, err := chain.blockStore.GetBlockHeaderByHeight(height)
   759  	if err != nil {
   760  		return err
   761  	}
   762  	//对应高度hash不相等就向后寻找分叉点
   763  	if !bytes.Equal(headers.Items[0].Hash, header.Hash) {
   764  		synlog.Info("ProcBlockHeader hash no equal", "height", height, "self hash", common.ToHex(header.Hash), "peer hash", common.ToHex(headers.Items[0].Hash))
   765  
   766  		if height > BackBlockNum {
   767  			err = chain.FetchBlockHeaders(height-BackBlockNum, height, peerid)
   768  		} else if height != 0 {
   769  			err = chain.FetchBlockHeaders(0, height, peerid)
   770  		}
   771  		if err != nil {
   772  			synlog.Info("ProcBlockHeader FetchBlockHeaders", "err", err)
   773  		}
   774  	}
   775  	return nil
   776  }
   777  
   778  //ProcBlockHeaders 多个headers消息的处理,主要用于寻找分叉节点
   779  func (chain *BlockChain) ProcBlockHeaders(headers *types.Headers, pid string) error {
   780  	var ForkHeight int64 = -1
   781  	var forkhash []byte
   782  	var err error
   783  	count := len(headers.Items)
   784  	tipheight := chain.bestChain.Height()
   785  
   786  	//循环找到分叉点
   787  	for i := count - 1; i >= 0; i-- {
   788  		exists := chain.bestChain.HaveBlock(headers.Items[i].Hash, headers.Items[i].Height)
   789  		if exists {
   790  			ForkHeight = headers.Items[i].Height
   791  			forkhash = headers.Items[i].Hash
   792  			break
   793  		}
   794  	}
   795  	if ForkHeight == -1 {
   796  		synlog.Error("ProcBlockHeaders do not find fork point ")
   797  		synlog.Error("ProcBlockHeaders start headerinfo", "height", headers.Items[0].Height, "hash", common.ToHex(headers.Items[0].Hash))
   798  		synlog.Error("ProcBlockHeaders end headerinfo", "height", headers.Items[count-1].Height, "hash", common.ToHex(headers.Items[count-1].Hash))
   799  
   800  		//回退5000个block之后不再回退了,直接返回错误
   801  		startheight := headers.Items[0].Height
   802  		if tipheight > startheight && (tipheight-startheight) > MaxRollBlockNum {
   803  			synlog.Error("ProcBlockHeaders Not Roll Back!", "selfheight", tipheight, "RollBackedhieght", startheight)
   804  			return types.ErrNotRollBack
   805  		}
   806  		//继续向后取指定数量的headers
   807  		height := headers.Items[0].Height
   808  		if height > BackBlockNum {
   809  			err = chain.FetchBlockHeaders(height-BackBlockNum, height, pid)
   810  		} else {
   811  			err = chain.FetchBlockHeaders(0, height, pid)
   812  		}
   813  		if err != nil {
   814  			synlog.Info("ProcBlockHeaders FetchBlockHeaders", "err", err)
   815  		}
   816  		return types.ErrContinueBack
   817  	}
   818  	synlog.Info("ProcBlockHeaders find fork point", "height", ForkHeight, "hash", common.ToHex(forkhash))
   819  
   820  	//获取此pid对应的peer信息,
   821  	peerinfo := chain.GetPeerInfo(pid)
   822  	if peerinfo == nil {
   823  		synlog.Error("ProcBlockHeaders GetPeerInfo is nil", "pid", pid)
   824  		return types.ErrPeerInfoIsNil
   825  	}
   826  
   827  	//从分叉节点高度继续请求block,从pid
   828  	peermaxheight := peerinfo.Height
   829  
   830  	//启动一个线程在后台获取分叉的blcok
   831  	if chain.downLoadTask.InProgress() {
   832  		synlog.Info("ProcBlockHeaders downLoadTask.InProgress")
   833  		return nil
   834  	}
   835  	//在快速下载block阶段不处理fork的处理
   836  	//如果在普通同步阶段出现了分叉
   837  	//需要暂定同步解决分叉回滚之后再继续开启普通同步
   838  	if chain.GetDownloadSyncStatus() == normalDownLoadMode {
   839  		if chain.syncTask.InProgress() {
   840  			err = chain.syncTask.Cancel()
   841  			synlog.Info("ProcBlockHeaders: cancel syncTask start fork process downLoadTask!", "err", err)
   842  		}
   843  		endHeight := peermaxheight
   844  		if tipheight < peermaxheight {
   845  			endHeight = tipheight + 1
   846  		}
   847  		go chain.ProcDownLoadBlocks(ForkHeight, endHeight, false, []string{pid})
   848  	}
   849  	return nil
   850  }
   851  
   852  //ProcAddBlockHeadersMsg 处理从peer获取的headers消息
   853  func (chain *BlockChain) ProcAddBlockHeadersMsg(headers *types.Headers, pid string) error {
   854  	if headers == nil {
   855  		return types.ErrInvalidParam
   856  	}
   857  	count := len(headers.Items)
   858  	synlog.Debug("ProcAddBlockHeadersMsg", "count", count, "pid", pid)
   859  	if count == 1 {
   860  		return chain.ProcBlockHeader(headers, pid)
   861  	}
   862  	return chain.ProcBlockHeaders(headers, pid)
   863  
   864  }
   865  
   866  //CheckTipBlockHash 在规定时间本链的高度没有增长,但peerlist中最新高度远远高于本节点高度,
   867  //可能当前链是在分支链上,需从指定最长链的peer向后请求指定数量的blockheader
   868  //请求bestchain.Height -BackBlockNum -- bestchain.Height的header
   869  //需要考虑收不到分叉之后的第一个广播block,这样就会导致后面的广播block都在孤儿节点中了。
   870  func (chain *BlockChain) CheckTipBlockHash() {
   871  	synlog.Debug("CheckTipBlockHash")
   872  	defer chain.tickerwg.Done()
   873  
   874  	//获取当前主链的高度
   875  	tipheight := chain.bestChain.Height()
   876  	tiphash := chain.bestChain.Tip().hash
   877  	laststorheight := chain.blockStore.Height()
   878  
   879  	if tipheight != laststorheight {
   880  		synlog.Error("CheckTipBlockHash", "tipheight", tipheight, "laststorheight", laststorheight)
   881  		return
   882  	}
   883  
   884  	maxpeer := chain.GetMaxPeerInfo()
   885  	if maxpeer == nil {
   886  		synlog.Error("CheckTipBlockHash GetMaxPeerInfo is nil")
   887  		return
   888  	}
   889  	peermaxheight := maxpeer.Height
   890  	pid := maxpeer.Name
   891  	peerhash := maxpeer.Hash
   892  	var Err error
   893  	//和最高的peer做tip block hash的校验
   894  	if peermaxheight > tipheight {
   895  		//从指定peer 请求BackBlockNum个blockheaders
   896  		synlog.Debug("CheckTipBlockHash >", "peermaxheight", peermaxheight, "tipheight", tipheight)
   897  		Err = chain.FetchBlockHeaders(tipheight, tipheight, pid)
   898  	} else if peermaxheight == tipheight {
   899  		// 直接tip block hash比较,如果不相等需要从peer向后去指定的headers,尝试寻找分叉点
   900  		if !bytes.Equal(tiphash, peerhash) {
   901  			if tipheight > BackBlockNum {
   902  				synlog.Debug("CheckTipBlockHash ==", "peermaxheight", peermaxheight, "tipheight", tipheight)
   903  				Err = chain.FetchBlockHeaders(tipheight-BackBlockNum, tipheight, pid)
   904  			} else {
   905  				synlog.Debug("CheckTipBlockHash !=", "peermaxheight", peermaxheight, "tipheight", tipheight)
   906  				Err = chain.FetchBlockHeaders(1, tipheight, pid)
   907  			}
   908  		}
   909  	} else {
   910  
   911  		header, err := chain.blockStore.GetBlockHeaderByHeight(peermaxheight)
   912  		if err != nil {
   913  			return
   914  		}
   915  		if !bytes.Equal(header.Hash, peerhash) {
   916  			if peermaxheight > BackBlockNum {
   917  				synlog.Debug("CheckTipBlockHash<!=", "peermaxheight", peermaxheight, "tipheight", tipheight)
   918  				Err = chain.FetchBlockHeaders(peermaxheight-BackBlockNum, peermaxheight, pid)
   919  			} else {
   920  				synlog.Debug("CheckTipBlockHash<!=", "peermaxheight", peermaxheight, "tipheight", tipheight)
   921  				Err = chain.FetchBlockHeaders(1, peermaxheight, pid)
   922  			}
   923  		}
   924  	}
   925  	if Err != nil {
   926  		synlog.Error("CheckTipBlockHash FetchBlockHeaders", "err", Err)
   927  	}
   928  }
   929  
   930  //IsCaughtUp 本节点是否已经追赶上主链高度,追赶上之后通知本节点的共识模块开始挖矿
   931  func (chain *BlockChain) IsCaughtUp() bool {
   932  
   933  	height := chain.GetBlockHeight()
   934  
   935  	//peerMaxBlklock.Lock()
   936  	//defer peerMaxBlklock.Unlock()
   937  	peers := chain.GetPeers()
   938  	// peer中只有自己节点,没有其他节点
   939  	if peers == nil {
   940  		synlog.Debug("IsCaughtUp has no peers")
   941  		return chain.cfg.SingleMode
   942  	}
   943  
   944  	var maxPeerHeight int64 = -1
   945  	peersNo := 0
   946  	for _, peer := range peers {
   947  		if peer != nil && maxPeerHeight < peer.Height {
   948  			ok := chain.IsFaultPeer(peer.Name)
   949  			if !ok {
   950  				maxPeerHeight = peer.Height
   951  			}
   952  		}
   953  		peersNo++
   954  	}
   955  
   956  	isCaughtUp := (height > 0 || types.Since(chain.startTime) > 60*time.Second) && (maxPeerHeight == 0 || (height >= maxPeerHeight && maxPeerHeight != -1))
   957  
   958  	synlog.Debug("IsCaughtUp", "IsCaughtUp ", isCaughtUp, "height", height, "maxPeerHeight", maxPeerHeight, "peersNo", peersNo)
   959  	return isCaughtUp
   960  }
   961  
   962  //GetNtpClockSyncStatus 获取ntp时间是否同步状态
   963  func (chain *BlockChain) GetNtpClockSyncStatus() bool {
   964  	chain.ntpClockSynclock.Lock()
   965  	defer chain.ntpClockSynclock.Unlock()
   966  	return chain.isNtpClockSync
   967  }
   968  
   969  //UpdateNtpClockSyncStatus 定时更新ntp时间同步状态
   970  func (chain *BlockChain) UpdateNtpClockSyncStatus(Sync bool) {
   971  	chain.ntpClockSynclock.Lock()
   972  	defer chain.ntpClockSynclock.Unlock()
   973  	chain.isNtpClockSync = Sync
   974  }
   975  
   976  //CheckBestChain 定时确保本节点在最优链上,定时向peer请求指定高度的header
   977  func (chain *BlockChain) CheckBestChain(isFirst bool) {
   978  	if !isFirst {
   979  		defer chain.tickerwg.Done()
   980  	}
   981  	peers := chain.GetPeers()
   982  	// peer中只有自己节点,没有其他节点
   983  	if peers == nil {
   984  		synlog.Debug("CheckBestChain has no peers")
   985  		return
   986  	}
   987  
   988  	//设置首次检测最优链的标志
   989  	atomic.CompareAndSwapInt32(&chain.firstcheckbestchain, 0, 1)
   990  
   991  	tipheight := chain.bestChain.Height()
   992  
   993  	chain.bestpeerlock.Lock()
   994  	defer chain.bestpeerlock.Unlock()
   995  
   996  	for _, peer := range peers {
   997  		bestpeer := chain.bestChainPeerList[peer.Name]
   998  		if bestpeer != nil {
   999  			bestpeer.Peer = peer
  1000  			bestpeer.Height = tipheight
  1001  			bestpeer.Hash = nil
  1002  			bestpeer.Td = nil
  1003  			bestpeer.ReqFlag = true
  1004  		} else {
  1005  			if peer.Height < tipheight {
  1006  				continue
  1007  			}
  1008  			var newbestpeer BestPeerInfo
  1009  			newbestpeer.Peer = peer
  1010  			newbestpeer.Height = tipheight
  1011  			newbestpeer.Hash = nil
  1012  			newbestpeer.Td = nil
  1013  			newbestpeer.ReqFlag = true
  1014  			newbestpeer.IsBestChain = false
  1015  			chain.bestChainPeerList[peer.Name] = &newbestpeer
  1016  		}
  1017  		synlog.Debug("CheckBestChain FetchBlockHeaders", "height", tipheight, "pid", peer.Name)
  1018  		err := chain.FetchBlockHeaders(tipheight, tipheight, peer.Name)
  1019  		if err != nil {
  1020  			synlog.Error("CheckBestChain FetchBlockHeaders", "height", tipheight, "pid", peer.Name)
  1021  		}
  1022  	}
  1023  }
  1024  
  1025  //GetBestChainPeer 获取最优节点
  1026  func (chain *BlockChain) GetBestChainPeer(pid string) *BestPeerInfo {
  1027  	chain.bestpeerlock.Lock()
  1028  	defer chain.bestpeerlock.Unlock()
  1029  	return chain.bestChainPeerList[pid]
  1030  }
  1031  
  1032  //isBestChainPeer 指定peer是不是最优链
  1033  func (chain *BlockChain) isBestChainPeer(pid string) bool {
  1034  	chain.bestpeerlock.Lock()
  1035  	defer chain.bestpeerlock.Unlock()
  1036  	peer := chain.bestChainPeerList[pid]
  1037  	if peer != nil && peer.IsBestChain {
  1038  		return true
  1039  	}
  1040  	return false
  1041  }
  1042  
  1043  //GetBestChainPids 定时确保本节点在最优链上,定时向peer请求指定高度的header
  1044  func (chain *BlockChain) GetBestChainPids() []string {
  1045  	var PeerPids []string
  1046  	chain.bestpeerlock.Lock()
  1047  	defer chain.bestpeerlock.Unlock()
  1048  
  1049  	peersmap := chain.GetPeersMap()
  1050  	for key, value := range chain.bestChainPeerList {
  1051  		if !peersmap[value.Peer.Name] {
  1052  			delete(chain.bestChainPeerList, value.Peer.Name)
  1053  			synlog.Debug("GetBestChainPids:delete", "peer", value.Peer.Name)
  1054  			continue
  1055  		}
  1056  		if value.IsBestChain {
  1057  			ok := chain.IsFaultPeer(value.Peer.Name)
  1058  			if !ok {
  1059  				PeerPids = append(PeerPids, key)
  1060  			}
  1061  		}
  1062  	}
  1063  	synlog.Debug("GetBestChainPids", "pids", PeerPids)
  1064  	return PeerPids
  1065  }
  1066  
  1067  //CheckBestChainProc 检查最优链
  1068  func (chain *BlockChain) CheckBestChainProc(headers *types.Headers, pid string) {
  1069  
  1070  	//获取本节点指定高度的blockhash
  1071  	blockhash, err := chain.blockStore.GetBlockHashByHeight(headers.Items[0].Height)
  1072  	if err != nil {
  1073  		synlog.Debug("CheckBestChainProc GetBlockHashByHeight", "Height", headers.Items[0].Height, "err", err)
  1074  		return
  1075  	}
  1076  
  1077  	chain.bestpeerlock.Lock()
  1078  	defer chain.bestpeerlock.Unlock()
  1079  
  1080  	bestchainpeer := chain.bestChainPeerList[pid]
  1081  	if bestchainpeer == nil {
  1082  		synlog.Debug("CheckBestChainProc bestChainPeerList is nil", "Height", headers.Items[0].Height, "pid", pid)
  1083  		return
  1084  	}
  1085  	//
  1086  	if bestchainpeer.Height == headers.Items[0].Height {
  1087  		bestchainpeer.Hash = headers.Items[0].Hash
  1088  		bestchainpeer.ReqFlag = false
  1089  		if bytes.Equal(headers.Items[0].Hash, blockhash) {
  1090  			bestchainpeer.IsBestChain = true
  1091  			synlog.Debug("CheckBestChainProc IsBestChain ", "Height", headers.Items[0].Height, "pid", pid)
  1092  		} else {
  1093  			bestchainpeer.IsBestChain = false
  1094  			synlog.Debug("CheckBestChainProc NotBestChain", "Height", headers.Items[0].Height, "pid", pid)
  1095  		}
  1096  	}
  1097  }
  1098  
  1099  // ChunkRecordSync 同步chunkrecord
  1100  func (chain *BlockChain) ChunkRecordSync() {
  1101  	curheight := chain.GetBlockHeight()
  1102  	peerMaxBlkHeight := chain.GetPeerMaxBlkHeight()
  1103  	recvChunk := chain.GetCurRecvChunkNum()
  1104  
  1105  	curShouldChunk, _, _ := chain.CalcChunkInfo(curheight)
  1106  	targetChunk, _, _ := chain.CalcSafetyChunkInfo(peerMaxBlkHeight)
  1107  	if targetChunk < 0 ||
  1108  		curShouldChunk >= targetChunk || //说明已同步上来了不需要再进行chunk请求
  1109  		recvChunk >= targetChunk {
  1110  		return
  1111  	}
  1112  
  1113  	//如果任务正常则不重复启动任务
  1114  	if chain.chunkRecordTask.InProgress() {
  1115  		synlog.Info("chain chunkRecordTask InProgress")
  1116  		return
  1117  	}
  1118  
  1119  	synlog.Info("ChunkRecordSync", "curheight", curheight, "peerMaxBlkHeight", peerMaxBlkHeight,
  1120  		"recvChunk", recvChunk, "curShouldChunk", curShouldChunk)
  1121  
  1122  	pids := chain.GetBestChainPids()
  1123  	if pids != nil {
  1124  		err := chain.FetchChunkRecords(recvChunk+1, targetChunk, pids)
  1125  		if err != nil {
  1126  			synlog.Error("ChunkRecordSync FetchChunkRecords", "err", err)
  1127  		}
  1128  	} else {
  1129  		synlog.Info("ChunkRecordSync GetBestChainPids is nil")
  1130  	}
  1131  }
  1132  
  1133  //FetchChunkRecords 从指定pid获取start到end之间的ChunkRecord,只需要获取存储归档索引 blockHeight--->chunkhash
  1134  func (chain *BlockChain) FetchChunkRecords(start int64, end int64, pid []string) (err error) {
  1135  	if chain.client == nil {
  1136  		synlog.Error("FetchChunkRecords chain client not bind message queue.")
  1137  		return types.ErrClientNotBindQueue
  1138  	}
  1139  
  1140  	synlog.Debug("FetchChunkRecords", "StartHeight", start, "EndHeight", end, "pid", pid)
  1141  
  1142  	count := end - start
  1143  	if count < 0 {
  1144  		return types.ErrStartBigThanEnd
  1145  	}
  1146  	reqRec := &types.ReqChunkRecords{
  1147  		Start:    start,
  1148  		End:      end,
  1149  		IsDetail: false,
  1150  		Pid:      pid,
  1151  	}
  1152  	var cb func()
  1153  	if count >= int64(MaxReqChunkRecord) { // 每次请求最大MaxReqChunkRecord个chunk的record
  1154  		reqRec.End = reqRec.Start + int64(MaxReqChunkRecord) - 1
  1155  		cb = func() {
  1156  			chain.ChunkRecordSync()
  1157  		}
  1158  	}
  1159  	// 目前数据量小可在一个节点下载多个chunk记录
  1160  	// TODO 后续可以多个节点并发下载
  1161  	err = chain.chunkRecordTask.Start(reqRec.Start, reqRec.End, cb, nil)
  1162  	if err != nil {
  1163  		return err
  1164  	}
  1165  
  1166  	msg := chain.client.NewMessage("p2p", types.EventGetChunkRecord, reqRec)
  1167  	err = chain.client.Send(msg, false)
  1168  	if err != nil {
  1169  		synlog.Error("FetchChunkRecords", "client.Send err:", err)
  1170  		return err
  1171  	}
  1172  	return err
  1173  }
  1174  
  1175  /*
  1176  FetchChunkBlock 函数功能:
  1177  通过向P2P模块送 EventGetChunkBlock(types.RequestGetBlock),向其他节点主动请求区块,
  1178  P2P区块收到这个消息后,会向blockchain 模块回复, EventReply。
  1179  其他节点如果有这个范围的区块,P2P模块收到其他节点发来的数据,
  1180  会发送送EventAddBlocks(types.Blocks) 给 blockchain 模块,
  1181  blockchain 模块回复 EventReply
  1182  */
  1183  func (chain *BlockChain) FetchChunkBlock(startHeight, endHeight int64, pid []string, isDownLoad bool) (err error) {
  1184  	if chain.client == nil {
  1185  		synlog.Error("FetchChunkBlock chain client not bind message queue.")
  1186  		return types.ErrClientNotBindQueue
  1187  	}
  1188  
  1189  	synlog.Debug("FetchChunkBlock input", "StartHeight", startHeight, "EndHeight", endHeight)
  1190  
  1191  	blockcount := endHeight - startHeight
  1192  	if blockcount < 0 {
  1193  		return types.ErrStartBigThanEnd
  1194  	}
  1195  	chunkNum, _, end := chain.CalcChunkInfo(startHeight)
  1196  
  1197  	var chunkhash []byte
  1198  	for i := 0; i < waitTimeDownLoad; i++ {
  1199  		chunkhash, err = chain.blockStore.getRecvChunkHash(chunkNum)
  1200  		if err != nil && isDownLoad { // downLoac模式下会进行多次尝试
  1201  			time.Sleep(time.Second)
  1202  			continue
  1203  		}
  1204  		break
  1205  	}
  1206  	if err != nil {
  1207  		return ErrNoChunkInfoToDownLoad
  1208  	}
  1209  
  1210  	// 以chunk为单位同步block
  1211  	var requestblock types.ChunkInfoMsg
  1212  	requestblock.ChunkHash = chunkhash
  1213  	requestblock.Start = startHeight
  1214  	requestblock.End = endHeight
  1215  	if endHeight > end {
  1216  		requestblock.End = end
  1217  	}
  1218  
  1219  	var cb func()
  1220  	var timeoutcb func(height int64)
  1221  	if isDownLoad {
  1222  		//还有区块需要请求,挂接钩子回调函数
  1223  		if requestblock.End < chain.downLoadInfo.EndHeight {
  1224  			cb = func() {
  1225  				chain.ReqDownLoadChunkBlocks()
  1226  			}
  1227  			timeoutcb = func(height int64) {
  1228  				chain.DownLoadChunkTimeOutProc(height)
  1229  			}
  1230  			chain.UpdateDownLoadStartHeight(requestblock.End + 1)
  1231  		} else { // 所有DownLoad block已请求结束,恢复DownLoadInfo为默认值
  1232  			chain.DefaultDownLoadInfo()
  1233  		}
  1234  		// chunk下载只是每次只下载一个chunk
  1235  		// TODO 后续再做并发请求下载
  1236  		err = chain.downLoadTask.Start(requestblock.Start, requestblock.End, cb, timeoutcb)
  1237  		if err != nil {
  1238  			return err
  1239  		}
  1240  	} else {
  1241  		if chain.GetPeerMaxBlkHeight()-requestblock.End > BackBlockNum {
  1242  			cb = func() {
  1243  				chain.SynBlocksFromPeers()
  1244  			}
  1245  		}
  1246  		// chunk下载只是每次只下载一个chunk
  1247  		// TODO 后续再做并发请求下载
  1248  		err = chain.syncTask.Start(requestblock.Start, requestblock.End, cb, timeoutcb)
  1249  		if err != nil {
  1250  			return err
  1251  		}
  1252  	}
  1253  	synlog.Info("FetchChunkBlock", "chunkNum", chunkNum, "Start", requestblock.Start, "End", requestblock.End, "isDownLoad", isDownLoad, "chunkhash", common.ToHex(chunkhash))
  1254  	msg := chain.client.NewMessage("p2p", types.EventGetChunkBlock, &requestblock)
  1255  	err = chain.client.Send(msg, false)
  1256  	if err != nil {
  1257  		synlog.Error("FetchChunkBlock", "client.Send err:", err)
  1258  		return err
  1259  	}
  1260  	return err
  1261  }