github.com/turingchain2020/turingchain@v1.1.21/util/healthcheck.go (about)

     1  // Copyright Turing Corp. 2018 All Rights Reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package util
     6  
     7  import (
     8  	"net"
     9  	"time"
    10  
    11  	"sync"
    12  
    13  	"github.com/turingchain2020/turingchain/client"
    14  	log "github.com/turingchain2020/turingchain/common/log/log15"
    15  	"github.com/turingchain2020/turingchain/queue"
    16  	"github.com/turingchain2020/turingchain/types"
    17  )
    18  
    19  var (
    20  	listenAddr            = "127.0.0.1:9675" //as server, should keep default 0.0.0.0
    21  	unSyncMaxTimes uint32 = 6                //max 6 times
    22  	checkInterval  uint32 = 5                // 5s
    23  )
    24  
    25  // HealthCheckServer  a node's health check server
    26  type HealthCheckServer struct {
    27  	api  client.QueueProtocolAPI
    28  	l    net.Listener
    29  	quit chan struct{}
    30  	wg   sync.WaitGroup
    31  }
    32  
    33  // Close NewHealthCheckServer close
    34  func (s *HealthCheckServer) Close() {
    35  	close(s.quit)
    36  	s.wg.Wait()
    37  	log.Info("healthCheck quit")
    38  }
    39  
    40  // NewHealthCheckServer new json rpcserver object
    41  func NewHealthCheckServer(c queue.Client) *HealthCheckServer {
    42  	if c == nil {
    43  		return nil
    44  	}
    45  	h := &HealthCheckServer{}
    46  	var err error
    47  	h.api, err = client.New(c, nil)
    48  	if err != nil {
    49  		return nil
    50  	}
    51  	h.quit = make(chan struct{})
    52  	return h
    53  }
    54  
    55  // Start HealthCheckServer start
    56  func (s *HealthCheckServer) Start(cfg *types.HealthCheck) {
    57  	if cfg != nil {
    58  		if cfg.ListenAddr != "" {
    59  			listenAddr = cfg.ListenAddr
    60  		}
    61  		if cfg.CheckInterval != 0 {
    62  			checkInterval = cfg.CheckInterval
    63  		}
    64  		if cfg.UnSyncMaxTimes != 0 {
    65  			unSyncMaxTimes = cfg.UnSyncMaxTimes
    66  		}
    67  	}
    68  	log.Info("healthCheck start ", "addr", listenAddr, "inter", checkInterval, "times", unSyncMaxTimes)
    69  	s.wg.Add(1)
    70  	go s.healthCheck()
    71  
    72  }
    73  
    74  func (s *HealthCheckServer) listen(on bool) error {
    75  	if on {
    76  		listener, err := net.Listen("tcp", listenAddr)
    77  		if err != nil {
    78  			return err
    79  		}
    80  		s.l = listener
    81  		log.Info("healthCheck listen open")
    82  		return nil
    83  	}
    84  
    85  	if s.l != nil {
    86  		err := s.l.Close()
    87  		if err != nil {
    88  			return err
    89  		}
    90  		log.Info("healthCheck listen close")
    91  		s.l = nil
    92  	}
    93  
    94  	return nil
    95  }
    96  
    97  func (s *HealthCheckServer) getHealth(sync bool) (bool, error) {
    98  	reply, err := s.api.IsSync()
    99  	if err != nil {
   100  		return false, err
   101  	}
   102  
   103  	peerList, err := s.api.PeerInfo(&types.P2PGetPeerReq{})
   104  	if err != nil {
   105  		return false, err
   106  	}
   107  
   108  	log.Debug("healthCheck tick", "peers", len(peerList.Peers), "isCaughtUp", reply.IsOk,
   109  		"health", len(peerList.Peers) > 1 && reply.IsOk, "listen", sync)
   110  
   111  	return len(peerList.Peers) > 1 && reply.IsOk, nil
   112  }
   113  
   114  func (s *HealthCheckServer) healthCheck() {
   115  	ticker := time.NewTicker(time.Second * time.Duration(checkInterval))
   116  	defer ticker.Stop()
   117  	defer s.wg.Done()
   118  
   119  	var sync bool
   120  	var unSyncTimes uint32
   121  
   122  	for {
   123  		select {
   124  		case <-s.quit:
   125  			if s.l != nil {
   126  				err := s.l.Close()
   127  				if err != nil {
   128  					log.Error("healthCheck ", "close err ", err)
   129  				}
   130  			}
   131  			if s.api != nil {
   132  				s.api.Close()
   133  			}
   134  			return
   135  		case <-ticker.C:
   136  			health, err := s.getHealth(sync)
   137  			if err != nil {
   138  				continue
   139  			}
   140  			//sync
   141  			if health {
   142  				if !sync {
   143  					err = s.listen(true)
   144  					if err != nil {
   145  						log.Error("healthCheck ", "listen open err", err.Error())
   146  						continue
   147  					}
   148  					sync = true
   149  				}
   150  				unSyncTimes = 0
   151  
   152  			} else {
   153  				if sync {
   154  					if unSyncTimes >= unSyncMaxTimes {
   155  						err = s.listen(false)
   156  						if err != nil {
   157  							log.Error("healthCheck ", "listen close err", err.Error())
   158  							continue
   159  						}
   160  						sync = false
   161  					}
   162  					unSyncTimes++
   163  				}
   164  			}
   165  		}
   166  	}
   167  }