vitess.io/vitess@v0.16.2/go/vt/vtorc/process/health.go (about)

     1  /*
     2     Copyright 2017 Shlomi Noach, GitHub Inc.
     3  
     4     Licensed under the Apache License, Version 2.0 (the "License");
     5     you may not use this file except in compliance with the License.
     6     You may obtain a copy of the License at
     7  
     8         http://www.apache.org/licenses/LICENSE-2.0
     9  
    10     Unless required by applicable law or agreed to in writing, software
    11     distributed under the License is distributed on an "AS IS" BASIS,
    12     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13     See the License for the specific language governing permissions and
    14     limitations under the License.
    15  */
    16  
    17  package process
    18  
    19  import (
    20  	"sync"
    21  	"sync/atomic"
    22  	"time"
    23  
    24  	"vitess.io/vitess/go/vt/log"
    25  
    26  	"vitess.io/vitess/go/vt/vtorc/config"
    27  	"vitess.io/vitess/go/vt/vtorc/util"
    28  
    29  	"github.com/patrickmn/go-cache"
    30  )
    31  
    32  var lastHealthCheckUnixNano int64
    33  var lastGoodHealthCheckUnixNano int64
    34  var LastContinousCheckHealthy int64
    35  
    36  var lastHealthCheckCache = cache.New(config.HealthPollSeconds*time.Second, time.Second)
    37  
    38  type NodeHealth struct {
    39  	Hostname        string
    40  	Token           string
    41  	AppVersion      string
    42  	FirstSeenActive string
    43  	LastSeenActive  string
    44  	ExtraInfo       string
    45  	Command         string
    46  	DBBackend       string
    47  
    48  	LastReported time.Time
    49  	onceHistory  sync.Once
    50  	onceUpdate   sync.Once
    51  }
    52  
    53  func NewNodeHealth() *NodeHealth {
    54  	return &NodeHealth{
    55  		Hostname: ThisHostname,
    56  		Token:    util.ProcessToken.Hash,
    57  	}
    58  }
    59  
    60  func (nodeHealth *NodeHealth) Update() *NodeHealth {
    61  	nodeHealth.onceUpdate.Do(func() {
    62  		nodeHealth.Hostname = ThisHostname
    63  		nodeHealth.Token = util.ProcessToken.Hash
    64  	})
    65  	nodeHealth.LastReported = time.Now()
    66  	return nodeHealth
    67  }
    68  
    69  var ThisNodeHealth = NewNodeHealth()
    70  
    71  type HealthStatus struct {
    72  	Healthy            bool
    73  	Hostname           string
    74  	Token              string
    75  	IsActiveNode       bool
    76  	ActiveNode         *NodeHealth
    77  	Error              error
    78  	AvailableNodes     [](*NodeHealth)
    79  	RaftLeader         string
    80  	IsRaftLeader       bool
    81  	RaftLeaderURI      string
    82  	RaftAdvertise      string
    83  	RaftHealthyMembers []string
    84  }
    85  
    86  type VTOrcExecutionMode string
    87  
    88  const (
    89  	VTOrcExecutionCliMode  VTOrcExecutionMode = "CLIMode"
    90  	VTOrcExecutionHTTPMode VTOrcExecutionMode = "HttpMode"
    91  )
    92  
    93  var continuousRegistrationOnce sync.Once
    94  
    95  func RegisterNode(nodeHealth *NodeHealth) (healthy bool, err error) {
    96  	nodeHealth.Update()
    97  	healthy, err = WriteRegisterNode(nodeHealth)
    98  	atomic.StoreInt64(&lastHealthCheckUnixNano, time.Now().UnixNano())
    99  	if healthy {
   100  		atomic.StoreInt64(&lastGoodHealthCheckUnixNano, time.Now().UnixNano())
   101  	}
   102  	return healthy, err
   103  }
   104  
   105  // HealthTest attempts to write to the backend database and get a result
   106  func HealthTest() (health *HealthStatus, err error) {
   107  	cacheKey := util.ProcessToken.Hash
   108  	if healthStatus, found := lastHealthCheckCache.Get(cacheKey); found {
   109  		return healthStatus.(*HealthStatus), nil
   110  	}
   111  
   112  	health = &HealthStatus{Healthy: false, Hostname: ThisHostname, Token: util.ProcessToken.Hash}
   113  	defer lastHealthCheckCache.Set(cacheKey, health, cache.DefaultExpiration)
   114  
   115  	healthy, err := RegisterNode(ThisNodeHealth)
   116  	if err != nil {
   117  		health.Error = err
   118  		log.Error(err)
   119  		return health, err
   120  	}
   121  	health.Healthy = healthy
   122  
   123  	if health.ActiveNode, health.IsActiveNode, err = ElectedNode(); err != nil {
   124  		health.Error = err
   125  		log.Error(err)
   126  		return health, err
   127  	}
   128  	health.AvailableNodes, _ = ReadAvailableNodes(true)
   129  
   130  	return health, nil
   131  }
   132  
   133  // ContinuousRegistration will continuously update the node_health
   134  // table showing that the current process is still running.
   135  func ContinuousRegistration(extraInfo string, command string) {
   136  	ThisNodeHealth.ExtraInfo = extraInfo
   137  	ThisNodeHealth.Command = command
   138  	continuousRegistrationOnce.Do(func() {
   139  		tickOperation := func() {
   140  			healthy, err := RegisterNode(ThisNodeHealth)
   141  			if err != nil {
   142  				log.Errorf("ContinuousRegistration: RegisterNode failed: %+v", err)
   143  			}
   144  			if healthy {
   145  				atomic.StoreInt64(&LastContinousCheckHealthy, 1)
   146  			} else {
   147  				atomic.StoreInt64(&LastContinousCheckHealthy, 0)
   148  			}
   149  		}
   150  		// First one is synchronous
   151  		tickOperation()
   152  		go func() {
   153  			registrationTick := time.Tick(config.HealthPollSeconds * time.Second)
   154  			for range registrationTick {
   155  				// We already run inside a go-routine so
   156  				// do not do this asynchronously.  If we
   157  				// get stuck then we don't want to fill up
   158  				// the backend pool with connections running
   159  				// this maintenance operation.
   160  				tickOperation()
   161  			}
   162  		}()
   163  	})
   164  }