github.com/justinjmoses/evergreen@v0.0.0-20170530173719-1d50e381ff0d/agent/comm/heartbeat.go (about)

     1  package comm
     2  
     3  import (
     4  	"time"
     5  
     6  	"github.com/mongodb/grip/slogger"
     7  )
     8  
     9  // HeartbeatTicker manages heartbeat communication with the API server
    10  type HeartbeatTicker struct {
    11  	// Number of consecutive failed heartbeats allowed before signaling a failure
    12  	MaxFailedHeartbeats int
    13  
    14  	// Period of time to wait between heartbeat attempts
    15  	Interval time.Duration
    16  
    17  	// Channel on which to notify of failed heartbeats or aborted task
    18  	SignalChan chan<- Signal
    19  
    20  	// A channel which, when closed, tells the heartbeat ticker should stop.
    21  	stop <-chan struct{}
    22  
    23  	// The current count of how many heartbeats have failed consecutively.
    24  	numFailed int
    25  
    26  	// Interface which handles sending the actual heartbeat over the network
    27  	TaskCommunicator
    28  
    29  	Logger *slogger.Logger
    30  }
    31  
    32  func NewHeartbeatTicker(stopper <-chan struct{}) *HeartbeatTicker {
    33  	// TODO replace the stopper channel with a context that we
    34  	// pass to StartHeartbeating and eliminate the special constructor.
    35  	return &HeartbeatTicker{stop: stopper}
    36  }
    37  
    38  func (hbt *HeartbeatTicker) StartHeartbeating() {
    39  	hbt.numFailed = 0
    40  
    41  	go func() {
    42  		ticker := time.NewTicker(hbt.Interval)
    43  		defer ticker.Stop()
    44  		for {
    45  			select {
    46  			case <-ticker.C:
    47  				abort, err := hbt.TaskCommunicator.Heartbeat()
    48  				if err != nil {
    49  					hbt.numFailed++
    50  					hbt.Logger.Logf(slogger.ERROR, "Error sending heartbeat (%v): %v", hbt.numFailed, err)
    51  				} else {
    52  					hbt.numFailed = 0
    53  				}
    54  				if hbt.numFailed == hbt.MaxFailedHeartbeats+1 {
    55  					hbt.Logger.Logf(slogger.ERROR, "Max heartbeats failed - trying to stop...")
    56  					hbt.SignalChan <- HeartbeatMaxFailed
    57  					return
    58  				}
    59  				if abort {
    60  					hbt.SignalChan <- AbortedByUser
    61  					return
    62  				}
    63  			case <-hbt.stop:
    64  				hbt.Logger.Logf(slogger.INFO, "Heartbeat ticker stopping.")
    65  				return
    66  			}
    67  		}
    68  	}()
    69  }