github.com/justinjmoses/evergreen@v0.0.0-20170530173719-1d50e381ff0d/monitor/task_flagging.go (about)

     1  package monitor
     2  
     3  import (
     4  	"time"
     5  
     6  	"github.com/evergreen-ci/evergreen/model/task"
     7  	"github.com/mongodb/grip"
     8  	"github.com/pkg/errors"
     9  )
    10  
    11  const (
    12  	// reasons for cleaning up a task
    13  	HeartbeatTimeout = "task heartbeat timed out"
    14  )
    15  
    16  var (
    17  	// threshold for a task's heartbeat to time out
    18  	HeartbeatTimeoutThreshold = 7 * time.Minute
    19  )
    20  
    21  // function that spits out a list of tasks that need to be stopped
    22  // and cleaned up
    23  type taskFlaggingFunc func() ([]doomedTaskWrapper, error)
    24  
    25  // wrapper for a task to be cleaned up. contains the task, as well as the
    26  // reason it is being cleaned up
    27  type doomedTaskWrapper struct {
    28  	// the task to be cleaned up
    29  	task task.Task
    30  	// why the task is being cleaned up
    31  	reason string
    32  }
    33  
    34  // flagTimedOutHeartbeats is a taskFlaggingFunc to flag any tasks whose
    35  // heartbeats have timed out
    36  func flagTimedOutHeartbeats() ([]doomedTaskWrapper, error) {
    37  	grip.Info("Finding tasks with timed-out heartbeats...")
    38  
    39  	// fetch any running tasks whose last heartbeat was too long in the past
    40  	threshold := time.Now().Add(-HeartbeatTimeoutThreshold)
    41  
    42  	tasks, err := task.Find(task.ByRunningLastHeartbeat(threshold))
    43  	if err != nil {
    44  		return nil, errors.Wrap(err,
    45  			"error finding tasks with timed-out heartbeats")
    46  	}
    47  
    48  	// convert to be returned
    49  	wrappers := make([]doomedTaskWrapper, 0, len(tasks))
    50  
    51  	for _, task := range tasks {
    52  		wrappers = append(wrappers, doomedTaskWrapper{task, HeartbeatTimeout})
    53  	}
    54  
    55  	grip.Infof("Found %d tasks whose heartbeats timed out", len(wrappers))
    56  
    57  	return wrappers, nil
    58  }