github.com/billybanfield/evergreen@v0.0.0-20170525200750-eeee692790f7/scheduler/deficit_based_host_allocator.go (about)

     1  package scheduler
     2  
     3  import (
     4  	"github.com/evergreen-ci/evergreen"
     5  	"github.com/evergreen-ci/evergreen/cloud/providers"
     6  	"github.com/evergreen-ci/evergreen/model/distro"
     7  	"github.com/evergreen-ci/evergreen/model/host"
     8  	"github.com/evergreen-ci/evergreen/util"
     9  	"github.com/mongodb/grip"
    10  	"github.com/pkg/errors"
    11  )
    12  
    13  // DeficitBasedHostAllocator uses the difference between the number of free hosts
    14  // and the number of tasks that need to be run as a metric for how many new
    15  // hosts need to be spun up
    16  type DeficitBasedHostAllocator struct{}
    17  
    18  // NewHostsNeeded decides how many new hosts are needed for a distro by seeing if
    19  // the number of tasks that need to be run for the distro is greater than the number
    20  // of hosts currently free to run a task. Returns a map of distro-># of hosts to spawn.
    21  func (self *DeficitBasedHostAllocator) NewHostsNeeded(
    22  	hostAllocatorData HostAllocatorData, settings *evergreen.Settings) (map[string]int, error) {
    23  
    24  	newHostsNeeded := make(map[string]int)
    25  
    26  	// now, for each distro, see if we need to spin up any new hosts
    27  	for distroId := range hostAllocatorData.taskQueueItems {
    28  		distro, ok := hostAllocatorData.distros[distroId]
    29  		if !ok {
    30  			return nil, errors.Errorf("No distro info available for distro %v",
    31  				distroId)
    32  		}
    33  		if distro.Id != distroId {
    34  			return nil, errors.Errorf("Bad mapping between task queue distro "+
    35  				"name and host allocator distro data: %v != %v", distro.Id,
    36  				distroId)
    37  		}
    38  
    39  		newHostsNeeded[distroId] = self.numNewHostsForDistro(
    40  			&hostAllocatorData, distro, settings)
    41  	}
    42  
    43  	return newHostsNeeded, nil
    44  }
    45  
    46  // numNewHostsForDistro determine how many new hosts should be spun up for an
    47  // individual distro
    48  func (self *DeficitBasedHostAllocator) numNewHostsForDistro(
    49  	hostAllocatorData *HostAllocatorData, distro distro.Distro, settings *evergreen.Settings) int {
    50  
    51  	cloudManager, err := providers.GetCloudManager(distro.Provider, settings)
    52  
    53  	if err != nil {
    54  		grip.Errorf("Couldn't get cloud manager for distro %s with provider %s: %+v",
    55  			distro.Id, distro.Provider, err)
    56  		return 0
    57  	}
    58  
    59  	can, err := cloudManager.CanSpawn()
    60  	if err != nil {
    61  		err = errors.Wrapf(err, "Couldn't check if cloud provider %s is spawnable",
    62  			distro.Provider)
    63  		grip.Error(err)
    64  		return 0
    65  	}
    66  	if !can {
    67  		return 0
    68  	}
    69  
    70  	existingDistroHosts := hostAllocatorData.existingDistroHosts[distro.Id]
    71  	runnableDistroTasks := hostAllocatorData.taskQueueItems[distro.Id]
    72  
    73  	freeHosts := make([]host.Host, 0, len(existingDistroHosts))
    74  	for _, existingDistroHost := range existingDistroHosts {
    75  		if existingDistroHost.RunningTask == "" {
    76  			freeHosts = append(freeHosts, existingDistroHost)
    77  		}
    78  	}
    79  
    80  	numNewHosts := util.Min(
    81  		// the deficit of available hosts vs. tasks to be run
    82  		len(runnableDistroTasks)-len(freeHosts),
    83  		// the maximum number of new hosts we're allowed to spin up
    84  		distro.PoolSize-len(existingDistroHosts),
    85  	)
    86  
    87  	// cap to zero as lower bound
    88  	if numNewHosts < 0 {
    89  		numNewHosts = 0
    90  	}
    91  	return numNewHosts
    92  }