github.com/pf-qiu/concourse/v6@v6.7.3-0.20201207032516-1f455d73275f/atc/worker/placement.go (about)

     1  package worker
     2  
     3  import (
     4  	"errors"
     5  	"fmt"
     6  	"math/rand"
     7  	"strings"
     8  	"time"
     9  
    10  	"code.cloudfoundry.org/lager"
    11  	"github.com/pf-qiu/concourse/v6/atc/db"
    12  )
    13  
    14  type ContainerPlacementStrategyOptions struct {
    15  	ContainerPlacementStrategy []string `long:"container-placement-strategy" default:"volume-locality" choice:"volume-locality" choice:"random" choice:"fewest-build-containers" choice:"limit-active-tasks" description:"Method by which a worker is selected during container placement. If multiple methods are specified, they will be applied in order. Random strategy should only be used alone."`
    16  	MaxActiveTasksPerWorker    int      `long:"max-active-tasks-per-worker" default:"0" description:"Maximum allowed number of active build tasks per worker. Has effect only when used with limit-active-tasks placement strategy. 0 means no limit."`
    17  }
    18  
    19  type ContainerPlacementStrategy interface {
    20  	//TODO: Don't pass around container metadata since it's not guaranteed to be deterministic.
    21  	// Change this after check containers stop being reused
    22  	Choose(lager.Logger, []Worker, ContainerSpec) (Worker, error)
    23  	ModifiesActiveTasks() bool
    24  }
    25  
    26  type ContainerPlacementStrategyChainNode interface {
    27  	Choose(lager.Logger, []Worker, ContainerSpec) ([]Worker, error)
    28  	ModifiesActiveTasks() bool
    29  }
    30  
    31  type containerPlacementStrategy struct {
    32  	nodes []ContainerPlacementStrategyChainNode
    33  }
    34  
    35  func NewContainerPlacementStrategy(opts ContainerPlacementStrategyOptions) (*containerPlacementStrategy, error) {
    36  	cps := &containerPlacementStrategy{nodes: []ContainerPlacementStrategyChainNode{}}
    37  	for _, strategy := range opts.ContainerPlacementStrategy {
    38  		strategy := strings.TrimSpace(strategy)
    39  		switch strategy {
    40  		case "random":
    41  			// Add nothing. Because an empty strategy chain equals to random strategy.
    42  		case "fewest-build-containers":
    43  			cps.nodes = append(cps.nodes, newFewestBuildContainersPlacementStrategy())
    44  		case "limit-active-tasks":
    45  			if opts.MaxActiveTasksPerWorker < 0 {
    46  				return nil, errors.New("max-active-tasks-per-worker must be greater or equal than 0")
    47  			}
    48  			cps.nodes = append(cps.nodes, newLimitActiveTasksPlacementStrategy(opts.MaxActiveTasksPerWorker))
    49  		case "volume-locality":
    50  			cps.nodes = append(cps.nodes, newVolumeLocalityPlacementStrategyNode())
    51  		default:
    52  			return nil, fmt.Errorf("invalid container placement strategy %s", strategy)
    53  		}
    54  	}
    55  	return cps, nil
    56  }
    57  
    58  func (strategy *containerPlacementStrategy) Choose(logger lager.Logger, workers []Worker, spec ContainerSpec) (Worker, error) {
    59  	var err error
    60  	for _, node := range strategy.nodes {
    61  		workers, err = node.Choose(logger, workers, spec)
    62  		if err != nil {
    63  			return nil, err
    64  		}
    65  		if len(workers) == 0 {
    66  			return nil, nil
    67  		}
    68  	}
    69  	if len(workers) == 1 {
    70  		return workers[0], nil
    71  	}
    72  
    73  	// If there are still multiple candidate, choose a random one.
    74  	r := rand.New(rand.NewSource(time.Now().UnixNano()))
    75  	return workers[r.Intn(len(workers))], nil
    76  }
    77  
    78  func (strategy *containerPlacementStrategy) ModifiesActiveTasks() bool {
    79  	for _, node := range strategy.nodes {
    80  		if node.ModifiesActiveTasks() {
    81  			return true
    82  		}
    83  	}
    84  	return false
    85  }
    86  
    87  func NewRandomPlacementStrategy() ContainerPlacementStrategy {
    88  	s, _ := NewContainerPlacementStrategy(ContainerPlacementStrategyOptions{ContainerPlacementStrategy: []string{"random"}})
    89  	return s
    90  }
    91  
    92  type VolumeLocalityPlacementStrategyNode struct{}
    93  
    94  func newVolumeLocalityPlacementStrategyNode() ContainerPlacementStrategyChainNode {
    95  	return &VolumeLocalityPlacementStrategyNode{}
    96  }
    97  
    98  func (strategy *VolumeLocalityPlacementStrategyNode) Choose(logger lager.Logger, workers []Worker, spec ContainerSpec) ([]Worker, error) {
    99  	workersByCount := map[int][]Worker{}
   100  	var highestCount int
   101  	for _, w := range workers {
   102  		candidateInputCount := 0
   103  
   104  		for _, inputSource := range spec.Inputs {
   105  			_, found, err := inputSource.Source().ExistsOn(logger, w)
   106  			if err != nil {
   107  				return nil, err
   108  			}
   109  
   110  			if found {
   111  				candidateInputCount++
   112  			}
   113  		}
   114  
   115  		workersByCount[candidateInputCount] = append(workersByCount[candidateInputCount], w)
   116  
   117  		if candidateInputCount >= highestCount {
   118  			highestCount = candidateInputCount
   119  		}
   120  	}
   121  
   122  	return workersByCount[highestCount], nil
   123  }
   124  
   125  func (strategy *VolumeLocalityPlacementStrategyNode) ModifiesActiveTasks() bool {
   126  	return false
   127  }
   128  
   129  type FewestBuildContainersPlacementStrategyNode struct{}
   130  
   131  func newFewestBuildContainersPlacementStrategy() ContainerPlacementStrategyChainNode {
   132  	return &FewestBuildContainersPlacementStrategyNode{}
   133  }
   134  
   135  func (strategy *FewestBuildContainersPlacementStrategyNode) Choose(logger lager.Logger, workers []Worker, spec ContainerSpec) ([]Worker, error) {
   136  	workersByWork := map[int][]Worker{}
   137  	var minWork int
   138  
   139  	for i, w := range workers {
   140  		work := w.BuildContainers()
   141  		workersByWork[work] = append(workersByWork[work], w)
   142  		if i == 0 || work < minWork {
   143  			minWork = work
   144  		}
   145  	}
   146  
   147  	return workersByWork[minWork], nil
   148  }
   149  
   150  func (strategy *FewestBuildContainersPlacementStrategyNode) ModifiesActiveTasks() bool {
   151  	return false
   152  }
   153  
   154  type LimitActiveTasksPlacementStrategyNode struct {
   155  	maxTasks int
   156  }
   157  
   158  func newLimitActiveTasksPlacementStrategy(maxTasks int) ContainerPlacementStrategyChainNode {
   159  	return &LimitActiveTasksPlacementStrategyNode{
   160  		maxTasks: maxTasks,
   161  	}
   162  }
   163  
   164  func (strategy *LimitActiveTasksPlacementStrategyNode) Choose(logger lager.Logger, workers []Worker, spec ContainerSpec) ([]Worker, error) {
   165  	workersByWork := map[int][]Worker{}
   166  	minActiveTasks := -1
   167  
   168  	for _, w := range workers {
   169  		activeTasks, err := w.ActiveTasks()
   170  		if err != nil {
   171  			logger.Error("Cannot retrive active tasks on worker. Skipping.", err)
   172  			continue
   173  		}
   174  
   175  		// If maxTasks == 0 or the step is not a task, ignore the number of active tasks and distribute the work evenly
   176  		if strategy.maxTasks > 0 && activeTasks >= strategy.maxTasks && spec.Type == db.ContainerTypeTask {
   177  			logger.Info("worker-busy")
   178  			continue
   179  		}
   180  
   181  		workersByWork[activeTasks] = append(workersByWork[activeTasks], w)
   182  		if minActiveTasks == -1 || activeTasks < minActiveTasks {
   183  			minActiveTasks = activeTasks
   184  		}
   185  	}
   186  
   187  	return workersByWork[minActiveTasks], nil
   188  }
   189  
   190  func (strategy *LimitActiveTasksPlacementStrategyNode) ModifiesActiveTasks() bool {
   191  	return true
   192  }