github.com/pf-qiu/concourse/v6@v6.7.3-0.20201207032516-1f455d73275f/atc/worker/placement.go (about) 1 package worker 2 3 import ( 4 "errors" 5 "fmt" 6 "math/rand" 7 "strings" 8 "time" 9 10 "code.cloudfoundry.org/lager" 11 "github.com/pf-qiu/concourse/v6/atc/db" 12 ) 13 14 type ContainerPlacementStrategyOptions struct { 15 ContainerPlacementStrategy []string `long:"container-placement-strategy" default:"volume-locality" choice:"volume-locality" choice:"random" choice:"fewest-build-containers" choice:"limit-active-tasks" description:"Method by which a worker is selected during container placement. If multiple methods are specified, they will be applied in order. Random strategy should only be used alone."` 16 MaxActiveTasksPerWorker int `long:"max-active-tasks-per-worker" default:"0" description:"Maximum allowed number of active build tasks per worker. Has effect only when used with limit-active-tasks placement strategy. 0 means no limit."` 17 } 18 19 type ContainerPlacementStrategy interface { 20 //TODO: Don't pass around container metadata since it's not guaranteed to be deterministic. 21 // Change this after check containers stop being reused 22 Choose(lager.Logger, []Worker, ContainerSpec) (Worker, error) 23 ModifiesActiveTasks() bool 24 } 25 26 type ContainerPlacementStrategyChainNode interface { 27 Choose(lager.Logger, []Worker, ContainerSpec) ([]Worker, error) 28 ModifiesActiveTasks() bool 29 } 30 31 type containerPlacementStrategy struct { 32 nodes []ContainerPlacementStrategyChainNode 33 } 34 35 func NewContainerPlacementStrategy(opts ContainerPlacementStrategyOptions) (*containerPlacementStrategy, error) { 36 cps := &containerPlacementStrategy{nodes: []ContainerPlacementStrategyChainNode{}} 37 for _, strategy := range opts.ContainerPlacementStrategy { 38 strategy := strings.TrimSpace(strategy) 39 switch strategy { 40 case "random": 41 // Add nothing. Because an empty strategy chain equals to random strategy. 42 case "fewest-build-containers": 43 cps.nodes = append(cps.nodes, newFewestBuildContainersPlacementStrategy()) 44 case "limit-active-tasks": 45 if opts.MaxActiveTasksPerWorker < 0 { 46 return nil, errors.New("max-active-tasks-per-worker must be greater or equal than 0") 47 } 48 cps.nodes = append(cps.nodes, newLimitActiveTasksPlacementStrategy(opts.MaxActiveTasksPerWorker)) 49 case "volume-locality": 50 cps.nodes = append(cps.nodes, newVolumeLocalityPlacementStrategyNode()) 51 default: 52 return nil, fmt.Errorf("invalid container placement strategy %s", strategy) 53 } 54 } 55 return cps, nil 56 } 57 58 func (strategy *containerPlacementStrategy) Choose(logger lager.Logger, workers []Worker, spec ContainerSpec) (Worker, error) { 59 var err error 60 for _, node := range strategy.nodes { 61 workers, err = node.Choose(logger, workers, spec) 62 if err != nil { 63 return nil, err 64 } 65 if len(workers) == 0 { 66 return nil, nil 67 } 68 } 69 if len(workers) == 1 { 70 return workers[0], nil 71 } 72 73 // If there are still multiple candidate, choose a random one. 74 r := rand.New(rand.NewSource(time.Now().UnixNano())) 75 return workers[r.Intn(len(workers))], nil 76 } 77 78 func (strategy *containerPlacementStrategy) ModifiesActiveTasks() bool { 79 for _, node := range strategy.nodes { 80 if node.ModifiesActiveTasks() { 81 return true 82 } 83 } 84 return false 85 } 86 87 func NewRandomPlacementStrategy() ContainerPlacementStrategy { 88 s, _ := NewContainerPlacementStrategy(ContainerPlacementStrategyOptions{ContainerPlacementStrategy: []string{"random"}}) 89 return s 90 } 91 92 type VolumeLocalityPlacementStrategyNode struct{} 93 94 func newVolumeLocalityPlacementStrategyNode() ContainerPlacementStrategyChainNode { 95 return &VolumeLocalityPlacementStrategyNode{} 96 } 97 98 func (strategy *VolumeLocalityPlacementStrategyNode) Choose(logger lager.Logger, workers []Worker, spec ContainerSpec) ([]Worker, error) { 99 workersByCount := map[int][]Worker{} 100 var highestCount int 101 for _, w := range workers { 102 candidateInputCount := 0 103 104 for _, inputSource := range spec.Inputs { 105 _, found, err := inputSource.Source().ExistsOn(logger, w) 106 if err != nil { 107 return nil, err 108 } 109 110 if found { 111 candidateInputCount++ 112 } 113 } 114 115 workersByCount[candidateInputCount] = append(workersByCount[candidateInputCount], w) 116 117 if candidateInputCount >= highestCount { 118 highestCount = candidateInputCount 119 } 120 } 121 122 return workersByCount[highestCount], nil 123 } 124 125 func (strategy *VolumeLocalityPlacementStrategyNode) ModifiesActiveTasks() bool { 126 return false 127 } 128 129 type FewestBuildContainersPlacementStrategyNode struct{} 130 131 func newFewestBuildContainersPlacementStrategy() ContainerPlacementStrategyChainNode { 132 return &FewestBuildContainersPlacementStrategyNode{} 133 } 134 135 func (strategy *FewestBuildContainersPlacementStrategyNode) Choose(logger lager.Logger, workers []Worker, spec ContainerSpec) ([]Worker, error) { 136 workersByWork := map[int][]Worker{} 137 var minWork int 138 139 for i, w := range workers { 140 work := w.BuildContainers() 141 workersByWork[work] = append(workersByWork[work], w) 142 if i == 0 || work < minWork { 143 minWork = work 144 } 145 } 146 147 return workersByWork[minWork], nil 148 } 149 150 func (strategy *FewestBuildContainersPlacementStrategyNode) ModifiesActiveTasks() bool { 151 return false 152 } 153 154 type LimitActiveTasksPlacementStrategyNode struct { 155 maxTasks int 156 } 157 158 func newLimitActiveTasksPlacementStrategy(maxTasks int) ContainerPlacementStrategyChainNode { 159 return &LimitActiveTasksPlacementStrategyNode{ 160 maxTasks: maxTasks, 161 } 162 } 163 164 func (strategy *LimitActiveTasksPlacementStrategyNode) Choose(logger lager.Logger, workers []Worker, spec ContainerSpec) ([]Worker, error) { 165 workersByWork := map[int][]Worker{} 166 minActiveTasks := -1 167 168 for _, w := range workers { 169 activeTasks, err := w.ActiveTasks() 170 if err != nil { 171 logger.Error("Cannot retrive active tasks on worker. Skipping.", err) 172 continue 173 } 174 175 // If maxTasks == 0 or the step is not a task, ignore the number of active tasks and distribute the work evenly 176 if strategy.maxTasks > 0 && activeTasks >= strategy.maxTasks && spec.Type == db.ContainerTypeTask { 177 logger.Info("worker-busy") 178 continue 179 } 180 181 workersByWork[activeTasks] = append(workersByWork[activeTasks], w) 182 if minActiveTasks == -1 || activeTasks < minActiveTasks { 183 minActiveTasks = activeTasks 184 } 185 } 186 187 return workersByWork[minActiveTasks], nil 188 } 189 190 func (strategy *LimitActiveTasksPlacementStrategyNode) ModifiesActiveTasks() bool { 191 return true 192 }