github.com/adityamillind98/nomad@v0.11.8/scheduler/spread.go (about) 1 package scheduler 2 3 import ( 4 "github.com/hashicorp/nomad/nomad/structs" 5 ) 6 7 const ( 8 // implicitTarget is used to represent any remaining attribute values 9 // when target percentages don't add up to 100 10 implicitTarget = "*" 11 ) 12 13 // SpreadIterator is used to spread allocations across a specified attribute 14 // according to preset weights 15 type SpreadIterator struct { 16 ctx Context 17 source RankIterator 18 job *structs.Job 19 tg *structs.TaskGroup 20 21 // jobSpreads is a slice of spread stored at the job level which apply 22 // to all task groups 23 jobSpreads []*structs.Spread 24 25 // tgSpreadInfo is a map per task group with precomputed 26 // values for desired counts and weight 27 tgSpreadInfo map[string]spreadAttributeMap 28 29 // sumSpreadWeights tracks the total weight across all spread 30 // stanzas 31 sumSpreadWeights int32 32 33 // hasSpread is used to early return when the job/task group 34 // does not have spread configured 35 hasSpread bool 36 37 // groupProperySets is a memoized map from task group to property sets. 38 // existing allocs are computed once, and allocs from the plan are updated 39 // when Reset is called 40 groupPropertySets map[string][]*propertySet 41 } 42 43 type spreadAttributeMap map[string]*spreadInfo 44 45 type spreadInfo struct { 46 weight int8 47 desiredCounts map[string]float64 48 } 49 50 func NewSpreadIterator(ctx Context, source RankIterator) *SpreadIterator { 51 iter := &SpreadIterator{ 52 ctx: ctx, 53 source: source, 54 groupPropertySets: make(map[string][]*propertySet), 55 tgSpreadInfo: make(map[string]spreadAttributeMap), 56 } 57 return iter 58 } 59 60 func (iter *SpreadIterator) Reset() { 61 iter.source.Reset() 62 for _, sets := range iter.groupPropertySets { 63 for _, ps := range sets { 64 ps.PopulateProposed() 65 } 66 } 67 } 68 69 func (iter *SpreadIterator) SetJob(job *structs.Job) { 70 iter.job = job 71 if job.Spreads != nil { 72 iter.jobSpreads = job.Spreads 73 } 74 } 75 76 func (iter *SpreadIterator) SetTaskGroup(tg *structs.TaskGroup) { 77 iter.tg = tg 78 79 // Build the property set at the taskgroup level 80 if _, ok := iter.groupPropertySets[tg.Name]; !ok { 81 // First add property sets that are at the job level for this task group 82 for _, spread := range iter.jobSpreads { 83 pset := NewPropertySet(iter.ctx, iter.job) 84 pset.SetTargetAttribute(spread.Attribute, tg.Name) 85 iter.groupPropertySets[tg.Name] = append(iter.groupPropertySets[tg.Name], pset) 86 } 87 88 // Include property sets at the task group level 89 for _, spread := range tg.Spreads { 90 pset := NewPropertySet(iter.ctx, iter.job) 91 pset.SetTargetAttribute(spread.Attribute, tg.Name) 92 iter.groupPropertySets[tg.Name] = append(iter.groupPropertySets[tg.Name], pset) 93 } 94 } 95 96 // Check if there are any spreads configured 97 iter.hasSpread = len(iter.groupPropertySets[tg.Name]) != 0 98 99 // Build tgSpreadInfo at the task group level 100 if _, ok := iter.tgSpreadInfo[tg.Name]; !ok { 101 iter.computeSpreadInfo(tg) 102 } 103 104 } 105 106 func (iter *SpreadIterator) hasSpreads() bool { 107 return iter.hasSpread 108 } 109 110 func (iter *SpreadIterator) Next() *RankedNode { 111 for { 112 option := iter.source.Next() 113 114 // Hot path if there is nothing to check 115 if option == nil || !iter.hasSpreads() { 116 return option 117 } 118 119 tgName := iter.tg.Name 120 propertySets := iter.groupPropertySets[tgName] 121 // Iterate over each spread attribute's property set and add a weighted score 122 totalSpreadScore := 0.0 123 for _, pset := range propertySets { 124 nValue, errorMsg, usedCount := pset.UsedCount(option.Node, tgName) 125 126 // Add one to include placement on this node in the scoring calculation 127 usedCount += 1 128 // Set score to -1 if there were errors in building this attribute 129 if errorMsg != "" { 130 iter.ctx.Logger().Named("spread").Debug("error building spread attributes for task group", "task_group", tgName, "error", errorMsg) 131 totalSpreadScore -= 1.0 132 continue 133 } 134 spreadAttributeMap := iter.tgSpreadInfo[tgName] 135 spreadDetails := spreadAttributeMap[pset.targetAttribute] 136 137 if len(spreadDetails.desiredCounts) == 0 { 138 // When desired counts map is empty the user didn't specify any targets 139 // Use even spreading scoring algorithm for this scenario 140 scoreBoost := evenSpreadScoreBoost(pset, option.Node) 141 totalSpreadScore += scoreBoost 142 } else { 143 // Get the desired count 144 desiredCount, ok := spreadDetails.desiredCounts[nValue] 145 if !ok { 146 // See if there is an implicit target 147 desiredCount, ok = spreadDetails.desiredCounts[implicitTarget] 148 if !ok { 149 // The desired count for this attribute is zero if it gets here 150 // so use the maximum possible penalty for this node 151 totalSpreadScore -= 1.0 152 continue 153 } 154 } 155 156 // Calculate the relative weight of this specific spread attribute 157 spreadWeight := float64(spreadDetails.weight) / float64(iter.sumSpreadWeights) 158 159 // Score Boost is proportional the difference between current and desired count 160 // It is negative when the used count is greater than the desired count 161 // It is multiplied with the spread weight to account for cases where the job has 162 // more than one spread attribute 163 scoreBoost := ((desiredCount - float64(usedCount)) / desiredCount) * spreadWeight 164 totalSpreadScore += scoreBoost 165 } 166 } 167 168 if totalSpreadScore != 0.0 { 169 option.Scores = append(option.Scores, totalSpreadScore) 170 iter.ctx.Metrics().ScoreNode(option.Node, "allocation-spread", totalSpreadScore) 171 } 172 return option 173 } 174 } 175 176 // evenSpreadScoreBoost is a scoring helper that calculates the score 177 // for the option when even spread is desired (all attribute values get equal preference) 178 func evenSpreadScoreBoost(pset *propertySet, option *structs.Node) float64 { 179 combinedUseMap := pset.GetCombinedUseMap() 180 if len(combinedUseMap) == 0 { 181 // Nothing placed yet, so return 0 as the score 182 return 0.0 183 } 184 // Get the nodes property value 185 nValue, ok := getProperty(option, pset.targetAttribute) 186 187 // Maximum possible penalty when the attribute isn't set on the node 188 if !ok { 189 return -1.0 190 } 191 currentAttributeCount := combinedUseMap[nValue] 192 minCount := uint64(0) 193 maxCount := uint64(0) 194 for _, value := range combinedUseMap { 195 if minCount == 0 || value < minCount { 196 minCount = value 197 } 198 if maxCount == 0 || value > maxCount { 199 maxCount = value 200 } 201 } 202 203 // calculate boost based on delta between the current and the minimum 204 var deltaBoost float64 205 if minCount == 0 { 206 deltaBoost = -1.0 207 } else { 208 delta := int(minCount - currentAttributeCount) 209 deltaBoost = float64(delta) / float64(minCount) 210 } 211 if currentAttributeCount != minCount { 212 // Boost based on delta between current and min 213 return deltaBoost 214 } else if minCount == maxCount { 215 // Maximum possible penalty when the distribution is even 216 return -1.0 217 } else if minCount == 0 { 218 // Current attribute count is equal to min and both are zero. This means no allocations 219 // were placed for this attribute value yet. Should get the maximum possible boost. 220 return 1.0 221 } 222 223 // Penalty based on delta from max value 224 delta := int(maxCount - minCount) 225 deltaBoost = float64(delta) / float64(minCount) 226 return deltaBoost 227 228 } 229 230 // computeSpreadInfo computes and stores percentages and total values 231 // from all spreads that apply to a specific task group 232 func (iter *SpreadIterator) computeSpreadInfo(tg *structs.TaskGroup) { 233 spreadInfos := make(spreadAttributeMap, len(tg.Spreads)) 234 totalCount := tg.Count 235 236 // Always combine any spread stanzas defined at the job level here 237 combinedSpreads := make([]*structs.Spread, 0, len(tg.Spreads)+len(iter.jobSpreads)) 238 combinedSpreads = append(combinedSpreads, tg.Spreads...) 239 combinedSpreads = append(combinedSpreads, iter.jobSpreads...) 240 for _, spread := range combinedSpreads { 241 si := &spreadInfo{weight: spread.Weight, desiredCounts: make(map[string]float64)} 242 sumDesiredCounts := 0.0 243 for _, st := range spread.SpreadTarget { 244 desiredCount := (float64(st.Percent) / float64(100)) * float64(totalCount) 245 si.desiredCounts[st.Value] = desiredCount 246 sumDesiredCounts += desiredCount 247 } 248 // Account for remaining count only if there is any spread targets 249 if sumDesiredCounts > 0 && sumDesiredCounts < float64(totalCount) { 250 remainingCount := float64(totalCount) - sumDesiredCounts 251 si.desiredCounts[implicitTarget] = remainingCount 252 } 253 spreadInfos[spread.Attribute] = si 254 iter.sumSpreadWeights += int32(spread.Weight) 255 } 256 iter.tgSpreadInfo[tg.Name] = spreadInfos 257 }