github.com/smithx10/nomad@v0.9.1-rc1/scheduler/propertyset.go (about) 1 package scheduler 2 3 import ( 4 "fmt" 5 "strconv" 6 7 log "github.com/hashicorp/go-hclog" 8 memdb "github.com/hashicorp/go-memdb" 9 "github.com/hashicorp/nomad/helper" 10 "github.com/hashicorp/nomad/nomad/structs" 11 ) 12 13 // propertySet is used to track the values used for a particular property. 14 type propertySet struct { 15 // ctx is used to lookup the plan and state 16 ctx Context 17 18 // logger is the logger for the property set 19 logger log.Logger 20 21 // jobID is the job we are operating on 22 jobID string 23 24 // namespace is the namespace of the job we are operating on 25 namespace string 26 27 // taskGroup is optionally set if the constraint is for a task group 28 taskGroup string 29 30 // targetAttribute is the attribute this property set is checking 31 targetAttribute string 32 33 // allowedCount is the allowed number of allocations that can have the 34 // distinct property 35 allowedCount uint64 36 37 // errorBuilding marks whether there was an error when building the property 38 // set 39 errorBuilding error 40 41 // existingValues is a mapping of the values of a property to the number of 42 // times the value has been used by pre-existing allocations. 43 existingValues map[string]uint64 44 45 // proposedValues is a mapping of the values of a property to the number of 46 // times the value has been used by proposed allocations. 47 proposedValues map[string]uint64 48 49 // clearedValues is a mapping of the values of a property to the number of 50 // times the value has been used by proposed stopped allocations. 51 clearedValues map[string]uint64 52 } 53 54 // NewPropertySet returns a new property set used to guarantee unique property 55 // values for new allocation placements. 56 func NewPropertySet(ctx Context, job *structs.Job) *propertySet { 57 p := &propertySet{ 58 ctx: ctx, 59 jobID: job.ID, 60 namespace: job.Namespace, 61 existingValues: make(map[string]uint64), 62 logger: ctx.Logger().Named("property_set"), 63 } 64 65 return p 66 } 67 68 // SetJobConstraint is used to parameterize the property set for a 69 // distinct_property constraint set at the job level. 70 func (p *propertySet) SetJobConstraint(constraint *structs.Constraint) { 71 p.setConstraint(constraint, "") 72 } 73 74 // SetTGConstraint is used to parameterize the property set for a 75 // distinct_property constraint set at the task group level. The inputs are the 76 // constraint and the task group name. 77 func (p *propertySet) SetTGConstraint(constraint *structs.Constraint, taskGroup string) { 78 p.setConstraint(constraint, taskGroup) 79 } 80 81 // setConstraint is a shared helper for setting a job or task group constraint. 82 func (p *propertySet) setConstraint(constraint *structs.Constraint, taskGroup string) { 83 var allowedCount uint64 84 // Determine the number of allowed allocations with the property. 85 if v := constraint.RTarget; v != "" { 86 c, err := strconv.ParseUint(v, 10, 64) 87 if err != nil { 88 p.errorBuilding = fmt.Errorf("failed to convert RTarget %q to uint64: %v", v, err) 89 p.logger.Error("failed to convert RTarget to uint64", "RTarget", v, "error", err) 90 return 91 } 92 93 allowedCount = c 94 } else { 95 allowedCount = 1 96 } 97 p.setTargetAttributeWithCount(constraint.LTarget, allowedCount, taskGroup) 98 } 99 100 // SetTargetAttribute is used to populate this property set without also storing allowed count 101 // This is used when evaluating spread stanzas 102 func (p *propertySet) SetTargetAttribute(targetAttribute string, taskGroup string) { 103 p.setTargetAttributeWithCount(targetAttribute, 0, taskGroup) 104 } 105 106 // setTargetAttributeWithCount is a shared helper for setting a job or task group attribute and allowedCount 107 // allowedCount can be zero when this is used in evaluating spread stanzas 108 func (p *propertySet) setTargetAttributeWithCount(targetAttribute string, allowedCount uint64, taskGroup string) { 109 // Store that this is for a task group 110 if taskGroup != "" { 111 p.taskGroup = taskGroup 112 } 113 114 // Store the constraint 115 p.targetAttribute = targetAttribute 116 117 p.allowedCount = allowedCount 118 119 // Determine the number of existing allocations that are using a property 120 // value 121 p.populateExisting() 122 123 // Populate the proposed when setting the constraint. We do this because 124 // when detecting if we can inplace update an allocation we stage an 125 // eviction and then select. This means the plan has an eviction before a 126 // single select has finished. 127 p.PopulateProposed() 128 } 129 130 // populateExisting is a helper shared when setting the constraint to populate 131 // the existing values. 132 func (p *propertySet) populateExisting() { 133 // Retrieve all previously placed allocations 134 ws := memdb.NewWatchSet() 135 allocs, err := p.ctx.State().AllocsByJob(ws, p.namespace, p.jobID, false) 136 if err != nil { 137 p.errorBuilding = fmt.Errorf("failed to get job's allocations: %v", err) 138 p.logger.Error("failed to get job's allocations", "job", p.jobID, "namespace", p.namespace, "error", err) 139 return 140 } 141 142 // Filter to the correct set of allocs 143 allocs = p.filterAllocs(allocs, true) 144 145 // Get all the nodes that have been used by the allocs 146 nodes, err := p.buildNodeMap(allocs) 147 if err != nil { 148 p.errorBuilding = err 149 p.logger.Error("failed to build node map", "error", err) 150 return 151 } 152 153 // Build existing properties map 154 p.populateProperties(allocs, nodes, p.existingValues) 155 } 156 157 // PopulateProposed populates the proposed values and recomputes any cleared 158 // value. It should be called whenever the plan is updated to ensure correct 159 // results when checking an option. 160 func (p *propertySet) PopulateProposed() { 161 162 // Reset the proposed properties 163 p.proposedValues = make(map[string]uint64) 164 p.clearedValues = make(map[string]uint64) 165 166 // Gather the set of proposed stops. 167 var stopping []*structs.Allocation 168 for _, updates := range p.ctx.Plan().NodeUpdate { 169 stopping = append(stopping, updates...) 170 } 171 stopping = p.filterAllocs(stopping, false) 172 173 // Gather the proposed allocations 174 var proposed []*structs.Allocation 175 for _, pallocs := range p.ctx.Plan().NodeAllocation { 176 proposed = append(proposed, pallocs...) 177 } 178 proposed = p.filterAllocs(proposed, true) 179 180 // Get the used nodes 181 both := make([]*structs.Allocation, 0, len(stopping)+len(proposed)) 182 both = append(both, stopping...) 183 both = append(both, proposed...) 184 nodes, err := p.buildNodeMap(both) 185 if err != nil { 186 p.errorBuilding = err 187 p.logger.Error("failed to build node map", "error", err) 188 return 189 } 190 191 // Populate the cleared values 192 p.populateProperties(stopping, nodes, p.clearedValues) 193 194 // Populate the proposed values 195 p.populateProperties(proposed, nodes, p.proposedValues) 196 197 // Remove any cleared value that is now being used by the proposed allocs 198 for value := range p.proposedValues { 199 current, ok := p.clearedValues[value] 200 if !ok { 201 continue 202 } else if current == 0 { 203 delete(p.clearedValues, value) 204 } else if current > 1 { 205 p.clearedValues[value]-- 206 } 207 } 208 } 209 210 // SatisfiesDistinctProperties checks if the option satisfies the 211 // distinct_property constraints given the existing placements and proposed 212 // placements. If the option does not satisfy the constraints an explanation is 213 // given. 214 func (p *propertySet) SatisfiesDistinctProperties(option *structs.Node, tg string) (bool, string) { 215 nValue, errorMsg, usedCount := p.UsedCount(option, tg) 216 if errorMsg != "" { 217 return false, errorMsg 218 } 219 // The property value has been used but within the number of allowed 220 // allocations. 221 if usedCount < p.allowedCount { 222 return true, "" 223 } 224 225 return false, fmt.Sprintf("distinct_property: %s=%s used by %d allocs", p.targetAttribute, nValue, usedCount) 226 } 227 228 // UsedCount returns the number of times the value of the attribute being tracked by this 229 // property set is used across current and proposed allocations. It also returns the resolved 230 // attribute value for the node, and an error message if it couldn't be resolved correctly 231 func (p *propertySet) UsedCount(option *structs.Node, tg string) (string, string, uint64) { 232 // Check if there was an error building 233 if p.errorBuilding != nil { 234 return "", p.errorBuilding.Error(), 0 235 } 236 237 // Get the nodes property value 238 nValue, ok := getProperty(option, p.targetAttribute) 239 if !ok { 240 return nValue, fmt.Sprintf("missing property %q", p.targetAttribute), 0 241 } 242 combinedUse := p.GetCombinedUseMap() 243 usedCount := combinedUse[nValue] 244 return nValue, "", usedCount 245 } 246 247 // GetCombinedUseMap counts how many times the property has been used by 248 // existing and proposed allocations. It also takes into account any stopped 249 // allocations 250 func (p *propertySet) GetCombinedUseMap() map[string]uint64 { 251 combinedUse := make(map[string]uint64, helper.IntMax(len(p.existingValues), len(p.proposedValues))) 252 for _, usedValues := range []map[string]uint64{p.existingValues, p.proposedValues} { 253 for propertyValue, usedCount := range usedValues { 254 combinedUse[propertyValue] += usedCount 255 } 256 } 257 258 // Go through and discount the combined count when the value has been 259 // cleared by a proposed stop. 260 for propertyValue, clearedCount := range p.clearedValues { 261 combined, ok := combinedUse[propertyValue] 262 if !ok { 263 continue 264 } 265 266 // Don't clear below 0. 267 if combined >= clearedCount { 268 combinedUse[propertyValue] = combined - clearedCount 269 } else { 270 combinedUse[propertyValue] = 0 271 } 272 } 273 return combinedUse 274 } 275 276 // filterAllocs filters a set of allocations to just be those that are running 277 // and if the property set is operation at a task group level, for allocations 278 // for that task group 279 func (p *propertySet) filterAllocs(allocs []*structs.Allocation, filterTerminal bool) []*structs.Allocation { 280 n := len(allocs) 281 for i := 0; i < n; i++ { 282 remove := false 283 if filterTerminal { 284 remove = allocs[i].TerminalStatus() 285 } 286 287 // If the constraint is on the task group filter the allocations to just 288 // those on the task group 289 if p.taskGroup != "" { 290 remove = remove || allocs[i].TaskGroup != p.taskGroup 291 } 292 293 if remove { 294 allocs[i], allocs[n-1] = allocs[n-1], nil 295 i-- 296 n-- 297 } 298 } 299 return allocs[:n] 300 } 301 302 // buildNodeMap takes a list of allocations and returns a map of the nodes used 303 // by those allocations 304 func (p *propertySet) buildNodeMap(allocs []*structs.Allocation) (map[string]*structs.Node, error) { 305 // Get all the nodes that have been used by the allocs 306 nodes := make(map[string]*structs.Node) 307 ws := memdb.NewWatchSet() 308 for _, alloc := range allocs { 309 if _, ok := nodes[alloc.NodeID]; ok { 310 continue 311 } 312 313 node, err := p.ctx.State().NodeByID(ws, alloc.NodeID) 314 if err != nil { 315 return nil, fmt.Errorf("failed to lookup node ID %q: %v", alloc.NodeID, err) 316 } 317 318 nodes[alloc.NodeID] = node 319 } 320 321 return nodes, nil 322 } 323 324 // populateProperties goes through all allocations and builds up the used 325 // properties from the nodes storing the results in the passed properties map. 326 func (p *propertySet) populateProperties(allocs []*structs.Allocation, nodes map[string]*structs.Node, 327 properties map[string]uint64) { 328 329 for _, alloc := range allocs { 330 nProperty, ok := getProperty(nodes[alloc.NodeID], p.targetAttribute) 331 if !ok { 332 continue 333 } 334 335 properties[nProperty]++ 336 } 337 } 338 339 // getProperty is used to lookup the property value on the node 340 func getProperty(n *structs.Node, property string) (string, bool) { 341 if n == nil || property == "" { 342 return "", false 343 } 344 345 val, ok := resolveTarget(property, n) 346 if !ok { 347 return "", false 348 } 349 nodeValue, ok := val.(string) 350 if !ok { 351 return "", false 352 } 353 354 return nodeValue, true 355 }