github.com/djenriquez/nomad-1@v0.8.1/scheduler/propertyset.go (about) 1 package scheduler 2 3 import ( 4 "fmt" 5 "strconv" 6 7 memdb "github.com/hashicorp/go-memdb" 8 "github.com/hashicorp/nomad/helper" 9 "github.com/hashicorp/nomad/nomad/structs" 10 ) 11 12 // propertySet is used to track the values used for a particular property. 13 type propertySet struct { 14 // ctx is used to lookup the plan and state 15 ctx Context 16 17 // jobID is the job we are operating on 18 jobID string 19 20 // namespace is the namespace of the job we are operating on 21 namespace string 22 23 // taskGroup is optionally set if the constraint is for a task group 24 taskGroup string 25 26 // constraint is the constraint this property set is checking 27 constraint *structs.Constraint 28 29 // allowedCount is the allowed number of allocations that can have the 30 // distinct property 31 allowedCount uint64 32 33 // errorBuilding marks whether there was an error when building the property 34 // set 35 errorBuilding error 36 37 // existingValues is a mapping of the values of a property to the number of 38 // times the value has been used by pre-existing allocations. 39 existingValues map[string]uint64 40 41 // proposedValues is a mapping of the values of a property to the number of 42 // times the value has been used by proposed allocations. 43 proposedValues map[string]uint64 44 45 // clearedValues is a mapping of the values of a property to the number of 46 // times the value has been used by proposed stopped allocations. 47 clearedValues map[string]uint64 48 } 49 50 // NewPropertySet returns a new property set used to guarantee unique property 51 // values for new allocation placements. 52 func NewPropertySet(ctx Context, job *structs.Job) *propertySet { 53 p := &propertySet{ 54 ctx: ctx, 55 jobID: job.ID, 56 namespace: job.Namespace, 57 existingValues: make(map[string]uint64), 58 } 59 60 return p 61 } 62 63 // SetJobConstraint is used to parameterize the property set for a 64 // distinct_property constraint set at the job level. 65 func (p *propertySet) SetJobConstraint(constraint *structs.Constraint) { 66 p.setConstraint(constraint, "") 67 } 68 69 // SetTGConstraint is used to parameterize the property set for a 70 // distinct_property constraint set at the task group level. The inputs are the 71 // constraint and the task group name. 72 func (p *propertySet) SetTGConstraint(constraint *structs.Constraint, taskGroup string) { 73 p.setConstraint(constraint, taskGroup) 74 } 75 76 // setConstraint is a shared helper for setting a job or task group constraint. 77 func (p *propertySet) setConstraint(constraint *structs.Constraint, taskGroup string) { 78 // Store that this is for a task group 79 if taskGroup != "" { 80 p.taskGroup = taskGroup 81 } 82 83 // Store the constraint 84 p.constraint = constraint 85 86 // Determine the number of allowed allocations with the property. 87 if v := constraint.RTarget; v != "" { 88 c, err := strconv.ParseUint(v, 10, 64) 89 if err != nil { 90 p.errorBuilding = fmt.Errorf("failed to convert RTarget %q to uint64: %v", v, err) 91 p.ctx.Logger().Printf("[ERR] scheduler.dynamic-constraint: %v", p.errorBuilding) 92 return 93 } 94 95 p.allowedCount = c 96 } else { 97 p.allowedCount = 1 98 } 99 100 // Determine the number of existing allocations that are using a property 101 // value 102 p.populateExisting(constraint) 103 104 // Populate the proposed when setting the constraint. We do this because 105 // when detecting if we can inplace update an allocation we stage an 106 // eviction and then select. This means the plan has an eviction before a 107 // single select has finished. 108 p.PopulateProposed() 109 } 110 111 // populateExisting is a helper shared when setting the constraint to populate 112 // the existing values. 113 func (p *propertySet) populateExisting(constraint *structs.Constraint) { 114 // Retrieve all previously placed allocations 115 ws := memdb.NewWatchSet() 116 allocs, err := p.ctx.State().AllocsByJob(ws, p.namespace, p.jobID, false) 117 if err != nil { 118 p.errorBuilding = fmt.Errorf("failed to get job's allocations: %v", err) 119 p.ctx.Logger().Printf("[ERR] scheduler.dynamic-constraint: %v", p.errorBuilding) 120 return 121 } 122 123 // Filter to the correct set of allocs 124 allocs = p.filterAllocs(allocs, true) 125 126 // Get all the nodes that have been used by the allocs 127 nodes, err := p.buildNodeMap(allocs) 128 if err != nil { 129 p.errorBuilding = err 130 p.ctx.Logger().Printf("[ERR] scheduler.dynamic-constraint: %v", err) 131 return 132 } 133 134 // Build existing properties map 135 p.populateProperties(allocs, nodes, p.existingValues) 136 } 137 138 // PopulateProposed populates the proposed values and recomputes any cleared 139 // value. It should be called whenever the plan is updated to ensure correct 140 // results when checking an option. 141 func (p *propertySet) PopulateProposed() { 142 143 // Reset the proposed properties 144 p.proposedValues = make(map[string]uint64) 145 p.clearedValues = make(map[string]uint64) 146 147 // Gather the set of proposed stops. 148 var stopping []*structs.Allocation 149 for _, updates := range p.ctx.Plan().NodeUpdate { 150 stopping = append(stopping, updates...) 151 } 152 stopping = p.filterAllocs(stopping, false) 153 154 // Gather the proposed allocations 155 var proposed []*structs.Allocation 156 for _, pallocs := range p.ctx.Plan().NodeAllocation { 157 proposed = append(proposed, pallocs...) 158 } 159 proposed = p.filterAllocs(proposed, true) 160 161 // Get the used nodes 162 both := make([]*structs.Allocation, 0, len(stopping)+len(proposed)) 163 both = append(both, stopping...) 164 both = append(both, proposed...) 165 nodes, err := p.buildNodeMap(both) 166 if err != nil { 167 p.errorBuilding = err 168 p.ctx.Logger().Printf("[ERR] scheduler.dynamic-constraint: %v", err) 169 return 170 } 171 172 // Populate the cleared values 173 p.populateProperties(stopping, nodes, p.clearedValues) 174 175 // Populate the proposed values 176 p.populateProperties(proposed, nodes, p.proposedValues) 177 178 // Remove any cleared value that is now being used by the proposed allocs 179 for value := range p.proposedValues { 180 current, ok := p.clearedValues[value] 181 if !ok { 182 continue 183 } else if current == 0 { 184 delete(p.clearedValues, value) 185 } else if current > 1 { 186 p.clearedValues[value]-- 187 } 188 } 189 } 190 191 // SatisfiesDistinctProperties checks if the option satisfies the 192 // distinct_property constraints given the existing placements and proposed 193 // placements. If the option does not satisfy the constraints an explanation is 194 // given. 195 func (p *propertySet) SatisfiesDistinctProperties(option *structs.Node, tg string) (bool, string) { 196 // Check if there was an error building 197 if p.errorBuilding != nil { 198 return false, p.errorBuilding.Error() 199 } 200 201 // Get the nodes property value 202 nValue, ok := getProperty(option, p.constraint.LTarget) 203 if !ok { 204 return false, fmt.Sprintf("missing property %q", p.constraint.LTarget) 205 } 206 207 // combine the counts of how many times the property has been used by 208 // existing and proposed allocations 209 combinedUse := make(map[string]uint64, helper.IntMax(len(p.existingValues), len(p.proposedValues))) 210 for _, usedValues := range []map[string]uint64{p.existingValues, p.proposedValues} { 211 for propertyValue, usedCount := range usedValues { 212 combinedUse[propertyValue] += usedCount 213 } 214 } 215 216 // Go through and discount the combined count when the value has been 217 // cleared by a proposed stop. 218 for propertyValue, clearedCount := range p.clearedValues { 219 combined, ok := combinedUse[propertyValue] 220 if !ok { 221 continue 222 } 223 224 // Don't clear below 0. 225 if combined >= clearedCount { 226 combinedUse[propertyValue] = combined - clearedCount 227 } else { 228 combinedUse[propertyValue] = 0 229 } 230 } 231 232 usedCount, used := combinedUse[nValue] 233 if !used { 234 // The property value has never been used so we can use it. 235 return true, "" 236 } 237 238 // The property value has been used but within the number of allowed 239 // allocations. 240 if usedCount < p.allowedCount { 241 return true, "" 242 } 243 244 return false, fmt.Sprintf("distinct_property: %s=%s used by %d allocs", p.constraint.LTarget, nValue, usedCount) 245 } 246 247 // filterAllocs filters a set of allocations to just be those that are running 248 // and if the property set is operation at a task group level, for allocations 249 // for that task group 250 func (p *propertySet) filterAllocs(allocs []*structs.Allocation, filterTerminal bool) []*structs.Allocation { 251 n := len(allocs) 252 for i := 0; i < n; i++ { 253 remove := false 254 if filterTerminal { 255 remove = allocs[i].TerminalStatus() 256 } 257 258 // If the constraint is on the task group filter the allocations to just 259 // those on the task group 260 if p.taskGroup != "" { 261 remove = remove || allocs[i].TaskGroup != p.taskGroup 262 } 263 264 if remove { 265 allocs[i], allocs[n-1] = allocs[n-1], nil 266 i-- 267 n-- 268 } 269 } 270 return allocs[:n] 271 } 272 273 // buildNodeMap takes a list of allocations and returns a map of the nodes used 274 // by those allocations 275 func (p *propertySet) buildNodeMap(allocs []*structs.Allocation) (map[string]*structs.Node, error) { 276 // Get all the nodes that have been used by the allocs 277 nodes := make(map[string]*structs.Node) 278 ws := memdb.NewWatchSet() 279 for _, alloc := range allocs { 280 if _, ok := nodes[alloc.NodeID]; ok { 281 continue 282 } 283 284 node, err := p.ctx.State().NodeByID(ws, alloc.NodeID) 285 if err != nil { 286 return nil, fmt.Errorf("failed to lookup node ID %q: %v", alloc.NodeID, err) 287 } 288 289 nodes[alloc.NodeID] = node 290 } 291 292 return nodes, nil 293 } 294 295 // populateProperties goes through all allocations and builds up the used 296 // properties from the nodes storing the results in the passed properties map. 297 func (p *propertySet) populateProperties(allocs []*structs.Allocation, nodes map[string]*structs.Node, 298 properties map[string]uint64) { 299 300 for _, alloc := range allocs { 301 nProperty, ok := getProperty(nodes[alloc.NodeID], p.constraint.LTarget) 302 if !ok { 303 continue 304 } 305 306 properties[nProperty]++ 307 } 308 } 309 310 // getProperty is used to lookup the property value on the node 311 func getProperty(n *structs.Node, property string) (string, bool) { 312 if n == nil || property == "" { 313 return "", false 314 } 315 316 val, ok := resolveConstraintTarget(property, n) 317 if !ok { 318 return "", false 319 } 320 nodeValue, ok := val.(string) 321 if !ok { 322 return "", false 323 } 324 325 return nodeValue, true 326 }