github.com/emate/nomad@v0.8.2-wo-binpacking/scheduler/feasible.go (about) 1 package scheduler 2 3 import ( 4 "fmt" 5 "reflect" 6 "regexp" 7 "strconv" 8 "strings" 9 10 "github.com/hashicorp/go-version" 11 "github.com/hashicorp/nomad/nomad/structs" 12 ) 13 14 // FeasibleIterator is used to iteratively yield nodes that 15 // match feasibility constraints. The iterators may manage 16 // some state for performance optimizations. 17 type FeasibleIterator interface { 18 // Next yields a feasible node or nil if exhausted 19 Next() *structs.Node 20 21 // Reset is invoked when an allocation has been placed 22 // to reset any stale state. 23 Reset() 24 } 25 26 // JobContextualIterator is an iterator that can have the job and task group set 27 // on it. 28 type ContextualIterator interface { 29 SetJob(*structs.Job) 30 SetTaskGroup(*structs.TaskGroup) 31 } 32 33 // FeasibilityChecker is used to check if a single node meets feasibility 34 // constraints. 35 type FeasibilityChecker interface { 36 Feasible(*structs.Node) bool 37 } 38 39 // StaticIterator is a FeasibleIterator which returns nodes 40 // in a static order. This is used at the base of the iterator 41 // chain only for testing due to deterministic behavior. 42 type StaticIterator struct { 43 ctx Context 44 nodes []*structs.Node 45 offset int 46 seen int 47 } 48 49 // NewStaticIterator constructs a random iterator from a list of nodes 50 func NewStaticIterator(ctx Context, nodes []*structs.Node) *StaticIterator { 51 iter := &StaticIterator{ 52 ctx: ctx, 53 nodes: nodes, 54 } 55 return iter 56 } 57 58 func (iter *StaticIterator) Next() *structs.Node { 59 // Check if exhausted 60 n := len(iter.nodes) 61 if iter.offset == n || iter.seen == n { 62 if iter.seen != n { 63 iter.offset = 0 64 } else { 65 return nil 66 } 67 } 68 69 // Return the next offset 70 offset := iter.offset 71 iter.offset += 1 72 iter.seen += 1 73 iter.ctx.Metrics().EvaluateNode() 74 return iter.nodes[offset] 75 } 76 77 func (iter *StaticIterator) Reset() { 78 iter.seen = 0 79 } 80 81 func (iter *StaticIterator) SetNodes(nodes []*structs.Node) { 82 iter.nodes = nodes 83 iter.offset = 0 84 iter.seen = 0 85 } 86 87 // NewRandomIterator constructs a static iterator from a list of nodes 88 // after applying the Fisher-Yates algorithm for a random shuffle. This 89 // is applied in-place 90 func NewRandomIterator(ctx Context, nodes []*structs.Node) *StaticIterator { 91 // shuffle with the Fisher-Yates algorithm 92 shuffleNodes(nodes) 93 94 // Create a static iterator 95 return NewStaticIterator(ctx, nodes) 96 } 97 98 // DriverChecker is a FeasibilityChecker which returns whether a node has the 99 // drivers necessary to scheduler a task group. 100 type DriverChecker struct { 101 ctx Context 102 drivers map[string]struct{} 103 } 104 105 // NewDriverChecker creates a DriverChecker from a set of drivers 106 func NewDriverChecker(ctx Context, drivers map[string]struct{}) *DriverChecker { 107 return &DriverChecker{ 108 ctx: ctx, 109 drivers: drivers, 110 } 111 } 112 113 func (c *DriverChecker) SetDrivers(d map[string]struct{}) { 114 c.drivers = d 115 } 116 117 func (c *DriverChecker) Feasible(option *structs.Node) bool { 118 // Use this node if possible 119 if c.hasDrivers(option) { 120 return true 121 } 122 c.ctx.Metrics().FilterNode(option, "missing drivers") 123 return false 124 } 125 126 // hasDrivers is used to check if the node has all the appropriate 127 // drivers for this task group. Drivers are registered as node attribute 128 // like "driver.docker=1" with their corresponding version. 129 func (c *DriverChecker) hasDrivers(option *structs.Node) bool { 130 for driver := range c.drivers { 131 driverStr := fmt.Sprintf("driver.%s", driver) 132 133 // COMPAT: Remove in 0.10: As of Nomad 0.8, nodes have a DriverInfo that 134 // corresponds with every driver. As a Nomad server might be on a later 135 // version than a Nomad client, we need to check for compatibility here 136 // to verify the client supports this. 137 if driverInfo, ok := option.Drivers[driver]; ok { 138 if driverInfo == nil { 139 c.ctx.Logger(). 140 Printf("[WARN] scheduler.DriverChecker: node %v has no driver info set for %v", 141 option.ID, driver) 142 return false 143 } 144 145 return driverInfo.Detected && driverInfo.Healthy 146 } 147 148 value, ok := option.Attributes[driverStr] 149 if !ok { 150 return false 151 } 152 153 enabled, err := strconv.ParseBool(value) 154 if err != nil { 155 c.ctx.Logger(). 156 Printf("[WARN] scheduler.DriverChecker: node %v has invalid driver setting %v: %v", 157 option.ID, driverStr, value) 158 return false 159 } 160 161 if !enabled { 162 return false 163 } 164 } 165 return true 166 } 167 168 // DistinctHostsIterator is a FeasibleIterator which returns nodes that pass the 169 // distinct_hosts constraint. The constraint ensures that multiple allocations 170 // do not exist on the same node. 171 type DistinctHostsIterator struct { 172 ctx Context 173 source FeasibleIterator 174 tg *structs.TaskGroup 175 job *structs.Job 176 177 // Store whether the Job or TaskGroup has a distinct_hosts constraints so 178 // they don't have to be calculated every time Next() is called. 179 tgDistinctHosts bool 180 jobDistinctHosts bool 181 } 182 183 // NewDistinctHostsIterator creates a DistinctHostsIterator from a source. 184 func NewDistinctHostsIterator(ctx Context, source FeasibleIterator) *DistinctHostsIterator { 185 return &DistinctHostsIterator{ 186 ctx: ctx, 187 source: source, 188 } 189 } 190 191 func (iter *DistinctHostsIterator) SetTaskGroup(tg *structs.TaskGroup) { 192 iter.tg = tg 193 iter.tgDistinctHosts = iter.hasDistinctHostsConstraint(tg.Constraints) 194 } 195 196 func (iter *DistinctHostsIterator) SetJob(job *structs.Job) { 197 iter.job = job 198 iter.jobDistinctHosts = iter.hasDistinctHostsConstraint(job.Constraints) 199 } 200 201 func (iter *DistinctHostsIterator) hasDistinctHostsConstraint(constraints []*structs.Constraint) bool { 202 for _, con := range constraints { 203 if con.Operand == structs.ConstraintDistinctHosts { 204 return true 205 } 206 } 207 208 return false 209 } 210 211 func (iter *DistinctHostsIterator) Next() *structs.Node { 212 for { 213 // Get the next option from the source 214 option := iter.source.Next() 215 216 // Hot-path if the option is nil or there are no distinct_hosts or 217 // distinct_property constraints. 218 hosts := iter.jobDistinctHosts || iter.tgDistinctHosts 219 if option == nil || !hosts { 220 return option 221 } 222 223 // Check if the host constraints are satisfied 224 if !iter.satisfiesDistinctHosts(option) { 225 iter.ctx.Metrics().FilterNode(option, structs.ConstraintDistinctHosts) 226 continue 227 } 228 229 return option 230 } 231 } 232 233 // satisfiesDistinctHosts checks if the node satisfies a distinct_hosts 234 // constraint either specified at the job level or the TaskGroup level. 235 func (iter *DistinctHostsIterator) satisfiesDistinctHosts(option *structs.Node) bool { 236 // Check if there is no constraint set. 237 if !(iter.jobDistinctHosts || iter.tgDistinctHosts) { 238 return true 239 } 240 241 // Get the proposed allocations 242 proposed, err := iter.ctx.ProposedAllocs(option.ID) 243 if err != nil { 244 iter.ctx.Logger().Printf( 245 "[ERR] scheduler.dynamic-constraint: failed to get proposed allocations: %v", err) 246 return false 247 } 248 249 // Skip the node if the task group has already been allocated on it. 250 for _, alloc := range proposed { 251 // If the job has a distinct_hosts constraint we only need an alloc 252 // collision on the JobID but if the constraint is on the TaskGroup then 253 // we need both a job and TaskGroup collision. 254 jobCollision := alloc.JobID == iter.job.ID 255 taskCollision := alloc.TaskGroup == iter.tg.Name 256 if iter.jobDistinctHosts && jobCollision || jobCollision && taskCollision { 257 return false 258 } 259 } 260 261 return true 262 } 263 264 func (iter *DistinctHostsIterator) Reset() { 265 iter.source.Reset() 266 } 267 268 // DistinctPropertyIterator is a FeasibleIterator which returns nodes that pass the 269 // distinct_property constraint. The constraint ensures that multiple allocations 270 // do not use the same value of the given property. 271 type DistinctPropertyIterator struct { 272 ctx Context 273 source FeasibleIterator 274 tg *structs.TaskGroup 275 job *structs.Job 276 277 hasDistinctPropertyConstraints bool 278 jobPropertySets []*propertySet 279 groupPropertySets map[string][]*propertySet 280 } 281 282 // NewDistinctPropertyIterator creates a DistinctPropertyIterator from a source. 283 func NewDistinctPropertyIterator(ctx Context, source FeasibleIterator) *DistinctPropertyIterator { 284 return &DistinctPropertyIterator{ 285 ctx: ctx, 286 source: source, 287 groupPropertySets: make(map[string][]*propertySet), 288 } 289 } 290 291 func (iter *DistinctPropertyIterator) SetTaskGroup(tg *structs.TaskGroup) { 292 iter.tg = tg 293 294 // Build the property set at the taskgroup level 295 if _, ok := iter.groupPropertySets[tg.Name]; !ok { 296 for _, c := range tg.Constraints { 297 if c.Operand != structs.ConstraintDistinctProperty { 298 continue 299 } 300 301 pset := NewPropertySet(iter.ctx, iter.job) 302 pset.SetTGConstraint(c, tg.Name) 303 iter.groupPropertySets[tg.Name] = append(iter.groupPropertySets[tg.Name], pset) 304 } 305 } 306 307 // Check if there is a distinct property 308 iter.hasDistinctPropertyConstraints = len(iter.jobPropertySets) != 0 || len(iter.groupPropertySets[tg.Name]) != 0 309 } 310 311 func (iter *DistinctPropertyIterator) SetJob(job *structs.Job) { 312 iter.job = job 313 314 // Build the property set at the job level 315 for _, c := range job.Constraints { 316 if c.Operand != structs.ConstraintDistinctProperty { 317 continue 318 } 319 320 pset := NewPropertySet(iter.ctx, job) 321 pset.SetJobConstraint(c) 322 iter.jobPropertySets = append(iter.jobPropertySets, pset) 323 } 324 } 325 326 func (iter *DistinctPropertyIterator) Next() *structs.Node { 327 for { 328 // Get the next option from the source 329 option := iter.source.Next() 330 331 // Hot path if there is nothing to check 332 if option == nil || !iter.hasDistinctPropertyConstraints { 333 return option 334 } 335 336 // Check if the constraints are met 337 if !iter.satisfiesProperties(option, iter.jobPropertySets) || 338 !iter.satisfiesProperties(option, iter.groupPropertySets[iter.tg.Name]) { 339 continue 340 } 341 342 return option 343 } 344 } 345 346 // satisfiesProperties returns whether the option satisfies the set of 347 // properties. If not it will be filtered. 348 func (iter *DistinctPropertyIterator) satisfiesProperties(option *structs.Node, set []*propertySet) bool { 349 for _, ps := range set { 350 if satisfies, reason := ps.SatisfiesDistinctProperties(option, iter.tg.Name); !satisfies { 351 iter.ctx.Metrics().FilterNode(option, reason) 352 return false 353 } 354 } 355 356 return true 357 } 358 359 func (iter *DistinctPropertyIterator) Reset() { 360 iter.source.Reset() 361 362 for _, ps := range iter.jobPropertySets { 363 ps.PopulateProposed() 364 } 365 366 for _, sets := range iter.groupPropertySets { 367 for _, ps := range sets { 368 ps.PopulateProposed() 369 } 370 } 371 } 372 373 // ConstraintChecker is a FeasibilityChecker which returns nodes that match a 374 // given set of constraints. This is used to filter on job, task group, and task 375 // constraints. 376 type ConstraintChecker struct { 377 ctx Context 378 constraints []*structs.Constraint 379 } 380 381 // NewConstraintChecker creates a ConstraintChecker for a set of constraints 382 func NewConstraintChecker(ctx Context, constraints []*structs.Constraint) *ConstraintChecker { 383 return &ConstraintChecker{ 384 ctx: ctx, 385 constraints: constraints, 386 } 387 } 388 389 func (c *ConstraintChecker) SetConstraints(constraints []*structs.Constraint) { 390 c.constraints = constraints 391 } 392 393 func (c *ConstraintChecker) Feasible(option *structs.Node) bool { 394 // Use this node if possible 395 for _, constraint := range c.constraints { 396 if !c.meetsConstraint(constraint, option) { 397 c.ctx.Metrics().FilterNode(option, constraint.String()) 398 return false 399 } 400 } 401 return true 402 } 403 404 func (c *ConstraintChecker) meetsConstraint(constraint *structs.Constraint, option *structs.Node) bool { 405 // Resolve the targets 406 lVal, ok := resolveConstraintTarget(constraint.LTarget, option) 407 if !ok { 408 return false 409 } 410 rVal, ok := resolveConstraintTarget(constraint.RTarget, option) 411 if !ok { 412 return false 413 } 414 415 // Check if satisfied 416 return checkConstraint(c.ctx, constraint.Operand, lVal, rVal) 417 } 418 419 // resolveConstraintTarget is used to resolve the LTarget and RTarget of a Constraint 420 func resolveConstraintTarget(target string, node *structs.Node) (interface{}, bool) { 421 // If no prefix, this must be a literal value 422 if !strings.HasPrefix(target, "${") { 423 return target, true 424 } 425 426 // Handle the interpolations 427 switch { 428 case "${node.unique.id}" == target: 429 return node.ID, true 430 431 case "${node.datacenter}" == target: 432 return node.Datacenter, true 433 434 case "${node.unique.name}" == target: 435 return node.Name, true 436 437 case "${node.class}" == target: 438 return node.NodeClass, true 439 440 case strings.HasPrefix(target, "${attr."): 441 attr := strings.TrimSuffix(strings.TrimPrefix(target, "${attr."), "}") 442 val, ok := node.Attributes[attr] 443 return val, ok 444 445 case strings.HasPrefix(target, "${meta."): 446 meta := strings.TrimSuffix(strings.TrimPrefix(target, "${meta."), "}") 447 val, ok := node.Meta[meta] 448 return val, ok 449 450 default: 451 return nil, false 452 } 453 } 454 455 // checkConstraint checks if a constraint is satisfied 456 func checkConstraint(ctx Context, operand string, lVal, rVal interface{}) bool { 457 // Check for constraints not handled by this checker. 458 switch operand { 459 case structs.ConstraintDistinctHosts, structs.ConstraintDistinctProperty: 460 return true 461 default: 462 break 463 } 464 465 switch operand { 466 case "=", "==", "is": 467 return reflect.DeepEqual(lVal, rVal) 468 case "!=", "not": 469 return !reflect.DeepEqual(lVal, rVal) 470 case "<", "<=", ">", ">=": 471 return checkLexicalOrder(operand, lVal, rVal) 472 case structs.ConstraintVersion: 473 return checkVersionConstraint(ctx, lVal, rVal) 474 case structs.ConstraintRegex: 475 return checkRegexpConstraint(ctx, lVal, rVal) 476 case structs.ConstraintSetContains: 477 return checkSetContainsConstraint(ctx, lVal, rVal) 478 default: 479 return false 480 } 481 } 482 483 // checkLexicalOrder is used to check for lexical ordering 484 func checkLexicalOrder(op string, lVal, rVal interface{}) bool { 485 // Ensure the values are strings 486 lStr, ok := lVal.(string) 487 if !ok { 488 return false 489 } 490 rStr, ok := rVal.(string) 491 if !ok { 492 return false 493 } 494 495 switch op { 496 case "<": 497 return lStr < rStr 498 case "<=": 499 return lStr <= rStr 500 case ">": 501 return lStr > rStr 502 case ">=": 503 return lStr >= rStr 504 default: 505 return false 506 } 507 } 508 509 // checkVersionConstraint is used to compare a version on the 510 // left hand side with a set of constraints on the right hand side 511 func checkVersionConstraint(ctx Context, lVal, rVal interface{}) bool { 512 // Parse the version 513 var versionStr string 514 switch v := lVal.(type) { 515 case string: 516 versionStr = v 517 case int: 518 versionStr = fmt.Sprintf("%d", v) 519 default: 520 return false 521 } 522 523 // Parse the version 524 vers, err := version.NewVersion(versionStr) 525 if err != nil { 526 return false 527 } 528 529 // Constraint must be a string 530 constraintStr, ok := rVal.(string) 531 if !ok { 532 return false 533 } 534 535 // Check the cache for a match 536 cache := ctx.ConstraintCache() 537 constraints := cache[constraintStr] 538 539 // Parse the constraints 540 if constraints == nil { 541 constraints, err = version.NewConstraint(constraintStr) 542 if err != nil { 543 return false 544 } 545 cache[constraintStr] = constraints 546 } 547 548 // Check the constraints against the version 549 return constraints.Check(vers) 550 } 551 552 // checkRegexpConstraint is used to compare a value on the 553 // left hand side with a regexp on the right hand side 554 func checkRegexpConstraint(ctx Context, lVal, rVal interface{}) bool { 555 // Ensure left-hand is string 556 lStr, ok := lVal.(string) 557 if !ok { 558 return false 559 } 560 561 // Regexp must be a string 562 regexpStr, ok := rVal.(string) 563 if !ok { 564 return false 565 } 566 567 // Check the cache 568 cache := ctx.RegexpCache() 569 re := cache[regexpStr] 570 571 // Parse the regexp 572 if re == nil { 573 var err error 574 re, err = regexp.Compile(regexpStr) 575 if err != nil { 576 return false 577 } 578 cache[regexpStr] = re 579 } 580 581 // Look for a match 582 return re.MatchString(lStr) 583 } 584 585 // checkSetContainsConstraint is used to see if the left hand side contains the 586 // string on the right hand side 587 func checkSetContainsConstraint(ctx Context, lVal, rVal interface{}) bool { 588 // Ensure left-hand is string 589 lStr, ok := lVal.(string) 590 if !ok { 591 return false 592 } 593 594 // Regexp must be a string 595 rStr, ok := rVal.(string) 596 if !ok { 597 return false 598 } 599 600 input := strings.Split(lStr, ",") 601 lookup := make(map[string]struct{}, len(input)) 602 for _, in := range input { 603 cleaned := strings.TrimSpace(in) 604 lookup[cleaned] = struct{}{} 605 } 606 607 for _, r := range strings.Split(rStr, ",") { 608 cleaned := strings.TrimSpace(r) 609 if _, ok := lookup[cleaned]; !ok { 610 return false 611 } 612 } 613 614 return true 615 } 616 617 // FeasibilityWrapper is a FeasibleIterator which wraps both job and task group 618 // FeasibilityCheckers in which feasibility checking can be skipped if the 619 // computed node class has previously been marked as eligible or ineligible. 620 type FeasibilityWrapper struct { 621 ctx Context 622 source FeasibleIterator 623 jobCheckers []FeasibilityChecker 624 tgCheckers []FeasibilityChecker 625 tg string 626 } 627 628 // NewFeasibilityWrapper returns a FeasibleIterator based on the passed source 629 // and FeasibilityCheckers. 630 func NewFeasibilityWrapper(ctx Context, source FeasibleIterator, 631 jobCheckers, tgCheckers []FeasibilityChecker) *FeasibilityWrapper { 632 return &FeasibilityWrapper{ 633 ctx: ctx, 634 source: source, 635 jobCheckers: jobCheckers, 636 tgCheckers: tgCheckers, 637 } 638 } 639 640 func (w *FeasibilityWrapper) SetTaskGroup(tg string) { 641 w.tg = tg 642 } 643 644 func (w *FeasibilityWrapper) Reset() { 645 w.source.Reset() 646 } 647 648 // Next returns an eligible node, only running the FeasibilityCheckers as needed 649 // based on the sources computed node class. 650 func (w *FeasibilityWrapper) Next() *structs.Node { 651 evalElig := w.ctx.Eligibility() 652 metrics := w.ctx.Metrics() 653 654 OUTER: 655 for { 656 // Get the next option from the source 657 option := w.source.Next() 658 if option == nil { 659 return nil 660 } 661 662 // Check if the job has been marked as eligible or ineligible. 663 jobEscaped, jobUnknown := false, false 664 switch evalElig.JobStatus(option.ComputedClass) { 665 case EvalComputedClassIneligible: 666 // Fast path the ineligible case 667 metrics.FilterNode(option, "computed class ineligible") 668 continue 669 case EvalComputedClassEscaped: 670 jobEscaped = true 671 case EvalComputedClassUnknown: 672 jobUnknown = true 673 } 674 675 // Run the job feasibility checks. 676 for _, check := range w.jobCheckers { 677 feasible := check.Feasible(option) 678 if !feasible { 679 // If the job hasn't escaped, set it to be ineligible since it 680 // failed a job check. 681 if !jobEscaped { 682 evalElig.SetJobEligibility(false, option.ComputedClass) 683 } 684 continue OUTER 685 } 686 } 687 688 // Set the job eligibility if the constraints weren't escaped and it 689 // hasn't been set before. 690 if !jobEscaped && jobUnknown { 691 evalElig.SetJobEligibility(true, option.ComputedClass) 692 } 693 694 // Check if the task group has been marked as eligible or ineligible. 695 tgEscaped, tgUnknown := false, false 696 switch evalElig.TaskGroupStatus(w.tg, option.ComputedClass) { 697 case EvalComputedClassIneligible: 698 // Fast path the ineligible case 699 metrics.FilterNode(option, "computed class ineligible") 700 continue 701 case EvalComputedClassEligible: 702 // Fast path the eligible case 703 return option 704 case EvalComputedClassEscaped: 705 tgEscaped = true 706 case EvalComputedClassUnknown: 707 tgUnknown = true 708 } 709 710 // Run the task group feasibility checks. 711 for _, check := range w.tgCheckers { 712 feasible := check.Feasible(option) 713 if !feasible { 714 // If the task group hasn't escaped, set it to be ineligible 715 // since it failed a check. 716 if !tgEscaped { 717 evalElig.SetTaskGroupEligibility(false, w.tg, option.ComputedClass) 718 } 719 continue OUTER 720 } 721 } 722 723 // Set the task group eligibility if the constraints weren't escaped and 724 // it hasn't been set before. 725 if !tgEscaped && tgUnknown { 726 evalElig.SetTaskGroupEligibility(true, w.tg, option.ComputedClass) 727 } 728 729 return option 730 } 731 }