github.com/anth0d/nomad@v0.0.0-20221214183521-ae3a0a2cad06/scheduler/stack_test.go (about) 1 package scheduler 2 3 import ( 4 "fmt" 5 "reflect" 6 "runtime" 7 "testing" 8 9 "github.com/hashicorp/nomad/ci" 10 "github.com/hashicorp/nomad/nomad/mock" 11 "github.com/hashicorp/nomad/nomad/structs" 12 "github.com/stretchr/testify/require" 13 ) 14 15 func BenchmarkServiceStack_With_ComputedClass(b *testing.B) { 16 // Key doesn't escape computed node class. 17 benchmarkServiceStack_MetaKeyConstraint(b, "key", 5000, 64) 18 } 19 20 func BenchmarkServiceStack_WithOut_ComputedClass(b *testing.B) { 21 // Key escapes computed node class. 22 benchmarkServiceStack_MetaKeyConstraint(b, "unique.key", 5000, 64) 23 } 24 25 // benchmarkServiceStack_MetaKeyConstraint creates the passed number of nodes 26 // and sets the meta data key to have nodePartitions number of values. It then 27 // benchmarks the stack by selecting a job that constrains against one of the 28 // partitions. 29 func benchmarkServiceStack_MetaKeyConstraint(b *testing.B, key string, numNodes, nodePartitions int) { 30 _, ctx := testContext(b) 31 stack := NewGenericStack(false, ctx) 32 33 // Create 4 classes of nodes. 34 nodes := make([]*structs.Node, numNodes) 35 for i := 0; i < numNodes; i++ { 36 n := mock.Node() 37 n.Meta[key] = fmt.Sprintf("%d", i%nodePartitions) 38 nodes[i] = n 39 } 40 stack.SetNodes(nodes) 41 42 // Create a job whose constraint meets two node classes. 43 job := mock.Job() 44 job.Constraints[0] = &structs.Constraint{ 45 LTarget: fmt.Sprintf("${meta.%v}", key), 46 RTarget: "1", 47 Operand: "<", 48 } 49 stack.SetJob(job) 50 51 b.ResetTimer() 52 selectOptions := &SelectOptions{} 53 for i := 0; i < b.N; i++ { 54 stack.Select(job.TaskGroups[0], selectOptions) 55 } 56 } 57 58 func TestServiceStack_SetNodes(t *testing.T) { 59 ci.Parallel(t) 60 61 _, ctx := testContext(t) 62 stack := NewGenericStack(false, ctx) 63 64 nodes := []*structs.Node{ 65 mock.Node(), 66 mock.Node(), 67 mock.Node(), 68 mock.Node(), 69 mock.Node(), 70 mock.Node(), 71 mock.Node(), 72 mock.Node(), 73 } 74 stack.SetNodes(nodes) 75 76 // Check that our scan limit is updated 77 if stack.limit.limit != 3 { 78 t.Fatalf("bad limit %d", stack.limit.limit) 79 } 80 81 out := collectFeasible(stack.source) 82 if !reflect.DeepEqual(out, nodes) { 83 t.Fatalf("bad: %#v", out) 84 } 85 } 86 87 func TestServiceStack_SetJob(t *testing.T) { 88 ci.Parallel(t) 89 90 _, ctx := testContext(t) 91 stack := NewGenericStack(false, ctx) 92 93 job := mock.Job() 94 stack.SetJob(job) 95 96 if stack.binPack.priority != job.Priority { 97 t.Fatalf("bad") 98 } 99 if !reflect.DeepEqual(stack.jobConstraint.constraints, job.Constraints) { 100 t.Fatalf("bad") 101 } 102 } 103 104 func TestServiceStack_Select_Size(t *testing.T) { 105 ci.Parallel(t) 106 107 _, ctx := testContext(t) 108 nodes := []*structs.Node{ 109 mock.Node(), 110 } 111 stack := NewGenericStack(false, ctx) 112 stack.SetNodes(nodes) 113 114 job := mock.Job() 115 stack.SetJob(job) 116 selectOptions := &SelectOptions{} 117 node := stack.Select(job.TaskGroups[0], selectOptions) 118 if node == nil { 119 t.Fatalf("missing node %#v", ctx.Metrics()) 120 } 121 122 // Note: On Windows time.Now currently has a best case granularity of 1ms. 123 // We skip the following assertion on Windows because this test usually 124 // runs too fast to measure an allocation time on Windows. 125 met := ctx.Metrics() 126 if runtime.GOOS != "windows" && met.AllocationTime == 0 { 127 t.Fatalf("missing time") 128 } 129 } 130 131 func TestServiceStack_Select_PreferringNodes(t *testing.T) { 132 ci.Parallel(t) 133 134 _, ctx := testContext(t) 135 nodes := []*structs.Node{ 136 mock.Node(), 137 } 138 stack := NewGenericStack(false, ctx) 139 stack.SetNodes(nodes) 140 141 job := mock.Job() 142 stack.SetJob(job) 143 144 // Create a preferred node 145 preferredNode := mock.Node() 146 prefNodes := []*structs.Node{preferredNode} 147 selectOptions := &SelectOptions{PreferredNodes: prefNodes} 148 option := stack.Select(job.TaskGroups[0], selectOptions) 149 if option == nil { 150 t.Fatalf("missing node %#v", ctx.Metrics()) 151 } 152 if option.Node.ID != preferredNode.ID { 153 t.Fatalf("expected: %v, actual: %v", option.Node.ID, preferredNode.ID) 154 } 155 156 // Make sure select doesn't have a side effect on preferred nodes 157 require.Equal(t, prefNodes, selectOptions.PreferredNodes) 158 159 // Change the preferred node's kernel to windows and ensure the allocations 160 // are placed elsewhere 161 preferredNode1 := preferredNode.Copy() 162 preferredNode1.Attributes["kernel.name"] = "windows" 163 preferredNode1.ComputeClass() 164 prefNodes1 := []*structs.Node{preferredNode1} 165 selectOptions = &SelectOptions{PreferredNodes: prefNodes1} 166 option = stack.Select(job.TaskGroups[0], selectOptions) 167 if option == nil { 168 t.Fatalf("missing node %#v", ctx.Metrics()) 169 } 170 171 if option.Node.ID != nodes[0].ID { 172 t.Fatalf("expected: %#v, actual: %#v", nodes[0], option.Node) 173 } 174 require.Equal(t, prefNodes1, selectOptions.PreferredNodes) 175 } 176 177 func TestServiceStack_Select_MetricsReset(t *testing.T) { 178 ci.Parallel(t) 179 180 _, ctx := testContext(t) 181 nodes := []*structs.Node{ 182 mock.Node(), 183 mock.Node(), 184 mock.Node(), 185 mock.Node(), 186 } 187 stack := NewGenericStack(false, ctx) 188 stack.SetNodes(nodes) 189 190 job := mock.Job() 191 stack.SetJob(job) 192 selectOptions := &SelectOptions{} 193 n1 := stack.Select(job.TaskGroups[0], selectOptions) 194 m1 := ctx.Metrics() 195 if n1 == nil { 196 t.Fatalf("missing node %#v", m1) 197 } 198 199 if m1.NodesEvaluated != 2 { 200 t.Fatalf("should only be 2") 201 } 202 203 n2 := stack.Select(job.TaskGroups[0], selectOptions) 204 m2 := ctx.Metrics() 205 if n2 == nil { 206 t.Fatalf("missing node %#v", m2) 207 } 208 209 // If we don't reset, this would be 4 210 if m2.NodesEvaluated != 2 { 211 t.Fatalf("should only be 2") 212 } 213 } 214 215 func TestServiceStack_Select_DriverFilter(t *testing.T) { 216 ci.Parallel(t) 217 218 _, ctx := testContext(t) 219 nodes := []*structs.Node{ 220 mock.Node(), 221 mock.Node(), 222 } 223 zero := nodes[0] 224 zero.Attributes["driver.foo"] = "1" 225 if err := zero.ComputeClass(); err != nil { 226 t.Fatalf("ComputedClass() failed: %v", err) 227 } 228 229 stack := NewGenericStack(false, ctx) 230 stack.SetNodes(nodes) 231 232 job := mock.Job() 233 job.TaskGroups[0].Tasks[0].Driver = "foo" 234 stack.SetJob(job) 235 236 selectOptions := &SelectOptions{} 237 node := stack.Select(job.TaskGroups[0], selectOptions) 238 if node == nil { 239 t.Fatalf("missing node %#v", ctx.Metrics()) 240 } 241 242 if node.Node != zero { 243 t.Fatalf("bad") 244 } 245 } 246 247 func TestServiceStack_Select_CSI(t *testing.T) { 248 ci.Parallel(t) 249 250 state, ctx := testContext(t) 251 nodes := []*structs.Node{ 252 mock.Node(), 253 mock.Node(), 254 } 255 256 // Create a volume in the state store 257 index := uint64(999) 258 v := structs.NewCSIVolume("foo[0]", index) 259 v.Namespace = structs.DefaultNamespace 260 v.AccessMode = structs.CSIVolumeAccessModeMultiNodeSingleWriter 261 v.AttachmentMode = structs.CSIVolumeAttachmentModeFilesystem 262 v.PluginID = "bar" 263 err := state.UpsertCSIVolume(999, []*structs.CSIVolume{v}) 264 require.NoError(t, err) 265 266 // Create a node with healthy fingerprints for both controller and node plugins 267 zero := nodes[0] 268 zero.CSIControllerPlugins = map[string]*structs.CSIInfo{"bar": { 269 PluginID: "bar", 270 Healthy: true, 271 RequiresTopologies: false, 272 ControllerInfo: &structs.CSIControllerInfo{ 273 SupportsReadOnlyAttach: true, 274 SupportsListVolumes: true, 275 }, 276 }} 277 zero.CSINodePlugins = map[string]*structs.CSIInfo{"bar": { 278 PluginID: "bar", 279 Healthy: true, 280 RequiresTopologies: false, 281 NodeInfo: &structs.CSINodeInfo{ 282 ID: zero.ID, 283 MaxVolumes: 2, 284 AccessibleTopology: nil, 285 RequiresNodeStageVolume: false, 286 }, 287 }} 288 289 // Add the node to the state store to index the healthy plugins and mark the volume "foo" healthy 290 err = state.UpsertNode(structs.MsgTypeTestSetup, 1000, zero) 291 require.NoError(t, err) 292 293 // Use the node to build the stack and test 294 if err := zero.ComputeClass(); err != nil { 295 t.Fatalf("ComputedClass() failed: %v", err) 296 } 297 298 stack := NewGenericStack(false, ctx) 299 stack.SetNodes(nodes) 300 301 job := mock.Job() 302 job.TaskGroups[0].Count = 2 303 job.TaskGroups[0].Volumes = map[string]*structs.VolumeRequest{"foo": { 304 Name: "bar", 305 Type: structs.VolumeTypeCSI, 306 Source: "foo", 307 ReadOnly: true, 308 PerAlloc: true, 309 }} 310 311 stack.SetJob(job) 312 313 selectOptions := &SelectOptions{ 314 AllocName: structs.AllocName(job.Name, job.TaskGroups[0].Name, 0)} 315 node := stack.Select(job.TaskGroups[0], selectOptions) 316 if node == nil { 317 t.Fatalf("missing node %#v", ctx.Metrics()) 318 } 319 320 if node.Node != zero { 321 t.Fatalf("bad") 322 } 323 } 324 325 func TestServiceStack_Select_ConstraintFilter(t *testing.T) { 326 ci.Parallel(t) 327 328 _, ctx := testContext(t) 329 nodes := []*structs.Node{ 330 mock.Node(), 331 mock.Node(), 332 } 333 zero := nodes[0] 334 zero.Attributes["kernel.name"] = "freebsd" 335 if err := zero.ComputeClass(); err != nil { 336 t.Fatalf("ComputedClass() failed: %v", err) 337 } 338 339 stack := NewGenericStack(false, ctx) 340 stack.SetNodes(nodes) 341 342 job := mock.Job() 343 job.Constraints[0].RTarget = "freebsd" 344 stack.SetJob(job) 345 selectOptions := &SelectOptions{} 346 node := stack.Select(job.TaskGroups[0], selectOptions) 347 if node == nil { 348 t.Fatalf("missing node %#v", ctx.Metrics()) 349 } 350 351 if node.Node != zero { 352 t.Fatalf("bad") 353 } 354 355 met := ctx.Metrics() 356 if met.NodesFiltered != 1 { 357 t.Fatalf("bad: %#v", met) 358 } 359 if met.ClassFiltered["linux-medium-pci"] != 1 { 360 t.Fatalf("bad: %#v", met) 361 } 362 if met.ConstraintFiltered["${attr.kernel.name} = freebsd"] != 1 { 363 t.Fatalf("bad: %#v", met) 364 } 365 } 366 367 func TestServiceStack_Select_BinPack_Overflow(t *testing.T) { 368 ci.Parallel(t) 369 370 _, ctx := testContext(t) 371 nodes := []*structs.Node{ 372 mock.Node(), 373 mock.Node(), 374 } 375 zero := nodes[0] 376 one := nodes[1] 377 one.ReservedResources = &structs.NodeReservedResources{ 378 Cpu: structs.NodeReservedCpuResources{ 379 CpuShares: one.NodeResources.Cpu.CpuShares, 380 }, 381 } 382 383 stack := NewGenericStack(false, ctx) 384 stack.SetNodes(nodes) 385 386 job := mock.Job() 387 stack.SetJob(job) 388 selectOptions := &SelectOptions{} 389 node := stack.Select(job.TaskGroups[0], selectOptions) 390 ctx.Metrics().PopulateScoreMetaData() 391 if node == nil { 392 t.Fatalf("missing node %#v", ctx.Metrics()) 393 } 394 395 if node.Node != zero { 396 t.Fatalf("bad") 397 } 398 399 met := ctx.Metrics() 400 if met.NodesExhausted != 1 { 401 t.Fatalf("bad: %#v", met) 402 } 403 if met.ClassExhausted["linux-medium-pci"] != 1 { 404 t.Fatalf("bad: %#v", met) 405 } 406 // Expect score metadata for one node 407 if len(met.ScoreMetaData) != 1 { 408 t.Fatalf("bad: %#v", met) 409 } 410 } 411 412 func TestSystemStack_SetNodes(t *testing.T) { 413 ci.Parallel(t) 414 415 _, ctx := testContext(t) 416 stack := NewSystemStack(false, ctx) 417 418 nodes := []*structs.Node{ 419 mock.Node(), 420 mock.Node(), 421 mock.Node(), 422 mock.Node(), 423 mock.Node(), 424 mock.Node(), 425 mock.Node(), 426 mock.Node(), 427 } 428 stack.SetNodes(nodes) 429 430 out := collectFeasible(stack.source) 431 if !reflect.DeepEqual(out, nodes) { 432 t.Fatalf("bad: %#v", out) 433 } 434 } 435 436 func TestSystemStack_SetJob(t *testing.T) { 437 ci.Parallel(t) 438 439 _, ctx := testContext(t) 440 stack := NewSystemStack(false, ctx) 441 442 job := mock.Job() 443 stack.SetJob(job) 444 445 if stack.binPack.priority != job.Priority { 446 t.Fatalf("bad") 447 } 448 if !reflect.DeepEqual(stack.jobConstraint.constraints, job.Constraints) { 449 t.Fatalf("bad") 450 } 451 } 452 453 func TestSystemStack_Select_Size(t *testing.T) { 454 ci.Parallel(t) 455 456 _, ctx := testContext(t) 457 nodes := []*structs.Node{mock.Node()} 458 stack := NewSystemStack(false, ctx) 459 stack.SetNodes(nodes) 460 461 job := mock.Job() 462 stack.SetJob(job) 463 selectOptions := &SelectOptions{} 464 node := stack.Select(job.TaskGroups[0], selectOptions) 465 if node == nil { 466 t.Fatalf("missing node %#v", ctx.Metrics()) 467 } 468 469 // Note: On Windows time.Now currently has a best case granularity of 1ms. 470 // We skip the following assertion on Windows because this test usually 471 // runs too fast to measure an allocation time on Windows. 472 met := ctx.Metrics() 473 if runtime.GOOS != "windows" && met.AllocationTime == 0 { 474 t.Fatalf("missing time") 475 } 476 } 477 478 func TestSystemStack_Select_MetricsReset(t *testing.T) { 479 ci.Parallel(t) 480 481 _, ctx := testContext(t) 482 nodes := []*structs.Node{ 483 mock.Node(), 484 mock.Node(), 485 mock.Node(), 486 mock.Node(), 487 } 488 stack := NewSystemStack(false, ctx) 489 stack.SetNodes(nodes) 490 491 job := mock.Job() 492 stack.SetJob(job) 493 selectOptions := &SelectOptions{} 494 n1 := stack.Select(job.TaskGroups[0], selectOptions) 495 m1 := ctx.Metrics() 496 if n1 == nil { 497 t.Fatalf("missing node %#v", m1) 498 } 499 500 if m1.NodesEvaluated != 1 { 501 t.Fatalf("should only be 1") 502 } 503 504 n2 := stack.Select(job.TaskGroups[0], selectOptions) 505 m2 := ctx.Metrics() 506 if n2 == nil { 507 t.Fatalf("missing node %#v", m2) 508 } 509 510 // If we don't reset, this would be 2 511 if m2.NodesEvaluated != 1 { 512 t.Fatalf("should only be 2") 513 } 514 } 515 516 func TestSystemStack_Select_DriverFilter(t *testing.T) { 517 ci.Parallel(t) 518 519 _, ctx := testContext(t) 520 nodes := []*structs.Node{ 521 mock.Node(), 522 } 523 zero := nodes[0] 524 zero.Attributes["driver.foo"] = "1" 525 526 stack := NewSystemStack(false, ctx) 527 stack.SetNodes(nodes) 528 529 job := mock.Job() 530 job.TaskGroups[0].Tasks[0].Driver = "foo" 531 stack.SetJob(job) 532 533 selectOptions := &SelectOptions{} 534 node := stack.Select(job.TaskGroups[0], selectOptions) 535 if node == nil { 536 t.Fatalf("missing node %#v", ctx.Metrics()) 537 } 538 539 if node.Node != zero { 540 t.Fatalf("bad") 541 } 542 543 zero.Attributes["driver.foo"] = "0" 544 if err := zero.ComputeClass(); err != nil { 545 t.Fatalf("ComputedClass() failed: %v", err) 546 } 547 548 stack = NewSystemStack(false, ctx) 549 stack.SetNodes(nodes) 550 stack.SetJob(job) 551 node = stack.Select(job.TaskGroups[0], selectOptions) 552 if node != nil { 553 t.Fatalf("node not filtered %#v", node) 554 } 555 } 556 557 func TestSystemStack_Select_ConstraintFilter(t *testing.T) { 558 ci.Parallel(t) 559 560 _, ctx := testContext(t) 561 nodes := []*structs.Node{ 562 mock.Node(), 563 mock.Node(), 564 } 565 zero := nodes[1] 566 zero.Attributes["kernel.name"] = "freebsd" 567 if err := zero.ComputeClass(); err != nil { 568 t.Fatalf("ComputedClass() failed: %v", err) 569 } 570 571 stack := NewSystemStack(false, ctx) 572 stack.SetNodes(nodes) 573 574 job := mock.Job() 575 job.Constraints[0].RTarget = "freebsd" 576 stack.SetJob(job) 577 578 selectOptions := &SelectOptions{} 579 node := stack.Select(job.TaskGroups[0], selectOptions) 580 if node == nil { 581 t.Fatalf("missing node %#v", ctx.Metrics()) 582 } 583 584 if node.Node != zero { 585 t.Fatalf("bad") 586 } 587 588 met := ctx.Metrics() 589 if met.NodesFiltered != 1 { 590 t.Fatalf("bad: %#v", met) 591 } 592 if met.ClassFiltered["linux-medium-pci"] != 1 { 593 t.Fatalf("bad: %#v", met) 594 } 595 if met.ConstraintFiltered["${attr.kernel.name} = freebsd"] != 1 { 596 t.Fatalf("bad: %#v", met) 597 } 598 } 599 600 func TestSystemStack_Select_BinPack_Overflow(t *testing.T) { 601 ci.Parallel(t) 602 603 _, ctx := testContext(t) 604 nodes := []*structs.Node{ 605 mock.Node(), 606 mock.Node(), 607 } 608 zero := nodes[0] 609 zero.ReservedResources = &structs.NodeReservedResources{ 610 Cpu: structs.NodeReservedCpuResources{ 611 CpuShares: zero.NodeResources.Cpu.CpuShares, 612 }, 613 } 614 one := nodes[1] 615 616 stack := NewSystemStack(false, ctx) 617 stack.SetNodes(nodes) 618 619 job := mock.Job() 620 stack.SetJob(job) 621 622 selectOptions := &SelectOptions{} 623 node := stack.Select(job.TaskGroups[0], selectOptions) 624 ctx.Metrics().PopulateScoreMetaData() 625 if node == nil { 626 t.Fatalf("missing node %#v", ctx.Metrics()) 627 } 628 629 if node.Node != one { 630 t.Fatalf("bad") 631 } 632 633 met := ctx.Metrics() 634 if met.NodesExhausted != 1 { 635 t.Fatalf("bad: %#v", met) 636 } 637 if met.ClassExhausted["linux-medium-pci"] != 1 { 638 t.Fatalf("bad: %#v", met) 639 } 640 // Should have two scores, one from bin packing and one from normalization 641 if len(met.ScoreMetaData) != 1 { 642 t.Fatalf("bad: %#v", met) 643 } 644 }