github.com/adityamillind98/nomad@v0.11.8/scheduler/stack_test.go (about) 1 package scheduler 2 3 import ( 4 "fmt" 5 "reflect" 6 "runtime" 7 "testing" 8 9 "github.com/hashicorp/nomad/nomad/mock" 10 "github.com/hashicorp/nomad/nomad/structs" 11 "github.com/stretchr/testify/require" 12 ) 13 14 func BenchmarkServiceStack_With_ComputedClass(b *testing.B) { 15 // Key doesn't escape computed node class. 16 benchmarkServiceStack_MetaKeyConstraint(b, "key", 5000, 64) 17 } 18 19 func BenchmarkServiceStack_WithOut_ComputedClass(b *testing.B) { 20 // Key escapes computed node class. 21 benchmarkServiceStack_MetaKeyConstraint(b, "unique.key", 5000, 64) 22 } 23 24 // benchmarkServiceStack_MetaKeyConstraint creates the passed number of nodes 25 // and sets the meta data key to have nodePartitions number of values. It then 26 // benchmarks the stack by selecting a job that constrains against one of the 27 // partitions. 28 func benchmarkServiceStack_MetaKeyConstraint(b *testing.B, key string, numNodes, nodePartitions int) { 29 _, ctx := testContext(b) 30 stack := NewGenericStack(false, ctx) 31 32 // Create 4 classes of nodes. 33 nodes := make([]*structs.Node, numNodes) 34 for i := 0; i < numNodes; i++ { 35 n := mock.Node() 36 n.Meta[key] = fmt.Sprintf("%d", i%nodePartitions) 37 nodes[i] = n 38 } 39 stack.SetNodes(nodes) 40 41 // Create a job whose constraint meets two node classes. 42 job := mock.Job() 43 job.Constraints[0] = &structs.Constraint{ 44 LTarget: fmt.Sprintf("${meta.%v}", key), 45 RTarget: "1", 46 Operand: "<", 47 } 48 stack.SetJob(job) 49 50 b.ResetTimer() 51 selectOptions := &SelectOptions{} 52 for i := 0; i < b.N; i++ { 53 stack.Select(job.TaskGroups[0], selectOptions) 54 } 55 } 56 57 func TestServiceStack_SetNodes(t *testing.T) { 58 _, ctx := testContext(t) 59 stack := NewGenericStack(false, ctx) 60 61 nodes := []*structs.Node{ 62 mock.Node(), 63 mock.Node(), 64 mock.Node(), 65 mock.Node(), 66 mock.Node(), 67 mock.Node(), 68 mock.Node(), 69 mock.Node(), 70 } 71 stack.SetNodes(nodes) 72 73 // Check that our scan limit is updated 74 if stack.limit.limit != 3 { 75 t.Fatalf("bad limit %d", stack.limit.limit) 76 } 77 78 out := collectFeasible(stack.source) 79 if !reflect.DeepEqual(out, nodes) { 80 t.Fatalf("bad: %#v", out) 81 } 82 } 83 84 func TestServiceStack_SetJob(t *testing.T) { 85 _, ctx := testContext(t) 86 stack := NewGenericStack(false, ctx) 87 88 job := mock.Job() 89 stack.SetJob(job) 90 91 if stack.binPack.priority != job.Priority { 92 t.Fatalf("bad") 93 } 94 if !reflect.DeepEqual(stack.jobConstraint.constraints, job.Constraints) { 95 t.Fatalf("bad") 96 } 97 } 98 99 func TestServiceStack_Select_Size(t *testing.T) { 100 _, ctx := testContext(t) 101 nodes := []*structs.Node{ 102 mock.Node(), 103 } 104 stack := NewGenericStack(false, ctx) 105 stack.SetNodes(nodes) 106 107 job := mock.Job() 108 stack.SetJob(job) 109 selectOptions := &SelectOptions{} 110 node := stack.Select(job.TaskGroups[0], selectOptions) 111 if node == nil { 112 t.Fatalf("missing node %#v", ctx.Metrics()) 113 } 114 115 // Note: On Windows time.Now currently has a best case granularity of 1ms. 116 // We skip the following assertion on Windows because this test usually 117 // runs too fast to measure an allocation time on Windows. 118 met := ctx.Metrics() 119 if runtime.GOOS != "windows" && met.AllocationTime == 0 { 120 t.Fatalf("missing time") 121 } 122 } 123 124 func TestServiceStack_Select_PreferringNodes(t *testing.T) { 125 _, ctx := testContext(t) 126 nodes := []*structs.Node{ 127 mock.Node(), 128 } 129 stack := NewGenericStack(false, ctx) 130 stack.SetNodes(nodes) 131 132 job := mock.Job() 133 stack.SetJob(job) 134 135 // Create a preferred node 136 preferredNode := mock.Node() 137 prefNodes := []*structs.Node{preferredNode} 138 selectOptions := &SelectOptions{PreferredNodes: prefNodes} 139 option := stack.Select(job.TaskGroups[0], selectOptions) 140 if option == nil { 141 t.Fatalf("missing node %#v", ctx.Metrics()) 142 } 143 if option.Node.ID != preferredNode.ID { 144 t.Fatalf("expected: %v, actual: %v", option.Node.ID, preferredNode.ID) 145 } 146 147 // Make sure select doesn't have a side effect on preferred nodes 148 require.Equal(t, prefNodes, selectOptions.PreferredNodes) 149 150 // Change the preferred node's kernel to windows and ensure the allocations 151 // are placed elsewhere 152 preferredNode1 := preferredNode.Copy() 153 preferredNode1.Attributes["kernel.name"] = "windows" 154 preferredNode1.ComputeClass() 155 prefNodes1 := []*structs.Node{preferredNode1} 156 selectOptions = &SelectOptions{PreferredNodes: prefNodes1} 157 option = stack.Select(job.TaskGroups[0], selectOptions) 158 if option == nil { 159 t.Fatalf("missing node %#v", ctx.Metrics()) 160 } 161 162 if option.Node.ID != nodes[0].ID { 163 t.Fatalf("expected: %#v, actual: %#v", nodes[0], option.Node) 164 } 165 require.Equal(t, prefNodes1, selectOptions.PreferredNodes) 166 } 167 168 func TestServiceStack_Select_MetricsReset(t *testing.T) { 169 _, ctx := testContext(t) 170 nodes := []*structs.Node{ 171 mock.Node(), 172 mock.Node(), 173 mock.Node(), 174 mock.Node(), 175 } 176 stack := NewGenericStack(false, ctx) 177 stack.SetNodes(nodes) 178 179 job := mock.Job() 180 stack.SetJob(job) 181 selectOptions := &SelectOptions{} 182 n1 := stack.Select(job.TaskGroups[0], selectOptions) 183 m1 := ctx.Metrics() 184 if n1 == nil { 185 t.Fatalf("missing node %#v", m1) 186 } 187 188 if m1.NodesEvaluated != 2 { 189 t.Fatalf("should only be 2") 190 } 191 192 n2 := stack.Select(job.TaskGroups[0], selectOptions) 193 m2 := ctx.Metrics() 194 if n2 == nil { 195 t.Fatalf("missing node %#v", m2) 196 } 197 198 // If we don't reset, this would be 4 199 if m2.NodesEvaluated != 2 { 200 t.Fatalf("should only be 2") 201 } 202 } 203 204 func TestServiceStack_Select_DriverFilter(t *testing.T) { 205 _, ctx := testContext(t) 206 nodes := []*structs.Node{ 207 mock.Node(), 208 mock.Node(), 209 } 210 zero := nodes[0] 211 zero.Attributes["driver.foo"] = "1" 212 if err := zero.ComputeClass(); err != nil { 213 t.Fatalf("ComputedClass() failed: %v", err) 214 } 215 216 stack := NewGenericStack(false, ctx) 217 stack.SetNodes(nodes) 218 219 job := mock.Job() 220 job.TaskGroups[0].Tasks[0].Driver = "foo" 221 stack.SetJob(job) 222 223 selectOptions := &SelectOptions{} 224 node := stack.Select(job.TaskGroups[0], selectOptions) 225 if node == nil { 226 t.Fatalf("missing node %#v", ctx.Metrics()) 227 } 228 229 if node.Node != zero { 230 t.Fatalf("bad") 231 } 232 } 233 234 func TestServiceStack_Select_CSI(t *testing.T) { 235 state, ctx := testContext(t) 236 nodes := []*structs.Node{ 237 mock.Node(), 238 mock.Node(), 239 } 240 241 // Create a volume in the state store 242 index := uint64(999) 243 v := structs.NewCSIVolume("foo", index) 244 v.Namespace = structs.DefaultNamespace 245 v.AccessMode = structs.CSIVolumeAccessModeMultiNodeSingleWriter 246 v.AttachmentMode = structs.CSIVolumeAttachmentModeFilesystem 247 v.PluginID = "bar" 248 err := state.CSIVolumeRegister(999, []*structs.CSIVolume{v}) 249 require.NoError(t, err) 250 251 // Create a node with healthy fingerprints for both controller and node plugins 252 zero := nodes[0] 253 zero.CSIControllerPlugins = map[string]*structs.CSIInfo{"bar": { 254 PluginID: "bar", 255 Healthy: true, 256 RequiresTopologies: false, 257 ControllerInfo: &structs.CSIControllerInfo{ 258 SupportsReadOnlyAttach: true, 259 SupportsListVolumes: true, 260 }, 261 }} 262 zero.CSINodePlugins = map[string]*structs.CSIInfo{"bar": { 263 PluginID: "bar", 264 Healthy: true, 265 RequiresTopologies: false, 266 NodeInfo: &structs.CSINodeInfo{ 267 ID: zero.ID, 268 MaxVolumes: 2, 269 AccessibleTopology: nil, 270 RequiresNodeStageVolume: false, 271 }, 272 }} 273 274 // Add the node to the state store to index the healthy plugins and mark the volume "foo" healthy 275 err = state.UpsertNode(1000, zero) 276 require.NoError(t, err) 277 278 // Use the node to build the stack and test 279 if err := zero.ComputeClass(); err != nil { 280 t.Fatalf("ComputedClass() failed: %v", err) 281 } 282 283 stack := NewGenericStack(false, ctx) 284 stack.SetNodes(nodes) 285 286 job := mock.Job() 287 job.TaskGroups[0].Volumes = map[string]*structs.VolumeRequest{"foo": { 288 Name: "bar", 289 Type: structs.VolumeTypeCSI, 290 Source: "foo", 291 ReadOnly: true, 292 }} 293 294 stack.SetJob(job) 295 296 selectOptions := &SelectOptions{} 297 node := stack.Select(job.TaskGroups[0], selectOptions) 298 if node == nil { 299 t.Fatalf("missing node %#v", ctx.Metrics()) 300 } 301 302 if node.Node != zero { 303 t.Fatalf("bad") 304 } 305 } 306 307 func TestServiceStack_Select_ConstraintFilter(t *testing.T) { 308 _, ctx := testContext(t) 309 nodes := []*structs.Node{ 310 mock.Node(), 311 mock.Node(), 312 } 313 zero := nodes[0] 314 zero.Attributes["kernel.name"] = "freebsd" 315 if err := zero.ComputeClass(); err != nil { 316 t.Fatalf("ComputedClass() failed: %v", err) 317 } 318 319 stack := NewGenericStack(false, ctx) 320 stack.SetNodes(nodes) 321 322 job := mock.Job() 323 job.Constraints[0].RTarget = "freebsd" 324 stack.SetJob(job) 325 selectOptions := &SelectOptions{} 326 node := stack.Select(job.TaskGroups[0], selectOptions) 327 if node == nil { 328 t.Fatalf("missing node %#v", ctx.Metrics()) 329 } 330 331 if node.Node != zero { 332 t.Fatalf("bad") 333 } 334 335 met := ctx.Metrics() 336 if met.NodesFiltered != 1 { 337 t.Fatalf("bad: %#v", met) 338 } 339 if met.ClassFiltered["linux-medium-pci"] != 1 { 340 t.Fatalf("bad: %#v", met) 341 } 342 if met.ConstraintFiltered["${attr.kernel.name} = freebsd"] != 1 { 343 t.Fatalf("bad: %#v", met) 344 } 345 } 346 347 func TestServiceStack_Select_BinPack_Overflow(t *testing.T) { 348 _, ctx := testContext(t) 349 nodes := []*structs.Node{ 350 mock.Node(), 351 mock.Node(), 352 } 353 zero := nodes[0] 354 one := nodes[1] 355 one.ReservedResources = &structs.NodeReservedResources{ 356 Cpu: structs.NodeReservedCpuResources{ 357 CpuShares: one.NodeResources.Cpu.CpuShares, 358 }, 359 } 360 361 stack := NewGenericStack(false, ctx) 362 stack.SetNodes(nodes) 363 364 job := mock.Job() 365 stack.SetJob(job) 366 selectOptions := &SelectOptions{} 367 node := stack.Select(job.TaskGroups[0], selectOptions) 368 ctx.Metrics().PopulateScoreMetaData() 369 if node == nil { 370 t.Fatalf("missing node %#v", ctx.Metrics()) 371 } 372 373 if node.Node != zero { 374 t.Fatalf("bad") 375 } 376 377 met := ctx.Metrics() 378 if met.NodesExhausted != 1 { 379 t.Fatalf("bad: %#v", met) 380 } 381 if met.ClassExhausted["linux-medium-pci"] != 1 { 382 t.Fatalf("bad: %#v", met) 383 } 384 // Expect score metadata for one node 385 if len(met.ScoreMetaData) != 1 { 386 t.Fatalf("bad: %#v", met) 387 } 388 } 389 390 func TestSystemStack_SetNodes(t *testing.T) { 391 _, ctx := testContext(t) 392 stack := NewSystemStack(ctx) 393 394 nodes := []*structs.Node{ 395 mock.Node(), 396 mock.Node(), 397 mock.Node(), 398 mock.Node(), 399 mock.Node(), 400 mock.Node(), 401 mock.Node(), 402 mock.Node(), 403 } 404 stack.SetNodes(nodes) 405 406 out := collectFeasible(stack.source) 407 if !reflect.DeepEqual(out, nodes) { 408 t.Fatalf("bad: %#v", out) 409 } 410 } 411 412 func TestSystemStack_SetJob(t *testing.T) { 413 _, ctx := testContext(t) 414 stack := NewSystemStack(ctx) 415 416 job := mock.Job() 417 stack.SetJob(job) 418 419 if stack.binPack.priority != job.Priority { 420 t.Fatalf("bad") 421 } 422 if !reflect.DeepEqual(stack.jobConstraint.constraints, job.Constraints) { 423 t.Fatalf("bad") 424 } 425 } 426 427 func TestSystemStack_Select_Size(t *testing.T) { 428 _, ctx := testContext(t) 429 nodes := []*structs.Node{mock.Node()} 430 stack := NewSystemStack(ctx) 431 stack.SetNodes(nodes) 432 433 job := mock.Job() 434 stack.SetJob(job) 435 selectOptions := &SelectOptions{} 436 node := stack.Select(job.TaskGroups[0], selectOptions) 437 if node == nil { 438 t.Fatalf("missing node %#v", ctx.Metrics()) 439 } 440 441 // Note: On Windows time.Now currently has a best case granularity of 1ms. 442 // We skip the following assertion on Windows because this test usually 443 // runs too fast to measure an allocation time on Windows. 444 met := ctx.Metrics() 445 if runtime.GOOS != "windows" && met.AllocationTime == 0 { 446 t.Fatalf("missing time") 447 } 448 } 449 450 func TestSystemStack_Select_MetricsReset(t *testing.T) { 451 _, ctx := testContext(t) 452 nodes := []*structs.Node{ 453 mock.Node(), 454 mock.Node(), 455 mock.Node(), 456 mock.Node(), 457 } 458 stack := NewSystemStack(ctx) 459 stack.SetNodes(nodes) 460 461 job := mock.Job() 462 stack.SetJob(job) 463 selectOptions := &SelectOptions{} 464 n1 := stack.Select(job.TaskGroups[0], selectOptions) 465 m1 := ctx.Metrics() 466 if n1 == nil { 467 t.Fatalf("missing node %#v", m1) 468 } 469 470 if m1.NodesEvaluated != 1 { 471 t.Fatalf("should only be 1") 472 } 473 474 n2 := stack.Select(job.TaskGroups[0], selectOptions) 475 m2 := ctx.Metrics() 476 if n2 == nil { 477 t.Fatalf("missing node %#v", m2) 478 } 479 480 // If we don't reset, this would be 2 481 if m2.NodesEvaluated != 1 { 482 t.Fatalf("should only be 2") 483 } 484 } 485 486 func TestSystemStack_Select_DriverFilter(t *testing.T) { 487 _, ctx := testContext(t) 488 nodes := []*structs.Node{ 489 mock.Node(), 490 } 491 zero := nodes[0] 492 zero.Attributes["driver.foo"] = "1" 493 494 stack := NewSystemStack(ctx) 495 stack.SetNodes(nodes) 496 497 job := mock.Job() 498 job.TaskGroups[0].Tasks[0].Driver = "foo" 499 stack.SetJob(job) 500 501 selectOptions := &SelectOptions{} 502 node := stack.Select(job.TaskGroups[0], selectOptions) 503 if node == nil { 504 t.Fatalf("missing node %#v", ctx.Metrics()) 505 } 506 507 if node.Node != zero { 508 t.Fatalf("bad") 509 } 510 511 zero.Attributes["driver.foo"] = "0" 512 if err := zero.ComputeClass(); err != nil { 513 t.Fatalf("ComputedClass() failed: %v", err) 514 } 515 516 stack = NewSystemStack(ctx) 517 stack.SetNodes(nodes) 518 stack.SetJob(job) 519 node = stack.Select(job.TaskGroups[0], selectOptions) 520 if node != nil { 521 t.Fatalf("node not filtered %#v", node) 522 } 523 } 524 525 func TestSystemStack_Select_ConstraintFilter(t *testing.T) { 526 _, ctx := testContext(t) 527 nodes := []*structs.Node{ 528 mock.Node(), 529 mock.Node(), 530 } 531 zero := nodes[1] 532 zero.Attributes["kernel.name"] = "freebsd" 533 if err := zero.ComputeClass(); err != nil { 534 t.Fatalf("ComputedClass() failed: %v", err) 535 } 536 537 stack := NewSystemStack(ctx) 538 stack.SetNodes(nodes) 539 540 job := mock.Job() 541 job.Constraints[0].RTarget = "freebsd" 542 stack.SetJob(job) 543 544 selectOptions := &SelectOptions{} 545 node := stack.Select(job.TaskGroups[0], selectOptions) 546 if node == nil { 547 t.Fatalf("missing node %#v", ctx.Metrics()) 548 } 549 550 if node.Node != zero { 551 t.Fatalf("bad") 552 } 553 554 met := ctx.Metrics() 555 if met.NodesFiltered != 1 { 556 t.Fatalf("bad: %#v", met) 557 } 558 if met.ClassFiltered["linux-medium-pci"] != 1 { 559 t.Fatalf("bad: %#v", met) 560 } 561 if met.ConstraintFiltered["${attr.kernel.name} = freebsd"] != 1 { 562 t.Fatalf("bad: %#v", met) 563 } 564 } 565 566 func TestSystemStack_Select_BinPack_Overflow(t *testing.T) { 567 _, ctx := testContext(t) 568 nodes := []*structs.Node{ 569 mock.Node(), 570 mock.Node(), 571 } 572 zero := nodes[0] 573 zero.ReservedResources = &structs.NodeReservedResources{ 574 Cpu: structs.NodeReservedCpuResources{ 575 CpuShares: zero.NodeResources.Cpu.CpuShares, 576 }, 577 } 578 one := nodes[1] 579 580 stack := NewSystemStack(ctx) 581 stack.SetNodes(nodes) 582 583 job := mock.Job() 584 stack.SetJob(job) 585 586 selectOptions := &SelectOptions{} 587 node := stack.Select(job.TaskGroups[0], selectOptions) 588 ctx.Metrics().PopulateScoreMetaData() 589 if node == nil { 590 t.Fatalf("missing node %#v", ctx.Metrics()) 591 } 592 593 if node.Node != one { 594 t.Fatalf("bad") 595 } 596 597 met := ctx.Metrics() 598 if met.NodesExhausted != 1 { 599 t.Fatalf("bad: %#v", met) 600 } 601 if met.ClassExhausted["linux-medium-pci"] != 1 { 602 t.Fatalf("bad: %#v", met) 603 } 604 // Should have two scores, one from bin packing and one from normalization 605 if len(met.ScoreMetaData) != 1 { 606 t.Fatalf("bad: %#v", met) 607 } 608 }