github.com/uchennaokeke444/nomad@v0.11.8/nomad/plan_apply_test.go (about) 1 package nomad 2 3 import ( 4 "reflect" 5 "testing" 6 "time" 7 8 memdb "github.com/hashicorp/go-memdb" 9 "github.com/hashicorp/nomad/helper/testlog" 10 "github.com/hashicorp/nomad/helper/uuid" 11 "github.com/hashicorp/nomad/nomad/mock" 12 "github.com/hashicorp/nomad/nomad/structs" 13 "github.com/hashicorp/nomad/testutil" 14 "github.com/hashicorp/raft" 15 "github.com/stretchr/testify/assert" 16 "github.com/stretchr/testify/require" 17 ) 18 19 const ( 20 // workerPoolSize is the size of the worker pool 21 workerPoolSize = 2 22 ) 23 24 // planWaitFuture is used to wait for the Raft future to complete 25 func planWaitFuture(future raft.ApplyFuture) (uint64, error) { 26 if err := future.Error(); err != nil { 27 return 0, err 28 } 29 return future.Index(), nil 30 } 31 32 func testRegisterNode(t *testing.T, s *Server, n *structs.Node) { 33 // Create the register request 34 req := &structs.NodeRegisterRequest{ 35 Node: n, 36 WriteRequest: structs.WriteRequest{Region: "global"}, 37 } 38 39 // Fetch the response 40 var resp structs.NodeUpdateResponse 41 if err := s.RPC("Node.Register", req, &resp); err != nil { 42 t.Fatalf("err: %v", err) 43 } 44 if resp.Index == 0 { 45 t.Fatalf("bad index: %d", resp.Index) 46 } 47 } 48 49 func testRegisterJob(t *testing.T, s *Server, j *structs.Job) { 50 // Create the register request 51 req := &structs.JobRegisterRequest{ 52 Job: j, 53 WriteRequest: structs.WriteRequest{Region: "global"}, 54 } 55 56 // Fetch the response 57 var resp structs.JobRegisterResponse 58 if err := s.RPC("Job.Register", req, &resp); err != nil { 59 t.Fatalf("err: %v", err) 60 } 61 if resp.Index == 0 { 62 t.Fatalf("bad index: %d", resp.Index) 63 } 64 } 65 66 // COMPAT 0.11: Tests the older unoptimized code path for applyPlan 67 func TestPlanApply_applyPlan(t *testing.T) { 68 t.Parallel() 69 70 s1, cleanupS1 := TestServer(t, nil) 71 defer cleanupS1() 72 testutil.WaitForLeader(t, s1.RPC) 73 74 // Register node 75 node := mock.Node() 76 testRegisterNode(t, s1, node) 77 78 // Register a fake deployment 79 oldDeployment := mock.Deployment() 80 if err := s1.State().UpsertDeployment(900, oldDeployment); err != nil { 81 t.Fatalf("UpsertDeployment failed: %v", err) 82 } 83 84 // Create a deployment 85 dnew := mock.Deployment() 86 87 // Create a deployment update for the old deployment id 88 desiredStatus, desiredStatusDescription := "foo", "bar" 89 updates := []*structs.DeploymentStatusUpdate{ 90 { 91 DeploymentID: oldDeployment.ID, 92 Status: desiredStatus, 93 StatusDescription: desiredStatusDescription, 94 }, 95 } 96 97 // Register alloc, deployment and deployment update 98 alloc := mock.Alloc() 99 s1.State().UpsertJobSummary(1000, mock.JobSummary(alloc.JobID)) 100 // Create an eval 101 eval := mock.Eval() 102 eval.JobID = alloc.JobID 103 if err := s1.State().UpsertEvals(1, []*structs.Evaluation{eval}); err != nil { 104 t.Fatalf("err: %v", err) 105 } 106 107 planRes := &structs.PlanResult{ 108 NodeAllocation: map[string][]*structs.Allocation{ 109 node.ID: {alloc}, 110 }, 111 Deployment: dnew, 112 DeploymentUpdates: updates, 113 } 114 115 // Snapshot the state 116 snap, err := s1.State().Snapshot() 117 if err != nil { 118 t.Fatalf("err: %v", err) 119 } 120 121 // Create the plan with a deployment 122 plan := &structs.Plan{ 123 Job: alloc.Job, 124 Deployment: dnew, 125 DeploymentUpdates: updates, 126 EvalID: eval.ID, 127 } 128 129 // Apply the plan 130 future, err := s1.applyPlan(plan, planRes, snap) 131 assert := assert.New(t) 132 assert.Nil(err) 133 134 // Verify our optimistic snapshot is updated 135 ws := memdb.NewWatchSet() 136 allocOut, err := snap.AllocByID(ws, alloc.ID) 137 assert.Nil(err) 138 assert.NotNil(allocOut) 139 140 deploymentOut, err := snap.DeploymentByID(ws, plan.Deployment.ID) 141 assert.Nil(err) 142 assert.NotNil(deploymentOut) 143 144 // Check plan does apply cleanly 145 index, err := planWaitFuture(future) 146 assert.Nil(err) 147 assert.NotEqual(0, index) 148 149 // Lookup the allocation 150 fsmState := s1.fsm.State() 151 allocOut, err = fsmState.AllocByID(ws, alloc.ID) 152 assert.Nil(err) 153 assert.NotNil(allocOut) 154 assert.True(allocOut.CreateTime > 0) 155 assert.True(allocOut.ModifyTime > 0) 156 assert.Equal(allocOut.CreateTime, allocOut.ModifyTime) 157 158 // Lookup the new deployment 159 dout, err := fsmState.DeploymentByID(ws, plan.Deployment.ID) 160 assert.Nil(err) 161 assert.NotNil(dout) 162 163 // Lookup the updated deployment 164 dout2, err := fsmState.DeploymentByID(ws, oldDeployment.ID) 165 assert.Nil(err) 166 assert.NotNil(dout2) 167 assert.Equal(desiredStatus, dout2.Status) 168 assert.Equal(desiredStatusDescription, dout2.StatusDescription) 169 170 // Lookup updated eval 171 evalOut, err := fsmState.EvalByID(ws, eval.ID) 172 assert.Nil(err) 173 assert.NotNil(evalOut) 174 assert.Equal(index, evalOut.ModifyIndex) 175 176 // Evict alloc, Register alloc2 177 allocEvict := new(structs.Allocation) 178 *allocEvict = *alloc 179 allocEvict.DesiredStatus = structs.AllocDesiredStatusEvict 180 job := allocEvict.Job 181 allocEvict.Job = nil 182 alloc2 := mock.Alloc() 183 s1.State().UpsertJobSummary(1500, mock.JobSummary(alloc2.JobID)) 184 planRes = &structs.PlanResult{ 185 NodeUpdate: map[string][]*structs.Allocation{ 186 node.ID: {allocEvict}, 187 }, 188 NodeAllocation: map[string][]*structs.Allocation{ 189 node.ID: {alloc2}, 190 }, 191 } 192 193 // Snapshot the state 194 snap, err = s1.State().Snapshot() 195 assert.Nil(err) 196 197 // Apply the plan 198 plan = &structs.Plan{ 199 Job: job, 200 EvalID: eval.ID, 201 } 202 future, err = s1.applyPlan(plan, planRes, snap) 203 assert.Nil(err) 204 205 // Check that our optimistic view is updated 206 out, _ := snap.AllocByID(ws, allocEvict.ID) 207 if out.DesiredStatus != structs.AllocDesiredStatusEvict && out.DesiredStatus != structs.AllocDesiredStatusStop { 208 assert.Equal(structs.AllocDesiredStatusEvict, out.DesiredStatus) 209 } 210 211 // Verify plan applies cleanly 212 index, err = planWaitFuture(future) 213 assert.Nil(err) 214 assert.NotEqual(0, index) 215 216 // Lookup the allocation 217 allocOut, err = s1.fsm.State().AllocByID(ws, alloc.ID) 218 assert.Nil(err) 219 if allocOut.DesiredStatus != structs.AllocDesiredStatusEvict && allocOut.DesiredStatus != structs.AllocDesiredStatusStop { 220 assert.Equal(structs.AllocDesiredStatusEvict, allocOut.DesiredStatus) 221 } 222 223 assert.NotNil(allocOut.Job) 224 assert.True(allocOut.ModifyTime > 0) 225 226 // Lookup the allocation 227 allocOut, err = s1.fsm.State().AllocByID(ws, alloc2.ID) 228 assert.Nil(err) 229 assert.NotNil(allocOut) 230 assert.NotNil(allocOut.Job) 231 232 // Lookup updated eval 233 evalOut, err = fsmState.EvalByID(ws, eval.ID) 234 assert.Nil(err) 235 assert.NotNil(evalOut) 236 assert.Equal(index, evalOut.ModifyIndex) 237 } 238 239 // Verifies that applyPlan properly updates the constituent objects in MemDB, 240 // when the plan contains normalized allocs. 241 func TestPlanApply_applyPlanWithNormalizedAllocs(t *testing.T) { 242 t.Parallel() 243 244 s1, cleanupS1 := TestServer(t, func(c *Config) { 245 c.Build = "0.9.2" 246 }) 247 defer cleanupS1() 248 testutil.WaitForLeader(t, s1.RPC) 249 250 // Register node 251 node := mock.Node() 252 testRegisterNode(t, s1, node) 253 254 // Register a fake deployment 255 oldDeployment := mock.Deployment() 256 if err := s1.State().UpsertDeployment(900, oldDeployment); err != nil { 257 t.Fatalf("UpsertDeployment failed: %v", err) 258 } 259 260 // Create a deployment 261 dnew := mock.Deployment() 262 263 // Create a deployment update for the old deployment id 264 desiredStatus, desiredStatusDescription := "foo", "bar" 265 updates := []*structs.DeploymentStatusUpdate{ 266 { 267 DeploymentID: oldDeployment.ID, 268 Status: desiredStatus, 269 StatusDescription: desiredStatusDescription, 270 }, 271 } 272 273 // Register allocs, deployment and deployment update 274 alloc := mock.Alloc() 275 stoppedAlloc := mock.Alloc() 276 stoppedAllocDiff := &structs.Allocation{ 277 ID: stoppedAlloc.ID, 278 DesiredDescription: "Desired Description", 279 ClientStatus: structs.AllocClientStatusLost, 280 } 281 preemptedAlloc := mock.Alloc() 282 preemptedAllocDiff := &structs.Allocation{ 283 ID: preemptedAlloc.ID, 284 PreemptedByAllocation: alloc.ID, 285 } 286 s1.State().UpsertJobSummary(1000, mock.JobSummary(alloc.JobID)) 287 s1.State().UpsertAllocs(1100, []*structs.Allocation{stoppedAlloc, preemptedAlloc}) 288 // Create an eval 289 eval := mock.Eval() 290 eval.JobID = alloc.JobID 291 if err := s1.State().UpsertEvals(1, []*structs.Evaluation{eval}); err != nil { 292 t.Fatalf("err: %v", err) 293 } 294 295 timestampBeforeCommit := time.Now().UTC().UnixNano() 296 planRes := &structs.PlanResult{ 297 NodeAllocation: map[string][]*structs.Allocation{ 298 node.ID: {alloc}, 299 }, 300 NodeUpdate: map[string][]*structs.Allocation{ 301 stoppedAlloc.NodeID: {stoppedAllocDiff}, 302 }, 303 NodePreemptions: map[string][]*structs.Allocation{ 304 preemptedAlloc.NodeID: {preemptedAllocDiff}, 305 }, 306 Deployment: dnew, 307 DeploymentUpdates: updates, 308 } 309 310 // Snapshot the state 311 snap, err := s1.State().Snapshot() 312 if err != nil { 313 t.Fatalf("err: %v", err) 314 } 315 316 // Create the plan with a deployment 317 plan := &structs.Plan{ 318 Job: alloc.Job, 319 Deployment: dnew, 320 DeploymentUpdates: updates, 321 EvalID: eval.ID, 322 } 323 324 require := require.New(t) 325 assert := assert.New(t) 326 327 // Apply the plan 328 future, err := s1.applyPlan(plan, planRes, snap) 329 require.NoError(err) 330 331 // Verify our optimistic snapshot is updated 332 ws := memdb.NewWatchSet() 333 allocOut, err := snap.AllocByID(ws, alloc.ID) 334 require.NoError(err) 335 require.NotNil(allocOut) 336 337 deploymentOut, err := snap.DeploymentByID(ws, plan.Deployment.ID) 338 require.NoError(err) 339 require.NotNil(deploymentOut) 340 341 // Check plan does apply cleanly 342 index, err := planWaitFuture(future) 343 require.NoError(err) 344 assert.NotEqual(0, index) 345 346 // Lookup the allocation 347 fsmState := s1.fsm.State() 348 allocOut, err = fsmState.AllocByID(ws, alloc.ID) 349 require.NoError(err) 350 require.NotNil(allocOut) 351 assert.True(allocOut.CreateTime > 0) 352 assert.True(allocOut.ModifyTime > 0) 353 assert.Equal(allocOut.CreateTime, allocOut.ModifyTime) 354 355 // Verify stopped alloc diff applied cleanly 356 updatedStoppedAlloc, err := fsmState.AllocByID(ws, stoppedAlloc.ID) 357 require.NoError(err) 358 require.NotNil(updatedStoppedAlloc) 359 assert.True(updatedStoppedAlloc.ModifyTime > timestampBeforeCommit) 360 assert.Equal(updatedStoppedAlloc.DesiredDescription, stoppedAllocDiff.DesiredDescription) 361 assert.Equal(updatedStoppedAlloc.ClientStatus, stoppedAllocDiff.ClientStatus) 362 assert.Equal(updatedStoppedAlloc.DesiredStatus, structs.AllocDesiredStatusStop) 363 364 // Verify preempted alloc diff applied cleanly 365 updatedPreemptedAlloc, err := fsmState.AllocByID(ws, preemptedAlloc.ID) 366 require.NoError(err) 367 require.NotNil(updatedPreemptedAlloc) 368 assert.True(updatedPreemptedAlloc.ModifyTime > timestampBeforeCommit) 369 assert.Equal(updatedPreemptedAlloc.DesiredDescription, 370 "Preempted by alloc ID "+preemptedAllocDiff.PreemptedByAllocation) 371 assert.Equal(updatedPreemptedAlloc.DesiredStatus, structs.AllocDesiredStatusEvict) 372 373 // Lookup the new deployment 374 dout, err := fsmState.DeploymentByID(ws, plan.Deployment.ID) 375 require.NoError(err) 376 require.NotNil(dout) 377 378 // Lookup the updated deployment 379 dout2, err := fsmState.DeploymentByID(ws, oldDeployment.ID) 380 require.NoError(err) 381 require.NotNil(dout2) 382 assert.Equal(desiredStatus, dout2.Status) 383 assert.Equal(desiredStatusDescription, dout2.StatusDescription) 384 385 // Lookup updated eval 386 evalOut, err := fsmState.EvalByID(ws, eval.ID) 387 require.NoError(err) 388 require.NotNil(evalOut) 389 assert.Equal(index, evalOut.ModifyIndex) 390 } 391 392 func TestPlanApply_EvalPlan_Simple(t *testing.T) { 393 t.Parallel() 394 state := testStateStore(t) 395 node := mock.Node() 396 state.UpsertNode(1000, node) 397 snap, _ := state.Snapshot() 398 399 alloc := mock.Alloc() 400 plan := &structs.Plan{ 401 Job: alloc.Job, 402 NodeAllocation: map[string][]*structs.Allocation{ 403 node.ID: {alloc}, 404 }, 405 Deployment: mock.Deployment(), 406 DeploymentUpdates: []*structs.DeploymentStatusUpdate{ 407 { 408 DeploymentID: uuid.Generate(), 409 Status: "foo", 410 StatusDescription: "bar", 411 }, 412 }, 413 } 414 415 pool := NewEvaluatePool(workerPoolSize, workerPoolBufferSize) 416 defer pool.Shutdown() 417 418 result, err := evaluatePlan(pool, snap, plan, testlog.HCLogger(t)) 419 if err != nil { 420 t.Fatalf("err: %v", err) 421 } 422 if result == nil { 423 t.Fatalf("missing result") 424 } 425 if !reflect.DeepEqual(result.NodeAllocation, plan.NodeAllocation) { 426 t.Fatalf("incorrect node allocations") 427 } 428 if !reflect.DeepEqual(result.Deployment, plan.Deployment) { 429 t.Fatalf("incorrect deployment") 430 } 431 if !reflect.DeepEqual(result.DeploymentUpdates, plan.DeploymentUpdates) { 432 t.Fatalf("incorrect deployment updates") 433 } 434 } 435 436 func TestPlanApply_EvalPlan_Preemption(t *testing.T) { 437 t.Parallel() 438 state := testStateStore(t) 439 node := mock.Node() 440 node.NodeResources = &structs.NodeResources{ 441 Cpu: structs.NodeCpuResources{ 442 CpuShares: 2000, 443 }, 444 Memory: structs.NodeMemoryResources{ 445 MemoryMB: 4192, 446 }, 447 Disk: structs.NodeDiskResources{ 448 DiskMB: 30 * 1024, 449 }, 450 Networks: []*structs.NetworkResource{ 451 { 452 Device: "eth0", 453 CIDR: "192.168.0.100/32", 454 MBits: 1000, 455 }, 456 }, 457 } 458 state.UpsertNode(1000, node) 459 460 preemptedAlloc := mock.Alloc() 461 preemptedAlloc.NodeID = node.ID 462 preemptedAlloc.AllocatedResources = &structs.AllocatedResources{ 463 Shared: structs.AllocatedSharedResources{ 464 DiskMB: 25 * 1024, 465 }, 466 Tasks: map[string]*structs.AllocatedTaskResources{ 467 "web": { 468 Cpu: structs.AllocatedCpuResources{ 469 CpuShares: 1500, 470 }, 471 Memory: structs.AllocatedMemoryResources{ 472 MemoryMB: 4000, 473 }, 474 Networks: []*structs.NetworkResource{ 475 { 476 Device: "eth0", 477 IP: "192.168.0.100", 478 ReservedPorts: []structs.Port{{Label: "admin", Value: 5000}}, 479 MBits: 800, 480 DynamicPorts: []structs.Port{{Label: "http", Value: 9876}}, 481 }, 482 }, 483 }, 484 }, 485 } 486 487 // Insert a preempted alloc such that the alloc will fit only after preemption 488 state.UpsertAllocs(1001, []*structs.Allocation{preemptedAlloc}) 489 490 alloc := mock.Alloc() 491 alloc.AllocatedResources = &structs.AllocatedResources{ 492 Shared: structs.AllocatedSharedResources{ 493 DiskMB: 24 * 1024, 494 }, 495 Tasks: map[string]*structs.AllocatedTaskResources{ 496 "web": { 497 Cpu: structs.AllocatedCpuResources{ 498 CpuShares: 1500, 499 }, 500 Memory: structs.AllocatedMemoryResources{ 501 MemoryMB: 3200, 502 }, 503 Networks: []*structs.NetworkResource{ 504 { 505 Device: "eth0", 506 IP: "192.168.0.100", 507 ReservedPorts: []structs.Port{{Label: "admin", Value: 5000}}, 508 MBits: 800, 509 DynamicPorts: []structs.Port{{Label: "http", Value: 9876}}, 510 }, 511 }, 512 }, 513 }, 514 } 515 plan := &structs.Plan{ 516 Job: alloc.Job, 517 NodeAllocation: map[string][]*structs.Allocation{ 518 node.ID: {alloc}, 519 }, 520 NodePreemptions: map[string][]*structs.Allocation{ 521 node.ID: {preemptedAlloc}, 522 }, 523 Deployment: mock.Deployment(), 524 DeploymentUpdates: []*structs.DeploymentStatusUpdate{ 525 { 526 DeploymentID: uuid.Generate(), 527 Status: "foo", 528 StatusDescription: "bar", 529 }, 530 }, 531 } 532 snap, _ := state.Snapshot() 533 534 pool := NewEvaluatePool(workerPoolSize, workerPoolBufferSize) 535 defer pool.Shutdown() 536 537 result, err := evaluatePlan(pool, snap, plan, testlog.HCLogger(t)) 538 539 require := require.New(t) 540 require.NoError(err) 541 require.NotNil(result) 542 543 require.Equal(result.NodeAllocation, plan.NodeAllocation) 544 require.Equal(result.Deployment, plan.Deployment) 545 require.Equal(result.DeploymentUpdates, plan.DeploymentUpdates) 546 require.Equal(result.NodePreemptions, plan.NodePreemptions) 547 548 } 549 550 func TestPlanApply_EvalPlan_Partial(t *testing.T) { 551 t.Parallel() 552 state := testStateStore(t) 553 node := mock.Node() 554 state.UpsertNode(1000, node) 555 node2 := mock.Node() 556 state.UpsertNode(1001, node2) 557 snap, _ := state.Snapshot() 558 559 alloc := mock.Alloc() 560 alloc2 := mock.Alloc() // Ensure alloc2 does not fit 561 alloc2.AllocatedResources = structs.NodeResourcesToAllocatedResources(node2.NodeResources) 562 563 // Create a deployment where the allocs are markeda as canaries 564 d := mock.Deployment() 565 d.TaskGroups["web"].PlacedCanaries = []string{alloc.ID, alloc2.ID} 566 567 plan := &structs.Plan{ 568 Job: alloc.Job, 569 NodeAllocation: map[string][]*structs.Allocation{ 570 node.ID: {alloc}, 571 node2.ID: {alloc2}, 572 }, 573 Deployment: d, 574 } 575 576 pool := NewEvaluatePool(workerPoolSize, workerPoolBufferSize) 577 defer pool.Shutdown() 578 579 result, err := evaluatePlan(pool, snap, plan, testlog.HCLogger(t)) 580 if err != nil { 581 t.Fatalf("err: %v", err) 582 } 583 if result == nil { 584 t.Fatalf("missing result") 585 } 586 587 if _, ok := result.NodeAllocation[node.ID]; !ok { 588 t.Fatalf("should allow alloc") 589 } 590 if _, ok := result.NodeAllocation[node2.ID]; ok { 591 t.Fatalf("should not allow alloc2") 592 } 593 594 // Check the deployment was updated 595 if result.Deployment == nil || len(result.Deployment.TaskGroups) == 0 { 596 t.Fatalf("bad: %v", result.Deployment) 597 } 598 placedCanaries := result.Deployment.TaskGroups["web"].PlacedCanaries 599 if len(placedCanaries) != 1 || placedCanaries[0] != alloc.ID { 600 t.Fatalf("bad: %v", placedCanaries) 601 } 602 603 if result.RefreshIndex != 1001 { 604 t.Fatalf("bad: %d", result.RefreshIndex) 605 } 606 } 607 608 func TestPlanApply_EvalPlan_Partial_AllAtOnce(t *testing.T) { 609 t.Parallel() 610 state := testStateStore(t) 611 node := mock.Node() 612 state.UpsertNode(1000, node) 613 node2 := mock.Node() 614 state.UpsertNode(1001, node2) 615 snap, _ := state.Snapshot() 616 617 alloc := mock.Alloc() 618 alloc2 := mock.Alloc() // Ensure alloc2 does not fit 619 alloc2.AllocatedResources = structs.NodeResourcesToAllocatedResources(node2.NodeResources) 620 plan := &structs.Plan{ 621 Job: alloc.Job, 622 AllAtOnce: true, // Require all to make progress 623 NodeAllocation: map[string][]*structs.Allocation{ 624 node.ID: {alloc}, 625 node2.ID: {alloc2}, 626 }, 627 Deployment: mock.Deployment(), 628 DeploymentUpdates: []*structs.DeploymentStatusUpdate{ 629 { 630 DeploymentID: uuid.Generate(), 631 Status: "foo", 632 StatusDescription: "bar", 633 }, 634 }, 635 } 636 637 pool := NewEvaluatePool(workerPoolSize, workerPoolBufferSize) 638 defer pool.Shutdown() 639 640 result, err := evaluatePlan(pool, snap, plan, testlog.HCLogger(t)) 641 if err != nil { 642 t.Fatalf("err: %v", err) 643 } 644 if result == nil { 645 t.Fatalf("missing result") 646 } 647 648 if len(result.NodeAllocation) != 0 { 649 t.Fatalf("should not alloc: %v", result.NodeAllocation) 650 } 651 if result.RefreshIndex != 1001 { 652 t.Fatalf("bad: %d", result.RefreshIndex) 653 } 654 if result.Deployment != nil || len(result.DeploymentUpdates) != 0 { 655 t.Fatalf("bad: %v", result) 656 } 657 } 658 659 func TestPlanApply_EvalNodePlan_Simple(t *testing.T) { 660 t.Parallel() 661 state := testStateStore(t) 662 node := mock.Node() 663 state.UpsertNode(1000, node) 664 snap, _ := state.Snapshot() 665 666 alloc := mock.Alloc() 667 plan := &structs.Plan{ 668 Job: alloc.Job, 669 NodeAllocation: map[string][]*structs.Allocation{ 670 node.ID: {alloc}, 671 }, 672 } 673 674 fit, reason, err := evaluateNodePlan(snap, plan, node.ID) 675 if err != nil { 676 t.Fatalf("err: %v", err) 677 } 678 if !fit { 679 t.Fatalf("bad") 680 } 681 if reason != "" { 682 t.Fatalf("bad") 683 } 684 } 685 686 func TestPlanApply_EvalNodePlan_NodeNotReady(t *testing.T) { 687 t.Parallel() 688 state := testStateStore(t) 689 node := mock.Node() 690 node.Status = structs.NodeStatusInit 691 state.UpsertNode(1000, node) 692 snap, _ := state.Snapshot() 693 694 alloc := mock.Alloc() 695 plan := &structs.Plan{ 696 Job: alloc.Job, 697 NodeAllocation: map[string][]*structs.Allocation{ 698 node.ID: {alloc}, 699 }, 700 } 701 702 fit, reason, err := evaluateNodePlan(snap, plan, node.ID) 703 if err != nil { 704 t.Fatalf("err: %v", err) 705 } 706 if fit { 707 t.Fatalf("bad") 708 } 709 if reason == "" { 710 t.Fatalf("bad") 711 } 712 } 713 714 func TestPlanApply_EvalNodePlan_NodeDrain(t *testing.T) { 715 t.Parallel() 716 state := testStateStore(t) 717 node := mock.Node() 718 node.Drain = true 719 state.UpsertNode(1000, node) 720 snap, _ := state.Snapshot() 721 722 alloc := mock.Alloc() 723 plan := &structs.Plan{ 724 Job: alloc.Job, 725 NodeAllocation: map[string][]*structs.Allocation{ 726 node.ID: {alloc}, 727 }, 728 } 729 730 fit, reason, err := evaluateNodePlan(snap, plan, node.ID) 731 if err != nil { 732 t.Fatalf("err: %v", err) 733 } 734 if fit { 735 t.Fatalf("bad") 736 } 737 if reason == "" { 738 t.Fatalf("bad") 739 } 740 } 741 742 func TestPlanApply_EvalNodePlan_NodeNotExist(t *testing.T) { 743 t.Parallel() 744 state := testStateStore(t) 745 snap, _ := state.Snapshot() 746 747 nodeID := "12345678-abcd-efab-cdef-123456789abc" 748 alloc := mock.Alloc() 749 plan := &structs.Plan{ 750 Job: alloc.Job, 751 NodeAllocation: map[string][]*structs.Allocation{ 752 nodeID: {alloc}, 753 }, 754 } 755 756 fit, reason, err := evaluateNodePlan(snap, plan, nodeID) 757 if err != nil { 758 t.Fatalf("err: %v", err) 759 } 760 if fit { 761 t.Fatalf("bad") 762 } 763 if reason == "" { 764 t.Fatalf("bad") 765 } 766 } 767 768 func TestPlanApply_EvalNodePlan_NodeFull(t *testing.T) { 769 t.Parallel() 770 alloc := mock.Alloc() 771 state := testStateStore(t) 772 node := mock.Node() 773 node.ReservedResources = nil 774 alloc.NodeID = node.ID 775 alloc.AllocatedResources = structs.NodeResourcesToAllocatedResources(node.NodeResources) 776 state.UpsertJobSummary(999, mock.JobSummary(alloc.JobID)) 777 state.UpsertNode(1000, node) 778 state.UpsertAllocs(1001, []*structs.Allocation{alloc}) 779 780 alloc2 := mock.Alloc() 781 alloc2.NodeID = node.ID 782 state.UpsertJobSummary(1200, mock.JobSummary(alloc2.JobID)) 783 784 snap, _ := state.Snapshot() 785 plan := &structs.Plan{ 786 Job: alloc.Job, 787 NodeAllocation: map[string][]*structs.Allocation{ 788 node.ID: {alloc2}, 789 }, 790 } 791 792 fit, reason, err := evaluateNodePlan(snap, plan, node.ID) 793 if err != nil { 794 t.Fatalf("err: %v", err) 795 } 796 if fit { 797 t.Fatalf("bad") 798 } 799 if reason == "" { 800 t.Fatalf("bad") 801 } 802 } 803 804 // Test that we detect device oversubscription 805 func TestPlanApply_EvalNodePlan_NodeFull_Device(t *testing.T) { 806 t.Parallel() 807 require := require.New(t) 808 alloc := mock.Alloc() 809 state := testStateStore(t) 810 node := mock.NvidiaNode() 811 node.ReservedResources = nil 812 813 nvidia0 := node.NodeResources.Devices[0].Instances[0].ID 814 815 // Have the allocation use a Nvidia device 816 alloc.NodeID = node.ID 817 alloc.AllocatedResources.Tasks["web"].Devices = []*structs.AllocatedDeviceResource{ 818 { 819 Type: "gpu", 820 Vendor: "nvidia", 821 Name: "1080ti", 822 DeviceIDs: []string{nvidia0}, 823 }, 824 } 825 826 state.UpsertJobSummary(999, mock.JobSummary(alloc.JobID)) 827 state.UpsertNode(1000, node) 828 state.UpsertAllocs(1001, []*structs.Allocation{alloc}) 829 830 // Alloc2 tries to use the same device 831 alloc2 := mock.Alloc() 832 alloc2.AllocatedResources.Tasks["web"].Networks = nil 833 alloc2.AllocatedResources.Tasks["web"].Devices = []*structs.AllocatedDeviceResource{ 834 { 835 Type: "gpu", 836 Vendor: "nvidia", 837 Name: "1080ti", 838 DeviceIDs: []string{nvidia0}, 839 }, 840 } 841 alloc2.NodeID = node.ID 842 state.UpsertJobSummary(1200, mock.JobSummary(alloc2.JobID)) 843 844 snap, _ := state.Snapshot() 845 plan := &structs.Plan{ 846 Job: alloc.Job, 847 NodeAllocation: map[string][]*structs.Allocation{ 848 node.ID: {alloc2}, 849 }, 850 } 851 852 fit, reason, err := evaluateNodePlan(snap, plan, node.ID) 853 require.NoError(err) 854 require.False(fit) 855 require.Equal("device oversubscribed", reason) 856 } 857 858 func TestPlanApply_EvalNodePlan_UpdateExisting(t *testing.T) { 859 t.Parallel() 860 alloc := mock.Alloc() 861 state := testStateStore(t) 862 node := mock.Node() 863 node.ReservedResources = nil 864 node.Reserved = nil 865 alloc.NodeID = node.ID 866 alloc.AllocatedResources = structs.NodeResourcesToAllocatedResources(node.NodeResources) 867 state.UpsertNode(1000, node) 868 state.UpsertAllocs(1001, []*structs.Allocation{alloc}) 869 snap, _ := state.Snapshot() 870 871 plan := &structs.Plan{ 872 Job: alloc.Job, 873 NodeAllocation: map[string][]*structs.Allocation{ 874 node.ID: {alloc}, 875 }, 876 } 877 878 fit, reason, err := evaluateNodePlan(snap, plan, node.ID) 879 if err != nil { 880 t.Fatalf("err: %v", err) 881 } 882 if !fit { 883 t.Fatalf("bad") 884 } 885 if reason != "" { 886 t.Fatalf("bad") 887 } 888 } 889 890 func TestPlanApply_EvalNodePlan_NodeFull_Evict(t *testing.T) { 891 t.Parallel() 892 alloc := mock.Alloc() 893 state := testStateStore(t) 894 node := mock.Node() 895 node.ReservedResources = nil 896 alloc.NodeID = node.ID 897 alloc.AllocatedResources = structs.NodeResourcesToAllocatedResources(node.NodeResources) 898 state.UpsertNode(1000, node) 899 state.UpsertAllocs(1001, []*structs.Allocation{alloc}) 900 snap, _ := state.Snapshot() 901 902 allocEvict := new(structs.Allocation) 903 *allocEvict = *alloc 904 allocEvict.DesiredStatus = structs.AllocDesiredStatusEvict 905 alloc2 := mock.Alloc() 906 plan := &structs.Plan{ 907 Job: alloc.Job, 908 NodeUpdate: map[string][]*structs.Allocation{ 909 node.ID: {allocEvict}, 910 }, 911 NodeAllocation: map[string][]*structs.Allocation{ 912 node.ID: {alloc2}, 913 }, 914 } 915 916 fit, reason, err := evaluateNodePlan(snap, plan, node.ID) 917 if err != nil { 918 t.Fatalf("err: %v", err) 919 } 920 if !fit { 921 t.Fatalf("bad") 922 } 923 if reason != "" { 924 t.Fatalf("bad") 925 } 926 } 927 928 func TestPlanApply_EvalNodePlan_NodeFull_AllocEvict(t *testing.T) { 929 t.Parallel() 930 alloc := mock.Alloc() 931 state := testStateStore(t) 932 node := mock.Node() 933 node.ReservedResources = nil 934 alloc.NodeID = node.ID 935 alloc.DesiredStatus = structs.AllocDesiredStatusEvict 936 alloc.AllocatedResources = structs.NodeResourcesToAllocatedResources(node.NodeResources) 937 state.UpsertNode(1000, node) 938 state.UpsertAllocs(1001, []*structs.Allocation{alloc}) 939 snap, _ := state.Snapshot() 940 941 alloc2 := mock.Alloc() 942 plan := &structs.Plan{ 943 Job: alloc.Job, 944 NodeAllocation: map[string][]*structs.Allocation{ 945 node.ID: {alloc2}, 946 }, 947 } 948 949 fit, reason, err := evaluateNodePlan(snap, plan, node.ID) 950 if err != nil { 951 t.Fatalf("err: %v", err) 952 } 953 if !fit { 954 t.Fatalf("bad") 955 } 956 if reason != "" { 957 t.Fatalf("bad") 958 } 959 } 960 961 func TestPlanApply_EvalNodePlan_NodeDown_EvictOnly(t *testing.T) { 962 t.Parallel() 963 alloc := mock.Alloc() 964 state := testStateStore(t) 965 node := mock.Node() 966 alloc.NodeID = node.ID 967 alloc.AllocatedResources = structs.NodeResourcesToAllocatedResources(node.NodeResources) 968 node.ReservedResources = nil 969 node.Status = structs.NodeStatusDown 970 state.UpsertNode(1000, node) 971 state.UpsertAllocs(1001, []*structs.Allocation{alloc}) 972 snap, _ := state.Snapshot() 973 974 allocEvict := new(structs.Allocation) 975 *allocEvict = *alloc 976 allocEvict.DesiredStatus = structs.AllocDesiredStatusEvict 977 plan := &structs.Plan{ 978 Job: alloc.Job, 979 NodeUpdate: map[string][]*structs.Allocation{ 980 node.ID: {allocEvict}, 981 }, 982 } 983 984 fit, reason, err := evaluateNodePlan(snap, plan, node.ID) 985 if err != nil { 986 t.Fatalf("err: %v", err) 987 } 988 if !fit { 989 t.Fatalf("bad") 990 } 991 if reason != "" { 992 t.Fatalf("bad") 993 } 994 }