github.com/diptanu/nomad@v0.5.7-0.20170516172507-d72e86cbe3d9/nomad/fsm_test.go (about) 1 package nomad 2 3 import ( 4 "bytes" 5 "fmt" 6 "os" 7 "reflect" 8 "testing" 9 "time" 10 11 memdb "github.com/hashicorp/go-memdb" 12 "github.com/hashicorp/nomad/nomad/mock" 13 "github.com/hashicorp/nomad/nomad/state" 14 "github.com/hashicorp/nomad/nomad/structs" 15 "github.com/hashicorp/nomad/testutil" 16 "github.com/hashicorp/raft" 17 ) 18 19 type MockSink struct { 20 *bytes.Buffer 21 cancel bool 22 } 23 24 func (m *MockSink) ID() string { 25 return "Mock" 26 } 27 28 func (m *MockSink) Cancel() error { 29 m.cancel = true 30 return nil 31 } 32 33 func (m *MockSink) Close() error { 34 return nil 35 } 36 37 func testStateStore(t *testing.T) *state.StateStore { 38 state, err := state.NewStateStore(os.Stderr) 39 if err != nil { 40 t.Fatalf("err: %v", err) 41 } 42 if state == nil { 43 t.Fatalf("missing state") 44 } 45 return state 46 } 47 48 func testFSM(t *testing.T) *nomadFSM { 49 p, _ := testPeriodicDispatcher() 50 broker := testBroker(t, 0) 51 blocked := NewBlockedEvals(broker) 52 fsm, err := NewFSM(broker, p, blocked, os.Stderr) 53 if err != nil { 54 t.Fatalf("err: %v", err) 55 } 56 if fsm == nil { 57 t.Fatalf("missing fsm") 58 } 59 return fsm 60 } 61 62 func makeLog(buf []byte) *raft.Log { 63 return &raft.Log{ 64 Index: 1, 65 Term: 1, 66 Type: raft.LogCommand, 67 Data: buf, 68 } 69 } 70 71 func TestFSM_UpsertNode(t *testing.T) { 72 fsm := testFSM(t) 73 fsm.blockedEvals.SetEnabled(true) 74 75 node := mock.Node() 76 77 // Mark an eval as blocked. 78 eval := mock.Eval() 79 eval.ClassEligibility = map[string]bool{node.ComputedClass: true} 80 fsm.blockedEvals.Block(eval) 81 82 req := structs.NodeRegisterRequest{ 83 Node: node, 84 } 85 buf, err := structs.Encode(structs.NodeRegisterRequestType, req) 86 if err != nil { 87 t.Fatalf("err: %v", err) 88 } 89 90 resp := fsm.Apply(makeLog(buf)) 91 if resp != nil { 92 t.Fatalf("resp: %v", resp) 93 } 94 95 // Verify we are registered 96 ws := memdb.NewWatchSet() 97 n, err := fsm.State().NodeByID(ws, req.Node.ID) 98 if err != nil { 99 t.Fatalf("err: %v", err) 100 } 101 if n == nil { 102 t.Fatalf("not found!") 103 } 104 if n.CreateIndex != 1 { 105 t.Fatalf("bad index: %d", node.CreateIndex) 106 } 107 108 tt := fsm.TimeTable() 109 index := tt.NearestIndex(time.Now().UTC()) 110 if index != 1 { 111 t.Fatalf("bad: %d", index) 112 } 113 114 // Verify the eval was unblocked. 115 testutil.WaitForResult(func() (bool, error) { 116 bStats := fsm.blockedEvals.Stats() 117 if bStats.TotalBlocked != 0 { 118 return false, fmt.Errorf("bad: %#v", bStats) 119 } 120 return true, nil 121 }, func(err error) { 122 t.Fatalf("err: %s", err) 123 }) 124 125 } 126 127 func TestFSM_DeregisterNode(t *testing.T) { 128 fsm := testFSM(t) 129 130 node := mock.Node() 131 req := structs.NodeRegisterRequest{ 132 Node: node, 133 } 134 buf, err := structs.Encode(structs.NodeRegisterRequestType, req) 135 if err != nil { 136 t.Fatalf("err: %v", err) 137 } 138 139 resp := fsm.Apply(makeLog(buf)) 140 if resp != nil { 141 t.Fatalf("resp: %v", resp) 142 } 143 144 req2 := structs.NodeDeregisterRequest{ 145 NodeID: node.ID, 146 } 147 buf, err = structs.Encode(structs.NodeDeregisterRequestType, req2) 148 if err != nil { 149 t.Fatalf("err: %v", err) 150 } 151 152 resp = fsm.Apply(makeLog(buf)) 153 if resp != nil { 154 t.Fatalf("resp: %v", resp) 155 } 156 157 // Verify we are NOT registered 158 ws := memdb.NewWatchSet() 159 node, err = fsm.State().NodeByID(ws, req.Node.ID) 160 if err != nil { 161 t.Fatalf("err: %v", err) 162 } 163 if node != nil { 164 t.Fatalf("node found!") 165 } 166 } 167 168 func TestFSM_UpdateNodeStatus(t *testing.T) { 169 fsm := testFSM(t) 170 fsm.blockedEvals.SetEnabled(true) 171 172 node := mock.Node() 173 req := structs.NodeRegisterRequest{ 174 Node: node, 175 } 176 buf, err := structs.Encode(structs.NodeRegisterRequestType, req) 177 if err != nil { 178 t.Fatalf("err: %v", err) 179 } 180 181 resp := fsm.Apply(makeLog(buf)) 182 if resp != nil { 183 t.Fatalf("resp: %v", resp) 184 } 185 186 // Mark an eval as blocked. 187 eval := mock.Eval() 188 eval.ClassEligibility = map[string]bool{node.ComputedClass: true} 189 fsm.blockedEvals.Block(eval) 190 191 req2 := structs.NodeUpdateStatusRequest{ 192 NodeID: node.ID, 193 Status: structs.NodeStatusReady, 194 } 195 buf, err = structs.Encode(structs.NodeUpdateStatusRequestType, req2) 196 if err != nil { 197 t.Fatalf("err: %v", err) 198 } 199 200 resp = fsm.Apply(makeLog(buf)) 201 if resp != nil { 202 t.Fatalf("resp: %v", resp) 203 } 204 205 // Verify the status is ready. 206 ws := memdb.NewWatchSet() 207 node, err = fsm.State().NodeByID(ws, req.Node.ID) 208 if err != nil { 209 t.Fatalf("err: %v", err) 210 } 211 if node.Status != structs.NodeStatusReady { 212 t.Fatalf("bad node: %#v", node) 213 } 214 215 // Verify the eval was unblocked. 216 testutil.WaitForResult(func() (bool, error) { 217 bStats := fsm.blockedEvals.Stats() 218 if bStats.TotalBlocked != 0 { 219 return false, fmt.Errorf("bad: %#v", bStats) 220 } 221 return true, nil 222 }, func(err error) { 223 t.Fatalf("err: %s", err) 224 }) 225 } 226 227 func TestFSM_UpdateNodeDrain(t *testing.T) { 228 fsm := testFSM(t) 229 230 node := mock.Node() 231 req := structs.NodeRegisterRequest{ 232 Node: node, 233 } 234 buf, err := structs.Encode(structs.NodeRegisterRequestType, req) 235 if err != nil { 236 t.Fatalf("err: %v", err) 237 } 238 239 resp := fsm.Apply(makeLog(buf)) 240 if resp != nil { 241 t.Fatalf("resp: %v", resp) 242 } 243 244 req2 := structs.NodeUpdateDrainRequest{ 245 NodeID: node.ID, 246 Drain: true, 247 } 248 buf, err = structs.Encode(structs.NodeUpdateDrainRequestType, req2) 249 if err != nil { 250 t.Fatalf("err: %v", err) 251 } 252 253 resp = fsm.Apply(makeLog(buf)) 254 if resp != nil { 255 t.Fatalf("resp: %v", resp) 256 } 257 258 // Verify we are NOT registered 259 ws := memdb.NewWatchSet() 260 node, err = fsm.State().NodeByID(ws, req.Node.ID) 261 if err != nil { 262 t.Fatalf("err: %v", err) 263 } 264 if !node.Drain { 265 t.Fatalf("bad node: %#v", node) 266 } 267 } 268 269 func TestFSM_RegisterJob(t *testing.T) { 270 fsm := testFSM(t) 271 272 job := mock.PeriodicJob() 273 req := structs.JobRegisterRequest{ 274 Job: job, 275 } 276 buf, err := structs.Encode(structs.JobRegisterRequestType, req) 277 if err != nil { 278 t.Fatalf("err: %v", err) 279 } 280 281 resp := fsm.Apply(makeLog(buf)) 282 if resp != nil { 283 t.Fatalf("resp: %v", resp) 284 } 285 286 // Verify we are registered 287 ws := memdb.NewWatchSet() 288 jobOut, err := fsm.State().JobByID(ws, req.Job.ID) 289 if err != nil { 290 t.Fatalf("err: %v", err) 291 } 292 if jobOut == nil { 293 t.Fatalf("not found!") 294 } 295 if jobOut.CreateIndex != 1 { 296 t.Fatalf("bad index: %d", jobOut.CreateIndex) 297 } 298 299 // Verify it was added to the periodic runner. 300 if _, ok := fsm.periodicDispatcher.tracked[job.ID]; !ok { 301 t.Fatal("job not added to periodic runner") 302 } 303 304 // Verify the launch time was tracked. 305 launchOut, err := fsm.State().PeriodicLaunchByID(ws, req.Job.ID) 306 if err != nil { 307 t.Fatalf("err: %v", err) 308 } 309 if launchOut == nil { 310 t.Fatalf("not found!") 311 } 312 if launchOut.Launch.IsZero() { 313 t.Fatalf("bad launch time: %v", launchOut.Launch) 314 } 315 } 316 317 func TestFSM_DeregisterJob_Purge(t *testing.T) { 318 fsm := testFSM(t) 319 320 job := mock.PeriodicJob() 321 req := structs.JobRegisterRequest{ 322 Job: job, 323 } 324 buf, err := structs.Encode(structs.JobRegisterRequestType, req) 325 if err != nil { 326 t.Fatalf("err: %v", err) 327 } 328 329 resp := fsm.Apply(makeLog(buf)) 330 if resp != nil { 331 t.Fatalf("resp: %v", resp) 332 } 333 334 req2 := structs.JobDeregisterRequest{ 335 JobID: job.ID, 336 Purge: true, 337 } 338 buf, err = structs.Encode(structs.JobDeregisterRequestType, req2) 339 if err != nil { 340 t.Fatalf("err: %v", err) 341 } 342 343 resp = fsm.Apply(makeLog(buf)) 344 if resp != nil { 345 t.Fatalf("resp: %v", resp) 346 } 347 348 // Verify we are NOT registered 349 ws := memdb.NewWatchSet() 350 jobOut, err := fsm.State().JobByID(ws, req.Job.ID) 351 if err != nil { 352 t.Fatalf("err: %v", err) 353 } 354 if jobOut != nil { 355 t.Fatalf("job found!") 356 } 357 358 // Verify it was removed from the periodic runner. 359 if _, ok := fsm.periodicDispatcher.tracked[job.ID]; ok { 360 t.Fatal("job not removed from periodic runner") 361 } 362 363 // Verify it was removed from the periodic launch table. 364 launchOut, err := fsm.State().PeriodicLaunchByID(ws, req.Job.ID) 365 if err != nil { 366 t.Fatalf("err: %v", err) 367 } 368 if launchOut != nil { 369 t.Fatalf("launch found!") 370 } 371 } 372 373 func TestFSM_DeregisterJob_NoPurge(t *testing.T) { 374 fsm := testFSM(t) 375 376 job := mock.PeriodicJob() 377 req := structs.JobRegisterRequest{ 378 Job: job, 379 } 380 buf, err := structs.Encode(structs.JobRegisterRequestType, req) 381 if err != nil { 382 t.Fatalf("err: %v", err) 383 } 384 385 resp := fsm.Apply(makeLog(buf)) 386 if resp != nil { 387 t.Fatalf("resp: %v", resp) 388 } 389 390 req2 := structs.JobDeregisterRequest{ 391 JobID: job.ID, 392 Purge: false, 393 } 394 buf, err = structs.Encode(structs.JobDeregisterRequestType, req2) 395 if err != nil { 396 t.Fatalf("err: %v", err) 397 } 398 399 resp = fsm.Apply(makeLog(buf)) 400 if resp != nil { 401 t.Fatalf("resp: %v", resp) 402 } 403 404 // Verify we are NOT registered 405 ws := memdb.NewWatchSet() 406 jobOut, err := fsm.State().JobByID(ws, req.Job.ID) 407 if err != nil { 408 t.Fatalf("err: %v", err) 409 } 410 if jobOut == nil { 411 t.Fatalf("job not found!") 412 } 413 if !jobOut.Stop { 414 t.Fatalf("job not stopped found!") 415 } 416 417 // Verify it was removed from the periodic runner. 418 if _, ok := fsm.periodicDispatcher.tracked[job.ID]; ok { 419 t.Fatal("job not removed from periodic runner") 420 } 421 422 // Verify it was removed from the periodic launch table. 423 launchOut, err := fsm.State().PeriodicLaunchByID(ws, req.Job.ID) 424 if err != nil { 425 t.Fatalf("err: %v", err) 426 } 427 if launchOut == nil { 428 t.Fatalf("launch not found!") 429 } 430 } 431 432 func TestFSM_UpdateEval(t *testing.T) { 433 fsm := testFSM(t) 434 fsm.evalBroker.SetEnabled(true) 435 436 req := structs.EvalUpdateRequest{ 437 Evals: []*structs.Evaluation{mock.Eval()}, 438 } 439 buf, err := structs.Encode(structs.EvalUpdateRequestType, req) 440 if err != nil { 441 t.Fatalf("err: %v", err) 442 } 443 444 resp := fsm.Apply(makeLog(buf)) 445 if resp != nil { 446 t.Fatalf("resp: %v", resp) 447 } 448 449 // Verify we are registered 450 ws := memdb.NewWatchSet() 451 eval, err := fsm.State().EvalByID(ws, req.Evals[0].ID) 452 if err != nil { 453 t.Fatalf("err: %v", err) 454 } 455 if eval == nil { 456 t.Fatalf("not found!") 457 } 458 if eval.CreateIndex != 1 { 459 t.Fatalf("bad index: %d", eval.CreateIndex) 460 } 461 462 // Verify enqueued 463 stats := fsm.evalBroker.Stats() 464 if stats.TotalReady != 1 { 465 t.Fatalf("bad: %#v %#v", stats, eval) 466 } 467 } 468 469 func TestFSM_UpdateEval_Blocked(t *testing.T) { 470 fsm := testFSM(t) 471 fsm.evalBroker.SetEnabled(true) 472 fsm.blockedEvals.SetEnabled(true) 473 474 // Create a blocked eval. 475 eval := mock.Eval() 476 eval.Status = structs.EvalStatusBlocked 477 478 req := structs.EvalUpdateRequest{ 479 Evals: []*structs.Evaluation{eval}, 480 } 481 buf, err := structs.Encode(structs.EvalUpdateRequestType, req) 482 if err != nil { 483 t.Fatalf("err: %v", err) 484 } 485 486 resp := fsm.Apply(makeLog(buf)) 487 if resp != nil { 488 t.Fatalf("resp: %v", resp) 489 } 490 491 // Verify we are registered 492 ws := memdb.NewWatchSet() 493 out, err := fsm.State().EvalByID(ws, eval.ID) 494 if err != nil { 495 t.Fatalf("err: %v", err) 496 } 497 if out == nil { 498 t.Fatalf("not found!") 499 } 500 if out.CreateIndex != 1 { 501 t.Fatalf("bad index: %d", out.CreateIndex) 502 } 503 504 // Verify the eval wasn't enqueued 505 stats := fsm.evalBroker.Stats() 506 if stats.TotalReady != 0 { 507 t.Fatalf("bad: %#v %#v", stats, out) 508 } 509 510 // Verify the eval was added to the blocked tracker. 511 bStats := fsm.blockedEvals.Stats() 512 if bStats.TotalBlocked != 1 { 513 t.Fatalf("bad: %#v %#v", bStats, out) 514 } 515 } 516 517 func TestFSM_UpdateEval_Untrack(t *testing.T) { 518 fsm := testFSM(t) 519 fsm.evalBroker.SetEnabled(true) 520 fsm.blockedEvals.SetEnabled(true) 521 522 // Mark an eval as blocked. 523 bEval := mock.Eval() 524 bEval.ClassEligibility = map[string]bool{"v1:123": true} 525 fsm.blockedEvals.Block(bEval) 526 527 // Create a successful eval for the same job 528 eval := mock.Eval() 529 eval.JobID = bEval.JobID 530 eval.Status = structs.EvalStatusComplete 531 532 req := structs.EvalUpdateRequest{ 533 Evals: []*structs.Evaluation{eval}, 534 } 535 buf, err := structs.Encode(structs.EvalUpdateRequestType, req) 536 if err != nil { 537 t.Fatalf("err: %v", err) 538 } 539 540 resp := fsm.Apply(makeLog(buf)) 541 if resp != nil { 542 t.Fatalf("resp: %v", resp) 543 } 544 545 // Verify we are registered 546 ws := memdb.NewWatchSet() 547 out, err := fsm.State().EvalByID(ws, eval.ID) 548 if err != nil { 549 t.Fatalf("err: %v", err) 550 } 551 if out == nil { 552 t.Fatalf("not found!") 553 } 554 if out.CreateIndex != 1 { 555 t.Fatalf("bad index: %d", out.CreateIndex) 556 } 557 558 // Verify the eval wasn't enqueued 559 stats := fsm.evalBroker.Stats() 560 if stats.TotalReady != 0 { 561 t.Fatalf("bad: %#v %#v", stats, out) 562 } 563 564 // Verify the eval was untracked in the blocked tracker. 565 bStats := fsm.blockedEvals.Stats() 566 if bStats.TotalBlocked != 0 { 567 t.Fatalf("bad: %#v %#v", bStats, out) 568 } 569 } 570 571 func TestFSM_UpdateEval_NoUntrack(t *testing.T) { 572 fsm := testFSM(t) 573 fsm.evalBroker.SetEnabled(true) 574 fsm.blockedEvals.SetEnabled(true) 575 576 // Mark an eval as blocked. 577 bEval := mock.Eval() 578 bEval.ClassEligibility = map[string]bool{"v1:123": true} 579 fsm.blockedEvals.Block(bEval) 580 581 // Create a successful eval for the same job but with placement failures 582 eval := mock.Eval() 583 eval.JobID = bEval.JobID 584 eval.Status = structs.EvalStatusComplete 585 eval.FailedTGAllocs = make(map[string]*structs.AllocMetric) 586 eval.FailedTGAllocs["test"] = new(structs.AllocMetric) 587 588 req := structs.EvalUpdateRequest{ 589 Evals: []*structs.Evaluation{eval}, 590 } 591 buf, err := structs.Encode(structs.EvalUpdateRequestType, req) 592 if err != nil { 593 t.Fatalf("err: %v", err) 594 } 595 596 resp := fsm.Apply(makeLog(buf)) 597 if resp != nil { 598 t.Fatalf("resp: %v", resp) 599 } 600 601 // Verify we are registered 602 ws := memdb.NewWatchSet() 603 out, err := fsm.State().EvalByID(ws, eval.ID) 604 if err != nil { 605 t.Fatalf("err: %v", err) 606 } 607 if out == nil { 608 t.Fatalf("not found!") 609 } 610 if out.CreateIndex != 1 { 611 t.Fatalf("bad index: %d", out.CreateIndex) 612 } 613 614 // Verify the eval wasn't enqueued 615 stats := fsm.evalBroker.Stats() 616 if stats.TotalReady != 0 { 617 t.Fatalf("bad: %#v %#v", stats, out) 618 } 619 620 // Verify the eval was not untracked in the blocked tracker. 621 bStats := fsm.blockedEvals.Stats() 622 if bStats.TotalBlocked != 1 { 623 t.Fatalf("bad: %#v %#v", bStats, out) 624 } 625 } 626 627 func TestFSM_DeleteEval(t *testing.T) { 628 fsm := testFSM(t) 629 630 eval := mock.Eval() 631 req := structs.EvalUpdateRequest{ 632 Evals: []*structs.Evaluation{eval}, 633 } 634 buf, err := structs.Encode(structs.EvalUpdateRequestType, req) 635 if err != nil { 636 t.Fatalf("err: %v", err) 637 } 638 639 resp := fsm.Apply(makeLog(buf)) 640 if resp != nil { 641 t.Fatalf("resp: %v", resp) 642 } 643 644 req2 := structs.EvalDeleteRequest{ 645 Evals: []string{eval.ID}, 646 } 647 buf, err = structs.Encode(structs.EvalDeleteRequestType, req2) 648 if err != nil { 649 t.Fatalf("err: %v", err) 650 } 651 652 resp = fsm.Apply(makeLog(buf)) 653 if resp != nil { 654 t.Fatalf("resp: %v", resp) 655 } 656 657 // Verify we are NOT registered 658 ws := memdb.NewWatchSet() 659 eval, err = fsm.State().EvalByID(ws, req.Evals[0].ID) 660 if err != nil { 661 t.Fatalf("err: %v", err) 662 } 663 if eval != nil { 664 t.Fatalf("eval found!") 665 } 666 } 667 668 func TestFSM_UpsertAllocs(t *testing.T) { 669 fsm := testFSM(t) 670 671 alloc := mock.Alloc() 672 fsm.State().UpsertJobSummary(1, mock.JobSummary(alloc.JobID)) 673 req := structs.AllocUpdateRequest{ 674 Alloc: []*structs.Allocation{alloc}, 675 } 676 buf, err := structs.Encode(structs.AllocUpdateRequestType, req) 677 if err != nil { 678 t.Fatalf("err: %v", err) 679 } 680 681 resp := fsm.Apply(makeLog(buf)) 682 if resp != nil { 683 t.Fatalf("resp: %v", resp) 684 } 685 686 // Verify we are registered 687 ws := memdb.NewWatchSet() 688 out, err := fsm.State().AllocByID(ws, alloc.ID) 689 if err != nil { 690 t.Fatalf("err: %v", err) 691 } 692 alloc.CreateIndex = out.CreateIndex 693 alloc.ModifyIndex = out.ModifyIndex 694 alloc.AllocModifyIndex = out.AllocModifyIndex 695 if !reflect.DeepEqual(alloc, out) { 696 t.Fatalf("bad: %#v %#v", alloc, out) 697 } 698 699 evictAlloc := new(structs.Allocation) 700 *evictAlloc = *alloc 701 evictAlloc.DesiredStatus = structs.AllocDesiredStatusEvict 702 req2 := structs.AllocUpdateRequest{ 703 Alloc: []*structs.Allocation{evictAlloc}, 704 } 705 buf, err = structs.Encode(structs.AllocUpdateRequestType, req2) 706 if err != nil { 707 t.Fatalf("err: %v", err) 708 } 709 710 resp = fsm.Apply(makeLog(buf)) 711 if resp != nil { 712 t.Fatalf("resp: %v", resp) 713 } 714 715 // Verify we are evicted 716 out, err = fsm.State().AllocByID(ws, alloc.ID) 717 if err != nil { 718 t.Fatalf("err: %v", err) 719 } 720 if out.DesiredStatus != structs.AllocDesiredStatusEvict { 721 t.Fatalf("alloc found!") 722 } 723 } 724 725 func TestFSM_UpsertAllocs_SharedJob(t *testing.T) { 726 fsm := testFSM(t) 727 728 alloc := mock.Alloc() 729 fsm.State().UpsertJobSummary(1, mock.JobSummary(alloc.JobID)) 730 job := alloc.Job 731 alloc.Job = nil 732 req := structs.AllocUpdateRequest{ 733 Job: job, 734 Alloc: []*structs.Allocation{alloc}, 735 } 736 buf, err := structs.Encode(structs.AllocUpdateRequestType, req) 737 if err != nil { 738 t.Fatalf("err: %v", err) 739 } 740 741 resp := fsm.Apply(makeLog(buf)) 742 if resp != nil { 743 t.Fatalf("resp: %v", resp) 744 } 745 746 // Verify we are registered 747 ws := memdb.NewWatchSet() 748 out, err := fsm.State().AllocByID(ws, alloc.ID) 749 if err != nil { 750 t.Fatalf("err: %v", err) 751 } 752 alloc.CreateIndex = out.CreateIndex 753 alloc.ModifyIndex = out.ModifyIndex 754 alloc.AllocModifyIndex = out.AllocModifyIndex 755 756 // Job should be re-attached 757 alloc.Job = job 758 if !reflect.DeepEqual(alloc, out) { 759 t.Fatalf("bad: %#v %#v", alloc, out) 760 } 761 762 // Ensure that the original job is used 763 evictAlloc := new(structs.Allocation) 764 *evictAlloc = *alloc 765 job = mock.Job() 766 job.Priority = 123 767 768 evictAlloc.Job = nil 769 evictAlloc.DesiredStatus = structs.AllocDesiredStatusEvict 770 req2 := structs.AllocUpdateRequest{ 771 Job: job, 772 Alloc: []*structs.Allocation{evictAlloc}, 773 } 774 buf, err = structs.Encode(structs.AllocUpdateRequestType, req2) 775 if err != nil { 776 t.Fatalf("err: %v", err) 777 } 778 779 resp = fsm.Apply(makeLog(buf)) 780 if resp != nil { 781 t.Fatalf("resp: %v", resp) 782 } 783 784 // Verify we are evicted 785 out, err = fsm.State().AllocByID(ws, alloc.ID) 786 if err != nil { 787 t.Fatalf("err: %v", err) 788 } 789 if out.DesiredStatus != structs.AllocDesiredStatusEvict { 790 t.Fatalf("alloc found!") 791 } 792 if out.Job == nil || out.Job.Priority == 123 { 793 t.Fatalf("bad job") 794 } 795 } 796 797 func TestFSM_UpsertAllocs_StrippedResources(t *testing.T) { 798 fsm := testFSM(t) 799 800 alloc := mock.Alloc() 801 fsm.State().UpsertJobSummary(1, mock.JobSummary(alloc.JobID)) 802 job := alloc.Job 803 resources := alloc.Resources 804 alloc.Resources = nil 805 req := structs.AllocUpdateRequest{ 806 Job: job, 807 Alloc: []*structs.Allocation{alloc}, 808 } 809 buf, err := structs.Encode(structs.AllocUpdateRequestType, req) 810 if err != nil { 811 t.Fatalf("err: %v", err) 812 } 813 814 resp := fsm.Apply(makeLog(buf)) 815 if resp != nil { 816 t.Fatalf("resp: %v", resp) 817 } 818 819 // Verify we are registered 820 ws := memdb.NewWatchSet() 821 out, err := fsm.State().AllocByID(ws, alloc.ID) 822 if err != nil { 823 t.Fatalf("err: %v", err) 824 } 825 alloc.CreateIndex = out.CreateIndex 826 alloc.ModifyIndex = out.ModifyIndex 827 alloc.AllocModifyIndex = out.AllocModifyIndex 828 829 // Resources should be recomputed 830 resources.DiskMB = alloc.Job.TaskGroups[0].EphemeralDisk.SizeMB 831 alloc.Resources = resources 832 if !reflect.DeepEqual(alloc, out) { 833 t.Fatalf("bad: %#v %#v", alloc, out) 834 } 835 } 836 837 func TestFSM_UpdateAllocFromClient_Unblock(t *testing.T) { 838 fsm := testFSM(t) 839 fsm.blockedEvals.SetEnabled(true) 840 state := fsm.State() 841 842 node := mock.Node() 843 state.UpsertNode(1, node) 844 845 // Mark an eval as blocked. 846 eval := mock.Eval() 847 eval.ClassEligibility = map[string]bool{node.ComputedClass: true} 848 fsm.blockedEvals.Block(eval) 849 850 bStats := fsm.blockedEvals.Stats() 851 if bStats.TotalBlocked != 1 { 852 t.Fatalf("bad: %#v", bStats) 853 } 854 855 // Create a completed eval 856 alloc := mock.Alloc() 857 alloc.NodeID = node.ID 858 alloc2 := mock.Alloc() 859 alloc2.NodeID = node.ID 860 state.UpsertJobSummary(8, mock.JobSummary(alloc.JobID)) 861 state.UpsertJobSummary(9, mock.JobSummary(alloc2.JobID)) 862 state.UpsertAllocs(10, []*structs.Allocation{alloc, alloc2}) 863 864 clientAlloc := new(structs.Allocation) 865 *clientAlloc = *alloc 866 clientAlloc.ClientStatus = structs.AllocClientStatusComplete 867 update2 := &structs.Allocation{ 868 ID: alloc2.ID, 869 ClientStatus: structs.AllocClientStatusRunning, 870 } 871 872 req := structs.AllocUpdateRequest{ 873 Alloc: []*structs.Allocation{clientAlloc, update2}, 874 } 875 buf, err := structs.Encode(structs.AllocClientUpdateRequestType, req) 876 if err != nil { 877 t.Fatalf("err: %v", err) 878 } 879 880 resp := fsm.Apply(makeLog(buf)) 881 if resp != nil { 882 t.Fatalf("resp: %v", resp) 883 } 884 885 // Verify we are updated 886 ws := memdb.NewWatchSet() 887 out, err := fsm.State().AllocByID(ws, alloc.ID) 888 if err != nil { 889 t.Fatalf("err: %v", err) 890 } 891 clientAlloc.CreateIndex = out.CreateIndex 892 clientAlloc.ModifyIndex = out.ModifyIndex 893 if !reflect.DeepEqual(clientAlloc, out) { 894 t.Fatalf("bad: %#v %#v", clientAlloc, out) 895 } 896 897 out, err = fsm.State().AllocByID(ws, alloc2.ID) 898 if err != nil { 899 t.Fatalf("err: %v", err) 900 } 901 alloc2.CreateIndex = out.CreateIndex 902 alloc2.ModifyIndex = out.ModifyIndex 903 alloc2.ClientStatus = structs.AllocClientStatusRunning 904 alloc2.TaskStates = nil 905 if !reflect.DeepEqual(alloc2, out) { 906 t.Fatalf("bad: %#v %#v", alloc2, out) 907 } 908 909 // Verify the eval was unblocked. 910 testutil.WaitForResult(func() (bool, error) { 911 bStats = fsm.blockedEvals.Stats() 912 if bStats.TotalBlocked != 0 { 913 return false, fmt.Errorf("bad: %#v %#v", bStats, out) 914 } 915 return true, nil 916 }, func(err error) { 917 t.Fatalf("err: %s", err) 918 }) 919 } 920 921 func TestFSM_UpdateAllocFromClient(t *testing.T) { 922 fsm := testFSM(t) 923 state := fsm.State() 924 925 alloc := mock.Alloc() 926 state.UpsertJobSummary(9, mock.JobSummary(alloc.JobID)) 927 state.UpsertAllocs(10, []*structs.Allocation{alloc}) 928 929 clientAlloc := new(structs.Allocation) 930 *clientAlloc = *alloc 931 clientAlloc.ClientStatus = structs.AllocClientStatusFailed 932 933 req := structs.AllocUpdateRequest{ 934 Alloc: []*structs.Allocation{clientAlloc}, 935 } 936 buf, err := structs.Encode(structs.AllocClientUpdateRequestType, req) 937 if err != nil { 938 t.Fatalf("err: %v", err) 939 } 940 941 resp := fsm.Apply(makeLog(buf)) 942 if resp != nil { 943 t.Fatalf("resp: %v", resp) 944 } 945 946 // Verify we are registered 947 ws := memdb.NewWatchSet() 948 out, err := fsm.State().AllocByID(ws, alloc.ID) 949 if err != nil { 950 t.Fatalf("err: %v", err) 951 } 952 clientAlloc.CreateIndex = out.CreateIndex 953 clientAlloc.ModifyIndex = out.ModifyIndex 954 if !reflect.DeepEqual(clientAlloc, out) { 955 t.Fatalf("err: %#v,%#v", clientAlloc, out) 956 } 957 } 958 959 func TestFSM_UpsertVaultAccessor(t *testing.T) { 960 fsm := testFSM(t) 961 fsm.blockedEvals.SetEnabled(true) 962 963 va := mock.VaultAccessor() 964 va2 := mock.VaultAccessor() 965 req := structs.VaultAccessorsRequest{ 966 Accessors: []*structs.VaultAccessor{va, va2}, 967 } 968 buf, err := structs.Encode(structs.VaultAccessorRegisterRequestType, req) 969 if err != nil { 970 t.Fatalf("err: %v", err) 971 } 972 973 resp := fsm.Apply(makeLog(buf)) 974 if resp != nil { 975 t.Fatalf("resp: %v", resp) 976 } 977 978 // Verify we are registered 979 ws := memdb.NewWatchSet() 980 out1, err := fsm.State().VaultAccessor(ws, va.Accessor) 981 if err != nil { 982 t.Fatalf("err: %v", err) 983 } 984 if out1 == nil { 985 t.Fatalf("not found!") 986 } 987 if out1.CreateIndex != 1 { 988 t.Fatalf("bad index: %d", out1.CreateIndex) 989 } 990 out2, err := fsm.State().VaultAccessor(ws, va2.Accessor) 991 if err != nil { 992 t.Fatalf("err: %v", err) 993 } 994 if out2 == nil { 995 t.Fatalf("not found!") 996 } 997 if out1.CreateIndex != 1 { 998 t.Fatalf("bad index: %d", out2.CreateIndex) 999 } 1000 1001 tt := fsm.TimeTable() 1002 index := tt.NearestIndex(time.Now().UTC()) 1003 if index != 1 { 1004 t.Fatalf("bad: %d", index) 1005 } 1006 } 1007 1008 func TestFSM_DeregisterVaultAccessor(t *testing.T) { 1009 fsm := testFSM(t) 1010 fsm.blockedEvals.SetEnabled(true) 1011 1012 va := mock.VaultAccessor() 1013 va2 := mock.VaultAccessor() 1014 accessors := []*structs.VaultAccessor{va, va2} 1015 1016 // Insert the accessors 1017 if err := fsm.State().UpsertVaultAccessor(1000, accessors); err != nil { 1018 t.Fatalf("bad: %v", err) 1019 } 1020 1021 req := structs.VaultAccessorsRequest{ 1022 Accessors: accessors, 1023 } 1024 buf, err := structs.Encode(structs.VaultAccessorDegisterRequestType, req) 1025 if err != nil { 1026 t.Fatalf("err: %v", err) 1027 } 1028 1029 resp := fsm.Apply(makeLog(buf)) 1030 if resp != nil { 1031 t.Fatalf("resp: %v", resp) 1032 } 1033 1034 ws := memdb.NewWatchSet() 1035 out1, err := fsm.State().VaultAccessor(ws, va.Accessor) 1036 if err != nil { 1037 t.Fatalf("err: %v", err) 1038 } 1039 if out1 != nil { 1040 t.Fatalf("not deleted!") 1041 } 1042 1043 tt := fsm.TimeTable() 1044 index := tt.NearestIndex(time.Now().UTC()) 1045 if index != 1 { 1046 t.Fatalf("bad: %d", index) 1047 } 1048 } 1049 1050 func testSnapshotRestore(t *testing.T, fsm *nomadFSM) *nomadFSM { 1051 // Snapshot 1052 snap, err := fsm.Snapshot() 1053 if err != nil { 1054 t.Fatalf("err: %v", err) 1055 } 1056 defer snap.Release() 1057 1058 // Persist 1059 buf := bytes.NewBuffer(nil) 1060 sink := &MockSink{buf, false} 1061 if err := snap.Persist(sink); err != nil { 1062 t.Fatalf("err: %v", err) 1063 } 1064 1065 // Try to restore on a new FSM 1066 fsm2 := testFSM(t) 1067 snap, err = fsm2.Snapshot() 1068 if err != nil { 1069 t.Fatalf("err: %v", err) 1070 } 1071 defer snap.Release() 1072 1073 abandonCh := fsm2.State().AbandonCh() 1074 1075 // Do a restore 1076 if err := fsm2.Restore(sink); err != nil { 1077 t.Fatalf("err: %v", err) 1078 } 1079 1080 select { 1081 case <-abandonCh: 1082 default: 1083 t.Fatalf("bad") 1084 } 1085 1086 return fsm2 1087 } 1088 1089 func TestFSM_SnapshotRestore_Nodes(t *testing.T) { 1090 // Add some state 1091 fsm := testFSM(t) 1092 state := fsm.State() 1093 node1 := mock.Node() 1094 state.UpsertNode(1000, node1) 1095 node2 := mock.Node() 1096 state.UpsertNode(1001, node2) 1097 1098 // Verify the contents 1099 fsm2 := testSnapshotRestore(t, fsm) 1100 state2 := fsm2.State() 1101 ws := memdb.NewWatchSet() 1102 out1, _ := state2.NodeByID(ws, node1.ID) 1103 out2, _ := state2.NodeByID(ws, node2.ID) 1104 if !reflect.DeepEqual(node1, out1) { 1105 t.Fatalf("bad: \n%#v\n%#v", out1, node1) 1106 } 1107 if !reflect.DeepEqual(node2, out2) { 1108 t.Fatalf("bad: \n%#v\n%#v", out2, node2) 1109 } 1110 } 1111 1112 func TestFSM_SnapshotRestore_Jobs(t *testing.T) { 1113 // Add some state 1114 fsm := testFSM(t) 1115 state := fsm.State() 1116 job1 := mock.Job() 1117 state.UpsertJob(1000, job1) 1118 job2 := mock.Job() 1119 state.UpsertJob(1001, job2) 1120 1121 // Verify the contents 1122 ws := memdb.NewWatchSet() 1123 fsm2 := testSnapshotRestore(t, fsm) 1124 state2 := fsm2.State() 1125 out1, _ := state2.JobByID(ws, job1.ID) 1126 out2, _ := state2.JobByID(ws, job2.ID) 1127 if !reflect.DeepEqual(job1, out1) { 1128 t.Fatalf("bad: \n%#v\n%#v", out1, job1) 1129 } 1130 if !reflect.DeepEqual(job2, out2) { 1131 t.Fatalf("bad: \n%#v\n%#v", out2, job2) 1132 } 1133 } 1134 1135 func TestFSM_SnapshotRestore_Evals(t *testing.T) { 1136 // Add some state 1137 fsm := testFSM(t) 1138 state := fsm.State() 1139 eval1 := mock.Eval() 1140 state.UpsertEvals(1000, []*structs.Evaluation{eval1}) 1141 eval2 := mock.Eval() 1142 state.UpsertEvals(1001, []*structs.Evaluation{eval2}) 1143 1144 // Verify the contents 1145 fsm2 := testSnapshotRestore(t, fsm) 1146 state2 := fsm2.State() 1147 ws := memdb.NewWatchSet() 1148 out1, _ := state2.EvalByID(ws, eval1.ID) 1149 out2, _ := state2.EvalByID(ws, eval2.ID) 1150 if !reflect.DeepEqual(eval1, out1) { 1151 t.Fatalf("bad: \n%#v\n%#v", out1, eval1) 1152 } 1153 if !reflect.DeepEqual(eval2, out2) { 1154 t.Fatalf("bad: \n%#v\n%#v", out2, eval2) 1155 } 1156 } 1157 1158 func TestFSM_SnapshotRestore_Allocs(t *testing.T) { 1159 // Add some state 1160 fsm := testFSM(t) 1161 state := fsm.State() 1162 alloc1 := mock.Alloc() 1163 alloc2 := mock.Alloc() 1164 state.UpsertJobSummary(998, mock.JobSummary(alloc1.JobID)) 1165 state.UpsertJobSummary(999, mock.JobSummary(alloc2.JobID)) 1166 state.UpsertAllocs(1000, []*structs.Allocation{alloc1}) 1167 state.UpsertAllocs(1001, []*structs.Allocation{alloc2}) 1168 1169 // Verify the contents 1170 fsm2 := testSnapshotRestore(t, fsm) 1171 state2 := fsm2.State() 1172 ws := memdb.NewWatchSet() 1173 out1, _ := state2.AllocByID(ws, alloc1.ID) 1174 out2, _ := state2.AllocByID(ws, alloc2.ID) 1175 if !reflect.DeepEqual(alloc1, out1) { 1176 t.Fatalf("bad: \n%#v\n%#v", out1, alloc1) 1177 } 1178 if !reflect.DeepEqual(alloc2, out2) { 1179 t.Fatalf("bad: \n%#v\n%#v", out2, alloc2) 1180 } 1181 } 1182 1183 func TestFSM_SnapshotRestore_Allocs_NoSharedResources(t *testing.T) { 1184 // Add some state 1185 fsm := testFSM(t) 1186 state := fsm.State() 1187 alloc1 := mock.Alloc() 1188 alloc2 := mock.Alloc() 1189 alloc1.SharedResources = nil 1190 alloc2.SharedResources = nil 1191 state.UpsertJobSummary(998, mock.JobSummary(alloc1.JobID)) 1192 state.UpsertJobSummary(999, mock.JobSummary(alloc2.JobID)) 1193 state.UpsertAllocs(1000, []*structs.Allocation{alloc1}) 1194 state.UpsertAllocs(1001, []*structs.Allocation{alloc2}) 1195 1196 // Verify the contents 1197 fsm2 := testSnapshotRestore(t, fsm) 1198 state2 := fsm2.State() 1199 ws := memdb.NewWatchSet() 1200 out1, _ := state2.AllocByID(ws, alloc1.ID) 1201 out2, _ := state2.AllocByID(ws, alloc2.ID) 1202 alloc1.SharedResources = &structs.Resources{DiskMB: 150} 1203 alloc2.SharedResources = &structs.Resources{DiskMB: 150} 1204 1205 if !reflect.DeepEqual(alloc1, out1) { 1206 t.Fatalf("bad: \n%#v\n%#v", out1, alloc1) 1207 } 1208 if !reflect.DeepEqual(alloc2, out2) { 1209 t.Fatalf("bad: \n%#v\n%#v", out2, alloc2) 1210 } 1211 } 1212 1213 func TestFSM_SnapshotRestore_Indexes(t *testing.T) { 1214 // Add some state 1215 fsm := testFSM(t) 1216 state := fsm.State() 1217 node1 := mock.Node() 1218 state.UpsertNode(1000, node1) 1219 1220 // Verify the contents 1221 fsm2 := testSnapshotRestore(t, fsm) 1222 state2 := fsm2.State() 1223 1224 index, err := state2.Index("nodes") 1225 if err != nil { 1226 t.Fatalf("err: %v", err) 1227 } 1228 if index != 1000 { 1229 t.Fatalf("bad: %d", index) 1230 } 1231 } 1232 1233 func TestFSM_SnapshotRestore_TimeTable(t *testing.T) { 1234 // Add some state 1235 fsm := testFSM(t) 1236 1237 tt := fsm.TimeTable() 1238 start := time.Now().UTC() 1239 tt.Witness(1000, start) 1240 tt.Witness(2000, start.Add(10*time.Minute)) 1241 1242 // Verify the contents 1243 fsm2 := testSnapshotRestore(t, fsm) 1244 1245 tt2 := fsm2.TimeTable() 1246 if tt2.NearestTime(1500) != start { 1247 t.Fatalf("bad") 1248 } 1249 if tt2.NearestIndex(start.Add(15*time.Minute)) != 2000 { 1250 t.Fatalf("bad") 1251 } 1252 } 1253 1254 func TestFSM_SnapshotRestore_PeriodicLaunches(t *testing.T) { 1255 // Add some state 1256 fsm := testFSM(t) 1257 state := fsm.State() 1258 job1 := mock.Job() 1259 launch1 := &structs.PeriodicLaunch{ID: job1.ID, Launch: time.Now()} 1260 state.UpsertPeriodicLaunch(1000, launch1) 1261 job2 := mock.Job() 1262 launch2 := &structs.PeriodicLaunch{ID: job2.ID, Launch: time.Now()} 1263 state.UpsertPeriodicLaunch(1001, launch2) 1264 1265 // Verify the contents 1266 fsm2 := testSnapshotRestore(t, fsm) 1267 state2 := fsm2.State() 1268 ws := memdb.NewWatchSet() 1269 out1, _ := state2.PeriodicLaunchByID(ws, launch1.ID) 1270 out2, _ := state2.PeriodicLaunchByID(ws, launch2.ID) 1271 if !reflect.DeepEqual(launch1, out1) { 1272 t.Fatalf("bad: \n%#v\n%#v", out1, job1) 1273 } 1274 if !reflect.DeepEqual(launch2, out2) { 1275 t.Fatalf("bad: \n%#v\n%#v", out2, job2) 1276 } 1277 } 1278 1279 func TestFSM_SnapshotRestore_JobSummary(t *testing.T) { 1280 // Add some state 1281 fsm := testFSM(t) 1282 state := fsm.State() 1283 1284 job1 := mock.Job() 1285 state.UpsertJob(1000, job1) 1286 ws := memdb.NewWatchSet() 1287 js1, _ := state.JobSummaryByID(ws, job1.ID) 1288 1289 job2 := mock.Job() 1290 state.UpsertJob(1001, job2) 1291 js2, _ := state.JobSummaryByID(ws, job2.ID) 1292 1293 // Verify the contents 1294 fsm2 := testSnapshotRestore(t, fsm) 1295 state2 := fsm2.State() 1296 out1, _ := state2.JobSummaryByID(ws, job1.ID) 1297 out2, _ := state2.JobSummaryByID(ws, job2.ID) 1298 if !reflect.DeepEqual(js1, out1) { 1299 t.Fatalf("bad: \n%#v\n%#v", js1, out1) 1300 } 1301 if !reflect.DeepEqual(js2, out2) { 1302 t.Fatalf("bad: \n%#v\n%#v", js2, out2) 1303 } 1304 } 1305 1306 func TestFSM_SnapshotRestore_VaultAccessors(t *testing.T) { 1307 // Add some state 1308 fsm := testFSM(t) 1309 state := fsm.State() 1310 a1 := mock.VaultAccessor() 1311 a2 := mock.VaultAccessor() 1312 state.UpsertVaultAccessor(1000, []*structs.VaultAccessor{a1, a2}) 1313 1314 // Verify the contents 1315 fsm2 := testSnapshotRestore(t, fsm) 1316 state2 := fsm2.State() 1317 ws := memdb.NewWatchSet() 1318 out1, _ := state2.VaultAccessor(ws, a1.Accessor) 1319 out2, _ := state2.VaultAccessor(ws, a2.Accessor) 1320 if !reflect.DeepEqual(a1, out1) { 1321 t.Fatalf("bad: \n%#v\n%#v", out1, a1) 1322 } 1323 if !reflect.DeepEqual(a2, out2) { 1324 t.Fatalf("bad: \n%#v\n%#v", out2, a2) 1325 } 1326 } 1327 1328 func TestFSM_SnapshotRestore_JobVersions(t *testing.T) { 1329 // Add some state 1330 fsm := testFSM(t) 1331 state := fsm.State() 1332 job1 := mock.Job() 1333 state.UpsertJob(1000, job1) 1334 job2 := mock.Job() 1335 job2.ID = job1.ID 1336 state.UpsertJob(1001, job2) 1337 1338 // Verify the contents 1339 ws := memdb.NewWatchSet() 1340 fsm2 := testSnapshotRestore(t, fsm) 1341 state2 := fsm2.State() 1342 out1, _ := state2.JobByIDAndVersion(ws, job1.ID, job1.Version) 1343 out2, _ := state2.JobByIDAndVersion(ws, job2.ID, job2.Version) 1344 if !reflect.DeepEqual(job1, out1) { 1345 t.Fatalf("bad: \n%#v\n%#v", out1, job1) 1346 } 1347 if !reflect.DeepEqual(job2, out2) { 1348 t.Fatalf("bad: \n%#v\n%#v", out2, job2) 1349 } 1350 if job2.Version != 1 { 1351 t.Fatalf("bad: \n%#v\n%#v", 1, job2) 1352 } 1353 } 1354 1355 func TestFSM_SnapshotRestore_Deployments(t *testing.T) { 1356 // Add some state 1357 fsm := testFSM(t) 1358 state := fsm.State() 1359 d1 := mock.Deployment() 1360 d2 := mock.Deployment() 1361 state.UpsertDeployment(1000, d1, false) 1362 state.UpsertDeployment(1001, d2, false) 1363 1364 // Verify the contents 1365 fsm2 := testSnapshotRestore(t, fsm) 1366 state2 := fsm2.State() 1367 ws := memdb.NewWatchSet() 1368 out1, _ := state2.DeploymentByID(ws, d1.ID) 1369 out2, _ := state2.DeploymentByID(ws, d2.ID) 1370 if !reflect.DeepEqual(d1, out1) { 1371 t.Fatalf("bad: \n%#v\n%#v", out1, d1) 1372 } 1373 if !reflect.DeepEqual(d2, out2) { 1374 t.Fatalf("bad: \n%#v\n%#v", out2, d2) 1375 } 1376 } 1377 1378 func TestFSM_SnapshotRestore_AddMissingSummary(t *testing.T) { 1379 // Add some state 1380 fsm := testFSM(t) 1381 state := fsm.State() 1382 1383 // make an allocation 1384 alloc := mock.Alloc() 1385 state.UpsertJob(1010, alloc.Job) 1386 state.UpsertAllocs(1011, []*structs.Allocation{alloc}) 1387 1388 // Delete the summary 1389 state.DeleteJobSummary(1040, alloc.Job.ID) 1390 1391 // Delete the index 1392 if err := state.RemoveIndex("job_summary"); err != nil { 1393 t.Fatalf("err: %v", err) 1394 } 1395 1396 fsm2 := testSnapshotRestore(t, fsm) 1397 state2 := fsm2.State() 1398 latestIndex, _ := state.LatestIndex() 1399 1400 ws := memdb.NewWatchSet() 1401 out, _ := state2.JobSummaryByID(ws, alloc.Job.ID) 1402 expected := structs.JobSummary{ 1403 JobID: alloc.Job.ID, 1404 Summary: map[string]structs.TaskGroupSummary{ 1405 "web": structs.TaskGroupSummary{ 1406 Starting: 1, 1407 }, 1408 }, 1409 CreateIndex: 1010, 1410 ModifyIndex: latestIndex, 1411 } 1412 if !reflect.DeepEqual(&expected, out) { 1413 t.Fatalf("expected: %#v, actual: %#v", &expected, out) 1414 } 1415 } 1416 1417 func TestFSM_ReconcileSummaries(t *testing.T) { 1418 // Add some state 1419 fsm := testFSM(t) 1420 state := fsm.State() 1421 1422 // Add a node 1423 node := mock.Node() 1424 state.UpsertNode(800, node) 1425 1426 // Make a job so that none of the tasks can be placed 1427 job1 := mock.Job() 1428 job1.TaskGroups[0].Tasks[0].Resources.CPU = 5000 1429 state.UpsertJob(1000, job1) 1430 1431 // make a job which can make partial progress 1432 alloc := mock.Alloc() 1433 alloc.NodeID = node.ID 1434 state.UpsertJob(1010, alloc.Job) 1435 state.UpsertAllocs(1011, []*structs.Allocation{alloc}) 1436 1437 // Delete the summaries 1438 state.DeleteJobSummary(1030, job1.ID) 1439 state.DeleteJobSummary(1040, alloc.Job.ID) 1440 1441 req := structs.GenericRequest{} 1442 buf, err := structs.Encode(structs.ReconcileJobSummariesRequestType, req) 1443 if err != nil { 1444 t.Fatalf("err: %v", err) 1445 } 1446 1447 resp := fsm.Apply(makeLog(buf)) 1448 if resp != nil { 1449 t.Fatalf("resp: %v", resp) 1450 } 1451 1452 ws := memdb.NewWatchSet() 1453 out1, _ := state.JobSummaryByID(ws, job1.ID) 1454 expected := structs.JobSummary{ 1455 JobID: job1.ID, 1456 Summary: map[string]structs.TaskGroupSummary{ 1457 "web": structs.TaskGroupSummary{ 1458 Queued: 10, 1459 }, 1460 }, 1461 CreateIndex: 1000, 1462 ModifyIndex: out1.ModifyIndex, 1463 } 1464 if !reflect.DeepEqual(&expected, out1) { 1465 t.Fatalf("expected: %#v, actual: %#v", &expected, out1) 1466 } 1467 1468 // This exercises the code path which adds the allocations made by the 1469 // planner and the number of unplaced allocations in the reconcile summaries 1470 // codepath 1471 out2, _ := state.JobSummaryByID(ws, alloc.Job.ID) 1472 expected = structs.JobSummary{ 1473 JobID: alloc.Job.ID, 1474 Summary: map[string]structs.TaskGroupSummary{ 1475 "web": structs.TaskGroupSummary{ 1476 Queued: 10, 1477 Starting: 1, 1478 }, 1479 }, 1480 CreateIndex: 1010, 1481 ModifyIndex: out2.ModifyIndex, 1482 } 1483 if !reflect.DeepEqual(&expected, out2) { 1484 t.Fatalf("expected: %#v, actual: %#v", &expected, out2) 1485 } 1486 } 1487 1488 func TestFSM_ApplyPlanResults(t *testing.T) { 1489 fsm := testFSM(t) 1490 1491 // Create the request and create a deployment 1492 alloc := mock.Alloc() 1493 job := alloc.Job 1494 alloc.Job = nil 1495 1496 d := mock.Deployment() 1497 d.JobID = job.ID 1498 d.JobModifyIndex = job.ModifyIndex 1499 d.JobVersion = job.Version 1500 1501 alloc.DeploymentID = d.ID 1502 1503 fsm.State().UpsertJobSummary(1, mock.JobSummary(alloc.JobID)) 1504 req := structs.ApplyPlanResultsRequest{ 1505 AllocUpdateRequest: structs.AllocUpdateRequest{ 1506 Job: job, 1507 Alloc: []*structs.Allocation{alloc}, 1508 }, 1509 CreatedDeployment: d, 1510 } 1511 buf, err := structs.Encode(structs.ApplyPlanResultsRequestType, req) 1512 if err != nil { 1513 t.Fatalf("err: %v", err) 1514 } 1515 1516 resp := fsm.Apply(makeLog(buf)) 1517 if resp != nil { 1518 t.Fatalf("resp: %v", resp) 1519 } 1520 1521 // Verify the allocation is registered 1522 ws := memdb.NewWatchSet() 1523 out, err := fsm.State().AllocByID(ws, alloc.ID) 1524 if err != nil { 1525 t.Fatalf("err: %v", err) 1526 } 1527 alloc.CreateIndex = out.CreateIndex 1528 alloc.ModifyIndex = out.ModifyIndex 1529 alloc.AllocModifyIndex = out.AllocModifyIndex 1530 1531 // Job should be re-attached 1532 alloc.Job = job 1533 if !reflect.DeepEqual(alloc, out) { 1534 t.Fatalf("bad: %#v %#v", alloc, out) 1535 } 1536 1537 dout, err := fsm.State().DeploymentByID(ws, d.ID) 1538 if err != nil { 1539 t.Fatalf("err: %v", err) 1540 } 1541 if tg, ok := dout.TaskGroups[alloc.TaskGroup]; !ok || tg.PlacedAllocs != 1 { 1542 t.Fatalf("err: %v %v", tg, err) 1543 } 1544 1545 // Ensure that the original job is used 1546 evictAlloc := alloc.Copy() 1547 job = mock.Job() 1548 job.Priority = 123 1549 1550 evictAlloc.Job = nil 1551 evictAlloc.DesiredStatus = structs.AllocDesiredStatusEvict 1552 req2 := structs.ApplyPlanResultsRequest{ 1553 AllocUpdateRequest: structs.AllocUpdateRequest{ 1554 Job: job, 1555 Alloc: []*structs.Allocation{evictAlloc}, 1556 }, 1557 } 1558 buf, err = structs.Encode(structs.ApplyPlanResultsRequestType, req2) 1559 if err != nil { 1560 t.Fatalf("err: %v", err) 1561 } 1562 1563 resp = fsm.Apply(makeLog(buf)) 1564 if resp != nil { 1565 t.Fatalf("resp: %v", resp) 1566 } 1567 1568 // Verify we are evicted 1569 out, err = fsm.State().AllocByID(ws, alloc.ID) 1570 if err != nil { 1571 t.Fatalf("err: %v", err) 1572 } 1573 if out.DesiredStatus != structs.AllocDesiredStatusEvict { 1574 t.Fatalf("alloc found!") 1575 } 1576 if out.Job == nil || out.Job.Priority == 123 { 1577 t.Fatalf("bad job") 1578 } 1579 }