github.com/mattyr/nomad@v0.3.3-0.20160919021406-3485a065154a/nomad/fsm_test.go (about) 1 package nomad 2 3 import ( 4 "bytes" 5 "fmt" 6 "os" 7 "reflect" 8 "testing" 9 "time" 10 11 "github.com/hashicorp/nomad/nomad/mock" 12 "github.com/hashicorp/nomad/nomad/state" 13 "github.com/hashicorp/nomad/nomad/structs" 14 "github.com/hashicorp/nomad/testutil" 15 "github.com/hashicorp/raft" 16 ) 17 18 type MockSink struct { 19 *bytes.Buffer 20 cancel bool 21 } 22 23 func (m *MockSink) ID() string { 24 return "Mock" 25 } 26 27 func (m *MockSink) Cancel() error { 28 m.cancel = true 29 return nil 30 } 31 32 func (m *MockSink) Close() error { 33 return nil 34 } 35 36 func testStateStore(t *testing.T) *state.StateStore { 37 state, err := state.NewStateStore(os.Stderr) 38 if err != nil { 39 t.Fatalf("err: %v", err) 40 } 41 if state == nil { 42 t.Fatalf("missing state") 43 } 44 return state 45 } 46 47 func testFSM(t *testing.T) *nomadFSM { 48 p, _ := testPeriodicDispatcher() 49 broker := testBroker(t, 0) 50 blocked := NewBlockedEvals(broker) 51 fsm, err := NewFSM(broker, p, blocked, os.Stderr) 52 if err != nil { 53 t.Fatalf("err: %v", err) 54 } 55 if fsm == nil { 56 t.Fatalf("missing fsm") 57 } 58 return fsm 59 } 60 61 func makeLog(buf []byte) *raft.Log { 62 return &raft.Log{ 63 Index: 1, 64 Term: 1, 65 Type: raft.LogCommand, 66 Data: buf, 67 } 68 } 69 70 func TestFSM_UpsertNode(t *testing.T) { 71 fsm := testFSM(t) 72 fsm.blockedEvals.SetEnabled(true) 73 74 node := mock.Node() 75 76 // Mark an eval as blocked. 77 eval := mock.Eval() 78 eval.ClassEligibility = map[string]bool{node.ComputedClass: true} 79 fsm.blockedEvals.Block(eval) 80 81 req := structs.NodeRegisterRequest{ 82 Node: node, 83 } 84 buf, err := structs.Encode(structs.NodeRegisterRequestType, req) 85 if err != nil { 86 t.Fatalf("err: %v", err) 87 } 88 89 resp := fsm.Apply(makeLog(buf)) 90 if resp != nil { 91 t.Fatalf("resp: %v", resp) 92 } 93 94 // Verify we are registered 95 n, err := fsm.State().NodeByID(req.Node.ID) 96 if err != nil { 97 t.Fatalf("err: %v", err) 98 } 99 if n == nil { 100 t.Fatalf("not found!") 101 } 102 if n.CreateIndex != 1 { 103 t.Fatalf("bad index: %d", node.CreateIndex) 104 } 105 106 tt := fsm.TimeTable() 107 index := tt.NearestIndex(time.Now().UTC()) 108 if index != 1 { 109 t.Fatalf("bad: %d", index) 110 } 111 112 // Verify the eval was unblocked. 113 testutil.WaitForResult(func() (bool, error) { 114 bStats := fsm.blockedEvals.Stats() 115 if bStats.TotalBlocked != 0 { 116 return false, fmt.Errorf("bad: %#v", bStats) 117 } 118 return true, nil 119 }, func(err error) { 120 t.Fatalf("err: %s", err) 121 }) 122 123 } 124 125 func TestFSM_DeregisterNode(t *testing.T) { 126 fsm := testFSM(t) 127 128 node := mock.Node() 129 req := structs.NodeRegisterRequest{ 130 Node: node, 131 } 132 buf, err := structs.Encode(structs.NodeRegisterRequestType, req) 133 if err != nil { 134 t.Fatalf("err: %v", err) 135 } 136 137 resp := fsm.Apply(makeLog(buf)) 138 if resp != nil { 139 t.Fatalf("resp: %v", resp) 140 } 141 142 req2 := structs.NodeDeregisterRequest{ 143 NodeID: node.ID, 144 } 145 buf, err = structs.Encode(structs.NodeDeregisterRequestType, req2) 146 if err != nil { 147 t.Fatalf("err: %v", err) 148 } 149 150 resp = fsm.Apply(makeLog(buf)) 151 if resp != nil { 152 t.Fatalf("resp: %v", resp) 153 } 154 155 // Verify we are NOT registered 156 node, err = fsm.State().NodeByID(req.Node.ID) 157 if err != nil { 158 t.Fatalf("err: %v", err) 159 } 160 if node != nil { 161 t.Fatalf("node found!") 162 } 163 } 164 165 func TestFSM_UpdateNodeStatus(t *testing.T) { 166 fsm := testFSM(t) 167 fsm.blockedEvals.SetEnabled(true) 168 169 node := mock.Node() 170 req := structs.NodeRegisterRequest{ 171 Node: node, 172 } 173 buf, err := structs.Encode(structs.NodeRegisterRequestType, req) 174 if err != nil { 175 t.Fatalf("err: %v", err) 176 } 177 178 resp := fsm.Apply(makeLog(buf)) 179 if resp != nil { 180 t.Fatalf("resp: %v", resp) 181 } 182 183 // Mark an eval as blocked. 184 eval := mock.Eval() 185 eval.ClassEligibility = map[string]bool{node.ComputedClass: true} 186 fsm.blockedEvals.Block(eval) 187 188 req2 := structs.NodeUpdateStatusRequest{ 189 NodeID: node.ID, 190 Status: structs.NodeStatusReady, 191 } 192 buf, err = structs.Encode(structs.NodeUpdateStatusRequestType, req2) 193 if err != nil { 194 t.Fatalf("err: %v", err) 195 } 196 197 resp = fsm.Apply(makeLog(buf)) 198 if resp != nil { 199 t.Fatalf("resp: %v", resp) 200 } 201 202 // Verify the status is ready. 203 node, err = fsm.State().NodeByID(req.Node.ID) 204 if err != nil { 205 t.Fatalf("err: %v", err) 206 } 207 if node.Status != structs.NodeStatusReady { 208 t.Fatalf("bad node: %#v", node) 209 } 210 211 // Verify the eval was unblocked. 212 testutil.WaitForResult(func() (bool, error) { 213 bStats := fsm.blockedEvals.Stats() 214 if bStats.TotalBlocked != 0 { 215 return false, fmt.Errorf("bad: %#v", bStats) 216 } 217 return true, nil 218 }, func(err error) { 219 t.Fatalf("err: %s", err) 220 }) 221 } 222 223 func TestFSM_UpdateNodeDrain(t *testing.T) { 224 fsm := testFSM(t) 225 226 node := mock.Node() 227 req := structs.NodeRegisterRequest{ 228 Node: node, 229 } 230 buf, err := structs.Encode(structs.NodeRegisterRequestType, req) 231 if err != nil { 232 t.Fatalf("err: %v", err) 233 } 234 235 resp := fsm.Apply(makeLog(buf)) 236 if resp != nil { 237 t.Fatalf("resp: %v", resp) 238 } 239 240 req2 := structs.NodeUpdateDrainRequest{ 241 NodeID: node.ID, 242 Drain: true, 243 } 244 buf, err = structs.Encode(structs.NodeUpdateDrainRequestType, req2) 245 if err != nil { 246 t.Fatalf("err: %v", err) 247 } 248 249 resp = fsm.Apply(makeLog(buf)) 250 if resp != nil { 251 t.Fatalf("resp: %v", resp) 252 } 253 254 // Verify we are NOT registered 255 node, err = fsm.State().NodeByID(req.Node.ID) 256 if err != nil { 257 t.Fatalf("err: %v", err) 258 } 259 if !node.Drain { 260 t.Fatalf("bad node: %#v", node) 261 } 262 } 263 264 func TestFSM_RegisterJob(t *testing.T) { 265 fsm := testFSM(t) 266 267 job := mock.PeriodicJob() 268 req := structs.JobRegisterRequest{ 269 Job: job, 270 } 271 buf, err := structs.Encode(structs.JobRegisterRequestType, req) 272 if err != nil { 273 t.Fatalf("err: %v", err) 274 } 275 276 resp := fsm.Apply(makeLog(buf)) 277 if resp != nil { 278 t.Fatalf("resp: %v", resp) 279 } 280 281 // Verify we are registered 282 jobOut, err := fsm.State().JobByID(req.Job.ID) 283 if err != nil { 284 t.Fatalf("err: %v", err) 285 } 286 if jobOut == nil { 287 t.Fatalf("not found!") 288 } 289 if jobOut.CreateIndex != 1 { 290 t.Fatalf("bad index: %d", jobOut.CreateIndex) 291 } 292 293 // Verify it was added to the periodic runner. 294 if _, ok := fsm.periodicDispatcher.tracked[job.ID]; !ok { 295 t.Fatal("job not added to periodic runner") 296 } 297 298 // Verify the launch time was tracked. 299 launchOut, err := fsm.State().PeriodicLaunchByID(req.Job.ID) 300 if err != nil { 301 t.Fatalf("err: %v", err) 302 } 303 if launchOut == nil { 304 t.Fatalf("not found!") 305 } 306 if launchOut.Launch.IsZero() { 307 t.Fatalf("bad launch time: %v", launchOut.Launch) 308 } 309 } 310 311 func TestFSM_DeregisterJob(t *testing.T) { 312 fsm := testFSM(t) 313 314 job := mock.PeriodicJob() 315 req := structs.JobRegisterRequest{ 316 Job: job, 317 } 318 buf, err := structs.Encode(structs.JobRegisterRequestType, req) 319 if err != nil { 320 t.Fatalf("err: %v", err) 321 } 322 323 resp := fsm.Apply(makeLog(buf)) 324 if resp != nil { 325 t.Fatalf("resp: %v", resp) 326 } 327 328 req2 := structs.JobDeregisterRequest{ 329 JobID: job.ID, 330 } 331 buf, err = structs.Encode(structs.JobDeregisterRequestType, req2) 332 if err != nil { 333 t.Fatalf("err: %v", err) 334 } 335 336 resp = fsm.Apply(makeLog(buf)) 337 if resp != nil { 338 t.Fatalf("resp: %v", resp) 339 } 340 341 // Verify we are NOT registered 342 jobOut, err := fsm.State().JobByID(req.Job.ID) 343 if err != nil { 344 t.Fatalf("err: %v", err) 345 } 346 if jobOut != nil { 347 t.Fatalf("job found!") 348 } 349 350 // Verify it was removed from the periodic runner. 351 if _, ok := fsm.periodicDispatcher.tracked[job.ID]; ok { 352 t.Fatal("job not removed from periodic runner") 353 } 354 355 // Verify it was removed from the periodic launch table. 356 launchOut, err := fsm.State().PeriodicLaunchByID(req.Job.ID) 357 if err != nil { 358 t.Fatalf("err: %v", err) 359 } 360 if launchOut != nil { 361 t.Fatalf("launch found!") 362 } 363 } 364 365 func TestFSM_UpdateEval(t *testing.T) { 366 fsm := testFSM(t) 367 fsm.evalBroker.SetEnabled(true) 368 369 req := structs.EvalUpdateRequest{ 370 Evals: []*structs.Evaluation{mock.Eval()}, 371 } 372 buf, err := structs.Encode(structs.EvalUpdateRequestType, req) 373 if err != nil { 374 t.Fatalf("err: %v", err) 375 } 376 377 resp := fsm.Apply(makeLog(buf)) 378 if resp != nil { 379 t.Fatalf("resp: %v", resp) 380 } 381 382 // Verify we are registered 383 eval, err := fsm.State().EvalByID(req.Evals[0].ID) 384 if err != nil { 385 t.Fatalf("err: %v", err) 386 } 387 if eval == nil { 388 t.Fatalf("not found!") 389 } 390 if eval.CreateIndex != 1 { 391 t.Fatalf("bad index: %d", eval.CreateIndex) 392 } 393 394 // Verify enqueued 395 stats := fsm.evalBroker.Stats() 396 if stats.TotalReady != 1 { 397 t.Fatalf("bad: %#v %#v", stats, eval) 398 } 399 } 400 401 func TestFSM_UpdateEval_Blocked(t *testing.T) { 402 fsm := testFSM(t) 403 fsm.evalBroker.SetEnabled(true) 404 fsm.blockedEvals.SetEnabled(true) 405 406 // Create a blocked eval. 407 eval := mock.Eval() 408 eval.Status = structs.EvalStatusBlocked 409 410 req := structs.EvalUpdateRequest{ 411 Evals: []*structs.Evaluation{eval}, 412 } 413 buf, err := structs.Encode(structs.EvalUpdateRequestType, req) 414 if err != nil { 415 t.Fatalf("err: %v", err) 416 } 417 418 resp := fsm.Apply(makeLog(buf)) 419 if resp != nil { 420 t.Fatalf("resp: %v", resp) 421 } 422 423 // Verify we are registered 424 out, err := fsm.State().EvalByID(eval.ID) 425 if err != nil { 426 t.Fatalf("err: %v", err) 427 } 428 if out == nil { 429 t.Fatalf("not found!") 430 } 431 if out.CreateIndex != 1 { 432 t.Fatalf("bad index: %d", out.CreateIndex) 433 } 434 435 // Verify the eval wasn't enqueued 436 stats := fsm.evalBroker.Stats() 437 if stats.TotalReady != 0 { 438 t.Fatalf("bad: %#v %#v", stats, out) 439 } 440 441 // Verify the eval was added to the blocked tracker. 442 bStats := fsm.blockedEvals.Stats() 443 if bStats.TotalBlocked != 1 { 444 t.Fatalf("bad: %#v %#v", bStats, out) 445 } 446 } 447 448 func TestFSM_DeleteEval(t *testing.T) { 449 fsm := testFSM(t) 450 451 eval := mock.Eval() 452 req := structs.EvalUpdateRequest{ 453 Evals: []*structs.Evaluation{eval}, 454 } 455 buf, err := structs.Encode(structs.EvalUpdateRequestType, req) 456 if err != nil { 457 t.Fatalf("err: %v", err) 458 } 459 460 resp := fsm.Apply(makeLog(buf)) 461 if resp != nil { 462 t.Fatalf("resp: %v", resp) 463 } 464 465 req2 := structs.EvalDeleteRequest{ 466 Evals: []string{eval.ID}, 467 } 468 buf, err = structs.Encode(structs.EvalDeleteRequestType, req2) 469 if err != nil { 470 t.Fatalf("err: %v", err) 471 } 472 473 resp = fsm.Apply(makeLog(buf)) 474 if resp != nil { 475 t.Fatalf("resp: %v", resp) 476 } 477 478 // Verify we are NOT registered 479 eval, err = fsm.State().EvalByID(req.Evals[0].ID) 480 if err != nil { 481 t.Fatalf("err: %v", err) 482 } 483 if eval != nil { 484 t.Fatalf("eval found!") 485 } 486 } 487 488 func TestFSM_UpsertAllocs(t *testing.T) { 489 fsm := testFSM(t) 490 491 alloc := mock.Alloc() 492 fsm.State().UpsertJobSummary(1, mock.JobSummary(alloc.JobID)) 493 req := structs.AllocUpdateRequest{ 494 Alloc: []*structs.Allocation{alloc}, 495 } 496 buf, err := structs.Encode(structs.AllocUpdateRequestType, req) 497 if err != nil { 498 t.Fatalf("err: %v", err) 499 } 500 501 resp := fsm.Apply(makeLog(buf)) 502 if resp != nil { 503 t.Fatalf("resp: %v", resp) 504 } 505 506 // Verify we are registered 507 out, err := fsm.State().AllocByID(alloc.ID) 508 if err != nil { 509 t.Fatalf("err: %v", err) 510 } 511 alloc.CreateIndex = out.CreateIndex 512 alloc.ModifyIndex = out.ModifyIndex 513 alloc.AllocModifyIndex = out.AllocModifyIndex 514 if !reflect.DeepEqual(alloc, out) { 515 t.Fatalf("bad: %#v %#v", alloc, out) 516 } 517 518 evictAlloc := new(structs.Allocation) 519 *evictAlloc = *alloc 520 evictAlloc.DesiredStatus = structs.AllocDesiredStatusEvict 521 req2 := structs.AllocUpdateRequest{ 522 Alloc: []*structs.Allocation{evictAlloc}, 523 } 524 buf, err = structs.Encode(structs.AllocUpdateRequestType, req2) 525 if err != nil { 526 t.Fatalf("err: %v", err) 527 } 528 529 resp = fsm.Apply(makeLog(buf)) 530 if resp != nil { 531 t.Fatalf("resp: %v", resp) 532 } 533 534 // Verify we are evicted 535 out, err = fsm.State().AllocByID(alloc.ID) 536 if err != nil { 537 t.Fatalf("err: %v", err) 538 } 539 if out.DesiredStatus != structs.AllocDesiredStatusEvict { 540 t.Fatalf("alloc found!") 541 } 542 } 543 544 func TestFSM_UpsertAllocs_SharedJob(t *testing.T) { 545 fsm := testFSM(t) 546 547 alloc := mock.Alloc() 548 fsm.State().UpsertJobSummary(1, mock.JobSummary(alloc.JobID)) 549 job := alloc.Job 550 alloc.Job = nil 551 req := structs.AllocUpdateRequest{ 552 Job: job, 553 Alloc: []*structs.Allocation{alloc}, 554 } 555 buf, err := structs.Encode(structs.AllocUpdateRequestType, req) 556 if err != nil { 557 t.Fatalf("err: %v", err) 558 } 559 560 resp := fsm.Apply(makeLog(buf)) 561 if resp != nil { 562 t.Fatalf("resp: %v", resp) 563 } 564 565 // Verify we are registered 566 out, err := fsm.State().AllocByID(alloc.ID) 567 if err != nil { 568 t.Fatalf("err: %v", err) 569 } 570 alloc.CreateIndex = out.CreateIndex 571 alloc.ModifyIndex = out.ModifyIndex 572 alloc.AllocModifyIndex = out.AllocModifyIndex 573 574 // Job should be re-attached 575 alloc.Job = job 576 if !reflect.DeepEqual(alloc, out) { 577 t.Fatalf("bad: %#v %#v", alloc, out) 578 } 579 580 // Ensure that the original job is used 581 evictAlloc := new(structs.Allocation) 582 *evictAlloc = *alloc 583 job = mock.Job() 584 job.Priority = 123 585 586 evictAlloc.Job = nil 587 evictAlloc.DesiredStatus = structs.AllocDesiredStatusEvict 588 req2 := structs.AllocUpdateRequest{ 589 Job: job, 590 Alloc: []*structs.Allocation{evictAlloc}, 591 } 592 buf, err = structs.Encode(structs.AllocUpdateRequestType, req2) 593 if err != nil { 594 t.Fatalf("err: %v", err) 595 } 596 597 resp = fsm.Apply(makeLog(buf)) 598 if resp != nil { 599 t.Fatalf("resp: %v", resp) 600 } 601 602 // Verify we are evicted 603 out, err = fsm.State().AllocByID(alloc.ID) 604 if err != nil { 605 t.Fatalf("err: %v", err) 606 } 607 if out.DesiredStatus != structs.AllocDesiredStatusEvict { 608 t.Fatalf("alloc found!") 609 } 610 if out.Job == nil || out.Job.Priority == 123 { 611 t.Fatalf("bad job") 612 } 613 } 614 615 func TestFSM_UpsertAllocs_StrippedResources(t *testing.T) { 616 fsm := testFSM(t) 617 618 alloc := mock.Alloc() 619 fsm.State().UpsertJobSummary(1, mock.JobSummary(alloc.JobID)) 620 job := alloc.Job 621 resources := alloc.Resources 622 alloc.Resources = nil 623 req := structs.AllocUpdateRequest{ 624 Job: job, 625 Alloc: []*structs.Allocation{alloc}, 626 } 627 buf, err := structs.Encode(structs.AllocUpdateRequestType, req) 628 if err != nil { 629 t.Fatalf("err: %v", err) 630 } 631 632 resp := fsm.Apply(makeLog(buf)) 633 if resp != nil { 634 t.Fatalf("resp: %v", resp) 635 } 636 637 // Verify we are registered 638 out, err := fsm.State().AllocByID(alloc.ID) 639 if err != nil { 640 t.Fatalf("err: %v", err) 641 } 642 alloc.CreateIndex = out.CreateIndex 643 alloc.ModifyIndex = out.ModifyIndex 644 alloc.AllocModifyIndex = out.AllocModifyIndex 645 646 // Resources should be recomputed 647 resources.DiskMB = alloc.Job.TaskGroups[0].EphemeralDisk.SizeMB 648 alloc.Resources = resources 649 if !reflect.DeepEqual(alloc, out) { 650 t.Fatalf("bad: %#v %#v", alloc, out) 651 } 652 } 653 654 func TestFSM_UpdateAllocFromClient_Unblock(t *testing.T) { 655 fsm := testFSM(t) 656 fsm.blockedEvals.SetEnabled(true) 657 state := fsm.State() 658 659 node := mock.Node() 660 state.UpsertNode(1, node) 661 662 // Mark an eval as blocked. 663 eval := mock.Eval() 664 eval.ClassEligibility = map[string]bool{node.ComputedClass: true} 665 fsm.blockedEvals.Block(eval) 666 667 bStats := fsm.blockedEvals.Stats() 668 if bStats.TotalBlocked != 1 { 669 t.Fatalf("bad: %#v", bStats) 670 } 671 672 // Create a completed eval 673 alloc := mock.Alloc() 674 alloc.NodeID = node.ID 675 alloc2 := mock.Alloc() 676 alloc2.NodeID = node.ID 677 state.UpsertJobSummary(8, mock.JobSummary(alloc.JobID)) 678 state.UpsertJobSummary(9, mock.JobSummary(alloc2.JobID)) 679 state.UpsertAllocs(10, []*structs.Allocation{alloc, alloc2}) 680 681 clientAlloc := new(structs.Allocation) 682 *clientAlloc = *alloc 683 clientAlloc.ClientStatus = structs.AllocClientStatusComplete 684 update2 := &structs.Allocation{ 685 ID: alloc2.ID, 686 ClientStatus: structs.AllocClientStatusRunning, 687 } 688 689 req := structs.AllocUpdateRequest{ 690 Alloc: []*structs.Allocation{clientAlloc, update2}, 691 } 692 buf, err := structs.Encode(structs.AllocClientUpdateRequestType, req) 693 if err != nil { 694 t.Fatalf("err: %v", err) 695 } 696 697 resp := fsm.Apply(makeLog(buf)) 698 if resp != nil { 699 t.Fatalf("resp: %v", resp) 700 } 701 702 // Verify we are updated 703 out, err := fsm.State().AllocByID(alloc.ID) 704 if err != nil { 705 t.Fatalf("err: %v", err) 706 } 707 clientAlloc.CreateIndex = out.CreateIndex 708 clientAlloc.ModifyIndex = out.ModifyIndex 709 if !reflect.DeepEqual(clientAlloc, out) { 710 t.Fatalf("bad: %#v %#v", clientAlloc, out) 711 } 712 713 out, err = fsm.State().AllocByID(alloc2.ID) 714 if err != nil { 715 t.Fatalf("err: %v", err) 716 } 717 alloc2.CreateIndex = out.CreateIndex 718 alloc2.ModifyIndex = out.ModifyIndex 719 alloc2.ClientStatus = structs.AllocClientStatusRunning 720 alloc2.TaskStates = nil 721 if !reflect.DeepEqual(alloc2, out) { 722 t.Fatalf("bad: %#v %#v", alloc2, out) 723 } 724 725 // Verify the eval was unblocked. 726 testutil.WaitForResult(func() (bool, error) { 727 bStats = fsm.blockedEvals.Stats() 728 if bStats.TotalBlocked != 0 { 729 return false, fmt.Errorf("bad: %#v %#v", bStats, out) 730 } 731 return true, nil 732 }, func(err error) { 733 t.Fatalf("err: %s", err) 734 }) 735 } 736 737 func TestFSM_UpdateAllocFromClient(t *testing.T) { 738 fsm := testFSM(t) 739 state := fsm.State() 740 741 alloc := mock.Alloc() 742 state.UpsertJobSummary(9, mock.JobSummary(alloc.JobID)) 743 state.UpsertAllocs(10, []*structs.Allocation{alloc}) 744 745 clientAlloc := new(structs.Allocation) 746 *clientAlloc = *alloc 747 clientAlloc.ClientStatus = structs.AllocClientStatusFailed 748 749 req := structs.AllocUpdateRequest{ 750 Alloc: []*structs.Allocation{clientAlloc}, 751 } 752 buf, err := structs.Encode(structs.AllocClientUpdateRequestType, req) 753 if err != nil { 754 t.Fatalf("err: %v", err) 755 } 756 757 resp := fsm.Apply(makeLog(buf)) 758 if resp != nil { 759 t.Fatalf("resp: %v", resp) 760 } 761 762 // Verify we are registered 763 out, err := fsm.State().AllocByID(alloc.ID) 764 if err != nil { 765 t.Fatalf("err: %v", err) 766 } 767 clientAlloc.CreateIndex = out.CreateIndex 768 clientAlloc.ModifyIndex = out.ModifyIndex 769 if !reflect.DeepEqual(clientAlloc, out) { 770 t.Fatalf("err: %#v,%#v", clientAlloc, out) 771 } 772 } 773 774 func TestFSM_UpsertVaultAccessor(t *testing.T) { 775 fsm := testFSM(t) 776 fsm.blockedEvals.SetEnabled(true) 777 778 va := mock.VaultAccessor() 779 va2 := mock.VaultAccessor() 780 req := structs.VaultAccessorsRequest{ 781 Accessors: []*structs.VaultAccessor{va, va2}, 782 } 783 buf, err := structs.Encode(structs.VaultAccessorRegisterRequestType, req) 784 if err != nil { 785 t.Fatalf("err: %v", err) 786 } 787 788 resp := fsm.Apply(makeLog(buf)) 789 if resp != nil { 790 t.Fatalf("resp: %v", resp) 791 } 792 793 // Verify we are registered 794 out1, err := fsm.State().VaultAccessor(va.Accessor) 795 if err != nil { 796 t.Fatalf("err: %v", err) 797 } 798 if out1 == nil { 799 t.Fatalf("not found!") 800 } 801 if out1.CreateIndex != 1 { 802 t.Fatalf("bad index: %d", out1.CreateIndex) 803 } 804 out2, err := fsm.State().VaultAccessor(va2.Accessor) 805 if err != nil { 806 t.Fatalf("err: %v", err) 807 } 808 if out2 == nil { 809 t.Fatalf("not found!") 810 } 811 if out1.CreateIndex != 1 { 812 t.Fatalf("bad index: %d", out2.CreateIndex) 813 } 814 815 tt := fsm.TimeTable() 816 index := tt.NearestIndex(time.Now().UTC()) 817 if index != 1 { 818 t.Fatalf("bad: %d", index) 819 } 820 } 821 822 func TestFSM_DeregisterVaultAccessor(t *testing.T) { 823 fsm := testFSM(t) 824 fsm.blockedEvals.SetEnabled(true) 825 826 va := mock.VaultAccessor() 827 va2 := mock.VaultAccessor() 828 accessors := []*structs.VaultAccessor{va, va2} 829 830 // Insert the accessors 831 if err := fsm.State().UpsertVaultAccessor(1000, accessors); err != nil { 832 t.Fatalf("bad: %v", err) 833 } 834 835 req := structs.VaultAccessorsRequest{ 836 Accessors: accessors, 837 } 838 buf, err := structs.Encode(structs.VaultAccessorDegisterRequestType, req) 839 if err != nil { 840 t.Fatalf("err: %v", err) 841 } 842 843 resp := fsm.Apply(makeLog(buf)) 844 if resp != nil { 845 t.Fatalf("resp: %v", resp) 846 } 847 848 out1, err := fsm.State().VaultAccessor(va.Accessor) 849 if err != nil { 850 t.Fatalf("err: %v", err) 851 } 852 if out1 != nil { 853 t.Fatalf("not deleted!") 854 } 855 856 tt := fsm.TimeTable() 857 index := tt.NearestIndex(time.Now().UTC()) 858 if index != 1 { 859 t.Fatalf("bad: %d", index) 860 } 861 } 862 863 func testSnapshotRestore(t *testing.T, fsm *nomadFSM) *nomadFSM { 864 // Snapshot 865 snap, err := fsm.Snapshot() 866 if err != nil { 867 t.Fatalf("err: %v", err) 868 } 869 defer snap.Release() 870 871 // Persist 872 buf := bytes.NewBuffer(nil) 873 sink := &MockSink{buf, false} 874 if err := snap.Persist(sink); err != nil { 875 t.Fatalf("err: %v", err) 876 } 877 878 // Try to restore on a new FSM 879 fsm2 := testFSM(t) 880 881 // Do a restore 882 if err := fsm2.Restore(sink); err != nil { 883 t.Fatalf("err: %v", err) 884 } 885 return fsm2 886 } 887 888 func TestFSM_SnapshotRestore_Nodes(t *testing.T) { 889 // Add some state 890 fsm := testFSM(t) 891 state := fsm.State() 892 node1 := mock.Node() 893 state.UpsertNode(1000, node1) 894 node2 := mock.Node() 895 state.UpsertNode(1001, node2) 896 897 // Verify the contents 898 fsm2 := testSnapshotRestore(t, fsm) 899 state2 := fsm2.State() 900 out1, _ := state2.NodeByID(node1.ID) 901 out2, _ := state2.NodeByID(node2.ID) 902 if !reflect.DeepEqual(node1, out1) { 903 t.Fatalf("bad: \n%#v\n%#v", out1, node1) 904 } 905 if !reflect.DeepEqual(node2, out2) { 906 t.Fatalf("bad: \n%#v\n%#v", out2, node2) 907 } 908 } 909 910 func TestFSM_SnapshotRestore_Jobs(t *testing.T) { 911 // Add some state 912 fsm := testFSM(t) 913 state := fsm.State() 914 job1 := mock.Job() 915 state.UpsertJob(1000, job1) 916 job2 := mock.Job() 917 state.UpsertJob(1001, job2) 918 919 // Verify the contents 920 fsm2 := testSnapshotRestore(t, fsm) 921 state2 := fsm2.State() 922 out1, _ := state2.JobByID(job1.ID) 923 out2, _ := state2.JobByID(job2.ID) 924 if !reflect.DeepEqual(job1, out1) { 925 t.Fatalf("bad: \n%#v\n%#v", out1, job1) 926 } 927 if !reflect.DeepEqual(job2, out2) { 928 t.Fatalf("bad: \n%#v\n%#v", out2, job2) 929 } 930 } 931 932 func TestFSM_SnapshotRestore_Evals(t *testing.T) { 933 // Add some state 934 fsm := testFSM(t) 935 state := fsm.State() 936 eval1 := mock.Eval() 937 state.UpsertEvals(1000, []*structs.Evaluation{eval1}) 938 eval2 := mock.Eval() 939 state.UpsertEvals(1001, []*structs.Evaluation{eval2}) 940 941 // Verify the contents 942 fsm2 := testSnapshotRestore(t, fsm) 943 state2 := fsm2.State() 944 out1, _ := state2.EvalByID(eval1.ID) 945 out2, _ := state2.EvalByID(eval2.ID) 946 if !reflect.DeepEqual(eval1, out1) { 947 t.Fatalf("bad: \n%#v\n%#v", out1, eval1) 948 } 949 if !reflect.DeepEqual(eval2, out2) { 950 t.Fatalf("bad: \n%#v\n%#v", out2, eval2) 951 } 952 } 953 954 func TestFSM_SnapshotRestore_Allocs(t *testing.T) { 955 // Add some state 956 fsm := testFSM(t) 957 state := fsm.State() 958 alloc1 := mock.Alloc() 959 alloc2 := mock.Alloc() 960 state.UpsertJobSummary(998, mock.JobSummary(alloc1.JobID)) 961 state.UpsertJobSummary(999, mock.JobSummary(alloc2.JobID)) 962 state.UpsertAllocs(1000, []*structs.Allocation{alloc1}) 963 state.UpsertAllocs(1001, []*structs.Allocation{alloc2}) 964 965 // Verify the contents 966 fsm2 := testSnapshotRestore(t, fsm) 967 state2 := fsm2.State() 968 out1, _ := state2.AllocByID(alloc1.ID) 969 out2, _ := state2.AllocByID(alloc2.ID) 970 if !reflect.DeepEqual(alloc1, out1) { 971 t.Fatalf("bad: \n%#v\n%#v", out1, alloc1) 972 } 973 if !reflect.DeepEqual(alloc2, out2) { 974 t.Fatalf("bad: \n%#v\n%#v", out2, alloc2) 975 } 976 } 977 978 func TestFSM_SnapshotRestore_Allocs_NoSharedResources(t *testing.T) { 979 // Add some state 980 fsm := testFSM(t) 981 state := fsm.State() 982 alloc1 := mock.Alloc() 983 alloc2 := mock.Alloc() 984 alloc1.SharedResources = nil 985 alloc2.SharedResources = nil 986 state.UpsertJobSummary(998, mock.JobSummary(alloc1.JobID)) 987 state.UpsertJobSummary(999, mock.JobSummary(alloc2.JobID)) 988 state.UpsertAllocs(1000, []*structs.Allocation{alloc1}) 989 state.UpsertAllocs(1001, []*structs.Allocation{alloc2}) 990 991 // Verify the contents 992 fsm2 := testSnapshotRestore(t, fsm) 993 state2 := fsm2.State() 994 out1, _ := state2.AllocByID(alloc1.ID) 995 out2, _ := state2.AllocByID(alloc2.ID) 996 alloc1.SharedResources = &structs.Resources{DiskMB: 150} 997 alloc2.SharedResources = &structs.Resources{DiskMB: 150} 998 999 if !reflect.DeepEqual(alloc1, out1) { 1000 t.Fatalf("bad: \n%#v\n%#v", out1, alloc1) 1001 } 1002 if !reflect.DeepEqual(alloc2, out2) { 1003 t.Fatalf("bad: \n%#v\n%#v", out2, alloc2) 1004 } 1005 } 1006 1007 func TestFSM_SnapshotRestore_Indexes(t *testing.T) { 1008 // Add some state 1009 fsm := testFSM(t) 1010 state := fsm.State() 1011 node1 := mock.Node() 1012 state.UpsertNode(1000, node1) 1013 1014 // Verify the contents 1015 fsm2 := testSnapshotRestore(t, fsm) 1016 state2 := fsm2.State() 1017 1018 index, err := state2.Index("nodes") 1019 if err != nil { 1020 t.Fatalf("err: %v", err) 1021 } 1022 if index != 1000 { 1023 t.Fatalf("bad: %d", index) 1024 } 1025 } 1026 1027 func TestFSM_SnapshotRestore_TimeTable(t *testing.T) { 1028 // Add some state 1029 fsm := testFSM(t) 1030 1031 tt := fsm.TimeTable() 1032 start := time.Now().UTC() 1033 tt.Witness(1000, start) 1034 tt.Witness(2000, start.Add(10*time.Minute)) 1035 1036 // Verify the contents 1037 fsm2 := testSnapshotRestore(t, fsm) 1038 1039 tt2 := fsm2.TimeTable() 1040 if tt2.NearestTime(1500) != start { 1041 t.Fatalf("bad") 1042 } 1043 if tt2.NearestIndex(start.Add(15*time.Minute)) != 2000 { 1044 t.Fatalf("bad") 1045 } 1046 } 1047 1048 func TestFSM_SnapshotRestore_PeriodicLaunches(t *testing.T) { 1049 // Add some state 1050 fsm := testFSM(t) 1051 state := fsm.State() 1052 job1 := mock.Job() 1053 launch1 := &structs.PeriodicLaunch{ID: job1.ID, Launch: time.Now()} 1054 state.UpsertPeriodicLaunch(1000, launch1) 1055 job2 := mock.Job() 1056 launch2 := &structs.PeriodicLaunch{ID: job2.ID, Launch: time.Now()} 1057 state.UpsertPeriodicLaunch(1001, launch2) 1058 1059 // Verify the contents 1060 fsm2 := testSnapshotRestore(t, fsm) 1061 state2 := fsm2.State() 1062 out1, _ := state2.PeriodicLaunchByID(launch1.ID) 1063 out2, _ := state2.PeriodicLaunchByID(launch2.ID) 1064 if !reflect.DeepEqual(launch1, out1) { 1065 t.Fatalf("bad: \n%#v\n%#v", out1, job1) 1066 } 1067 if !reflect.DeepEqual(launch2, out2) { 1068 t.Fatalf("bad: \n%#v\n%#v", out2, job2) 1069 } 1070 } 1071 1072 func TestFSM_SnapshotRestore_JobSummary(t *testing.T) { 1073 // Add some state 1074 fsm := testFSM(t) 1075 state := fsm.State() 1076 1077 job1 := mock.Job() 1078 state.UpsertJob(1000, job1) 1079 js1, _ := state.JobSummaryByID(job1.ID) 1080 1081 job2 := mock.Job() 1082 state.UpsertJob(1001, job2) 1083 js2, _ := state.JobSummaryByID(job2.ID) 1084 1085 // Verify the contents 1086 fsm2 := testSnapshotRestore(t, fsm) 1087 state2 := fsm2.State() 1088 out1, _ := state2.JobSummaryByID(job1.ID) 1089 out2, _ := state2.JobSummaryByID(job2.ID) 1090 if !reflect.DeepEqual(js1, out1) { 1091 t.Fatalf("bad: \n%#v\n%#v", js1, out1) 1092 } 1093 if !reflect.DeepEqual(js2, out2) { 1094 t.Fatalf("bad: \n%#v\n%#v", js2, out2) 1095 } 1096 } 1097 1098 func TestFSM_SnapshotRestore_VaultAccessors(t *testing.T) { 1099 // Add some state 1100 fsm := testFSM(t) 1101 state := fsm.State() 1102 a1 := mock.VaultAccessor() 1103 a2 := mock.VaultAccessor() 1104 state.UpsertVaultAccessor(1000, []*structs.VaultAccessor{a1, a2}) 1105 1106 // Verify the contents 1107 fsm2 := testSnapshotRestore(t, fsm) 1108 state2 := fsm2.State() 1109 out1, _ := state2.VaultAccessor(a1.Accessor) 1110 out2, _ := state2.VaultAccessor(a2.Accessor) 1111 if !reflect.DeepEqual(a1, out1) { 1112 t.Fatalf("bad: \n%#v\n%#v", out1, a1) 1113 } 1114 if !reflect.DeepEqual(a2, out2) { 1115 t.Fatalf("bad: \n%#v\n%#v", out2, a2) 1116 } 1117 } 1118 1119 func TestFSM_SnapshotRestore_AddMissingSummary(t *testing.T) { 1120 // Add some state 1121 fsm := testFSM(t) 1122 state := fsm.State() 1123 1124 // make an allocation 1125 alloc := mock.Alloc() 1126 state.UpsertJob(1010, alloc.Job) 1127 state.UpsertAllocs(1011, []*structs.Allocation{alloc}) 1128 1129 // Delete the summary 1130 state.DeleteJobSummary(1040, alloc.Job.ID) 1131 1132 // Delete the index 1133 if err := state.RemoveIndex("job_summary"); err != nil { 1134 t.Fatalf("err: %v", err) 1135 } 1136 1137 fsm2 := testSnapshotRestore(t, fsm) 1138 state2 := fsm2.State() 1139 latestIndex, _ := state.LatestIndex() 1140 1141 out, _ := state2.JobSummaryByID(alloc.Job.ID) 1142 expected := structs.JobSummary{ 1143 JobID: alloc.Job.ID, 1144 Summary: map[string]structs.TaskGroupSummary{ 1145 "web": structs.TaskGroupSummary{ 1146 Starting: 1, 1147 }, 1148 }, 1149 CreateIndex: 1010, 1150 ModifyIndex: latestIndex, 1151 } 1152 if !reflect.DeepEqual(&expected, out) { 1153 t.Fatalf("expected: %#v, actual: %#v", &expected, out) 1154 } 1155 } 1156 1157 func TestFSM_ReconcileSummaries(t *testing.T) { 1158 // Add some state 1159 fsm := testFSM(t) 1160 state := fsm.State() 1161 1162 // Add a node 1163 node := mock.Node() 1164 state.UpsertNode(800, node) 1165 1166 // Make a job so that none of the tasks can be placed 1167 job1 := mock.Job() 1168 job1.TaskGroups[0].Tasks[0].Resources.CPU = 5000 1169 state.UpsertJob(1000, job1) 1170 1171 // make a job which can make partial progress 1172 alloc := mock.Alloc() 1173 alloc.NodeID = node.ID 1174 state.UpsertJob(1010, alloc.Job) 1175 state.UpsertAllocs(1011, []*structs.Allocation{alloc}) 1176 1177 // Delete the summaries 1178 state.DeleteJobSummary(1030, job1.ID) 1179 state.DeleteJobSummary(1040, alloc.Job.ID) 1180 1181 req := structs.GenericRequest{} 1182 buf, err := structs.Encode(structs.ReconcileJobSummariesRequestType, req) 1183 if err != nil { 1184 t.Fatalf("err: %v", err) 1185 } 1186 1187 resp := fsm.Apply(makeLog(buf)) 1188 if resp != nil { 1189 t.Fatalf("resp: %v", resp) 1190 } 1191 1192 out1, _ := state.JobSummaryByID(job1.ID) 1193 expected := structs.JobSummary{ 1194 JobID: job1.ID, 1195 Summary: map[string]structs.TaskGroupSummary{ 1196 "web": structs.TaskGroupSummary{ 1197 Queued: 10, 1198 }, 1199 }, 1200 CreateIndex: 1000, 1201 ModifyIndex: out1.ModifyIndex, 1202 } 1203 if !reflect.DeepEqual(&expected, out1) { 1204 t.Fatalf("expected: %#v, actual: %#v", &expected, out1) 1205 } 1206 1207 // This exercises the code path which adds the allocations made by the 1208 // planner and the number of unplaced allocations in the reconcile summaries 1209 // codepath 1210 out2, _ := state.JobSummaryByID(alloc.Job.ID) 1211 expected = structs.JobSummary{ 1212 JobID: alloc.Job.ID, 1213 Summary: map[string]structs.TaskGroupSummary{ 1214 "web": structs.TaskGroupSummary{ 1215 Queued: 10, 1216 Starting: 1, 1217 }, 1218 }, 1219 CreateIndex: 1010, 1220 ModifyIndex: out2.ModifyIndex, 1221 } 1222 if !reflect.DeepEqual(&expected, out2) { 1223 t.Fatalf("expected: %#v, actual: %#v", &expected, out2) 1224 } 1225 }