github.com/ThomasObenaus/nomad@v0.11.1/nomad/fsm_test.go (about) 1 package nomad 2 3 import ( 4 "bytes" 5 "fmt" 6 "reflect" 7 "strings" 8 "testing" 9 "time" 10 11 "github.com/google/go-cmp/cmp" 12 memdb "github.com/hashicorp/go-memdb" 13 "github.com/hashicorp/nomad/helper" 14 "github.com/hashicorp/nomad/helper/testlog" 15 "github.com/hashicorp/nomad/helper/uuid" 16 "github.com/hashicorp/nomad/nomad/mock" 17 "github.com/hashicorp/nomad/nomad/state" 18 "github.com/hashicorp/nomad/nomad/structs" 19 "github.com/hashicorp/nomad/testutil" 20 "github.com/hashicorp/raft" 21 "github.com/kr/pretty" 22 "github.com/stretchr/testify/assert" 23 "github.com/stretchr/testify/require" 24 ) 25 26 type MockSink struct { 27 *bytes.Buffer 28 cancel bool 29 } 30 31 func (m *MockSink) ID() string { 32 return "Mock" 33 } 34 35 func (m *MockSink) Cancel() error { 36 m.cancel = true 37 return nil 38 } 39 40 func (m *MockSink) Close() error { 41 return nil 42 } 43 44 func testStateStore(t *testing.T) *state.StateStore { 45 return state.TestStateStore(t) 46 } 47 48 func testFSM(t *testing.T) *nomadFSM { 49 broker := testBroker(t, 0) 50 dispatcher, _ := testPeriodicDispatcher(t) 51 logger := testlog.HCLogger(t) 52 fsmConfig := &FSMConfig{ 53 EvalBroker: broker, 54 Periodic: dispatcher, 55 Blocked: NewBlockedEvals(broker, logger), 56 Logger: logger, 57 Region: "global", 58 } 59 fsm, err := NewFSM(fsmConfig) 60 if err != nil { 61 t.Fatalf("err: %v", err) 62 } 63 if fsm == nil { 64 t.Fatalf("missing fsm") 65 } 66 state.TestInitState(t, fsm.state) 67 return fsm 68 } 69 70 func makeLog(buf []byte) *raft.Log { 71 return &raft.Log{ 72 Index: 1, 73 Term: 1, 74 Type: raft.LogCommand, 75 Data: buf, 76 } 77 } 78 79 func TestFSM_UpsertNodeEvents(t *testing.T) { 80 t.Parallel() 81 require := require.New(t) 82 fsm := testFSM(t) 83 state := fsm.State() 84 85 node := mock.Node() 86 87 err := state.UpsertNode(1000, node) 88 if err != nil { 89 t.Fatalf("err: %v", err) 90 } 91 92 nodeEvent := &structs.NodeEvent{ 93 Message: "Heartbeating failed", 94 Subsystem: "Heartbeat", 95 Timestamp: time.Now(), 96 } 97 98 nodeEvents := []*structs.NodeEvent{nodeEvent} 99 allEvents := map[string][]*structs.NodeEvent{node.ID: nodeEvents} 100 101 req := structs.EmitNodeEventsRequest{ 102 NodeEvents: allEvents, 103 WriteRequest: structs.WriteRequest{Region: "global"}, 104 } 105 buf, err := structs.Encode(structs.UpsertNodeEventsType, req) 106 require.Nil(err) 107 108 // the response in this case will be an error 109 resp := fsm.Apply(makeLog(buf)) 110 require.Nil(resp) 111 112 ws := memdb.NewWatchSet() 113 out, err := state.NodeByID(ws, node.ID) 114 require.Nil(err) 115 116 require.Equal(2, len(out.Events)) 117 118 first := out.Events[1] 119 require.Equal(uint64(1), first.CreateIndex) 120 require.Equal("Heartbeating failed", first.Message) 121 } 122 123 func TestFSM_UpsertNode(t *testing.T) { 124 t.Parallel() 125 fsm := testFSM(t) 126 fsm.blockedEvals.SetEnabled(true) 127 128 node := mock.Node() 129 130 // Mark an eval as blocked. 131 eval := mock.Eval() 132 eval.ClassEligibility = map[string]bool{node.ComputedClass: true} 133 fsm.blockedEvals.Block(eval) 134 135 req := structs.NodeRegisterRequest{ 136 Node: node, 137 } 138 buf, err := structs.Encode(structs.NodeRegisterRequestType, req) 139 if err != nil { 140 t.Fatalf("err: %v", err) 141 } 142 143 resp := fsm.Apply(makeLog(buf)) 144 if resp != nil { 145 t.Fatalf("resp: %v", resp) 146 } 147 148 // Verify we are registered 149 ws := memdb.NewWatchSet() 150 n, err := fsm.State().NodeByID(ws, req.Node.ID) 151 if err != nil { 152 t.Fatalf("err: %v", err) 153 } 154 if n == nil { 155 t.Fatalf("not found!") 156 } 157 if n.CreateIndex != 1 { 158 t.Fatalf("bad index: %d", node.CreateIndex) 159 } 160 161 tt := fsm.TimeTable() 162 index := tt.NearestIndex(time.Now().UTC()) 163 if index != 1 { 164 t.Fatalf("bad: %d", index) 165 } 166 167 // Verify the eval was unblocked. 168 testutil.WaitForResult(func() (bool, error) { 169 bStats := fsm.blockedEvals.Stats() 170 if bStats.TotalBlocked != 0 { 171 return false, fmt.Errorf("bad: %#v", bStats) 172 } 173 return true, nil 174 }, func(err error) { 175 t.Fatalf("err: %s", err) 176 }) 177 178 } 179 180 func TestFSM_UpsertNode_Canonicalize(t *testing.T) { 181 t.Parallel() 182 require := require.New(t) 183 184 fsm := testFSM(t) 185 fsm.blockedEvals.SetEnabled(true) 186 187 // Setup a node without eligibility 188 node := mock.Node() 189 node.SchedulingEligibility = "" 190 191 req := structs.NodeRegisterRequest{ 192 Node: node, 193 } 194 buf, err := structs.Encode(structs.NodeRegisterRequestType, req) 195 require.Nil(err) 196 197 resp := fsm.Apply(makeLog(buf)) 198 require.Nil(resp) 199 200 // Verify we are registered 201 ws := memdb.NewWatchSet() 202 n, err := fsm.State().NodeByID(ws, req.Node.ID) 203 require.Nil(err) 204 require.NotNil(n) 205 require.EqualValues(1, n.CreateIndex) 206 require.Equal(structs.NodeSchedulingEligible, n.SchedulingEligibility) 207 } 208 209 func TestFSM_DeregisterNode(t *testing.T) { 210 t.Parallel() 211 fsm := testFSM(t) 212 213 node := mock.Node() 214 req := structs.NodeRegisterRequest{ 215 Node: node, 216 } 217 buf, err := structs.Encode(structs.NodeRegisterRequestType, req) 218 if err != nil { 219 t.Fatalf("err: %v", err) 220 } 221 222 resp := fsm.Apply(makeLog(buf)) 223 if resp != nil { 224 t.Fatalf("resp: %v", resp) 225 } 226 227 req2 := structs.NodeBatchDeregisterRequest{ 228 NodeIDs: []string{node.ID}, 229 } 230 buf, err = structs.Encode(structs.NodeBatchDeregisterRequestType, req2) 231 if err != nil { 232 t.Fatalf("err: %v", err) 233 } 234 235 resp = fsm.Apply(makeLog(buf)) 236 if resp != nil { 237 t.Fatalf("resp: %v", resp) 238 } 239 240 // Verify we are NOT registered 241 ws := memdb.NewWatchSet() 242 node, err = fsm.State().NodeByID(ws, req.Node.ID) 243 if err != nil { 244 t.Fatalf("err: %v", err) 245 } 246 if node != nil { 247 t.Fatalf("node found!") 248 } 249 } 250 251 func TestFSM_UpdateNodeStatus(t *testing.T) { 252 t.Parallel() 253 require := require.New(t) 254 fsm := testFSM(t) 255 fsm.blockedEvals.SetEnabled(true) 256 257 node := mock.Node() 258 req := structs.NodeRegisterRequest{ 259 Node: node, 260 } 261 buf, err := structs.Encode(structs.NodeRegisterRequestType, req) 262 require.NoError(err) 263 264 resp := fsm.Apply(makeLog(buf)) 265 require.Nil(resp) 266 267 // Mark an eval as blocked. 268 eval := mock.Eval() 269 eval.ClassEligibility = map[string]bool{node.ComputedClass: true} 270 fsm.blockedEvals.Block(eval) 271 272 event := &structs.NodeEvent{ 273 Message: "Node ready foo", 274 Subsystem: structs.NodeEventSubsystemCluster, 275 Timestamp: time.Now(), 276 } 277 req2 := structs.NodeUpdateStatusRequest{ 278 NodeID: node.ID, 279 Status: structs.NodeStatusReady, 280 NodeEvent: event, 281 } 282 buf, err = structs.Encode(structs.NodeUpdateStatusRequestType, req2) 283 require.NoError(err) 284 285 resp = fsm.Apply(makeLog(buf)) 286 require.Nil(resp) 287 288 // Verify the status is ready. 289 ws := memdb.NewWatchSet() 290 node, err = fsm.State().NodeByID(ws, req.Node.ID) 291 require.NoError(err) 292 require.Equal(structs.NodeStatusReady, node.Status) 293 require.Len(node.Events, 2) 294 require.Equal(event.Message, node.Events[1].Message) 295 296 // Verify the eval was unblocked. 297 testutil.WaitForResult(func() (bool, error) { 298 bStats := fsm.blockedEvals.Stats() 299 if bStats.TotalBlocked != 0 { 300 return false, fmt.Errorf("bad: %#v", bStats) 301 } 302 return true, nil 303 }, func(err error) { 304 t.Fatalf("err: %s", err) 305 }) 306 } 307 308 func TestFSM_BatchUpdateNodeDrain(t *testing.T) { 309 t.Parallel() 310 require := require.New(t) 311 fsm := testFSM(t) 312 313 node := mock.Node() 314 req := structs.NodeRegisterRequest{ 315 Node: node, 316 } 317 buf, err := structs.Encode(structs.NodeRegisterRequestType, req) 318 require.Nil(err) 319 320 resp := fsm.Apply(makeLog(buf)) 321 require.Nil(resp) 322 323 strategy := &structs.DrainStrategy{ 324 DrainSpec: structs.DrainSpec{ 325 Deadline: 10 * time.Second, 326 }, 327 } 328 event := &structs.NodeEvent{ 329 Message: "Drain strategy enabled", 330 Subsystem: structs.NodeEventSubsystemDrain, 331 Timestamp: time.Now(), 332 } 333 req2 := structs.BatchNodeUpdateDrainRequest{ 334 Updates: map[string]*structs.DrainUpdate{ 335 node.ID: { 336 DrainStrategy: strategy, 337 }, 338 }, 339 NodeEvents: map[string]*structs.NodeEvent{ 340 node.ID: event, 341 }, 342 } 343 buf, err = structs.Encode(structs.BatchNodeUpdateDrainRequestType, req2) 344 require.Nil(err) 345 346 resp = fsm.Apply(makeLog(buf)) 347 require.Nil(resp) 348 349 // Verify drain is set 350 ws := memdb.NewWatchSet() 351 node, err = fsm.State().NodeByID(ws, req.Node.ID) 352 require.Nil(err) 353 require.True(node.Drain) 354 require.Equal(node.DrainStrategy, strategy) 355 require.Len(node.Events, 2) 356 } 357 358 func TestFSM_UpdateNodeDrain(t *testing.T) { 359 t.Parallel() 360 require := require.New(t) 361 fsm := testFSM(t) 362 363 node := mock.Node() 364 req := structs.NodeRegisterRequest{ 365 Node: node, 366 } 367 buf, err := structs.Encode(structs.NodeRegisterRequestType, req) 368 require.Nil(err) 369 370 resp := fsm.Apply(makeLog(buf)) 371 require.Nil(resp) 372 373 strategy := &structs.DrainStrategy{ 374 DrainSpec: structs.DrainSpec{ 375 Deadline: 10 * time.Second, 376 }, 377 } 378 req2 := structs.NodeUpdateDrainRequest{ 379 NodeID: node.ID, 380 DrainStrategy: strategy, 381 NodeEvent: &structs.NodeEvent{ 382 Message: "Drain strategy enabled", 383 Subsystem: structs.NodeEventSubsystemDrain, 384 Timestamp: time.Now(), 385 }, 386 } 387 buf, err = structs.Encode(structs.NodeUpdateDrainRequestType, req2) 388 require.Nil(err) 389 390 resp = fsm.Apply(makeLog(buf)) 391 require.Nil(resp) 392 393 // Verify we are NOT registered 394 ws := memdb.NewWatchSet() 395 node, err = fsm.State().NodeByID(ws, req.Node.ID) 396 require.Nil(err) 397 require.True(node.Drain) 398 require.Equal(node.DrainStrategy, strategy) 399 require.Len(node.Events, 2) 400 } 401 402 func TestFSM_UpdateNodeDrain_Pre08_Compatibility(t *testing.T) { 403 t.Parallel() 404 require := require.New(t) 405 fsm := testFSM(t) 406 407 // Force a node into the state store without eligiblity 408 node := mock.Node() 409 node.SchedulingEligibility = "" 410 require.Nil(fsm.State().UpsertNode(1, node)) 411 412 // Do an old style drain 413 req := structs.NodeUpdateDrainRequest{ 414 NodeID: node.ID, 415 Drain: true, 416 } 417 buf, err := structs.Encode(structs.NodeUpdateDrainRequestType, req) 418 require.Nil(err) 419 420 resp := fsm.Apply(makeLog(buf)) 421 require.Nil(resp) 422 423 // Verify we have upgraded to a force drain 424 ws := memdb.NewWatchSet() 425 node, err = fsm.State().NodeByID(ws, req.NodeID) 426 require.Nil(err) 427 require.True(node.Drain) 428 429 expected := &structs.DrainStrategy{ 430 DrainSpec: structs.DrainSpec{ 431 Deadline: -1 * time.Second, 432 }, 433 } 434 require.Equal(expected, node.DrainStrategy) 435 } 436 437 func TestFSM_UpdateNodeEligibility(t *testing.T) { 438 t.Parallel() 439 require := require.New(t) 440 fsm := testFSM(t) 441 442 node := mock.Node() 443 req := structs.NodeRegisterRequest{ 444 Node: node, 445 } 446 buf, err := structs.Encode(structs.NodeRegisterRequestType, req) 447 require.Nil(err) 448 449 resp := fsm.Apply(makeLog(buf)) 450 require.Nil(resp) 451 452 event := &structs.NodeEvent{ 453 Message: "Node marked as ineligible", 454 Subsystem: structs.NodeEventSubsystemCluster, 455 Timestamp: time.Now(), 456 } 457 458 // Set the eligibility 459 req2 := structs.NodeUpdateEligibilityRequest{ 460 NodeID: node.ID, 461 Eligibility: structs.NodeSchedulingIneligible, 462 NodeEvent: event, 463 } 464 buf, err = structs.Encode(structs.NodeUpdateEligibilityRequestType, req2) 465 require.Nil(err) 466 467 resp = fsm.Apply(makeLog(buf)) 468 require.Nil(resp) 469 470 // Lookup the node and check 471 node, err = fsm.State().NodeByID(nil, req.Node.ID) 472 require.Nil(err) 473 require.Equal(node.SchedulingEligibility, structs.NodeSchedulingIneligible) 474 require.Len(node.Events, 2) 475 require.Equal(event.Message, node.Events[1].Message) 476 477 // Update the drain 478 strategy := &structs.DrainStrategy{ 479 DrainSpec: structs.DrainSpec{ 480 Deadline: 10 * time.Second, 481 }, 482 } 483 req3 := structs.NodeUpdateDrainRequest{ 484 NodeID: node.ID, 485 DrainStrategy: strategy, 486 } 487 buf, err = structs.Encode(structs.NodeUpdateDrainRequestType, req3) 488 require.Nil(err) 489 resp = fsm.Apply(makeLog(buf)) 490 require.Nil(resp) 491 492 // Try forcing eligibility 493 req4 := structs.NodeUpdateEligibilityRequest{ 494 NodeID: node.ID, 495 Eligibility: structs.NodeSchedulingEligible, 496 } 497 buf, err = structs.Encode(structs.NodeUpdateEligibilityRequestType, req4) 498 require.Nil(err) 499 500 resp = fsm.Apply(makeLog(buf)) 501 require.NotNil(resp) 502 err, ok := resp.(error) 503 require.True(ok) 504 require.Contains(err.Error(), "draining") 505 } 506 507 func TestFSM_UpdateNodeEligibility_Unblock(t *testing.T) { 508 t.Parallel() 509 require := require.New(t) 510 fsm := testFSM(t) 511 512 node := mock.Node() 513 req := structs.NodeRegisterRequest{ 514 Node: node, 515 } 516 buf, err := structs.Encode(structs.NodeRegisterRequestType, req) 517 require.Nil(err) 518 519 resp := fsm.Apply(makeLog(buf)) 520 require.Nil(resp) 521 522 // Set the eligibility 523 req2 := structs.NodeUpdateEligibilityRequest{ 524 NodeID: node.ID, 525 Eligibility: structs.NodeSchedulingIneligible, 526 } 527 buf, err = structs.Encode(structs.NodeUpdateEligibilityRequestType, req2) 528 require.Nil(err) 529 530 resp = fsm.Apply(makeLog(buf)) 531 require.Nil(resp) 532 533 // Mark an eval as blocked. 534 eval := mock.Eval() 535 eval.ClassEligibility = map[string]bool{node.ComputedClass: true} 536 fsm.blockedEvals.Block(eval) 537 538 // Set eligible 539 req4 := structs.NodeUpdateEligibilityRequest{ 540 NodeID: node.ID, 541 Eligibility: structs.NodeSchedulingEligible, 542 } 543 buf, err = structs.Encode(structs.NodeUpdateEligibilityRequestType, req4) 544 require.Nil(err) 545 546 resp = fsm.Apply(makeLog(buf)) 547 require.Nil(resp) 548 549 // Verify the eval was unblocked. 550 testutil.WaitForResult(func() (bool, error) { 551 bStats := fsm.blockedEvals.Stats() 552 if bStats.TotalBlocked != 0 { 553 return false, fmt.Errorf("bad: %#v", bStats) 554 } 555 return true, nil 556 }, func(err error) { 557 t.Fatalf("err: %s", err) 558 }) 559 } 560 561 func TestFSM_RegisterJob(t *testing.T) { 562 t.Parallel() 563 fsm := testFSM(t) 564 565 job := mock.PeriodicJob() 566 req := structs.JobRegisterRequest{ 567 Job: job, 568 WriteRequest: structs.WriteRequest{ 569 Namespace: job.Namespace, 570 }, 571 } 572 buf, err := structs.Encode(structs.JobRegisterRequestType, req) 573 if err != nil { 574 t.Fatalf("err: %v", err) 575 } 576 577 resp := fsm.Apply(makeLog(buf)) 578 if resp != nil { 579 t.Fatalf("resp: %v", resp) 580 } 581 582 // Verify we are registered 583 ws := memdb.NewWatchSet() 584 jobOut, err := fsm.State().JobByID(ws, req.Namespace, req.Job.ID) 585 if err != nil { 586 t.Fatalf("err: %v", err) 587 } 588 if jobOut == nil { 589 t.Fatalf("not found!") 590 } 591 if jobOut.CreateIndex != 1 { 592 t.Fatalf("bad index: %d", jobOut.CreateIndex) 593 } 594 595 // Verify it was added to the periodic runner. 596 tuple := structs.NamespacedID{ 597 ID: job.ID, 598 Namespace: job.Namespace, 599 } 600 if _, ok := fsm.periodicDispatcher.tracked[tuple]; !ok { 601 t.Fatal("job not added to periodic runner") 602 } 603 604 // Verify the launch time was tracked. 605 launchOut, err := fsm.State().PeriodicLaunchByID(ws, req.Namespace, req.Job.ID) 606 if err != nil { 607 t.Fatalf("err: %v", err) 608 } 609 if launchOut == nil { 610 t.Fatalf("not found!") 611 } 612 if launchOut.Launch.IsZero() { 613 t.Fatalf("bad launch time: %v", launchOut.Launch) 614 } 615 } 616 617 func TestFSM_RegisterPeriodicJob_NonLeader(t *testing.T) { 618 t.Parallel() 619 fsm := testFSM(t) 620 621 // Disable the dispatcher 622 fsm.periodicDispatcher.SetEnabled(false) 623 624 job := mock.PeriodicJob() 625 req := structs.JobRegisterRequest{ 626 Job: job, 627 WriteRequest: structs.WriteRequest{ 628 Namespace: job.Namespace, 629 }, 630 } 631 buf, err := structs.Encode(structs.JobRegisterRequestType, req) 632 if err != nil { 633 t.Fatalf("err: %v", err) 634 } 635 636 resp := fsm.Apply(makeLog(buf)) 637 if resp != nil { 638 t.Fatalf("resp: %v", resp) 639 } 640 641 // Verify we are registered 642 ws := memdb.NewWatchSet() 643 jobOut, err := fsm.State().JobByID(ws, req.Namespace, req.Job.ID) 644 if err != nil { 645 t.Fatalf("err: %v", err) 646 } 647 if jobOut == nil { 648 t.Fatalf("not found!") 649 } 650 if jobOut.CreateIndex != 1 { 651 t.Fatalf("bad index: %d", jobOut.CreateIndex) 652 } 653 654 // Verify it wasn't added to the periodic runner. 655 tuple := structs.NamespacedID{ 656 ID: job.ID, 657 Namespace: job.Namespace, 658 } 659 if _, ok := fsm.periodicDispatcher.tracked[tuple]; ok { 660 t.Fatal("job added to periodic runner") 661 } 662 663 // Verify the launch time was tracked. 664 launchOut, err := fsm.State().PeriodicLaunchByID(ws, req.Namespace, req.Job.ID) 665 if err != nil { 666 t.Fatalf("err: %v", err) 667 } 668 if launchOut == nil { 669 t.Fatalf("not found!") 670 } 671 if launchOut.Launch.IsZero() { 672 t.Fatalf("bad launch time: %v", launchOut.Launch) 673 } 674 } 675 676 func TestFSM_RegisterJob_BadNamespace(t *testing.T) { 677 t.Parallel() 678 fsm := testFSM(t) 679 680 job := mock.Job() 681 job.Namespace = "foo" 682 req := structs.JobRegisterRequest{ 683 Job: job, 684 WriteRequest: structs.WriteRequest{ 685 Namespace: job.Namespace, 686 }, 687 } 688 buf, err := structs.Encode(structs.JobRegisterRequestType, req) 689 if err != nil { 690 t.Fatalf("err: %v", err) 691 } 692 693 resp := fsm.Apply(makeLog(buf)) 694 if resp == nil { 695 t.Fatalf("no resp: %v", resp) 696 } 697 err, ok := resp.(error) 698 if !ok { 699 t.Fatalf("resp not of error type: %T %v", resp, resp) 700 } 701 if !strings.Contains(err.Error(), "nonexistent namespace") { 702 t.Fatalf("bad error: %v", err) 703 } 704 705 // Verify we are not registered 706 ws := memdb.NewWatchSet() 707 jobOut, err := fsm.State().JobByID(ws, req.Namespace, req.Job.ID) 708 if err != nil { 709 t.Fatalf("err: %v", err) 710 } 711 if jobOut != nil { 712 t.Fatalf("job found!") 713 } 714 } 715 716 func TestFSM_DeregisterJob_Purge(t *testing.T) { 717 t.Parallel() 718 fsm := testFSM(t) 719 720 job := mock.PeriodicJob() 721 req := structs.JobRegisterRequest{ 722 Job: job, 723 WriteRequest: structs.WriteRequest{ 724 Namespace: job.Namespace, 725 }, 726 } 727 buf, err := structs.Encode(structs.JobRegisterRequestType, req) 728 if err != nil { 729 t.Fatalf("err: %v", err) 730 } 731 732 resp := fsm.Apply(makeLog(buf)) 733 if resp != nil { 734 t.Fatalf("resp: %v", resp) 735 } 736 737 req2 := structs.JobDeregisterRequest{ 738 JobID: job.ID, 739 Purge: true, 740 WriteRequest: structs.WriteRequest{ 741 Namespace: job.Namespace, 742 }, 743 } 744 buf, err = structs.Encode(structs.JobDeregisterRequestType, req2) 745 if err != nil { 746 t.Fatalf("err: %v", err) 747 } 748 749 resp = fsm.Apply(makeLog(buf)) 750 if resp != nil { 751 t.Fatalf("resp: %v", resp) 752 } 753 754 // Verify we are NOT registered 755 ws := memdb.NewWatchSet() 756 jobOut, err := fsm.State().JobByID(ws, req.Namespace, req.Job.ID) 757 if err != nil { 758 t.Fatalf("err: %v", err) 759 } 760 if jobOut != nil { 761 t.Fatalf("job found!") 762 } 763 764 // Verify it was removed from the periodic runner. 765 tuple := structs.NamespacedID{ 766 ID: job.ID, 767 Namespace: job.Namespace, 768 } 769 if _, ok := fsm.periodicDispatcher.tracked[tuple]; ok { 770 t.Fatal("job not removed from periodic runner") 771 } 772 773 // Verify it was removed from the periodic launch table. 774 launchOut, err := fsm.State().PeriodicLaunchByID(ws, req.Namespace, req.Job.ID) 775 if err != nil { 776 t.Fatalf("err: %v", err) 777 } 778 if launchOut != nil { 779 t.Fatalf("launch found!") 780 } 781 } 782 783 func TestFSM_DeregisterJob_NoPurge(t *testing.T) { 784 t.Parallel() 785 fsm := testFSM(t) 786 787 job := mock.PeriodicJob() 788 req := structs.JobRegisterRequest{ 789 Job: job, 790 WriteRequest: structs.WriteRequest{ 791 Namespace: job.Namespace, 792 }, 793 } 794 buf, err := structs.Encode(structs.JobRegisterRequestType, req) 795 if err != nil { 796 t.Fatalf("err: %v", err) 797 } 798 799 resp := fsm.Apply(makeLog(buf)) 800 if resp != nil { 801 t.Fatalf("resp: %v", resp) 802 } 803 804 req2 := structs.JobDeregisterRequest{ 805 JobID: job.ID, 806 Purge: false, 807 WriteRequest: structs.WriteRequest{ 808 Namespace: job.Namespace, 809 }, 810 } 811 buf, err = structs.Encode(structs.JobDeregisterRequestType, req2) 812 if err != nil { 813 t.Fatalf("err: %v", err) 814 } 815 816 resp = fsm.Apply(makeLog(buf)) 817 if resp != nil { 818 t.Fatalf("resp: %v", resp) 819 } 820 821 // Verify we are NOT registered 822 ws := memdb.NewWatchSet() 823 jobOut, err := fsm.State().JobByID(ws, req.Namespace, req.Job.ID) 824 if err != nil { 825 t.Fatalf("err: %v", err) 826 } 827 if jobOut == nil { 828 t.Fatalf("job not found!") 829 } 830 if !jobOut.Stop { 831 t.Fatalf("job not stopped found!") 832 } 833 834 // Verify it was removed from the periodic runner. 835 tuple := structs.NamespacedID{ 836 ID: job.ID, 837 Namespace: job.Namespace, 838 } 839 if _, ok := fsm.periodicDispatcher.tracked[tuple]; ok { 840 t.Fatal("job not removed from periodic runner") 841 } 842 843 // Verify it was removed from the periodic launch table. 844 launchOut, err := fsm.State().PeriodicLaunchByID(ws, req.Namespace, req.Job.ID) 845 if err != nil { 846 t.Fatalf("err: %v", err) 847 } 848 if launchOut == nil { 849 t.Fatalf("launch not found!") 850 } 851 } 852 853 func TestFSM_BatchDeregisterJob(t *testing.T) { 854 t.Parallel() 855 require := require.New(t) 856 fsm := testFSM(t) 857 858 job := mock.PeriodicJob() 859 req := structs.JobRegisterRequest{ 860 Job: job, 861 WriteRequest: structs.WriteRequest{ 862 Namespace: job.Namespace, 863 }, 864 } 865 buf, err := structs.Encode(structs.JobRegisterRequestType, req) 866 require.Nil(err) 867 resp := fsm.Apply(makeLog(buf)) 868 require.Nil(resp) 869 870 job2 := mock.Job() 871 req2 := structs.JobRegisterRequest{ 872 Job: job2, 873 WriteRequest: structs.WriteRequest{ 874 Namespace: job2.Namespace, 875 }, 876 } 877 878 buf, err = structs.Encode(structs.JobRegisterRequestType, req2) 879 require.Nil(err) 880 resp = fsm.Apply(makeLog(buf)) 881 require.Nil(resp) 882 883 req3 := structs.JobBatchDeregisterRequest{ 884 Jobs: map[structs.NamespacedID]*structs.JobDeregisterOptions{ 885 { 886 ID: job.ID, 887 Namespace: job.Namespace, 888 }: {}, 889 { 890 ID: job2.ID, 891 Namespace: job2.Namespace, 892 }: { 893 Purge: true, 894 }, 895 }, 896 WriteRequest: structs.WriteRequest{ 897 Namespace: job.Namespace, 898 }, 899 } 900 buf, err = structs.Encode(structs.JobBatchDeregisterRequestType, req3) 901 require.Nil(err) 902 903 resp = fsm.Apply(makeLog(buf)) 904 require.Nil(resp) 905 906 // Verify we are NOT registered 907 ws := memdb.NewWatchSet() 908 jobOut, err := fsm.State().JobByID(ws, req.Namespace, req.Job.ID) 909 require.Nil(err) 910 require.NotNil(jobOut) 911 require.True(jobOut.Stop) 912 913 // Verify it was removed from the periodic runner. 914 tuple := structs.NamespacedID{ 915 ID: job.ID, 916 Namespace: job.Namespace, 917 } 918 require.NotContains(fsm.periodicDispatcher.tracked, tuple) 919 920 // Verify it was not removed from the periodic launch table. 921 launchOut, err := fsm.State().PeriodicLaunchByID(ws, job.Namespace, job.ID) 922 require.Nil(err) 923 require.NotNil(launchOut) 924 925 // Verify the other jbo was purged 926 jobOut2, err := fsm.State().JobByID(ws, job2.Namespace, job2.ID) 927 require.Nil(err) 928 require.Nil(jobOut2) 929 } 930 931 func TestFSM_UpdateEval(t *testing.T) { 932 t.Parallel() 933 fsm := testFSM(t) 934 fsm.evalBroker.SetEnabled(true) 935 936 req := structs.EvalUpdateRequest{ 937 Evals: []*structs.Evaluation{mock.Eval()}, 938 } 939 buf, err := structs.Encode(structs.EvalUpdateRequestType, req) 940 if err != nil { 941 t.Fatalf("err: %v", err) 942 } 943 944 resp := fsm.Apply(makeLog(buf)) 945 if resp != nil { 946 t.Fatalf("resp: %v", resp) 947 } 948 949 // Verify we are registered 950 ws := memdb.NewWatchSet() 951 eval, err := fsm.State().EvalByID(ws, req.Evals[0].ID) 952 if err != nil { 953 t.Fatalf("err: %v", err) 954 } 955 if eval == nil { 956 t.Fatalf("not found!") 957 } 958 if eval.CreateIndex != 1 { 959 t.Fatalf("bad index: %d", eval.CreateIndex) 960 } 961 962 // Verify enqueued 963 stats := fsm.evalBroker.Stats() 964 if stats.TotalReady != 1 { 965 t.Fatalf("bad: %#v %#v", stats, eval) 966 } 967 } 968 969 func TestFSM_UpdateEval_Blocked(t *testing.T) { 970 t.Parallel() 971 fsm := testFSM(t) 972 fsm.evalBroker.SetEnabled(true) 973 fsm.blockedEvals.SetEnabled(true) 974 975 // Create a blocked eval. 976 eval := mock.Eval() 977 eval.Status = structs.EvalStatusBlocked 978 979 req := structs.EvalUpdateRequest{ 980 Evals: []*structs.Evaluation{eval}, 981 } 982 buf, err := structs.Encode(structs.EvalUpdateRequestType, req) 983 if err != nil { 984 t.Fatalf("err: %v", err) 985 } 986 987 resp := fsm.Apply(makeLog(buf)) 988 if resp != nil { 989 t.Fatalf("resp: %v", resp) 990 } 991 992 // Verify we are registered 993 ws := memdb.NewWatchSet() 994 out, err := fsm.State().EvalByID(ws, eval.ID) 995 if err != nil { 996 t.Fatalf("err: %v", err) 997 } 998 if out == nil { 999 t.Fatalf("not found!") 1000 } 1001 if out.CreateIndex != 1 { 1002 t.Fatalf("bad index: %d", out.CreateIndex) 1003 } 1004 1005 // Verify the eval wasn't enqueued 1006 stats := fsm.evalBroker.Stats() 1007 if stats.TotalReady != 0 { 1008 t.Fatalf("bad: %#v %#v", stats, out) 1009 } 1010 1011 // Verify the eval was added to the blocked tracker. 1012 bStats := fsm.blockedEvals.Stats() 1013 if bStats.TotalBlocked != 1 { 1014 t.Fatalf("bad: %#v %#v", bStats, out) 1015 } 1016 } 1017 1018 func TestFSM_UpdateEval_Untrack(t *testing.T) { 1019 t.Parallel() 1020 fsm := testFSM(t) 1021 fsm.evalBroker.SetEnabled(true) 1022 fsm.blockedEvals.SetEnabled(true) 1023 1024 // Mark an eval as blocked. 1025 bEval := mock.Eval() 1026 bEval.ClassEligibility = map[string]bool{"v1:123": true} 1027 fsm.blockedEvals.Block(bEval) 1028 1029 // Create a successful eval for the same job 1030 eval := mock.Eval() 1031 eval.JobID = bEval.JobID 1032 eval.Status = structs.EvalStatusComplete 1033 1034 req := structs.EvalUpdateRequest{ 1035 Evals: []*structs.Evaluation{eval}, 1036 } 1037 buf, err := structs.Encode(structs.EvalUpdateRequestType, req) 1038 if err != nil { 1039 t.Fatalf("err: %v", err) 1040 } 1041 1042 resp := fsm.Apply(makeLog(buf)) 1043 if resp != nil { 1044 t.Fatalf("resp: %v", resp) 1045 } 1046 1047 // Verify we are registered 1048 ws := memdb.NewWatchSet() 1049 out, err := fsm.State().EvalByID(ws, eval.ID) 1050 if err != nil { 1051 t.Fatalf("err: %v", err) 1052 } 1053 if out == nil { 1054 t.Fatalf("not found!") 1055 } 1056 if out.CreateIndex != 1 { 1057 t.Fatalf("bad index: %d", out.CreateIndex) 1058 } 1059 1060 // Verify the eval wasn't enqueued 1061 stats := fsm.evalBroker.Stats() 1062 if stats.TotalReady != 0 { 1063 t.Fatalf("bad: %#v %#v", stats, out) 1064 } 1065 1066 // Verify the eval was untracked in the blocked tracker. 1067 bStats := fsm.blockedEvals.Stats() 1068 if bStats.TotalBlocked != 0 { 1069 t.Fatalf("bad: %#v %#v", bStats, out) 1070 } 1071 } 1072 1073 func TestFSM_UpdateEval_NoUntrack(t *testing.T) { 1074 t.Parallel() 1075 fsm := testFSM(t) 1076 fsm.evalBroker.SetEnabled(true) 1077 fsm.blockedEvals.SetEnabled(true) 1078 1079 // Mark an eval as blocked. 1080 bEval := mock.Eval() 1081 bEval.ClassEligibility = map[string]bool{"v1:123": true} 1082 fsm.blockedEvals.Block(bEval) 1083 1084 // Create a successful eval for the same job but with placement failures 1085 eval := mock.Eval() 1086 eval.JobID = bEval.JobID 1087 eval.Status = structs.EvalStatusComplete 1088 eval.FailedTGAllocs = make(map[string]*structs.AllocMetric) 1089 eval.FailedTGAllocs["test"] = new(structs.AllocMetric) 1090 1091 req := structs.EvalUpdateRequest{ 1092 Evals: []*structs.Evaluation{eval}, 1093 } 1094 buf, err := structs.Encode(structs.EvalUpdateRequestType, req) 1095 if err != nil { 1096 t.Fatalf("err: %v", err) 1097 } 1098 1099 resp := fsm.Apply(makeLog(buf)) 1100 if resp != nil { 1101 t.Fatalf("resp: %v", resp) 1102 } 1103 1104 // Verify we are registered 1105 ws := memdb.NewWatchSet() 1106 out, err := fsm.State().EvalByID(ws, eval.ID) 1107 if err != nil { 1108 t.Fatalf("err: %v", err) 1109 } 1110 if out == nil { 1111 t.Fatalf("not found!") 1112 } 1113 if out.CreateIndex != 1 { 1114 t.Fatalf("bad index: %d", out.CreateIndex) 1115 } 1116 1117 // Verify the eval wasn't enqueued 1118 stats := fsm.evalBroker.Stats() 1119 if stats.TotalReady != 0 { 1120 t.Fatalf("bad: %#v %#v", stats, out) 1121 } 1122 1123 // Verify the eval was not untracked in the blocked tracker. 1124 bStats := fsm.blockedEvals.Stats() 1125 if bStats.TotalBlocked != 1 { 1126 t.Fatalf("bad: %#v %#v", bStats, out) 1127 } 1128 } 1129 1130 func TestFSM_DeleteEval(t *testing.T) { 1131 t.Parallel() 1132 fsm := testFSM(t) 1133 1134 eval := mock.Eval() 1135 req := structs.EvalUpdateRequest{ 1136 Evals: []*structs.Evaluation{eval}, 1137 } 1138 buf, err := structs.Encode(structs.EvalUpdateRequestType, req) 1139 if err != nil { 1140 t.Fatalf("err: %v", err) 1141 } 1142 1143 resp := fsm.Apply(makeLog(buf)) 1144 if resp != nil { 1145 t.Fatalf("resp: %v", resp) 1146 } 1147 1148 req2 := structs.EvalDeleteRequest{ 1149 Evals: []string{eval.ID}, 1150 } 1151 buf, err = structs.Encode(structs.EvalDeleteRequestType, req2) 1152 if err != nil { 1153 t.Fatalf("err: %v", err) 1154 } 1155 1156 resp = fsm.Apply(makeLog(buf)) 1157 if resp != nil { 1158 t.Fatalf("resp: %v", resp) 1159 } 1160 1161 // Verify we are NOT registered 1162 ws := memdb.NewWatchSet() 1163 eval, err = fsm.State().EvalByID(ws, req.Evals[0].ID) 1164 if err != nil { 1165 t.Fatalf("err: %v", err) 1166 } 1167 if eval != nil { 1168 t.Fatalf("eval found!") 1169 } 1170 } 1171 1172 func TestFSM_UpsertAllocs(t *testing.T) { 1173 t.Parallel() 1174 fsm := testFSM(t) 1175 1176 alloc := mock.Alloc() 1177 alloc.Resources = &structs.Resources{} // COMPAT(0.11): Remove in 0.11, used to bypass resource creation in state store 1178 fsm.State().UpsertJobSummary(1, mock.JobSummary(alloc.JobID)) 1179 req := structs.AllocUpdateRequest{ 1180 Alloc: []*structs.Allocation{alloc}, 1181 } 1182 buf, err := structs.Encode(structs.AllocUpdateRequestType, req) 1183 if err != nil { 1184 t.Fatalf("err: %v", err) 1185 } 1186 1187 resp := fsm.Apply(makeLog(buf)) 1188 if resp != nil { 1189 t.Fatalf("resp: %v", resp) 1190 } 1191 1192 // Verify we are registered 1193 ws := memdb.NewWatchSet() 1194 out, err := fsm.State().AllocByID(ws, alloc.ID) 1195 if err != nil { 1196 t.Fatalf("err: %v", err) 1197 } 1198 alloc.CreateIndex = out.CreateIndex 1199 alloc.ModifyIndex = out.ModifyIndex 1200 alloc.AllocModifyIndex = out.AllocModifyIndex 1201 if !reflect.DeepEqual(alloc, out) { 1202 t.Fatalf("bad: %#v %#v", alloc, out) 1203 } 1204 1205 evictAlloc := new(structs.Allocation) 1206 *evictAlloc = *alloc 1207 evictAlloc.DesiredStatus = structs.AllocDesiredStatusEvict 1208 req2 := structs.AllocUpdateRequest{ 1209 Alloc: []*structs.Allocation{evictAlloc}, 1210 } 1211 buf, err = structs.Encode(structs.AllocUpdateRequestType, req2) 1212 if err != nil { 1213 t.Fatalf("err: %v", err) 1214 } 1215 1216 resp = fsm.Apply(makeLog(buf)) 1217 if resp != nil { 1218 t.Fatalf("resp: %v", resp) 1219 } 1220 1221 // Verify we are evicted 1222 out, err = fsm.State().AllocByID(ws, alloc.ID) 1223 if err != nil { 1224 t.Fatalf("err: %v", err) 1225 } 1226 if out.DesiredStatus != structs.AllocDesiredStatusEvict { 1227 t.Fatalf("alloc found!") 1228 } 1229 } 1230 1231 func TestFSM_UpsertAllocs_SharedJob(t *testing.T) { 1232 t.Parallel() 1233 fsm := testFSM(t) 1234 1235 alloc := mock.Alloc() 1236 alloc.Resources = &structs.Resources{} // COMPAT(0.11): Remove in 0.11, used to bypass resource creation in state store 1237 fsm.State().UpsertJobSummary(1, mock.JobSummary(alloc.JobID)) 1238 job := alloc.Job 1239 alloc.Job = nil 1240 req := structs.AllocUpdateRequest{ 1241 Job: job, 1242 Alloc: []*structs.Allocation{alloc}, 1243 } 1244 buf, err := structs.Encode(structs.AllocUpdateRequestType, req) 1245 if err != nil { 1246 t.Fatalf("err: %v", err) 1247 } 1248 1249 resp := fsm.Apply(makeLog(buf)) 1250 if resp != nil { 1251 t.Fatalf("resp: %v", resp) 1252 } 1253 1254 // Verify we are registered 1255 ws := memdb.NewWatchSet() 1256 out, err := fsm.State().AllocByID(ws, alloc.ID) 1257 if err != nil { 1258 t.Fatalf("err: %v", err) 1259 } 1260 alloc.CreateIndex = out.CreateIndex 1261 alloc.ModifyIndex = out.ModifyIndex 1262 alloc.AllocModifyIndex = out.AllocModifyIndex 1263 1264 // Job should be re-attached 1265 alloc.Job = job 1266 require.Equal(t, alloc, out) 1267 1268 // Ensure that the original job is used 1269 evictAlloc := new(structs.Allocation) 1270 *evictAlloc = *alloc 1271 job = mock.Job() 1272 job.Priority = 123 1273 1274 evictAlloc.Job = nil 1275 evictAlloc.DesiredStatus = structs.AllocDesiredStatusEvict 1276 req2 := structs.AllocUpdateRequest{ 1277 Job: job, 1278 Alloc: []*structs.Allocation{evictAlloc}, 1279 } 1280 buf, err = structs.Encode(structs.AllocUpdateRequestType, req2) 1281 if err != nil { 1282 t.Fatalf("err: %v", err) 1283 } 1284 1285 resp = fsm.Apply(makeLog(buf)) 1286 if resp != nil { 1287 t.Fatalf("resp: %v", resp) 1288 } 1289 1290 // Verify we are evicted 1291 out, err = fsm.State().AllocByID(ws, alloc.ID) 1292 if err != nil { 1293 t.Fatalf("err: %v", err) 1294 } 1295 if out.DesiredStatus != structs.AllocDesiredStatusEvict { 1296 t.Fatalf("alloc found!") 1297 } 1298 if out.Job == nil || out.Job.Priority == 123 { 1299 t.Fatalf("bad job") 1300 } 1301 } 1302 1303 // COMPAT(0.11): Remove in 0.11 1304 func TestFSM_UpsertAllocs_StrippedResources(t *testing.T) { 1305 t.Parallel() 1306 fsm := testFSM(t) 1307 1308 alloc := mock.Alloc() 1309 alloc.Resources = &structs.Resources{ 1310 CPU: 500, 1311 MemoryMB: 256, 1312 DiskMB: 150, 1313 Networks: []*structs.NetworkResource{ 1314 { 1315 Device: "eth0", 1316 IP: "192.168.0.100", 1317 ReservedPorts: []structs.Port{{Label: "admin", Value: 5000}}, 1318 MBits: 50, 1319 DynamicPorts: []structs.Port{{Label: "http"}}, 1320 }, 1321 }, 1322 } 1323 alloc.TaskResources = map[string]*structs.Resources{ 1324 "web": { 1325 CPU: 500, 1326 MemoryMB: 256, 1327 Networks: []*structs.NetworkResource{ 1328 { 1329 Device: "eth0", 1330 IP: "192.168.0.100", 1331 ReservedPorts: []structs.Port{{Label: "admin", Value: 5000}}, 1332 MBits: 50, 1333 DynamicPorts: []structs.Port{{Label: "http", Value: 9876}}, 1334 }, 1335 }, 1336 }, 1337 } 1338 alloc.SharedResources = &structs.Resources{ 1339 DiskMB: 150, 1340 } 1341 1342 // Need to remove mock dynamic port from alloc as it won't be computed 1343 // in this test 1344 alloc.TaskResources["web"].Networks[0].DynamicPorts[0].Value = 0 1345 1346 fsm.State().UpsertJobSummary(1, mock.JobSummary(alloc.JobID)) 1347 job := alloc.Job 1348 origResources := alloc.Resources 1349 alloc.Resources = nil 1350 req := structs.AllocUpdateRequest{ 1351 Job: job, 1352 Alloc: []*structs.Allocation{alloc}, 1353 } 1354 buf, err := structs.Encode(structs.AllocUpdateRequestType, req) 1355 if err != nil { 1356 t.Fatalf("err: %v", err) 1357 } 1358 1359 resp := fsm.Apply(makeLog(buf)) 1360 if resp != nil { 1361 t.Fatalf("resp: %v", resp) 1362 } 1363 1364 // Verify we are registered 1365 ws := memdb.NewWatchSet() 1366 out, err := fsm.State().AllocByID(ws, alloc.ID) 1367 if err != nil { 1368 t.Fatalf("err: %v", err) 1369 } 1370 alloc.CreateIndex = out.CreateIndex 1371 alloc.ModifyIndex = out.ModifyIndex 1372 alloc.AllocModifyIndex = out.AllocModifyIndex 1373 1374 // Resources should be recomputed 1375 origResources.DiskMB = alloc.Job.TaskGroups[0].EphemeralDisk.SizeMB 1376 alloc.Resources = origResources 1377 if !reflect.DeepEqual(alloc, out) { 1378 t.Fatalf("not equal: % #v", pretty.Diff(alloc, out)) 1379 } 1380 } 1381 1382 // TestFSM_UpsertAllocs_Canonicalize asserts that allocations are Canonicalized 1383 // to handle logs emited by servers running old versions 1384 func TestFSM_UpsertAllocs_Canonicalize(t *testing.T) { 1385 t.Parallel() 1386 fsm := testFSM(t) 1387 1388 alloc := mock.Alloc() 1389 alloc.Resources = &structs.Resources{} // COMPAT(0.11): Remove in 0.11, used to bypass resource creation in state store 1390 alloc.AllocatedResources = nil 1391 1392 // pre-assert that our mock populates old field 1393 require.NotEmpty(t, alloc.TaskResources) 1394 1395 fsm.State().UpsertJobSummary(1, mock.JobSummary(alloc.JobID)) 1396 req := structs.AllocUpdateRequest{ 1397 Alloc: []*structs.Allocation{alloc}, 1398 } 1399 buf, err := structs.Encode(structs.AllocUpdateRequestType, req) 1400 require.NoError(t, err) 1401 1402 resp := fsm.Apply(makeLog(buf)) 1403 require.Nil(t, resp) 1404 1405 // Verify we are registered 1406 ws := memdb.NewWatchSet() 1407 out, err := fsm.State().AllocByID(ws, alloc.ID) 1408 require.NoError(t, err) 1409 1410 require.NotNil(t, out.AllocatedResources) 1411 require.Contains(t, out.AllocatedResources.Tasks, "web") 1412 1413 expected := alloc.Copy() 1414 expected.Canonicalize() 1415 expected.CreateIndex = out.CreateIndex 1416 expected.ModifyIndex = out.ModifyIndex 1417 expected.AllocModifyIndex = out.AllocModifyIndex 1418 require.Equal(t, expected, out) 1419 } 1420 1421 func TestFSM_UpdateAllocFromClient_Unblock(t *testing.T) { 1422 t.Parallel() 1423 fsm := testFSM(t) 1424 fsm.blockedEvals.SetEnabled(true) 1425 state := fsm.State() 1426 1427 node := mock.Node() 1428 state.UpsertNode(1, node) 1429 1430 // Mark an eval as blocked. 1431 eval := mock.Eval() 1432 eval.ClassEligibility = map[string]bool{node.ComputedClass: true} 1433 fsm.blockedEvals.Block(eval) 1434 1435 bStats := fsm.blockedEvals.Stats() 1436 if bStats.TotalBlocked != 1 { 1437 t.Fatalf("bad: %#v", bStats) 1438 } 1439 1440 // Create a completed eval 1441 alloc := mock.Alloc() 1442 alloc.NodeID = node.ID 1443 alloc2 := mock.Alloc() 1444 alloc2.NodeID = node.ID 1445 state.UpsertJobSummary(8, mock.JobSummary(alloc.JobID)) 1446 state.UpsertJobSummary(9, mock.JobSummary(alloc2.JobID)) 1447 state.UpsertAllocs(10, []*structs.Allocation{alloc, alloc2}) 1448 1449 clientAlloc := new(structs.Allocation) 1450 *clientAlloc = *alloc 1451 clientAlloc.ClientStatus = structs.AllocClientStatusComplete 1452 update2 := &structs.Allocation{ 1453 ID: alloc2.ID, 1454 ClientStatus: structs.AllocClientStatusRunning, 1455 } 1456 1457 req := structs.AllocUpdateRequest{ 1458 Alloc: []*structs.Allocation{clientAlloc, update2}, 1459 } 1460 buf, err := structs.Encode(structs.AllocClientUpdateRequestType, req) 1461 if err != nil { 1462 t.Fatalf("err: %v", err) 1463 } 1464 1465 resp := fsm.Apply(makeLog(buf)) 1466 if resp != nil { 1467 t.Fatalf("resp: %v", resp) 1468 } 1469 1470 // Verify we are updated 1471 ws := memdb.NewWatchSet() 1472 out, err := fsm.State().AllocByID(ws, alloc.ID) 1473 if err != nil { 1474 t.Fatalf("err: %v", err) 1475 } 1476 clientAlloc.CreateIndex = out.CreateIndex 1477 clientAlloc.ModifyIndex = out.ModifyIndex 1478 if !reflect.DeepEqual(clientAlloc, out) { 1479 t.Fatalf("bad: %#v %#v", clientAlloc, out) 1480 } 1481 1482 out, err = fsm.State().AllocByID(ws, alloc2.ID) 1483 if err != nil { 1484 t.Fatalf("err: %v", err) 1485 } 1486 alloc2.CreateIndex = out.CreateIndex 1487 alloc2.ModifyIndex = out.ModifyIndex 1488 alloc2.ClientStatus = structs.AllocClientStatusRunning 1489 alloc2.TaskStates = nil 1490 if !reflect.DeepEqual(alloc2, out) { 1491 t.Fatalf("bad: %#v %#v", alloc2, out) 1492 } 1493 1494 // Verify the eval was unblocked. 1495 testutil.WaitForResult(func() (bool, error) { 1496 bStats = fsm.blockedEvals.Stats() 1497 if bStats.TotalBlocked != 0 { 1498 return false, fmt.Errorf("bad: %#v %#v", bStats, out) 1499 } 1500 return true, nil 1501 }, func(err error) { 1502 t.Fatalf("err: %s", err) 1503 }) 1504 } 1505 1506 func TestFSM_UpdateAllocFromClient(t *testing.T) { 1507 t.Parallel() 1508 fsm := testFSM(t) 1509 state := fsm.State() 1510 require := require.New(t) 1511 1512 alloc := mock.Alloc() 1513 state.UpsertJobSummary(9, mock.JobSummary(alloc.JobID)) 1514 state.UpsertAllocs(10, []*structs.Allocation{alloc}) 1515 1516 clientAlloc := new(structs.Allocation) 1517 *clientAlloc = *alloc 1518 clientAlloc.ClientStatus = structs.AllocClientStatusFailed 1519 1520 eval := mock.Eval() 1521 eval.JobID = alloc.JobID 1522 eval.TriggeredBy = structs.EvalTriggerRetryFailedAlloc 1523 eval.Type = alloc.Job.Type 1524 1525 req := structs.AllocUpdateRequest{ 1526 Alloc: []*structs.Allocation{clientAlloc}, 1527 Evals: []*structs.Evaluation{eval}, 1528 } 1529 buf, err := structs.Encode(structs.AllocClientUpdateRequestType, req) 1530 require.Nil(err) 1531 1532 resp := fsm.Apply(makeLog(buf)) 1533 require.Nil(resp) 1534 1535 // Verify we are registered 1536 ws := memdb.NewWatchSet() 1537 out, err := fsm.State().AllocByID(ws, alloc.ID) 1538 require.Nil(err) 1539 clientAlloc.CreateIndex = out.CreateIndex 1540 clientAlloc.ModifyIndex = out.ModifyIndex 1541 require.Equal(clientAlloc, out) 1542 1543 // Verify eval was inserted 1544 ws = memdb.NewWatchSet() 1545 evals, err := fsm.State().EvalsByJob(ws, eval.Namespace, eval.JobID) 1546 require.Nil(err) 1547 require.Equal(1, len(evals)) 1548 res := evals[0] 1549 eval.CreateIndex = res.CreateIndex 1550 eval.ModifyIndex = res.ModifyIndex 1551 require.Equal(eval, res) 1552 } 1553 1554 func TestFSM_UpdateAllocDesiredTransition(t *testing.T) { 1555 t.Parallel() 1556 fsm := testFSM(t) 1557 state := fsm.State() 1558 require := require.New(t) 1559 1560 alloc := mock.Alloc() 1561 alloc2 := mock.Alloc() 1562 alloc2.Job = alloc.Job 1563 alloc2.JobID = alloc.JobID 1564 state.UpsertJobSummary(9, mock.JobSummary(alloc.JobID)) 1565 state.UpsertAllocs(10, []*structs.Allocation{alloc, alloc2}) 1566 1567 t1 := &structs.DesiredTransition{ 1568 Migrate: helper.BoolToPtr(true), 1569 } 1570 1571 eval := &structs.Evaluation{ 1572 ID: uuid.Generate(), 1573 Namespace: alloc.Namespace, 1574 Priority: alloc.Job.Priority, 1575 Type: alloc.Job.Type, 1576 TriggeredBy: structs.EvalTriggerNodeDrain, 1577 JobID: alloc.Job.ID, 1578 JobModifyIndex: alloc.Job.ModifyIndex, 1579 Status: structs.EvalStatusPending, 1580 } 1581 req := structs.AllocUpdateDesiredTransitionRequest{ 1582 Allocs: map[string]*structs.DesiredTransition{ 1583 alloc.ID: t1, 1584 alloc2.ID: t1, 1585 }, 1586 Evals: []*structs.Evaluation{eval}, 1587 } 1588 buf, err := structs.Encode(structs.AllocUpdateDesiredTransitionRequestType, req) 1589 require.Nil(err) 1590 1591 resp := fsm.Apply(makeLog(buf)) 1592 require.Nil(resp) 1593 1594 // Verify we are registered 1595 ws := memdb.NewWatchSet() 1596 out1, err := fsm.State().AllocByID(ws, alloc.ID) 1597 require.Nil(err) 1598 out2, err := fsm.State().AllocByID(ws, alloc2.ID) 1599 require.Nil(err) 1600 evalOut, err := fsm.State().EvalByID(ws, eval.ID) 1601 require.Nil(err) 1602 require.NotNil(evalOut) 1603 require.Equal(eval.ID, evalOut.ID) 1604 1605 require.NotNil(out1.DesiredTransition.Migrate) 1606 require.NotNil(out2.DesiredTransition.Migrate) 1607 require.True(*out1.DesiredTransition.Migrate) 1608 require.True(*out2.DesiredTransition.Migrate) 1609 } 1610 1611 func TestFSM_UpsertVaultAccessor(t *testing.T) { 1612 t.Parallel() 1613 fsm := testFSM(t) 1614 fsm.blockedEvals.SetEnabled(true) 1615 1616 va := mock.VaultAccessor() 1617 va2 := mock.VaultAccessor() 1618 req := structs.VaultAccessorsRequest{ 1619 Accessors: []*structs.VaultAccessor{va, va2}, 1620 } 1621 buf, err := structs.Encode(structs.VaultAccessorRegisterRequestType, req) 1622 if err != nil { 1623 t.Fatalf("err: %v", err) 1624 } 1625 1626 resp := fsm.Apply(makeLog(buf)) 1627 if resp != nil { 1628 t.Fatalf("resp: %v", resp) 1629 } 1630 1631 // Verify we are registered 1632 ws := memdb.NewWatchSet() 1633 out1, err := fsm.State().VaultAccessor(ws, va.Accessor) 1634 if err != nil { 1635 t.Fatalf("err: %v", err) 1636 } 1637 if out1 == nil { 1638 t.Fatalf("not found!") 1639 } 1640 if out1.CreateIndex != 1 { 1641 t.Fatalf("bad index: %d", out1.CreateIndex) 1642 } 1643 out2, err := fsm.State().VaultAccessor(ws, va2.Accessor) 1644 if err != nil { 1645 t.Fatalf("err: %v", err) 1646 } 1647 if out2 == nil { 1648 t.Fatalf("not found!") 1649 } 1650 if out1.CreateIndex != 1 { 1651 t.Fatalf("bad index: %d", out2.CreateIndex) 1652 } 1653 1654 tt := fsm.TimeTable() 1655 index := tt.NearestIndex(time.Now().UTC()) 1656 if index != 1 { 1657 t.Fatalf("bad: %d", index) 1658 } 1659 } 1660 1661 func TestFSM_DeregisterVaultAccessor(t *testing.T) { 1662 t.Parallel() 1663 fsm := testFSM(t) 1664 fsm.blockedEvals.SetEnabled(true) 1665 1666 va := mock.VaultAccessor() 1667 va2 := mock.VaultAccessor() 1668 accessors := []*structs.VaultAccessor{va, va2} 1669 1670 // Insert the accessors 1671 if err := fsm.State().UpsertVaultAccessor(1000, accessors); err != nil { 1672 t.Fatalf("bad: %v", err) 1673 } 1674 1675 req := structs.VaultAccessorsRequest{ 1676 Accessors: accessors, 1677 } 1678 buf, err := structs.Encode(structs.VaultAccessorDeregisterRequestType, req) 1679 if err != nil { 1680 t.Fatalf("err: %v", err) 1681 } 1682 1683 resp := fsm.Apply(makeLog(buf)) 1684 if resp != nil { 1685 t.Fatalf("resp: %v", resp) 1686 } 1687 1688 ws := memdb.NewWatchSet() 1689 out1, err := fsm.State().VaultAccessor(ws, va.Accessor) 1690 if err != nil { 1691 t.Fatalf("err: %v", err) 1692 } 1693 if out1 != nil { 1694 t.Fatalf("not deleted!") 1695 } 1696 1697 tt := fsm.TimeTable() 1698 index := tt.NearestIndex(time.Now().UTC()) 1699 if index != 1 { 1700 t.Fatalf("bad: %d", index) 1701 } 1702 } 1703 1704 func TestFSM_UpsertSITokenAccessor(t *testing.T) { 1705 t.Parallel() 1706 r := require.New(t) 1707 1708 fsm := testFSM(t) 1709 fsm.blockedEvals.SetEnabled(true) 1710 1711 a1 := mock.SITokenAccessor() 1712 a2 := mock.SITokenAccessor() 1713 request := structs.SITokenAccessorsRequest{ 1714 Accessors: []*structs.SITokenAccessor{a1, a2}, 1715 } 1716 buf, err := structs.Encode(structs.ServiceIdentityAccessorRegisterRequestType, request) 1717 r.NoError(err) 1718 1719 response := fsm.Apply(makeLog(buf)) 1720 r.Nil(response) 1721 1722 // Verify the accessors got registered 1723 ws := memdb.NewWatchSet() 1724 result1, err := fsm.State().SITokenAccessor(ws, a1.AccessorID) 1725 r.NoError(err) 1726 r.NotNil(result1) 1727 r.Equal(uint64(1), result1.CreateIndex) 1728 1729 result2, err := fsm.State().SITokenAccessor(ws, a2.AccessorID) 1730 r.NoError(err) 1731 r.NotNil(result2) 1732 r.Equal(uint64(1), result2.CreateIndex) 1733 1734 tt := fsm.TimeTable() 1735 latestIndex := tt.NearestIndex(time.Now()) 1736 r.Equal(uint64(1), latestIndex) 1737 } 1738 1739 func TestFSM_DeregisterSITokenAccessor(t *testing.T) { 1740 t.Parallel() 1741 r := require.New(t) 1742 1743 fsm := testFSM(t) 1744 fsm.blockedEvals.SetEnabled(true) 1745 1746 a1 := mock.SITokenAccessor() 1747 a2 := mock.SITokenAccessor() 1748 accessors := []*structs.SITokenAccessor{a1, a2} 1749 var err error 1750 1751 // Insert the accessors 1752 err = fsm.State().UpsertSITokenAccessors(1000, accessors) 1753 r.NoError(err) 1754 1755 request := structs.SITokenAccessorsRequest{Accessors: accessors} 1756 buf, err := structs.Encode(structs.ServiceIdentityAccessorDeregisterRequestType, request) 1757 r.NoError(err) 1758 1759 response := fsm.Apply(makeLog(buf)) 1760 r.Nil(response) 1761 1762 ws := memdb.NewWatchSet() 1763 1764 result1, err := fsm.State().SITokenAccessor(ws, a1.AccessorID) 1765 r.NoError(err) 1766 r.Nil(result1) // should have been deleted 1767 1768 result2, err := fsm.State().SITokenAccessor(ws, a2.AccessorID) 1769 r.NoError(err) 1770 r.Nil(result2) // should have been deleted 1771 1772 tt := fsm.TimeTable() 1773 latestIndex := tt.NearestIndex(time.Now()) 1774 r.Equal(uint64(1), latestIndex) 1775 } 1776 1777 func TestFSM_ApplyPlanResults(t *testing.T) { 1778 t.Parallel() 1779 fsm := testFSM(t) 1780 fsm.evalBroker.SetEnabled(true) 1781 // Create the request and create a deployment 1782 alloc := mock.Alloc() 1783 alloc.Resources = &structs.Resources{} // COMPAT(0.11): Remove in 0.11, used to bypass resource creation in state store 1784 job := alloc.Job 1785 alloc.Job = nil 1786 1787 d := mock.Deployment() 1788 d.JobID = job.ID 1789 d.JobModifyIndex = job.ModifyIndex 1790 d.JobVersion = job.Version 1791 1792 alloc.DeploymentID = d.ID 1793 1794 eval := mock.Eval() 1795 eval.JobID = job.ID 1796 fsm.State().UpsertEvals(1, []*structs.Evaluation{eval}) 1797 1798 fsm.State().UpsertJobSummary(1, mock.JobSummary(alloc.JobID)) 1799 1800 // set up preempted jobs and allocs 1801 job1 := mock.Job() 1802 job2 := mock.Job() 1803 1804 alloc1 := mock.Alloc() 1805 alloc1.Job = job1 1806 alloc1.JobID = job1.ID 1807 alloc1.PreemptedByAllocation = alloc.ID 1808 1809 alloc2 := mock.Alloc() 1810 alloc2.Job = job2 1811 alloc2.JobID = job2.ID 1812 alloc2.PreemptedByAllocation = alloc.ID 1813 1814 fsm.State().UpsertAllocs(1, []*structs.Allocation{alloc1, alloc2}) 1815 1816 // evals for preempted jobs 1817 eval1 := mock.Eval() 1818 eval1.JobID = job1.ID 1819 1820 eval2 := mock.Eval() 1821 eval2.JobID = job2.ID 1822 1823 req := structs.ApplyPlanResultsRequest{ 1824 AllocUpdateRequest: structs.AllocUpdateRequest{ 1825 Job: job, 1826 Alloc: []*structs.Allocation{alloc}, 1827 }, 1828 Deployment: d, 1829 EvalID: eval.ID, 1830 NodePreemptions: []*structs.Allocation{alloc1, alloc2}, 1831 PreemptionEvals: []*structs.Evaluation{eval1, eval2}, 1832 } 1833 buf, err := structs.Encode(structs.ApplyPlanResultsRequestType, req) 1834 if err != nil { 1835 t.Fatalf("err: %v", err) 1836 } 1837 1838 resp := fsm.Apply(makeLog(buf)) 1839 if resp != nil { 1840 t.Fatalf("resp: %v", resp) 1841 } 1842 1843 // Verify the allocation is registered 1844 ws := memdb.NewWatchSet() 1845 assert := assert.New(t) 1846 out, err := fsm.State().AllocByID(ws, alloc.ID) 1847 assert.Nil(err) 1848 alloc.CreateIndex = out.CreateIndex 1849 alloc.ModifyIndex = out.ModifyIndex 1850 alloc.AllocModifyIndex = out.AllocModifyIndex 1851 1852 // Job should be re-attached 1853 alloc.Job = job 1854 assert.Equal(alloc, out) 1855 1856 // Verify that evals for preempted jobs have been created 1857 e1, err := fsm.State().EvalByID(ws, eval1.ID) 1858 require := require.New(t) 1859 require.Nil(err) 1860 require.NotNil(e1) 1861 1862 e2, err := fsm.State().EvalByID(ws, eval2.ID) 1863 require.Nil(err) 1864 require.NotNil(e2) 1865 1866 // Verify that eval broker has both evals 1867 _, ok := fsm.evalBroker.evals[e1.ID] 1868 require.True(ok) 1869 1870 _, ok = fsm.evalBroker.evals[e1.ID] 1871 require.True(ok) 1872 1873 dout, err := fsm.State().DeploymentByID(ws, d.ID) 1874 assert.Nil(err) 1875 tg, ok := dout.TaskGroups[alloc.TaskGroup] 1876 assert.True(ok) 1877 assert.NotNil(tg) 1878 assert.Equal(1, tg.PlacedAllocs) 1879 1880 // Ensure that the original job is used 1881 evictAlloc := alloc.Copy() 1882 job = mock.Job() 1883 job.Priority = 123 1884 eval = mock.Eval() 1885 eval.JobID = job.ID 1886 1887 fsm.State().UpsertEvals(2, []*structs.Evaluation{eval}) 1888 1889 evictAlloc.Job = nil 1890 evictAlloc.DesiredStatus = structs.AllocDesiredStatusEvict 1891 req2 := structs.ApplyPlanResultsRequest{ 1892 AllocUpdateRequest: structs.AllocUpdateRequest{ 1893 Job: job, 1894 Alloc: []*structs.Allocation{evictAlloc}, 1895 }, 1896 EvalID: eval.ID, 1897 } 1898 buf, err = structs.Encode(structs.ApplyPlanResultsRequestType, req2) 1899 assert.Nil(err) 1900 1901 log := makeLog(buf) 1902 //set the index to something other than 1 1903 log.Index = 25 1904 resp = fsm.Apply(log) 1905 assert.Nil(resp) 1906 1907 // Verify we are evicted 1908 out, err = fsm.State().AllocByID(ws, alloc.ID) 1909 assert.Nil(err) 1910 assert.Equal(structs.AllocDesiredStatusEvict, out.DesiredStatus) 1911 assert.NotNil(out.Job) 1912 assert.NotEqual(123, out.Job.Priority) 1913 1914 evalOut, err := fsm.State().EvalByID(ws, eval.ID) 1915 assert.Nil(err) 1916 assert.Equal(log.Index, evalOut.ModifyIndex) 1917 1918 } 1919 1920 func TestFSM_DeploymentStatusUpdate(t *testing.T) { 1921 t.Parallel() 1922 fsm := testFSM(t) 1923 fsm.evalBroker.SetEnabled(true) 1924 state := fsm.State() 1925 1926 // Upsert a deployment 1927 d := mock.Deployment() 1928 if err := state.UpsertDeployment(1, d); err != nil { 1929 t.Fatalf("bad: %v", err) 1930 } 1931 1932 // Create a request to update the deployment, create an eval and job 1933 e := mock.Eval() 1934 j := mock.Job() 1935 status, desc := structs.DeploymentStatusFailed, "foo" 1936 req := &structs.DeploymentStatusUpdateRequest{ 1937 DeploymentUpdate: &structs.DeploymentStatusUpdate{ 1938 DeploymentID: d.ID, 1939 Status: status, 1940 StatusDescription: desc, 1941 }, 1942 Job: j, 1943 Eval: e, 1944 } 1945 buf, err := structs.Encode(structs.DeploymentStatusUpdateRequestType, req) 1946 if err != nil { 1947 t.Fatalf("err: %v", err) 1948 } 1949 resp := fsm.Apply(makeLog(buf)) 1950 if resp != nil { 1951 t.Fatalf("resp: %v", resp) 1952 } 1953 1954 // Check that the status was updated properly 1955 ws := memdb.NewWatchSet() 1956 dout, err := state.DeploymentByID(ws, d.ID) 1957 if err != nil { 1958 t.Fatalf("bad: %v", err) 1959 } 1960 if dout.Status != status || dout.StatusDescription != desc { 1961 t.Fatalf("bad: %#v", dout) 1962 } 1963 1964 // Check that the evaluation was created 1965 eout, _ := state.EvalByID(ws, e.ID) 1966 if err != nil { 1967 t.Fatalf("bad: %v", err) 1968 } 1969 if eout == nil { 1970 t.Fatalf("bad: %#v", eout) 1971 } 1972 1973 // Check that the job was created 1974 jout, _ := state.JobByID(ws, j.Namespace, j.ID) 1975 if err != nil { 1976 t.Fatalf("bad: %v", err) 1977 } 1978 if jout == nil { 1979 t.Fatalf("bad: %#v", jout) 1980 } 1981 1982 // Assert the eval was enqueued 1983 stats := fsm.evalBroker.Stats() 1984 if stats.TotalReady != 1 { 1985 t.Fatalf("bad: %#v %#v", stats, e) 1986 } 1987 } 1988 1989 func TestFSM_JobStabilityUpdate(t *testing.T) { 1990 t.Parallel() 1991 fsm := testFSM(t) 1992 fsm.evalBroker.SetEnabled(true) 1993 state := fsm.State() 1994 1995 // Upsert a deployment 1996 job := mock.Job() 1997 if err := state.UpsertJob(1, job); err != nil { 1998 t.Fatalf("bad: %v", err) 1999 } 2000 2001 // Create a request to update the job to stable 2002 req := &structs.JobStabilityRequest{ 2003 JobID: job.ID, 2004 JobVersion: job.Version, 2005 Stable: true, 2006 WriteRequest: structs.WriteRequest{ 2007 Namespace: job.Namespace, 2008 }, 2009 } 2010 buf, err := structs.Encode(structs.JobStabilityRequestType, req) 2011 if err != nil { 2012 t.Fatalf("err: %v", err) 2013 } 2014 resp := fsm.Apply(makeLog(buf)) 2015 if resp != nil { 2016 t.Fatalf("resp: %v", resp) 2017 } 2018 2019 // Check that the stability was updated properly 2020 ws := memdb.NewWatchSet() 2021 jout, _ := state.JobByIDAndVersion(ws, job.Namespace, job.ID, job.Version) 2022 if err != nil { 2023 t.Fatalf("bad: %v", err) 2024 } 2025 if jout == nil || !jout.Stable { 2026 t.Fatalf("bad: %#v", jout) 2027 } 2028 } 2029 2030 func TestFSM_DeploymentPromotion(t *testing.T) { 2031 t.Parallel() 2032 fsm := testFSM(t) 2033 fsm.evalBroker.SetEnabled(true) 2034 state := fsm.State() 2035 2036 // Create a job with two task groups 2037 j := mock.Job() 2038 tg1 := j.TaskGroups[0] 2039 tg2 := tg1.Copy() 2040 tg2.Name = "foo" 2041 j.TaskGroups = append(j.TaskGroups, tg2) 2042 if err := state.UpsertJob(1, j); err != nil { 2043 t.Fatalf("bad: %v", err) 2044 } 2045 2046 // Create a deployment 2047 d := mock.Deployment() 2048 d.JobID = j.ID 2049 d.TaskGroups = map[string]*structs.DeploymentState{ 2050 "web": { 2051 DesiredTotal: 10, 2052 DesiredCanaries: 1, 2053 }, 2054 "foo": { 2055 DesiredTotal: 10, 2056 DesiredCanaries: 1, 2057 }, 2058 } 2059 if err := state.UpsertDeployment(2, d); err != nil { 2060 t.Fatalf("bad: %v", err) 2061 } 2062 2063 // Create a set of allocations 2064 c1 := mock.Alloc() 2065 c1.JobID = j.ID 2066 c1.DeploymentID = d.ID 2067 d.TaskGroups[c1.TaskGroup].PlacedCanaries = append(d.TaskGroups[c1.TaskGroup].PlacedCanaries, c1.ID) 2068 c1.DeploymentStatus = &structs.AllocDeploymentStatus{ 2069 Healthy: helper.BoolToPtr(true), 2070 } 2071 c2 := mock.Alloc() 2072 c2.JobID = j.ID 2073 c2.DeploymentID = d.ID 2074 d.TaskGroups[c2.TaskGroup].PlacedCanaries = append(d.TaskGroups[c2.TaskGroup].PlacedCanaries, c2.ID) 2075 c2.TaskGroup = tg2.Name 2076 c2.DeploymentStatus = &structs.AllocDeploymentStatus{ 2077 Healthy: helper.BoolToPtr(true), 2078 } 2079 2080 if err := state.UpsertAllocs(3, []*structs.Allocation{c1, c2}); err != nil { 2081 t.Fatalf("err: %v", err) 2082 } 2083 2084 // Create an eval 2085 e := mock.Eval() 2086 2087 // Promote the canaries 2088 req := &structs.ApplyDeploymentPromoteRequest{ 2089 DeploymentPromoteRequest: structs.DeploymentPromoteRequest{ 2090 DeploymentID: d.ID, 2091 All: true, 2092 }, 2093 Eval: e, 2094 } 2095 buf, err := structs.Encode(structs.DeploymentPromoteRequestType, req) 2096 if err != nil { 2097 t.Fatalf("err: %v", err) 2098 } 2099 resp := fsm.Apply(makeLog(buf)) 2100 if resp != nil { 2101 t.Fatalf("resp: %v", resp) 2102 } 2103 2104 // Check that the status per task group was updated properly 2105 ws := memdb.NewWatchSet() 2106 dout, err := state.DeploymentByID(ws, d.ID) 2107 if err != nil { 2108 t.Fatalf("bad: %v", err) 2109 } 2110 if len(dout.TaskGroups) != 2 { 2111 t.Fatalf("bad: %#v", dout.TaskGroups) 2112 } 2113 for tg, state := range dout.TaskGroups { 2114 if !state.Promoted { 2115 t.Fatalf("bad: group %q not promoted %#v", tg, state) 2116 } 2117 } 2118 2119 // Check that the evaluation was created 2120 eout, _ := state.EvalByID(ws, e.ID) 2121 if err != nil { 2122 t.Fatalf("bad: %v", err) 2123 } 2124 if eout == nil { 2125 t.Fatalf("bad: %#v", eout) 2126 } 2127 2128 // Assert the eval was enqueued 2129 stats := fsm.evalBroker.Stats() 2130 if stats.TotalReady != 1 { 2131 t.Fatalf("bad: %#v %#v", stats, e) 2132 } 2133 } 2134 2135 func TestFSM_DeploymentAllocHealth(t *testing.T) { 2136 t.Parallel() 2137 fsm := testFSM(t) 2138 fsm.evalBroker.SetEnabled(true) 2139 state := fsm.State() 2140 2141 // Insert a deployment 2142 d := mock.Deployment() 2143 if err := state.UpsertDeployment(1, d); err != nil { 2144 t.Fatalf("bad: %v", err) 2145 } 2146 2147 // Insert two allocations 2148 a1 := mock.Alloc() 2149 a1.DeploymentID = d.ID 2150 a2 := mock.Alloc() 2151 a2.DeploymentID = d.ID 2152 if err := state.UpsertAllocs(2, []*structs.Allocation{a1, a2}); err != nil { 2153 t.Fatalf("bad: %v", err) 2154 } 2155 2156 // Create a job to roll back to 2157 j := mock.Job() 2158 2159 // Create an eval that should be upserted 2160 e := mock.Eval() 2161 2162 // Create a status update for the deployment 2163 status, desc := structs.DeploymentStatusFailed, "foo" 2164 u := &structs.DeploymentStatusUpdate{ 2165 DeploymentID: d.ID, 2166 Status: status, 2167 StatusDescription: desc, 2168 } 2169 2170 // Set health against the deployment 2171 req := &structs.ApplyDeploymentAllocHealthRequest{ 2172 DeploymentAllocHealthRequest: structs.DeploymentAllocHealthRequest{ 2173 DeploymentID: d.ID, 2174 HealthyAllocationIDs: []string{a1.ID}, 2175 UnhealthyAllocationIDs: []string{a2.ID}, 2176 }, 2177 Job: j, 2178 Eval: e, 2179 DeploymentUpdate: u, 2180 } 2181 buf, err := structs.Encode(structs.DeploymentAllocHealthRequestType, req) 2182 if err != nil { 2183 t.Fatalf("err: %v", err) 2184 } 2185 resp := fsm.Apply(makeLog(buf)) 2186 if resp != nil { 2187 t.Fatalf("resp: %v", resp) 2188 } 2189 2190 // Check that the status was updated properly 2191 ws := memdb.NewWatchSet() 2192 dout, err := state.DeploymentByID(ws, d.ID) 2193 if err != nil { 2194 t.Fatalf("bad: %v", err) 2195 } 2196 if dout.Status != status || dout.StatusDescription != desc { 2197 t.Fatalf("bad: %#v", dout) 2198 } 2199 2200 // Check that the evaluation was created 2201 eout, _ := state.EvalByID(ws, e.ID) 2202 if err != nil { 2203 t.Fatalf("bad: %v", err) 2204 } 2205 if eout == nil { 2206 t.Fatalf("bad: %#v", eout) 2207 } 2208 2209 // Check that the job was created 2210 jout, _ := state.JobByID(ws, j.Namespace, j.ID) 2211 if err != nil { 2212 t.Fatalf("bad: %v", err) 2213 } 2214 if jout == nil { 2215 t.Fatalf("bad: %#v", jout) 2216 } 2217 2218 // Check the status of the allocs 2219 out1, err := state.AllocByID(ws, a1.ID) 2220 if err != nil { 2221 t.Fatalf("err: %v", err) 2222 } 2223 out2, err := state.AllocByID(ws, a2.ID) 2224 if err != nil { 2225 t.Fatalf("err: %v", err) 2226 } 2227 2228 if !out1.DeploymentStatus.IsHealthy() { 2229 t.Fatalf("bad: alloc %q not healthy", out1.ID) 2230 } 2231 if !out2.DeploymentStatus.IsUnhealthy() { 2232 t.Fatalf("bad: alloc %q not unhealthy", out2.ID) 2233 } 2234 2235 // Assert the eval was enqueued 2236 stats := fsm.evalBroker.Stats() 2237 if stats.TotalReady != 1 { 2238 t.Fatalf("bad: %#v %#v", stats, e) 2239 } 2240 } 2241 2242 func TestFSM_DeleteDeployment(t *testing.T) { 2243 t.Parallel() 2244 fsm := testFSM(t) 2245 state := fsm.State() 2246 2247 // Upsert a deployments 2248 d := mock.Deployment() 2249 if err := state.UpsertDeployment(1, d); err != nil { 2250 t.Fatalf("bad: %v", err) 2251 } 2252 2253 req := structs.DeploymentDeleteRequest{ 2254 Deployments: []string{d.ID}, 2255 } 2256 buf, err := structs.Encode(structs.DeploymentDeleteRequestType, req) 2257 if err != nil { 2258 t.Fatalf("err: %v", err) 2259 } 2260 2261 resp := fsm.Apply(makeLog(buf)) 2262 if resp != nil { 2263 t.Fatalf("resp: %v", resp) 2264 } 2265 2266 // Verify we are NOT registered 2267 ws := memdb.NewWatchSet() 2268 deployment, err := state.DeploymentByID(ws, d.ID) 2269 if err != nil { 2270 t.Fatalf("err: %v", err) 2271 } 2272 if deployment != nil { 2273 t.Fatalf("deployment found!") 2274 } 2275 } 2276 2277 func TestFSM_UpsertACLPolicies(t *testing.T) { 2278 t.Parallel() 2279 fsm := testFSM(t) 2280 2281 policy := mock.ACLPolicy() 2282 req := structs.ACLPolicyUpsertRequest{ 2283 Policies: []*structs.ACLPolicy{policy}, 2284 } 2285 buf, err := structs.Encode(structs.ACLPolicyUpsertRequestType, req) 2286 if err != nil { 2287 t.Fatalf("err: %v", err) 2288 } 2289 2290 resp := fsm.Apply(makeLog(buf)) 2291 if resp != nil { 2292 t.Fatalf("resp: %v", resp) 2293 } 2294 2295 // Verify we are registered 2296 ws := memdb.NewWatchSet() 2297 out, err := fsm.State().ACLPolicyByName(ws, policy.Name) 2298 assert.Nil(t, err) 2299 assert.NotNil(t, out) 2300 } 2301 2302 func TestFSM_DeleteACLPolicies(t *testing.T) { 2303 t.Parallel() 2304 fsm := testFSM(t) 2305 2306 policy := mock.ACLPolicy() 2307 err := fsm.State().UpsertACLPolicies(1000, []*structs.ACLPolicy{policy}) 2308 assert.Nil(t, err) 2309 2310 req := structs.ACLPolicyDeleteRequest{ 2311 Names: []string{policy.Name}, 2312 } 2313 buf, err := structs.Encode(structs.ACLPolicyDeleteRequestType, req) 2314 if err != nil { 2315 t.Fatalf("err: %v", err) 2316 } 2317 2318 resp := fsm.Apply(makeLog(buf)) 2319 if resp != nil { 2320 t.Fatalf("resp: %v", resp) 2321 } 2322 2323 // Verify we are NOT registered 2324 ws := memdb.NewWatchSet() 2325 out, err := fsm.State().ACLPolicyByName(ws, policy.Name) 2326 assert.Nil(t, err) 2327 assert.Nil(t, out) 2328 } 2329 2330 func TestFSM_BootstrapACLTokens(t *testing.T) { 2331 t.Parallel() 2332 fsm := testFSM(t) 2333 2334 token := mock.ACLToken() 2335 req := structs.ACLTokenBootstrapRequest{ 2336 Token: token, 2337 } 2338 buf, err := structs.Encode(structs.ACLTokenBootstrapRequestType, req) 2339 if err != nil { 2340 t.Fatalf("err: %v", err) 2341 } 2342 2343 resp := fsm.Apply(makeLog(buf)) 2344 if resp != nil { 2345 t.Fatalf("resp: %v", resp) 2346 } 2347 2348 // Verify we are registered 2349 out, err := fsm.State().ACLTokenByAccessorID(nil, token.AccessorID) 2350 assert.Nil(t, err) 2351 assert.NotNil(t, out) 2352 2353 // Test with reset 2354 token2 := mock.ACLToken() 2355 req = structs.ACLTokenBootstrapRequest{ 2356 Token: token2, 2357 ResetIndex: out.CreateIndex, 2358 } 2359 buf, err = structs.Encode(structs.ACLTokenBootstrapRequestType, req) 2360 if err != nil { 2361 t.Fatalf("err: %v", err) 2362 } 2363 2364 resp = fsm.Apply(makeLog(buf)) 2365 if resp != nil { 2366 t.Fatalf("resp: %v", resp) 2367 } 2368 2369 // Verify we are registered 2370 out2, err := fsm.State().ACLTokenByAccessorID(nil, token2.AccessorID) 2371 assert.Nil(t, err) 2372 assert.NotNil(t, out2) 2373 } 2374 2375 func TestFSM_UpsertACLTokens(t *testing.T) { 2376 t.Parallel() 2377 fsm := testFSM(t) 2378 2379 token := mock.ACLToken() 2380 req := structs.ACLTokenUpsertRequest{ 2381 Tokens: []*structs.ACLToken{token}, 2382 } 2383 buf, err := structs.Encode(structs.ACLTokenUpsertRequestType, req) 2384 if err != nil { 2385 t.Fatalf("err: %v", err) 2386 } 2387 2388 resp := fsm.Apply(makeLog(buf)) 2389 if resp != nil { 2390 t.Fatalf("resp: %v", resp) 2391 } 2392 2393 // Verify we are registered 2394 ws := memdb.NewWatchSet() 2395 out, err := fsm.State().ACLTokenByAccessorID(ws, token.AccessorID) 2396 assert.Nil(t, err) 2397 assert.NotNil(t, out) 2398 } 2399 2400 func TestFSM_DeleteACLTokens(t *testing.T) { 2401 t.Parallel() 2402 fsm := testFSM(t) 2403 2404 token := mock.ACLToken() 2405 err := fsm.State().UpsertACLTokens(1000, []*structs.ACLToken{token}) 2406 assert.Nil(t, err) 2407 2408 req := structs.ACLTokenDeleteRequest{ 2409 AccessorIDs: []string{token.AccessorID}, 2410 } 2411 buf, err := structs.Encode(structs.ACLTokenDeleteRequestType, req) 2412 if err != nil { 2413 t.Fatalf("err: %v", err) 2414 } 2415 2416 resp := fsm.Apply(makeLog(buf)) 2417 if resp != nil { 2418 t.Fatalf("resp: %v", resp) 2419 } 2420 2421 // Verify we are NOT registered 2422 ws := memdb.NewWatchSet() 2423 out, err := fsm.State().ACLTokenByAccessorID(ws, token.AccessorID) 2424 assert.Nil(t, err) 2425 assert.Nil(t, out) 2426 } 2427 2428 func testSnapshotRestore(t *testing.T, fsm *nomadFSM) *nomadFSM { 2429 // Snapshot 2430 snap, err := fsm.Snapshot() 2431 if err != nil { 2432 t.Fatalf("err: %v", err) 2433 } 2434 defer snap.Release() 2435 2436 // Persist 2437 buf := bytes.NewBuffer(nil) 2438 sink := &MockSink{buf, false} 2439 if err := snap.Persist(sink); err != nil { 2440 t.Fatalf("err: %v", err) 2441 } 2442 2443 // Try to restore on a new FSM 2444 fsm2 := testFSM(t) 2445 snap, err = fsm2.Snapshot() 2446 if err != nil { 2447 t.Fatalf("err: %v", err) 2448 } 2449 defer snap.Release() 2450 2451 abandonCh := fsm2.State().AbandonCh() 2452 2453 // Do a restore 2454 if err := fsm2.Restore(sink); err != nil { 2455 t.Fatalf("err: %v", err) 2456 } 2457 2458 select { 2459 case <-abandonCh: 2460 default: 2461 t.Fatalf("bad") 2462 } 2463 2464 return fsm2 2465 } 2466 2467 func TestFSM_SnapshotRestore_Nodes(t *testing.T) { 2468 t.Parallel() 2469 // Add some state 2470 fsm := testFSM(t) 2471 state := fsm.State() 2472 node1 := mock.Node() 2473 state.UpsertNode(1000, node1) 2474 2475 // Upgrade this node 2476 node2 := mock.Node() 2477 node2.SchedulingEligibility = "" 2478 state.UpsertNode(1001, node2) 2479 2480 // Verify the contents 2481 fsm2 := testSnapshotRestore(t, fsm) 2482 state2 := fsm2.State() 2483 out1, _ := state2.NodeByID(nil, node1.ID) 2484 out2, _ := state2.NodeByID(nil, node2.ID) 2485 node2.SchedulingEligibility = structs.NodeSchedulingEligible 2486 if !reflect.DeepEqual(node1, out1) { 2487 t.Fatalf("bad: \n%#v\n%#v", out1, node1) 2488 } 2489 if !reflect.DeepEqual(node2, out2) { 2490 t.Fatalf("bad: \n%#v\n%#v", out2, node2) 2491 } 2492 } 2493 2494 func TestFSM_SnapshotRestore_Jobs(t *testing.T) { 2495 t.Parallel() 2496 // Add some state 2497 fsm := testFSM(t) 2498 state := fsm.State() 2499 job1 := mock.Job() 2500 state.UpsertJob(1000, job1) 2501 job2 := mock.Job() 2502 state.UpsertJob(1001, job2) 2503 2504 // Verify the contents 2505 ws := memdb.NewWatchSet() 2506 fsm2 := testSnapshotRestore(t, fsm) 2507 state2 := fsm2.State() 2508 out1, _ := state2.JobByID(ws, job1.Namespace, job1.ID) 2509 out2, _ := state2.JobByID(ws, job2.Namespace, job2.ID) 2510 if !reflect.DeepEqual(job1, out1) { 2511 t.Fatalf("bad: \n%#v\n%#v", out1, job1) 2512 } 2513 if !reflect.DeepEqual(job2, out2) { 2514 t.Fatalf("bad: \n%#v\n%#v", out2, job2) 2515 } 2516 } 2517 2518 func TestFSM_SnapshotRestore_Evals(t *testing.T) { 2519 t.Parallel() 2520 // Add some state 2521 fsm := testFSM(t) 2522 state := fsm.State() 2523 eval1 := mock.Eval() 2524 state.UpsertEvals(1000, []*structs.Evaluation{eval1}) 2525 eval2 := mock.Eval() 2526 state.UpsertEvals(1001, []*structs.Evaluation{eval2}) 2527 2528 // Verify the contents 2529 fsm2 := testSnapshotRestore(t, fsm) 2530 state2 := fsm2.State() 2531 ws := memdb.NewWatchSet() 2532 out1, _ := state2.EvalByID(ws, eval1.ID) 2533 out2, _ := state2.EvalByID(ws, eval2.ID) 2534 if !reflect.DeepEqual(eval1, out1) { 2535 t.Fatalf("bad: \n%#v\n%#v", out1, eval1) 2536 } 2537 if !reflect.DeepEqual(eval2, out2) { 2538 t.Fatalf("bad: \n%#v\n%#v", out2, eval2) 2539 } 2540 } 2541 2542 func TestFSM_SnapshotRestore_Allocs(t *testing.T) { 2543 t.Parallel() 2544 // Add some state 2545 fsm := testFSM(t) 2546 state := fsm.State() 2547 alloc1 := mock.Alloc() 2548 alloc2 := mock.Alloc() 2549 state.UpsertJobSummary(998, mock.JobSummary(alloc1.JobID)) 2550 state.UpsertJobSummary(999, mock.JobSummary(alloc2.JobID)) 2551 state.UpsertAllocs(1000, []*structs.Allocation{alloc1}) 2552 state.UpsertAllocs(1001, []*structs.Allocation{alloc2}) 2553 2554 // Verify the contents 2555 fsm2 := testSnapshotRestore(t, fsm) 2556 state2 := fsm2.State() 2557 ws := memdb.NewWatchSet() 2558 out1, _ := state2.AllocByID(ws, alloc1.ID) 2559 out2, _ := state2.AllocByID(ws, alloc2.ID) 2560 if !reflect.DeepEqual(alloc1, out1) { 2561 t.Fatalf("bad: \n%#v\n%#v", out1, alloc1) 2562 } 2563 if !reflect.DeepEqual(alloc2, out2) { 2564 t.Fatalf("bad: \n%#v\n%#v", out2, alloc2) 2565 } 2566 } 2567 2568 func TestFSM_SnapshotRestore_Allocs_Canonicalize(t *testing.T) { 2569 t.Parallel() 2570 // Add some state 2571 fsm := testFSM(t) 2572 state := fsm.State() 2573 alloc := mock.Alloc() 2574 2575 // remove old versions to force migration path 2576 alloc.AllocatedResources = nil 2577 2578 state.UpsertJobSummary(998, mock.JobSummary(alloc.JobID)) 2579 state.UpsertAllocs(1000, []*structs.Allocation{alloc}) 2580 2581 // Verify the contents 2582 fsm2 := testSnapshotRestore(t, fsm) 2583 state2 := fsm2.State() 2584 ws := memdb.NewWatchSet() 2585 out, err := state2.AllocByID(ws, alloc.ID) 2586 require.NoError(t, err) 2587 2588 require.NotNil(t, out.AllocatedResources) 2589 require.Contains(t, out.AllocatedResources.Tasks, "web") 2590 2591 alloc.Canonicalize() 2592 require.Equal(t, alloc, out) 2593 } 2594 2595 func TestFSM_SnapshotRestore_Indexes(t *testing.T) { 2596 t.Parallel() 2597 // Add some state 2598 fsm := testFSM(t) 2599 state := fsm.State() 2600 node1 := mock.Node() 2601 state.UpsertNode(1000, node1) 2602 2603 // Verify the contents 2604 fsm2 := testSnapshotRestore(t, fsm) 2605 state2 := fsm2.State() 2606 2607 index, err := state2.Index("nodes") 2608 if err != nil { 2609 t.Fatalf("err: %v", err) 2610 } 2611 if index != 1000 { 2612 t.Fatalf("bad: %d", index) 2613 } 2614 } 2615 2616 func TestFSM_SnapshotRestore_TimeTable(t *testing.T) { 2617 t.Parallel() 2618 // Add some state 2619 fsm := testFSM(t) 2620 2621 tt := fsm.TimeTable() 2622 start := time.Now().UTC() 2623 tt.Witness(1000, start) 2624 tt.Witness(2000, start.Add(10*time.Minute)) 2625 2626 // Verify the contents 2627 fsm2 := testSnapshotRestore(t, fsm) 2628 2629 tt2 := fsm2.TimeTable() 2630 if tt2.NearestTime(1500) != start { 2631 t.Fatalf("bad") 2632 } 2633 if tt2.NearestIndex(start.Add(15*time.Minute)) != 2000 { 2634 t.Fatalf("bad") 2635 } 2636 } 2637 2638 func TestFSM_SnapshotRestore_PeriodicLaunches(t *testing.T) { 2639 t.Parallel() 2640 // Add some state 2641 fsm := testFSM(t) 2642 state := fsm.State() 2643 job1 := mock.Job() 2644 launch1 := &structs.PeriodicLaunch{ 2645 ID: job1.ID, 2646 Namespace: job1.Namespace, 2647 Launch: time.Now(), 2648 } 2649 state.UpsertPeriodicLaunch(1000, launch1) 2650 job2 := mock.Job() 2651 launch2 := &structs.PeriodicLaunch{ 2652 ID: job2.ID, 2653 Namespace: job2.Namespace, 2654 Launch: time.Now(), 2655 } 2656 state.UpsertPeriodicLaunch(1001, launch2) 2657 2658 // Verify the contents 2659 fsm2 := testSnapshotRestore(t, fsm) 2660 state2 := fsm2.State() 2661 ws := memdb.NewWatchSet() 2662 out1, _ := state2.PeriodicLaunchByID(ws, launch1.Namespace, launch1.ID) 2663 out2, _ := state2.PeriodicLaunchByID(ws, launch2.Namespace, launch2.ID) 2664 2665 if !cmp.Equal(launch1, out1) { 2666 t.Fatalf("bad: %v", cmp.Diff(launch1, out1)) 2667 } 2668 if !cmp.Equal(launch2, out2) { 2669 t.Fatalf("bad: %v", cmp.Diff(launch2, out2)) 2670 } 2671 } 2672 2673 func TestFSM_SnapshotRestore_JobSummary(t *testing.T) { 2674 t.Parallel() 2675 // Add some state 2676 fsm := testFSM(t) 2677 state := fsm.State() 2678 2679 job1 := mock.Job() 2680 state.UpsertJob(1000, job1) 2681 ws := memdb.NewWatchSet() 2682 js1, _ := state.JobSummaryByID(ws, job1.Namespace, job1.ID) 2683 2684 job2 := mock.Job() 2685 state.UpsertJob(1001, job2) 2686 js2, _ := state.JobSummaryByID(ws, job2.Namespace, job2.ID) 2687 2688 // Verify the contents 2689 fsm2 := testSnapshotRestore(t, fsm) 2690 state2 := fsm2.State() 2691 out1, _ := state2.JobSummaryByID(ws, job1.Namespace, job1.ID) 2692 out2, _ := state2.JobSummaryByID(ws, job2.Namespace, job2.ID) 2693 if !reflect.DeepEqual(js1, out1) { 2694 t.Fatalf("bad: \n%#v\n%#v", js1, out1) 2695 } 2696 if !reflect.DeepEqual(js2, out2) { 2697 t.Fatalf("bad: \n%#v\n%#v", js2, out2) 2698 } 2699 } 2700 2701 func TestFSM_SnapshotRestore_VaultAccessors(t *testing.T) { 2702 t.Parallel() 2703 // Add some state 2704 fsm := testFSM(t) 2705 state := fsm.State() 2706 a1 := mock.VaultAccessor() 2707 a2 := mock.VaultAccessor() 2708 state.UpsertVaultAccessor(1000, []*structs.VaultAccessor{a1, a2}) 2709 2710 // Verify the contents 2711 fsm2 := testSnapshotRestore(t, fsm) 2712 state2 := fsm2.State() 2713 ws := memdb.NewWatchSet() 2714 out1, _ := state2.VaultAccessor(ws, a1.Accessor) 2715 out2, _ := state2.VaultAccessor(ws, a2.Accessor) 2716 if !reflect.DeepEqual(a1, out1) { 2717 t.Fatalf("bad: \n%#v\n%#v", out1, a1) 2718 } 2719 if !reflect.DeepEqual(a2, out2) { 2720 t.Fatalf("bad: \n%#v\n%#v", out2, a2) 2721 } 2722 } 2723 2724 func TestFSM_SnapshotRestore_JobVersions(t *testing.T) { 2725 t.Parallel() 2726 // Add some state 2727 fsm := testFSM(t) 2728 state := fsm.State() 2729 job1 := mock.Job() 2730 state.UpsertJob(1000, job1) 2731 job2 := mock.Job() 2732 job2.ID = job1.ID 2733 state.UpsertJob(1001, job2) 2734 2735 // Verify the contents 2736 ws := memdb.NewWatchSet() 2737 fsm2 := testSnapshotRestore(t, fsm) 2738 state2 := fsm2.State() 2739 out1, _ := state2.JobByIDAndVersion(ws, job1.Namespace, job1.ID, job1.Version) 2740 out2, _ := state2.JobByIDAndVersion(ws, job2.Namespace, job2.ID, job2.Version) 2741 if !reflect.DeepEqual(job1, out1) { 2742 t.Fatalf("bad: \n%#v\n%#v", out1, job1) 2743 } 2744 if !reflect.DeepEqual(job2, out2) { 2745 t.Fatalf("bad: \n%#v\n%#v", out2, job2) 2746 } 2747 if job2.Version != 1 { 2748 t.Fatalf("bad: \n%#v\n%#v", 1, job2) 2749 } 2750 } 2751 2752 func TestFSM_SnapshotRestore_Deployments(t *testing.T) { 2753 t.Parallel() 2754 // Add some state 2755 fsm := testFSM(t) 2756 state := fsm.State() 2757 d1 := mock.Deployment() 2758 d2 := mock.Deployment() 2759 2760 j := mock.Job() 2761 d1.JobID = j.ID 2762 d2.JobID = j.ID 2763 2764 state.UpsertJob(999, j) 2765 state.UpsertDeployment(1000, d1) 2766 state.UpsertDeployment(1001, d2) 2767 2768 // Verify the contents 2769 fsm2 := testSnapshotRestore(t, fsm) 2770 state2 := fsm2.State() 2771 ws := memdb.NewWatchSet() 2772 out1, _ := state2.DeploymentByID(ws, d1.ID) 2773 out2, _ := state2.DeploymentByID(ws, d2.ID) 2774 if !reflect.DeepEqual(d1, out1) { 2775 t.Fatalf("bad: \n%#v\n%#v", out1, d1) 2776 } 2777 if !reflect.DeepEqual(d2, out2) { 2778 t.Fatalf("bad: \n%#v\n%#v", out2, d2) 2779 } 2780 } 2781 2782 func TestFSM_SnapshotRestore_ACLPolicy(t *testing.T) { 2783 t.Parallel() 2784 // Add some state 2785 fsm := testFSM(t) 2786 state := fsm.State() 2787 p1 := mock.ACLPolicy() 2788 p2 := mock.ACLPolicy() 2789 state.UpsertACLPolicies(1000, []*structs.ACLPolicy{p1, p2}) 2790 2791 // Verify the contents 2792 fsm2 := testSnapshotRestore(t, fsm) 2793 state2 := fsm2.State() 2794 ws := memdb.NewWatchSet() 2795 out1, _ := state2.ACLPolicyByName(ws, p1.Name) 2796 out2, _ := state2.ACLPolicyByName(ws, p2.Name) 2797 assert.Equal(t, p1, out1) 2798 assert.Equal(t, p2, out2) 2799 } 2800 2801 func TestFSM_SnapshotRestore_ACLTokens(t *testing.T) { 2802 t.Parallel() 2803 // Add some state 2804 fsm := testFSM(t) 2805 state := fsm.State() 2806 tk1 := mock.ACLToken() 2807 tk2 := mock.ACLToken() 2808 state.UpsertACLTokens(1000, []*structs.ACLToken{tk1, tk2}) 2809 2810 // Verify the contents 2811 fsm2 := testSnapshotRestore(t, fsm) 2812 state2 := fsm2.State() 2813 ws := memdb.NewWatchSet() 2814 out1, _ := state2.ACLTokenByAccessorID(ws, tk1.AccessorID) 2815 out2, _ := state2.ACLTokenByAccessorID(ws, tk2.AccessorID) 2816 assert.Equal(t, tk1, out1) 2817 assert.Equal(t, tk2, out2) 2818 } 2819 2820 func TestFSM_SnapshotRestore_SchedulerConfiguration(t *testing.T) { 2821 t.Parallel() 2822 // Add some state 2823 fsm := testFSM(t) 2824 state := fsm.State() 2825 schedConfig := &structs.SchedulerConfiguration{ 2826 PreemptionConfig: structs.PreemptionConfig{ 2827 SystemSchedulerEnabled: true, 2828 }, 2829 } 2830 state.SchedulerSetConfig(1000, schedConfig) 2831 2832 // Verify the contents 2833 require := require.New(t) 2834 fsm2 := testSnapshotRestore(t, fsm) 2835 state2 := fsm2.State() 2836 index, out, err := state2.SchedulerConfig() 2837 require.Nil(err) 2838 require.EqualValues(1000, index) 2839 require.Equal(schedConfig, out) 2840 } 2841 2842 func TestFSM_SnapshotRestore_ClusterMetadata(t *testing.T) { 2843 t.Parallel() 2844 2845 fsm := testFSM(t) 2846 state := fsm.State() 2847 clusterID := "12345678-1234-1234-1234-1234567890" 2848 now := time.Now().UnixNano() 2849 meta := &structs.ClusterMetadata{ClusterID: clusterID, CreateTime: now} 2850 state.ClusterSetMetadata(1000, meta) 2851 2852 // Verify the contents 2853 require := require.New(t) 2854 fsm2 := testSnapshotRestore(t, fsm) 2855 state2 := fsm2.State() 2856 out, err := state2.ClusterMetadata() 2857 require.NoError(err) 2858 require.Equal(clusterID, out.ClusterID) 2859 } 2860 2861 func TestFSM_ReconcileSummaries(t *testing.T) { 2862 t.Parallel() 2863 // Add some state 2864 fsm := testFSM(t) 2865 state := fsm.State() 2866 2867 // Add a node 2868 node := mock.Node() 2869 state.UpsertNode(800, node) 2870 2871 // Make a job so that none of the tasks can be placed 2872 job1 := mock.Job() 2873 job1.TaskGroups[0].Tasks[0].Resources.CPU = 5000 2874 state.UpsertJob(1000, job1) 2875 2876 // make a job which can make partial progress 2877 alloc := mock.Alloc() 2878 alloc.NodeID = node.ID 2879 state.UpsertJob(1010, alloc.Job) 2880 state.UpsertAllocs(1011, []*structs.Allocation{alloc}) 2881 2882 // Delete the summaries 2883 state.DeleteJobSummary(1030, job1.Namespace, job1.ID) 2884 state.DeleteJobSummary(1040, alloc.Namespace, alloc.Job.ID) 2885 2886 req := structs.GenericRequest{} 2887 buf, err := structs.Encode(structs.ReconcileJobSummariesRequestType, req) 2888 if err != nil { 2889 t.Fatalf("err: %v", err) 2890 } 2891 2892 resp := fsm.Apply(makeLog(buf)) 2893 if resp != nil { 2894 t.Fatalf("resp: %v", resp) 2895 } 2896 2897 ws := memdb.NewWatchSet() 2898 out1, _ := state.JobSummaryByID(ws, job1.Namespace, job1.ID) 2899 expected := structs.JobSummary{ 2900 JobID: job1.ID, 2901 Namespace: job1.Namespace, 2902 Summary: map[string]structs.TaskGroupSummary{ 2903 "web": { 2904 Queued: 10, 2905 }, 2906 }, 2907 CreateIndex: 1000, 2908 ModifyIndex: out1.ModifyIndex, 2909 } 2910 if !reflect.DeepEqual(&expected, out1) { 2911 t.Fatalf("expected: %#v, actual: %#v", &expected, out1) 2912 } 2913 2914 // This exercises the code path which adds the allocations made by the 2915 // planner and the number of unplaced allocations in the reconcile summaries 2916 // codepath 2917 out2, _ := state.JobSummaryByID(ws, alloc.Namespace, alloc.Job.ID) 2918 expected = structs.JobSummary{ 2919 JobID: alloc.Job.ID, 2920 Namespace: alloc.Job.Namespace, 2921 Summary: map[string]structs.TaskGroupSummary{ 2922 "web": { 2923 Queued: 9, 2924 Starting: 1, 2925 }, 2926 }, 2927 CreateIndex: 1010, 2928 ModifyIndex: out2.ModifyIndex, 2929 } 2930 if !reflect.DeepEqual(&expected, out2) { 2931 t.Fatalf("Diff % #v", pretty.Diff(&expected, out2)) 2932 } 2933 } 2934 2935 // COMPAT: Remove in 0.11 2936 func TestFSM_ReconcileParentJobSummary(t *testing.T) { 2937 // This test exercises code to handle https://github.com/hashicorp/nomad/issues/3886 2938 t.Parallel() 2939 2940 require := require.New(t) 2941 // Add some state 2942 fsm := testFSM(t) 2943 state := fsm.State() 2944 2945 // Add a node 2946 node := mock.Node() 2947 state.UpsertNode(800, node) 2948 2949 // Make a parameterized job 2950 job1 := mock.BatchJob() 2951 job1.ID = "test" 2952 job1.ParameterizedJob = &structs.ParameterizedJobConfig{ 2953 Payload: "random", 2954 } 2955 job1.TaskGroups[0].Count = 1 2956 state.UpsertJob(1000, job1) 2957 2958 // Make a child job 2959 childJob := job1.Copy() 2960 childJob.ID = job1.ID + "dispatch-23423423" 2961 childJob.ParentID = job1.ID 2962 childJob.Dispatched = true 2963 childJob.Status = structs.JobStatusRunning 2964 2965 // Create an alloc for child job 2966 alloc := mock.Alloc() 2967 alloc.NodeID = node.ID 2968 alloc.Job = childJob 2969 alloc.JobID = childJob.ID 2970 alloc.ClientStatus = structs.AllocClientStatusRunning 2971 2972 state.UpsertJob(1010, childJob) 2973 state.UpsertAllocs(1011, []*structs.Allocation{alloc}) 2974 2975 // Make the summary incorrect in the state store 2976 summary, err := state.JobSummaryByID(nil, job1.Namespace, job1.ID) 2977 require.Nil(err) 2978 2979 summary.Children = nil 2980 summary.Summary = make(map[string]structs.TaskGroupSummary) 2981 summary.Summary["web"] = structs.TaskGroupSummary{ 2982 Queued: 1, 2983 } 2984 2985 req := structs.GenericRequest{} 2986 buf, err := structs.Encode(structs.ReconcileJobSummariesRequestType, req) 2987 require.Nil(err) 2988 2989 resp := fsm.Apply(makeLog(buf)) 2990 require.Nil(resp) 2991 2992 ws := memdb.NewWatchSet() 2993 out1, _ := state.JobSummaryByID(ws, job1.Namespace, job1.ID) 2994 expected := structs.JobSummary{ 2995 JobID: job1.ID, 2996 Namespace: job1.Namespace, 2997 Summary: make(map[string]structs.TaskGroupSummary), 2998 CreateIndex: 1000, 2999 ModifyIndex: out1.ModifyIndex, 3000 Children: &structs.JobChildrenSummary{ 3001 Running: 1, 3002 }, 3003 } 3004 require.Equal(&expected, out1) 3005 } 3006 3007 func TestFSM_LeakedDeployments(t *testing.T) { 3008 t.Parallel() 3009 require := require.New(t) 3010 3011 // Add some state 3012 fsm := testFSM(t) 3013 state := fsm.State() 3014 d := mock.Deployment() 3015 require.NoError(state.UpsertDeployment(1000, d)) 3016 3017 // Verify the contents 3018 fsm2 := testSnapshotRestore(t, fsm) 3019 state2 := fsm2.State() 3020 out, _ := state2.DeploymentByID(nil, d.ID) 3021 require.NotNil(out) 3022 require.Equal(structs.DeploymentStatusCancelled, out.Status) 3023 } 3024 3025 func TestFSM_Autopilot(t *testing.T) { 3026 t.Parallel() 3027 fsm := testFSM(t) 3028 3029 // Set the autopilot config using a request. 3030 req := structs.AutopilotSetConfigRequest{ 3031 Datacenter: "dc1", 3032 Config: structs.AutopilotConfig{ 3033 CleanupDeadServers: true, 3034 LastContactThreshold: 10 * time.Second, 3035 MaxTrailingLogs: 300, 3036 MinQuorum: 3, 3037 }, 3038 } 3039 buf, err := structs.Encode(structs.AutopilotRequestType, req) 3040 if err != nil { 3041 t.Fatalf("err: %v", err) 3042 } 3043 resp := fsm.Apply(makeLog(buf)) 3044 if _, ok := resp.(error); ok { 3045 t.Fatalf("bad: %v", resp) 3046 } 3047 3048 // Verify key is set directly in the state store. 3049 _, config, err := fsm.state.AutopilotConfig() 3050 if err != nil { 3051 t.Fatalf("err: %v", err) 3052 } 3053 if config.CleanupDeadServers != req.Config.CleanupDeadServers { 3054 t.Fatalf("bad: %v", config.CleanupDeadServers) 3055 } 3056 if config.LastContactThreshold != req.Config.LastContactThreshold { 3057 t.Fatalf("bad: %v", config.LastContactThreshold) 3058 } 3059 if config.MaxTrailingLogs != req.Config.MaxTrailingLogs { 3060 t.Fatalf("bad: %v", config.MaxTrailingLogs) 3061 } 3062 if config.MinQuorum != req.Config.MinQuorum { 3063 t.Fatalf("bad: %v", config.MinQuorum) 3064 } 3065 3066 // Now use CAS and provide an old index 3067 req.CAS = true 3068 req.Config.CleanupDeadServers = false 3069 req.Config.ModifyIndex = config.ModifyIndex - 1 3070 buf, err = structs.Encode(structs.AutopilotRequestType, req) 3071 if err != nil { 3072 t.Fatalf("err: %v", err) 3073 } 3074 resp = fsm.Apply(makeLog(buf)) 3075 if _, ok := resp.(error); ok { 3076 t.Fatalf("bad: %v", resp) 3077 } 3078 3079 _, config, err = fsm.state.AutopilotConfig() 3080 if err != nil { 3081 t.Fatalf("err: %v", err) 3082 } 3083 if !config.CleanupDeadServers { 3084 t.Fatalf("bad: %v", config.CleanupDeadServers) 3085 } 3086 } 3087 3088 func TestFSM_SchedulerConfig(t *testing.T) { 3089 t.Parallel() 3090 fsm := testFSM(t) 3091 3092 require := require.New(t) 3093 3094 // Set the scheduler config using a request. 3095 req := structs.SchedulerSetConfigRequest{ 3096 Config: structs.SchedulerConfiguration{ 3097 PreemptionConfig: structs.PreemptionConfig{ 3098 SystemSchedulerEnabled: true, 3099 BatchSchedulerEnabled: true, 3100 }, 3101 }, 3102 } 3103 buf, err := structs.Encode(structs.SchedulerConfigRequestType, req) 3104 require.Nil(err) 3105 3106 resp := fsm.Apply(makeLog(buf)) 3107 if _, ok := resp.(error); ok { 3108 t.Fatalf("bad: %v", resp) 3109 } 3110 3111 // Verify key is set directly in the state store. 3112 _, config, err := fsm.state.SchedulerConfig() 3113 require.Nil(err) 3114 3115 require.Equal(config.PreemptionConfig.SystemSchedulerEnabled, req.Config.PreemptionConfig.SystemSchedulerEnabled) 3116 require.Equal(config.PreemptionConfig.BatchSchedulerEnabled, req.Config.PreemptionConfig.BatchSchedulerEnabled) 3117 3118 // Now use CAS and provide an old index 3119 req.CAS = true 3120 req.Config.PreemptionConfig = structs.PreemptionConfig{SystemSchedulerEnabled: false, BatchSchedulerEnabled: false} 3121 req.Config.ModifyIndex = config.ModifyIndex - 1 3122 buf, err = structs.Encode(structs.SchedulerConfigRequestType, req) 3123 require.Nil(err) 3124 3125 resp = fsm.Apply(makeLog(buf)) 3126 if _, ok := resp.(error); ok { 3127 t.Fatalf("bad: %v", resp) 3128 } 3129 3130 _, config, err = fsm.state.SchedulerConfig() 3131 require.Nil(err) 3132 // Verify that preemption is still enabled 3133 require.True(config.PreemptionConfig.SystemSchedulerEnabled) 3134 require.True(config.PreemptionConfig.BatchSchedulerEnabled) 3135 } 3136 3137 func TestFSM_ClusterMetadata(t *testing.T) { 3138 t.Parallel() 3139 r := require.New(t) 3140 3141 fsm := testFSM(t) 3142 clusterID := "12345678-1234-1234-1234-1234567890" 3143 now := time.Now().UnixNano() 3144 meta := structs.ClusterMetadata{ 3145 ClusterID: clusterID, 3146 CreateTime: now, 3147 } 3148 buf, err := structs.Encode(structs.ClusterMetadataRequestType, meta) 3149 r.NoError(err) 3150 3151 result := fsm.Apply(makeLog(buf)) 3152 r.Nil(result) 3153 3154 // Verify the clusterID is set directly in the state store 3155 storedMetadata, err := fsm.state.ClusterMetadata() 3156 r.NoError(err) 3157 r.Equal(clusterID, storedMetadata.ClusterID) 3158 3159 // Check that the sanity check prevents accidental UUID regeneration 3160 erroneous := structs.ClusterMetadata{ 3161 ClusterID: "99999999-9999-9999-9999-9999999999", 3162 } 3163 buf, err = structs.Encode(structs.ClusterMetadataRequestType, erroneous) 3164 r.NoError(err) 3165 3166 result = fsm.Apply(makeLog(buf)) 3167 r.Error(result.(error)) 3168 3169 storedMetadata, err = fsm.state.ClusterMetadata() 3170 r.NoError(err) 3171 r.Equal(clusterID, storedMetadata.ClusterID) 3172 r.Equal(now, storedMetadata.CreateTime) 3173 }