github.com/huiliang/nomad@v0.2.1-0.20151124023127-7a8b664699ff/nomad/fsm_test.go (about) 1 package nomad 2 3 import ( 4 "bytes" 5 "os" 6 "reflect" 7 "testing" 8 "time" 9 10 "github.com/hashicorp/nomad/nomad/mock" 11 "github.com/hashicorp/nomad/nomad/state" 12 "github.com/hashicorp/nomad/nomad/structs" 13 "github.com/hashicorp/raft" 14 ) 15 16 type MockSink struct { 17 *bytes.Buffer 18 cancel bool 19 } 20 21 func (m *MockSink) ID() string { 22 return "Mock" 23 } 24 25 func (m *MockSink) Cancel() error { 26 m.cancel = true 27 return nil 28 } 29 30 func (m *MockSink) Close() error { 31 return nil 32 } 33 34 func testStateStore(t *testing.T) *state.StateStore { 35 state, err := state.NewStateStore(os.Stderr) 36 if err != nil { 37 t.Fatalf("err: %v", err) 38 } 39 if state == nil { 40 t.Fatalf("missing state") 41 } 42 return state 43 } 44 45 func testFSM(t *testing.T) *nomadFSM { 46 fsm, err := NewFSM(testBroker(t, 0), os.Stderr) 47 if err != nil { 48 t.Fatalf("err: %v", err) 49 } 50 if fsm == nil { 51 t.Fatalf("missing fsm") 52 } 53 return fsm 54 } 55 56 func makeLog(buf []byte) *raft.Log { 57 return &raft.Log{ 58 Index: 1, 59 Term: 1, 60 Type: raft.LogCommand, 61 Data: buf, 62 } 63 } 64 65 func TestFSM_UpsertNode(t *testing.T) { 66 fsm := testFSM(t) 67 68 req := structs.NodeRegisterRequest{ 69 Node: mock.Node(), 70 } 71 buf, err := structs.Encode(structs.NodeRegisterRequestType, req) 72 if err != nil { 73 t.Fatalf("err: %v", err) 74 } 75 76 resp := fsm.Apply(makeLog(buf)) 77 if resp != nil { 78 t.Fatalf("resp: %v", resp) 79 } 80 81 // Verify we are registered 82 node, err := fsm.State().NodeByID(req.Node.ID) 83 if err != nil { 84 t.Fatalf("err: %v", err) 85 } 86 if node == nil { 87 t.Fatalf("not found!") 88 } 89 if node.CreateIndex != 1 { 90 t.Fatalf("bad index: %d", node.CreateIndex) 91 } 92 93 tt := fsm.TimeTable() 94 index := tt.NearestIndex(time.Now().UTC()) 95 if index != 1 { 96 t.Fatalf("bad: %d", index) 97 } 98 } 99 100 func TestFSM_DeregisterNode(t *testing.T) { 101 fsm := testFSM(t) 102 103 node := mock.Node() 104 req := structs.NodeRegisterRequest{ 105 Node: node, 106 } 107 buf, err := structs.Encode(structs.NodeRegisterRequestType, req) 108 if err != nil { 109 t.Fatalf("err: %v", err) 110 } 111 112 resp := fsm.Apply(makeLog(buf)) 113 if resp != nil { 114 t.Fatalf("resp: %v", resp) 115 } 116 117 req2 := structs.NodeDeregisterRequest{ 118 NodeID: node.ID, 119 } 120 buf, err = structs.Encode(structs.NodeDeregisterRequestType, req2) 121 if err != nil { 122 t.Fatalf("err: %v", err) 123 } 124 125 resp = fsm.Apply(makeLog(buf)) 126 if resp != nil { 127 t.Fatalf("resp: %v", resp) 128 } 129 130 // Verify we are NOT registered 131 node, err = fsm.State().NodeByID(req.Node.ID) 132 if err != nil { 133 t.Fatalf("err: %v", err) 134 } 135 if node != nil { 136 t.Fatalf("node found!") 137 } 138 } 139 140 func TestFSM_UpdateNodeStatus(t *testing.T) { 141 fsm := testFSM(t) 142 143 node := mock.Node() 144 req := structs.NodeRegisterRequest{ 145 Node: node, 146 } 147 buf, err := structs.Encode(structs.NodeRegisterRequestType, req) 148 if err != nil { 149 t.Fatalf("err: %v", err) 150 } 151 152 resp := fsm.Apply(makeLog(buf)) 153 if resp != nil { 154 t.Fatalf("resp: %v", resp) 155 } 156 157 req2 := structs.NodeUpdateStatusRequest{ 158 NodeID: node.ID, 159 Status: structs.NodeStatusReady, 160 } 161 buf, err = structs.Encode(structs.NodeUpdateStatusRequestType, req2) 162 if err != nil { 163 t.Fatalf("err: %v", err) 164 } 165 166 resp = fsm.Apply(makeLog(buf)) 167 if resp != nil { 168 t.Fatalf("resp: %v", resp) 169 } 170 171 // Verify we are NOT registered 172 node, err = fsm.State().NodeByID(req.Node.ID) 173 if err != nil { 174 t.Fatalf("err: %v", err) 175 } 176 if node.Status != structs.NodeStatusReady { 177 t.Fatalf("bad node: %#v", node) 178 } 179 } 180 181 func TestFSM_UpdateNodeDrain(t *testing.T) { 182 fsm := testFSM(t) 183 184 node := mock.Node() 185 req := structs.NodeRegisterRequest{ 186 Node: node, 187 } 188 buf, err := structs.Encode(structs.NodeRegisterRequestType, req) 189 if err != nil { 190 t.Fatalf("err: %v", err) 191 } 192 193 resp := fsm.Apply(makeLog(buf)) 194 if resp != nil { 195 t.Fatalf("resp: %v", resp) 196 } 197 198 req2 := structs.NodeUpdateDrainRequest{ 199 NodeID: node.ID, 200 Drain: true, 201 } 202 buf, err = structs.Encode(structs.NodeUpdateDrainRequestType, req2) 203 if err != nil { 204 t.Fatalf("err: %v", err) 205 } 206 207 resp = fsm.Apply(makeLog(buf)) 208 if resp != nil { 209 t.Fatalf("resp: %v", resp) 210 } 211 212 // Verify we are NOT registered 213 node, err = fsm.State().NodeByID(req.Node.ID) 214 if err != nil { 215 t.Fatalf("err: %v", err) 216 } 217 if !node.Drain { 218 t.Fatalf("bad node: %#v", node) 219 } 220 } 221 222 func TestFSM_RegisterJob(t *testing.T) { 223 fsm := testFSM(t) 224 225 req := structs.JobRegisterRequest{ 226 Job: mock.Job(), 227 } 228 buf, err := structs.Encode(structs.JobRegisterRequestType, req) 229 if err != nil { 230 t.Fatalf("err: %v", err) 231 } 232 233 resp := fsm.Apply(makeLog(buf)) 234 if resp != nil { 235 t.Fatalf("resp: %v", resp) 236 } 237 238 // Verify we are registered 239 job, err := fsm.State().JobByID(req.Job.ID) 240 if err != nil { 241 t.Fatalf("err: %v", err) 242 } 243 if job == nil { 244 t.Fatalf("not found!") 245 } 246 if job.CreateIndex != 1 { 247 t.Fatalf("bad index: %d", job.CreateIndex) 248 } 249 } 250 251 func TestFSM_DeregisterJob(t *testing.T) { 252 fsm := testFSM(t) 253 254 job := mock.Job() 255 req := structs.JobRegisterRequest{ 256 Job: job, 257 } 258 buf, err := structs.Encode(structs.JobRegisterRequestType, req) 259 if err != nil { 260 t.Fatalf("err: %v", err) 261 } 262 263 resp := fsm.Apply(makeLog(buf)) 264 if resp != nil { 265 t.Fatalf("resp: %v", resp) 266 } 267 268 req2 := structs.JobDeregisterRequest{ 269 JobID: job.ID, 270 } 271 buf, err = structs.Encode(structs.JobDeregisterRequestType, req2) 272 if err != nil { 273 t.Fatalf("err: %v", err) 274 } 275 276 resp = fsm.Apply(makeLog(buf)) 277 if resp != nil { 278 t.Fatalf("resp: %v", resp) 279 } 280 281 // Verify we are NOT registered 282 job, err = fsm.State().JobByID(req.Job.ID) 283 if err != nil { 284 t.Fatalf("err: %v", err) 285 } 286 if job != nil { 287 t.Fatalf("job found!") 288 } 289 } 290 291 func TestFSM_UpdateEval(t *testing.T) { 292 fsm := testFSM(t) 293 fsm.evalBroker.SetEnabled(true) 294 295 req := structs.EvalUpdateRequest{ 296 Evals: []*structs.Evaluation{mock.Eval()}, 297 } 298 buf, err := structs.Encode(structs.EvalUpdateRequestType, req) 299 if err != nil { 300 t.Fatalf("err: %v", err) 301 } 302 303 resp := fsm.Apply(makeLog(buf)) 304 if resp != nil { 305 t.Fatalf("resp: %v", resp) 306 } 307 308 // Verify we are registered 309 eval, err := fsm.State().EvalByID(req.Evals[0].ID) 310 if err != nil { 311 t.Fatalf("err: %v", err) 312 } 313 if eval == nil { 314 t.Fatalf("not found!") 315 } 316 if eval.CreateIndex != 1 { 317 t.Fatalf("bad index: %d", eval.CreateIndex) 318 } 319 320 // Verify enqueued 321 stats := fsm.evalBroker.Stats() 322 if stats.TotalReady != 1 { 323 t.Fatalf("bad: %#v %#v", stats, eval) 324 } 325 } 326 327 func TestFSM_DeleteEval(t *testing.T) { 328 fsm := testFSM(t) 329 330 eval := mock.Eval() 331 req := structs.EvalUpdateRequest{ 332 Evals: []*structs.Evaluation{eval}, 333 } 334 buf, err := structs.Encode(structs.EvalUpdateRequestType, req) 335 if err != nil { 336 t.Fatalf("err: %v", err) 337 } 338 339 resp := fsm.Apply(makeLog(buf)) 340 if resp != nil { 341 t.Fatalf("resp: %v", resp) 342 } 343 344 req2 := structs.EvalDeleteRequest{ 345 Evals: []string{eval.ID}, 346 } 347 buf, err = structs.Encode(structs.EvalDeleteRequestType, req2) 348 if err != nil { 349 t.Fatalf("err: %v", err) 350 } 351 352 resp = fsm.Apply(makeLog(buf)) 353 if resp != nil { 354 t.Fatalf("resp: %v", resp) 355 } 356 357 // Verify we are NOT registered 358 eval, err = fsm.State().EvalByID(req.Evals[0].ID) 359 if err != nil { 360 t.Fatalf("err: %v", err) 361 } 362 if eval != nil { 363 t.Fatalf("eval found!") 364 } 365 } 366 367 func TestFSM_UpsertAllocs(t *testing.T) { 368 fsm := testFSM(t) 369 370 alloc := mock.Alloc() 371 req := structs.AllocUpdateRequest{ 372 Alloc: []*structs.Allocation{alloc}, 373 } 374 buf, err := structs.Encode(structs.AllocUpdateRequestType, req) 375 if err != nil { 376 t.Fatalf("err: %v", err) 377 } 378 379 resp := fsm.Apply(makeLog(buf)) 380 if resp != nil { 381 t.Fatalf("resp: %v", resp) 382 } 383 384 // Verify we are registered 385 out, err := fsm.State().AllocByID(alloc.ID) 386 if err != nil { 387 t.Fatalf("err: %v", err) 388 } 389 alloc.CreateIndex = out.CreateIndex 390 alloc.ModifyIndex = out.ModifyIndex 391 if !reflect.DeepEqual(alloc, out) { 392 t.Fatalf("bad: %#v %#v", alloc, out) 393 } 394 395 evictAlloc := new(structs.Allocation) 396 *evictAlloc = *alloc 397 evictAlloc.DesiredStatus = structs.AllocDesiredStatusEvict 398 req2 := structs.AllocUpdateRequest{ 399 Alloc: []*structs.Allocation{evictAlloc}, 400 } 401 buf, err = structs.Encode(structs.AllocUpdateRequestType, req2) 402 if err != nil { 403 t.Fatalf("err: %v", err) 404 } 405 406 resp = fsm.Apply(makeLog(buf)) 407 if resp != nil { 408 t.Fatalf("resp: %v", resp) 409 } 410 411 // Verify we are evicted 412 out, err = fsm.State().AllocByID(alloc.ID) 413 if err != nil { 414 t.Fatalf("err: %v", err) 415 } 416 if out.DesiredStatus != structs.AllocDesiredStatusEvict { 417 t.Fatalf("alloc found!") 418 } 419 } 420 421 func TestFSM_UpdateAllocFromClient(t *testing.T) { 422 fsm := testFSM(t) 423 state := fsm.State() 424 425 alloc := mock.Alloc() 426 state.UpsertAllocs(1, []*structs.Allocation{alloc}) 427 428 clientAlloc := new(structs.Allocation) 429 *clientAlloc = *alloc 430 clientAlloc.ClientStatus = structs.AllocClientStatusFailed 431 432 req := structs.AllocUpdateRequest{ 433 Alloc: []*structs.Allocation{clientAlloc}, 434 } 435 buf, err := structs.Encode(structs.AllocClientUpdateRequestType, req) 436 if err != nil { 437 t.Fatalf("err: %v", err) 438 } 439 440 resp := fsm.Apply(makeLog(buf)) 441 if resp != nil { 442 t.Fatalf("resp: %v", resp) 443 } 444 445 // Verify we are registered 446 out, err := fsm.State().AllocByID(alloc.ID) 447 if err != nil { 448 t.Fatalf("err: %v", err) 449 } 450 clientAlloc.CreateIndex = out.CreateIndex 451 clientAlloc.ModifyIndex = out.ModifyIndex 452 if !reflect.DeepEqual(clientAlloc, out) { 453 t.Fatalf("bad: %#v %#v", clientAlloc, out) 454 } 455 } 456 457 func testSnapshotRestore(t *testing.T, fsm *nomadFSM) *nomadFSM { 458 // Snapshot 459 snap, err := fsm.Snapshot() 460 if err != nil { 461 t.Fatalf("err: %v", err) 462 } 463 defer snap.Release() 464 465 // Persist 466 buf := bytes.NewBuffer(nil) 467 sink := &MockSink{buf, false} 468 if err := snap.Persist(sink); err != nil { 469 t.Fatalf("err: %v", err) 470 } 471 472 // Try to restore on a new FSM 473 fsm2 := testFSM(t) 474 475 // Do a restore 476 if err := fsm2.Restore(sink); err != nil { 477 t.Fatalf("err: %v", err) 478 } 479 return fsm2 480 } 481 482 func TestFSM_SnapshotRestore_Nodes(t *testing.T) { 483 // Add some state 484 fsm := testFSM(t) 485 state := fsm.State() 486 node1 := mock.Node() 487 state.UpsertNode(1000, node1) 488 node2 := mock.Node() 489 state.UpsertNode(1001, node2) 490 491 // Verify the contents 492 fsm2 := testSnapshotRestore(t, fsm) 493 state2 := fsm2.State() 494 out1, _ := state2.NodeByID(node1.ID) 495 out2, _ := state2.NodeByID(node2.ID) 496 if !reflect.DeepEqual(node1, out1) { 497 t.Fatalf("bad: \n%#v\n%#v", out1, node1) 498 } 499 if !reflect.DeepEqual(node2, out2) { 500 t.Fatalf("bad: \n%#v\n%#v", out2, node2) 501 } 502 } 503 504 func TestFSM_SnapshotRestore_Jobs(t *testing.T) { 505 // Add some state 506 fsm := testFSM(t) 507 state := fsm.State() 508 job1 := mock.Job() 509 state.UpsertJob(1000, job1) 510 job2 := mock.Job() 511 state.UpsertJob(1001, job2) 512 513 // Verify the contents 514 fsm2 := testSnapshotRestore(t, fsm) 515 state2 := fsm2.State() 516 out1, _ := state2.JobByID(job1.ID) 517 out2, _ := state2.JobByID(job2.ID) 518 if !reflect.DeepEqual(job1, out1) { 519 t.Fatalf("bad: \n%#v\n%#v", out1, job1) 520 } 521 if !reflect.DeepEqual(job2, out2) { 522 t.Fatalf("bad: \n%#v\n%#v", out2, job2) 523 } 524 } 525 526 func TestFSM_SnapshotRestore_Evals(t *testing.T) { 527 // Add some state 528 fsm := testFSM(t) 529 state := fsm.State() 530 eval1 := mock.Eval() 531 state.UpsertEvals(1000, []*structs.Evaluation{eval1}) 532 eval2 := mock.Eval() 533 state.UpsertEvals(1001, []*structs.Evaluation{eval2}) 534 535 // Verify the contents 536 fsm2 := testSnapshotRestore(t, fsm) 537 state2 := fsm2.State() 538 out1, _ := state2.EvalByID(eval1.ID) 539 out2, _ := state2.EvalByID(eval2.ID) 540 if !reflect.DeepEqual(eval1, out1) { 541 t.Fatalf("bad: \n%#v\n%#v", out1, eval1) 542 } 543 if !reflect.DeepEqual(eval2, out2) { 544 t.Fatalf("bad: \n%#v\n%#v", out2, eval2) 545 } 546 } 547 548 func TestFSM_SnapshotRestore_Allocs(t *testing.T) { 549 // Add some state 550 fsm := testFSM(t) 551 state := fsm.State() 552 alloc1 := mock.Alloc() 553 state.UpsertAllocs(1000, []*structs.Allocation{alloc1}) 554 alloc2 := mock.Alloc() 555 state.UpsertAllocs(1001, []*structs.Allocation{alloc2}) 556 557 // Verify the contents 558 fsm2 := testSnapshotRestore(t, fsm) 559 state2 := fsm2.State() 560 out1, _ := state2.AllocByID(alloc1.ID) 561 out2, _ := state2.AllocByID(alloc2.ID) 562 if !reflect.DeepEqual(alloc1, out1) { 563 t.Fatalf("bad: \n%#v\n%#v", out1, alloc1) 564 } 565 if !reflect.DeepEqual(alloc2, out2) { 566 t.Fatalf("bad: \n%#v\n%#v", out2, alloc2) 567 } 568 } 569 570 func TestFSM_SnapshotRestore_Indexes(t *testing.T) { 571 // Add some state 572 fsm := testFSM(t) 573 state := fsm.State() 574 node1 := mock.Node() 575 state.UpsertNode(1000, node1) 576 577 // Verify the contents 578 fsm2 := testSnapshotRestore(t, fsm) 579 state2 := fsm2.State() 580 581 index, err := state2.Index("nodes") 582 if err != nil { 583 t.Fatalf("err: %v", err) 584 } 585 if index != 1000 { 586 t.Fatalf("bad: %d", index) 587 } 588 } 589 590 func TestFSM_SnapshotRestore_TimeTable(t *testing.T) { 591 // Add some state 592 fsm := testFSM(t) 593 594 tt := fsm.TimeTable() 595 start := time.Now().UTC() 596 tt.Witness(1000, start) 597 tt.Witness(2000, start.Add(10*time.Minute)) 598 599 // Verify the contents 600 fsm2 := testSnapshotRestore(t, fsm) 601 602 tt2 := fsm2.TimeTable() 603 if tt2.NearestTime(1500) != start { 604 t.Fatalf("bad") 605 } 606 if tt2.NearestIndex(start.Add(15*time.Minute)) != 2000 { 607 t.Fatalf("bad") 608 } 609 }