github.com/huiliang/nomad@v0.2.1-0.20151124023127-7a8b664699ff/nomad/fsm.go (about) 1 package nomad 2 3 import ( 4 "fmt" 5 "io" 6 "log" 7 "time" 8 9 "github.com/armon/go-metrics" 10 "github.com/hashicorp/go-msgpack/codec" 11 "github.com/hashicorp/nomad/nomad/state" 12 "github.com/hashicorp/nomad/nomad/structs" 13 "github.com/hashicorp/raft" 14 ) 15 16 const ( 17 // timeTableGranularity is the granularity of index to time tracking 18 timeTableGranularity = 5 * time.Minute 19 20 // timeTableLimit is the maximum limit of our tracking 21 timeTableLimit = 72 * time.Hour 22 ) 23 24 // SnapshotType is prefixed to a record in the FSM snapshot 25 // so that we can determine the type for restore 26 type SnapshotType byte 27 28 const ( 29 NodeSnapshot SnapshotType = iota 30 JobSnapshot 31 IndexSnapshot 32 EvalSnapshot 33 AllocSnapshot 34 TimeTableSnapshot 35 ) 36 37 // nomadFSM implements a finite state machine that is used 38 // along with Raft to provide strong consistency. We implement 39 // this outside the Server to avoid exposing this outside the package. 40 type nomadFSM struct { 41 evalBroker *EvalBroker 42 logOutput io.Writer 43 logger *log.Logger 44 state *state.StateStore 45 timetable *TimeTable 46 } 47 48 // nomadSnapshot is used to provide a snapshot of the current 49 // state in a way that can be accessed concurrently with operations 50 // that may modify the live state. 51 type nomadSnapshot struct { 52 snap *state.StateSnapshot 53 timetable *TimeTable 54 } 55 56 // snapshotHeader is the first entry in our snapshot 57 type snapshotHeader struct { 58 } 59 60 // NewFSMPath is used to construct a new FSM with a blank state 61 func NewFSM(evalBroker *EvalBroker, logOutput io.Writer) (*nomadFSM, error) { 62 // Create a state store 63 state, err := state.NewStateStore(logOutput) 64 if err != nil { 65 return nil, err 66 } 67 68 fsm := &nomadFSM{ 69 evalBroker: evalBroker, 70 logOutput: logOutput, 71 logger: log.New(logOutput, "", log.LstdFlags), 72 state: state, 73 timetable: NewTimeTable(timeTableGranularity, timeTableLimit), 74 } 75 return fsm, nil 76 } 77 78 // Close is used to cleanup resources associated with the FSM 79 func (n *nomadFSM) Close() error { 80 return nil 81 } 82 83 // State is used to return a handle to the current state 84 func (n *nomadFSM) State() *state.StateStore { 85 return n.state 86 } 87 88 // TimeTable returns the time table of transactions 89 func (n *nomadFSM) TimeTable() *TimeTable { 90 return n.timetable 91 } 92 93 func (n *nomadFSM) Apply(log *raft.Log) interface{} { 94 buf := log.Data 95 msgType := structs.MessageType(buf[0]) 96 97 // Witness this write 98 n.timetable.Witness(log.Index, time.Now().UTC()) 99 100 // Check if this message type should be ignored when unknown. This is 101 // used so that new commands can be added with developer control if older 102 // versions can safely ignore the command, or if they should crash. 103 ignoreUnknown := false 104 if msgType&structs.IgnoreUnknownTypeFlag == structs.IgnoreUnknownTypeFlag { 105 msgType &= ^structs.IgnoreUnknownTypeFlag 106 ignoreUnknown = true 107 } 108 109 switch msgType { 110 case structs.NodeRegisterRequestType: 111 return n.applyUpsertNode(buf[1:], log.Index) 112 case structs.NodeDeregisterRequestType: 113 return n.applyDeregisterNode(buf[1:], log.Index) 114 case structs.NodeUpdateStatusRequestType: 115 return n.applyStatusUpdate(buf[1:], log.Index) 116 case structs.NodeUpdateDrainRequestType: 117 return n.applyDrainUpdate(buf[1:], log.Index) 118 case structs.JobRegisterRequestType: 119 return n.applyUpsertJob(buf[1:], log.Index) 120 case structs.JobDeregisterRequestType: 121 return n.applyDeregisterJob(buf[1:], log.Index) 122 case structs.EvalUpdateRequestType: 123 return n.applyUpdateEval(buf[1:], log.Index) 124 case structs.EvalDeleteRequestType: 125 return n.applyDeleteEval(buf[1:], log.Index) 126 case structs.AllocUpdateRequestType: 127 return n.applyAllocUpdate(buf[1:], log.Index) 128 case structs.AllocClientUpdateRequestType: 129 return n.applyAllocClientUpdate(buf[1:], log.Index) 130 default: 131 if ignoreUnknown { 132 n.logger.Printf("[WARN] nomad.fsm: ignoring unknown message type (%d), upgrade to newer version", msgType) 133 return nil 134 } else { 135 panic(fmt.Errorf("failed to apply request: %#v", buf)) 136 } 137 } 138 } 139 140 func (n *nomadFSM) applyUpsertNode(buf []byte, index uint64) interface{} { 141 defer metrics.MeasureSince([]string{"nomad", "fsm", "register_node"}, time.Now()) 142 var req structs.NodeRegisterRequest 143 if err := structs.Decode(buf, &req); err != nil { 144 panic(fmt.Errorf("failed to decode request: %v", err)) 145 } 146 147 if err := n.state.UpsertNode(index, req.Node); err != nil { 148 n.logger.Printf("[ERR] nomad.fsm: UpsertNode failed: %v", err) 149 return err 150 } 151 return nil 152 } 153 154 func (n *nomadFSM) applyDeregisterNode(buf []byte, index uint64) interface{} { 155 defer metrics.MeasureSince([]string{"nomad", "fsm", "deregister_node"}, time.Now()) 156 var req structs.NodeDeregisterRequest 157 if err := structs.Decode(buf, &req); err != nil { 158 panic(fmt.Errorf("failed to decode request: %v", err)) 159 } 160 161 if err := n.state.DeleteNode(index, req.NodeID); err != nil { 162 n.logger.Printf("[ERR] nomad.fsm: DeleteNode failed: %v", err) 163 return err 164 } 165 return nil 166 } 167 168 func (n *nomadFSM) applyStatusUpdate(buf []byte, index uint64) interface{} { 169 defer metrics.MeasureSince([]string{"nomad", "fsm", "node_status_update"}, time.Now()) 170 var req structs.NodeUpdateStatusRequest 171 if err := structs.Decode(buf, &req); err != nil { 172 panic(fmt.Errorf("failed to decode request: %v", err)) 173 } 174 175 if err := n.state.UpdateNodeStatus(index, req.NodeID, req.Status); err != nil { 176 n.logger.Printf("[ERR] nomad.fsm: UpdateNodeStatus failed: %v", err) 177 return err 178 } 179 return nil 180 } 181 182 func (n *nomadFSM) applyDrainUpdate(buf []byte, index uint64) interface{} { 183 defer metrics.MeasureSince([]string{"nomad", "fsm", "node_drain_update"}, time.Now()) 184 var req structs.NodeUpdateDrainRequest 185 if err := structs.Decode(buf, &req); err != nil { 186 panic(fmt.Errorf("failed to decode request: %v", err)) 187 } 188 189 if err := n.state.UpdateNodeDrain(index, req.NodeID, req.Drain); err != nil { 190 n.logger.Printf("[ERR] nomad.fsm: UpdateNodeDrain failed: %v", err) 191 return err 192 } 193 return nil 194 } 195 196 func (n *nomadFSM) applyUpsertJob(buf []byte, index uint64) interface{} { 197 defer metrics.MeasureSince([]string{"nomad", "fsm", "register_job"}, time.Now()) 198 var req structs.JobRegisterRequest 199 if err := structs.Decode(buf, &req); err != nil { 200 panic(fmt.Errorf("failed to decode request: %v", err)) 201 } 202 203 if err := n.state.UpsertJob(index, req.Job); err != nil { 204 n.logger.Printf("[ERR] nomad.fsm: UpsertJob failed: %v", err) 205 return err 206 } 207 return nil 208 } 209 210 func (n *nomadFSM) applyDeregisterJob(buf []byte, index uint64) interface{} { 211 defer metrics.MeasureSince([]string{"nomad", "fsm", "deregister_job"}, time.Now()) 212 var req structs.JobDeregisterRequest 213 if err := structs.Decode(buf, &req); err != nil { 214 panic(fmt.Errorf("failed to decode request: %v", err)) 215 } 216 217 if err := n.state.DeleteJob(index, req.JobID); err != nil { 218 n.logger.Printf("[ERR] nomad.fsm: DeleteJob failed: %v", err) 219 return err 220 } 221 return nil 222 } 223 224 func (n *nomadFSM) applyUpdateEval(buf []byte, index uint64) interface{} { 225 defer metrics.MeasureSince([]string{"nomad", "fsm", "update_eval"}, time.Now()) 226 var req structs.EvalUpdateRequest 227 if err := structs.Decode(buf, &req); err != nil { 228 panic(fmt.Errorf("failed to decode request: %v", err)) 229 } 230 231 if err := n.state.UpsertEvals(index, req.Evals); err != nil { 232 n.logger.Printf("[ERR] nomad.fsm: UpsertEvals failed: %v", err) 233 return err 234 } 235 236 for _, eval := range req.Evals { 237 if eval.ShouldEnqueue() { 238 if err := n.evalBroker.Enqueue(eval); err != nil { 239 n.logger.Printf("[ERR] nomad.fsm: failed to enqueue evaluation %s: %v", eval.ID, err) 240 return err 241 } 242 } 243 } 244 return nil 245 } 246 247 func (n *nomadFSM) applyDeleteEval(buf []byte, index uint64) interface{} { 248 defer metrics.MeasureSince([]string{"nomad", "fsm", "delete_eval"}, time.Now()) 249 var req structs.EvalDeleteRequest 250 if err := structs.Decode(buf, &req); err != nil { 251 panic(fmt.Errorf("failed to decode request: %v", err)) 252 } 253 254 if err := n.state.DeleteEval(index, req.Evals, req.Allocs); err != nil { 255 n.logger.Printf("[ERR] nomad.fsm: DeleteEval failed: %v", err) 256 return err 257 } 258 return nil 259 } 260 261 func (n *nomadFSM) applyAllocUpdate(buf []byte, index uint64) interface{} { 262 defer metrics.MeasureSince([]string{"nomad", "fsm", "alloc_update"}, time.Now()) 263 var req structs.AllocUpdateRequest 264 if err := structs.Decode(buf, &req); err != nil { 265 panic(fmt.Errorf("failed to decode request: %v", err)) 266 } 267 268 if err := n.state.UpsertAllocs(index, req.Alloc); err != nil { 269 n.logger.Printf("[ERR] nomad.fsm: UpsertAllocs failed: %v", err) 270 return err 271 } 272 return nil 273 } 274 275 func (n *nomadFSM) applyAllocClientUpdate(buf []byte, index uint64) interface{} { 276 defer metrics.MeasureSince([]string{"nomad", "fsm", "alloc_client_update"}, time.Now()) 277 var req structs.AllocUpdateRequest 278 if err := structs.Decode(buf, &req); err != nil { 279 panic(fmt.Errorf("failed to decode request: %v", err)) 280 } 281 if len(req.Alloc) == 0 { 282 return nil 283 } 284 285 if err := n.state.UpdateAllocFromClient(index, req.Alloc[0]); err != nil { 286 n.logger.Printf("[ERR] nomad.fsm: UpdateAllocFromClient failed: %v", err) 287 return err 288 } 289 return nil 290 } 291 292 func (n *nomadFSM) Snapshot() (raft.FSMSnapshot, error) { 293 // Create a new snapshot 294 snap, err := n.state.Snapshot() 295 if err != nil { 296 return nil, err 297 } 298 299 ns := &nomadSnapshot{ 300 snap: snap, 301 timetable: n.timetable, 302 } 303 return ns, nil 304 } 305 306 func (n *nomadFSM) Restore(old io.ReadCloser) error { 307 defer old.Close() 308 309 // Create a new state store 310 newState, err := state.NewStateStore(n.logOutput) 311 if err != nil { 312 return err 313 } 314 n.state = newState 315 316 // Start the state restore 317 restore, err := newState.Restore() 318 if err != nil { 319 return err 320 } 321 defer restore.Abort() 322 323 // Create a decoder 324 dec := codec.NewDecoder(old, structs.MsgpackHandle) 325 326 // Read in the header 327 var header snapshotHeader 328 if err := dec.Decode(&header); err != nil { 329 return err 330 } 331 332 // Populate the new state 333 msgType := make([]byte, 1) 334 for { 335 // Read the message type 336 _, err := old.Read(msgType) 337 if err == io.EOF { 338 break 339 } else if err != nil { 340 return err 341 } 342 343 // Decode 344 switch SnapshotType(msgType[0]) { 345 case TimeTableSnapshot: 346 if err := n.timetable.Deserialize(dec); err != nil { 347 return fmt.Errorf("time table deserialize failed: %v", err) 348 } 349 350 case NodeSnapshot: 351 node := new(structs.Node) 352 if err := dec.Decode(node); err != nil { 353 return err 354 } 355 if err := restore.NodeRestore(node); err != nil { 356 return err 357 } 358 359 case JobSnapshot: 360 job := new(structs.Job) 361 if err := dec.Decode(job); err != nil { 362 return err 363 } 364 if err := restore.JobRestore(job); err != nil { 365 return err 366 } 367 368 case EvalSnapshot: 369 eval := new(structs.Evaluation) 370 if err := dec.Decode(eval); err != nil { 371 return err 372 } 373 if err := restore.EvalRestore(eval); err != nil { 374 return err 375 } 376 377 case AllocSnapshot: 378 alloc := new(structs.Allocation) 379 if err := dec.Decode(alloc); err != nil { 380 return err 381 } 382 if err := restore.AllocRestore(alloc); err != nil { 383 return err 384 } 385 386 case IndexSnapshot: 387 idx := new(state.IndexEntry) 388 if err := dec.Decode(idx); err != nil { 389 return err 390 } 391 if err := restore.IndexRestore(idx); err != nil { 392 return err 393 } 394 395 default: 396 return fmt.Errorf("Unrecognized snapshot type: %v", msgType) 397 } 398 } 399 400 // Commit the state restore 401 restore.Commit() 402 return nil 403 } 404 405 func (s *nomadSnapshot) Persist(sink raft.SnapshotSink) error { 406 defer metrics.MeasureSince([]string{"nomad", "fsm", "persist"}, time.Now()) 407 // Register the nodes 408 encoder := codec.NewEncoder(sink, structs.MsgpackHandle) 409 410 // Write the header 411 header := snapshotHeader{} 412 if err := encoder.Encode(&header); err != nil { 413 sink.Cancel() 414 return err 415 } 416 417 // Write the time table 418 sink.Write([]byte{byte(TimeTableSnapshot)}) 419 if err := s.timetable.Serialize(encoder); err != nil { 420 sink.Cancel() 421 return err 422 } 423 424 // Write all the data out 425 if err := s.persistIndexes(sink, encoder); err != nil { 426 sink.Cancel() 427 return err 428 } 429 if err := s.persistNodes(sink, encoder); err != nil { 430 sink.Cancel() 431 return err 432 } 433 if err := s.persistJobs(sink, encoder); err != nil { 434 sink.Cancel() 435 return err 436 } 437 if err := s.persistEvals(sink, encoder); err != nil { 438 sink.Cancel() 439 return err 440 } 441 if err := s.persistAllocs(sink, encoder); err != nil { 442 sink.Cancel() 443 return err 444 } 445 return nil 446 } 447 448 func (s *nomadSnapshot) persistIndexes(sink raft.SnapshotSink, 449 encoder *codec.Encoder) error { 450 // Get all the indexes 451 iter, err := s.snap.Indexes() 452 if err != nil { 453 return err 454 } 455 456 for { 457 // Get the next item 458 raw := iter.Next() 459 if raw == nil { 460 break 461 } 462 463 // Prepare the request struct 464 idx := raw.(*state.IndexEntry) 465 466 // Write out a node registration 467 sink.Write([]byte{byte(IndexSnapshot)}) 468 if err := encoder.Encode(idx); err != nil { 469 return err 470 } 471 } 472 return nil 473 } 474 475 func (s *nomadSnapshot) persistNodes(sink raft.SnapshotSink, 476 encoder *codec.Encoder) error { 477 // Get all the nodes 478 nodes, err := s.snap.Nodes() 479 if err != nil { 480 return err 481 } 482 483 for { 484 // Get the next item 485 raw := nodes.Next() 486 if raw == nil { 487 break 488 } 489 490 // Prepare the request struct 491 node := raw.(*structs.Node) 492 493 // Write out a node registration 494 sink.Write([]byte{byte(NodeSnapshot)}) 495 if err := encoder.Encode(node); err != nil { 496 return err 497 } 498 } 499 return nil 500 } 501 502 func (s *nomadSnapshot) persistJobs(sink raft.SnapshotSink, 503 encoder *codec.Encoder) error { 504 // Get all the jobs 505 jobs, err := s.snap.Jobs() 506 if err != nil { 507 return err 508 } 509 510 for { 511 // Get the next item 512 raw := jobs.Next() 513 if raw == nil { 514 break 515 } 516 517 // Prepare the request struct 518 job := raw.(*structs.Job) 519 520 // Write out a job registration 521 sink.Write([]byte{byte(JobSnapshot)}) 522 if err := encoder.Encode(job); err != nil { 523 return err 524 } 525 } 526 return nil 527 } 528 529 func (s *nomadSnapshot) persistEvals(sink raft.SnapshotSink, 530 encoder *codec.Encoder) error { 531 // Get all the evaluations 532 evals, err := s.snap.Evals() 533 if err != nil { 534 return err 535 } 536 537 for { 538 // Get the next item 539 raw := evals.Next() 540 if raw == nil { 541 break 542 } 543 544 // Prepare the request struct 545 eval := raw.(*structs.Evaluation) 546 547 // Write out the evaluation 548 sink.Write([]byte{byte(EvalSnapshot)}) 549 if err := encoder.Encode(eval); err != nil { 550 return err 551 } 552 } 553 return nil 554 } 555 556 func (s *nomadSnapshot) persistAllocs(sink raft.SnapshotSink, 557 encoder *codec.Encoder) error { 558 // Get all the allocations 559 allocs, err := s.snap.Allocs() 560 if err != nil { 561 return err 562 } 563 564 for { 565 // Get the next item 566 raw := allocs.Next() 567 if raw == nil { 568 break 569 } 570 571 // Prepare the request struct 572 alloc := raw.(*structs.Allocation) 573 574 // Write out the evaluation 575 sink.Write([]byte{byte(AllocSnapshot)}) 576 if err := encoder.Encode(alloc); err != nil { 577 return err 578 } 579 } 580 return nil 581 } 582 583 // Release is a no-op, as we just need to GC the pointer 584 // to the state store snapshot. There is nothing to explicitly 585 // cleanup. 586 func (s *nomadSnapshot) Release() {}