github.com/kardianos/nomad@v0.1.3-0.20151022182107-b13df73ee850/nomad/fsm.go (about) 1 package nomad 2 3 import ( 4 "fmt" 5 "io" 6 "log" 7 "time" 8 9 "github.com/armon/go-metrics" 10 "github.com/hashicorp/go-msgpack/codec" 11 "github.com/hashicorp/nomad/nomad/state" 12 "github.com/hashicorp/nomad/nomad/structs" 13 "github.com/hashicorp/raft" 14 ) 15 16 var ( 17 msgpackHandle = &codec.MsgpackHandle{ 18 RawToString: true, 19 WriteExt: true, 20 } 21 ) 22 23 const ( 24 // timeTableGranularity is the granularity of index to time tracking 25 timeTableGranularity = 5 * time.Minute 26 27 // timeTableLimit is the maximum limit of our tracking 28 timeTableLimit = 72 * time.Hour 29 ) 30 31 // SnapshotType is prefixed to a record in the FSM snapshot 32 // so that we can determine the type for restore 33 type SnapshotType byte 34 35 const ( 36 NodeSnapshot SnapshotType = iota 37 JobSnapshot 38 IndexSnapshot 39 EvalSnapshot 40 AllocSnapshot 41 TimeTableSnapshot 42 ) 43 44 // nomadFSM implements a finite state machine that is used 45 // along with Raft to provide strong consistency. We implement 46 // this outside the Server to avoid exposing this outside the package. 47 type nomadFSM struct { 48 evalBroker *EvalBroker 49 logOutput io.Writer 50 logger *log.Logger 51 state *state.StateStore 52 timetable *TimeTable 53 } 54 55 // nomadSnapshot is used to provide a snapshot of the current 56 // state in a way that can be accessed concurrently with operations 57 // that may modify the live state. 58 type nomadSnapshot struct { 59 snap *state.StateSnapshot 60 timetable *TimeTable 61 } 62 63 // snapshotHeader is the first entry in our snapshot 64 type snapshotHeader struct { 65 } 66 67 // NewFSMPath is used to construct a new FSM with a blank state 68 func NewFSM(evalBroker *EvalBroker, logOutput io.Writer) (*nomadFSM, error) { 69 // Create a state store 70 state, err := state.NewStateStore(logOutput) 71 if err != nil { 72 return nil, err 73 } 74 75 fsm := &nomadFSM{ 76 evalBroker: evalBroker, 77 logOutput: logOutput, 78 logger: log.New(logOutput, "", log.LstdFlags), 79 state: state, 80 timetable: NewTimeTable(timeTableGranularity, timeTableLimit), 81 } 82 return fsm, nil 83 } 84 85 // Close is used to cleanup resources associated with the FSM 86 func (n *nomadFSM) Close() error { 87 return nil 88 } 89 90 // State is used to return a handle to the current state 91 func (n *nomadFSM) State() *state.StateStore { 92 return n.state 93 } 94 95 // TimeTable returns the time table of transactions 96 func (n *nomadFSM) TimeTable() *TimeTable { 97 return n.timetable 98 } 99 100 func (n *nomadFSM) Apply(log *raft.Log) interface{} { 101 buf := log.Data 102 msgType := structs.MessageType(buf[0]) 103 104 // Witness this write 105 n.timetable.Witness(log.Index, time.Now().UTC()) 106 107 // Check if this message type should be ignored when unknown. This is 108 // used so that new commands can be added with developer control if older 109 // versions can safely ignore the command, or if they should crash. 110 ignoreUnknown := false 111 if msgType&structs.IgnoreUnknownTypeFlag == structs.IgnoreUnknownTypeFlag { 112 msgType &= ^structs.IgnoreUnknownTypeFlag 113 ignoreUnknown = true 114 } 115 116 switch msgType { 117 case structs.NodeRegisterRequestType: 118 return n.applyUpsertNode(buf[1:], log.Index) 119 case structs.NodeDeregisterRequestType: 120 return n.applyDeregisterNode(buf[1:], log.Index) 121 case structs.NodeUpdateStatusRequestType: 122 return n.applyStatusUpdate(buf[1:], log.Index) 123 case structs.NodeUpdateDrainRequestType: 124 return n.applyDrainUpdate(buf[1:], log.Index) 125 case structs.JobRegisterRequestType: 126 return n.applyUpsertJob(buf[1:], log.Index) 127 case structs.JobDeregisterRequestType: 128 return n.applyDeregisterJob(buf[1:], log.Index) 129 case structs.EvalUpdateRequestType: 130 return n.applyUpdateEval(buf[1:], log.Index) 131 case structs.EvalDeleteRequestType: 132 return n.applyDeleteEval(buf[1:], log.Index) 133 case structs.AllocUpdateRequestType: 134 return n.applyAllocUpdate(buf[1:], log.Index) 135 case structs.AllocClientUpdateRequestType: 136 return n.applyAllocClientUpdate(buf[1:], log.Index) 137 default: 138 if ignoreUnknown { 139 n.logger.Printf("[WARN] nomad.fsm: ignoring unknown message type (%d), upgrade to newer version", msgType) 140 return nil 141 } else { 142 panic(fmt.Errorf("failed to apply request: %#v", buf)) 143 } 144 } 145 } 146 147 func (n *nomadFSM) applyUpsertNode(buf []byte, index uint64) interface{} { 148 defer metrics.MeasureSince([]string{"nomad", "fsm", "register_node"}, time.Now()) 149 var req structs.NodeRegisterRequest 150 if err := structs.Decode(buf, &req); err != nil { 151 panic(fmt.Errorf("failed to decode request: %v", err)) 152 } 153 154 if err := n.state.UpsertNode(index, req.Node); err != nil { 155 n.logger.Printf("[ERR] nomad.fsm: UpsertNode failed: %v", err) 156 return err 157 } 158 return nil 159 } 160 161 func (n *nomadFSM) applyDeregisterNode(buf []byte, index uint64) interface{} { 162 defer metrics.MeasureSince([]string{"nomad", "fsm", "deregister_node"}, time.Now()) 163 var req structs.NodeDeregisterRequest 164 if err := structs.Decode(buf, &req); err != nil { 165 panic(fmt.Errorf("failed to decode request: %v", err)) 166 } 167 168 if err := n.state.DeleteNode(index, req.NodeID); err != nil { 169 n.logger.Printf("[ERR] nomad.fsm: DeleteNode failed: %v", err) 170 return err 171 } 172 return nil 173 } 174 175 func (n *nomadFSM) applyStatusUpdate(buf []byte, index uint64) interface{} { 176 defer metrics.MeasureSince([]string{"nomad", "fsm", "node_status_update"}, time.Now()) 177 var req structs.NodeUpdateStatusRequest 178 if err := structs.Decode(buf, &req); err != nil { 179 panic(fmt.Errorf("failed to decode request: %v", err)) 180 } 181 182 if err := n.state.UpdateNodeStatus(index, req.NodeID, req.Status); err != nil { 183 n.logger.Printf("[ERR] nomad.fsm: UpdateNodeStatus failed: %v", err) 184 return err 185 } 186 return nil 187 } 188 189 func (n *nomadFSM) applyDrainUpdate(buf []byte, index uint64) interface{} { 190 defer metrics.MeasureSince([]string{"nomad", "fsm", "node_drain_update"}, time.Now()) 191 var req structs.NodeUpdateDrainRequest 192 if err := structs.Decode(buf, &req); err != nil { 193 panic(fmt.Errorf("failed to decode request: %v", err)) 194 } 195 196 if err := n.state.UpdateNodeDrain(index, req.NodeID, req.Drain); err != nil { 197 n.logger.Printf("[ERR] nomad.fsm: UpdateNodeDrain failed: %v", err) 198 return err 199 } 200 return nil 201 } 202 203 func (n *nomadFSM) applyUpsertJob(buf []byte, index uint64) interface{} { 204 defer metrics.MeasureSince([]string{"nomad", "fsm", "register_job"}, time.Now()) 205 var req structs.JobRegisterRequest 206 if err := structs.Decode(buf, &req); err != nil { 207 panic(fmt.Errorf("failed to decode request: %v", err)) 208 } 209 210 if err := n.state.UpsertJob(index, req.Job); err != nil { 211 n.logger.Printf("[ERR] nomad.fsm: UpsertJob failed: %v", err) 212 return err 213 } 214 return nil 215 } 216 217 func (n *nomadFSM) applyDeregisterJob(buf []byte, index uint64) interface{} { 218 defer metrics.MeasureSince([]string{"nomad", "fsm", "deregister_job"}, time.Now()) 219 var req structs.JobDeregisterRequest 220 if err := structs.Decode(buf, &req); err != nil { 221 panic(fmt.Errorf("failed to decode request: %v", err)) 222 } 223 224 if err := n.state.DeleteJob(index, req.JobID); err != nil { 225 n.logger.Printf("[ERR] nomad.fsm: DeleteJob failed: %v", err) 226 return err 227 } 228 return nil 229 } 230 231 func (n *nomadFSM) applyUpdateEval(buf []byte, index uint64) interface{} { 232 defer metrics.MeasureSince([]string{"nomad", "fsm", "update_eval"}, time.Now()) 233 var req structs.EvalUpdateRequest 234 if err := structs.Decode(buf, &req); err != nil { 235 panic(fmt.Errorf("failed to decode request: %v", err)) 236 } 237 238 if err := n.state.UpsertEvals(index, req.Evals); err != nil { 239 n.logger.Printf("[ERR] nomad.fsm: UpsertEvals failed: %v", err) 240 return err 241 } 242 243 for _, eval := range req.Evals { 244 if eval.ShouldEnqueue() { 245 if err := n.evalBroker.Enqueue(eval); err != nil { 246 n.logger.Printf("[ERR] nomad.fsm: failed to enqueue evaluation %s: %v", eval.ID, err) 247 return err 248 } 249 } 250 } 251 return nil 252 } 253 254 func (n *nomadFSM) applyDeleteEval(buf []byte, index uint64) interface{} { 255 defer metrics.MeasureSince([]string{"nomad", "fsm", "delete_eval"}, time.Now()) 256 var req structs.EvalDeleteRequest 257 if err := structs.Decode(buf, &req); err != nil { 258 panic(fmt.Errorf("failed to decode request: %v", err)) 259 } 260 261 if err := n.state.DeleteEval(index, req.Evals, req.Allocs); err != nil { 262 n.logger.Printf("[ERR] nomad.fsm: DeleteEval failed: %v", err) 263 return err 264 } 265 return nil 266 } 267 268 func (n *nomadFSM) applyAllocUpdate(buf []byte, index uint64) interface{} { 269 defer metrics.MeasureSince([]string{"nomad", "fsm", "alloc_update"}, time.Now()) 270 var req structs.AllocUpdateRequest 271 if err := structs.Decode(buf, &req); err != nil { 272 panic(fmt.Errorf("failed to decode request: %v", err)) 273 } 274 275 if err := n.state.UpsertAllocs(index, req.Alloc); err != nil { 276 n.logger.Printf("[ERR] nomad.fsm: UpsertAllocs failed: %v", err) 277 return err 278 } 279 return nil 280 } 281 282 func (n *nomadFSM) applyAllocClientUpdate(buf []byte, index uint64) interface{} { 283 defer metrics.MeasureSince([]string{"nomad", "fsm", "alloc_client_update"}, time.Now()) 284 var req structs.AllocUpdateRequest 285 if err := structs.Decode(buf, &req); err != nil { 286 panic(fmt.Errorf("failed to decode request: %v", err)) 287 } 288 if len(req.Alloc) == 0 { 289 return nil 290 } 291 292 if err := n.state.UpdateAllocFromClient(index, req.Alloc[0]); err != nil { 293 n.logger.Printf("[ERR] nomad.fsm: UpdateAllocFromClient failed: %v", err) 294 return err 295 } 296 return nil 297 } 298 299 func (n *nomadFSM) Snapshot() (raft.FSMSnapshot, error) { 300 // Create a new snapshot 301 snap, err := n.state.Snapshot() 302 if err != nil { 303 return nil, err 304 } 305 306 ns := &nomadSnapshot{ 307 snap: snap, 308 timetable: n.timetable, 309 } 310 return ns, nil 311 } 312 313 func (n *nomadFSM) Restore(old io.ReadCloser) error { 314 defer old.Close() 315 316 // Create a new state store 317 newState, err := state.NewStateStore(n.logOutput) 318 if err != nil { 319 return err 320 } 321 n.state = newState 322 323 // Start the state restore 324 restore, err := newState.Restore() 325 if err != nil { 326 return err 327 } 328 defer restore.Abort() 329 330 // Create a decoder 331 dec := codec.NewDecoder(old, msgpackHandle) 332 333 // Read in the header 334 var header snapshotHeader 335 if err := dec.Decode(&header); err != nil { 336 return err 337 } 338 339 // Populate the new state 340 msgType := make([]byte, 1) 341 for { 342 // Read the message type 343 _, err := old.Read(msgType) 344 if err == io.EOF { 345 break 346 } else if err != nil { 347 return err 348 } 349 350 // Decode 351 switch SnapshotType(msgType[0]) { 352 case TimeTableSnapshot: 353 if err := n.timetable.Deserialize(dec); err != nil { 354 return fmt.Errorf("time table deserialize failed: %v", err) 355 } 356 357 case NodeSnapshot: 358 node := new(structs.Node) 359 if err := dec.Decode(node); err != nil { 360 return err 361 } 362 if err := restore.NodeRestore(node); err != nil { 363 return err 364 } 365 366 case JobSnapshot: 367 job := new(structs.Job) 368 if err := dec.Decode(job); err != nil { 369 return err 370 } 371 if err := restore.JobRestore(job); err != nil { 372 return err 373 } 374 375 case EvalSnapshot: 376 eval := new(structs.Evaluation) 377 if err := dec.Decode(eval); err != nil { 378 return err 379 } 380 if err := restore.EvalRestore(eval); err != nil { 381 return err 382 } 383 384 case AllocSnapshot: 385 alloc := new(structs.Allocation) 386 if err := dec.Decode(alloc); err != nil { 387 return err 388 } 389 if err := restore.AllocRestore(alloc); err != nil { 390 return err 391 } 392 393 case IndexSnapshot: 394 idx := new(state.IndexEntry) 395 if err := dec.Decode(idx); err != nil { 396 return err 397 } 398 if err := restore.IndexRestore(idx); err != nil { 399 return err 400 } 401 402 default: 403 return fmt.Errorf("Unrecognized snapshot type: %v", msgType) 404 } 405 } 406 407 // Commit the state restore 408 restore.Commit() 409 return nil 410 } 411 412 func (s *nomadSnapshot) Persist(sink raft.SnapshotSink) error { 413 defer metrics.MeasureSince([]string{"nomad", "fsm", "persist"}, time.Now()) 414 // Register the nodes 415 encoder := codec.NewEncoder(sink, msgpackHandle) 416 417 // Write the header 418 header := snapshotHeader{} 419 if err := encoder.Encode(&header); err != nil { 420 sink.Cancel() 421 return err 422 } 423 424 // Write the time table 425 sink.Write([]byte{byte(TimeTableSnapshot)}) 426 if err := s.timetable.Serialize(encoder); err != nil { 427 sink.Cancel() 428 return err 429 } 430 431 // Write all the data out 432 if err := s.persistIndexes(sink, encoder); err != nil { 433 sink.Cancel() 434 return err 435 } 436 if err := s.persistNodes(sink, encoder); err != nil { 437 sink.Cancel() 438 return err 439 } 440 if err := s.persistJobs(sink, encoder); err != nil { 441 sink.Cancel() 442 return err 443 } 444 if err := s.persistEvals(sink, encoder); err != nil { 445 sink.Cancel() 446 return err 447 } 448 if err := s.persistAllocs(sink, encoder); err != nil { 449 sink.Cancel() 450 return err 451 } 452 return nil 453 } 454 455 func (s *nomadSnapshot) persistIndexes(sink raft.SnapshotSink, 456 encoder *codec.Encoder) error { 457 // Get all the indexes 458 iter, err := s.snap.Indexes() 459 if err != nil { 460 return err 461 } 462 463 for { 464 // Get the next item 465 raw := iter.Next() 466 if raw == nil { 467 break 468 } 469 470 // Prepare the request struct 471 idx := raw.(*state.IndexEntry) 472 473 // Write out a node registration 474 sink.Write([]byte{byte(IndexSnapshot)}) 475 if err := encoder.Encode(idx); err != nil { 476 return err 477 } 478 } 479 return nil 480 } 481 482 func (s *nomadSnapshot) persistNodes(sink raft.SnapshotSink, 483 encoder *codec.Encoder) error { 484 // Get all the nodes 485 nodes, err := s.snap.Nodes() 486 if err != nil { 487 return err 488 } 489 490 for { 491 // Get the next item 492 raw := nodes.Next() 493 if raw == nil { 494 break 495 } 496 497 // Prepare the request struct 498 node := raw.(*structs.Node) 499 500 // Write out a node registration 501 sink.Write([]byte{byte(NodeSnapshot)}) 502 if err := encoder.Encode(node); err != nil { 503 return err 504 } 505 } 506 return nil 507 } 508 509 func (s *nomadSnapshot) persistJobs(sink raft.SnapshotSink, 510 encoder *codec.Encoder) error { 511 // Get all the jobs 512 jobs, err := s.snap.Jobs() 513 if err != nil { 514 return err 515 } 516 517 for { 518 // Get the next item 519 raw := jobs.Next() 520 if raw == nil { 521 break 522 } 523 524 // Prepare the request struct 525 job := raw.(*structs.Job) 526 527 // Write out a job registration 528 sink.Write([]byte{byte(JobSnapshot)}) 529 if err := encoder.Encode(job); err != nil { 530 return err 531 } 532 } 533 return nil 534 } 535 536 func (s *nomadSnapshot) persistEvals(sink raft.SnapshotSink, 537 encoder *codec.Encoder) error { 538 // Get all the evaluations 539 evals, err := s.snap.Evals() 540 if err != nil { 541 return err 542 } 543 544 for { 545 // Get the next item 546 raw := evals.Next() 547 if raw == nil { 548 break 549 } 550 551 // Prepare the request struct 552 eval := raw.(*structs.Evaluation) 553 554 // Write out the evaluation 555 sink.Write([]byte{byte(EvalSnapshot)}) 556 if err := encoder.Encode(eval); err != nil { 557 return err 558 } 559 } 560 return nil 561 } 562 563 func (s *nomadSnapshot) persistAllocs(sink raft.SnapshotSink, 564 encoder *codec.Encoder) error { 565 // Get all the allocations 566 allocs, err := s.snap.Allocs() 567 if err != nil { 568 return err 569 } 570 571 for { 572 // Get the next item 573 raw := allocs.Next() 574 if raw == nil { 575 break 576 } 577 578 // Prepare the request struct 579 alloc := raw.(*structs.Allocation) 580 581 // Write out the evaluation 582 sink.Write([]byte{byte(AllocSnapshot)}) 583 if err := encoder.Encode(alloc); err != nil { 584 return err 585 } 586 } 587 return nil 588 } 589 590 // Release is a no-op, as we just need to GC the pointer 591 // to the state store snapshot. There is nothing to explicitly 592 // cleanup. 593 func (s *nomadSnapshot) Release() {}