github.com/hhrutter/nomad@v0.6.0-rc2.0.20170723054333-80c4b03f0705/nomad/state/state_store.go (about) 1 package state 2 3 import ( 4 "fmt" 5 "io" 6 "log" 7 8 "github.com/hashicorp/go-memdb" 9 multierror "github.com/hashicorp/go-multierror" 10 "github.com/hashicorp/nomad/helper" 11 "github.com/hashicorp/nomad/nomad/structs" 12 ) 13 14 // IndexEntry is used with the "index" table 15 // for managing the latest Raft index affecting a table. 16 type IndexEntry struct { 17 Key string 18 Value uint64 19 } 20 21 // The StateStore is responsible for maintaining all the Nomad 22 // state. It is manipulated by the FSM which maintains consistency 23 // through the use of Raft. The goals of the StateStore are to provide 24 // high concurrency for read operations without blocking writes, and 25 // to provide write availability in the face of reads. EVERY object 26 // returned as a result of a read against the state store should be 27 // considered a constant and NEVER modified in place. 28 type StateStore struct { 29 logger *log.Logger 30 db *memdb.MemDB 31 32 // abandonCh is used to signal watchers that this state store has been 33 // abandoned (usually during a restore). This is only ever closed. 34 abandonCh chan struct{} 35 } 36 37 // NewStateStore is used to create a new state store 38 func NewStateStore(logOutput io.Writer) (*StateStore, error) { 39 // Create the MemDB 40 db, err := memdb.NewMemDB(stateStoreSchema()) 41 if err != nil { 42 return nil, fmt.Errorf("state store setup failed: %v", err) 43 } 44 45 // Create the state store 46 s := &StateStore{ 47 logger: log.New(logOutput, "", log.LstdFlags), 48 db: db, 49 abandonCh: make(chan struct{}), 50 } 51 return s, nil 52 } 53 54 // Snapshot is used to create a point in time snapshot. Because 55 // we use MemDB, we just need to snapshot the state of the underlying 56 // database. 57 func (s *StateStore) Snapshot() (*StateSnapshot, error) { 58 snap := &StateSnapshot{ 59 StateStore: StateStore{ 60 logger: s.logger, 61 db: s.db.Snapshot(), 62 }, 63 } 64 return snap, nil 65 } 66 67 // Restore is used to optimize the efficiency of rebuilding 68 // state by minimizing the number of transactions and checking 69 // overhead. 70 func (s *StateStore) Restore() (*StateRestore, error) { 71 txn := s.db.Txn(true) 72 r := &StateRestore{ 73 txn: txn, 74 } 75 return r, nil 76 } 77 78 // AbandonCh returns a channel you can wait on to know if the state store was 79 // abandoned. 80 func (s *StateStore) AbandonCh() <-chan struct{} { 81 return s.abandonCh 82 } 83 84 // Abandon is used to signal that the given state store has been abandoned. 85 // Calling this more than one time will panic. 86 func (s *StateStore) Abandon() { 87 close(s.abandonCh) 88 } 89 90 // UpsertPlanResults is used to upsert the results of a plan. 91 func (s *StateStore) UpsertPlanResults(index uint64, results *structs.ApplyPlanResultsRequest) error { 92 txn := s.db.Txn(true) 93 defer txn.Abort() 94 95 // Upsert the newly created or updated deployment 96 if results.Deployment != nil { 97 if err := s.upsertDeploymentImpl(index, results.Deployment, txn); err != nil { 98 return err 99 } 100 } 101 102 // Update the status of deployments effected by the plan. 103 if len(results.DeploymentUpdates) != 0 { 104 s.upsertDeploymentUpdates(index, results.DeploymentUpdates, txn) 105 } 106 107 // Attach the job to all the allocations. It is pulled out in the payload to 108 // avoid the redundancy of encoding, but should be denormalized prior to 109 // being inserted into MemDB. 110 structs.DenormalizeAllocationJobs(results.Job, results.Alloc) 111 112 // Calculate the total resources of allocations. It is pulled out in the 113 // payload to avoid encoding something that can be computed, but should be 114 // denormalized prior to being inserted into MemDB. 115 for _, alloc := range results.Alloc { 116 if alloc.Resources != nil { 117 continue 118 } 119 120 alloc.Resources = new(structs.Resources) 121 for _, task := range alloc.TaskResources { 122 alloc.Resources.Add(task) 123 } 124 125 // Add the shared resources 126 alloc.Resources.Add(alloc.SharedResources) 127 } 128 129 // Upsert the allocations 130 if err := s.upsertAllocsImpl(index, results.Alloc, txn); err != nil { 131 return err 132 } 133 134 txn.Commit() 135 return nil 136 } 137 138 // upsertDeploymentUpdates updates the deployments given the passed status 139 // updates. 140 func (s *StateStore) upsertDeploymentUpdates(index uint64, updates []*structs.DeploymentStatusUpdate, txn *memdb.Txn) error { 141 for _, u := range updates { 142 if err := s.updateDeploymentStatusImpl(index, u, txn); err != nil { 143 return err 144 } 145 } 146 147 return nil 148 } 149 150 // UpsertJobSummary upserts a job summary into the state store. 151 func (s *StateStore) UpsertJobSummary(index uint64, jobSummary *structs.JobSummary) error { 152 txn := s.db.Txn(true) 153 defer txn.Abort() 154 155 // Check if the job summary already exists 156 existing, err := txn.First("job_summary", "id", jobSummary.JobID) 157 if err != nil { 158 return fmt.Errorf("job summary lookup failed: %v", err) 159 } 160 161 // Setup the indexes correctly 162 if existing != nil { 163 jobSummary.CreateIndex = existing.(*structs.JobSummary).CreateIndex 164 jobSummary.ModifyIndex = index 165 } else { 166 jobSummary.CreateIndex = index 167 jobSummary.ModifyIndex = index 168 } 169 170 // Update the index 171 if err := txn.Insert("job_summary", jobSummary); err != nil { 172 return err 173 } 174 175 // Update the indexes table for job summary 176 if err := txn.Insert("index", &IndexEntry{"job_summary", index}); err != nil { 177 return fmt.Errorf("index update failed: %v", err) 178 } 179 180 txn.Commit() 181 return nil 182 } 183 184 // DeleteJobSummary deletes the job summary with the given ID. This is for 185 // testing purposes only. 186 func (s *StateStore) DeleteJobSummary(index uint64, id string) error { 187 txn := s.db.Txn(true) 188 defer txn.Abort() 189 190 // Delete the job summary 191 if _, err := txn.DeleteAll("job_summary", "id", id); err != nil { 192 return fmt.Errorf("deleting job summary failed: %v", err) 193 } 194 if err := txn.Insert("index", &IndexEntry{"job_summary", index}); err != nil { 195 return fmt.Errorf("index update failed: %v", err) 196 } 197 txn.Commit() 198 return nil 199 } 200 201 // UpsertDeployment is used to insert a new deployment. If cancelPrior is set to 202 // true, all prior deployments for the same job will be cancelled. 203 func (s *StateStore) UpsertDeployment(index uint64, deployment *structs.Deployment) error { 204 txn := s.db.Txn(true) 205 defer txn.Abort() 206 if err := s.upsertDeploymentImpl(index, deployment, txn); err != nil { 207 return err 208 } 209 txn.Commit() 210 return nil 211 } 212 213 func (s *StateStore) upsertDeploymentImpl(index uint64, deployment *structs.Deployment, txn *memdb.Txn) error { 214 // Check if the deployment already exists 215 existing, err := txn.First("deployment", "id", deployment.ID) 216 if err != nil { 217 return fmt.Errorf("deployment lookup failed: %v", err) 218 } 219 220 // Setup the indexes correctly 221 if existing != nil { 222 deployment.CreateIndex = existing.(*structs.Deployment).CreateIndex 223 deployment.ModifyIndex = index 224 } else { 225 deployment.CreateIndex = index 226 deployment.ModifyIndex = index 227 } 228 229 // Insert the deployment 230 if err := txn.Insert("deployment", deployment); err != nil { 231 return err 232 } 233 234 // Update the indexes table for deployment 235 if err := txn.Insert("index", &IndexEntry{"deployment", index}); err != nil { 236 return fmt.Errorf("index update failed: %v", err) 237 } 238 239 // If the deployment is being marked as complete, set the job to stable. 240 if deployment.Status == structs.DeploymentStatusSuccessful { 241 if err := s.updateJobStabilityImpl(index, deployment.JobID, deployment.JobVersion, true, txn); err != nil { 242 return fmt.Errorf("failed to update job stability: %v", err) 243 } 244 } 245 246 return nil 247 } 248 249 func (s *StateStore) Deployments(ws memdb.WatchSet) (memdb.ResultIterator, error) { 250 txn := s.db.Txn(false) 251 252 // Walk the entire deployments table 253 iter, err := txn.Get("deployment", "id") 254 if err != nil { 255 return nil, err 256 } 257 258 ws.Add(iter.WatchCh()) 259 return iter, nil 260 } 261 262 func (s *StateStore) DeploymentsByIDPrefix(ws memdb.WatchSet, deploymentID string) (memdb.ResultIterator, error) { 263 txn := s.db.Txn(false) 264 265 // Walk the entire deployments table 266 iter, err := txn.Get("deployment", "id_prefix", deploymentID) 267 if err != nil { 268 return nil, err 269 } 270 271 ws.Add(iter.WatchCh()) 272 return iter, nil 273 } 274 275 func (s *StateStore) DeploymentByID(ws memdb.WatchSet, deploymentID string) (*structs.Deployment, error) { 276 txn := s.db.Txn(false) 277 return s.deploymentByIDImpl(ws, deploymentID, txn) 278 } 279 280 func (s *StateStore) deploymentByIDImpl(ws memdb.WatchSet, deploymentID string, txn *memdb.Txn) (*structs.Deployment, error) { 281 watchCh, existing, err := txn.FirstWatch("deployment", "id", deploymentID) 282 if err != nil { 283 return nil, fmt.Errorf("deployment lookup failed: %v", err) 284 } 285 ws.Add(watchCh) 286 287 if existing != nil { 288 return existing.(*structs.Deployment), nil 289 } 290 291 return nil, nil 292 } 293 294 func (s *StateStore) DeploymentsByJobID(ws memdb.WatchSet, jobID string) ([]*structs.Deployment, error) { 295 txn := s.db.Txn(false) 296 297 // Get an iterator over the deployments 298 iter, err := txn.Get("deployment", "job", jobID) 299 if err != nil { 300 return nil, err 301 } 302 303 ws.Add(iter.WatchCh()) 304 305 var out []*structs.Deployment 306 for { 307 raw := iter.Next() 308 if raw == nil { 309 break 310 } 311 312 d := raw.(*structs.Deployment) 313 out = append(out, d) 314 } 315 316 return out, nil 317 } 318 319 // LatestDeploymentByJobID returns the latest deployment for the given job. The 320 // latest is determined strictly by CreateIndex. 321 func (s *StateStore) LatestDeploymentByJobID(ws memdb.WatchSet, jobID string) (*structs.Deployment, error) { 322 txn := s.db.Txn(false) 323 324 // Get an iterator over the deployments 325 iter, err := txn.Get("deployment", "job", jobID) 326 if err != nil { 327 return nil, err 328 } 329 330 ws.Add(iter.WatchCh()) 331 332 var out *structs.Deployment 333 for { 334 raw := iter.Next() 335 if raw == nil { 336 break 337 } 338 339 d := raw.(*structs.Deployment) 340 if out == nil || out.CreateIndex < d.CreateIndex { 341 out = d 342 } 343 } 344 345 return out, nil 346 } 347 348 // DeleteDeployment is used to delete a set of deployments by ID 349 func (s *StateStore) DeleteDeployment(index uint64, deploymentIDs []string) error { 350 txn := s.db.Txn(true) 351 defer txn.Abort() 352 353 if len(deploymentIDs) == 0 { 354 return nil 355 } 356 357 for _, deploymentID := range deploymentIDs { 358 // Lookup the deployment 359 existing, err := txn.First("deployment", "id", deploymentID) 360 if err != nil { 361 return fmt.Errorf("deployment lookup failed: %v", err) 362 } 363 if existing == nil { 364 return fmt.Errorf("deployment not found") 365 } 366 367 // Delete the deployment 368 if err := txn.Delete("deployment", existing); err != nil { 369 return fmt.Errorf("deployment delete failed: %v", err) 370 } 371 } 372 373 if err := txn.Insert("index", &IndexEntry{"deployment", index}); err != nil { 374 return fmt.Errorf("index update failed: %v", err) 375 } 376 377 txn.Commit() 378 return nil 379 } 380 381 // UpsertNode is used to register a node or update a node definition 382 // This is assumed to be triggered by the client, so we retain the value 383 // of drain which is set by the scheduler. 384 func (s *StateStore) UpsertNode(index uint64, node *structs.Node) error { 385 txn := s.db.Txn(true) 386 defer txn.Abort() 387 388 // Check if the node already exists 389 existing, err := txn.First("nodes", "id", node.ID) 390 if err != nil { 391 return fmt.Errorf("node lookup failed: %v", err) 392 } 393 394 // Setup the indexes correctly 395 if existing != nil { 396 exist := existing.(*structs.Node) 397 node.CreateIndex = exist.CreateIndex 398 node.ModifyIndex = index 399 node.Drain = exist.Drain // Retain the drain mode 400 } else { 401 node.CreateIndex = index 402 node.ModifyIndex = index 403 } 404 405 // Insert the node 406 if err := txn.Insert("nodes", node); err != nil { 407 return fmt.Errorf("node insert failed: %v", err) 408 } 409 if err := txn.Insert("index", &IndexEntry{"nodes", index}); err != nil { 410 return fmt.Errorf("index update failed: %v", err) 411 } 412 413 txn.Commit() 414 return nil 415 } 416 417 // DeleteNode is used to deregister a node 418 func (s *StateStore) DeleteNode(index uint64, nodeID string) error { 419 txn := s.db.Txn(true) 420 defer txn.Abort() 421 422 // Lookup the node 423 existing, err := txn.First("nodes", "id", nodeID) 424 if err != nil { 425 return fmt.Errorf("node lookup failed: %v", err) 426 } 427 if existing == nil { 428 return fmt.Errorf("node not found") 429 } 430 431 // Delete the node 432 if err := txn.Delete("nodes", existing); err != nil { 433 return fmt.Errorf("node delete failed: %v", err) 434 } 435 if err := txn.Insert("index", &IndexEntry{"nodes", index}); err != nil { 436 return fmt.Errorf("index update failed: %v", err) 437 } 438 439 txn.Commit() 440 return nil 441 } 442 443 // UpdateNodeStatus is used to update the status of a node 444 func (s *StateStore) UpdateNodeStatus(index uint64, nodeID, status string) error { 445 txn := s.db.Txn(true) 446 defer txn.Abort() 447 448 // Lookup the node 449 existing, err := txn.First("nodes", "id", nodeID) 450 if err != nil { 451 return fmt.Errorf("node lookup failed: %v", err) 452 } 453 if existing == nil { 454 return fmt.Errorf("node not found") 455 } 456 457 // Copy the existing node 458 existingNode := existing.(*structs.Node) 459 copyNode := new(structs.Node) 460 *copyNode = *existingNode 461 462 // Update the status in the copy 463 copyNode.Status = status 464 copyNode.ModifyIndex = index 465 466 // Insert the node 467 if err := txn.Insert("nodes", copyNode); err != nil { 468 return fmt.Errorf("node update failed: %v", err) 469 } 470 if err := txn.Insert("index", &IndexEntry{"nodes", index}); err != nil { 471 return fmt.Errorf("index update failed: %v", err) 472 } 473 474 txn.Commit() 475 return nil 476 } 477 478 // UpdateNodeDrain is used to update the drain of a node 479 func (s *StateStore) UpdateNodeDrain(index uint64, nodeID string, drain bool) error { 480 txn := s.db.Txn(true) 481 defer txn.Abort() 482 483 // Lookup the node 484 existing, err := txn.First("nodes", "id", nodeID) 485 if err != nil { 486 return fmt.Errorf("node lookup failed: %v", err) 487 } 488 if existing == nil { 489 return fmt.Errorf("node not found") 490 } 491 492 // Copy the existing node 493 existingNode := existing.(*structs.Node) 494 copyNode := new(structs.Node) 495 *copyNode = *existingNode 496 497 // Update the drain in the copy 498 copyNode.Drain = drain 499 copyNode.ModifyIndex = index 500 501 // Insert the node 502 if err := txn.Insert("nodes", copyNode); err != nil { 503 return fmt.Errorf("node update failed: %v", err) 504 } 505 if err := txn.Insert("index", &IndexEntry{"nodes", index}); err != nil { 506 return fmt.Errorf("index update failed: %v", err) 507 } 508 509 txn.Commit() 510 return nil 511 } 512 513 // NodeByID is used to lookup a node by ID 514 func (s *StateStore) NodeByID(ws memdb.WatchSet, nodeID string) (*structs.Node, error) { 515 txn := s.db.Txn(false) 516 517 watchCh, existing, err := txn.FirstWatch("nodes", "id", nodeID) 518 if err != nil { 519 return nil, fmt.Errorf("node lookup failed: %v", err) 520 } 521 ws.Add(watchCh) 522 523 if existing != nil { 524 return existing.(*structs.Node), nil 525 } 526 return nil, nil 527 } 528 529 // NodesByIDPrefix is used to lookup nodes by prefix 530 func (s *StateStore) NodesByIDPrefix(ws memdb.WatchSet, nodeID string) (memdb.ResultIterator, error) { 531 txn := s.db.Txn(false) 532 533 iter, err := txn.Get("nodes", "id_prefix", nodeID) 534 if err != nil { 535 return nil, fmt.Errorf("node lookup failed: %v", err) 536 } 537 ws.Add(iter.WatchCh()) 538 539 return iter, nil 540 } 541 542 // Nodes returns an iterator over all the nodes 543 func (s *StateStore) Nodes(ws memdb.WatchSet) (memdb.ResultIterator, error) { 544 txn := s.db.Txn(false) 545 546 // Walk the entire nodes table 547 iter, err := txn.Get("nodes", "id") 548 if err != nil { 549 return nil, err 550 } 551 ws.Add(iter.WatchCh()) 552 return iter, nil 553 } 554 555 // UpsertJob is used to register a job or update a job definition 556 func (s *StateStore) UpsertJob(index uint64, job *structs.Job) error { 557 txn := s.db.Txn(true) 558 defer txn.Abort() 559 if err := s.upsertJobImpl(index, job, false, txn); err != nil { 560 return err 561 } 562 txn.Commit() 563 return nil 564 } 565 566 // upsertJobImpl is the inplementation for registering a job or updating a job definition 567 func (s *StateStore) upsertJobImpl(index uint64, job *structs.Job, keepVersion bool, txn *memdb.Txn) error { 568 // Check if the job already exists 569 existing, err := txn.First("jobs", "id", job.ID) 570 if err != nil { 571 return fmt.Errorf("job lookup failed: %v", err) 572 } 573 574 // Setup the indexes correctly 575 if existing != nil { 576 job.CreateIndex = existing.(*structs.Job).CreateIndex 577 job.ModifyIndex = index 578 579 // Bump the version unless asked to keep it. This should only be done 580 // when changing an internal field such as Stable. A spec change should 581 // always come with a version bump 582 if !keepVersion { 583 job.JobModifyIndex = index 584 job.Version = existing.(*structs.Job).Version + 1 585 } 586 587 // Compute the job status 588 var err error 589 job.Status, err = s.getJobStatus(txn, job, false) 590 if err != nil { 591 return fmt.Errorf("setting job status for %q failed: %v", job.ID, err) 592 } 593 } else { 594 job.CreateIndex = index 595 job.ModifyIndex = index 596 job.JobModifyIndex = index 597 job.Version = 0 598 599 if err := s.setJobStatus(index, txn, job, false, ""); err != nil { 600 return fmt.Errorf("setting job status for %q failed: %v", job.ID, err) 601 } 602 603 // Have to get the job again since it could have been updated 604 updated, err := txn.First("jobs", "id", job.ID) 605 if err != nil { 606 return fmt.Errorf("job lookup failed: %v", err) 607 } 608 if updated != nil { 609 job = updated.(*structs.Job) 610 } 611 } 612 613 if err := s.updateSummaryWithJob(index, job, txn); err != nil { 614 return fmt.Errorf("unable to create job summary: %v", err) 615 } 616 617 if err := s.upsertJobVersion(index, job, txn); err != nil { 618 return fmt.Errorf("unable to upsert job into job_version table: %v", err) 619 } 620 621 // Create the EphemeralDisk if it's nil by adding up DiskMB from task resources. 622 // COMPAT 0.4.1 -> 0.5 623 s.addEphemeralDiskToTaskGroups(job) 624 625 // Insert the job 626 if err := txn.Insert("jobs", job); err != nil { 627 return fmt.Errorf("job insert failed: %v", err) 628 } 629 if err := txn.Insert("index", &IndexEntry{"jobs", index}); err != nil { 630 return fmt.Errorf("index update failed: %v", err) 631 } 632 633 return nil 634 } 635 636 // DeleteJob is used to deregister a job 637 func (s *StateStore) DeleteJob(index uint64, jobID string) error { 638 txn := s.db.Txn(true) 639 defer txn.Abort() 640 641 // Lookup the node 642 existing, err := txn.First("jobs", "id", jobID) 643 if err != nil { 644 return fmt.Errorf("job lookup failed: %v", err) 645 } 646 if existing == nil { 647 return fmt.Errorf("job not found") 648 } 649 650 // Check if we should update a parent job summary 651 job := existing.(*structs.Job) 652 if job.ParentID != "" { 653 summaryRaw, err := txn.First("job_summary", "id", job.ParentID) 654 if err != nil { 655 return fmt.Errorf("unable to retrieve summary for parent job: %v", err) 656 } 657 658 // Only continue if the summary exists. It could not exist if the parent 659 // job was removed 660 if summaryRaw != nil { 661 existing := summaryRaw.(*structs.JobSummary) 662 pSummary := existing.Copy() 663 if pSummary.Children != nil { 664 665 modified := false 666 switch job.Status { 667 case structs.JobStatusPending: 668 pSummary.Children.Pending-- 669 pSummary.Children.Dead++ 670 modified = true 671 case structs.JobStatusRunning: 672 pSummary.Children.Running-- 673 pSummary.Children.Dead++ 674 modified = true 675 case structs.JobStatusDead: 676 default: 677 return fmt.Errorf("unknown old job status %q", job.Status) 678 } 679 680 if modified { 681 // Update the modify index 682 pSummary.ModifyIndex = index 683 684 // Insert the summary 685 if err := txn.Insert("job_summary", pSummary); err != nil { 686 return fmt.Errorf("job summary insert failed: %v", err) 687 } 688 if err := txn.Insert("index", &IndexEntry{"job_summary", index}); err != nil { 689 return fmt.Errorf("index update failed: %v", err) 690 } 691 } 692 } 693 } 694 } 695 696 // Delete the job 697 if err := txn.Delete("jobs", existing); err != nil { 698 return fmt.Errorf("job delete failed: %v", err) 699 } 700 if err := txn.Insert("index", &IndexEntry{"jobs", index}); err != nil { 701 return fmt.Errorf("index update failed: %v", err) 702 } 703 704 // Delete the job versions 705 if err := s.deleteJobVersions(index, job, txn); err != nil { 706 return err 707 } 708 709 // Delete the job summary 710 if _, err = txn.DeleteAll("job_summary", "id", jobID); err != nil { 711 return fmt.Errorf("deleing job summary failed: %v", err) 712 } 713 if err := txn.Insert("index", &IndexEntry{"job_summary", index}); err != nil { 714 return fmt.Errorf("index update failed: %v", err) 715 } 716 717 txn.Commit() 718 return nil 719 } 720 721 // deleteJobVersions deletes all versions of the given job. 722 func (s *StateStore) deleteJobVersions(index uint64, job *structs.Job, txn *memdb.Txn) error { 723 iter, err := txn.Get("job_version", "id_prefix", job.ID) 724 if err != nil { 725 return err 726 } 727 728 for { 729 raw := iter.Next() 730 if raw == nil { 731 break 732 } 733 734 // Ensure the ID is an exact match 735 j := raw.(*structs.Job) 736 if j.ID != job.ID { 737 continue 738 } 739 740 if _, err = txn.DeleteAll("job_version", "id", job.ID, job.Version); err != nil { 741 return fmt.Errorf("deleting job versions failed: %v", err) 742 } 743 } 744 745 if err := txn.Insert("index", &IndexEntry{"job_summary", index}); err != nil { 746 return fmt.Errorf("index update failed: %v", err) 747 } 748 749 return nil 750 } 751 752 // upsertJobVersion inserts a job into its historic version table and limits the 753 // number of job versions that are tracked. 754 func (s *StateStore) upsertJobVersion(index uint64, job *structs.Job, txn *memdb.Txn) error { 755 // Insert the job 756 if err := txn.Insert("job_version", job); err != nil { 757 return fmt.Errorf("failed to insert job into job_version table: %v", err) 758 } 759 760 if err := txn.Insert("index", &IndexEntry{"job_version", index}); err != nil { 761 return fmt.Errorf("index update failed: %v", err) 762 } 763 764 // Get all the historic jobs for this ID 765 all, err := s.jobVersionByID(txn, nil, job.ID) 766 if err != nil { 767 return fmt.Errorf("failed to look up job versions for %q: %v", job.ID, err) 768 } 769 770 // If we are below the limit there is no GCing to be done 771 if len(all) <= structs.JobTrackedVersions { 772 return nil 773 } 774 775 // We have to delete a historic job to make room. 776 // Find index of the highest versioned stable job 777 stableIdx := -1 778 for i, j := range all { 779 if j.Stable { 780 stableIdx = i 781 break 782 } 783 } 784 785 // If the stable job is the oldest version, do a swap to bring it into the 786 // keep set. 787 max := structs.JobTrackedVersions 788 if stableIdx == max { 789 all[max-1], all[max] = all[max], all[max-1] 790 } 791 792 // Delete the job outside of the set that are being kept. 793 d := all[max] 794 if err := txn.Delete("job_version", d); err != nil { 795 return fmt.Errorf("failed to delete job %v (%d) from job_version", d.ID, d.Version) 796 } 797 798 return nil 799 } 800 801 // JobByID is used to lookup a job by its ID. JobByID returns the current/latest job 802 // version. 803 func (s *StateStore) JobByID(ws memdb.WatchSet, id string) (*structs.Job, error) { 804 txn := s.db.Txn(false) 805 806 watchCh, existing, err := txn.FirstWatch("jobs", "id", id) 807 if err != nil { 808 return nil, fmt.Errorf("job lookup failed: %v", err) 809 } 810 ws.Add(watchCh) 811 812 if existing != nil { 813 return existing.(*structs.Job), nil 814 } 815 return nil, nil 816 } 817 818 // JobsByIDPrefix is used to lookup a job by prefix 819 func (s *StateStore) JobsByIDPrefix(ws memdb.WatchSet, id string) (memdb.ResultIterator, error) { 820 txn := s.db.Txn(false) 821 822 iter, err := txn.Get("jobs", "id_prefix", id) 823 if err != nil { 824 return nil, fmt.Errorf("job lookup failed: %v", err) 825 } 826 827 ws.Add(iter.WatchCh()) 828 829 return iter, nil 830 } 831 832 // JobVersionsByID returns all the tracked versions of a job. 833 func (s *StateStore) JobVersionsByID(ws memdb.WatchSet, id string) ([]*structs.Job, error) { 834 txn := s.db.Txn(false) 835 return s.jobVersionByID(txn, &ws, id) 836 } 837 838 // jobVersionByID is the underlying implementation for retrieving all tracked 839 // versions of a job and is called under an existing transaction. A watch set 840 // can optionally be passed in to add the job histories to the watch set. 841 func (s *StateStore) jobVersionByID(txn *memdb.Txn, ws *memdb.WatchSet, id string) ([]*structs.Job, error) { 842 // Get all the historic jobs for this ID 843 iter, err := txn.Get("job_version", "id_prefix", id) 844 if err != nil { 845 return nil, err 846 } 847 848 if ws != nil { 849 ws.Add(iter.WatchCh()) 850 } 851 852 var all []*structs.Job 853 for { 854 raw := iter.Next() 855 if raw == nil { 856 break 857 } 858 859 // Ensure the ID is an exact match 860 j := raw.(*structs.Job) 861 if j.ID != id { 862 continue 863 } 864 865 all = append(all, j) 866 } 867 868 // Reverse so that highest versions first 869 for i, j := 0, len(all)-1; i < j; i, j = i+1, j-1 { 870 all[i], all[j] = all[j], all[i] 871 } 872 873 return all, nil 874 } 875 876 // JobByIDAndVersion returns the job identified by its ID and Version. The 877 // passed watchset may be nil. 878 func (s *StateStore) JobByIDAndVersion(ws memdb.WatchSet, id string, version uint64) (*structs.Job, error) { 879 txn := s.db.Txn(false) 880 return s.jobByIDAndVersionImpl(ws, id, version, txn) 881 } 882 883 // jobByIDAndVersionImpl returns the job identified by its ID and Version. The 884 // passed watchset may be nil. 885 func (s *StateStore) jobByIDAndVersionImpl(ws memdb.WatchSet, id string, version uint64, txn *memdb.Txn) (*structs.Job, error) { 886 watchCh, existing, err := txn.FirstWatch("job_version", "id", id, version) 887 if err != nil { 888 return nil, err 889 } 890 891 if ws != nil { 892 ws.Add(watchCh) 893 } 894 895 if existing != nil { 896 job := existing.(*structs.Job) 897 return job, nil 898 } 899 900 return nil, nil 901 } 902 903 func (s *StateStore) JobVersions(ws memdb.WatchSet) (memdb.ResultIterator, error) { 904 txn := s.db.Txn(false) 905 906 // Walk the entire deployments table 907 iter, err := txn.Get("job_version", "id") 908 if err != nil { 909 return nil, err 910 } 911 912 ws.Add(iter.WatchCh()) 913 return iter, nil 914 } 915 916 // Jobs returns an iterator over all the jobs 917 func (s *StateStore) Jobs(ws memdb.WatchSet) (memdb.ResultIterator, error) { 918 txn := s.db.Txn(false) 919 920 // Walk the entire jobs table 921 iter, err := txn.Get("jobs", "id") 922 if err != nil { 923 return nil, err 924 } 925 926 ws.Add(iter.WatchCh()) 927 928 return iter, nil 929 } 930 931 // JobsByPeriodic returns an iterator over all the periodic or non-periodic jobs. 932 func (s *StateStore) JobsByPeriodic(ws memdb.WatchSet, periodic bool) (memdb.ResultIterator, error) { 933 txn := s.db.Txn(false) 934 935 iter, err := txn.Get("jobs", "periodic", periodic) 936 if err != nil { 937 return nil, err 938 } 939 940 ws.Add(iter.WatchCh()) 941 942 return iter, nil 943 } 944 945 // JobsByScheduler returns an iterator over all the jobs with the specific 946 // scheduler type. 947 func (s *StateStore) JobsByScheduler(ws memdb.WatchSet, schedulerType string) (memdb.ResultIterator, error) { 948 txn := s.db.Txn(false) 949 950 // Return an iterator for jobs with the specific type. 951 iter, err := txn.Get("jobs", "type", schedulerType) 952 if err != nil { 953 return nil, err 954 } 955 956 ws.Add(iter.WatchCh()) 957 958 return iter, nil 959 } 960 961 // JobsByGC returns an iterator over all jobs eligible or uneligible for garbage 962 // collection. 963 func (s *StateStore) JobsByGC(ws memdb.WatchSet, gc bool) (memdb.ResultIterator, error) { 964 txn := s.db.Txn(false) 965 966 iter, err := txn.Get("jobs", "gc", gc) 967 if err != nil { 968 return nil, err 969 } 970 971 ws.Add(iter.WatchCh()) 972 973 return iter, nil 974 } 975 976 // JobSummary returns a job summary object which matches a specific id. 977 func (s *StateStore) JobSummaryByID(ws memdb.WatchSet, jobID string) (*structs.JobSummary, error) { 978 txn := s.db.Txn(false) 979 980 watchCh, existing, err := txn.FirstWatch("job_summary", "id", jobID) 981 if err != nil { 982 return nil, err 983 } 984 985 ws.Add(watchCh) 986 987 if existing != nil { 988 summary := existing.(*structs.JobSummary) 989 return summary, nil 990 } 991 992 return nil, nil 993 } 994 995 // JobSummaries walks the entire job summary table and returns all the job 996 // summary objects 997 func (s *StateStore) JobSummaries(ws memdb.WatchSet) (memdb.ResultIterator, error) { 998 txn := s.db.Txn(false) 999 1000 iter, err := txn.Get("job_summary", "id") 1001 if err != nil { 1002 return nil, err 1003 } 1004 1005 ws.Add(iter.WatchCh()) 1006 1007 return iter, nil 1008 } 1009 1010 // JobSummaryByPrefix is used to look up Job Summary by id prefix 1011 func (s *StateStore) JobSummaryByPrefix(ws memdb.WatchSet, id string) (memdb.ResultIterator, error) { 1012 txn := s.db.Txn(false) 1013 1014 iter, err := txn.Get("job_summary", "id_prefix", id) 1015 if err != nil { 1016 return nil, fmt.Errorf("eval lookup failed: %v", err) 1017 } 1018 1019 ws.Add(iter.WatchCh()) 1020 1021 return iter, nil 1022 } 1023 1024 // UpsertPeriodicLaunch is used to register a launch or update it. 1025 func (s *StateStore) UpsertPeriodicLaunch(index uint64, launch *structs.PeriodicLaunch) error { 1026 txn := s.db.Txn(true) 1027 defer txn.Abort() 1028 1029 // Check if the job already exists 1030 existing, err := txn.First("periodic_launch", "id", launch.ID) 1031 if err != nil { 1032 return fmt.Errorf("periodic launch lookup failed: %v", err) 1033 } 1034 1035 // Setup the indexes correctly 1036 if existing != nil { 1037 launch.CreateIndex = existing.(*structs.PeriodicLaunch).CreateIndex 1038 launch.ModifyIndex = index 1039 } else { 1040 launch.CreateIndex = index 1041 launch.ModifyIndex = index 1042 } 1043 1044 // Insert the job 1045 if err := txn.Insert("periodic_launch", launch); err != nil { 1046 return fmt.Errorf("launch insert failed: %v", err) 1047 } 1048 if err := txn.Insert("index", &IndexEntry{"periodic_launch", index}); err != nil { 1049 return fmt.Errorf("index update failed: %v", err) 1050 } 1051 1052 txn.Commit() 1053 return nil 1054 } 1055 1056 // DeletePeriodicLaunch is used to delete the periodic launch 1057 func (s *StateStore) DeletePeriodicLaunch(index uint64, jobID string) error { 1058 txn := s.db.Txn(true) 1059 defer txn.Abort() 1060 1061 // Lookup the launch 1062 existing, err := txn.First("periodic_launch", "id", jobID) 1063 if err != nil { 1064 return fmt.Errorf("launch lookup failed: %v", err) 1065 } 1066 if existing == nil { 1067 return fmt.Errorf("launch not found") 1068 } 1069 1070 // Delete the launch 1071 if err := txn.Delete("periodic_launch", existing); err != nil { 1072 return fmt.Errorf("launch delete failed: %v", err) 1073 } 1074 if err := txn.Insert("index", &IndexEntry{"periodic_launch", index}); err != nil { 1075 return fmt.Errorf("index update failed: %v", err) 1076 } 1077 1078 txn.Commit() 1079 return nil 1080 } 1081 1082 // PeriodicLaunchByID is used to lookup a periodic launch by the periodic job 1083 // ID. 1084 func (s *StateStore) PeriodicLaunchByID(ws memdb.WatchSet, id string) (*structs.PeriodicLaunch, error) { 1085 txn := s.db.Txn(false) 1086 1087 watchCh, existing, err := txn.FirstWatch("periodic_launch", "id", id) 1088 if err != nil { 1089 return nil, fmt.Errorf("periodic launch lookup failed: %v", err) 1090 } 1091 1092 ws.Add(watchCh) 1093 1094 if existing != nil { 1095 return existing.(*structs.PeriodicLaunch), nil 1096 } 1097 return nil, nil 1098 } 1099 1100 // PeriodicLaunches returns an iterator over all the periodic launches 1101 func (s *StateStore) PeriodicLaunches(ws memdb.WatchSet) (memdb.ResultIterator, error) { 1102 txn := s.db.Txn(false) 1103 1104 // Walk the entire table 1105 iter, err := txn.Get("periodic_launch", "id") 1106 if err != nil { 1107 return nil, err 1108 } 1109 1110 ws.Add(iter.WatchCh()) 1111 1112 return iter, nil 1113 } 1114 1115 // UpsertEvals is used to upsert a set of evaluations 1116 func (s *StateStore) UpsertEvals(index uint64, evals []*structs.Evaluation) error { 1117 txn := s.db.Txn(true) 1118 defer txn.Abort() 1119 1120 // Do a nested upsert 1121 jobs := make(map[string]string, len(evals)) 1122 for _, eval := range evals { 1123 if err := s.nestedUpsertEval(txn, index, eval); err != nil { 1124 return err 1125 } 1126 1127 jobs[eval.JobID] = "" 1128 } 1129 1130 // Set the job's status 1131 if err := s.setJobStatuses(index, txn, jobs, false); err != nil { 1132 return fmt.Errorf("setting job status failed: %v", err) 1133 } 1134 1135 txn.Commit() 1136 return nil 1137 } 1138 1139 // nestedUpsertEvaluation is used to nest an evaluation upsert within a transaction 1140 func (s *StateStore) nestedUpsertEval(txn *memdb.Txn, index uint64, eval *structs.Evaluation) error { 1141 // Lookup the evaluation 1142 existing, err := txn.First("evals", "id", eval.ID) 1143 if err != nil { 1144 return fmt.Errorf("eval lookup failed: %v", err) 1145 } 1146 1147 // Update the indexes 1148 if existing != nil { 1149 eval.CreateIndex = existing.(*structs.Evaluation).CreateIndex 1150 eval.ModifyIndex = index 1151 } else { 1152 eval.CreateIndex = index 1153 eval.ModifyIndex = index 1154 } 1155 1156 // Update the job summary 1157 summaryRaw, err := txn.First("job_summary", "id", eval.JobID) 1158 if err != nil { 1159 return fmt.Errorf("job summary lookup failed: %v", err) 1160 } 1161 if summaryRaw != nil { 1162 js := summaryRaw.(*structs.JobSummary).Copy() 1163 hasSummaryChanged := false 1164 for tg, num := range eval.QueuedAllocations { 1165 if summary, ok := js.Summary[tg]; ok { 1166 if summary.Queued != num { 1167 summary.Queued = num 1168 js.Summary[tg] = summary 1169 hasSummaryChanged = true 1170 } 1171 } else { 1172 s.logger.Printf("[ERR] state_store: unable to update queued for job %q and task group %q", eval.JobID, tg) 1173 } 1174 } 1175 1176 // Insert the job summary 1177 if hasSummaryChanged { 1178 js.ModifyIndex = index 1179 if err := txn.Insert("job_summary", js); err != nil { 1180 return fmt.Errorf("job summary insert failed: %v", err) 1181 } 1182 if err := txn.Insert("index", &IndexEntry{"job_summary", index}); err != nil { 1183 return fmt.Errorf("index update failed: %v", err) 1184 } 1185 } 1186 } 1187 1188 // Check if the job has any blocked evaluations and cancel them 1189 if eval.Status == structs.EvalStatusComplete && len(eval.FailedTGAllocs) == 0 { 1190 // Get the blocked evaluation for a job if it exists 1191 iter, err := txn.Get("evals", "job", eval.JobID, structs.EvalStatusBlocked) 1192 if err != nil { 1193 return fmt.Errorf("failed to get blocked evals for job %q: %v", eval.JobID, err) 1194 } 1195 1196 var blocked []*structs.Evaluation 1197 for { 1198 raw := iter.Next() 1199 if raw == nil { 1200 break 1201 } 1202 blocked = append(blocked, raw.(*structs.Evaluation)) 1203 } 1204 1205 // Go through and update the evals 1206 for _, eval := range blocked { 1207 newEval := eval.Copy() 1208 newEval.Status = structs.EvalStatusCancelled 1209 newEval.StatusDescription = fmt.Sprintf("evaluation %q successful", newEval.ID) 1210 newEval.ModifyIndex = index 1211 if err := txn.Insert("evals", newEval); err != nil { 1212 return fmt.Errorf("eval insert failed: %v", err) 1213 } 1214 } 1215 } 1216 1217 // Insert the eval 1218 if err := txn.Insert("evals", eval); err != nil { 1219 return fmt.Errorf("eval insert failed: %v", err) 1220 } 1221 if err := txn.Insert("index", &IndexEntry{"evals", index}); err != nil { 1222 return fmt.Errorf("index update failed: %v", err) 1223 } 1224 return nil 1225 } 1226 1227 // DeleteEval is used to delete an evaluation 1228 func (s *StateStore) DeleteEval(index uint64, evals []string, allocs []string) error { 1229 txn := s.db.Txn(true) 1230 defer txn.Abort() 1231 1232 jobs := make(map[string]string, len(evals)) 1233 for _, eval := range evals { 1234 existing, err := txn.First("evals", "id", eval) 1235 if err != nil { 1236 return fmt.Errorf("eval lookup failed: %v", err) 1237 } 1238 if existing == nil { 1239 continue 1240 } 1241 if err := txn.Delete("evals", existing); err != nil { 1242 return fmt.Errorf("eval delete failed: %v", err) 1243 } 1244 jobID := existing.(*structs.Evaluation).JobID 1245 jobs[jobID] = "" 1246 } 1247 1248 for _, alloc := range allocs { 1249 existing, err := txn.First("allocs", "id", alloc) 1250 if err != nil { 1251 return fmt.Errorf("alloc lookup failed: %v", err) 1252 } 1253 if existing == nil { 1254 continue 1255 } 1256 if err := txn.Delete("allocs", existing); err != nil { 1257 return fmt.Errorf("alloc delete failed: %v", err) 1258 } 1259 } 1260 1261 // Update the indexes 1262 if err := txn.Insert("index", &IndexEntry{"evals", index}); err != nil { 1263 return fmt.Errorf("index update failed: %v", err) 1264 } 1265 if err := txn.Insert("index", &IndexEntry{"allocs", index}); err != nil { 1266 return fmt.Errorf("index update failed: %v", err) 1267 } 1268 1269 // Set the job's status 1270 if err := s.setJobStatuses(index, txn, jobs, true); err != nil { 1271 return fmt.Errorf("setting job status failed: %v", err) 1272 } 1273 1274 txn.Commit() 1275 return nil 1276 } 1277 1278 // EvalByID is used to lookup an eval by its ID 1279 func (s *StateStore) EvalByID(ws memdb.WatchSet, id string) (*structs.Evaluation, error) { 1280 txn := s.db.Txn(false) 1281 1282 watchCh, existing, err := txn.FirstWatch("evals", "id", id) 1283 if err != nil { 1284 return nil, fmt.Errorf("eval lookup failed: %v", err) 1285 } 1286 1287 ws.Add(watchCh) 1288 1289 if existing != nil { 1290 return existing.(*structs.Evaluation), nil 1291 } 1292 return nil, nil 1293 } 1294 1295 // EvalsByIDPrefix is used to lookup evaluations by prefix 1296 func (s *StateStore) EvalsByIDPrefix(ws memdb.WatchSet, id string) (memdb.ResultIterator, error) { 1297 txn := s.db.Txn(false) 1298 1299 iter, err := txn.Get("evals", "id_prefix", id) 1300 if err != nil { 1301 return nil, fmt.Errorf("eval lookup failed: %v", err) 1302 } 1303 1304 ws.Add(iter.WatchCh()) 1305 1306 return iter, nil 1307 } 1308 1309 // EvalsByJob returns all the evaluations by job id 1310 func (s *StateStore) EvalsByJob(ws memdb.WatchSet, jobID string) ([]*structs.Evaluation, error) { 1311 txn := s.db.Txn(false) 1312 1313 // Get an iterator over the node allocations 1314 iter, err := txn.Get("evals", "job_prefix", jobID) 1315 if err != nil { 1316 return nil, err 1317 } 1318 1319 ws.Add(iter.WatchCh()) 1320 1321 var out []*structs.Evaluation 1322 for { 1323 raw := iter.Next() 1324 if raw == nil { 1325 break 1326 } 1327 1328 e := raw.(*structs.Evaluation) 1329 1330 // Filter non-exact matches 1331 if e.JobID != jobID { 1332 continue 1333 } 1334 1335 out = append(out, e) 1336 } 1337 return out, nil 1338 } 1339 1340 // Evals returns an iterator over all the evaluations 1341 func (s *StateStore) Evals(ws memdb.WatchSet) (memdb.ResultIterator, error) { 1342 txn := s.db.Txn(false) 1343 1344 // Walk the entire table 1345 iter, err := txn.Get("evals", "id") 1346 if err != nil { 1347 return nil, err 1348 } 1349 1350 ws.Add(iter.WatchCh()) 1351 1352 return iter, nil 1353 } 1354 1355 // UpdateAllocsFromClient is used to update an allocation based on input 1356 // from a client. While the schedulers are the authority on the allocation for 1357 // most things, some updates are authoritative from the client. Specifically, 1358 // the desired state comes from the schedulers, while the actual state comes 1359 // from clients. 1360 func (s *StateStore) UpdateAllocsFromClient(index uint64, allocs []*structs.Allocation) error { 1361 txn := s.db.Txn(true) 1362 defer txn.Abort() 1363 1364 // Handle each of the updated allocations 1365 for _, alloc := range allocs { 1366 if err := s.nestedUpdateAllocFromClient(txn, index, alloc); err != nil { 1367 return err 1368 } 1369 } 1370 1371 // Update the indexes 1372 if err := txn.Insert("index", &IndexEntry{"allocs", index}); err != nil { 1373 return fmt.Errorf("index update failed: %v", err) 1374 } 1375 1376 txn.Commit() 1377 return nil 1378 } 1379 1380 // nestedUpdateAllocFromClient is used to nest an update of an allocation with client status 1381 func (s *StateStore) nestedUpdateAllocFromClient(txn *memdb.Txn, index uint64, alloc *structs.Allocation) error { 1382 // Look for existing alloc 1383 existing, err := txn.First("allocs", "id", alloc.ID) 1384 if err != nil { 1385 return fmt.Errorf("alloc lookup failed: %v", err) 1386 } 1387 1388 // Nothing to do if this does not exist 1389 if existing == nil { 1390 return nil 1391 } 1392 exist := existing.(*structs.Allocation) 1393 1394 // Copy everything from the existing allocation 1395 copyAlloc := exist.Copy() 1396 1397 // Pull in anything the client is the authority on 1398 copyAlloc.ClientStatus = alloc.ClientStatus 1399 copyAlloc.ClientDescription = alloc.ClientDescription 1400 copyAlloc.TaskStates = alloc.TaskStates 1401 copyAlloc.DeploymentStatus = alloc.DeploymentStatus 1402 1403 // Update the modify index 1404 copyAlloc.ModifyIndex = index 1405 1406 // TODO TEST 1407 if err := s.updateDeploymentWithAlloc(index, copyAlloc, exist, txn); err != nil { 1408 return fmt.Errorf("error updating deployment: %v", err) 1409 } 1410 1411 if err := s.updateSummaryWithAlloc(index, copyAlloc, exist, txn); err != nil { 1412 return fmt.Errorf("error updating job summary: %v", err) 1413 } 1414 1415 // Update the allocation 1416 if err := txn.Insert("allocs", copyAlloc); err != nil { 1417 return fmt.Errorf("alloc insert failed: %v", err) 1418 } 1419 1420 // Set the job's status 1421 forceStatus := "" 1422 if !copyAlloc.TerminalStatus() { 1423 forceStatus = structs.JobStatusRunning 1424 } 1425 jobs := map[string]string{exist.JobID: forceStatus} 1426 if err := s.setJobStatuses(index, txn, jobs, false); err != nil { 1427 return fmt.Errorf("setting job status failed: %v", err) 1428 } 1429 return nil 1430 } 1431 1432 // UpsertAllocs is used to evict a set of allocations and allocate new ones at 1433 // the same time. 1434 func (s *StateStore) UpsertAllocs(index uint64, allocs []*structs.Allocation) error { 1435 txn := s.db.Txn(true) 1436 defer txn.Abort() 1437 if err := s.upsertAllocsImpl(index, allocs, txn); err != nil { 1438 return err 1439 } 1440 txn.Commit() 1441 return nil 1442 } 1443 1444 // upsertAllocs is the actual implementation of UpsertAllocs so that it may be 1445 // used with an existing transaction. 1446 func (s *StateStore) upsertAllocsImpl(index uint64, allocs []*structs.Allocation, txn *memdb.Txn) error { 1447 // Handle the allocations 1448 jobs := make(map[string]string, 1) 1449 for _, alloc := range allocs { 1450 existing, err := txn.First("allocs", "id", alloc.ID) 1451 if err != nil { 1452 return fmt.Errorf("alloc lookup failed: %v", err) 1453 } 1454 exist, _ := existing.(*structs.Allocation) 1455 1456 if exist == nil { 1457 alloc.CreateIndex = index 1458 alloc.ModifyIndex = index 1459 alloc.AllocModifyIndex = index 1460 1461 // Issue https://github.com/hashicorp/nomad/issues/2583 uncovered 1462 // the a race between a forced garbage collection and the scheduler 1463 // marking an allocation as terminal. The issue is that the 1464 // allocation from the scheduler has its job normalized and the FSM 1465 // will only denormalize if the allocation is not terminal. However 1466 // if the allocation is garbage collected, that will result in a 1467 // allocation being upserted for the first time without a job 1468 // attached. By returning an error here, it will cause the FSM to 1469 // error, causing the plan_apply to error and thus causing the 1470 // evaluation to be failed. This will force an index refresh that 1471 // should solve this issue. 1472 if alloc.Job == nil { 1473 return fmt.Errorf("attempting to upsert allocation %q without a job", alloc.ID) 1474 } 1475 } else { 1476 alloc.CreateIndex = exist.CreateIndex 1477 alloc.ModifyIndex = index 1478 alloc.AllocModifyIndex = index 1479 1480 // Keep the clients task states 1481 alloc.TaskStates = exist.TaskStates 1482 1483 // If the scheduler is marking this allocation as lost we do not 1484 // want to reuse the status of the existing allocation. 1485 if alloc.ClientStatus != structs.AllocClientStatusLost { 1486 alloc.ClientStatus = exist.ClientStatus 1487 alloc.ClientDescription = exist.ClientDescription 1488 } 1489 1490 // The job has been denormalized so re-attach the original job 1491 if alloc.Job == nil { 1492 alloc.Job = exist.Job 1493 } 1494 } 1495 1496 if err := s.updateDeploymentWithAlloc(index, alloc, exist, txn); err != nil { 1497 return fmt.Errorf("error updating deployment: %v", err) 1498 } 1499 1500 if err := s.updateSummaryWithAlloc(index, alloc, exist, txn); err != nil { 1501 return fmt.Errorf("error updating job summary: %v", err) 1502 } 1503 1504 // Create the EphemeralDisk if it's nil by adding up DiskMB from task resources. 1505 // COMPAT 0.4.1 -> 0.5 1506 if alloc.Job != nil { 1507 s.addEphemeralDiskToTaskGroups(alloc.Job) 1508 } 1509 1510 if err := txn.Insert("allocs", alloc); err != nil { 1511 return fmt.Errorf("alloc insert failed: %v", err) 1512 } 1513 1514 // If the allocation is running, force the job to running status. 1515 forceStatus := "" 1516 if !alloc.TerminalStatus() { 1517 forceStatus = structs.JobStatusRunning 1518 } 1519 jobs[alloc.JobID] = forceStatus 1520 } 1521 1522 // Update the indexes 1523 if err := txn.Insert("index", &IndexEntry{"allocs", index}); err != nil { 1524 return fmt.Errorf("index update failed: %v", err) 1525 } 1526 1527 // Set the job's status 1528 if err := s.setJobStatuses(index, txn, jobs, false); err != nil { 1529 return fmt.Errorf("setting job status failed: %v", err) 1530 } 1531 1532 return nil 1533 } 1534 1535 // AllocByID is used to lookup an allocation by its ID 1536 func (s *StateStore) AllocByID(ws memdb.WatchSet, id string) (*structs.Allocation, error) { 1537 txn := s.db.Txn(false) 1538 1539 watchCh, existing, err := txn.FirstWatch("allocs", "id", id) 1540 if err != nil { 1541 return nil, fmt.Errorf("alloc lookup failed: %v", err) 1542 } 1543 1544 ws.Add(watchCh) 1545 1546 if existing != nil { 1547 return existing.(*structs.Allocation), nil 1548 } 1549 return nil, nil 1550 } 1551 1552 // AllocsByIDPrefix is used to lookup allocs by prefix 1553 func (s *StateStore) AllocsByIDPrefix(ws memdb.WatchSet, id string) (memdb.ResultIterator, error) { 1554 txn := s.db.Txn(false) 1555 1556 iter, err := txn.Get("allocs", "id_prefix", id) 1557 if err != nil { 1558 return nil, fmt.Errorf("alloc lookup failed: %v", err) 1559 } 1560 1561 ws.Add(iter.WatchCh()) 1562 1563 return iter, nil 1564 } 1565 1566 // AllocsByNode returns all the allocations by node 1567 func (s *StateStore) AllocsByNode(ws memdb.WatchSet, node string) ([]*structs.Allocation, error) { 1568 txn := s.db.Txn(false) 1569 1570 // Get an iterator over the node allocations, using only the 1571 // node prefix which ignores the terminal status 1572 iter, err := txn.Get("allocs", "node_prefix", node) 1573 if err != nil { 1574 return nil, err 1575 } 1576 1577 ws.Add(iter.WatchCh()) 1578 1579 var out []*structs.Allocation 1580 for { 1581 raw := iter.Next() 1582 if raw == nil { 1583 break 1584 } 1585 out = append(out, raw.(*structs.Allocation)) 1586 } 1587 return out, nil 1588 } 1589 1590 // AllocsByNode returns all the allocations by node and terminal status 1591 func (s *StateStore) AllocsByNodeTerminal(ws memdb.WatchSet, node string, terminal bool) ([]*structs.Allocation, error) { 1592 txn := s.db.Txn(false) 1593 1594 // Get an iterator over the node allocations 1595 iter, err := txn.Get("allocs", "node", node, terminal) 1596 if err != nil { 1597 return nil, err 1598 } 1599 1600 ws.Add(iter.WatchCh()) 1601 1602 var out []*structs.Allocation 1603 for { 1604 raw := iter.Next() 1605 if raw == nil { 1606 break 1607 } 1608 out = append(out, raw.(*structs.Allocation)) 1609 } 1610 return out, nil 1611 } 1612 1613 // AllocsByJob returns all the allocations by job id 1614 func (s *StateStore) AllocsByJob(ws memdb.WatchSet, jobID string, all bool) ([]*structs.Allocation, error) { 1615 txn := s.db.Txn(false) 1616 1617 // Get the job 1618 var job *structs.Job 1619 rawJob, err := txn.First("jobs", "id", jobID) 1620 if err != nil { 1621 return nil, err 1622 } 1623 if rawJob != nil { 1624 job = rawJob.(*structs.Job) 1625 } 1626 1627 // Get an iterator over the node allocations 1628 iter, err := txn.Get("allocs", "job", jobID) 1629 if err != nil { 1630 return nil, err 1631 } 1632 1633 ws.Add(iter.WatchCh()) 1634 1635 var out []*structs.Allocation 1636 for { 1637 raw := iter.Next() 1638 if raw == nil { 1639 break 1640 } 1641 1642 alloc := raw.(*structs.Allocation) 1643 // If the allocation belongs to a job with the same ID but a different 1644 // create index and we are not getting all the allocations whose Jobs 1645 // matches the same Job ID then we skip it 1646 if !all && job != nil && alloc.Job.CreateIndex != job.CreateIndex { 1647 continue 1648 } 1649 out = append(out, raw.(*structs.Allocation)) 1650 } 1651 return out, nil 1652 } 1653 1654 // AllocsByEval returns all the allocations by eval id 1655 func (s *StateStore) AllocsByEval(ws memdb.WatchSet, evalID string) ([]*structs.Allocation, error) { 1656 txn := s.db.Txn(false) 1657 1658 // Get an iterator over the eval allocations 1659 iter, err := txn.Get("allocs", "eval", evalID) 1660 if err != nil { 1661 return nil, err 1662 } 1663 1664 ws.Add(iter.WatchCh()) 1665 1666 var out []*structs.Allocation 1667 for { 1668 raw := iter.Next() 1669 if raw == nil { 1670 break 1671 } 1672 out = append(out, raw.(*structs.Allocation)) 1673 } 1674 return out, nil 1675 } 1676 1677 // AllocsByDeployment returns all the allocations by deployment id 1678 func (s *StateStore) AllocsByDeployment(ws memdb.WatchSet, deploymentID string) ([]*structs.Allocation, error) { 1679 txn := s.db.Txn(false) 1680 1681 // Get an iterator over the deployments allocations 1682 iter, err := txn.Get("allocs", "deployment", deploymentID) 1683 if err != nil { 1684 return nil, err 1685 } 1686 1687 ws.Add(iter.WatchCh()) 1688 1689 var out []*structs.Allocation 1690 for { 1691 raw := iter.Next() 1692 if raw == nil { 1693 break 1694 } 1695 out = append(out, raw.(*structs.Allocation)) 1696 } 1697 return out, nil 1698 } 1699 1700 // Allocs returns an iterator over all the evaluations 1701 func (s *StateStore) Allocs(ws memdb.WatchSet) (memdb.ResultIterator, error) { 1702 txn := s.db.Txn(false) 1703 1704 // Walk the entire table 1705 iter, err := txn.Get("allocs", "id") 1706 if err != nil { 1707 return nil, err 1708 } 1709 1710 ws.Add(iter.WatchCh()) 1711 1712 return iter, nil 1713 } 1714 1715 // UpsertVaultAccessors is used to register a set of Vault Accessors 1716 func (s *StateStore) UpsertVaultAccessor(index uint64, accessors []*structs.VaultAccessor) error { 1717 txn := s.db.Txn(true) 1718 defer txn.Abort() 1719 1720 for _, accessor := range accessors { 1721 // Set the create index 1722 accessor.CreateIndex = index 1723 1724 // Insert the accessor 1725 if err := txn.Insert("vault_accessors", accessor); err != nil { 1726 return fmt.Errorf("accessor insert failed: %v", err) 1727 } 1728 } 1729 1730 if err := txn.Insert("index", &IndexEntry{"vault_accessors", index}); err != nil { 1731 return fmt.Errorf("index update failed: %v", err) 1732 } 1733 1734 txn.Commit() 1735 return nil 1736 } 1737 1738 // DeleteVaultAccessors is used to delete a set of Vault Accessors 1739 func (s *StateStore) DeleteVaultAccessors(index uint64, accessors []*structs.VaultAccessor) error { 1740 txn := s.db.Txn(true) 1741 defer txn.Abort() 1742 1743 // Lookup the accessor 1744 for _, accessor := range accessors { 1745 // Delete the accessor 1746 if err := txn.Delete("vault_accessors", accessor); err != nil { 1747 return fmt.Errorf("accessor delete failed: %v", err) 1748 } 1749 } 1750 1751 if err := txn.Insert("index", &IndexEntry{"vault_accessors", index}); err != nil { 1752 return fmt.Errorf("index update failed: %v", err) 1753 } 1754 1755 txn.Commit() 1756 return nil 1757 } 1758 1759 // VaultAccessor returns the given Vault accessor 1760 func (s *StateStore) VaultAccessor(ws memdb.WatchSet, accessor string) (*structs.VaultAccessor, error) { 1761 txn := s.db.Txn(false) 1762 1763 watchCh, existing, err := txn.FirstWatch("vault_accessors", "id", accessor) 1764 if err != nil { 1765 return nil, fmt.Errorf("accessor lookup failed: %v", err) 1766 } 1767 1768 ws.Add(watchCh) 1769 1770 if existing != nil { 1771 return existing.(*structs.VaultAccessor), nil 1772 } 1773 1774 return nil, nil 1775 } 1776 1777 // VaultAccessors returns an iterator of Vault accessors. 1778 func (s *StateStore) VaultAccessors(ws memdb.WatchSet) (memdb.ResultIterator, error) { 1779 txn := s.db.Txn(false) 1780 1781 iter, err := txn.Get("vault_accessors", "id") 1782 if err != nil { 1783 return nil, err 1784 } 1785 1786 ws.Add(iter.WatchCh()) 1787 1788 return iter, nil 1789 } 1790 1791 // VaultAccessorsByAlloc returns all the Vault accessors by alloc id 1792 func (s *StateStore) VaultAccessorsByAlloc(ws memdb.WatchSet, allocID string) ([]*structs.VaultAccessor, error) { 1793 txn := s.db.Txn(false) 1794 1795 // Get an iterator over the accessors 1796 iter, err := txn.Get("vault_accessors", "alloc_id", allocID) 1797 if err != nil { 1798 return nil, err 1799 } 1800 1801 ws.Add(iter.WatchCh()) 1802 1803 var out []*structs.VaultAccessor 1804 for { 1805 raw := iter.Next() 1806 if raw == nil { 1807 break 1808 } 1809 out = append(out, raw.(*structs.VaultAccessor)) 1810 } 1811 return out, nil 1812 } 1813 1814 // VaultAccessorsByNode returns all the Vault accessors by node id 1815 func (s *StateStore) VaultAccessorsByNode(ws memdb.WatchSet, nodeID string) ([]*structs.VaultAccessor, error) { 1816 txn := s.db.Txn(false) 1817 1818 // Get an iterator over the accessors 1819 iter, err := txn.Get("vault_accessors", "node_id", nodeID) 1820 if err != nil { 1821 return nil, err 1822 } 1823 1824 ws.Add(iter.WatchCh()) 1825 1826 var out []*structs.VaultAccessor 1827 for { 1828 raw := iter.Next() 1829 if raw == nil { 1830 break 1831 } 1832 out = append(out, raw.(*structs.VaultAccessor)) 1833 } 1834 return out, nil 1835 } 1836 1837 // UpdateDeploymentStatus is used to make deployment status updates and 1838 // potentially make a evaluation 1839 func (s *StateStore) UpdateDeploymentStatus(index uint64, req *structs.DeploymentStatusUpdateRequest) error { 1840 txn := s.db.Txn(true) 1841 defer txn.Abort() 1842 1843 if err := s.updateDeploymentStatusImpl(index, req.DeploymentUpdate, txn); err != nil { 1844 return err 1845 } 1846 1847 // Upsert the job if necessary 1848 if req.Job != nil { 1849 if err := s.upsertJobImpl(index, req.Job, false, txn); err != nil { 1850 return err 1851 } 1852 } 1853 1854 // Upsert the optional eval 1855 if req.Eval != nil { 1856 if err := s.nestedUpsertEval(txn, index, req.Eval); err != nil { 1857 return err 1858 } 1859 } 1860 1861 txn.Commit() 1862 return nil 1863 } 1864 1865 // updateDeploymentStatusImpl is used to make deployment status updates 1866 func (s *StateStore) updateDeploymentStatusImpl(index uint64, u *structs.DeploymentStatusUpdate, txn *memdb.Txn) error { 1867 // Retrieve deployment 1868 ws := memdb.NewWatchSet() 1869 deployment, err := s.deploymentByIDImpl(ws, u.DeploymentID, txn) 1870 if err != nil { 1871 return err 1872 } else if deployment == nil { 1873 return fmt.Errorf("Deployment ID %q couldn't be updated as it does not exist", u.DeploymentID) 1874 } else if !deployment.Active() { 1875 return fmt.Errorf("Deployment %q has terminal status %q:", deployment.ID, deployment.Status) 1876 } 1877 1878 // Apply the new status 1879 copy := deployment.Copy() 1880 copy.Status = u.Status 1881 copy.StatusDescription = u.StatusDescription 1882 copy.ModifyIndex = index 1883 1884 // Insert the deployment 1885 if err := txn.Insert("deployment", copy); err != nil { 1886 return err 1887 } 1888 1889 // Update the index 1890 if err := txn.Insert("index", &IndexEntry{"deployment", index}); err != nil { 1891 return fmt.Errorf("index update failed: %v", err) 1892 } 1893 1894 // If the deployment is being marked as complete, set the job to stable. 1895 if copy.Status == structs.DeploymentStatusSuccessful { 1896 if err := s.updateJobStabilityImpl(index, copy.JobID, copy.JobVersion, true, txn); err != nil { 1897 return fmt.Errorf("failed to update job stability: %v", err) 1898 } 1899 } 1900 1901 return nil 1902 } 1903 1904 // UpdateJobStability updates the stability of the given job and version to the 1905 // desired status. 1906 func (s *StateStore) UpdateJobStability(index uint64, jobID string, jobVersion uint64, stable bool) error { 1907 txn := s.db.Txn(true) 1908 defer txn.Abort() 1909 1910 if err := s.updateJobStabilityImpl(index, jobID, jobVersion, stable, txn); err != nil { 1911 return err 1912 } 1913 1914 txn.Commit() 1915 return nil 1916 } 1917 1918 // updateJobStabilityImpl updates the stability of the given job and version 1919 func (s *StateStore) updateJobStabilityImpl(index uint64, jobID string, jobVersion uint64, stable bool, txn *memdb.Txn) error { 1920 // Get the job that is referenced 1921 job, err := s.jobByIDAndVersionImpl(nil, jobID, jobVersion, txn) 1922 if err != nil { 1923 return err 1924 } 1925 1926 // Has already been cleared, nothing to do 1927 if job == nil { 1928 return nil 1929 } 1930 1931 // If the job already has the desired stability, nothing to do 1932 if job.Stable == stable { 1933 return nil 1934 } 1935 1936 copy := job.Copy() 1937 copy.Stable = stable 1938 return s.upsertJobImpl(index, copy, true, txn) 1939 } 1940 1941 // UpdateDeploymentPromotion is used to promote canaries in a deployment and 1942 // potentially make a evaluation 1943 func (s *StateStore) UpdateDeploymentPromotion(index uint64, req *structs.ApplyDeploymentPromoteRequest) error { 1944 txn := s.db.Txn(true) 1945 defer txn.Abort() 1946 1947 // Retrieve deployment and ensure it is not terminal and is active 1948 ws := memdb.NewWatchSet() 1949 deployment, err := s.deploymentByIDImpl(ws, req.DeploymentID, txn) 1950 if err != nil { 1951 return err 1952 } else if deployment == nil { 1953 return fmt.Errorf("Deployment ID %q couldn't be updated as it does not exist", req.DeploymentID) 1954 } else if !deployment.Active() { 1955 return fmt.Errorf("Deployment %q has terminal status %q:", deployment.ID, deployment.Status) 1956 } 1957 1958 // Retrieve effected allocations 1959 iter, err := txn.Get("allocs", "deployment", req.DeploymentID) 1960 if err != nil { 1961 return err 1962 } 1963 1964 groupIndex := make(map[string]struct{}, len(req.Groups)) 1965 for _, g := range req.Groups { 1966 groupIndex[g] = struct{}{} 1967 } 1968 1969 canaryIndex := make(map[string]struct{}, len(deployment.TaskGroups)) 1970 for _, state := range deployment.TaskGroups { 1971 for _, c := range state.PlacedCanaries { 1972 canaryIndex[c] = struct{}{} 1973 } 1974 } 1975 1976 var unhealthyErr multierror.Error 1977 for { 1978 raw := iter.Next() 1979 if raw == nil { 1980 break 1981 } 1982 1983 alloc := raw.(*structs.Allocation) 1984 1985 // Check that the alloc is a canary 1986 if _, ok := canaryIndex[alloc.ID]; !ok { 1987 continue 1988 } 1989 1990 // Check that the canary is part of a group being promoted 1991 if _, ok := groupIndex[alloc.TaskGroup]; !req.All && !ok { 1992 continue 1993 } 1994 1995 // Ensure the canaries are healthy 1996 if !alloc.DeploymentStatus.IsHealthy() { 1997 multierror.Append(&unhealthyErr, fmt.Errorf("Canary allocation %q for group %q is not healthy", alloc.ID, alloc.TaskGroup)) 1998 continue 1999 } 2000 } 2001 2002 if err := unhealthyErr.ErrorOrNil(); err != nil { 2003 return err 2004 } 2005 2006 // Update deployment 2007 copy := deployment.Copy() 2008 copy.ModifyIndex = index 2009 for tg, status := range copy.TaskGroups { 2010 _, ok := groupIndex[tg] 2011 if !req.All && !ok { 2012 continue 2013 } 2014 2015 status.Promoted = true 2016 } 2017 2018 // If the deployment no longer needs promotion, update its status 2019 if !copy.RequiresPromotion() && copy.Status == structs.DeploymentStatusRunning { 2020 copy.StatusDescription = structs.DeploymentStatusDescriptionRunning 2021 } 2022 2023 // Insert the deployment 2024 if err := s.upsertDeploymentImpl(index, copy, txn); err != nil { 2025 return err 2026 } 2027 2028 // Upsert the optional eval 2029 if req.Eval != nil { 2030 if err := s.nestedUpsertEval(txn, index, req.Eval); err != nil { 2031 return err 2032 } 2033 } 2034 2035 txn.Commit() 2036 return nil 2037 } 2038 2039 // UpdateDeploymentAllocHealth is used to update the health of allocations as 2040 // part of the deployment and potentially make a evaluation 2041 func (s *StateStore) UpdateDeploymentAllocHealth(index uint64, req *structs.ApplyDeploymentAllocHealthRequest) error { 2042 txn := s.db.Txn(true) 2043 defer txn.Abort() 2044 2045 // Retrieve deployment and ensure it is not terminal and is active 2046 ws := memdb.NewWatchSet() 2047 deployment, err := s.deploymentByIDImpl(ws, req.DeploymentID, txn) 2048 if err != nil { 2049 return err 2050 } else if deployment == nil { 2051 return fmt.Errorf("Deployment ID %q couldn't be updated as it does not exist", req.DeploymentID) 2052 } else if !deployment.Active() { 2053 return fmt.Errorf("Deployment %q has terminal status %q:", deployment.ID, deployment.Status) 2054 } 2055 2056 // Update the health status of each allocation 2057 if total := len(req.HealthyAllocationIDs) + len(req.UnhealthyAllocationIDs); total != 0 { 2058 setAllocHealth := func(id string, healthy bool) error { 2059 existing, err := txn.First("allocs", "id", id) 2060 if err != nil { 2061 return fmt.Errorf("alloc %q lookup failed: %v", id, err) 2062 } 2063 if existing == nil { 2064 return fmt.Errorf("unknown alloc %q", id) 2065 } 2066 2067 old := existing.(*structs.Allocation) 2068 if old.DeploymentID != req.DeploymentID { 2069 return fmt.Errorf("alloc %q is not part of deployment %q", id, req.DeploymentID) 2070 } 2071 2072 // Set the health 2073 copy := old.Copy() 2074 if copy.DeploymentStatus == nil { 2075 copy.DeploymentStatus = &structs.AllocDeploymentStatus{} 2076 } 2077 copy.DeploymentStatus.Healthy = helper.BoolToPtr(healthy) 2078 copy.DeploymentStatus.ModifyIndex = index 2079 2080 if err := s.updateDeploymentWithAlloc(index, copy, old, txn); err != nil { 2081 return fmt.Errorf("error updating deployment: %v", err) 2082 } 2083 2084 if err := txn.Insert("allocs", copy); err != nil { 2085 return fmt.Errorf("alloc insert failed: %v", err) 2086 } 2087 2088 return nil 2089 } 2090 2091 for _, id := range req.HealthyAllocationIDs { 2092 if err := setAllocHealth(id, true); err != nil { 2093 return err 2094 } 2095 } 2096 for _, id := range req.UnhealthyAllocationIDs { 2097 if err := setAllocHealth(id, false); err != nil { 2098 return err 2099 } 2100 } 2101 2102 // Update the indexes 2103 if err := txn.Insert("index", &IndexEntry{"allocs", index}); err != nil { 2104 return fmt.Errorf("index update failed: %v", err) 2105 } 2106 } 2107 2108 // Update the deployment status as needed. 2109 if req.DeploymentUpdate != nil { 2110 if err := s.updateDeploymentStatusImpl(index, req.DeploymentUpdate, txn); err != nil { 2111 return err 2112 } 2113 } 2114 2115 // Upsert the job if necessary 2116 if req.Job != nil { 2117 if err := s.upsertJobImpl(index, req.Job, false, txn); err != nil { 2118 return err 2119 } 2120 } 2121 2122 // Upsert the optional eval 2123 if req.Eval != nil { 2124 if err := s.nestedUpsertEval(txn, index, req.Eval); err != nil { 2125 return err 2126 } 2127 } 2128 2129 txn.Commit() 2130 return nil 2131 } 2132 2133 // LastIndex returns the greatest index value for all indexes 2134 func (s *StateStore) LatestIndex() (uint64, error) { 2135 indexes, err := s.Indexes() 2136 if err != nil { 2137 return 0, err 2138 } 2139 2140 var max uint64 = 0 2141 for { 2142 raw := indexes.Next() 2143 if raw == nil { 2144 break 2145 } 2146 2147 // Prepare the request struct 2148 idx := raw.(*IndexEntry) 2149 2150 // Determine the max 2151 if idx.Value > max { 2152 max = idx.Value 2153 } 2154 } 2155 2156 return max, nil 2157 } 2158 2159 // Index finds the matching index value 2160 func (s *StateStore) Index(name string) (uint64, error) { 2161 txn := s.db.Txn(false) 2162 2163 // Lookup the first matching index 2164 out, err := txn.First("index", "id", name) 2165 if err != nil { 2166 return 0, err 2167 } 2168 if out == nil { 2169 return 0, nil 2170 } 2171 return out.(*IndexEntry).Value, nil 2172 } 2173 2174 // RemoveIndex is a helper method to remove an index for testing purposes 2175 func (s *StateStore) RemoveIndex(name string) error { 2176 txn := s.db.Txn(true) 2177 defer txn.Abort() 2178 2179 if _, err := txn.DeleteAll("index", "id", name); err != nil { 2180 return err 2181 } 2182 2183 txn.Commit() 2184 return nil 2185 } 2186 2187 // Indexes returns an iterator over all the indexes 2188 func (s *StateStore) Indexes() (memdb.ResultIterator, error) { 2189 txn := s.db.Txn(false) 2190 2191 // Walk the entire nodes table 2192 iter, err := txn.Get("index", "id") 2193 if err != nil { 2194 return nil, err 2195 } 2196 return iter, nil 2197 } 2198 2199 // ReconcileJobSummaries re-creates summaries for all jobs present in the state 2200 // store 2201 func (s *StateStore) ReconcileJobSummaries(index uint64) error { 2202 txn := s.db.Txn(true) 2203 defer txn.Abort() 2204 2205 // Get all the jobs 2206 iter, err := txn.Get("jobs", "id") 2207 if err != nil { 2208 return err 2209 } 2210 for { 2211 rawJob := iter.Next() 2212 if rawJob == nil { 2213 break 2214 } 2215 job := rawJob.(*structs.Job) 2216 2217 // Create a job summary for the job 2218 summary := &structs.JobSummary{ 2219 JobID: job.ID, 2220 Summary: make(map[string]structs.TaskGroupSummary), 2221 } 2222 for _, tg := range job.TaskGroups { 2223 summary.Summary[tg.Name] = structs.TaskGroupSummary{} 2224 } 2225 2226 // Find all the allocations for the jobs 2227 iterAllocs, err := txn.Get("allocs", "job", job.ID) 2228 if err != nil { 2229 return err 2230 } 2231 2232 // Calculate the summary for the job 2233 for { 2234 rawAlloc := iterAllocs.Next() 2235 if rawAlloc == nil { 2236 break 2237 } 2238 alloc := rawAlloc.(*structs.Allocation) 2239 2240 // Ignore the allocation if it doesn't belong to the currently 2241 // registered job. The allocation is checked because of issue #2304 2242 if alloc.Job == nil || alloc.Job.CreateIndex != job.CreateIndex { 2243 continue 2244 } 2245 2246 tg := summary.Summary[alloc.TaskGroup] 2247 switch alloc.ClientStatus { 2248 case structs.AllocClientStatusFailed: 2249 tg.Failed += 1 2250 case structs.AllocClientStatusLost: 2251 tg.Lost += 1 2252 case structs.AllocClientStatusComplete: 2253 tg.Complete += 1 2254 case structs.AllocClientStatusRunning: 2255 tg.Running += 1 2256 case structs.AllocClientStatusPending: 2257 tg.Starting += 1 2258 default: 2259 s.logger.Printf("[ERR] state_store: invalid client status: %v in allocation %q", alloc.ClientStatus, alloc.ID) 2260 } 2261 summary.Summary[alloc.TaskGroup] = tg 2262 } 2263 2264 // Set the create index of the summary same as the job's create index 2265 // and the modify index to the current index 2266 summary.CreateIndex = job.CreateIndex 2267 summary.ModifyIndex = index 2268 2269 // Insert the job summary 2270 if err := txn.Insert("job_summary", summary); err != nil { 2271 return fmt.Errorf("error inserting job summary: %v", err) 2272 } 2273 } 2274 2275 // Update the indexes table for job summary 2276 if err := txn.Insert("index", &IndexEntry{"job_summary", index}); err != nil { 2277 return fmt.Errorf("index update failed: %v", err) 2278 } 2279 txn.Commit() 2280 return nil 2281 } 2282 2283 // setJobStatuses is a helper for calling setJobStatus on multiple jobs by ID. 2284 // It takes a map of job IDs to an optional forceStatus string. It returns an 2285 // error if the job doesn't exist or setJobStatus fails. 2286 func (s *StateStore) setJobStatuses(index uint64, txn *memdb.Txn, 2287 jobs map[string]string, evalDelete bool) error { 2288 for job, forceStatus := range jobs { 2289 existing, err := txn.First("jobs", "id", job) 2290 if err != nil { 2291 return fmt.Errorf("job lookup failed: %v", err) 2292 } 2293 2294 if existing == nil { 2295 continue 2296 } 2297 2298 if err := s.setJobStatus(index, txn, existing.(*structs.Job), evalDelete, forceStatus); err != nil { 2299 return err 2300 } 2301 } 2302 2303 return nil 2304 } 2305 2306 // setJobStatus sets the status of the job by looking up associated evaluations 2307 // and allocations. evalDelete should be set to true if setJobStatus is being 2308 // called because an evaluation is being deleted (potentially because of garbage 2309 // collection). If forceStatus is non-empty, the job's status will be set to the 2310 // passed status. 2311 func (s *StateStore) setJobStatus(index uint64, txn *memdb.Txn, 2312 job *structs.Job, evalDelete bool, forceStatus string) error { 2313 2314 // Capture the current status so we can check if there is a change 2315 oldStatus := job.Status 2316 if index == job.CreateIndex { 2317 oldStatus = "" 2318 } 2319 newStatus := forceStatus 2320 2321 // If forceStatus is not set, compute the jobs status. 2322 if forceStatus == "" { 2323 var err error 2324 newStatus, err = s.getJobStatus(txn, job, evalDelete) 2325 if err != nil { 2326 return err 2327 } 2328 } 2329 2330 // Fast-path if nothing has changed. 2331 if oldStatus == newStatus { 2332 return nil 2333 } 2334 2335 // Copy and update the existing job 2336 updated := job.Copy() 2337 updated.Status = newStatus 2338 updated.ModifyIndex = index 2339 2340 // Insert the job 2341 if err := txn.Insert("jobs", updated); err != nil { 2342 return fmt.Errorf("job insert failed: %v", err) 2343 } 2344 if err := txn.Insert("index", &IndexEntry{"jobs", index}); err != nil { 2345 return fmt.Errorf("index update failed: %v", err) 2346 } 2347 2348 // Update the children summary 2349 if updated.ParentID != "" { 2350 // Try to update the summary of the parent job summary 2351 summaryRaw, err := txn.First("job_summary", "id", updated.ParentID) 2352 if err != nil { 2353 return fmt.Errorf("unable to retrieve summary for parent job: %v", err) 2354 } 2355 2356 // Only continue if the summary exists. It could not exist if the parent 2357 // job was removed 2358 if summaryRaw != nil { 2359 existing := summaryRaw.(*structs.JobSummary) 2360 pSummary := existing.Copy() 2361 if pSummary.Children == nil { 2362 pSummary.Children = new(structs.JobChildrenSummary) 2363 } 2364 2365 // Determine the transistion and update the correct fields 2366 children := pSummary.Children 2367 2368 // Decrement old status 2369 if oldStatus != "" { 2370 switch oldStatus { 2371 case structs.JobStatusPending: 2372 children.Pending-- 2373 case structs.JobStatusRunning: 2374 children.Running-- 2375 case structs.JobStatusDead: 2376 children.Dead-- 2377 default: 2378 return fmt.Errorf("unknown old job status %q", oldStatus) 2379 } 2380 } 2381 2382 // Increment new status 2383 switch newStatus { 2384 case structs.JobStatusPending: 2385 children.Pending++ 2386 case structs.JobStatusRunning: 2387 children.Running++ 2388 case structs.JobStatusDead: 2389 children.Dead++ 2390 default: 2391 return fmt.Errorf("unknown new job status %q", newStatus) 2392 } 2393 2394 // Update the index 2395 pSummary.ModifyIndex = index 2396 2397 // Insert the summary 2398 if err := txn.Insert("job_summary", pSummary); err != nil { 2399 return fmt.Errorf("job summary insert failed: %v", err) 2400 } 2401 if err := txn.Insert("index", &IndexEntry{"job_summary", index}); err != nil { 2402 return fmt.Errorf("index update failed: %v", err) 2403 } 2404 } 2405 } 2406 2407 return nil 2408 } 2409 2410 func (s *StateStore) getJobStatus(txn *memdb.Txn, job *structs.Job, evalDelete bool) (string, error) { 2411 allocs, err := txn.Get("allocs", "job", job.ID) 2412 if err != nil { 2413 return "", err 2414 } 2415 2416 // If there is a non-terminal allocation, the job is running. 2417 hasAlloc := false 2418 for alloc := allocs.Next(); alloc != nil; alloc = allocs.Next() { 2419 hasAlloc = true 2420 if !alloc.(*structs.Allocation).TerminalStatus() { 2421 return structs.JobStatusRunning, nil 2422 } 2423 } 2424 2425 evals, err := txn.Get("evals", "job_prefix", job.ID) 2426 if err != nil { 2427 return "", err 2428 } 2429 2430 hasEval := false 2431 for raw := evals.Next(); raw != nil; raw = evals.Next() { 2432 e := raw.(*structs.Evaluation) 2433 2434 // Filter non-exact matches 2435 if e.JobID != job.ID { 2436 continue 2437 } 2438 2439 hasEval = true 2440 if !e.TerminalStatus() { 2441 return structs.JobStatusPending, nil 2442 } 2443 } 2444 2445 // system jobs are running until explicitly stopped (which is handled elsewhere) 2446 if job.Type == structs.JobTypeSystem { 2447 if job.Stop { 2448 return structs.JobStatusDead, nil 2449 } 2450 2451 // Pending until at least one eval has completed 2452 return structs.JobStatusRunning, nil 2453 } 2454 2455 // The job is dead if all the allocations and evals are terminal or if there 2456 // are no evals because of garbage collection. 2457 if evalDelete || hasEval || hasAlloc { 2458 return structs.JobStatusDead, nil 2459 } 2460 2461 // If there are no allocations or evaluations it is a new job. If the 2462 // job is periodic or is a parameterized job, we mark it as running as 2463 // it will never have an allocation/evaluation against it. 2464 if job.IsPeriodic() || job.IsParameterized() { 2465 // If the job is stopped mark it as dead 2466 if job.Stop { 2467 return structs.JobStatusDead, nil 2468 } 2469 2470 return structs.JobStatusRunning, nil 2471 } 2472 return structs.JobStatusPending, nil 2473 } 2474 2475 // updateSummaryWithJob creates or updates job summaries when new jobs are 2476 // upserted or existing ones are updated 2477 func (s *StateStore) updateSummaryWithJob(index uint64, job *structs.Job, 2478 txn *memdb.Txn) error { 2479 2480 // Update the job summary 2481 summaryRaw, err := txn.First("job_summary", "id", job.ID) 2482 if err != nil { 2483 return fmt.Errorf("job summary lookup failed: %v", err) 2484 } 2485 2486 // Get the summary or create if necessary 2487 var summary *structs.JobSummary 2488 hasSummaryChanged := false 2489 if summaryRaw != nil { 2490 summary = summaryRaw.(*structs.JobSummary).Copy() 2491 } else { 2492 summary = &structs.JobSummary{ 2493 JobID: job.ID, 2494 Summary: make(map[string]structs.TaskGroupSummary), 2495 Children: new(structs.JobChildrenSummary), 2496 CreateIndex: index, 2497 } 2498 hasSummaryChanged = true 2499 } 2500 2501 for _, tg := range job.TaskGroups { 2502 if _, ok := summary.Summary[tg.Name]; !ok { 2503 newSummary := structs.TaskGroupSummary{ 2504 Complete: 0, 2505 Failed: 0, 2506 Running: 0, 2507 Starting: 0, 2508 } 2509 summary.Summary[tg.Name] = newSummary 2510 hasSummaryChanged = true 2511 } 2512 } 2513 2514 // The job summary has changed, so update the modify index. 2515 if hasSummaryChanged { 2516 summary.ModifyIndex = index 2517 2518 // Update the indexes table for job summary 2519 if err := txn.Insert("index", &IndexEntry{"job_summary", index}); err != nil { 2520 return fmt.Errorf("index update failed: %v", err) 2521 } 2522 if err := txn.Insert("job_summary", summary); err != nil { 2523 return err 2524 } 2525 } 2526 2527 return nil 2528 } 2529 2530 // updateDeploymentWithAlloc is used to update the deployment state associated 2531 // with the given allocation. The passed alloc may be updated if the deployment 2532 // status has changed to capture the modify index at which it has changed. 2533 func (s *StateStore) updateDeploymentWithAlloc(index uint64, alloc, existing *structs.Allocation, txn *memdb.Txn) error { 2534 // Nothing to do if the allocation is not associated with a deployment 2535 if alloc.DeploymentID == "" { 2536 return nil 2537 } 2538 2539 // Get the deployment 2540 ws := memdb.NewWatchSet() 2541 deployment, err := s.deploymentByIDImpl(ws, alloc.DeploymentID, txn) 2542 if err != nil { 2543 return err 2544 } 2545 if deployment == nil { 2546 return nil 2547 } 2548 2549 // Retrieve the deployment state object 2550 _, ok := deployment.TaskGroups[alloc.TaskGroup] 2551 if !ok { 2552 // If the task group isn't part of the deployment, the task group wasn't 2553 // part of a rolling update so nothing to do 2554 return nil 2555 } 2556 2557 // Do not modify in-place. Instead keep track of what must be done 2558 placed := 0 2559 healthy := 0 2560 unhealthy := 0 2561 2562 // If there was no existing allocation, this is a placement and we increment 2563 // the placement 2564 existingHealthSet := existing != nil && existing.DeploymentStatus != nil && existing.DeploymentStatus.Healthy != nil 2565 allocHealthSet := alloc.DeploymentStatus != nil && alloc.DeploymentStatus.Healthy != nil 2566 if existing == nil || existing.DeploymentID != alloc.DeploymentID { 2567 placed++ 2568 } else if !existingHealthSet && allocHealthSet { 2569 if *alloc.DeploymentStatus.Healthy { 2570 healthy++ 2571 } else { 2572 unhealthy++ 2573 } 2574 } else if existingHealthSet && allocHealthSet { 2575 // See if it has gone from healthy to unhealthy 2576 if *existing.DeploymentStatus.Healthy && !*alloc.DeploymentStatus.Healthy { 2577 healthy-- 2578 unhealthy++ 2579 } 2580 } 2581 2582 // Nothing to do 2583 if placed == 0 && healthy == 0 && unhealthy == 0 { 2584 return nil 2585 } 2586 2587 // Update the allocation's deployment status modify index 2588 if alloc.DeploymentStatus != nil && healthy+unhealthy != 0 { 2589 alloc.DeploymentStatus.ModifyIndex = index 2590 } 2591 2592 // Create a copy of the deployment object 2593 deploymentCopy := deployment.Copy() 2594 deploymentCopy.ModifyIndex = index 2595 2596 state := deploymentCopy.TaskGroups[alloc.TaskGroup] 2597 state.PlacedAllocs += placed 2598 state.HealthyAllocs += healthy 2599 state.UnhealthyAllocs += unhealthy 2600 2601 // Upsert the deployment 2602 if err := s.upsertDeploymentImpl(index, deploymentCopy, txn); err != nil { 2603 return err 2604 } 2605 2606 return nil 2607 } 2608 2609 // updateSummaryWithAlloc updates the job summary when allocations are updated 2610 // or inserted 2611 func (s *StateStore) updateSummaryWithAlloc(index uint64, alloc *structs.Allocation, 2612 existingAlloc *structs.Allocation, txn *memdb.Txn) error { 2613 2614 // We don't have to update the summary if the job is missing 2615 if alloc.Job == nil { 2616 return nil 2617 } 2618 2619 summaryRaw, err := txn.First("job_summary", "id", alloc.JobID) 2620 if err != nil { 2621 return fmt.Errorf("unable to lookup job summary for job id %q: %v", alloc.JobID, err) 2622 } 2623 2624 if summaryRaw == nil { 2625 // Check if the job is de-registered 2626 rawJob, err := txn.First("jobs", "id", alloc.JobID) 2627 if err != nil { 2628 return fmt.Errorf("unable to query job: %v", err) 2629 } 2630 2631 // If the job is de-registered then we skip updating it's summary 2632 if rawJob == nil { 2633 return nil 2634 } 2635 2636 return fmt.Errorf("job summary for job %q is not present", alloc.JobID) 2637 } 2638 2639 // Get a copy of the existing summary 2640 jobSummary := summaryRaw.(*structs.JobSummary).Copy() 2641 2642 // Not updating the job summary because the allocation doesn't belong to the 2643 // currently registered job 2644 if jobSummary.CreateIndex != alloc.Job.CreateIndex { 2645 return nil 2646 } 2647 2648 tgSummary, ok := jobSummary.Summary[alloc.TaskGroup] 2649 if !ok { 2650 return fmt.Errorf("unable to find task group in the job summary: %v", alloc.TaskGroup) 2651 } 2652 2653 summaryChanged := false 2654 if existingAlloc == nil { 2655 switch alloc.DesiredStatus { 2656 case structs.AllocDesiredStatusStop, structs.AllocDesiredStatusEvict: 2657 s.logger.Printf("[ERR] state_store: new allocation inserted into state store with id: %v and state: %v", 2658 alloc.ID, alloc.DesiredStatus) 2659 } 2660 switch alloc.ClientStatus { 2661 case structs.AllocClientStatusPending: 2662 tgSummary.Starting += 1 2663 if tgSummary.Queued > 0 { 2664 tgSummary.Queued -= 1 2665 } 2666 summaryChanged = true 2667 case structs.AllocClientStatusRunning, structs.AllocClientStatusFailed, 2668 structs.AllocClientStatusComplete: 2669 s.logger.Printf("[ERR] state_store: new allocation inserted into state store with id: %v and state: %v", 2670 alloc.ID, alloc.ClientStatus) 2671 } 2672 } else if existingAlloc.ClientStatus != alloc.ClientStatus { 2673 // Incrementing the client of the bin of the current state 2674 switch alloc.ClientStatus { 2675 case structs.AllocClientStatusRunning: 2676 tgSummary.Running += 1 2677 case structs.AllocClientStatusFailed: 2678 tgSummary.Failed += 1 2679 case structs.AllocClientStatusPending: 2680 tgSummary.Starting += 1 2681 case structs.AllocClientStatusComplete: 2682 tgSummary.Complete += 1 2683 case structs.AllocClientStatusLost: 2684 tgSummary.Lost += 1 2685 } 2686 2687 // Decrementing the count of the bin of the last state 2688 switch existingAlloc.ClientStatus { 2689 case structs.AllocClientStatusRunning: 2690 tgSummary.Running -= 1 2691 case structs.AllocClientStatusPending: 2692 tgSummary.Starting -= 1 2693 case structs.AllocClientStatusLost: 2694 tgSummary.Lost -= 1 2695 case structs.AllocClientStatusFailed, structs.AllocClientStatusComplete: 2696 default: 2697 s.logger.Printf("[ERR] state_store: invalid old state of allocation with id: %v, and state: %v", 2698 existingAlloc.ID, existingAlloc.ClientStatus) 2699 } 2700 summaryChanged = true 2701 } 2702 jobSummary.Summary[alloc.TaskGroup] = tgSummary 2703 2704 if summaryChanged { 2705 jobSummary.ModifyIndex = index 2706 2707 // Update the indexes table for job summary 2708 if err := txn.Insert("index", &IndexEntry{"job_summary", index}); err != nil { 2709 return fmt.Errorf("index update failed: %v", err) 2710 } 2711 2712 if err := txn.Insert("job_summary", jobSummary); err != nil { 2713 return fmt.Errorf("updating job summary failed: %v", err) 2714 } 2715 } 2716 2717 return nil 2718 } 2719 2720 // addEphemeralDiskToTaskGroups adds missing EphemeralDisk objects to TaskGroups 2721 func (s *StateStore) addEphemeralDiskToTaskGroups(job *structs.Job) { 2722 for _, tg := range job.TaskGroups { 2723 var diskMB int 2724 for _, task := range tg.Tasks { 2725 if task.Resources != nil { 2726 diskMB += task.Resources.DiskMB 2727 task.Resources.DiskMB = 0 2728 } 2729 } 2730 if tg.EphemeralDisk != nil { 2731 continue 2732 } 2733 tg.EphemeralDisk = &structs.EphemeralDisk{ 2734 SizeMB: diskMB, 2735 } 2736 } 2737 } 2738 2739 // StateSnapshot is used to provide a point-in-time snapshot 2740 type StateSnapshot struct { 2741 StateStore 2742 } 2743 2744 // StateRestore is used to optimize the performance when 2745 // restoring state by only using a single large transaction 2746 // instead of thousands of sub transactions 2747 type StateRestore struct { 2748 txn *memdb.Txn 2749 } 2750 2751 // Abort is used to abort the restore operation 2752 func (s *StateRestore) Abort() { 2753 s.txn.Abort() 2754 } 2755 2756 // Commit is used to commit the restore operation 2757 func (s *StateRestore) Commit() { 2758 s.txn.Commit() 2759 } 2760 2761 // NodeRestore is used to restore a node 2762 func (r *StateRestore) NodeRestore(node *structs.Node) error { 2763 if err := r.txn.Insert("nodes", node); err != nil { 2764 return fmt.Errorf("node insert failed: %v", err) 2765 } 2766 return nil 2767 } 2768 2769 // JobRestore is used to restore a job 2770 func (r *StateRestore) JobRestore(job *structs.Job) error { 2771 // Create the EphemeralDisk if it's nil by adding up DiskMB from task resources. 2772 // COMPAT 0.4.1 -> 0.5 2773 r.addEphemeralDiskToTaskGroups(job) 2774 2775 if err := r.txn.Insert("jobs", job); err != nil { 2776 return fmt.Errorf("job insert failed: %v", err) 2777 } 2778 return nil 2779 } 2780 2781 // EvalRestore is used to restore an evaluation 2782 func (r *StateRestore) EvalRestore(eval *structs.Evaluation) error { 2783 if err := r.txn.Insert("evals", eval); err != nil { 2784 return fmt.Errorf("eval insert failed: %v", err) 2785 } 2786 return nil 2787 } 2788 2789 // AllocRestore is used to restore an allocation 2790 func (r *StateRestore) AllocRestore(alloc *structs.Allocation) error { 2791 // Set the shared resources if it's not present 2792 // COMPAT 0.4.1 -> 0.5 2793 if alloc.SharedResources == nil { 2794 alloc.SharedResources = &structs.Resources{ 2795 DiskMB: alloc.Resources.DiskMB, 2796 } 2797 } 2798 2799 // Create the EphemeralDisk if it's nil by adding up DiskMB from task resources. 2800 if alloc.Job != nil { 2801 r.addEphemeralDiskToTaskGroups(alloc.Job) 2802 } 2803 2804 if err := r.txn.Insert("allocs", alloc); err != nil { 2805 return fmt.Errorf("alloc insert failed: %v", err) 2806 } 2807 return nil 2808 } 2809 2810 // IndexRestore is used to restore an index 2811 func (r *StateRestore) IndexRestore(idx *IndexEntry) error { 2812 if err := r.txn.Insert("index", idx); err != nil { 2813 return fmt.Errorf("index insert failed: %v", err) 2814 } 2815 return nil 2816 } 2817 2818 // PeriodicLaunchRestore is used to restore a periodic launch. 2819 func (r *StateRestore) PeriodicLaunchRestore(launch *structs.PeriodicLaunch) error { 2820 if err := r.txn.Insert("periodic_launch", launch); err != nil { 2821 return fmt.Errorf("periodic launch insert failed: %v", err) 2822 } 2823 return nil 2824 } 2825 2826 // JobSummaryRestore is used to restore a job summary 2827 func (r *StateRestore) JobSummaryRestore(jobSummary *structs.JobSummary) error { 2828 if err := r.txn.Insert("job_summary", jobSummary); err != nil { 2829 return fmt.Errorf("job summary insert failed: %v", err) 2830 } 2831 return nil 2832 } 2833 2834 // JobVersionRestore is used to restore a job version 2835 func (r *StateRestore) JobVersionRestore(version *structs.Job) error { 2836 if err := r.txn.Insert("job_version", version); err != nil { 2837 return fmt.Errorf("job version insert failed: %v", err) 2838 } 2839 return nil 2840 } 2841 2842 // DeploymentRestore is used to restore a deployment 2843 func (r *StateRestore) DeploymentRestore(deployment *structs.Deployment) error { 2844 if err := r.txn.Insert("deployment", deployment); err != nil { 2845 return fmt.Errorf("deployment insert failed: %v", err) 2846 } 2847 return nil 2848 } 2849 2850 // VaultAccessorRestore is used to restore a vault accessor 2851 func (r *StateRestore) VaultAccessorRestore(accessor *structs.VaultAccessor) error { 2852 if err := r.txn.Insert("vault_accessors", accessor); err != nil { 2853 return fmt.Errorf("vault accessor insert failed: %v", err) 2854 } 2855 return nil 2856 } 2857 2858 // addEphemeralDiskToTaskGroups adds missing EphemeralDisk objects to TaskGroups 2859 func (r *StateRestore) addEphemeralDiskToTaskGroups(job *structs.Job) { 2860 for _, tg := range job.TaskGroups { 2861 if tg.EphemeralDisk != nil { 2862 continue 2863 } 2864 var sizeMB int 2865 for _, task := range tg.Tasks { 2866 if task.Resources != nil { 2867 sizeMB += task.Resources.DiskMB 2868 task.Resources.DiskMB = 0 2869 } 2870 } 2871 tg.EphemeralDisk = &structs.EphemeralDisk{ 2872 SizeMB: sizeMB, 2873 } 2874 } 2875 }