gopkg.in/hashicorp/nomad.v0@v0.11.8/nomad/state/state_store.go (about) 1 package state 2 3 import ( 4 "context" 5 "fmt" 6 "reflect" 7 "sort" 8 "time" 9 10 log "github.com/hashicorp/go-hclog" 11 memdb "github.com/hashicorp/go-memdb" 12 multierror "github.com/hashicorp/go-multierror" 13 "github.com/pkg/errors" 14 15 "github.com/hashicorp/nomad/helper" 16 "github.com/hashicorp/nomad/nomad/structs" 17 ) 18 19 // Txn is a transaction against a state store. 20 // This can be a read or write transaction. 21 type Txn = *memdb.Txn 22 23 const ( 24 // NodeRegisterEventReregistered is the message used when the node becomes 25 // reregistered. 26 NodeRegisterEventRegistered = "Node registered" 27 28 // NodeRegisterEventReregistered is the message used when the node becomes 29 // reregistered. 30 NodeRegisterEventReregistered = "Node re-registered" 31 ) 32 33 // IndexEntry is used with the "index" table 34 // for managing the latest Raft index affecting a table. 35 type IndexEntry struct { 36 Key string 37 Value uint64 38 } 39 40 // StateStoreConfig is used to configure a new state store 41 type StateStoreConfig struct { 42 // Logger is used to output the state store's logs 43 Logger log.Logger 44 45 // Region is the region of the server embedding the state store. 46 Region string 47 } 48 49 // The StateStore is responsible for maintaining all the Nomad 50 // state. It is manipulated by the FSM which maintains consistency 51 // through the use of Raft. The goals of the StateStore are to provide 52 // high concurrency for read operations without blocking writes, and 53 // to provide write availability in the face of reads. EVERY object 54 // returned as a result of a read against the state store should be 55 // considered a constant and NEVER modified in place. 56 type StateStore struct { 57 logger log.Logger 58 db *memdb.MemDB 59 60 // config is the passed in configuration 61 config *StateStoreConfig 62 63 // abandonCh is used to signal watchers that this state store has been 64 // abandoned (usually during a restore). This is only ever closed. 65 abandonCh chan struct{} 66 } 67 68 // NewStateStore is used to create a new state store 69 func NewStateStore(config *StateStoreConfig) (*StateStore, error) { 70 // Create the MemDB 71 db, err := memdb.NewMemDB(stateStoreSchema()) 72 if err != nil { 73 return nil, fmt.Errorf("state store setup failed: %v", err) 74 } 75 76 // Create the state store 77 s := &StateStore{ 78 logger: config.Logger.Named("state_store"), 79 db: db, 80 config: config, 81 abandonCh: make(chan struct{}), 82 } 83 return s, nil 84 } 85 86 // Config returns the state store configuration. 87 func (s *StateStore) Config() *StateStoreConfig { 88 return s.config 89 } 90 91 // Snapshot is used to create a point in time snapshot. Because 92 // we use MemDB, we just need to snapshot the state of the underlying 93 // database. 94 func (s *StateStore) Snapshot() (*StateSnapshot, error) { 95 snap := &StateSnapshot{ 96 StateStore: StateStore{ 97 logger: s.logger, 98 config: s.config, 99 db: s.db.Snapshot(), 100 }, 101 } 102 return snap, nil 103 } 104 105 // SnapshotMinIndex is used to create a state snapshot where the index is 106 // guaranteed to be greater than or equal to the index parameter. 107 // 108 // Some server operations (such as scheduling) exchange objects via RPC 109 // concurrent with Raft log application, so they must ensure the state store 110 // snapshot they are operating on is at or after the index the objects 111 // retrieved via RPC were applied to the Raft log at. 112 // 113 // Callers should maintain their own timer metric as the time this method 114 // blocks indicates Raft log application latency relative to scheduling. 115 func (s *StateStore) SnapshotMinIndex(ctx context.Context, index uint64) (*StateSnapshot, error) { 116 // Ported from work.go:waitForIndex prior to 0.9 117 118 const backoffBase = 20 * time.Millisecond 119 const backoffLimit = 1 * time.Second 120 var retries uint 121 var retryTimer *time.Timer 122 123 // XXX: Potential optimization is to set up a watch on the state 124 // store's index table and only unblock via a trigger rather than 125 // polling. 126 for { 127 // Get the states current index 128 snapshotIndex, err := s.LatestIndex() 129 if err != nil { 130 return nil, fmt.Errorf("failed to determine state store's index: %v", err) 131 } 132 133 // We only need the FSM state to be as recent as the given index 134 if snapshotIndex >= index { 135 return s.Snapshot() 136 } 137 138 // Exponential back off 139 retries++ 140 if retryTimer == nil { 141 // First retry, start at baseline 142 retryTimer = time.NewTimer(backoffBase) 143 } else { 144 // Subsequent retry, reset timer 145 deadline := 1 << (2 * retries) * backoffBase 146 if deadline > backoffLimit { 147 deadline = backoffLimit 148 } 149 retryTimer.Reset(deadline) 150 } 151 152 select { 153 case <-ctx.Done(): 154 return nil, ctx.Err() 155 case <-retryTimer.C: 156 } 157 } 158 } 159 160 // Restore is used to optimize the efficiency of rebuilding 161 // state by minimizing the number of transactions and checking 162 // overhead. 163 func (s *StateStore) Restore() (*StateRestore, error) { 164 txn := s.db.Txn(true) 165 r := &StateRestore{ 166 txn: txn, 167 } 168 return r, nil 169 } 170 171 // AbandonCh returns a channel you can wait on to know if the state store was 172 // abandoned. 173 func (s *StateStore) AbandonCh() <-chan struct{} { 174 return s.abandonCh 175 } 176 177 // Abandon is used to signal that the given state store has been abandoned. 178 // Calling this more than one time will panic. 179 func (s *StateStore) Abandon() { 180 close(s.abandonCh) 181 } 182 183 // QueryFn is the definition of a function that can be used to implement a basic 184 // blocking query against the state store. 185 type QueryFn func(memdb.WatchSet, *StateStore) (resp interface{}, index uint64, err error) 186 187 // BlockingQuery takes a query function and runs the function until the minimum 188 // query index is met or until the passed context is cancelled. 189 func (s *StateStore) BlockingQuery(query QueryFn, minIndex uint64, ctx context.Context) ( 190 resp interface{}, index uint64, err error) { 191 192 RUN_QUERY: 193 // We capture the state store and its abandon channel but pass a snapshot to 194 // the blocking query function. We operate on the snapshot to allow separate 195 // calls to the state store not all wrapped within the same transaction. 196 abandonCh := s.AbandonCh() 197 snap, _ := s.Snapshot() 198 stateSnap := &snap.StateStore 199 200 // We can skip all watch tracking if this isn't a blocking query. 201 var ws memdb.WatchSet 202 if minIndex > 0 { 203 ws = memdb.NewWatchSet() 204 205 // This channel will be closed if a snapshot is restored and the 206 // whole state store is abandoned. 207 ws.Add(abandonCh) 208 } 209 210 resp, index, err = query(ws, stateSnap) 211 if err != nil { 212 return nil, index, err 213 } 214 215 // We haven't reached the min-index yet. 216 if minIndex > 0 && index <= minIndex { 217 if err := ws.WatchCtx(ctx); err != nil { 218 return nil, index, err 219 } 220 221 goto RUN_QUERY 222 } 223 224 return resp, index, nil 225 } 226 227 // UpsertPlanResults is used to upsert the results of a plan. 228 func (s *StateStore) UpsertPlanResults(index uint64, results *structs.ApplyPlanResultsRequest) error { 229 snapshot, err := s.Snapshot() 230 if err != nil { 231 return err 232 } 233 234 allocsStopped, err := snapshot.DenormalizeAllocationDiffSlice(results.AllocsStopped) 235 if err != nil { 236 return err 237 } 238 239 allocsPreempted, err := snapshot.DenormalizeAllocationDiffSlice(results.AllocsPreempted) 240 if err != nil { 241 return err 242 } 243 244 // COMPAT 0.11: Remove this denormalization when NodePreemptions is removed 245 results.NodePreemptions, err = snapshot.DenormalizeAllocationSlice(results.NodePreemptions) 246 if err != nil { 247 return err 248 } 249 250 txn := s.db.Txn(true) 251 defer txn.Abort() 252 253 // Upsert the newly created or updated deployment 254 if results.Deployment != nil { 255 if err := s.upsertDeploymentImpl(index, results.Deployment, txn); err != nil { 256 return err 257 } 258 } 259 260 // Update the status of deployments effected by the plan. 261 if len(results.DeploymentUpdates) != 0 { 262 s.upsertDeploymentUpdates(index, results.DeploymentUpdates, txn) 263 } 264 265 if results.EvalID != "" { 266 // Update the modify index of the eval id 267 if err := s.updateEvalModifyIndex(txn, index, results.EvalID); err != nil { 268 return err 269 } 270 } 271 272 numAllocs := 0 273 if len(results.Alloc) > 0 || len(results.NodePreemptions) > 0 { 274 // COMPAT 0.11: This branch will be removed, when Alloc is removed 275 // Attach the job to all the allocations. It is pulled out in the payload to 276 // avoid the redundancy of encoding, but should be denormalized prior to 277 // being inserted into MemDB. 278 addComputedAllocAttrs(results.Alloc, results.Job) 279 numAllocs = len(results.Alloc) + len(results.NodePreemptions) 280 } else { 281 // Attach the job to all the allocations. It is pulled out in the payload to 282 // avoid the redundancy of encoding, but should be denormalized prior to 283 // being inserted into MemDB. 284 addComputedAllocAttrs(results.AllocsUpdated, results.Job) 285 numAllocs = len(allocsStopped) + len(results.AllocsUpdated) + len(allocsPreempted) 286 } 287 288 allocsToUpsert := make([]*structs.Allocation, 0, numAllocs) 289 290 // COMPAT 0.11: Both these appends should be removed when Alloc and NodePreemptions are removed 291 allocsToUpsert = append(allocsToUpsert, results.Alloc...) 292 allocsToUpsert = append(allocsToUpsert, results.NodePreemptions...) 293 294 allocsToUpsert = append(allocsToUpsert, allocsStopped...) 295 allocsToUpsert = append(allocsToUpsert, results.AllocsUpdated...) 296 allocsToUpsert = append(allocsToUpsert, allocsPreempted...) 297 298 // handle upgrade path 299 for _, alloc := range allocsToUpsert { 300 alloc.Canonicalize() 301 } 302 303 if err := s.upsertAllocsImpl(index, allocsToUpsert, txn); err != nil { 304 return err 305 } 306 307 // Upsert followup evals for allocs that were preempted 308 for _, eval := range results.PreemptionEvals { 309 if err := s.nestedUpsertEval(txn, index, eval); err != nil { 310 return err 311 } 312 } 313 314 txn.Commit() 315 return nil 316 } 317 318 // addComputedAllocAttrs adds the computed/derived attributes to the allocation. 319 // This method is used when an allocation is being denormalized. 320 func addComputedAllocAttrs(allocs []*structs.Allocation, job *structs.Job) { 321 structs.DenormalizeAllocationJobs(job, allocs) 322 323 // COMPAT(0.11): Remove in 0.11 324 // Calculate the total resources of allocations. It is pulled out in the 325 // payload to avoid encoding something that can be computed, but should be 326 // denormalized prior to being inserted into MemDB. 327 for _, alloc := range allocs { 328 if alloc.Resources != nil { 329 continue 330 } 331 332 alloc.Resources = new(structs.Resources) 333 for _, task := range alloc.TaskResources { 334 alloc.Resources.Add(task) 335 } 336 337 // Add the shared resources 338 alloc.Resources.Add(alloc.SharedResources) 339 } 340 } 341 342 // upsertDeploymentUpdates updates the deployments given the passed status 343 // updates. 344 func (s *StateStore) upsertDeploymentUpdates(index uint64, updates []*structs.DeploymentStatusUpdate, txn *memdb.Txn) error { 345 for _, u := range updates { 346 if err := s.updateDeploymentStatusImpl(index, u, txn); err != nil { 347 return err 348 } 349 } 350 351 return nil 352 } 353 354 // UpsertJobSummary upserts a job summary into the state store. 355 func (s *StateStore) UpsertJobSummary(index uint64, jobSummary *structs.JobSummary) error { 356 txn := s.db.Txn(true) 357 defer txn.Abort() 358 359 // Check if the job summary already exists 360 existing, err := txn.First("job_summary", "id", jobSummary.Namespace, jobSummary.JobID) 361 if err != nil { 362 return fmt.Errorf("job summary lookup failed: %v", err) 363 } 364 365 // Setup the indexes correctly 366 if existing != nil { 367 jobSummary.CreateIndex = existing.(*structs.JobSummary).CreateIndex 368 jobSummary.ModifyIndex = index 369 } else { 370 jobSummary.CreateIndex = index 371 jobSummary.ModifyIndex = index 372 } 373 374 // Update the index 375 if err := txn.Insert("job_summary", jobSummary); err != nil { 376 return err 377 } 378 379 // Update the indexes table for job summary 380 if err := txn.Insert("index", &IndexEntry{"job_summary", index}); err != nil { 381 return fmt.Errorf("index update failed: %v", err) 382 } 383 384 txn.Commit() 385 return nil 386 } 387 388 // DeleteJobSummary deletes the job summary with the given ID. This is for 389 // testing purposes only. 390 func (s *StateStore) DeleteJobSummary(index uint64, namespace, id string) error { 391 txn := s.db.Txn(true) 392 defer txn.Abort() 393 394 // Delete the job summary 395 if _, err := txn.DeleteAll("job_summary", "id", namespace, id); err != nil { 396 return fmt.Errorf("deleting job summary failed: %v", err) 397 } 398 if err := txn.Insert("index", &IndexEntry{"job_summary", index}); err != nil { 399 return fmt.Errorf("index update failed: %v", err) 400 } 401 txn.Commit() 402 return nil 403 } 404 405 // UpsertDeployment is used to insert a new deployment. If cancelPrior is set to 406 // true, all prior deployments for the same job will be cancelled. 407 func (s *StateStore) UpsertDeployment(index uint64, deployment *structs.Deployment) error { 408 txn := s.db.Txn(true) 409 defer txn.Abort() 410 if err := s.upsertDeploymentImpl(index, deployment, txn); err != nil { 411 return err 412 } 413 txn.Commit() 414 return nil 415 } 416 417 func (s *StateStore) upsertDeploymentImpl(index uint64, deployment *structs.Deployment, txn *memdb.Txn) error { 418 // Check if the deployment already exists 419 existing, err := txn.First("deployment", "id", deployment.ID) 420 if err != nil { 421 return fmt.Errorf("deployment lookup failed: %v", err) 422 } 423 424 // Setup the indexes correctly 425 if existing != nil { 426 deployment.CreateIndex = existing.(*structs.Deployment).CreateIndex 427 deployment.ModifyIndex = index 428 } else { 429 deployment.CreateIndex = index 430 deployment.ModifyIndex = index 431 } 432 433 // Insert the deployment 434 if err := txn.Insert("deployment", deployment); err != nil { 435 return err 436 } 437 438 // Update the indexes table for deployment 439 if err := txn.Insert("index", &IndexEntry{"deployment", index}); err != nil { 440 return fmt.Errorf("index update failed: %v", err) 441 } 442 443 // If the deployment is being marked as complete, set the job to stable. 444 if deployment.Status == structs.DeploymentStatusSuccessful { 445 if err := s.updateJobStabilityImpl(index, deployment.Namespace, deployment.JobID, deployment.JobVersion, true, txn); err != nil { 446 return fmt.Errorf("failed to update job stability: %v", err) 447 } 448 } 449 450 return nil 451 } 452 453 func (s *StateStore) Deployments(ws memdb.WatchSet) (memdb.ResultIterator, error) { 454 txn := s.db.Txn(false) 455 456 // Walk the entire deployments table 457 iter, err := txn.Get("deployment", "id") 458 if err != nil { 459 return nil, err 460 } 461 462 ws.Add(iter.WatchCh()) 463 return iter, nil 464 } 465 466 func (s *StateStore) DeploymentsByNamespace(ws memdb.WatchSet, namespace string) (memdb.ResultIterator, error) { 467 txn := s.db.Txn(false) 468 469 // Walk the entire deployments table 470 iter, err := txn.Get("deployment", "namespace", namespace) 471 if err != nil { 472 return nil, err 473 } 474 475 ws.Add(iter.WatchCh()) 476 return iter, nil 477 } 478 479 func (s *StateStore) DeploymentsByIDPrefix(ws memdb.WatchSet, namespace, deploymentID string) (memdb.ResultIterator, error) { 480 txn := s.db.Txn(false) 481 482 // Walk the entire deployments table 483 iter, err := txn.Get("deployment", "id_prefix", deploymentID) 484 if err != nil { 485 return nil, err 486 } 487 488 ws.Add(iter.WatchCh()) 489 490 // Wrap the iterator in a filter 491 wrap := memdb.NewFilterIterator(iter, deploymentNamespaceFilter(namespace)) 492 return wrap, nil 493 } 494 495 // deploymentNamespaceFilter returns a filter function that filters all 496 // deployment not in the given namespace. 497 func deploymentNamespaceFilter(namespace string) func(interface{}) bool { 498 return func(raw interface{}) bool { 499 d, ok := raw.(*structs.Deployment) 500 if !ok { 501 return true 502 } 503 504 return d.Namespace != namespace 505 } 506 } 507 508 func (s *StateStore) DeploymentByID(ws memdb.WatchSet, deploymentID string) (*structs.Deployment, error) { 509 txn := s.db.Txn(false) 510 return s.deploymentByIDImpl(ws, deploymentID, txn) 511 } 512 513 func (s *StateStore) deploymentByIDImpl(ws memdb.WatchSet, deploymentID string, txn *memdb.Txn) (*structs.Deployment, error) { 514 watchCh, existing, err := txn.FirstWatch("deployment", "id", deploymentID) 515 if err != nil { 516 return nil, fmt.Errorf("deployment lookup failed: %v", err) 517 } 518 ws.Add(watchCh) 519 520 if existing != nil { 521 return existing.(*structs.Deployment), nil 522 } 523 524 return nil, nil 525 } 526 527 func (s *StateStore) DeploymentsByJobID(ws memdb.WatchSet, namespace, jobID string, all bool) ([]*structs.Deployment, error) { 528 txn := s.db.Txn(false) 529 530 var job *structs.Job 531 // Read job from state store 532 _, existing, err := txn.FirstWatch("jobs", "id", namespace, jobID) 533 if err != nil { 534 return nil, fmt.Errorf("job lookup failed: %v", err) 535 } 536 if existing != nil { 537 job = existing.(*structs.Job) 538 } 539 540 // Get an iterator over the deployments 541 iter, err := txn.Get("deployment", "job", namespace, jobID) 542 if err != nil { 543 return nil, err 544 } 545 546 ws.Add(iter.WatchCh()) 547 548 var out []*structs.Deployment 549 for { 550 raw := iter.Next() 551 if raw == nil { 552 break 553 } 554 d := raw.(*structs.Deployment) 555 556 // If the allocation belongs to a job with the same ID but a different 557 // create index and we are not getting all the allocations whose Jobs 558 // matches the same Job ID then we skip it 559 if !all && job != nil && d.JobCreateIndex != job.CreateIndex { 560 continue 561 } 562 out = append(out, d) 563 } 564 565 return out, nil 566 } 567 568 // LatestDeploymentByJobID returns the latest deployment for the given job. The 569 // latest is determined strictly by CreateIndex. 570 func (s *StateStore) LatestDeploymentByJobID(ws memdb.WatchSet, namespace, jobID string) (*structs.Deployment, error) { 571 txn := s.db.Txn(false) 572 573 // Get an iterator over the deployments 574 iter, err := txn.Get("deployment", "job", namespace, jobID) 575 if err != nil { 576 return nil, err 577 } 578 579 ws.Add(iter.WatchCh()) 580 581 var out *structs.Deployment 582 for { 583 raw := iter.Next() 584 if raw == nil { 585 break 586 } 587 588 d := raw.(*structs.Deployment) 589 if out == nil || out.CreateIndex < d.CreateIndex { 590 out = d 591 } 592 } 593 594 return out, nil 595 } 596 597 // DeleteDeployment is used to delete a set of deployments by ID 598 func (s *StateStore) DeleteDeployment(index uint64, deploymentIDs []string) error { 599 txn := s.db.Txn(true) 600 defer txn.Abort() 601 602 if len(deploymentIDs) == 0 { 603 return nil 604 } 605 606 for _, deploymentID := range deploymentIDs { 607 // Lookup the deployment 608 existing, err := txn.First("deployment", "id", deploymentID) 609 if err != nil { 610 return fmt.Errorf("deployment lookup failed: %v", err) 611 } 612 if existing == nil { 613 return fmt.Errorf("deployment not found") 614 } 615 616 // Delete the deployment 617 if err := txn.Delete("deployment", existing); err != nil { 618 return fmt.Errorf("deployment delete failed: %v", err) 619 } 620 } 621 622 if err := txn.Insert("index", &IndexEntry{"deployment", index}); err != nil { 623 return fmt.Errorf("index update failed: %v", err) 624 } 625 626 txn.Commit() 627 return nil 628 } 629 630 // UpsertScalingEvent is used to insert a new scaling event. 631 // Only the most recent JobTrackedScalingEvents will be kept. 632 func (s *StateStore) UpsertScalingEvent(index uint64, req *structs.ScalingEventRequest) error { 633 txn := s.db.Txn(true) 634 defer txn.Abort() 635 636 // Get the existing events 637 existing, err := txn.First("scaling_event", "id", req.Namespace, req.JobID) 638 if err != nil { 639 return fmt.Errorf("scaling event lookup failed: %v", err) 640 } 641 642 var jobEvents *structs.JobScalingEvents 643 if existing != nil { 644 jobEvents = existing.(*structs.JobScalingEvents) 645 } else { 646 jobEvents = &structs.JobScalingEvents{ 647 Namespace: req.Namespace, 648 JobID: req.JobID, 649 ScalingEvents: make(map[string][]*structs.ScalingEvent), 650 } 651 } 652 653 jobEvents.ModifyIndex = index 654 req.ScalingEvent.CreateIndex = index 655 656 events := jobEvents.ScalingEvents[req.TaskGroup] 657 // Prepend this latest event 658 events = append( 659 []*structs.ScalingEvent{req.ScalingEvent}, 660 events..., 661 ) 662 // Truncate older events 663 if len(events) > structs.JobTrackedScalingEvents { 664 events = events[0:structs.JobTrackedScalingEvents] 665 } 666 jobEvents.ScalingEvents[req.TaskGroup] = events 667 668 // Insert the new event 669 if err := txn.Insert("scaling_event", jobEvents); err != nil { 670 return fmt.Errorf("scaling event insert failed: %v", err) 671 } 672 673 // Update the indexes table for scaling_event 674 if err := txn.Insert("index", &IndexEntry{"scaling_event", index}); err != nil { 675 return fmt.Errorf("index update failed: %v", err) 676 } 677 678 txn.Commit() 679 return nil 680 } 681 682 // ScalingEvents returns an iterator over all the job scaling events 683 func (s *StateStore) ScalingEvents(ws memdb.WatchSet) (memdb.ResultIterator, error) { 684 txn := s.db.Txn(false) 685 686 // Walk the entire scaling_event table 687 iter, err := txn.Get("scaling_event", "id") 688 if err != nil { 689 return nil, err 690 } 691 692 ws.Add(iter.WatchCh()) 693 694 return iter, nil 695 } 696 697 func (s *StateStore) ScalingEventsByJob(ws memdb.WatchSet, namespace, jobID string) (map[string][]*structs.ScalingEvent, uint64, error) { 698 txn := s.db.Txn(false) 699 700 watchCh, existing, err := txn.FirstWatch("scaling_event", "id", namespace, jobID) 701 if err != nil { 702 return nil, 0, fmt.Errorf("job scaling events lookup failed: %v", err) 703 } 704 ws.Add(watchCh) 705 706 if existing != nil { 707 events := existing.(*structs.JobScalingEvents) 708 return events.ScalingEvents, events.ModifyIndex, nil 709 } 710 return nil, 0, nil 711 } 712 713 // UpsertNode is used to register a node or update a node definition 714 // This is assumed to be triggered by the client, so we retain the value 715 // of drain/eligibility which is set by the scheduler. 716 func (s *StateStore) UpsertNode(index uint64, node *structs.Node) error { 717 txn := s.db.Txn(true) 718 defer txn.Abort() 719 720 // Check if the node already exists 721 existing, err := txn.First("nodes", "id", node.ID) 722 if err != nil { 723 return fmt.Errorf("node lookup failed: %v", err) 724 } 725 726 // Setup the indexes correctly 727 if existing != nil { 728 exist := existing.(*structs.Node) 729 node.CreateIndex = exist.CreateIndex 730 node.ModifyIndex = index 731 732 // Retain node events that have already been set on the node 733 node.Events = exist.Events 734 735 // If we are transitioning from down, record the re-registration 736 if exist.Status == structs.NodeStatusDown && node.Status != structs.NodeStatusDown { 737 appendNodeEvents(index, node, []*structs.NodeEvent{ 738 structs.NewNodeEvent().SetSubsystem(structs.NodeEventSubsystemCluster). 739 SetMessage(NodeRegisterEventReregistered). 740 SetTimestamp(time.Unix(node.StatusUpdatedAt, 0))}) 741 } 742 743 node.Drain = exist.Drain // Retain the drain mode 744 node.SchedulingEligibility = exist.SchedulingEligibility // Retain the eligibility 745 node.DrainStrategy = exist.DrainStrategy // Retain the drain strategy 746 } else { 747 // Because this is the first time the node is being registered, we should 748 // also create a node registration event 749 nodeEvent := structs.NewNodeEvent().SetSubsystem(structs.NodeEventSubsystemCluster). 750 SetMessage(NodeRegisterEventRegistered). 751 SetTimestamp(time.Unix(node.StatusUpdatedAt, 0)) 752 node.Events = []*structs.NodeEvent{nodeEvent} 753 node.CreateIndex = index 754 node.ModifyIndex = index 755 } 756 757 // Insert the node 758 if err := txn.Insert("nodes", node); err != nil { 759 return fmt.Errorf("node insert failed: %v", err) 760 } 761 if err := txn.Insert("index", &IndexEntry{"nodes", index}); err != nil { 762 return fmt.Errorf("index update failed: %v", err) 763 } 764 if err := upsertNodeCSIPlugins(txn, node, index); err != nil { 765 return fmt.Errorf("csi plugin update failed: %v", err) 766 } 767 768 txn.Commit() 769 return nil 770 } 771 772 // DeleteNode deregisters a batch of nodes 773 func (s *StateStore) DeleteNode(index uint64, nodes []string) error { 774 if len(nodes) == 0 { 775 return fmt.Errorf("node ids missing") 776 } 777 778 txn := s.db.Txn(true) 779 defer txn.Abort() 780 781 for _, nodeID := range nodes { 782 existing, err := txn.First("nodes", "id", nodeID) 783 if err != nil { 784 return fmt.Errorf("node lookup failed: %s: %v", nodeID, err) 785 } 786 if existing == nil { 787 return fmt.Errorf("node not found: %s", nodeID) 788 } 789 790 // Delete the node 791 if err := txn.Delete("nodes", existing); err != nil { 792 return fmt.Errorf("node delete failed: %s: %v", nodeID, err) 793 } 794 795 node := existing.(*structs.Node) 796 if err := deleteNodeCSIPlugins(txn, node, index); err != nil { 797 return fmt.Errorf("csi plugin delete failed: %v", err) 798 } 799 } 800 801 if err := txn.Insert("index", &IndexEntry{"nodes", index}); err != nil { 802 return fmt.Errorf("index update failed: %v", err) 803 } 804 805 txn.Commit() 806 return nil 807 } 808 809 // UpdateNodeStatus is used to update the status of a node 810 func (s *StateStore) UpdateNodeStatus(index uint64, nodeID, status string, updatedAt int64, event *structs.NodeEvent) error { 811 txn := s.db.Txn(true) 812 defer txn.Abort() 813 814 // Lookup the node 815 existing, err := txn.First("nodes", "id", nodeID) 816 if err != nil { 817 return fmt.Errorf("node lookup failed: %v", err) 818 } 819 if existing == nil { 820 return fmt.Errorf("node not found") 821 } 822 823 // Copy the existing node 824 existingNode := existing.(*structs.Node) 825 copyNode := existingNode.Copy() 826 copyNode.StatusUpdatedAt = updatedAt 827 828 // Add the event if given 829 if event != nil { 830 appendNodeEvents(index, copyNode, []*structs.NodeEvent{event}) 831 } 832 833 // Update the status in the copy 834 copyNode.Status = status 835 copyNode.ModifyIndex = index 836 837 // Insert the node 838 if err := txn.Insert("nodes", copyNode); err != nil { 839 return fmt.Errorf("node update failed: %v", err) 840 } 841 if err := txn.Insert("index", &IndexEntry{"nodes", index}); err != nil { 842 return fmt.Errorf("index update failed: %v", err) 843 } 844 845 txn.Commit() 846 return nil 847 } 848 849 // BatchUpdateNodeDrain is used to update the drain of a node set of nodes 850 func (s *StateStore) BatchUpdateNodeDrain(index uint64, updatedAt int64, updates map[string]*structs.DrainUpdate, events map[string]*structs.NodeEvent) error { 851 txn := s.db.Txn(true) 852 defer txn.Abort() 853 for node, update := range updates { 854 if err := s.updateNodeDrainImpl(txn, index, node, update.DrainStrategy, update.MarkEligible, updatedAt, events[node]); err != nil { 855 return err 856 } 857 } 858 txn.Commit() 859 return nil 860 } 861 862 // UpdateNodeDrain is used to update the drain of a node 863 func (s *StateStore) UpdateNodeDrain(index uint64, nodeID string, 864 drain *structs.DrainStrategy, markEligible bool, updatedAt int64, event *structs.NodeEvent) error { 865 866 txn := s.db.Txn(true) 867 defer txn.Abort() 868 if err := s.updateNodeDrainImpl(txn, index, nodeID, drain, markEligible, updatedAt, event); err != nil { 869 return err 870 } 871 txn.Commit() 872 return nil 873 } 874 875 func (s *StateStore) updateNodeDrainImpl(txn *memdb.Txn, index uint64, nodeID string, 876 drain *structs.DrainStrategy, markEligible bool, updatedAt int64, event *structs.NodeEvent) error { 877 878 // Lookup the node 879 existing, err := txn.First("nodes", "id", nodeID) 880 if err != nil { 881 return fmt.Errorf("node lookup failed: %v", err) 882 } 883 if existing == nil { 884 return fmt.Errorf("node not found") 885 } 886 887 // Copy the existing node 888 existingNode := existing.(*structs.Node) 889 copyNode := existingNode.Copy() 890 copyNode.StatusUpdatedAt = updatedAt 891 892 // Add the event if given 893 if event != nil { 894 appendNodeEvents(index, copyNode, []*structs.NodeEvent{event}) 895 } 896 897 // Update the drain in the copy 898 copyNode.Drain = drain != nil // COMPAT: Remove in Nomad 0.10 899 copyNode.DrainStrategy = drain 900 if drain != nil { 901 copyNode.SchedulingEligibility = structs.NodeSchedulingIneligible 902 } else if markEligible { 903 copyNode.SchedulingEligibility = structs.NodeSchedulingEligible 904 } 905 906 copyNode.ModifyIndex = index 907 908 // Insert the node 909 if err := txn.Insert("nodes", copyNode); err != nil { 910 return fmt.Errorf("node update failed: %v", err) 911 } 912 if err := txn.Insert("index", &IndexEntry{"nodes", index}); err != nil { 913 return fmt.Errorf("index update failed: %v", err) 914 } 915 916 return nil 917 } 918 919 // UpdateNodeEligibility is used to update the scheduling eligibility of a node 920 func (s *StateStore) UpdateNodeEligibility(index uint64, nodeID string, eligibility string, updatedAt int64, event *structs.NodeEvent) error { 921 922 txn := s.db.Txn(true) 923 defer txn.Abort() 924 925 // Lookup the node 926 existing, err := txn.First("nodes", "id", nodeID) 927 if err != nil { 928 return fmt.Errorf("node lookup failed: %v", err) 929 } 930 if existing == nil { 931 return fmt.Errorf("node not found") 932 } 933 934 // Copy the existing node 935 existingNode := existing.(*structs.Node) 936 copyNode := existingNode.Copy() 937 copyNode.StatusUpdatedAt = updatedAt 938 939 // Add the event if given 940 if event != nil { 941 appendNodeEvents(index, copyNode, []*structs.NodeEvent{event}) 942 } 943 944 // Check if this is a valid action 945 if copyNode.DrainStrategy != nil && eligibility == structs.NodeSchedulingEligible { 946 return fmt.Errorf("can not set node's scheduling eligibility to eligible while it is draining") 947 } 948 949 // Update the eligibility in the copy 950 copyNode.SchedulingEligibility = eligibility 951 copyNode.ModifyIndex = index 952 953 // Insert the node 954 if err := txn.Insert("nodes", copyNode); err != nil { 955 return fmt.Errorf("node update failed: %v", err) 956 } 957 if err := txn.Insert("index", &IndexEntry{"nodes", index}); err != nil { 958 return fmt.Errorf("index update failed: %v", err) 959 } 960 961 txn.Commit() 962 return nil 963 } 964 965 // UpsertNodeEvents adds the node events to the nodes, rotating events as 966 // necessary. 967 func (s *StateStore) UpsertNodeEvents(index uint64, nodeEvents map[string][]*structs.NodeEvent) error { 968 txn := s.db.Txn(true) 969 defer txn.Abort() 970 971 for nodeID, events := range nodeEvents { 972 if err := s.upsertNodeEvents(index, nodeID, events, txn); err != nil { 973 return err 974 } 975 } 976 977 txn.Commit() 978 return nil 979 } 980 981 // upsertNodeEvent upserts a node event for a respective node. It also maintains 982 // that a fixed number of node events are ever stored simultaneously, deleting 983 // older events once this bound has been reached. 984 func (s *StateStore) upsertNodeEvents(index uint64, nodeID string, events []*structs.NodeEvent, txn *memdb.Txn) error { 985 // Lookup the node 986 existing, err := txn.First("nodes", "id", nodeID) 987 if err != nil { 988 return fmt.Errorf("node lookup failed: %v", err) 989 } 990 if existing == nil { 991 return fmt.Errorf("node not found") 992 } 993 994 // Copy the existing node 995 existingNode := existing.(*structs.Node) 996 copyNode := existingNode.Copy() 997 appendNodeEvents(index, copyNode, events) 998 999 // Insert the node 1000 if err := txn.Insert("nodes", copyNode); err != nil { 1001 return fmt.Errorf("node update failed: %v", err) 1002 } 1003 if err := txn.Insert("index", &IndexEntry{"nodes", index}); err != nil { 1004 return fmt.Errorf("index update failed: %v", err) 1005 } 1006 1007 return nil 1008 } 1009 1010 // appendNodeEvents is a helper that takes a node and new events and appends 1011 // them, pruning older events as needed. 1012 func appendNodeEvents(index uint64, node *structs.Node, events []*structs.NodeEvent) { 1013 // Add the events, updating the indexes 1014 for _, e := range events { 1015 e.CreateIndex = index 1016 node.Events = append(node.Events, e) 1017 } 1018 1019 // Keep node events pruned to not exceed the max allowed 1020 if l := len(node.Events); l > structs.MaxRetainedNodeEvents { 1021 delta := l - structs.MaxRetainedNodeEvents 1022 node.Events = node.Events[delta:] 1023 } 1024 } 1025 1026 // upsertNodeCSIPlugins indexes csi plugins for volume retrieval, with health. It's called 1027 // on upsertNodeEvents, so that event driven health changes are updated 1028 func upsertNodeCSIPlugins(txn *memdb.Txn, node *structs.Node, index uint64) error { 1029 1030 loop := func(info *structs.CSIInfo) error { 1031 raw, err := txn.First("csi_plugins", "id", info.PluginID) 1032 if err != nil { 1033 return fmt.Errorf("csi_plugin lookup error: %s %v", info.PluginID, err) 1034 } 1035 1036 var plug *structs.CSIPlugin 1037 if raw != nil { 1038 plug = raw.(*structs.CSIPlugin).Copy() 1039 } else { 1040 if !info.Healthy { 1041 // we don't want to create new plugins for unhealthy 1042 // allocs, otherwise we'd recreate the plugin when we 1043 // get the update for the alloc becoming terminal 1044 return nil 1045 } 1046 plug = structs.NewCSIPlugin(info.PluginID, index) 1047 plug.Provider = info.Provider 1048 plug.Version = info.ProviderVersion 1049 } 1050 1051 err = plug.AddPlugin(node.ID, info) 1052 if err != nil { 1053 return err 1054 } 1055 1056 plug.ModifyIndex = index 1057 1058 err = txn.Insert("csi_plugins", plug) 1059 if err != nil { 1060 return fmt.Errorf("csi_plugins insert error: %v", err) 1061 } 1062 1063 return nil 1064 } 1065 1066 inUseController := map[string]struct{}{} 1067 inUseNode := map[string]struct{}{} 1068 1069 for _, info := range node.CSIControllerPlugins { 1070 err := loop(info) 1071 if err != nil { 1072 return err 1073 } 1074 inUseController[info.PluginID] = struct{}{} 1075 } 1076 1077 for _, info := range node.CSINodePlugins { 1078 err := loop(info) 1079 if err != nil { 1080 return err 1081 } 1082 inUseNode[info.PluginID] = struct{}{} 1083 } 1084 1085 // remove the client node from any plugin that's not 1086 // running on it. 1087 iter, err := txn.Get("csi_plugins", "id") 1088 if err != nil { 1089 return fmt.Errorf("csi_plugins lookup failed: %v", err) 1090 } 1091 for { 1092 raw := iter.Next() 1093 if raw == nil { 1094 break 1095 } 1096 plug := raw.(*structs.CSIPlugin) 1097 1098 var hadDelete bool 1099 if _, ok := inUseController[plug.ID]; !ok { 1100 if _, asController := plug.Controllers[node.ID]; asController { 1101 err := plug.DeleteNodeForType(node.ID, structs.CSIPluginTypeController) 1102 if err != nil { 1103 return err 1104 } 1105 hadDelete = true 1106 } 1107 } 1108 if _, ok := inUseNode[plug.ID]; !ok { 1109 if _, asNode := plug.Nodes[node.ID]; asNode { 1110 err := plug.DeleteNodeForType(node.ID, structs.CSIPluginTypeNode) 1111 if err != nil { 1112 return err 1113 } 1114 hadDelete = true 1115 } 1116 } 1117 // we check this flag both for performance and to make sure we 1118 // don't delete a plugin when registering a node plugin but 1119 // no controller 1120 if hadDelete { 1121 err = updateOrGCPlugin(index, txn, plug) 1122 if err != nil { 1123 return err 1124 } 1125 } 1126 } 1127 1128 if err := txn.Insert("index", &IndexEntry{"csi_plugins", index}); err != nil { 1129 return fmt.Errorf("index update failed: %v", err) 1130 } 1131 1132 return nil 1133 } 1134 1135 // deleteNodeCSIPlugins cleans up CSIInfo node health status, called in DeleteNode 1136 func deleteNodeCSIPlugins(txn *memdb.Txn, node *structs.Node, index uint64) error { 1137 if len(node.CSIControllerPlugins) == 0 && len(node.CSINodePlugins) == 0 { 1138 return nil 1139 } 1140 1141 names := map[string]struct{}{} 1142 for _, info := range node.CSIControllerPlugins { 1143 names[info.PluginID] = struct{}{} 1144 } 1145 for _, info := range node.CSINodePlugins { 1146 names[info.PluginID] = struct{}{} 1147 } 1148 1149 for id := range names { 1150 raw, err := txn.First("csi_plugins", "id", id) 1151 if err != nil { 1152 return fmt.Errorf("csi_plugins lookup error %s: %v", id, err) 1153 } 1154 if raw == nil { 1155 return fmt.Errorf("csi_plugins missing plugin %s", id) 1156 } 1157 1158 plug := raw.(*structs.CSIPlugin).Copy() 1159 err = plug.DeleteNode(node.ID) 1160 if err != nil { 1161 return err 1162 } 1163 err = updateOrGCPlugin(index, txn, plug) 1164 if err != nil { 1165 return err 1166 } 1167 } 1168 1169 if err := txn.Insert("index", &IndexEntry{"csi_plugins", index}); err != nil { 1170 return fmt.Errorf("index update failed: %v", err) 1171 } 1172 1173 return nil 1174 } 1175 1176 // updateOrGCPlugin updates a plugin but will delete it if the plugin is empty 1177 func updateOrGCPlugin(index uint64, txn *memdb.Txn, plug *structs.CSIPlugin) error { 1178 plug.ModifyIndex = index 1179 1180 if plug.IsEmpty() { 1181 err := txn.Delete("csi_plugins", plug) 1182 if err != nil { 1183 return fmt.Errorf("csi_plugins delete error: %v", err) 1184 } 1185 } else { 1186 err := txn.Insert("csi_plugins", plug) 1187 if err != nil { 1188 return fmt.Errorf("csi_plugins update error %s: %v", plug.ID, err) 1189 } 1190 } 1191 return nil 1192 } 1193 1194 // deleteJobFromPlugin removes the allocations of this job from any plugins the job is 1195 // running, possibly deleting the plugin if it's no longer in use. It's called in DeleteJobTxn 1196 func (s *StateStore) deleteJobFromPlugin(index uint64, txn *memdb.Txn, job *structs.Job) error { 1197 ws := memdb.NewWatchSet() 1198 allocs, err := s.AllocsByJob(ws, job.Namespace, job.ID, false) 1199 if err != nil { 1200 return fmt.Errorf("error getting allocations: %v", err) 1201 } 1202 1203 type pair struct { 1204 pluginID string 1205 alloc *structs.Allocation 1206 } 1207 1208 plugAllocs := []*pair{} 1209 plugins := map[string]*structs.CSIPlugin{} 1210 1211 for _, a := range allocs { 1212 tg := a.Job.LookupTaskGroup(a.TaskGroup) 1213 for _, t := range tg.Tasks { 1214 if t.CSIPluginConfig != nil { 1215 plugAllocs = append(plugAllocs, &pair{ 1216 pluginID: t.CSIPluginConfig.ID, 1217 alloc: a, 1218 }) 1219 } 1220 } 1221 } 1222 1223 for _, x := range plugAllocs { 1224 plug, ok := plugins[x.pluginID] 1225 1226 if !ok { 1227 plug, err = s.CSIPluginByID(ws, x.pluginID) 1228 if err != nil { 1229 return fmt.Errorf("error getting plugin: %s, %v", x.pluginID, err) 1230 } 1231 if plug == nil { 1232 return fmt.Errorf("plugin missing: %s %v", x.pluginID, err) 1233 } 1234 // only copy once, so we update the same plugin on each alloc 1235 plugins[x.pluginID] = plug.Copy() 1236 plug = plugins[x.pluginID] 1237 } 1238 1239 err := plug.DeleteAlloc(x.alloc.ID, x.alloc.NodeID) 1240 if err != nil { 1241 return err 1242 } 1243 } 1244 1245 for _, plug := range plugins { 1246 err = updateOrGCPlugin(index, txn, plug) 1247 if err != nil { 1248 return err 1249 } 1250 } 1251 1252 if err = txn.Insert("index", &IndexEntry{"csi_plugins", index}); err != nil { 1253 return fmt.Errorf("index update failed: %v", err) 1254 } 1255 1256 return nil 1257 } 1258 1259 // NodeByID is used to lookup a node by ID 1260 func (s *StateStore) NodeByID(ws memdb.WatchSet, nodeID string) (*structs.Node, error) { 1261 txn := s.db.Txn(false) 1262 1263 watchCh, existing, err := txn.FirstWatch("nodes", "id", nodeID) 1264 if err != nil { 1265 return nil, fmt.Errorf("node lookup failed: %v", err) 1266 } 1267 ws.Add(watchCh) 1268 1269 if existing != nil { 1270 return existing.(*structs.Node), nil 1271 } 1272 return nil, nil 1273 } 1274 1275 // NodesByIDPrefix is used to lookup nodes by prefix 1276 func (s *StateStore) NodesByIDPrefix(ws memdb.WatchSet, nodeID string) (memdb.ResultIterator, error) { 1277 txn := s.db.Txn(false) 1278 1279 iter, err := txn.Get("nodes", "id_prefix", nodeID) 1280 if err != nil { 1281 return nil, fmt.Errorf("node lookup failed: %v", err) 1282 } 1283 ws.Add(iter.WatchCh()) 1284 1285 return iter, nil 1286 } 1287 1288 // NodeBySecretID is used to lookup a node by SecretID 1289 func (s *StateStore) NodeBySecretID(ws memdb.WatchSet, secretID string) (*structs.Node, error) { 1290 txn := s.db.Txn(false) 1291 1292 watchCh, existing, err := txn.FirstWatch("nodes", "secret_id", secretID) 1293 if err != nil { 1294 return nil, fmt.Errorf("node lookup by SecretID failed: %v", err) 1295 } 1296 ws.Add(watchCh) 1297 1298 if existing != nil { 1299 return existing.(*structs.Node), nil 1300 } 1301 return nil, nil 1302 } 1303 1304 // Nodes returns an iterator over all the nodes 1305 func (s *StateStore) Nodes(ws memdb.WatchSet) (memdb.ResultIterator, error) { 1306 txn := s.db.Txn(false) 1307 1308 // Walk the entire nodes table 1309 iter, err := txn.Get("nodes", "id") 1310 if err != nil { 1311 return nil, err 1312 } 1313 ws.Add(iter.WatchCh()) 1314 return iter, nil 1315 } 1316 1317 // UpsertJob is used to register a job or update a job definition 1318 func (s *StateStore) UpsertJob(index uint64, job *structs.Job) error { 1319 txn := s.db.Txn(true) 1320 defer txn.Abort() 1321 if err := s.upsertJobImpl(index, job, false, txn); err != nil { 1322 return err 1323 } 1324 txn.Commit() 1325 return nil 1326 } 1327 1328 // UpsertJobTxn is used to register a job or update a job definition, like UpsertJob, 1329 // but in a transaction. Useful for when making multiple modifications atomically 1330 func (s *StateStore) UpsertJobTxn(index uint64, job *structs.Job, txn Txn) error { 1331 return s.upsertJobImpl(index, job, false, txn) 1332 } 1333 1334 // upsertJobImpl is the implementation for registering a job or updating a job definition 1335 func (s *StateStore) upsertJobImpl(index uint64, job *structs.Job, keepVersion bool, txn *memdb.Txn) error { 1336 // Assert the namespace exists 1337 if exists, err := s.namespaceExists(txn, job.Namespace); err != nil { 1338 return err 1339 } else if !exists { 1340 return fmt.Errorf("job %q is in nonexistent namespace %q", job.ID, job.Namespace) 1341 } 1342 1343 // Check if the job already exists 1344 existing, err := txn.First("jobs", "id", job.Namespace, job.ID) 1345 if err != nil { 1346 return fmt.Errorf("job lookup failed: %v", err) 1347 } 1348 1349 // Setup the indexes correctly 1350 if existing != nil { 1351 job.CreateIndex = existing.(*structs.Job).CreateIndex 1352 job.ModifyIndex = index 1353 1354 // Bump the version unless asked to keep it. This should only be done 1355 // when changing an internal field such as Stable. A spec change should 1356 // always come with a version bump 1357 if !keepVersion { 1358 job.JobModifyIndex = index 1359 job.Version = existing.(*structs.Job).Version + 1 1360 } 1361 1362 // Compute the job status 1363 var err error 1364 job.Status, err = s.getJobStatus(txn, job, false) 1365 if err != nil { 1366 return fmt.Errorf("setting job status for %q failed: %v", job.ID, err) 1367 } 1368 } else { 1369 job.CreateIndex = index 1370 job.ModifyIndex = index 1371 job.JobModifyIndex = index 1372 job.Version = 0 1373 1374 if err := s.setJobStatus(index, txn, job, false, ""); err != nil { 1375 return fmt.Errorf("setting job status for %q failed: %v", job.ID, err) 1376 } 1377 1378 // Have to get the job again since it could have been updated 1379 updated, err := txn.First("jobs", "id", job.Namespace, job.ID) 1380 if err != nil { 1381 return fmt.Errorf("job lookup failed: %v", err) 1382 } 1383 if updated != nil { 1384 job = updated.(*structs.Job) 1385 } 1386 } 1387 1388 if err := s.updateSummaryWithJob(index, job, txn); err != nil { 1389 return fmt.Errorf("unable to create job summary: %v", err) 1390 } 1391 1392 if err := s.upsertJobVersion(index, job, txn); err != nil { 1393 return fmt.Errorf("unable to upsert job into job_version table: %v", err) 1394 } 1395 1396 if err := s.updateJobScalingPolicies(index, job, txn); err != nil { 1397 return fmt.Errorf("unable to update job scaling policies: %v", err) 1398 } 1399 1400 // Insert the job 1401 if err := txn.Insert("jobs", job); err != nil { 1402 return fmt.Errorf("job insert failed: %v", err) 1403 } 1404 if err := txn.Insert("index", &IndexEntry{"jobs", index}); err != nil { 1405 return fmt.Errorf("index update failed: %v", err) 1406 } 1407 1408 return nil 1409 } 1410 1411 // DeleteJob is used to deregister a job 1412 func (s *StateStore) DeleteJob(index uint64, namespace, jobID string) error { 1413 txn := s.db.Txn(true) 1414 defer txn.Abort() 1415 1416 err := s.DeleteJobTxn(index, namespace, jobID, txn) 1417 if err == nil { 1418 txn.Commit() 1419 } 1420 return err 1421 } 1422 1423 // DeleteJobTxn is used to deregister a job, like DeleteJob, 1424 // but in a transaction. Useful for when making multiple modifications atomically 1425 func (s *StateStore) DeleteJobTxn(index uint64, namespace, jobID string, txn Txn) error { 1426 // Lookup the node 1427 existing, err := txn.First("jobs", "id", namespace, jobID) 1428 if err != nil { 1429 return fmt.Errorf("job lookup failed: %v", err) 1430 } 1431 if existing == nil { 1432 return fmt.Errorf("job not found") 1433 } 1434 1435 // Check if we should update a parent job summary 1436 job := existing.(*structs.Job) 1437 if job.ParentID != "" { 1438 summaryRaw, err := txn.First("job_summary", "id", namespace, job.ParentID) 1439 if err != nil { 1440 return fmt.Errorf("unable to retrieve summary for parent job: %v", err) 1441 } 1442 1443 // Only continue if the summary exists. It could not exist if the parent 1444 // job was removed 1445 if summaryRaw != nil { 1446 existing := summaryRaw.(*structs.JobSummary) 1447 pSummary := existing.Copy() 1448 if pSummary.Children != nil { 1449 1450 modified := false 1451 switch job.Status { 1452 case structs.JobStatusPending: 1453 pSummary.Children.Pending-- 1454 pSummary.Children.Dead++ 1455 modified = true 1456 case structs.JobStatusRunning: 1457 pSummary.Children.Running-- 1458 pSummary.Children.Dead++ 1459 modified = true 1460 case structs.JobStatusDead: 1461 default: 1462 return fmt.Errorf("unknown old job status %q", job.Status) 1463 } 1464 1465 if modified { 1466 // Update the modify index 1467 pSummary.ModifyIndex = index 1468 1469 // Insert the summary 1470 if err := txn.Insert("job_summary", pSummary); err != nil { 1471 return fmt.Errorf("job summary insert failed: %v", err) 1472 } 1473 if err := txn.Insert("index", &IndexEntry{"job_summary", index}); err != nil { 1474 return fmt.Errorf("index update failed: %v", err) 1475 } 1476 } 1477 } 1478 } 1479 } 1480 1481 // Delete the job 1482 if err := txn.Delete("jobs", existing); err != nil { 1483 return fmt.Errorf("job delete failed: %v", err) 1484 } 1485 if err := txn.Insert("index", &IndexEntry{"jobs", index}); err != nil { 1486 return fmt.Errorf("index update failed: %v", err) 1487 } 1488 1489 // Delete the job versions 1490 if err := s.deleteJobVersions(index, job, txn); err != nil { 1491 return err 1492 } 1493 1494 // Delete the job summary 1495 if _, err = txn.DeleteAll("job_summary", "id", namespace, jobID); err != nil { 1496 return fmt.Errorf("deleting job summary failed: %v", err) 1497 } 1498 if err := txn.Insert("index", &IndexEntry{"job_summary", index}); err != nil { 1499 return fmt.Errorf("index update failed: %v", err) 1500 } 1501 1502 // Delete any remaining job scaling policies 1503 if err := s.deleteJobScalingPolicies(index, job, txn); err != nil { 1504 return fmt.Errorf("deleting job scaling policies failed: %v", err) 1505 } 1506 1507 // Delete the scaling events 1508 if _, err = txn.DeleteAll("scaling_event", "id", namespace, jobID); err != nil { 1509 return fmt.Errorf("deleting job scaling events failed: %v", err) 1510 } 1511 if err := txn.Insert("index", &IndexEntry{"scaling_event", index}); err != nil { 1512 return fmt.Errorf("index update failed: %v", err) 1513 } 1514 1515 // Cleanup plugins registered by this job 1516 err = s.deleteJobFromPlugin(index, txn, job) 1517 if err != nil { 1518 return fmt.Errorf("deleting job from plugin: %v", err) 1519 } 1520 1521 return nil 1522 } 1523 1524 // deleteJobScalingPolicies deletes any scaling policies associated with the job 1525 func (s *StateStore) deleteJobScalingPolicies(index uint64, job *structs.Job, txn *memdb.Txn) error { 1526 numDeletedScalingPolicies, err := txn.DeleteAll("scaling_policy", "target_prefix", job.Namespace, job.ID) 1527 if err != nil { 1528 return fmt.Errorf("deleting job scaling policies failed: %v", err) 1529 } 1530 if numDeletedScalingPolicies > 0 { 1531 if err := txn.Insert("index", &IndexEntry{"scaling_policy", index}); err != nil { 1532 return fmt.Errorf("index update failed: %v", err) 1533 } 1534 } 1535 return nil 1536 } 1537 1538 // deleteJobVersions deletes all versions of the given job. 1539 func (s *StateStore) deleteJobVersions(index uint64, job *structs.Job, txn *memdb.Txn) error { 1540 iter, err := txn.Get("job_version", "id_prefix", job.Namespace, job.ID) 1541 if err != nil { 1542 return err 1543 } 1544 1545 // Put them into a slice so there are no safety concerns while actually 1546 // performing the deletes 1547 jobs := []*structs.Job{} 1548 for { 1549 raw := iter.Next() 1550 if raw == nil { 1551 break 1552 } 1553 1554 // Ensure the ID is an exact match 1555 j := raw.(*structs.Job) 1556 if j.ID != job.ID { 1557 continue 1558 } 1559 1560 jobs = append(jobs, j) 1561 } 1562 1563 // Do the deletes 1564 for _, j := range jobs { 1565 if err := txn.Delete("job_version", j); err != nil { 1566 return fmt.Errorf("deleting job versions failed: %v", err) 1567 } 1568 } 1569 1570 if err := txn.Insert("index", &IndexEntry{"job_version", index}); err != nil { 1571 return fmt.Errorf("index update failed: %v", err) 1572 } 1573 1574 return nil 1575 } 1576 1577 // upsertJobVersion inserts a job into its historic version table and limits the 1578 // number of job versions that are tracked. 1579 func (s *StateStore) upsertJobVersion(index uint64, job *structs.Job, txn *memdb.Txn) error { 1580 // Insert the job 1581 if err := txn.Insert("job_version", job); err != nil { 1582 return fmt.Errorf("failed to insert job into job_version table: %v", err) 1583 } 1584 1585 if err := txn.Insert("index", &IndexEntry{"job_version", index}); err != nil { 1586 return fmt.Errorf("index update failed: %v", err) 1587 } 1588 1589 // Get all the historic jobs for this ID 1590 all, err := s.jobVersionByID(txn, nil, job.Namespace, job.ID) 1591 if err != nil { 1592 return fmt.Errorf("failed to look up job versions for %q: %v", job.ID, err) 1593 } 1594 1595 // If we are below the limit there is no GCing to be done 1596 if len(all) <= structs.JobTrackedVersions { 1597 return nil 1598 } 1599 1600 // We have to delete a historic job to make room. 1601 // Find index of the highest versioned stable job 1602 stableIdx := -1 1603 for i, j := range all { 1604 if j.Stable { 1605 stableIdx = i 1606 break 1607 } 1608 } 1609 1610 // If the stable job is the oldest version, do a swap to bring it into the 1611 // keep set. 1612 max := structs.JobTrackedVersions 1613 if stableIdx == max { 1614 all[max-1], all[max] = all[max], all[max-1] 1615 } 1616 1617 // Delete the job outside of the set that are being kept. 1618 d := all[max] 1619 if err := txn.Delete("job_version", d); err != nil { 1620 return fmt.Errorf("failed to delete job %v (%d) from job_version", d.ID, d.Version) 1621 } 1622 1623 return nil 1624 } 1625 1626 // JobByID is used to lookup a job by its ID. JobByID returns the current/latest job 1627 // version. 1628 func (s *StateStore) JobByID(ws memdb.WatchSet, namespace, id string) (*structs.Job, error) { 1629 txn := s.db.Txn(false) 1630 return s.JobByIDTxn(ws, namespace, id, txn) 1631 } 1632 1633 // JobByIDTxn is used to lookup a job by its ID, like JobByID. JobByID returns the job version 1634 // accessible through in the transaction 1635 func (s *StateStore) JobByIDTxn(ws memdb.WatchSet, namespace, id string, txn Txn) (*structs.Job, error) { 1636 watchCh, existing, err := txn.FirstWatch("jobs", "id", namespace, id) 1637 if err != nil { 1638 return nil, fmt.Errorf("job lookup failed: %v", err) 1639 } 1640 ws.Add(watchCh) 1641 1642 if existing != nil { 1643 return existing.(*structs.Job), nil 1644 } 1645 return nil, nil 1646 } 1647 1648 // JobsByIDPrefix is used to lookup a job by prefix 1649 func (s *StateStore) JobsByIDPrefix(ws memdb.WatchSet, namespace, id string) (memdb.ResultIterator, error) { 1650 txn := s.db.Txn(false) 1651 1652 iter, err := txn.Get("jobs", "id_prefix", namespace, id) 1653 if err != nil { 1654 return nil, fmt.Errorf("job lookup failed: %v", err) 1655 } 1656 1657 ws.Add(iter.WatchCh()) 1658 1659 return iter, nil 1660 } 1661 1662 // JobVersionsByID returns all the tracked versions of a job. 1663 func (s *StateStore) JobVersionsByID(ws memdb.WatchSet, namespace, id string) ([]*structs.Job, error) { 1664 txn := s.db.Txn(false) 1665 1666 return s.jobVersionByID(txn, &ws, namespace, id) 1667 } 1668 1669 // jobVersionByID is the underlying implementation for retrieving all tracked 1670 // versions of a job and is called under an existing transaction. A watch set 1671 // can optionally be passed in to add the job histories to the watch set. 1672 func (s *StateStore) jobVersionByID(txn *memdb.Txn, ws *memdb.WatchSet, namespace, id string) ([]*structs.Job, error) { 1673 // Get all the historic jobs for this ID 1674 iter, err := txn.Get("job_version", "id_prefix", namespace, id) 1675 if err != nil { 1676 return nil, err 1677 } 1678 1679 if ws != nil { 1680 ws.Add(iter.WatchCh()) 1681 } 1682 1683 var all []*structs.Job 1684 for { 1685 raw := iter.Next() 1686 if raw == nil { 1687 break 1688 } 1689 1690 // Ensure the ID is an exact match 1691 j := raw.(*structs.Job) 1692 if j.ID != id { 1693 continue 1694 } 1695 1696 all = append(all, j) 1697 } 1698 1699 // Sort in reverse order so that the highest version is first 1700 sort.Slice(all, func(i, j int) bool { 1701 return all[i].Version > all[j].Version 1702 }) 1703 1704 return all, nil 1705 } 1706 1707 // JobByIDAndVersion returns the job identified by its ID and Version. The 1708 // passed watchset may be nil. 1709 func (s *StateStore) JobByIDAndVersion(ws memdb.WatchSet, namespace, id string, version uint64) (*structs.Job, error) { 1710 txn := s.db.Txn(false) 1711 return s.jobByIDAndVersionImpl(ws, namespace, id, version, txn) 1712 } 1713 1714 // jobByIDAndVersionImpl returns the job identified by its ID and Version. The 1715 // passed watchset may be nil. 1716 func (s *StateStore) jobByIDAndVersionImpl(ws memdb.WatchSet, namespace, id string, 1717 version uint64, txn *memdb.Txn) (*structs.Job, error) { 1718 1719 watchCh, existing, err := txn.FirstWatch("job_version", "id", namespace, id, version) 1720 if err != nil { 1721 return nil, err 1722 } 1723 1724 if ws != nil { 1725 ws.Add(watchCh) 1726 } 1727 1728 if existing != nil { 1729 job := existing.(*structs.Job) 1730 return job, nil 1731 } 1732 1733 return nil, nil 1734 } 1735 1736 func (s *StateStore) JobVersions(ws memdb.WatchSet) (memdb.ResultIterator, error) { 1737 txn := s.db.Txn(false) 1738 1739 // Walk the entire deployments table 1740 iter, err := txn.Get("job_version", "id") 1741 if err != nil { 1742 return nil, err 1743 } 1744 1745 ws.Add(iter.WatchCh()) 1746 return iter, nil 1747 } 1748 1749 // Jobs returns an iterator over all the jobs 1750 func (s *StateStore) Jobs(ws memdb.WatchSet) (memdb.ResultIterator, error) { 1751 txn := s.db.Txn(false) 1752 1753 // Walk the entire jobs table 1754 iter, err := txn.Get("jobs", "id") 1755 if err != nil { 1756 return nil, err 1757 } 1758 1759 ws.Add(iter.WatchCh()) 1760 1761 return iter, nil 1762 } 1763 1764 // JobsByNamespace returns an iterator over all the jobs for the given namespace 1765 func (s *StateStore) JobsByNamespace(ws memdb.WatchSet, namespace string) (memdb.ResultIterator, error) { 1766 txn := s.db.Txn(false) 1767 return s.jobsByNamespaceImpl(ws, namespace, txn) 1768 } 1769 1770 // jobsByNamespaceImpl returns an iterator over all the jobs for the given namespace 1771 func (s *StateStore) jobsByNamespaceImpl(ws memdb.WatchSet, namespace string, txn *memdb.Txn) (memdb.ResultIterator, error) { 1772 // Walk the entire jobs table 1773 iter, err := txn.Get("jobs", "id_prefix", namespace, "") 1774 if err != nil { 1775 return nil, err 1776 } 1777 1778 ws.Add(iter.WatchCh()) 1779 1780 return iter, nil 1781 } 1782 1783 // JobsByPeriodic returns an iterator over all the periodic or non-periodic jobs. 1784 func (s *StateStore) JobsByPeriodic(ws memdb.WatchSet, periodic bool) (memdb.ResultIterator, error) { 1785 txn := s.db.Txn(false) 1786 1787 iter, err := txn.Get("jobs", "periodic", periodic) 1788 if err != nil { 1789 return nil, err 1790 } 1791 1792 ws.Add(iter.WatchCh()) 1793 1794 return iter, nil 1795 } 1796 1797 // JobsByScheduler returns an iterator over all the jobs with the specific 1798 // scheduler type. 1799 func (s *StateStore) JobsByScheduler(ws memdb.WatchSet, schedulerType string) (memdb.ResultIterator, error) { 1800 txn := s.db.Txn(false) 1801 1802 // Return an iterator for jobs with the specific type. 1803 iter, err := txn.Get("jobs", "type", schedulerType) 1804 if err != nil { 1805 return nil, err 1806 } 1807 1808 ws.Add(iter.WatchCh()) 1809 1810 return iter, nil 1811 } 1812 1813 // JobsByGC returns an iterator over all jobs eligible or uneligible for garbage 1814 // collection. 1815 func (s *StateStore) JobsByGC(ws memdb.WatchSet, gc bool) (memdb.ResultIterator, error) { 1816 txn := s.db.Txn(false) 1817 1818 iter, err := txn.Get("jobs", "gc", gc) 1819 if err != nil { 1820 return nil, err 1821 } 1822 1823 ws.Add(iter.WatchCh()) 1824 1825 return iter, nil 1826 } 1827 1828 // JobSummary returns a job summary object which matches a specific id. 1829 func (s *StateStore) JobSummaryByID(ws memdb.WatchSet, namespace, jobID string) (*structs.JobSummary, error) { 1830 txn := s.db.Txn(false) 1831 1832 watchCh, existing, err := txn.FirstWatch("job_summary", "id", namespace, jobID) 1833 if err != nil { 1834 return nil, err 1835 } 1836 1837 ws.Add(watchCh) 1838 1839 if existing != nil { 1840 summary := existing.(*structs.JobSummary) 1841 return summary, nil 1842 } 1843 1844 return nil, nil 1845 } 1846 1847 // JobSummaries walks the entire job summary table and returns all the job 1848 // summary objects 1849 func (s *StateStore) JobSummaries(ws memdb.WatchSet) (memdb.ResultIterator, error) { 1850 txn := s.db.Txn(false) 1851 1852 iter, err := txn.Get("job_summary", "id") 1853 if err != nil { 1854 return nil, err 1855 } 1856 1857 ws.Add(iter.WatchCh()) 1858 1859 return iter, nil 1860 } 1861 1862 // JobSummaryByPrefix is used to look up Job Summary by id prefix 1863 func (s *StateStore) JobSummaryByPrefix(ws memdb.WatchSet, namespace, id string) (memdb.ResultIterator, error) { 1864 txn := s.db.Txn(false) 1865 1866 iter, err := txn.Get("job_summary", "id_prefix", namespace, id) 1867 if err != nil { 1868 return nil, fmt.Errorf("job_summary lookup failed: %v", err) 1869 } 1870 1871 ws.Add(iter.WatchCh()) 1872 1873 return iter, nil 1874 } 1875 1876 // CSIVolumeRegister adds a volume to the server store, failing if it already exists 1877 func (s *StateStore) CSIVolumeRegister(index uint64, volumes []*structs.CSIVolume) error { 1878 txn := s.db.Txn(true) 1879 defer txn.Abort() 1880 1881 for _, v := range volumes { 1882 if exists, err := s.namespaceExists(txn, v.Namespace); err != nil { 1883 return err 1884 } else if !exists { 1885 return fmt.Errorf("volume %s is in nonexistent namespace %s", v.ID, v.Namespace) 1886 } 1887 1888 // Check for volume existence 1889 obj, err := txn.First("csi_volumes", "id", v.Namespace, v.ID) 1890 if err != nil { 1891 return fmt.Errorf("volume existence check error: %v", err) 1892 } 1893 if obj != nil { 1894 // Allow some properties of a volume to be updated in place, but 1895 // prevent accidentally overwriting important properties, or 1896 // overwriting a volume in use 1897 old, ok := obj.(*structs.CSIVolume) 1898 if ok && 1899 old.InUse() || 1900 old.ExternalID != v.ExternalID || 1901 old.PluginID != v.PluginID || 1902 old.Provider != v.Provider { 1903 return fmt.Errorf("volume exists: %s", v.ID) 1904 } 1905 } 1906 1907 if v.CreateIndex == 0 { 1908 v.CreateIndex = index 1909 v.ModifyIndex = index 1910 } 1911 1912 err = txn.Insert("csi_volumes", v) 1913 if err != nil { 1914 return fmt.Errorf("volume insert: %v", err) 1915 } 1916 } 1917 1918 if err := txn.Insert("index", &IndexEntry{"csi_volumes", index}); err != nil { 1919 return fmt.Errorf("index update failed: %v", err) 1920 } 1921 1922 txn.Commit() 1923 return nil 1924 } 1925 1926 // CSIVolumes returns the unfiltered list of all volumes 1927 func (s *StateStore) CSIVolumes(ws memdb.WatchSet) (memdb.ResultIterator, error) { 1928 txn := s.db.Txn(false) 1929 defer txn.Abort() 1930 1931 iter, err := txn.Get("csi_volumes", "id") 1932 if err != nil { 1933 return nil, fmt.Errorf("csi_volumes lookup failed: %v", err) 1934 } 1935 1936 ws.Add(iter.WatchCh()) 1937 1938 return iter, nil 1939 } 1940 1941 // CSIVolumeByID is used to lookup a single volume. Returns a copy of the volume 1942 // because its plugins are denormalized to provide accurate Health. 1943 func (s *StateStore) CSIVolumeByID(ws memdb.WatchSet, namespace, id string) (*structs.CSIVolume, error) { 1944 txn := s.db.Txn(false) 1945 1946 watchCh, obj, err := txn.FirstWatch("csi_volumes", "id_prefix", namespace, id) 1947 if err != nil { 1948 return nil, fmt.Errorf("volume lookup failed: %s %v", id, err) 1949 } 1950 ws.Add(watchCh) 1951 1952 if obj == nil { 1953 return nil, nil 1954 } 1955 1956 vol := obj.(*structs.CSIVolume) 1957 return s.CSIVolumeDenormalizePlugins(ws, vol.Copy()) 1958 } 1959 1960 // CSIVolumes looks up csi_volumes by pluginID 1961 func (s *StateStore) CSIVolumesByPluginID(ws memdb.WatchSet, namespace, pluginID string) (memdb.ResultIterator, error) { 1962 txn := s.db.Txn(false) 1963 1964 iter, err := txn.Get("csi_volumes", "plugin_id", pluginID) 1965 if err != nil { 1966 return nil, fmt.Errorf("volume lookup failed: %v", err) 1967 } 1968 1969 // Filter the iterator by namespace 1970 f := func(raw interface{}) bool { 1971 v, ok := raw.(*structs.CSIVolume) 1972 if !ok { 1973 return false 1974 } 1975 return v.Namespace != namespace 1976 } 1977 1978 wrap := memdb.NewFilterIterator(iter, f) 1979 return wrap, nil 1980 } 1981 1982 // CSIVolumesByIDPrefix supports search 1983 func (s *StateStore) CSIVolumesByIDPrefix(ws memdb.WatchSet, namespace, volumeID string) (memdb.ResultIterator, error) { 1984 txn := s.db.Txn(false) 1985 1986 iter, err := txn.Get("csi_volumes", "id_prefix", namespace, volumeID) 1987 if err != nil { 1988 return nil, err 1989 } 1990 1991 ws.Add(iter.WatchCh()) 1992 return iter, nil 1993 } 1994 1995 // CSIVolumesByNodeID looks up CSIVolumes in use on a node 1996 func (s *StateStore) CSIVolumesByNodeID(ws memdb.WatchSet, nodeID string) (memdb.ResultIterator, error) { 1997 allocs, err := s.AllocsByNode(ws, nodeID) 1998 if err != nil { 1999 return nil, fmt.Errorf("alloc lookup failed: %v", err) 2000 } 2001 2002 // Find volume ids for CSI volumes in running allocs, or allocs that we desire to run 2003 ids := map[string]string{} // Map volumeID to Namespace 2004 for _, a := range allocs { 2005 tg := a.Job.LookupTaskGroup(a.TaskGroup) 2006 2007 if !(a.DesiredStatus == structs.AllocDesiredStatusRun || 2008 a.ClientStatus == structs.AllocClientStatusRunning) || 2009 len(tg.Volumes) == 0 { 2010 continue 2011 } 2012 2013 for _, v := range tg.Volumes { 2014 if v.Type != structs.VolumeTypeCSI { 2015 continue 2016 } 2017 ids[v.Source] = a.Namespace 2018 } 2019 } 2020 2021 // Lookup the raw CSIVolumes to match the other list interfaces 2022 iter := NewSliceIterator() 2023 txn := s.db.Txn(false) 2024 for id, namespace := range ids { 2025 raw, err := txn.First("csi_volumes", "id", namespace, id) 2026 if err != nil { 2027 return nil, fmt.Errorf("volume lookup failed: %s %v", id, err) 2028 } 2029 iter.Add(raw) 2030 } 2031 2032 return iter, nil 2033 } 2034 2035 // CSIVolumesByNamespace looks up the entire csi_volumes table 2036 func (s *StateStore) CSIVolumesByNamespace(ws memdb.WatchSet, namespace string) (memdb.ResultIterator, error) { 2037 txn := s.db.Txn(false) 2038 2039 iter, err := txn.Get("csi_volumes", "id_prefix", namespace, "") 2040 if err != nil { 2041 return nil, fmt.Errorf("volume lookup failed: %v", err) 2042 } 2043 ws.Add(iter.WatchCh()) 2044 2045 return iter, nil 2046 } 2047 2048 // CSIVolumeClaim updates the volume's claim count and allocation list 2049 func (s *StateStore) CSIVolumeClaim(index uint64, namespace, id string, claim *structs.CSIVolumeClaim) error { 2050 txn := s.db.Txn(true) 2051 defer txn.Abort() 2052 ws := memdb.NewWatchSet() 2053 2054 row, err := txn.First("csi_volumes", "id", namespace, id) 2055 if err != nil { 2056 return fmt.Errorf("volume lookup failed: %s: %v", id, err) 2057 } 2058 if row == nil { 2059 return fmt.Errorf("volume not found: %s", id) 2060 } 2061 2062 orig, ok := row.(*structs.CSIVolume) 2063 if !ok { 2064 return fmt.Errorf("volume row conversion error") 2065 } 2066 2067 var alloc *structs.Allocation 2068 if claim.Mode != structs.CSIVolumeClaimRelease { 2069 alloc, err = s.AllocByID(ws, claim.AllocationID) 2070 if err != nil { 2071 s.logger.Error("AllocByID failed", "error", err) 2072 return fmt.Errorf(structs.ErrUnknownAllocationPrefix) 2073 } 2074 if alloc == nil { 2075 s.logger.Error("AllocByID failed to find alloc", "alloc_id", claim.AllocationID) 2076 if err != nil { 2077 return fmt.Errorf(structs.ErrUnknownAllocationPrefix) 2078 } 2079 } 2080 } 2081 2082 volume, err := s.CSIVolumeDenormalizePlugins(ws, orig.Copy()) 2083 if err != nil { 2084 return err 2085 } 2086 2087 volume, err = s.CSIVolumeDenormalize(ws, volume) 2088 if err != nil { 2089 return err 2090 } 2091 2092 // in the case of a job deregistration, there will be no allocation ID 2093 // for the claim but we still want to write an updated index to the volume 2094 // so that volume reaping is triggered 2095 if claim.AllocationID != "" { 2096 err = volume.Claim(claim, alloc) 2097 if err != nil { 2098 return err 2099 } 2100 } 2101 2102 volume.ModifyIndex = index 2103 2104 if err = txn.Insert("csi_volumes", volume); err != nil { 2105 return fmt.Errorf("volume update failed: %s: %v", id, err) 2106 } 2107 2108 if err = txn.Insert("index", &IndexEntry{"csi_volumes", index}); err != nil { 2109 return fmt.Errorf("index update failed: %v", err) 2110 } 2111 2112 txn.Commit() 2113 return nil 2114 } 2115 2116 // CSIVolumeDeregister removes the volume from the server 2117 func (s *StateStore) CSIVolumeDeregister(index uint64, namespace string, ids []string) error { 2118 txn := s.db.Txn(true) 2119 defer txn.Abort() 2120 2121 for _, id := range ids { 2122 existing, err := txn.First("csi_volumes", "id_prefix", namespace, id) 2123 if err != nil { 2124 return fmt.Errorf("volume lookup failed: %s: %v", id, err) 2125 } 2126 2127 if existing == nil { 2128 return fmt.Errorf("volume not found: %s", id) 2129 } 2130 2131 vol, ok := existing.(*structs.CSIVolume) 2132 if !ok { 2133 return fmt.Errorf("volume row conversion error: %s", id) 2134 } 2135 2136 if vol.InUse() { 2137 return fmt.Errorf("volume in use: %s", id) 2138 } 2139 2140 if err = txn.Delete("csi_volumes", existing); err != nil { 2141 return fmt.Errorf("volume delete failed: %s: %v", id, err) 2142 } 2143 } 2144 2145 if err := txn.Insert("index", &IndexEntry{"csi_volumes", index}); err != nil { 2146 return fmt.Errorf("index update failed: %v", err) 2147 } 2148 2149 txn.Commit() 2150 return nil 2151 } 2152 2153 // CSIVolumeDenormalizePlugins returns a CSIVolume with current health and plugins, but 2154 // without allocations 2155 // Use this for current volume metadata, handling lists of volumes 2156 // Use CSIVolumeDenormalize for volumes containing both health and current allocations 2157 func (s *StateStore) CSIVolumeDenormalizePlugins(ws memdb.WatchSet, vol *structs.CSIVolume) (*structs.CSIVolume, error) { 2158 if vol == nil { 2159 return nil, nil 2160 } 2161 // Lookup CSIPlugin, the health records, and calculate volume health 2162 txn := s.db.Txn(false) 2163 defer txn.Abort() 2164 2165 plug, err := s.CSIPluginByID(ws, vol.PluginID) 2166 if err != nil { 2167 return nil, fmt.Errorf("plugin lookup error: %s %v", vol.PluginID, err) 2168 } 2169 if plug == nil { 2170 vol.ControllersHealthy = 0 2171 vol.NodesHealthy = 0 2172 vol.Schedulable = false 2173 return vol, nil 2174 } 2175 2176 vol.Provider = plug.Provider 2177 vol.ProviderVersion = plug.Version 2178 vol.ControllerRequired = plug.ControllerRequired 2179 vol.ControllersHealthy = plug.ControllersHealthy 2180 vol.NodesHealthy = plug.NodesHealthy 2181 // This number is incorrect! The expected number of node plugins is actually this + 2182 // the number of blocked evaluations for the jobs controlling these plugins 2183 vol.ControllersExpected = len(plug.Controllers) 2184 vol.NodesExpected = len(plug.Nodes) 2185 2186 vol.Schedulable = vol.NodesHealthy > 0 2187 if vol.ControllerRequired { 2188 vol.Schedulable = vol.ControllersHealthy > 0 && vol.Schedulable 2189 } 2190 2191 return vol, nil 2192 } 2193 2194 // CSIVolumeDenormalize returns a CSIVolume with allocations 2195 func (s *StateStore) CSIVolumeDenormalize(ws memdb.WatchSet, vol *structs.CSIVolume) (*structs.CSIVolume, error) { 2196 for id := range vol.ReadAllocs { 2197 a, err := s.AllocByID(ws, id) 2198 if err != nil { 2199 return nil, err 2200 } 2201 if a != nil { 2202 vol.ReadAllocs[id] = a 2203 // COMPAT(1.0): the CSIVolumeClaim fields were added 2204 // after 0.11.1, so claims made before that may be 2205 // missing this value. (same for WriteAlloc below) 2206 if _, ok := vol.ReadClaims[id]; !ok { 2207 vol.ReadClaims[id] = &structs.CSIVolumeClaim{ 2208 AllocationID: a.ID, 2209 NodeID: a.NodeID, 2210 Mode: structs.CSIVolumeClaimRead, 2211 State: structs.CSIVolumeClaimStateTaken, 2212 } 2213 } 2214 } 2215 } 2216 2217 for id := range vol.WriteAllocs { 2218 a, err := s.AllocByID(ws, id) 2219 if err != nil { 2220 return nil, err 2221 } 2222 if a != nil { 2223 vol.WriteAllocs[id] = a 2224 if _, ok := vol.WriteClaims[id]; !ok { 2225 vol.WriteClaims[id] = &structs.CSIVolumeClaim{ 2226 AllocationID: a.ID, 2227 NodeID: a.NodeID, 2228 Mode: structs.CSIVolumeClaimWrite, 2229 State: structs.CSIVolumeClaimStateTaken, 2230 } 2231 } 2232 } 2233 } 2234 2235 return vol, nil 2236 } 2237 2238 // CSIPlugins returns the unfiltered list of all plugin health status 2239 func (s *StateStore) CSIPlugins(ws memdb.WatchSet) (memdb.ResultIterator, error) { 2240 txn := s.db.Txn(false) 2241 defer txn.Abort() 2242 2243 iter, err := txn.Get("csi_plugins", "id") 2244 if err != nil { 2245 return nil, fmt.Errorf("csi_plugins lookup failed: %v", err) 2246 } 2247 2248 ws.Add(iter.WatchCh()) 2249 2250 return iter, nil 2251 } 2252 2253 // CSIPluginsByIDPrefix supports search 2254 func (s *StateStore) CSIPluginsByIDPrefix(ws memdb.WatchSet, pluginID string) (memdb.ResultIterator, error) { 2255 txn := s.db.Txn(false) 2256 2257 iter, err := txn.Get("csi_plugins", "id_prefix", pluginID) 2258 if err != nil { 2259 return nil, err 2260 } 2261 2262 ws.Add(iter.WatchCh()) 2263 2264 return iter, nil 2265 } 2266 2267 // CSIPluginByID returns the one named CSIPlugin 2268 func (s *StateStore) CSIPluginByID(ws memdb.WatchSet, id string) (*structs.CSIPlugin, error) { 2269 txn := s.db.Txn(false) 2270 defer txn.Abort() 2271 2272 raw, err := txn.First("csi_plugins", "id_prefix", id) 2273 if err != nil { 2274 return nil, fmt.Errorf("csi_plugin lookup failed: %s %v", id, err) 2275 } 2276 2277 if raw == nil { 2278 return nil, nil 2279 } 2280 2281 plug := raw.(*structs.CSIPlugin) 2282 2283 return plug, nil 2284 } 2285 2286 // CSIPluginDenormalize returns a CSIPlugin with allocation details 2287 func (s *StateStore) CSIPluginDenormalize(ws memdb.WatchSet, plug *structs.CSIPlugin) (*structs.CSIPlugin, error) { 2288 if plug == nil { 2289 return nil, nil 2290 } 2291 2292 // Get the unique list of allocation ids 2293 ids := map[string]struct{}{} 2294 for _, info := range plug.Controllers { 2295 ids[info.AllocID] = struct{}{} 2296 } 2297 for _, info := range plug.Nodes { 2298 ids[info.AllocID] = struct{}{} 2299 } 2300 2301 for id := range ids { 2302 alloc, err := s.AllocByID(ws, id) 2303 if err != nil { 2304 return nil, err 2305 } 2306 if alloc == nil { 2307 continue 2308 } 2309 plug.Allocations = append(plug.Allocations, alloc.Stub()) 2310 } 2311 2312 return plug, nil 2313 } 2314 2315 // UpsertCSIPlugin writes the plugin to the state store. Note: there 2316 // is currently no raft message for this, as it's intended to support 2317 // testing use cases. 2318 func (s *StateStore) UpsertCSIPlugin(index uint64, plug *structs.CSIPlugin) error { 2319 txn := s.db.Txn(true) 2320 defer txn.Abort() 2321 2322 existing, err := txn.First("csi_plugins", "id", plug.ID) 2323 if err != nil { 2324 return fmt.Errorf("csi_plugin lookup error: %s %v", plug.ID, err) 2325 } 2326 2327 plug.ModifyIndex = index 2328 if existing != nil { 2329 plug.CreateIndex = existing.(*structs.CSIPlugin).CreateIndex 2330 } 2331 2332 err = txn.Insert("csi_plugins", plug) 2333 if err != nil { 2334 return fmt.Errorf("csi_plugins insert error: %v", err) 2335 } 2336 if err := txn.Insert("index", &IndexEntry{"csi_plugins", index}); err != nil { 2337 return fmt.Errorf("index update failed: %v", err) 2338 } 2339 txn.Commit() 2340 return nil 2341 } 2342 2343 // DeleteCSIPlugin deletes the plugin if it's not in use. 2344 func (s *StateStore) DeleteCSIPlugin(index uint64, id string) error { 2345 txn := s.db.Txn(true) 2346 defer txn.Abort() 2347 ws := memdb.NewWatchSet() 2348 2349 plug, err := s.CSIPluginByID(ws, id) 2350 if err != nil { 2351 return err 2352 } 2353 2354 if plug == nil { 2355 return nil 2356 } 2357 2358 plug, err = s.CSIPluginDenormalize(ws, plug.Copy()) 2359 if err != nil { 2360 return err 2361 } 2362 if !plug.IsEmpty() { 2363 return fmt.Errorf("plugin in use") 2364 } 2365 2366 err = txn.Delete("csi_plugins", plug) 2367 if err != nil { 2368 return fmt.Errorf("csi_plugins delete error: %v", err) 2369 } 2370 txn.Commit() 2371 return nil 2372 } 2373 2374 // UpsertPeriodicLaunch is used to register a launch or update it. 2375 func (s *StateStore) UpsertPeriodicLaunch(index uint64, launch *structs.PeriodicLaunch) error { 2376 txn := s.db.Txn(true) 2377 defer txn.Abort() 2378 2379 // Check if the job already exists 2380 existing, err := txn.First("periodic_launch", "id", launch.Namespace, launch.ID) 2381 if err != nil { 2382 return fmt.Errorf("periodic launch lookup failed: %v", err) 2383 } 2384 2385 // Setup the indexes correctly 2386 if existing != nil { 2387 launch.CreateIndex = existing.(*structs.PeriodicLaunch).CreateIndex 2388 launch.ModifyIndex = index 2389 } else { 2390 launch.CreateIndex = index 2391 launch.ModifyIndex = index 2392 } 2393 2394 // Insert the job 2395 if err := txn.Insert("periodic_launch", launch); err != nil { 2396 return fmt.Errorf("launch insert failed: %v", err) 2397 } 2398 if err := txn.Insert("index", &IndexEntry{"periodic_launch", index}); err != nil { 2399 return fmt.Errorf("index update failed: %v", err) 2400 } 2401 2402 txn.Commit() 2403 return nil 2404 } 2405 2406 // DeletePeriodicLaunch is used to delete the periodic launch 2407 func (s *StateStore) DeletePeriodicLaunch(index uint64, namespace, jobID string) error { 2408 txn := s.db.Txn(true) 2409 defer txn.Abort() 2410 2411 err := s.DeletePeriodicLaunchTxn(index, namespace, jobID, txn) 2412 if err == nil { 2413 txn.Commit() 2414 } 2415 return err 2416 } 2417 2418 // DeletePeriodicLaunchTxn is used to delete the periodic launch, like DeletePeriodicLaunch 2419 // but in a transaction. Useful for when making multiple modifications atomically 2420 func (s *StateStore) DeletePeriodicLaunchTxn(index uint64, namespace, jobID string, txn Txn) error { 2421 // Lookup the launch 2422 existing, err := txn.First("periodic_launch", "id", namespace, jobID) 2423 if err != nil { 2424 return fmt.Errorf("launch lookup failed: %v", err) 2425 } 2426 if existing == nil { 2427 return fmt.Errorf("launch not found") 2428 } 2429 2430 // Delete the launch 2431 if err := txn.Delete("periodic_launch", existing); err != nil { 2432 return fmt.Errorf("launch delete failed: %v", err) 2433 } 2434 if err := txn.Insert("index", &IndexEntry{"periodic_launch", index}); err != nil { 2435 return fmt.Errorf("index update failed: %v", err) 2436 } 2437 2438 return nil 2439 } 2440 2441 // PeriodicLaunchByID is used to lookup a periodic launch by the periodic job 2442 // ID. 2443 func (s *StateStore) PeriodicLaunchByID(ws memdb.WatchSet, namespace, id string) (*structs.PeriodicLaunch, error) { 2444 txn := s.db.Txn(false) 2445 2446 watchCh, existing, err := txn.FirstWatch("periodic_launch", "id", namespace, id) 2447 if err != nil { 2448 return nil, fmt.Errorf("periodic launch lookup failed: %v", err) 2449 } 2450 2451 ws.Add(watchCh) 2452 2453 if existing != nil { 2454 return existing.(*structs.PeriodicLaunch), nil 2455 } 2456 return nil, nil 2457 } 2458 2459 // PeriodicLaunches returns an iterator over all the periodic launches 2460 func (s *StateStore) PeriodicLaunches(ws memdb.WatchSet) (memdb.ResultIterator, error) { 2461 txn := s.db.Txn(false) 2462 2463 // Walk the entire table 2464 iter, err := txn.Get("periodic_launch", "id") 2465 if err != nil { 2466 return nil, err 2467 } 2468 2469 ws.Add(iter.WatchCh()) 2470 2471 return iter, nil 2472 } 2473 2474 // UpsertEvals is used to upsert a set of evaluations 2475 func (s *StateStore) UpsertEvals(index uint64, evals []*structs.Evaluation) error { 2476 txn := s.db.Txn(true) 2477 defer txn.Abort() 2478 2479 err := s.UpsertEvalsTxn(index, evals, txn) 2480 if err == nil { 2481 txn.Commit() 2482 } 2483 return err 2484 } 2485 2486 // UpsertEvals is used to upsert a set of evaluations, like UpsertEvals 2487 // but in a transaction. Useful for when making multiple modifications atomically 2488 func (s *StateStore) UpsertEvalsTxn(index uint64, evals []*structs.Evaluation, txn Txn) error { 2489 // Do a nested upsert 2490 jobs := make(map[structs.NamespacedID]string, len(evals)) 2491 for _, eval := range evals { 2492 if err := s.nestedUpsertEval(txn, index, eval); err != nil { 2493 return err 2494 } 2495 2496 tuple := structs.NamespacedID{ 2497 ID: eval.JobID, 2498 Namespace: eval.Namespace, 2499 } 2500 jobs[tuple] = "" 2501 } 2502 2503 // Set the job's status 2504 if err := s.setJobStatuses(index, txn, jobs, false); err != nil { 2505 return fmt.Errorf("setting job status failed: %v", err) 2506 } 2507 2508 return nil 2509 } 2510 2511 // nestedUpsertEvaluation is used to nest an evaluation upsert within a transaction 2512 func (s *StateStore) nestedUpsertEval(txn *memdb.Txn, index uint64, eval *structs.Evaluation) error { 2513 // Lookup the evaluation 2514 existing, err := txn.First("evals", "id", eval.ID) 2515 if err != nil { 2516 return fmt.Errorf("eval lookup failed: %v", err) 2517 } 2518 2519 // Update the indexes 2520 if existing != nil { 2521 eval.CreateIndex = existing.(*structs.Evaluation).CreateIndex 2522 eval.ModifyIndex = index 2523 } else { 2524 eval.CreateIndex = index 2525 eval.ModifyIndex = index 2526 } 2527 2528 // Update the job summary 2529 summaryRaw, err := txn.First("job_summary", "id", eval.Namespace, eval.JobID) 2530 if err != nil { 2531 return fmt.Errorf("job summary lookup failed: %v", err) 2532 } 2533 if summaryRaw != nil { 2534 js := summaryRaw.(*structs.JobSummary).Copy() 2535 hasSummaryChanged := false 2536 for tg, num := range eval.QueuedAllocations { 2537 if summary, ok := js.Summary[tg]; ok { 2538 if summary.Queued != num { 2539 summary.Queued = num 2540 js.Summary[tg] = summary 2541 hasSummaryChanged = true 2542 } 2543 } else { 2544 s.logger.Error("unable to update queued for job and task group", "job_id", eval.JobID, "task_group", tg, "namespace", eval.Namespace) 2545 } 2546 } 2547 2548 // Insert the job summary 2549 if hasSummaryChanged { 2550 js.ModifyIndex = index 2551 if err := txn.Insert("job_summary", js); err != nil { 2552 return fmt.Errorf("job summary insert failed: %v", err) 2553 } 2554 if err := txn.Insert("index", &IndexEntry{"job_summary", index}); err != nil { 2555 return fmt.Errorf("index update failed: %v", err) 2556 } 2557 } 2558 } 2559 2560 // Check if the job has any blocked evaluations and cancel them 2561 if eval.Status == structs.EvalStatusComplete && len(eval.FailedTGAllocs) == 0 { 2562 // Get the blocked evaluation for a job if it exists 2563 iter, err := txn.Get("evals", "job", eval.Namespace, eval.JobID, structs.EvalStatusBlocked) 2564 if err != nil { 2565 return fmt.Errorf("failed to get blocked evals for job %q in namespace %q: %v", eval.JobID, eval.Namespace, err) 2566 } 2567 2568 var blocked []*structs.Evaluation 2569 for { 2570 raw := iter.Next() 2571 if raw == nil { 2572 break 2573 } 2574 blocked = append(blocked, raw.(*structs.Evaluation)) 2575 } 2576 2577 // Go through and update the evals 2578 for _, eval := range blocked { 2579 newEval := eval.Copy() 2580 newEval.Status = structs.EvalStatusCancelled 2581 newEval.StatusDescription = fmt.Sprintf("evaluation %q successful", newEval.ID) 2582 newEval.ModifyIndex = index 2583 2584 if err := txn.Insert("evals", newEval); err != nil { 2585 return fmt.Errorf("eval insert failed: %v", err) 2586 } 2587 } 2588 } 2589 2590 // Insert the eval 2591 if err := txn.Insert("evals", eval); err != nil { 2592 return fmt.Errorf("eval insert failed: %v", err) 2593 } 2594 if err := txn.Insert("index", &IndexEntry{"evals", index}); err != nil { 2595 return fmt.Errorf("index update failed: %v", err) 2596 } 2597 return nil 2598 } 2599 2600 // updateEvalModifyIndex is used to update the modify index of an evaluation that has been 2601 // through a scheduler pass. This is done as part of plan apply. It ensures that when a subsequent 2602 // scheduler workers process a re-queued evaluation it sees any partial updates from the plan apply. 2603 func (s *StateStore) updateEvalModifyIndex(txn *memdb.Txn, index uint64, evalID string) error { 2604 // Lookup the evaluation 2605 existing, err := txn.First("evals", "id", evalID) 2606 if err != nil { 2607 return fmt.Errorf("eval lookup failed: %v", err) 2608 } 2609 if existing == nil { 2610 s.logger.Error("unable to find eval", "eval_id", evalID) 2611 return fmt.Errorf("unable to find eval id %q", evalID) 2612 } 2613 eval := existing.(*structs.Evaluation).Copy() 2614 // Update the indexes 2615 eval.ModifyIndex = index 2616 2617 // Insert the eval 2618 if err := txn.Insert("evals", eval); err != nil { 2619 return fmt.Errorf("eval insert failed: %v", err) 2620 } 2621 if err := txn.Insert("index", &IndexEntry{"evals", index}); err != nil { 2622 return fmt.Errorf("index update failed: %v", err) 2623 } 2624 return nil 2625 } 2626 2627 // DeleteEval is used to delete an evaluation 2628 func (s *StateStore) DeleteEval(index uint64, evals []string, allocs []string) error { 2629 txn := s.db.Txn(true) 2630 defer txn.Abort() 2631 2632 jobs := make(map[structs.NamespacedID]string, len(evals)) 2633 for _, eval := range evals { 2634 existing, err := txn.First("evals", "id", eval) 2635 if err != nil { 2636 return fmt.Errorf("eval lookup failed: %v", err) 2637 } 2638 if existing == nil { 2639 continue 2640 } 2641 if err := txn.Delete("evals", existing); err != nil { 2642 return fmt.Errorf("eval delete failed: %v", err) 2643 } 2644 eval := existing.(*structs.Evaluation) 2645 2646 tuple := structs.NamespacedID{ 2647 ID: eval.JobID, 2648 Namespace: eval.Namespace, 2649 } 2650 jobs[tuple] = "" 2651 } 2652 2653 for _, alloc := range allocs { 2654 raw, err := txn.First("allocs", "id", alloc) 2655 if err != nil { 2656 return fmt.Errorf("alloc lookup failed: %v", err) 2657 } 2658 if raw == nil { 2659 continue 2660 } 2661 if err := txn.Delete("allocs", raw); err != nil { 2662 return fmt.Errorf("alloc delete failed: %v", err) 2663 } 2664 } 2665 2666 // Update the indexes 2667 if err := txn.Insert("index", &IndexEntry{"evals", index}); err != nil { 2668 return fmt.Errorf("index update failed: %v", err) 2669 } 2670 if err := txn.Insert("index", &IndexEntry{"allocs", index}); err != nil { 2671 return fmt.Errorf("index update failed: %v", err) 2672 } 2673 2674 // Set the job's status 2675 if err := s.setJobStatuses(index, txn, jobs, true); err != nil { 2676 return fmt.Errorf("setting job status failed: %v", err) 2677 } 2678 2679 txn.Commit() 2680 return nil 2681 } 2682 2683 // EvalByID is used to lookup an eval by its ID 2684 func (s *StateStore) EvalByID(ws memdb.WatchSet, id string) (*structs.Evaluation, error) { 2685 txn := s.db.Txn(false) 2686 2687 watchCh, existing, err := txn.FirstWatch("evals", "id", id) 2688 if err != nil { 2689 return nil, fmt.Errorf("eval lookup failed: %v", err) 2690 } 2691 2692 ws.Add(watchCh) 2693 2694 if existing != nil { 2695 return existing.(*structs.Evaluation), nil 2696 } 2697 return nil, nil 2698 } 2699 2700 // EvalsByIDPrefix is used to lookup evaluations by prefix in a particular 2701 // namespace 2702 func (s *StateStore) EvalsByIDPrefix(ws memdb.WatchSet, namespace, id string) (memdb.ResultIterator, error) { 2703 txn := s.db.Txn(false) 2704 2705 // Get an iterator over all evals by the id prefix 2706 iter, err := txn.Get("evals", "id_prefix", id) 2707 if err != nil { 2708 return nil, fmt.Errorf("eval lookup failed: %v", err) 2709 } 2710 2711 ws.Add(iter.WatchCh()) 2712 2713 // Wrap the iterator in a filter 2714 wrap := memdb.NewFilterIterator(iter, evalNamespaceFilter(namespace)) 2715 return wrap, nil 2716 } 2717 2718 // evalNamespaceFilter returns a filter function that filters all evaluations 2719 // not in the given namespace. 2720 func evalNamespaceFilter(namespace string) func(interface{}) bool { 2721 return func(raw interface{}) bool { 2722 eval, ok := raw.(*structs.Evaluation) 2723 if !ok { 2724 return true 2725 } 2726 2727 return eval.Namespace != namespace 2728 } 2729 } 2730 2731 // EvalsByJob returns all the evaluations by job id 2732 func (s *StateStore) EvalsByJob(ws memdb.WatchSet, namespace, jobID string) ([]*structs.Evaluation, error) { 2733 txn := s.db.Txn(false) 2734 2735 // Get an iterator over the node allocations 2736 iter, err := txn.Get("evals", "job_prefix", namespace, jobID) 2737 if err != nil { 2738 return nil, err 2739 } 2740 2741 ws.Add(iter.WatchCh()) 2742 2743 var out []*structs.Evaluation 2744 for { 2745 raw := iter.Next() 2746 if raw == nil { 2747 break 2748 } 2749 2750 e := raw.(*structs.Evaluation) 2751 2752 // Filter non-exact matches 2753 if e.JobID != jobID { 2754 continue 2755 } 2756 2757 out = append(out, e) 2758 } 2759 return out, nil 2760 } 2761 2762 // Evals returns an iterator over all the evaluations 2763 func (s *StateStore) Evals(ws memdb.WatchSet) (memdb.ResultIterator, error) { 2764 txn := s.db.Txn(false) 2765 2766 // Walk the entire table 2767 iter, err := txn.Get("evals", "id") 2768 if err != nil { 2769 return nil, err 2770 } 2771 2772 ws.Add(iter.WatchCh()) 2773 2774 return iter, nil 2775 } 2776 2777 // EvalsByNamespace returns an iterator over all the evaluations in the given 2778 // namespace 2779 func (s *StateStore) EvalsByNamespace(ws memdb.WatchSet, namespace string) (memdb.ResultIterator, error) { 2780 txn := s.db.Txn(false) 2781 2782 // Walk the entire table 2783 iter, err := txn.Get("evals", "namespace", namespace) 2784 if err != nil { 2785 return nil, err 2786 } 2787 2788 ws.Add(iter.WatchCh()) 2789 2790 return iter, nil 2791 } 2792 2793 // UpdateAllocsFromClient is used to update an allocation based on input 2794 // from a client. While the schedulers are the authority on the allocation for 2795 // most things, some updates are authoritative from the client. Specifically, 2796 // the desired state comes from the schedulers, while the actual state comes 2797 // from clients. 2798 func (s *StateStore) UpdateAllocsFromClient(index uint64, allocs []*structs.Allocation) error { 2799 txn := s.db.Txn(true) 2800 defer txn.Abort() 2801 2802 // Handle each of the updated allocations 2803 for _, alloc := range allocs { 2804 if err := s.nestedUpdateAllocFromClient(txn, index, alloc); err != nil { 2805 return err 2806 } 2807 } 2808 2809 // Update the indexes 2810 if err := txn.Insert("index", &IndexEntry{"allocs", index}); err != nil { 2811 return fmt.Errorf("index update failed: %v", err) 2812 } 2813 2814 txn.Commit() 2815 return nil 2816 } 2817 2818 // nestedUpdateAllocFromClient is used to nest an update of an allocation with client status 2819 func (s *StateStore) nestedUpdateAllocFromClient(txn *memdb.Txn, index uint64, alloc *structs.Allocation) error { 2820 // Look for existing alloc 2821 existing, err := txn.First("allocs", "id", alloc.ID) 2822 if err != nil { 2823 return fmt.Errorf("alloc lookup failed: %v", err) 2824 } 2825 2826 // Nothing to do if this does not exist 2827 if existing == nil { 2828 return nil 2829 } 2830 exist := existing.(*structs.Allocation) 2831 2832 // Copy everything from the existing allocation 2833 copyAlloc := exist.Copy() 2834 2835 // Pull in anything the client is the authority on 2836 copyAlloc.ClientStatus = alloc.ClientStatus 2837 copyAlloc.ClientDescription = alloc.ClientDescription 2838 copyAlloc.TaskStates = alloc.TaskStates 2839 2840 // The client can only set its deployment health and timestamp, so just take 2841 // those 2842 if copyAlloc.DeploymentStatus != nil && alloc.DeploymentStatus != nil { 2843 oldHasHealthy := copyAlloc.DeploymentStatus.HasHealth() 2844 newHasHealthy := alloc.DeploymentStatus.HasHealth() 2845 2846 // We got new health information from the client 2847 if newHasHealthy && (!oldHasHealthy || *copyAlloc.DeploymentStatus.Healthy != *alloc.DeploymentStatus.Healthy) { 2848 // Updated deployment health and timestamp 2849 copyAlloc.DeploymentStatus.Healthy = helper.BoolToPtr(*alloc.DeploymentStatus.Healthy) 2850 copyAlloc.DeploymentStatus.Timestamp = alloc.DeploymentStatus.Timestamp 2851 copyAlloc.DeploymentStatus.ModifyIndex = index 2852 } 2853 } else if alloc.DeploymentStatus != nil { 2854 // First time getting a deployment status so copy everything and just 2855 // set the index 2856 copyAlloc.DeploymentStatus = alloc.DeploymentStatus.Copy() 2857 copyAlloc.DeploymentStatus.ModifyIndex = index 2858 } 2859 2860 // Update the modify index 2861 copyAlloc.ModifyIndex = index 2862 2863 // Update the modify time 2864 copyAlloc.ModifyTime = alloc.ModifyTime 2865 2866 if err := s.updateDeploymentWithAlloc(index, copyAlloc, exist, txn); err != nil { 2867 return fmt.Errorf("error updating deployment: %v", err) 2868 } 2869 2870 if err := s.updateSummaryWithAlloc(index, copyAlloc, exist, txn); err != nil { 2871 return fmt.Errorf("error updating job summary: %v", err) 2872 } 2873 2874 if err := s.updateEntWithAlloc(index, copyAlloc, exist, txn); err != nil { 2875 return err 2876 } 2877 2878 if err := s.updatePluginWithAlloc(index, copyAlloc, txn); err != nil { 2879 return err 2880 } 2881 2882 // Update the allocation 2883 if err := txn.Insert("allocs", copyAlloc); err != nil { 2884 return fmt.Errorf("alloc insert failed: %v", err) 2885 } 2886 2887 // Set the job's status 2888 forceStatus := "" 2889 if !copyAlloc.TerminalStatus() { 2890 forceStatus = structs.JobStatusRunning 2891 } 2892 2893 tuple := structs.NamespacedID{ 2894 ID: exist.JobID, 2895 Namespace: exist.Namespace, 2896 } 2897 jobs := map[structs.NamespacedID]string{tuple: forceStatus} 2898 2899 if err := s.setJobStatuses(index, txn, jobs, false); err != nil { 2900 return fmt.Errorf("setting job status failed: %v", err) 2901 } 2902 return nil 2903 } 2904 2905 // UpsertAllocs is used to evict a set of allocations and allocate new ones at 2906 // the same time. 2907 func (s *StateStore) UpsertAllocs(index uint64, allocs []*structs.Allocation) error { 2908 txn := s.db.Txn(true) 2909 defer txn.Abort() 2910 if err := s.upsertAllocsImpl(index, allocs, txn); err != nil { 2911 return err 2912 } 2913 txn.Commit() 2914 return nil 2915 } 2916 2917 // upsertAllocs is the actual implementation of UpsertAllocs so that it may be 2918 // used with an existing transaction. 2919 func (s *StateStore) upsertAllocsImpl(index uint64, allocs []*structs.Allocation, txn *memdb.Txn) error { 2920 // Handle the allocations 2921 jobs := make(map[structs.NamespacedID]string, 1) 2922 for _, alloc := range allocs { 2923 existing, err := txn.First("allocs", "id", alloc.ID) 2924 if err != nil { 2925 return fmt.Errorf("alloc lookup failed: %v", err) 2926 } 2927 exist, _ := existing.(*structs.Allocation) 2928 2929 if exist == nil { 2930 alloc.CreateIndex = index 2931 alloc.ModifyIndex = index 2932 alloc.AllocModifyIndex = index 2933 if alloc.DeploymentStatus != nil { 2934 alloc.DeploymentStatus.ModifyIndex = index 2935 } 2936 2937 // Issue https://github.com/hashicorp/nomad/issues/2583 uncovered 2938 // the a race between a forced garbage collection and the scheduler 2939 // marking an allocation as terminal. The issue is that the 2940 // allocation from the scheduler has its job normalized and the FSM 2941 // will only denormalize if the allocation is not terminal. However 2942 // if the allocation is garbage collected, that will result in a 2943 // allocation being upserted for the first time without a job 2944 // attached. By returning an error here, it will cause the FSM to 2945 // error, causing the plan_apply to error and thus causing the 2946 // evaluation to be failed. This will force an index refresh that 2947 // should solve this issue. 2948 if alloc.Job == nil { 2949 return fmt.Errorf("attempting to upsert allocation %q without a job", alloc.ID) 2950 } 2951 } else { 2952 alloc.CreateIndex = exist.CreateIndex 2953 alloc.ModifyIndex = index 2954 alloc.AllocModifyIndex = index 2955 2956 // Keep the clients task states 2957 alloc.TaskStates = exist.TaskStates 2958 2959 // If the scheduler is marking this allocation as lost we do not 2960 // want to reuse the status of the existing allocation. 2961 if alloc.ClientStatus != structs.AllocClientStatusLost { 2962 alloc.ClientStatus = exist.ClientStatus 2963 alloc.ClientDescription = exist.ClientDescription 2964 } 2965 2966 // The job has been denormalized so re-attach the original job 2967 if alloc.Job == nil { 2968 alloc.Job = exist.Job 2969 } 2970 } 2971 2972 // OPTIMIZATION: 2973 // These should be given a map of new to old allocation and the updates 2974 // should be one on all changes. The current implementation causes O(n) 2975 // lookups/copies/insertions rather than O(1) 2976 if err := s.updateDeploymentWithAlloc(index, alloc, exist, txn); err != nil { 2977 return fmt.Errorf("error updating deployment: %v", err) 2978 } 2979 2980 if err := s.updateSummaryWithAlloc(index, alloc, exist, txn); err != nil { 2981 return fmt.Errorf("error updating job summary: %v", err) 2982 } 2983 2984 if err := s.updateEntWithAlloc(index, alloc, exist, txn); err != nil { 2985 return err 2986 } 2987 2988 if err := s.updatePluginWithAlloc(index, alloc, txn); err != nil { 2989 return err 2990 } 2991 2992 if err := txn.Insert("allocs", alloc); err != nil { 2993 return fmt.Errorf("alloc insert failed: %v", err) 2994 } 2995 2996 if alloc.PreviousAllocation != "" { 2997 prevAlloc, err := txn.First("allocs", "id", alloc.PreviousAllocation) 2998 if err != nil { 2999 return fmt.Errorf("alloc lookup failed: %v", err) 3000 } 3001 existingPrevAlloc, _ := prevAlloc.(*structs.Allocation) 3002 if existingPrevAlloc != nil { 3003 prevAllocCopy := existingPrevAlloc.Copy() 3004 prevAllocCopy.NextAllocation = alloc.ID 3005 prevAllocCopy.ModifyIndex = index 3006 if err := txn.Insert("allocs", prevAllocCopy); err != nil { 3007 return fmt.Errorf("alloc insert failed: %v", err) 3008 } 3009 } 3010 } 3011 3012 // If the allocation is running, force the job to running status. 3013 forceStatus := "" 3014 if !alloc.TerminalStatus() { 3015 forceStatus = structs.JobStatusRunning 3016 } 3017 3018 tuple := structs.NamespacedID{ 3019 ID: alloc.JobID, 3020 Namespace: alloc.Namespace, 3021 } 3022 jobs[tuple] = forceStatus 3023 } 3024 3025 // Update the indexes 3026 if err := txn.Insert("index", &IndexEntry{"allocs", index}); err != nil { 3027 return fmt.Errorf("index update failed: %v", err) 3028 } 3029 3030 // Set the job's status 3031 if err := s.setJobStatuses(index, txn, jobs, false); err != nil { 3032 return fmt.Errorf("setting job status failed: %v", err) 3033 } 3034 3035 return nil 3036 } 3037 3038 // UpdateAllocsDesiredTransitions is used to update a set of allocations 3039 // desired transitions. 3040 func (s *StateStore) UpdateAllocsDesiredTransitions(index uint64, allocs map[string]*structs.DesiredTransition, 3041 evals []*structs.Evaluation) error { 3042 3043 txn := s.db.Txn(true) 3044 defer txn.Abort() 3045 3046 // Handle each of the updated allocations 3047 for id, transition := range allocs { 3048 if err := s.nestedUpdateAllocDesiredTransition(txn, index, id, transition); err != nil { 3049 return err 3050 } 3051 } 3052 3053 for _, eval := range evals { 3054 if err := s.nestedUpsertEval(txn, index, eval); err != nil { 3055 return err 3056 } 3057 } 3058 3059 // Update the indexes 3060 if err := txn.Insert("index", &IndexEntry{"allocs", index}); err != nil { 3061 return fmt.Errorf("index update failed: %v", err) 3062 } 3063 3064 txn.Commit() 3065 return nil 3066 } 3067 3068 // nestedUpdateAllocDesiredTransition is used to nest an update of an 3069 // allocations desired transition 3070 func (s *StateStore) nestedUpdateAllocDesiredTransition( 3071 txn *memdb.Txn, index uint64, allocID string, 3072 transition *structs.DesiredTransition) error { 3073 3074 // Look for existing alloc 3075 existing, err := txn.First("allocs", "id", allocID) 3076 if err != nil { 3077 return fmt.Errorf("alloc lookup failed: %v", err) 3078 } 3079 3080 // Nothing to do if this does not exist 3081 if existing == nil { 3082 return nil 3083 } 3084 exist := existing.(*structs.Allocation) 3085 3086 // Copy everything from the existing allocation 3087 copyAlloc := exist.Copy() 3088 3089 // Merge the desired transitions 3090 copyAlloc.DesiredTransition.Merge(transition) 3091 3092 // Update the modify index 3093 copyAlloc.ModifyIndex = index 3094 3095 // Update the allocation 3096 if err := txn.Insert("allocs", copyAlloc); err != nil { 3097 return fmt.Errorf("alloc insert failed: %v", err) 3098 } 3099 3100 return nil 3101 } 3102 3103 // AllocByID is used to lookup an allocation by its ID 3104 func (s *StateStore) AllocByID(ws memdb.WatchSet, id string) (*structs.Allocation, error) { 3105 txn := s.db.Txn(false) 3106 3107 watchCh, existing, err := txn.FirstWatch("allocs", "id", id) 3108 if err != nil { 3109 return nil, fmt.Errorf("alloc lookup failed: %v", err) 3110 } 3111 3112 ws.Add(watchCh) 3113 3114 if existing != nil { 3115 return existing.(*structs.Allocation), nil 3116 } 3117 return nil, nil 3118 } 3119 3120 // AllocsByIDPrefix is used to lookup allocs by prefix 3121 func (s *StateStore) AllocsByIDPrefix(ws memdb.WatchSet, namespace, id string) (memdb.ResultIterator, error) { 3122 txn := s.db.Txn(false) 3123 3124 iter, err := txn.Get("allocs", "id_prefix", id) 3125 if err != nil { 3126 return nil, fmt.Errorf("alloc lookup failed: %v", err) 3127 } 3128 3129 ws.Add(iter.WatchCh()) 3130 3131 // Wrap the iterator in a filter 3132 wrap := memdb.NewFilterIterator(iter, allocNamespaceFilter(namespace)) 3133 return wrap, nil 3134 } 3135 3136 // allocNamespaceFilter returns a filter function that filters all allocations 3137 // not in the given namespace. 3138 func allocNamespaceFilter(namespace string) func(interface{}) bool { 3139 return func(raw interface{}) bool { 3140 alloc, ok := raw.(*structs.Allocation) 3141 if !ok { 3142 return true 3143 } 3144 3145 return alloc.Namespace != namespace 3146 } 3147 } 3148 3149 // AllocsByNode returns all the allocations by node 3150 func (s *StateStore) AllocsByNode(ws memdb.WatchSet, node string) ([]*structs.Allocation, error) { 3151 txn := s.db.Txn(false) 3152 3153 // Get an iterator over the node allocations, using only the 3154 // node prefix which ignores the terminal status 3155 iter, err := txn.Get("allocs", "node_prefix", node) 3156 if err != nil { 3157 return nil, err 3158 } 3159 3160 ws.Add(iter.WatchCh()) 3161 3162 var out []*structs.Allocation 3163 for { 3164 raw := iter.Next() 3165 if raw == nil { 3166 break 3167 } 3168 out = append(out, raw.(*structs.Allocation)) 3169 } 3170 return out, nil 3171 } 3172 3173 // AllocsByNode returns all the allocations by node and terminal status 3174 func (s *StateStore) AllocsByNodeTerminal(ws memdb.WatchSet, node string, terminal bool) ([]*structs.Allocation, error) { 3175 txn := s.db.Txn(false) 3176 3177 // Get an iterator over the node allocations 3178 iter, err := txn.Get("allocs", "node", node, terminal) 3179 if err != nil { 3180 return nil, err 3181 } 3182 3183 ws.Add(iter.WatchCh()) 3184 3185 var out []*structs.Allocation 3186 for { 3187 raw := iter.Next() 3188 if raw == nil { 3189 break 3190 } 3191 out = append(out, raw.(*structs.Allocation)) 3192 } 3193 return out, nil 3194 } 3195 3196 // AllocsByJob returns allocations by job id 3197 func (s *StateStore) AllocsByJob(ws memdb.WatchSet, namespace, jobID string, anyCreateIndex bool) ([]*structs.Allocation, error) { 3198 txn := s.db.Txn(false) 3199 3200 // Get the job 3201 var job *structs.Job 3202 rawJob, err := txn.First("jobs", "id", namespace, jobID) 3203 if err != nil { 3204 return nil, err 3205 } 3206 if rawJob != nil { 3207 job = rawJob.(*structs.Job) 3208 } 3209 3210 // Get an iterator over the node allocations 3211 iter, err := txn.Get("allocs", "job", namespace, jobID) 3212 if err != nil { 3213 return nil, err 3214 } 3215 3216 ws.Add(iter.WatchCh()) 3217 3218 var out []*structs.Allocation 3219 for { 3220 raw := iter.Next() 3221 if raw == nil { 3222 break 3223 } 3224 3225 alloc := raw.(*structs.Allocation) 3226 // If the allocation belongs to a job with the same ID but a different 3227 // create index and we are not getting all the allocations whose Jobs 3228 // matches the same Job ID then we skip it 3229 if !anyCreateIndex && job != nil && alloc.Job.CreateIndex != job.CreateIndex { 3230 continue 3231 } 3232 out = append(out, raw.(*structs.Allocation)) 3233 } 3234 return out, nil 3235 } 3236 3237 // AllocsByEval returns all the allocations by eval id 3238 func (s *StateStore) AllocsByEval(ws memdb.WatchSet, evalID string) ([]*structs.Allocation, error) { 3239 txn := s.db.Txn(false) 3240 3241 // Get an iterator over the eval allocations 3242 iter, err := txn.Get("allocs", "eval", evalID) 3243 if err != nil { 3244 return nil, err 3245 } 3246 3247 ws.Add(iter.WatchCh()) 3248 3249 var out []*structs.Allocation 3250 for { 3251 raw := iter.Next() 3252 if raw == nil { 3253 break 3254 } 3255 out = append(out, raw.(*structs.Allocation)) 3256 } 3257 return out, nil 3258 } 3259 3260 // AllocsByDeployment returns all the allocations by deployment id 3261 func (s *StateStore) AllocsByDeployment(ws memdb.WatchSet, deploymentID string) ([]*structs.Allocation, error) { 3262 txn := s.db.Txn(false) 3263 3264 // Get an iterator over the deployments allocations 3265 iter, err := txn.Get("allocs", "deployment", deploymentID) 3266 if err != nil { 3267 return nil, err 3268 } 3269 3270 ws.Add(iter.WatchCh()) 3271 3272 var out []*structs.Allocation 3273 for { 3274 raw := iter.Next() 3275 if raw == nil { 3276 break 3277 } 3278 out = append(out, raw.(*structs.Allocation)) 3279 } 3280 return out, nil 3281 } 3282 3283 // Allocs returns an iterator over all the evaluations 3284 func (s *StateStore) Allocs(ws memdb.WatchSet) (memdb.ResultIterator, error) { 3285 txn := s.db.Txn(false) 3286 3287 // Walk the entire table 3288 iter, err := txn.Get("allocs", "id") 3289 if err != nil { 3290 return nil, err 3291 } 3292 3293 ws.Add(iter.WatchCh()) 3294 3295 return iter, nil 3296 } 3297 3298 // AllocsByNamespace returns an iterator over all the allocations in the 3299 // namespace 3300 func (s *StateStore) AllocsByNamespace(ws memdb.WatchSet, namespace string) (memdb.ResultIterator, error) { 3301 txn := s.db.Txn(false) 3302 return s.allocsByNamespaceImpl(ws, txn, namespace) 3303 } 3304 3305 // allocsByNamespaceImpl returns an iterator over all the allocations in the 3306 // namespace 3307 func (s *StateStore) allocsByNamespaceImpl(ws memdb.WatchSet, txn *memdb.Txn, namespace string) (memdb.ResultIterator, error) { 3308 // Walk the entire table 3309 iter, err := txn.Get("allocs", "namespace", namespace) 3310 if err != nil { 3311 return nil, err 3312 } 3313 3314 ws.Add(iter.WatchCh()) 3315 3316 return iter, nil 3317 } 3318 3319 // UpsertVaultAccessors is used to register a set of Vault Accessors 3320 func (s *StateStore) UpsertVaultAccessor(index uint64, accessors []*structs.VaultAccessor) error { 3321 txn := s.db.Txn(true) 3322 defer txn.Abort() 3323 3324 for _, accessor := range accessors { 3325 // Set the create index 3326 accessor.CreateIndex = index 3327 3328 // Insert the accessor 3329 if err := txn.Insert("vault_accessors", accessor); err != nil { 3330 return fmt.Errorf("accessor insert failed: %v", err) 3331 } 3332 } 3333 3334 if err := txn.Insert("index", &IndexEntry{"vault_accessors", index}); err != nil { 3335 return fmt.Errorf("index update failed: %v", err) 3336 } 3337 3338 txn.Commit() 3339 return nil 3340 } 3341 3342 // DeleteVaultAccessors is used to delete a set of Vault Accessors 3343 func (s *StateStore) DeleteVaultAccessors(index uint64, accessors []*structs.VaultAccessor) error { 3344 txn := s.db.Txn(true) 3345 defer txn.Abort() 3346 3347 // Lookup the accessor 3348 for _, accessor := range accessors { 3349 // Delete the accessor 3350 if err := txn.Delete("vault_accessors", accessor); err != nil { 3351 return fmt.Errorf("accessor delete failed: %v", err) 3352 } 3353 } 3354 3355 if err := txn.Insert("index", &IndexEntry{"vault_accessors", index}); err != nil { 3356 return fmt.Errorf("index update failed: %v", err) 3357 } 3358 3359 txn.Commit() 3360 return nil 3361 } 3362 3363 // VaultAccessor returns the given Vault accessor 3364 func (s *StateStore) VaultAccessor(ws memdb.WatchSet, accessor string) (*structs.VaultAccessor, error) { 3365 txn := s.db.Txn(false) 3366 3367 watchCh, existing, err := txn.FirstWatch("vault_accessors", "id", accessor) 3368 if err != nil { 3369 return nil, fmt.Errorf("accessor lookup failed: %v", err) 3370 } 3371 3372 ws.Add(watchCh) 3373 3374 if existing != nil { 3375 return existing.(*structs.VaultAccessor), nil 3376 } 3377 3378 return nil, nil 3379 } 3380 3381 // VaultAccessors returns an iterator of Vault accessors. 3382 func (s *StateStore) VaultAccessors(ws memdb.WatchSet) (memdb.ResultIterator, error) { 3383 txn := s.db.Txn(false) 3384 3385 iter, err := txn.Get("vault_accessors", "id") 3386 if err != nil { 3387 return nil, err 3388 } 3389 3390 ws.Add(iter.WatchCh()) 3391 3392 return iter, nil 3393 } 3394 3395 // VaultAccessorsByAlloc returns all the Vault accessors by alloc id 3396 func (s *StateStore) VaultAccessorsByAlloc(ws memdb.WatchSet, allocID string) ([]*structs.VaultAccessor, error) { 3397 txn := s.db.Txn(false) 3398 3399 // Get an iterator over the accessors 3400 iter, err := txn.Get("vault_accessors", "alloc_id", allocID) 3401 if err != nil { 3402 return nil, err 3403 } 3404 3405 ws.Add(iter.WatchCh()) 3406 3407 var out []*structs.VaultAccessor 3408 for { 3409 raw := iter.Next() 3410 if raw == nil { 3411 break 3412 } 3413 out = append(out, raw.(*structs.VaultAccessor)) 3414 } 3415 return out, nil 3416 } 3417 3418 // VaultAccessorsByNode returns all the Vault accessors by node id 3419 func (s *StateStore) VaultAccessorsByNode(ws memdb.WatchSet, nodeID string) ([]*structs.VaultAccessor, error) { 3420 txn := s.db.Txn(false) 3421 3422 // Get an iterator over the accessors 3423 iter, err := txn.Get("vault_accessors", "node_id", nodeID) 3424 if err != nil { 3425 return nil, err 3426 } 3427 3428 ws.Add(iter.WatchCh()) 3429 3430 var out []*structs.VaultAccessor 3431 for { 3432 raw := iter.Next() 3433 if raw == nil { 3434 break 3435 } 3436 out = append(out, raw.(*structs.VaultAccessor)) 3437 } 3438 return out, nil 3439 } 3440 3441 func indexEntry(table string, index uint64) *IndexEntry { 3442 return &IndexEntry{ 3443 Key: table, 3444 Value: index, 3445 } 3446 } 3447 3448 const siTokenAccessorTable = "si_token_accessors" 3449 3450 // UpsertSITokenAccessors is used to register a set of Service Identity token accessors. 3451 func (s *StateStore) UpsertSITokenAccessors(index uint64, accessors []*structs.SITokenAccessor) error { 3452 txn := s.db.Txn(true) 3453 defer txn.Abort() 3454 3455 for _, accessor := range accessors { 3456 // set the create index 3457 accessor.CreateIndex = index 3458 3459 // insert the accessor 3460 if err := txn.Insert(siTokenAccessorTable, accessor); err != nil { 3461 return errors.Wrap(err, "accessor insert failed") 3462 } 3463 } 3464 3465 // update the index for this table 3466 if err := txn.Insert("index", indexEntry(siTokenAccessorTable, index)); err != nil { 3467 return errors.Wrap(err, "index update failed") 3468 } 3469 3470 txn.Commit() 3471 return nil 3472 } 3473 3474 // DeleteSITokenAccessors is used to delete a set of Service Identity token accessors. 3475 func (s *StateStore) DeleteSITokenAccessors(index uint64, accessors []*structs.SITokenAccessor) error { 3476 txn := s.db.Txn(true) 3477 defer txn.Abort() 3478 3479 // Lookup each accessor 3480 for _, accessor := range accessors { 3481 // Delete the accessor 3482 if err := txn.Delete(siTokenAccessorTable, accessor); err != nil { 3483 return errors.Wrap(err, "accessor delete failed") 3484 } 3485 } 3486 3487 // update the index for this table 3488 if err := txn.Insert("index", indexEntry(siTokenAccessorTable, index)); err != nil { 3489 return errors.Wrap(err, "index update failed") 3490 } 3491 3492 txn.Commit() 3493 return nil 3494 } 3495 3496 // SITokenAccessor returns the given Service Identity token accessor. 3497 func (s *StateStore) SITokenAccessor(ws memdb.WatchSet, accessorID string) (*structs.SITokenAccessor, error) { 3498 txn := s.db.Txn(false) 3499 defer txn.Abort() 3500 3501 watchCh, existing, err := txn.FirstWatch(siTokenAccessorTable, "id", accessorID) 3502 if err != nil { 3503 return nil, errors.Wrap(err, "accessor lookup failed") 3504 } 3505 3506 ws.Add(watchCh) 3507 3508 if existing != nil { 3509 return existing.(*structs.SITokenAccessor), nil 3510 } 3511 3512 return nil, nil 3513 } 3514 3515 // SITokenAccessors returns an iterator of Service Identity token accessors. 3516 func (s *StateStore) SITokenAccessors(ws memdb.WatchSet) (memdb.ResultIterator, error) { 3517 txn := s.db.Txn(false) 3518 defer txn.Abort() 3519 3520 iter, err := txn.Get(siTokenAccessorTable, "id") 3521 if err != nil { 3522 return nil, err 3523 } 3524 3525 ws.Add(iter.WatchCh()) 3526 3527 return iter, nil 3528 } 3529 3530 // SITokenAccessorsByAlloc returns all the Service Identity token accessors by alloc ID. 3531 func (s *StateStore) SITokenAccessorsByAlloc(ws memdb.WatchSet, allocID string) ([]*structs.SITokenAccessor, error) { 3532 txn := s.db.Txn(false) 3533 defer txn.Abort() 3534 3535 // Get an iterator over the accessors 3536 iter, err := txn.Get(siTokenAccessorTable, "alloc_id", allocID) 3537 if err != nil { 3538 return nil, err 3539 } 3540 3541 ws.Add(iter.WatchCh()) 3542 3543 var result []*structs.SITokenAccessor 3544 for raw := iter.Next(); raw != nil; raw = iter.Next() { 3545 result = append(result, raw.(*structs.SITokenAccessor)) 3546 } 3547 3548 return result, nil 3549 } 3550 3551 // SITokenAccessorsByNode returns all the Service Identity token accessors by node ID. 3552 func (s *StateStore) SITokenAccessorsByNode(ws memdb.WatchSet, nodeID string) ([]*structs.SITokenAccessor, error) { 3553 txn := s.db.Txn(false) 3554 defer txn.Abort() 3555 3556 // Get an iterator over the accessors 3557 iter, err := txn.Get(siTokenAccessorTable, "node_id", nodeID) 3558 if err != nil { 3559 return nil, err 3560 } 3561 3562 ws.Add(iter.WatchCh()) 3563 3564 var result []*structs.SITokenAccessor 3565 for raw := iter.Next(); raw != nil; raw = iter.Next() { 3566 result = append(result, raw.(*structs.SITokenAccessor)) 3567 } 3568 3569 return result, nil 3570 } 3571 3572 // UpdateDeploymentStatus is used to make deployment status updates and 3573 // potentially make a evaluation 3574 func (s *StateStore) UpdateDeploymentStatus(index uint64, req *structs.DeploymentStatusUpdateRequest) error { 3575 txn := s.db.Txn(true) 3576 defer txn.Abort() 3577 3578 if err := s.updateDeploymentStatusImpl(index, req.DeploymentUpdate, txn); err != nil { 3579 return err 3580 } 3581 3582 // Upsert the job if necessary 3583 if req.Job != nil { 3584 if err := s.upsertJobImpl(index, req.Job, false, txn); err != nil { 3585 return err 3586 } 3587 } 3588 3589 // Upsert the optional eval 3590 if req.Eval != nil { 3591 if err := s.nestedUpsertEval(txn, index, req.Eval); err != nil { 3592 return err 3593 } 3594 } 3595 3596 txn.Commit() 3597 return nil 3598 } 3599 3600 // updateDeploymentStatusImpl is used to make deployment status updates 3601 func (s *StateStore) updateDeploymentStatusImpl(index uint64, u *structs.DeploymentStatusUpdate, txn *memdb.Txn) error { 3602 // Retrieve deployment 3603 ws := memdb.NewWatchSet() 3604 deployment, err := s.deploymentByIDImpl(ws, u.DeploymentID, txn) 3605 if err != nil { 3606 return err 3607 } else if deployment == nil { 3608 return fmt.Errorf("Deployment ID %q couldn't be updated as it does not exist", u.DeploymentID) 3609 } else if !deployment.Active() { 3610 return fmt.Errorf("Deployment %q has terminal status %q:", deployment.ID, deployment.Status) 3611 } 3612 3613 // Apply the new status 3614 copy := deployment.Copy() 3615 copy.Status = u.Status 3616 copy.StatusDescription = u.StatusDescription 3617 copy.ModifyIndex = index 3618 3619 // Insert the deployment 3620 if err := txn.Insert("deployment", copy); err != nil { 3621 return err 3622 } 3623 3624 // Update the index 3625 if err := txn.Insert("index", &IndexEntry{"deployment", index}); err != nil { 3626 return fmt.Errorf("index update failed: %v", err) 3627 } 3628 3629 // If the deployment is being marked as complete, set the job to stable. 3630 if copy.Status == structs.DeploymentStatusSuccessful { 3631 if err := s.updateJobStabilityImpl(index, copy.Namespace, copy.JobID, copy.JobVersion, true, txn); err != nil { 3632 return fmt.Errorf("failed to update job stability: %v", err) 3633 } 3634 } 3635 3636 return nil 3637 } 3638 3639 // UpdateJobStability updates the stability of the given job and version to the 3640 // desired status. 3641 func (s *StateStore) UpdateJobStability(index uint64, namespace, jobID string, jobVersion uint64, stable bool) error { 3642 txn := s.db.Txn(true) 3643 defer txn.Abort() 3644 3645 if err := s.updateJobStabilityImpl(index, namespace, jobID, jobVersion, stable, txn); err != nil { 3646 return err 3647 } 3648 3649 txn.Commit() 3650 return nil 3651 } 3652 3653 // updateJobStabilityImpl updates the stability of the given job and version 3654 func (s *StateStore) updateJobStabilityImpl(index uint64, namespace, jobID string, jobVersion uint64, stable bool, txn *memdb.Txn) error { 3655 // Get the job that is referenced 3656 job, err := s.jobByIDAndVersionImpl(nil, namespace, jobID, jobVersion, txn) 3657 if err != nil { 3658 return err 3659 } 3660 3661 // Has already been cleared, nothing to do 3662 if job == nil { 3663 return nil 3664 } 3665 3666 // If the job already has the desired stability, nothing to do 3667 if job.Stable == stable { 3668 return nil 3669 } 3670 3671 copy := job.Copy() 3672 copy.Stable = stable 3673 return s.upsertJobImpl(index, copy, true, txn) 3674 } 3675 3676 // UpdateDeploymentPromotion is used to promote canaries in a deployment and 3677 // potentially make a evaluation 3678 func (s *StateStore) UpdateDeploymentPromotion(index uint64, req *structs.ApplyDeploymentPromoteRequest) error { 3679 txn := s.db.Txn(true) 3680 defer txn.Abort() 3681 3682 // Retrieve deployment and ensure it is not terminal and is active 3683 ws := memdb.NewWatchSet() 3684 deployment, err := s.deploymentByIDImpl(ws, req.DeploymentID, txn) 3685 if err != nil { 3686 return err 3687 } else if deployment == nil { 3688 return fmt.Errorf("Deployment ID %q couldn't be updated as it does not exist", req.DeploymentID) 3689 } else if !deployment.Active() { 3690 return fmt.Errorf("Deployment %q has terminal status %q:", deployment.ID, deployment.Status) 3691 } 3692 3693 // Retrieve effected allocations 3694 iter, err := txn.Get("allocs", "deployment", req.DeploymentID) 3695 if err != nil { 3696 return err 3697 } 3698 3699 // groupIndex is a map of groups being promoted 3700 groupIndex := make(map[string]struct{}, len(req.Groups)) 3701 for _, g := range req.Groups { 3702 groupIndex[g] = struct{}{} 3703 } 3704 3705 // canaryIndex is the set of placed canaries in the deployment 3706 canaryIndex := make(map[string]struct{}, len(deployment.TaskGroups)) 3707 for _, state := range deployment.TaskGroups { 3708 for _, c := range state.PlacedCanaries { 3709 canaryIndex[c] = struct{}{} 3710 } 3711 } 3712 3713 // healthyCounts is a mapping of group to the number of healthy canaries 3714 healthyCounts := make(map[string]int, len(deployment.TaskGroups)) 3715 3716 // promotable is the set of allocations that we can move from canary to 3717 // non-canary 3718 var promotable []*structs.Allocation 3719 3720 for { 3721 raw := iter.Next() 3722 if raw == nil { 3723 break 3724 } 3725 3726 alloc := raw.(*structs.Allocation) 3727 3728 // Check that the alloc is a canary 3729 if _, ok := canaryIndex[alloc.ID]; !ok { 3730 continue 3731 } 3732 3733 // Check that the canary is part of a group being promoted 3734 if _, ok := groupIndex[alloc.TaskGroup]; !req.All && !ok { 3735 continue 3736 } 3737 3738 // Ensure the canaries are healthy 3739 if alloc.TerminalStatus() || !alloc.DeploymentStatus.IsHealthy() { 3740 continue 3741 } 3742 3743 healthyCounts[alloc.TaskGroup]++ 3744 promotable = append(promotable, alloc) 3745 } 3746 3747 // Determine if we have enough healthy allocations 3748 var unhealthyErr multierror.Error 3749 for tg, state := range deployment.TaskGroups { 3750 if _, ok := groupIndex[tg]; !req.All && !ok { 3751 continue 3752 } 3753 3754 need := state.DesiredCanaries 3755 if need == 0 { 3756 continue 3757 } 3758 3759 if have := healthyCounts[tg]; have < need { 3760 multierror.Append(&unhealthyErr, fmt.Errorf("Task group %q has %d/%d healthy allocations", tg, have, need)) 3761 } 3762 } 3763 3764 if err := unhealthyErr.ErrorOrNil(); err != nil { 3765 return err 3766 } 3767 3768 // Update deployment 3769 copy := deployment.Copy() 3770 copy.ModifyIndex = index 3771 for tg, status := range copy.TaskGroups { 3772 _, ok := groupIndex[tg] 3773 if !req.All && !ok { 3774 continue 3775 } 3776 3777 status.Promoted = true 3778 } 3779 3780 // If the deployment no longer needs promotion, update its status 3781 if !copy.RequiresPromotion() && copy.Status == structs.DeploymentStatusRunning { 3782 copy.StatusDescription = structs.DeploymentStatusDescriptionRunning 3783 } 3784 3785 // Insert the deployment 3786 if err := s.upsertDeploymentImpl(index, copy, txn); err != nil { 3787 return err 3788 } 3789 3790 // Upsert the optional eval 3791 if req.Eval != nil { 3792 if err := s.nestedUpsertEval(txn, index, req.Eval); err != nil { 3793 return err 3794 } 3795 } 3796 3797 // For each promotable allocation remove the canary field 3798 for _, alloc := range promotable { 3799 promoted := alloc.Copy() 3800 promoted.DeploymentStatus.Canary = false 3801 promoted.DeploymentStatus.ModifyIndex = index 3802 promoted.ModifyIndex = index 3803 promoted.AllocModifyIndex = index 3804 3805 if err := txn.Insert("allocs", promoted); err != nil { 3806 return fmt.Errorf("alloc insert failed: %v", err) 3807 } 3808 } 3809 3810 // Update the alloc index 3811 if err := txn.Insert("index", &IndexEntry{"allocs", index}); err != nil { 3812 return fmt.Errorf("index update failed: %v", err) 3813 } 3814 3815 txn.Commit() 3816 return nil 3817 } 3818 3819 // UpdateDeploymentAllocHealth is used to update the health of allocations as 3820 // part of the deployment and potentially make a evaluation 3821 func (s *StateStore) UpdateDeploymentAllocHealth(index uint64, req *structs.ApplyDeploymentAllocHealthRequest) error { 3822 txn := s.db.Txn(true) 3823 defer txn.Abort() 3824 3825 // Retrieve deployment and ensure it is not terminal and is active 3826 ws := memdb.NewWatchSet() 3827 deployment, err := s.deploymentByIDImpl(ws, req.DeploymentID, txn) 3828 if err != nil { 3829 return err 3830 } else if deployment == nil { 3831 return fmt.Errorf("Deployment ID %q couldn't be updated as it does not exist", req.DeploymentID) 3832 } else if !deployment.Active() { 3833 return fmt.Errorf("Deployment %q has terminal status %q:", deployment.ID, deployment.Status) 3834 } 3835 3836 // Update the health status of each allocation 3837 if total := len(req.HealthyAllocationIDs) + len(req.UnhealthyAllocationIDs); total != 0 { 3838 setAllocHealth := func(id string, healthy bool, ts time.Time) error { 3839 existing, err := txn.First("allocs", "id", id) 3840 if err != nil { 3841 return fmt.Errorf("alloc %q lookup failed: %v", id, err) 3842 } 3843 if existing == nil { 3844 return fmt.Errorf("unknown alloc %q", id) 3845 } 3846 3847 old := existing.(*structs.Allocation) 3848 if old.DeploymentID != req.DeploymentID { 3849 return fmt.Errorf("alloc %q is not part of deployment %q", id, req.DeploymentID) 3850 } 3851 3852 // Set the health 3853 copy := old.Copy() 3854 if copy.DeploymentStatus == nil { 3855 copy.DeploymentStatus = &structs.AllocDeploymentStatus{} 3856 } 3857 copy.DeploymentStatus.Healthy = helper.BoolToPtr(healthy) 3858 copy.DeploymentStatus.Timestamp = ts 3859 copy.DeploymentStatus.ModifyIndex = index 3860 copy.ModifyIndex = index 3861 3862 if err := s.updateDeploymentWithAlloc(index, copy, old, txn); err != nil { 3863 return fmt.Errorf("error updating deployment: %v", err) 3864 } 3865 3866 if err := txn.Insert("allocs", copy); err != nil { 3867 return fmt.Errorf("alloc insert failed: %v", err) 3868 } 3869 3870 return nil 3871 } 3872 3873 for _, id := range req.HealthyAllocationIDs { 3874 if err := setAllocHealth(id, true, req.Timestamp); err != nil { 3875 return err 3876 } 3877 } 3878 for _, id := range req.UnhealthyAllocationIDs { 3879 if err := setAllocHealth(id, false, req.Timestamp); err != nil { 3880 return err 3881 } 3882 } 3883 3884 // Update the indexes 3885 if err := txn.Insert("index", &IndexEntry{"allocs", index}); err != nil { 3886 return fmt.Errorf("index update failed: %v", err) 3887 } 3888 } 3889 3890 // Update the deployment status as needed. 3891 if req.DeploymentUpdate != nil { 3892 if err := s.updateDeploymentStatusImpl(index, req.DeploymentUpdate, txn); err != nil { 3893 return err 3894 } 3895 } 3896 3897 // Upsert the job if necessary 3898 if req.Job != nil { 3899 if err := s.upsertJobImpl(index, req.Job, false, txn); err != nil { 3900 return err 3901 } 3902 } 3903 3904 // Upsert the optional eval 3905 if req.Eval != nil { 3906 if err := s.nestedUpsertEval(txn, index, req.Eval); err != nil { 3907 return err 3908 } 3909 } 3910 3911 txn.Commit() 3912 return nil 3913 } 3914 3915 // LastIndex returns the greatest index value for all indexes 3916 func (s *StateStore) LatestIndex() (uint64, error) { 3917 indexes, err := s.Indexes() 3918 if err != nil { 3919 return 0, err 3920 } 3921 3922 var max uint64 = 0 3923 for { 3924 raw := indexes.Next() 3925 if raw == nil { 3926 break 3927 } 3928 3929 // Prepare the request struct 3930 idx := raw.(*IndexEntry) 3931 3932 // Determine the max 3933 if idx.Value > max { 3934 max = idx.Value 3935 } 3936 } 3937 3938 return max, nil 3939 } 3940 3941 // Index finds the matching index value 3942 func (s *StateStore) Index(name string) (uint64, error) { 3943 txn := s.db.Txn(false) 3944 3945 // Lookup the first matching index 3946 out, err := txn.First("index", "id", name) 3947 if err != nil { 3948 return 0, err 3949 } 3950 if out == nil { 3951 return 0, nil 3952 } 3953 return out.(*IndexEntry).Value, nil 3954 } 3955 3956 // RemoveIndex is a helper method to remove an index for testing purposes 3957 func (s *StateStore) RemoveIndex(name string) error { 3958 txn := s.db.Txn(true) 3959 defer txn.Abort() 3960 3961 if _, err := txn.DeleteAll("index", "id", name); err != nil { 3962 return err 3963 } 3964 3965 txn.Commit() 3966 return nil 3967 } 3968 3969 // Indexes returns an iterator over all the indexes 3970 func (s *StateStore) Indexes() (memdb.ResultIterator, error) { 3971 txn := s.db.Txn(false) 3972 3973 // Walk the entire nodes table 3974 iter, err := txn.Get("index", "id") 3975 if err != nil { 3976 return nil, err 3977 } 3978 return iter, nil 3979 } 3980 3981 // ReconcileJobSummaries re-creates summaries for all jobs present in the state 3982 // store 3983 func (s *StateStore) ReconcileJobSummaries(index uint64) error { 3984 txn := s.db.Txn(true) 3985 defer txn.Abort() 3986 3987 // Get all the jobs 3988 iter, err := txn.Get("jobs", "id") 3989 if err != nil { 3990 return err 3991 } 3992 // COMPAT: Remove after 0.11 3993 // Iterate over jobs to build a list of parent jobs and their children 3994 parentMap := make(map[string][]*structs.Job) 3995 for { 3996 rawJob := iter.Next() 3997 if rawJob == nil { 3998 break 3999 } 4000 job := rawJob.(*structs.Job) 4001 if job.ParentID != "" { 4002 children := parentMap[job.ParentID] 4003 children = append(children, job) 4004 parentMap[job.ParentID] = children 4005 } 4006 } 4007 4008 // Get all the jobs again 4009 iter, err = txn.Get("jobs", "id") 4010 if err != nil { 4011 return err 4012 } 4013 4014 for { 4015 rawJob := iter.Next() 4016 if rawJob == nil { 4017 break 4018 } 4019 job := rawJob.(*structs.Job) 4020 4021 if job.IsParameterized() || job.IsPeriodic() { 4022 // COMPAT: Remove after 0.11 4023 4024 // The following block of code fixes incorrect child summaries due to a bug 4025 // See https://github.com/hashicorp/nomad/issues/3886 for details 4026 rawSummary, err := txn.First("job_summary", "id", job.Namespace, job.ID) 4027 if err != nil { 4028 return err 4029 } 4030 if rawSummary == nil { 4031 continue 4032 } 4033 4034 oldSummary := rawSummary.(*structs.JobSummary) 4035 4036 // Create an empty summary 4037 summary := &structs.JobSummary{ 4038 JobID: job.ID, 4039 Namespace: job.Namespace, 4040 Summary: make(map[string]structs.TaskGroupSummary), 4041 Children: &structs.JobChildrenSummary{}, 4042 } 4043 4044 // Iterate over children of this job if any to fix summary counts 4045 children := parentMap[job.ID] 4046 for _, childJob := range children { 4047 switch childJob.Status { 4048 case structs.JobStatusPending: 4049 summary.Children.Pending++ 4050 case structs.JobStatusDead: 4051 summary.Children.Dead++ 4052 case structs.JobStatusRunning: 4053 summary.Children.Running++ 4054 } 4055 } 4056 4057 // Insert the job summary if its different 4058 if !reflect.DeepEqual(summary, oldSummary) { 4059 // Set the create index of the summary same as the job's create index 4060 // and the modify index to the current index 4061 summary.CreateIndex = job.CreateIndex 4062 summary.ModifyIndex = index 4063 4064 if err := txn.Insert("job_summary", summary); err != nil { 4065 return fmt.Errorf("error inserting job summary: %v", err) 4066 } 4067 } 4068 4069 // Done with handling a parent job, continue to next 4070 continue 4071 } 4072 4073 // Create a job summary for the job 4074 summary := &structs.JobSummary{ 4075 JobID: job.ID, 4076 Namespace: job.Namespace, 4077 Summary: make(map[string]structs.TaskGroupSummary), 4078 } 4079 for _, tg := range job.TaskGroups { 4080 summary.Summary[tg.Name] = structs.TaskGroupSummary{} 4081 } 4082 4083 // Find all the allocations for the jobs 4084 iterAllocs, err := txn.Get("allocs", "job", job.Namespace, job.ID) 4085 if err != nil { 4086 return err 4087 } 4088 4089 // Calculate the summary for the job 4090 for { 4091 rawAlloc := iterAllocs.Next() 4092 if rawAlloc == nil { 4093 break 4094 } 4095 alloc := rawAlloc.(*structs.Allocation) 4096 4097 // Ignore the allocation if it doesn't belong to the currently 4098 // registered job. The allocation is checked because of issue #2304 4099 if alloc.Job == nil || alloc.Job.CreateIndex != job.CreateIndex { 4100 continue 4101 } 4102 4103 tg := summary.Summary[alloc.TaskGroup] 4104 switch alloc.ClientStatus { 4105 case structs.AllocClientStatusFailed: 4106 tg.Failed += 1 4107 case structs.AllocClientStatusLost: 4108 tg.Lost += 1 4109 case structs.AllocClientStatusComplete: 4110 tg.Complete += 1 4111 case structs.AllocClientStatusRunning: 4112 tg.Running += 1 4113 case structs.AllocClientStatusPending: 4114 tg.Starting += 1 4115 default: 4116 s.logger.Error("invalid client status set on allocation", "client_status", alloc.ClientStatus, "alloc_id", alloc.ID) 4117 } 4118 summary.Summary[alloc.TaskGroup] = tg 4119 } 4120 4121 // Set the create index of the summary same as the job's create index 4122 // and the modify index to the current index 4123 summary.CreateIndex = job.CreateIndex 4124 summary.ModifyIndex = index 4125 4126 // Insert the job summary 4127 if err := txn.Insert("job_summary", summary); err != nil { 4128 return fmt.Errorf("error inserting job summary: %v", err) 4129 } 4130 } 4131 4132 // Update the indexes table for job summary 4133 if err := txn.Insert("index", &IndexEntry{"job_summary", index}); err != nil { 4134 return fmt.Errorf("index update failed: %v", err) 4135 } 4136 txn.Commit() 4137 return nil 4138 } 4139 4140 // setJobStatuses is a helper for calling setJobStatus on multiple jobs by ID. 4141 // It takes a map of job IDs to an optional forceStatus string. It returns an 4142 // error if the job doesn't exist or setJobStatus fails. 4143 func (s *StateStore) setJobStatuses(index uint64, txn *memdb.Txn, 4144 jobs map[structs.NamespacedID]string, evalDelete bool) error { 4145 for tuple, forceStatus := range jobs { 4146 4147 existing, err := txn.First("jobs", "id", tuple.Namespace, tuple.ID) 4148 if err != nil { 4149 return fmt.Errorf("job lookup failed: %v", err) 4150 } 4151 4152 if existing == nil { 4153 continue 4154 } 4155 4156 if err := s.setJobStatus(index, txn, existing.(*structs.Job), evalDelete, forceStatus); err != nil { 4157 return err 4158 } 4159 } 4160 4161 return nil 4162 } 4163 4164 // setJobStatus sets the status of the job by looking up associated evaluations 4165 // and allocations. evalDelete should be set to true if setJobStatus is being 4166 // called because an evaluation is being deleted (potentially because of garbage 4167 // collection). If forceStatus is non-empty, the job's status will be set to the 4168 // passed status. 4169 func (s *StateStore) setJobStatus(index uint64, txn *memdb.Txn, 4170 job *structs.Job, evalDelete bool, forceStatus string) error { 4171 4172 // Capture the current status so we can check if there is a change 4173 oldStatus := job.Status 4174 if index == job.CreateIndex { 4175 oldStatus = "" 4176 } 4177 newStatus := forceStatus 4178 4179 // If forceStatus is not set, compute the jobs status. 4180 if forceStatus == "" { 4181 var err error 4182 newStatus, err = s.getJobStatus(txn, job, evalDelete) 4183 if err != nil { 4184 return err 4185 } 4186 } 4187 4188 // Fast-path if nothing has changed. 4189 if oldStatus == newStatus { 4190 return nil 4191 } 4192 4193 // Copy and update the existing job 4194 updated := job.Copy() 4195 updated.Status = newStatus 4196 updated.ModifyIndex = index 4197 4198 // Insert the job 4199 if err := txn.Insert("jobs", updated); err != nil { 4200 return fmt.Errorf("job insert failed: %v", err) 4201 } 4202 if err := txn.Insert("index", &IndexEntry{"jobs", index}); err != nil { 4203 return fmt.Errorf("index update failed: %v", err) 4204 } 4205 4206 // Update the children summary 4207 if updated.ParentID != "" { 4208 // Try to update the summary of the parent job summary 4209 summaryRaw, err := txn.First("job_summary", "id", updated.Namespace, updated.ParentID) 4210 if err != nil { 4211 return fmt.Errorf("unable to retrieve summary for parent job: %v", err) 4212 } 4213 4214 // Only continue if the summary exists. It could not exist if the parent 4215 // job was removed 4216 if summaryRaw != nil { 4217 existing := summaryRaw.(*structs.JobSummary) 4218 pSummary := existing.Copy() 4219 if pSummary.Children == nil { 4220 pSummary.Children = new(structs.JobChildrenSummary) 4221 } 4222 4223 // Determine the transition and update the correct fields 4224 children := pSummary.Children 4225 4226 // Decrement old status 4227 if oldStatus != "" { 4228 switch oldStatus { 4229 case structs.JobStatusPending: 4230 children.Pending-- 4231 case structs.JobStatusRunning: 4232 children.Running-- 4233 case structs.JobStatusDead: 4234 children.Dead-- 4235 default: 4236 return fmt.Errorf("unknown old job status %q", oldStatus) 4237 } 4238 } 4239 4240 // Increment new status 4241 switch newStatus { 4242 case structs.JobStatusPending: 4243 children.Pending++ 4244 case structs.JobStatusRunning: 4245 children.Running++ 4246 case structs.JobStatusDead: 4247 children.Dead++ 4248 default: 4249 return fmt.Errorf("unknown new job status %q", newStatus) 4250 } 4251 4252 // Update the index 4253 pSummary.ModifyIndex = index 4254 4255 // Insert the summary 4256 if err := txn.Insert("job_summary", pSummary); err != nil { 4257 return fmt.Errorf("job summary insert failed: %v", err) 4258 } 4259 if err := txn.Insert("index", &IndexEntry{"job_summary", index}); err != nil { 4260 return fmt.Errorf("index update failed: %v", err) 4261 } 4262 } 4263 } 4264 4265 return nil 4266 } 4267 4268 func (s *StateStore) getJobStatus(txn *memdb.Txn, job *structs.Job, evalDelete bool) (string, error) { 4269 // System, Periodic and Parameterized jobs are running until explicitly 4270 // stopped 4271 if job.Type == structs.JobTypeSystem || job.IsParameterized() || job.IsPeriodic() { 4272 if job.Stop { 4273 return structs.JobStatusDead, nil 4274 } 4275 4276 return structs.JobStatusRunning, nil 4277 } 4278 4279 allocs, err := txn.Get("allocs", "job", job.Namespace, job.ID) 4280 if err != nil { 4281 return "", err 4282 } 4283 4284 // If there is a non-terminal allocation, the job is running. 4285 hasAlloc := false 4286 for alloc := allocs.Next(); alloc != nil; alloc = allocs.Next() { 4287 hasAlloc = true 4288 if !alloc.(*structs.Allocation).TerminalStatus() { 4289 return structs.JobStatusRunning, nil 4290 } 4291 } 4292 4293 evals, err := txn.Get("evals", "job_prefix", job.Namespace, job.ID) 4294 if err != nil { 4295 return "", err 4296 } 4297 4298 hasEval := false 4299 for raw := evals.Next(); raw != nil; raw = evals.Next() { 4300 e := raw.(*structs.Evaluation) 4301 4302 // Filter non-exact matches 4303 if e.JobID != job.ID { 4304 continue 4305 } 4306 4307 hasEval = true 4308 if !e.TerminalStatus() { 4309 return structs.JobStatusPending, nil 4310 } 4311 } 4312 4313 // The job is dead if all the allocations and evals are terminal or if there 4314 // are no evals because of garbage collection. 4315 if evalDelete || hasEval || hasAlloc { 4316 return structs.JobStatusDead, nil 4317 } 4318 4319 return structs.JobStatusPending, nil 4320 } 4321 4322 // updateSummaryWithJob creates or updates job summaries when new jobs are 4323 // upserted or existing ones are updated 4324 func (s *StateStore) updateSummaryWithJob(index uint64, job *structs.Job, 4325 txn *memdb.Txn) error { 4326 4327 // Update the job summary 4328 summaryRaw, err := txn.First("job_summary", "id", job.Namespace, job.ID) 4329 if err != nil { 4330 return fmt.Errorf("job summary lookup failed: %v", err) 4331 } 4332 4333 // Get the summary or create if necessary 4334 var summary *structs.JobSummary 4335 hasSummaryChanged := false 4336 if summaryRaw != nil { 4337 summary = summaryRaw.(*structs.JobSummary).Copy() 4338 } else { 4339 summary = &structs.JobSummary{ 4340 JobID: job.ID, 4341 Namespace: job.Namespace, 4342 Summary: make(map[string]structs.TaskGroupSummary), 4343 Children: new(structs.JobChildrenSummary), 4344 CreateIndex: index, 4345 } 4346 hasSummaryChanged = true 4347 } 4348 4349 for _, tg := range job.TaskGroups { 4350 if _, ok := summary.Summary[tg.Name]; !ok { 4351 newSummary := structs.TaskGroupSummary{ 4352 Complete: 0, 4353 Failed: 0, 4354 Running: 0, 4355 Starting: 0, 4356 } 4357 summary.Summary[tg.Name] = newSummary 4358 hasSummaryChanged = true 4359 } 4360 } 4361 4362 // The job summary has changed, so update the modify index. 4363 if hasSummaryChanged { 4364 summary.ModifyIndex = index 4365 4366 // Update the indexes table for job summary 4367 if err := txn.Insert("index", &IndexEntry{"job_summary", index}); err != nil { 4368 return fmt.Errorf("index update failed: %v", err) 4369 } 4370 if err := txn.Insert("job_summary", summary); err != nil { 4371 return err 4372 } 4373 } 4374 4375 return nil 4376 } 4377 4378 // updateJobScalingPolicies upserts any scaling policies contained in the job and removes 4379 // any previous scaling policies that were removed from the job 4380 func (s *StateStore) updateJobScalingPolicies(index uint64, job *structs.Job, txn *memdb.Txn) error { 4381 4382 ws := memdb.NewWatchSet() 4383 4384 if job.Stop { 4385 if err := s.deleteJobScalingPolicies(index, job, txn); err != nil { 4386 return fmt.Errorf("deleting job scaling policies failed: %v", err) 4387 } 4388 return nil 4389 } 4390 4391 scalingPolicies := job.GetScalingPolicies() 4392 newTargets := map[string]struct{}{} 4393 for _, p := range scalingPolicies { 4394 newTargets[p.Target[structs.ScalingTargetGroup]] = struct{}{} 4395 } 4396 // find existing policies that need to be deleted 4397 deletedPolicies := []string{} 4398 iter, err := s.ScalingPoliciesByJobTxn(ws, job.Namespace, job.ID, txn) 4399 if err != nil { 4400 return fmt.Errorf("ScalingPoliciesByJob lookup failed: %v", err) 4401 } 4402 for { 4403 raw := iter.Next() 4404 if raw == nil { 4405 break 4406 } 4407 oldPolicy := raw.(*structs.ScalingPolicy) 4408 if _, ok := newTargets[oldPolicy.Target[structs.ScalingTargetGroup]]; !ok { 4409 deletedPolicies = append(deletedPolicies, oldPolicy.ID) 4410 } 4411 } 4412 err = s.DeleteScalingPoliciesTxn(index, deletedPolicies, txn) 4413 if err != nil { 4414 return fmt.Errorf("DeleteScalingPolicies of removed policies failed: %v", err) 4415 } 4416 4417 err = s.UpsertScalingPoliciesTxn(index, scalingPolicies, txn) 4418 if err != nil { 4419 return fmt.Errorf("UpsertScalingPolicies of policies failed: %v", err) 4420 } 4421 4422 return nil 4423 } 4424 4425 // updateDeploymentWithAlloc is used to update the deployment state associated 4426 // with the given allocation. The passed alloc may be updated if the deployment 4427 // status has changed to capture the modify index at which it has changed. 4428 func (s *StateStore) updateDeploymentWithAlloc(index uint64, alloc, existing *structs.Allocation, txn *memdb.Txn) error { 4429 // Nothing to do if the allocation is not associated with a deployment 4430 if alloc.DeploymentID == "" { 4431 return nil 4432 } 4433 4434 // Get the deployment 4435 ws := memdb.NewWatchSet() 4436 deployment, err := s.deploymentByIDImpl(ws, alloc.DeploymentID, txn) 4437 if err != nil { 4438 return err 4439 } 4440 if deployment == nil { 4441 return nil 4442 } 4443 4444 // Retrieve the deployment state object 4445 _, ok := deployment.TaskGroups[alloc.TaskGroup] 4446 if !ok { 4447 // If the task group isn't part of the deployment, the task group wasn't 4448 // part of a rolling update so nothing to do 4449 return nil 4450 } 4451 4452 // Do not modify in-place. Instead keep track of what must be done 4453 placed := 0 4454 healthy := 0 4455 unhealthy := 0 4456 4457 // If there was no existing allocation, this is a placement and we increment 4458 // the placement 4459 existingHealthSet := existing != nil && existing.DeploymentStatus.HasHealth() 4460 allocHealthSet := alloc.DeploymentStatus.HasHealth() 4461 if existing == nil || existing.DeploymentID != alloc.DeploymentID { 4462 placed++ 4463 } else if !existingHealthSet && allocHealthSet { 4464 if *alloc.DeploymentStatus.Healthy { 4465 healthy++ 4466 } else { 4467 unhealthy++ 4468 } 4469 } else if existingHealthSet && allocHealthSet { 4470 // See if it has gone from healthy to unhealthy 4471 if *existing.DeploymentStatus.Healthy && !*alloc.DeploymentStatus.Healthy { 4472 healthy-- 4473 unhealthy++ 4474 } 4475 } 4476 4477 // Nothing to do 4478 if placed == 0 && healthy == 0 && unhealthy == 0 { 4479 return nil 4480 } 4481 4482 // Update the allocation's deployment status modify index 4483 if alloc.DeploymentStatus != nil && healthy+unhealthy != 0 { 4484 alloc.DeploymentStatus.ModifyIndex = index 4485 } 4486 4487 // Create a copy of the deployment object 4488 deploymentCopy := deployment.Copy() 4489 deploymentCopy.ModifyIndex = index 4490 4491 state := deploymentCopy.TaskGroups[alloc.TaskGroup] 4492 state.PlacedAllocs += placed 4493 state.HealthyAllocs += healthy 4494 state.UnhealthyAllocs += unhealthy 4495 4496 // Ensure PlacedCanaries accurately reflects the alloc canary status 4497 if alloc.DeploymentStatus != nil && alloc.DeploymentStatus.Canary { 4498 found := false 4499 for _, canary := range state.PlacedCanaries { 4500 if alloc.ID == canary { 4501 found = true 4502 break 4503 } 4504 } 4505 if !found { 4506 state.PlacedCanaries = append(state.PlacedCanaries, alloc.ID) 4507 } 4508 } 4509 4510 // Update the progress deadline 4511 if pd := state.ProgressDeadline; pd != 0 { 4512 // If we are the first placed allocation for the deployment start the progress deadline. 4513 if placed != 0 && state.RequireProgressBy.IsZero() { 4514 // Use modify time instead of create time because we may in-place 4515 // update the allocation to be part of a new deployment. 4516 state.RequireProgressBy = time.Unix(0, alloc.ModifyTime).Add(pd) 4517 } else if healthy != 0 { 4518 if d := alloc.DeploymentStatus.Timestamp.Add(pd); d.After(state.RequireProgressBy) { 4519 state.RequireProgressBy = d 4520 } 4521 } 4522 } 4523 4524 // Upsert the deployment 4525 if err := s.upsertDeploymentImpl(index, deploymentCopy, txn); err != nil { 4526 return err 4527 } 4528 4529 return nil 4530 } 4531 4532 // updateSummaryWithAlloc updates the job summary when allocations are updated 4533 // or inserted 4534 func (s *StateStore) updateSummaryWithAlloc(index uint64, alloc *structs.Allocation, 4535 existingAlloc *structs.Allocation, txn *memdb.Txn) error { 4536 4537 // We don't have to update the summary if the job is missing 4538 if alloc.Job == nil { 4539 return nil 4540 } 4541 4542 summaryRaw, err := txn.First("job_summary", "id", alloc.Namespace, alloc.JobID) 4543 if err != nil { 4544 return fmt.Errorf("unable to lookup job summary for job id %q in namespace %q: %v", alloc.JobID, alloc.Namespace, err) 4545 } 4546 4547 if summaryRaw == nil { 4548 // Check if the job is de-registered 4549 rawJob, err := txn.First("jobs", "id", alloc.Namespace, alloc.JobID) 4550 if err != nil { 4551 return fmt.Errorf("unable to query job: %v", err) 4552 } 4553 4554 // If the job is de-registered then we skip updating it's summary 4555 if rawJob == nil { 4556 return nil 4557 } 4558 4559 return fmt.Errorf("job summary for job %q in namespace %q is not present", alloc.JobID, alloc.Namespace) 4560 } 4561 4562 // Get a copy of the existing summary 4563 jobSummary := summaryRaw.(*structs.JobSummary).Copy() 4564 4565 // Not updating the job summary because the allocation doesn't belong to the 4566 // currently registered job 4567 if jobSummary.CreateIndex != alloc.Job.CreateIndex { 4568 return nil 4569 } 4570 4571 tgSummary, ok := jobSummary.Summary[alloc.TaskGroup] 4572 if !ok { 4573 return fmt.Errorf("unable to find task group in the job summary: %v", alloc.TaskGroup) 4574 } 4575 4576 summaryChanged := false 4577 if existingAlloc == nil { 4578 switch alloc.DesiredStatus { 4579 case structs.AllocDesiredStatusStop, structs.AllocDesiredStatusEvict: 4580 s.logger.Error("new allocation inserted into state store with bad desired status", 4581 "alloc_id", alloc.ID, "desired_status", alloc.DesiredStatus) 4582 } 4583 switch alloc.ClientStatus { 4584 case structs.AllocClientStatusPending: 4585 tgSummary.Starting += 1 4586 if tgSummary.Queued > 0 { 4587 tgSummary.Queued -= 1 4588 } 4589 summaryChanged = true 4590 case structs.AllocClientStatusRunning, structs.AllocClientStatusFailed, 4591 structs.AllocClientStatusComplete: 4592 s.logger.Error("new allocation inserted into state store with bad client status", 4593 "alloc_id", alloc.ID, "client_status", alloc.ClientStatus) 4594 } 4595 } else if existingAlloc.ClientStatus != alloc.ClientStatus { 4596 // Incrementing the client of the bin of the current state 4597 switch alloc.ClientStatus { 4598 case structs.AllocClientStatusRunning: 4599 tgSummary.Running += 1 4600 case structs.AllocClientStatusFailed: 4601 tgSummary.Failed += 1 4602 case structs.AllocClientStatusPending: 4603 tgSummary.Starting += 1 4604 case structs.AllocClientStatusComplete: 4605 tgSummary.Complete += 1 4606 case structs.AllocClientStatusLost: 4607 tgSummary.Lost += 1 4608 } 4609 4610 // Decrementing the count of the bin of the last state 4611 switch existingAlloc.ClientStatus { 4612 case structs.AllocClientStatusRunning: 4613 if tgSummary.Running > 0 { 4614 tgSummary.Running -= 1 4615 } 4616 case structs.AllocClientStatusPending: 4617 if tgSummary.Starting > 0 { 4618 tgSummary.Starting -= 1 4619 } 4620 case structs.AllocClientStatusLost: 4621 if tgSummary.Lost > 0 { 4622 tgSummary.Lost -= 1 4623 } 4624 case structs.AllocClientStatusFailed, structs.AllocClientStatusComplete: 4625 default: 4626 s.logger.Error("invalid old client status for allocation", 4627 "alloc_id", existingAlloc.ID, "client_status", existingAlloc.ClientStatus) 4628 } 4629 summaryChanged = true 4630 } 4631 jobSummary.Summary[alloc.TaskGroup] = tgSummary 4632 4633 if summaryChanged { 4634 jobSummary.ModifyIndex = index 4635 4636 // Update the indexes table for job summary 4637 if err := txn.Insert("index", &IndexEntry{"job_summary", index}); err != nil { 4638 return fmt.Errorf("index update failed: %v", err) 4639 } 4640 4641 if err := txn.Insert("job_summary", jobSummary); err != nil { 4642 return fmt.Errorf("updating job summary failed: %v", err) 4643 } 4644 } 4645 4646 return nil 4647 } 4648 4649 // updatePluginWithAlloc updates the CSI plugins for an alloc when the 4650 // allocation is updated or inserted with a terminal server status. 4651 func (s *StateStore) updatePluginWithAlloc(index uint64, alloc *structs.Allocation, 4652 txn *memdb.Txn) error { 4653 if !alloc.ServerTerminalStatus() { 4654 return nil 4655 } 4656 4657 ws := memdb.NewWatchSet() 4658 tg := alloc.Job.LookupTaskGroup(alloc.TaskGroup) 4659 for _, t := range tg.Tasks { 4660 if t.CSIPluginConfig != nil { 4661 pluginID := t.CSIPluginConfig.ID 4662 plug, err := s.CSIPluginByID(ws, pluginID) 4663 if err != nil { 4664 return err 4665 } 4666 if plug == nil { 4667 // plugin may not have been created because it never 4668 // became healthy, just move on 4669 return nil 4670 } 4671 err = plug.DeleteAlloc(alloc.ID, alloc.NodeID) 4672 if err != nil { 4673 return err 4674 } 4675 err = updateOrGCPlugin(index, txn, plug) 4676 if err != nil { 4677 return err 4678 } 4679 } 4680 } 4681 4682 return nil 4683 } 4684 4685 // UpsertACLPolicies is used to create or update a set of ACL policies 4686 func (s *StateStore) UpsertACLPolicies(index uint64, policies []*structs.ACLPolicy) error { 4687 txn := s.db.Txn(true) 4688 defer txn.Abort() 4689 4690 for _, policy := range policies { 4691 // Ensure the policy hash is non-nil. This should be done outside the state store 4692 // for performance reasons, but we check here for defense in depth. 4693 if len(policy.Hash) == 0 { 4694 policy.SetHash() 4695 } 4696 4697 // Check if the policy already exists 4698 existing, err := txn.First("acl_policy", "id", policy.Name) 4699 if err != nil { 4700 return fmt.Errorf("policy lookup failed: %v", err) 4701 } 4702 4703 // Update all the indexes 4704 if existing != nil { 4705 policy.CreateIndex = existing.(*structs.ACLPolicy).CreateIndex 4706 policy.ModifyIndex = index 4707 } else { 4708 policy.CreateIndex = index 4709 policy.ModifyIndex = index 4710 } 4711 4712 // Update the policy 4713 if err := txn.Insert("acl_policy", policy); err != nil { 4714 return fmt.Errorf("upserting policy failed: %v", err) 4715 } 4716 } 4717 4718 // Update the indexes tabl 4719 if err := txn.Insert("index", &IndexEntry{"acl_policy", index}); err != nil { 4720 return fmt.Errorf("index update failed: %v", err) 4721 } 4722 4723 txn.Commit() 4724 return nil 4725 } 4726 4727 // DeleteACLPolicies deletes the policies with the given names 4728 func (s *StateStore) DeleteACLPolicies(index uint64, names []string) error { 4729 txn := s.db.Txn(true) 4730 defer txn.Abort() 4731 4732 // Delete the policy 4733 for _, name := range names { 4734 if _, err := txn.DeleteAll("acl_policy", "id", name); err != nil { 4735 return fmt.Errorf("deleting acl policy failed: %v", err) 4736 } 4737 } 4738 if err := txn.Insert("index", &IndexEntry{"acl_policy", index}); err != nil { 4739 return fmt.Errorf("index update failed: %v", err) 4740 } 4741 txn.Commit() 4742 return nil 4743 } 4744 4745 // ACLPolicyByName is used to lookup a policy by name 4746 func (s *StateStore) ACLPolicyByName(ws memdb.WatchSet, name string) (*structs.ACLPolicy, error) { 4747 txn := s.db.Txn(false) 4748 4749 watchCh, existing, err := txn.FirstWatch("acl_policy", "id", name) 4750 if err != nil { 4751 return nil, fmt.Errorf("acl policy lookup failed: %v", err) 4752 } 4753 ws.Add(watchCh) 4754 4755 if existing != nil { 4756 return existing.(*structs.ACLPolicy), nil 4757 } 4758 return nil, nil 4759 } 4760 4761 // ACLPolicyByNamePrefix is used to lookup policies by prefix 4762 func (s *StateStore) ACLPolicyByNamePrefix(ws memdb.WatchSet, prefix string) (memdb.ResultIterator, error) { 4763 txn := s.db.Txn(false) 4764 4765 iter, err := txn.Get("acl_policy", "id_prefix", prefix) 4766 if err != nil { 4767 return nil, fmt.Errorf("acl policy lookup failed: %v", err) 4768 } 4769 ws.Add(iter.WatchCh()) 4770 4771 return iter, nil 4772 } 4773 4774 // ACLPolicies returns an iterator over all the acl policies 4775 func (s *StateStore) ACLPolicies(ws memdb.WatchSet) (memdb.ResultIterator, error) { 4776 txn := s.db.Txn(false) 4777 4778 // Walk the entire table 4779 iter, err := txn.Get("acl_policy", "id") 4780 if err != nil { 4781 return nil, err 4782 } 4783 ws.Add(iter.WatchCh()) 4784 return iter, nil 4785 } 4786 4787 // UpsertACLTokens is used to create or update a set of ACL tokens 4788 func (s *StateStore) UpsertACLTokens(index uint64, tokens []*structs.ACLToken) error { 4789 txn := s.db.Txn(true) 4790 defer txn.Abort() 4791 4792 for _, token := range tokens { 4793 // Ensure the policy hash is non-nil. This should be done outside the state store 4794 // for performance reasons, but we check here for defense in depth. 4795 if len(token.Hash) == 0 { 4796 token.SetHash() 4797 } 4798 4799 // Check if the token already exists 4800 existing, err := txn.First("acl_token", "id", token.AccessorID) 4801 if err != nil { 4802 return fmt.Errorf("token lookup failed: %v", err) 4803 } 4804 4805 // Update all the indexes 4806 if existing != nil { 4807 existTK := existing.(*structs.ACLToken) 4808 token.CreateIndex = existTK.CreateIndex 4809 token.ModifyIndex = index 4810 4811 // Do not allow SecretID or create time to change 4812 token.SecretID = existTK.SecretID 4813 token.CreateTime = existTK.CreateTime 4814 4815 } else { 4816 token.CreateIndex = index 4817 token.ModifyIndex = index 4818 } 4819 4820 // Update the token 4821 if err := txn.Insert("acl_token", token); err != nil { 4822 return fmt.Errorf("upserting token failed: %v", err) 4823 } 4824 } 4825 4826 // Update the indexes table 4827 if err := txn.Insert("index", &IndexEntry{"acl_token", index}); err != nil { 4828 return fmt.Errorf("index update failed: %v", err) 4829 } 4830 txn.Commit() 4831 return nil 4832 } 4833 4834 // DeleteACLTokens deletes the tokens with the given accessor ids 4835 func (s *StateStore) DeleteACLTokens(index uint64, ids []string) error { 4836 txn := s.db.Txn(true) 4837 defer txn.Abort() 4838 4839 // Delete the tokens 4840 for _, id := range ids { 4841 if _, err := txn.DeleteAll("acl_token", "id", id); err != nil { 4842 return fmt.Errorf("deleting acl token failed: %v", err) 4843 } 4844 } 4845 if err := txn.Insert("index", &IndexEntry{"acl_token", index}); err != nil { 4846 return fmt.Errorf("index update failed: %v", err) 4847 } 4848 txn.Commit() 4849 return nil 4850 } 4851 4852 // ACLTokenByAccessorID is used to lookup a token by accessor ID 4853 func (s *StateStore) ACLTokenByAccessorID(ws memdb.WatchSet, id string) (*structs.ACLToken, error) { 4854 if id == "" { 4855 return nil, fmt.Errorf("acl token lookup failed: missing accessor id") 4856 } 4857 4858 txn := s.db.Txn(false) 4859 4860 watchCh, existing, err := txn.FirstWatch("acl_token", "id", id) 4861 if err != nil { 4862 return nil, fmt.Errorf("acl token lookup failed: %v", err) 4863 } 4864 ws.Add(watchCh) 4865 4866 if existing != nil { 4867 return existing.(*structs.ACLToken), nil 4868 } 4869 return nil, nil 4870 } 4871 4872 // ACLTokenBySecretID is used to lookup a token by secret ID 4873 func (s *StateStore) ACLTokenBySecretID(ws memdb.WatchSet, secretID string) (*structs.ACLToken, error) { 4874 if secretID == "" { 4875 return nil, fmt.Errorf("acl token lookup failed: missing secret id") 4876 } 4877 4878 txn := s.db.Txn(false) 4879 4880 watchCh, existing, err := txn.FirstWatch("acl_token", "secret", secretID) 4881 if err != nil { 4882 return nil, fmt.Errorf("acl token lookup failed: %v", err) 4883 } 4884 ws.Add(watchCh) 4885 4886 if existing != nil { 4887 return existing.(*structs.ACLToken), nil 4888 } 4889 return nil, nil 4890 } 4891 4892 // ACLTokenByAccessorIDPrefix is used to lookup tokens by prefix 4893 func (s *StateStore) ACLTokenByAccessorIDPrefix(ws memdb.WatchSet, prefix string) (memdb.ResultIterator, error) { 4894 txn := s.db.Txn(false) 4895 4896 iter, err := txn.Get("acl_token", "id_prefix", prefix) 4897 if err != nil { 4898 return nil, fmt.Errorf("acl token lookup failed: %v", err) 4899 } 4900 ws.Add(iter.WatchCh()) 4901 return iter, nil 4902 } 4903 4904 // ACLTokens returns an iterator over all the tokens 4905 func (s *StateStore) ACLTokens(ws memdb.WatchSet) (memdb.ResultIterator, error) { 4906 txn := s.db.Txn(false) 4907 4908 // Walk the entire table 4909 iter, err := txn.Get("acl_token", "id") 4910 if err != nil { 4911 return nil, err 4912 } 4913 ws.Add(iter.WatchCh()) 4914 return iter, nil 4915 } 4916 4917 // ACLTokensByGlobal returns an iterator over all the tokens filtered by global value 4918 func (s *StateStore) ACLTokensByGlobal(ws memdb.WatchSet, globalVal bool) (memdb.ResultIterator, error) { 4919 txn := s.db.Txn(false) 4920 4921 // Walk the entire table 4922 iter, err := txn.Get("acl_token", "global", globalVal) 4923 if err != nil { 4924 return nil, err 4925 } 4926 ws.Add(iter.WatchCh()) 4927 return iter, nil 4928 } 4929 4930 // CanBootstrapACLToken checks if bootstrapping is possible and returns the reset index 4931 func (s *StateStore) CanBootstrapACLToken() (bool, uint64, error) { 4932 txn := s.db.Txn(false) 4933 4934 // Lookup the bootstrap sentinel 4935 out, err := txn.First("index", "id", "acl_token_bootstrap") 4936 if err != nil { 4937 return false, 0, err 4938 } 4939 4940 // No entry, we haven't bootstrapped yet 4941 if out == nil { 4942 return true, 0, nil 4943 } 4944 4945 // Return the reset index if we've already bootstrapped 4946 return false, out.(*IndexEntry).Value, nil 4947 } 4948 4949 // BootstrapACLToken is used to create an initial ACL token 4950 func (s *StateStore) BootstrapACLTokens(index, resetIndex uint64, token *structs.ACLToken) error { 4951 txn := s.db.Txn(true) 4952 defer txn.Abort() 4953 4954 // Check if we have already done a bootstrap 4955 existing, err := txn.First("index", "id", "acl_token_bootstrap") 4956 if err != nil { 4957 return fmt.Errorf("bootstrap check failed: %v", err) 4958 } 4959 if existing != nil { 4960 if resetIndex == 0 { 4961 return fmt.Errorf("ACL bootstrap already done") 4962 } else if resetIndex != existing.(*IndexEntry).Value { 4963 return fmt.Errorf("Invalid reset index for ACL bootstrap") 4964 } 4965 } 4966 4967 // Update the Create/Modify time 4968 token.CreateIndex = index 4969 token.ModifyIndex = index 4970 4971 // Insert the token 4972 if err := txn.Insert("acl_token", token); err != nil { 4973 return fmt.Errorf("upserting token failed: %v", err) 4974 } 4975 4976 // Update the indexes table, prevents future bootstrap until reset 4977 if err := txn.Insert("index", &IndexEntry{"acl_token", index}); err != nil { 4978 return fmt.Errorf("index update failed: %v", err) 4979 } 4980 if err := txn.Insert("index", &IndexEntry{"acl_token_bootstrap", index}); err != nil { 4981 return fmt.Errorf("index update failed: %v", err) 4982 } 4983 txn.Commit() 4984 return nil 4985 } 4986 4987 // SchedulerConfig is used to get the current Scheduler configuration. 4988 func (s *StateStore) SchedulerConfig() (uint64, *structs.SchedulerConfiguration, error) { 4989 tx := s.db.Txn(false) 4990 defer tx.Abort() 4991 4992 // Get the scheduler config 4993 c, err := tx.First("scheduler_config", "id") 4994 if err != nil { 4995 return 0, nil, fmt.Errorf("failed scheduler config lookup: %s", err) 4996 } 4997 4998 config, ok := c.(*structs.SchedulerConfiguration) 4999 if !ok { 5000 return 0, nil, nil 5001 } 5002 5003 return config.ModifyIndex, config, nil 5004 } 5005 5006 // SchedulerSetConfig is used to set the current Scheduler configuration. 5007 func (s *StateStore) SchedulerSetConfig(idx uint64, config *structs.SchedulerConfiguration) error { 5008 tx := s.db.Txn(true) 5009 defer tx.Abort() 5010 5011 s.schedulerSetConfigTxn(idx, tx, config) 5012 5013 tx.Commit() 5014 return nil 5015 } 5016 5017 func (s *StateStore) ClusterMetadata() (*structs.ClusterMetadata, error) { 5018 txn := s.db.Txn(false) 5019 defer txn.Abort() 5020 5021 // Get the cluster metadata 5022 m, err := txn.First("cluster_meta", "id") 5023 if err != nil { 5024 return nil, errors.Wrap(err, "failed cluster metadata lookup") 5025 } 5026 5027 if m != nil { 5028 return m.(*structs.ClusterMetadata), nil 5029 } 5030 5031 return nil, nil 5032 } 5033 5034 func (s *StateStore) ClusterSetMetadata(index uint64, meta *structs.ClusterMetadata) error { 5035 txn := s.db.Txn(true) 5036 defer txn.Abort() 5037 5038 if err := s.setClusterMetadata(txn, meta); err != nil { 5039 return errors.Wrap(err, "set cluster metadata failed") 5040 } 5041 5042 txn.Commit() 5043 return nil 5044 } 5045 5046 // WithWriteTransaction executes the passed function within a write transaction, 5047 // and returns its result. If the invocation returns no error, the transaction 5048 // is committed; otherwise, it's aborted. 5049 func (s *StateStore) WithWriteTransaction(fn func(Txn) error) error { 5050 tx := s.db.Txn(true) 5051 defer tx.Abort() 5052 5053 err := fn(tx) 5054 if err == nil { 5055 tx.Commit() 5056 } 5057 return err 5058 } 5059 5060 // SchedulerCASConfig is used to update the scheduler configuration with a 5061 // given Raft index. If the CAS index specified is not equal to the last observed index 5062 // for the config, then the call is a noop. 5063 func (s *StateStore) SchedulerCASConfig(idx, cidx uint64, config *structs.SchedulerConfiguration) (bool, error) { 5064 tx := s.db.Txn(true) 5065 defer tx.Abort() 5066 5067 // Check for an existing config 5068 existing, err := tx.First("scheduler_config", "id") 5069 if err != nil { 5070 return false, fmt.Errorf("failed scheduler config lookup: %s", err) 5071 } 5072 5073 // If the existing index does not match the provided CAS 5074 // index arg, then we shouldn't update anything and can safely 5075 // return early here. 5076 e, ok := existing.(*structs.SchedulerConfiguration) 5077 if !ok || (e != nil && e.ModifyIndex != cidx) { 5078 return false, nil 5079 } 5080 5081 s.schedulerSetConfigTxn(idx, tx, config) 5082 5083 tx.Commit() 5084 return true, nil 5085 } 5086 5087 func (s *StateStore) schedulerSetConfigTxn(idx uint64, tx *memdb.Txn, config *structs.SchedulerConfiguration) error { 5088 // Check for an existing config 5089 existing, err := tx.First("scheduler_config", "id") 5090 if err != nil { 5091 return fmt.Errorf("failed scheduler config lookup: %s", err) 5092 } 5093 5094 // Set the indexes. 5095 if existing != nil { 5096 config.CreateIndex = existing.(*structs.SchedulerConfiguration).CreateIndex 5097 } else { 5098 config.CreateIndex = idx 5099 } 5100 config.ModifyIndex = idx 5101 5102 if err := tx.Insert("scheduler_config", config); err != nil { 5103 return fmt.Errorf("failed updating scheduler config: %s", err) 5104 } 5105 return nil 5106 } 5107 5108 func (s *StateStore) setClusterMetadata(txn *memdb.Txn, meta *structs.ClusterMetadata) error { 5109 // Check for an existing config, if it exists, sanity check the cluster ID matches 5110 existing, err := txn.First("cluster_meta", "id") 5111 if err != nil { 5112 return fmt.Errorf("failed cluster meta lookup: %v", err) 5113 } 5114 5115 if existing != nil { 5116 existingClusterID := existing.(*structs.ClusterMetadata).ClusterID 5117 if meta.ClusterID != existingClusterID && existingClusterID != "" { 5118 // there is a bug in cluster ID detection 5119 return fmt.Errorf("refusing to set new cluster id, previous: %s, new: %s", existingClusterID, meta.ClusterID) 5120 } 5121 } 5122 5123 // update is technically a noop, unless someday we add more / mutable fields 5124 if err := txn.Insert("cluster_meta", meta); err != nil { 5125 return fmt.Errorf("set cluster metadata failed: %v", err) 5126 } 5127 5128 return nil 5129 } 5130 5131 // UpsertScalingPolicy is used to insert a new scaling policy. 5132 func (s *StateStore) UpsertScalingPolicies(index uint64, scalingPolicies []*structs.ScalingPolicy) error { 5133 txn := s.db.Txn(true) 5134 defer txn.Abort() 5135 5136 if err := s.UpsertScalingPoliciesTxn(index, scalingPolicies, txn); err != nil { 5137 return err 5138 } 5139 5140 txn.Commit() 5141 return nil 5142 } 5143 5144 // upsertScalingPolicy is used to insert a new scaling policy. 5145 func (s *StateStore) UpsertScalingPoliciesTxn(index uint64, scalingPolicies []*structs.ScalingPolicy, 5146 txn *memdb.Txn) error { 5147 5148 hadUpdates := false 5149 5150 for _, policy := range scalingPolicies { 5151 // Check if the scaling policy already exists 5152 existing, err := txn.First("scaling_policy", "target", 5153 policy.Target[structs.ScalingTargetNamespace], 5154 policy.Target[structs.ScalingTargetJob], 5155 policy.Target[structs.ScalingTargetGroup]) 5156 if err != nil { 5157 return fmt.Errorf("scaling policy lookup failed: %v", err) 5158 } 5159 5160 // Setup the indexes correctly 5161 if existing != nil { 5162 p := existing.(*structs.ScalingPolicy) 5163 if !p.Diff(policy) { 5164 continue 5165 } 5166 policy.ID = p.ID 5167 policy.CreateIndex = p.CreateIndex 5168 policy.ModifyIndex = index 5169 } else { 5170 // policy.ID must have been set already in Job.Register before log apply 5171 policy.CreateIndex = index 5172 policy.ModifyIndex = index 5173 } 5174 5175 // Insert the scaling policy 5176 hadUpdates = true 5177 if err := txn.Insert("scaling_policy", policy); err != nil { 5178 return err 5179 } 5180 } 5181 5182 // Update the indexes table for scaling policy 5183 if hadUpdates { 5184 if err := txn.Insert("index", &IndexEntry{"scaling_policy", index}); err != nil { 5185 return fmt.Errorf("index update failed: %v", err) 5186 } 5187 } 5188 5189 return nil 5190 } 5191 5192 func (s *StateStore) DeleteScalingPolicies(index uint64, ids []string) error { 5193 txn := s.db.Txn(true) 5194 defer txn.Abort() 5195 5196 err := s.DeleteScalingPoliciesTxn(index, ids, txn) 5197 if err == nil { 5198 txn.Commit() 5199 } 5200 5201 return err 5202 } 5203 5204 // DeleteScalingPolicies is used to delete a set of scaling policies by ID 5205 func (s *StateStore) DeleteScalingPoliciesTxn(index uint64, ids []string, txn *memdb.Txn) error { 5206 if len(ids) == 0 { 5207 return nil 5208 } 5209 5210 for _, id := range ids { 5211 // Lookup the scaling policy 5212 existing, err := txn.First("scaling_policy", "id", id) 5213 if err != nil { 5214 return fmt.Errorf("scaling policy lookup failed: %v", err) 5215 } 5216 if existing == nil { 5217 return fmt.Errorf("scaling policy not found") 5218 } 5219 5220 // Delete the scaling policy 5221 if err := txn.Delete("scaling_policy", existing); err != nil { 5222 return fmt.Errorf("scaling policy delete failed: %v", err) 5223 } 5224 } 5225 5226 if err := txn.Insert("index", &IndexEntry{"scaling_policy", index}); err != nil { 5227 return fmt.Errorf("index update failed: %v", err) 5228 } 5229 5230 return nil 5231 } 5232 5233 // ScalingPolicies returns an iterator over all the scaling policies 5234 func (s *StateStore) ScalingPolicies(ws memdb.WatchSet) (memdb.ResultIterator, error) { 5235 txn := s.db.Txn(false) 5236 5237 // Walk the entire scaling_policy table 5238 iter, err := txn.Get("scaling_policy", "id") 5239 if err != nil { 5240 return nil, err 5241 } 5242 5243 ws.Add(iter.WatchCh()) 5244 5245 return iter, nil 5246 } 5247 5248 func (s *StateStore) ScalingPoliciesByNamespace(ws memdb.WatchSet, namespace string) (memdb.ResultIterator, error) { 5249 txn := s.db.Txn(false) 5250 5251 iter, err := txn.Get("scaling_policy", "target_prefix", namespace) 5252 if err != nil { 5253 return nil, err 5254 } 5255 5256 ws.Add(iter.WatchCh()) 5257 return iter, nil 5258 } 5259 5260 func (s *StateStore) ScalingPoliciesByJob(ws memdb.WatchSet, namespace, jobID string) (memdb.ResultIterator, error) { 5261 txn := s.db.Txn(false) 5262 return s.ScalingPoliciesByJobTxn(ws, namespace, jobID, txn) 5263 } 5264 5265 func (s *StateStore) ScalingPoliciesByJobTxn(ws memdb.WatchSet, namespace, jobID string, 5266 txn *memdb.Txn) (memdb.ResultIterator, error) { 5267 5268 iter, err := txn.Get("scaling_policy", "target_prefix", namespace, jobID) 5269 if err != nil { 5270 return nil, err 5271 } 5272 5273 ws.Add(iter.WatchCh()) 5274 return iter, nil 5275 } 5276 5277 func (s *StateStore) ScalingPolicyByID(ws memdb.WatchSet, id string) (*structs.ScalingPolicy, error) { 5278 txn := s.db.Txn(false) 5279 5280 watchCh, existing, err := txn.FirstWatch("scaling_policy", "id", id) 5281 if err != nil { 5282 return nil, fmt.Errorf("scaling_policy lookup failed: %v", err) 5283 } 5284 ws.Add(watchCh) 5285 5286 if existing != nil { 5287 return existing.(*structs.ScalingPolicy), nil 5288 } 5289 5290 return nil, nil 5291 } 5292 5293 func (s *StateStore) ScalingPolicyByTarget(ws memdb.WatchSet, target map[string]string) (*structs.ScalingPolicy, 5294 error) { 5295 txn := s.db.Txn(false) 5296 5297 // currently, only scaling policy type is against a task group 5298 namespace := target[structs.ScalingTargetNamespace] 5299 job := target[structs.ScalingTargetJob] 5300 group := target[structs.ScalingTargetGroup] 5301 5302 watchCh, existing, err := txn.FirstWatch("scaling_policy", "target", namespace, job, group) 5303 if err != nil { 5304 return nil, fmt.Errorf("scaling_policy lookup failed: %v", err) 5305 } 5306 ws.Add(watchCh) 5307 5308 if existing != nil { 5309 return existing.(*structs.ScalingPolicy), nil 5310 } 5311 5312 return nil, nil 5313 } 5314 5315 // StateSnapshot is used to provide a point-in-time snapshot 5316 type StateSnapshot struct { 5317 StateStore 5318 } 5319 5320 // DenormalizeAllocationsMap takes in a map of nodes to allocations, and queries the 5321 // Allocation for each of the Allocation diffs and merges the updated attributes with 5322 // the existing Allocation, and attaches the Job provided 5323 func (s *StateSnapshot) DenormalizeAllocationsMap(nodeAllocations map[string][]*structs.Allocation) error { 5324 for nodeID, allocs := range nodeAllocations { 5325 denormalizedAllocs, err := s.DenormalizeAllocationSlice(allocs) 5326 if err != nil { 5327 return err 5328 } 5329 5330 nodeAllocations[nodeID] = denormalizedAllocs 5331 } 5332 return nil 5333 } 5334 5335 // DenormalizeAllocationSlice queries the Allocation for each allocation diff 5336 // represented as an Allocation and merges the updated attributes with the existing 5337 // Allocation, and attaches the Job provided. 5338 // 5339 // This should only be called on terminal allocs, particularly stopped or preempted allocs 5340 func (s *StateSnapshot) DenormalizeAllocationSlice(allocs []*structs.Allocation) ([]*structs.Allocation, error) { 5341 allocDiffs := make([]*structs.AllocationDiff, len(allocs)) 5342 for i, alloc := range allocs { 5343 allocDiffs[i] = alloc.AllocationDiff() 5344 } 5345 5346 return s.DenormalizeAllocationDiffSlice(allocDiffs) 5347 } 5348 5349 // DenormalizeAllocationDiffSlice queries the Allocation for each AllocationDiff and merges 5350 // the updated attributes with the existing Allocation, and attaches the Job provided. 5351 // 5352 // This should only be called on terminal alloc, particularly stopped or preempted allocs 5353 func (s *StateSnapshot) DenormalizeAllocationDiffSlice(allocDiffs []*structs.AllocationDiff) ([]*structs.Allocation, error) { 5354 // Output index for denormalized Allocations 5355 j := 0 5356 5357 denormalizedAllocs := make([]*structs.Allocation, len(allocDiffs)) 5358 for _, allocDiff := range allocDiffs { 5359 alloc, err := s.AllocByID(nil, allocDiff.ID) 5360 if err != nil { 5361 return nil, fmt.Errorf("alloc lookup failed: %v", err) 5362 } 5363 if alloc == nil { 5364 return nil, fmt.Errorf("alloc %v doesn't exist", allocDiff.ID) 5365 } 5366 5367 // Merge the updates to the Allocation. Don't update alloc.Job for terminal allocs 5368 // so alloc refers to the latest Job view before destruction and to ease handler implementations 5369 allocCopy := alloc.Copy() 5370 5371 if allocDiff.PreemptedByAllocation != "" { 5372 allocCopy.PreemptedByAllocation = allocDiff.PreemptedByAllocation 5373 allocCopy.DesiredDescription = getPreemptedAllocDesiredDescription(allocDiff.PreemptedByAllocation) 5374 allocCopy.DesiredStatus = structs.AllocDesiredStatusEvict 5375 } else { 5376 // If alloc is a stopped alloc 5377 allocCopy.DesiredDescription = allocDiff.DesiredDescription 5378 allocCopy.DesiredStatus = structs.AllocDesiredStatusStop 5379 if allocDiff.ClientStatus != "" { 5380 allocCopy.ClientStatus = allocDiff.ClientStatus 5381 } 5382 if allocDiff.FollowupEvalID != "" { 5383 allocCopy.FollowupEvalID = allocDiff.FollowupEvalID 5384 } 5385 } 5386 if allocDiff.ModifyTime != 0 { 5387 allocCopy.ModifyTime = allocDiff.ModifyTime 5388 } 5389 5390 // Update the allocDiff in the slice to equal the denormalized alloc 5391 denormalizedAllocs[j] = allocCopy 5392 j++ 5393 } 5394 // Retain only the denormalized Allocations in the slice 5395 denormalizedAllocs = denormalizedAllocs[:j] 5396 return denormalizedAllocs, nil 5397 } 5398 5399 func getPreemptedAllocDesiredDescription(PreemptedByAllocID string) string { 5400 return fmt.Sprintf("Preempted by alloc ID %v", PreemptedByAllocID) 5401 } 5402 5403 // StateRestore is used to optimize the performance when 5404 // restoring state by only using a single large transaction 5405 // instead of thousands of sub transactions 5406 type StateRestore struct { 5407 txn *memdb.Txn 5408 } 5409 5410 // Abort is used to abort the restore operation 5411 func (s *StateRestore) Abort() { 5412 s.txn.Abort() 5413 } 5414 5415 // Commit is used to commit the restore operation 5416 func (s *StateRestore) Commit() { 5417 s.txn.Commit() 5418 } 5419 5420 // NodeRestore is used to restore a node 5421 func (r *StateRestore) NodeRestore(node *structs.Node) error { 5422 if err := r.txn.Insert("nodes", node); err != nil { 5423 return fmt.Errorf("node insert failed: %v", err) 5424 } 5425 return nil 5426 } 5427 5428 // JobRestore is used to restore a job 5429 func (r *StateRestore) JobRestore(job *structs.Job) error { 5430 if err := r.txn.Insert("jobs", job); err != nil { 5431 return fmt.Errorf("job insert failed: %v", err) 5432 } 5433 return nil 5434 } 5435 5436 // EvalRestore is used to restore an evaluation 5437 func (r *StateRestore) EvalRestore(eval *structs.Evaluation) error { 5438 if err := r.txn.Insert("evals", eval); err != nil { 5439 return fmt.Errorf("eval insert failed: %v", err) 5440 } 5441 return nil 5442 } 5443 5444 // AllocRestore is used to restore an allocation 5445 func (r *StateRestore) AllocRestore(alloc *structs.Allocation) error { 5446 if err := r.txn.Insert("allocs", alloc); err != nil { 5447 return fmt.Errorf("alloc insert failed: %v", err) 5448 } 5449 return nil 5450 } 5451 5452 // IndexRestore is used to restore an index 5453 func (r *StateRestore) IndexRestore(idx *IndexEntry) error { 5454 if err := r.txn.Insert("index", idx); err != nil { 5455 return fmt.Errorf("index insert failed: %v", err) 5456 } 5457 return nil 5458 } 5459 5460 // PeriodicLaunchRestore is used to restore a periodic launch. 5461 func (r *StateRestore) PeriodicLaunchRestore(launch *structs.PeriodicLaunch) error { 5462 if err := r.txn.Insert("periodic_launch", launch); err != nil { 5463 return fmt.Errorf("periodic launch insert failed: %v", err) 5464 } 5465 return nil 5466 } 5467 5468 // JobSummaryRestore is used to restore a job summary 5469 func (r *StateRestore) JobSummaryRestore(jobSummary *structs.JobSummary) error { 5470 if err := r.txn.Insert("job_summary", jobSummary); err != nil { 5471 return fmt.Errorf("job summary insert failed: %v", err) 5472 } 5473 return nil 5474 } 5475 5476 // JobVersionRestore is used to restore a job version 5477 func (r *StateRestore) JobVersionRestore(version *structs.Job) error { 5478 if err := r.txn.Insert("job_version", version); err != nil { 5479 return fmt.Errorf("job version insert failed: %v", err) 5480 } 5481 return nil 5482 } 5483 5484 // DeploymentRestore is used to restore a deployment 5485 func (r *StateRestore) DeploymentRestore(deployment *structs.Deployment) error { 5486 if err := r.txn.Insert("deployment", deployment); err != nil { 5487 return fmt.Errorf("deployment insert failed: %v", err) 5488 } 5489 return nil 5490 } 5491 5492 // VaultAccessorRestore is used to restore a vault accessor 5493 func (r *StateRestore) VaultAccessorRestore(accessor *structs.VaultAccessor) error { 5494 if err := r.txn.Insert("vault_accessors", accessor); err != nil { 5495 return fmt.Errorf("vault accessor insert failed: %v", err) 5496 } 5497 return nil 5498 } 5499 5500 // SITokenAccessorRestore is used to restore an SI token accessor 5501 func (r *StateRestore) SITokenAccessorRestore(accessor *structs.SITokenAccessor) error { 5502 if err := r.txn.Insert(siTokenAccessorTable, accessor); err != nil { 5503 return errors.Wrap(err, "si token accessor insert failed") 5504 } 5505 return nil 5506 } 5507 5508 // ACLPolicyRestore is used to restore an ACL policy 5509 func (r *StateRestore) ACLPolicyRestore(policy *structs.ACLPolicy) error { 5510 if err := r.txn.Insert("acl_policy", policy); err != nil { 5511 return fmt.Errorf("inserting acl policy failed: %v", err) 5512 } 5513 return nil 5514 } 5515 5516 // ACLTokenRestore is used to restore an ACL token 5517 func (r *StateRestore) ACLTokenRestore(token *structs.ACLToken) error { 5518 if err := r.txn.Insert("acl_token", token); err != nil { 5519 return fmt.Errorf("inserting acl token failed: %v", err) 5520 } 5521 return nil 5522 } 5523 5524 func (r *StateRestore) SchedulerConfigRestore(schedConfig *structs.SchedulerConfiguration) error { 5525 if err := r.txn.Insert("scheduler_config", schedConfig); err != nil { 5526 return fmt.Errorf("inserting scheduler config failed: %s", err) 5527 } 5528 return nil 5529 } 5530 5531 func (r *StateRestore) ClusterMetadataRestore(meta *structs.ClusterMetadata) error { 5532 if err := r.txn.Insert("cluster_meta", meta); err != nil { 5533 return fmt.Errorf("inserting cluster meta failed: %v", err) 5534 } 5535 return nil 5536 } 5537 5538 // ScalingPolicyRestore is used to restore a scaling policy 5539 func (r *StateRestore) ScalingPolicyRestore(scalingPolicy *structs.ScalingPolicy) error { 5540 if err := r.txn.Insert("scaling_policy", scalingPolicy); err != nil { 5541 return fmt.Errorf("scaling policy insert failed: %v", err) 5542 } 5543 return nil 5544 } 5545 5546 // CSIPluginRestore is used to restore a CSI plugin 5547 func (r *StateRestore) CSIPluginRestore(plugin *structs.CSIPlugin) error { 5548 if err := r.txn.Insert("csi_plugins", plugin); err != nil { 5549 return fmt.Errorf("csi plugin insert failed: %v", err) 5550 } 5551 return nil 5552 } 5553 5554 // CSIVolumeRestore is used to restore a CSI volume 5555 func (r *StateRestore) CSIVolumeRestore(volume *structs.CSIVolume) error { 5556 if err := r.txn.Insert("csi_volumes", volume); err != nil { 5557 return fmt.Errorf("csi volume insert failed: %v", err) 5558 } 5559 return nil 5560 } 5561 5562 func (r *StateRestore) ScalingEventsRestore(jobEvents *structs.JobScalingEvents) error { 5563 if err := r.txn.Insert("scaling_event", jobEvents); err != nil { 5564 return fmt.Errorf("scaling event insert failed: %v", err) 5565 } 5566 return nil 5567 }