github.com/ferranbt/nomad@v0.9.3-0.20190607002617-85c449b7667c/nomad/structs/structs.go (about) 1 package structs 2 3 import ( 4 "bytes" 5 "container/heap" 6 "crypto/md5" 7 "crypto/sha1" 8 "crypto/sha256" 9 "crypto/sha512" 10 "encoding/base32" 11 "encoding/base64" 12 "encoding/hex" 13 "errors" 14 "fmt" 15 "io" 16 "math" 17 "net" 18 "net/url" 19 "os" 20 "path/filepath" 21 "reflect" 22 "regexp" 23 "sort" 24 "strconv" 25 "strings" 26 "time" 27 28 "github.com/gorhill/cronexpr" 29 "github.com/hashicorp/consul/api" 30 hcodec "github.com/hashicorp/go-msgpack/codec" 31 "github.com/hashicorp/go-multierror" 32 "github.com/hashicorp/go-version" 33 "github.com/hashicorp/nomad/acl" 34 "github.com/hashicorp/nomad/helper" 35 "github.com/hashicorp/nomad/helper/args" 36 "github.com/hashicorp/nomad/helper/uuid" 37 "github.com/hashicorp/nomad/lib/kheap" 38 psstructs "github.com/hashicorp/nomad/plugins/shared/structs" 39 "github.com/mitchellh/copystructure" 40 "github.com/ugorji/go/codec" 41 "golang.org/x/crypto/blake2b" 42 ) 43 44 var ( 45 // validPolicyName is used to validate a policy name 46 validPolicyName = regexp.MustCompile("^[a-zA-Z0-9-]{1,128}$") 47 48 // b32 is a lowercase base32 encoding for use in URL friendly service hashes 49 b32 = base32.NewEncoding(strings.ToLower("abcdefghijklmnopqrstuvwxyz234567")) 50 ) 51 52 type MessageType uint8 53 54 const ( 55 NodeRegisterRequestType MessageType = iota 56 NodeDeregisterRequestType 57 NodeUpdateStatusRequestType 58 NodeUpdateDrainRequestType 59 JobRegisterRequestType 60 JobDeregisterRequestType 61 EvalUpdateRequestType 62 EvalDeleteRequestType 63 AllocUpdateRequestType 64 AllocClientUpdateRequestType 65 ReconcileJobSummariesRequestType 66 VaultAccessorRegisterRequestType 67 VaultAccessorDeregisterRequestType 68 ApplyPlanResultsRequestType 69 DeploymentStatusUpdateRequestType 70 DeploymentPromoteRequestType 71 DeploymentAllocHealthRequestType 72 DeploymentDeleteRequestType 73 JobStabilityRequestType 74 ACLPolicyUpsertRequestType 75 ACLPolicyDeleteRequestType 76 ACLTokenUpsertRequestType 77 ACLTokenDeleteRequestType 78 ACLTokenBootstrapRequestType 79 AutopilotRequestType 80 UpsertNodeEventsType 81 JobBatchDeregisterRequestType 82 AllocUpdateDesiredTransitionRequestType 83 NodeUpdateEligibilityRequestType 84 BatchNodeUpdateDrainRequestType 85 SchedulerConfigRequestType 86 ) 87 88 const ( 89 // IgnoreUnknownTypeFlag is set along with a MessageType 90 // to indicate that the message type can be safely ignored 91 // if it is not recognized. This is for future proofing, so 92 // that new commands can be added in a way that won't cause 93 // old servers to crash when the FSM attempts to process them. 94 IgnoreUnknownTypeFlag MessageType = 128 95 96 // ApiMajorVersion is returned as part of the Status.Version request. 97 // It should be incremented anytime the APIs are changed in a way 98 // that would break clients for sane client versioning. 99 ApiMajorVersion = 1 100 101 // ApiMinorVersion is returned as part of the Status.Version request. 102 // It should be incremented anytime the APIs are changed to allow 103 // for sane client versioning. Minor changes should be compatible 104 // within the major version. 105 ApiMinorVersion = 1 106 107 ProtocolVersion = "protocol" 108 APIMajorVersion = "api.major" 109 APIMinorVersion = "api.minor" 110 111 GetterModeAny = "any" 112 GetterModeFile = "file" 113 GetterModeDir = "dir" 114 115 // maxPolicyDescriptionLength limits a policy description length 116 maxPolicyDescriptionLength = 256 117 118 // maxTokenNameLength limits a ACL token name length 119 maxTokenNameLength = 256 120 121 // ACLClientToken and ACLManagementToken are the only types of tokens 122 ACLClientToken = "client" 123 ACLManagementToken = "management" 124 125 // DefaultNamespace is the default namespace. 126 DefaultNamespace = "default" 127 DefaultNamespaceDescription = "Default shared namespace" 128 129 // JitterFraction is a the limit to the amount of jitter we apply 130 // to a user specified MaxQueryTime. We divide the specified time by 131 // the fraction. So 16 == 6.25% limit of jitter. This jitter is also 132 // applied to RPCHoldTimeout. 133 JitterFraction = 16 134 135 // MaxRetainedNodeEvents is the maximum number of node events that will be 136 // retained for a single node 137 MaxRetainedNodeEvents = 10 138 139 // MaxRetainedNodeScores is the number of top scoring nodes for which we 140 // retain scoring metadata 141 MaxRetainedNodeScores = 5 142 143 // Normalized scorer name 144 NormScorerName = "normalized-score" 145 ) 146 147 // Context defines the scope in which a search for Nomad object operates, and 148 // is also used to query the matching index value for this context 149 type Context string 150 151 const ( 152 Allocs Context = "allocs" 153 Deployments Context = "deployment" 154 Evals Context = "evals" 155 Jobs Context = "jobs" 156 Nodes Context = "nodes" 157 Namespaces Context = "namespaces" 158 Quotas Context = "quotas" 159 All Context = "all" 160 ) 161 162 // NamespacedID is a tuple of an ID and a namespace 163 type NamespacedID struct { 164 ID string 165 Namespace string 166 } 167 168 // NewNamespacedID returns a new namespaced ID given the ID and namespace 169 func NewNamespacedID(id, ns string) NamespacedID { 170 return NamespacedID{ 171 ID: id, 172 Namespace: ns, 173 } 174 } 175 176 func (n NamespacedID) String() string { 177 return fmt.Sprintf("<ns: %q, id: %q>", n.Namespace, n.ID) 178 } 179 180 // RPCInfo is used to describe common information about query 181 type RPCInfo interface { 182 RequestRegion() string 183 IsRead() bool 184 AllowStaleRead() bool 185 IsForwarded() bool 186 SetForwarded() 187 } 188 189 // InternalRpcInfo allows adding internal RPC metadata to an RPC. This struct 190 // should NOT be replicated in the API package as it is internal only. 191 type InternalRpcInfo struct { 192 // Forwarded marks whether the RPC has been forwarded. 193 Forwarded bool 194 } 195 196 // IsForwarded returns whether the RPC is forwarded from another server. 197 func (i *InternalRpcInfo) IsForwarded() bool { 198 return i.Forwarded 199 } 200 201 // SetForwarded marks that the RPC is being forwarded from another server. 202 func (i *InternalRpcInfo) SetForwarded() { 203 i.Forwarded = true 204 } 205 206 // QueryOptions is used to specify various flags for read queries 207 type QueryOptions struct { 208 // The target region for this query 209 Region string 210 211 // Namespace is the target namespace for the query. 212 Namespace string 213 214 // If set, wait until query exceeds given index. Must be provided 215 // with MaxQueryTime. 216 MinQueryIndex uint64 217 218 // Provided with MinQueryIndex to wait for change. 219 MaxQueryTime time.Duration 220 221 // If set, any follower can service the request. Results 222 // may be arbitrarily stale. 223 AllowStale bool 224 225 // If set, used as prefix for resource list searches 226 Prefix string 227 228 // AuthToken is secret portion of the ACL token used for the request 229 AuthToken string 230 231 InternalRpcInfo 232 } 233 234 func (q QueryOptions) RequestRegion() string { 235 return q.Region 236 } 237 238 func (q QueryOptions) RequestNamespace() string { 239 if q.Namespace == "" { 240 return DefaultNamespace 241 } 242 return q.Namespace 243 } 244 245 // QueryOption only applies to reads, so always true 246 func (q QueryOptions) IsRead() bool { 247 return true 248 } 249 250 func (q QueryOptions) AllowStaleRead() bool { 251 return q.AllowStale 252 } 253 254 type WriteRequest struct { 255 // The target region for this write 256 Region string 257 258 // Namespace is the target namespace for the write. 259 Namespace string 260 261 // AuthToken is secret portion of the ACL token used for the request 262 AuthToken string 263 264 InternalRpcInfo 265 } 266 267 func (w WriteRequest) RequestRegion() string { 268 // The target region for this request 269 return w.Region 270 } 271 272 func (w WriteRequest) RequestNamespace() string { 273 if w.Namespace == "" { 274 return DefaultNamespace 275 } 276 return w.Namespace 277 } 278 279 // WriteRequest only applies to writes, always false 280 func (w WriteRequest) IsRead() bool { 281 return false 282 } 283 284 func (w WriteRequest) AllowStaleRead() bool { 285 return false 286 } 287 288 // QueryMeta allows a query response to include potentially 289 // useful metadata about a query 290 type QueryMeta struct { 291 // This is the index associated with the read 292 Index uint64 293 294 // If AllowStale is used, this is time elapsed since 295 // last contact between the follower and leader. This 296 // can be used to gauge staleness. 297 LastContact time.Duration 298 299 // Used to indicate if there is a known leader node 300 KnownLeader bool 301 } 302 303 // WriteMeta allows a write response to include potentially 304 // useful metadata about the write 305 type WriteMeta struct { 306 // This is the index associated with the write 307 Index uint64 308 } 309 310 // NodeRegisterRequest is used for Node.Register endpoint 311 // to register a node as being a schedulable entity. 312 type NodeRegisterRequest struct { 313 Node *Node 314 NodeEvent *NodeEvent 315 WriteRequest 316 } 317 318 // NodeDeregisterRequest is used for Node.Deregister endpoint 319 // to deregister a node as being a schedulable entity. 320 type NodeDeregisterRequest struct { 321 NodeID string 322 WriteRequest 323 } 324 325 // NodeServerInfo is used to in NodeUpdateResponse to return Nomad server 326 // information used in RPC server lists. 327 type NodeServerInfo struct { 328 // RPCAdvertiseAddr is the IP endpoint that a Nomad Server wishes to 329 // be contacted at for RPCs. 330 RPCAdvertiseAddr string 331 332 // RpcMajorVersion is the major version number the Nomad Server 333 // supports 334 RPCMajorVersion int32 335 336 // RpcMinorVersion is the minor version number the Nomad Server 337 // supports 338 RPCMinorVersion int32 339 340 // Datacenter is the datacenter that a Nomad server belongs to 341 Datacenter string 342 } 343 344 // NodeUpdateStatusRequest is used for Node.UpdateStatus endpoint 345 // to update the status of a node. 346 type NodeUpdateStatusRequest struct { 347 NodeID string 348 Status string 349 NodeEvent *NodeEvent 350 UpdatedAt int64 351 WriteRequest 352 } 353 354 // NodeUpdateDrainRequest is used for updating the drain strategy 355 type NodeUpdateDrainRequest struct { 356 NodeID string 357 DrainStrategy *DrainStrategy 358 359 // COMPAT Remove in version 0.10 360 // As part of Nomad 0.8 we have deprecated the drain boolean in favor of a 361 // drain strategy but we need to handle the upgrade path where the Raft log 362 // contains drain updates with just the drain boolean being manipulated. 363 Drain bool 364 365 // MarkEligible marks the node as eligible if removing the drain strategy. 366 MarkEligible bool 367 368 // NodeEvent is the event added to the node 369 NodeEvent *NodeEvent 370 371 // UpdatedAt represents server time of receiving request 372 UpdatedAt int64 373 374 WriteRequest 375 } 376 377 // BatchNodeUpdateDrainRequest is used for updating the drain strategy for a 378 // batch of nodes 379 type BatchNodeUpdateDrainRequest struct { 380 // Updates is a mapping of nodes to their updated drain strategy 381 Updates map[string]*DrainUpdate 382 383 // NodeEvents is a mapping of the node to the event to add to the node 384 NodeEvents map[string]*NodeEvent 385 386 // UpdatedAt represents server time of receiving request 387 UpdatedAt int64 388 389 WriteRequest 390 } 391 392 // DrainUpdate is used to update the drain of a node 393 type DrainUpdate struct { 394 // DrainStrategy is the new strategy for the node 395 DrainStrategy *DrainStrategy 396 397 // MarkEligible marks the node as eligible if removing the drain strategy. 398 MarkEligible bool 399 } 400 401 // NodeUpdateEligibilityRequest is used for updating the scheduling eligibility 402 type NodeUpdateEligibilityRequest struct { 403 NodeID string 404 Eligibility string 405 406 // NodeEvent is the event added to the node 407 NodeEvent *NodeEvent 408 409 // UpdatedAt represents server time of receiving request 410 UpdatedAt int64 411 412 WriteRequest 413 } 414 415 // NodeEvaluateRequest is used to re-evaluate the node 416 type NodeEvaluateRequest struct { 417 NodeID string 418 WriteRequest 419 } 420 421 // NodeSpecificRequest is used when we just need to specify a target node 422 type NodeSpecificRequest struct { 423 NodeID string 424 SecretID string 425 QueryOptions 426 } 427 428 // SearchResponse is used to return matches and information about whether 429 // the match list is truncated specific to each type of context. 430 type SearchResponse struct { 431 // Map of context types to ids which match a specified prefix 432 Matches map[Context][]string 433 434 // Truncations indicates whether the matches for a particular context have 435 // been truncated 436 Truncations map[Context]bool 437 438 QueryMeta 439 } 440 441 // SearchRequest is used to parameterize a request, and returns a 442 // list of matches made up of jobs, allocations, evaluations, and/or nodes, 443 // along with whether or not the information returned is truncated. 444 type SearchRequest struct { 445 // Prefix is what ids are matched to. I.e, if the given prefix were 446 // "a", potential matches might be "abcd" or "aabb" 447 Prefix string 448 449 // Context is the type that can be matched against. A context can be a job, 450 // node, evaluation, allocation, or empty (indicated every context should be 451 // matched) 452 Context Context 453 454 QueryOptions 455 } 456 457 // JobRegisterRequest is used for Job.Register endpoint 458 // to register a job as being a schedulable entity. 459 type JobRegisterRequest struct { 460 Job *Job 461 462 // If EnforceIndex is set then the job will only be registered if the passed 463 // JobModifyIndex matches the current Jobs index. If the index is zero, the 464 // register only occurs if the job is new. 465 EnforceIndex bool 466 JobModifyIndex uint64 467 468 // PolicyOverride is set when the user is attempting to override any policies 469 PolicyOverride bool 470 471 WriteRequest 472 } 473 474 // JobDeregisterRequest is used for Job.Deregister endpoint 475 // to deregister a job as being a schedulable entity. 476 type JobDeregisterRequest struct { 477 JobID string 478 479 // Purge controls whether the deregister purges the job from the system or 480 // whether the job is just marked as stopped and will be removed by the 481 // garbage collector 482 Purge bool 483 484 WriteRequest 485 } 486 487 // JobBatchDeregisterRequest is used to batch deregister jobs and upsert 488 // evaluations. 489 type JobBatchDeregisterRequest struct { 490 // Jobs is the set of jobs to deregister 491 Jobs map[NamespacedID]*JobDeregisterOptions 492 493 // Evals is the set of evaluations to create. 494 Evals []*Evaluation 495 496 WriteRequest 497 } 498 499 // JobDeregisterOptions configures how a job is deregistered. 500 type JobDeregisterOptions struct { 501 // Purge controls whether the deregister purges the job from the system or 502 // whether the job is just marked as stopped and will be removed by the 503 // garbage collector 504 Purge bool 505 } 506 507 // JobEvaluateRequest is used when we just need to re-evaluate a target job 508 type JobEvaluateRequest struct { 509 JobID string 510 EvalOptions EvalOptions 511 WriteRequest 512 } 513 514 // EvalOptions is used to encapsulate options when forcing a job evaluation 515 type EvalOptions struct { 516 ForceReschedule bool 517 } 518 519 // JobSpecificRequest is used when we just need to specify a target job 520 type JobSpecificRequest struct { 521 JobID string 522 All bool 523 QueryOptions 524 } 525 526 // JobListRequest is used to parameterize a list request 527 type JobListRequest struct { 528 QueryOptions 529 } 530 531 // JobPlanRequest is used for the Job.Plan endpoint to trigger a dry-run 532 // evaluation of the Job. 533 type JobPlanRequest struct { 534 Job *Job 535 Diff bool // Toggles an annotated diff 536 // PolicyOverride is set when the user is attempting to override any policies 537 PolicyOverride bool 538 WriteRequest 539 } 540 541 // JobSummaryRequest is used when we just need to get a specific job summary 542 type JobSummaryRequest struct { 543 JobID string 544 QueryOptions 545 } 546 547 // JobDispatchRequest is used to dispatch a job based on a parameterized job 548 type JobDispatchRequest struct { 549 JobID string 550 Payload []byte 551 Meta map[string]string 552 WriteRequest 553 } 554 555 // JobValidateRequest is used to validate a job 556 type JobValidateRequest struct { 557 Job *Job 558 WriteRequest 559 } 560 561 // JobRevertRequest is used to revert a job to a prior version. 562 type JobRevertRequest struct { 563 // JobID is the ID of the job being reverted 564 JobID string 565 566 // JobVersion the version to revert to. 567 JobVersion uint64 568 569 // EnforcePriorVersion if set will enforce that the job is at the given 570 // version before reverting. 571 EnforcePriorVersion *uint64 572 573 // VaultToken is the Vault token that proves the submitter of the job revert 574 // has access to any Vault policies specified in the targeted job version. This 575 // field is only used to transfer the token and is not stored after the Job 576 // revert. 577 VaultToken string 578 579 WriteRequest 580 } 581 582 // JobStabilityRequest is used to marked a job as stable. 583 type JobStabilityRequest struct { 584 // Job to set the stability on 585 JobID string 586 JobVersion uint64 587 588 // Set the stability 589 Stable bool 590 WriteRequest 591 } 592 593 // JobStabilityResponse is the response when marking a job as stable. 594 type JobStabilityResponse struct { 595 WriteMeta 596 } 597 598 // NodeListRequest is used to parameterize a list request 599 type NodeListRequest struct { 600 QueryOptions 601 } 602 603 // EvalUpdateRequest is used for upserting evaluations. 604 type EvalUpdateRequest struct { 605 Evals []*Evaluation 606 EvalToken string 607 WriteRequest 608 } 609 610 // EvalDeleteRequest is used for deleting an evaluation. 611 type EvalDeleteRequest struct { 612 Evals []string 613 Allocs []string 614 WriteRequest 615 } 616 617 // EvalSpecificRequest is used when we just need to specify a target evaluation 618 type EvalSpecificRequest struct { 619 EvalID string 620 QueryOptions 621 } 622 623 // EvalAckRequest is used to Ack/Nack a specific evaluation 624 type EvalAckRequest struct { 625 EvalID string 626 Token string 627 WriteRequest 628 } 629 630 // EvalDequeueRequest is used when we want to dequeue an evaluation 631 type EvalDequeueRequest struct { 632 Schedulers []string 633 Timeout time.Duration 634 SchedulerVersion uint16 635 WriteRequest 636 } 637 638 // EvalListRequest is used to list the evaluations 639 type EvalListRequest struct { 640 QueryOptions 641 } 642 643 // PlanRequest is used to submit an allocation plan to the leader 644 type PlanRequest struct { 645 Plan *Plan 646 WriteRequest 647 } 648 649 // ApplyPlanResultsRequest is used by the planner to apply a Raft transaction 650 // committing the result of a plan. 651 type ApplyPlanResultsRequest struct { 652 // AllocUpdateRequest holds the allocation updates to be made by the 653 // scheduler. 654 AllocUpdateRequest 655 656 // Deployment is the deployment created or updated as a result of a 657 // scheduling event. 658 Deployment *Deployment 659 660 // DeploymentUpdates is a set of status updates to apply to the given 661 // deployments. This allows the scheduler to cancel any unneeded deployment 662 // because the job is stopped or the update block is removed. 663 DeploymentUpdates []*DeploymentStatusUpdate 664 665 // EvalID is the eval ID of the plan being applied. The modify index of the 666 // evaluation is updated as part of applying the plan to ensure that subsequent 667 // scheduling events for the same job will wait for the index that last produced 668 // state changes. This is necessary for blocked evaluations since they can be 669 // processed many times, potentially making state updates, without the state of 670 // the evaluation itself being updated. 671 EvalID string 672 673 // COMPAT 0.11 674 // NodePreemptions is a slice of allocations from other lower priority jobs 675 // that are preempted. Preempted allocations are marked as evicted. 676 // Deprecated: Replaced with AllocsPreempted which contains only the diff 677 NodePreemptions []*Allocation 678 679 // AllocsPreempted is a slice of allocation diffs from other lower priority jobs 680 // that are preempted. Preempted allocations are marked as evicted. 681 AllocsPreempted []*AllocationDiff 682 683 // PreemptionEvals is a slice of follow up evals for jobs whose allocations 684 // have been preempted to place allocs in this plan 685 PreemptionEvals []*Evaluation 686 } 687 688 // AllocUpdateRequest is used to submit changes to allocations, either 689 // to cause evictions or to assign new allocations. Both can be done 690 // within a single transaction 691 type AllocUpdateRequest struct { 692 // COMPAT 0.11 693 // Alloc is the list of new allocations to assign 694 // Deprecated: Replaced with two separate slices, one containing stopped allocations 695 // and another containing updated allocations 696 Alloc []*Allocation 697 698 // Allocations to stop. Contains only the diff, not the entire allocation 699 AllocsStopped []*AllocationDiff 700 701 // New or updated allocations 702 AllocsUpdated []*Allocation 703 704 // Evals is the list of new evaluations to create 705 // Evals are valid only when used in the Raft RPC 706 Evals []*Evaluation 707 708 // Job is the shared parent job of the allocations. 709 // It is pulled out since it is common to reduce payload size. 710 Job *Job 711 712 WriteRequest 713 } 714 715 // AllocUpdateDesiredTransitionRequest is used to submit changes to allocations 716 // desired transition state. 717 type AllocUpdateDesiredTransitionRequest struct { 718 // Allocs is the mapping of allocation ids to their desired state 719 // transition 720 Allocs map[string]*DesiredTransition 721 722 // Evals is the set of evaluations to create 723 Evals []*Evaluation 724 725 WriteRequest 726 } 727 728 // AllocStopRequest is used to stop and reschedule a running Allocation. 729 type AllocStopRequest struct { 730 AllocID string 731 732 WriteRequest 733 } 734 735 // AllocStopResponse is the response to an `AllocStopRequest` 736 type AllocStopResponse struct { 737 // EvalID is the id of the follow up evalution for the rescheduled alloc. 738 EvalID string 739 740 WriteMeta 741 } 742 743 // AllocListRequest is used to request a list of allocations 744 type AllocListRequest struct { 745 QueryOptions 746 } 747 748 // AllocSpecificRequest is used to query a specific allocation 749 type AllocSpecificRequest struct { 750 AllocID string 751 QueryOptions 752 } 753 754 // AllocSignalRequest is used to signal a specific allocation 755 type AllocSignalRequest struct { 756 AllocID string 757 Task string 758 Signal string 759 QueryOptions 760 } 761 762 // AllocsGetRequest is used to query a set of allocations 763 type AllocsGetRequest struct { 764 AllocIDs []string 765 QueryOptions 766 } 767 768 // AllocRestartRequest is used to restart a specific allocations tasks. 769 type AllocRestartRequest struct { 770 AllocID string 771 TaskName string 772 773 QueryOptions 774 } 775 776 // PeriodicForceRequest is used to force a specific periodic job. 777 type PeriodicForceRequest struct { 778 JobID string 779 WriteRequest 780 } 781 782 // ServerMembersResponse has the list of servers in a cluster 783 type ServerMembersResponse struct { 784 ServerName string 785 ServerRegion string 786 ServerDC string 787 Members []*ServerMember 788 } 789 790 // ServerMember holds information about a Nomad server agent in a cluster 791 type ServerMember struct { 792 Name string 793 Addr net.IP 794 Port uint16 795 Tags map[string]string 796 Status string 797 ProtocolMin uint8 798 ProtocolMax uint8 799 ProtocolCur uint8 800 DelegateMin uint8 801 DelegateMax uint8 802 DelegateCur uint8 803 } 804 805 // DeriveVaultTokenRequest is used to request wrapped Vault tokens for the 806 // following tasks in the given allocation 807 type DeriveVaultTokenRequest struct { 808 NodeID string 809 SecretID string 810 AllocID string 811 Tasks []string 812 QueryOptions 813 } 814 815 // VaultAccessorsRequest is used to operate on a set of Vault accessors 816 type VaultAccessorsRequest struct { 817 Accessors []*VaultAccessor 818 } 819 820 // VaultAccessor is a reference to a created Vault token on behalf of 821 // an allocation's task. 822 type VaultAccessor struct { 823 AllocID string 824 Task string 825 NodeID string 826 Accessor string 827 CreationTTL int 828 829 // Raft Indexes 830 CreateIndex uint64 831 } 832 833 // DeriveVaultTokenResponse returns the wrapped tokens for each requested task 834 type DeriveVaultTokenResponse struct { 835 // Tasks is a mapping between the task name and the wrapped token 836 Tasks map[string]string 837 838 // Error stores any error that occurred. Errors are stored here so we can 839 // communicate whether it is retriable 840 Error *RecoverableError 841 842 QueryMeta 843 } 844 845 // GenericRequest is used to request where no 846 // specific information is needed. 847 type GenericRequest struct { 848 QueryOptions 849 } 850 851 // DeploymentListRequest is used to list the deployments 852 type DeploymentListRequest struct { 853 QueryOptions 854 } 855 856 // DeploymentDeleteRequest is used for deleting deployments. 857 type DeploymentDeleteRequest struct { 858 Deployments []string 859 WriteRequest 860 } 861 862 // DeploymentStatusUpdateRequest is used to update the status of a deployment as 863 // well as optionally creating an evaluation atomically. 864 type DeploymentStatusUpdateRequest struct { 865 // Eval, if set, is used to create an evaluation at the same time as 866 // updating the status of a deployment. 867 Eval *Evaluation 868 869 // DeploymentUpdate is a status update to apply to the given 870 // deployment. 871 DeploymentUpdate *DeploymentStatusUpdate 872 873 // Job is used to optionally upsert a job. This is used when setting the 874 // allocation health results in a deployment failure and the deployment 875 // auto-reverts to the latest stable job. 876 Job *Job 877 } 878 879 // DeploymentAllocHealthRequest is used to set the health of a set of 880 // allocations as part of a deployment. 881 type DeploymentAllocHealthRequest struct { 882 DeploymentID string 883 884 // Marks these allocations as healthy, allow further allocations 885 // to be rolled. 886 HealthyAllocationIDs []string 887 888 // Any unhealthy allocations fail the deployment 889 UnhealthyAllocationIDs []string 890 891 WriteRequest 892 } 893 894 // ApplyDeploymentAllocHealthRequest is used to apply an alloc health request via Raft 895 type ApplyDeploymentAllocHealthRequest struct { 896 DeploymentAllocHealthRequest 897 898 // Timestamp is the timestamp to use when setting the allocations health. 899 Timestamp time.Time 900 901 // An optional field to update the status of a deployment 902 DeploymentUpdate *DeploymentStatusUpdate 903 904 // Job is used to optionally upsert a job. This is used when setting the 905 // allocation health results in a deployment failure and the deployment 906 // auto-reverts to the latest stable job. 907 Job *Job 908 909 // An optional evaluation to create after promoting the canaries 910 Eval *Evaluation 911 } 912 913 // DeploymentPromoteRequest is used to promote task groups in a deployment 914 type DeploymentPromoteRequest struct { 915 DeploymentID string 916 917 // All is to promote all task groups 918 All bool 919 920 // Groups is used to set the promotion status per task group 921 Groups []string 922 923 WriteRequest 924 } 925 926 // ApplyDeploymentPromoteRequest is used to apply a promotion request via Raft 927 type ApplyDeploymentPromoteRequest struct { 928 DeploymentPromoteRequest 929 930 // An optional evaluation to create after promoting the canaries 931 Eval *Evaluation 932 } 933 934 // DeploymentPauseRequest is used to pause a deployment 935 type DeploymentPauseRequest struct { 936 DeploymentID string 937 938 // Pause sets the pause status 939 Pause bool 940 941 WriteRequest 942 } 943 944 // DeploymentSpecificRequest is used to make a request specific to a particular 945 // deployment 946 type DeploymentSpecificRequest struct { 947 DeploymentID string 948 QueryOptions 949 } 950 951 // DeploymentFailRequest is used to fail a particular deployment 952 type DeploymentFailRequest struct { 953 DeploymentID string 954 WriteRequest 955 } 956 957 // SingleDeploymentResponse is used to respond with a single deployment 958 type SingleDeploymentResponse struct { 959 Deployment *Deployment 960 QueryMeta 961 } 962 963 // GenericResponse is used to respond to a request where no 964 // specific response information is needed. 965 type GenericResponse struct { 966 WriteMeta 967 } 968 969 // VersionResponse is used for the Status.Version response 970 type VersionResponse struct { 971 Build string 972 Versions map[string]int 973 QueryMeta 974 } 975 976 // JobRegisterResponse is used to respond to a job registration 977 type JobRegisterResponse struct { 978 EvalID string 979 EvalCreateIndex uint64 980 JobModifyIndex uint64 981 982 // Warnings contains any warnings about the given job. These may include 983 // deprecation warnings. 984 Warnings string 985 986 QueryMeta 987 } 988 989 // JobDeregisterResponse is used to respond to a job deregistration 990 type JobDeregisterResponse struct { 991 EvalID string 992 EvalCreateIndex uint64 993 JobModifyIndex uint64 994 QueryMeta 995 } 996 997 // JobBatchDeregisterResponse is used to respond to a batch job deregistration 998 type JobBatchDeregisterResponse struct { 999 // JobEvals maps the job to its created evaluation 1000 JobEvals map[NamespacedID]string 1001 QueryMeta 1002 } 1003 1004 // JobValidateResponse is the response from validate request 1005 type JobValidateResponse struct { 1006 // DriverConfigValidated indicates whether the agent validated the driver 1007 // config 1008 DriverConfigValidated bool 1009 1010 // ValidationErrors is a list of validation errors 1011 ValidationErrors []string 1012 1013 // Error is a string version of any error that may have occurred 1014 Error string 1015 1016 // Warnings contains any warnings about the given job. These may include 1017 // deprecation warnings. 1018 Warnings string 1019 } 1020 1021 // NodeUpdateResponse is used to respond to a node update 1022 type NodeUpdateResponse struct { 1023 HeartbeatTTL time.Duration 1024 EvalIDs []string 1025 EvalCreateIndex uint64 1026 NodeModifyIndex uint64 1027 1028 // LeaderRPCAddr is the RPC address of the current Raft Leader. If 1029 // empty, the current Nomad Server is in the minority of a partition. 1030 LeaderRPCAddr string 1031 1032 // NumNodes is the number of Nomad nodes attached to this quorum of 1033 // Nomad Servers at the time of the response. This value can 1034 // fluctuate based on the health of the cluster between heartbeats. 1035 NumNodes int32 1036 1037 // Servers is the full list of known Nomad servers in the local 1038 // region. 1039 Servers []*NodeServerInfo 1040 1041 QueryMeta 1042 } 1043 1044 // NodeDrainUpdateResponse is used to respond to a node drain update 1045 type NodeDrainUpdateResponse struct { 1046 NodeModifyIndex uint64 1047 EvalIDs []string 1048 EvalCreateIndex uint64 1049 WriteMeta 1050 } 1051 1052 // NodeEligibilityUpdateResponse is used to respond to a node eligibility update 1053 type NodeEligibilityUpdateResponse struct { 1054 NodeModifyIndex uint64 1055 EvalIDs []string 1056 EvalCreateIndex uint64 1057 WriteMeta 1058 } 1059 1060 // NodeAllocsResponse is used to return allocs for a single node 1061 type NodeAllocsResponse struct { 1062 Allocs []*Allocation 1063 QueryMeta 1064 } 1065 1066 // NodeClientAllocsResponse is used to return allocs meta data for a single node 1067 type NodeClientAllocsResponse struct { 1068 Allocs map[string]uint64 1069 1070 // MigrateTokens are used when ACLs are enabled to allow cross node, 1071 // authenticated access to sticky volumes 1072 MigrateTokens map[string]string 1073 1074 QueryMeta 1075 } 1076 1077 // SingleNodeResponse is used to return a single node 1078 type SingleNodeResponse struct { 1079 Node *Node 1080 QueryMeta 1081 } 1082 1083 // NodeListResponse is used for a list request 1084 type NodeListResponse struct { 1085 Nodes []*NodeListStub 1086 QueryMeta 1087 } 1088 1089 // SingleJobResponse is used to return a single job 1090 type SingleJobResponse struct { 1091 Job *Job 1092 QueryMeta 1093 } 1094 1095 // JobSummaryResponse is used to return a single job summary 1096 type JobSummaryResponse struct { 1097 JobSummary *JobSummary 1098 QueryMeta 1099 } 1100 1101 type JobDispatchResponse struct { 1102 DispatchedJobID string 1103 EvalID string 1104 EvalCreateIndex uint64 1105 JobCreateIndex uint64 1106 WriteMeta 1107 } 1108 1109 // JobListResponse is used for a list request 1110 type JobListResponse struct { 1111 Jobs []*JobListStub 1112 QueryMeta 1113 } 1114 1115 // JobVersionsRequest is used to get a jobs versions 1116 type JobVersionsRequest struct { 1117 JobID string 1118 Diffs bool 1119 QueryOptions 1120 } 1121 1122 // JobVersionsResponse is used for a job get versions request 1123 type JobVersionsResponse struct { 1124 Versions []*Job 1125 Diffs []*JobDiff 1126 QueryMeta 1127 } 1128 1129 // JobPlanResponse is used to respond to a job plan request 1130 type JobPlanResponse struct { 1131 // Annotations stores annotations explaining decisions the scheduler made. 1132 Annotations *PlanAnnotations 1133 1134 // FailedTGAllocs is the placement failures per task group. 1135 FailedTGAllocs map[string]*AllocMetric 1136 1137 // JobModifyIndex is the modification index of the job. The value can be 1138 // used when running `nomad run` to ensure that the Job wasn’t modified 1139 // since the last plan. If the job is being created, the value is zero. 1140 JobModifyIndex uint64 1141 1142 // CreatedEvals is the set of evaluations created by the scheduler. The 1143 // reasons for this can be rolling-updates or blocked evals. 1144 CreatedEvals []*Evaluation 1145 1146 // Diff contains the diff of the job and annotations on whether the change 1147 // causes an in-place update or create/destroy 1148 Diff *JobDiff 1149 1150 // NextPeriodicLaunch is the time duration till the job would be launched if 1151 // submitted. 1152 NextPeriodicLaunch time.Time 1153 1154 // Warnings contains any warnings about the given job. These may include 1155 // deprecation warnings. 1156 Warnings string 1157 1158 WriteMeta 1159 } 1160 1161 // SingleAllocResponse is used to return a single allocation 1162 type SingleAllocResponse struct { 1163 Alloc *Allocation 1164 QueryMeta 1165 } 1166 1167 // AllocsGetResponse is used to return a set of allocations 1168 type AllocsGetResponse struct { 1169 Allocs []*Allocation 1170 QueryMeta 1171 } 1172 1173 // JobAllocationsResponse is used to return the allocations for a job 1174 type JobAllocationsResponse struct { 1175 Allocations []*AllocListStub 1176 QueryMeta 1177 } 1178 1179 // JobEvaluationsResponse is used to return the evaluations for a job 1180 type JobEvaluationsResponse struct { 1181 Evaluations []*Evaluation 1182 QueryMeta 1183 } 1184 1185 // SingleEvalResponse is used to return a single evaluation 1186 type SingleEvalResponse struct { 1187 Eval *Evaluation 1188 QueryMeta 1189 } 1190 1191 // EvalDequeueResponse is used to return from a dequeue 1192 type EvalDequeueResponse struct { 1193 Eval *Evaluation 1194 Token string 1195 1196 // WaitIndex is the Raft index the worker should wait until invoking the 1197 // scheduler. 1198 WaitIndex uint64 1199 1200 QueryMeta 1201 } 1202 1203 // GetWaitIndex is used to retrieve the Raft index in which state should be at 1204 // or beyond before invoking the scheduler. 1205 func (e *EvalDequeueResponse) GetWaitIndex() uint64 { 1206 // Prefer the wait index sent. This will be populated on all responses from 1207 // 0.7.0 and above 1208 if e.WaitIndex != 0 { 1209 return e.WaitIndex 1210 } else if e.Eval != nil { 1211 return e.Eval.ModifyIndex 1212 } 1213 1214 // This should never happen 1215 return 1 1216 } 1217 1218 // PlanResponse is used to return from a PlanRequest 1219 type PlanResponse struct { 1220 Result *PlanResult 1221 WriteMeta 1222 } 1223 1224 // AllocListResponse is used for a list request 1225 type AllocListResponse struct { 1226 Allocations []*AllocListStub 1227 QueryMeta 1228 } 1229 1230 // DeploymentListResponse is used for a list request 1231 type DeploymentListResponse struct { 1232 Deployments []*Deployment 1233 QueryMeta 1234 } 1235 1236 // EvalListResponse is used for a list request 1237 type EvalListResponse struct { 1238 Evaluations []*Evaluation 1239 QueryMeta 1240 } 1241 1242 // EvalAllocationsResponse is used to return the allocations for an evaluation 1243 type EvalAllocationsResponse struct { 1244 Allocations []*AllocListStub 1245 QueryMeta 1246 } 1247 1248 // PeriodicForceResponse is used to respond to a periodic job force launch 1249 type PeriodicForceResponse struct { 1250 EvalID string 1251 EvalCreateIndex uint64 1252 WriteMeta 1253 } 1254 1255 // DeploymentUpdateResponse is used to respond to a deployment change. The 1256 // response will include the modify index of the deployment as well as details 1257 // of any triggered evaluation. 1258 type DeploymentUpdateResponse struct { 1259 EvalID string 1260 EvalCreateIndex uint64 1261 DeploymentModifyIndex uint64 1262 1263 // RevertedJobVersion is the version the job was reverted to. If unset, the 1264 // job wasn't reverted 1265 RevertedJobVersion *uint64 1266 1267 WriteMeta 1268 } 1269 1270 // NodeConnQueryResponse is used to respond to a query of whether a server has 1271 // a connection to a specific Node 1272 type NodeConnQueryResponse struct { 1273 // Connected indicates whether a connection to the Client exists 1274 Connected bool 1275 1276 // Established marks the time at which the connection was established 1277 Established time.Time 1278 1279 QueryMeta 1280 } 1281 1282 // EmitNodeEventsRequest is a request to update the node events source 1283 // with a new client-side event 1284 type EmitNodeEventsRequest struct { 1285 // NodeEvents are a map where the key is a node id, and value is a list of 1286 // events for that node 1287 NodeEvents map[string][]*NodeEvent 1288 1289 WriteRequest 1290 } 1291 1292 // EmitNodeEventsResponse is a response to the client about the status of 1293 // the node event source update. 1294 type EmitNodeEventsResponse struct { 1295 WriteMeta 1296 } 1297 1298 const ( 1299 NodeEventSubsystemDrain = "Drain" 1300 NodeEventSubsystemDriver = "Driver" 1301 NodeEventSubsystemHeartbeat = "Heartbeat" 1302 NodeEventSubsystemCluster = "Cluster" 1303 ) 1304 1305 // NodeEvent is a single unit representing a node’s state change 1306 type NodeEvent struct { 1307 Message string 1308 Subsystem string 1309 Details map[string]string 1310 Timestamp time.Time 1311 CreateIndex uint64 1312 } 1313 1314 func (ne *NodeEvent) String() string { 1315 var details []string 1316 for k, v := range ne.Details { 1317 details = append(details, fmt.Sprintf("%s: %s", k, v)) 1318 } 1319 1320 return fmt.Sprintf("Message: %s, Subsystem: %s, Details: %s, Timestamp: %s", ne.Message, ne.Subsystem, strings.Join(details, ","), ne.Timestamp.String()) 1321 } 1322 1323 func (ne *NodeEvent) Copy() *NodeEvent { 1324 c := new(NodeEvent) 1325 *c = *ne 1326 c.Details = helper.CopyMapStringString(ne.Details) 1327 return c 1328 } 1329 1330 // NewNodeEvent generates a new node event storing the current time as the 1331 // timestamp 1332 func NewNodeEvent() *NodeEvent { 1333 return &NodeEvent{Timestamp: time.Now()} 1334 } 1335 1336 // SetMessage is used to set the message on the node event 1337 func (ne *NodeEvent) SetMessage(msg string) *NodeEvent { 1338 ne.Message = msg 1339 return ne 1340 } 1341 1342 // SetSubsystem is used to set the subsystem on the node event 1343 func (ne *NodeEvent) SetSubsystem(sys string) *NodeEvent { 1344 ne.Subsystem = sys 1345 return ne 1346 } 1347 1348 // SetTimestamp is used to set the timestamp on the node event 1349 func (ne *NodeEvent) SetTimestamp(ts time.Time) *NodeEvent { 1350 ne.Timestamp = ts 1351 return ne 1352 } 1353 1354 // AddDetail is used to add a detail to the node event 1355 func (ne *NodeEvent) AddDetail(k, v string) *NodeEvent { 1356 if ne.Details == nil { 1357 ne.Details = make(map[string]string, 1) 1358 } 1359 ne.Details[k] = v 1360 return ne 1361 } 1362 1363 const ( 1364 NodeStatusInit = "initializing" 1365 NodeStatusReady = "ready" 1366 NodeStatusDown = "down" 1367 ) 1368 1369 // ShouldDrainNode checks if a given node status should trigger an 1370 // evaluation. Some states don't require any further action. 1371 func ShouldDrainNode(status string) bool { 1372 switch status { 1373 case NodeStatusInit, NodeStatusReady: 1374 return false 1375 case NodeStatusDown: 1376 return true 1377 default: 1378 panic(fmt.Sprintf("unhandled node status %s", status)) 1379 } 1380 } 1381 1382 // ValidNodeStatus is used to check if a node status is valid 1383 func ValidNodeStatus(status string) bool { 1384 switch status { 1385 case NodeStatusInit, NodeStatusReady, NodeStatusDown: 1386 return true 1387 default: 1388 return false 1389 } 1390 } 1391 1392 const ( 1393 // NodeSchedulingEligible and Ineligible marks the node as eligible or not, 1394 // respectively, for receiving allocations. This is orthoginal to the node 1395 // status being ready. 1396 NodeSchedulingEligible = "eligible" 1397 NodeSchedulingIneligible = "ineligible" 1398 ) 1399 1400 // DrainSpec describes a Node's desired drain behavior. 1401 type DrainSpec struct { 1402 // Deadline is the duration after StartTime when the remaining 1403 // allocations on a draining Node should be told to stop. 1404 Deadline time.Duration 1405 1406 // IgnoreSystemJobs allows systems jobs to remain on the node even though it 1407 // has been marked for draining. 1408 IgnoreSystemJobs bool 1409 } 1410 1411 // DrainStrategy describes a Node's drain behavior. 1412 type DrainStrategy struct { 1413 // DrainSpec is the user declared drain specification 1414 DrainSpec 1415 1416 // ForceDeadline is the deadline time for the drain after which drains will 1417 // be forced 1418 ForceDeadline time.Time 1419 } 1420 1421 func (d *DrainStrategy) Copy() *DrainStrategy { 1422 if d == nil { 1423 return nil 1424 } 1425 1426 nd := new(DrainStrategy) 1427 *nd = *d 1428 return nd 1429 } 1430 1431 // DeadlineTime returns a boolean whether the drain strategy allows an infinite 1432 // duration or otherwise the deadline time. The force drain is captured by the 1433 // deadline time being in the past. 1434 func (d *DrainStrategy) DeadlineTime() (infinite bool, deadline time.Time) { 1435 // Treat the nil case as a force drain so during an upgrade where a node may 1436 // not have a drain strategy but has Drain set to true, it is treated as a 1437 // force to mimick old behavior. 1438 if d == nil { 1439 return false, time.Time{} 1440 } 1441 1442 ns := d.Deadline.Nanoseconds() 1443 switch { 1444 case ns < 0: // Force 1445 return false, time.Time{} 1446 case ns == 0: // Infinite 1447 return true, time.Time{} 1448 default: 1449 return false, d.ForceDeadline 1450 } 1451 } 1452 1453 func (d *DrainStrategy) Equal(o *DrainStrategy) bool { 1454 if d == nil && o == nil { 1455 return true 1456 } else if o != nil && d == nil { 1457 return false 1458 } else if d != nil && o == nil { 1459 return false 1460 } 1461 1462 // Compare values 1463 if d.ForceDeadline != o.ForceDeadline { 1464 return false 1465 } else if d.Deadline != o.Deadline { 1466 return false 1467 } else if d.IgnoreSystemJobs != o.IgnoreSystemJobs { 1468 return false 1469 } 1470 1471 return true 1472 } 1473 1474 // Node is a representation of a schedulable client node 1475 type Node struct { 1476 // ID is a unique identifier for the node. It can be constructed 1477 // by doing a concatenation of the Name and Datacenter as a simple 1478 // approach. Alternatively a UUID may be used. 1479 ID string 1480 1481 // SecretID is an ID that is only known by the Node and the set of Servers. 1482 // It is not accessible via the API and is used to authenticate nodes 1483 // conducting privileged activities. 1484 SecretID string 1485 1486 // Datacenter for this node 1487 Datacenter string 1488 1489 // Node name 1490 Name string 1491 1492 // HTTPAddr is the address on which the Nomad client is listening for http 1493 // requests 1494 HTTPAddr string 1495 1496 // TLSEnabled indicates if the Agent has TLS enabled for the HTTP API 1497 TLSEnabled bool 1498 1499 // Attributes is an arbitrary set of key/value 1500 // data that can be used for constraints. Examples 1501 // include "kernel.name=linux", "arch=386", "driver.docker=1", 1502 // "docker.runtime=1.8.3" 1503 Attributes map[string]string 1504 1505 // NodeResources captures the available resources on the client. 1506 NodeResources *NodeResources 1507 1508 // ReservedResources captures the set resources on the client that are 1509 // reserved from scheduling. 1510 ReservedResources *NodeReservedResources 1511 1512 // Resources is the available resources on the client. 1513 // For example 'cpu=2' 'memory=2048' 1514 // COMPAT(0.10): Remove in 0.10 1515 Resources *Resources 1516 1517 // Reserved is the set of resources that are reserved, 1518 // and should be subtracted from the total resources for 1519 // the purposes of scheduling. This may be provide certain 1520 // high-watermark tolerances or because of external schedulers 1521 // consuming resources. 1522 Reserved *Resources 1523 1524 // Links are used to 'link' this client to external 1525 // systems. For example 'consul=foo.dc1' 'aws=i-83212' 1526 // 'ami=ami-123' 1527 Links map[string]string 1528 1529 // Meta is used to associate arbitrary metadata with this 1530 // client. This is opaque to Nomad. 1531 Meta map[string]string 1532 1533 // NodeClass is an opaque identifier used to group nodes 1534 // together for the purpose of determining scheduling pressure. 1535 NodeClass string 1536 1537 // ComputedClass is a unique id that identifies nodes with a common set of 1538 // attributes and capabilities. 1539 ComputedClass string 1540 1541 // COMPAT: Remove in Nomad 0.9 1542 // Drain is controlled by the servers, and not the client. 1543 // If true, no jobs will be scheduled to this node, and existing 1544 // allocations will be drained. Superceded by DrainStrategy in Nomad 1545 // 0.8 but kept for backward compat. 1546 Drain bool 1547 1548 // DrainStrategy determines the node's draining behavior. Will be nil 1549 // when Drain=false. 1550 DrainStrategy *DrainStrategy 1551 1552 // SchedulingEligibility determines whether this node will receive new 1553 // placements. 1554 SchedulingEligibility string 1555 1556 // Status of this node 1557 Status string 1558 1559 // StatusDescription is meant to provide more human useful information 1560 StatusDescription string 1561 1562 // StatusUpdatedAt is the time stamp at which the state of the node was 1563 // updated 1564 StatusUpdatedAt int64 1565 1566 // Events is the most recent set of events generated for the node, 1567 // retaining only MaxRetainedNodeEvents number at a time 1568 Events []*NodeEvent 1569 1570 // Drivers is a map of driver names to current driver information 1571 Drivers map[string]*DriverInfo 1572 1573 // Raft Indexes 1574 CreateIndex uint64 1575 ModifyIndex uint64 1576 } 1577 1578 // Ready returns true if the node is ready for running allocations 1579 func (n *Node) Ready() bool { 1580 // Drain is checked directly to support pre-0.8 Node data 1581 return n.Status == NodeStatusReady && !n.Drain && n.SchedulingEligibility == NodeSchedulingEligible 1582 } 1583 1584 func (n *Node) Canonicalize() { 1585 if n == nil { 1586 return 1587 } 1588 1589 // COMPAT Remove in 0.10 1590 // In v0.8.0 we introduced scheduling eligibility, so we need to set it for 1591 // upgrading nodes 1592 if n.SchedulingEligibility == "" { 1593 if n.Drain { 1594 n.SchedulingEligibility = NodeSchedulingIneligible 1595 } else { 1596 n.SchedulingEligibility = NodeSchedulingEligible 1597 } 1598 } 1599 } 1600 1601 func (n *Node) Copy() *Node { 1602 if n == nil { 1603 return nil 1604 } 1605 nn := new(Node) 1606 *nn = *n 1607 nn.Attributes = helper.CopyMapStringString(nn.Attributes) 1608 nn.Resources = nn.Resources.Copy() 1609 nn.Reserved = nn.Reserved.Copy() 1610 nn.NodeResources = nn.NodeResources.Copy() 1611 nn.ReservedResources = nn.ReservedResources.Copy() 1612 nn.Links = helper.CopyMapStringString(nn.Links) 1613 nn.Meta = helper.CopyMapStringString(nn.Meta) 1614 nn.Events = copyNodeEvents(n.Events) 1615 nn.DrainStrategy = nn.DrainStrategy.Copy() 1616 nn.Drivers = copyNodeDrivers(n.Drivers) 1617 return nn 1618 } 1619 1620 // copyNodeEvents is a helper to copy a list of NodeEvent's 1621 func copyNodeEvents(events []*NodeEvent) []*NodeEvent { 1622 l := len(events) 1623 if l == 0 { 1624 return nil 1625 } 1626 1627 c := make([]*NodeEvent, l) 1628 for i, event := range events { 1629 c[i] = event.Copy() 1630 } 1631 return c 1632 } 1633 1634 // copyNodeDrivers is a helper to copy a map of DriverInfo 1635 func copyNodeDrivers(drivers map[string]*DriverInfo) map[string]*DriverInfo { 1636 l := len(drivers) 1637 if l == 0 { 1638 return nil 1639 } 1640 1641 c := make(map[string]*DriverInfo, l) 1642 for driver, info := range drivers { 1643 c[driver] = info.Copy() 1644 } 1645 return c 1646 } 1647 1648 // TerminalStatus returns if the current status is terminal and 1649 // will no longer transition. 1650 func (n *Node) TerminalStatus() bool { 1651 switch n.Status { 1652 case NodeStatusDown: 1653 return true 1654 default: 1655 return false 1656 } 1657 } 1658 1659 // COMPAT(0.11): Remove in 0.11 1660 // ComparableReservedResources returns the reserved resouces on the node 1661 // handling upgrade paths. Reserved networks must be handled separately. After 1662 // 0.11 calls to this should be replaced with: 1663 // node.ReservedResources.Comparable() 1664 func (n *Node) ComparableReservedResources() *ComparableResources { 1665 // See if we can no-op 1666 if n.Reserved == nil && n.ReservedResources == nil { 1667 return nil 1668 } 1669 1670 // Node already has 0.9+ behavior 1671 if n.ReservedResources != nil { 1672 return n.ReservedResources.Comparable() 1673 } 1674 1675 // Upgrade path 1676 return &ComparableResources{ 1677 Flattened: AllocatedTaskResources{ 1678 Cpu: AllocatedCpuResources{ 1679 CpuShares: int64(n.Reserved.CPU), 1680 }, 1681 Memory: AllocatedMemoryResources{ 1682 MemoryMB: int64(n.Reserved.MemoryMB), 1683 }, 1684 }, 1685 Shared: AllocatedSharedResources{ 1686 DiskMB: int64(n.Reserved.DiskMB), 1687 }, 1688 } 1689 } 1690 1691 // COMPAT(0.11): Remove in 0.11 1692 // ComparableResources returns the resouces on the node 1693 // handling upgrade paths. Networking must be handled separately. After 0.11 1694 // calls to this should be replaced with: node.NodeResources.Comparable() 1695 func (n *Node) ComparableResources() *ComparableResources { 1696 // Node already has 0.9+ behavior 1697 if n.NodeResources != nil { 1698 return n.NodeResources.Comparable() 1699 } 1700 1701 // Upgrade path 1702 return &ComparableResources{ 1703 Flattened: AllocatedTaskResources{ 1704 Cpu: AllocatedCpuResources{ 1705 CpuShares: int64(n.Resources.CPU), 1706 }, 1707 Memory: AllocatedMemoryResources{ 1708 MemoryMB: int64(n.Resources.MemoryMB), 1709 }, 1710 }, 1711 Shared: AllocatedSharedResources{ 1712 DiskMB: int64(n.Resources.DiskMB), 1713 }, 1714 } 1715 } 1716 1717 // Stub returns a summarized version of the node 1718 func (n *Node) Stub() *NodeListStub { 1719 1720 addr, _, _ := net.SplitHostPort(n.HTTPAddr) 1721 1722 return &NodeListStub{ 1723 Address: addr, 1724 ID: n.ID, 1725 Datacenter: n.Datacenter, 1726 Name: n.Name, 1727 NodeClass: n.NodeClass, 1728 Version: n.Attributes["nomad.version"], 1729 Drain: n.Drain, 1730 SchedulingEligibility: n.SchedulingEligibility, 1731 Status: n.Status, 1732 StatusDescription: n.StatusDescription, 1733 Drivers: n.Drivers, 1734 CreateIndex: n.CreateIndex, 1735 ModifyIndex: n.ModifyIndex, 1736 } 1737 } 1738 1739 // NodeListStub is used to return a subset of job information 1740 // for the job list 1741 type NodeListStub struct { 1742 Address string 1743 ID string 1744 Datacenter string 1745 Name string 1746 NodeClass string 1747 Version string 1748 Drain bool 1749 SchedulingEligibility string 1750 Status string 1751 StatusDescription string 1752 Drivers map[string]*DriverInfo 1753 CreateIndex uint64 1754 ModifyIndex uint64 1755 } 1756 1757 // Resources is used to define the resources available 1758 // on a client 1759 type Resources struct { 1760 CPU int 1761 MemoryMB int 1762 DiskMB int 1763 IOPS int // COMPAT(0.10): Only being used to issue warnings 1764 Networks Networks 1765 Devices ResourceDevices 1766 } 1767 1768 const ( 1769 BytesInMegabyte = 1024 * 1024 1770 ) 1771 1772 // DefaultResources is a small resources object that contains the 1773 // default resources requests that we will provide to an object. 1774 // --- THIS FUNCTION IS REPLICATED IN api/resources.go and should 1775 // be kept in sync. 1776 func DefaultResources() *Resources { 1777 return &Resources{ 1778 CPU: 100, 1779 MemoryMB: 300, 1780 } 1781 } 1782 1783 // MinResources is a small resources object that contains the 1784 // absolute minimum resources that we will provide to an object. 1785 // This should not be confused with the defaults which are 1786 // provided in Canonicalize() --- THIS FUNCTION IS REPLICATED IN 1787 // api/resources.go and should be kept in sync. 1788 func MinResources() *Resources { 1789 return &Resources{ 1790 CPU: 20, 1791 MemoryMB: 10, 1792 } 1793 } 1794 1795 // DiskInBytes returns the amount of disk resources in bytes. 1796 func (r *Resources) DiskInBytes() int64 { 1797 return int64(r.DiskMB * BytesInMegabyte) 1798 } 1799 1800 func (r *Resources) Validate() error { 1801 var mErr multierror.Error 1802 if err := r.MeetsMinResources(); err != nil { 1803 mErr.Errors = append(mErr.Errors, err) 1804 } 1805 1806 // Ensure the task isn't asking for disk resources 1807 if r.DiskMB > 0 { 1808 mErr.Errors = append(mErr.Errors, errors.New("Task can't ask for disk resources, they have to be specified at the task group level.")) 1809 } 1810 1811 for i, d := range r.Devices { 1812 if err := d.Validate(); err != nil { 1813 mErr.Errors = append(mErr.Errors, fmt.Errorf("device %d failed validation: %v", i+1, err)) 1814 } 1815 } 1816 1817 return mErr.ErrorOrNil() 1818 } 1819 1820 // Merge merges this resource with another resource. 1821 // COMPAT(0.10): Remove in 0.10 1822 func (r *Resources) Merge(other *Resources) { 1823 if other.CPU != 0 { 1824 r.CPU = other.CPU 1825 } 1826 if other.MemoryMB != 0 { 1827 r.MemoryMB = other.MemoryMB 1828 } 1829 if other.DiskMB != 0 { 1830 r.DiskMB = other.DiskMB 1831 } 1832 if len(other.Networks) != 0 { 1833 r.Networks = other.Networks 1834 } 1835 if len(other.Devices) != 0 { 1836 r.Devices = other.Devices 1837 } 1838 } 1839 1840 // COMPAT(0.10): Remove in 0.10 1841 func (r *Resources) Equals(o *Resources) bool { 1842 if r == o { 1843 return true 1844 } 1845 if r == nil || o == nil { 1846 return false 1847 } 1848 return r.CPU == o.CPU && 1849 r.MemoryMB == o.MemoryMB && 1850 r.DiskMB == o.DiskMB && 1851 r.IOPS == o.IOPS && 1852 r.Networks.Equals(&o.Networks) && 1853 r.Devices.Equals(&o.Devices) 1854 } 1855 1856 // COMPAT(0.10): Remove in 0.10 1857 // ResourceDevices are part of Resources 1858 type ResourceDevices []*RequestedDevice 1859 1860 // COMPAT(0.10): Remove in 0.10 1861 // Equals ResourceDevices as set keyed by Name 1862 func (d *ResourceDevices) Equals(o *ResourceDevices) bool { 1863 if d == o { 1864 return true 1865 } 1866 if d == nil || o == nil { 1867 return false 1868 } 1869 if len(*d) != len(*o) { 1870 return false 1871 } 1872 m := make(map[string]*RequestedDevice, len(*d)) 1873 for _, e := range *d { 1874 m[e.Name] = e 1875 } 1876 for _, oe := range *o { 1877 de, ok := m[oe.Name] 1878 if !ok || !de.Equals(oe) { 1879 return false 1880 } 1881 } 1882 return true 1883 } 1884 1885 // COMPAT(0.10): Remove in 0.10 1886 func (r *Resources) Canonicalize() { 1887 // Ensure that an empty and nil slices are treated the same to avoid scheduling 1888 // problems since we use reflect DeepEquals. 1889 if len(r.Networks) == 0 { 1890 r.Networks = nil 1891 } 1892 if len(r.Devices) == 0 { 1893 r.Devices = nil 1894 } 1895 1896 for _, n := range r.Networks { 1897 n.Canonicalize() 1898 } 1899 } 1900 1901 // MeetsMinResources returns an error if the resources specified are less than 1902 // the minimum allowed. 1903 // This is based on the minimums defined in the Resources type 1904 // COMPAT(0.10): Remove in 0.10 1905 func (r *Resources) MeetsMinResources() error { 1906 var mErr multierror.Error 1907 minResources := MinResources() 1908 if r.CPU < minResources.CPU { 1909 mErr.Errors = append(mErr.Errors, fmt.Errorf("minimum CPU value is %d; got %d", minResources.CPU, r.CPU)) 1910 } 1911 if r.MemoryMB < minResources.MemoryMB { 1912 mErr.Errors = append(mErr.Errors, fmt.Errorf("minimum MemoryMB value is %d; got %d", minResources.MemoryMB, r.MemoryMB)) 1913 } 1914 for i, n := range r.Networks { 1915 if err := n.MeetsMinResources(); err != nil { 1916 mErr.Errors = append(mErr.Errors, fmt.Errorf("network resource at index %d failed: %v", i, err)) 1917 } 1918 } 1919 1920 return mErr.ErrorOrNil() 1921 } 1922 1923 // Copy returns a deep copy of the resources 1924 func (r *Resources) Copy() *Resources { 1925 if r == nil { 1926 return nil 1927 } 1928 newR := new(Resources) 1929 *newR = *r 1930 1931 // Copy the network objects 1932 if r.Networks != nil { 1933 n := len(r.Networks) 1934 newR.Networks = make([]*NetworkResource, n) 1935 for i := 0; i < n; i++ { 1936 newR.Networks[i] = r.Networks[i].Copy() 1937 } 1938 } 1939 1940 // Copy the devices 1941 if r.Devices != nil { 1942 n := len(r.Devices) 1943 newR.Devices = make([]*RequestedDevice, n) 1944 for i := 0; i < n; i++ { 1945 newR.Devices[i] = r.Devices[i].Copy() 1946 } 1947 } 1948 1949 return newR 1950 } 1951 1952 // NetIndex finds the matching net index using device name 1953 // COMPAT(0.10): Remove in 0.10 1954 func (r *Resources) NetIndex(n *NetworkResource) int { 1955 return r.Networks.NetIndex(n) 1956 } 1957 1958 // Superset checks if one set of resources is a superset 1959 // of another. This ignores network resources, and the NetworkIndex 1960 // should be used for that. 1961 // COMPAT(0.10): Remove in 0.10 1962 func (r *Resources) Superset(other *Resources) (bool, string) { 1963 if r.CPU < other.CPU { 1964 return false, "cpu" 1965 } 1966 if r.MemoryMB < other.MemoryMB { 1967 return false, "memory" 1968 } 1969 if r.DiskMB < other.DiskMB { 1970 return false, "disk" 1971 } 1972 return true, "" 1973 } 1974 1975 // Add adds the resources of the delta to this, potentially 1976 // returning an error if not possible. 1977 // COMPAT(0.10): Remove in 0.10 1978 func (r *Resources) Add(delta *Resources) error { 1979 if delta == nil { 1980 return nil 1981 } 1982 r.CPU += delta.CPU 1983 r.MemoryMB += delta.MemoryMB 1984 r.DiskMB += delta.DiskMB 1985 1986 for _, n := range delta.Networks { 1987 // Find the matching interface by IP or CIDR 1988 idx := r.NetIndex(n) 1989 if idx == -1 { 1990 r.Networks = append(r.Networks, n.Copy()) 1991 } else { 1992 r.Networks[idx].Add(n) 1993 } 1994 } 1995 return nil 1996 } 1997 1998 // COMPAT(0.10): Remove in 0.10 1999 func (r *Resources) GoString() string { 2000 return fmt.Sprintf("*%#v", *r) 2001 } 2002 2003 type Port struct { 2004 Label string 2005 Value int 2006 } 2007 2008 // NetworkResource is used to represent available network 2009 // resources 2010 type NetworkResource struct { 2011 Device string // Name of the device 2012 CIDR string // CIDR block of addresses 2013 IP string // Host IP address 2014 MBits int // Throughput 2015 ReservedPorts []Port // Host Reserved ports 2016 DynamicPorts []Port // Host Dynamically assigned ports 2017 } 2018 2019 func (nr *NetworkResource) Equals(other *NetworkResource) bool { 2020 if nr.Device != other.Device { 2021 return false 2022 } 2023 2024 if nr.CIDR != other.CIDR { 2025 return false 2026 } 2027 2028 if nr.IP != other.IP { 2029 return false 2030 } 2031 2032 if nr.MBits != other.MBits { 2033 return false 2034 } 2035 2036 if len(nr.ReservedPorts) != len(other.ReservedPorts) { 2037 return false 2038 } 2039 2040 for i, port := range nr.ReservedPorts { 2041 if len(other.ReservedPorts) <= i { 2042 return false 2043 } 2044 if port != other.ReservedPorts[i] { 2045 return false 2046 } 2047 } 2048 2049 if len(nr.DynamicPorts) != len(other.DynamicPorts) { 2050 return false 2051 } 2052 for i, port := range nr.DynamicPorts { 2053 if len(other.DynamicPorts) <= i { 2054 return false 2055 } 2056 if port != other.DynamicPorts[i] { 2057 return false 2058 } 2059 } 2060 return true 2061 } 2062 2063 func (n *NetworkResource) Canonicalize() { 2064 // Ensure that an empty and nil slices are treated the same to avoid scheduling 2065 // problems since we use reflect DeepEquals. 2066 if len(n.ReservedPorts) == 0 { 2067 n.ReservedPorts = nil 2068 } 2069 if len(n.DynamicPorts) == 0 { 2070 n.DynamicPorts = nil 2071 } 2072 } 2073 2074 // MeetsMinResources returns an error if the resources specified are less than 2075 // the minimum allowed. 2076 func (n *NetworkResource) MeetsMinResources() error { 2077 var mErr multierror.Error 2078 if n.MBits < 1 { 2079 mErr.Errors = append(mErr.Errors, fmt.Errorf("minimum MBits value is 1; got %d", n.MBits)) 2080 } 2081 return mErr.ErrorOrNil() 2082 } 2083 2084 // Copy returns a deep copy of the network resource 2085 func (n *NetworkResource) Copy() *NetworkResource { 2086 if n == nil { 2087 return nil 2088 } 2089 newR := new(NetworkResource) 2090 *newR = *n 2091 if n.ReservedPorts != nil { 2092 newR.ReservedPorts = make([]Port, len(n.ReservedPorts)) 2093 copy(newR.ReservedPorts, n.ReservedPorts) 2094 } 2095 if n.DynamicPorts != nil { 2096 newR.DynamicPorts = make([]Port, len(n.DynamicPorts)) 2097 copy(newR.DynamicPorts, n.DynamicPorts) 2098 } 2099 return newR 2100 } 2101 2102 // Add adds the resources of the delta to this, potentially 2103 // returning an error if not possible. 2104 func (n *NetworkResource) Add(delta *NetworkResource) { 2105 if len(delta.ReservedPorts) > 0 { 2106 n.ReservedPorts = append(n.ReservedPorts, delta.ReservedPorts...) 2107 } 2108 n.MBits += delta.MBits 2109 n.DynamicPorts = append(n.DynamicPorts, delta.DynamicPorts...) 2110 } 2111 2112 func (n *NetworkResource) GoString() string { 2113 return fmt.Sprintf("*%#v", *n) 2114 } 2115 2116 // PortLabels returns a map of port labels to their assigned host ports. 2117 func (n *NetworkResource) PortLabels() map[string]int { 2118 num := len(n.ReservedPorts) + len(n.DynamicPorts) 2119 labelValues := make(map[string]int, num) 2120 for _, port := range n.ReservedPorts { 2121 labelValues[port.Label] = port.Value 2122 } 2123 for _, port := range n.DynamicPorts { 2124 labelValues[port.Label] = port.Value 2125 } 2126 return labelValues 2127 } 2128 2129 // Networks defined for a task on the Resources struct. 2130 type Networks []*NetworkResource 2131 2132 // Port assignment and IP for the given label or empty values. 2133 func (ns Networks) Port(label string) (string, int) { 2134 for _, n := range ns { 2135 for _, p := range n.ReservedPorts { 2136 if p.Label == label { 2137 return n.IP, p.Value 2138 } 2139 } 2140 for _, p := range n.DynamicPorts { 2141 if p.Label == label { 2142 return n.IP, p.Value 2143 } 2144 } 2145 } 2146 return "", 0 2147 } 2148 2149 func (ns Networks) NetIndex(n *NetworkResource) int { 2150 for idx, net := range ns { 2151 if net.Device == n.Device { 2152 return idx 2153 } 2154 } 2155 return -1 2156 } 2157 2158 // RequestedDevice is used to request a device for a task. 2159 type RequestedDevice struct { 2160 // Name is the request name. The possible values are as follows: 2161 // * <type>: A single value only specifies the type of request. 2162 // * <vendor>/<type>: A single slash delimiter assumes the vendor and type of device is specified. 2163 // * <vendor>/<type>/<name>: Two slash delimiters assume vendor, type and specific model are specified. 2164 // 2165 // Examples are as follows: 2166 // * "gpu" 2167 // * "nvidia/gpu" 2168 // * "nvidia/gpu/GTX2080Ti" 2169 Name string 2170 2171 // Count is the number of requested devices 2172 Count uint64 2173 2174 // Constraints are a set of constraints to apply when selecting the device 2175 // to use. 2176 Constraints Constraints 2177 2178 // Affinities are a set of affinites to apply when selecting the device 2179 // to use. 2180 Affinities Affinities 2181 } 2182 2183 func (r *RequestedDevice) Equals(o *RequestedDevice) bool { 2184 if r == o { 2185 return true 2186 } 2187 if r == nil || o == nil { 2188 return false 2189 } 2190 return r.Name == o.Name && 2191 r.Count == o.Count && 2192 r.Constraints.Equals(&o.Constraints) && 2193 r.Affinities.Equals(&o.Affinities) 2194 } 2195 2196 func (r *RequestedDevice) Copy() *RequestedDevice { 2197 if r == nil { 2198 return nil 2199 } 2200 2201 nr := *r 2202 nr.Constraints = CopySliceConstraints(nr.Constraints) 2203 nr.Affinities = CopySliceAffinities(nr.Affinities) 2204 2205 return &nr 2206 } 2207 2208 func (r *RequestedDevice) ID() *DeviceIdTuple { 2209 if r == nil || r.Name == "" { 2210 return nil 2211 } 2212 2213 parts := strings.SplitN(r.Name, "/", 3) 2214 switch len(parts) { 2215 case 1: 2216 return &DeviceIdTuple{ 2217 Type: parts[0], 2218 } 2219 case 2: 2220 return &DeviceIdTuple{ 2221 Vendor: parts[0], 2222 Type: parts[1], 2223 } 2224 default: 2225 return &DeviceIdTuple{ 2226 Vendor: parts[0], 2227 Type: parts[1], 2228 Name: parts[2], 2229 } 2230 } 2231 } 2232 2233 func (r *RequestedDevice) Validate() error { 2234 if r == nil { 2235 return nil 2236 } 2237 2238 var mErr multierror.Error 2239 if r.Name == "" { 2240 multierror.Append(&mErr, errors.New("device name must be given as one of the following: type, vendor/type, or vendor/type/name")) 2241 } 2242 2243 for idx, constr := range r.Constraints { 2244 // Ensure that the constraint doesn't use an operand we do not allow 2245 switch constr.Operand { 2246 case ConstraintDistinctHosts, ConstraintDistinctProperty: 2247 outer := fmt.Errorf("Constraint %d validation failed: using unsupported operand %q", idx+1, constr.Operand) 2248 multierror.Append(&mErr, outer) 2249 default: 2250 if err := constr.Validate(); err != nil { 2251 outer := fmt.Errorf("Constraint %d validation failed: %s", idx+1, err) 2252 multierror.Append(&mErr, outer) 2253 } 2254 } 2255 } 2256 for idx, affinity := range r.Affinities { 2257 if err := affinity.Validate(); err != nil { 2258 outer := fmt.Errorf("Affinity %d validation failed: %s", idx+1, err) 2259 multierror.Append(&mErr, outer) 2260 } 2261 } 2262 2263 return mErr.ErrorOrNil() 2264 } 2265 2266 // NodeResources is used to define the resources available on a client node. 2267 type NodeResources struct { 2268 Cpu NodeCpuResources 2269 Memory NodeMemoryResources 2270 Disk NodeDiskResources 2271 Networks Networks 2272 Devices []*NodeDeviceResource 2273 } 2274 2275 func (n *NodeResources) Copy() *NodeResources { 2276 if n == nil { 2277 return nil 2278 } 2279 2280 newN := new(NodeResources) 2281 *newN = *n 2282 2283 // Copy the networks 2284 if n.Networks != nil { 2285 networks := len(n.Networks) 2286 newN.Networks = make([]*NetworkResource, networks) 2287 for i := 0; i < networks; i++ { 2288 newN.Networks[i] = n.Networks[i].Copy() 2289 } 2290 } 2291 2292 // Copy the devices 2293 if n.Devices != nil { 2294 devices := len(n.Devices) 2295 newN.Devices = make([]*NodeDeviceResource, devices) 2296 for i := 0; i < devices; i++ { 2297 newN.Devices[i] = n.Devices[i].Copy() 2298 } 2299 } 2300 2301 return newN 2302 } 2303 2304 // Comparable returns a comparable version of the nodes resources. This 2305 // conversion can be lossy so care must be taken when using it. 2306 func (n *NodeResources) Comparable() *ComparableResources { 2307 if n == nil { 2308 return nil 2309 } 2310 2311 c := &ComparableResources{ 2312 Flattened: AllocatedTaskResources{ 2313 Cpu: AllocatedCpuResources{ 2314 CpuShares: n.Cpu.CpuShares, 2315 }, 2316 Memory: AllocatedMemoryResources{ 2317 MemoryMB: n.Memory.MemoryMB, 2318 }, 2319 Networks: n.Networks, 2320 }, 2321 Shared: AllocatedSharedResources{ 2322 DiskMB: n.Disk.DiskMB, 2323 }, 2324 } 2325 return c 2326 } 2327 2328 func (n *NodeResources) Merge(o *NodeResources) { 2329 if o == nil { 2330 return 2331 } 2332 2333 n.Cpu.Merge(&o.Cpu) 2334 n.Memory.Merge(&o.Memory) 2335 n.Disk.Merge(&o.Disk) 2336 2337 if len(o.Networks) != 0 { 2338 n.Networks = o.Networks 2339 } 2340 2341 if len(o.Devices) != 0 { 2342 n.Devices = o.Devices 2343 } 2344 } 2345 2346 func (n *NodeResources) Equals(o *NodeResources) bool { 2347 if o == nil && n == nil { 2348 return true 2349 } else if o == nil { 2350 return false 2351 } else if n == nil { 2352 return false 2353 } 2354 2355 if !n.Cpu.Equals(&o.Cpu) { 2356 return false 2357 } 2358 if !n.Memory.Equals(&o.Memory) { 2359 return false 2360 } 2361 if !n.Disk.Equals(&o.Disk) { 2362 return false 2363 } 2364 if !n.Networks.Equals(&o.Networks) { 2365 return false 2366 } 2367 2368 // Check the devices 2369 if !DevicesEquals(n.Devices, o.Devices) { 2370 return false 2371 } 2372 2373 return true 2374 } 2375 2376 // Equals equates Networks as a set 2377 func (n *Networks) Equals(o *Networks) bool { 2378 if n == o { 2379 return true 2380 } 2381 if n == nil || o == nil { 2382 return false 2383 } 2384 if len(*n) != len(*o) { 2385 return false 2386 } 2387 SETEQUALS: 2388 for _, ne := range *n { 2389 for _, oe := range *o { 2390 if ne.Equals(oe) { 2391 continue SETEQUALS 2392 } 2393 } 2394 return false 2395 } 2396 return true 2397 } 2398 2399 // DevicesEquals returns true if the two device arrays are set equal 2400 func DevicesEquals(d1, d2 []*NodeDeviceResource) bool { 2401 if len(d1) != len(d2) { 2402 return false 2403 } 2404 idMap := make(map[DeviceIdTuple]*NodeDeviceResource, len(d1)) 2405 for _, d := range d1 { 2406 idMap[*d.ID()] = d 2407 } 2408 for _, otherD := range d2 { 2409 if d, ok := idMap[*otherD.ID()]; !ok || !d.Equals(otherD) { 2410 return false 2411 } 2412 } 2413 2414 return true 2415 } 2416 2417 // NodeCpuResources captures the CPU resources of the node. 2418 type NodeCpuResources struct { 2419 // CpuShares is the CPU shares available. This is calculated by number of 2420 // cores multiplied by the core frequency. 2421 CpuShares int64 2422 } 2423 2424 func (n *NodeCpuResources) Merge(o *NodeCpuResources) { 2425 if o == nil { 2426 return 2427 } 2428 2429 if o.CpuShares != 0 { 2430 n.CpuShares = o.CpuShares 2431 } 2432 } 2433 2434 func (n *NodeCpuResources) Equals(o *NodeCpuResources) bool { 2435 if o == nil && n == nil { 2436 return true 2437 } else if o == nil { 2438 return false 2439 } else if n == nil { 2440 return false 2441 } 2442 2443 if n.CpuShares != o.CpuShares { 2444 return false 2445 } 2446 2447 return true 2448 } 2449 2450 // NodeMemoryResources captures the memory resources of the node 2451 type NodeMemoryResources struct { 2452 // MemoryMB is the total available memory on the node 2453 MemoryMB int64 2454 } 2455 2456 func (n *NodeMemoryResources) Merge(o *NodeMemoryResources) { 2457 if o == nil { 2458 return 2459 } 2460 2461 if o.MemoryMB != 0 { 2462 n.MemoryMB = o.MemoryMB 2463 } 2464 } 2465 2466 func (n *NodeMemoryResources) Equals(o *NodeMemoryResources) bool { 2467 if o == nil && n == nil { 2468 return true 2469 } else if o == nil { 2470 return false 2471 } else if n == nil { 2472 return false 2473 } 2474 2475 if n.MemoryMB != o.MemoryMB { 2476 return false 2477 } 2478 2479 return true 2480 } 2481 2482 // NodeDiskResources captures the disk resources of the node 2483 type NodeDiskResources struct { 2484 // DiskMB is the total available disk space on the node 2485 DiskMB int64 2486 } 2487 2488 func (n *NodeDiskResources) Merge(o *NodeDiskResources) { 2489 if o == nil { 2490 return 2491 } 2492 if o.DiskMB != 0 { 2493 n.DiskMB = o.DiskMB 2494 } 2495 } 2496 2497 func (n *NodeDiskResources) Equals(o *NodeDiskResources) bool { 2498 if o == nil && n == nil { 2499 return true 2500 } else if o == nil { 2501 return false 2502 } else if n == nil { 2503 return false 2504 } 2505 2506 if n.DiskMB != o.DiskMB { 2507 return false 2508 } 2509 2510 return true 2511 } 2512 2513 // DeviceIdTuple is the tuple that identifies a device 2514 type DeviceIdTuple struct { 2515 Vendor string 2516 Type string 2517 Name string 2518 } 2519 2520 func (d *DeviceIdTuple) String() string { 2521 if d == nil { 2522 return "" 2523 } 2524 2525 return fmt.Sprintf("%s/%s/%s", d.Vendor, d.Type, d.Name) 2526 } 2527 2528 // Matches returns if this Device ID is a superset of the passed ID. 2529 func (id *DeviceIdTuple) Matches(other *DeviceIdTuple) bool { 2530 if other == nil { 2531 return false 2532 } 2533 2534 if other.Name != "" && other.Name != id.Name { 2535 return false 2536 } 2537 2538 if other.Vendor != "" && other.Vendor != id.Vendor { 2539 return false 2540 } 2541 2542 if other.Type != "" && other.Type != id.Type { 2543 return false 2544 } 2545 2546 return true 2547 } 2548 2549 // Equals returns if this Device ID is the same as the passed ID. 2550 func (id *DeviceIdTuple) Equals(o *DeviceIdTuple) bool { 2551 if id == nil && o == nil { 2552 return true 2553 } else if id == nil || o == nil { 2554 return false 2555 } 2556 2557 return o.Vendor == id.Vendor && o.Type == id.Type && o.Name == id.Name 2558 } 2559 2560 // NodeDeviceResource captures a set of devices sharing a common 2561 // vendor/type/device_name tuple. 2562 type NodeDeviceResource struct { 2563 Vendor string 2564 Type string 2565 Name string 2566 Instances []*NodeDevice 2567 Attributes map[string]*psstructs.Attribute 2568 } 2569 2570 func (n *NodeDeviceResource) ID() *DeviceIdTuple { 2571 if n == nil { 2572 return nil 2573 } 2574 2575 return &DeviceIdTuple{ 2576 Vendor: n.Vendor, 2577 Type: n.Type, 2578 Name: n.Name, 2579 } 2580 } 2581 2582 func (n *NodeDeviceResource) Copy() *NodeDeviceResource { 2583 if n == nil { 2584 return nil 2585 } 2586 2587 // Copy the primitives 2588 nn := *n 2589 2590 // Copy the device instances 2591 if l := len(nn.Instances); l != 0 { 2592 nn.Instances = make([]*NodeDevice, 0, l) 2593 for _, d := range n.Instances { 2594 nn.Instances = append(nn.Instances, d.Copy()) 2595 } 2596 } 2597 2598 // Copy the Attributes 2599 nn.Attributes = psstructs.CopyMapStringAttribute(nn.Attributes) 2600 2601 return &nn 2602 } 2603 2604 func (n *NodeDeviceResource) Equals(o *NodeDeviceResource) bool { 2605 if o == nil && n == nil { 2606 return true 2607 } else if o == nil { 2608 return false 2609 } else if n == nil { 2610 return false 2611 } 2612 2613 if n.Vendor != o.Vendor { 2614 return false 2615 } else if n.Type != o.Type { 2616 return false 2617 } else if n.Name != o.Name { 2618 return false 2619 } 2620 2621 // Check the attributes 2622 if len(n.Attributes) != len(o.Attributes) { 2623 return false 2624 } 2625 for k, v := range n.Attributes { 2626 if otherV, ok := o.Attributes[k]; !ok || v != otherV { 2627 return false 2628 } 2629 } 2630 2631 // Check the instances 2632 if len(n.Instances) != len(o.Instances) { 2633 return false 2634 } 2635 idMap := make(map[string]*NodeDevice, len(n.Instances)) 2636 for _, d := range n.Instances { 2637 idMap[d.ID] = d 2638 } 2639 for _, otherD := range o.Instances { 2640 if d, ok := idMap[otherD.ID]; !ok || !d.Equals(otherD) { 2641 return false 2642 } 2643 } 2644 2645 return true 2646 } 2647 2648 // NodeDevice is an instance of a particular device. 2649 type NodeDevice struct { 2650 // ID is the ID of the device. 2651 ID string 2652 2653 // Healthy captures whether the device is healthy. 2654 Healthy bool 2655 2656 // HealthDescription is used to provide a human readable description of why 2657 // the device may be unhealthy. 2658 HealthDescription string 2659 2660 // Locality stores HW locality information for the node to optionally be 2661 // used when making placement decisions. 2662 Locality *NodeDeviceLocality 2663 } 2664 2665 func (n *NodeDevice) Equals(o *NodeDevice) bool { 2666 if o == nil && n == nil { 2667 return true 2668 } else if o == nil { 2669 return false 2670 } else if n == nil { 2671 return false 2672 } 2673 2674 if n.ID != o.ID { 2675 return false 2676 } else if n.Healthy != o.Healthy { 2677 return false 2678 } else if n.HealthDescription != o.HealthDescription { 2679 return false 2680 } else if !n.Locality.Equals(o.Locality) { 2681 return false 2682 } 2683 2684 return false 2685 } 2686 2687 func (n *NodeDevice) Copy() *NodeDevice { 2688 if n == nil { 2689 return nil 2690 } 2691 2692 // Copy the primitives 2693 nn := *n 2694 2695 // Copy the locality 2696 nn.Locality = nn.Locality.Copy() 2697 2698 return &nn 2699 } 2700 2701 // NodeDeviceLocality stores information about the devices hardware locality on 2702 // the node. 2703 type NodeDeviceLocality struct { 2704 // PciBusID is the PCI Bus ID for the device. 2705 PciBusID string 2706 } 2707 2708 func (n *NodeDeviceLocality) Equals(o *NodeDeviceLocality) bool { 2709 if o == nil && n == nil { 2710 return true 2711 } else if o == nil { 2712 return false 2713 } else if n == nil { 2714 return false 2715 } 2716 2717 if n.PciBusID != o.PciBusID { 2718 return false 2719 } 2720 2721 return true 2722 } 2723 2724 func (n *NodeDeviceLocality) Copy() *NodeDeviceLocality { 2725 if n == nil { 2726 return nil 2727 } 2728 2729 // Copy the primitives 2730 nn := *n 2731 return &nn 2732 } 2733 2734 // NodeReservedResources is used to capture the resources on a client node that 2735 // should be reserved and not made available to jobs. 2736 type NodeReservedResources struct { 2737 Cpu NodeReservedCpuResources 2738 Memory NodeReservedMemoryResources 2739 Disk NodeReservedDiskResources 2740 Networks NodeReservedNetworkResources 2741 } 2742 2743 func (n *NodeReservedResources) Copy() *NodeReservedResources { 2744 if n == nil { 2745 return nil 2746 } 2747 newN := new(NodeReservedResources) 2748 *newN = *n 2749 return newN 2750 } 2751 2752 // Comparable returns a comparable version of the node's reserved resources. The 2753 // returned resources doesn't contain any network information. This conversion 2754 // can be lossy so care must be taken when using it. 2755 func (n *NodeReservedResources) Comparable() *ComparableResources { 2756 if n == nil { 2757 return nil 2758 } 2759 2760 c := &ComparableResources{ 2761 Flattened: AllocatedTaskResources{ 2762 Cpu: AllocatedCpuResources{ 2763 CpuShares: n.Cpu.CpuShares, 2764 }, 2765 Memory: AllocatedMemoryResources{ 2766 MemoryMB: n.Memory.MemoryMB, 2767 }, 2768 }, 2769 Shared: AllocatedSharedResources{ 2770 DiskMB: n.Disk.DiskMB, 2771 }, 2772 } 2773 return c 2774 } 2775 2776 // NodeReservedCpuResources captures the reserved CPU resources of the node. 2777 type NodeReservedCpuResources struct { 2778 CpuShares int64 2779 } 2780 2781 // NodeReservedMemoryResources captures the reserved memory resources of the node. 2782 type NodeReservedMemoryResources struct { 2783 MemoryMB int64 2784 } 2785 2786 // NodeReservedDiskResources captures the reserved disk resources of the node. 2787 type NodeReservedDiskResources struct { 2788 DiskMB int64 2789 } 2790 2791 // NodeReservedNetworkResources captures the reserved network resources of the node. 2792 type NodeReservedNetworkResources struct { 2793 // ReservedHostPorts is the set of ports reserved on all host network 2794 // interfaces. Its format is a comma separate list of integers or integer 2795 // ranges. (80,443,1000-2000,2005) 2796 ReservedHostPorts string 2797 } 2798 2799 // ParsePortHostPorts returns the reserved host ports. 2800 func (n *NodeReservedNetworkResources) ParseReservedHostPorts() ([]uint64, error) { 2801 return ParsePortRanges(n.ReservedHostPorts) 2802 } 2803 2804 // AllocatedResources is the set of resources to be used by an allocation. 2805 type AllocatedResources struct { 2806 // Tasks is a mapping of task name to the resources for the task. 2807 Tasks map[string]*AllocatedTaskResources 2808 2809 // Shared is the set of resource that are shared by all tasks in the group. 2810 Shared AllocatedSharedResources 2811 } 2812 2813 func (a *AllocatedResources) Copy() *AllocatedResources { 2814 if a == nil { 2815 return nil 2816 } 2817 newA := new(AllocatedResources) 2818 *newA = *a 2819 2820 if a.Tasks != nil { 2821 tr := make(map[string]*AllocatedTaskResources, len(newA.Tasks)) 2822 for task, resource := range newA.Tasks { 2823 tr[task] = resource.Copy() 2824 } 2825 newA.Tasks = tr 2826 } 2827 2828 return newA 2829 } 2830 2831 // Comparable returns a comparable version of the allocations allocated 2832 // resources. This conversion can be lossy so care must be taken when using it. 2833 func (a *AllocatedResources) Comparable() *ComparableResources { 2834 if a == nil { 2835 return nil 2836 } 2837 2838 c := &ComparableResources{ 2839 Shared: a.Shared, 2840 } 2841 for _, r := range a.Tasks { 2842 c.Flattened.Add(r) 2843 } 2844 return c 2845 } 2846 2847 // OldTaskResources returns the pre-0.9.0 map of task resources 2848 func (a *AllocatedResources) OldTaskResources() map[string]*Resources { 2849 m := make(map[string]*Resources, len(a.Tasks)) 2850 for name, res := range a.Tasks { 2851 m[name] = &Resources{ 2852 CPU: int(res.Cpu.CpuShares), 2853 MemoryMB: int(res.Memory.MemoryMB), 2854 Networks: res.Networks, 2855 } 2856 } 2857 2858 return m 2859 } 2860 2861 // AllocatedTaskResources are the set of resources allocated to a task. 2862 type AllocatedTaskResources struct { 2863 Cpu AllocatedCpuResources 2864 Memory AllocatedMemoryResources 2865 Networks Networks 2866 Devices []*AllocatedDeviceResource 2867 } 2868 2869 func (a *AllocatedTaskResources) Copy() *AllocatedTaskResources { 2870 if a == nil { 2871 return nil 2872 } 2873 newA := new(AllocatedTaskResources) 2874 *newA = *a 2875 2876 // Copy the networks 2877 if a.Networks != nil { 2878 n := len(a.Networks) 2879 newA.Networks = make([]*NetworkResource, n) 2880 for i := 0; i < n; i++ { 2881 newA.Networks[i] = a.Networks[i].Copy() 2882 } 2883 } 2884 2885 // Copy the devices 2886 if newA.Devices != nil { 2887 n := len(a.Devices) 2888 newA.Devices = make([]*AllocatedDeviceResource, n) 2889 for i := 0; i < n; i++ { 2890 newA.Devices[i] = a.Devices[i].Copy() 2891 } 2892 } 2893 2894 return newA 2895 } 2896 2897 // NetIndex finds the matching net index using device name 2898 func (a *AllocatedTaskResources) NetIndex(n *NetworkResource) int { 2899 return a.Networks.NetIndex(n) 2900 } 2901 2902 func (a *AllocatedTaskResources) Add(delta *AllocatedTaskResources) { 2903 if delta == nil { 2904 return 2905 } 2906 2907 a.Cpu.Add(&delta.Cpu) 2908 a.Memory.Add(&delta.Memory) 2909 2910 for _, n := range delta.Networks { 2911 // Find the matching interface by IP or CIDR 2912 idx := a.NetIndex(n) 2913 if idx == -1 { 2914 a.Networks = append(a.Networks, n.Copy()) 2915 } else { 2916 a.Networks[idx].Add(n) 2917 } 2918 } 2919 2920 for _, d := range delta.Devices { 2921 // Find the matching device 2922 idx := AllocatedDevices(a.Devices).Index(d) 2923 if idx == -1 { 2924 a.Devices = append(a.Devices, d.Copy()) 2925 } else { 2926 a.Devices[idx].Add(d) 2927 } 2928 } 2929 } 2930 2931 // Comparable turns AllocatedTaskResources into ComparableResources 2932 // as a helper step in preemption 2933 func (a *AllocatedTaskResources) Comparable() *ComparableResources { 2934 ret := &ComparableResources{ 2935 Flattened: AllocatedTaskResources{ 2936 Cpu: AllocatedCpuResources{ 2937 CpuShares: a.Cpu.CpuShares, 2938 }, 2939 Memory: AllocatedMemoryResources{ 2940 MemoryMB: a.Memory.MemoryMB, 2941 }, 2942 }, 2943 } 2944 if len(a.Networks) > 0 { 2945 for _, net := range a.Networks { 2946 ret.Flattened.Networks = append(ret.Flattened.Networks, net) 2947 } 2948 } 2949 return ret 2950 } 2951 2952 // Subtract only subtracts CPU and Memory resources. Network utilization 2953 // is managed separately in NetworkIndex 2954 func (a *AllocatedTaskResources) Subtract(delta *AllocatedTaskResources) { 2955 if delta == nil { 2956 return 2957 } 2958 2959 a.Cpu.Subtract(&delta.Cpu) 2960 a.Memory.Subtract(&delta.Memory) 2961 } 2962 2963 // AllocatedSharedResources are the set of resources allocated to a task group. 2964 type AllocatedSharedResources struct { 2965 DiskMB int64 2966 } 2967 2968 func (a *AllocatedSharedResources) Add(delta *AllocatedSharedResources) { 2969 if delta == nil { 2970 return 2971 } 2972 2973 a.DiskMB += delta.DiskMB 2974 } 2975 2976 func (a *AllocatedSharedResources) Subtract(delta *AllocatedSharedResources) { 2977 if delta == nil { 2978 return 2979 } 2980 2981 a.DiskMB -= delta.DiskMB 2982 } 2983 2984 // AllocatedCpuResources captures the allocated CPU resources. 2985 type AllocatedCpuResources struct { 2986 CpuShares int64 2987 } 2988 2989 func (a *AllocatedCpuResources) Add(delta *AllocatedCpuResources) { 2990 if delta == nil { 2991 return 2992 } 2993 2994 a.CpuShares += delta.CpuShares 2995 } 2996 2997 func (a *AllocatedCpuResources) Subtract(delta *AllocatedCpuResources) { 2998 if delta == nil { 2999 return 3000 } 3001 3002 a.CpuShares -= delta.CpuShares 3003 } 3004 3005 // AllocatedMemoryResources captures the allocated memory resources. 3006 type AllocatedMemoryResources struct { 3007 MemoryMB int64 3008 } 3009 3010 func (a *AllocatedMemoryResources) Add(delta *AllocatedMemoryResources) { 3011 if delta == nil { 3012 return 3013 } 3014 3015 a.MemoryMB += delta.MemoryMB 3016 } 3017 3018 func (a *AllocatedMemoryResources) Subtract(delta *AllocatedMemoryResources) { 3019 if delta == nil { 3020 return 3021 } 3022 3023 a.MemoryMB -= delta.MemoryMB 3024 } 3025 3026 type AllocatedDevices []*AllocatedDeviceResource 3027 3028 // Index finds the matching index using the passed device. If not found, -1 is 3029 // returned. 3030 func (a AllocatedDevices) Index(d *AllocatedDeviceResource) int { 3031 if d == nil { 3032 return -1 3033 } 3034 3035 for i, o := range a { 3036 if o.ID().Equals(d.ID()) { 3037 return i 3038 } 3039 } 3040 3041 return -1 3042 } 3043 3044 // AllocatedDeviceResource captures a set of allocated devices. 3045 type AllocatedDeviceResource struct { 3046 // Vendor, Type, and Name are used to select the plugin to request the 3047 // device IDs from. 3048 Vendor string 3049 Type string 3050 Name string 3051 3052 // DeviceIDs is the set of allocated devices 3053 DeviceIDs []string 3054 } 3055 3056 func (a *AllocatedDeviceResource) ID() *DeviceIdTuple { 3057 if a == nil { 3058 return nil 3059 } 3060 3061 return &DeviceIdTuple{ 3062 Vendor: a.Vendor, 3063 Type: a.Type, 3064 Name: a.Name, 3065 } 3066 } 3067 3068 func (a *AllocatedDeviceResource) Add(delta *AllocatedDeviceResource) { 3069 if delta == nil { 3070 return 3071 } 3072 3073 a.DeviceIDs = append(a.DeviceIDs, delta.DeviceIDs...) 3074 } 3075 3076 func (a *AllocatedDeviceResource) Copy() *AllocatedDeviceResource { 3077 if a == nil { 3078 return a 3079 } 3080 3081 na := *a 3082 3083 // Copy the devices 3084 na.DeviceIDs = make([]string, len(a.DeviceIDs)) 3085 for i, id := range a.DeviceIDs { 3086 na.DeviceIDs[i] = id 3087 } 3088 3089 return &na 3090 } 3091 3092 // ComparableResources is the set of resources allocated to a task group but 3093 // not keyed by Task, making it easier to compare. 3094 type ComparableResources struct { 3095 Flattened AllocatedTaskResources 3096 Shared AllocatedSharedResources 3097 } 3098 3099 func (c *ComparableResources) Add(delta *ComparableResources) { 3100 if delta == nil { 3101 return 3102 } 3103 3104 c.Flattened.Add(&delta.Flattened) 3105 c.Shared.Add(&delta.Shared) 3106 } 3107 3108 func (c *ComparableResources) Subtract(delta *ComparableResources) { 3109 if delta == nil { 3110 return 3111 } 3112 3113 c.Flattened.Subtract(&delta.Flattened) 3114 c.Shared.Subtract(&delta.Shared) 3115 } 3116 3117 func (c *ComparableResources) Copy() *ComparableResources { 3118 if c == nil { 3119 return nil 3120 } 3121 newR := new(ComparableResources) 3122 *newR = *c 3123 return newR 3124 } 3125 3126 // Superset checks if one set of resources is a superset of another. This 3127 // ignores network resources, and the NetworkIndex should be used for that. 3128 func (c *ComparableResources) Superset(other *ComparableResources) (bool, string) { 3129 if c.Flattened.Cpu.CpuShares < other.Flattened.Cpu.CpuShares { 3130 return false, "cpu" 3131 } 3132 if c.Flattened.Memory.MemoryMB < other.Flattened.Memory.MemoryMB { 3133 return false, "memory" 3134 } 3135 if c.Shared.DiskMB < other.Shared.DiskMB { 3136 return false, "disk" 3137 } 3138 return true, "" 3139 } 3140 3141 // allocated finds the matching net index using device name 3142 func (c *ComparableResources) NetIndex(n *NetworkResource) int { 3143 return c.Flattened.Networks.NetIndex(n) 3144 } 3145 3146 const ( 3147 // JobTypeNomad is reserved for internal system tasks and is 3148 // always handled by the CoreScheduler. 3149 JobTypeCore = "_core" 3150 JobTypeService = "service" 3151 JobTypeBatch = "batch" 3152 JobTypeSystem = "system" 3153 ) 3154 3155 const ( 3156 JobStatusPending = "pending" // Pending means the job is waiting on scheduling 3157 JobStatusRunning = "running" // Running means the job has non-terminal allocations 3158 JobStatusDead = "dead" // Dead means all evaluation's and allocations are terminal 3159 ) 3160 3161 const ( 3162 // JobMinPriority is the minimum allowed priority 3163 JobMinPriority = 1 3164 3165 // JobDefaultPriority is the default priority if not 3166 // not specified. 3167 JobDefaultPriority = 50 3168 3169 // JobMaxPriority is the maximum allowed priority 3170 JobMaxPriority = 100 3171 3172 // Ensure CoreJobPriority is higher than any user 3173 // specified job so that it gets priority. This is important 3174 // for the system to remain healthy. 3175 CoreJobPriority = JobMaxPriority * 2 3176 3177 // JobTrackedVersions is the number of historic job versions that are 3178 // kept. 3179 JobTrackedVersions = 6 3180 ) 3181 3182 // Job is the scope of a scheduling request to Nomad. It is the largest 3183 // scoped object, and is a named collection of task groups. Each task group 3184 // is further composed of tasks. A task group (TG) is the unit of scheduling 3185 // however. 3186 type Job struct { 3187 // Stop marks whether the user has stopped the job. A stopped job will 3188 // have all created allocations stopped and acts as a way to stop a job 3189 // without purging it from the system. This allows existing allocs to be 3190 // queried and the job to be inspected as it is being killed. 3191 Stop bool 3192 3193 // Region is the Nomad region that handles scheduling this job 3194 Region string 3195 3196 // Namespace is the namespace the job is submitted into. 3197 Namespace string 3198 3199 // ID is a unique identifier for the job per region. It can be 3200 // specified hierarchically like LineOfBiz/OrgName/Team/Project 3201 ID string 3202 3203 // ParentID is the unique identifier of the job that spawned this job. 3204 ParentID string 3205 3206 // Name is the logical name of the job used to refer to it. This is unique 3207 // per region, but not unique globally. 3208 Name string 3209 3210 // Type is used to control various behaviors about the job. Most jobs 3211 // are service jobs, meaning they are expected to be long lived. 3212 // Some jobs are batch oriented meaning they run and then terminate. 3213 // This can be extended in the future to support custom schedulers. 3214 Type string 3215 3216 // Priority is used to control scheduling importance and if this job 3217 // can preempt other jobs. 3218 Priority int 3219 3220 // AllAtOnce is used to control if incremental scheduling of task groups 3221 // is allowed or if we must do a gang scheduling of the entire job. This 3222 // can slow down larger jobs if resources are not available. 3223 AllAtOnce bool 3224 3225 // Datacenters contains all the datacenters this job is allowed to span 3226 Datacenters []string 3227 3228 // Constraints can be specified at a job level and apply to 3229 // all the task groups and tasks. 3230 Constraints []*Constraint 3231 3232 // Affinities can be specified at the job level to express 3233 // scheduling preferences that apply to all groups and tasks 3234 Affinities []*Affinity 3235 3236 // Spread can be specified at the job level to express spreading 3237 // allocations across a desired attribute, such as datacenter 3238 Spreads []*Spread 3239 3240 // TaskGroups are the collections of task groups that this job needs 3241 // to run. Each task group is an atomic unit of scheduling and placement. 3242 TaskGroups []*TaskGroup 3243 3244 // See agent.ApiJobToStructJob 3245 // Update provides defaults for the TaskGroup Update stanzas 3246 Update UpdateStrategy 3247 3248 // Periodic is used to define the interval the job is run at. 3249 Periodic *PeriodicConfig 3250 3251 // ParameterizedJob is used to specify the job as a parameterized job 3252 // for dispatching. 3253 ParameterizedJob *ParameterizedJobConfig 3254 3255 // Dispatched is used to identify if the Job has been dispatched from a 3256 // parameterized job. 3257 Dispatched bool 3258 3259 // Payload is the payload supplied when the job was dispatched. 3260 Payload []byte 3261 3262 // Meta is used to associate arbitrary metadata with this 3263 // job. This is opaque to Nomad. 3264 Meta map[string]string 3265 3266 // VaultToken is the Vault token that proves the submitter of the job has 3267 // access to the specified Vault policies. This field is only used to 3268 // transfer the token and is not stored after Job submission. 3269 VaultToken string 3270 3271 // Job status 3272 Status string 3273 3274 // StatusDescription is meant to provide more human useful information 3275 StatusDescription string 3276 3277 // Stable marks a job as stable. Stability is only defined on "service" and 3278 // "system" jobs. The stability of a job will be set automatically as part 3279 // of a deployment and can be manually set via APIs. 3280 Stable bool 3281 3282 // Version is a monotonically increasing version number that is incremented 3283 // on each job register. 3284 Version uint64 3285 3286 // SubmitTime is the time at which the job was submitted as a UnixNano in 3287 // UTC 3288 SubmitTime int64 3289 3290 // Raft Indexes 3291 CreateIndex uint64 3292 ModifyIndex uint64 3293 JobModifyIndex uint64 3294 } 3295 3296 // NamespacedID returns the namespaced id useful for logging 3297 func (j *Job) NamespacedID() *NamespacedID { 3298 return &NamespacedID{ 3299 ID: j.ID, 3300 Namespace: j.Namespace, 3301 } 3302 } 3303 3304 // Canonicalize is used to canonicalize fields in the Job. This should be called 3305 // when registering a Job. A set of warnings are returned if the job was changed 3306 // in anyway that the user should be made aware of. 3307 func (j *Job) Canonicalize() (warnings error) { 3308 if j == nil { 3309 return nil 3310 } 3311 3312 var mErr multierror.Error 3313 // Ensure that an empty and nil map are treated the same to avoid scheduling 3314 // problems since we use reflect DeepEquals. 3315 if len(j.Meta) == 0 { 3316 j.Meta = nil 3317 } 3318 3319 // Ensure the job is in a namespace. 3320 if j.Namespace == "" { 3321 j.Namespace = DefaultNamespace 3322 } 3323 3324 for _, tg := range j.TaskGroups { 3325 tg.Canonicalize(j) 3326 } 3327 3328 if j.ParameterizedJob != nil { 3329 j.ParameterizedJob.Canonicalize() 3330 } 3331 3332 if j.Periodic != nil { 3333 j.Periodic.Canonicalize() 3334 } 3335 3336 return mErr.ErrorOrNil() 3337 } 3338 3339 // Copy returns a deep copy of the Job. It is expected that callers use recover. 3340 // This job can panic if the deep copy failed as it uses reflection. 3341 func (j *Job) Copy() *Job { 3342 if j == nil { 3343 return nil 3344 } 3345 nj := new(Job) 3346 *nj = *j 3347 nj.Datacenters = helper.CopySliceString(nj.Datacenters) 3348 nj.Constraints = CopySliceConstraints(nj.Constraints) 3349 nj.Affinities = CopySliceAffinities(nj.Affinities) 3350 3351 if j.TaskGroups != nil { 3352 tgs := make([]*TaskGroup, len(nj.TaskGroups)) 3353 for i, tg := range nj.TaskGroups { 3354 tgs[i] = tg.Copy() 3355 } 3356 nj.TaskGroups = tgs 3357 } 3358 3359 nj.Periodic = nj.Periodic.Copy() 3360 nj.Meta = helper.CopyMapStringString(nj.Meta) 3361 nj.ParameterizedJob = nj.ParameterizedJob.Copy() 3362 return nj 3363 } 3364 3365 // Validate is used to sanity check a job input 3366 func (j *Job) Validate() error { 3367 var mErr multierror.Error 3368 3369 if j.Region == "" { 3370 mErr.Errors = append(mErr.Errors, errors.New("Missing job region")) 3371 } 3372 if j.ID == "" { 3373 mErr.Errors = append(mErr.Errors, errors.New("Missing job ID")) 3374 } else if strings.Contains(j.ID, " ") { 3375 mErr.Errors = append(mErr.Errors, errors.New("Job ID contains a space")) 3376 } 3377 if j.Name == "" { 3378 mErr.Errors = append(mErr.Errors, errors.New("Missing job name")) 3379 } 3380 if j.Namespace == "" { 3381 mErr.Errors = append(mErr.Errors, errors.New("Job must be in a namespace")) 3382 } 3383 switch j.Type { 3384 case JobTypeCore, JobTypeService, JobTypeBatch, JobTypeSystem: 3385 case "": 3386 mErr.Errors = append(mErr.Errors, errors.New("Missing job type")) 3387 default: 3388 mErr.Errors = append(mErr.Errors, fmt.Errorf("Invalid job type: %q", j.Type)) 3389 } 3390 if j.Priority < JobMinPriority || j.Priority > JobMaxPriority { 3391 mErr.Errors = append(mErr.Errors, fmt.Errorf("Job priority must be between [%d, %d]", JobMinPriority, JobMaxPriority)) 3392 } 3393 if len(j.Datacenters) == 0 { 3394 mErr.Errors = append(mErr.Errors, errors.New("Missing job datacenters")) 3395 } else { 3396 for _, v := range j.Datacenters { 3397 if v == "" { 3398 mErr.Errors = append(mErr.Errors, errors.New("Job datacenter must be non-empty string")) 3399 } 3400 } 3401 } 3402 if len(j.TaskGroups) == 0 { 3403 mErr.Errors = append(mErr.Errors, errors.New("Missing job task groups")) 3404 } 3405 for idx, constr := range j.Constraints { 3406 if err := constr.Validate(); err != nil { 3407 outer := fmt.Errorf("Constraint %d validation failed: %s", idx+1, err) 3408 mErr.Errors = append(mErr.Errors, outer) 3409 } 3410 } 3411 if j.Type == JobTypeSystem { 3412 if j.Affinities != nil { 3413 mErr.Errors = append(mErr.Errors, fmt.Errorf("System jobs may not have an affinity stanza")) 3414 } 3415 } else { 3416 for idx, affinity := range j.Affinities { 3417 if err := affinity.Validate(); err != nil { 3418 outer := fmt.Errorf("Affinity %d validation failed: %s", idx+1, err) 3419 mErr.Errors = append(mErr.Errors, outer) 3420 } 3421 } 3422 } 3423 3424 if j.Type == JobTypeSystem { 3425 if j.Spreads != nil { 3426 mErr.Errors = append(mErr.Errors, fmt.Errorf("System jobs may not have a spread stanza")) 3427 } 3428 } else { 3429 for idx, spread := range j.Spreads { 3430 if err := spread.Validate(); err != nil { 3431 outer := fmt.Errorf("Spread %d validation failed: %s", idx+1, err) 3432 mErr.Errors = append(mErr.Errors, outer) 3433 } 3434 } 3435 } 3436 3437 // Check for duplicate task groups 3438 taskGroups := make(map[string]int) 3439 for idx, tg := range j.TaskGroups { 3440 if tg.Name == "" { 3441 mErr.Errors = append(mErr.Errors, fmt.Errorf("Job task group %d missing name", idx+1)) 3442 } else if existing, ok := taskGroups[tg.Name]; ok { 3443 mErr.Errors = append(mErr.Errors, fmt.Errorf("Job task group %d redefines '%s' from group %d", idx+1, tg.Name, existing+1)) 3444 } else { 3445 taskGroups[tg.Name] = idx 3446 } 3447 3448 if j.Type == "system" && tg.Count > 1 { 3449 mErr.Errors = append(mErr.Errors, 3450 fmt.Errorf("Job task group %s has count %d. Count cannot exceed 1 with system scheduler", 3451 tg.Name, tg.Count)) 3452 } 3453 } 3454 3455 // Validate the task group 3456 for _, tg := range j.TaskGroups { 3457 if err := tg.Validate(j); err != nil { 3458 outer := fmt.Errorf("Task group %s validation failed: %v", tg.Name, err) 3459 mErr.Errors = append(mErr.Errors, outer) 3460 } 3461 } 3462 3463 // Validate periodic is only used with batch jobs. 3464 if j.IsPeriodic() && j.Periodic.Enabled { 3465 if j.Type != JobTypeBatch { 3466 mErr.Errors = append(mErr.Errors, 3467 fmt.Errorf("Periodic can only be used with %q scheduler", JobTypeBatch)) 3468 } 3469 3470 if err := j.Periodic.Validate(); err != nil { 3471 mErr.Errors = append(mErr.Errors, err) 3472 } 3473 } 3474 3475 if j.IsParameterized() { 3476 if j.Type != JobTypeBatch { 3477 mErr.Errors = append(mErr.Errors, 3478 fmt.Errorf("Parameterized job can only be used with %q scheduler", JobTypeBatch)) 3479 } 3480 3481 if err := j.ParameterizedJob.Validate(); err != nil { 3482 mErr.Errors = append(mErr.Errors, err) 3483 } 3484 } 3485 3486 return mErr.ErrorOrNil() 3487 } 3488 3489 // Warnings returns a list of warnings that may be from dubious settings or 3490 // deprecation warnings. 3491 func (j *Job) Warnings() error { 3492 var mErr multierror.Error 3493 3494 // Check the groups 3495 ap := 0 3496 for _, tg := range j.TaskGroups { 3497 if err := tg.Warnings(j); err != nil { 3498 outer := fmt.Errorf("Group %q has warnings: %v", tg.Name, err) 3499 mErr.Errors = append(mErr.Errors, outer) 3500 } 3501 if tg.Update != nil && tg.Update.AutoPromote { 3502 ap += 1 3503 } 3504 } 3505 3506 // Check AutoPromote, should be all or none 3507 if ap > 0 && ap < len(j.TaskGroups) { 3508 err := fmt.Errorf("auto_promote must be true for all groups to enable automatic promotion") 3509 mErr.Errors = append(mErr.Errors, err) 3510 } 3511 3512 return mErr.ErrorOrNil() 3513 } 3514 3515 // LookupTaskGroup finds a task group by name 3516 func (j *Job) LookupTaskGroup(name string) *TaskGroup { 3517 for _, tg := range j.TaskGroups { 3518 if tg.Name == name { 3519 return tg 3520 } 3521 } 3522 return nil 3523 } 3524 3525 // CombinedTaskMeta takes a TaskGroup and Task name and returns the combined 3526 // meta data for the task. When joining Job, Group and Task Meta, the precedence 3527 // is by deepest scope (Task > Group > Job). 3528 func (j *Job) CombinedTaskMeta(groupName, taskName string) map[string]string { 3529 group := j.LookupTaskGroup(groupName) 3530 if group == nil { 3531 return nil 3532 } 3533 3534 task := group.LookupTask(taskName) 3535 if task == nil { 3536 return nil 3537 } 3538 3539 meta := helper.CopyMapStringString(task.Meta) 3540 if meta == nil { 3541 meta = make(map[string]string, len(group.Meta)+len(j.Meta)) 3542 } 3543 3544 // Add the group specific meta 3545 for k, v := range group.Meta { 3546 if _, ok := meta[k]; !ok { 3547 meta[k] = v 3548 } 3549 } 3550 3551 // Add the job specific meta 3552 for k, v := range j.Meta { 3553 if _, ok := meta[k]; !ok { 3554 meta[k] = v 3555 } 3556 } 3557 3558 return meta 3559 } 3560 3561 // Stopped returns if a job is stopped. 3562 func (j *Job) Stopped() bool { 3563 return j == nil || j.Stop 3564 } 3565 3566 // HasUpdateStrategy returns if any task group in the job has an update strategy 3567 func (j *Job) HasUpdateStrategy() bool { 3568 for _, tg := range j.TaskGroups { 3569 if tg.Update != nil { 3570 return true 3571 } 3572 } 3573 3574 return false 3575 } 3576 3577 // Stub is used to return a summary of the job 3578 func (j *Job) Stub(summary *JobSummary) *JobListStub { 3579 return &JobListStub{ 3580 ID: j.ID, 3581 ParentID: j.ParentID, 3582 Name: j.Name, 3583 Datacenters: j.Datacenters, 3584 Type: j.Type, 3585 Priority: j.Priority, 3586 Periodic: j.IsPeriodic(), 3587 ParameterizedJob: j.IsParameterized(), 3588 Stop: j.Stop, 3589 Status: j.Status, 3590 StatusDescription: j.StatusDescription, 3591 CreateIndex: j.CreateIndex, 3592 ModifyIndex: j.ModifyIndex, 3593 JobModifyIndex: j.JobModifyIndex, 3594 SubmitTime: j.SubmitTime, 3595 JobSummary: summary, 3596 } 3597 } 3598 3599 // IsPeriodic returns whether a job is periodic. 3600 func (j *Job) IsPeriodic() bool { 3601 return j.Periodic != nil 3602 } 3603 3604 // IsPeriodicActive returns whether the job is an active periodic job that will 3605 // create child jobs 3606 func (j *Job) IsPeriodicActive() bool { 3607 return j.IsPeriodic() && j.Periodic.Enabled && !j.Stopped() && !j.IsParameterized() 3608 } 3609 3610 // IsParameterized returns whether a job is parameterized job. 3611 func (j *Job) IsParameterized() bool { 3612 return j.ParameterizedJob != nil && !j.Dispatched 3613 } 3614 3615 // VaultPolicies returns the set of Vault policies per task group, per task 3616 func (j *Job) VaultPolicies() map[string]map[string]*Vault { 3617 policies := make(map[string]map[string]*Vault, len(j.TaskGroups)) 3618 3619 for _, tg := range j.TaskGroups { 3620 tgPolicies := make(map[string]*Vault, len(tg.Tasks)) 3621 3622 for _, task := range tg.Tasks { 3623 if task.Vault == nil { 3624 continue 3625 } 3626 3627 tgPolicies[task.Name] = task.Vault 3628 } 3629 3630 if len(tgPolicies) != 0 { 3631 policies[tg.Name] = tgPolicies 3632 } 3633 } 3634 3635 return policies 3636 } 3637 3638 // RequiredSignals returns a mapping of task groups to tasks to their required 3639 // set of signals 3640 func (j *Job) RequiredSignals() map[string]map[string][]string { 3641 signals := make(map[string]map[string][]string) 3642 3643 for _, tg := range j.TaskGroups { 3644 for _, task := range tg.Tasks { 3645 // Use this local one as a set 3646 taskSignals := make(map[string]struct{}) 3647 3648 // Check if the Vault change mode uses signals 3649 if task.Vault != nil && task.Vault.ChangeMode == VaultChangeModeSignal { 3650 taskSignals[task.Vault.ChangeSignal] = struct{}{} 3651 } 3652 3653 // If a user has specified a KillSignal, add it to required signals 3654 if task.KillSignal != "" { 3655 taskSignals[task.KillSignal] = struct{}{} 3656 } 3657 3658 // Check if any template change mode uses signals 3659 for _, t := range task.Templates { 3660 if t.ChangeMode != TemplateChangeModeSignal { 3661 continue 3662 } 3663 3664 taskSignals[t.ChangeSignal] = struct{}{} 3665 } 3666 3667 // Flatten and sort the signals 3668 l := len(taskSignals) 3669 if l == 0 { 3670 continue 3671 } 3672 3673 flat := make([]string, 0, l) 3674 for sig := range taskSignals { 3675 flat = append(flat, sig) 3676 } 3677 3678 sort.Strings(flat) 3679 tgSignals, ok := signals[tg.Name] 3680 if !ok { 3681 tgSignals = make(map[string][]string) 3682 signals[tg.Name] = tgSignals 3683 } 3684 tgSignals[task.Name] = flat 3685 } 3686 3687 } 3688 3689 return signals 3690 } 3691 3692 // SpecChanged determines if the functional specification has changed between 3693 // two job versions. 3694 func (j *Job) SpecChanged(new *Job) bool { 3695 if j == nil { 3696 return new != nil 3697 } 3698 3699 // Create a copy of the new job 3700 c := new.Copy() 3701 3702 // Update the new job so we can do a reflect 3703 c.Status = j.Status 3704 c.StatusDescription = j.StatusDescription 3705 c.Stable = j.Stable 3706 c.Version = j.Version 3707 c.CreateIndex = j.CreateIndex 3708 c.ModifyIndex = j.ModifyIndex 3709 c.JobModifyIndex = j.JobModifyIndex 3710 c.SubmitTime = j.SubmitTime 3711 3712 // Deep equals the jobs 3713 return !reflect.DeepEqual(j, c) 3714 } 3715 3716 func (j *Job) SetSubmitTime() { 3717 j.SubmitTime = time.Now().UTC().UnixNano() 3718 } 3719 3720 // JobListStub is used to return a subset of job information 3721 // for the job list 3722 type JobListStub struct { 3723 ID string 3724 ParentID string 3725 Name string 3726 Datacenters []string 3727 Type string 3728 Priority int 3729 Periodic bool 3730 ParameterizedJob bool 3731 Stop bool 3732 Status string 3733 StatusDescription string 3734 JobSummary *JobSummary 3735 CreateIndex uint64 3736 ModifyIndex uint64 3737 JobModifyIndex uint64 3738 SubmitTime int64 3739 } 3740 3741 // JobSummary summarizes the state of the allocations of a job 3742 type JobSummary struct { 3743 // JobID is the ID of the job the summary is for 3744 JobID string 3745 3746 // Namespace is the namespace of the job and its summary 3747 Namespace string 3748 3749 // Summary contains the summary per task group for the Job 3750 Summary map[string]TaskGroupSummary 3751 3752 // Children contains a summary for the children of this job. 3753 Children *JobChildrenSummary 3754 3755 // Raft Indexes 3756 CreateIndex uint64 3757 ModifyIndex uint64 3758 } 3759 3760 // Copy returns a new copy of JobSummary 3761 func (js *JobSummary) Copy() *JobSummary { 3762 newJobSummary := new(JobSummary) 3763 *newJobSummary = *js 3764 newTGSummary := make(map[string]TaskGroupSummary, len(js.Summary)) 3765 for k, v := range js.Summary { 3766 newTGSummary[k] = v 3767 } 3768 newJobSummary.Summary = newTGSummary 3769 newJobSummary.Children = newJobSummary.Children.Copy() 3770 return newJobSummary 3771 } 3772 3773 // JobChildrenSummary contains the summary of children job statuses 3774 type JobChildrenSummary struct { 3775 Pending int64 3776 Running int64 3777 Dead int64 3778 } 3779 3780 // Copy returns a new copy of a JobChildrenSummary 3781 func (jc *JobChildrenSummary) Copy() *JobChildrenSummary { 3782 if jc == nil { 3783 return nil 3784 } 3785 3786 njc := new(JobChildrenSummary) 3787 *njc = *jc 3788 return njc 3789 } 3790 3791 // TaskGroup summarizes the state of all the allocations of a particular 3792 // TaskGroup 3793 type TaskGroupSummary struct { 3794 Queued int 3795 Complete int 3796 Failed int 3797 Running int 3798 Starting int 3799 Lost int 3800 } 3801 3802 const ( 3803 // Checks uses any registered health check state in combination with task 3804 // states to determine if a allocation is healthy. 3805 UpdateStrategyHealthCheck_Checks = "checks" 3806 3807 // TaskStates uses the task states of an allocation to determine if the 3808 // allocation is healthy. 3809 UpdateStrategyHealthCheck_TaskStates = "task_states" 3810 3811 // Manual allows the operator to manually signal to Nomad when an 3812 // allocations is healthy. This allows more advanced health checking that is 3813 // outside of the scope of Nomad. 3814 UpdateStrategyHealthCheck_Manual = "manual" 3815 ) 3816 3817 var ( 3818 // DefaultUpdateStrategy provides a baseline that can be used to upgrade 3819 // jobs with the old policy or for populating field defaults. 3820 DefaultUpdateStrategy = &UpdateStrategy{ 3821 Stagger: 30 * time.Second, 3822 MaxParallel: 1, 3823 HealthCheck: UpdateStrategyHealthCheck_Checks, 3824 MinHealthyTime: 10 * time.Second, 3825 HealthyDeadline: 5 * time.Minute, 3826 ProgressDeadline: 10 * time.Minute, 3827 AutoRevert: false, 3828 AutoPromote: false, 3829 Canary: 0, 3830 } 3831 ) 3832 3833 // UpdateStrategy is used to modify how updates are done 3834 type UpdateStrategy struct { 3835 // Stagger is used to determine the rate at which allocations are migrated 3836 // due to down or draining nodes. 3837 Stagger time.Duration 3838 3839 // MaxParallel is how many updates can be done in parallel 3840 MaxParallel int 3841 3842 // HealthCheck specifies the mechanism in which allocations are marked 3843 // healthy or unhealthy as part of a deployment. 3844 HealthCheck string 3845 3846 // MinHealthyTime is the minimum time an allocation must be in the healthy 3847 // state before it is marked as healthy, unblocking more allocations to be 3848 // rolled. 3849 MinHealthyTime time.Duration 3850 3851 // HealthyDeadline is the time in which an allocation must be marked as 3852 // healthy before it is automatically transitioned to unhealthy. This time 3853 // period doesn't count against the MinHealthyTime. 3854 HealthyDeadline time.Duration 3855 3856 // ProgressDeadline is the time in which an allocation as part of the 3857 // deployment must transition to healthy. If no allocation becomes healthy 3858 // after the deadline, the deployment is marked as failed. If the deadline 3859 // is zero, the first failure causes the deployment to fail. 3860 ProgressDeadline time.Duration 3861 3862 // AutoRevert declares that if a deployment fails because of unhealthy 3863 // allocations, there should be an attempt to auto-revert the job to a 3864 // stable version. 3865 AutoRevert bool 3866 3867 // AutoPromote declares that the deployment should be promoted when all canaries are 3868 // healthy 3869 AutoPromote bool 3870 3871 // Canary is the number of canaries to deploy when a change to the task 3872 // group is detected. 3873 Canary int 3874 } 3875 3876 func (u *UpdateStrategy) Copy() *UpdateStrategy { 3877 if u == nil { 3878 return nil 3879 } 3880 3881 copy := new(UpdateStrategy) 3882 *copy = *u 3883 return copy 3884 } 3885 3886 func (u *UpdateStrategy) Validate() error { 3887 if u == nil { 3888 return nil 3889 } 3890 3891 var mErr multierror.Error 3892 switch u.HealthCheck { 3893 case UpdateStrategyHealthCheck_Checks, UpdateStrategyHealthCheck_TaskStates, UpdateStrategyHealthCheck_Manual: 3894 default: 3895 multierror.Append(&mErr, fmt.Errorf("Invalid health check given: %q", u.HealthCheck)) 3896 } 3897 3898 if u.MaxParallel < 1 { 3899 multierror.Append(&mErr, fmt.Errorf("Max parallel can not be less than one: %d < 1", u.MaxParallel)) 3900 } 3901 if u.Canary < 0 { 3902 multierror.Append(&mErr, fmt.Errorf("Canary count can not be less than zero: %d < 0", u.Canary)) 3903 } 3904 if u.Canary == 0 && u.AutoPromote { 3905 multierror.Append(&mErr, fmt.Errorf("Auto Promote requires a Canary count greater than zero")) 3906 } 3907 if u.MinHealthyTime < 0 { 3908 multierror.Append(&mErr, fmt.Errorf("Minimum healthy time may not be less than zero: %v", u.MinHealthyTime)) 3909 } 3910 if u.HealthyDeadline <= 0 { 3911 multierror.Append(&mErr, fmt.Errorf("Healthy deadline must be greater than zero: %v", u.HealthyDeadline)) 3912 } 3913 if u.ProgressDeadline < 0 { 3914 multierror.Append(&mErr, fmt.Errorf("Progress deadline must be zero or greater: %v", u.ProgressDeadline)) 3915 } 3916 if u.MinHealthyTime >= u.HealthyDeadline { 3917 multierror.Append(&mErr, fmt.Errorf("Minimum healthy time must be less than healthy deadline: %v > %v", u.MinHealthyTime, u.HealthyDeadline)) 3918 } 3919 if u.ProgressDeadline != 0 && u.HealthyDeadline >= u.ProgressDeadline { 3920 multierror.Append(&mErr, fmt.Errorf("Healthy deadline must be less than progress deadline: %v > %v", u.HealthyDeadline, u.ProgressDeadline)) 3921 } 3922 if u.Stagger <= 0 { 3923 multierror.Append(&mErr, fmt.Errorf("Stagger must be greater than zero: %v", u.Stagger)) 3924 } 3925 3926 return mErr.ErrorOrNil() 3927 } 3928 3929 // TODO(alexdadgar): Remove once no longer used by the scheduler. 3930 // Rolling returns if a rolling strategy should be used 3931 func (u *UpdateStrategy) Rolling() bool { 3932 return u.Stagger > 0 && u.MaxParallel > 0 3933 } 3934 3935 const ( 3936 // PeriodicSpecCron is used for a cron spec. 3937 PeriodicSpecCron = "cron" 3938 3939 // PeriodicSpecTest is only used by unit tests. It is a sorted, comma 3940 // separated list of unix timestamps at which to launch. 3941 PeriodicSpecTest = "_internal_test" 3942 ) 3943 3944 // Periodic defines the interval a job should be run at. 3945 type PeriodicConfig struct { 3946 // Enabled determines if the job should be run periodically. 3947 Enabled bool 3948 3949 // Spec specifies the interval the job should be run as. It is parsed based 3950 // on the SpecType. 3951 Spec string 3952 3953 // SpecType defines the format of the spec. 3954 SpecType string 3955 3956 // ProhibitOverlap enforces that spawned jobs do not run in parallel. 3957 ProhibitOverlap bool 3958 3959 // TimeZone is the user specified string that determines the time zone to 3960 // launch against. The time zones must be specified from IANA Time Zone 3961 // database, such as "America/New_York". 3962 // Reference: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones 3963 // Reference: https://www.iana.org/time-zones 3964 TimeZone string 3965 3966 // location is the time zone to evaluate the launch time against 3967 location *time.Location 3968 } 3969 3970 func (p *PeriodicConfig) Copy() *PeriodicConfig { 3971 if p == nil { 3972 return nil 3973 } 3974 np := new(PeriodicConfig) 3975 *np = *p 3976 return np 3977 } 3978 3979 func (p *PeriodicConfig) Validate() error { 3980 if !p.Enabled { 3981 return nil 3982 } 3983 3984 var mErr multierror.Error 3985 if p.Spec == "" { 3986 multierror.Append(&mErr, fmt.Errorf("Must specify a spec")) 3987 } 3988 3989 // Check if we got a valid time zone 3990 if p.TimeZone != "" { 3991 if _, err := time.LoadLocation(p.TimeZone); err != nil { 3992 multierror.Append(&mErr, fmt.Errorf("Invalid time zone %q: %v", p.TimeZone, err)) 3993 } 3994 } 3995 3996 switch p.SpecType { 3997 case PeriodicSpecCron: 3998 // Validate the cron spec 3999 if _, err := cronexpr.Parse(p.Spec); err != nil { 4000 multierror.Append(&mErr, fmt.Errorf("Invalid cron spec %q: %v", p.Spec, err)) 4001 } 4002 case PeriodicSpecTest: 4003 // No-op 4004 default: 4005 multierror.Append(&mErr, fmt.Errorf("Unknown periodic specification type %q", p.SpecType)) 4006 } 4007 4008 return mErr.ErrorOrNil() 4009 } 4010 4011 func (p *PeriodicConfig) Canonicalize() { 4012 // Load the location 4013 l, err := time.LoadLocation(p.TimeZone) 4014 if err != nil { 4015 p.location = time.UTC 4016 } 4017 4018 p.location = l 4019 } 4020 4021 // CronParseNext is a helper that parses the next time for the given expression 4022 // but captures any panic that may occur in the underlying library. 4023 func CronParseNext(e *cronexpr.Expression, fromTime time.Time, spec string) (t time.Time, err error) { 4024 defer func() { 4025 if recover() != nil { 4026 t = time.Time{} 4027 err = fmt.Errorf("failed parsing cron expression: %q", spec) 4028 } 4029 }() 4030 4031 return e.Next(fromTime), nil 4032 } 4033 4034 // Next returns the closest time instant matching the spec that is after the 4035 // passed time. If no matching instance exists, the zero value of time.Time is 4036 // returned. The `time.Location` of the returned value matches that of the 4037 // passed time. 4038 func (p *PeriodicConfig) Next(fromTime time.Time) (time.Time, error) { 4039 switch p.SpecType { 4040 case PeriodicSpecCron: 4041 if e, err := cronexpr.Parse(p.Spec); err == nil { 4042 return CronParseNext(e, fromTime, p.Spec) 4043 } 4044 case PeriodicSpecTest: 4045 split := strings.Split(p.Spec, ",") 4046 if len(split) == 1 && split[0] == "" { 4047 return time.Time{}, nil 4048 } 4049 4050 // Parse the times 4051 times := make([]time.Time, len(split)) 4052 for i, s := range split { 4053 unix, err := strconv.Atoi(s) 4054 if err != nil { 4055 return time.Time{}, nil 4056 } 4057 4058 times[i] = time.Unix(int64(unix), 0) 4059 } 4060 4061 // Find the next match 4062 for _, next := range times { 4063 if fromTime.Before(next) { 4064 return next, nil 4065 } 4066 } 4067 } 4068 4069 return time.Time{}, nil 4070 } 4071 4072 // GetLocation returns the location to use for determining the time zone to run 4073 // the periodic job against. 4074 func (p *PeriodicConfig) GetLocation() *time.Location { 4075 // Jobs pre 0.5.5 will not have this 4076 if p.location != nil { 4077 return p.location 4078 } 4079 4080 return time.UTC 4081 } 4082 4083 const ( 4084 // PeriodicLaunchSuffix is the string appended to the periodic jobs ID 4085 // when launching derived instances of it. 4086 PeriodicLaunchSuffix = "/periodic-" 4087 ) 4088 4089 // PeriodicLaunch tracks the last launch time of a periodic job. 4090 type PeriodicLaunch struct { 4091 ID string // ID of the periodic job. 4092 Namespace string // Namespace of the periodic job 4093 Launch time.Time // The last launch time. 4094 4095 // Raft Indexes 4096 CreateIndex uint64 4097 ModifyIndex uint64 4098 } 4099 4100 const ( 4101 DispatchPayloadForbidden = "forbidden" 4102 DispatchPayloadOptional = "optional" 4103 DispatchPayloadRequired = "required" 4104 4105 // DispatchLaunchSuffix is the string appended to the parameterized job's ID 4106 // when dispatching instances of it. 4107 DispatchLaunchSuffix = "/dispatch-" 4108 ) 4109 4110 // ParameterizedJobConfig is used to configure the parameterized job 4111 type ParameterizedJobConfig struct { 4112 // Payload configure the payload requirements 4113 Payload string 4114 4115 // MetaRequired is metadata keys that must be specified by the dispatcher 4116 MetaRequired []string 4117 4118 // MetaOptional is metadata keys that may be specified by the dispatcher 4119 MetaOptional []string 4120 } 4121 4122 func (d *ParameterizedJobConfig) Validate() error { 4123 var mErr multierror.Error 4124 switch d.Payload { 4125 case DispatchPayloadOptional, DispatchPayloadRequired, DispatchPayloadForbidden: 4126 default: 4127 multierror.Append(&mErr, fmt.Errorf("Unknown payload requirement: %q", d.Payload)) 4128 } 4129 4130 // Check that the meta configurations are disjoint sets 4131 disjoint, offending := helper.SliceSetDisjoint(d.MetaRequired, d.MetaOptional) 4132 if !disjoint { 4133 multierror.Append(&mErr, fmt.Errorf("Required and optional meta keys should be disjoint. Following keys exist in both: %v", offending)) 4134 } 4135 4136 return mErr.ErrorOrNil() 4137 } 4138 4139 func (d *ParameterizedJobConfig) Canonicalize() { 4140 if d.Payload == "" { 4141 d.Payload = DispatchPayloadOptional 4142 } 4143 } 4144 4145 func (d *ParameterizedJobConfig) Copy() *ParameterizedJobConfig { 4146 if d == nil { 4147 return nil 4148 } 4149 nd := new(ParameterizedJobConfig) 4150 *nd = *d 4151 nd.MetaOptional = helper.CopySliceString(nd.MetaOptional) 4152 nd.MetaRequired = helper.CopySliceString(nd.MetaRequired) 4153 return nd 4154 } 4155 4156 // DispatchedID returns an ID appropriate for a job dispatched against a 4157 // particular parameterized job 4158 func DispatchedID(templateID string, t time.Time) string { 4159 u := uuid.Generate()[:8] 4160 return fmt.Sprintf("%s%s%d-%s", templateID, DispatchLaunchSuffix, t.Unix(), u) 4161 } 4162 4163 // DispatchPayloadConfig configures how a task gets its input from a job dispatch 4164 type DispatchPayloadConfig struct { 4165 // File specifies a relative path to where the input data should be written 4166 File string 4167 } 4168 4169 func (d *DispatchPayloadConfig) Copy() *DispatchPayloadConfig { 4170 if d == nil { 4171 return nil 4172 } 4173 nd := new(DispatchPayloadConfig) 4174 *nd = *d 4175 return nd 4176 } 4177 4178 func (d *DispatchPayloadConfig) Validate() error { 4179 // Verify the destination doesn't escape 4180 escaped, err := PathEscapesAllocDir("task/local/", d.File) 4181 if err != nil { 4182 return fmt.Errorf("invalid destination path: %v", err) 4183 } else if escaped { 4184 return fmt.Errorf("destination escapes allocation directory") 4185 } 4186 4187 return nil 4188 } 4189 4190 var ( 4191 // These default restart policies needs to be in sync with 4192 // Canonicalize in api/tasks.go 4193 4194 DefaultServiceJobRestartPolicy = RestartPolicy{ 4195 Delay: 15 * time.Second, 4196 Attempts: 2, 4197 Interval: 30 * time.Minute, 4198 Mode: RestartPolicyModeFail, 4199 } 4200 DefaultBatchJobRestartPolicy = RestartPolicy{ 4201 Delay: 15 * time.Second, 4202 Attempts: 3, 4203 Interval: 24 * time.Hour, 4204 Mode: RestartPolicyModeFail, 4205 } 4206 ) 4207 4208 var ( 4209 // These default reschedule policies needs to be in sync with 4210 // NewDefaultReschedulePolicy in api/tasks.go 4211 4212 DefaultServiceJobReschedulePolicy = ReschedulePolicy{ 4213 Delay: 30 * time.Second, 4214 DelayFunction: "exponential", 4215 MaxDelay: 1 * time.Hour, 4216 Unlimited: true, 4217 } 4218 DefaultBatchJobReschedulePolicy = ReschedulePolicy{ 4219 Attempts: 1, 4220 Interval: 24 * time.Hour, 4221 Delay: 5 * time.Second, 4222 DelayFunction: "constant", 4223 } 4224 ) 4225 4226 const ( 4227 // RestartPolicyModeDelay causes an artificial delay till the next interval is 4228 // reached when the specified attempts have been reached in the interval. 4229 RestartPolicyModeDelay = "delay" 4230 4231 // RestartPolicyModeFail causes a job to fail if the specified number of 4232 // attempts are reached within an interval. 4233 RestartPolicyModeFail = "fail" 4234 4235 // RestartPolicyMinInterval is the minimum interval that is accepted for a 4236 // restart policy. 4237 RestartPolicyMinInterval = 5 * time.Second 4238 4239 // ReasonWithinPolicy describes restart events that are within policy 4240 ReasonWithinPolicy = "Restart within policy" 4241 ) 4242 4243 // RestartPolicy configures how Tasks are restarted when they crash or fail. 4244 type RestartPolicy struct { 4245 // Attempts is the number of restart that will occur in an interval. 4246 Attempts int 4247 4248 // Interval is a duration in which we can limit the number of restarts 4249 // within. 4250 Interval time.Duration 4251 4252 // Delay is the time between a failure and a restart. 4253 Delay time.Duration 4254 4255 // Mode controls what happens when the task restarts more than attempt times 4256 // in an interval. 4257 Mode string 4258 } 4259 4260 func (r *RestartPolicy) Copy() *RestartPolicy { 4261 if r == nil { 4262 return nil 4263 } 4264 nrp := new(RestartPolicy) 4265 *nrp = *r 4266 return nrp 4267 } 4268 4269 func (r *RestartPolicy) Validate() error { 4270 var mErr multierror.Error 4271 switch r.Mode { 4272 case RestartPolicyModeDelay, RestartPolicyModeFail: 4273 default: 4274 multierror.Append(&mErr, fmt.Errorf("Unsupported restart mode: %q", r.Mode)) 4275 } 4276 4277 // Check for ambiguous/confusing settings 4278 if r.Attempts == 0 && r.Mode != RestartPolicyModeFail { 4279 multierror.Append(&mErr, fmt.Errorf("Restart policy %q with %d attempts is ambiguous", r.Mode, r.Attempts)) 4280 } 4281 4282 if r.Interval.Nanoseconds() < RestartPolicyMinInterval.Nanoseconds() { 4283 multierror.Append(&mErr, fmt.Errorf("Interval can not be less than %v (got %v)", RestartPolicyMinInterval, r.Interval)) 4284 } 4285 if time.Duration(r.Attempts)*r.Delay > r.Interval { 4286 multierror.Append(&mErr, 4287 fmt.Errorf("Nomad can't restart the TaskGroup %v times in an interval of %v with a delay of %v", r.Attempts, r.Interval, r.Delay)) 4288 } 4289 return mErr.ErrorOrNil() 4290 } 4291 4292 func NewRestartPolicy(jobType string) *RestartPolicy { 4293 switch jobType { 4294 case JobTypeService, JobTypeSystem: 4295 rp := DefaultServiceJobRestartPolicy 4296 return &rp 4297 case JobTypeBatch: 4298 rp := DefaultBatchJobRestartPolicy 4299 return &rp 4300 } 4301 return nil 4302 } 4303 4304 const ReschedulePolicyMinInterval = 15 * time.Second 4305 const ReschedulePolicyMinDelay = 5 * time.Second 4306 4307 var RescheduleDelayFunctions = [...]string{"constant", "exponential", "fibonacci"} 4308 4309 // ReschedulePolicy configures how Tasks are rescheduled when they crash or fail. 4310 type ReschedulePolicy struct { 4311 // Attempts limits the number of rescheduling attempts that can occur in an interval. 4312 Attempts int 4313 4314 // Interval is a duration in which we can limit the number of reschedule attempts. 4315 Interval time.Duration 4316 4317 // Delay is a minimum duration to wait between reschedule attempts. 4318 // The delay function determines how much subsequent reschedule attempts are delayed by. 4319 Delay time.Duration 4320 4321 // DelayFunction determines how the delay progressively changes on subsequent reschedule 4322 // attempts. Valid values are "exponential", "constant", and "fibonacci". 4323 DelayFunction string 4324 4325 // MaxDelay is an upper bound on the delay. 4326 MaxDelay time.Duration 4327 4328 // Unlimited allows infinite rescheduling attempts. Only allowed when delay is set 4329 // between reschedule attempts. 4330 Unlimited bool 4331 } 4332 4333 func (r *ReschedulePolicy) Copy() *ReschedulePolicy { 4334 if r == nil { 4335 return nil 4336 } 4337 nrp := new(ReschedulePolicy) 4338 *nrp = *r 4339 return nrp 4340 } 4341 4342 func (r *ReschedulePolicy) Enabled() bool { 4343 enabled := r != nil && (r.Attempts > 0 || r.Unlimited) 4344 return enabled 4345 } 4346 4347 // Validate uses different criteria to validate the reschedule policy 4348 // Delay must be a minimum of 5 seconds 4349 // Delay Ceiling is ignored if Delay Function is "constant" 4350 // Number of possible attempts is validated, given the interval, delay and delay function 4351 func (r *ReschedulePolicy) Validate() error { 4352 if !r.Enabled() { 4353 return nil 4354 } 4355 var mErr multierror.Error 4356 // Check for ambiguous/confusing settings 4357 if r.Attempts > 0 { 4358 if r.Interval <= 0 { 4359 multierror.Append(&mErr, fmt.Errorf("Interval must be a non zero value if Attempts > 0")) 4360 } 4361 if r.Unlimited { 4362 multierror.Append(&mErr, fmt.Errorf("Reschedule Policy with Attempts = %v, Interval = %v, "+ 4363 "and Unlimited = %v is ambiguous", r.Attempts, r.Interval, r.Unlimited)) 4364 multierror.Append(&mErr, errors.New("If Attempts >0, Unlimited cannot also be set to true")) 4365 } 4366 } 4367 4368 delayPreCheck := true 4369 // Delay should be bigger than the default 4370 if r.Delay.Nanoseconds() < ReschedulePolicyMinDelay.Nanoseconds() { 4371 multierror.Append(&mErr, fmt.Errorf("Delay cannot be less than %v (got %v)", ReschedulePolicyMinDelay, r.Delay)) 4372 delayPreCheck = false 4373 } 4374 4375 // Must use a valid delay function 4376 if !isValidDelayFunction(r.DelayFunction) { 4377 multierror.Append(&mErr, fmt.Errorf("Invalid delay function %q, must be one of %q", r.DelayFunction, RescheduleDelayFunctions)) 4378 delayPreCheck = false 4379 } 4380 4381 // Validate MaxDelay if not using linear delay progression 4382 if r.DelayFunction != "constant" { 4383 if r.MaxDelay.Nanoseconds() < ReschedulePolicyMinDelay.Nanoseconds() { 4384 multierror.Append(&mErr, fmt.Errorf("Max Delay cannot be less than %v (got %v)", ReschedulePolicyMinDelay, r.Delay)) 4385 delayPreCheck = false 4386 } 4387 if r.MaxDelay < r.Delay { 4388 multierror.Append(&mErr, fmt.Errorf("Max Delay cannot be less than Delay %v (got %v)", r.Delay, r.MaxDelay)) 4389 delayPreCheck = false 4390 } 4391 4392 } 4393 4394 // Validate Interval and other delay parameters if attempts are limited 4395 if !r.Unlimited { 4396 if r.Interval.Nanoseconds() < ReschedulePolicyMinInterval.Nanoseconds() { 4397 multierror.Append(&mErr, fmt.Errorf("Interval cannot be less than %v (got %v)", ReschedulePolicyMinInterval, r.Interval)) 4398 } 4399 if !delayPreCheck { 4400 // We can't cross validate the rest of the delay params if delayPreCheck fails, so return early 4401 return mErr.ErrorOrNil() 4402 } 4403 crossValidationErr := r.validateDelayParams() 4404 if crossValidationErr != nil { 4405 multierror.Append(&mErr, crossValidationErr) 4406 } 4407 } 4408 return mErr.ErrorOrNil() 4409 } 4410 4411 func isValidDelayFunction(delayFunc string) bool { 4412 for _, value := range RescheduleDelayFunctions { 4413 if value == delayFunc { 4414 return true 4415 } 4416 } 4417 return false 4418 } 4419 4420 func (r *ReschedulePolicy) validateDelayParams() error { 4421 ok, possibleAttempts, recommendedInterval := r.viableAttempts() 4422 if ok { 4423 return nil 4424 } 4425 var mErr multierror.Error 4426 if r.DelayFunction == "constant" { 4427 multierror.Append(&mErr, fmt.Errorf("Nomad can only make %v attempts in %v with initial delay %v and "+ 4428 "delay function %q", possibleAttempts, r.Interval, r.Delay, r.DelayFunction)) 4429 } else { 4430 multierror.Append(&mErr, fmt.Errorf("Nomad can only make %v attempts in %v with initial delay %v, "+ 4431 "delay function %q, and delay ceiling %v", possibleAttempts, r.Interval, r.Delay, r.DelayFunction, r.MaxDelay)) 4432 } 4433 multierror.Append(&mErr, fmt.Errorf("Set the interval to at least %v to accommodate %v attempts", recommendedInterval.Round(time.Second), r.Attempts)) 4434 return mErr.ErrorOrNil() 4435 } 4436 4437 func (r *ReschedulePolicy) viableAttempts() (bool, int, time.Duration) { 4438 var possibleAttempts int 4439 var recommendedInterval time.Duration 4440 valid := true 4441 switch r.DelayFunction { 4442 case "constant": 4443 recommendedInterval = time.Duration(r.Attempts) * r.Delay 4444 if r.Interval < recommendedInterval { 4445 possibleAttempts = int(r.Interval / r.Delay) 4446 valid = false 4447 } 4448 case "exponential": 4449 for i := 0; i < r.Attempts; i++ { 4450 nextDelay := time.Duration(math.Pow(2, float64(i))) * r.Delay 4451 if nextDelay > r.MaxDelay { 4452 nextDelay = r.MaxDelay 4453 recommendedInterval += nextDelay 4454 } else { 4455 recommendedInterval = nextDelay 4456 } 4457 if recommendedInterval < r.Interval { 4458 possibleAttempts++ 4459 } 4460 } 4461 if possibleAttempts < r.Attempts { 4462 valid = false 4463 } 4464 case "fibonacci": 4465 var slots []time.Duration 4466 slots = append(slots, r.Delay) 4467 slots = append(slots, r.Delay) 4468 reachedCeiling := false 4469 for i := 2; i < r.Attempts; i++ { 4470 var nextDelay time.Duration 4471 if reachedCeiling { 4472 //switch to linear 4473 nextDelay = slots[i-1] + r.MaxDelay 4474 } else { 4475 nextDelay = slots[i-1] + slots[i-2] 4476 if nextDelay > r.MaxDelay { 4477 nextDelay = r.MaxDelay 4478 reachedCeiling = true 4479 } 4480 } 4481 slots = append(slots, nextDelay) 4482 } 4483 recommendedInterval = slots[len(slots)-1] 4484 if r.Interval < recommendedInterval { 4485 valid = false 4486 // calculate possible attempts 4487 for i := 0; i < len(slots); i++ { 4488 if slots[i] > r.Interval { 4489 possibleAttempts = i 4490 break 4491 } 4492 } 4493 } 4494 default: 4495 return false, 0, 0 4496 } 4497 if possibleAttempts < 0 { // can happen if delay is bigger than interval 4498 possibleAttempts = 0 4499 } 4500 return valid, possibleAttempts, recommendedInterval 4501 } 4502 4503 func NewReschedulePolicy(jobType string) *ReschedulePolicy { 4504 switch jobType { 4505 case JobTypeService: 4506 rp := DefaultServiceJobReschedulePolicy 4507 return &rp 4508 case JobTypeBatch: 4509 rp := DefaultBatchJobReschedulePolicy 4510 return &rp 4511 } 4512 return nil 4513 } 4514 4515 const ( 4516 MigrateStrategyHealthChecks = "checks" 4517 MigrateStrategyHealthStates = "task_states" 4518 ) 4519 4520 type MigrateStrategy struct { 4521 MaxParallel int 4522 HealthCheck string 4523 MinHealthyTime time.Duration 4524 HealthyDeadline time.Duration 4525 } 4526 4527 // DefaultMigrateStrategy is used for backwards compat with pre-0.8 Allocations 4528 // that lack an update strategy. 4529 // 4530 // This function should match its counterpart in api/tasks.go 4531 func DefaultMigrateStrategy() *MigrateStrategy { 4532 return &MigrateStrategy{ 4533 MaxParallel: 1, 4534 HealthCheck: MigrateStrategyHealthChecks, 4535 MinHealthyTime: 10 * time.Second, 4536 HealthyDeadline: 5 * time.Minute, 4537 } 4538 } 4539 4540 func (m *MigrateStrategy) Validate() error { 4541 var mErr multierror.Error 4542 4543 if m.MaxParallel < 0 { 4544 multierror.Append(&mErr, fmt.Errorf("MaxParallel must be >= 0 but found %d", m.MaxParallel)) 4545 } 4546 4547 switch m.HealthCheck { 4548 case MigrateStrategyHealthChecks, MigrateStrategyHealthStates: 4549 // ok 4550 case "": 4551 if m.MaxParallel > 0 { 4552 multierror.Append(&mErr, fmt.Errorf("Missing HealthCheck")) 4553 } 4554 default: 4555 multierror.Append(&mErr, fmt.Errorf("Invalid HealthCheck: %q", m.HealthCheck)) 4556 } 4557 4558 if m.MinHealthyTime < 0 { 4559 multierror.Append(&mErr, fmt.Errorf("MinHealthyTime is %s and must be >= 0", m.MinHealthyTime)) 4560 } 4561 4562 if m.HealthyDeadline < 0 { 4563 multierror.Append(&mErr, fmt.Errorf("HealthyDeadline is %s and must be >= 0", m.HealthyDeadline)) 4564 } 4565 4566 if m.MinHealthyTime > m.HealthyDeadline { 4567 multierror.Append(&mErr, fmt.Errorf("MinHealthyTime must be less than HealthyDeadline")) 4568 } 4569 4570 return mErr.ErrorOrNil() 4571 } 4572 4573 // TaskGroup is an atomic unit of placement. Each task group belongs to 4574 // a job and may contain any number of tasks. A task group support running 4575 // in many replicas using the same configuration.. 4576 type TaskGroup struct { 4577 // Name of the task group 4578 Name string 4579 4580 // Count is the number of replicas of this task group that should 4581 // be scheduled. 4582 Count int 4583 4584 // Update is used to control the update strategy for this task group 4585 Update *UpdateStrategy 4586 4587 // Migrate is used to control the migration strategy for this task group 4588 Migrate *MigrateStrategy 4589 4590 // Constraints can be specified at a task group level and apply to 4591 // all the tasks contained. 4592 Constraints []*Constraint 4593 4594 //RestartPolicy of a TaskGroup 4595 RestartPolicy *RestartPolicy 4596 4597 // Tasks are the collection of tasks that this task group needs to run 4598 Tasks []*Task 4599 4600 // EphemeralDisk is the disk resources that the task group requests 4601 EphemeralDisk *EphemeralDisk 4602 4603 // Meta is used to associate arbitrary metadata with this 4604 // task group. This is opaque to Nomad. 4605 Meta map[string]string 4606 4607 // ReschedulePolicy is used to configure how the scheduler should 4608 // retry failed allocations. 4609 ReschedulePolicy *ReschedulePolicy 4610 4611 // Affinities can be specified at the task group level to express 4612 // scheduling preferences. 4613 Affinities []*Affinity 4614 4615 // Spread can be specified at the task group level to express spreading 4616 // allocations across a desired attribute, such as datacenter 4617 Spreads []*Spread 4618 } 4619 4620 func (tg *TaskGroup) Copy() *TaskGroup { 4621 if tg == nil { 4622 return nil 4623 } 4624 ntg := new(TaskGroup) 4625 *ntg = *tg 4626 ntg.Update = ntg.Update.Copy() 4627 ntg.Constraints = CopySliceConstraints(ntg.Constraints) 4628 ntg.RestartPolicy = ntg.RestartPolicy.Copy() 4629 ntg.ReschedulePolicy = ntg.ReschedulePolicy.Copy() 4630 ntg.Affinities = CopySliceAffinities(ntg.Affinities) 4631 ntg.Spreads = CopySliceSpreads(ntg.Spreads) 4632 4633 if tg.Tasks != nil { 4634 tasks := make([]*Task, len(ntg.Tasks)) 4635 for i, t := range ntg.Tasks { 4636 tasks[i] = t.Copy() 4637 } 4638 ntg.Tasks = tasks 4639 } 4640 4641 ntg.Meta = helper.CopyMapStringString(ntg.Meta) 4642 4643 if tg.EphemeralDisk != nil { 4644 ntg.EphemeralDisk = tg.EphemeralDisk.Copy() 4645 } 4646 return ntg 4647 } 4648 4649 // Canonicalize is used to canonicalize fields in the TaskGroup. 4650 func (tg *TaskGroup) Canonicalize(job *Job) { 4651 // Ensure that an empty and nil map are treated the same to avoid scheduling 4652 // problems since we use reflect DeepEquals. 4653 if len(tg.Meta) == 0 { 4654 tg.Meta = nil 4655 } 4656 4657 // Set the default restart policy. 4658 if tg.RestartPolicy == nil { 4659 tg.RestartPolicy = NewRestartPolicy(job.Type) 4660 } 4661 4662 if tg.ReschedulePolicy == nil { 4663 tg.ReschedulePolicy = NewReschedulePolicy(job.Type) 4664 } 4665 4666 // Canonicalize Migrate for service jobs 4667 if job.Type == JobTypeService && tg.Migrate == nil { 4668 tg.Migrate = DefaultMigrateStrategy() 4669 } 4670 4671 // Set a default ephemeral disk object if the user has not requested for one 4672 if tg.EphemeralDisk == nil { 4673 tg.EphemeralDisk = DefaultEphemeralDisk() 4674 } 4675 4676 for _, task := range tg.Tasks { 4677 task.Canonicalize(job, tg) 4678 } 4679 } 4680 4681 // Validate is used to sanity check a task group 4682 func (tg *TaskGroup) Validate(j *Job) error { 4683 var mErr multierror.Error 4684 if tg.Name == "" { 4685 mErr.Errors = append(mErr.Errors, errors.New("Missing task group name")) 4686 } 4687 if tg.Count < 0 { 4688 mErr.Errors = append(mErr.Errors, errors.New("Task group count can't be negative")) 4689 } 4690 if len(tg.Tasks) == 0 { 4691 mErr.Errors = append(mErr.Errors, errors.New("Missing tasks for task group")) 4692 } 4693 for idx, constr := range tg.Constraints { 4694 if err := constr.Validate(); err != nil { 4695 outer := fmt.Errorf("Constraint %d validation failed: %s", idx+1, err) 4696 mErr.Errors = append(mErr.Errors, outer) 4697 } 4698 } 4699 if j.Type == JobTypeSystem { 4700 if tg.Affinities != nil { 4701 mErr.Errors = append(mErr.Errors, fmt.Errorf("System jobs may not have an affinity stanza")) 4702 } 4703 } else { 4704 for idx, affinity := range tg.Affinities { 4705 if err := affinity.Validate(); err != nil { 4706 outer := fmt.Errorf("Affinity %d validation failed: %s", idx+1, err) 4707 mErr.Errors = append(mErr.Errors, outer) 4708 } 4709 } 4710 } 4711 4712 if tg.RestartPolicy != nil { 4713 if err := tg.RestartPolicy.Validate(); err != nil { 4714 mErr.Errors = append(mErr.Errors, err) 4715 } 4716 } else { 4717 mErr.Errors = append(mErr.Errors, fmt.Errorf("Task Group %v should have a restart policy", tg.Name)) 4718 } 4719 4720 if j.Type == JobTypeSystem { 4721 if tg.Spreads != nil { 4722 mErr.Errors = append(mErr.Errors, fmt.Errorf("System jobs may not have a spread stanza")) 4723 } 4724 } else { 4725 for idx, spread := range tg.Spreads { 4726 if err := spread.Validate(); err != nil { 4727 outer := fmt.Errorf("Spread %d validation failed: %s", idx+1, err) 4728 mErr.Errors = append(mErr.Errors, outer) 4729 } 4730 } 4731 } 4732 4733 if j.Type == JobTypeSystem { 4734 if tg.ReschedulePolicy != nil { 4735 mErr.Errors = append(mErr.Errors, fmt.Errorf("System jobs should not have a reschedule policy")) 4736 } 4737 } else { 4738 if tg.ReschedulePolicy != nil { 4739 if err := tg.ReschedulePolicy.Validate(); err != nil { 4740 mErr.Errors = append(mErr.Errors, err) 4741 } 4742 } else { 4743 mErr.Errors = append(mErr.Errors, fmt.Errorf("Task Group %v should have a reschedule policy", tg.Name)) 4744 } 4745 } 4746 4747 if tg.EphemeralDisk != nil { 4748 if err := tg.EphemeralDisk.Validate(); err != nil { 4749 mErr.Errors = append(mErr.Errors, err) 4750 } 4751 } else { 4752 mErr.Errors = append(mErr.Errors, fmt.Errorf("Task Group %v should have an ephemeral disk object", tg.Name)) 4753 } 4754 4755 // Validate the update strategy 4756 if u := tg.Update; u != nil { 4757 switch j.Type { 4758 case JobTypeService, JobTypeSystem: 4759 default: 4760 mErr.Errors = append(mErr.Errors, fmt.Errorf("Job type %q does not allow update block", j.Type)) 4761 } 4762 if err := u.Validate(); err != nil { 4763 mErr.Errors = append(mErr.Errors, err) 4764 } 4765 } 4766 4767 // Validate the migration strategy 4768 switch j.Type { 4769 case JobTypeService: 4770 if tg.Migrate != nil { 4771 if err := tg.Migrate.Validate(); err != nil { 4772 mErr.Errors = append(mErr.Errors, err) 4773 } 4774 } 4775 default: 4776 if tg.Migrate != nil { 4777 mErr.Errors = append(mErr.Errors, fmt.Errorf("Job type %q does not allow migrate block", j.Type)) 4778 } 4779 } 4780 4781 // Check for duplicate tasks, that there is only leader task if any, 4782 // and no duplicated static ports 4783 tasks := make(map[string]int) 4784 staticPorts := make(map[int]string) 4785 leaderTasks := 0 4786 for idx, task := range tg.Tasks { 4787 if task.Name == "" { 4788 mErr.Errors = append(mErr.Errors, fmt.Errorf("Task %d missing name", idx+1)) 4789 } else if existing, ok := tasks[task.Name]; ok { 4790 mErr.Errors = append(mErr.Errors, fmt.Errorf("Task %d redefines '%s' from task %d", idx+1, task.Name, existing+1)) 4791 } else { 4792 tasks[task.Name] = idx 4793 } 4794 4795 if task.Leader { 4796 leaderTasks++ 4797 } 4798 4799 if task.Resources == nil { 4800 continue 4801 } 4802 4803 for _, net := range task.Resources.Networks { 4804 for _, port := range net.ReservedPorts { 4805 if other, ok := staticPorts[port.Value]; ok { 4806 err := fmt.Errorf("Static port %d already reserved by %s", port.Value, other) 4807 mErr.Errors = append(mErr.Errors, err) 4808 } else { 4809 staticPorts[port.Value] = fmt.Sprintf("%s:%s", task.Name, port.Label) 4810 } 4811 } 4812 } 4813 } 4814 4815 if leaderTasks > 1 { 4816 mErr.Errors = append(mErr.Errors, fmt.Errorf("Only one task may be marked as leader")) 4817 } 4818 4819 // Validate the tasks 4820 for _, task := range tg.Tasks { 4821 if err := task.Validate(tg.EphemeralDisk, j.Type); err != nil { 4822 outer := fmt.Errorf("Task %s validation failed: %v", task.Name, err) 4823 mErr.Errors = append(mErr.Errors, outer) 4824 } 4825 } 4826 return mErr.ErrorOrNil() 4827 } 4828 4829 // Warnings returns a list of warnings that may be from dubious settings or 4830 // deprecation warnings. 4831 func (tg *TaskGroup) Warnings(j *Job) error { 4832 var mErr multierror.Error 4833 4834 // Validate the update strategy 4835 if u := tg.Update; u != nil { 4836 // Check the counts are appropriate 4837 if u.MaxParallel > tg.Count { 4838 mErr.Errors = append(mErr.Errors, 4839 fmt.Errorf("Update max parallel count is greater than task group count (%d > %d). "+ 4840 "A destructive change would result in the simultaneous replacement of all allocations.", u.MaxParallel, tg.Count)) 4841 } 4842 } 4843 4844 for _, t := range tg.Tasks { 4845 if err := t.Warnings(); err != nil { 4846 err = multierror.Prefix(err, fmt.Sprintf("Task %q:", t.Name)) 4847 mErr.Errors = append(mErr.Errors, err) 4848 } 4849 } 4850 4851 return mErr.ErrorOrNil() 4852 } 4853 4854 // LookupTask finds a task by name 4855 func (tg *TaskGroup) LookupTask(name string) *Task { 4856 for _, t := range tg.Tasks { 4857 if t.Name == name { 4858 return t 4859 } 4860 } 4861 return nil 4862 } 4863 4864 func (tg *TaskGroup) GoString() string { 4865 return fmt.Sprintf("*%#v", *tg) 4866 } 4867 4868 // CheckRestart describes if and when a task should be restarted based on 4869 // failing health checks. 4870 type CheckRestart struct { 4871 Limit int // Restart task after this many unhealthy intervals 4872 Grace time.Duration // Grace time to give tasks after starting to get healthy 4873 IgnoreWarnings bool // If true treat checks in `warning` as passing 4874 } 4875 4876 func (c *CheckRestart) Copy() *CheckRestart { 4877 if c == nil { 4878 return nil 4879 } 4880 4881 nc := new(CheckRestart) 4882 *nc = *c 4883 return nc 4884 } 4885 4886 func (c *CheckRestart) Validate() error { 4887 if c == nil { 4888 return nil 4889 } 4890 4891 var mErr multierror.Error 4892 if c.Limit < 0 { 4893 mErr.Errors = append(mErr.Errors, fmt.Errorf("limit must be greater than or equal to 0 but found %d", c.Limit)) 4894 } 4895 4896 if c.Grace < 0 { 4897 mErr.Errors = append(mErr.Errors, fmt.Errorf("grace period must be greater than or equal to 0 but found %d", c.Grace)) 4898 } 4899 4900 return mErr.ErrorOrNil() 4901 } 4902 4903 const ( 4904 ServiceCheckHTTP = "http" 4905 ServiceCheckTCP = "tcp" 4906 ServiceCheckScript = "script" 4907 ServiceCheckGRPC = "grpc" 4908 4909 // minCheckInterval is the minimum check interval permitted. Consul 4910 // currently has its MinInterval set to 1s. Mirror that here for 4911 // consistency. 4912 minCheckInterval = 1 * time.Second 4913 4914 // minCheckTimeout is the minimum check timeout permitted for Consul 4915 // script TTL checks. 4916 minCheckTimeout = 1 * time.Second 4917 ) 4918 4919 // The ServiceCheck data model represents the consul health check that 4920 // Nomad registers for a Task 4921 type ServiceCheck struct { 4922 Name string // Name of the check, defaults to id 4923 Type string // Type of the check - tcp, http, docker and script 4924 Command string // Command is the command to run for script checks 4925 Args []string // Args is a list of arguments for script checks 4926 Path string // path of the health check url for http type check 4927 Protocol string // Protocol to use if check is http, defaults to http 4928 PortLabel string // The port to use for tcp/http checks 4929 AddressMode string // 'host' to use host ip:port or 'driver' to use driver's 4930 Interval time.Duration // Interval of the check 4931 Timeout time.Duration // Timeout of the response from the check before consul fails the check 4932 InitialStatus string // Initial status of the check 4933 TLSSkipVerify bool // Skip TLS verification when Protocol=https 4934 Method string // HTTP Method to use (GET by default) 4935 Header map[string][]string // HTTP Headers for Consul to set when making HTTP checks 4936 CheckRestart *CheckRestart // If and when a task should be restarted based on checks 4937 GRPCService string // Service for GRPC checks 4938 GRPCUseTLS bool // Whether or not to use TLS for GRPC checks 4939 } 4940 4941 func (sc *ServiceCheck) Copy() *ServiceCheck { 4942 if sc == nil { 4943 return nil 4944 } 4945 nsc := new(ServiceCheck) 4946 *nsc = *sc 4947 nsc.Args = helper.CopySliceString(sc.Args) 4948 nsc.Header = helper.CopyMapStringSliceString(sc.Header) 4949 nsc.CheckRestart = sc.CheckRestart.Copy() 4950 return nsc 4951 } 4952 4953 func (sc *ServiceCheck) Canonicalize(serviceName string) { 4954 // Ensure empty maps/slices are treated as null to avoid scheduling 4955 // issues when using DeepEquals. 4956 if len(sc.Args) == 0 { 4957 sc.Args = nil 4958 } 4959 4960 if len(sc.Header) == 0 { 4961 sc.Header = nil 4962 } else { 4963 for k, v := range sc.Header { 4964 if len(v) == 0 { 4965 sc.Header[k] = nil 4966 } 4967 } 4968 } 4969 4970 if sc.Name == "" { 4971 sc.Name = fmt.Sprintf("service: %q check", serviceName) 4972 } 4973 } 4974 4975 // validate a Service's ServiceCheck 4976 func (sc *ServiceCheck) validate() error { 4977 // Validate Type 4978 switch strings.ToLower(sc.Type) { 4979 case ServiceCheckGRPC: 4980 case ServiceCheckTCP: 4981 case ServiceCheckHTTP: 4982 if sc.Path == "" { 4983 return fmt.Errorf("http type must have a valid http path") 4984 } 4985 url, err := url.Parse(sc.Path) 4986 if err != nil { 4987 return fmt.Errorf("http type must have a valid http path") 4988 } 4989 if url.IsAbs() { 4990 return fmt.Errorf("http type must have a relative http path") 4991 } 4992 4993 case ServiceCheckScript: 4994 if sc.Command == "" { 4995 return fmt.Errorf("script type must have a valid script path") 4996 } 4997 4998 default: 4999 return fmt.Errorf(`invalid type (%+q), must be one of "http", "tcp", or "script" type`, sc.Type) 5000 } 5001 5002 // Validate interval and timeout 5003 if sc.Interval == 0 { 5004 return fmt.Errorf("missing required value interval. Interval cannot be less than %v", minCheckInterval) 5005 } else if sc.Interval < minCheckInterval { 5006 return fmt.Errorf("interval (%v) cannot be lower than %v", sc.Interval, minCheckInterval) 5007 } 5008 5009 if sc.Timeout == 0 { 5010 return fmt.Errorf("missing required value timeout. Timeout cannot be less than %v", minCheckInterval) 5011 } else if sc.Timeout < minCheckTimeout { 5012 return fmt.Errorf("timeout (%v) is lower than required minimum timeout %v", sc.Timeout, minCheckInterval) 5013 } 5014 5015 // Validate InitialStatus 5016 switch sc.InitialStatus { 5017 case "": 5018 case api.HealthPassing: 5019 case api.HealthWarning: 5020 case api.HealthCritical: 5021 default: 5022 return fmt.Errorf(`invalid initial check state (%s), must be one of %q, %q, %q or empty`, sc.InitialStatus, api.HealthPassing, api.HealthWarning, api.HealthCritical) 5023 5024 } 5025 5026 // Validate AddressMode 5027 switch sc.AddressMode { 5028 case "", AddressModeHost, AddressModeDriver: 5029 // Ok 5030 case AddressModeAuto: 5031 return fmt.Errorf("invalid address_mode %q - %s only valid for services", sc.AddressMode, AddressModeAuto) 5032 default: 5033 return fmt.Errorf("invalid address_mode %q", sc.AddressMode) 5034 } 5035 5036 return sc.CheckRestart.Validate() 5037 } 5038 5039 // RequiresPort returns whether the service check requires the task has a port. 5040 func (sc *ServiceCheck) RequiresPort() bool { 5041 switch sc.Type { 5042 case ServiceCheckGRPC, ServiceCheckHTTP, ServiceCheckTCP: 5043 return true 5044 default: 5045 return false 5046 } 5047 } 5048 5049 // TriggersRestarts returns true if this check should be watched and trigger a restart 5050 // on failure. 5051 func (sc *ServiceCheck) TriggersRestarts() bool { 5052 return sc.CheckRestart != nil && sc.CheckRestart.Limit > 0 5053 } 5054 5055 // Hash all ServiceCheck fields and the check's corresponding service ID to 5056 // create an identifier. The identifier is not guaranteed to be unique as if 5057 // the PortLabel is blank, the Service's PortLabel will be used after Hash is 5058 // called. 5059 func (sc *ServiceCheck) Hash(serviceID string) string { 5060 h := sha1.New() 5061 io.WriteString(h, serviceID) 5062 io.WriteString(h, sc.Name) 5063 io.WriteString(h, sc.Type) 5064 io.WriteString(h, sc.Command) 5065 io.WriteString(h, strings.Join(sc.Args, "")) 5066 io.WriteString(h, sc.Path) 5067 io.WriteString(h, sc.Protocol) 5068 io.WriteString(h, sc.PortLabel) 5069 io.WriteString(h, sc.Interval.String()) 5070 io.WriteString(h, sc.Timeout.String()) 5071 io.WriteString(h, sc.Method) 5072 // Only include TLSSkipVerify if set to maintain ID stability with Nomad <0.6 5073 if sc.TLSSkipVerify { 5074 io.WriteString(h, "true") 5075 } 5076 5077 // Since map iteration order isn't stable we need to write k/v pairs to 5078 // a slice and sort it before hashing. 5079 if len(sc.Header) > 0 { 5080 headers := make([]string, 0, len(sc.Header)) 5081 for k, v := range sc.Header { 5082 headers = append(headers, k+strings.Join(v, "")) 5083 } 5084 sort.Strings(headers) 5085 io.WriteString(h, strings.Join(headers, "")) 5086 } 5087 5088 // Only include AddressMode if set to maintain ID stability with Nomad <0.7.1 5089 if len(sc.AddressMode) > 0 { 5090 io.WriteString(h, sc.AddressMode) 5091 } 5092 5093 // Only include GRPC if set to maintain ID stability with Nomad <0.8.4 5094 if sc.GRPCService != "" { 5095 io.WriteString(h, sc.GRPCService) 5096 } 5097 if sc.GRPCUseTLS { 5098 io.WriteString(h, "true") 5099 } 5100 5101 return fmt.Sprintf("%x", h.Sum(nil)) 5102 } 5103 5104 const ( 5105 AddressModeAuto = "auto" 5106 AddressModeHost = "host" 5107 AddressModeDriver = "driver" 5108 ) 5109 5110 // Service represents a Consul service definition in Nomad 5111 type Service struct { 5112 // Name of the service registered with Consul. Consul defaults the 5113 // Name to ServiceID if not specified. The Name if specified is used 5114 // as one of the seed values when generating a Consul ServiceID. 5115 Name string 5116 5117 // PortLabel is either the numeric port number or the `host:port`. 5118 // To specify the port number using the host's Consul Advertise 5119 // address, specify an empty host in the PortLabel (e.g. `:port`). 5120 PortLabel string 5121 5122 // AddressMode specifies whether or not to use the host ip:port for 5123 // this service. 5124 AddressMode string 5125 5126 Tags []string // List of tags for the service 5127 CanaryTags []string // List of tags for the service when it is a canary 5128 Checks []*ServiceCheck // List of checks associated with the service 5129 } 5130 5131 func (s *Service) Copy() *Service { 5132 if s == nil { 5133 return nil 5134 } 5135 ns := new(Service) 5136 *ns = *s 5137 ns.Tags = helper.CopySliceString(ns.Tags) 5138 ns.CanaryTags = helper.CopySliceString(ns.CanaryTags) 5139 5140 if s.Checks != nil { 5141 checks := make([]*ServiceCheck, len(ns.Checks)) 5142 for i, c := range ns.Checks { 5143 checks[i] = c.Copy() 5144 } 5145 ns.Checks = checks 5146 } 5147 5148 return ns 5149 } 5150 5151 // Canonicalize interpolates values of Job, Task Group and Task in the Service 5152 // Name. This also generates check names, service id and check ids. 5153 func (s *Service) Canonicalize(job string, taskGroup string, task string) { 5154 // Ensure empty lists are treated as null to avoid scheduler issues when 5155 // using DeepEquals 5156 if len(s.Tags) == 0 { 5157 s.Tags = nil 5158 } 5159 if len(s.CanaryTags) == 0 { 5160 s.CanaryTags = nil 5161 } 5162 if len(s.Checks) == 0 { 5163 s.Checks = nil 5164 } 5165 5166 s.Name = args.ReplaceEnv(s.Name, map[string]string{ 5167 "JOB": job, 5168 "TASKGROUP": taskGroup, 5169 "TASK": task, 5170 "BASE": fmt.Sprintf("%s-%s-%s", job, taskGroup, task), 5171 }, 5172 ) 5173 5174 for _, check := range s.Checks { 5175 check.Canonicalize(s.Name) 5176 } 5177 } 5178 5179 // Validate checks if the Check definition is valid 5180 func (s *Service) Validate() error { 5181 var mErr multierror.Error 5182 5183 // Ensure the service name is valid per the below RFCs but make an exception 5184 // for our interpolation syntax by first stripping any environment variables from the name 5185 5186 serviceNameStripped := args.ReplaceEnvWithPlaceHolder(s.Name, "ENV-VAR") 5187 5188 if err := s.ValidateName(serviceNameStripped); err != nil { 5189 mErr.Errors = append(mErr.Errors, fmt.Errorf("service name must be valid per RFC 1123 and can contain only alphanumeric characters or dashes: %q", s.Name)) 5190 } 5191 5192 switch s.AddressMode { 5193 case "", AddressModeAuto, AddressModeHost, AddressModeDriver: 5194 // OK 5195 default: 5196 mErr.Errors = append(mErr.Errors, fmt.Errorf("service address_mode must be %q, %q, or %q; not %q", AddressModeAuto, AddressModeHost, AddressModeDriver, s.AddressMode)) 5197 } 5198 5199 for _, c := range s.Checks { 5200 if s.PortLabel == "" && c.PortLabel == "" && c.RequiresPort() { 5201 mErr.Errors = append(mErr.Errors, fmt.Errorf("check %s invalid: check requires a port but neither check nor service %+q have a port", c.Name, s.Name)) 5202 continue 5203 } 5204 5205 if err := c.validate(); err != nil { 5206 mErr.Errors = append(mErr.Errors, fmt.Errorf("check %s invalid: %v", c.Name, err)) 5207 } 5208 } 5209 5210 return mErr.ErrorOrNil() 5211 } 5212 5213 // ValidateName checks if the services Name is valid and should be called after 5214 // the name has been interpolated 5215 func (s *Service) ValidateName(name string) error { 5216 // Ensure the service name is valid per RFC-952 §1 5217 // (https://tools.ietf.org/html/rfc952), RFC-1123 §2.1 5218 // (https://tools.ietf.org/html/rfc1123), and RFC-2782 5219 // (https://tools.ietf.org/html/rfc2782). 5220 re := regexp.MustCompile(`^(?i:[a-z0-9]|[a-z0-9][a-z0-9\-]{0,61}[a-z0-9])$`) 5221 if !re.MatchString(name) { 5222 return fmt.Errorf("service name must be valid per RFC 1123 and can contain only alphanumeric characters or dashes and must be no longer than 63 characters: %q", name) 5223 } 5224 return nil 5225 } 5226 5227 // Hash returns a base32 encoded hash of a Service's contents excluding checks 5228 // as they're hashed independently. 5229 func (s *Service) Hash(allocID, taskName string, canary bool) string { 5230 h := sha1.New() 5231 io.WriteString(h, allocID) 5232 io.WriteString(h, taskName) 5233 io.WriteString(h, s.Name) 5234 io.WriteString(h, s.PortLabel) 5235 io.WriteString(h, s.AddressMode) 5236 for _, tag := range s.Tags { 5237 io.WriteString(h, tag) 5238 } 5239 for _, tag := range s.CanaryTags { 5240 io.WriteString(h, tag) 5241 } 5242 5243 // Vary ID on whether or not CanaryTags will be used 5244 if canary { 5245 h.Write([]byte("Canary")) 5246 } 5247 5248 // Base32 is used for encoding the hash as sha1 hashes can always be 5249 // encoded without padding, only 4 bytes larger than base64, and saves 5250 // 8 bytes vs hex. Since these hashes are used in Consul URLs it's nice 5251 // to have a reasonably compact URL-safe representation. 5252 return b32.EncodeToString(h.Sum(nil)) 5253 } 5254 5255 const ( 5256 // DefaultKillTimeout is the default timeout between signaling a task it 5257 // will be killed and killing it. 5258 DefaultKillTimeout = 5 * time.Second 5259 ) 5260 5261 // LogConfig provides configuration for log rotation 5262 type LogConfig struct { 5263 MaxFiles int 5264 MaxFileSizeMB int 5265 } 5266 5267 // DefaultLogConfig returns the default LogConfig values. 5268 func DefaultLogConfig() *LogConfig { 5269 return &LogConfig{ 5270 MaxFiles: 10, 5271 MaxFileSizeMB: 10, 5272 } 5273 } 5274 5275 // Validate returns an error if the log config specified are less than 5276 // the minimum allowed. 5277 func (l *LogConfig) Validate() error { 5278 var mErr multierror.Error 5279 if l.MaxFiles < 1 { 5280 mErr.Errors = append(mErr.Errors, fmt.Errorf("minimum number of files is 1; got %d", l.MaxFiles)) 5281 } 5282 if l.MaxFileSizeMB < 1 { 5283 mErr.Errors = append(mErr.Errors, fmt.Errorf("minimum file size is 1MB; got %d", l.MaxFileSizeMB)) 5284 } 5285 return mErr.ErrorOrNil() 5286 } 5287 5288 // Task is a single process typically that is executed as part of a task group. 5289 type Task struct { 5290 // Name of the task 5291 Name string 5292 5293 // Driver is used to control which driver is used 5294 Driver string 5295 5296 // User is used to determine which user will run the task. It defaults to 5297 // the same user the Nomad client is being run as. 5298 User string 5299 5300 // Config is provided to the driver to initialize 5301 Config map[string]interface{} 5302 5303 // Map of environment variables to be used by the driver 5304 Env map[string]string 5305 5306 // List of service definitions exposed by the Task 5307 Services []*Service 5308 5309 // Vault is used to define the set of Vault policies that this task should 5310 // have access to. 5311 Vault *Vault 5312 5313 // Templates are the set of templates to be rendered for the task. 5314 Templates []*Template 5315 5316 // Constraints can be specified at a task level and apply only to 5317 // the particular task. 5318 Constraints []*Constraint 5319 5320 // Affinities can be specified at the task level to express 5321 // scheduling preferences 5322 Affinities []*Affinity 5323 5324 // Resources is the resources needed by this task 5325 Resources *Resources 5326 5327 // DispatchPayload configures how the task retrieves its input from a dispatch 5328 DispatchPayload *DispatchPayloadConfig 5329 5330 // Meta is used to associate arbitrary metadata with this 5331 // task. This is opaque to Nomad. 5332 Meta map[string]string 5333 5334 // KillTimeout is the time between signaling a task that it will be 5335 // killed and killing it. 5336 KillTimeout time.Duration 5337 5338 // LogConfig provides configuration for log rotation 5339 LogConfig *LogConfig 5340 5341 // Artifacts is a list of artifacts to download and extract before running 5342 // the task. 5343 Artifacts []*TaskArtifact 5344 5345 // Leader marks the task as the leader within the group. When the leader 5346 // task exits, other tasks will be gracefully terminated. 5347 Leader bool 5348 5349 // ShutdownDelay is the duration of the delay between deregistering a 5350 // task from Consul and sending it a signal to shutdown. See #2441 5351 ShutdownDelay time.Duration 5352 5353 // The kill signal to use for the task. This is an optional specification, 5354 5355 // KillSignal is the kill signal to use for the task. This is an optional 5356 // specification and defaults to SIGINT 5357 KillSignal string 5358 } 5359 5360 func (t *Task) Copy() *Task { 5361 if t == nil { 5362 return nil 5363 } 5364 nt := new(Task) 5365 *nt = *t 5366 nt.Env = helper.CopyMapStringString(nt.Env) 5367 5368 if t.Services != nil { 5369 services := make([]*Service, len(nt.Services)) 5370 for i, s := range nt.Services { 5371 services[i] = s.Copy() 5372 } 5373 nt.Services = services 5374 } 5375 5376 nt.Constraints = CopySliceConstraints(nt.Constraints) 5377 nt.Affinities = CopySliceAffinities(nt.Affinities) 5378 5379 nt.Vault = nt.Vault.Copy() 5380 nt.Resources = nt.Resources.Copy() 5381 nt.Meta = helper.CopyMapStringString(nt.Meta) 5382 nt.DispatchPayload = nt.DispatchPayload.Copy() 5383 5384 if t.Artifacts != nil { 5385 artifacts := make([]*TaskArtifact, 0, len(t.Artifacts)) 5386 for _, a := range nt.Artifacts { 5387 artifacts = append(artifacts, a.Copy()) 5388 } 5389 nt.Artifacts = artifacts 5390 } 5391 5392 if i, err := copystructure.Copy(nt.Config); err != nil { 5393 panic(err.Error()) 5394 } else { 5395 nt.Config = i.(map[string]interface{}) 5396 } 5397 5398 if t.Templates != nil { 5399 templates := make([]*Template, len(t.Templates)) 5400 for i, tmpl := range nt.Templates { 5401 templates[i] = tmpl.Copy() 5402 } 5403 nt.Templates = templates 5404 } 5405 5406 return nt 5407 } 5408 5409 // Canonicalize canonicalizes fields in the task. 5410 func (t *Task) Canonicalize(job *Job, tg *TaskGroup) { 5411 // Ensure that an empty and nil map are treated the same to avoid scheduling 5412 // problems since we use reflect DeepEquals. 5413 if len(t.Meta) == 0 { 5414 t.Meta = nil 5415 } 5416 if len(t.Config) == 0 { 5417 t.Config = nil 5418 } 5419 if len(t.Env) == 0 { 5420 t.Env = nil 5421 } 5422 5423 for _, service := range t.Services { 5424 service.Canonicalize(job.Name, tg.Name, t.Name) 5425 } 5426 5427 // If Resources are nil initialize them to defaults, otherwise canonicalize 5428 if t.Resources == nil { 5429 t.Resources = DefaultResources() 5430 } else { 5431 t.Resources.Canonicalize() 5432 } 5433 5434 // Set the default timeout if it is not specified. 5435 if t.KillTimeout == 0 { 5436 t.KillTimeout = DefaultKillTimeout 5437 } 5438 5439 if t.Vault != nil { 5440 t.Vault.Canonicalize() 5441 } 5442 5443 for _, template := range t.Templates { 5444 template.Canonicalize() 5445 } 5446 } 5447 5448 func (t *Task) GoString() string { 5449 return fmt.Sprintf("*%#v", *t) 5450 } 5451 5452 // Validate is used to sanity check a task 5453 func (t *Task) Validate(ephemeralDisk *EphemeralDisk, jobType string) error { 5454 var mErr multierror.Error 5455 if t.Name == "" { 5456 mErr.Errors = append(mErr.Errors, errors.New("Missing task name")) 5457 } 5458 if strings.ContainsAny(t.Name, `/\`) { 5459 // We enforce this so that when creating the directory on disk it will 5460 // not have any slashes. 5461 mErr.Errors = append(mErr.Errors, errors.New("Task name cannot include slashes")) 5462 } 5463 if t.Driver == "" { 5464 mErr.Errors = append(mErr.Errors, errors.New("Missing task driver")) 5465 } 5466 if t.KillTimeout < 0 { 5467 mErr.Errors = append(mErr.Errors, errors.New("KillTimeout must be a positive value")) 5468 } 5469 if t.ShutdownDelay < 0 { 5470 mErr.Errors = append(mErr.Errors, errors.New("ShutdownDelay must be a positive value")) 5471 } 5472 5473 // Validate the resources. 5474 if t.Resources == nil { 5475 mErr.Errors = append(mErr.Errors, errors.New("Missing task resources")) 5476 } else if err := t.Resources.Validate(); err != nil { 5477 mErr.Errors = append(mErr.Errors, err) 5478 } 5479 5480 // Validate the log config 5481 if t.LogConfig == nil { 5482 mErr.Errors = append(mErr.Errors, errors.New("Missing Log Config")) 5483 } else if err := t.LogConfig.Validate(); err != nil { 5484 mErr.Errors = append(mErr.Errors, err) 5485 } 5486 5487 for idx, constr := range t.Constraints { 5488 if err := constr.Validate(); err != nil { 5489 outer := fmt.Errorf("Constraint %d validation failed: %s", idx+1, err) 5490 mErr.Errors = append(mErr.Errors, outer) 5491 } 5492 5493 switch constr.Operand { 5494 case ConstraintDistinctHosts, ConstraintDistinctProperty: 5495 outer := fmt.Errorf("Constraint %d has disallowed Operand at task level: %s", idx+1, constr.Operand) 5496 mErr.Errors = append(mErr.Errors, outer) 5497 } 5498 } 5499 5500 if jobType == JobTypeSystem { 5501 if t.Affinities != nil { 5502 mErr.Errors = append(mErr.Errors, fmt.Errorf("System jobs may not have an affinity stanza")) 5503 } 5504 } else { 5505 for idx, affinity := range t.Affinities { 5506 if err := affinity.Validate(); err != nil { 5507 outer := fmt.Errorf("Affinity %d validation failed: %s", idx+1, err) 5508 mErr.Errors = append(mErr.Errors, outer) 5509 } 5510 } 5511 } 5512 5513 // Validate Services 5514 if err := validateServices(t); err != nil { 5515 mErr.Errors = append(mErr.Errors, err) 5516 } 5517 5518 if t.LogConfig != nil && ephemeralDisk != nil { 5519 logUsage := (t.LogConfig.MaxFiles * t.LogConfig.MaxFileSizeMB) 5520 if ephemeralDisk.SizeMB <= logUsage { 5521 mErr.Errors = append(mErr.Errors, 5522 fmt.Errorf("log storage (%d MB) must be less than requested disk capacity (%d MB)", 5523 logUsage, ephemeralDisk.SizeMB)) 5524 } 5525 } 5526 5527 for idx, artifact := range t.Artifacts { 5528 if err := artifact.Validate(); err != nil { 5529 outer := fmt.Errorf("Artifact %d validation failed: %v", idx+1, err) 5530 mErr.Errors = append(mErr.Errors, outer) 5531 } 5532 } 5533 5534 if t.Vault != nil { 5535 if err := t.Vault.Validate(); err != nil { 5536 mErr.Errors = append(mErr.Errors, fmt.Errorf("Vault validation failed: %v", err)) 5537 } 5538 } 5539 5540 destinations := make(map[string]int, len(t.Templates)) 5541 for idx, tmpl := range t.Templates { 5542 if err := tmpl.Validate(); err != nil { 5543 outer := fmt.Errorf("Template %d validation failed: %s", idx+1, err) 5544 mErr.Errors = append(mErr.Errors, outer) 5545 } 5546 5547 if other, ok := destinations[tmpl.DestPath]; ok { 5548 outer := fmt.Errorf("Template %d has same destination as %d", idx+1, other) 5549 mErr.Errors = append(mErr.Errors, outer) 5550 } else { 5551 destinations[tmpl.DestPath] = idx + 1 5552 } 5553 } 5554 5555 // Validate the dispatch payload block if there 5556 if t.DispatchPayload != nil { 5557 if err := t.DispatchPayload.Validate(); err != nil { 5558 mErr.Errors = append(mErr.Errors, fmt.Errorf("Dispatch Payload validation failed: %v", err)) 5559 } 5560 } 5561 5562 return mErr.ErrorOrNil() 5563 } 5564 5565 // validateServices takes a task and validates the services within it are valid 5566 // and reference ports that exist. 5567 func validateServices(t *Task) error { 5568 var mErr multierror.Error 5569 5570 // Ensure that services don't ask for nonexistent ports and their names are 5571 // unique. 5572 servicePorts := make(map[string]map[string]struct{}) 5573 addServicePort := func(label, service string) { 5574 if _, ok := servicePorts[label]; !ok { 5575 servicePorts[label] = map[string]struct{}{} 5576 } 5577 servicePorts[label][service] = struct{}{} 5578 } 5579 knownServices := make(map[string]struct{}) 5580 for i, service := range t.Services { 5581 if err := service.Validate(); err != nil { 5582 outer := fmt.Errorf("service[%d] %+q validation failed: %s", i, service.Name, err) 5583 mErr.Errors = append(mErr.Errors, outer) 5584 } 5585 5586 // Ensure that services with the same name are not being registered for 5587 // the same port 5588 if _, ok := knownServices[service.Name+service.PortLabel]; ok { 5589 mErr.Errors = append(mErr.Errors, fmt.Errorf("service %q is duplicate", service.Name)) 5590 } 5591 knownServices[service.Name+service.PortLabel] = struct{}{} 5592 5593 if service.PortLabel != "" { 5594 if service.AddressMode == "driver" { 5595 // Numeric port labels are valid for address_mode=driver 5596 _, err := strconv.Atoi(service.PortLabel) 5597 if err != nil { 5598 // Not a numeric port label, add it to list to check 5599 addServicePort(service.PortLabel, service.Name) 5600 } 5601 } else { 5602 addServicePort(service.PortLabel, service.Name) 5603 } 5604 } 5605 5606 // Ensure that check names are unique and have valid ports 5607 knownChecks := make(map[string]struct{}) 5608 for _, check := range service.Checks { 5609 if _, ok := knownChecks[check.Name]; ok { 5610 mErr.Errors = append(mErr.Errors, fmt.Errorf("check %q is duplicate", check.Name)) 5611 } 5612 knownChecks[check.Name] = struct{}{} 5613 5614 if !check.RequiresPort() { 5615 // No need to continue validating check if it doesn't need a port 5616 continue 5617 } 5618 5619 effectivePort := check.PortLabel 5620 if effectivePort == "" { 5621 // Inherits from service 5622 effectivePort = service.PortLabel 5623 } 5624 5625 if effectivePort == "" { 5626 mErr.Errors = append(mErr.Errors, fmt.Errorf("check %q is missing a port", check.Name)) 5627 continue 5628 } 5629 5630 isNumeric := false 5631 portNumber, err := strconv.Atoi(effectivePort) 5632 if err == nil { 5633 isNumeric = true 5634 } 5635 5636 // Numeric ports are fine for address_mode = "driver" 5637 if check.AddressMode == "driver" && isNumeric { 5638 if portNumber <= 0 { 5639 mErr.Errors = append(mErr.Errors, fmt.Errorf("check %q has invalid numeric port %d", check.Name, portNumber)) 5640 } 5641 continue 5642 } 5643 5644 if isNumeric { 5645 mErr.Errors = append(mErr.Errors, fmt.Errorf(`check %q cannot use a numeric port %d without setting address_mode="driver"`, check.Name, portNumber)) 5646 continue 5647 } 5648 5649 // PortLabel must exist, report errors by its parent service 5650 addServicePort(effectivePort, service.Name) 5651 } 5652 } 5653 5654 // Get the set of port labels. 5655 portLabels := make(map[string]struct{}) 5656 if t.Resources != nil { 5657 for _, network := range t.Resources.Networks { 5658 ports := network.PortLabels() 5659 for portLabel := range ports { 5660 portLabels[portLabel] = struct{}{} 5661 } 5662 } 5663 } 5664 5665 // Iterate over a sorted list of keys to make error listings stable 5666 keys := make([]string, 0, len(servicePorts)) 5667 for p := range servicePorts { 5668 keys = append(keys, p) 5669 } 5670 sort.Strings(keys) 5671 5672 // Ensure all ports referenced in services exist. 5673 for _, servicePort := range keys { 5674 services := servicePorts[servicePort] 5675 _, ok := portLabels[servicePort] 5676 if !ok { 5677 names := make([]string, 0, len(services)) 5678 for name := range services { 5679 names = append(names, name) 5680 } 5681 5682 // Keep order deterministic 5683 sort.Strings(names) 5684 joined := strings.Join(names, ", ") 5685 err := fmt.Errorf("port label %q referenced by services %v does not exist", servicePort, joined) 5686 mErr.Errors = append(mErr.Errors, err) 5687 } 5688 } 5689 5690 // Ensure address mode is valid 5691 return mErr.ErrorOrNil() 5692 } 5693 5694 func (t *Task) Warnings() error { 5695 var mErr multierror.Error 5696 5697 // Validate the resources 5698 if t.Resources != nil && t.Resources.IOPS != 0 { 5699 mErr.Errors = append(mErr.Errors, fmt.Errorf("IOPS has been deprecated as of Nomad 0.9.0. Please remove IOPS from resource stanza.")) 5700 } 5701 5702 return mErr.ErrorOrNil() 5703 } 5704 5705 const ( 5706 // TemplateChangeModeNoop marks that no action should be taken if the 5707 // template is re-rendered 5708 TemplateChangeModeNoop = "noop" 5709 5710 // TemplateChangeModeSignal marks that the task should be signaled if the 5711 // template is re-rendered 5712 TemplateChangeModeSignal = "signal" 5713 5714 // TemplateChangeModeRestart marks that the task should be restarted if the 5715 // template is re-rendered 5716 TemplateChangeModeRestart = "restart" 5717 ) 5718 5719 var ( 5720 // TemplateChangeModeInvalidError is the error for when an invalid change 5721 // mode is given 5722 TemplateChangeModeInvalidError = errors.New("Invalid change mode. Must be one of the following: noop, signal, restart") 5723 ) 5724 5725 // Template represents a template configuration to be rendered for a given task 5726 type Template struct { 5727 // SourcePath is the path to the template to be rendered 5728 SourcePath string 5729 5730 // DestPath is the path to where the template should be rendered 5731 DestPath string 5732 5733 // EmbeddedTmpl store the raw template. This is useful for smaller templates 5734 // where they are embedded in the job file rather than sent as an artifact 5735 EmbeddedTmpl string 5736 5737 // ChangeMode indicates what should be done if the template is re-rendered 5738 ChangeMode string 5739 5740 // ChangeSignal is the signal that should be sent if the change mode 5741 // requires it. 5742 ChangeSignal string 5743 5744 // Splay is used to avoid coordinated restarts of processes by applying a 5745 // random wait between 0 and the given splay value before signalling the 5746 // application of a change 5747 Splay time.Duration 5748 5749 // Perms is the permission the file should be written out with. 5750 Perms string 5751 5752 // LeftDelim and RightDelim are optional configurations to control what 5753 // delimiter is utilized when parsing the template. 5754 LeftDelim string 5755 RightDelim string 5756 5757 // Envvars enables exposing the template as environment variables 5758 // instead of as a file. The template must be of the form: 5759 // 5760 // VAR_NAME_1={{ key service/my-key }} 5761 // VAR_NAME_2=raw string and {{ env "attr.kernel.name" }} 5762 // 5763 // Lines will be split on the initial "=" with the first part being the 5764 // key name and the second part the value. 5765 // Empty lines and lines starting with # will be ignored, but to avoid 5766 // escaping issues #s within lines will not be treated as comments. 5767 Envvars bool 5768 5769 // VaultGrace is the grace duration between lease renewal and reacquiring a 5770 // secret. If the lease of a secret is less than the grace, a new secret is 5771 // acquired. 5772 VaultGrace time.Duration 5773 } 5774 5775 // DefaultTemplate returns a default template. 5776 func DefaultTemplate() *Template { 5777 return &Template{ 5778 ChangeMode: TemplateChangeModeRestart, 5779 Splay: 5 * time.Second, 5780 Perms: "0644", 5781 } 5782 } 5783 5784 func (t *Template) Copy() *Template { 5785 if t == nil { 5786 return nil 5787 } 5788 copy := new(Template) 5789 *copy = *t 5790 return copy 5791 } 5792 5793 func (t *Template) Canonicalize() { 5794 if t.ChangeSignal != "" { 5795 t.ChangeSignal = strings.ToUpper(t.ChangeSignal) 5796 } 5797 } 5798 5799 func (t *Template) Validate() error { 5800 var mErr multierror.Error 5801 5802 // Verify we have something to render 5803 if t.SourcePath == "" && t.EmbeddedTmpl == "" { 5804 multierror.Append(&mErr, fmt.Errorf("Must specify a source path or have an embedded template")) 5805 } 5806 5807 // Verify we can render somewhere 5808 if t.DestPath == "" { 5809 multierror.Append(&mErr, fmt.Errorf("Must specify a destination for the template")) 5810 } 5811 5812 // Verify the destination doesn't escape 5813 escaped, err := PathEscapesAllocDir("task", t.DestPath) 5814 if err != nil { 5815 mErr.Errors = append(mErr.Errors, fmt.Errorf("invalid destination path: %v", err)) 5816 } else if escaped { 5817 mErr.Errors = append(mErr.Errors, fmt.Errorf("destination escapes allocation directory")) 5818 } 5819 5820 // Verify a proper change mode 5821 switch t.ChangeMode { 5822 case TemplateChangeModeNoop, TemplateChangeModeRestart: 5823 case TemplateChangeModeSignal: 5824 if t.ChangeSignal == "" { 5825 multierror.Append(&mErr, fmt.Errorf("Must specify signal value when change mode is signal")) 5826 } 5827 if t.Envvars { 5828 multierror.Append(&mErr, fmt.Errorf("cannot use signals with env var templates")) 5829 } 5830 default: 5831 multierror.Append(&mErr, TemplateChangeModeInvalidError) 5832 } 5833 5834 // Verify the splay is positive 5835 if t.Splay < 0 { 5836 multierror.Append(&mErr, fmt.Errorf("Must specify positive splay value")) 5837 } 5838 5839 // Verify the permissions 5840 if t.Perms != "" { 5841 if _, err := strconv.ParseUint(t.Perms, 8, 12); err != nil { 5842 multierror.Append(&mErr, fmt.Errorf("Failed to parse %q as octal: %v", t.Perms, err)) 5843 } 5844 } 5845 5846 if t.VaultGrace.Nanoseconds() < 0 { 5847 multierror.Append(&mErr, fmt.Errorf("Vault grace must be greater than zero: %v < 0", t.VaultGrace)) 5848 } 5849 5850 return mErr.ErrorOrNil() 5851 } 5852 5853 // Set of possible states for a task. 5854 const ( 5855 TaskStatePending = "pending" // The task is waiting to be run. 5856 TaskStateRunning = "running" // The task is currently running. 5857 TaskStateDead = "dead" // Terminal state of task. 5858 ) 5859 5860 // TaskState tracks the current state of a task and events that caused state 5861 // transitions. 5862 type TaskState struct { 5863 // The current state of the task. 5864 State string 5865 5866 // Failed marks a task as having failed 5867 Failed bool 5868 5869 // Restarts is the number of times the task has restarted 5870 Restarts uint64 5871 5872 // LastRestart is the time the task last restarted. It is updated each time the 5873 // task restarts 5874 LastRestart time.Time 5875 5876 // StartedAt is the time the task is started. It is updated each time the 5877 // task starts 5878 StartedAt time.Time 5879 5880 // FinishedAt is the time at which the task transitioned to dead and will 5881 // not be started again. 5882 FinishedAt time.Time 5883 5884 // Series of task events that transition the state of the task. 5885 Events []*TaskEvent 5886 } 5887 5888 // NewTaskState returns a TaskState initialized in the Pending state. 5889 func NewTaskState() *TaskState { 5890 return &TaskState{ 5891 State: TaskStatePending, 5892 } 5893 } 5894 5895 // Canonicalize ensures the TaskState has a State set. It should default to 5896 // Pending. 5897 func (ts *TaskState) Canonicalize() { 5898 if ts.State == "" { 5899 ts.State = TaskStatePending 5900 } 5901 } 5902 5903 func (ts *TaskState) Copy() *TaskState { 5904 if ts == nil { 5905 return nil 5906 } 5907 copy := new(TaskState) 5908 *copy = *ts 5909 5910 if ts.Events != nil { 5911 copy.Events = make([]*TaskEvent, len(ts.Events)) 5912 for i, e := range ts.Events { 5913 copy.Events[i] = e.Copy() 5914 } 5915 } 5916 return copy 5917 } 5918 5919 // Successful returns whether a task finished successfully. This doesn't really 5920 // have meaning on a non-batch allocation because a service and system 5921 // allocation should not finish. 5922 func (ts *TaskState) Successful() bool { 5923 return ts.State == TaskStateDead && !ts.Failed 5924 } 5925 5926 const ( 5927 // TaskSetupFailure indicates that the task could not be started due to a 5928 // a setup failure. 5929 TaskSetupFailure = "Setup Failure" 5930 5931 // TaskDriveFailure indicates that the task could not be started due to a 5932 // failure in the driver. 5933 TaskDriverFailure = "Driver Failure" 5934 5935 // TaskReceived signals that the task has been pulled by the client at the 5936 // given timestamp. 5937 TaskReceived = "Received" 5938 5939 // TaskFailedValidation indicates the task was invalid and as such was not 5940 // run. 5941 TaskFailedValidation = "Failed Validation" 5942 5943 // TaskStarted signals that the task was started and its timestamp can be 5944 // used to determine the running length of the task. 5945 TaskStarted = "Started" 5946 5947 // TaskTerminated indicates that the task was started and exited. 5948 TaskTerminated = "Terminated" 5949 5950 // TaskKilling indicates a kill signal has been sent to the task. 5951 TaskKilling = "Killing" 5952 5953 // TaskKilled indicates a user has killed the task. 5954 TaskKilled = "Killed" 5955 5956 // TaskRestarting indicates that task terminated and is being restarted. 5957 TaskRestarting = "Restarting" 5958 5959 // TaskNotRestarting indicates that the task has failed and is not being 5960 // restarted because it has exceeded its restart policy. 5961 TaskNotRestarting = "Not Restarting" 5962 5963 // TaskRestartSignal indicates that the task has been signalled to be 5964 // restarted 5965 TaskRestartSignal = "Restart Signaled" 5966 5967 // TaskSignaling indicates that the task is being signalled. 5968 TaskSignaling = "Signaling" 5969 5970 // TaskDownloadingArtifacts means the task is downloading the artifacts 5971 // specified in the task. 5972 TaskDownloadingArtifacts = "Downloading Artifacts" 5973 5974 // TaskArtifactDownloadFailed indicates that downloading the artifacts 5975 // failed. 5976 TaskArtifactDownloadFailed = "Failed Artifact Download" 5977 5978 // TaskBuildingTaskDir indicates that the task directory/chroot is being 5979 // built. 5980 TaskBuildingTaskDir = "Building Task Directory" 5981 5982 // TaskSetup indicates the task runner is setting up the task environment 5983 TaskSetup = "Task Setup" 5984 5985 // TaskDiskExceeded indicates that one of the tasks in a taskgroup has 5986 // exceeded the requested disk resources. 5987 TaskDiskExceeded = "Disk Resources Exceeded" 5988 5989 // TaskSiblingFailed indicates that a sibling task in the task group has 5990 // failed. 5991 TaskSiblingFailed = "Sibling Task Failed" 5992 5993 // TaskDriverMessage is an informational event message emitted by 5994 // drivers such as when they're performing a long running action like 5995 // downloading an image. 5996 TaskDriverMessage = "Driver" 5997 5998 // TaskLeaderDead indicates that the leader task within the has finished. 5999 TaskLeaderDead = "Leader Task Dead" 6000 6001 // TaskHookFailed indicates that one of the hooks for a task failed. 6002 TaskHookFailed = "Task hook failed" 6003 6004 // TaskRestoreFailed indicates Nomad was unable to reattach to a 6005 // restored task. 6006 TaskRestoreFailed = "Failed Restoring Task" 6007 ) 6008 6009 // TaskEvent is an event that effects the state of a task and contains meta-data 6010 // appropriate to the events type. 6011 type TaskEvent struct { 6012 Type string 6013 Time int64 // Unix Nanosecond timestamp 6014 6015 Message string // A possible message explaining the termination of the task. 6016 6017 // DisplayMessage is a human friendly message about the event 6018 DisplayMessage string 6019 6020 // Details is a map with annotated info about the event 6021 Details map[string]string 6022 6023 // DEPRECATION NOTICE: The following fields are deprecated and will be removed 6024 // in a future release. Field values are available in the Details map. 6025 6026 // FailsTask marks whether this event fails the task. 6027 // Deprecated, use Details["fails_task"] to access this. 6028 FailsTask bool 6029 6030 // Restart fields. 6031 // Deprecated, use Details["restart_reason"] to access this. 6032 RestartReason string 6033 6034 // Setup Failure fields. 6035 // Deprecated, use Details["setup_error"] to access this. 6036 SetupError string 6037 6038 // Driver Failure fields. 6039 // Deprecated, use Details["driver_error"] to access this. 6040 DriverError string // A driver error occurred while starting the task. 6041 6042 // Task Terminated Fields. 6043 6044 // Deprecated, use Details["exit_code"] to access this. 6045 ExitCode int // The exit code of the task. 6046 6047 // Deprecated, use Details["signal"] to access this. 6048 Signal int // The signal that terminated the task. 6049 6050 // Killing fields 6051 // Deprecated, use Details["kill_timeout"] to access this. 6052 KillTimeout time.Duration 6053 6054 // Task Killed Fields. 6055 // Deprecated, use Details["kill_error"] to access this. 6056 KillError string // Error killing the task. 6057 6058 // KillReason is the reason the task was killed 6059 // Deprecated, use Details["kill_reason"] to access this. 6060 KillReason string 6061 6062 // TaskRestarting fields. 6063 // Deprecated, use Details["start_delay"] to access this. 6064 StartDelay int64 // The sleep period before restarting the task in unix nanoseconds. 6065 6066 // Artifact Download fields 6067 // Deprecated, use Details["download_error"] to access this. 6068 DownloadError string // Error downloading artifacts 6069 6070 // Validation fields 6071 // Deprecated, use Details["validation_error"] to access this. 6072 ValidationError string // Validation error 6073 6074 // The maximum allowed task disk size. 6075 // Deprecated, use Details["disk_limit"] to access this. 6076 DiskLimit int64 6077 6078 // Name of the sibling task that caused termination of the task that 6079 // the TaskEvent refers to. 6080 // Deprecated, use Details["failed_sibling"] to access this. 6081 FailedSibling string 6082 6083 // VaultError is the error from token renewal 6084 // Deprecated, use Details["vault_renewal_error"] to access this. 6085 VaultError string 6086 6087 // TaskSignalReason indicates the reason the task is being signalled. 6088 // Deprecated, use Details["task_signal_reason"] to access this. 6089 TaskSignalReason string 6090 6091 // TaskSignal is the signal that was sent to the task 6092 // Deprecated, use Details["task_signal"] to access this. 6093 TaskSignal string 6094 6095 // DriverMessage indicates a driver action being taken. 6096 // Deprecated, use Details["driver_message"] to access this. 6097 DriverMessage string 6098 6099 // GenericSource is the source of a message. 6100 // Deprecated, is redundant with event type. 6101 GenericSource string 6102 } 6103 6104 func (event *TaskEvent) PopulateEventDisplayMessage() { 6105 // Build up the description based on the event type. 6106 if event == nil { //TODO(preetha) needs investigation alloc_runner's Run method sends a nil event when sigterming nomad. Why? 6107 return 6108 } 6109 6110 if event.DisplayMessage != "" { 6111 return 6112 } 6113 6114 var desc string 6115 switch event.Type { 6116 case TaskSetup: 6117 desc = event.Message 6118 case TaskStarted: 6119 desc = "Task started by client" 6120 case TaskReceived: 6121 desc = "Task received by client" 6122 case TaskFailedValidation: 6123 if event.ValidationError != "" { 6124 desc = event.ValidationError 6125 } else { 6126 desc = "Validation of task failed" 6127 } 6128 case TaskSetupFailure: 6129 if event.SetupError != "" { 6130 desc = event.SetupError 6131 } else { 6132 desc = "Task setup failed" 6133 } 6134 case TaskDriverFailure: 6135 if event.DriverError != "" { 6136 desc = event.DriverError 6137 } else { 6138 desc = "Failed to start task" 6139 } 6140 case TaskDownloadingArtifacts: 6141 desc = "Client is downloading artifacts" 6142 case TaskArtifactDownloadFailed: 6143 if event.DownloadError != "" { 6144 desc = event.DownloadError 6145 } else { 6146 desc = "Failed to download artifacts" 6147 } 6148 case TaskKilling: 6149 if event.KillReason != "" { 6150 desc = event.KillReason 6151 } else if event.KillTimeout != 0 { 6152 desc = fmt.Sprintf("Sent interrupt. Waiting %v before force killing", event.KillTimeout) 6153 } else { 6154 desc = "Sent interrupt" 6155 } 6156 case TaskKilled: 6157 if event.KillError != "" { 6158 desc = event.KillError 6159 } else { 6160 desc = "Task successfully killed" 6161 } 6162 case TaskTerminated: 6163 var parts []string 6164 parts = append(parts, fmt.Sprintf("Exit Code: %d", event.ExitCode)) 6165 6166 if event.Signal != 0 { 6167 parts = append(parts, fmt.Sprintf("Signal: %d", event.Signal)) 6168 } 6169 6170 if event.Message != "" { 6171 parts = append(parts, fmt.Sprintf("Exit Message: %q", event.Message)) 6172 } 6173 desc = strings.Join(parts, ", ") 6174 case TaskRestarting: 6175 in := fmt.Sprintf("Task restarting in %v", time.Duration(event.StartDelay)) 6176 if event.RestartReason != "" && event.RestartReason != ReasonWithinPolicy { 6177 desc = fmt.Sprintf("%s - %s", event.RestartReason, in) 6178 } else { 6179 desc = in 6180 } 6181 case TaskNotRestarting: 6182 if event.RestartReason != "" { 6183 desc = event.RestartReason 6184 } else { 6185 desc = "Task exceeded restart policy" 6186 } 6187 case TaskSiblingFailed: 6188 if event.FailedSibling != "" { 6189 desc = fmt.Sprintf("Task's sibling %q failed", event.FailedSibling) 6190 } else { 6191 desc = "Task's sibling failed" 6192 } 6193 case TaskSignaling: 6194 sig := event.TaskSignal 6195 reason := event.TaskSignalReason 6196 6197 if sig == "" && reason == "" { 6198 desc = "Task being sent a signal" 6199 } else if sig == "" { 6200 desc = reason 6201 } else if reason == "" { 6202 desc = fmt.Sprintf("Task being sent signal %v", sig) 6203 } else { 6204 desc = fmt.Sprintf("Task being sent signal %v: %v", sig, reason) 6205 } 6206 case TaskRestartSignal: 6207 if event.RestartReason != "" { 6208 desc = event.RestartReason 6209 } else { 6210 desc = "Task signaled to restart" 6211 } 6212 case TaskDriverMessage: 6213 desc = event.DriverMessage 6214 case TaskLeaderDead: 6215 desc = "Leader Task in Group dead" 6216 default: 6217 desc = event.Message 6218 } 6219 6220 event.DisplayMessage = desc 6221 } 6222 6223 func (te *TaskEvent) GoString() string { 6224 return fmt.Sprintf("%v - %v", te.Time, te.Type) 6225 } 6226 6227 // SetDisplayMessage sets the display message of TaskEvent 6228 func (te *TaskEvent) SetDisplayMessage(msg string) *TaskEvent { 6229 te.DisplayMessage = msg 6230 return te 6231 } 6232 6233 // SetMessage sets the message of TaskEvent 6234 func (te *TaskEvent) SetMessage(msg string) *TaskEvent { 6235 te.Message = msg 6236 te.Details["message"] = msg 6237 return te 6238 } 6239 6240 func (te *TaskEvent) Copy() *TaskEvent { 6241 if te == nil { 6242 return nil 6243 } 6244 copy := new(TaskEvent) 6245 *copy = *te 6246 return copy 6247 } 6248 6249 func NewTaskEvent(event string) *TaskEvent { 6250 return &TaskEvent{ 6251 Type: event, 6252 Time: time.Now().UnixNano(), 6253 Details: make(map[string]string), 6254 } 6255 } 6256 6257 // SetSetupError is used to store an error that occurred while setting up the 6258 // task 6259 func (e *TaskEvent) SetSetupError(err error) *TaskEvent { 6260 if err != nil { 6261 e.SetupError = err.Error() 6262 e.Details["setup_error"] = err.Error() 6263 } 6264 return e 6265 } 6266 6267 func (e *TaskEvent) SetFailsTask() *TaskEvent { 6268 e.FailsTask = true 6269 e.Details["fails_task"] = "true" 6270 return e 6271 } 6272 6273 func (e *TaskEvent) SetDriverError(err error) *TaskEvent { 6274 if err != nil { 6275 e.DriverError = err.Error() 6276 e.Details["driver_error"] = err.Error() 6277 } 6278 return e 6279 } 6280 6281 func (e *TaskEvent) SetExitCode(c int) *TaskEvent { 6282 e.ExitCode = c 6283 e.Details["exit_code"] = fmt.Sprintf("%d", c) 6284 return e 6285 } 6286 6287 func (e *TaskEvent) SetSignal(s int) *TaskEvent { 6288 e.Signal = s 6289 e.Details["signal"] = fmt.Sprintf("%d", s) 6290 return e 6291 } 6292 6293 func (e *TaskEvent) SetSignalText(s string) *TaskEvent { 6294 e.Details["signal"] = s 6295 return e 6296 } 6297 6298 func (e *TaskEvent) SetExitMessage(err error) *TaskEvent { 6299 if err != nil { 6300 e.Message = err.Error() 6301 e.Details["exit_message"] = err.Error() 6302 } 6303 return e 6304 } 6305 6306 func (e *TaskEvent) SetKillError(err error) *TaskEvent { 6307 if err != nil { 6308 e.KillError = err.Error() 6309 e.Details["kill_error"] = err.Error() 6310 } 6311 return e 6312 } 6313 6314 func (e *TaskEvent) SetKillReason(r string) *TaskEvent { 6315 e.KillReason = r 6316 e.Details["kill_reason"] = r 6317 return e 6318 } 6319 6320 func (e *TaskEvent) SetRestartDelay(delay time.Duration) *TaskEvent { 6321 e.StartDelay = int64(delay) 6322 e.Details["start_delay"] = fmt.Sprintf("%d", delay) 6323 return e 6324 } 6325 6326 func (e *TaskEvent) SetRestartReason(reason string) *TaskEvent { 6327 e.RestartReason = reason 6328 e.Details["restart_reason"] = reason 6329 return e 6330 } 6331 6332 func (e *TaskEvent) SetTaskSignalReason(r string) *TaskEvent { 6333 e.TaskSignalReason = r 6334 e.Details["task_signal_reason"] = r 6335 return e 6336 } 6337 6338 func (e *TaskEvent) SetTaskSignal(s os.Signal) *TaskEvent { 6339 e.TaskSignal = s.String() 6340 e.Details["task_signal"] = s.String() 6341 return e 6342 } 6343 6344 func (e *TaskEvent) SetDownloadError(err error) *TaskEvent { 6345 if err != nil { 6346 e.DownloadError = err.Error() 6347 e.Details["download_error"] = err.Error() 6348 } 6349 return e 6350 } 6351 6352 func (e *TaskEvent) SetValidationError(err error) *TaskEvent { 6353 if err != nil { 6354 e.ValidationError = err.Error() 6355 e.Details["validation_error"] = err.Error() 6356 } 6357 return e 6358 } 6359 6360 func (e *TaskEvent) SetKillTimeout(timeout time.Duration) *TaskEvent { 6361 e.KillTimeout = timeout 6362 e.Details["kill_timeout"] = timeout.String() 6363 return e 6364 } 6365 6366 func (e *TaskEvent) SetDiskLimit(limit int64) *TaskEvent { 6367 e.DiskLimit = limit 6368 e.Details["disk_limit"] = fmt.Sprintf("%d", limit) 6369 return e 6370 } 6371 6372 func (e *TaskEvent) SetFailedSibling(sibling string) *TaskEvent { 6373 e.FailedSibling = sibling 6374 e.Details["failed_sibling"] = sibling 6375 return e 6376 } 6377 6378 func (e *TaskEvent) SetVaultRenewalError(err error) *TaskEvent { 6379 if err != nil { 6380 e.VaultError = err.Error() 6381 e.Details["vault_renewal_error"] = err.Error() 6382 } 6383 return e 6384 } 6385 6386 func (e *TaskEvent) SetDriverMessage(m string) *TaskEvent { 6387 e.DriverMessage = m 6388 e.Details["driver_message"] = m 6389 return e 6390 } 6391 6392 func (e *TaskEvent) SetOOMKilled(oom bool) *TaskEvent { 6393 e.Details["oom_killed"] = strconv.FormatBool(oom) 6394 return e 6395 } 6396 6397 // TaskArtifact is an artifact to download before running the task. 6398 type TaskArtifact struct { 6399 // GetterSource is the source to download an artifact using go-getter 6400 GetterSource string 6401 6402 // GetterOptions are options to use when downloading the artifact using 6403 // go-getter. 6404 GetterOptions map[string]string 6405 6406 // GetterMode is the go-getter.ClientMode for fetching resources. 6407 // Defaults to "any" but can be set to "file" or "dir". 6408 GetterMode string 6409 6410 // RelativeDest is the download destination given relative to the task's 6411 // directory. 6412 RelativeDest string 6413 } 6414 6415 func (ta *TaskArtifact) Copy() *TaskArtifact { 6416 if ta == nil { 6417 return nil 6418 } 6419 nta := new(TaskArtifact) 6420 *nta = *ta 6421 nta.GetterOptions = helper.CopyMapStringString(ta.GetterOptions) 6422 return nta 6423 } 6424 6425 func (ta *TaskArtifact) GoString() string { 6426 return fmt.Sprintf("%+v", ta) 6427 } 6428 6429 // Hash creates a unique identifier for a TaskArtifact as the same GetterSource 6430 // may be specified multiple times with different destinations. 6431 func (ta *TaskArtifact) Hash() string { 6432 hash, err := blake2b.New256(nil) 6433 if err != nil { 6434 panic(err) 6435 } 6436 6437 hash.Write([]byte(ta.GetterSource)) 6438 6439 // Must iterate over keys in a consistent order 6440 keys := make([]string, 0, len(ta.GetterOptions)) 6441 for k := range ta.GetterOptions { 6442 keys = append(keys, k) 6443 } 6444 sort.Strings(keys) 6445 for _, k := range keys { 6446 hash.Write([]byte(k)) 6447 hash.Write([]byte(ta.GetterOptions[k])) 6448 } 6449 6450 hash.Write([]byte(ta.GetterMode)) 6451 hash.Write([]byte(ta.RelativeDest)) 6452 return base64.RawStdEncoding.EncodeToString(hash.Sum(nil)) 6453 } 6454 6455 // PathEscapesAllocDir returns if the given path escapes the allocation 6456 // directory. The prefix allows adding a prefix if the path will be joined, for 6457 // example a "task/local" prefix may be provided if the path will be joined 6458 // against that prefix. 6459 func PathEscapesAllocDir(prefix, path string) (bool, error) { 6460 // Verify the destination doesn't escape the tasks directory 6461 alloc, err := filepath.Abs(filepath.Join("/", "alloc-dir/", "alloc-id/")) 6462 if err != nil { 6463 return false, err 6464 } 6465 abs, err := filepath.Abs(filepath.Join(alloc, prefix, path)) 6466 if err != nil { 6467 return false, err 6468 } 6469 rel, err := filepath.Rel(alloc, abs) 6470 if err != nil { 6471 return false, err 6472 } 6473 6474 return strings.HasPrefix(rel, ".."), nil 6475 } 6476 6477 func (ta *TaskArtifact) Validate() error { 6478 // Verify the source 6479 var mErr multierror.Error 6480 if ta.GetterSource == "" { 6481 mErr.Errors = append(mErr.Errors, fmt.Errorf("source must be specified")) 6482 } 6483 6484 switch ta.GetterMode { 6485 case "": 6486 // Default to any 6487 ta.GetterMode = GetterModeAny 6488 case GetterModeAny, GetterModeFile, GetterModeDir: 6489 // Ok 6490 default: 6491 mErr.Errors = append(mErr.Errors, fmt.Errorf("invalid artifact mode %q; must be one of: %s, %s, %s", 6492 ta.GetterMode, GetterModeAny, GetterModeFile, GetterModeDir)) 6493 } 6494 6495 escaped, err := PathEscapesAllocDir("task", ta.RelativeDest) 6496 if err != nil { 6497 mErr.Errors = append(mErr.Errors, fmt.Errorf("invalid destination path: %v", err)) 6498 } else if escaped { 6499 mErr.Errors = append(mErr.Errors, fmt.Errorf("destination escapes allocation directory")) 6500 } 6501 6502 if err := ta.validateChecksum(); err != nil { 6503 mErr.Errors = append(mErr.Errors, err) 6504 } 6505 6506 return mErr.ErrorOrNil() 6507 } 6508 6509 func (ta *TaskArtifact) validateChecksum() error { 6510 check, ok := ta.GetterOptions["checksum"] 6511 if !ok { 6512 return nil 6513 } 6514 6515 // Job struct validation occurs before interpolation resolution can be effective. 6516 // Skip checking if checksum contain variable reference, and artifacts fetching will 6517 // eventually fail, if checksum is indeed invalid. 6518 if args.ContainsEnv(check) { 6519 return nil 6520 } 6521 6522 check = strings.TrimSpace(check) 6523 if check == "" { 6524 return fmt.Errorf("checksum value cannot be empty") 6525 } 6526 6527 parts := strings.Split(check, ":") 6528 if l := len(parts); l != 2 { 6529 return fmt.Errorf(`checksum must be given as "type:value"; got %q`, check) 6530 } 6531 6532 checksumVal := parts[1] 6533 checksumBytes, err := hex.DecodeString(checksumVal) 6534 if err != nil { 6535 return fmt.Errorf("invalid checksum: %v", err) 6536 } 6537 6538 checksumType := parts[0] 6539 expectedLength := 0 6540 switch checksumType { 6541 case "md5": 6542 expectedLength = md5.Size 6543 case "sha1": 6544 expectedLength = sha1.Size 6545 case "sha256": 6546 expectedLength = sha256.Size 6547 case "sha512": 6548 expectedLength = sha512.Size 6549 default: 6550 return fmt.Errorf("unsupported checksum type: %s", checksumType) 6551 } 6552 6553 if len(checksumBytes) != expectedLength { 6554 return fmt.Errorf("invalid %s checksum: %v", checksumType, checksumVal) 6555 } 6556 6557 return nil 6558 } 6559 6560 const ( 6561 ConstraintDistinctProperty = "distinct_property" 6562 ConstraintDistinctHosts = "distinct_hosts" 6563 ConstraintRegex = "regexp" 6564 ConstraintVersion = "version" 6565 ConstraintSetContains = "set_contains" 6566 ConstraintSetContainsAll = "set_contains_all" 6567 ConstraintSetContainsAny = "set_contains_any" 6568 ConstraintAttributeIsSet = "is_set" 6569 ConstraintAttributeIsNotSet = "is_not_set" 6570 ) 6571 6572 // Constraints are used to restrict placement options. 6573 type Constraint struct { 6574 LTarget string // Left-hand target 6575 RTarget string // Right-hand target 6576 Operand string // Constraint operand (<=, <, =, !=, >, >=), contains, near 6577 str string // Memoized string 6578 } 6579 6580 // Equal checks if two constraints are equal 6581 func (c *Constraint) Equals(o *Constraint) bool { 6582 return c == o || 6583 c.LTarget == o.LTarget && 6584 c.RTarget == o.RTarget && 6585 c.Operand == o.Operand 6586 } 6587 6588 func (c *Constraint) Equal(o *Constraint) bool { 6589 return c.Equals(o) 6590 } 6591 6592 func (c *Constraint) Copy() *Constraint { 6593 if c == nil { 6594 return nil 6595 } 6596 nc := new(Constraint) 6597 *nc = *c 6598 return nc 6599 } 6600 6601 func (c *Constraint) String() string { 6602 if c.str != "" { 6603 return c.str 6604 } 6605 c.str = fmt.Sprintf("%s %s %s", c.LTarget, c.Operand, c.RTarget) 6606 return c.str 6607 } 6608 6609 func (c *Constraint) Validate() error { 6610 var mErr multierror.Error 6611 if c.Operand == "" { 6612 mErr.Errors = append(mErr.Errors, errors.New("Missing constraint operand")) 6613 } 6614 6615 // requireLtarget specifies whether the constraint requires an LTarget to be 6616 // provided. 6617 requireLtarget := true 6618 6619 // Perform additional validation based on operand 6620 switch c.Operand { 6621 case ConstraintDistinctHosts: 6622 requireLtarget = false 6623 case ConstraintSetContainsAll, ConstraintSetContainsAny, ConstraintSetContains: 6624 if c.RTarget == "" { 6625 mErr.Errors = append(mErr.Errors, fmt.Errorf("Set contains constraint requires an RTarget")) 6626 } 6627 case ConstraintRegex: 6628 if _, err := regexp.Compile(c.RTarget); err != nil { 6629 mErr.Errors = append(mErr.Errors, fmt.Errorf("Regular expression failed to compile: %v", err)) 6630 } 6631 case ConstraintVersion: 6632 if _, err := version.NewConstraint(c.RTarget); err != nil { 6633 mErr.Errors = append(mErr.Errors, fmt.Errorf("Version constraint is invalid: %v", err)) 6634 } 6635 case ConstraintDistinctProperty: 6636 // If a count is set, make sure it is convertible to a uint64 6637 if c.RTarget != "" { 6638 count, err := strconv.ParseUint(c.RTarget, 10, 64) 6639 if err != nil { 6640 mErr.Errors = append(mErr.Errors, fmt.Errorf("Failed to convert RTarget %q to uint64: %v", c.RTarget, err)) 6641 } else if count < 1 { 6642 mErr.Errors = append(mErr.Errors, fmt.Errorf("Distinct Property must have an allowed count of 1 or greater: %d < 1", count)) 6643 } 6644 } 6645 case ConstraintAttributeIsSet, ConstraintAttributeIsNotSet: 6646 if c.RTarget != "" { 6647 mErr.Errors = append(mErr.Errors, fmt.Errorf("Operator %q does not support an RTarget", c.Operand)) 6648 } 6649 case "=", "==", "is", "!=", "not", "<", "<=", ">", ">=": 6650 if c.RTarget == "" { 6651 mErr.Errors = append(mErr.Errors, fmt.Errorf("Operator %q requires an RTarget", c.Operand)) 6652 } 6653 default: 6654 mErr.Errors = append(mErr.Errors, fmt.Errorf("Unknown constraint type %q", c.Operand)) 6655 } 6656 6657 // Ensure we have an LTarget for the constraints that need one 6658 if requireLtarget && c.LTarget == "" { 6659 mErr.Errors = append(mErr.Errors, fmt.Errorf("No LTarget provided but is required by constraint")) 6660 } 6661 6662 return mErr.ErrorOrNil() 6663 } 6664 6665 type Constraints []*Constraint 6666 6667 // Equals compares Constraints as a set 6668 func (xs *Constraints) Equals(ys *Constraints) bool { 6669 if xs == ys { 6670 return true 6671 } 6672 if xs == nil || ys == nil { 6673 return false 6674 } 6675 if len(*xs) != len(*ys) { 6676 return false 6677 } 6678 SETEQUALS: 6679 for _, x := range *xs { 6680 for _, y := range *ys { 6681 if x.Equals(y) { 6682 continue SETEQUALS 6683 } 6684 } 6685 return false 6686 } 6687 return true 6688 } 6689 6690 // Affinity is used to score placement options based on a weight 6691 type Affinity struct { 6692 LTarget string // Left-hand target 6693 RTarget string // Right-hand target 6694 Operand string // Affinity operand (<=, <, =, !=, >, >=), set_contains_all, set_contains_any 6695 Weight int8 // Weight applied to nodes that match the affinity. Can be negative 6696 str string // Memoized string 6697 } 6698 6699 // Equal checks if two affinities are equal 6700 func (a *Affinity) Equals(o *Affinity) bool { 6701 return a == o || 6702 a.LTarget == o.LTarget && 6703 a.RTarget == o.RTarget && 6704 a.Operand == o.Operand && 6705 a.Weight == o.Weight 6706 } 6707 6708 func (a *Affinity) Equal(o *Affinity) bool { 6709 return a.Equals(o) 6710 } 6711 6712 func (a *Affinity) Copy() *Affinity { 6713 if a == nil { 6714 return nil 6715 } 6716 na := new(Affinity) 6717 *na = *a 6718 return na 6719 } 6720 6721 func (a *Affinity) String() string { 6722 if a.str != "" { 6723 return a.str 6724 } 6725 a.str = fmt.Sprintf("%s %s %s %v", a.LTarget, a.Operand, a.RTarget, a.Weight) 6726 return a.str 6727 } 6728 6729 func (a *Affinity) Validate() error { 6730 var mErr multierror.Error 6731 if a.Operand == "" { 6732 mErr.Errors = append(mErr.Errors, errors.New("Missing affinity operand")) 6733 } 6734 6735 // Perform additional validation based on operand 6736 switch a.Operand { 6737 case ConstraintSetContainsAll, ConstraintSetContainsAny, ConstraintSetContains: 6738 if a.RTarget == "" { 6739 mErr.Errors = append(mErr.Errors, fmt.Errorf("Set contains operators require an RTarget")) 6740 } 6741 case ConstraintRegex: 6742 if _, err := regexp.Compile(a.RTarget); err != nil { 6743 mErr.Errors = append(mErr.Errors, fmt.Errorf("Regular expression failed to compile: %v", err)) 6744 } 6745 case ConstraintVersion: 6746 if _, err := version.NewConstraint(a.RTarget); err != nil { 6747 mErr.Errors = append(mErr.Errors, fmt.Errorf("Version affinity is invalid: %v", err)) 6748 } 6749 case "=", "==", "is", "!=", "not", "<", "<=", ">", ">=": 6750 if a.RTarget == "" { 6751 mErr.Errors = append(mErr.Errors, fmt.Errorf("Operator %q requires an RTarget", a.Operand)) 6752 } 6753 default: 6754 mErr.Errors = append(mErr.Errors, fmt.Errorf("Unknown affinity operator %q", a.Operand)) 6755 } 6756 6757 // Ensure we have an LTarget 6758 if a.LTarget == "" { 6759 mErr.Errors = append(mErr.Errors, fmt.Errorf("No LTarget provided but is required")) 6760 } 6761 6762 // Ensure that weight is between -100 and 100, and not zero 6763 if a.Weight == 0 { 6764 mErr.Errors = append(mErr.Errors, fmt.Errorf("Affinity weight cannot be zero")) 6765 } 6766 6767 if a.Weight > 100 || a.Weight < -100 { 6768 mErr.Errors = append(mErr.Errors, fmt.Errorf("Affinity weight must be within the range [-100,100]")) 6769 } 6770 6771 return mErr.ErrorOrNil() 6772 } 6773 6774 // Spread is used to specify desired distribution of allocations according to weight 6775 type Spread struct { 6776 // Attribute is the node attribute used as the spread criteria 6777 Attribute string 6778 6779 // Weight is the relative weight of this spread, useful when there are multiple 6780 // spread and affinities 6781 Weight int8 6782 6783 // SpreadTarget is used to describe desired percentages for each attribute value 6784 SpreadTarget []*SpreadTarget 6785 6786 // Memoized string representation 6787 str string 6788 } 6789 6790 type Affinities []*Affinity 6791 6792 // Equals compares Affinities as a set 6793 func (xs *Affinities) Equals(ys *Affinities) bool { 6794 if xs == ys { 6795 return true 6796 } 6797 if xs == nil || ys == nil { 6798 return false 6799 } 6800 if len(*xs) != len(*ys) { 6801 return false 6802 } 6803 SETEQUALS: 6804 for _, x := range *xs { 6805 for _, y := range *ys { 6806 if x.Equals(y) { 6807 continue SETEQUALS 6808 } 6809 } 6810 return false 6811 } 6812 return true 6813 } 6814 6815 func (s *Spread) Copy() *Spread { 6816 if s == nil { 6817 return nil 6818 } 6819 ns := new(Spread) 6820 *ns = *s 6821 6822 ns.SpreadTarget = CopySliceSpreadTarget(s.SpreadTarget) 6823 return ns 6824 } 6825 6826 func (s *Spread) String() string { 6827 if s.str != "" { 6828 return s.str 6829 } 6830 s.str = fmt.Sprintf("%s %s %v", s.Attribute, s.SpreadTarget, s.Weight) 6831 return s.str 6832 } 6833 6834 func (s *Spread) Validate() error { 6835 var mErr multierror.Error 6836 if s.Attribute == "" { 6837 mErr.Errors = append(mErr.Errors, errors.New("Missing spread attribute")) 6838 } 6839 if s.Weight <= 0 || s.Weight > 100 { 6840 mErr.Errors = append(mErr.Errors, errors.New("Spread stanza must have a positive weight from 0 to 100")) 6841 } 6842 seen := make(map[string]struct{}) 6843 sumPercent := uint32(0) 6844 6845 for _, target := range s.SpreadTarget { 6846 // Make sure there are no duplicates 6847 _, ok := seen[target.Value] 6848 if !ok { 6849 seen[target.Value] = struct{}{} 6850 } else { 6851 mErr.Errors = append(mErr.Errors, errors.New(fmt.Sprintf("Spread target value %q already defined", target.Value))) 6852 } 6853 if target.Percent < 0 || target.Percent > 100 { 6854 mErr.Errors = append(mErr.Errors, errors.New(fmt.Sprintf("Spread target percentage for value %q must be between 0 and 100", target.Value))) 6855 } 6856 sumPercent += uint32(target.Percent) 6857 } 6858 if sumPercent > 100 { 6859 mErr.Errors = append(mErr.Errors, errors.New(fmt.Sprintf("Sum of spread target percentages must not be greater than 100%%; got %d%%", sumPercent))) 6860 } 6861 return mErr.ErrorOrNil() 6862 } 6863 6864 // SpreadTarget is used to specify desired percentages for each attribute value 6865 type SpreadTarget struct { 6866 // Value is a single attribute value, like "dc1" 6867 Value string 6868 6869 // Percent is the desired percentage of allocs 6870 Percent uint8 6871 6872 // Memoized string representation 6873 str string 6874 } 6875 6876 func (s *SpreadTarget) Copy() *SpreadTarget { 6877 if s == nil { 6878 return nil 6879 } 6880 6881 ns := new(SpreadTarget) 6882 *ns = *s 6883 return ns 6884 } 6885 6886 func (s *SpreadTarget) String() string { 6887 if s.str != "" { 6888 return s.str 6889 } 6890 s.str = fmt.Sprintf("%q %v%%", s.Value, s.Percent) 6891 return s.str 6892 } 6893 6894 // EphemeralDisk is an ephemeral disk object 6895 type EphemeralDisk struct { 6896 // Sticky indicates whether the allocation is sticky to a node 6897 Sticky bool 6898 6899 // SizeMB is the size of the local disk 6900 SizeMB int 6901 6902 // Migrate determines if Nomad client should migrate the allocation dir for 6903 // sticky allocations 6904 Migrate bool 6905 } 6906 6907 // DefaultEphemeralDisk returns a EphemeralDisk with default configurations 6908 func DefaultEphemeralDisk() *EphemeralDisk { 6909 return &EphemeralDisk{ 6910 SizeMB: 300, 6911 } 6912 } 6913 6914 // Validate validates EphemeralDisk 6915 func (d *EphemeralDisk) Validate() error { 6916 if d.SizeMB < 10 { 6917 return fmt.Errorf("minimum DiskMB value is 10; got %d", d.SizeMB) 6918 } 6919 return nil 6920 } 6921 6922 // Copy copies the EphemeralDisk struct and returns a new one 6923 func (d *EphemeralDisk) Copy() *EphemeralDisk { 6924 ld := new(EphemeralDisk) 6925 *ld = *d 6926 return ld 6927 } 6928 6929 var ( 6930 // VaultUnrecoverableError matches unrecoverable errors returned by a Vault 6931 // server 6932 VaultUnrecoverableError = regexp.MustCompile(`Code:\s+40(0|3|4)`) 6933 ) 6934 6935 const ( 6936 // VaultChangeModeNoop takes no action when a new token is retrieved. 6937 VaultChangeModeNoop = "noop" 6938 6939 // VaultChangeModeSignal signals the task when a new token is retrieved. 6940 VaultChangeModeSignal = "signal" 6941 6942 // VaultChangeModeRestart restarts the task when a new token is retrieved. 6943 VaultChangeModeRestart = "restart" 6944 ) 6945 6946 // Vault stores the set of permissions a task needs access to from Vault. 6947 type Vault struct { 6948 // Policies is the set of policies that the task needs access to 6949 Policies []string 6950 6951 // Env marks whether the Vault Token should be exposed as an environment 6952 // variable 6953 Env bool 6954 6955 // ChangeMode is used to configure the task's behavior when the Vault 6956 // token changes because the original token could not be renewed in time. 6957 ChangeMode string 6958 6959 // ChangeSignal is the signal sent to the task when a new token is 6960 // retrieved. This is only valid when using the signal change mode. 6961 ChangeSignal string 6962 } 6963 6964 func DefaultVaultBlock() *Vault { 6965 return &Vault{ 6966 Env: true, 6967 ChangeMode: VaultChangeModeRestart, 6968 } 6969 } 6970 6971 // Copy returns a copy of this Vault block. 6972 func (v *Vault) Copy() *Vault { 6973 if v == nil { 6974 return nil 6975 } 6976 6977 nv := new(Vault) 6978 *nv = *v 6979 return nv 6980 } 6981 6982 func (v *Vault) Canonicalize() { 6983 if v.ChangeSignal != "" { 6984 v.ChangeSignal = strings.ToUpper(v.ChangeSignal) 6985 } 6986 } 6987 6988 // Validate returns if the Vault block is valid. 6989 func (v *Vault) Validate() error { 6990 if v == nil { 6991 return nil 6992 } 6993 6994 var mErr multierror.Error 6995 if len(v.Policies) == 0 { 6996 multierror.Append(&mErr, fmt.Errorf("Policy list cannot be empty")) 6997 } 6998 6999 for _, p := range v.Policies { 7000 if p == "root" { 7001 multierror.Append(&mErr, fmt.Errorf("Can not specify \"root\" policy")) 7002 } 7003 } 7004 7005 switch v.ChangeMode { 7006 case VaultChangeModeSignal: 7007 if v.ChangeSignal == "" { 7008 multierror.Append(&mErr, fmt.Errorf("Signal must be specified when using change mode %q", VaultChangeModeSignal)) 7009 } 7010 case VaultChangeModeNoop, VaultChangeModeRestart: 7011 default: 7012 multierror.Append(&mErr, fmt.Errorf("Unknown change mode %q", v.ChangeMode)) 7013 } 7014 7015 return mErr.ErrorOrNil() 7016 } 7017 7018 const ( 7019 // DeploymentStatuses are the various states a deployment can be be in 7020 DeploymentStatusRunning = "running" 7021 DeploymentStatusPaused = "paused" 7022 DeploymentStatusFailed = "failed" 7023 DeploymentStatusSuccessful = "successful" 7024 DeploymentStatusCancelled = "cancelled" 7025 7026 // TODO Statuses and Descriptions do not match 1:1 and we sometimes use the Description as a status flag 7027 7028 // DeploymentStatusDescriptions are the various descriptions of the states a 7029 // deployment can be in. 7030 DeploymentStatusDescriptionRunning = "Deployment is running" 7031 DeploymentStatusDescriptionRunningNeedsPromotion = "Deployment is running but requires manual promotion" 7032 DeploymentStatusDescriptionRunningAutoPromotion = "Deployment is running pending automatic promotion" 7033 DeploymentStatusDescriptionPaused = "Deployment is paused" 7034 DeploymentStatusDescriptionSuccessful = "Deployment completed successfully" 7035 DeploymentStatusDescriptionStoppedJob = "Cancelled because job is stopped" 7036 DeploymentStatusDescriptionNewerJob = "Cancelled due to newer version of job" 7037 DeploymentStatusDescriptionFailedAllocations = "Failed due to unhealthy allocations" 7038 DeploymentStatusDescriptionProgressDeadline = "Failed due to progress deadline" 7039 DeploymentStatusDescriptionFailedByUser = "Deployment marked as failed" 7040 ) 7041 7042 // DeploymentStatusDescriptionRollback is used to get the status description of 7043 // a deployment when rolling back to an older job. 7044 func DeploymentStatusDescriptionRollback(baseDescription string, jobVersion uint64) string { 7045 return fmt.Sprintf("%s - rolling back to job version %d", baseDescription, jobVersion) 7046 } 7047 7048 // DeploymentStatusDescriptionRollbackNoop is used to get the status description of 7049 // a deployment when rolling back is not possible because it has the same specification 7050 func DeploymentStatusDescriptionRollbackNoop(baseDescription string, jobVersion uint64) string { 7051 return fmt.Sprintf("%s - not rolling back to stable job version %d as current job has same specification", baseDescription, jobVersion) 7052 } 7053 7054 // DeploymentStatusDescriptionNoRollbackTarget is used to get the status description of 7055 // a deployment when there is no target to rollback to but autorevert is desired. 7056 func DeploymentStatusDescriptionNoRollbackTarget(baseDescription string) string { 7057 return fmt.Sprintf("%s - no stable job version to auto revert to", baseDescription) 7058 } 7059 7060 // Deployment is the object that represents a job deployment which is used to 7061 // transition a job between versions. 7062 type Deployment struct { 7063 // ID is a generated UUID for the deployment 7064 ID string 7065 7066 // Namespace is the namespace the deployment is created in 7067 Namespace string 7068 7069 // JobID is the job the deployment is created for 7070 JobID string 7071 7072 // JobVersion is the version of the job at which the deployment is tracking 7073 JobVersion uint64 7074 7075 // JobModifyIndex is the ModifyIndex of the job which the deployment is 7076 // tracking. 7077 JobModifyIndex uint64 7078 7079 // JobSpecModifyIndex is the JobModifyIndex of the job which the 7080 // deployment is tracking. 7081 JobSpecModifyIndex uint64 7082 7083 // JobCreateIndex is the create index of the job which the deployment is 7084 // tracking. It is needed so that if the job gets stopped and reran we can 7085 // present the correct list of deployments for the job and not old ones. 7086 JobCreateIndex uint64 7087 7088 // TaskGroups is the set of task groups effected by the deployment and their 7089 // current deployment status. 7090 TaskGroups map[string]*DeploymentState 7091 7092 // The status of the deployment 7093 Status string 7094 7095 // StatusDescription allows a human readable description of the deployment 7096 // status. 7097 StatusDescription string 7098 7099 CreateIndex uint64 7100 ModifyIndex uint64 7101 } 7102 7103 // NewDeployment creates a new deployment given the job. 7104 func NewDeployment(job *Job) *Deployment { 7105 return &Deployment{ 7106 ID: uuid.Generate(), 7107 Namespace: job.Namespace, 7108 JobID: job.ID, 7109 JobVersion: job.Version, 7110 JobModifyIndex: job.ModifyIndex, 7111 JobSpecModifyIndex: job.JobModifyIndex, 7112 JobCreateIndex: job.CreateIndex, 7113 Status: DeploymentStatusRunning, 7114 StatusDescription: DeploymentStatusDescriptionRunning, 7115 TaskGroups: make(map[string]*DeploymentState, len(job.TaskGroups)), 7116 } 7117 } 7118 7119 func (d *Deployment) Copy() *Deployment { 7120 if d == nil { 7121 return nil 7122 } 7123 7124 c := &Deployment{} 7125 *c = *d 7126 7127 c.TaskGroups = nil 7128 if l := len(d.TaskGroups); d.TaskGroups != nil { 7129 c.TaskGroups = make(map[string]*DeploymentState, l) 7130 for tg, s := range d.TaskGroups { 7131 c.TaskGroups[tg] = s.Copy() 7132 } 7133 } 7134 7135 return c 7136 } 7137 7138 // Active returns whether the deployment is active or terminal. 7139 func (d *Deployment) Active() bool { 7140 switch d.Status { 7141 case DeploymentStatusRunning, DeploymentStatusPaused: 7142 return true 7143 default: 7144 return false 7145 } 7146 } 7147 7148 // GetID is a helper for getting the ID when the object may be nil 7149 func (d *Deployment) GetID() string { 7150 if d == nil { 7151 return "" 7152 } 7153 return d.ID 7154 } 7155 7156 // HasPlacedCanaries returns whether the deployment has placed canaries 7157 func (d *Deployment) HasPlacedCanaries() bool { 7158 if d == nil || len(d.TaskGroups) == 0 { 7159 return false 7160 } 7161 for _, group := range d.TaskGroups { 7162 if len(group.PlacedCanaries) != 0 { 7163 return true 7164 } 7165 } 7166 return false 7167 } 7168 7169 // RequiresPromotion returns whether the deployment requires promotion to 7170 // continue 7171 func (d *Deployment) RequiresPromotion() bool { 7172 if d == nil || len(d.TaskGroups) == 0 || d.Status != DeploymentStatusRunning { 7173 return false 7174 } 7175 for _, group := range d.TaskGroups { 7176 if group.DesiredCanaries > 0 && !group.Promoted { 7177 return true 7178 } 7179 } 7180 return false 7181 } 7182 7183 // HasAutoPromote determines if all taskgroups are marked auto_promote 7184 func (d *Deployment) HasAutoPromote() bool { 7185 if d == nil || len(d.TaskGroups) == 0 || d.Status != DeploymentStatusRunning { 7186 return false 7187 } 7188 for _, group := range d.TaskGroups { 7189 if !group.AutoPromote { 7190 return false 7191 } 7192 } 7193 return true 7194 } 7195 7196 func (d *Deployment) GoString() string { 7197 base := fmt.Sprintf("Deployment ID %q for job %q has status %q (%v):", d.ID, d.JobID, d.Status, d.StatusDescription) 7198 for group, state := range d.TaskGroups { 7199 base += fmt.Sprintf("\nTask Group %q has state:\n%#v", group, state) 7200 } 7201 return base 7202 } 7203 7204 // DeploymentState tracks the state of a deployment for a given task group. 7205 type DeploymentState struct { 7206 // AutoRevert marks whether the task group has indicated the job should be 7207 // reverted on failure 7208 AutoRevert bool 7209 7210 // AutoPromote marks promotion triggered automatically by healthy canaries 7211 // copied from TaskGroup UpdateStrategy in scheduler.reconcile 7212 AutoPromote bool 7213 7214 // ProgressDeadline is the deadline by which an allocation must transition 7215 // to healthy before the deployment is considered failed. 7216 ProgressDeadline time.Duration 7217 7218 // RequireProgressBy is the time by which an allocation must transition 7219 // to healthy before the deployment is considered failed. 7220 RequireProgressBy time.Time 7221 7222 // Promoted marks whether the canaries have been promoted 7223 Promoted bool 7224 7225 // PlacedCanaries is the set of placed canary allocations 7226 PlacedCanaries []string 7227 7228 // DesiredCanaries is the number of canaries that should be created. 7229 DesiredCanaries int 7230 7231 // DesiredTotal is the total number of allocations that should be created as 7232 // part of the deployment. 7233 DesiredTotal int 7234 7235 // PlacedAllocs is the number of allocations that have been placed 7236 PlacedAllocs int 7237 7238 // HealthyAllocs is the number of allocations that have been marked healthy. 7239 HealthyAllocs int 7240 7241 // UnhealthyAllocs are allocations that have been marked as unhealthy. 7242 UnhealthyAllocs int 7243 } 7244 7245 func (d *DeploymentState) GoString() string { 7246 base := fmt.Sprintf("\tDesired Total: %d", d.DesiredTotal) 7247 base += fmt.Sprintf("\n\tDesired Canaries: %d", d.DesiredCanaries) 7248 base += fmt.Sprintf("\n\tPlaced Canaries: %#v", d.PlacedCanaries) 7249 base += fmt.Sprintf("\n\tPromoted: %v", d.Promoted) 7250 base += fmt.Sprintf("\n\tPlaced: %d", d.PlacedAllocs) 7251 base += fmt.Sprintf("\n\tHealthy: %d", d.HealthyAllocs) 7252 base += fmt.Sprintf("\n\tUnhealthy: %d", d.UnhealthyAllocs) 7253 base += fmt.Sprintf("\n\tAutoRevert: %v", d.AutoRevert) 7254 base += fmt.Sprintf("\n\tAutoPromote: %v", d.AutoPromote) 7255 return base 7256 } 7257 7258 func (d *DeploymentState) Copy() *DeploymentState { 7259 c := &DeploymentState{} 7260 *c = *d 7261 c.PlacedCanaries = helper.CopySliceString(d.PlacedCanaries) 7262 return c 7263 } 7264 7265 // DeploymentStatusUpdate is used to update the status of a given deployment 7266 type DeploymentStatusUpdate struct { 7267 // DeploymentID is the ID of the deployment to update 7268 DeploymentID string 7269 7270 // Status is the new status of the deployment. 7271 Status string 7272 7273 // StatusDescription is the new status description of the deployment. 7274 StatusDescription string 7275 } 7276 7277 // RescheduleTracker encapsulates previous reschedule events 7278 type RescheduleTracker struct { 7279 Events []*RescheduleEvent 7280 } 7281 7282 func (rt *RescheduleTracker) Copy() *RescheduleTracker { 7283 if rt == nil { 7284 return nil 7285 } 7286 nt := &RescheduleTracker{} 7287 *nt = *rt 7288 rescheduleEvents := make([]*RescheduleEvent, 0, len(rt.Events)) 7289 for _, tracker := range rt.Events { 7290 rescheduleEvents = append(rescheduleEvents, tracker.Copy()) 7291 } 7292 nt.Events = rescheduleEvents 7293 return nt 7294 } 7295 7296 // RescheduleEvent is used to keep track of previous attempts at rescheduling an allocation 7297 type RescheduleEvent struct { 7298 // RescheduleTime is the timestamp of a reschedule attempt 7299 RescheduleTime int64 7300 7301 // PrevAllocID is the ID of the previous allocation being restarted 7302 PrevAllocID string 7303 7304 // PrevNodeID is the node ID of the previous allocation 7305 PrevNodeID string 7306 7307 // Delay is the reschedule delay associated with the attempt 7308 Delay time.Duration 7309 } 7310 7311 func NewRescheduleEvent(rescheduleTime int64, prevAllocID string, prevNodeID string, delay time.Duration) *RescheduleEvent { 7312 return &RescheduleEvent{RescheduleTime: rescheduleTime, 7313 PrevAllocID: prevAllocID, 7314 PrevNodeID: prevNodeID, 7315 Delay: delay} 7316 } 7317 7318 func (re *RescheduleEvent) Copy() *RescheduleEvent { 7319 if re == nil { 7320 return nil 7321 } 7322 copy := new(RescheduleEvent) 7323 *copy = *re 7324 return copy 7325 } 7326 7327 // DesiredTransition is used to mark an allocation as having a desired state 7328 // transition. This information can be used by the scheduler to make the 7329 // correct decision. 7330 type DesiredTransition struct { 7331 // Migrate is used to indicate that this allocation should be stopped and 7332 // migrated to another node. 7333 Migrate *bool 7334 7335 // Reschedule is used to indicate that this allocation is eligible to be 7336 // rescheduled. Most allocations are automatically eligible for 7337 // rescheduling, so this field is only required when an allocation is not 7338 // automatically eligible. An example is an allocation that is part of a 7339 // deployment. 7340 Reschedule *bool 7341 7342 // ForceReschedule is used to indicate that this allocation must be rescheduled. 7343 // This field is only used when operators want to force a placement even if 7344 // a failed allocation is not eligible to be rescheduled 7345 ForceReschedule *bool 7346 } 7347 7348 // Merge merges the two desired transitions, preferring the values from the 7349 // passed in object. 7350 func (d *DesiredTransition) Merge(o *DesiredTransition) { 7351 if o.Migrate != nil { 7352 d.Migrate = o.Migrate 7353 } 7354 7355 if o.Reschedule != nil { 7356 d.Reschedule = o.Reschedule 7357 } 7358 7359 if o.ForceReschedule != nil { 7360 d.ForceReschedule = o.ForceReschedule 7361 } 7362 } 7363 7364 // ShouldMigrate returns whether the transition object dictates a migration. 7365 func (d *DesiredTransition) ShouldMigrate() bool { 7366 return d.Migrate != nil && *d.Migrate 7367 } 7368 7369 // ShouldReschedule returns whether the transition object dictates a 7370 // rescheduling. 7371 func (d *DesiredTransition) ShouldReschedule() bool { 7372 return d.Reschedule != nil && *d.Reschedule 7373 } 7374 7375 // ShouldForceReschedule returns whether the transition object dictates a 7376 // forced rescheduling. 7377 func (d *DesiredTransition) ShouldForceReschedule() bool { 7378 if d == nil { 7379 return false 7380 } 7381 return d.ForceReschedule != nil && *d.ForceReschedule 7382 } 7383 7384 const ( 7385 AllocDesiredStatusRun = "run" // Allocation should run 7386 AllocDesiredStatusStop = "stop" // Allocation should stop 7387 AllocDesiredStatusEvict = "evict" // Allocation should stop, and was evicted 7388 ) 7389 7390 const ( 7391 AllocClientStatusPending = "pending" 7392 AllocClientStatusRunning = "running" 7393 AllocClientStatusComplete = "complete" 7394 AllocClientStatusFailed = "failed" 7395 AllocClientStatusLost = "lost" 7396 ) 7397 7398 // Allocation is used to allocate the placement of a task group to a node. 7399 type Allocation struct { 7400 // msgpack omit empty fields during serialization 7401 _struct bool `codec:",omitempty"` // nolint: structcheck 7402 7403 // ID of the allocation (UUID) 7404 ID string 7405 7406 // Namespace is the namespace the allocation is created in 7407 Namespace string 7408 7409 // ID of the evaluation that generated this allocation 7410 EvalID string 7411 7412 // Name is a logical name of the allocation. 7413 Name string 7414 7415 // NodeID is the node this is being placed on 7416 NodeID string 7417 7418 // NodeName is the name of the node this is being placed on. 7419 NodeName string 7420 7421 // Job is the parent job of the task group being allocated. 7422 // This is copied at allocation time to avoid issues if the job 7423 // definition is updated. 7424 JobID string 7425 Job *Job 7426 7427 // TaskGroup is the name of the task group that should be run 7428 TaskGroup string 7429 7430 // COMPAT(0.11): Remove in 0.11 7431 // Resources is the total set of resources allocated as part 7432 // of this allocation of the task group. Dynamic ports will be set by 7433 // the scheduler. 7434 Resources *Resources 7435 7436 // COMPAT(0.11): Remove in 0.11 7437 // SharedResources are the resources that are shared by all the tasks in an 7438 // allocation 7439 SharedResources *Resources 7440 7441 // COMPAT(0.11): Remove in 0.11 7442 // TaskResources is the set of resources allocated to each 7443 // task. These should sum to the total Resources. Dynamic ports will be 7444 // set by the scheduler. 7445 TaskResources map[string]*Resources 7446 7447 // AllocatedResources is the total resources allocated for the task group. 7448 AllocatedResources *AllocatedResources 7449 7450 // Metrics associated with this allocation 7451 Metrics *AllocMetric 7452 7453 // Desired Status of the allocation on the client 7454 DesiredStatus string 7455 7456 // DesiredStatusDescription is meant to provide more human useful information 7457 DesiredDescription string 7458 7459 // DesiredTransition is used to indicate that a state transition 7460 // is desired for a given reason. 7461 DesiredTransition DesiredTransition 7462 7463 // Status of the allocation on the client 7464 ClientStatus string 7465 7466 // ClientStatusDescription is meant to provide more human useful information 7467 ClientDescription string 7468 7469 // TaskStates stores the state of each task, 7470 TaskStates map[string]*TaskState 7471 7472 // PreviousAllocation is the allocation that this allocation is replacing 7473 PreviousAllocation string 7474 7475 // NextAllocation is the allocation that this allocation is being replaced by 7476 NextAllocation string 7477 7478 // DeploymentID identifies an allocation as being created from a 7479 // particular deployment 7480 DeploymentID string 7481 7482 // DeploymentStatus captures the status of the allocation as part of the 7483 // given deployment 7484 DeploymentStatus *AllocDeploymentStatus 7485 7486 // RescheduleTrackers captures details of previous reschedule attempts of the allocation 7487 RescheduleTracker *RescheduleTracker 7488 7489 // FollowupEvalID captures a follow up evaluation created to handle a failed allocation 7490 // that can be rescheduled in the future 7491 FollowupEvalID string 7492 7493 // PreemptedAllocations captures IDs of any allocations that were preempted 7494 // in order to place this allocation 7495 PreemptedAllocations []string 7496 7497 // PreemptedByAllocation tracks the alloc ID of the allocation that caused this allocation 7498 // to stop running because it got preempted 7499 PreemptedByAllocation string 7500 7501 // Raft Indexes 7502 CreateIndex uint64 7503 ModifyIndex uint64 7504 7505 // AllocModifyIndex is not updated when the client updates allocations. This 7506 // lets the client pull only the allocs updated by the server. 7507 AllocModifyIndex uint64 7508 7509 // CreateTime is the time the allocation has finished scheduling and been 7510 // verified by the plan applier. 7511 CreateTime int64 7512 7513 // ModifyTime is the time the allocation was last updated. 7514 ModifyTime int64 7515 } 7516 7517 // Index returns the index of the allocation. If the allocation is from a task 7518 // group with count greater than 1, there will be multiple allocations for it. 7519 func (a *Allocation) Index() uint { 7520 l := len(a.Name) 7521 prefix := len(a.JobID) + len(a.TaskGroup) + 2 7522 if l <= 3 || l <= prefix { 7523 return uint(0) 7524 } 7525 7526 strNum := a.Name[prefix : len(a.Name)-1] 7527 num, _ := strconv.Atoi(strNum) 7528 return uint(num) 7529 } 7530 7531 // Copy provides a copy of the allocation and deep copies the job 7532 func (a *Allocation) Copy() *Allocation { 7533 return a.copyImpl(true) 7534 } 7535 7536 // CopySkipJob provides a copy of the allocation but doesn't deep copy the job 7537 func (a *Allocation) CopySkipJob() *Allocation { 7538 return a.copyImpl(false) 7539 } 7540 7541 func (a *Allocation) copyImpl(job bool) *Allocation { 7542 if a == nil { 7543 return nil 7544 } 7545 na := new(Allocation) 7546 *na = *a 7547 7548 if job { 7549 na.Job = na.Job.Copy() 7550 } 7551 7552 na.AllocatedResources = na.AllocatedResources.Copy() 7553 na.Resources = na.Resources.Copy() 7554 na.SharedResources = na.SharedResources.Copy() 7555 7556 if a.TaskResources != nil { 7557 tr := make(map[string]*Resources, len(na.TaskResources)) 7558 for task, resource := range na.TaskResources { 7559 tr[task] = resource.Copy() 7560 } 7561 na.TaskResources = tr 7562 } 7563 7564 na.Metrics = na.Metrics.Copy() 7565 na.DeploymentStatus = na.DeploymentStatus.Copy() 7566 7567 if a.TaskStates != nil { 7568 ts := make(map[string]*TaskState, len(na.TaskStates)) 7569 for task, state := range na.TaskStates { 7570 ts[task] = state.Copy() 7571 } 7572 na.TaskStates = ts 7573 } 7574 7575 na.RescheduleTracker = a.RescheduleTracker.Copy() 7576 na.PreemptedAllocations = helper.CopySliceString(a.PreemptedAllocations) 7577 return na 7578 } 7579 7580 // TerminalStatus returns if the desired or actual status is terminal and 7581 // will no longer transition. 7582 func (a *Allocation) TerminalStatus() bool { 7583 // First check the desired state and if that isn't terminal, check client 7584 // state. 7585 return a.ServerTerminalStatus() || a.ClientTerminalStatus() 7586 } 7587 7588 // ServerTerminalStatus returns true if the desired state of the allocation is terminal 7589 func (a *Allocation) ServerTerminalStatus() bool { 7590 switch a.DesiredStatus { 7591 case AllocDesiredStatusStop, AllocDesiredStatusEvict: 7592 return true 7593 default: 7594 return false 7595 } 7596 } 7597 7598 // ClientTerminalStatus returns if the client status is terminal and will no longer transition 7599 func (a *Allocation) ClientTerminalStatus() bool { 7600 switch a.ClientStatus { 7601 case AllocClientStatusComplete, AllocClientStatusFailed, AllocClientStatusLost: 7602 return true 7603 default: 7604 return false 7605 } 7606 } 7607 7608 // ShouldReschedule returns if the allocation is eligible to be rescheduled according 7609 // to its status and ReschedulePolicy given its failure time 7610 func (a *Allocation) ShouldReschedule(reschedulePolicy *ReschedulePolicy, failTime time.Time) bool { 7611 // First check the desired state 7612 switch a.DesiredStatus { 7613 case AllocDesiredStatusStop, AllocDesiredStatusEvict: 7614 return false 7615 default: 7616 } 7617 switch a.ClientStatus { 7618 case AllocClientStatusFailed: 7619 return a.RescheduleEligible(reschedulePolicy, failTime) 7620 default: 7621 return false 7622 } 7623 } 7624 7625 // RescheduleEligible returns if the allocation is eligible to be rescheduled according 7626 // to its ReschedulePolicy and the current state of its reschedule trackers 7627 func (a *Allocation) RescheduleEligible(reschedulePolicy *ReschedulePolicy, failTime time.Time) bool { 7628 if reschedulePolicy == nil { 7629 return false 7630 } 7631 attempts := reschedulePolicy.Attempts 7632 interval := reschedulePolicy.Interval 7633 enabled := attempts > 0 || reschedulePolicy.Unlimited 7634 if !enabled { 7635 return false 7636 } 7637 if reschedulePolicy.Unlimited { 7638 return true 7639 } 7640 // Early return true if there are no attempts yet and the number of allowed attempts is > 0 7641 if (a.RescheduleTracker == nil || len(a.RescheduleTracker.Events) == 0) && attempts > 0 { 7642 return true 7643 } 7644 attempted := 0 7645 for j := len(a.RescheduleTracker.Events) - 1; j >= 0; j-- { 7646 lastAttempt := a.RescheduleTracker.Events[j].RescheduleTime 7647 timeDiff := failTime.UTC().UnixNano() - lastAttempt 7648 if timeDiff < interval.Nanoseconds() { 7649 attempted += 1 7650 } 7651 } 7652 return attempted < attempts 7653 } 7654 7655 // LastEventTime is the time of the last task event in the allocation. 7656 // It is used to determine allocation failure time. If the FinishedAt field 7657 // is not set, the alloc's modify time is used 7658 func (a *Allocation) LastEventTime() time.Time { 7659 var lastEventTime time.Time 7660 if a.TaskStates != nil { 7661 for _, s := range a.TaskStates { 7662 if lastEventTime.IsZero() || s.FinishedAt.After(lastEventTime) { 7663 lastEventTime = s.FinishedAt 7664 } 7665 } 7666 } 7667 7668 if lastEventTime.IsZero() { 7669 return time.Unix(0, a.ModifyTime).UTC() 7670 } 7671 return lastEventTime 7672 } 7673 7674 // ReschedulePolicy returns the reschedule policy based on the task group 7675 func (a *Allocation) ReschedulePolicy() *ReschedulePolicy { 7676 tg := a.Job.LookupTaskGroup(a.TaskGroup) 7677 if tg == nil { 7678 return nil 7679 } 7680 return tg.ReschedulePolicy 7681 } 7682 7683 // NextRescheduleTime returns a time on or after which the allocation is eligible to be rescheduled, 7684 // and whether the next reschedule time is within policy's interval if the policy doesn't allow unlimited reschedules 7685 func (a *Allocation) NextRescheduleTime() (time.Time, bool) { 7686 failTime := a.LastEventTime() 7687 reschedulePolicy := a.ReschedulePolicy() 7688 if a.DesiredStatus == AllocDesiredStatusStop || a.ClientStatus != AllocClientStatusFailed || failTime.IsZero() || reschedulePolicy == nil { 7689 return time.Time{}, false 7690 } 7691 7692 nextDelay := a.NextDelay() 7693 nextRescheduleTime := failTime.Add(nextDelay) 7694 rescheduleEligible := reschedulePolicy.Unlimited || (reschedulePolicy.Attempts > 0 && a.RescheduleTracker == nil) 7695 if reschedulePolicy.Attempts > 0 && a.RescheduleTracker != nil && a.RescheduleTracker.Events != nil { 7696 // Check for eligibility based on the interval if max attempts is set 7697 attempted := 0 7698 for j := len(a.RescheduleTracker.Events) - 1; j >= 0; j-- { 7699 lastAttempt := a.RescheduleTracker.Events[j].RescheduleTime 7700 timeDiff := failTime.UTC().UnixNano() - lastAttempt 7701 if timeDiff < reschedulePolicy.Interval.Nanoseconds() { 7702 attempted += 1 7703 } 7704 } 7705 rescheduleEligible = attempted < reschedulePolicy.Attempts && nextDelay < reschedulePolicy.Interval 7706 } 7707 return nextRescheduleTime, rescheduleEligible 7708 } 7709 7710 // NextDelay returns a duration after which the allocation can be rescheduled. 7711 // It is calculated according to the delay function and previous reschedule attempts. 7712 func (a *Allocation) NextDelay() time.Duration { 7713 policy := a.ReschedulePolicy() 7714 // Can be nil if the task group was updated to remove its reschedule policy 7715 if policy == nil { 7716 return 0 7717 } 7718 delayDur := policy.Delay 7719 if a.RescheduleTracker == nil || a.RescheduleTracker.Events == nil || len(a.RescheduleTracker.Events) == 0 { 7720 return delayDur 7721 } 7722 events := a.RescheduleTracker.Events 7723 switch policy.DelayFunction { 7724 case "exponential": 7725 delayDur = a.RescheduleTracker.Events[len(a.RescheduleTracker.Events)-1].Delay * 2 7726 case "fibonacci": 7727 if len(events) >= 2 { 7728 fibN1Delay := events[len(events)-1].Delay 7729 fibN2Delay := events[len(events)-2].Delay 7730 // Handle reset of delay ceiling which should cause 7731 // a new series to start 7732 if fibN2Delay == policy.MaxDelay && fibN1Delay == policy.Delay { 7733 delayDur = fibN1Delay 7734 } else { 7735 delayDur = fibN1Delay + fibN2Delay 7736 } 7737 } 7738 default: 7739 return delayDur 7740 } 7741 if policy.MaxDelay > 0 && delayDur > policy.MaxDelay { 7742 delayDur = policy.MaxDelay 7743 // check if delay needs to be reset 7744 7745 lastRescheduleEvent := a.RescheduleTracker.Events[len(a.RescheduleTracker.Events)-1] 7746 timeDiff := a.LastEventTime().UTC().UnixNano() - lastRescheduleEvent.RescheduleTime 7747 if timeDiff > delayDur.Nanoseconds() { 7748 delayDur = policy.Delay 7749 } 7750 7751 } 7752 7753 return delayDur 7754 } 7755 7756 // Terminated returns if the allocation is in a terminal state on a client. 7757 func (a *Allocation) Terminated() bool { 7758 if a.ClientStatus == AllocClientStatusFailed || 7759 a.ClientStatus == AllocClientStatusComplete || 7760 a.ClientStatus == AllocClientStatusLost { 7761 return true 7762 } 7763 return false 7764 } 7765 7766 // RanSuccessfully returns whether the client has ran the allocation and all 7767 // tasks finished successfully. Critically this function returns whether the 7768 // allocation has ran to completion and not just that the alloc has converged to 7769 // its desired state. That is to say that a batch allocation must have finished 7770 // with exit code 0 on all task groups. This doesn't really have meaning on a 7771 // non-batch allocation because a service and system allocation should not 7772 // finish. 7773 func (a *Allocation) RanSuccessfully() bool { 7774 // Handle the case the client hasn't started the allocation. 7775 if len(a.TaskStates) == 0 { 7776 return false 7777 } 7778 7779 // Check to see if all the tasks finished successfully in the allocation 7780 allSuccess := true 7781 for _, state := range a.TaskStates { 7782 allSuccess = allSuccess && state.Successful() 7783 } 7784 7785 return allSuccess 7786 } 7787 7788 // ShouldMigrate returns if the allocation needs data migration 7789 func (a *Allocation) ShouldMigrate() bool { 7790 if a.PreviousAllocation == "" { 7791 return false 7792 } 7793 7794 if a.DesiredStatus == AllocDesiredStatusStop || a.DesiredStatus == AllocDesiredStatusEvict { 7795 return false 7796 } 7797 7798 tg := a.Job.LookupTaskGroup(a.TaskGroup) 7799 7800 // if the task group is nil or the ephemeral disk block isn't present then 7801 // we won't migrate 7802 if tg == nil || tg.EphemeralDisk == nil { 7803 return false 7804 } 7805 7806 // We won't migrate any data is the user hasn't enabled migration or the 7807 // disk is not marked as sticky 7808 if !tg.EphemeralDisk.Migrate || !tg.EphemeralDisk.Sticky { 7809 return false 7810 } 7811 7812 return true 7813 } 7814 7815 // SetEventDisplayMessage populates the display message if its not already set, 7816 // a temporary fix to handle old allocations that don't have it. 7817 // This method will be removed in a future release. 7818 func (a *Allocation) SetEventDisplayMessages() { 7819 setDisplayMsg(a.TaskStates) 7820 } 7821 7822 // COMPAT(0.11): Remove in 0.11 7823 // ComparableResources returns the resouces on the allocation 7824 // handling upgrade paths. After 0.11 calls to this should be replaced with: 7825 // alloc.AllocatedResources.Comparable() 7826 func (a *Allocation) ComparableResources() *ComparableResources { 7827 // ALloc already has 0.9+ behavior 7828 if a.AllocatedResources != nil { 7829 return a.AllocatedResources.Comparable() 7830 } 7831 7832 var resources *Resources 7833 if a.Resources != nil { 7834 resources = a.Resources 7835 } else if a.TaskResources != nil { 7836 resources = new(Resources) 7837 resources.Add(a.SharedResources) 7838 for _, taskResource := range a.TaskResources { 7839 resources.Add(taskResource) 7840 } 7841 } 7842 7843 // Upgrade path 7844 return &ComparableResources{ 7845 Flattened: AllocatedTaskResources{ 7846 Cpu: AllocatedCpuResources{ 7847 CpuShares: int64(resources.CPU), 7848 }, 7849 Memory: AllocatedMemoryResources{ 7850 MemoryMB: int64(resources.MemoryMB), 7851 }, 7852 Networks: resources.Networks, 7853 }, 7854 Shared: AllocatedSharedResources{ 7855 DiskMB: int64(resources.DiskMB), 7856 }, 7857 } 7858 } 7859 7860 // LookupTask by name from the Allocation. Returns nil if the Job is not set, the 7861 // TaskGroup does not exist, or the task name cannot be found. 7862 func (a *Allocation) LookupTask(name string) *Task { 7863 if a.Job == nil { 7864 return nil 7865 } 7866 7867 tg := a.Job.LookupTaskGroup(a.TaskGroup) 7868 if tg == nil { 7869 return nil 7870 } 7871 7872 return tg.LookupTask(name) 7873 } 7874 7875 // Stub returns a list stub for the allocation 7876 func (a *Allocation) Stub() *AllocListStub { 7877 return &AllocListStub{ 7878 ID: a.ID, 7879 EvalID: a.EvalID, 7880 Name: a.Name, 7881 Namespace: a.Namespace, 7882 NodeID: a.NodeID, 7883 NodeName: a.NodeName, 7884 JobID: a.JobID, 7885 JobType: a.Job.Type, 7886 JobVersion: a.Job.Version, 7887 TaskGroup: a.TaskGroup, 7888 DesiredStatus: a.DesiredStatus, 7889 DesiredDescription: a.DesiredDescription, 7890 ClientStatus: a.ClientStatus, 7891 ClientDescription: a.ClientDescription, 7892 DesiredTransition: a.DesiredTransition, 7893 TaskStates: a.TaskStates, 7894 DeploymentStatus: a.DeploymentStatus, 7895 FollowupEvalID: a.FollowupEvalID, 7896 RescheduleTracker: a.RescheduleTracker, 7897 PreemptedAllocations: a.PreemptedAllocations, 7898 PreemptedByAllocation: a.PreemptedByAllocation, 7899 CreateIndex: a.CreateIndex, 7900 ModifyIndex: a.ModifyIndex, 7901 CreateTime: a.CreateTime, 7902 ModifyTime: a.ModifyTime, 7903 } 7904 } 7905 7906 // AllocationDiff converts an Allocation type to an AllocationDiff type 7907 // If at any time, modification are made to AllocationDiff so that an 7908 // Allocation can no longer be safely converted to AllocationDiff, 7909 // this method should be changed accordingly. 7910 func (a *Allocation) AllocationDiff() *AllocationDiff { 7911 return (*AllocationDiff)(a) 7912 } 7913 7914 // AllocationDiff is another named type for Allocation (to use the same fields), 7915 // which is used to represent the delta for an Allocation. If you need a method 7916 // defined on the al 7917 type AllocationDiff Allocation 7918 7919 // AllocListStub is used to return a subset of alloc information 7920 type AllocListStub struct { 7921 ID string 7922 EvalID string 7923 Name string 7924 Namespace string 7925 NodeID string 7926 NodeName string 7927 JobID string 7928 JobType string 7929 JobVersion uint64 7930 TaskGroup string 7931 DesiredStatus string 7932 DesiredDescription string 7933 ClientStatus string 7934 ClientDescription string 7935 DesiredTransition DesiredTransition 7936 TaskStates map[string]*TaskState 7937 DeploymentStatus *AllocDeploymentStatus 7938 FollowupEvalID string 7939 RescheduleTracker *RescheduleTracker 7940 PreemptedAllocations []string 7941 PreemptedByAllocation string 7942 CreateIndex uint64 7943 ModifyIndex uint64 7944 CreateTime int64 7945 ModifyTime int64 7946 } 7947 7948 // SetEventDisplayMessage populates the display message if its not already set, 7949 // a temporary fix to handle old allocations that don't have it. 7950 // This method will be removed in a future release. 7951 func (a *AllocListStub) SetEventDisplayMessages() { 7952 setDisplayMsg(a.TaskStates) 7953 } 7954 7955 func setDisplayMsg(taskStates map[string]*TaskState) { 7956 if taskStates != nil { 7957 for _, taskState := range taskStates { 7958 for _, event := range taskState.Events { 7959 event.PopulateEventDisplayMessage() 7960 } 7961 } 7962 } 7963 } 7964 7965 // AllocMetric is used to track various metrics while attempting 7966 // to make an allocation. These are used to debug a job, or to better 7967 // understand the pressure within the system. 7968 type AllocMetric struct { 7969 // NodesEvaluated is the number of nodes that were evaluated 7970 NodesEvaluated int 7971 7972 // NodesFiltered is the number of nodes filtered due to a constraint 7973 NodesFiltered int 7974 7975 // NodesAvailable is the number of nodes available for evaluation per DC. 7976 NodesAvailable map[string]int 7977 7978 // ClassFiltered is the number of nodes filtered by class 7979 ClassFiltered map[string]int 7980 7981 // ConstraintFiltered is the number of failures caused by constraint 7982 ConstraintFiltered map[string]int 7983 7984 // NodesExhausted is the number of nodes skipped due to being 7985 // exhausted of at least one resource 7986 NodesExhausted int 7987 7988 // ClassExhausted is the number of nodes exhausted by class 7989 ClassExhausted map[string]int 7990 7991 // DimensionExhausted provides the count by dimension or reason 7992 DimensionExhausted map[string]int 7993 7994 // QuotaExhausted provides the exhausted dimensions 7995 QuotaExhausted []string 7996 7997 // Scores is the scores of the final few nodes remaining 7998 // for placement. The top score is typically selected. 7999 // Deprecated: Replaced by ScoreMetaData in Nomad 0.9 8000 Scores map[string]float64 8001 8002 // ScoreMetaData is a slice of top scoring nodes displayed in the CLI 8003 ScoreMetaData []*NodeScoreMeta 8004 8005 // nodeScoreMeta is used to keep scores for a single node id. It is cleared out after 8006 // we receive normalized score during the last step of the scoring stack. 8007 nodeScoreMeta *NodeScoreMeta 8008 8009 // topScores is used to maintain a heap of the top K nodes with 8010 // the highest normalized score 8011 topScores *kheap.ScoreHeap 8012 8013 // AllocationTime is a measure of how long the allocation 8014 // attempt took. This can affect performance and SLAs. 8015 AllocationTime time.Duration 8016 8017 // CoalescedFailures indicates the number of other 8018 // allocations that were coalesced into this failed allocation. 8019 // This is to prevent creating many failed allocations for a 8020 // single task group. 8021 CoalescedFailures int 8022 } 8023 8024 func (a *AllocMetric) Copy() *AllocMetric { 8025 if a == nil { 8026 return nil 8027 } 8028 na := new(AllocMetric) 8029 *na = *a 8030 na.NodesAvailable = helper.CopyMapStringInt(na.NodesAvailable) 8031 na.ClassFiltered = helper.CopyMapStringInt(na.ClassFiltered) 8032 na.ConstraintFiltered = helper.CopyMapStringInt(na.ConstraintFiltered) 8033 na.ClassExhausted = helper.CopyMapStringInt(na.ClassExhausted) 8034 na.DimensionExhausted = helper.CopyMapStringInt(na.DimensionExhausted) 8035 na.QuotaExhausted = helper.CopySliceString(na.QuotaExhausted) 8036 na.Scores = helper.CopyMapStringFloat64(na.Scores) 8037 na.ScoreMetaData = CopySliceNodeScoreMeta(na.ScoreMetaData) 8038 return na 8039 } 8040 8041 func (a *AllocMetric) EvaluateNode() { 8042 a.NodesEvaluated += 1 8043 } 8044 8045 func (a *AllocMetric) FilterNode(node *Node, constraint string) { 8046 a.NodesFiltered += 1 8047 if node != nil && node.NodeClass != "" { 8048 if a.ClassFiltered == nil { 8049 a.ClassFiltered = make(map[string]int) 8050 } 8051 a.ClassFiltered[node.NodeClass] += 1 8052 } 8053 if constraint != "" { 8054 if a.ConstraintFiltered == nil { 8055 a.ConstraintFiltered = make(map[string]int) 8056 } 8057 a.ConstraintFiltered[constraint] += 1 8058 } 8059 } 8060 8061 func (a *AllocMetric) ExhaustedNode(node *Node, dimension string) { 8062 a.NodesExhausted += 1 8063 if node != nil && node.NodeClass != "" { 8064 if a.ClassExhausted == nil { 8065 a.ClassExhausted = make(map[string]int) 8066 } 8067 a.ClassExhausted[node.NodeClass] += 1 8068 } 8069 if dimension != "" { 8070 if a.DimensionExhausted == nil { 8071 a.DimensionExhausted = make(map[string]int) 8072 } 8073 a.DimensionExhausted[dimension] += 1 8074 } 8075 } 8076 8077 func (a *AllocMetric) ExhaustQuota(dimensions []string) { 8078 if a.QuotaExhausted == nil { 8079 a.QuotaExhausted = make([]string, 0, len(dimensions)) 8080 } 8081 8082 a.QuotaExhausted = append(a.QuotaExhausted, dimensions...) 8083 } 8084 8085 // ScoreNode is used to gather top K scoring nodes in a heap 8086 func (a *AllocMetric) ScoreNode(node *Node, name string, score float64) { 8087 // Create nodeScoreMeta lazily if its the first time or if its a new node 8088 if a.nodeScoreMeta == nil || a.nodeScoreMeta.NodeID != node.ID { 8089 a.nodeScoreMeta = &NodeScoreMeta{ 8090 NodeID: node.ID, 8091 Scores: make(map[string]float64), 8092 } 8093 } 8094 if name == NormScorerName { 8095 a.nodeScoreMeta.NormScore = score 8096 // Once we have the normalized score we can push to the heap 8097 // that tracks top K by normalized score 8098 8099 // Create the heap if its not there already 8100 if a.topScores == nil { 8101 a.topScores = kheap.NewScoreHeap(MaxRetainedNodeScores) 8102 } 8103 heap.Push(a.topScores, a.nodeScoreMeta) 8104 8105 // Clear out this entry because its now in the heap 8106 a.nodeScoreMeta = nil 8107 } else { 8108 a.nodeScoreMeta.Scores[name] = score 8109 } 8110 } 8111 8112 // PopulateScoreMetaData populates a map of scorer to scoring metadata 8113 // The map is populated by popping elements from a heap of top K scores 8114 // maintained per scorer 8115 func (a *AllocMetric) PopulateScoreMetaData() { 8116 if a.topScores == nil { 8117 return 8118 } 8119 8120 if a.ScoreMetaData == nil { 8121 a.ScoreMetaData = make([]*NodeScoreMeta, a.topScores.Len()) 8122 } 8123 heapItems := a.topScores.GetItemsReverse() 8124 for i, item := range heapItems { 8125 a.ScoreMetaData[i] = item.(*NodeScoreMeta) 8126 } 8127 } 8128 8129 // NodeScoreMeta captures scoring meta data derived from 8130 // different scoring factors. 8131 type NodeScoreMeta struct { 8132 NodeID string 8133 Scores map[string]float64 8134 NormScore float64 8135 } 8136 8137 func (s *NodeScoreMeta) Copy() *NodeScoreMeta { 8138 if s == nil { 8139 return nil 8140 } 8141 ns := new(NodeScoreMeta) 8142 *ns = *s 8143 return ns 8144 } 8145 8146 func (s *NodeScoreMeta) String() string { 8147 return fmt.Sprintf("%s %f %v", s.NodeID, s.NormScore, s.Scores) 8148 } 8149 8150 func (s *NodeScoreMeta) Score() float64 { 8151 return s.NormScore 8152 } 8153 8154 func (s *NodeScoreMeta) Data() interface{} { 8155 return s 8156 } 8157 8158 // AllocDeploymentStatus captures the status of the allocation as part of the 8159 // deployment. This can include things like if the allocation has been marked as 8160 // healthy. 8161 type AllocDeploymentStatus struct { 8162 // Healthy marks whether the allocation has been marked healthy or unhealthy 8163 // as part of a deployment. It can be unset if it has neither been marked 8164 // healthy or unhealthy. 8165 Healthy *bool 8166 8167 // Timestamp is the time at which the health status was set. 8168 Timestamp time.Time 8169 8170 // Canary marks whether the allocation is a canary or not. A canary that has 8171 // been promoted will have this field set to false. 8172 Canary bool 8173 8174 // ModifyIndex is the raft index in which the deployment status was last 8175 // changed. 8176 ModifyIndex uint64 8177 } 8178 8179 // HasHealth returns true if the allocation has its health set. 8180 func (a *AllocDeploymentStatus) HasHealth() bool { 8181 return a != nil && a.Healthy != nil 8182 } 8183 8184 // IsHealthy returns if the allocation is marked as healthy as part of a 8185 // deployment 8186 func (a *AllocDeploymentStatus) IsHealthy() bool { 8187 if a == nil { 8188 return false 8189 } 8190 8191 return a.Healthy != nil && *a.Healthy 8192 } 8193 8194 // IsUnhealthy returns if the allocation is marked as unhealthy as part of a 8195 // deployment 8196 func (a *AllocDeploymentStatus) IsUnhealthy() bool { 8197 if a == nil { 8198 return false 8199 } 8200 8201 return a.Healthy != nil && !*a.Healthy 8202 } 8203 8204 // IsCanary returns if the allocation is marked as a canary 8205 func (a *AllocDeploymentStatus) IsCanary() bool { 8206 if a == nil { 8207 return false 8208 } 8209 8210 return a.Canary 8211 } 8212 8213 func (a *AllocDeploymentStatus) Copy() *AllocDeploymentStatus { 8214 if a == nil { 8215 return nil 8216 } 8217 8218 c := new(AllocDeploymentStatus) 8219 *c = *a 8220 8221 if a.Healthy != nil { 8222 c.Healthy = helper.BoolToPtr(*a.Healthy) 8223 } 8224 8225 return c 8226 } 8227 8228 const ( 8229 EvalStatusBlocked = "blocked" 8230 EvalStatusPending = "pending" 8231 EvalStatusComplete = "complete" 8232 EvalStatusFailed = "failed" 8233 EvalStatusCancelled = "canceled" 8234 ) 8235 8236 const ( 8237 EvalTriggerJobRegister = "job-register" 8238 EvalTriggerJobDeregister = "job-deregister" 8239 EvalTriggerPeriodicJob = "periodic-job" 8240 EvalTriggerNodeDrain = "node-drain" 8241 EvalTriggerNodeUpdate = "node-update" 8242 EvalTriggerAllocStop = "alloc-stop" 8243 EvalTriggerScheduled = "scheduled" 8244 EvalTriggerRollingUpdate = "rolling-update" 8245 EvalTriggerDeploymentWatcher = "deployment-watcher" 8246 EvalTriggerFailedFollowUp = "failed-follow-up" 8247 EvalTriggerMaxPlans = "max-plan-attempts" 8248 EvalTriggerRetryFailedAlloc = "alloc-failure" 8249 EvalTriggerQueuedAllocs = "queued-allocs" 8250 EvalTriggerPreemption = "preemption" 8251 ) 8252 8253 const ( 8254 // CoreJobEvalGC is used for the garbage collection of evaluations 8255 // and allocations. We periodically scan evaluations in a terminal state, 8256 // in which all the corresponding allocations are also terminal. We 8257 // delete these out of the system to bound the state. 8258 CoreJobEvalGC = "eval-gc" 8259 8260 // CoreJobNodeGC is used for the garbage collection of failed nodes. 8261 // We periodically scan nodes in a terminal state, and if they have no 8262 // corresponding allocations we delete these out of the system. 8263 CoreJobNodeGC = "node-gc" 8264 8265 // CoreJobJobGC is used for the garbage collection of eligible jobs. We 8266 // periodically scan garbage collectible jobs and check if both their 8267 // evaluations and allocations are terminal. If so, we delete these out of 8268 // the system. 8269 CoreJobJobGC = "job-gc" 8270 8271 // CoreJobDeploymentGC is used for the garbage collection of eligible 8272 // deployments. We periodically scan garbage collectible deployments and 8273 // check if they are terminal. If so, we delete these out of the system. 8274 CoreJobDeploymentGC = "deployment-gc" 8275 8276 // CoreJobForceGC is used to force garbage collection of all GCable objects. 8277 CoreJobForceGC = "force-gc" 8278 ) 8279 8280 // Evaluation is used anytime we need to apply business logic as a result 8281 // of a change to our desired state (job specification) or the emergent state 8282 // (registered nodes). When the inputs change, we need to "evaluate" them, 8283 // potentially taking action (allocation of work) or doing nothing if the state 8284 // of the world does not require it. 8285 type Evaluation struct { 8286 // msgpack omit empty fields during serialization 8287 _struct bool `codec:",omitempty"` // nolint: structcheck 8288 8289 // ID is a randomly generated UUID used for this evaluation. This 8290 // is assigned upon the creation of the evaluation. 8291 ID string 8292 8293 // Namespace is the namespace the evaluation is created in 8294 Namespace string 8295 8296 // Priority is used to control scheduling importance and if this job 8297 // can preempt other jobs. 8298 Priority int 8299 8300 // Type is used to control which schedulers are available to handle 8301 // this evaluation. 8302 Type string 8303 8304 // TriggeredBy is used to give some insight into why this Eval 8305 // was created. (Job change, node failure, alloc failure, etc). 8306 TriggeredBy string 8307 8308 // JobID is the job this evaluation is scoped to. Evaluations cannot 8309 // be run in parallel for a given JobID, so we serialize on this. 8310 JobID string 8311 8312 // JobModifyIndex is the modify index of the job at the time 8313 // the evaluation was created 8314 JobModifyIndex uint64 8315 8316 // NodeID is the node that was affected triggering the evaluation. 8317 NodeID string 8318 8319 // NodeModifyIndex is the modify index of the node at the time 8320 // the evaluation was created 8321 NodeModifyIndex uint64 8322 8323 // DeploymentID is the ID of the deployment that triggered the evaluation. 8324 DeploymentID string 8325 8326 // Status of the evaluation 8327 Status string 8328 8329 // StatusDescription is meant to provide more human useful information 8330 StatusDescription string 8331 8332 // Wait is a minimum wait time for running the eval. This is used to 8333 // support a rolling upgrade in versions prior to 0.7.0 8334 // Deprecated 8335 Wait time.Duration 8336 8337 // WaitUntil is the time when this eval should be run. This is used to 8338 // supported delayed rescheduling of failed allocations 8339 WaitUntil time.Time 8340 8341 // NextEval is the evaluation ID for the eval created to do a followup. 8342 // This is used to support rolling upgrades and failed-follow-up evals, where 8343 // we need a chain of evaluations. 8344 NextEval string 8345 8346 // PreviousEval is the evaluation ID for the eval creating this one to do a followup. 8347 // This is used to support rolling upgrades and failed-follow-up evals, where 8348 // we need a chain of evaluations. 8349 PreviousEval string 8350 8351 // BlockedEval is the evaluation ID for a created blocked eval. A 8352 // blocked eval will be created if all allocations could not be placed due 8353 // to constraints or lacking resources. 8354 BlockedEval string 8355 8356 // FailedTGAllocs are task groups which have allocations that could not be 8357 // made, but the metrics are persisted so that the user can use the feedback 8358 // to determine the cause. 8359 FailedTGAllocs map[string]*AllocMetric 8360 8361 // ClassEligibility tracks computed node classes that have been explicitly 8362 // marked as eligible or ineligible. 8363 ClassEligibility map[string]bool 8364 8365 // QuotaLimitReached marks whether a quota limit was reached for the 8366 // evaluation. 8367 QuotaLimitReached string 8368 8369 // EscapedComputedClass marks whether the job has constraints that are not 8370 // captured by computed node classes. 8371 EscapedComputedClass bool 8372 8373 // AnnotatePlan triggers the scheduler to provide additional annotations 8374 // during the evaluation. This should not be set during normal operations. 8375 AnnotatePlan bool 8376 8377 // QueuedAllocations is the number of unplaced allocations at the time the 8378 // evaluation was processed. The map is keyed by Task Group names. 8379 QueuedAllocations map[string]int 8380 8381 // LeaderACL provides the ACL token to when issuing RPCs back to the 8382 // leader. This will be a valid management token as long as the leader is 8383 // active. This should not ever be exposed via the API. 8384 LeaderACL string 8385 8386 // SnapshotIndex is the Raft index of the snapshot used to process the 8387 // evaluation. The index will either be set when it has gone through the 8388 // scheduler or if a blocked evaluation is being created. The index is set 8389 // in this case so we can determine if an early unblocking is required since 8390 // capacity has changed since the evaluation was created. This can result in 8391 // the SnapshotIndex being less than the CreateIndex. 8392 SnapshotIndex uint64 8393 8394 // Raft Indexes 8395 CreateIndex uint64 8396 ModifyIndex uint64 8397 } 8398 8399 // TerminalStatus returns if the current status is terminal and 8400 // will no longer transition. 8401 func (e *Evaluation) TerminalStatus() bool { 8402 switch e.Status { 8403 case EvalStatusComplete, EvalStatusFailed, EvalStatusCancelled: 8404 return true 8405 default: 8406 return false 8407 } 8408 } 8409 8410 func (e *Evaluation) GoString() string { 8411 return fmt.Sprintf("<Eval %q JobID: %q Namespace: %q>", e.ID, e.JobID, e.Namespace) 8412 } 8413 8414 func (e *Evaluation) Copy() *Evaluation { 8415 if e == nil { 8416 return nil 8417 } 8418 ne := new(Evaluation) 8419 *ne = *e 8420 8421 // Copy ClassEligibility 8422 if e.ClassEligibility != nil { 8423 classes := make(map[string]bool, len(e.ClassEligibility)) 8424 for class, elig := range e.ClassEligibility { 8425 classes[class] = elig 8426 } 8427 ne.ClassEligibility = classes 8428 } 8429 8430 // Copy FailedTGAllocs 8431 if e.FailedTGAllocs != nil { 8432 failedTGs := make(map[string]*AllocMetric, len(e.FailedTGAllocs)) 8433 for tg, metric := range e.FailedTGAllocs { 8434 failedTGs[tg] = metric.Copy() 8435 } 8436 ne.FailedTGAllocs = failedTGs 8437 } 8438 8439 // Copy queued allocations 8440 if e.QueuedAllocations != nil { 8441 queuedAllocations := make(map[string]int, len(e.QueuedAllocations)) 8442 for tg, num := range e.QueuedAllocations { 8443 queuedAllocations[tg] = num 8444 } 8445 ne.QueuedAllocations = queuedAllocations 8446 } 8447 8448 return ne 8449 } 8450 8451 // ShouldEnqueue checks if a given evaluation should be enqueued into the 8452 // eval_broker 8453 func (e *Evaluation) ShouldEnqueue() bool { 8454 switch e.Status { 8455 case EvalStatusPending: 8456 return true 8457 case EvalStatusComplete, EvalStatusFailed, EvalStatusBlocked, EvalStatusCancelled: 8458 return false 8459 default: 8460 panic(fmt.Sprintf("unhandled evaluation (%s) status %s", e.ID, e.Status)) 8461 } 8462 } 8463 8464 // ShouldBlock checks if a given evaluation should be entered into the blocked 8465 // eval tracker. 8466 func (e *Evaluation) ShouldBlock() bool { 8467 switch e.Status { 8468 case EvalStatusBlocked: 8469 return true 8470 case EvalStatusComplete, EvalStatusFailed, EvalStatusPending, EvalStatusCancelled: 8471 return false 8472 default: 8473 panic(fmt.Sprintf("unhandled evaluation (%s) status %s", e.ID, e.Status)) 8474 } 8475 } 8476 8477 // MakePlan is used to make a plan from the given evaluation 8478 // for a given Job 8479 func (e *Evaluation) MakePlan(j *Job) *Plan { 8480 p := &Plan{ 8481 EvalID: e.ID, 8482 Priority: e.Priority, 8483 Job: j, 8484 NodeUpdate: make(map[string][]*Allocation), 8485 NodeAllocation: make(map[string][]*Allocation), 8486 NodePreemptions: make(map[string][]*Allocation), 8487 } 8488 if j != nil { 8489 p.AllAtOnce = j.AllAtOnce 8490 } 8491 return p 8492 } 8493 8494 // NextRollingEval creates an evaluation to followup this eval for rolling updates 8495 func (e *Evaluation) NextRollingEval(wait time.Duration) *Evaluation { 8496 return &Evaluation{ 8497 ID: uuid.Generate(), 8498 Namespace: e.Namespace, 8499 Priority: e.Priority, 8500 Type: e.Type, 8501 TriggeredBy: EvalTriggerRollingUpdate, 8502 JobID: e.JobID, 8503 JobModifyIndex: e.JobModifyIndex, 8504 Status: EvalStatusPending, 8505 Wait: wait, 8506 PreviousEval: e.ID, 8507 } 8508 } 8509 8510 // CreateBlockedEval creates a blocked evaluation to followup this eval to place any 8511 // failed allocations. It takes the classes marked explicitly eligible or 8512 // ineligible, whether the job has escaped computed node classes and whether the 8513 // quota limit was reached. 8514 func (e *Evaluation) CreateBlockedEval(classEligibility map[string]bool, 8515 escaped bool, quotaReached string) *Evaluation { 8516 8517 return &Evaluation{ 8518 ID: uuid.Generate(), 8519 Namespace: e.Namespace, 8520 Priority: e.Priority, 8521 Type: e.Type, 8522 TriggeredBy: EvalTriggerQueuedAllocs, 8523 JobID: e.JobID, 8524 JobModifyIndex: e.JobModifyIndex, 8525 Status: EvalStatusBlocked, 8526 PreviousEval: e.ID, 8527 ClassEligibility: classEligibility, 8528 EscapedComputedClass: escaped, 8529 QuotaLimitReached: quotaReached, 8530 } 8531 } 8532 8533 // CreateFailedFollowUpEval creates a follow up evaluation when the current one 8534 // has been marked as failed because it has hit the delivery limit and will not 8535 // be retried by the eval_broker. Callers should copy the created eval's ID to 8536 // into the old eval's NextEval field. 8537 func (e *Evaluation) CreateFailedFollowUpEval(wait time.Duration) *Evaluation { 8538 return &Evaluation{ 8539 ID: uuid.Generate(), 8540 Namespace: e.Namespace, 8541 Priority: e.Priority, 8542 Type: e.Type, 8543 TriggeredBy: EvalTriggerFailedFollowUp, 8544 JobID: e.JobID, 8545 JobModifyIndex: e.JobModifyIndex, 8546 Status: EvalStatusPending, 8547 Wait: wait, 8548 PreviousEval: e.ID, 8549 } 8550 } 8551 8552 // Plan is used to submit a commit plan for task allocations. These 8553 // are submitted to the leader which verifies that resources have 8554 // not been overcommitted before admitting the plan. 8555 type Plan struct { 8556 // msgpack omit empty fields during serialization 8557 _struct bool `codec:",omitempty"` // nolint: structcheck 8558 8559 // EvalID is the evaluation ID this plan is associated with 8560 EvalID string 8561 8562 // EvalToken is used to prevent a split-brain processing of 8563 // an evaluation. There should only be a single scheduler running 8564 // an Eval at a time, but this could be violated after a leadership 8565 // transition. This unique token is used to reject plans that are 8566 // being submitted from a different leader. 8567 EvalToken string 8568 8569 // Priority is the priority of the upstream job 8570 Priority int 8571 8572 // AllAtOnce is used to control if incremental scheduling of task groups 8573 // is allowed or if we must do a gang scheduling of the entire job. 8574 // If this is false, a plan may be partially applied. Otherwise, the 8575 // entire plan must be able to make progress. 8576 AllAtOnce bool 8577 8578 // Job is the parent job of all the allocations in the Plan. 8579 // Since a Plan only involves a single Job, we can reduce the size 8580 // of the plan by only including it once. 8581 Job *Job 8582 8583 // NodeUpdate contains all the allocations for each node. For each node, 8584 // this is a list of the allocations to update to either stop or evict. 8585 NodeUpdate map[string][]*Allocation 8586 8587 // NodeAllocation contains all the allocations for each node. 8588 // The evicts must be considered prior to the allocations. 8589 NodeAllocation map[string][]*Allocation 8590 8591 // Annotations contains annotations by the scheduler to be used by operators 8592 // to understand the decisions made by the scheduler. 8593 Annotations *PlanAnnotations 8594 8595 // Deployment is the deployment created or updated by the scheduler that 8596 // should be applied by the planner. 8597 Deployment *Deployment 8598 8599 // DeploymentUpdates is a set of status updates to apply to the given 8600 // deployments. This allows the scheduler to cancel any unneeded deployment 8601 // because the job is stopped or the update block is removed. 8602 DeploymentUpdates []*DeploymentStatusUpdate 8603 8604 // NodePreemptions is a map from node id to a set of allocations from other 8605 // lower priority jobs that are preempted. Preempted allocations are marked 8606 // as evicted. 8607 NodePreemptions map[string][]*Allocation 8608 } 8609 8610 // AppendStoppedAlloc marks an allocation to be stopped. The clientStatus of the 8611 // allocation may be optionally set by passing in a non-empty value. 8612 func (p *Plan) AppendStoppedAlloc(alloc *Allocation, desiredDesc, clientStatus string) { 8613 newAlloc := new(Allocation) 8614 *newAlloc = *alloc 8615 8616 // If the job is not set in the plan we are deregistering a job so we 8617 // extract the job from the allocation. 8618 if p.Job == nil && newAlloc.Job != nil { 8619 p.Job = newAlloc.Job 8620 } 8621 8622 // Normalize the job 8623 newAlloc.Job = nil 8624 8625 // Strip the resources as it can be rebuilt. 8626 newAlloc.Resources = nil 8627 8628 newAlloc.DesiredStatus = AllocDesiredStatusStop 8629 newAlloc.DesiredDescription = desiredDesc 8630 8631 if clientStatus != "" { 8632 newAlloc.ClientStatus = clientStatus 8633 } 8634 8635 node := alloc.NodeID 8636 existing := p.NodeUpdate[node] 8637 p.NodeUpdate[node] = append(existing, newAlloc) 8638 } 8639 8640 // AppendPreemptedAlloc is used to append an allocation that's being preempted to the plan. 8641 // To minimize the size of the plan, this only sets a minimal set of fields in the allocation 8642 func (p *Plan) AppendPreemptedAlloc(alloc *Allocation, preemptingAllocID string) { 8643 newAlloc := &Allocation{} 8644 newAlloc.ID = alloc.ID 8645 newAlloc.JobID = alloc.JobID 8646 newAlloc.Namespace = alloc.Namespace 8647 newAlloc.DesiredStatus = AllocDesiredStatusEvict 8648 newAlloc.PreemptedByAllocation = preemptingAllocID 8649 8650 desiredDesc := fmt.Sprintf("Preempted by alloc ID %v", preemptingAllocID) 8651 newAlloc.DesiredDescription = desiredDesc 8652 8653 // TaskResources are needed by the plan applier to check if allocations fit 8654 // after removing preempted allocations 8655 if alloc.AllocatedResources != nil { 8656 newAlloc.AllocatedResources = alloc.AllocatedResources 8657 } else { 8658 // COMPAT Remove in version 0.11 8659 newAlloc.TaskResources = alloc.TaskResources 8660 newAlloc.SharedResources = alloc.SharedResources 8661 } 8662 8663 // Append this alloc to slice for this node 8664 node := alloc.NodeID 8665 existing := p.NodePreemptions[node] 8666 p.NodePreemptions[node] = append(existing, newAlloc) 8667 } 8668 8669 func (p *Plan) PopUpdate(alloc *Allocation) { 8670 existing := p.NodeUpdate[alloc.NodeID] 8671 n := len(existing) 8672 if n > 0 && existing[n-1].ID == alloc.ID { 8673 existing = existing[:n-1] 8674 if len(existing) > 0 { 8675 p.NodeUpdate[alloc.NodeID] = existing 8676 } else { 8677 delete(p.NodeUpdate, alloc.NodeID) 8678 } 8679 } 8680 } 8681 8682 func (p *Plan) AppendAlloc(alloc *Allocation) { 8683 node := alloc.NodeID 8684 existing := p.NodeAllocation[node] 8685 8686 // Normalize the job 8687 alloc.Job = nil 8688 8689 p.NodeAllocation[node] = append(existing, alloc) 8690 } 8691 8692 // IsNoOp checks if this plan would do nothing 8693 func (p *Plan) IsNoOp() bool { 8694 return len(p.NodeUpdate) == 0 && 8695 len(p.NodeAllocation) == 0 && 8696 p.Deployment == nil && 8697 len(p.DeploymentUpdates) == 0 8698 } 8699 8700 // NormalizeAllocations normalizes allocations to remove fields that can 8701 // be fetched from the MemDB instead of sending over the wire 8702 func (p *Plan) NormalizeAllocations() { 8703 for _, allocs := range p.NodeUpdate { 8704 for i, alloc := range allocs { 8705 allocs[i] = &Allocation{ 8706 ID: alloc.ID, 8707 DesiredDescription: alloc.DesiredDescription, 8708 ClientStatus: alloc.ClientStatus, 8709 } 8710 } 8711 } 8712 8713 for _, allocs := range p.NodePreemptions { 8714 for i, alloc := range allocs { 8715 allocs[i] = &Allocation{ 8716 ID: alloc.ID, 8717 PreemptedByAllocation: alloc.PreemptedByAllocation, 8718 } 8719 } 8720 } 8721 } 8722 8723 // PlanResult is the result of a plan submitted to the leader. 8724 type PlanResult struct { 8725 // NodeUpdate contains all the updates that were committed. 8726 NodeUpdate map[string][]*Allocation 8727 8728 // NodeAllocation contains all the allocations that were committed. 8729 NodeAllocation map[string][]*Allocation 8730 8731 // Deployment is the deployment that was committed. 8732 Deployment *Deployment 8733 8734 // DeploymentUpdates is the set of deployment updates that were committed. 8735 DeploymentUpdates []*DeploymentStatusUpdate 8736 8737 // NodePreemptions is a map from node id to a set of allocations from other 8738 // lower priority jobs that are preempted. Preempted allocations are marked 8739 // as stopped. 8740 NodePreemptions map[string][]*Allocation 8741 8742 // RefreshIndex is the index the worker should refresh state up to. 8743 // This allows all evictions and allocations to be materialized. 8744 // If any allocations were rejected due to stale data (node state, 8745 // over committed) this can be used to force a worker refresh. 8746 RefreshIndex uint64 8747 8748 // AllocIndex is the Raft index in which the evictions and 8749 // allocations took place. This is used for the write index. 8750 AllocIndex uint64 8751 } 8752 8753 // IsNoOp checks if this plan result would do nothing 8754 func (p *PlanResult) IsNoOp() bool { 8755 return len(p.NodeUpdate) == 0 && len(p.NodeAllocation) == 0 && 8756 len(p.DeploymentUpdates) == 0 && p.Deployment == nil 8757 } 8758 8759 // FullCommit is used to check if all the allocations in a plan 8760 // were committed as part of the result. Returns if there was 8761 // a match, and the number of expected and actual allocations. 8762 func (p *PlanResult) FullCommit(plan *Plan) (bool, int, int) { 8763 expected := 0 8764 actual := 0 8765 for name, allocList := range plan.NodeAllocation { 8766 didAlloc, _ := p.NodeAllocation[name] 8767 expected += len(allocList) 8768 actual += len(didAlloc) 8769 } 8770 return actual == expected, expected, actual 8771 } 8772 8773 // PlanAnnotations holds annotations made by the scheduler to give further debug 8774 // information to operators. 8775 type PlanAnnotations struct { 8776 // DesiredTGUpdates is the set of desired updates per task group. 8777 DesiredTGUpdates map[string]*DesiredUpdates 8778 8779 // PreemptedAllocs is the set of allocations to be preempted to make the placement successful. 8780 PreemptedAllocs []*AllocListStub 8781 } 8782 8783 // DesiredUpdates is the set of changes the scheduler would like to make given 8784 // sufficient resources and cluster capacity. 8785 type DesiredUpdates struct { 8786 Ignore uint64 8787 Place uint64 8788 Migrate uint64 8789 Stop uint64 8790 InPlaceUpdate uint64 8791 DestructiveUpdate uint64 8792 Canary uint64 8793 Preemptions uint64 8794 } 8795 8796 func (d *DesiredUpdates) GoString() string { 8797 return fmt.Sprintf("(place %d) (inplace %d) (destructive %d) (stop %d) (migrate %d) (ignore %d) (canary %d)", 8798 d.Place, d.InPlaceUpdate, d.DestructiveUpdate, d.Stop, d.Migrate, d.Ignore, d.Canary) 8799 } 8800 8801 // msgpackHandle is a shared handle for encoding/decoding of structs 8802 var MsgpackHandle = func() *codec.MsgpackHandle { 8803 h := &codec.MsgpackHandle{RawToString: true} 8804 8805 // Sets the default type for decoding a map into a nil interface{}. 8806 // This is necessary in particular because we store the driver configs as a 8807 // nil interface{}. 8808 h.MapType = reflect.TypeOf(map[string]interface{}(nil)) 8809 return h 8810 }() 8811 8812 var ( 8813 // JsonHandle and JsonHandlePretty are the codec handles to JSON encode 8814 // structs. The pretty handle will add indents for easier human consumption. 8815 JsonHandle = &codec.JsonHandle{ 8816 HTMLCharsAsIs: true, 8817 } 8818 JsonHandlePretty = &codec.JsonHandle{ 8819 HTMLCharsAsIs: true, 8820 Indent: 4, 8821 } 8822 ) 8823 8824 // TODO Figure out if we can remove this. This is our fork that is just way 8825 // behind. I feel like its original purpose was to pin at a stable version but 8826 // now we can accomplish this with vendoring. 8827 var HashiMsgpackHandle = func() *hcodec.MsgpackHandle { 8828 h := &hcodec.MsgpackHandle{RawToString: true} 8829 8830 // Sets the default type for decoding a map into a nil interface{}. 8831 // This is necessary in particular because we store the driver configs as a 8832 // nil interface{}. 8833 h.MapType = reflect.TypeOf(map[string]interface{}(nil)) 8834 return h 8835 }() 8836 8837 // Decode is used to decode a MsgPack encoded object 8838 func Decode(buf []byte, out interface{}) error { 8839 return codec.NewDecoder(bytes.NewReader(buf), MsgpackHandle).Decode(out) 8840 } 8841 8842 // Encode is used to encode a MsgPack object with type prefix 8843 func Encode(t MessageType, msg interface{}) ([]byte, error) { 8844 var buf bytes.Buffer 8845 buf.WriteByte(uint8(t)) 8846 err := codec.NewEncoder(&buf, MsgpackHandle).Encode(msg) 8847 return buf.Bytes(), err 8848 } 8849 8850 // KeyringResponse is a unified key response and can be used for install, 8851 // remove, use, as well as listing key queries. 8852 type KeyringResponse struct { 8853 Messages map[string]string 8854 Keys map[string]int 8855 NumNodes int 8856 } 8857 8858 // KeyringRequest is request objects for serf key operations. 8859 type KeyringRequest struct { 8860 Key string 8861 } 8862 8863 // RecoverableError wraps an error and marks whether it is recoverable and could 8864 // be retried or it is fatal. 8865 type RecoverableError struct { 8866 Err string 8867 Recoverable bool 8868 } 8869 8870 // NewRecoverableError is used to wrap an error and mark it as recoverable or 8871 // not. 8872 func NewRecoverableError(e error, recoverable bool) error { 8873 if e == nil { 8874 return nil 8875 } 8876 8877 return &RecoverableError{ 8878 Err: e.Error(), 8879 Recoverable: recoverable, 8880 } 8881 } 8882 8883 // WrapRecoverable wraps an existing error in a new RecoverableError with a new 8884 // message. If the error was recoverable before the returned error is as well; 8885 // otherwise it is unrecoverable. 8886 func WrapRecoverable(msg string, err error) error { 8887 return &RecoverableError{Err: msg, Recoverable: IsRecoverable(err)} 8888 } 8889 8890 func (r *RecoverableError) Error() string { 8891 return r.Err 8892 } 8893 8894 func (r *RecoverableError) IsRecoverable() bool { 8895 return r.Recoverable 8896 } 8897 8898 func (r *RecoverableError) IsUnrecoverable() bool { 8899 return !r.Recoverable 8900 } 8901 8902 // Recoverable is an interface for errors to implement to indicate whether or 8903 // not they are fatal or recoverable. 8904 type Recoverable interface { 8905 error 8906 IsRecoverable() bool 8907 } 8908 8909 // IsRecoverable returns true if error is a RecoverableError with 8910 // Recoverable=true. Otherwise false is returned. 8911 func IsRecoverable(e error) bool { 8912 if re, ok := e.(Recoverable); ok { 8913 return re.IsRecoverable() 8914 } 8915 return false 8916 } 8917 8918 // WrappedServerError wraps an error and satisfies 8919 // both the Recoverable and the ServerSideError interfaces 8920 type WrappedServerError struct { 8921 Err error 8922 } 8923 8924 // NewWrappedServerError is used to create a wrapped server side error 8925 func NewWrappedServerError(e error) error { 8926 return &WrappedServerError{ 8927 Err: e, 8928 } 8929 } 8930 8931 func (r *WrappedServerError) IsRecoverable() bool { 8932 return IsRecoverable(r.Err) 8933 } 8934 8935 func (r *WrappedServerError) Error() string { 8936 return r.Err.Error() 8937 } 8938 8939 func (r *WrappedServerError) IsServerSide() bool { 8940 return true 8941 } 8942 8943 // ServerSideError is an interface for errors to implement to indicate 8944 // errors occurring after the request makes it to a server 8945 type ServerSideError interface { 8946 error 8947 IsServerSide() bool 8948 } 8949 8950 // IsServerSide returns true if error is a wrapped 8951 // server side error 8952 func IsServerSide(e error) bool { 8953 if se, ok := e.(ServerSideError); ok { 8954 return se.IsServerSide() 8955 } 8956 return false 8957 } 8958 8959 // ACLPolicy is used to represent an ACL policy 8960 type ACLPolicy struct { 8961 Name string // Unique name 8962 Description string // Human readable 8963 Rules string // HCL or JSON format 8964 Hash []byte 8965 CreateIndex uint64 8966 ModifyIndex uint64 8967 } 8968 8969 // SetHash is used to compute and set the hash of the ACL policy 8970 func (c *ACLPolicy) SetHash() []byte { 8971 // Initialize a 256bit Blake2 hash (32 bytes) 8972 hash, err := blake2b.New256(nil) 8973 if err != nil { 8974 panic(err) 8975 } 8976 8977 // Write all the user set fields 8978 hash.Write([]byte(c.Name)) 8979 hash.Write([]byte(c.Description)) 8980 hash.Write([]byte(c.Rules)) 8981 8982 // Finalize the hash 8983 hashVal := hash.Sum(nil) 8984 8985 // Set and return the hash 8986 c.Hash = hashVal 8987 return hashVal 8988 } 8989 8990 func (a *ACLPolicy) Stub() *ACLPolicyListStub { 8991 return &ACLPolicyListStub{ 8992 Name: a.Name, 8993 Description: a.Description, 8994 Hash: a.Hash, 8995 CreateIndex: a.CreateIndex, 8996 ModifyIndex: a.ModifyIndex, 8997 } 8998 } 8999 9000 func (a *ACLPolicy) Validate() error { 9001 var mErr multierror.Error 9002 if !validPolicyName.MatchString(a.Name) { 9003 err := fmt.Errorf("invalid name '%s'", a.Name) 9004 mErr.Errors = append(mErr.Errors, err) 9005 } 9006 if _, err := acl.Parse(a.Rules); err != nil { 9007 err = fmt.Errorf("failed to parse rules: %v", err) 9008 mErr.Errors = append(mErr.Errors, err) 9009 } 9010 if len(a.Description) > maxPolicyDescriptionLength { 9011 err := fmt.Errorf("description longer than %d", maxPolicyDescriptionLength) 9012 mErr.Errors = append(mErr.Errors, err) 9013 } 9014 return mErr.ErrorOrNil() 9015 } 9016 9017 // ACLPolicyListStub is used to for listing ACL policies 9018 type ACLPolicyListStub struct { 9019 Name string 9020 Description string 9021 Hash []byte 9022 CreateIndex uint64 9023 ModifyIndex uint64 9024 } 9025 9026 // ACLPolicyListRequest is used to request a list of policies 9027 type ACLPolicyListRequest struct { 9028 QueryOptions 9029 } 9030 9031 // ACLPolicySpecificRequest is used to query a specific policy 9032 type ACLPolicySpecificRequest struct { 9033 Name string 9034 QueryOptions 9035 } 9036 9037 // ACLPolicySetRequest is used to query a set of policies 9038 type ACLPolicySetRequest struct { 9039 Names []string 9040 QueryOptions 9041 } 9042 9043 // ACLPolicyListResponse is used for a list request 9044 type ACLPolicyListResponse struct { 9045 Policies []*ACLPolicyListStub 9046 QueryMeta 9047 } 9048 9049 // SingleACLPolicyResponse is used to return a single policy 9050 type SingleACLPolicyResponse struct { 9051 Policy *ACLPolicy 9052 QueryMeta 9053 } 9054 9055 // ACLPolicySetResponse is used to return a set of policies 9056 type ACLPolicySetResponse struct { 9057 Policies map[string]*ACLPolicy 9058 QueryMeta 9059 } 9060 9061 // ACLPolicyDeleteRequest is used to delete a set of policies 9062 type ACLPolicyDeleteRequest struct { 9063 Names []string 9064 WriteRequest 9065 } 9066 9067 // ACLPolicyUpsertRequest is used to upsert a set of policies 9068 type ACLPolicyUpsertRequest struct { 9069 Policies []*ACLPolicy 9070 WriteRequest 9071 } 9072 9073 // ACLToken represents a client token which is used to Authenticate 9074 type ACLToken struct { 9075 AccessorID string // Public Accessor ID (UUID) 9076 SecretID string // Secret ID, private (UUID) 9077 Name string // Human friendly name 9078 Type string // Client or Management 9079 Policies []string // Policies this token ties to 9080 Global bool // Global or Region local 9081 Hash []byte 9082 CreateTime time.Time // Time of creation 9083 CreateIndex uint64 9084 ModifyIndex uint64 9085 } 9086 9087 var ( 9088 // AnonymousACLToken is used no SecretID is provided, and the 9089 // request is made anonymously. 9090 AnonymousACLToken = &ACLToken{ 9091 AccessorID: "anonymous", 9092 Name: "Anonymous Token", 9093 Type: ACLClientToken, 9094 Policies: []string{"anonymous"}, 9095 Global: false, 9096 } 9097 ) 9098 9099 type ACLTokenListStub struct { 9100 AccessorID string 9101 Name string 9102 Type string 9103 Policies []string 9104 Global bool 9105 Hash []byte 9106 CreateTime time.Time 9107 CreateIndex uint64 9108 ModifyIndex uint64 9109 } 9110 9111 // SetHash is used to compute and set the hash of the ACL token 9112 func (a *ACLToken) SetHash() []byte { 9113 // Initialize a 256bit Blake2 hash (32 bytes) 9114 hash, err := blake2b.New256(nil) 9115 if err != nil { 9116 panic(err) 9117 } 9118 9119 // Write all the user set fields 9120 hash.Write([]byte(a.Name)) 9121 hash.Write([]byte(a.Type)) 9122 for _, policyName := range a.Policies { 9123 hash.Write([]byte(policyName)) 9124 } 9125 if a.Global { 9126 hash.Write([]byte("global")) 9127 } else { 9128 hash.Write([]byte("local")) 9129 } 9130 9131 // Finalize the hash 9132 hashVal := hash.Sum(nil) 9133 9134 // Set and return the hash 9135 a.Hash = hashVal 9136 return hashVal 9137 } 9138 9139 func (a *ACLToken) Stub() *ACLTokenListStub { 9140 return &ACLTokenListStub{ 9141 AccessorID: a.AccessorID, 9142 Name: a.Name, 9143 Type: a.Type, 9144 Policies: a.Policies, 9145 Global: a.Global, 9146 Hash: a.Hash, 9147 CreateTime: a.CreateTime, 9148 CreateIndex: a.CreateIndex, 9149 ModifyIndex: a.ModifyIndex, 9150 } 9151 } 9152 9153 // Validate is used to sanity check a token 9154 func (a *ACLToken) Validate() error { 9155 var mErr multierror.Error 9156 if len(a.Name) > maxTokenNameLength { 9157 mErr.Errors = append(mErr.Errors, fmt.Errorf("token name too long")) 9158 } 9159 switch a.Type { 9160 case ACLClientToken: 9161 if len(a.Policies) == 0 { 9162 mErr.Errors = append(mErr.Errors, fmt.Errorf("client token missing policies")) 9163 } 9164 case ACLManagementToken: 9165 if len(a.Policies) != 0 { 9166 mErr.Errors = append(mErr.Errors, fmt.Errorf("management token cannot be associated with policies")) 9167 } 9168 default: 9169 mErr.Errors = append(mErr.Errors, fmt.Errorf("token type must be client or management")) 9170 } 9171 return mErr.ErrorOrNil() 9172 } 9173 9174 // PolicySubset checks if a given set of policies is a subset of the token 9175 func (a *ACLToken) PolicySubset(policies []string) bool { 9176 // Hot-path the management tokens, superset of all policies. 9177 if a.Type == ACLManagementToken { 9178 return true 9179 } 9180 associatedPolicies := make(map[string]struct{}, len(a.Policies)) 9181 for _, policy := range a.Policies { 9182 associatedPolicies[policy] = struct{}{} 9183 } 9184 for _, policy := range policies { 9185 if _, ok := associatedPolicies[policy]; !ok { 9186 return false 9187 } 9188 } 9189 return true 9190 } 9191 9192 // ACLTokenListRequest is used to request a list of tokens 9193 type ACLTokenListRequest struct { 9194 GlobalOnly bool 9195 QueryOptions 9196 } 9197 9198 // ACLTokenSpecificRequest is used to query a specific token 9199 type ACLTokenSpecificRequest struct { 9200 AccessorID string 9201 QueryOptions 9202 } 9203 9204 // ACLTokenSetRequest is used to query a set of tokens 9205 type ACLTokenSetRequest struct { 9206 AccessorIDS []string 9207 QueryOptions 9208 } 9209 9210 // ACLTokenListResponse is used for a list request 9211 type ACLTokenListResponse struct { 9212 Tokens []*ACLTokenListStub 9213 QueryMeta 9214 } 9215 9216 // SingleACLTokenResponse is used to return a single token 9217 type SingleACLTokenResponse struct { 9218 Token *ACLToken 9219 QueryMeta 9220 } 9221 9222 // ACLTokenSetResponse is used to return a set of token 9223 type ACLTokenSetResponse struct { 9224 Tokens map[string]*ACLToken // Keyed by Accessor ID 9225 QueryMeta 9226 } 9227 9228 // ResolveACLTokenRequest is used to resolve a specific token 9229 type ResolveACLTokenRequest struct { 9230 SecretID string 9231 QueryOptions 9232 } 9233 9234 // ResolveACLTokenResponse is used to resolve a single token 9235 type ResolveACLTokenResponse struct { 9236 Token *ACLToken 9237 QueryMeta 9238 } 9239 9240 // ACLTokenDeleteRequest is used to delete a set of tokens 9241 type ACLTokenDeleteRequest struct { 9242 AccessorIDs []string 9243 WriteRequest 9244 } 9245 9246 // ACLTokenBootstrapRequest is used to bootstrap ACLs 9247 type ACLTokenBootstrapRequest struct { 9248 Token *ACLToken // Not client specifiable 9249 ResetIndex uint64 // Reset index is used to clear the bootstrap token 9250 WriteRequest 9251 } 9252 9253 // ACLTokenUpsertRequest is used to upsert a set of tokens 9254 type ACLTokenUpsertRequest struct { 9255 Tokens []*ACLToken 9256 WriteRequest 9257 } 9258 9259 // ACLTokenUpsertResponse is used to return from an ACLTokenUpsertRequest 9260 type ACLTokenUpsertResponse struct { 9261 Tokens []*ACLToken 9262 WriteMeta 9263 }