github.com/hhrutter/nomad@v0.6.0-rc2.0.20170723054333-80c4b03f0705/command/status.go (about) 1 package command 2 3 import ( 4 "fmt" 5 "sort" 6 "strings" 7 "time" 8 9 "github.com/hashicorp/nomad/api" 10 "github.com/hashicorp/nomad/nomad/structs" 11 ) 12 13 const ( 14 // maxFailedTGs is the maximum number of task groups we show failure reasons 15 // for before defering to eval-status 16 maxFailedTGs = 5 17 ) 18 19 type StatusCommand struct { 20 Meta 21 length int 22 evals bool 23 allAllocs bool 24 verbose bool 25 } 26 27 func (c *StatusCommand) Help() string { 28 helpText := ` 29 Usage: nomad status [options] <job> 30 31 Display status information about jobs. If no job ID is given, 32 a list of all known jobs will be dumped. 33 34 General Options: 35 36 ` + generalOptionsUsage() + ` 37 38 Status Options: 39 40 -short 41 Display short output. Used only when a single job is being 42 queried, and drops verbose information about allocations. 43 44 -evals 45 Display the evaluations associated with the job. 46 47 -all-allocs 48 Display all allocations matching the job ID, including those from an older 49 instance of the job. 50 51 -verbose 52 Display full information. 53 ` 54 return strings.TrimSpace(helpText) 55 } 56 57 func (c *StatusCommand) Synopsis() string { 58 return "Display status information about jobs" 59 } 60 61 func (c *StatusCommand) Run(args []string) int { 62 var short bool 63 64 flags := c.Meta.FlagSet("status", FlagSetClient) 65 flags.Usage = func() { c.Ui.Output(c.Help()) } 66 flags.BoolVar(&short, "short", false, "") 67 flags.BoolVar(&c.evals, "evals", false, "") 68 flags.BoolVar(&c.allAllocs, "all-allocs", false, "") 69 flags.BoolVar(&c.verbose, "verbose", false, "") 70 71 if err := flags.Parse(args); err != nil { 72 return 1 73 } 74 75 // Check that we either got no jobs or exactly one. 76 args = flags.Args() 77 if len(args) > 1 { 78 c.Ui.Error(c.Help()) 79 return 1 80 } 81 82 // Truncate the id unless full length is requested 83 c.length = shortId 84 if c.verbose { 85 c.length = fullId 86 } 87 88 // Get the HTTP client 89 client, err := c.Meta.Client() 90 if err != nil { 91 c.Ui.Error(fmt.Sprintf("Error initializing client: %s", err)) 92 return 1 93 } 94 95 // Invoke list mode if no job ID. 96 if len(args) == 0 { 97 jobs, _, err := client.Jobs().List(nil) 98 if err != nil { 99 c.Ui.Error(fmt.Sprintf("Error querying jobs: %s", err)) 100 return 1 101 } 102 103 if len(jobs) == 0 { 104 // No output if we have no jobs 105 c.Ui.Output("No running jobs") 106 } else { 107 c.Ui.Output(createStatusListOutput(jobs)) 108 } 109 return 0 110 } 111 112 // Try querying the job 113 jobID := args[0] 114 jobs, _, err := client.Jobs().PrefixList(jobID) 115 if err != nil { 116 c.Ui.Error(fmt.Sprintf("Error querying job: %s", err)) 117 return 1 118 } 119 if len(jobs) == 0 { 120 c.Ui.Error(fmt.Sprintf("No job(s) with prefix or id %q found", jobID)) 121 return 1 122 } 123 if len(jobs) > 1 && strings.TrimSpace(jobID) != jobs[0].ID { 124 c.Ui.Error(fmt.Sprintf("Prefix matched multiple jobs\n\n%s", createStatusListOutput(jobs))) 125 return 1 126 } 127 // Prefix lookup matched a single job 128 job, _, err := client.Jobs().Info(jobs[0].ID, nil) 129 if err != nil { 130 c.Ui.Error(fmt.Sprintf("Error querying job: %s", err)) 131 return 1 132 } 133 134 periodic := job.IsPeriodic() 135 parameterized := job.IsParameterized() 136 137 // Format the job info 138 basic := []string{ 139 fmt.Sprintf("ID|%s", *job.ID), 140 fmt.Sprintf("Name|%s", *job.Name), 141 fmt.Sprintf("Submit Date|%s", formatTime(time.Unix(0, *job.SubmitTime))), 142 fmt.Sprintf("Type|%s", *job.Type), 143 fmt.Sprintf("Priority|%d", *job.Priority), 144 fmt.Sprintf("Datacenters|%s", strings.Join(job.Datacenters, ",")), 145 fmt.Sprintf("Status|%s", getStatusString(*job.Status, *job.Stop)), 146 fmt.Sprintf("Periodic|%v", periodic), 147 fmt.Sprintf("Parameterized|%v", parameterized), 148 } 149 150 if periodic && !parameterized { 151 if *job.Stop { 152 basic = append(basic, fmt.Sprintf("Next Periodic Launch|none (job stopped)")) 153 } else { 154 location, err := job.Periodic.GetLocation() 155 if err == nil { 156 now := time.Now().In(location) 157 next := job.Periodic.Next(now) 158 basic = append(basic, fmt.Sprintf("Next Periodic Launch|%s", 159 fmt.Sprintf("%s (%s from now)", 160 formatTime(next), formatTimeDifference(now, next, time.Second)))) 161 } 162 } 163 } 164 165 c.Ui.Output(formatKV(basic)) 166 167 // Exit early 168 if short { 169 return 0 170 } 171 172 // Print periodic job information 173 if periodic && !parameterized { 174 if err := c.outputPeriodicInfo(client, job); err != nil { 175 c.Ui.Error(err.Error()) 176 return 1 177 } 178 } else if parameterized { 179 if err := c.outputParameterizedInfo(client, job); err != nil { 180 c.Ui.Error(err.Error()) 181 return 1 182 } 183 } else { 184 if err := c.outputJobInfo(client, job); err != nil { 185 c.Ui.Error(err.Error()) 186 return 1 187 } 188 } 189 190 return 0 191 } 192 193 // outputPeriodicInfo prints information about the passed periodic job. If a 194 // request fails, an error is returned. 195 func (c *StatusCommand) outputPeriodicInfo(client *api.Client, job *api.Job) error { 196 // Output the summary 197 if err := c.outputJobSummary(client, job); err != nil { 198 return err 199 } 200 201 // Generate the prefix that matches launched jobs from the periodic job. 202 prefix := fmt.Sprintf("%s%s", *job.ID, structs.PeriodicLaunchSuffix) 203 children, _, err := client.Jobs().PrefixList(prefix) 204 if err != nil { 205 return fmt.Errorf("Error querying job: %s", err) 206 } 207 208 if len(children) == 0 { 209 c.Ui.Output("\nNo instances of periodic job found") 210 return nil 211 } 212 213 out := make([]string, 1) 214 out[0] = "ID|Status" 215 for _, child := range children { 216 // Ensure that we are only showing jobs whose parent is the requested 217 // job. 218 if child.ParentID != *job.ID { 219 continue 220 } 221 222 out = append(out, fmt.Sprintf("%s|%s", 223 child.ID, 224 child.Status)) 225 } 226 227 c.Ui.Output(c.Colorize().Color("\n[bold]Previously Launched Jobs[reset]")) 228 c.Ui.Output(formatList(out)) 229 return nil 230 } 231 232 // outputParameterizedInfo prints information about a parameterized job. If a 233 // request fails, an error is returned. 234 func (c *StatusCommand) outputParameterizedInfo(client *api.Client, job *api.Job) error { 235 // Output parameterized job details 236 c.Ui.Output(c.Colorize().Color("\n[bold]Parameterized Job[reset]")) 237 parameterizedJob := make([]string, 3) 238 parameterizedJob[0] = fmt.Sprintf("Payload|%s", job.ParameterizedJob.Payload) 239 parameterizedJob[1] = fmt.Sprintf("Required Metadata|%v", strings.Join(job.ParameterizedJob.MetaRequired, ", ")) 240 parameterizedJob[2] = fmt.Sprintf("Optional Metadata|%v", strings.Join(job.ParameterizedJob.MetaOptional, ", ")) 241 c.Ui.Output(formatKV(parameterizedJob)) 242 243 // Output the summary 244 if err := c.outputJobSummary(client, job); err != nil { 245 return err 246 } 247 248 // Generate the prefix that matches launched jobs from the parameterized job. 249 prefix := fmt.Sprintf("%s%s", *job.ID, structs.DispatchLaunchSuffix) 250 children, _, err := client.Jobs().PrefixList(prefix) 251 if err != nil { 252 return fmt.Errorf("Error querying job: %s", err) 253 } 254 255 if len(children) == 0 { 256 c.Ui.Output("\nNo dispatched instances of parameterized job found") 257 return nil 258 } 259 260 out := make([]string, 1) 261 out[0] = "ID|Status" 262 for _, child := range children { 263 // Ensure that we are only showing jobs whose parent is the requested 264 // job. 265 if child.ParentID != *job.ID { 266 continue 267 } 268 269 out = append(out, fmt.Sprintf("%s|%s", 270 child.ID, 271 child.Status)) 272 } 273 274 c.Ui.Output(c.Colorize().Color("\n[bold]Dispatched Jobs[reset]")) 275 c.Ui.Output(formatList(out)) 276 return nil 277 } 278 279 // outputJobInfo prints information about the passed non-periodic job. If a 280 // request fails, an error is returned. 281 func (c *StatusCommand) outputJobInfo(client *api.Client, job *api.Job) error { 282 283 // Query the allocations 284 jobAllocs, _, err := client.Jobs().Allocations(*job.ID, c.allAllocs, nil) 285 if err != nil { 286 return fmt.Errorf("Error querying job allocations: %s", err) 287 } 288 289 // Query the evaluations 290 jobEvals, _, err := client.Jobs().Evaluations(*job.ID, nil) 291 if err != nil { 292 return fmt.Errorf("Error querying job evaluations: %s", err) 293 } 294 295 latestDeployment, _, err := client.Jobs().LatestDeployment(*job.ID, nil) 296 if err != nil { 297 return fmt.Errorf("Error querying latest job deployment: %s", err) 298 } 299 300 // Output the summary 301 if err := c.outputJobSummary(client, job); err != nil { 302 return err 303 } 304 305 // Determine latest evaluation with failures whose follow up hasn't 306 // completed, this is done while formatting 307 var latestFailedPlacement *api.Evaluation 308 blockedEval := false 309 310 // Format the evals 311 evals := make([]string, len(jobEvals)+1) 312 evals[0] = "ID|Priority|Triggered By|Status|Placement Failures" 313 for i, eval := range jobEvals { 314 failures, _ := evalFailureStatus(eval) 315 evals[i+1] = fmt.Sprintf("%s|%d|%s|%s|%s", 316 limit(eval.ID, c.length), 317 eval.Priority, 318 eval.TriggeredBy, 319 eval.Status, 320 failures, 321 ) 322 323 if eval.Status == "blocked" { 324 blockedEval = true 325 } 326 327 if len(eval.FailedTGAllocs) == 0 { 328 // Skip evals without failures 329 continue 330 } 331 332 if latestFailedPlacement == nil || latestFailedPlacement.CreateIndex < eval.CreateIndex { 333 latestFailedPlacement = eval 334 } 335 } 336 337 if c.verbose || c.evals { 338 c.Ui.Output(c.Colorize().Color("\n[bold]Evaluations[reset]")) 339 c.Ui.Output(formatList(evals)) 340 } 341 342 if blockedEval && latestFailedPlacement != nil { 343 c.outputFailedPlacements(latestFailedPlacement) 344 } 345 346 if latestDeployment != nil { 347 c.Ui.Output(c.Colorize().Color("\n[bold]Latest Deployment[reset]")) 348 c.Ui.Output(c.Colorize().Color(c.formatDeployment(latestDeployment))) 349 } 350 351 // Format the allocs 352 c.Ui.Output(c.Colorize().Color("\n[bold]Allocations[reset]")) 353 c.Ui.Output(formatAllocListStubs(jobAllocs, c.verbose, c.length)) 354 return nil 355 } 356 357 func (c *StatusCommand) formatDeployment(d *api.Deployment) string { 358 // Format the high-level elements 359 high := []string{ 360 fmt.Sprintf("ID|%s", limit(d.ID, c.length)), 361 fmt.Sprintf("Status|%s", d.Status), 362 fmt.Sprintf("Description|%s", d.StatusDescription), 363 } 364 365 base := formatKV(high) 366 if len(d.TaskGroups) == 0 { 367 return base 368 } 369 base += "\n\n[bold]Deployed[reset]\n" 370 base += formatDeploymentGroups(d, c.length) 371 return base 372 } 373 374 func formatAllocListStubs(stubs []*api.AllocationListStub, verbose bool, uuidLength int) string { 375 if len(stubs) == 0 { 376 return "No allocations placed" 377 } 378 379 allocs := make([]string, len(stubs)+1) 380 if verbose { 381 allocs[0] = "ID|Eval ID|Node ID|Task Group|Version|Desired|Status|Created At" 382 for i, alloc := range stubs { 383 allocs[i+1] = fmt.Sprintf("%s|%s|%s|%s|%d|%s|%s|%s", 384 limit(alloc.ID, uuidLength), 385 limit(alloc.EvalID, uuidLength), 386 limit(alloc.NodeID, uuidLength), 387 alloc.TaskGroup, 388 alloc.JobVersion, 389 alloc.DesiredStatus, 390 alloc.ClientStatus, 391 formatUnixNanoTime(alloc.CreateTime)) 392 } 393 } else { 394 allocs[0] = "ID|Node ID|Task Group|Version|Desired|Status|Created At" 395 for i, alloc := range stubs { 396 allocs[i+1] = fmt.Sprintf("%s|%s|%s|%d|%s|%s|%s", 397 limit(alloc.ID, uuidLength), 398 limit(alloc.NodeID, uuidLength), 399 alloc.TaskGroup, 400 alloc.JobVersion, 401 alloc.DesiredStatus, 402 alloc.ClientStatus, 403 formatUnixNanoTime(alloc.CreateTime)) 404 } 405 } 406 407 return formatList(allocs) 408 } 409 410 func formatAllocList(allocations []*api.Allocation, verbose bool, uuidLength int) string { 411 if len(allocations) == 0 { 412 return "No allocations placed" 413 } 414 415 allocs := make([]string, len(allocations)+1) 416 if verbose { 417 allocs[0] = "ID|Eval ID|Node ID|Task Group|Version|Desired|Status|Created At" 418 for i, alloc := range allocations { 419 allocs[i+1] = fmt.Sprintf("%s|%s|%s|%s|%d|%s|%s|%s", 420 limit(alloc.ID, uuidLength), 421 limit(alloc.EvalID, uuidLength), 422 limit(alloc.NodeID, uuidLength), 423 alloc.TaskGroup, 424 *alloc.Job.Version, 425 alloc.DesiredStatus, 426 alloc.ClientStatus, 427 formatUnixNanoTime(alloc.CreateTime)) 428 } 429 } else { 430 allocs[0] = "ID|Node ID|Task Group|Version|Desired|Status|Created At" 431 for i, alloc := range allocations { 432 allocs[i+1] = fmt.Sprintf("%s|%s|%s|%d|%s|%s|%s", 433 limit(alloc.ID, uuidLength), 434 limit(alloc.NodeID, uuidLength), 435 alloc.TaskGroup, 436 *alloc.Job.Version, 437 alloc.DesiredStatus, 438 alloc.ClientStatus, 439 formatUnixNanoTime(alloc.CreateTime)) 440 } 441 } 442 443 return formatList(allocs) 444 } 445 446 // outputJobSummary displays the given jobs summary and children job summary 447 // where appropriate 448 func (c *StatusCommand) outputJobSummary(client *api.Client, job *api.Job) error { 449 // Query the summary 450 summary, _, err := client.Jobs().Summary(*job.ID, nil) 451 if err != nil { 452 return fmt.Errorf("Error querying job summary: %s", err) 453 } 454 455 if summary == nil { 456 return nil 457 } 458 459 periodic := job.IsPeriodic() 460 parameterizedJob := job.IsParameterized() 461 462 // Print the summary 463 if !periodic && !parameterizedJob { 464 c.Ui.Output(c.Colorize().Color("\n[bold]Summary[reset]")) 465 summaries := make([]string, len(summary.Summary)+1) 466 summaries[0] = "Task Group|Queued|Starting|Running|Failed|Complete|Lost" 467 taskGroups := make([]string, 0, len(summary.Summary)) 468 for taskGroup := range summary.Summary { 469 taskGroups = append(taskGroups, taskGroup) 470 } 471 sort.Strings(taskGroups) 472 for idx, taskGroup := range taskGroups { 473 tgs := summary.Summary[taskGroup] 474 summaries[idx+1] = fmt.Sprintf("%s|%d|%d|%d|%d|%d|%d", 475 taskGroup, tgs.Queued, tgs.Starting, 476 tgs.Running, tgs.Failed, 477 tgs.Complete, tgs.Lost, 478 ) 479 } 480 c.Ui.Output(formatList(summaries)) 481 } 482 483 // Always display the summary if we are periodic or parameterized, but 484 // only display if the summary is non-zero on normal jobs 485 if summary.Children != nil && (parameterizedJob || periodic || summary.Children.Sum() > 0) { 486 if parameterizedJob { 487 c.Ui.Output(c.Colorize().Color("\n[bold]Parameterized Job Summary[reset]")) 488 } else { 489 c.Ui.Output(c.Colorize().Color("\n[bold]Children Job Summary[reset]")) 490 } 491 summaries := make([]string, 2) 492 summaries[0] = "Pending|Running|Dead" 493 summaries[1] = fmt.Sprintf("%d|%d|%d", 494 summary.Children.Pending, summary.Children.Running, summary.Children.Dead) 495 c.Ui.Output(formatList(summaries)) 496 } 497 498 return nil 499 } 500 501 func (c *StatusCommand) outputFailedPlacements(failedEval *api.Evaluation) { 502 if failedEval == nil || len(failedEval.FailedTGAllocs) == 0 { 503 return 504 } 505 506 c.Ui.Output(c.Colorize().Color("\n[bold]Placement Failure[reset]")) 507 508 sorted := sortedTaskGroupFromMetrics(failedEval.FailedTGAllocs) 509 for i, tg := range sorted { 510 if i >= maxFailedTGs { 511 break 512 } 513 514 c.Ui.Output(fmt.Sprintf("Task Group %q:", tg)) 515 metrics := failedEval.FailedTGAllocs[tg] 516 c.Ui.Output(formatAllocMetrics(metrics, false, " ")) 517 if i != len(sorted)-1 { 518 c.Ui.Output("") 519 } 520 } 521 522 if len(sorted) > maxFailedTGs { 523 trunc := fmt.Sprintf("\nPlacement failures truncated. To see remainder run:\nnomad eval-status %s", failedEval.ID) 524 c.Ui.Output(trunc) 525 } 526 } 527 528 // list general information about a list of jobs 529 func createStatusListOutput(jobs []*api.JobListStub) string { 530 out := make([]string, len(jobs)+1) 531 out[0] = "ID|Type|Priority|Status|Submit Date" 532 for i, job := range jobs { 533 out[i+1] = fmt.Sprintf("%s|%s|%d|%s|%s", 534 job.ID, 535 getTypeString(job), 536 job.Priority, 537 getStatusString(job.Status, job.Stop), 538 formatTime(time.Unix(0, job.SubmitTime))) 539 } 540 return formatList(out) 541 } 542 543 func getTypeString(job *api.JobListStub) string { 544 t := job.Type 545 546 if job.Periodic { 547 t += "/periodic" 548 } 549 550 if job.ParameterizedJob { 551 t += "/parameterized" 552 } 553 554 return t 555 } 556 557 func getStatusString(status string, stop bool) string { 558 if stop { 559 return fmt.Sprintf("%s (stopped)", status) 560 } 561 return status 562 }