github.com/diptanu/nomad@v0.5.7-0.20170516172507-d72e86cbe3d9/command/status.go (about) 1 package command 2 3 import ( 4 "fmt" 5 "sort" 6 "strings" 7 "time" 8 9 "github.com/hashicorp/nomad/api" 10 "github.com/hashicorp/nomad/nomad/structs" 11 ) 12 13 const ( 14 // maxFailedTGs is the maximum number of task groups we show failure reasons 15 // for before defering to eval-status 16 maxFailedTGs = 5 17 ) 18 19 type StatusCommand struct { 20 Meta 21 length int 22 evals bool 23 allAllocs bool 24 verbose bool 25 } 26 27 func (c *StatusCommand) Help() string { 28 helpText := ` 29 Usage: nomad status [options] <job> 30 31 Display status information about jobs. If no job ID is given, 32 a list of all known jobs will be dumped. 33 34 General Options: 35 36 ` + generalOptionsUsage() + ` 37 38 Status Options: 39 40 -short 41 Display short output. Used only when a single job is being 42 queried, and drops verbose information about allocations. 43 44 -evals 45 Display the evaluations associated with the job. 46 47 -all-allocs 48 Display all allocations matching the job ID, including those from an older 49 instance of the job. 50 51 -verbose 52 Display full information. 53 ` 54 return strings.TrimSpace(helpText) 55 } 56 57 func (c *StatusCommand) Synopsis() string { 58 return "Display status information about jobs" 59 } 60 61 func (c *StatusCommand) Run(args []string) int { 62 var short bool 63 64 flags := c.Meta.FlagSet("status", FlagSetClient) 65 flags.Usage = func() { c.Ui.Output(c.Help()) } 66 flags.BoolVar(&short, "short", false, "") 67 flags.BoolVar(&c.evals, "evals", false, "") 68 flags.BoolVar(&c.allAllocs, "all-allocs", false, "") 69 flags.BoolVar(&c.verbose, "verbose", false, "") 70 71 if err := flags.Parse(args); err != nil { 72 return 1 73 } 74 75 // Check that we either got no jobs or exactly one. 76 args = flags.Args() 77 if len(args) > 1 { 78 c.Ui.Error(c.Help()) 79 return 1 80 } 81 82 // Truncate the id unless full length is requested 83 c.length = shortId 84 if c.verbose { 85 c.length = fullId 86 } 87 88 // Get the HTTP client 89 client, err := c.Meta.Client() 90 if err != nil { 91 c.Ui.Error(fmt.Sprintf("Error initializing client: %s", err)) 92 return 1 93 } 94 95 // Invoke list mode if no job ID. 96 if len(args) == 0 { 97 jobs, _, err := client.Jobs().List(nil) 98 if err != nil { 99 c.Ui.Error(fmt.Sprintf("Error querying jobs: %s", err)) 100 return 1 101 } 102 103 if len(jobs) == 0 { 104 // No output if we have no jobs 105 c.Ui.Output("No running jobs") 106 } else { 107 c.Ui.Output(createStatusListOutput(jobs)) 108 } 109 return 0 110 } 111 112 // Try querying the job 113 jobID := args[0] 114 jobs, _, err := client.Jobs().PrefixList(jobID) 115 if err != nil { 116 c.Ui.Error(fmt.Sprintf("Error querying job: %s", err)) 117 return 1 118 } 119 if len(jobs) == 0 { 120 c.Ui.Error(fmt.Sprintf("No job(s) with prefix or id %q found", jobID)) 121 return 1 122 } 123 if len(jobs) > 1 && strings.TrimSpace(jobID) != jobs[0].ID { 124 c.Ui.Output(fmt.Sprintf("Prefix matched multiple jobs\n\n%s", createStatusListOutput(jobs))) 125 return 0 126 } 127 // Prefix lookup matched a single job 128 job, _, err := client.Jobs().Info(jobs[0].ID, nil) 129 if err != nil { 130 c.Ui.Error(fmt.Sprintf("Error querying job: %s", err)) 131 return 1 132 } 133 134 periodic := job.IsPeriodic() 135 parameterized := job.IsParameterized() 136 137 // Format the job info 138 basic := []string{ 139 fmt.Sprintf("ID|%s", *job.ID), 140 fmt.Sprintf("Name|%s", *job.Name), 141 fmt.Sprintf("Type|%s", *job.Type), 142 fmt.Sprintf("Priority|%d", *job.Priority), 143 fmt.Sprintf("Datacenters|%s", strings.Join(job.Datacenters, ",")), 144 fmt.Sprintf("Status|%s", getStatusString(*job.Status, *job.Stop)), 145 fmt.Sprintf("Periodic|%v", periodic), 146 fmt.Sprintf("Parameterized|%v", parameterized), 147 } 148 149 if periodic && !parameterized { 150 if *job.Stop { 151 basic = append(basic, fmt.Sprintf("Next Periodic Launch|none (job stopped)")) 152 } else { 153 location, err := job.Periodic.GetLocation() 154 if err == nil { 155 now := time.Now().In(location) 156 next := job.Periodic.Next(now) 157 basic = append(basic, fmt.Sprintf("Next Periodic Launch|%s", 158 fmt.Sprintf("%s (%s from now)", 159 formatTime(next), formatTimeDifference(now, next, time.Second)))) 160 } 161 } 162 } 163 164 c.Ui.Output(formatKV(basic)) 165 166 // Exit early 167 if short { 168 return 0 169 } 170 171 // Print periodic job information 172 if periodic && !parameterized { 173 if err := c.outputPeriodicInfo(client, job); err != nil { 174 c.Ui.Error(err.Error()) 175 return 1 176 } 177 } else if parameterized { 178 if err := c.outputParameterizedInfo(client, job); err != nil { 179 c.Ui.Error(err.Error()) 180 return 1 181 } 182 } else { 183 if err := c.outputJobInfo(client, job); err != nil { 184 c.Ui.Error(err.Error()) 185 return 1 186 } 187 } 188 189 return 0 190 } 191 192 // outputPeriodicInfo prints information about the passed periodic job. If a 193 // request fails, an error is returned. 194 func (c *StatusCommand) outputPeriodicInfo(client *api.Client, job *api.Job) error { 195 // Output the summary 196 if err := c.outputJobSummary(client, job); err != nil { 197 return err 198 } 199 200 // Generate the prefix that matches launched jobs from the periodic job. 201 prefix := fmt.Sprintf("%s%s", *job.ID, structs.PeriodicLaunchSuffix) 202 children, _, err := client.Jobs().PrefixList(prefix) 203 if err != nil { 204 return fmt.Errorf("Error querying job: %s", err) 205 } 206 207 if len(children) == 0 { 208 c.Ui.Output("\nNo instances of periodic job found") 209 return nil 210 } 211 212 out := make([]string, 1) 213 out[0] = "ID|Status" 214 for _, child := range children { 215 // Ensure that we are only showing jobs whose parent is the requested 216 // job. 217 if child.ParentID != *job.ID { 218 continue 219 } 220 221 out = append(out, fmt.Sprintf("%s|%s", 222 child.ID, 223 child.Status)) 224 } 225 226 c.Ui.Output(c.Colorize().Color("\n[bold]Previously Launched Jobs[reset]")) 227 c.Ui.Output(formatList(out)) 228 return nil 229 } 230 231 // outputParameterizedInfo prints information about a parameterized job. If a 232 // request fails, an error is returned. 233 func (c *StatusCommand) outputParameterizedInfo(client *api.Client, job *api.Job) error { 234 // Output parameterized job details 235 c.Ui.Output(c.Colorize().Color("\n[bold]Parameterized Job[reset]")) 236 parameterizedJob := make([]string, 3) 237 parameterizedJob[0] = fmt.Sprintf("Payload|%s", job.ParameterizedJob.Payload) 238 parameterizedJob[1] = fmt.Sprintf("Required Metadata|%v", strings.Join(job.ParameterizedJob.MetaRequired, ", ")) 239 parameterizedJob[2] = fmt.Sprintf("Optional Metadata|%v", strings.Join(job.ParameterizedJob.MetaOptional, ", ")) 240 c.Ui.Output(formatKV(parameterizedJob)) 241 242 // Output the summary 243 if err := c.outputJobSummary(client, job); err != nil { 244 return err 245 } 246 247 // Generate the prefix that matches launched jobs from the parameterized job. 248 prefix := fmt.Sprintf("%s%s", *job.ID, structs.DispatchLaunchSuffix) 249 children, _, err := client.Jobs().PrefixList(prefix) 250 if err != nil { 251 return fmt.Errorf("Error querying job: %s", err) 252 } 253 254 if len(children) == 0 { 255 c.Ui.Output("\nNo dispatched instances of parameterized job found") 256 return nil 257 } 258 259 out := make([]string, 1) 260 out[0] = "ID|Status" 261 for _, child := range children { 262 // Ensure that we are only showing jobs whose parent is the requested 263 // job. 264 if child.ParentID != *job.ID { 265 continue 266 } 267 268 out = append(out, fmt.Sprintf("%s|%s", 269 child.ID, 270 child.Status)) 271 } 272 273 c.Ui.Output(c.Colorize().Color("\n[bold]Dispatched Jobs[reset]")) 274 c.Ui.Output(formatList(out)) 275 return nil 276 } 277 278 // outputJobInfo prints information about the passed non-periodic job. If a 279 // request fails, an error is returned. 280 func (c *StatusCommand) outputJobInfo(client *api.Client, job *api.Job) error { 281 var evals, allocs []string 282 283 // Query the allocations 284 jobAllocs, _, err := client.Jobs().Allocations(*job.ID, c.allAllocs, nil) 285 if err != nil { 286 return fmt.Errorf("Error querying job allocations: %s", err) 287 } 288 289 // Query the evaluations 290 jobEvals, _, err := client.Jobs().Evaluations(*job.ID, nil) 291 if err != nil { 292 return fmt.Errorf("Error querying job evaluations: %s", err) 293 } 294 295 // Output the summary 296 if err := c.outputJobSummary(client, job); err != nil { 297 return err 298 } 299 300 // Determine latest evaluation with failures whose follow up hasn't 301 // completed, this is done while formatting 302 var latestFailedPlacement *api.Evaluation 303 blockedEval := false 304 305 // Format the evals 306 evals = make([]string, len(jobEvals)+1) 307 evals[0] = "ID|Priority|Triggered By|Status|Placement Failures" 308 for i, eval := range jobEvals { 309 failures, _ := evalFailureStatus(eval) 310 evals[i+1] = fmt.Sprintf("%s|%d|%s|%s|%s", 311 limit(eval.ID, c.length), 312 eval.Priority, 313 eval.TriggeredBy, 314 eval.Status, 315 failures, 316 ) 317 318 if eval.Status == "blocked" { 319 blockedEval = true 320 } 321 322 if len(eval.FailedTGAllocs) == 0 { 323 // Skip evals without failures 324 continue 325 } 326 327 if latestFailedPlacement == nil || latestFailedPlacement.CreateIndex < eval.CreateIndex { 328 latestFailedPlacement = eval 329 } 330 } 331 332 if c.verbose || c.evals { 333 c.Ui.Output(c.Colorize().Color("\n[bold]Evaluations[reset]")) 334 c.Ui.Output(formatList(evals)) 335 } 336 337 if blockedEval && latestFailedPlacement != nil { 338 c.outputFailedPlacements(latestFailedPlacement) 339 } 340 341 // Format the allocs 342 c.Ui.Output(c.Colorize().Color("\n[bold]Allocations[reset]")) 343 if len(jobAllocs) > 0 { 344 allocs = make([]string, len(jobAllocs)+1) 345 allocs[0] = "ID|Eval ID|Node ID|Task Group|Desired|Status|Created At" 346 for i, alloc := range jobAllocs { 347 allocs[i+1] = fmt.Sprintf("%s|%s|%s|%s|%s|%s|%s", 348 limit(alloc.ID, c.length), 349 limit(alloc.EvalID, c.length), 350 limit(alloc.NodeID, c.length), 351 alloc.TaskGroup, 352 alloc.DesiredStatus, 353 alloc.ClientStatus, 354 formatUnixNanoTime(alloc.CreateTime)) 355 } 356 357 c.Ui.Output(formatList(allocs)) 358 } else { 359 c.Ui.Output("No allocations placed") 360 } 361 return nil 362 } 363 364 // outputJobSummary displays the given jobs summary and children job summary 365 // where appropriate 366 func (c *StatusCommand) outputJobSummary(client *api.Client, job *api.Job) error { 367 // Query the summary 368 summary, _, err := client.Jobs().Summary(*job.ID, nil) 369 if err != nil { 370 return fmt.Errorf("Error querying job summary: %s", err) 371 } 372 373 if summary == nil { 374 return nil 375 } 376 377 periodic := job.IsPeriodic() 378 parameterizedJob := job.IsParameterized() 379 380 // Print the summary 381 if !periodic && !parameterizedJob { 382 c.Ui.Output(c.Colorize().Color("\n[bold]Summary[reset]")) 383 summaries := make([]string, len(summary.Summary)+1) 384 summaries[0] = "Task Group|Queued|Starting|Running|Failed|Complete|Lost" 385 taskGroups := make([]string, 0, len(summary.Summary)) 386 for taskGroup := range summary.Summary { 387 taskGroups = append(taskGroups, taskGroup) 388 } 389 sort.Strings(taskGroups) 390 for idx, taskGroup := range taskGroups { 391 tgs := summary.Summary[taskGroup] 392 summaries[idx+1] = fmt.Sprintf("%s|%d|%d|%d|%d|%d|%d", 393 taskGroup, tgs.Queued, tgs.Starting, 394 tgs.Running, tgs.Failed, 395 tgs.Complete, tgs.Lost, 396 ) 397 } 398 c.Ui.Output(formatList(summaries)) 399 } 400 401 // Always display the summary if we are periodic or parameterized, but 402 // only display if the summary is non-zero on normal jobs 403 if summary.Children != nil && (parameterizedJob || periodic || summary.Children.Sum() > 0) { 404 if parameterizedJob { 405 c.Ui.Output(c.Colorize().Color("\n[bold]Parameterized Job Summary[reset]")) 406 } else { 407 c.Ui.Output(c.Colorize().Color("\n[bold]Children Job Summary[reset]")) 408 } 409 summaries := make([]string, 2) 410 summaries[0] = "Pending|Running|Dead" 411 summaries[1] = fmt.Sprintf("%d|%d|%d", 412 summary.Children.Pending, summary.Children.Running, summary.Children.Dead) 413 c.Ui.Output(formatList(summaries)) 414 } 415 416 return nil 417 } 418 419 func (c *StatusCommand) outputFailedPlacements(failedEval *api.Evaluation) { 420 if failedEval == nil || len(failedEval.FailedTGAllocs) == 0 { 421 return 422 } 423 424 c.Ui.Output(c.Colorize().Color("\n[bold]Placement Failure[reset]")) 425 426 sorted := sortedTaskGroupFromMetrics(failedEval.FailedTGAllocs) 427 for i, tg := range sorted { 428 if i >= maxFailedTGs { 429 break 430 } 431 432 c.Ui.Output(fmt.Sprintf("Task Group %q:", tg)) 433 metrics := failedEval.FailedTGAllocs[tg] 434 c.Ui.Output(formatAllocMetrics(metrics, false, " ")) 435 if i != len(sorted)-1 { 436 c.Ui.Output("") 437 } 438 } 439 440 if len(sorted) > maxFailedTGs { 441 trunc := fmt.Sprintf("\nPlacement failures truncated. To see remainder run:\nnomad eval-status %s", failedEval.ID) 442 c.Ui.Output(trunc) 443 } 444 } 445 446 // list general information about a list of jobs 447 func createStatusListOutput(jobs []*api.JobListStub) string { 448 out := make([]string, len(jobs)+1) 449 out[0] = "ID|Type|Priority|Status" 450 for i, job := range jobs { 451 out[i+1] = fmt.Sprintf("%s|%s|%d|%s", 452 job.ID, 453 getTypeString(job), 454 job.Priority, 455 getStatusString(job.Status, job.Stop)) 456 } 457 return formatList(out) 458 } 459 460 func getTypeString(job *api.JobListStub) string { 461 t := job.Type 462 463 if job.Periodic { 464 t += "/periodic" 465 } 466 467 if job.ParameterizedJob { 468 t += "/parameterized" 469 } 470 471 return t 472 } 473 474 func getStatusString(status string, stop bool) string { 475 if stop { 476 return fmt.Sprintf("%s (stopped)", status) 477 } 478 return status 479 }