github.com/hooklift/nomad@v0.5.7-0.20170407200202-db11e7dd7b55/command/status.go (about) 1 package command 2 3 import ( 4 "fmt" 5 "sort" 6 "strings" 7 "time" 8 9 "github.com/hashicorp/nomad/api" 10 "github.com/hashicorp/nomad/nomad/structs" 11 ) 12 13 const ( 14 // maxFailedTGs is the maximum number of task groups we show failure reasons 15 // for before defering to eval-status 16 maxFailedTGs = 5 17 ) 18 19 type StatusCommand struct { 20 Meta 21 length int 22 evals bool 23 allAllocs bool 24 verbose bool 25 } 26 27 func (c *StatusCommand) Help() string { 28 helpText := ` 29 Usage: nomad status [options] <job> 30 31 Display status information about jobs. If no job ID is given, 32 a list of all known jobs will be dumped. 33 34 General Options: 35 36 ` + generalOptionsUsage() + ` 37 38 Status Options: 39 40 -short 41 Display short output. Used only when a single job is being 42 queried, and drops verbose information about allocations. 43 44 -evals 45 Display the evaluations associated with the job. 46 47 -all-allocs 48 Display all allocations matching the job ID, including those from an older 49 instance of the job. 50 51 -verbose 52 Display full information. 53 ` 54 return strings.TrimSpace(helpText) 55 } 56 57 func (c *StatusCommand) Synopsis() string { 58 return "Display status information about jobs" 59 } 60 61 func (c *StatusCommand) Run(args []string) int { 62 var short bool 63 64 flags := c.Meta.FlagSet("status", FlagSetClient) 65 flags.Usage = func() { c.Ui.Output(c.Help()) } 66 flags.BoolVar(&short, "short", false, "") 67 flags.BoolVar(&c.evals, "evals", false, "") 68 flags.BoolVar(&c.allAllocs, "all-allocs", false, "") 69 flags.BoolVar(&c.verbose, "verbose", false, "") 70 71 if err := flags.Parse(args); err != nil { 72 return 1 73 } 74 75 // Check that we either got no jobs or exactly one. 76 args = flags.Args() 77 if len(args) > 1 { 78 c.Ui.Error(c.Help()) 79 return 1 80 } 81 82 // Truncate the id unless full length is requested 83 c.length = shortId 84 if c.verbose { 85 c.length = fullId 86 } 87 88 // Get the HTTP client 89 client, err := c.Meta.Client() 90 if err != nil { 91 c.Ui.Error(fmt.Sprintf("Error initializing client: %s", err)) 92 return 1 93 } 94 95 // Invoke list mode if no job ID. 96 if len(args) == 0 { 97 jobs, _, err := client.Jobs().List(nil) 98 if err != nil { 99 c.Ui.Error(fmt.Sprintf("Error querying jobs: %s", err)) 100 return 1 101 } 102 103 if len(jobs) == 0 { 104 // No output if we have no jobs 105 c.Ui.Output("No running jobs") 106 } else { 107 c.Ui.Output(createStatusListOutput(jobs)) 108 } 109 return 0 110 } 111 112 // Try querying the job 113 jobID := args[0] 114 jobs, _, err := client.Jobs().PrefixList(jobID) 115 if err != nil { 116 c.Ui.Error(fmt.Sprintf("Error querying job: %s", err)) 117 return 1 118 } 119 if len(jobs) == 0 { 120 c.Ui.Error(fmt.Sprintf("No job(s) with prefix or id %q found", jobID)) 121 return 1 122 } 123 if len(jobs) > 1 && strings.TrimSpace(jobID) != jobs[0].ID { 124 c.Ui.Output(fmt.Sprintf("Prefix matched multiple jobs\n\n%s", createStatusListOutput(jobs))) 125 return 0 126 } 127 // Prefix lookup matched a single job 128 job, _, err := client.Jobs().Info(jobs[0].ID, nil) 129 if err != nil { 130 c.Ui.Error(fmt.Sprintf("Error querying job: %s", err)) 131 return 1 132 } 133 134 periodic := job.IsPeriodic() 135 parameterized := job.IsParameterized() 136 137 // Format the job info 138 basic := []string{ 139 fmt.Sprintf("ID|%s", *job.ID), 140 fmt.Sprintf("Name|%s", *job.Name), 141 fmt.Sprintf("Type|%s", *job.Type), 142 fmt.Sprintf("Priority|%d", *job.Priority), 143 fmt.Sprintf("Datacenters|%s", strings.Join(job.Datacenters, ",")), 144 fmt.Sprintf("Status|%s", *job.Status), 145 fmt.Sprintf("Periodic|%v", periodic), 146 fmt.Sprintf("Parameterized|%v", parameterized), 147 } 148 149 if periodic && !parameterized { 150 location, err := job.Periodic.GetLocation() 151 if err == nil { 152 now := time.Now().In(location) 153 next := job.Periodic.Next(now) 154 basic = append(basic, fmt.Sprintf("Next Periodic Launch|%s", 155 fmt.Sprintf("%s (%s from now)", 156 formatTime(next), formatTimeDifference(now, next, time.Second)))) 157 } 158 } 159 160 c.Ui.Output(formatKV(basic)) 161 162 // Exit early 163 if short { 164 return 0 165 } 166 167 // Print periodic job information 168 if periodic && !parameterized { 169 if err := c.outputPeriodicInfo(client, job); err != nil { 170 c.Ui.Error(err.Error()) 171 return 1 172 } 173 } else if parameterized { 174 if err := c.outputParameterizedInfo(client, job); err != nil { 175 c.Ui.Error(err.Error()) 176 return 1 177 } 178 } else { 179 if err := c.outputJobInfo(client, job); err != nil { 180 c.Ui.Error(err.Error()) 181 return 1 182 } 183 } 184 185 return 0 186 } 187 188 // outputPeriodicInfo prints information about the passed periodic job. If a 189 // request fails, an error is returned. 190 func (c *StatusCommand) outputPeriodicInfo(client *api.Client, job *api.Job) error { 191 // Output the summary 192 if err := c.outputJobSummary(client, job); err != nil { 193 return err 194 } 195 196 // Generate the prefix that matches launched jobs from the periodic job. 197 prefix := fmt.Sprintf("%s%s", *job.ID, structs.PeriodicLaunchSuffix) 198 children, _, err := client.Jobs().PrefixList(prefix) 199 if err != nil { 200 return fmt.Errorf("Error querying job: %s", err) 201 } 202 203 if len(children) == 0 { 204 c.Ui.Output("\nNo instances of periodic job found") 205 return nil 206 } 207 208 out := make([]string, 1) 209 out[0] = "ID|Status" 210 for _, child := range children { 211 // Ensure that we are only showing jobs whose parent is the requested 212 // job. 213 if child.ParentID != *job.ID { 214 continue 215 } 216 217 out = append(out, fmt.Sprintf("%s|%s", 218 child.ID, 219 child.Status)) 220 } 221 222 c.Ui.Output(c.Colorize().Color("\n[bold]Previously Launched Jobs[reset]")) 223 c.Ui.Output(formatList(out)) 224 return nil 225 } 226 227 // outputParameterizedInfo prints information about a parameterized job. If a 228 // request fails, an error is returned. 229 func (c *StatusCommand) outputParameterizedInfo(client *api.Client, job *api.Job) error { 230 // Output parameterized job details 231 c.Ui.Output(c.Colorize().Color("\n[bold]Parameterized Job[reset]")) 232 parameterizedJob := make([]string, 3) 233 parameterizedJob[0] = fmt.Sprintf("Payload|%s", job.ParameterizedJob.Payload) 234 parameterizedJob[1] = fmt.Sprintf("Required Metadata|%v", strings.Join(job.ParameterizedJob.MetaRequired, ", ")) 235 parameterizedJob[2] = fmt.Sprintf("Optional Metadata|%v", strings.Join(job.ParameterizedJob.MetaOptional, ", ")) 236 c.Ui.Output(formatKV(parameterizedJob)) 237 238 // Output the summary 239 if err := c.outputJobSummary(client, job); err != nil { 240 return err 241 } 242 243 // Generate the prefix that matches launched jobs from the parameterized job. 244 prefix := fmt.Sprintf("%s%s", *job.ID, structs.DispatchLaunchSuffix) 245 children, _, err := client.Jobs().PrefixList(prefix) 246 if err != nil { 247 return fmt.Errorf("Error querying job: %s", err) 248 } 249 250 if len(children) == 0 { 251 c.Ui.Output("\nNo dispatched instances of parameterized job found") 252 return nil 253 } 254 255 out := make([]string, 1) 256 out[0] = "ID|Status" 257 for _, child := range children { 258 // Ensure that we are only showing jobs whose parent is the requested 259 // job. 260 if child.ParentID != *job.ID { 261 continue 262 } 263 264 out = append(out, fmt.Sprintf("%s|%s", 265 child.ID, 266 child.Status)) 267 } 268 269 c.Ui.Output(c.Colorize().Color("\n[bold]Dispatched Jobs[reset]")) 270 c.Ui.Output(formatList(out)) 271 return nil 272 } 273 274 // outputJobInfo prints information about the passed non-periodic job. If a 275 // request fails, an error is returned. 276 func (c *StatusCommand) outputJobInfo(client *api.Client, job *api.Job) error { 277 var evals, allocs []string 278 279 // Query the allocations 280 jobAllocs, _, err := client.Jobs().Allocations(*job.ID, c.allAllocs, nil) 281 if err != nil { 282 return fmt.Errorf("Error querying job allocations: %s", err) 283 } 284 285 // Query the evaluations 286 jobEvals, _, err := client.Jobs().Evaluations(*job.ID, nil) 287 if err != nil { 288 return fmt.Errorf("Error querying job evaluations: %s", err) 289 } 290 291 // Output the summary 292 if err := c.outputJobSummary(client, job); err != nil { 293 return err 294 } 295 296 // Determine latest evaluation with failures whose follow up hasn't 297 // completed, this is done while formatting 298 var latestFailedPlacement *api.Evaluation 299 blockedEval := false 300 301 // Format the evals 302 evals = make([]string, len(jobEvals)+1) 303 evals[0] = "ID|Priority|Triggered By|Status|Placement Failures" 304 for i, eval := range jobEvals { 305 failures, _ := evalFailureStatus(eval) 306 evals[i+1] = fmt.Sprintf("%s|%d|%s|%s|%s", 307 limit(eval.ID, c.length), 308 eval.Priority, 309 eval.TriggeredBy, 310 eval.Status, 311 failures, 312 ) 313 314 if eval.Status == "blocked" { 315 blockedEval = true 316 } 317 318 if len(eval.FailedTGAllocs) == 0 { 319 // Skip evals without failures 320 continue 321 } 322 323 if latestFailedPlacement == nil || latestFailedPlacement.CreateIndex < eval.CreateIndex { 324 latestFailedPlacement = eval 325 } 326 } 327 328 if c.verbose || c.evals { 329 c.Ui.Output(c.Colorize().Color("\n[bold]Evaluations[reset]")) 330 c.Ui.Output(formatList(evals)) 331 } 332 333 if blockedEval && latestFailedPlacement != nil { 334 c.outputFailedPlacements(latestFailedPlacement) 335 } 336 337 // Format the allocs 338 c.Ui.Output(c.Colorize().Color("\n[bold]Allocations[reset]")) 339 if len(jobAllocs) > 0 { 340 allocs = make([]string, len(jobAllocs)+1) 341 allocs[0] = "ID|Eval ID|Node ID|Task Group|Desired|Status|Created At" 342 for i, alloc := range jobAllocs { 343 allocs[i+1] = fmt.Sprintf("%s|%s|%s|%s|%s|%s|%s", 344 limit(alloc.ID, c.length), 345 limit(alloc.EvalID, c.length), 346 limit(alloc.NodeID, c.length), 347 alloc.TaskGroup, 348 alloc.DesiredStatus, 349 alloc.ClientStatus, 350 formatUnixNanoTime(alloc.CreateTime)) 351 } 352 353 c.Ui.Output(formatList(allocs)) 354 } else { 355 c.Ui.Output("No allocations placed") 356 } 357 return nil 358 } 359 360 // outputJobSummary displays the given jobs summary and children job summary 361 // where appropriate 362 func (c *StatusCommand) outputJobSummary(client *api.Client, job *api.Job) error { 363 // Query the summary 364 summary, _, err := client.Jobs().Summary(*job.ID, nil) 365 if err != nil { 366 return fmt.Errorf("Error querying job summary: %s", err) 367 } 368 369 if summary == nil { 370 return nil 371 } 372 373 periodic := job.IsPeriodic() 374 parameterizedJob := job.IsParameterized() 375 376 // Print the summary 377 if !periodic && !parameterizedJob { 378 c.Ui.Output(c.Colorize().Color("\n[bold]Summary[reset]")) 379 summaries := make([]string, len(summary.Summary)+1) 380 summaries[0] = "Task Group|Queued|Starting|Running|Failed|Complete|Lost" 381 taskGroups := make([]string, 0, len(summary.Summary)) 382 for taskGroup := range summary.Summary { 383 taskGroups = append(taskGroups, taskGroup) 384 } 385 sort.Strings(taskGroups) 386 for idx, taskGroup := range taskGroups { 387 tgs := summary.Summary[taskGroup] 388 summaries[idx+1] = fmt.Sprintf("%s|%d|%d|%d|%d|%d|%d", 389 taskGroup, tgs.Queued, tgs.Starting, 390 tgs.Running, tgs.Failed, 391 tgs.Complete, tgs.Lost, 392 ) 393 } 394 c.Ui.Output(formatList(summaries)) 395 } 396 397 // Always display the summary if we are periodic or parameterized, but 398 // only display if the summary is non-zero on normal jobs 399 if summary.Children != nil && (parameterizedJob || periodic || summary.Children.Sum() > 0) { 400 if parameterizedJob { 401 c.Ui.Output(c.Colorize().Color("\n[bold]Parameterized Job Summary[reset]")) 402 } else { 403 c.Ui.Output(c.Colorize().Color("\n[bold]Children Job Summary[reset]")) 404 } 405 summaries := make([]string, 2) 406 summaries[0] = "Pending|Running|Dead" 407 summaries[1] = fmt.Sprintf("%d|%d|%d", 408 summary.Children.Pending, summary.Children.Running, summary.Children.Dead) 409 c.Ui.Output(formatList(summaries)) 410 } 411 412 return nil 413 } 414 415 func (c *StatusCommand) outputFailedPlacements(failedEval *api.Evaluation) { 416 if failedEval == nil || len(failedEval.FailedTGAllocs) == 0 { 417 return 418 } 419 420 c.Ui.Output(c.Colorize().Color("\n[bold]Placement Failure[reset]")) 421 422 sorted := sortedTaskGroupFromMetrics(failedEval.FailedTGAllocs) 423 for i, tg := range sorted { 424 if i >= maxFailedTGs { 425 break 426 } 427 428 c.Ui.Output(fmt.Sprintf("Task Group %q:", tg)) 429 metrics := failedEval.FailedTGAllocs[tg] 430 c.Ui.Output(formatAllocMetrics(metrics, false, " ")) 431 if i != len(sorted)-1 { 432 c.Ui.Output("") 433 } 434 } 435 436 if len(sorted) > maxFailedTGs { 437 trunc := fmt.Sprintf("\nPlacement failures truncated. To see remainder run:\nnomad eval-status %s", failedEval.ID) 438 c.Ui.Output(trunc) 439 } 440 } 441 442 // list general information about a list of jobs 443 func createStatusListOutput(jobs []*api.JobListStub) string { 444 out := make([]string, len(jobs)+1) 445 out[0] = "ID|Type|Priority|Status" 446 for i, job := range jobs { 447 out[i+1] = fmt.Sprintf("%s|%s|%d|%s", 448 job.ID, 449 job.Type, 450 job.Priority, 451 job.Status) 452 } 453 return formatList(out) 454 }