github.com/blixtra/nomad@v0.7.2-0.20171221000451-da9a1d7bb050/command/alloc_status.go (about) 1 package command 2 3 import ( 4 "fmt" 5 "math" 6 "sort" 7 "strconv" 8 "strings" 9 "time" 10 11 humanize "github.com/dustin/go-humanize" 12 13 "github.com/hashicorp/nomad/api" 14 "github.com/hashicorp/nomad/api/contexts" 15 "github.com/hashicorp/nomad/client" 16 "github.com/posener/complete" 17 ) 18 19 type AllocStatusCommand struct { 20 Meta 21 } 22 23 func (c *AllocStatusCommand) Help() string { 24 helpText := ` 25 Usage: nomad alloc-status [options] <allocation> 26 27 Display information about existing allocations and its tasks. This command can 28 be used to inspect the current status of an allocation, including its running 29 status, metadata, and verbose failure messages reported by internal 30 subsystems. 31 32 General Options: 33 34 ` + generalOptionsUsage() + ` 35 36 Alloc Status Options: 37 38 -short 39 Display short output. Shows only the most recent task event. 40 41 -stats 42 Display detailed resource usage statistics. 43 44 -verbose 45 Show full information. 46 47 -json 48 Output the allocation in its JSON format. 49 50 -t 51 Format and display allocation using a Go template. 52 ` 53 54 return strings.TrimSpace(helpText) 55 } 56 57 func (c *AllocStatusCommand) Synopsis() string { 58 return "Display allocation status information and metadata" 59 } 60 61 func (c *AllocStatusCommand) AutocompleteFlags() complete.Flags { 62 return mergeAutocompleteFlags(c.Meta.AutocompleteFlags(FlagSetClient), 63 complete.Flags{ 64 "-short": complete.PredictNothing, 65 "-verbose": complete.PredictNothing, 66 "-json": complete.PredictNothing, 67 "-t": complete.PredictAnything, 68 }) 69 } 70 71 func (c *AllocStatusCommand) AutocompleteArgs() complete.Predictor { 72 return complete.PredictFunc(func(a complete.Args) []string { 73 client, err := c.Meta.Client() 74 if err != nil { 75 return nil 76 } 77 78 resp, _, err := client.Search().PrefixSearch(a.Last, contexts.Allocs, nil) 79 if err != nil { 80 return []string{} 81 } 82 return resp.Matches[contexts.Allocs] 83 }) 84 } 85 86 func (c *AllocStatusCommand) Run(args []string) int { 87 var short, displayStats, verbose, json bool 88 var tmpl string 89 90 flags := c.Meta.FlagSet("alloc-status", FlagSetClient) 91 flags.Usage = func() { c.Ui.Output(c.Help()) } 92 flags.BoolVar(&short, "short", false, "") 93 flags.BoolVar(&verbose, "verbose", false, "") 94 flags.BoolVar(&displayStats, "stats", false, "") 95 flags.BoolVar(&json, "json", false, "") 96 flags.StringVar(&tmpl, "t", "", "") 97 98 if err := flags.Parse(args); err != nil { 99 return 1 100 } 101 102 // Check that we got exactly one allocation ID 103 args = flags.Args() 104 105 // Get the HTTP client 106 client, err := c.Meta.Client() 107 if err != nil { 108 c.Ui.Error(fmt.Sprintf("Error initializing client: %s", err)) 109 return 1 110 } 111 112 // If args not specified but output format is specified, format and output the allocations data list 113 if len(args) == 0 && json || len(tmpl) > 0 { 114 allocs, _, err := client.Allocations().List(nil) 115 if err != nil { 116 c.Ui.Error(fmt.Sprintf("Error querying allocations: %v", err)) 117 return 1 118 } 119 120 out, err := Format(json, tmpl, allocs) 121 if err != nil { 122 c.Ui.Error(err.Error()) 123 return 1 124 } 125 126 c.Ui.Output(out) 127 return 0 128 } 129 130 if len(args) != 1 { 131 c.Ui.Error(c.Help()) 132 return 1 133 } 134 allocID := args[0] 135 136 // Truncate the id unless full length is requested 137 length := shortId 138 if verbose { 139 length = fullId 140 } 141 142 // Query the allocation info 143 if len(allocID) == 1 { 144 c.Ui.Error(fmt.Sprintf("Identifier must contain at least two characters.")) 145 return 1 146 } 147 148 allocID = sanatizeUUIDPrefix(allocID) 149 allocs, _, err := client.Allocations().PrefixList(allocID) 150 if err != nil { 151 c.Ui.Error(fmt.Sprintf("Error querying allocation: %v", err)) 152 return 1 153 } 154 if len(allocs) == 0 { 155 c.Ui.Error(fmt.Sprintf("No allocation(s) with prefix or id %q found", allocID)) 156 return 1 157 } 158 if len(allocs) > 1 { 159 out := formatAllocListStubs(allocs, verbose, length) 160 c.Ui.Output(fmt.Sprintf("Prefix matched multiple allocations\n\n%s", out)) 161 return 0 162 } 163 // Prefix lookup matched a single allocation 164 alloc, _, err := client.Allocations().Info(allocs[0].ID, nil) 165 if err != nil { 166 c.Ui.Error(fmt.Sprintf("Error querying allocation: %s", err)) 167 return 1 168 } 169 170 // If output format is specified, format and output the data 171 if json || len(tmpl) > 0 { 172 out, err := Format(json, tmpl, alloc) 173 if err != nil { 174 c.Ui.Error(err.Error()) 175 return 1 176 } 177 178 c.Ui.Output(out) 179 return 0 180 } 181 182 // Format the allocation data 183 output, err := formatAllocBasicInfo(alloc, client, length, verbose) 184 if err != nil { 185 c.Ui.Error(err.Error()) 186 return 1 187 } 188 c.Ui.Output(output) 189 190 if short { 191 c.shortTaskStatus(alloc) 192 } else { 193 var statsErr error 194 var stats *api.AllocResourceUsage 195 stats, statsErr = client.Allocations().Stats(alloc, nil) 196 if statsErr != nil { 197 c.Ui.Output("") 198 if statsErr != api.NodeDownErr { 199 c.Ui.Error(fmt.Sprintf("Couldn't retrieve stats (HINT: ensure Client.Advertise.HTTP is set): %v", statsErr)) 200 } else { 201 c.Ui.Output("Omitting resource statistics since the node is down.") 202 } 203 } 204 c.outputTaskDetails(alloc, stats, displayStats) 205 } 206 207 // Format the detailed status 208 if verbose { 209 c.Ui.Output(c.Colorize().Color("\n[bold]Placement Metrics[reset]")) 210 c.Ui.Output(formatAllocMetrics(alloc.Metrics, true, " ")) 211 } 212 213 return 0 214 } 215 216 func formatAllocBasicInfo(alloc *api.Allocation, client *api.Client, uuidLength int, verbose bool) (string, error) { 217 var formattedCreateTime, formattedModifyTime string 218 219 if verbose { 220 formattedCreateTime = formatUnixNanoTime(alloc.CreateTime) 221 formattedModifyTime = formatUnixNanoTime(alloc.ModifyTime) 222 } else { 223 formattedCreateTime = prettyTimeDiff(time.Unix(0, alloc.CreateTime), time.Now()) 224 formattedModifyTime = prettyTimeDiff(time.Unix(0, alloc.ModifyTime), time.Now()) 225 } 226 227 basic := []string{ 228 fmt.Sprintf("ID|%s", limit(alloc.ID, uuidLength)), 229 fmt.Sprintf("Eval ID|%s", limit(alloc.EvalID, uuidLength)), 230 fmt.Sprintf("Name|%s", alloc.Name), 231 fmt.Sprintf("Node ID|%s", limit(alloc.NodeID, uuidLength)), 232 fmt.Sprintf("Job ID|%s", alloc.JobID), 233 fmt.Sprintf("Job Version|%d", getVersion(alloc.Job)), 234 fmt.Sprintf("Client Status|%s", alloc.ClientStatus), 235 fmt.Sprintf("Client Description|%s", alloc.ClientDescription), 236 fmt.Sprintf("Desired Status|%s", alloc.DesiredStatus), 237 fmt.Sprintf("Desired Description|%s", alloc.DesiredDescription), 238 fmt.Sprintf("Created|%s", formattedCreateTime), 239 fmt.Sprintf("Modified|%s", formattedModifyTime), 240 } 241 242 if alloc.DeploymentID != "" { 243 health := "unset" 244 if alloc.DeploymentStatus != nil && alloc.DeploymentStatus.Healthy != nil { 245 if *alloc.DeploymentStatus.Healthy { 246 health = "healthy" 247 } else { 248 health = "unhealthy" 249 } 250 } 251 252 basic = append(basic, 253 fmt.Sprintf("Deployment ID|%s", limit(alloc.DeploymentID, uuidLength)), 254 fmt.Sprintf("Deployment Health|%s", health)) 255 256 // Check if this allocation is a canary 257 deployment, _, err := client.Deployments().Info(alloc.DeploymentID, nil) 258 if err != nil { 259 return "", fmt.Errorf("Error querying deployment %q: %s", alloc.DeploymentID, err) 260 } 261 262 canary := false 263 if state, ok := deployment.TaskGroups[alloc.TaskGroup]; ok { 264 for _, id := range state.PlacedCanaries { 265 if id == alloc.ID { 266 canary = true 267 break 268 } 269 } 270 } 271 272 if canary { 273 basic = append(basic, fmt.Sprintf("Canary|%v", true)) 274 } 275 } 276 277 if verbose { 278 basic = append(basic, 279 fmt.Sprintf("Evaluated Nodes|%d", alloc.Metrics.NodesEvaluated), 280 fmt.Sprintf("Filtered Nodes|%d", alloc.Metrics.NodesFiltered), 281 fmt.Sprintf("Exhausted Nodes|%d", alloc.Metrics.NodesExhausted), 282 fmt.Sprintf("Allocation Time|%s", alloc.Metrics.AllocationTime), 283 fmt.Sprintf("Failures|%d", alloc.Metrics.CoalescedFailures)) 284 } 285 286 return formatKV(basic), nil 287 } 288 289 // outputTaskDetails prints task details for each task in the allocation, 290 // optionally printing verbose statistics if displayStats is set 291 func (c *AllocStatusCommand) outputTaskDetails(alloc *api.Allocation, stats *api.AllocResourceUsage, displayStats bool) { 292 for task := range c.sortedTaskStateIterator(alloc.TaskStates) { 293 state := alloc.TaskStates[task] 294 c.Ui.Output(c.Colorize().Color(fmt.Sprintf("\n[bold]Task %q is %q[reset]", task, state.State))) 295 c.outputTaskResources(alloc, task, stats, displayStats) 296 c.Ui.Output("") 297 c.outputTaskStatus(state) 298 } 299 } 300 301 func formatTaskTimes(t time.Time) string { 302 if t.IsZero() { 303 return "N/A" 304 } 305 306 return formatTime(t) 307 } 308 309 // outputTaskStatus prints out a list of the most recent events for the given 310 // task state. 311 func (c *AllocStatusCommand) outputTaskStatus(state *api.TaskState) { 312 basic := []string{ 313 fmt.Sprintf("Started At|%s", formatTaskTimes(state.StartedAt)), 314 fmt.Sprintf("Finished At|%s", formatTaskTimes(state.FinishedAt)), 315 fmt.Sprintf("Total Restarts|%d", state.Restarts), 316 fmt.Sprintf("Last Restart|%s", formatTaskTimes(state.LastRestart))} 317 318 c.Ui.Output("Task Events:") 319 c.Ui.Output(formatKV(basic)) 320 c.Ui.Output("") 321 322 c.Ui.Output("Recent Events:") 323 events := make([]string, len(state.Events)+1) 324 events[0] = "Time|Type|Description" 325 326 size := len(state.Events) 327 for i, event := range state.Events { 328 msg := event.DisplayMessage 329 if msg == "" { 330 msg = buildDisplayMessage(event) 331 } 332 formattedTime := formatUnixNanoTime(event.Time) 333 events[size-i] = fmt.Sprintf("%s|%s|%s", formattedTime, event.Type, msg) 334 // Reverse order so we are sorted by time 335 } 336 c.Ui.Output(formatList(events)) 337 } 338 339 func buildDisplayMessage(event *api.TaskEvent) string { 340 // Build up the description based on the event type. 341 var desc string 342 switch event.Type { 343 case api.TaskSetup: 344 desc = event.Message 345 case api.TaskStarted: 346 desc = "Task started by client" 347 case api.TaskReceived: 348 desc = "Task received by client" 349 case api.TaskFailedValidation: 350 if event.ValidationError != "" { 351 desc = event.ValidationError 352 } else { 353 desc = "Validation of task failed" 354 } 355 case api.TaskSetupFailure: 356 if event.SetupError != "" { 357 desc = event.SetupError 358 } else { 359 desc = "Task setup failed" 360 } 361 case api.TaskDriverFailure: 362 if event.DriverError != "" { 363 desc = event.DriverError 364 } else { 365 desc = "Failed to start task" 366 } 367 case api.TaskDownloadingArtifacts: 368 desc = "Client is downloading artifacts" 369 case api.TaskArtifactDownloadFailed: 370 if event.DownloadError != "" { 371 desc = event.DownloadError 372 } else { 373 desc = "Failed to download artifacts" 374 } 375 case api.TaskKilling: 376 if event.KillReason != "" { 377 desc = fmt.Sprintf("Killing task: %v", event.KillReason) 378 } else if event.KillTimeout != 0 { 379 desc = fmt.Sprintf("Sent interrupt. Waiting %v before force killing", event.KillTimeout) 380 } else { 381 desc = "Sent interrupt" 382 } 383 case api.TaskKilled: 384 if event.KillError != "" { 385 desc = event.KillError 386 } else { 387 desc = "Task successfully killed" 388 } 389 case api.TaskTerminated: 390 var parts []string 391 parts = append(parts, fmt.Sprintf("Exit Code: %d", event.ExitCode)) 392 393 if event.Signal != 0 { 394 parts = append(parts, fmt.Sprintf("Signal: %d", event.Signal)) 395 } 396 397 if event.Message != "" { 398 parts = append(parts, fmt.Sprintf("Exit Message: %q", event.Message)) 399 } 400 desc = strings.Join(parts, ", ") 401 case api.TaskRestarting: 402 in := fmt.Sprintf("Task restarting in %v", time.Duration(event.StartDelay)) 403 if event.RestartReason != "" && event.RestartReason != client.ReasonWithinPolicy { 404 desc = fmt.Sprintf("%s - %s", event.RestartReason, in) 405 } else { 406 desc = in 407 } 408 case api.TaskNotRestarting: 409 if event.RestartReason != "" { 410 desc = event.RestartReason 411 } else { 412 desc = "Task exceeded restart policy" 413 } 414 case api.TaskSiblingFailed: 415 if event.FailedSibling != "" { 416 desc = fmt.Sprintf("Task's sibling %q failed", event.FailedSibling) 417 } else { 418 desc = "Task's sibling failed" 419 } 420 case api.TaskSignaling: 421 sig := event.TaskSignal 422 reason := event.TaskSignalReason 423 424 if sig == "" && reason == "" { 425 desc = "Task being sent a signal" 426 } else if sig == "" { 427 desc = reason 428 } else if reason == "" { 429 desc = fmt.Sprintf("Task being sent signal %v", sig) 430 } else { 431 desc = fmt.Sprintf("Task being sent signal %v: %v", sig, reason) 432 } 433 case api.TaskRestartSignal: 434 if event.RestartReason != "" { 435 desc = event.RestartReason 436 } else { 437 desc = "Task signaled to restart" 438 } 439 case api.TaskDriverMessage: 440 desc = event.DriverMessage 441 case api.TaskLeaderDead: 442 desc = "Leader Task in Group dead" 443 default: 444 desc = event.Message 445 } 446 447 return desc 448 } 449 450 // outputTaskResources prints the task resources for the passed task and if 451 // displayStats is set, verbose resource usage statistics 452 func (c *AllocStatusCommand) outputTaskResources(alloc *api.Allocation, task string, stats *api.AllocResourceUsage, displayStats bool) { 453 resource, ok := alloc.TaskResources[task] 454 if !ok { 455 return 456 } 457 458 c.Ui.Output("Task Resources") 459 var addr []string 460 for _, nw := range resource.Networks { 461 ports := append(nw.DynamicPorts, nw.ReservedPorts...) 462 for _, port := range ports { 463 addr = append(addr, fmt.Sprintf("%v: %v:%v\n", port.Label, nw.IP, port.Value)) 464 } 465 } 466 var resourcesOutput []string 467 resourcesOutput = append(resourcesOutput, "CPU|Memory|Disk|IOPS|Addresses") 468 firstAddr := "" 469 if len(addr) > 0 { 470 firstAddr = addr[0] 471 } 472 473 // Display the rolled up stats. If possible prefer the live statistics 474 cpuUsage := strconv.Itoa(*resource.CPU) 475 memUsage := humanize.IBytes(uint64(*resource.MemoryMB * bytesPerMegabyte)) 476 if stats != nil { 477 if ru, ok := stats.Tasks[task]; ok && ru != nil && ru.ResourceUsage != nil { 478 if cs := ru.ResourceUsage.CpuStats; cs != nil { 479 cpuUsage = fmt.Sprintf("%v/%v", math.Floor(cs.TotalTicks), cpuUsage) 480 } 481 if ms := ru.ResourceUsage.MemoryStats; ms != nil { 482 memUsage = fmt.Sprintf("%v/%v", humanize.IBytes(ms.RSS), memUsage) 483 } 484 } 485 } 486 resourcesOutput = append(resourcesOutput, fmt.Sprintf("%v MHz|%v|%v|%v|%v", 487 cpuUsage, 488 memUsage, 489 humanize.IBytes(uint64(*alloc.Resources.DiskMB*bytesPerMegabyte)), 490 *resource.IOPS, 491 firstAddr)) 492 for i := 1; i < len(addr); i++ { 493 resourcesOutput = append(resourcesOutput, fmt.Sprintf("||||%v", addr[i])) 494 } 495 c.Ui.Output(formatListWithSpaces(resourcesOutput)) 496 497 if stats != nil { 498 if ru, ok := stats.Tasks[task]; ok && ru != nil && displayStats && ru.ResourceUsage != nil { 499 c.Ui.Output("") 500 c.outputVerboseResourceUsage(task, ru.ResourceUsage) 501 } 502 } 503 } 504 505 // outputVerboseResourceUsage outputs the verbose resource usage for the passed 506 // task 507 func (c *AllocStatusCommand) outputVerboseResourceUsage(task string, resourceUsage *api.ResourceUsage) { 508 memoryStats := resourceUsage.MemoryStats 509 cpuStats := resourceUsage.CpuStats 510 if memoryStats != nil && len(memoryStats.Measured) > 0 { 511 c.Ui.Output("Memory Stats") 512 513 // Sort the measured stats 514 sort.Strings(memoryStats.Measured) 515 516 var measuredStats []string 517 for _, measured := range memoryStats.Measured { 518 switch measured { 519 case "RSS": 520 measuredStats = append(measuredStats, humanize.IBytes(memoryStats.RSS)) 521 case "Cache": 522 measuredStats = append(measuredStats, humanize.IBytes(memoryStats.Cache)) 523 case "Swap": 524 measuredStats = append(measuredStats, humanize.IBytes(memoryStats.Swap)) 525 case "Max Usage": 526 measuredStats = append(measuredStats, humanize.IBytes(memoryStats.MaxUsage)) 527 case "Kernel Usage": 528 measuredStats = append(measuredStats, humanize.IBytes(memoryStats.KernelUsage)) 529 case "Kernel Max Usage": 530 measuredStats = append(measuredStats, humanize.IBytes(memoryStats.KernelMaxUsage)) 531 } 532 } 533 534 out := make([]string, 2) 535 out[0] = strings.Join(memoryStats.Measured, "|") 536 out[1] = strings.Join(measuredStats, "|") 537 c.Ui.Output(formatList(out)) 538 c.Ui.Output("") 539 } 540 541 if cpuStats != nil && len(cpuStats.Measured) > 0 { 542 c.Ui.Output("CPU Stats") 543 544 // Sort the measured stats 545 sort.Strings(cpuStats.Measured) 546 547 var measuredStats []string 548 for _, measured := range cpuStats.Measured { 549 switch measured { 550 case "Percent": 551 percent := strconv.FormatFloat(cpuStats.Percent, 'f', 2, 64) 552 measuredStats = append(measuredStats, fmt.Sprintf("%v%%", percent)) 553 case "Throttled Periods": 554 measuredStats = append(measuredStats, fmt.Sprintf("%v", cpuStats.ThrottledPeriods)) 555 case "Throttled Time": 556 measuredStats = append(measuredStats, fmt.Sprintf("%v", cpuStats.ThrottledTime)) 557 case "User Mode": 558 percent := strconv.FormatFloat(cpuStats.UserMode, 'f', 2, 64) 559 measuredStats = append(measuredStats, fmt.Sprintf("%v%%", percent)) 560 case "System Mode": 561 percent := strconv.FormatFloat(cpuStats.SystemMode, 'f', 2, 64) 562 measuredStats = append(measuredStats, fmt.Sprintf("%v%%", percent)) 563 } 564 } 565 566 out := make([]string, 2) 567 out[0] = strings.Join(cpuStats.Measured, "|") 568 out[1] = strings.Join(measuredStats, "|") 569 c.Ui.Output(formatList(out)) 570 } 571 } 572 573 // shortTaskStatus prints out the current state of each task. 574 func (c *AllocStatusCommand) shortTaskStatus(alloc *api.Allocation) { 575 tasks := make([]string, 0, len(alloc.TaskStates)+1) 576 tasks = append(tasks, "Name|State|Last Event|Time") 577 for task := range c.sortedTaskStateIterator(alloc.TaskStates) { 578 state := alloc.TaskStates[task] 579 lastState := state.State 580 var lastEvent, lastTime string 581 582 l := len(state.Events) 583 if l != 0 { 584 last := state.Events[l-1] 585 lastEvent = last.Type 586 lastTime = formatUnixNanoTime(last.Time) 587 } 588 589 tasks = append(tasks, fmt.Sprintf("%s|%s|%s|%s", 590 task, lastState, lastEvent, lastTime)) 591 } 592 593 c.Ui.Output(c.Colorize().Color("\n[bold]Tasks[reset]")) 594 c.Ui.Output(formatList(tasks)) 595 } 596 597 // sortedTaskStateIterator is a helper that takes the task state map and returns a 598 // channel that returns the keys in a sorted order. 599 func (c *AllocStatusCommand) sortedTaskStateIterator(m map[string]*api.TaskState) <-chan string { 600 output := make(chan string, len(m)) 601 keys := make([]string, len(m)) 602 i := 0 603 for k := range m { 604 keys[i] = k 605 i++ 606 } 607 sort.Strings(keys) 608 609 for _, key := range keys { 610 output <- key 611 } 612 613 close(output) 614 return output 615 }