github.com/diptanu/nomad@v0.5.7-0.20170516172507-d72e86cbe3d9/command/alloc_status.go (about) 1 package command 2 3 import ( 4 "fmt" 5 "math" 6 "sort" 7 "strconv" 8 "strings" 9 "time" 10 11 "github.com/dustin/go-humanize" 12 "github.com/mitchellh/colorstring" 13 14 "github.com/hashicorp/nomad/api" 15 "github.com/hashicorp/nomad/client" 16 ) 17 18 type AllocStatusCommand struct { 19 Meta 20 color *colorstring.Colorize 21 } 22 23 func (c *AllocStatusCommand) Help() string { 24 helpText := ` 25 Usage: nomad alloc-status [options] <allocation> 26 27 Display information about existing allocations and its tasks. This command can 28 be used to inspect the current status of an allocation, including its running 29 status, metadata, and verbose failure messages reported by internal 30 subsystems. 31 32 General Options: 33 34 ` + generalOptionsUsage() + ` 35 36 Alloc Status Options: 37 38 -short 39 Display short output. Shows only the most recent task event. 40 41 -stats 42 Display detailed resource usage statistics. 43 44 -verbose 45 Show full information. 46 47 -json 48 Output the allocation in its JSON format. 49 50 -t 51 Format and display allocation using a Go template. 52 ` 53 54 return strings.TrimSpace(helpText) 55 } 56 57 func (c *AllocStatusCommand) Synopsis() string { 58 return "Display allocation status information and metadata" 59 } 60 61 func (c *AllocStatusCommand) Run(args []string) int { 62 var short, displayStats, verbose, json bool 63 var tmpl string 64 65 flags := c.Meta.FlagSet("alloc-status", FlagSetClient) 66 flags.Usage = func() { c.Ui.Output(c.Help()) } 67 flags.BoolVar(&short, "short", false, "") 68 flags.BoolVar(&verbose, "verbose", false, "") 69 flags.BoolVar(&displayStats, "stats", false, "") 70 flags.BoolVar(&json, "json", false, "") 71 flags.StringVar(&tmpl, "t", "", "") 72 73 if err := flags.Parse(args); err != nil { 74 return 1 75 } 76 77 // Check that we got exactly one allocation ID 78 args = flags.Args() 79 80 // Get the HTTP client 81 client, err := c.Meta.Client() 82 if err != nil { 83 c.Ui.Error(fmt.Sprintf("Error initializing client: %s", err)) 84 return 1 85 } 86 87 // If args not specified but output format is specified, format and output the allocations data list 88 if len(args) == 0 { 89 var format string 90 if json && len(tmpl) > 0 { 91 c.Ui.Error("Both -json and -t are not allowed") 92 return 1 93 } else if json { 94 format = "json" 95 } else if len(tmpl) > 0 { 96 format = "template" 97 } 98 if len(format) > 0 { 99 allocs, _, err := client.Allocations().List(nil) 100 if err != nil { 101 c.Ui.Error(fmt.Sprintf("Error querying allocations: %v", err)) 102 return 1 103 } 104 // Return nothing if no allocations found 105 if len(allocs) == 0 { 106 return 0 107 } 108 109 f, err := DataFormat(format, tmpl) 110 if err != nil { 111 c.Ui.Error(fmt.Sprintf("Error getting formatter: %s", err)) 112 return 1 113 } 114 115 out, err := f.TransformData(allocs) 116 if err != nil { 117 c.Ui.Error(fmt.Sprintf("Error formatting the data: %s", err)) 118 return 1 119 } 120 c.Ui.Output(out) 121 return 0 122 } 123 } 124 125 if len(args) != 1 { 126 c.Ui.Error(c.Help()) 127 return 1 128 } 129 allocID := args[0] 130 131 // Truncate the id unless full length is requested 132 length := shortId 133 if verbose { 134 length = fullId 135 } 136 137 // Query the allocation info 138 if len(allocID) == 1 { 139 c.Ui.Error(fmt.Sprintf("Identifier must contain at least two characters.")) 140 return 1 141 } 142 if len(allocID)%2 == 1 { 143 // Identifiers must be of even length, so we strip off the last byte 144 // to provide a consistent user experience. 145 allocID = allocID[:len(allocID)-1] 146 } 147 148 allocs, _, err := client.Allocations().PrefixList(allocID) 149 if err != nil { 150 c.Ui.Error(fmt.Sprintf("Error querying allocation: %v", err)) 151 return 1 152 } 153 if len(allocs) == 0 { 154 c.Ui.Error(fmt.Sprintf("No allocation(s) with prefix or id %q found", allocID)) 155 return 1 156 } 157 if len(allocs) > 1 { 158 // Format the allocs 159 out := make([]string, len(allocs)+1) 160 out[0] = "ID|Eval ID|Job ID|Task Group|Desired Status|Client Status" 161 for i, alloc := range allocs { 162 out[i+1] = fmt.Sprintf("%s|%s|%s|%s|%s|%s", 163 limit(alloc.ID, length), 164 limit(alloc.EvalID, length), 165 alloc.JobID, 166 alloc.TaskGroup, 167 alloc.DesiredStatus, 168 alloc.ClientStatus, 169 ) 170 } 171 c.Ui.Output(fmt.Sprintf("Prefix matched multiple allocations\n\n%s", formatList(out))) 172 return 0 173 } 174 // Prefix lookup matched a single allocation 175 alloc, _, err := client.Allocations().Info(allocs[0].ID, nil) 176 if err != nil { 177 c.Ui.Error(fmt.Sprintf("Error querying allocation: %s", err)) 178 return 1 179 } 180 181 // If output format is specified, format and output the data 182 var format string 183 if json && len(tmpl) > 0 { 184 c.Ui.Error("Both -json and -t are not allowed") 185 return 1 186 } else if json { 187 format = "json" 188 } else if len(tmpl) > 0 { 189 format = "template" 190 } 191 if len(format) > 0 { 192 f, err := DataFormat(format, tmpl) 193 if err != nil { 194 c.Ui.Error(fmt.Sprintf("Error getting formatter: %s", err)) 195 return 1 196 } 197 198 out, err := f.TransformData(alloc) 199 if err != nil { 200 c.Ui.Error(fmt.Sprintf("Error formatting the data: %s", err)) 201 return 1 202 } 203 c.Ui.Output(out) 204 return 0 205 } 206 207 // Format the allocation data 208 basic := []string{ 209 fmt.Sprintf("ID|%s", limit(alloc.ID, length)), 210 fmt.Sprintf("Eval ID|%s", limit(alloc.EvalID, length)), 211 fmt.Sprintf("Name|%s", alloc.Name), 212 fmt.Sprintf("Node ID|%s", limit(alloc.NodeID, length)), 213 fmt.Sprintf("Job ID|%s", alloc.JobID), 214 fmt.Sprintf("Client Status|%s", alloc.ClientStatus), 215 fmt.Sprintf("Client Description|%s", alloc.ClientDescription), 216 fmt.Sprintf("Desired Status|%s", alloc.DesiredStatus), 217 fmt.Sprintf("Desired Description|%s", alloc.DesiredDescription), 218 fmt.Sprintf("Created At|%s", formatUnixNanoTime(alloc.CreateTime)), 219 } 220 221 if verbose { 222 basic = append(basic, 223 fmt.Sprintf("Evaluated Nodes|%d", alloc.Metrics.NodesEvaluated), 224 fmt.Sprintf("Filtered Nodes|%d", alloc.Metrics.NodesFiltered), 225 fmt.Sprintf("Exhausted Nodes|%d", alloc.Metrics.NodesExhausted), 226 fmt.Sprintf("Allocation Time|%s", alloc.Metrics.AllocationTime), 227 fmt.Sprintf("Failures|%d", alloc.Metrics.CoalescedFailures)) 228 } 229 c.Ui.Output(formatKV(basic)) 230 231 if short { 232 c.shortTaskStatus(alloc) 233 } else { 234 var statsErr error 235 var stats *api.AllocResourceUsage 236 stats, statsErr = client.Allocations().Stats(alloc, nil) 237 if statsErr != nil { 238 c.Ui.Output("") 239 if statsErr != api.NodeDownErr { 240 c.Ui.Error(fmt.Sprintf("Couldn't retrieve stats (HINT: ensure Client.Advertise.HTTP is set): %v", statsErr)) 241 } else { 242 c.Ui.Output("Omitting resource statistics since the node is down.") 243 } 244 } 245 c.outputTaskDetails(alloc, stats, displayStats) 246 } 247 248 // Format the detailed status 249 if verbose { 250 c.Ui.Output(c.Colorize().Color("\n[bold]Placement Metrics[reset]")) 251 c.Ui.Output(formatAllocMetrics(alloc.Metrics, true, " ")) 252 } 253 254 return 0 255 } 256 257 // outputTaskDetails prints task details for each task in the allocation, 258 // optionally printing verbose statistics if displayStats is set 259 func (c *AllocStatusCommand) outputTaskDetails(alloc *api.Allocation, stats *api.AllocResourceUsage, displayStats bool) { 260 for task := range c.sortedTaskStateIterator(alloc.TaskStates) { 261 state := alloc.TaskStates[task] 262 c.Ui.Output(c.Colorize().Color(fmt.Sprintf("\n[bold]Task %q is %q[reset]", task, state.State))) 263 c.outputTaskResources(alloc, task, stats, displayStats) 264 c.Ui.Output("") 265 c.outputTaskStatus(state) 266 } 267 } 268 269 // outputTaskStatus prints out a list of the most recent events for the given 270 // task state. 271 func (c *AllocStatusCommand) outputTaskStatus(state *api.TaskState) { 272 c.Ui.Output("Recent Events:") 273 events := make([]string, len(state.Events)+1) 274 events[0] = "Time|Type|Description" 275 276 size := len(state.Events) 277 for i, event := range state.Events { 278 formatedTime := formatUnixNanoTime(event.Time) 279 280 // Build up the description based on the event type. 281 var desc string 282 switch event.Type { 283 case api.TaskSetup: 284 desc = event.Message 285 case api.TaskStarted: 286 desc = "Task started by client" 287 case api.TaskReceived: 288 desc = "Task received by client" 289 case api.TaskFailedValidation: 290 if event.ValidationError != "" { 291 desc = event.ValidationError 292 } else { 293 desc = "Validation of task failed" 294 } 295 case api.TaskSetupFailure: 296 if event.SetupError != "" { 297 desc = event.SetupError 298 } else { 299 desc = "Task setup failed" 300 } 301 case api.TaskDriverFailure: 302 if event.DriverError != "" { 303 desc = event.DriverError 304 } else { 305 desc = "Failed to start task" 306 } 307 case api.TaskDownloadingArtifacts: 308 desc = "Client is downloading artifacts" 309 case api.TaskArtifactDownloadFailed: 310 if event.DownloadError != "" { 311 desc = event.DownloadError 312 } else { 313 desc = "Failed to download artifacts" 314 } 315 case api.TaskKilling: 316 if event.KillReason != "" { 317 desc = fmt.Sprintf("Killing task: %v", event.KillReason) 318 } else if event.KillTimeout != 0 { 319 desc = fmt.Sprintf("Sent interrupt. Waiting %v before force killing", event.KillTimeout) 320 } else { 321 desc = "Sent interrupt" 322 } 323 case api.TaskKilled: 324 if event.KillError != "" { 325 desc = event.KillError 326 } else { 327 desc = "Task successfully killed" 328 } 329 case api.TaskTerminated: 330 var parts []string 331 parts = append(parts, fmt.Sprintf("Exit Code: %d", event.ExitCode)) 332 333 if event.Signal != 0 { 334 parts = append(parts, fmt.Sprintf("Signal: %d", event.Signal)) 335 } 336 337 if event.Message != "" { 338 parts = append(parts, fmt.Sprintf("Exit Message: %q", event.Message)) 339 } 340 desc = strings.Join(parts, ", ") 341 case api.TaskRestarting: 342 in := fmt.Sprintf("Task restarting in %v", time.Duration(event.StartDelay)) 343 if event.RestartReason != "" && event.RestartReason != client.ReasonWithinPolicy { 344 desc = fmt.Sprintf("%s - %s", event.RestartReason, in) 345 } else { 346 desc = in 347 } 348 case api.TaskNotRestarting: 349 if event.RestartReason != "" { 350 desc = event.RestartReason 351 } else { 352 desc = "Task exceeded restart policy" 353 } 354 case api.TaskSiblingFailed: 355 if event.FailedSibling != "" { 356 desc = fmt.Sprintf("Task's sibling %q failed", event.FailedSibling) 357 } else { 358 desc = "Task's sibling failed" 359 } 360 case api.TaskSignaling: 361 sig := event.TaskSignal 362 reason := event.TaskSignalReason 363 364 if sig == "" && reason == "" { 365 desc = "Task being sent a signal" 366 } else if sig == "" { 367 desc = reason 368 } else if reason == "" { 369 desc = fmt.Sprintf("Task being sent signal %v", sig) 370 } else { 371 desc = fmt.Sprintf("Task being sent signal %v: %v", sig, reason) 372 } 373 case api.TaskRestartSignal: 374 if event.RestartReason != "" { 375 desc = event.RestartReason 376 } else { 377 desc = "Task signaled to restart" 378 } 379 case api.TaskDriverMessage: 380 desc = event.DriverMessage 381 case api.TaskLeaderDead: 382 desc = "Leader Task in Group dead" 383 } 384 385 // Reverse order so we are sorted by time 386 events[size-i] = fmt.Sprintf("%s|%s|%s", formatedTime, event.Type, desc) 387 } 388 c.Ui.Output(formatList(events)) 389 } 390 391 // outputTaskResources prints the task resources for the passed task and if 392 // displayStats is set, verbose resource usage statistics 393 func (c *AllocStatusCommand) outputTaskResources(alloc *api.Allocation, task string, stats *api.AllocResourceUsage, displayStats bool) { 394 resource, ok := alloc.TaskResources[task] 395 if !ok { 396 return 397 } 398 399 c.Ui.Output("Task Resources") 400 var addr []string 401 for _, nw := range resource.Networks { 402 ports := append(nw.DynamicPorts, nw.ReservedPorts...) 403 for _, port := range ports { 404 addr = append(addr, fmt.Sprintf("%v: %v:%v\n", port.Label, nw.IP, port.Value)) 405 } 406 } 407 var resourcesOutput []string 408 resourcesOutput = append(resourcesOutput, "CPU|Memory|Disk|IOPS|Addresses") 409 firstAddr := "" 410 if len(addr) > 0 { 411 firstAddr = addr[0] 412 } 413 414 // Display the rolled up stats. If possible prefer the live statistics 415 cpuUsage := strconv.Itoa(*resource.CPU) 416 memUsage := humanize.IBytes(uint64(*resource.MemoryMB * bytesPerMegabyte)) 417 if stats != nil { 418 if ru, ok := stats.Tasks[task]; ok && ru != nil && ru.ResourceUsage != nil { 419 if cs := ru.ResourceUsage.CpuStats; cs != nil { 420 cpuUsage = fmt.Sprintf("%v/%v", math.Floor(cs.TotalTicks), cpuUsage) 421 } 422 if ms := ru.ResourceUsage.MemoryStats; ms != nil { 423 memUsage = fmt.Sprintf("%v/%v", humanize.IBytes(ms.RSS), memUsage) 424 } 425 } 426 } 427 resourcesOutput = append(resourcesOutput, fmt.Sprintf("%v MHz|%v|%v|%v|%v", 428 cpuUsage, 429 memUsage, 430 humanize.IBytes(uint64(*alloc.Resources.DiskMB*bytesPerMegabyte)), 431 *resource.IOPS, 432 firstAddr)) 433 for i := 1; i < len(addr); i++ { 434 resourcesOutput = append(resourcesOutput, fmt.Sprintf("||||%v", addr[i])) 435 } 436 c.Ui.Output(formatListWithSpaces(resourcesOutput)) 437 438 if stats != nil { 439 if ru, ok := stats.Tasks[task]; ok && ru != nil && displayStats && ru.ResourceUsage != nil { 440 c.Ui.Output("") 441 c.outputVerboseResourceUsage(task, ru.ResourceUsage) 442 } 443 } 444 } 445 446 // outputVerboseResourceUsage outputs the verbose resource usage for the passed 447 // task 448 func (c *AllocStatusCommand) outputVerboseResourceUsage(task string, resourceUsage *api.ResourceUsage) { 449 memoryStats := resourceUsage.MemoryStats 450 cpuStats := resourceUsage.CpuStats 451 if memoryStats != nil && len(memoryStats.Measured) > 0 { 452 c.Ui.Output("Memory Stats") 453 454 // Sort the measured stats 455 sort.Strings(memoryStats.Measured) 456 457 var measuredStats []string 458 for _, measured := range memoryStats.Measured { 459 switch measured { 460 case "RSS": 461 measuredStats = append(measuredStats, humanize.IBytes(memoryStats.RSS)) 462 case "Cache": 463 measuredStats = append(measuredStats, humanize.IBytes(memoryStats.Cache)) 464 case "Swap": 465 measuredStats = append(measuredStats, humanize.IBytes(memoryStats.Swap)) 466 case "Max Usage": 467 measuredStats = append(measuredStats, humanize.IBytes(memoryStats.MaxUsage)) 468 case "Kernel Usage": 469 measuredStats = append(measuredStats, humanize.IBytes(memoryStats.KernelUsage)) 470 case "Kernel Max Usage": 471 measuredStats = append(measuredStats, humanize.IBytes(memoryStats.KernelMaxUsage)) 472 } 473 } 474 475 out := make([]string, 2) 476 out[0] = strings.Join(memoryStats.Measured, "|") 477 out[1] = strings.Join(measuredStats, "|") 478 c.Ui.Output(formatList(out)) 479 c.Ui.Output("") 480 } 481 482 if cpuStats != nil && len(cpuStats.Measured) > 0 { 483 c.Ui.Output("CPU Stats") 484 485 // Sort the measured stats 486 sort.Strings(cpuStats.Measured) 487 488 var measuredStats []string 489 for _, measured := range cpuStats.Measured { 490 switch measured { 491 case "Percent": 492 percent := strconv.FormatFloat(cpuStats.Percent, 'f', 2, 64) 493 measuredStats = append(measuredStats, fmt.Sprintf("%v%%", percent)) 494 case "Throttled Periods": 495 measuredStats = append(measuredStats, fmt.Sprintf("%v", cpuStats.ThrottledPeriods)) 496 case "Throttled Time": 497 measuredStats = append(measuredStats, fmt.Sprintf("%v", cpuStats.ThrottledTime)) 498 case "User Mode": 499 percent := strconv.FormatFloat(cpuStats.UserMode, 'f', 2, 64) 500 measuredStats = append(measuredStats, fmt.Sprintf("%v%%", percent)) 501 case "System Mode": 502 percent := strconv.FormatFloat(cpuStats.SystemMode, 'f', 2, 64) 503 measuredStats = append(measuredStats, fmt.Sprintf("%v%%", percent)) 504 } 505 } 506 507 out := make([]string, 2) 508 out[0] = strings.Join(cpuStats.Measured, "|") 509 out[1] = strings.Join(measuredStats, "|") 510 c.Ui.Output(formatList(out)) 511 } 512 } 513 514 // shortTaskStatus prints out the current state of each task. 515 func (c *AllocStatusCommand) shortTaskStatus(alloc *api.Allocation) { 516 tasks := make([]string, 0, len(alloc.TaskStates)+1) 517 tasks = append(tasks, "Name|State|Last Event|Time") 518 for task := range c.sortedTaskStateIterator(alloc.TaskStates) { 519 state := alloc.TaskStates[task] 520 lastState := state.State 521 var lastEvent, lastTime string 522 523 l := len(state.Events) 524 if l != 0 { 525 last := state.Events[l-1] 526 lastEvent = last.Type 527 lastTime = formatUnixNanoTime(last.Time) 528 } 529 530 tasks = append(tasks, fmt.Sprintf("%s|%s|%s|%s", 531 task, lastState, lastEvent, lastTime)) 532 } 533 534 c.Ui.Output(c.Colorize().Color("\n[bold]Tasks[reset]")) 535 c.Ui.Output(formatList(tasks)) 536 } 537 538 // sortedTaskStateIterator is a helper that takes the task state map and returns a 539 // channel that returns the keys in a sorted order. 540 func (c *AllocStatusCommand) sortedTaskStateIterator(m map[string]*api.TaskState) <-chan string { 541 output := make(chan string, len(m)) 542 keys := make([]string, len(m)) 543 i := 0 544 for k := range m { 545 keys[i] = k 546 i++ 547 } 548 sort.Strings(keys) 549 550 for _, key := range keys { 551 output <- key 552 } 553 554 close(output) 555 return output 556 }