github.com/mattyr/nomad@v0.3.3-0.20160919021406-3485a065154a/command/alloc_status.go (about) 1 package command 2 3 import ( 4 "fmt" 5 "math" 6 "sort" 7 "strconv" 8 "strings" 9 "time" 10 11 "github.com/dustin/go-humanize" 12 "github.com/mitchellh/colorstring" 13 14 "github.com/hashicorp/nomad/api" 15 "github.com/hashicorp/nomad/client" 16 ) 17 18 type AllocStatusCommand struct { 19 Meta 20 color *colorstring.Colorize 21 } 22 23 func (c *AllocStatusCommand) Help() string { 24 helpText := ` 25 Usage: nomad alloc-status [options] <allocation> 26 27 Display information about existing allocations and its tasks. This command can 28 be used to inspect the current status of all allocation, including its running 29 status, metadata, and verbose failure messages reported by internal 30 subsystems. 31 32 General Options: 33 34 ` + generalOptionsUsage() + ` 35 36 Alloc Status Options: 37 38 -short 39 Display short output. Shows only the most recent task event. 40 41 -stats 42 Display detailed resource usage statistics. 43 44 -verbose 45 Show full information. 46 47 -json 48 Output the allocation in its JSON format. 49 50 -t 51 Format and display allocation using a Go template. 52 ` 53 54 return strings.TrimSpace(helpText) 55 } 56 57 func (c *AllocStatusCommand) Synopsis() string { 58 return "Display allocation status information and metadata" 59 } 60 61 func (c *AllocStatusCommand) Run(args []string) int { 62 var short, displayStats, verbose, json bool 63 var tmpl string 64 65 flags := c.Meta.FlagSet("alloc-status", FlagSetClient) 66 flags.Usage = func() { c.Ui.Output(c.Help()) } 67 flags.BoolVar(&short, "short", false, "") 68 flags.BoolVar(&verbose, "verbose", false, "") 69 flags.BoolVar(&displayStats, "stats", false, "") 70 flags.BoolVar(&json, "json", false, "") 71 flags.StringVar(&tmpl, "t", "", "") 72 73 if err := flags.Parse(args); err != nil { 74 return 1 75 } 76 77 // Check that we got exactly one allocation ID 78 args = flags.Args() 79 80 // Get the HTTP client 81 client, err := c.Meta.Client() 82 if err != nil { 83 c.Ui.Error(fmt.Sprintf("Error initializing client: %s", err)) 84 return 1 85 } 86 87 // If args not specified but output format is specified, format and output the allocations data list 88 if len(args) == 0 { 89 var format string 90 if json && len(tmpl) > 0 { 91 c.Ui.Error("Both -json and -t are not allowed") 92 return 1 93 } else if json { 94 format = "json" 95 } else if len(tmpl) > 0 { 96 format = "template" 97 } 98 if len(format) > 0 { 99 allocs, _, err := client.Allocations().List(nil) 100 if err != nil { 101 c.Ui.Error(fmt.Sprintf("Error querying allocations: %v", err)) 102 return 1 103 } 104 // Return nothing if no allocations found 105 if len(allocs) == 0 { 106 return 0 107 } 108 109 f, err := DataFormat(format, tmpl) 110 if err != nil { 111 c.Ui.Error(fmt.Sprintf("Error getting formatter: %s", err)) 112 return 1 113 } 114 115 out, err := f.TransformData(allocs) 116 if err != nil { 117 c.Ui.Error(fmt.Sprintf("Error formatting the data: %s", err)) 118 return 1 119 } 120 c.Ui.Output(out) 121 return 0 122 } 123 } 124 125 if len(args) != 1 { 126 c.Ui.Error(c.Help()) 127 return 1 128 } 129 allocID := args[0] 130 131 // Truncate the id unless full length is requested 132 length := shortId 133 if verbose { 134 length = fullId 135 } 136 137 // Query the allocation info 138 if len(allocID) == 1 { 139 c.Ui.Error(fmt.Sprintf("Identifier must contain at least two characters.")) 140 return 1 141 } 142 if len(allocID)%2 == 1 { 143 // Identifiers must be of even length, so we strip off the last byte 144 // to provide a consistent user experience. 145 allocID = allocID[:len(allocID)-1] 146 } 147 148 allocs, _, err := client.Allocations().PrefixList(allocID) 149 if err != nil { 150 c.Ui.Error(fmt.Sprintf("Error querying allocation: %v", err)) 151 return 1 152 } 153 if len(allocs) == 0 { 154 c.Ui.Error(fmt.Sprintf("No allocation(s) with prefix or id %q found", allocID)) 155 return 1 156 } 157 if len(allocs) > 1 { 158 // Format the allocs 159 out := make([]string, len(allocs)+1) 160 out[0] = "ID|Eval ID|Job ID|Task Group|Desired Status|Client Status" 161 for i, alloc := range allocs { 162 out[i+1] = fmt.Sprintf("%s|%s|%s|%s|%s|%s", 163 limit(alloc.ID, length), 164 limit(alloc.EvalID, length), 165 alloc.JobID, 166 alloc.TaskGroup, 167 alloc.DesiredStatus, 168 alloc.ClientStatus, 169 ) 170 } 171 c.Ui.Output(fmt.Sprintf("Prefix matched multiple allocations\n\n%s", formatList(out))) 172 return 0 173 } 174 // Prefix lookup matched a single allocation 175 alloc, _, err := client.Allocations().Info(allocs[0].ID, nil) 176 if err != nil { 177 c.Ui.Error(fmt.Sprintf("Error querying allocation: %s", err)) 178 return 1 179 } 180 181 // If output format is specified, format and output the data 182 var format string 183 if json && len(tmpl) > 0 { 184 c.Ui.Error("Both -json and -t are not allowed") 185 return 1 186 } else if json { 187 format = "json" 188 } else if len(tmpl) > 0 { 189 format = "template" 190 } 191 if len(format) > 0 { 192 f, err := DataFormat(format, tmpl) 193 if err != nil { 194 c.Ui.Error(fmt.Sprintf("Error getting formatter: %s", err)) 195 return 1 196 } 197 198 out, err := f.TransformData(alloc) 199 if err != nil { 200 c.Ui.Error(fmt.Sprintf("Error formatting the data: %s", err)) 201 return 1 202 } 203 c.Ui.Output(out) 204 return 0 205 } 206 207 // Format the allocation data 208 basic := []string{ 209 fmt.Sprintf("ID|%s", limit(alloc.ID, length)), 210 fmt.Sprintf("Eval ID|%s", limit(alloc.EvalID, length)), 211 fmt.Sprintf("Name|%s", alloc.Name), 212 fmt.Sprintf("Node ID|%s", limit(alloc.NodeID, length)), 213 fmt.Sprintf("Job ID|%s", alloc.JobID), 214 fmt.Sprintf("Client Status|%s", alloc.ClientStatus), 215 fmt.Sprintf("Created At|%s", formatUnixNanoTime(alloc.CreateTime)), 216 } 217 218 if verbose { 219 basic = append(basic, 220 fmt.Sprintf("Evaluated Nodes|%d", alloc.Metrics.NodesEvaluated), 221 fmt.Sprintf("Filtered Nodes|%d", alloc.Metrics.NodesFiltered), 222 fmt.Sprintf("Exhausted Nodes|%d", alloc.Metrics.NodesExhausted), 223 fmt.Sprintf("Allocation Time|%s", alloc.Metrics.AllocationTime), 224 fmt.Sprintf("Failures|%d", alloc.Metrics.CoalescedFailures)) 225 } 226 c.Ui.Output(formatKV(basic)) 227 228 if short { 229 c.shortTaskStatus(alloc) 230 } else { 231 var statsErr error 232 var stats *api.AllocResourceUsage 233 stats, statsErr = client.Allocations().Stats(alloc, nil) 234 if statsErr != nil { 235 c.Ui.Output("") 236 c.Ui.Error(fmt.Sprintf("couldn't retrieve stats (HINT: ensure Client.Advertise.HTTP is set): %v", statsErr)) 237 } 238 c.outputTaskDetails(alloc, stats, displayStats) 239 } 240 241 // Format the detailed status 242 if verbose { 243 c.Ui.Output(c.Colorize().Color("\n[bold]Placement Metrics[reset]")) 244 c.Ui.Output(formatAllocMetrics(alloc.Metrics, true, " ")) 245 } 246 247 return 0 248 } 249 250 // outputTaskDetails prints task details for each task in the allocation, 251 // optionally printing verbose statistics if displayStats is set 252 func (c *AllocStatusCommand) outputTaskDetails(alloc *api.Allocation, stats *api.AllocResourceUsage, displayStats bool) { 253 for task := range c.sortedTaskStateIterator(alloc.TaskStates) { 254 state := alloc.TaskStates[task] 255 c.Ui.Output(c.Colorize().Color(fmt.Sprintf("\n[bold]Task %q is %q[reset]", task, state.State))) 256 c.outputTaskResources(alloc, task, stats, displayStats) 257 c.Ui.Output("") 258 c.outputTaskStatus(state) 259 } 260 } 261 262 // outputTaskStatus prints out a list of the most recent events for the given 263 // task state. 264 func (c *AllocStatusCommand) outputTaskStatus(state *api.TaskState) { 265 c.Ui.Output("Recent Events:") 266 events := make([]string, len(state.Events)+1) 267 events[0] = "Time|Type|Description" 268 269 size := len(state.Events) 270 for i, event := range state.Events { 271 formatedTime := formatUnixNanoTime(event.Time) 272 273 // Build up the description based on the event type. 274 var desc string 275 switch event.Type { 276 case api.TaskStarted: 277 desc = "Task started by client" 278 case api.TaskReceived: 279 desc = "Task received by client" 280 case api.TaskFailedValidation: 281 if event.ValidationError != "" { 282 desc = event.ValidationError 283 } else { 284 desc = "Validation of task failed" 285 } 286 case api.TaskDriverFailure: 287 if event.DriverError != "" { 288 desc = event.DriverError 289 } else { 290 desc = "Failed to start task" 291 } 292 case api.TaskDownloadingArtifacts: 293 desc = "Client is downloading artifacts" 294 case api.TaskArtifactDownloadFailed: 295 if event.DownloadError != "" { 296 desc = event.DownloadError 297 } else { 298 desc = "Failed to download artifacts" 299 } 300 case api.TaskKilling: 301 if event.KillTimeout != 0 { 302 desc = fmt.Sprintf("Sent interrupt. Waiting %v before force killing", event.KillTimeout) 303 } else { 304 desc = "Sent interrupt" 305 } 306 case api.TaskKilled: 307 if event.KillError != "" { 308 desc = event.KillError 309 } else { 310 desc = "Task successfully killed" 311 } 312 case api.TaskTerminated: 313 var parts []string 314 parts = append(parts, fmt.Sprintf("Exit Code: %d", event.ExitCode)) 315 316 if event.Signal != 0 { 317 parts = append(parts, fmt.Sprintf("Signal: %d", event.Signal)) 318 } 319 320 if event.Message != "" { 321 parts = append(parts, fmt.Sprintf("Exit Message: %q", event.Message)) 322 } 323 desc = strings.Join(parts, ", ") 324 case api.TaskRestarting: 325 in := fmt.Sprintf("Task restarting in %v", time.Duration(event.StartDelay)) 326 if event.RestartReason != "" && event.RestartReason != client.ReasonWithinPolicy { 327 desc = fmt.Sprintf("%s - %s", event.RestartReason, in) 328 } else { 329 desc = in 330 } 331 case api.TaskNotRestarting: 332 if event.RestartReason != "" { 333 desc = event.RestartReason 334 } else { 335 desc = "Task exceeded restart policy" 336 } 337 } 338 339 // Reverse order so we are sorted by time 340 events[size-i] = fmt.Sprintf("%s|%s|%s", formatedTime, event.Type, desc) 341 } 342 c.Ui.Output(formatList(events)) 343 } 344 345 // outputTaskResources prints the task resources for the passed task and if 346 // displayStats is set, verbose resource usage statistics 347 func (c *AllocStatusCommand) outputTaskResources(alloc *api.Allocation, task string, stats *api.AllocResourceUsage, displayStats bool) { 348 resource, ok := alloc.TaskResources[task] 349 if !ok { 350 return 351 } 352 353 c.Ui.Output("Task Resources") 354 var addr []string 355 for _, nw := range resource.Networks { 356 ports := append(nw.DynamicPorts, nw.ReservedPorts...) 357 for _, port := range ports { 358 addr = append(addr, fmt.Sprintf("%v: %v:%v\n", port.Label, nw.IP, port.Value)) 359 } 360 } 361 var resourcesOutput []string 362 resourcesOutput = append(resourcesOutput, "CPU|Memory|Disk|IOPS|Addresses") 363 firstAddr := "" 364 if len(addr) > 0 { 365 firstAddr = addr[0] 366 } 367 368 // Display the rolled up stats. If possible prefer the live stastics 369 cpuUsage := strconv.Itoa(resource.CPU) 370 memUsage := humanize.IBytes(uint64(resource.MemoryMB * bytesPerMegabyte)) 371 if ru, ok := stats.Tasks[task]; ok && ru != nil && ru.ResourceUsage != nil { 372 if cs := ru.ResourceUsage.CpuStats; cs != nil { 373 cpuUsage = fmt.Sprintf("%v/%v", math.Floor(cs.TotalTicks), resource.CPU) 374 } 375 if ms := ru.ResourceUsage.MemoryStats; ms != nil { 376 memUsage = fmt.Sprintf("%v/%v", humanize.IBytes(ms.RSS), memUsage) 377 } 378 } 379 resourcesOutput = append(resourcesOutput, fmt.Sprintf("%v MHz|%v|%v|%v|%v", 380 cpuUsage, 381 memUsage, 382 humanize.IBytes(uint64(resource.DiskMB*bytesPerMegabyte)), 383 resource.IOPS, 384 firstAddr)) 385 for i := 1; i < len(addr); i++ { 386 resourcesOutput = append(resourcesOutput, fmt.Sprintf("||||%v", addr[i])) 387 } 388 c.Ui.Output(formatListWithSpaces(resourcesOutput)) 389 390 if ru, ok := stats.Tasks[task]; ok && ru != nil && displayStats && ru.ResourceUsage != nil { 391 c.Ui.Output("") 392 c.outputVerboseResourceUsage(task, ru.ResourceUsage) 393 } 394 } 395 396 // outputVerboseResourceUsage outputs the verbose resource usage for the passed 397 // task 398 func (c *AllocStatusCommand) outputVerboseResourceUsage(task string, resourceUsage *api.ResourceUsage) { 399 memoryStats := resourceUsage.MemoryStats 400 cpuStats := resourceUsage.CpuStats 401 if memoryStats != nil && len(memoryStats.Measured) > 0 { 402 c.Ui.Output("Memory Stats") 403 404 // Sort the measured stats 405 sort.Strings(memoryStats.Measured) 406 407 var measuredStats []string 408 for _, measured := range memoryStats.Measured { 409 switch measured { 410 case "RSS": 411 measuredStats = append(measuredStats, humanize.IBytes(memoryStats.RSS)) 412 case "Cache": 413 measuredStats = append(measuredStats, humanize.IBytes(memoryStats.Cache)) 414 case "Swap": 415 measuredStats = append(measuredStats, humanize.IBytes(memoryStats.Swap)) 416 case "Max Usage": 417 measuredStats = append(measuredStats, humanize.IBytes(memoryStats.MaxUsage)) 418 case "Kernel Usage": 419 measuredStats = append(measuredStats, humanize.IBytes(memoryStats.KernelUsage)) 420 case "Kernel Max Usage": 421 measuredStats = append(measuredStats, humanize.IBytes(memoryStats.KernelMaxUsage)) 422 } 423 } 424 425 out := make([]string, 2) 426 out[0] = strings.Join(memoryStats.Measured, "|") 427 out[1] = strings.Join(measuredStats, "|") 428 c.Ui.Output(formatList(out)) 429 c.Ui.Output("") 430 } 431 432 if cpuStats != nil && len(cpuStats.Measured) > 0 { 433 c.Ui.Output("CPU Stats") 434 435 // Sort the measured stats 436 sort.Strings(cpuStats.Measured) 437 438 var measuredStats []string 439 for _, measured := range cpuStats.Measured { 440 switch measured { 441 case "Percent": 442 percent := strconv.FormatFloat(cpuStats.Percent, 'f', 2, 64) 443 measuredStats = append(measuredStats, fmt.Sprintf("%v%%", percent)) 444 case "Throttled Periods": 445 measuredStats = append(measuredStats, fmt.Sprintf("%v", cpuStats.ThrottledPeriods)) 446 case "Throttled Time": 447 measuredStats = append(measuredStats, fmt.Sprintf("%v", cpuStats.ThrottledTime)) 448 case "User Mode": 449 percent := strconv.FormatFloat(cpuStats.UserMode, 'f', 2, 64) 450 measuredStats = append(measuredStats, fmt.Sprintf("%v%%", percent)) 451 case "System Mode": 452 percent := strconv.FormatFloat(cpuStats.SystemMode, 'f', 2, 64) 453 measuredStats = append(measuredStats, fmt.Sprintf("%v%%", percent)) 454 } 455 } 456 457 out := make([]string, 2) 458 out[0] = strings.Join(cpuStats.Measured, "|") 459 out[1] = strings.Join(measuredStats, "|") 460 c.Ui.Output(formatList(out)) 461 } 462 } 463 464 // shortTaskStatus prints out the current state of each task. 465 func (c *AllocStatusCommand) shortTaskStatus(alloc *api.Allocation) { 466 tasks := make([]string, 0, len(alloc.TaskStates)+1) 467 tasks = append(tasks, "Name|State|Last Event|Time") 468 for task := range c.sortedTaskStateIterator(alloc.TaskStates) { 469 state := alloc.TaskStates[task] 470 lastState := state.State 471 var lastEvent, lastTime string 472 473 l := len(state.Events) 474 if l != 0 { 475 last := state.Events[l-1] 476 lastEvent = last.Type 477 lastTime = formatUnixNanoTime(last.Time) 478 } 479 480 tasks = append(tasks, fmt.Sprintf("%s|%s|%s|%s", 481 task, lastState, lastEvent, lastTime)) 482 } 483 484 c.Ui.Output(c.Colorize().Color("\n[bold]Tasks[reset]")) 485 c.Ui.Output(formatList(tasks)) 486 } 487 488 // sortedTaskStateIterator is a helper that takes the task state map and returns a 489 // channel that returns the keys in a sorted order. 490 func (c *AllocStatusCommand) sortedTaskStateIterator(m map[string]*api.TaskState) <-chan string { 491 output := make(chan string, len(m)) 492 keys := make([]string, len(m)) 493 i := 0 494 for k := range m { 495 keys[i] = k 496 i++ 497 } 498 sort.Strings(keys) 499 500 for _, key := range keys { 501 output <- key 502 } 503 504 close(output) 505 return output 506 }