github.com/mattyr/nomad@v0.3.3-0.20160919021406-3485a065154a/command/node_status.go (about) 1 package command 2 3 import ( 4 "fmt" 5 "math" 6 "sort" 7 "strings" 8 "time" 9 10 "github.com/dustin/go-humanize" 11 "github.com/mitchellh/colorstring" 12 13 "github.com/hashicorp/nomad/api" 14 ) 15 16 const ( 17 // floatFormat is a format string for formatting floats. 18 floatFormat = "#,###.##" 19 20 // bytesPerMegabyte is the number of bytes per MB 21 bytesPerMegabyte = 1024 * 1024 22 ) 23 24 type NodeStatusCommand struct { 25 Meta 26 color *colorstring.Colorize 27 length int 28 short bool 29 verbose bool 30 list_allocs bool 31 self bool 32 stats bool 33 json bool 34 tmpl string 35 } 36 37 func (c *NodeStatusCommand) Help() string { 38 helpText := ` 39 Usage: nomad node-status [options] <node> 40 41 Display status information about a given node. The list of nodes 42 returned includes only nodes which jobs may be scheduled to, and 43 includes status and other high-level information. 44 45 If a node ID is passed, information for that specific node will be displayed, 46 including resource usage statistics. If no node ID's are passed, then a 47 short-hand list of all nodes will be displayed. The -self flag is useful to 48 quickly access the status of the local node. 49 50 General Options: 51 52 ` + generalOptionsUsage() + ` 53 54 Node Status Options: 55 56 -self 57 Query the status of the local node. 58 59 -stats 60 Display detailed resource usage statistics. 61 62 -allocs 63 Display a count of running allocations for each node. 64 65 -short 66 Display short output. Used only when a single node is being 67 queried, and drops verbose output about node allocations. 68 69 -verbose 70 Display full information. 71 72 -json 73 Output the node in its JSON format. 74 75 -t 76 Format and display node using a Go template. 77 ` 78 return strings.TrimSpace(helpText) 79 } 80 81 func (c *NodeStatusCommand) Synopsis() string { 82 return "Display status information about nodes" 83 } 84 85 func (c *NodeStatusCommand) Run(args []string) int { 86 87 flags := c.Meta.FlagSet("node-status", FlagSetClient) 88 flags.Usage = func() { c.Ui.Output(c.Help()) } 89 flags.BoolVar(&c.short, "short", false, "") 90 flags.BoolVar(&c.verbose, "verbose", false, "") 91 flags.BoolVar(&c.list_allocs, "allocs", false, "") 92 flags.BoolVar(&c.self, "self", false, "") 93 flags.BoolVar(&c.stats, "stats", false, "") 94 flags.BoolVar(&c.json, "json", false, "") 95 flags.StringVar(&c.tmpl, "t", "", "") 96 97 if err := flags.Parse(args); err != nil { 98 return 1 99 } 100 101 // Check that we got either a single node or none 102 args = flags.Args() 103 if len(args) > 1 { 104 c.Ui.Error(c.Help()) 105 return 1 106 } 107 108 // Truncate the id unless full length is requested 109 c.length = shortId 110 if c.verbose { 111 c.length = fullId 112 } 113 114 // Get the HTTP client 115 client, err := c.Meta.Client() 116 if err != nil { 117 c.Ui.Error(fmt.Sprintf("Error initializing client: %s", err)) 118 return 1 119 } 120 121 // Use list mode if no node name was provided 122 if len(args) == 0 && !c.self { 123 // If output format is specified, format and output the node data list 124 var format string 125 if c.json && len(c.tmpl) > 0 { 126 c.Ui.Error("Both -json and -t are not allowed") 127 return 1 128 } else if c.json { 129 format = "json" 130 } else if len(c.tmpl) > 0 { 131 format = "template" 132 } 133 134 // Query the node info 135 nodes, _, err := client.Nodes().List(nil) 136 if err != nil { 137 c.Ui.Error(fmt.Sprintf("Error querying node status: %s", err)) 138 return 1 139 } 140 141 // Return nothing if no nodes found 142 if len(nodes) == 0 { 143 return 0 144 } 145 146 if len(format) > 0 { 147 f, err := DataFormat(format, c.tmpl) 148 if err != nil { 149 c.Ui.Error(fmt.Sprintf("Error getting formatter: %s", err)) 150 return 1 151 } 152 153 out, err := f.TransformData(nodes) 154 if err != nil { 155 c.Ui.Error(fmt.Sprintf("Error formatting the data: %s", err)) 156 return 1 157 } 158 c.Ui.Output(out) 159 return 0 160 } 161 162 // Format the nodes list 163 out := make([]string, len(nodes)+1) 164 if c.list_allocs { 165 out[0] = "ID|DC|Name|Class|Drain|Status|Running Allocs" 166 } else { 167 out[0] = "ID|DC|Name|Class|Drain|Status" 168 } 169 170 for i, node := range nodes { 171 if c.list_allocs { 172 numAllocs, err := getRunningAllocs(client, node.ID) 173 if err != nil { 174 c.Ui.Error(fmt.Sprintf("Error querying node allocations: %s", err)) 175 return 1 176 } 177 out[i+1] = fmt.Sprintf("%s|%s|%s|%s|%v|%s|%v", 178 limit(node.ID, c.length), 179 node.Datacenter, 180 node.Name, 181 node.NodeClass, 182 node.Drain, 183 node.Status, 184 len(numAllocs)) 185 } else { 186 out[i+1] = fmt.Sprintf("%s|%s|%s|%s|%v|%s", 187 limit(node.ID, c.length), 188 node.Datacenter, 189 node.Name, 190 node.NodeClass, 191 node.Drain, 192 node.Status) 193 } 194 } 195 196 // Dump the output 197 c.Ui.Output(formatList(out)) 198 return 0 199 } 200 201 // Query the specific node 202 nodeID := "" 203 if !c.self { 204 nodeID = args[0] 205 } else { 206 var err error 207 if nodeID, err = getLocalNodeID(client); err != nil { 208 c.Ui.Error(err.Error()) 209 return 1 210 } 211 } 212 if len(nodeID) == 1 { 213 c.Ui.Error(fmt.Sprintf("Identifier must contain at least two characters.")) 214 return 1 215 } 216 if len(nodeID)%2 == 1 { 217 // Identifiers must be of even length, so we strip off the last byte 218 // to provide a consistent user experience. 219 nodeID = nodeID[:len(nodeID)-1] 220 } 221 222 nodes, _, err := client.Nodes().PrefixList(nodeID) 223 if err != nil { 224 c.Ui.Error(fmt.Sprintf("Error querying node info: %s", err)) 225 return 1 226 } 227 // Return error if no nodes are found 228 if len(nodes) == 0 { 229 c.Ui.Error(fmt.Sprintf("No node(s) with prefix %q found", nodeID)) 230 return 1 231 } 232 if len(nodes) > 1 { 233 // Format the nodes list that matches the prefix so that the user 234 // can create a more specific request 235 out := make([]string, len(nodes)+1) 236 out[0] = "ID|DC|Name|Class|Drain|Status" 237 for i, node := range nodes { 238 out[i+1] = fmt.Sprintf("%s|%s|%s|%s|%v|%s", 239 limit(node.ID, c.length), 240 node.Datacenter, 241 node.Name, 242 node.NodeClass, 243 node.Drain, 244 node.Status) 245 } 246 // Dump the output 247 c.Ui.Output(fmt.Sprintf("Prefix matched multiple nodes\n\n%s", formatList(out))) 248 return 0 249 } 250 // Prefix lookup matched a single node 251 node, _, err := client.Nodes().Info(nodes[0].ID, nil) 252 if err != nil { 253 c.Ui.Error(fmt.Sprintf("Error querying node info: %s", err)) 254 return 1 255 } 256 257 // If output format is specified, format and output the data 258 var format string 259 if c.json && len(c.tmpl) > 0 { 260 c.Ui.Error("Both -json and -t are not allowed") 261 return 1 262 } else if c.json { 263 format = "json" 264 } else if len(c.tmpl) > 0 { 265 format = "template" 266 } 267 if len(format) > 0 { 268 f, err := DataFormat(format, c.tmpl) 269 if err != nil { 270 c.Ui.Error(fmt.Sprintf("Error getting formatter: %s", err)) 271 return 1 272 } 273 274 out, err := f.TransformData(node) 275 if err != nil { 276 c.Ui.Error(fmt.Sprintf("Error formatting the data: %s", err)) 277 return 1 278 } 279 c.Ui.Output(out) 280 return 0 281 } 282 283 return c.formatNode(client, node) 284 } 285 286 func (c *NodeStatusCommand) formatNode(client *api.Client, node *api.Node) int { 287 // Format the header output 288 basic := []string{ 289 fmt.Sprintf("ID|%s", limit(node.ID, c.length)), 290 fmt.Sprintf("Name|%s", node.Name), 291 fmt.Sprintf("Class|%s", node.NodeClass), 292 fmt.Sprintf("DC|%s", node.Datacenter), 293 fmt.Sprintf("Drain|%v", node.Drain), 294 fmt.Sprintf("Status|%s", node.Status), 295 } 296 297 if c.short { 298 c.Ui.Output(c.Colorize().Color(formatKV(basic))) 299 } else { 300 // Get the host stats 301 hostStats, nodeStatsErr := client.Nodes().Stats(node.ID, nil) 302 if nodeStatsErr != nil { 303 c.Ui.Output("") 304 c.Ui.Error(fmt.Sprintf("error fetching node stats (HINT: ensure Client.Advertise.HTTP is set): %v", nodeStatsErr)) 305 } 306 if hostStats != nil { 307 uptime := time.Duration(hostStats.Uptime * uint64(time.Second)) 308 basic = append(basic, fmt.Sprintf("Uptime|%s", uptime.String())) 309 } 310 c.Ui.Output(c.Colorize().Color(formatKV(basic))) 311 312 // Get list of running allocations on the node 313 runningAllocs, err := getRunningAllocs(client, node.ID) 314 if err != nil { 315 c.Ui.Error(fmt.Sprintf("Error querying node for running allocations: %s", err)) 316 return 1 317 } 318 319 allocatedResources := getAllocatedResources(client, runningAllocs, node) 320 c.Ui.Output(c.Colorize().Color("\n[bold]Allocated Resources[reset]")) 321 c.Ui.Output(formatList(allocatedResources)) 322 323 actualResources, err := getActualResources(client, runningAllocs, node) 324 if err == nil { 325 c.Ui.Output(c.Colorize().Color("\n[bold]Allocation Resource Utilization[reset]")) 326 c.Ui.Output(formatList(actualResources)) 327 } 328 329 hostResources, err := getHostResources(hostStats, node) 330 if err != nil { 331 c.Ui.Output("") 332 c.Ui.Error(fmt.Sprintf("error fetching node stats (HINT: ensure Client.Advertise.HTTP is set): %v", err)) 333 } 334 if err == nil { 335 c.Ui.Output(c.Colorize().Color("\n[bold]Host Resource Utilization[reset]")) 336 c.Ui.Output(formatList(hostResources)) 337 } 338 339 if hostStats != nil && c.stats { 340 c.Ui.Output(c.Colorize().Color("\n[bold]CPU Stats[reset]")) 341 c.printCpuStats(hostStats) 342 c.Ui.Output(c.Colorize().Color("\n[bold]Memory Stats[reset]")) 343 c.printMemoryStats(hostStats) 344 c.Ui.Output(c.Colorize().Color("\n[bold]Disk Stats[reset]")) 345 c.printDiskStats(hostStats) 346 } 347 } 348 349 allocs, err := getAllocs(client, node, c.length) 350 if err != nil { 351 c.Ui.Error(fmt.Sprintf("Error querying node allocations: %s", err)) 352 return 1 353 } 354 355 if len(allocs) > 1 { 356 c.Ui.Output(c.Colorize().Color("\n[bold]Allocations[reset]")) 357 c.Ui.Output(formatList(allocs)) 358 } 359 360 if c.verbose { 361 c.formatAttributes(node) 362 } 363 return 0 364 365 } 366 367 func (c *NodeStatusCommand) formatAttributes(node *api.Node) { 368 // Print the attributes 369 keys := make([]string, len(node.Attributes)) 370 for k := range node.Attributes { 371 keys = append(keys, k) 372 } 373 sort.Strings(keys) 374 375 var attributes []string 376 for _, k := range keys { 377 if k != "" { 378 attributes = append(attributes, fmt.Sprintf("%s|%s", k, node.Attributes[k])) 379 } 380 } 381 c.Ui.Output(c.Colorize().Color("\n[bold]Attributes[reset]")) 382 c.Ui.Output(formatKV(attributes)) 383 } 384 385 func (c *NodeStatusCommand) printCpuStats(hostStats *api.HostStats) { 386 l := len(hostStats.CPU) 387 for i, cpuStat := range hostStats.CPU { 388 cpuStatsAttr := make([]string, 4) 389 cpuStatsAttr[0] = fmt.Sprintf("CPU|%v", cpuStat.CPU) 390 cpuStatsAttr[1] = fmt.Sprintf("User|%v%%", humanize.FormatFloat(floatFormat, cpuStat.User)) 391 cpuStatsAttr[2] = fmt.Sprintf("System|%v%%", humanize.FormatFloat(floatFormat, cpuStat.System)) 392 cpuStatsAttr[3] = fmt.Sprintf("Idle|%v%%", humanize.FormatFloat(floatFormat, cpuStat.Idle)) 393 c.Ui.Output(formatKV(cpuStatsAttr)) 394 if i+1 < l { 395 c.Ui.Output("") 396 } 397 } 398 } 399 400 func (c *NodeStatusCommand) printMemoryStats(hostStats *api.HostStats) { 401 memoryStat := hostStats.Memory 402 memStatsAttr := make([]string, 4) 403 memStatsAttr[0] = fmt.Sprintf("Total|%v", humanize.IBytes(memoryStat.Total)) 404 memStatsAttr[1] = fmt.Sprintf("Available|%v", humanize.IBytes(memoryStat.Available)) 405 memStatsAttr[2] = fmt.Sprintf("Used|%v", humanize.IBytes(memoryStat.Used)) 406 memStatsAttr[3] = fmt.Sprintf("Free|%v", humanize.IBytes(memoryStat.Free)) 407 c.Ui.Output(formatKV(memStatsAttr)) 408 } 409 410 func (c *NodeStatusCommand) printDiskStats(hostStats *api.HostStats) { 411 l := len(hostStats.DiskStats) 412 for i, diskStat := range hostStats.DiskStats { 413 diskStatsAttr := make([]string, 7) 414 diskStatsAttr[0] = fmt.Sprintf("Device|%s", diskStat.Device) 415 diskStatsAttr[1] = fmt.Sprintf("MountPoint|%s", diskStat.Mountpoint) 416 diskStatsAttr[2] = fmt.Sprintf("Size|%s", humanize.IBytes(diskStat.Size)) 417 diskStatsAttr[3] = fmt.Sprintf("Used|%s", humanize.IBytes(diskStat.Used)) 418 diskStatsAttr[4] = fmt.Sprintf("Available|%s", humanize.IBytes(diskStat.Available)) 419 diskStatsAttr[5] = fmt.Sprintf("Used Percent|%v%%", humanize.FormatFloat(floatFormat, diskStat.UsedPercent)) 420 diskStatsAttr[6] = fmt.Sprintf("Inodes Percent|%v%%", humanize.FormatFloat(floatFormat, diskStat.InodesUsedPercent)) 421 c.Ui.Output(formatKV(diskStatsAttr)) 422 if i+1 < l { 423 c.Ui.Output("") 424 } 425 } 426 } 427 428 // getRunningAllocs returns a slice of allocation id's running on the node 429 func getRunningAllocs(client *api.Client, nodeID string) ([]*api.Allocation, error) { 430 var allocs []*api.Allocation 431 432 // Query the node allocations 433 nodeAllocs, _, err := client.Nodes().Allocations(nodeID, nil) 434 // Filter list to only running allocations 435 for _, alloc := range nodeAllocs { 436 if alloc.ClientStatus == "running" { 437 allocs = append(allocs, alloc) 438 } 439 } 440 return allocs, err 441 } 442 443 // getAllocs returns information about every running allocation on the node 444 func getAllocs(client *api.Client, node *api.Node, length int) ([]string, error) { 445 var allocs []string 446 // Query the node allocations 447 nodeAllocs, _, err := client.Nodes().Allocations(node.ID, nil) 448 // Format the allocations 449 allocs = make([]string, len(nodeAllocs)+1) 450 allocs[0] = "ID|Eval ID|Job ID|Task Group|Desired Status|Client Status" 451 for i, alloc := range nodeAllocs { 452 allocs[i+1] = fmt.Sprintf("%s|%s|%s|%s|%s|%s", 453 limit(alloc.ID, length), 454 limit(alloc.EvalID, length), 455 alloc.JobID, 456 alloc.TaskGroup, 457 alloc.DesiredStatus, 458 alloc.ClientStatus) 459 } 460 return allocs, err 461 } 462 463 // getAllocatedResources returns the resource usage of the node. 464 func getAllocatedResources(client *api.Client, runningAllocs []*api.Allocation, node *api.Node) []string { 465 // Compute the total 466 total := computeNodeTotalResources(node) 467 468 // Get Resources 469 var cpu, mem, disk, iops int 470 for _, alloc := range runningAllocs { 471 cpu += alloc.Resources.CPU 472 mem += alloc.Resources.MemoryMB 473 disk += alloc.Resources.DiskMB 474 iops += alloc.Resources.IOPS 475 } 476 477 resources := make([]string, 2) 478 resources[0] = "CPU|Memory|Disk|IOPS" 479 resources[1] = fmt.Sprintf("%v/%v MHz|%v/%v|%v/%v|%v/%v", 480 cpu, 481 total.CPU, 482 humanize.IBytes(uint64(mem*bytesPerMegabyte)), 483 humanize.IBytes(uint64(total.MemoryMB*bytesPerMegabyte)), 484 humanize.IBytes(uint64(disk*bytesPerMegabyte)), 485 humanize.IBytes(uint64(total.DiskMB*bytesPerMegabyte)), 486 iops, 487 total.IOPS) 488 489 return resources 490 } 491 492 // computeNodeTotalResources returns the total allocatable resources (resources 493 // minus reserved) 494 func computeNodeTotalResources(node *api.Node) api.Resources { 495 total := api.Resources{} 496 497 r := node.Resources 498 res := node.Reserved 499 if res == nil { 500 res = &api.Resources{} 501 } 502 total.CPU = r.CPU - res.CPU 503 total.MemoryMB = r.MemoryMB - res.MemoryMB 504 total.DiskMB = r.DiskMB - res.DiskMB 505 total.IOPS = r.IOPS - res.IOPS 506 return total 507 } 508 509 // getActualResources returns the actual resource usage of the allocations. 510 func getActualResources(client *api.Client, runningAllocs []*api.Allocation, node *api.Node) ([]string, error) { 511 // Compute the total 512 total := computeNodeTotalResources(node) 513 514 // Get Resources 515 var cpu float64 516 var mem uint64 517 for _, alloc := range runningAllocs { 518 // Make the call to the client to get the actual usage. 519 stats, err := client.Allocations().Stats(alloc, nil) 520 if err != nil { 521 return nil, err 522 } 523 524 cpu += stats.ResourceUsage.CpuStats.TotalTicks 525 mem += stats.ResourceUsage.MemoryStats.RSS 526 } 527 528 resources := make([]string, 2) 529 resources[0] = "CPU|Memory" 530 resources[1] = fmt.Sprintf("%v/%v MHz|%v/%v", 531 math.Floor(cpu), 532 total.CPU, 533 humanize.IBytes(mem), 534 humanize.IBytes(uint64(total.MemoryMB*bytesPerMegabyte))) 535 536 return resources, nil 537 } 538 539 // getHostResources returns the actual resource usage of the node. 540 func getHostResources(hostStats *api.HostStats, node *api.Node) ([]string, error) { 541 if hostStats == nil { 542 return nil, fmt.Errorf("actual resource usage not present") 543 } 544 var resources []string 545 546 // calculate disk usage 547 storageDevice := node.Attributes["unique.storage.volume"] 548 var diskUsed, diskSize uint64 549 var physical bool 550 for _, disk := range hostStats.DiskStats { 551 if disk.Device == storageDevice { 552 diskUsed = disk.Used 553 diskSize = disk.Size 554 physical = true 555 } 556 } 557 558 resources = make([]string, 2) 559 resources[0] = "CPU|Memory|Disk" 560 if physical { 561 resources[1] = fmt.Sprintf("%v/%v MHz|%v/%v|%v/%v", 562 math.Floor(hostStats.CPUTicksConsumed), 563 node.Resources.CPU, 564 humanize.IBytes(hostStats.Memory.Used), 565 humanize.IBytes(hostStats.Memory.Total), 566 humanize.IBytes(diskUsed), 567 humanize.IBytes(diskSize), 568 ) 569 } else { 570 // If non-physical device are used, output device name only, 571 // since nomad doesn't collect the stats data. 572 resources[1] = fmt.Sprintf("%v/%v MHz|%v/%v|(%s)", 573 math.Floor(hostStats.CPUTicksConsumed), 574 node.Resources.CPU, 575 humanize.IBytes(hostStats.Memory.Used), 576 humanize.IBytes(hostStats.Memory.Total), 577 storageDevice, 578 ) 579 } 580 return resources, nil 581 }