github.com/hooklift/nomad@v0.5.7-0.20170407200202-db11e7dd7b55/command/node_status.go (about) 1 package command 2 3 import ( 4 "fmt" 5 "math" 6 "sort" 7 "strings" 8 "time" 9 10 "github.com/dustin/go-humanize" 11 "github.com/mitchellh/colorstring" 12 13 "github.com/hashicorp/nomad/api" 14 "github.com/hashicorp/nomad/helper" 15 ) 16 17 const ( 18 // floatFormat is a format string for formatting floats. 19 floatFormat = "#,###.##" 20 21 // bytesPerMegabyte is the number of bytes per MB 22 bytesPerMegabyte = 1024 * 1024 23 ) 24 25 type NodeStatusCommand struct { 26 Meta 27 color *colorstring.Colorize 28 length int 29 short bool 30 verbose bool 31 list_allocs bool 32 self bool 33 stats bool 34 json bool 35 tmpl string 36 } 37 38 func (c *NodeStatusCommand) Help() string { 39 helpText := ` 40 Usage: nomad node-status [options] <node> 41 42 Display status information about a given node. The list of nodes 43 returned includes only nodes which jobs may be scheduled to, and 44 includes status and other high-level information. 45 46 If a node ID is passed, information for that specific node will be displayed, 47 including resource usage statistics. If no node ID's are passed, then a 48 short-hand list of all nodes will be displayed. The -self flag is useful to 49 quickly access the status of the local node. 50 51 General Options: 52 53 ` + generalOptionsUsage() + ` 54 55 Node Status Options: 56 57 -self 58 Query the status of the local node. 59 60 -stats 61 Display detailed resource usage statistics. 62 63 -allocs 64 Display a count of running allocations for each node. 65 66 -short 67 Display short output. Used only when a single node is being 68 queried, and drops verbose output about node allocations. 69 70 -verbose 71 Display full information. 72 73 -json 74 Output the node in its JSON format. 75 76 -t 77 Format and display node using a Go template. 78 ` 79 return strings.TrimSpace(helpText) 80 } 81 82 func (c *NodeStatusCommand) Synopsis() string { 83 return "Display status information about nodes" 84 } 85 86 func (c *NodeStatusCommand) Run(args []string) int { 87 88 flags := c.Meta.FlagSet("node-status", FlagSetClient) 89 flags.Usage = func() { c.Ui.Output(c.Help()) } 90 flags.BoolVar(&c.short, "short", false, "") 91 flags.BoolVar(&c.verbose, "verbose", false, "") 92 flags.BoolVar(&c.list_allocs, "allocs", false, "") 93 flags.BoolVar(&c.self, "self", false, "") 94 flags.BoolVar(&c.stats, "stats", false, "") 95 flags.BoolVar(&c.json, "json", false, "") 96 flags.StringVar(&c.tmpl, "t", "", "") 97 98 if err := flags.Parse(args); err != nil { 99 return 1 100 } 101 102 // Check that we got either a single node or none 103 args = flags.Args() 104 if len(args) > 1 { 105 c.Ui.Error(c.Help()) 106 return 1 107 } 108 109 // Truncate the id unless full length is requested 110 c.length = shortId 111 if c.verbose { 112 c.length = fullId 113 } 114 115 // Get the HTTP client 116 client, err := c.Meta.Client() 117 if err != nil { 118 c.Ui.Error(fmt.Sprintf("Error initializing client: %s", err)) 119 return 1 120 } 121 122 // Use list mode if no node name was provided 123 if len(args) == 0 && !c.self { 124 // If output format is specified, format and output the node data list 125 var format string 126 if c.json && len(c.tmpl) > 0 { 127 c.Ui.Error("Both -json and -t are not allowed") 128 return 1 129 } else if c.json { 130 format = "json" 131 } else if len(c.tmpl) > 0 { 132 format = "template" 133 } 134 135 // Query the node info 136 nodes, _, err := client.Nodes().List(nil) 137 if err != nil { 138 c.Ui.Error(fmt.Sprintf("Error querying node status: %s", err)) 139 return 1 140 } 141 142 // Return nothing if no nodes found 143 if len(nodes) == 0 { 144 return 0 145 } 146 147 if len(format) > 0 { 148 f, err := DataFormat(format, c.tmpl) 149 if err != nil { 150 c.Ui.Error(fmt.Sprintf("Error getting formatter: %s", err)) 151 return 1 152 } 153 154 out, err := f.TransformData(nodes) 155 if err != nil { 156 c.Ui.Error(fmt.Sprintf("Error formatting the data: %s", err)) 157 return 1 158 } 159 c.Ui.Output(out) 160 return 0 161 } 162 163 // Format the nodes list 164 out := make([]string, len(nodes)+1) 165 if c.list_allocs { 166 out[0] = "ID|DC|Name|Class|Drain|Status|Running Allocs" 167 } else { 168 out[0] = "ID|DC|Name|Class|Drain|Status" 169 } 170 171 for i, node := range nodes { 172 if c.list_allocs { 173 numAllocs, err := getRunningAllocs(client, node.ID) 174 if err != nil { 175 c.Ui.Error(fmt.Sprintf("Error querying node allocations: %s", err)) 176 return 1 177 } 178 out[i+1] = fmt.Sprintf("%s|%s|%s|%s|%v|%s|%v", 179 limit(node.ID, c.length), 180 node.Datacenter, 181 node.Name, 182 node.NodeClass, 183 node.Drain, 184 node.Status, 185 len(numAllocs)) 186 } else { 187 out[i+1] = fmt.Sprintf("%s|%s|%s|%s|%v|%s", 188 limit(node.ID, c.length), 189 node.Datacenter, 190 node.Name, 191 node.NodeClass, 192 node.Drain, 193 node.Status) 194 } 195 } 196 197 // Dump the output 198 c.Ui.Output(formatList(out)) 199 return 0 200 } 201 202 // Query the specific node 203 nodeID := "" 204 if !c.self { 205 nodeID = args[0] 206 } else { 207 var err error 208 if nodeID, err = getLocalNodeID(client); err != nil { 209 c.Ui.Error(err.Error()) 210 return 1 211 } 212 } 213 if len(nodeID) == 1 { 214 c.Ui.Error(fmt.Sprintf("Identifier must contain at least two characters.")) 215 return 1 216 } 217 if len(nodeID)%2 == 1 { 218 // Identifiers must be of even length, so we strip off the last byte 219 // to provide a consistent user experience. 220 nodeID = nodeID[:len(nodeID)-1] 221 } 222 223 nodes, _, err := client.Nodes().PrefixList(nodeID) 224 if err != nil { 225 c.Ui.Error(fmt.Sprintf("Error querying node info: %s", err)) 226 return 1 227 } 228 // Return error if no nodes are found 229 if len(nodes) == 0 { 230 c.Ui.Error(fmt.Sprintf("No node(s) with prefix %q found", nodeID)) 231 return 1 232 } 233 if len(nodes) > 1 { 234 // Format the nodes list that matches the prefix so that the user 235 // can create a more specific request 236 out := make([]string, len(nodes)+1) 237 out[0] = "ID|DC|Name|Class|Drain|Status" 238 for i, node := range nodes { 239 out[i+1] = fmt.Sprintf("%s|%s|%s|%s|%v|%s", 240 limit(node.ID, c.length), 241 node.Datacenter, 242 node.Name, 243 node.NodeClass, 244 node.Drain, 245 node.Status) 246 } 247 // Dump the output 248 c.Ui.Output(fmt.Sprintf("Prefix matched multiple nodes\n\n%s", formatList(out))) 249 return 0 250 } 251 // Prefix lookup matched a single node 252 node, _, err := client.Nodes().Info(nodes[0].ID, nil) 253 if err != nil { 254 c.Ui.Error(fmt.Sprintf("Error querying node info: %s", err)) 255 return 1 256 } 257 258 // If output format is specified, format and output the data 259 var format string 260 if c.json && len(c.tmpl) > 0 { 261 c.Ui.Error("Both -json and -t are not allowed") 262 return 1 263 } else if c.json { 264 format = "json" 265 } else if len(c.tmpl) > 0 { 266 format = "template" 267 } 268 if len(format) > 0 { 269 f, err := DataFormat(format, c.tmpl) 270 if err != nil { 271 c.Ui.Error(fmt.Sprintf("Error getting formatter: %s", err)) 272 return 1 273 } 274 275 out, err := f.TransformData(node) 276 if err != nil { 277 c.Ui.Error(fmt.Sprintf("Error formatting the data: %s", err)) 278 return 1 279 } 280 c.Ui.Output(out) 281 return 0 282 } 283 284 return c.formatNode(client, node) 285 } 286 287 func nodeDrivers(n *api.Node) []string { 288 var drivers []string 289 for k, v := range n.Attributes { 290 // driver.docker = 1 291 parts := strings.Split(k, ".") 292 if len(parts) != 2 { 293 continue 294 } else if parts[0] != "driver" { 295 continue 296 } else if v != "1" { 297 continue 298 } 299 300 drivers = append(drivers, parts[1]) 301 } 302 303 sort.Strings(drivers) 304 return drivers 305 } 306 307 func (c *NodeStatusCommand) formatNode(client *api.Client, node *api.Node) int { 308 // Format the header output 309 basic := []string{ 310 fmt.Sprintf("ID|%s", limit(node.ID, c.length)), 311 fmt.Sprintf("Name|%s", node.Name), 312 fmt.Sprintf("Class|%s", node.NodeClass), 313 fmt.Sprintf("DC|%s", node.Datacenter), 314 fmt.Sprintf("Drain|%v", node.Drain), 315 fmt.Sprintf("Status|%s", node.Status), 316 fmt.Sprintf("Drivers|%s", strings.Join(nodeDrivers(node), ",")), 317 } 318 319 if c.short { 320 c.Ui.Output(c.Colorize().Color(formatKV(basic))) 321 } else { 322 // Get the host stats 323 hostStats, nodeStatsErr := client.Nodes().Stats(node.ID, nil) 324 if nodeStatsErr != nil { 325 c.Ui.Output("") 326 c.Ui.Error(fmt.Sprintf("error fetching node stats (HINT: ensure Client.Advertise.HTTP is set): %v", nodeStatsErr)) 327 } 328 if hostStats != nil { 329 uptime := time.Duration(hostStats.Uptime * uint64(time.Second)) 330 basic = append(basic, fmt.Sprintf("Uptime|%s", uptime.String())) 331 } 332 c.Ui.Output(c.Colorize().Color(formatKV(basic))) 333 334 // Get list of running allocations on the node 335 runningAllocs, err := getRunningAllocs(client, node.ID) 336 if err != nil { 337 c.Ui.Error(fmt.Sprintf("Error querying node for running allocations: %s", err)) 338 return 1 339 } 340 341 allocatedResources := getAllocatedResources(client, runningAllocs, node) 342 c.Ui.Output(c.Colorize().Color("\n[bold]Allocated Resources[reset]")) 343 c.Ui.Output(formatList(allocatedResources)) 344 345 actualResources, err := getActualResources(client, runningAllocs, node) 346 if err == nil { 347 c.Ui.Output(c.Colorize().Color("\n[bold]Allocation Resource Utilization[reset]")) 348 c.Ui.Output(formatList(actualResources)) 349 } 350 351 hostResources, err := getHostResources(hostStats, node) 352 if err != nil { 353 c.Ui.Output("") 354 c.Ui.Error(fmt.Sprintf("error fetching node stats (HINT: ensure Client.Advertise.HTTP is set): %v", err)) 355 } 356 if err == nil { 357 c.Ui.Output(c.Colorize().Color("\n[bold]Host Resource Utilization[reset]")) 358 c.Ui.Output(formatList(hostResources)) 359 } 360 361 if hostStats != nil && c.stats { 362 c.Ui.Output(c.Colorize().Color("\n[bold]CPU Stats[reset]")) 363 c.printCpuStats(hostStats) 364 c.Ui.Output(c.Colorize().Color("\n[bold]Memory Stats[reset]")) 365 c.printMemoryStats(hostStats) 366 c.Ui.Output(c.Colorize().Color("\n[bold]Disk Stats[reset]")) 367 c.printDiskStats(hostStats) 368 } 369 } 370 371 allocs, err := getAllocs(client, node, c.length) 372 if err != nil { 373 c.Ui.Error(fmt.Sprintf("Error querying node allocations: %s", err)) 374 return 1 375 } 376 377 if len(allocs) > 1 { 378 c.Ui.Output(c.Colorize().Color("\n[bold]Allocations[reset]")) 379 c.Ui.Output(formatList(allocs)) 380 } 381 382 if c.verbose { 383 c.formatAttributes(node) 384 c.formatMeta(node) 385 } 386 return 0 387 388 } 389 390 func (c *NodeStatusCommand) formatAttributes(node *api.Node) { 391 // Print the attributes 392 keys := make([]string, len(node.Attributes)) 393 for k := range node.Attributes { 394 keys = append(keys, k) 395 } 396 sort.Strings(keys) 397 398 var attributes []string 399 for _, k := range keys { 400 if k != "" { 401 attributes = append(attributes, fmt.Sprintf("%s|%s", k, node.Attributes[k])) 402 } 403 } 404 c.Ui.Output(c.Colorize().Color("\n[bold]Attributes[reset]")) 405 c.Ui.Output(formatKV(attributes)) 406 } 407 408 func (c *NodeStatusCommand) formatMeta(node *api.Node) { 409 // Print the meta 410 keys := make([]string, 0, len(node.Meta)) 411 for k := range node.Meta { 412 keys = append(keys, k) 413 } 414 sort.Strings(keys) 415 416 var meta []string 417 for _, k := range keys { 418 if k != "" { 419 meta = append(meta, fmt.Sprintf("%s|%s", k, node.Meta[k])) 420 } 421 } 422 c.Ui.Output(c.Colorize().Color("\n[bold]Meta[reset]")) 423 c.Ui.Output(formatKV(meta)) 424 } 425 426 func (c *NodeStatusCommand) printCpuStats(hostStats *api.HostStats) { 427 l := len(hostStats.CPU) 428 for i, cpuStat := range hostStats.CPU { 429 cpuStatsAttr := make([]string, 4) 430 cpuStatsAttr[0] = fmt.Sprintf("CPU|%v", cpuStat.CPU) 431 cpuStatsAttr[1] = fmt.Sprintf("User|%v%%", humanize.FormatFloat(floatFormat, cpuStat.User)) 432 cpuStatsAttr[2] = fmt.Sprintf("System|%v%%", humanize.FormatFloat(floatFormat, cpuStat.System)) 433 cpuStatsAttr[3] = fmt.Sprintf("Idle|%v%%", humanize.FormatFloat(floatFormat, cpuStat.Idle)) 434 c.Ui.Output(formatKV(cpuStatsAttr)) 435 if i+1 < l { 436 c.Ui.Output("") 437 } 438 } 439 } 440 441 func (c *NodeStatusCommand) printMemoryStats(hostStats *api.HostStats) { 442 memoryStat := hostStats.Memory 443 memStatsAttr := make([]string, 4) 444 memStatsAttr[0] = fmt.Sprintf("Total|%v", humanize.IBytes(memoryStat.Total)) 445 memStatsAttr[1] = fmt.Sprintf("Available|%v", humanize.IBytes(memoryStat.Available)) 446 memStatsAttr[2] = fmt.Sprintf("Used|%v", humanize.IBytes(memoryStat.Used)) 447 memStatsAttr[3] = fmt.Sprintf("Free|%v", humanize.IBytes(memoryStat.Free)) 448 c.Ui.Output(formatKV(memStatsAttr)) 449 } 450 451 func (c *NodeStatusCommand) printDiskStats(hostStats *api.HostStats) { 452 l := len(hostStats.DiskStats) 453 for i, diskStat := range hostStats.DiskStats { 454 diskStatsAttr := make([]string, 7) 455 diskStatsAttr[0] = fmt.Sprintf("Device|%s", diskStat.Device) 456 diskStatsAttr[1] = fmt.Sprintf("MountPoint|%s", diskStat.Mountpoint) 457 diskStatsAttr[2] = fmt.Sprintf("Size|%s", humanize.IBytes(diskStat.Size)) 458 diskStatsAttr[3] = fmt.Sprintf("Used|%s", humanize.IBytes(diskStat.Used)) 459 diskStatsAttr[4] = fmt.Sprintf("Available|%s", humanize.IBytes(diskStat.Available)) 460 diskStatsAttr[5] = fmt.Sprintf("Used Percent|%v%%", humanize.FormatFloat(floatFormat, diskStat.UsedPercent)) 461 diskStatsAttr[6] = fmt.Sprintf("Inodes Percent|%v%%", humanize.FormatFloat(floatFormat, diskStat.InodesUsedPercent)) 462 c.Ui.Output(formatKV(diskStatsAttr)) 463 if i+1 < l { 464 c.Ui.Output("") 465 } 466 } 467 } 468 469 // getRunningAllocs returns a slice of allocation id's running on the node 470 func getRunningAllocs(client *api.Client, nodeID string) ([]*api.Allocation, error) { 471 var allocs []*api.Allocation 472 473 // Query the node allocations 474 nodeAllocs, _, err := client.Nodes().Allocations(nodeID, nil) 475 // Filter list to only running allocations 476 for _, alloc := range nodeAllocs { 477 if alloc.ClientStatus == "running" { 478 allocs = append(allocs, alloc) 479 } 480 } 481 return allocs, err 482 } 483 484 // getAllocs returns information about every running allocation on the node 485 func getAllocs(client *api.Client, node *api.Node, length int) ([]string, error) { 486 var allocs []string 487 // Query the node allocations 488 nodeAllocs, _, err := client.Nodes().Allocations(node.ID, nil) 489 // Format the allocations 490 allocs = make([]string, len(nodeAllocs)+1) 491 allocs[0] = "ID|Eval ID|Job ID|Task Group|Desired Status|Client Status" 492 for i, alloc := range nodeAllocs { 493 allocs[i+1] = fmt.Sprintf("%s|%s|%s|%s|%s|%s", 494 limit(alloc.ID, length), 495 limit(alloc.EvalID, length), 496 alloc.JobID, 497 alloc.TaskGroup, 498 alloc.DesiredStatus, 499 alloc.ClientStatus) 500 } 501 return allocs, err 502 } 503 504 // getAllocatedResources returns the resource usage of the node. 505 func getAllocatedResources(client *api.Client, runningAllocs []*api.Allocation, node *api.Node) []string { 506 // Compute the total 507 total := computeNodeTotalResources(node) 508 509 // Get Resources 510 var cpu, mem, disk, iops int 511 for _, alloc := range runningAllocs { 512 cpu += *alloc.Resources.CPU 513 mem += *alloc.Resources.MemoryMB 514 disk += *alloc.Resources.DiskMB 515 iops += *alloc.Resources.IOPS 516 } 517 518 resources := make([]string, 2) 519 resources[0] = "CPU|Memory|Disk|IOPS" 520 resources[1] = fmt.Sprintf("%d/%d MHz|%s/%s|%s/%s|%d/%d", 521 cpu, 522 *total.CPU, 523 humanize.IBytes(uint64(mem*bytesPerMegabyte)), 524 humanize.IBytes(uint64(*total.MemoryMB*bytesPerMegabyte)), 525 humanize.IBytes(uint64(disk*bytesPerMegabyte)), 526 humanize.IBytes(uint64(*total.DiskMB*bytesPerMegabyte)), 527 iops, 528 *total.IOPS) 529 530 return resources 531 } 532 533 // computeNodeTotalResources returns the total allocatable resources (resources 534 // minus reserved) 535 func computeNodeTotalResources(node *api.Node) api.Resources { 536 total := api.Resources{} 537 538 r := node.Resources 539 res := node.Reserved 540 if res == nil { 541 res = &api.Resources{} 542 } 543 total.CPU = helper.IntToPtr(*r.CPU - *res.CPU) 544 total.MemoryMB = helper.IntToPtr(*r.MemoryMB - *res.MemoryMB) 545 total.DiskMB = helper.IntToPtr(*r.DiskMB - *res.DiskMB) 546 total.IOPS = helper.IntToPtr(*r.IOPS - *res.IOPS) 547 return total 548 } 549 550 // getActualResources returns the actual resource usage of the allocations. 551 func getActualResources(client *api.Client, runningAllocs []*api.Allocation, node *api.Node) ([]string, error) { 552 // Compute the total 553 total := computeNodeTotalResources(node) 554 555 // Get Resources 556 var cpu float64 557 var mem uint64 558 for _, alloc := range runningAllocs { 559 // Make the call to the client to get the actual usage. 560 stats, err := client.Allocations().Stats(alloc, nil) 561 if err != nil { 562 return nil, err 563 } 564 565 cpu += stats.ResourceUsage.CpuStats.TotalTicks 566 mem += stats.ResourceUsage.MemoryStats.RSS 567 } 568 569 resources := make([]string, 2) 570 resources[0] = "CPU|Memory" 571 resources[1] = fmt.Sprintf("%v/%d MHz|%v/%v", 572 math.Floor(cpu), 573 *total.CPU, 574 humanize.IBytes(mem), 575 humanize.IBytes(uint64(*total.MemoryMB*bytesPerMegabyte))) 576 577 return resources, nil 578 } 579 580 // getHostResources returns the actual resource usage of the node. 581 func getHostResources(hostStats *api.HostStats, node *api.Node) ([]string, error) { 582 if hostStats == nil { 583 return nil, fmt.Errorf("actual resource usage not present") 584 } 585 var resources []string 586 587 // calculate disk usage 588 storageDevice := node.Attributes["unique.storage.volume"] 589 var diskUsed, diskSize uint64 590 var physical bool 591 for _, disk := range hostStats.DiskStats { 592 if disk.Device == storageDevice { 593 diskUsed = disk.Used 594 diskSize = disk.Size 595 physical = true 596 } 597 } 598 599 resources = make([]string, 2) 600 resources[0] = "CPU|Memory|Disk" 601 if physical { 602 resources[1] = fmt.Sprintf("%v/%d MHz|%s/%s|%s/%s", 603 math.Floor(hostStats.CPUTicksConsumed), 604 *node.Resources.CPU, 605 humanize.IBytes(hostStats.Memory.Used), 606 humanize.IBytes(hostStats.Memory.Total), 607 humanize.IBytes(diskUsed), 608 humanize.IBytes(diskSize), 609 ) 610 } else { 611 // If non-physical device are used, output device name only, 612 // since nomad doesn't collect the stats data. 613 resources[1] = fmt.Sprintf("%v/%d MHz|%s/%s|(%s)", 614 math.Floor(hostStats.CPUTicksConsumed), 615 *node.Resources.CPU, 616 humanize.IBytes(hostStats.Memory.Used), 617 humanize.IBytes(hostStats.Memory.Total), 618 storageDevice, 619 ) 620 } 621 return resources, nil 622 }