github.com/jrxfive/nomad@v0.6.1-0.20170802162750-1fef470e89bf/command/node_status.go (about) 1 package command 2 3 import ( 4 "fmt" 5 "math" 6 "sort" 7 "strings" 8 "time" 9 10 "github.com/dustin/go-humanize" 11 "github.com/mitchellh/colorstring" 12 13 "github.com/hashicorp/nomad/api" 14 "github.com/hashicorp/nomad/helper" 15 ) 16 17 const ( 18 // floatFormat is a format string for formatting floats. 19 floatFormat = "#,###.##" 20 21 // bytesPerMegabyte is the number of bytes per MB 22 bytesPerMegabyte = 1024 * 1024 23 ) 24 25 type NodeStatusCommand struct { 26 Meta 27 color *colorstring.Colorize 28 length int 29 short bool 30 verbose bool 31 list_allocs bool 32 self bool 33 stats bool 34 json bool 35 tmpl string 36 } 37 38 func (c *NodeStatusCommand) Help() string { 39 helpText := ` 40 Usage: nomad node-status [options] <node> 41 42 Display status information about a given node. The list of nodes 43 returned includes only nodes which jobs may be scheduled to, and 44 includes status and other high-level information. 45 46 If a node ID is passed, information for that specific node will be displayed, 47 including resource usage statistics. If no node ID's are passed, then a 48 short-hand list of all nodes will be displayed. The -self flag is useful to 49 quickly access the status of the local node. 50 51 General Options: 52 53 ` + generalOptionsUsage() + ` 54 55 Node Status Options: 56 57 -self 58 Query the status of the local node. 59 60 -stats 61 Display detailed resource usage statistics. 62 63 -allocs 64 Display a count of running allocations for each node. 65 66 -short 67 Display short output. Used only when a single node is being 68 queried, and drops verbose output about node allocations. 69 70 -verbose 71 Display full information. 72 73 -json 74 Output the node in its JSON format. 75 76 -t 77 Format and display node using a Go template. 78 ` 79 return strings.TrimSpace(helpText) 80 } 81 82 func (c *NodeStatusCommand) Synopsis() string { 83 return "Display status information about nodes" 84 } 85 86 func (c *NodeStatusCommand) Run(args []string) int { 87 88 flags := c.Meta.FlagSet("node-status", FlagSetClient) 89 flags.Usage = func() { c.Ui.Output(c.Help()) } 90 flags.BoolVar(&c.short, "short", false, "") 91 flags.BoolVar(&c.verbose, "verbose", false, "") 92 flags.BoolVar(&c.list_allocs, "allocs", false, "") 93 flags.BoolVar(&c.self, "self", false, "") 94 flags.BoolVar(&c.stats, "stats", false, "") 95 flags.BoolVar(&c.json, "json", false, "") 96 flags.StringVar(&c.tmpl, "t", "", "") 97 98 if err := flags.Parse(args); err != nil { 99 return 1 100 } 101 102 // Check that we got either a single node or none 103 args = flags.Args() 104 if len(args) > 1 { 105 c.Ui.Error(c.Help()) 106 return 1 107 } 108 109 // Truncate the id unless full length is requested 110 c.length = shortId 111 if c.verbose { 112 c.length = fullId 113 } 114 115 // Get the HTTP client 116 client, err := c.Meta.Client() 117 if err != nil { 118 c.Ui.Error(fmt.Sprintf("Error initializing client: %s", err)) 119 return 1 120 } 121 122 // Use list mode if no node name was provided 123 if len(args) == 0 && !c.self { 124 125 // Query the node info 126 nodes, _, err := client.Nodes().List(nil) 127 if err != nil { 128 c.Ui.Error(fmt.Sprintf("Error querying node status: %s", err)) 129 return 1 130 } 131 132 // If output format is specified, format and output the node data list 133 if c.json || len(c.tmpl) > 0 { 134 out, err := Format(c.json, c.tmpl, nodes) 135 if err != nil { 136 c.Ui.Error(err.Error()) 137 return 1 138 } 139 140 c.Ui.Output(out) 141 return 0 142 } 143 144 // Return nothing if no nodes found 145 if len(nodes) == 0 { 146 return 0 147 } 148 149 // Format the nodes list 150 out := make([]string, len(nodes)+1) 151 if c.list_allocs { 152 out[0] = "ID|DC|Name|Class|Drain|Status|Running Allocs" 153 } else { 154 out[0] = "ID|DC|Name|Class|Drain|Status" 155 } 156 157 for i, node := range nodes { 158 if c.list_allocs { 159 numAllocs, err := getRunningAllocs(client, node.ID) 160 if err != nil { 161 c.Ui.Error(fmt.Sprintf("Error querying node allocations: %s", err)) 162 return 1 163 } 164 out[i+1] = fmt.Sprintf("%s|%s|%s|%s|%v|%s|%v", 165 limit(node.ID, c.length), 166 node.Datacenter, 167 node.Name, 168 node.NodeClass, 169 node.Drain, 170 node.Status, 171 len(numAllocs)) 172 } else { 173 out[i+1] = fmt.Sprintf("%s|%s|%s|%s|%v|%s", 174 limit(node.ID, c.length), 175 node.Datacenter, 176 node.Name, 177 node.NodeClass, 178 node.Drain, 179 node.Status) 180 } 181 } 182 183 // Dump the output 184 c.Ui.Output(formatList(out)) 185 return 0 186 } 187 188 // Query the specific node 189 nodeID := "" 190 if !c.self { 191 nodeID = args[0] 192 } else { 193 var err error 194 if nodeID, err = getLocalNodeID(client); err != nil { 195 c.Ui.Error(err.Error()) 196 return 1 197 } 198 } 199 if len(nodeID) == 1 { 200 c.Ui.Error(fmt.Sprintf("Identifier must contain at least two characters.")) 201 return 1 202 } 203 if len(nodeID)%2 == 1 { 204 // Identifiers must be of even length, so we strip off the last byte 205 // to provide a consistent user experience. 206 nodeID = nodeID[:len(nodeID)-1] 207 } 208 209 nodes, _, err := client.Nodes().PrefixList(nodeID) 210 if err != nil { 211 c.Ui.Error(fmt.Sprintf("Error querying node info: %s", err)) 212 return 1 213 } 214 // Return error if no nodes are found 215 if len(nodes) == 0 { 216 c.Ui.Error(fmt.Sprintf("No node(s) with prefix %q found", nodeID)) 217 return 1 218 } 219 if len(nodes) > 1 { 220 // Format the nodes list that matches the prefix so that the user 221 // can create a more specific request 222 out := make([]string, len(nodes)+1) 223 out[0] = "ID|DC|Name|Class|Drain|Status" 224 for i, node := range nodes { 225 out[i+1] = fmt.Sprintf("%s|%s|%s|%s|%v|%s", 226 limit(node.ID, c.length), 227 node.Datacenter, 228 node.Name, 229 node.NodeClass, 230 node.Drain, 231 node.Status) 232 } 233 // Dump the output 234 c.Ui.Error(fmt.Sprintf("Prefix matched multiple nodes\n\n%s", formatList(out))) 235 return 1 236 } 237 // Prefix lookup matched a single node 238 node, _, err := client.Nodes().Info(nodes[0].ID, nil) 239 if err != nil { 240 c.Ui.Error(fmt.Sprintf("Error querying node info: %s", err)) 241 return 1 242 } 243 244 // If output format is specified, format and output the data 245 if c.json || len(c.tmpl) > 0 { 246 out, err := Format(c.json, c.tmpl, node) 247 if err != nil { 248 c.Ui.Error(err.Error()) 249 return 1 250 } 251 252 c.Ui.Output(out) 253 return 0 254 } 255 256 return c.formatNode(client, node) 257 } 258 259 func nodeDrivers(n *api.Node) []string { 260 var drivers []string 261 for k, v := range n.Attributes { 262 // driver.docker = 1 263 parts := strings.Split(k, ".") 264 if len(parts) != 2 { 265 continue 266 } else if parts[0] != "driver" { 267 continue 268 } else if v != "1" { 269 continue 270 } 271 272 drivers = append(drivers, parts[1]) 273 } 274 275 sort.Strings(drivers) 276 return drivers 277 } 278 279 func (c *NodeStatusCommand) formatNode(client *api.Client, node *api.Node) int { 280 // Format the header output 281 basic := []string{ 282 fmt.Sprintf("ID|%s", limit(node.ID, c.length)), 283 fmt.Sprintf("Name|%s", node.Name), 284 fmt.Sprintf("Class|%s", node.NodeClass), 285 fmt.Sprintf("DC|%s", node.Datacenter), 286 fmt.Sprintf("Drain|%v", node.Drain), 287 fmt.Sprintf("Status|%s", node.Status), 288 fmt.Sprintf("Drivers|%s", strings.Join(nodeDrivers(node), ",")), 289 } 290 291 if c.short { 292 c.Ui.Output(c.Colorize().Color(formatKV(basic))) 293 } else { 294 // Get the host stats 295 hostStats, nodeStatsErr := client.Nodes().Stats(node.ID, nil) 296 if nodeStatsErr != nil { 297 c.Ui.Output("") 298 c.Ui.Error(fmt.Sprintf("error fetching node stats (HINT: ensure Client.Advertise.HTTP is set): %v", nodeStatsErr)) 299 } 300 if hostStats != nil { 301 uptime := time.Duration(hostStats.Uptime * uint64(time.Second)) 302 basic = append(basic, fmt.Sprintf("Uptime|%s", uptime.String())) 303 } 304 c.Ui.Output(c.Colorize().Color(formatKV(basic))) 305 306 // Get list of running allocations on the node 307 runningAllocs, err := getRunningAllocs(client, node.ID) 308 if err != nil { 309 c.Ui.Error(fmt.Sprintf("Error querying node for running allocations: %s", err)) 310 return 1 311 } 312 313 allocatedResources := getAllocatedResources(client, runningAllocs, node) 314 c.Ui.Output(c.Colorize().Color("\n[bold]Allocated Resources[reset]")) 315 c.Ui.Output(formatList(allocatedResources)) 316 317 actualResources, err := getActualResources(client, runningAllocs, node) 318 if err == nil { 319 c.Ui.Output(c.Colorize().Color("\n[bold]Allocation Resource Utilization[reset]")) 320 c.Ui.Output(formatList(actualResources)) 321 } 322 323 hostResources, err := getHostResources(hostStats, node) 324 if err != nil { 325 c.Ui.Output("") 326 c.Ui.Error(fmt.Sprintf("error fetching node stats (HINT: ensure Client.Advertise.HTTP is set): %v", err)) 327 } 328 if err == nil { 329 c.Ui.Output(c.Colorize().Color("\n[bold]Host Resource Utilization[reset]")) 330 c.Ui.Output(formatList(hostResources)) 331 } 332 333 if hostStats != nil && c.stats { 334 c.Ui.Output(c.Colorize().Color("\n[bold]CPU Stats[reset]")) 335 c.printCpuStats(hostStats) 336 c.Ui.Output(c.Colorize().Color("\n[bold]Memory Stats[reset]")) 337 c.printMemoryStats(hostStats) 338 c.Ui.Output(c.Colorize().Color("\n[bold]Disk Stats[reset]")) 339 c.printDiskStats(hostStats) 340 } 341 } 342 343 nodeAllocs, _, err := client.Nodes().Allocations(node.ID, nil) 344 if err != nil { 345 c.Ui.Error(fmt.Sprintf("Error querying node allocations: %s", err)) 346 return 1 347 } 348 349 c.Ui.Output(c.Colorize().Color("\n[bold]Allocations[reset]")) 350 c.Ui.Output(formatAllocList(nodeAllocs, c.verbose, c.length)) 351 352 if c.verbose { 353 c.formatAttributes(node) 354 c.formatMeta(node) 355 } 356 return 0 357 358 } 359 360 func (c *NodeStatusCommand) formatAttributes(node *api.Node) { 361 // Print the attributes 362 keys := make([]string, len(node.Attributes)) 363 for k := range node.Attributes { 364 keys = append(keys, k) 365 } 366 sort.Strings(keys) 367 368 var attributes []string 369 for _, k := range keys { 370 if k != "" { 371 attributes = append(attributes, fmt.Sprintf("%s|%s", k, node.Attributes[k])) 372 } 373 } 374 c.Ui.Output(c.Colorize().Color("\n[bold]Attributes[reset]")) 375 c.Ui.Output(formatKV(attributes)) 376 } 377 378 func (c *NodeStatusCommand) formatMeta(node *api.Node) { 379 // Print the meta 380 keys := make([]string, 0, len(node.Meta)) 381 for k := range node.Meta { 382 keys = append(keys, k) 383 } 384 sort.Strings(keys) 385 386 var meta []string 387 for _, k := range keys { 388 if k != "" { 389 meta = append(meta, fmt.Sprintf("%s|%s", k, node.Meta[k])) 390 } 391 } 392 c.Ui.Output(c.Colorize().Color("\n[bold]Meta[reset]")) 393 c.Ui.Output(formatKV(meta)) 394 } 395 396 func (c *NodeStatusCommand) printCpuStats(hostStats *api.HostStats) { 397 l := len(hostStats.CPU) 398 for i, cpuStat := range hostStats.CPU { 399 cpuStatsAttr := make([]string, 4) 400 cpuStatsAttr[0] = fmt.Sprintf("CPU|%v", cpuStat.CPU) 401 cpuStatsAttr[1] = fmt.Sprintf("User|%v%%", humanize.FormatFloat(floatFormat, cpuStat.User)) 402 cpuStatsAttr[2] = fmt.Sprintf("System|%v%%", humanize.FormatFloat(floatFormat, cpuStat.System)) 403 cpuStatsAttr[3] = fmt.Sprintf("Idle|%v%%", humanize.FormatFloat(floatFormat, cpuStat.Idle)) 404 c.Ui.Output(formatKV(cpuStatsAttr)) 405 if i+1 < l { 406 c.Ui.Output("") 407 } 408 } 409 } 410 411 func (c *NodeStatusCommand) printMemoryStats(hostStats *api.HostStats) { 412 memoryStat := hostStats.Memory 413 memStatsAttr := make([]string, 4) 414 memStatsAttr[0] = fmt.Sprintf("Total|%v", humanize.IBytes(memoryStat.Total)) 415 memStatsAttr[1] = fmt.Sprintf("Available|%v", humanize.IBytes(memoryStat.Available)) 416 memStatsAttr[2] = fmt.Sprintf("Used|%v", humanize.IBytes(memoryStat.Used)) 417 memStatsAttr[3] = fmt.Sprintf("Free|%v", humanize.IBytes(memoryStat.Free)) 418 c.Ui.Output(formatKV(memStatsAttr)) 419 } 420 421 func (c *NodeStatusCommand) printDiskStats(hostStats *api.HostStats) { 422 l := len(hostStats.DiskStats) 423 for i, diskStat := range hostStats.DiskStats { 424 diskStatsAttr := make([]string, 7) 425 diskStatsAttr[0] = fmt.Sprintf("Device|%s", diskStat.Device) 426 diskStatsAttr[1] = fmt.Sprintf("MountPoint|%s", diskStat.Mountpoint) 427 diskStatsAttr[2] = fmt.Sprintf("Size|%s", humanize.IBytes(diskStat.Size)) 428 diskStatsAttr[3] = fmt.Sprintf("Used|%s", humanize.IBytes(diskStat.Used)) 429 diskStatsAttr[4] = fmt.Sprintf("Available|%s", humanize.IBytes(diskStat.Available)) 430 diskStatsAttr[5] = fmt.Sprintf("Used Percent|%v%%", humanize.FormatFloat(floatFormat, diskStat.UsedPercent)) 431 diskStatsAttr[6] = fmt.Sprintf("Inodes Percent|%v%%", humanize.FormatFloat(floatFormat, diskStat.InodesUsedPercent)) 432 c.Ui.Output(formatKV(diskStatsAttr)) 433 if i+1 < l { 434 c.Ui.Output("") 435 } 436 } 437 } 438 439 // getRunningAllocs returns a slice of allocation id's running on the node 440 func getRunningAllocs(client *api.Client, nodeID string) ([]*api.Allocation, error) { 441 var allocs []*api.Allocation 442 443 // Query the node allocations 444 nodeAllocs, _, err := client.Nodes().Allocations(nodeID, nil) 445 // Filter list to only running allocations 446 for _, alloc := range nodeAllocs { 447 if alloc.ClientStatus == "running" { 448 allocs = append(allocs, alloc) 449 } 450 } 451 return allocs, err 452 } 453 454 // getAllocatedResources returns the resource usage of the node. 455 func getAllocatedResources(client *api.Client, runningAllocs []*api.Allocation, node *api.Node) []string { 456 // Compute the total 457 total := computeNodeTotalResources(node) 458 459 // Get Resources 460 var cpu, mem, disk, iops int 461 for _, alloc := range runningAllocs { 462 cpu += *alloc.Resources.CPU 463 mem += *alloc.Resources.MemoryMB 464 disk += *alloc.Resources.DiskMB 465 iops += *alloc.Resources.IOPS 466 } 467 468 resources := make([]string, 2) 469 resources[0] = "CPU|Memory|Disk|IOPS" 470 resources[1] = fmt.Sprintf("%d/%d MHz|%s/%s|%s/%s|%d/%d", 471 cpu, 472 *total.CPU, 473 humanize.IBytes(uint64(mem*bytesPerMegabyte)), 474 humanize.IBytes(uint64(*total.MemoryMB*bytesPerMegabyte)), 475 humanize.IBytes(uint64(disk*bytesPerMegabyte)), 476 humanize.IBytes(uint64(*total.DiskMB*bytesPerMegabyte)), 477 iops, 478 *total.IOPS) 479 480 return resources 481 } 482 483 // computeNodeTotalResources returns the total allocatable resources (resources 484 // minus reserved) 485 func computeNodeTotalResources(node *api.Node) api.Resources { 486 total := api.Resources{} 487 488 r := node.Resources 489 res := node.Reserved 490 if res == nil { 491 res = &api.Resources{} 492 } 493 total.CPU = helper.IntToPtr(*r.CPU - *res.CPU) 494 total.MemoryMB = helper.IntToPtr(*r.MemoryMB - *res.MemoryMB) 495 total.DiskMB = helper.IntToPtr(*r.DiskMB - *res.DiskMB) 496 total.IOPS = helper.IntToPtr(*r.IOPS - *res.IOPS) 497 return total 498 } 499 500 // getActualResources returns the actual resource usage of the allocations. 501 func getActualResources(client *api.Client, runningAllocs []*api.Allocation, node *api.Node) ([]string, error) { 502 // Compute the total 503 total := computeNodeTotalResources(node) 504 505 // Get Resources 506 var cpu float64 507 var mem uint64 508 for _, alloc := range runningAllocs { 509 // Make the call to the client to get the actual usage. 510 stats, err := client.Allocations().Stats(alloc, nil) 511 if err != nil { 512 return nil, err 513 } 514 515 cpu += stats.ResourceUsage.CpuStats.TotalTicks 516 mem += stats.ResourceUsage.MemoryStats.RSS 517 } 518 519 resources := make([]string, 2) 520 resources[0] = "CPU|Memory" 521 resources[1] = fmt.Sprintf("%v/%d MHz|%v/%v", 522 math.Floor(cpu), 523 *total.CPU, 524 humanize.IBytes(mem), 525 humanize.IBytes(uint64(*total.MemoryMB*bytesPerMegabyte))) 526 527 return resources, nil 528 } 529 530 // getHostResources returns the actual resource usage of the node. 531 func getHostResources(hostStats *api.HostStats, node *api.Node) ([]string, error) { 532 if hostStats == nil { 533 return nil, fmt.Errorf("actual resource usage not present") 534 } 535 var resources []string 536 537 // calculate disk usage 538 storageDevice := node.Attributes["unique.storage.volume"] 539 var diskUsed, diskSize uint64 540 var physical bool 541 for _, disk := range hostStats.DiskStats { 542 if disk.Device == storageDevice { 543 diskUsed = disk.Used 544 diskSize = disk.Size 545 physical = true 546 } 547 } 548 549 resources = make([]string, 2) 550 resources[0] = "CPU|Memory|Disk" 551 if physical { 552 resources[1] = fmt.Sprintf("%v/%d MHz|%s/%s|%s/%s", 553 math.Floor(hostStats.CPUTicksConsumed), 554 *node.Resources.CPU, 555 humanize.IBytes(hostStats.Memory.Used), 556 humanize.IBytes(hostStats.Memory.Total), 557 humanize.IBytes(diskUsed), 558 humanize.IBytes(diskSize), 559 ) 560 } else { 561 // If non-physical device are used, output device name only, 562 // since nomad doesn't collect the stats data. 563 resources[1] = fmt.Sprintf("%v/%d MHz|%s/%s|(%s)", 564 math.Floor(hostStats.CPUTicksConsumed), 565 *node.Resources.CPU, 566 humanize.IBytes(hostStats.Memory.Used), 567 humanize.IBytes(hostStats.Memory.Total), 568 storageDevice, 569 ) 570 } 571 return resources, nil 572 }