github.com/blixtra/nomad@v0.7.2-0.20171221000451-da9a1d7bb050/command/node_status.go (about) 1 package command 2 3 import ( 4 "fmt" 5 "math" 6 "sort" 7 "strings" 8 "time" 9 10 humanize "github.com/dustin/go-humanize" 11 "github.com/posener/complete" 12 13 "github.com/hashicorp/nomad/api" 14 "github.com/hashicorp/nomad/api/contexts" 15 "github.com/hashicorp/nomad/helper" 16 ) 17 18 const ( 19 // floatFormat is a format string for formatting floats. 20 floatFormat = "#,###.##" 21 22 // bytesPerMegabyte is the number of bytes per MB 23 bytesPerMegabyte = 1024 * 1024 24 ) 25 26 type NodeStatusCommand struct { 27 Meta 28 length int 29 short bool 30 verbose bool 31 list_allocs bool 32 self bool 33 stats bool 34 json bool 35 tmpl string 36 } 37 38 func (c *NodeStatusCommand) Help() string { 39 helpText := ` 40 Usage: nomad node-status [options] <node> 41 42 Display status information about a given node. The list of nodes 43 returned includes only nodes which jobs may be scheduled to, and 44 includes status and other high-level information. 45 46 If a node ID is passed, information for that specific node will be displayed, 47 including resource usage statistics. If no node ID's are passed, then a 48 short-hand list of all nodes will be displayed. The -self flag is useful to 49 quickly access the status of the local node. 50 51 General Options: 52 53 ` + generalOptionsUsage() + ` 54 55 Node Status Options: 56 57 -self 58 Query the status of the local node. 59 60 -stats 61 Display detailed resource usage statistics. 62 63 -allocs 64 Display a count of running allocations for each node. 65 66 -short 67 Display short output. Used only when a single node is being 68 queried, and drops verbose output about node allocations. 69 70 -verbose 71 Display full information. 72 73 -json 74 Output the node in its JSON format. 75 76 -t 77 Format and display node using a Go template. 78 ` 79 return strings.TrimSpace(helpText) 80 } 81 82 func (c *NodeStatusCommand) Synopsis() string { 83 return "Display status information about nodes" 84 } 85 86 func (c *NodeStatusCommand) AutocompleteFlags() complete.Flags { 87 return mergeAutocompleteFlags(c.Meta.AutocompleteFlags(FlagSetClient), 88 complete.Flags{ 89 "-allocs": complete.PredictNothing, 90 "-json": complete.PredictNothing, 91 "-self": complete.PredictNothing, 92 "-short": complete.PredictNothing, 93 "-stats": complete.PredictNothing, 94 "-t": complete.PredictAnything, 95 "-verbose": complete.PredictNothing, 96 }) 97 } 98 99 func (c *NodeStatusCommand) AutocompleteArgs() complete.Predictor { 100 return complete.PredictFunc(func(a complete.Args) []string { 101 client, err := c.Meta.Client() 102 if err != nil { 103 return nil 104 } 105 106 resp, _, err := client.Search().PrefixSearch(a.Last, contexts.Nodes, nil) 107 if err != nil { 108 return []string{} 109 } 110 return resp.Matches[contexts.Nodes] 111 }) 112 } 113 114 func (c *NodeStatusCommand) Run(args []string) int { 115 116 flags := c.Meta.FlagSet("node-status", FlagSetClient) 117 flags.Usage = func() { c.Ui.Output(c.Help()) } 118 flags.BoolVar(&c.short, "short", false, "") 119 flags.BoolVar(&c.verbose, "verbose", false, "") 120 flags.BoolVar(&c.list_allocs, "allocs", false, "") 121 flags.BoolVar(&c.self, "self", false, "") 122 flags.BoolVar(&c.stats, "stats", false, "") 123 flags.BoolVar(&c.json, "json", false, "") 124 flags.StringVar(&c.tmpl, "t", "", "") 125 126 if err := flags.Parse(args); err != nil { 127 return 1 128 } 129 130 // Check that we got either a single node or none 131 args = flags.Args() 132 if len(args) > 1 { 133 c.Ui.Error(c.Help()) 134 return 1 135 } 136 137 // Truncate the id unless full length is requested 138 c.length = shortId 139 if c.verbose { 140 c.length = fullId 141 } 142 143 // Get the HTTP client 144 client, err := c.Meta.Client() 145 if err != nil { 146 c.Ui.Error(fmt.Sprintf("Error initializing client: %s", err)) 147 return 1 148 } 149 150 // Use list mode if no node name was provided 151 if len(args) == 0 && !c.self { 152 153 // Query the node info 154 nodes, _, err := client.Nodes().List(nil) 155 if err != nil { 156 c.Ui.Error(fmt.Sprintf("Error querying node status: %s", err)) 157 return 1 158 } 159 160 // If output format is specified, format and output the node data list 161 if c.json || len(c.tmpl) > 0 { 162 out, err := Format(c.json, c.tmpl, nodes) 163 if err != nil { 164 c.Ui.Error(err.Error()) 165 return 1 166 } 167 168 c.Ui.Output(out) 169 return 0 170 } 171 172 // Return nothing if no nodes found 173 if len(nodes) == 0 { 174 return 0 175 } 176 177 // Format the nodes list 178 out := make([]string, len(nodes)+1) 179 180 out[0] = "ID|DC|Name|Class|" 181 182 if c.verbose { 183 out[0] += "Version|" 184 } 185 186 out[0] += "Drain|Status" 187 188 if c.list_allocs { 189 out[0] += "|Running Allocs" 190 } 191 192 for i, node := range nodes { 193 out[i+1] = fmt.Sprintf("%s|%s|%s|%s", 194 limit(node.ID, c.length), 195 node.Datacenter, 196 node.Name, 197 node.NodeClass) 198 if c.verbose { 199 out[i+1] += fmt.Sprintf("|%s", 200 node.Version) 201 } 202 out[i+1] += fmt.Sprintf("|%v|%s", 203 node.Drain, 204 node.Status) 205 if c.list_allocs { 206 numAllocs, err := getRunningAllocs(client, node.ID) 207 if err != nil { 208 c.Ui.Error(fmt.Sprintf("Error querying node allocations: %s", err)) 209 return 1 210 } 211 out[i+1] += fmt.Sprintf("|%v", 212 len(numAllocs)) 213 } 214 } 215 216 // Dump the output 217 c.Ui.Output(formatList(out)) 218 return 0 219 } 220 221 // Query the specific node 222 var nodeID string 223 if !c.self { 224 nodeID = args[0] 225 } else { 226 var err error 227 if nodeID, err = getLocalNodeID(client); err != nil { 228 c.Ui.Error(err.Error()) 229 return 1 230 } 231 } 232 if len(nodeID) == 1 { 233 c.Ui.Error(fmt.Sprintf("Identifier must contain at least two characters.")) 234 return 1 235 } 236 237 nodeID = sanatizeUUIDPrefix(nodeID) 238 nodes, _, err := client.Nodes().PrefixList(nodeID) 239 if err != nil { 240 c.Ui.Error(fmt.Sprintf("Error querying node info: %s", err)) 241 return 1 242 } 243 // Return error if no nodes are found 244 if len(nodes) == 0 { 245 c.Ui.Error(fmt.Sprintf("No node(s) with prefix %q found", nodeID)) 246 return 1 247 } 248 if len(nodes) > 1 { 249 // Format the nodes list that matches the prefix so that the user 250 // can create a more specific request 251 out := make([]string, len(nodes)+1) 252 out[0] = "ID|DC|Name|Class|Drain|Status" 253 for i, node := range nodes { 254 out[i+1] = fmt.Sprintf("%s|%s|%s|%s|%v|%s", 255 limit(node.ID, c.length), 256 node.Datacenter, 257 node.Name, 258 node.NodeClass, 259 node.Drain, 260 node.Status) 261 } 262 // Dump the output 263 c.Ui.Error(fmt.Sprintf("Prefix matched multiple nodes\n\n%s", formatList(out))) 264 return 1 265 } 266 // Prefix lookup matched a single node 267 node, _, err := client.Nodes().Info(nodes[0].ID, nil) 268 if err != nil { 269 c.Ui.Error(fmt.Sprintf("Error querying node info: %s", err)) 270 return 1 271 } 272 273 // If output format is specified, format and output the data 274 if c.json || len(c.tmpl) > 0 { 275 out, err := Format(c.json, c.tmpl, node) 276 if err != nil { 277 c.Ui.Error(err.Error()) 278 return 1 279 } 280 281 c.Ui.Output(out) 282 return 0 283 } 284 285 return c.formatNode(client, node) 286 } 287 288 func nodeDrivers(n *api.Node) []string { 289 var drivers []string 290 for k, v := range n.Attributes { 291 // driver.docker = 1 292 parts := strings.Split(k, ".") 293 if len(parts) != 2 { 294 continue 295 } else if parts[0] != "driver" { 296 continue 297 } else if v != "1" { 298 continue 299 } 300 301 drivers = append(drivers, parts[1]) 302 } 303 304 sort.Strings(drivers) 305 return drivers 306 } 307 308 func (c *NodeStatusCommand) formatNode(client *api.Client, node *api.Node) int { 309 // Format the header output 310 basic := []string{ 311 fmt.Sprintf("ID|%s", limit(node.ID, c.length)), 312 fmt.Sprintf("Name|%s", node.Name), 313 fmt.Sprintf("Class|%s", node.NodeClass), 314 fmt.Sprintf("DC|%s", node.Datacenter), 315 fmt.Sprintf("Drain|%v", node.Drain), 316 fmt.Sprintf("Status|%s", node.Status), 317 fmt.Sprintf("Drivers|%s", strings.Join(nodeDrivers(node), ",")), 318 } 319 320 if c.short { 321 c.Ui.Output(c.Colorize().Color(formatKV(basic))) 322 } else { 323 // Get the host stats 324 hostStats, nodeStatsErr := client.Nodes().Stats(node.ID, nil) 325 if nodeStatsErr != nil { 326 c.Ui.Output("") 327 c.Ui.Error(fmt.Sprintf("error fetching node stats (HINT: ensure Client.Advertise.HTTP is set): %v", nodeStatsErr)) 328 } 329 if hostStats != nil { 330 uptime := time.Duration(hostStats.Uptime * uint64(time.Second)) 331 basic = append(basic, fmt.Sprintf("Uptime|%s", uptime.String())) 332 } 333 c.Ui.Output(c.Colorize().Color(formatKV(basic))) 334 335 // Get list of running allocations on the node 336 runningAllocs, err := getRunningAllocs(client, node.ID) 337 if err != nil { 338 c.Ui.Error(fmt.Sprintf("Error querying node for running allocations: %s", err)) 339 return 1 340 } 341 342 allocatedResources := getAllocatedResources(client, runningAllocs, node) 343 c.Ui.Output(c.Colorize().Color("\n[bold]Allocated Resources[reset]")) 344 c.Ui.Output(formatList(allocatedResources)) 345 346 actualResources, err := getActualResources(client, runningAllocs, node) 347 if err == nil { 348 c.Ui.Output(c.Colorize().Color("\n[bold]Allocation Resource Utilization[reset]")) 349 c.Ui.Output(formatList(actualResources)) 350 } 351 352 hostResources, err := getHostResources(hostStats, node) 353 if err != nil { 354 c.Ui.Output("") 355 c.Ui.Error(fmt.Sprintf("error fetching node stats (HINT: ensure Client.Advertise.HTTP is set): %v", err)) 356 } 357 if err == nil { 358 c.Ui.Output(c.Colorize().Color("\n[bold]Host Resource Utilization[reset]")) 359 c.Ui.Output(formatList(hostResources)) 360 } 361 362 if hostStats != nil && c.stats { 363 c.Ui.Output(c.Colorize().Color("\n[bold]CPU Stats[reset]")) 364 c.printCpuStats(hostStats) 365 c.Ui.Output(c.Colorize().Color("\n[bold]Memory Stats[reset]")) 366 c.printMemoryStats(hostStats) 367 c.Ui.Output(c.Colorize().Color("\n[bold]Disk Stats[reset]")) 368 c.printDiskStats(hostStats) 369 } 370 } 371 372 nodeAllocs, _, err := client.Nodes().Allocations(node.ID, nil) 373 if err != nil { 374 c.Ui.Error(fmt.Sprintf("Error querying node allocations: %s", err)) 375 return 1 376 } 377 378 c.Ui.Output(c.Colorize().Color("\n[bold]Allocations[reset]")) 379 c.Ui.Output(formatAllocList(nodeAllocs, c.verbose, c.length)) 380 381 if c.verbose { 382 c.formatAttributes(node) 383 c.formatMeta(node) 384 } 385 return 0 386 387 } 388 389 func (c *NodeStatusCommand) formatAttributes(node *api.Node) { 390 // Print the attributes 391 keys := make([]string, len(node.Attributes)) 392 for k := range node.Attributes { 393 keys = append(keys, k) 394 } 395 sort.Strings(keys) 396 397 var attributes []string 398 for _, k := range keys { 399 if k != "" { 400 attributes = append(attributes, fmt.Sprintf("%s|%s", k, node.Attributes[k])) 401 } 402 } 403 c.Ui.Output(c.Colorize().Color("\n[bold]Attributes[reset]")) 404 c.Ui.Output(formatKV(attributes)) 405 } 406 407 func (c *NodeStatusCommand) formatMeta(node *api.Node) { 408 // Print the meta 409 keys := make([]string, 0, len(node.Meta)) 410 for k := range node.Meta { 411 keys = append(keys, k) 412 } 413 sort.Strings(keys) 414 415 var meta []string 416 for _, k := range keys { 417 if k != "" { 418 meta = append(meta, fmt.Sprintf("%s|%s", k, node.Meta[k])) 419 } 420 } 421 c.Ui.Output(c.Colorize().Color("\n[bold]Meta[reset]")) 422 c.Ui.Output(formatKV(meta)) 423 } 424 425 func (c *NodeStatusCommand) printCpuStats(hostStats *api.HostStats) { 426 l := len(hostStats.CPU) 427 for i, cpuStat := range hostStats.CPU { 428 cpuStatsAttr := make([]string, 4) 429 cpuStatsAttr[0] = fmt.Sprintf("CPU|%v", cpuStat.CPU) 430 cpuStatsAttr[1] = fmt.Sprintf("User|%v%%", humanize.FormatFloat(floatFormat, cpuStat.User)) 431 cpuStatsAttr[2] = fmt.Sprintf("System|%v%%", humanize.FormatFloat(floatFormat, cpuStat.System)) 432 cpuStatsAttr[3] = fmt.Sprintf("Idle|%v%%", humanize.FormatFloat(floatFormat, cpuStat.Idle)) 433 c.Ui.Output(formatKV(cpuStatsAttr)) 434 if i+1 < l { 435 c.Ui.Output("") 436 } 437 } 438 } 439 440 func (c *NodeStatusCommand) printMemoryStats(hostStats *api.HostStats) { 441 memoryStat := hostStats.Memory 442 memStatsAttr := make([]string, 4) 443 memStatsAttr[0] = fmt.Sprintf("Total|%v", humanize.IBytes(memoryStat.Total)) 444 memStatsAttr[1] = fmt.Sprintf("Available|%v", humanize.IBytes(memoryStat.Available)) 445 memStatsAttr[2] = fmt.Sprintf("Used|%v", humanize.IBytes(memoryStat.Used)) 446 memStatsAttr[3] = fmt.Sprintf("Free|%v", humanize.IBytes(memoryStat.Free)) 447 c.Ui.Output(formatKV(memStatsAttr)) 448 } 449 450 func (c *NodeStatusCommand) printDiskStats(hostStats *api.HostStats) { 451 l := len(hostStats.DiskStats) 452 for i, diskStat := range hostStats.DiskStats { 453 diskStatsAttr := make([]string, 7) 454 diskStatsAttr[0] = fmt.Sprintf("Device|%s", diskStat.Device) 455 diskStatsAttr[1] = fmt.Sprintf("MountPoint|%s", diskStat.Mountpoint) 456 diskStatsAttr[2] = fmt.Sprintf("Size|%s", humanize.IBytes(diskStat.Size)) 457 diskStatsAttr[3] = fmt.Sprintf("Used|%s", humanize.IBytes(diskStat.Used)) 458 diskStatsAttr[4] = fmt.Sprintf("Available|%s", humanize.IBytes(diskStat.Available)) 459 diskStatsAttr[5] = fmt.Sprintf("Used Percent|%v%%", humanize.FormatFloat(floatFormat, diskStat.UsedPercent)) 460 diskStatsAttr[6] = fmt.Sprintf("Inodes Percent|%v%%", humanize.FormatFloat(floatFormat, diskStat.InodesUsedPercent)) 461 c.Ui.Output(formatKV(diskStatsAttr)) 462 if i+1 < l { 463 c.Ui.Output("") 464 } 465 } 466 } 467 468 // getRunningAllocs returns a slice of allocation id's running on the node 469 func getRunningAllocs(client *api.Client, nodeID string) ([]*api.Allocation, error) { 470 var allocs []*api.Allocation 471 472 // Query the node allocations 473 nodeAllocs, _, err := client.Nodes().Allocations(nodeID, nil) 474 // Filter list to only running allocations 475 for _, alloc := range nodeAllocs { 476 if alloc.ClientStatus == "running" { 477 allocs = append(allocs, alloc) 478 } 479 } 480 return allocs, err 481 } 482 483 // getAllocatedResources returns the resource usage of the node. 484 func getAllocatedResources(client *api.Client, runningAllocs []*api.Allocation, node *api.Node) []string { 485 // Compute the total 486 total := computeNodeTotalResources(node) 487 488 // Get Resources 489 var cpu, mem, disk, iops int 490 for _, alloc := range runningAllocs { 491 cpu += *alloc.Resources.CPU 492 mem += *alloc.Resources.MemoryMB 493 disk += *alloc.Resources.DiskMB 494 iops += *alloc.Resources.IOPS 495 } 496 497 resources := make([]string, 2) 498 resources[0] = "CPU|Memory|Disk|IOPS" 499 resources[1] = fmt.Sprintf("%d/%d MHz|%s/%s|%s/%s|%d/%d", 500 cpu, 501 *total.CPU, 502 humanize.IBytes(uint64(mem*bytesPerMegabyte)), 503 humanize.IBytes(uint64(*total.MemoryMB*bytesPerMegabyte)), 504 humanize.IBytes(uint64(disk*bytesPerMegabyte)), 505 humanize.IBytes(uint64(*total.DiskMB*bytesPerMegabyte)), 506 iops, 507 *total.IOPS) 508 509 return resources 510 } 511 512 // computeNodeTotalResources returns the total allocatable resources (resources 513 // minus reserved) 514 func computeNodeTotalResources(node *api.Node) api.Resources { 515 total := api.Resources{} 516 517 r := node.Resources 518 res := node.Reserved 519 if res == nil { 520 res = &api.Resources{} 521 } 522 total.CPU = helper.IntToPtr(*r.CPU - *res.CPU) 523 total.MemoryMB = helper.IntToPtr(*r.MemoryMB - *res.MemoryMB) 524 total.DiskMB = helper.IntToPtr(*r.DiskMB - *res.DiskMB) 525 total.IOPS = helper.IntToPtr(*r.IOPS - *res.IOPS) 526 return total 527 } 528 529 // getActualResources returns the actual resource usage of the allocations. 530 func getActualResources(client *api.Client, runningAllocs []*api.Allocation, node *api.Node) ([]string, error) { 531 // Compute the total 532 total := computeNodeTotalResources(node) 533 534 // Get Resources 535 var cpu float64 536 var mem uint64 537 for _, alloc := range runningAllocs { 538 // Make the call to the client to get the actual usage. 539 stats, err := client.Allocations().Stats(alloc, nil) 540 if err != nil { 541 return nil, err 542 } 543 544 cpu += stats.ResourceUsage.CpuStats.TotalTicks 545 mem += stats.ResourceUsage.MemoryStats.RSS 546 } 547 548 resources := make([]string, 2) 549 resources[0] = "CPU|Memory" 550 resources[1] = fmt.Sprintf("%v/%d MHz|%v/%v", 551 math.Floor(cpu), 552 *total.CPU, 553 humanize.IBytes(mem), 554 humanize.IBytes(uint64(*total.MemoryMB*bytesPerMegabyte))) 555 556 return resources, nil 557 } 558 559 // getHostResources returns the actual resource usage of the node. 560 func getHostResources(hostStats *api.HostStats, node *api.Node) ([]string, error) { 561 if hostStats == nil { 562 return nil, fmt.Errorf("actual resource usage not present") 563 } 564 var resources []string 565 566 // calculate disk usage 567 storageDevice := node.Attributes["unique.storage.volume"] 568 var diskUsed, diskSize uint64 569 var physical bool 570 for _, disk := range hostStats.DiskStats { 571 if disk.Device == storageDevice { 572 diskUsed = disk.Used 573 diskSize = disk.Size 574 physical = true 575 } 576 } 577 578 resources = make([]string, 2) 579 resources[0] = "CPU|Memory|Disk" 580 if physical { 581 resources[1] = fmt.Sprintf("%v/%d MHz|%s/%s|%s/%s", 582 math.Floor(hostStats.CPUTicksConsumed), 583 *node.Resources.CPU, 584 humanize.IBytes(hostStats.Memory.Used), 585 humanize.IBytes(hostStats.Memory.Total), 586 humanize.IBytes(diskUsed), 587 humanize.IBytes(diskSize), 588 ) 589 } else { 590 // If non-physical device are used, output device name only, 591 // since nomad doesn't collect the stats data. 592 resources[1] = fmt.Sprintf("%v/%d MHz|%s/%s|(%s)", 593 math.Floor(hostStats.CPUTicksConsumed), 594 *node.Resources.CPU, 595 humanize.IBytes(hostStats.Memory.Used), 596 humanize.IBytes(hostStats.Memory.Total), 597 storageDevice, 598 ) 599 } 600 return resources, nil 601 }