github.com/iqoqo/nomad@v0.11.3-0.20200911112621-d7021c74d101/command/node_status.go (about) 1 package command 2 3 import ( 4 "fmt" 5 "math" 6 "sort" 7 "strconv" 8 "strings" 9 "time" 10 11 humanize "github.com/dustin/go-humanize" 12 "github.com/hashicorp/nomad/api" 13 "github.com/hashicorp/nomad/api/contexts" 14 "github.com/hashicorp/nomad/helper" 15 "github.com/posener/complete" 16 ) 17 18 const ( 19 // floatFormat is a format string for formatting floats. 20 floatFormat = "#,###.##" 21 22 // bytesPerMegabyte is the number of bytes per MB 23 bytesPerMegabyte = 1024 * 1024 24 ) 25 26 type NodeStatusCommand struct { 27 Meta 28 length int 29 short bool 30 verbose bool 31 list_allocs bool 32 self bool 33 stats bool 34 json bool 35 tmpl string 36 tokenLookup bool 37 } 38 39 func (c *NodeStatusCommand) Help() string { 40 helpText := ` 41 Usage: nomad node status [options] <node> 42 43 Display status information about a given node. The list of nodes 44 returned includes only nodes which jobs may be scheduled to, and 45 includes status and other high-level information. 46 47 If a node ID is passed, information for that specific node will be displayed, 48 including resource usage statistics. If no node ID's are passed, then a 49 short-hand list of all nodes will be displayed. The -self flag is useful to 50 quickly access the status of the local node. 51 52 General Options: 53 54 ` + generalOptionsUsage() + ` 55 56 Node Status Options: 57 58 -self 59 Query the status of the local node. 60 61 -stats 62 Display detailed resource usage statistics. 63 64 -allocs 65 Display a count of running allocations for each node. 66 67 -short 68 Display short output. Used only when a single node is being 69 queried, and drops verbose output about node allocations. 70 71 -verbose 72 Display full information. 73 74 -json 75 Output the node in its JSON format. 76 77 -t 78 Format and display node using a Go template. 79 80 -tokenLookup 81 Treat the argument as token lookup. 82 ` 83 return strings.TrimSpace(helpText) 84 } 85 86 func (c *NodeStatusCommand) Synopsis() string { 87 return "Display status information about nodes" 88 } 89 90 func (c *NodeStatusCommand) AutocompleteFlags() complete.Flags { 91 return mergeAutocompleteFlags(c.Meta.AutocompleteFlags(FlagSetClient), 92 complete.Flags{ 93 "-allocs": complete.PredictNothing, 94 "-json": complete.PredictNothing, 95 "-self": complete.PredictNothing, 96 "-short": complete.PredictNothing, 97 "-stats": complete.PredictNothing, 98 "-t": complete.PredictAnything, 99 "-token-lookup": complete.PredictAnything, 100 "-verbose": complete.PredictNothing, 101 }) 102 } 103 104 func (c *NodeStatusCommand) AutocompleteArgs() complete.Predictor { 105 return complete.PredictFunc(func(a complete.Args) []string { 106 client, err := c.Meta.Client() 107 if err != nil { 108 return nil 109 } 110 111 resp, _, err := client.Search().PrefixSearch(a.Last, contexts.Nodes, nil) 112 if err != nil { 113 return []string{} 114 } 115 return resp.Matches[contexts.Nodes] 116 }) 117 } 118 119 func (c *NodeStatusCommand) Name() string { return "node-status" } 120 121 func (c *NodeStatusCommand) Run(args []string) int { 122 123 flags := c.Meta.FlagSet(c.Name(), FlagSetClient) 124 flags.Usage = func() { c.Ui.Output(c.Help()) } 125 flags.BoolVar(&c.short, "short", false, "") 126 flags.BoolVar(&c.verbose, "verbose", false, "") 127 flags.BoolVar(&c.list_allocs, "allocs", false, "") 128 flags.BoolVar(&c.self, "self", false, "") 129 flags.BoolVar(&c.stats, "stats", false, "") 130 flags.BoolVar(&c.json, "json", false, "") 131 flags.StringVar(&c.tmpl, "t", "", "") 132 flags.BoolVar(&c.tokenLookup, "token-lookup", false, "") 133 134 if err := flags.Parse(args); err != nil { 135 return 1 136 } 137 138 // Check that we got either a single node or none 139 args = flags.Args() 140 if len(args) > 1 { 141 c.Ui.Error("This command takes either one or no arguments") 142 c.Ui.Error(commandErrorText(c)) 143 return 1 144 } 145 146 // Truncate the id unless full length is requested 147 c.length = shortId 148 if c.verbose { 149 c.length = fullId 150 } 151 152 // Get the HTTP client 153 client, err := c.Meta.Client() 154 if err != nil { 155 c.Ui.Error(fmt.Sprintf("Error initializing client: %s", err)) 156 return 1 157 } 158 159 // Use list mode if no node name was provided 160 if len(args) == 0 && !c.self { 161 162 // Query the node info 163 nodes, _, err := client.Nodes().List(nil) 164 if err != nil { 165 c.Ui.Error(fmt.Sprintf("Error querying node status: %s", err)) 166 return 1 167 } 168 169 // If output format is specified, format and output the node data list 170 if c.json || len(c.tmpl) > 0 { 171 out, err := Format(c.json, c.tmpl, nodes) 172 if err != nil { 173 c.Ui.Error(err.Error()) 174 return 1 175 } 176 177 c.Ui.Output(out) 178 return 0 179 } 180 181 // Return nothing if no nodes found 182 if len(nodes) == 0 { 183 return 0 184 } 185 186 // Format the nodes list 187 out := make([]string, len(nodes)+1) 188 189 out[0] = "ID|DC|Name|Class|" 190 191 if c.verbose { 192 out[0] += "Address|Version|" 193 } 194 195 out[0] += "Drain|Eligibility|Status" 196 197 if c.list_allocs { 198 out[0] += "|Running Allocs" 199 } 200 201 for i, node := range nodes { 202 out[i+1] = fmt.Sprintf("%s|%s|%s|%s", 203 limit(node.ID, c.length), 204 node.Datacenter, 205 node.Name, 206 node.NodeClass) 207 if c.verbose { 208 out[i+1] += fmt.Sprintf("|%s|%s", 209 node.Address, node.Version) 210 } 211 out[i+1] += fmt.Sprintf("|%v|%s|%s", 212 node.Drain, 213 node.SchedulingEligibility, 214 node.Status) 215 216 if c.list_allocs { 217 numAllocs, err := getRunningAllocs(client, node.ID) 218 if err != nil { 219 c.Ui.Error(fmt.Sprintf("Error querying node allocations: %s", err)) 220 return 1 221 } 222 out[i+1] += fmt.Sprintf("|%v", 223 len(numAllocs)) 224 } 225 } 226 227 // Dump the output 228 c.Ui.Output(formatList(out)) 229 return 0 230 } 231 232 // Query the specific node 233 var nodeID = args[0] 234 if c.tokenLookup { 235 nodes, _, err := client.Nodes().TokenList(nodeID) 236 if err != nil { 237 c.Ui.Error(err.Error()) 238 return 1 239 } 240 c.Ui.Output(formatNodeStubList(nodes, c.verbose)) 241 return 0 242 } else if c.self { 243 var err error 244 if nodeID, err = getLocalNodeID(client); err != nil { 245 c.Ui.Error(err.Error()) 246 return 1 247 } 248 } 249 if len(nodeID) == 1 { 250 c.Ui.Error(fmt.Sprintf("Identifier must contain at least two characters.")) 251 return 1 252 } 253 254 nodeID = sanitizeUUIDPrefix(nodeID) 255 nodes, _, err := client.Nodes().PrefixList(nodeID) 256 if err != nil { 257 c.Ui.Error(fmt.Sprintf("Error querying node info: %s", err)) 258 return 1 259 } 260 // Return error if no nodes are found 261 if len(nodes) == 0 { 262 c.Ui.Error(fmt.Sprintf("No node(s) with prefix %q found", nodeID)) 263 return 1 264 } 265 if len(nodes) > 1 { 266 // Dump the output 267 c.Ui.Error(fmt.Sprintf("Prefix matched multiple nodes\n\n%s", 268 formatNodeStubList(nodes, c.verbose))) 269 return 1 270 } 271 272 // Prefix lookup matched a single node 273 node, _, err := client.Nodes().Info(nodes[0].ID, nil) 274 if err != nil { 275 c.Ui.Error(fmt.Sprintf("Error querying node info: %s", err)) 276 return 1 277 } 278 279 // If output format is specified, format and output the data 280 if c.json || len(c.tmpl) > 0 { 281 out, err := Format(c.json, c.tmpl, node) 282 if err != nil { 283 c.Ui.Error(err.Error()) 284 return 1 285 } 286 287 c.Ui.Output(out) 288 return 0 289 } 290 291 return c.formatNode(client, node) 292 } 293 294 func nodeDrivers(n *api.Node) []string { 295 var drivers []string 296 for k, v := range n.Attributes { 297 // driver.docker = 1 298 parts := strings.Split(k, ".") 299 if len(parts) != 2 { 300 continue 301 } else if parts[0] != "driver" { 302 continue 303 } else if v != "1" { 304 continue 305 } 306 307 drivers = append(drivers, parts[1]) 308 } 309 310 sort.Strings(drivers) 311 return drivers 312 } 313 314 func nodeCSIControllerNames(n *api.Node) []string { 315 var names []string 316 for name := range n.CSIControllerPlugins { 317 names = append(names, name) 318 } 319 sort.Strings(names) 320 return names 321 } 322 323 func nodeCSINodeNames(n *api.Node) []string { 324 var names []string 325 for name := range n.CSINodePlugins { 326 names = append(names, name) 327 } 328 sort.Strings(names) 329 return names 330 } 331 332 func nodeCSIVolumeNames(n *api.Node, allocs []*api.Allocation) []string { 333 var names []string 334 for _, alloc := range allocs { 335 tg := alloc.GetTaskGroup() 336 if tg == nil || len(tg.Volumes) == 0 { 337 continue 338 } 339 340 for _, v := range tg.Volumes { 341 names = append(names, v.Name) 342 } 343 } 344 sort.Strings(names) 345 return names 346 } 347 348 func nodeVolumeNames(n *api.Node) []string { 349 var volumes []string 350 for name := range n.HostVolumes { 351 volumes = append(volumes, name) 352 } 353 354 sort.Strings(volumes) 355 return volumes 356 } 357 358 func formatDrain(n *api.Node) string { 359 if n.DrainStrategy != nil { 360 b := new(strings.Builder) 361 b.WriteString("true") 362 if n.DrainStrategy.DrainSpec.Deadline.Nanoseconds() < 0 { 363 b.WriteString("; force drain") 364 } else if n.DrainStrategy.ForceDeadline.IsZero() { 365 b.WriteString("; no deadline") 366 } else { 367 fmt.Fprintf(b, "; %s deadline", formatTime(n.DrainStrategy.ForceDeadline)) 368 } 369 370 if n.DrainStrategy.IgnoreSystemJobs { 371 b.WriteString("; ignoring system jobs") 372 } 373 return b.String() 374 } 375 376 return strconv.FormatBool(n.Drain) 377 } 378 379 func (c *NodeStatusCommand) formatNode(client *api.Client, node *api.Node) int { 380 // Make one API call for allocations 381 nodeAllocs, _, err := client.Nodes().Allocations(node.ID, nil) 382 if err != nil { 383 c.Ui.Error(fmt.Sprintf("Error querying node allocations: %s", err)) 384 return 1 385 } 386 387 var runningAllocs []*api.Allocation 388 for _, alloc := range nodeAllocs { 389 if alloc.ClientStatus == "running" { 390 runningAllocs = append(runningAllocs, alloc) 391 } 392 } 393 394 // Format the header output 395 basic := []string{ 396 fmt.Sprintf("ID|%s", node.ID), 397 fmt.Sprintf("Name|%s", node.Name), 398 fmt.Sprintf("Class|%s", node.NodeClass), 399 fmt.Sprintf("DC|%s", node.Datacenter), 400 fmt.Sprintf("Drain|%v", formatDrain(node)), 401 fmt.Sprintf("Eligibility|%s", node.SchedulingEligibility), 402 fmt.Sprintf("Status|%s", node.Status), 403 fmt.Sprintf("CSI Controllers|%s", strings.Join(nodeCSIControllerNames(node), ",")), 404 fmt.Sprintf("CSI Drivers|%s", strings.Join(nodeCSINodeNames(node), ",")), 405 } 406 407 if c.short { 408 basic = append(basic, fmt.Sprintf("Host Volumes|%s", strings.Join(nodeVolumeNames(node), ","))) 409 basic = append(basic, fmt.Sprintf("CSI Volumes|%s", strings.Join(nodeCSIVolumeNames(node, runningAllocs), ","))) 410 basic = append(basic, fmt.Sprintf("Drivers|%s", strings.Join(nodeDrivers(node), ","))) 411 c.Ui.Output(c.Colorize().Color(formatKV(basic))) 412 413 // Output alloc info 414 if err := c.outputAllocInfo(node, nodeAllocs); err != nil { 415 c.Ui.Error(fmt.Sprintf("%s", err)) 416 return 1 417 } 418 419 return 0 420 } 421 422 // Get the host stats 423 hostStats, nodeStatsErr := client.Nodes().Stats(node.ID, nil) 424 if nodeStatsErr != nil { 425 c.Ui.Output("") 426 c.Ui.Error(fmt.Sprintf("error fetching node stats: %v", nodeStatsErr)) 427 } 428 if hostStats != nil { 429 uptime := time.Duration(hostStats.Uptime * uint64(time.Second)) 430 basic = append(basic, fmt.Sprintf("Uptime|%s", uptime.String())) 431 } 432 433 // When we're not running in verbose mode, then also include host volumes and 434 // driver info in the basic output 435 if !c.verbose { 436 basic = append(basic, fmt.Sprintf("Host Volumes|%s", strings.Join(nodeVolumeNames(node), ","))) 437 basic = append(basic, fmt.Sprintf("CSI Volumes|%s", strings.Join(nodeCSIVolumeNames(node, runningAllocs), ","))) 438 driverStatus := fmt.Sprintf("Driver Status| %s", c.outputTruncatedNodeDriverInfo(node)) 439 basic = append(basic, driverStatus) 440 } 441 442 // Output the basic info 443 c.Ui.Output(c.Colorize().Color(formatKV(basic))) 444 445 // If we're running in verbose mode, include full host volume and driver info 446 if c.verbose { 447 c.outputNodeVolumeInfo(node) 448 c.outputNodeCSIVolumeInfo(client, node, runningAllocs) 449 c.outputNodeDriverInfo(node) 450 } 451 452 // Emit node events 453 c.outputNodeStatusEvents(node) 454 455 // Get list of running allocations on the node 456 allocatedResources := getAllocatedResources(client, runningAllocs, node) 457 c.Ui.Output(c.Colorize().Color("\n[bold]Allocated Resources[reset]")) 458 c.Ui.Output(formatList(allocatedResources)) 459 460 actualResources, err := getActualResources(client, runningAllocs, node) 461 if err == nil { 462 c.Ui.Output(c.Colorize().Color("\n[bold]Allocation Resource Utilization[reset]")) 463 c.Ui.Output(formatList(actualResources)) 464 } 465 466 hostResources, err := getHostResources(hostStats, node) 467 if err != nil { 468 c.Ui.Output("") 469 c.Ui.Error(fmt.Sprintf("error fetching node stats: %v", err)) 470 } 471 if err == nil { 472 c.Ui.Output(c.Colorize().Color("\n[bold]Host Resource Utilization[reset]")) 473 c.Ui.Output(formatList(hostResources)) 474 } 475 476 if err == nil && node.NodeResources != nil && len(node.NodeResources.Devices) > 0 { 477 c.Ui.Output(c.Colorize().Color("\n[bold]Device Resource Utilization[reset]")) 478 c.Ui.Output(formatList(getDeviceResourcesForNode(hostStats.DeviceStats, node))) 479 } 480 if hostStats != nil && c.stats { 481 c.Ui.Output(c.Colorize().Color("\n[bold]CPU Stats[reset]")) 482 c.printCpuStats(hostStats) 483 c.Ui.Output(c.Colorize().Color("\n[bold]Memory Stats[reset]")) 484 c.printMemoryStats(hostStats) 485 c.Ui.Output(c.Colorize().Color("\n[bold]Disk Stats[reset]")) 486 c.printDiskStats(hostStats) 487 if len(hostStats.DeviceStats) > 0 { 488 c.Ui.Output(c.Colorize().Color("\n[bold]Device Stats[reset]")) 489 printDeviceStats(c.Ui, hostStats.DeviceStats) 490 } 491 } 492 493 if err := c.outputAllocInfo(node, nodeAllocs); err != nil { 494 c.Ui.Error(fmt.Sprintf("%s", err)) 495 return 1 496 } 497 498 return 0 499 } 500 501 func (c *NodeStatusCommand) outputAllocInfo(node *api.Node, nodeAllocs []*api.Allocation) error { 502 c.Ui.Output(c.Colorize().Color("\n[bold]Allocations[reset]")) 503 c.Ui.Output(formatAllocList(nodeAllocs, c.verbose, c.length)) 504 505 if c.verbose { 506 c.formatAttributes(node) 507 c.formatDeviceAttributes(node) 508 c.formatMeta(node) 509 } 510 511 return nil 512 } 513 514 func (c *NodeStatusCommand) outputTruncatedNodeDriverInfo(node *api.Node) string { 515 drivers := make([]string, 0, len(node.Drivers)) 516 517 for driverName, driverInfo := range node.Drivers { 518 if !driverInfo.Detected { 519 continue 520 } 521 522 if !driverInfo.Healthy { 523 drivers = append(drivers, fmt.Sprintf("%s (unhealthy)", driverName)) 524 } else { 525 drivers = append(drivers, driverName) 526 } 527 } 528 sort.Strings(drivers) 529 return strings.Trim(strings.Join(drivers, ","), ", ") 530 } 531 532 func (c *NodeStatusCommand) outputNodeVolumeInfo(node *api.Node) { 533 c.Ui.Output(c.Colorize().Color("\n[bold]Host Volumes")) 534 535 names := make([]string, 0, len(node.HostVolumes)) 536 for name := range node.HostVolumes { 537 names = append(names, name) 538 } 539 sort.Strings(names) 540 541 output := make([]string, 0, len(names)+1) 542 output = append(output, "Name|ReadOnly|Source") 543 544 for _, volName := range names { 545 info := node.HostVolumes[volName] 546 output = append(output, fmt.Sprintf("%s|%v|%s", volName, info.ReadOnly, info.Path)) 547 } 548 c.Ui.Output(formatList(output)) 549 } 550 551 func (c *NodeStatusCommand) outputNodeCSIVolumeInfo(client *api.Client, node *api.Node, runningAllocs []*api.Allocation) { 552 c.Ui.Output(c.Colorize().Color("\n[bold]CSI Volumes")) 553 554 // Duplicate nodeCSIVolumeNames to sort by name but also index volume names to ids 555 var names []string 556 requests := map[string]*api.VolumeRequest{} 557 for _, alloc := range runningAllocs { 558 tg := alloc.GetTaskGroup() 559 if tg == nil || len(tg.Volumes) == 0 { 560 continue 561 } 562 563 for _, v := range tg.Volumes { 564 names = append(names, v.Name) 565 requests[v.Source] = v 566 } 567 } 568 if len(names) == 0 { 569 return 570 } 571 sort.Strings(names) 572 573 // Fetch the volume objects with current status 574 // Ignore an error, all we're going to do is omit the volumes 575 volumes := map[string]*api.CSIVolumeListStub{} 576 vs, _ := client.Nodes().CSIVolumes(node.ID, nil) 577 for _, v := range vs { 578 n := requests[v.ID].Name 579 volumes[n] = v 580 } 581 582 // Output the volumes in name order 583 output := make([]string, 0, len(names)+1) 584 output = append(output, "ID|Name|Plugin ID|Schedulable|Provider|Access Mode") 585 for _, name := range names { 586 v := volumes[name] 587 output = append(output, fmt.Sprintf( 588 "%s|%s|%s|%t|%s|%s", 589 v.ID, 590 name, 591 v.PluginID, 592 v.Schedulable, 593 v.Provider, 594 v.AccessMode, 595 )) 596 } 597 598 c.Ui.Output(formatList(output)) 599 } 600 601 func (c *NodeStatusCommand) outputNodeDriverInfo(node *api.Node) { 602 c.Ui.Output(c.Colorize().Color("\n[bold]Drivers")) 603 604 size := len(node.Drivers) 605 nodeDrivers := make([]string, 0, size+1) 606 607 nodeDrivers = append(nodeDrivers, "Driver|Detected|Healthy|Message|Time") 608 609 drivers := make([]string, 0, len(node.Drivers)) 610 for driver := range node.Drivers { 611 drivers = append(drivers, driver) 612 } 613 sort.Strings(drivers) 614 615 for _, driver := range drivers { 616 info := node.Drivers[driver] 617 timestamp := formatTime(info.UpdateTime) 618 nodeDrivers = append(nodeDrivers, fmt.Sprintf("%s|%v|%v|%s|%s", driver, info.Detected, info.Healthy, info.HealthDescription, timestamp)) 619 } 620 c.Ui.Output(formatList(nodeDrivers)) 621 } 622 623 func (c *NodeStatusCommand) outputNodeStatusEvents(node *api.Node) { 624 c.Ui.Output(c.Colorize().Color("\n[bold]Node Events")) 625 c.outputNodeEvent(node.Events) 626 } 627 628 func (c *NodeStatusCommand) outputNodeEvent(events []*api.NodeEvent) { 629 size := len(events) 630 nodeEvents := make([]string, size+1) 631 if c.verbose { 632 nodeEvents[0] = "Time|Subsystem|Message|Details" 633 } else { 634 nodeEvents[0] = "Time|Subsystem|Message" 635 } 636 637 for i, event := range events { 638 timestamp := formatTime(event.Timestamp) 639 subsystem := formatEventSubsystem(event.Subsystem, event.Details["driver"]) 640 msg := event.Message 641 if c.verbose { 642 details := formatEventDetails(event.Details) 643 nodeEvents[size-i] = fmt.Sprintf("%s|%s|%s|%s", timestamp, subsystem, msg, details) 644 } else { 645 nodeEvents[size-i] = fmt.Sprintf("%s|%s|%s", timestamp, subsystem, msg) 646 } 647 } 648 c.Ui.Output(formatList(nodeEvents)) 649 } 650 651 func formatEventSubsystem(subsystem, driverName string) string { 652 if driverName == "" { 653 return subsystem 654 } 655 656 // If this event is for a driver, append the driver name to make the message 657 // clearer 658 return fmt.Sprintf("Driver: %s", driverName) 659 } 660 661 func formatEventDetails(details map[string]string) string { 662 output := make([]string, 0, len(details)) 663 for k, v := range details { 664 output = append(output, fmt.Sprintf("%s: %s", k, v)) 665 } 666 return strings.Join(output, ", ") 667 } 668 669 func (c *NodeStatusCommand) formatAttributes(node *api.Node) { 670 // Print the attributes 671 keys := make([]string, len(node.Attributes)) 672 for k := range node.Attributes { 673 keys = append(keys, k) 674 } 675 sort.Strings(keys) 676 677 var attributes []string 678 for _, k := range keys { 679 if k != "" { 680 attributes = append(attributes, fmt.Sprintf("%s|%s", k, node.Attributes[k])) 681 } 682 } 683 c.Ui.Output(c.Colorize().Color("\n[bold]Attributes[reset]")) 684 c.Ui.Output(formatKV(attributes)) 685 } 686 687 func (c *NodeStatusCommand) formatDeviceAttributes(node *api.Node) { 688 if node.NodeResources == nil { 689 return 690 } 691 devices := node.NodeResources.Devices 692 if len(devices) == 0 { 693 return 694 } 695 696 sort.Slice(devices, func(i, j int) bool { 697 return devices[i].ID() < devices[j].ID() 698 }) 699 700 first := true 701 for _, d := range devices { 702 if len(d.Attributes) == 0 { 703 continue 704 } 705 706 if first { 707 c.Ui.Output("\nDevice Group Attributes") 708 first = false 709 } else { 710 c.Ui.Output("") 711 } 712 c.Ui.Output(formatKV(getDeviceAttributes(d))) 713 } 714 } 715 716 func (c *NodeStatusCommand) formatMeta(node *api.Node) { 717 // Print the meta 718 keys := make([]string, 0, len(node.Meta)) 719 for k := range node.Meta { 720 keys = append(keys, k) 721 } 722 sort.Strings(keys) 723 724 var meta []string 725 for _, k := range keys { 726 if k != "" { 727 meta = append(meta, fmt.Sprintf("%s|%s", k, node.Meta[k])) 728 } 729 } 730 c.Ui.Output(c.Colorize().Color("\n[bold]Meta[reset]")) 731 c.Ui.Output(formatKV(meta)) 732 } 733 734 func (c *NodeStatusCommand) printCpuStats(hostStats *api.HostStats) { 735 l := len(hostStats.CPU) 736 for i, cpuStat := range hostStats.CPU { 737 cpuStatsAttr := make([]string, 4) 738 cpuStatsAttr[0] = fmt.Sprintf("CPU|%v", cpuStat.CPU) 739 cpuStatsAttr[1] = fmt.Sprintf("User|%v%%", humanize.FormatFloat(floatFormat, cpuStat.User)) 740 cpuStatsAttr[2] = fmt.Sprintf("System|%v%%", humanize.FormatFloat(floatFormat, cpuStat.System)) 741 cpuStatsAttr[3] = fmt.Sprintf("Idle|%v%%", humanize.FormatFloat(floatFormat, cpuStat.Idle)) 742 c.Ui.Output(formatKV(cpuStatsAttr)) 743 if i+1 < l { 744 c.Ui.Output("") 745 } 746 } 747 } 748 749 func (c *NodeStatusCommand) printMemoryStats(hostStats *api.HostStats) { 750 memoryStat := hostStats.Memory 751 memStatsAttr := make([]string, 4) 752 memStatsAttr[0] = fmt.Sprintf("Total|%v", humanize.IBytes(memoryStat.Total)) 753 memStatsAttr[1] = fmt.Sprintf("Available|%v", humanize.IBytes(memoryStat.Available)) 754 memStatsAttr[2] = fmt.Sprintf("Used|%v", humanize.IBytes(memoryStat.Used)) 755 memStatsAttr[3] = fmt.Sprintf("Free|%v", humanize.IBytes(memoryStat.Free)) 756 c.Ui.Output(formatKV(memStatsAttr)) 757 } 758 759 func (c *NodeStatusCommand) printDiskStats(hostStats *api.HostStats) { 760 l := len(hostStats.DiskStats) 761 for i, diskStat := range hostStats.DiskStats { 762 diskStatsAttr := make([]string, 7) 763 diskStatsAttr[0] = fmt.Sprintf("Device|%s", diskStat.Device) 764 diskStatsAttr[1] = fmt.Sprintf("MountPoint|%s", diskStat.Mountpoint) 765 diskStatsAttr[2] = fmt.Sprintf("Size|%s", humanize.IBytes(diskStat.Size)) 766 diskStatsAttr[3] = fmt.Sprintf("Used|%s", humanize.IBytes(diskStat.Used)) 767 diskStatsAttr[4] = fmt.Sprintf("Available|%s", humanize.IBytes(diskStat.Available)) 768 diskStatsAttr[5] = fmt.Sprintf("Used Percent|%v%%", humanize.FormatFloat(floatFormat, diskStat.UsedPercent)) 769 diskStatsAttr[6] = fmt.Sprintf("Inodes Percent|%v%%", humanize.FormatFloat(floatFormat, diskStat.InodesUsedPercent)) 770 c.Ui.Output(formatKV(diskStatsAttr)) 771 if i+1 < l { 772 c.Ui.Output("") 773 } 774 } 775 } 776 777 // getRunningAllocs returns a slice of allocation id's running on the node 778 func getRunningAllocs(client *api.Client, nodeID string) ([]*api.Allocation, error) { 779 var allocs []*api.Allocation 780 781 // Query the node allocations 782 nodeAllocs, _, err := client.Nodes().Allocations(nodeID, nil) 783 // Filter list to only running allocations 784 for _, alloc := range nodeAllocs { 785 if alloc.ClientStatus == "running" { 786 allocs = append(allocs, alloc) 787 } 788 } 789 return allocs, err 790 } 791 792 // getAllocatedResources returns the resource usage of the node. 793 func getAllocatedResources(client *api.Client, runningAllocs []*api.Allocation, node *api.Node) []string { 794 // Compute the total 795 total := computeNodeTotalResources(node) 796 797 // Get Resources 798 var cpu, mem, disk int 799 for _, alloc := range runningAllocs { 800 cpu += *alloc.Resources.CPU 801 mem += *alloc.Resources.MemoryMB 802 disk += *alloc.Resources.DiskMB 803 } 804 805 resources := make([]string, 2) 806 resources[0] = "CPU|Memory|Disk" 807 resources[1] = fmt.Sprintf("%d/%d MHz|%s/%s|%s/%s", 808 cpu, 809 *total.CPU, 810 humanize.IBytes(uint64(mem*bytesPerMegabyte)), 811 humanize.IBytes(uint64(*total.MemoryMB*bytesPerMegabyte)), 812 humanize.IBytes(uint64(disk*bytesPerMegabyte)), 813 humanize.IBytes(uint64(*total.DiskMB*bytesPerMegabyte))) 814 815 return resources 816 } 817 818 // computeNodeTotalResources returns the total allocatable resources (resources 819 // minus reserved) 820 func computeNodeTotalResources(node *api.Node) api.Resources { 821 total := api.Resources{} 822 823 r := node.Resources 824 res := node.Reserved 825 if res == nil { 826 res = &api.Resources{} 827 } 828 total.CPU = helper.IntToPtr(*r.CPU - *res.CPU) 829 total.MemoryMB = helper.IntToPtr(*r.MemoryMB - *res.MemoryMB) 830 total.DiskMB = helper.IntToPtr(*r.DiskMB - *res.DiskMB) 831 return total 832 } 833 834 // getActualResources returns the actual resource usage of the allocations. 835 func getActualResources(client *api.Client, runningAllocs []*api.Allocation, node *api.Node) ([]string, error) { 836 // Compute the total 837 total := computeNodeTotalResources(node) 838 839 // Get Resources 840 var cpu float64 841 var mem uint64 842 for _, alloc := range runningAllocs { 843 // Make the call to the client to get the actual usage. 844 stats, err := client.Allocations().Stats(alloc, nil) 845 if err != nil { 846 return nil, err 847 } 848 849 cpu += stats.ResourceUsage.CpuStats.TotalTicks 850 mem += stats.ResourceUsage.MemoryStats.RSS 851 } 852 853 resources := make([]string, 2) 854 resources[0] = "CPU|Memory" 855 resources[1] = fmt.Sprintf("%v/%d MHz|%v/%v", 856 math.Floor(cpu), 857 *total.CPU, 858 humanize.IBytes(mem), 859 humanize.IBytes(uint64(*total.MemoryMB*bytesPerMegabyte))) 860 861 return resources, nil 862 } 863 864 // getHostResources returns the actual resource usage of the node. 865 func getHostResources(hostStats *api.HostStats, node *api.Node) ([]string, error) { 866 if hostStats == nil { 867 return nil, fmt.Errorf("actual resource usage not present") 868 } 869 var resources []string 870 871 // calculate disk usage 872 storageDevice := node.Attributes["unique.storage.volume"] 873 var diskUsed, diskSize uint64 874 var physical bool 875 for _, disk := range hostStats.DiskStats { 876 if disk.Device == storageDevice { 877 diskUsed = disk.Used 878 diskSize = disk.Size 879 physical = true 880 } 881 } 882 883 resources = make([]string, 2) 884 resources[0] = "CPU|Memory|Disk" 885 if physical { 886 resources[1] = fmt.Sprintf("%v/%d MHz|%s/%s|%s/%s", 887 math.Floor(hostStats.CPUTicksConsumed), 888 *node.Resources.CPU, 889 humanize.IBytes(hostStats.Memory.Used), 890 humanize.IBytes(hostStats.Memory.Total), 891 humanize.IBytes(diskUsed), 892 humanize.IBytes(diskSize), 893 ) 894 } else { 895 // If non-physical device are used, output device name only, 896 // since nomad doesn't collect the stats data. 897 resources[1] = fmt.Sprintf("%v/%d MHz|%s/%s|(%s)", 898 math.Floor(hostStats.CPUTicksConsumed), 899 *node.Resources.CPU, 900 humanize.IBytes(hostStats.Memory.Used), 901 humanize.IBytes(hostStats.Memory.Total), 902 storageDevice, 903 ) 904 } 905 return resources, nil 906 } 907 908 // formatNodeStubList is used to return a table format of a list of node stubs. 909 func formatNodeStubList(nodes []*api.NodeListStub, verbose bool) string { 910 // Return error if no nodes are found 911 if len(nodes) == 0 { 912 return "" 913 } 914 // Truncate the id unless full length is requested 915 length := shortId 916 if verbose { 917 length = fullId 918 } 919 920 // Format the nodes list that matches the prefix so that the user 921 // can create a more specific request 922 out := make([]string, len(nodes)+1) 923 out[0] = "ID|DC|Name|Class|Drain|Eligibility|Status" 924 for i, node := range nodes { 925 out[i+1] = fmt.Sprintf("%s|%s|%s|%s|%v|%s|%s", 926 limit(node.ID, length), 927 node.Datacenter, 928 node.Name, 929 node.NodeClass, 930 node.Drain, 931 node.SchedulingEligibility, 932 node.Status) 933 } 934 935 return formatList(out) 936 }