github.com/Ilhicas/nomad@v1.0.4-0.20210304152020-e86851182bc3/command/node_status.go (about) 1 package command 2 3 import ( 4 "fmt" 5 "math" 6 "sort" 7 "strconv" 8 "strings" 9 "time" 10 11 humanize "github.com/dustin/go-humanize" 12 "github.com/hashicorp/nomad/api" 13 "github.com/hashicorp/nomad/api/contexts" 14 "github.com/hashicorp/nomad/helper" 15 "github.com/hashicorp/nomad/nomad/structs" 16 "github.com/posener/complete" 17 ) 18 19 const ( 20 // floatFormat is a format string for formatting floats. 21 floatFormat = "#,###.##" 22 23 // bytesPerMegabyte is the number of bytes per MB 24 bytesPerMegabyte = 1024 * 1024 25 ) 26 27 type NodeStatusCommand struct { 28 Meta 29 length int 30 short bool 31 verbose bool 32 list_allocs bool 33 self bool 34 stats bool 35 json bool 36 tmpl string 37 } 38 39 func (c *NodeStatusCommand) Help() string { 40 helpText := ` 41 Usage: nomad node status [options] <node> 42 43 Display status information about a given node. The list of nodes 44 returned includes only nodes which jobs may be scheduled to, and 45 includes status and other high-level information. 46 47 If a node ID is passed, information for that specific node will be displayed, 48 including resource usage statistics. If no node ID's are passed, then a 49 short-hand list of all nodes will be displayed. The -self flag is useful to 50 quickly access the status of the local node. 51 52 If ACLs are enabled, this option requires a token with the 'node:read' 53 capability. 54 55 General Options: 56 57 ` + generalOptionsUsage(usageOptsDefault|usageOptsNoNamespace) + ` 58 59 Node Status Options: 60 61 -self 62 Query the status of the local node. 63 64 -stats 65 Display detailed resource usage statistics. 66 67 -allocs 68 Display a count of running allocations for each node. 69 70 -short 71 Display short output. Used only when a single node is being 72 queried, and drops verbose output about node allocations. 73 74 -verbose 75 Display full information. 76 77 -json 78 Output the node in its JSON format. 79 80 -t 81 Format and display node using a Go template. 82 ` 83 return strings.TrimSpace(helpText) 84 } 85 86 func (c *NodeStatusCommand) Synopsis() string { 87 return "Display status information about nodes" 88 } 89 90 func (c *NodeStatusCommand) AutocompleteFlags() complete.Flags { 91 return mergeAutocompleteFlags(c.Meta.AutocompleteFlags(FlagSetClient), 92 complete.Flags{ 93 "-allocs": complete.PredictNothing, 94 "-json": complete.PredictNothing, 95 "-self": complete.PredictNothing, 96 "-short": complete.PredictNothing, 97 "-stats": complete.PredictNothing, 98 "-t": complete.PredictAnything, 99 "-verbose": complete.PredictNothing, 100 }) 101 } 102 103 func (c *NodeStatusCommand) AutocompleteArgs() complete.Predictor { 104 return complete.PredictFunc(func(a complete.Args) []string { 105 client, err := c.Meta.Client() 106 if err != nil { 107 return nil 108 } 109 110 resp, _, err := client.Search().PrefixSearch(a.Last, contexts.Nodes, nil) 111 if err != nil { 112 return []string{} 113 } 114 return resp.Matches[contexts.Nodes] 115 }) 116 } 117 118 func (c *NodeStatusCommand) Name() string { return "node-status" } 119 120 func (c *NodeStatusCommand) Run(args []string) int { 121 122 flags := c.Meta.FlagSet(c.Name(), FlagSetClient) 123 flags.Usage = func() { c.Ui.Output(c.Help()) } 124 flags.BoolVar(&c.short, "short", false, "") 125 flags.BoolVar(&c.verbose, "verbose", false, "") 126 flags.BoolVar(&c.list_allocs, "allocs", false, "") 127 flags.BoolVar(&c.self, "self", false, "") 128 flags.BoolVar(&c.stats, "stats", false, "") 129 flags.BoolVar(&c.json, "json", false, "") 130 flags.StringVar(&c.tmpl, "t", "", "") 131 132 if err := flags.Parse(args); err != nil { 133 return 1 134 } 135 136 // Check that we got either a single node or none 137 args = flags.Args() 138 if len(args) > 1 { 139 c.Ui.Error("This command takes either one or no arguments") 140 c.Ui.Error(commandErrorText(c)) 141 return 1 142 } 143 144 // Truncate the id unless full length is requested 145 c.length = shortId 146 if c.verbose { 147 c.length = fullId 148 } 149 150 // Get the HTTP client 151 client, err := c.Meta.Client() 152 if err != nil { 153 c.Ui.Error(fmt.Sprintf("Error initializing client: %s", err)) 154 return 1 155 } 156 157 // Use list mode if no node name was provided 158 if len(args) == 0 && !c.self { 159 160 // Query the node info 161 nodes, _, err := client.Nodes().List(nil) 162 if err != nil { 163 c.Ui.Error(fmt.Sprintf("Error querying node status: %s", err)) 164 return 1 165 } 166 167 // If output format is specified, format and output the node data list 168 if c.json || len(c.tmpl) > 0 { 169 out, err := Format(c.json, c.tmpl, nodes) 170 if err != nil { 171 c.Ui.Error(err.Error()) 172 return 1 173 } 174 175 c.Ui.Output(out) 176 return 0 177 } 178 179 // Return nothing if no nodes found 180 if len(nodes) == 0 { 181 return 0 182 } 183 184 // Format the nodes list 185 out := make([]string, len(nodes)+1) 186 187 out[0] = "ID|DC|Name|Class|" 188 189 if c.verbose { 190 out[0] += "Address|Version|" 191 } 192 193 out[0] += "Drain|Eligibility|Status" 194 195 if c.list_allocs { 196 out[0] += "|Running Allocs" 197 } 198 199 for i, node := range nodes { 200 out[i+1] = fmt.Sprintf("%s|%s|%s|%s", 201 limit(node.ID, c.length), 202 node.Datacenter, 203 node.Name, 204 node.NodeClass) 205 if c.verbose { 206 out[i+1] += fmt.Sprintf("|%s|%s", 207 node.Address, node.Version) 208 } 209 out[i+1] += fmt.Sprintf("|%v|%s|%s", 210 node.Drain, 211 node.SchedulingEligibility, 212 node.Status) 213 214 if c.list_allocs { 215 numAllocs, err := getRunningAllocs(client, node.ID) 216 if err != nil { 217 c.Ui.Error(fmt.Sprintf("Error querying node allocations: %s", err)) 218 return 1 219 } 220 out[i+1] += fmt.Sprintf("|%v", 221 len(numAllocs)) 222 } 223 } 224 225 // Dump the output 226 c.Ui.Output(formatList(out)) 227 return 0 228 } 229 230 // Query the specific node 231 var nodeID string 232 if !c.self { 233 nodeID = args[0] 234 } else { 235 var err error 236 if nodeID, err = getLocalNodeID(client); err != nil { 237 c.Ui.Error(err.Error()) 238 return 1 239 } 240 } 241 if len(nodeID) == 1 { 242 c.Ui.Error("Identifier must contain at least two characters.") 243 return 1 244 } 245 246 nodeID = sanitizeUUIDPrefix(nodeID) 247 nodes, _, err := client.Nodes().PrefixList(nodeID) 248 if err != nil { 249 c.Ui.Error(fmt.Sprintf("Error querying node info: %s", err)) 250 return 1 251 } 252 // Return error if no nodes are found 253 if len(nodes) == 0 { 254 c.Ui.Error(fmt.Sprintf("No node(s) with prefix %q found", nodeID)) 255 return 1 256 } 257 if len(nodes) > 1 { 258 // Dump the output 259 c.Ui.Error(fmt.Sprintf("Prefix matched multiple nodes\n\n%s", 260 formatNodeStubList(nodes, c.verbose))) 261 return 1 262 } 263 264 // Prefix lookup matched a single node 265 node, _, err := client.Nodes().Info(nodes[0].ID, nil) 266 if err != nil { 267 c.Ui.Error(fmt.Sprintf("Error querying node info: %s", err)) 268 return 1 269 } 270 271 // If output format is specified, format and output the data 272 if c.json || len(c.tmpl) > 0 { 273 out, err := Format(c.json, c.tmpl, node) 274 if err != nil { 275 c.Ui.Error(err.Error()) 276 return 1 277 } 278 279 c.Ui.Output(out) 280 return 0 281 } 282 283 return c.formatNode(client, node) 284 } 285 286 func nodeDrivers(n *api.Node) []string { 287 var drivers []string 288 for k, v := range n.Attributes { 289 // driver.docker = 1 290 parts := strings.Split(k, ".") 291 if len(parts) != 2 { 292 continue 293 } else if parts[0] != "driver" { 294 continue 295 } else if v != "1" { 296 continue 297 } 298 299 drivers = append(drivers, parts[1]) 300 } 301 302 sort.Strings(drivers) 303 return drivers 304 } 305 306 func nodeCSIControllerNames(n *api.Node) []string { 307 var names []string 308 for name := range n.CSIControllerPlugins { 309 names = append(names, name) 310 } 311 sort.Strings(names) 312 return names 313 } 314 315 func nodeCSINodeNames(n *api.Node) []string { 316 var names []string 317 for name := range n.CSINodePlugins { 318 names = append(names, name) 319 } 320 sort.Strings(names) 321 return names 322 } 323 324 func nodeCSIVolumeNames(n *api.Node, allocs []*api.Allocation) []string { 325 var names []string 326 for _, alloc := range allocs { 327 tg := alloc.GetTaskGroup() 328 if tg == nil || len(tg.Volumes) == 0 { 329 continue 330 } 331 332 for _, v := range tg.Volumes { 333 if v.Type == structs.VolumeTypeCSI { 334 names = append(names, v.Name) 335 } 336 } 337 } 338 sort.Strings(names) 339 return names 340 } 341 342 func nodeVolumeNames(n *api.Node) []string { 343 var volumes []string 344 for name := range n.HostVolumes { 345 volumes = append(volumes, name) 346 } 347 348 sort.Strings(volumes) 349 return volumes 350 } 351 352 func formatDrain(n *api.Node) string { 353 if n.DrainStrategy != nil { 354 b := new(strings.Builder) 355 b.WriteString("true") 356 if n.DrainStrategy.DrainSpec.Deadline.Nanoseconds() < 0 { 357 b.WriteString("; force drain") 358 } else if n.DrainStrategy.ForceDeadline.IsZero() { 359 b.WriteString("; no deadline") 360 } else { 361 fmt.Fprintf(b, "; %s deadline", formatTime(n.DrainStrategy.ForceDeadline)) 362 } 363 364 if n.DrainStrategy.IgnoreSystemJobs { 365 b.WriteString("; ignoring system jobs") 366 } 367 return b.String() 368 } 369 370 return strconv.FormatBool(n.Drain) 371 } 372 373 func (c *NodeStatusCommand) formatNode(client *api.Client, node *api.Node) int { 374 // Make one API call for allocations 375 nodeAllocs, _, err := client.Nodes().Allocations(node.ID, nil) 376 if err != nil { 377 c.Ui.Error(fmt.Sprintf("Error querying node allocations: %s", err)) 378 return 1 379 } 380 381 var runningAllocs []*api.Allocation 382 for _, alloc := range nodeAllocs { 383 if alloc.ClientStatus == "running" { 384 runningAllocs = append(runningAllocs, alloc) 385 } 386 } 387 388 // Format the header output 389 basic := []string{ 390 fmt.Sprintf("ID|%s", node.ID), 391 fmt.Sprintf("Name|%s", node.Name), 392 fmt.Sprintf("Class|%s", node.NodeClass), 393 fmt.Sprintf("DC|%s", node.Datacenter), 394 fmt.Sprintf("Drain|%v", formatDrain(node)), 395 fmt.Sprintf("Eligibility|%s", node.SchedulingEligibility), 396 fmt.Sprintf("Status|%s", node.Status), 397 fmt.Sprintf("CSI Controllers|%s", strings.Join(nodeCSIControllerNames(node), ",")), 398 fmt.Sprintf("CSI Drivers|%s", strings.Join(nodeCSINodeNames(node), ",")), 399 } 400 401 if c.short { 402 basic = append(basic, fmt.Sprintf("Host Volumes|%s", strings.Join(nodeVolumeNames(node), ","))) 403 basic = append(basic, fmt.Sprintf("CSI Volumes|%s", strings.Join(nodeCSIVolumeNames(node, runningAllocs), ","))) 404 basic = append(basic, fmt.Sprintf("Drivers|%s", strings.Join(nodeDrivers(node), ","))) 405 c.Ui.Output(c.Colorize().Color(formatKV(basic))) 406 407 // Output alloc info 408 if err := c.outputAllocInfo(node, nodeAllocs); err != nil { 409 c.Ui.Error(fmt.Sprintf("%s", err)) 410 return 1 411 } 412 413 return 0 414 } 415 416 // Get the host stats 417 hostStats, nodeStatsErr := client.Nodes().Stats(node.ID, nil) 418 if nodeStatsErr != nil { 419 c.Ui.Output("") 420 c.Ui.Error(fmt.Sprintf("error fetching node stats: %v", nodeStatsErr)) 421 } 422 if hostStats != nil { 423 uptime := time.Duration(hostStats.Uptime * uint64(time.Second)) 424 basic = append(basic, fmt.Sprintf("Uptime|%s", uptime.String())) 425 } 426 427 // When we're not running in verbose mode, then also include host volumes and 428 // driver info in the basic output 429 if !c.verbose { 430 basic = append(basic, fmt.Sprintf("Host Volumes|%s", strings.Join(nodeVolumeNames(node), ","))) 431 basic = append(basic, fmt.Sprintf("CSI Volumes|%s", strings.Join(nodeCSIVolumeNames(node, runningAllocs), ","))) 432 driverStatus := fmt.Sprintf("Driver Status| %s", c.outputTruncatedNodeDriverInfo(node)) 433 basic = append(basic, driverStatus) 434 } 435 436 // Output the basic info 437 c.Ui.Output(c.Colorize().Color(formatKV(basic))) 438 439 // If we're running in verbose mode, include full host volume and driver info 440 if c.verbose { 441 c.outputNodeVolumeInfo(node) 442 c.outputNodeCSIVolumeInfo(client, node, runningAllocs) 443 c.outputNodeDriverInfo(node) 444 } 445 446 // Emit node events 447 c.outputNodeStatusEvents(node) 448 449 // Get list of running allocations on the node 450 allocatedResources := getAllocatedResources(client, runningAllocs, node) 451 c.Ui.Output(c.Colorize().Color("\n[bold]Allocated Resources[reset]")) 452 c.Ui.Output(formatList(allocatedResources)) 453 454 actualResources, err := getActualResources(client, runningAllocs, node) 455 if err == nil { 456 c.Ui.Output(c.Colorize().Color("\n[bold]Allocation Resource Utilization[reset]")) 457 c.Ui.Output(formatList(actualResources)) 458 } 459 460 hostResources, err := getHostResources(hostStats, node) 461 if err != nil { 462 c.Ui.Output("") 463 c.Ui.Error(fmt.Sprintf("error fetching node stats: %v", err)) 464 } 465 if err == nil { 466 c.Ui.Output(c.Colorize().Color("\n[bold]Host Resource Utilization[reset]")) 467 c.Ui.Output(formatList(hostResources)) 468 } 469 470 if err == nil && node.NodeResources != nil && len(node.NodeResources.Devices) > 0 { 471 c.Ui.Output(c.Colorize().Color("\n[bold]Device Resource Utilization[reset]")) 472 c.Ui.Output(formatList(getDeviceResourcesForNode(hostStats.DeviceStats, node))) 473 } 474 if hostStats != nil && c.stats { 475 c.Ui.Output(c.Colorize().Color("\n[bold]CPU Stats[reset]")) 476 c.printCpuStats(hostStats) 477 c.Ui.Output(c.Colorize().Color("\n[bold]Memory Stats[reset]")) 478 c.printMemoryStats(hostStats) 479 c.Ui.Output(c.Colorize().Color("\n[bold]Disk Stats[reset]")) 480 c.printDiskStats(hostStats) 481 if len(hostStats.DeviceStats) > 0 { 482 c.Ui.Output(c.Colorize().Color("\n[bold]Device Stats[reset]")) 483 printDeviceStats(c.Ui, hostStats.DeviceStats) 484 } 485 } 486 487 if err := c.outputAllocInfo(node, nodeAllocs); err != nil { 488 c.Ui.Error(fmt.Sprintf("%s", err)) 489 return 1 490 } 491 492 return 0 493 } 494 495 func (c *NodeStatusCommand) outputAllocInfo(node *api.Node, nodeAllocs []*api.Allocation) error { 496 c.Ui.Output(c.Colorize().Color("\n[bold]Allocations[reset]")) 497 c.Ui.Output(formatAllocList(nodeAllocs, c.verbose, c.length)) 498 499 if c.verbose { 500 c.formatAttributes(node) 501 c.formatDeviceAttributes(node) 502 c.formatMeta(node) 503 } 504 505 return nil 506 } 507 508 func (c *NodeStatusCommand) outputTruncatedNodeDriverInfo(node *api.Node) string { 509 drivers := make([]string, 0, len(node.Drivers)) 510 511 for driverName, driverInfo := range node.Drivers { 512 if !driverInfo.Detected { 513 continue 514 } 515 516 if !driverInfo.Healthy { 517 drivers = append(drivers, fmt.Sprintf("%s (unhealthy)", driverName)) 518 } else { 519 drivers = append(drivers, driverName) 520 } 521 } 522 sort.Strings(drivers) 523 return strings.Trim(strings.Join(drivers, ","), ", ") 524 } 525 526 func (c *NodeStatusCommand) outputNodeVolumeInfo(node *api.Node) { 527 528 names := make([]string, 0, len(node.HostVolumes)) 529 for name := range node.HostVolumes { 530 names = append(names, name) 531 } 532 sort.Strings(names) 533 534 output := make([]string, 0, len(names)+1) 535 output = append(output, "Name|ReadOnly|Source") 536 537 if len(names) > 0 { 538 c.Ui.Output(c.Colorize().Color("\n[bold]Host Volumes")) 539 for _, volName := range names { 540 info := node.HostVolumes[volName] 541 output = append(output, fmt.Sprintf("%s|%v|%s", volName, info.ReadOnly, info.Path)) 542 } 543 c.Ui.Output(formatList(output)) 544 } 545 } 546 547 func (c *NodeStatusCommand) outputNodeCSIVolumeInfo(client *api.Client, node *api.Node, runningAllocs []*api.Allocation) { 548 549 // Duplicate nodeCSIVolumeNames to sort by name but also index volume names to ids 550 var names []string 551 requests := map[string]*api.VolumeRequest{} 552 for _, alloc := range runningAllocs { 553 tg := alloc.GetTaskGroup() 554 if tg == nil || len(tg.Volumes) == 0 { 555 continue 556 } 557 558 for _, v := range tg.Volumes { 559 if v.Type == structs.VolumeTypeCSI { 560 names = append(names, v.Name) 561 requests[v.Source] = v 562 } 563 } 564 } 565 if len(names) == 0 { 566 return 567 } 568 sort.Strings(names) 569 570 // Fetch the volume objects with current status 571 // Ignore an error, all we're going to do is omit the volumes 572 volumes := map[string]*api.CSIVolumeListStub{} 573 vs, _ := client.Nodes().CSIVolumes(node.ID, nil) 574 for _, v := range vs { 575 n, ok := requests[v.ID] 576 if ok { 577 volumes[n.Name] = v 578 } 579 } 580 581 if len(names) > 0 { 582 c.Ui.Output(c.Colorize().Color("\n[bold]CSI Volumes")) 583 584 // Output the volumes in name order 585 output := make([]string, 0, len(names)+1) 586 output = append(output, "ID|Name|Plugin ID|Schedulable|Provider|Access Mode") 587 for _, name := range names { 588 v, ok := volumes[name] 589 if ok { 590 output = append(output, fmt.Sprintf( 591 "%s|%s|%s|%t|%s|%s", 592 v.ID, 593 name, 594 v.PluginID, 595 v.Schedulable, 596 v.Provider, 597 v.AccessMode, 598 )) 599 } 600 } 601 602 c.Ui.Output(formatList(output)) 603 } 604 } 605 606 func (c *NodeStatusCommand) outputNodeDriverInfo(node *api.Node) { 607 c.Ui.Output(c.Colorize().Color("\n[bold]Drivers")) 608 609 size := len(node.Drivers) 610 nodeDrivers := make([]string, 0, size+1) 611 612 nodeDrivers = append(nodeDrivers, "Driver|Detected|Healthy|Message|Time") 613 614 drivers := make([]string, 0, len(node.Drivers)) 615 for driver := range node.Drivers { 616 drivers = append(drivers, driver) 617 } 618 sort.Strings(drivers) 619 620 for _, driver := range drivers { 621 info := node.Drivers[driver] 622 timestamp := formatTime(info.UpdateTime) 623 nodeDrivers = append(nodeDrivers, fmt.Sprintf("%s|%v|%v|%s|%s", driver, info.Detected, info.Healthy, info.HealthDescription, timestamp)) 624 } 625 c.Ui.Output(formatList(nodeDrivers)) 626 } 627 628 func (c *NodeStatusCommand) outputNodeStatusEvents(node *api.Node) { 629 c.Ui.Output(c.Colorize().Color("\n[bold]Node Events")) 630 c.outputNodeEvent(node.Events) 631 } 632 633 func (c *NodeStatusCommand) outputNodeEvent(events []*api.NodeEvent) { 634 size := len(events) 635 nodeEvents := make([]string, size+1) 636 if c.verbose { 637 nodeEvents[0] = "Time|Subsystem|Message|Details" 638 } else { 639 nodeEvents[0] = "Time|Subsystem|Message" 640 } 641 642 for i, event := range events { 643 timestamp := formatTime(event.Timestamp) 644 subsystem := formatEventSubsystem(event.Subsystem, event.Details["driver"]) 645 msg := event.Message 646 if c.verbose { 647 details := formatEventDetails(event.Details) 648 nodeEvents[size-i] = fmt.Sprintf("%s|%s|%s|%s", timestamp, subsystem, msg, details) 649 } else { 650 nodeEvents[size-i] = fmt.Sprintf("%s|%s|%s", timestamp, subsystem, msg) 651 } 652 } 653 c.Ui.Output(formatList(nodeEvents)) 654 } 655 656 func formatEventSubsystem(subsystem, driverName string) string { 657 if driverName == "" { 658 return subsystem 659 } 660 661 // If this event is for a driver, append the driver name to make the message 662 // clearer 663 return fmt.Sprintf("Driver: %s", driverName) 664 } 665 666 func formatEventDetails(details map[string]string) string { 667 output := make([]string, 0, len(details)) 668 for k, v := range details { 669 output = append(output, fmt.Sprintf("%s: %s", k, v)) 670 } 671 return strings.Join(output, ", ") 672 } 673 674 func (c *NodeStatusCommand) formatAttributes(node *api.Node) { 675 // Print the attributes 676 keys := make([]string, len(node.Attributes)) 677 for k := range node.Attributes { 678 keys = append(keys, k) 679 } 680 sort.Strings(keys) 681 682 var attributes []string 683 for _, k := range keys { 684 if k != "" { 685 attributes = append(attributes, fmt.Sprintf("%s|%s", k, node.Attributes[k])) 686 } 687 } 688 c.Ui.Output(c.Colorize().Color("\n[bold]Attributes[reset]")) 689 c.Ui.Output(formatKV(attributes)) 690 } 691 692 func (c *NodeStatusCommand) formatDeviceAttributes(node *api.Node) { 693 if node.NodeResources == nil { 694 return 695 } 696 devices := node.NodeResources.Devices 697 if len(devices) == 0 { 698 return 699 } 700 701 sort.Slice(devices, func(i, j int) bool { 702 return devices[i].ID() < devices[j].ID() 703 }) 704 705 first := true 706 for _, d := range devices { 707 if len(d.Attributes) == 0 { 708 continue 709 } 710 711 if first { 712 c.Ui.Output("\nDevice Group Attributes") 713 first = false 714 } else { 715 c.Ui.Output("") 716 } 717 c.Ui.Output(formatKV(getDeviceAttributes(d))) 718 } 719 } 720 721 func (c *NodeStatusCommand) formatMeta(node *api.Node) { 722 // Print the meta 723 keys := make([]string, 0, len(node.Meta)) 724 for k := range node.Meta { 725 keys = append(keys, k) 726 } 727 sort.Strings(keys) 728 729 var meta []string 730 for _, k := range keys { 731 if k != "" { 732 meta = append(meta, fmt.Sprintf("%s|%s", k, node.Meta[k])) 733 } 734 } 735 c.Ui.Output(c.Colorize().Color("\n[bold]Meta[reset]")) 736 c.Ui.Output(formatKV(meta)) 737 } 738 739 func (c *NodeStatusCommand) printCpuStats(hostStats *api.HostStats) { 740 l := len(hostStats.CPU) 741 for i, cpuStat := range hostStats.CPU { 742 cpuStatsAttr := make([]string, 4) 743 cpuStatsAttr[0] = fmt.Sprintf("CPU|%v", cpuStat.CPU) 744 cpuStatsAttr[1] = fmt.Sprintf("User|%v%%", humanize.FormatFloat(floatFormat, cpuStat.User)) 745 cpuStatsAttr[2] = fmt.Sprintf("System|%v%%", humanize.FormatFloat(floatFormat, cpuStat.System)) 746 cpuStatsAttr[3] = fmt.Sprintf("Idle|%v%%", humanize.FormatFloat(floatFormat, cpuStat.Idle)) 747 c.Ui.Output(formatKV(cpuStatsAttr)) 748 if i+1 < l { 749 c.Ui.Output("") 750 } 751 } 752 } 753 754 func (c *NodeStatusCommand) printMemoryStats(hostStats *api.HostStats) { 755 memoryStat := hostStats.Memory 756 memStatsAttr := make([]string, 4) 757 memStatsAttr[0] = fmt.Sprintf("Total|%v", humanize.IBytes(memoryStat.Total)) 758 memStatsAttr[1] = fmt.Sprintf("Available|%v", humanize.IBytes(memoryStat.Available)) 759 memStatsAttr[2] = fmt.Sprintf("Used|%v", humanize.IBytes(memoryStat.Used)) 760 memStatsAttr[3] = fmt.Sprintf("Free|%v", humanize.IBytes(memoryStat.Free)) 761 c.Ui.Output(formatKV(memStatsAttr)) 762 } 763 764 func (c *NodeStatusCommand) printDiskStats(hostStats *api.HostStats) { 765 l := len(hostStats.DiskStats) 766 for i, diskStat := range hostStats.DiskStats { 767 diskStatsAttr := make([]string, 7) 768 diskStatsAttr[0] = fmt.Sprintf("Device|%s", diskStat.Device) 769 diskStatsAttr[1] = fmt.Sprintf("MountPoint|%s", diskStat.Mountpoint) 770 diskStatsAttr[2] = fmt.Sprintf("Size|%s", humanize.IBytes(diskStat.Size)) 771 diskStatsAttr[3] = fmt.Sprintf("Used|%s", humanize.IBytes(diskStat.Used)) 772 diskStatsAttr[4] = fmt.Sprintf("Available|%s", humanize.IBytes(diskStat.Available)) 773 diskStatsAttr[5] = fmt.Sprintf("Used Percent|%v%%", humanize.FormatFloat(floatFormat, diskStat.UsedPercent)) 774 diskStatsAttr[6] = fmt.Sprintf("Inodes Percent|%v%%", humanize.FormatFloat(floatFormat, diskStat.InodesUsedPercent)) 775 c.Ui.Output(formatKV(diskStatsAttr)) 776 if i+1 < l { 777 c.Ui.Output("") 778 } 779 } 780 } 781 782 // getRunningAllocs returns a slice of allocation id's running on the node 783 func getRunningAllocs(client *api.Client, nodeID string) ([]*api.Allocation, error) { 784 var allocs []*api.Allocation 785 786 // Query the node allocations 787 nodeAllocs, _, err := client.Nodes().Allocations(nodeID, nil) 788 // Filter list to only running allocations 789 for _, alloc := range nodeAllocs { 790 if alloc.ClientStatus == "running" { 791 allocs = append(allocs, alloc) 792 } 793 } 794 return allocs, err 795 } 796 797 // getAllocatedResources returns the resource usage of the node. 798 func getAllocatedResources(client *api.Client, runningAllocs []*api.Allocation, node *api.Node) []string { 799 // Compute the total 800 total := computeNodeTotalResources(node) 801 802 // Get Resources 803 var cpu, mem, disk int 804 for _, alloc := range runningAllocs { 805 cpu += *alloc.Resources.CPU 806 mem += *alloc.Resources.MemoryMB 807 disk += *alloc.Resources.DiskMB 808 } 809 810 resources := make([]string, 2) 811 resources[0] = "CPU|Memory|Disk" 812 resources[1] = fmt.Sprintf("%d/%d MHz|%s/%s|%s/%s", 813 cpu, 814 *total.CPU, 815 humanize.IBytes(uint64(mem*bytesPerMegabyte)), 816 humanize.IBytes(uint64(*total.MemoryMB*bytesPerMegabyte)), 817 humanize.IBytes(uint64(disk*bytesPerMegabyte)), 818 humanize.IBytes(uint64(*total.DiskMB*bytesPerMegabyte))) 819 820 return resources 821 } 822 823 // computeNodeTotalResources returns the total allocatable resources (resources 824 // minus reserved) 825 func computeNodeTotalResources(node *api.Node) api.Resources { 826 total := api.Resources{} 827 828 r := node.Resources 829 res := node.Reserved 830 if res == nil { 831 res = &api.Resources{} 832 } 833 total.CPU = helper.IntToPtr(*r.CPU - *res.CPU) 834 total.MemoryMB = helper.IntToPtr(*r.MemoryMB - *res.MemoryMB) 835 total.DiskMB = helper.IntToPtr(*r.DiskMB - *res.DiskMB) 836 return total 837 } 838 839 // getActualResources returns the actual resource usage of the allocations. 840 func getActualResources(client *api.Client, runningAllocs []*api.Allocation, node *api.Node) ([]string, error) { 841 // Compute the total 842 total := computeNodeTotalResources(node) 843 844 // Get Resources 845 var cpu float64 846 var mem uint64 847 for _, alloc := range runningAllocs { 848 // Make the call to the client to get the actual usage. 849 stats, err := client.Allocations().Stats(alloc, nil) 850 if err != nil { 851 return nil, err 852 } 853 854 cpu += stats.ResourceUsage.CpuStats.TotalTicks 855 mem += stats.ResourceUsage.MemoryStats.RSS 856 } 857 858 resources := make([]string, 2) 859 resources[0] = "CPU|Memory" 860 resources[1] = fmt.Sprintf("%v/%d MHz|%v/%v", 861 math.Floor(cpu), 862 *total.CPU, 863 humanize.IBytes(mem), 864 humanize.IBytes(uint64(*total.MemoryMB*bytesPerMegabyte))) 865 866 return resources, nil 867 } 868 869 // getHostResources returns the actual resource usage of the node. 870 func getHostResources(hostStats *api.HostStats, node *api.Node) ([]string, error) { 871 if hostStats == nil { 872 return nil, fmt.Errorf("actual resource usage not present") 873 } 874 var resources []string 875 876 // calculate disk usage 877 storageDevice := node.Attributes["unique.storage.volume"] 878 var diskUsed, diskSize uint64 879 var physical bool 880 for _, disk := range hostStats.DiskStats { 881 if disk.Device == storageDevice { 882 diskUsed = disk.Used 883 diskSize = disk.Size 884 physical = true 885 } 886 } 887 888 resources = make([]string, 2) 889 resources[0] = "CPU|Memory|Disk" 890 if physical { 891 resources[1] = fmt.Sprintf("%v/%d MHz|%s/%s|%s/%s", 892 math.Floor(hostStats.CPUTicksConsumed), 893 *node.Resources.CPU, 894 humanize.IBytes(hostStats.Memory.Used), 895 humanize.IBytes(hostStats.Memory.Total), 896 humanize.IBytes(diskUsed), 897 humanize.IBytes(diskSize), 898 ) 899 } else { 900 // If non-physical device are used, output device name only, 901 // since nomad doesn't collect the stats data. 902 resources[1] = fmt.Sprintf("%v/%d MHz|%s/%s|(%s)", 903 math.Floor(hostStats.CPUTicksConsumed), 904 *node.Resources.CPU, 905 humanize.IBytes(hostStats.Memory.Used), 906 humanize.IBytes(hostStats.Memory.Total), 907 storageDevice, 908 ) 909 } 910 return resources, nil 911 } 912 913 // formatNodeStubList is used to return a table format of a list of node stubs. 914 func formatNodeStubList(nodes []*api.NodeListStub, verbose bool) string { 915 // Return error if no nodes are found 916 if len(nodes) == 0 { 917 return "" 918 } 919 // Truncate the id unless full length is requested 920 length := shortId 921 if verbose { 922 length = fullId 923 } 924 925 // Format the nodes list that matches the prefix so that the user 926 // can create a more specific request 927 out := make([]string, len(nodes)+1) 928 out[0] = "ID|DC|Name|Class|Drain|Eligibility|Status" 929 for i, node := range nodes { 930 out[i+1] = fmt.Sprintf("%s|%s|%s|%s|%v|%s|%s", 931 limit(node.ID, length), 932 node.Datacenter, 933 node.Name, 934 node.NodeClass, 935 node.Drain, 936 node.SchedulingEligibility, 937 node.Status) 938 } 939 940 return formatList(out) 941 }