github.com/aminovpavel/nomad@v0.11.8/command/node_status.go (about) 1 package command 2 3 import ( 4 "fmt" 5 "math" 6 "sort" 7 "strconv" 8 "strings" 9 "time" 10 11 humanize "github.com/dustin/go-humanize" 12 "github.com/hashicorp/nomad/api" 13 "github.com/hashicorp/nomad/api/contexts" 14 "github.com/hashicorp/nomad/helper" 15 "github.com/posener/complete" 16 ) 17 18 const ( 19 // floatFormat is a format string for formatting floats. 20 floatFormat = "#,###.##" 21 22 // bytesPerMegabyte is the number of bytes per MB 23 bytesPerMegabyte = 1024 * 1024 24 ) 25 26 type NodeStatusCommand struct { 27 Meta 28 length int 29 short bool 30 verbose bool 31 list_allocs bool 32 self bool 33 stats bool 34 json bool 35 tmpl string 36 } 37 38 func (c *NodeStatusCommand) Help() string { 39 helpText := ` 40 Usage: nomad node status [options] <node> 41 42 Display status information about a given node. The list of nodes 43 returned includes only nodes which jobs may be scheduled to, and 44 includes status and other high-level information. 45 46 If a node ID is passed, information for that specific node will be displayed, 47 including resource usage statistics. If no node ID's are passed, then a 48 short-hand list of all nodes will be displayed. The -self flag is useful to 49 quickly access the status of the local node. 50 51 General Options: 52 53 ` + generalOptionsUsage() + ` 54 55 Node Status Options: 56 57 -self 58 Query the status of the local node. 59 60 -stats 61 Display detailed resource usage statistics. 62 63 -allocs 64 Display a count of running allocations for each node. 65 66 -short 67 Display short output. Used only when a single node is being 68 queried, and drops verbose output about node allocations. 69 70 -verbose 71 Display full information. 72 73 -json 74 Output the node in its JSON format. 75 76 -t 77 Format and display node using a Go template. 78 ` 79 return strings.TrimSpace(helpText) 80 } 81 82 func (c *NodeStatusCommand) Synopsis() string { 83 return "Display status information about nodes" 84 } 85 86 func (c *NodeStatusCommand) AutocompleteFlags() complete.Flags { 87 return mergeAutocompleteFlags(c.Meta.AutocompleteFlags(FlagSetClient), 88 complete.Flags{ 89 "-allocs": complete.PredictNothing, 90 "-json": complete.PredictNothing, 91 "-self": complete.PredictNothing, 92 "-short": complete.PredictNothing, 93 "-stats": complete.PredictNothing, 94 "-t": complete.PredictAnything, 95 "-verbose": complete.PredictNothing, 96 }) 97 } 98 99 func (c *NodeStatusCommand) AutocompleteArgs() complete.Predictor { 100 return complete.PredictFunc(func(a complete.Args) []string { 101 client, err := c.Meta.Client() 102 if err != nil { 103 return nil 104 } 105 106 resp, _, err := client.Search().PrefixSearch(a.Last, contexts.Nodes, nil) 107 if err != nil { 108 return []string{} 109 } 110 return resp.Matches[contexts.Nodes] 111 }) 112 } 113 114 func (c *NodeStatusCommand) Name() string { return "node-status" } 115 116 func (c *NodeStatusCommand) Run(args []string) int { 117 118 flags := c.Meta.FlagSet(c.Name(), FlagSetClient) 119 flags.Usage = func() { c.Ui.Output(c.Help()) } 120 flags.BoolVar(&c.short, "short", false, "") 121 flags.BoolVar(&c.verbose, "verbose", false, "") 122 flags.BoolVar(&c.list_allocs, "allocs", false, "") 123 flags.BoolVar(&c.self, "self", false, "") 124 flags.BoolVar(&c.stats, "stats", false, "") 125 flags.BoolVar(&c.json, "json", false, "") 126 flags.StringVar(&c.tmpl, "t", "", "") 127 128 if err := flags.Parse(args); err != nil { 129 return 1 130 } 131 132 // Check that we got either a single node or none 133 args = flags.Args() 134 if len(args) > 1 { 135 c.Ui.Error("This command takes either one or no arguments") 136 c.Ui.Error(commandErrorText(c)) 137 return 1 138 } 139 140 // Truncate the id unless full length is requested 141 c.length = shortId 142 if c.verbose { 143 c.length = fullId 144 } 145 146 // Get the HTTP client 147 client, err := c.Meta.Client() 148 if err != nil { 149 c.Ui.Error(fmt.Sprintf("Error initializing client: %s", err)) 150 return 1 151 } 152 153 // Use list mode if no node name was provided 154 if len(args) == 0 && !c.self { 155 156 // Query the node info 157 nodes, _, err := client.Nodes().List(nil) 158 if err != nil { 159 c.Ui.Error(fmt.Sprintf("Error querying node status: %s", err)) 160 return 1 161 } 162 163 // If output format is specified, format and output the node data list 164 if c.json || len(c.tmpl) > 0 { 165 out, err := Format(c.json, c.tmpl, nodes) 166 if err != nil { 167 c.Ui.Error(err.Error()) 168 return 1 169 } 170 171 c.Ui.Output(out) 172 return 0 173 } 174 175 // Return nothing if no nodes found 176 if len(nodes) == 0 { 177 return 0 178 } 179 180 // Format the nodes list 181 out := make([]string, len(nodes)+1) 182 183 out[0] = "ID|DC|Name|Class|" 184 185 if c.verbose { 186 out[0] += "Address|Version|" 187 } 188 189 out[0] += "Drain|Eligibility|Status" 190 191 if c.list_allocs { 192 out[0] += "|Running Allocs" 193 } 194 195 for i, node := range nodes { 196 out[i+1] = fmt.Sprintf("%s|%s|%s|%s", 197 limit(node.ID, c.length), 198 node.Datacenter, 199 node.Name, 200 node.NodeClass) 201 if c.verbose { 202 out[i+1] += fmt.Sprintf("|%s|%s", 203 node.Address, node.Version) 204 } 205 out[i+1] += fmt.Sprintf("|%v|%s|%s", 206 node.Drain, 207 node.SchedulingEligibility, 208 node.Status) 209 210 if c.list_allocs { 211 numAllocs, err := getRunningAllocs(client, node.ID) 212 if err != nil { 213 c.Ui.Error(fmt.Sprintf("Error querying node allocations: %s", err)) 214 return 1 215 } 216 out[i+1] += fmt.Sprintf("|%v", 217 len(numAllocs)) 218 } 219 } 220 221 // Dump the output 222 c.Ui.Output(formatList(out)) 223 return 0 224 } 225 226 // Query the specific node 227 var nodeID string 228 if !c.self { 229 nodeID = args[0] 230 } else { 231 var err error 232 if nodeID, err = getLocalNodeID(client); err != nil { 233 c.Ui.Error(err.Error()) 234 return 1 235 } 236 } 237 if len(nodeID) == 1 { 238 c.Ui.Error(fmt.Sprintf("Identifier must contain at least two characters.")) 239 return 1 240 } 241 242 nodeID = sanitizeUUIDPrefix(nodeID) 243 nodes, _, err := client.Nodes().PrefixList(nodeID) 244 if err != nil { 245 c.Ui.Error(fmt.Sprintf("Error querying node info: %s", err)) 246 return 1 247 } 248 // Return error if no nodes are found 249 if len(nodes) == 0 { 250 c.Ui.Error(fmt.Sprintf("No node(s) with prefix %q found", nodeID)) 251 return 1 252 } 253 if len(nodes) > 1 { 254 // Dump the output 255 c.Ui.Error(fmt.Sprintf("Prefix matched multiple nodes\n\n%s", 256 formatNodeStubList(nodes, c.verbose))) 257 return 1 258 } 259 260 // Prefix lookup matched a single node 261 node, _, err := client.Nodes().Info(nodes[0].ID, nil) 262 if err != nil { 263 c.Ui.Error(fmt.Sprintf("Error querying node info: %s", err)) 264 return 1 265 } 266 267 // If output format is specified, format and output the data 268 if c.json || len(c.tmpl) > 0 { 269 out, err := Format(c.json, c.tmpl, node) 270 if err != nil { 271 c.Ui.Error(err.Error()) 272 return 1 273 } 274 275 c.Ui.Output(out) 276 return 0 277 } 278 279 return c.formatNode(client, node) 280 } 281 282 func nodeDrivers(n *api.Node) []string { 283 var drivers []string 284 for k, v := range n.Attributes { 285 // driver.docker = 1 286 parts := strings.Split(k, ".") 287 if len(parts) != 2 { 288 continue 289 } else if parts[0] != "driver" { 290 continue 291 } else if v != "1" { 292 continue 293 } 294 295 drivers = append(drivers, parts[1]) 296 } 297 298 sort.Strings(drivers) 299 return drivers 300 } 301 302 func nodeCSIControllerNames(n *api.Node) []string { 303 var names []string 304 for name := range n.CSIControllerPlugins { 305 names = append(names, name) 306 } 307 sort.Strings(names) 308 return names 309 } 310 311 func nodeCSINodeNames(n *api.Node) []string { 312 var names []string 313 for name := range n.CSINodePlugins { 314 names = append(names, name) 315 } 316 sort.Strings(names) 317 return names 318 } 319 320 func nodeCSIVolumeNames(n *api.Node, allocs []*api.Allocation) []string { 321 var names []string 322 for _, alloc := range allocs { 323 tg := alloc.GetTaskGroup() 324 if tg == nil || len(tg.Volumes) == 0 { 325 continue 326 } 327 328 for _, v := range tg.Volumes { 329 names = append(names, v.Name) 330 } 331 } 332 sort.Strings(names) 333 return names 334 } 335 336 func nodeVolumeNames(n *api.Node) []string { 337 var volumes []string 338 for name := range n.HostVolumes { 339 volumes = append(volumes, name) 340 } 341 342 sort.Strings(volumes) 343 return volumes 344 } 345 346 func formatDrain(n *api.Node) string { 347 if n.DrainStrategy != nil { 348 b := new(strings.Builder) 349 b.WriteString("true") 350 if n.DrainStrategy.DrainSpec.Deadline.Nanoseconds() < 0 { 351 b.WriteString("; force drain") 352 } else if n.DrainStrategy.ForceDeadline.IsZero() { 353 b.WriteString("; no deadline") 354 } else { 355 fmt.Fprintf(b, "; %s deadline", formatTime(n.DrainStrategy.ForceDeadline)) 356 } 357 358 if n.DrainStrategy.IgnoreSystemJobs { 359 b.WriteString("; ignoring system jobs") 360 } 361 return b.String() 362 } 363 364 return strconv.FormatBool(n.Drain) 365 } 366 367 func (c *NodeStatusCommand) formatNode(client *api.Client, node *api.Node) int { 368 // Make one API call for allocations 369 nodeAllocs, _, err := client.Nodes().Allocations(node.ID, nil) 370 if err != nil { 371 c.Ui.Error(fmt.Sprintf("Error querying node allocations: %s", err)) 372 return 1 373 } 374 375 var runningAllocs []*api.Allocation 376 for _, alloc := range nodeAllocs { 377 if alloc.ClientStatus == "running" { 378 runningAllocs = append(runningAllocs, alloc) 379 } 380 } 381 382 // Format the header output 383 basic := []string{ 384 fmt.Sprintf("ID|%s", node.ID), 385 fmt.Sprintf("Name|%s", node.Name), 386 fmt.Sprintf("Class|%s", node.NodeClass), 387 fmt.Sprintf("DC|%s", node.Datacenter), 388 fmt.Sprintf("Drain|%v", formatDrain(node)), 389 fmt.Sprintf("Eligibility|%s", node.SchedulingEligibility), 390 fmt.Sprintf("Status|%s", node.Status), 391 fmt.Sprintf("CSI Controllers|%s", strings.Join(nodeCSIControllerNames(node), ",")), 392 fmt.Sprintf("CSI Drivers|%s", strings.Join(nodeCSINodeNames(node), ",")), 393 } 394 395 if c.short { 396 basic = append(basic, fmt.Sprintf("Host Volumes|%s", strings.Join(nodeVolumeNames(node), ","))) 397 basic = append(basic, fmt.Sprintf("CSI Volumes|%s", strings.Join(nodeCSIVolumeNames(node, runningAllocs), ","))) 398 basic = append(basic, fmt.Sprintf("Drivers|%s", strings.Join(nodeDrivers(node), ","))) 399 c.Ui.Output(c.Colorize().Color(formatKV(basic))) 400 401 // Output alloc info 402 if err := c.outputAllocInfo(node, nodeAllocs); err != nil { 403 c.Ui.Error(fmt.Sprintf("%s", err)) 404 return 1 405 } 406 407 return 0 408 } 409 410 // Get the host stats 411 hostStats, nodeStatsErr := client.Nodes().Stats(node.ID, nil) 412 if nodeStatsErr != nil { 413 c.Ui.Output("") 414 c.Ui.Error(fmt.Sprintf("error fetching node stats: %v", nodeStatsErr)) 415 } 416 if hostStats != nil { 417 uptime := time.Duration(hostStats.Uptime * uint64(time.Second)) 418 basic = append(basic, fmt.Sprintf("Uptime|%s", uptime.String())) 419 } 420 421 // When we're not running in verbose mode, then also include host volumes and 422 // driver info in the basic output 423 if !c.verbose { 424 basic = append(basic, fmt.Sprintf("Host Volumes|%s", strings.Join(nodeVolumeNames(node), ","))) 425 basic = append(basic, fmt.Sprintf("CSI Volumes|%s", strings.Join(nodeCSIVolumeNames(node, runningAllocs), ","))) 426 driverStatus := fmt.Sprintf("Driver Status| %s", c.outputTruncatedNodeDriverInfo(node)) 427 basic = append(basic, driverStatus) 428 } 429 430 // Output the basic info 431 c.Ui.Output(c.Colorize().Color(formatKV(basic))) 432 433 // If we're running in verbose mode, include full host volume and driver info 434 if c.verbose { 435 c.outputNodeVolumeInfo(node) 436 c.outputNodeCSIVolumeInfo(client, node, runningAllocs) 437 c.outputNodeDriverInfo(node) 438 } 439 440 // Emit node events 441 c.outputNodeStatusEvents(node) 442 443 // Get list of running allocations on the node 444 allocatedResources := getAllocatedResources(client, runningAllocs, node) 445 c.Ui.Output(c.Colorize().Color("\n[bold]Allocated Resources[reset]")) 446 c.Ui.Output(formatList(allocatedResources)) 447 448 actualResources, err := getActualResources(client, runningAllocs, node) 449 if err == nil { 450 c.Ui.Output(c.Colorize().Color("\n[bold]Allocation Resource Utilization[reset]")) 451 c.Ui.Output(formatList(actualResources)) 452 } 453 454 hostResources, err := getHostResources(hostStats, node) 455 if err != nil { 456 c.Ui.Output("") 457 c.Ui.Error(fmt.Sprintf("error fetching node stats: %v", err)) 458 } 459 if err == nil { 460 c.Ui.Output(c.Colorize().Color("\n[bold]Host Resource Utilization[reset]")) 461 c.Ui.Output(formatList(hostResources)) 462 } 463 464 if err == nil && node.NodeResources != nil && len(node.NodeResources.Devices) > 0 { 465 c.Ui.Output(c.Colorize().Color("\n[bold]Device Resource Utilization[reset]")) 466 c.Ui.Output(formatList(getDeviceResourcesForNode(hostStats.DeviceStats, node))) 467 } 468 if hostStats != nil && c.stats { 469 c.Ui.Output(c.Colorize().Color("\n[bold]CPU Stats[reset]")) 470 c.printCpuStats(hostStats) 471 c.Ui.Output(c.Colorize().Color("\n[bold]Memory Stats[reset]")) 472 c.printMemoryStats(hostStats) 473 c.Ui.Output(c.Colorize().Color("\n[bold]Disk Stats[reset]")) 474 c.printDiskStats(hostStats) 475 if len(hostStats.DeviceStats) > 0 { 476 c.Ui.Output(c.Colorize().Color("\n[bold]Device Stats[reset]")) 477 printDeviceStats(c.Ui, hostStats.DeviceStats) 478 } 479 } 480 481 if err := c.outputAllocInfo(node, nodeAllocs); err != nil { 482 c.Ui.Error(fmt.Sprintf("%s", err)) 483 return 1 484 } 485 486 return 0 487 } 488 489 func (c *NodeStatusCommand) outputAllocInfo(node *api.Node, nodeAllocs []*api.Allocation) error { 490 c.Ui.Output(c.Colorize().Color("\n[bold]Allocations[reset]")) 491 c.Ui.Output(formatAllocList(nodeAllocs, c.verbose, c.length)) 492 493 if c.verbose { 494 c.formatAttributes(node) 495 c.formatDeviceAttributes(node) 496 c.formatMeta(node) 497 } 498 499 return nil 500 } 501 502 func (c *NodeStatusCommand) outputTruncatedNodeDriverInfo(node *api.Node) string { 503 drivers := make([]string, 0, len(node.Drivers)) 504 505 for driverName, driverInfo := range node.Drivers { 506 if !driverInfo.Detected { 507 continue 508 } 509 510 if !driverInfo.Healthy { 511 drivers = append(drivers, fmt.Sprintf("%s (unhealthy)", driverName)) 512 } else { 513 drivers = append(drivers, driverName) 514 } 515 } 516 sort.Strings(drivers) 517 return strings.Trim(strings.Join(drivers, ","), ", ") 518 } 519 520 func (c *NodeStatusCommand) outputNodeVolumeInfo(node *api.Node) { 521 c.Ui.Output(c.Colorize().Color("\n[bold]Host Volumes")) 522 523 names := make([]string, 0, len(node.HostVolumes)) 524 for name := range node.HostVolumes { 525 names = append(names, name) 526 } 527 sort.Strings(names) 528 529 output := make([]string, 0, len(names)+1) 530 output = append(output, "Name|ReadOnly|Source") 531 532 for _, volName := range names { 533 info := node.HostVolumes[volName] 534 output = append(output, fmt.Sprintf("%s|%v|%s", volName, info.ReadOnly, info.Path)) 535 } 536 c.Ui.Output(formatList(output)) 537 } 538 539 func (c *NodeStatusCommand) outputNodeCSIVolumeInfo(client *api.Client, node *api.Node, runningAllocs []*api.Allocation) { 540 c.Ui.Output(c.Colorize().Color("\n[bold]CSI Volumes")) 541 542 // Duplicate nodeCSIVolumeNames to sort by name but also index volume names to ids 543 var names []string 544 requests := map[string]*api.VolumeRequest{} 545 for _, alloc := range runningAllocs { 546 tg := alloc.GetTaskGroup() 547 if tg == nil || len(tg.Volumes) == 0 { 548 continue 549 } 550 551 for _, v := range tg.Volumes { 552 names = append(names, v.Name) 553 requests[v.Source] = v 554 } 555 } 556 if len(names) == 0 { 557 return 558 } 559 sort.Strings(names) 560 561 // Fetch the volume objects with current status 562 // Ignore an error, all we're going to do is omit the volumes 563 volumes := map[string]*api.CSIVolumeListStub{} 564 vs, _ := client.Nodes().CSIVolumes(node.ID, nil) 565 for _, v := range vs { 566 n := requests[v.ID].Name 567 volumes[n] = v 568 } 569 570 // Output the volumes in name order 571 output := make([]string, 0, len(names)+1) 572 output = append(output, "ID|Name|Plugin ID|Schedulable|Provider|Access Mode") 573 for _, name := range names { 574 v := volumes[name] 575 output = append(output, fmt.Sprintf( 576 "%s|%s|%s|%t|%s|%s", 577 v.ID, 578 name, 579 v.PluginID, 580 v.Schedulable, 581 v.Provider, 582 v.AccessMode, 583 )) 584 } 585 586 c.Ui.Output(formatList(output)) 587 } 588 589 func (c *NodeStatusCommand) outputNodeDriverInfo(node *api.Node) { 590 c.Ui.Output(c.Colorize().Color("\n[bold]Drivers")) 591 592 size := len(node.Drivers) 593 nodeDrivers := make([]string, 0, size+1) 594 595 nodeDrivers = append(nodeDrivers, "Driver|Detected|Healthy|Message|Time") 596 597 drivers := make([]string, 0, len(node.Drivers)) 598 for driver := range node.Drivers { 599 drivers = append(drivers, driver) 600 } 601 sort.Strings(drivers) 602 603 for _, driver := range drivers { 604 info := node.Drivers[driver] 605 timestamp := formatTime(info.UpdateTime) 606 nodeDrivers = append(nodeDrivers, fmt.Sprintf("%s|%v|%v|%s|%s", driver, info.Detected, info.Healthy, info.HealthDescription, timestamp)) 607 } 608 c.Ui.Output(formatList(nodeDrivers)) 609 } 610 611 func (c *NodeStatusCommand) outputNodeStatusEvents(node *api.Node) { 612 c.Ui.Output(c.Colorize().Color("\n[bold]Node Events")) 613 c.outputNodeEvent(node.Events) 614 } 615 616 func (c *NodeStatusCommand) outputNodeEvent(events []*api.NodeEvent) { 617 size := len(events) 618 nodeEvents := make([]string, size+1) 619 if c.verbose { 620 nodeEvents[0] = "Time|Subsystem|Message|Details" 621 } else { 622 nodeEvents[0] = "Time|Subsystem|Message" 623 } 624 625 for i, event := range events { 626 timestamp := formatTime(event.Timestamp) 627 subsystem := formatEventSubsystem(event.Subsystem, event.Details["driver"]) 628 msg := event.Message 629 if c.verbose { 630 details := formatEventDetails(event.Details) 631 nodeEvents[size-i] = fmt.Sprintf("%s|%s|%s|%s", timestamp, subsystem, msg, details) 632 } else { 633 nodeEvents[size-i] = fmt.Sprintf("%s|%s|%s", timestamp, subsystem, msg) 634 } 635 } 636 c.Ui.Output(formatList(nodeEvents)) 637 } 638 639 func formatEventSubsystem(subsystem, driverName string) string { 640 if driverName == "" { 641 return subsystem 642 } 643 644 // If this event is for a driver, append the driver name to make the message 645 // clearer 646 return fmt.Sprintf("Driver: %s", driverName) 647 } 648 649 func formatEventDetails(details map[string]string) string { 650 output := make([]string, 0, len(details)) 651 for k, v := range details { 652 output = append(output, fmt.Sprintf("%s: %s", k, v)) 653 } 654 return strings.Join(output, ", ") 655 } 656 657 func (c *NodeStatusCommand) formatAttributes(node *api.Node) { 658 // Print the attributes 659 keys := make([]string, len(node.Attributes)) 660 for k := range node.Attributes { 661 keys = append(keys, k) 662 } 663 sort.Strings(keys) 664 665 var attributes []string 666 for _, k := range keys { 667 if k != "" { 668 attributes = append(attributes, fmt.Sprintf("%s|%s", k, node.Attributes[k])) 669 } 670 } 671 c.Ui.Output(c.Colorize().Color("\n[bold]Attributes[reset]")) 672 c.Ui.Output(formatKV(attributes)) 673 } 674 675 func (c *NodeStatusCommand) formatDeviceAttributes(node *api.Node) { 676 if node.NodeResources == nil { 677 return 678 } 679 devices := node.NodeResources.Devices 680 if len(devices) == 0 { 681 return 682 } 683 684 sort.Slice(devices, func(i, j int) bool { 685 return devices[i].ID() < devices[j].ID() 686 }) 687 688 first := true 689 for _, d := range devices { 690 if len(d.Attributes) == 0 { 691 continue 692 } 693 694 if first { 695 c.Ui.Output("\nDevice Group Attributes") 696 first = false 697 } else { 698 c.Ui.Output("") 699 } 700 c.Ui.Output(formatKV(getDeviceAttributes(d))) 701 } 702 } 703 704 func (c *NodeStatusCommand) formatMeta(node *api.Node) { 705 // Print the meta 706 keys := make([]string, 0, len(node.Meta)) 707 for k := range node.Meta { 708 keys = append(keys, k) 709 } 710 sort.Strings(keys) 711 712 var meta []string 713 for _, k := range keys { 714 if k != "" { 715 meta = append(meta, fmt.Sprintf("%s|%s", k, node.Meta[k])) 716 } 717 } 718 c.Ui.Output(c.Colorize().Color("\n[bold]Meta[reset]")) 719 c.Ui.Output(formatKV(meta)) 720 } 721 722 func (c *NodeStatusCommand) printCpuStats(hostStats *api.HostStats) { 723 l := len(hostStats.CPU) 724 for i, cpuStat := range hostStats.CPU { 725 cpuStatsAttr := make([]string, 4) 726 cpuStatsAttr[0] = fmt.Sprintf("CPU|%v", cpuStat.CPU) 727 cpuStatsAttr[1] = fmt.Sprintf("User|%v%%", humanize.FormatFloat(floatFormat, cpuStat.User)) 728 cpuStatsAttr[2] = fmt.Sprintf("System|%v%%", humanize.FormatFloat(floatFormat, cpuStat.System)) 729 cpuStatsAttr[3] = fmt.Sprintf("Idle|%v%%", humanize.FormatFloat(floatFormat, cpuStat.Idle)) 730 c.Ui.Output(formatKV(cpuStatsAttr)) 731 if i+1 < l { 732 c.Ui.Output("") 733 } 734 } 735 } 736 737 func (c *NodeStatusCommand) printMemoryStats(hostStats *api.HostStats) { 738 memoryStat := hostStats.Memory 739 memStatsAttr := make([]string, 4) 740 memStatsAttr[0] = fmt.Sprintf("Total|%v", humanize.IBytes(memoryStat.Total)) 741 memStatsAttr[1] = fmt.Sprintf("Available|%v", humanize.IBytes(memoryStat.Available)) 742 memStatsAttr[2] = fmt.Sprintf("Used|%v", humanize.IBytes(memoryStat.Used)) 743 memStatsAttr[3] = fmt.Sprintf("Free|%v", humanize.IBytes(memoryStat.Free)) 744 c.Ui.Output(formatKV(memStatsAttr)) 745 } 746 747 func (c *NodeStatusCommand) printDiskStats(hostStats *api.HostStats) { 748 l := len(hostStats.DiskStats) 749 for i, diskStat := range hostStats.DiskStats { 750 diskStatsAttr := make([]string, 7) 751 diskStatsAttr[0] = fmt.Sprintf("Device|%s", diskStat.Device) 752 diskStatsAttr[1] = fmt.Sprintf("MountPoint|%s", diskStat.Mountpoint) 753 diskStatsAttr[2] = fmt.Sprintf("Size|%s", humanize.IBytes(diskStat.Size)) 754 diskStatsAttr[3] = fmt.Sprintf("Used|%s", humanize.IBytes(diskStat.Used)) 755 diskStatsAttr[4] = fmt.Sprintf("Available|%s", humanize.IBytes(diskStat.Available)) 756 diskStatsAttr[5] = fmt.Sprintf("Used Percent|%v%%", humanize.FormatFloat(floatFormat, diskStat.UsedPercent)) 757 diskStatsAttr[6] = fmt.Sprintf("Inodes Percent|%v%%", humanize.FormatFloat(floatFormat, diskStat.InodesUsedPercent)) 758 c.Ui.Output(formatKV(diskStatsAttr)) 759 if i+1 < l { 760 c.Ui.Output("") 761 } 762 } 763 } 764 765 // getRunningAllocs returns a slice of allocation id's running on the node 766 func getRunningAllocs(client *api.Client, nodeID string) ([]*api.Allocation, error) { 767 var allocs []*api.Allocation 768 769 // Query the node allocations 770 nodeAllocs, _, err := client.Nodes().Allocations(nodeID, nil) 771 // Filter list to only running allocations 772 for _, alloc := range nodeAllocs { 773 if alloc.ClientStatus == "running" { 774 allocs = append(allocs, alloc) 775 } 776 } 777 return allocs, err 778 } 779 780 // getAllocatedResources returns the resource usage of the node. 781 func getAllocatedResources(client *api.Client, runningAllocs []*api.Allocation, node *api.Node) []string { 782 // Compute the total 783 total := computeNodeTotalResources(node) 784 785 // Get Resources 786 var cpu, mem, disk int 787 for _, alloc := range runningAllocs { 788 cpu += *alloc.Resources.CPU 789 mem += *alloc.Resources.MemoryMB 790 disk += *alloc.Resources.DiskMB 791 } 792 793 resources := make([]string, 2) 794 resources[0] = "CPU|Memory|Disk" 795 resources[1] = fmt.Sprintf("%d/%d MHz|%s/%s|%s/%s", 796 cpu, 797 *total.CPU, 798 humanize.IBytes(uint64(mem*bytesPerMegabyte)), 799 humanize.IBytes(uint64(*total.MemoryMB*bytesPerMegabyte)), 800 humanize.IBytes(uint64(disk*bytesPerMegabyte)), 801 humanize.IBytes(uint64(*total.DiskMB*bytesPerMegabyte))) 802 803 return resources 804 } 805 806 // computeNodeTotalResources returns the total allocatable resources (resources 807 // minus reserved) 808 func computeNodeTotalResources(node *api.Node) api.Resources { 809 total := api.Resources{} 810 811 r := node.Resources 812 res := node.Reserved 813 if res == nil { 814 res = &api.Resources{} 815 } 816 total.CPU = helper.IntToPtr(*r.CPU - *res.CPU) 817 total.MemoryMB = helper.IntToPtr(*r.MemoryMB - *res.MemoryMB) 818 total.DiskMB = helper.IntToPtr(*r.DiskMB - *res.DiskMB) 819 return total 820 } 821 822 // getActualResources returns the actual resource usage of the allocations. 823 func getActualResources(client *api.Client, runningAllocs []*api.Allocation, node *api.Node) ([]string, error) { 824 // Compute the total 825 total := computeNodeTotalResources(node) 826 827 // Get Resources 828 var cpu float64 829 var mem uint64 830 for _, alloc := range runningAllocs { 831 // Make the call to the client to get the actual usage. 832 stats, err := client.Allocations().Stats(alloc, nil) 833 if err != nil { 834 return nil, err 835 } 836 837 cpu += stats.ResourceUsage.CpuStats.TotalTicks 838 mem += stats.ResourceUsage.MemoryStats.RSS 839 } 840 841 resources := make([]string, 2) 842 resources[0] = "CPU|Memory" 843 resources[1] = fmt.Sprintf("%v/%d MHz|%v/%v", 844 math.Floor(cpu), 845 *total.CPU, 846 humanize.IBytes(mem), 847 humanize.IBytes(uint64(*total.MemoryMB*bytesPerMegabyte))) 848 849 return resources, nil 850 } 851 852 // getHostResources returns the actual resource usage of the node. 853 func getHostResources(hostStats *api.HostStats, node *api.Node) ([]string, error) { 854 if hostStats == nil { 855 return nil, fmt.Errorf("actual resource usage not present") 856 } 857 var resources []string 858 859 // calculate disk usage 860 storageDevice := node.Attributes["unique.storage.volume"] 861 var diskUsed, diskSize uint64 862 var physical bool 863 for _, disk := range hostStats.DiskStats { 864 if disk.Device == storageDevice { 865 diskUsed = disk.Used 866 diskSize = disk.Size 867 physical = true 868 } 869 } 870 871 resources = make([]string, 2) 872 resources[0] = "CPU|Memory|Disk" 873 if physical { 874 resources[1] = fmt.Sprintf("%v/%d MHz|%s/%s|%s/%s", 875 math.Floor(hostStats.CPUTicksConsumed), 876 *node.Resources.CPU, 877 humanize.IBytes(hostStats.Memory.Used), 878 humanize.IBytes(hostStats.Memory.Total), 879 humanize.IBytes(diskUsed), 880 humanize.IBytes(diskSize), 881 ) 882 } else { 883 // If non-physical device are used, output device name only, 884 // since nomad doesn't collect the stats data. 885 resources[1] = fmt.Sprintf("%v/%d MHz|%s/%s|(%s)", 886 math.Floor(hostStats.CPUTicksConsumed), 887 *node.Resources.CPU, 888 humanize.IBytes(hostStats.Memory.Used), 889 humanize.IBytes(hostStats.Memory.Total), 890 storageDevice, 891 ) 892 } 893 return resources, nil 894 } 895 896 // formatNodeStubList is used to return a table format of a list of node stubs. 897 func formatNodeStubList(nodes []*api.NodeListStub, verbose bool) string { 898 // Return error if no nodes are found 899 if len(nodes) == 0 { 900 return "" 901 } 902 // Truncate the id unless full length is requested 903 length := shortId 904 if verbose { 905 length = fullId 906 } 907 908 // Format the nodes list that matches the prefix so that the user 909 // can create a more specific request 910 out := make([]string, len(nodes)+1) 911 out[0] = "ID|DC|Name|Class|Drain|Eligibility|Status" 912 for i, node := range nodes { 913 out[i+1] = fmt.Sprintf("%s|%s|%s|%s|%v|%s|%s", 914 limit(node.ID, length), 915 node.Datacenter, 916 node.Name, 917 node.NodeClass, 918 node.Drain, 919 node.SchedulingEligibility, 920 node.Status) 921 } 922 923 return formatList(out) 924 }