github.com/smithx10/nomad@v0.9.1-rc1/command/alloc_status.go (about) 1 package command 2 3 import ( 4 "fmt" 5 "math" 6 "sort" 7 "strconv" 8 "strings" 9 "time" 10 11 humanize "github.com/dustin/go-humanize" 12 "github.com/hashicorp/nomad/api" 13 "github.com/hashicorp/nomad/api/contexts" 14 "github.com/hashicorp/nomad/client/allocrunner/taskrunner/restarts" 15 "github.com/posener/complete" 16 ) 17 18 type AllocStatusCommand struct { 19 Meta 20 } 21 22 func (c *AllocStatusCommand) Help() string { 23 helpText := ` 24 Usage: nomad alloc status [options] <allocation> 25 26 Display information about existing allocations and its tasks. This command can 27 be used to inspect the current status of an allocation, including its running 28 status, metadata, and verbose failure messages reported by internal 29 subsystems. 30 31 General Options: 32 33 ` + generalOptionsUsage() + ` 34 35 Alloc Status Options: 36 37 -short 38 Display short output. Shows only the most recent task event. 39 40 -stats 41 Display detailed resource usage statistics. 42 43 -verbose 44 Show full information. 45 46 -json 47 Output the allocation in its JSON format. 48 49 -t 50 Format and display allocation using a Go template. 51 ` 52 53 return strings.TrimSpace(helpText) 54 } 55 56 func (c *AllocStatusCommand) Synopsis() string { 57 return "Display allocation status information and metadata" 58 } 59 60 func (c *AllocStatusCommand) AutocompleteFlags() complete.Flags { 61 return mergeAutocompleteFlags(c.Meta.AutocompleteFlags(FlagSetClient), 62 complete.Flags{ 63 "-short": complete.PredictNothing, 64 "-verbose": complete.PredictNothing, 65 "-json": complete.PredictNothing, 66 "-t": complete.PredictAnything, 67 }) 68 } 69 70 func (c *AllocStatusCommand) AutocompleteArgs() complete.Predictor { 71 return complete.PredictFunc(func(a complete.Args) []string { 72 client, err := c.Meta.Client() 73 if err != nil { 74 return nil 75 } 76 77 resp, _, err := client.Search().PrefixSearch(a.Last, contexts.Allocs, nil) 78 if err != nil { 79 return []string{} 80 } 81 return resp.Matches[contexts.Allocs] 82 }) 83 } 84 85 func (c *AllocStatusCommand) Name() string { return "alloc status" } 86 87 func (c *AllocStatusCommand) Run(args []string) int { 88 var short, displayStats, verbose, json bool 89 var tmpl string 90 91 flags := c.Meta.FlagSet(c.Name(), FlagSetClient) 92 flags.Usage = func() { c.Ui.Output(c.Help()) } 93 flags.BoolVar(&short, "short", false, "") 94 flags.BoolVar(&verbose, "verbose", false, "") 95 flags.BoolVar(&displayStats, "stats", false, "") 96 flags.BoolVar(&json, "json", false, "") 97 flags.StringVar(&tmpl, "t", "", "") 98 99 if err := flags.Parse(args); err != nil { 100 return 1 101 } 102 103 // Check that we got exactly one allocation ID 104 args = flags.Args() 105 106 // Get the HTTP client 107 client, err := c.Meta.Client() 108 if err != nil { 109 c.Ui.Error(fmt.Sprintf("Error initializing client: %s", err)) 110 return 1 111 } 112 113 // If args not specified but output format is specified, format and output the allocations data list 114 if len(args) == 0 && json || len(tmpl) > 0 { 115 allocs, _, err := client.Allocations().List(nil) 116 if err != nil { 117 c.Ui.Error(fmt.Sprintf("Error querying allocations: %v", err)) 118 return 1 119 } 120 121 out, err := Format(json, tmpl, allocs) 122 if err != nil { 123 c.Ui.Error(err.Error()) 124 return 1 125 } 126 127 c.Ui.Output(out) 128 return 0 129 } 130 131 if len(args) != 1 { 132 c.Ui.Error("This command takes one of the following argument conditions:") 133 c.Ui.Error(" * A single <allocation>") 134 c.Ui.Error(" * No arguments, with output format specified") 135 c.Ui.Error(commandErrorText(c)) 136 return 1 137 } 138 allocID := args[0] 139 140 // Truncate the id unless full length is requested 141 length := shortId 142 if verbose { 143 length = fullId 144 } 145 146 // Query the allocation info 147 if len(allocID) == 1 { 148 c.Ui.Error(fmt.Sprintf("Identifier must contain at least two characters.")) 149 return 1 150 } 151 152 allocID = sanitizeUUIDPrefix(allocID) 153 allocs, _, err := client.Allocations().PrefixList(allocID) 154 if err != nil { 155 c.Ui.Error(fmt.Sprintf("Error querying allocation: %v", err)) 156 return 1 157 } 158 if len(allocs) == 0 { 159 c.Ui.Error(fmt.Sprintf("No allocation(s) with prefix or id %q found", allocID)) 160 return 1 161 } 162 if len(allocs) > 1 { 163 out := formatAllocListStubs(allocs, verbose, length) 164 c.Ui.Output(fmt.Sprintf("Prefix matched multiple allocations\n\n%s", out)) 165 return 0 166 } 167 // Prefix lookup matched a single allocation 168 alloc, _, err := client.Allocations().Info(allocs[0].ID, nil) 169 if err != nil { 170 c.Ui.Error(fmt.Sprintf("Error querying allocation: %s", err)) 171 return 1 172 } 173 174 // If output format is specified, format and output the data 175 if json || len(tmpl) > 0 { 176 out, err := Format(json, tmpl, alloc) 177 if err != nil { 178 c.Ui.Error(err.Error()) 179 return 1 180 } 181 182 c.Ui.Output(out) 183 return 0 184 } 185 186 // Format the allocation data 187 output, err := formatAllocBasicInfo(alloc, client, length, verbose) 188 if err != nil { 189 c.Ui.Error(err.Error()) 190 return 1 191 } 192 c.Ui.Output(output) 193 194 if short { 195 c.shortTaskStatus(alloc) 196 } else { 197 var statsErr error 198 var stats *api.AllocResourceUsage 199 stats, statsErr = client.Allocations().Stats(alloc, nil) 200 if statsErr != nil { 201 c.Ui.Output("") 202 if statsErr != api.NodeDownErr { 203 c.Ui.Error(fmt.Sprintf("Couldn't retrieve stats: %v", statsErr)) 204 } else { 205 c.Ui.Output("Omitting resource statistics since the node is down.") 206 } 207 } 208 c.outputTaskDetails(alloc, stats, displayStats) 209 } 210 211 // Format the detailed status 212 if verbose { 213 c.Ui.Output(c.Colorize().Color("\n[bold]Placement Metrics[reset]")) 214 c.Ui.Output(formatAllocMetrics(alloc.Metrics, true, " ")) 215 } 216 217 return 0 218 } 219 220 func formatAllocBasicInfo(alloc *api.Allocation, client *api.Client, uuidLength int, verbose bool) (string, error) { 221 var formattedCreateTime, formattedModifyTime string 222 223 if verbose { 224 formattedCreateTime = formatUnixNanoTime(alloc.CreateTime) 225 formattedModifyTime = formatUnixNanoTime(alloc.ModifyTime) 226 } else { 227 formattedCreateTime = prettyTimeDiff(time.Unix(0, alloc.CreateTime), time.Now()) 228 formattedModifyTime = prettyTimeDiff(time.Unix(0, alloc.ModifyTime), time.Now()) 229 } 230 231 basic := []string{ 232 fmt.Sprintf("ID|%s", limit(alloc.ID, uuidLength)), 233 fmt.Sprintf("Eval ID|%s", limit(alloc.EvalID, uuidLength)), 234 fmt.Sprintf("Name|%s", alloc.Name), 235 fmt.Sprintf("Node ID|%s", limit(alloc.NodeID, uuidLength)), 236 fmt.Sprintf("Job ID|%s", alloc.JobID), 237 fmt.Sprintf("Job Version|%d", getVersion(alloc.Job)), 238 fmt.Sprintf("Client Status|%s", alloc.ClientStatus), 239 fmt.Sprintf("Client Description|%s", alloc.ClientDescription), 240 fmt.Sprintf("Desired Status|%s", alloc.DesiredStatus), 241 fmt.Sprintf("Desired Description|%s", alloc.DesiredDescription), 242 fmt.Sprintf("Created|%s", formattedCreateTime), 243 fmt.Sprintf("Modified|%s", formattedModifyTime), 244 } 245 246 if alloc.DeploymentID != "" { 247 health := "unset" 248 canary := false 249 if alloc.DeploymentStatus != nil { 250 if alloc.DeploymentStatus.Healthy != nil { 251 if *alloc.DeploymentStatus.Healthy { 252 health = "healthy" 253 } else { 254 health = "unhealthy" 255 } 256 } 257 258 canary = alloc.DeploymentStatus.Canary 259 } 260 261 basic = append(basic, 262 fmt.Sprintf("Deployment ID|%s", limit(alloc.DeploymentID, uuidLength)), 263 fmt.Sprintf("Deployment Health|%s", health)) 264 if canary { 265 basic = append(basic, fmt.Sprintf("Canary|%v", true)) 266 } 267 } 268 269 if alloc.RescheduleTracker != nil && len(alloc.RescheduleTracker.Events) > 0 { 270 attempts, total := alloc.RescheduleInfo(time.Unix(0, alloc.ModifyTime)) 271 // Show this section only if the reschedule policy limits the number of attempts 272 if total > 0 { 273 reschedInfo := fmt.Sprintf("Reschedule Attempts|%d/%d", attempts, total) 274 basic = append(basic, reschedInfo) 275 } 276 } 277 if alloc.NextAllocation != "" { 278 basic = append(basic, 279 fmt.Sprintf("Replacement Alloc ID|%s", limit(alloc.NextAllocation, uuidLength))) 280 } 281 if alloc.FollowupEvalID != "" { 282 nextEvalTime := futureEvalTimePretty(alloc.FollowupEvalID, client) 283 if nextEvalTime != "" { 284 basic = append(basic, 285 fmt.Sprintf("Reschedule Eligibility|%s", nextEvalTime)) 286 } 287 } 288 289 if verbose { 290 basic = append(basic, 291 fmt.Sprintf("Evaluated Nodes|%d", alloc.Metrics.NodesEvaluated), 292 fmt.Sprintf("Filtered Nodes|%d", alloc.Metrics.NodesFiltered), 293 fmt.Sprintf("Exhausted Nodes|%d", alloc.Metrics.NodesExhausted), 294 fmt.Sprintf("Allocation Time|%s", alloc.Metrics.AllocationTime), 295 fmt.Sprintf("Failures|%d", alloc.Metrics.CoalescedFailures)) 296 } 297 298 return formatKV(basic), nil 299 } 300 301 // futureEvalTimePretty returns when the eval is eligible to reschedule 302 // relative to current time, based on the WaitUntil field 303 func futureEvalTimePretty(evalID string, client *api.Client) string { 304 evaluation, _, err := client.Evaluations().Info(evalID, nil) 305 // Eval time is not a critical output, 306 // don't return it on errors, if its not set or already in the past 307 if err != nil || evaluation.WaitUntil.IsZero() || time.Now().After(evaluation.WaitUntil) { 308 return "" 309 } 310 return prettyTimeDiff(evaluation.WaitUntil, time.Now()) 311 } 312 313 // outputTaskDetails prints task details for each task in the allocation, 314 // optionally printing verbose statistics if displayStats is set 315 func (c *AllocStatusCommand) outputTaskDetails(alloc *api.Allocation, stats *api.AllocResourceUsage, displayStats bool) { 316 for task := range c.sortedTaskStateIterator(alloc.TaskStates) { 317 state := alloc.TaskStates[task] 318 c.Ui.Output(c.Colorize().Color(fmt.Sprintf("\n[bold]Task %q is %q[reset]", task, state.State))) 319 c.outputTaskResources(alloc, task, stats, displayStats) 320 c.Ui.Output("") 321 c.outputTaskStatus(state) 322 } 323 } 324 325 func formatTaskTimes(t time.Time) string { 326 if t.IsZero() { 327 return "N/A" 328 } 329 330 return formatTime(t) 331 } 332 333 // outputTaskStatus prints out a list of the most recent events for the given 334 // task state. 335 func (c *AllocStatusCommand) outputTaskStatus(state *api.TaskState) { 336 basic := []string{ 337 fmt.Sprintf("Started At|%s", formatTaskTimes(state.StartedAt)), 338 fmt.Sprintf("Finished At|%s", formatTaskTimes(state.FinishedAt)), 339 fmt.Sprintf("Total Restarts|%d", state.Restarts), 340 fmt.Sprintf("Last Restart|%s", formatTaskTimes(state.LastRestart))} 341 342 c.Ui.Output("Task Events:") 343 c.Ui.Output(formatKV(basic)) 344 c.Ui.Output("") 345 346 c.Ui.Output("Recent Events:") 347 events := make([]string, len(state.Events)+1) 348 events[0] = "Time|Type|Description" 349 350 size := len(state.Events) 351 for i, event := range state.Events { 352 msg := event.DisplayMessage 353 if msg == "" { 354 msg = buildDisplayMessage(event) 355 } 356 formattedTime := formatUnixNanoTime(event.Time) 357 events[size-i] = fmt.Sprintf("%s|%s|%s", formattedTime, event.Type, msg) 358 // Reverse order so we are sorted by time 359 } 360 c.Ui.Output(formatList(events)) 361 } 362 363 func buildDisplayMessage(event *api.TaskEvent) string { 364 // Build up the description based on the event type. 365 var desc string 366 switch event.Type { 367 case api.TaskSetup: 368 desc = event.Message 369 case api.TaskStarted: 370 desc = "Task started by client" 371 case api.TaskReceived: 372 desc = "Task received by client" 373 case api.TaskFailedValidation: 374 if event.ValidationError != "" { 375 desc = event.ValidationError 376 } else { 377 desc = "Validation of task failed" 378 } 379 case api.TaskSetupFailure: 380 if event.SetupError != "" { 381 desc = event.SetupError 382 } else { 383 desc = "Task setup failed" 384 } 385 case api.TaskDriverFailure: 386 if event.DriverError != "" { 387 desc = event.DriverError 388 } else { 389 desc = "Failed to start task" 390 } 391 case api.TaskDownloadingArtifacts: 392 desc = "Client is downloading artifacts" 393 case api.TaskArtifactDownloadFailed: 394 if event.DownloadError != "" { 395 desc = event.DownloadError 396 } else { 397 desc = "Failed to download artifacts" 398 } 399 case api.TaskKilling: 400 if event.KillReason != "" { 401 desc = fmt.Sprintf("Killing task: %v", event.KillReason) 402 } else if event.KillTimeout != 0 { 403 desc = fmt.Sprintf("Sent interrupt. Waiting %v before force killing", event.KillTimeout) 404 } else { 405 desc = "Sent interrupt" 406 } 407 case api.TaskKilled: 408 if event.KillError != "" { 409 desc = event.KillError 410 } else { 411 desc = "Task successfully killed" 412 } 413 case api.TaskTerminated: 414 var parts []string 415 parts = append(parts, fmt.Sprintf("Exit Code: %d", event.ExitCode)) 416 417 if event.Signal != 0 { 418 parts = append(parts, fmt.Sprintf("Signal: %d", event.Signal)) 419 } 420 421 if event.Message != "" { 422 parts = append(parts, fmt.Sprintf("Exit Message: %q", event.Message)) 423 } 424 desc = strings.Join(parts, ", ") 425 case api.TaskRestarting: 426 in := fmt.Sprintf("Task restarting in %v", time.Duration(event.StartDelay)) 427 if event.RestartReason != "" && event.RestartReason != restarts.ReasonWithinPolicy { 428 desc = fmt.Sprintf("%s - %s", event.RestartReason, in) 429 } else { 430 desc = in 431 } 432 case api.TaskNotRestarting: 433 if event.RestartReason != "" { 434 desc = event.RestartReason 435 } else { 436 desc = "Task exceeded restart policy" 437 } 438 case api.TaskSiblingFailed: 439 if event.FailedSibling != "" { 440 desc = fmt.Sprintf("Task's sibling %q failed", event.FailedSibling) 441 } else { 442 desc = "Task's sibling failed" 443 } 444 case api.TaskSignaling: 445 sig := event.TaskSignal 446 reason := event.TaskSignalReason 447 448 if sig == "" && reason == "" { 449 desc = "Task being sent a signal" 450 } else if sig == "" { 451 desc = reason 452 } else if reason == "" { 453 desc = fmt.Sprintf("Task being sent signal %v", sig) 454 } else { 455 desc = fmt.Sprintf("Task being sent signal %v: %v", sig, reason) 456 } 457 case api.TaskRestartSignal: 458 if event.RestartReason != "" { 459 desc = event.RestartReason 460 } else { 461 desc = "Task signaled to restart" 462 } 463 case api.TaskDriverMessage: 464 desc = event.DriverMessage 465 case api.TaskLeaderDead: 466 desc = "Leader Task in Group dead" 467 default: 468 desc = event.Message 469 } 470 471 return desc 472 } 473 474 // outputTaskResources prints the task resources for the passed task and if 475 // displayStats is set, verbose resource usage statistics 476 func (c *AllocStatusCommand) outputTaskResources(alloc *api.Allocation, task string, stats *api.AllocResourceUsage, displayStats bool) { 477 resource, ok := alloc.TaskResources[task] 478 if !ok { 479 return 480 } 481 482 c.Ui.Output("Task Resources") 483 var addr []string 484 for _, nw := range resource.Networks { 485 ports := append(nw.DynamicPorts, nw.ReservedPorts...) 486 for _, port := range ports { 487 addr = append(addr, fmt.Sprintf("%v: %v:%v\n", port.Label, nw.IP, port.Value)) 488 } 489 } 490 491 var resourcesOutput []string 492 resourcesOutput = append(resourcesOutput, "CPU|Memory|Disk|Addresses") 493 firstAddr := "" 494 if len(addr) > 0 { 495 firstAddr = addr[0] 496 } 497 498 // Display the rolled up stats. If possible prefer the live statistics 499 cpuUsage := strconv.Itoa(*resource.CPU) 500 memUsage := humanize.IBytes(uint64(*resource.MemoryMB * bytesPerMegabyte)) 501 var deviceStats []*api.DeviceGroupStats 502 503 if stats != nil { 504 if ru, ok := stats.Tasks[task]; ok && ru != nil && ru.ResourceUsage != nil { 505 if cs := ru.ResourceUsage.CpuStats; cs != nil { 506 cpuUsage = fmt.Sprintf("%v/%v", math.Floor(cs.TotalTicks), cpuUsage) 507 } 508 if ms := ru.ResourceUsage.MemoryStats; ms != nil { 509 memUsage = fmt.Sprintf("%v/%v", humanize.IBytes(ms.RSS), memUsage) 510 } 511 deviceStats = ru.ResourceUsage.DeviceStats 512 } 513 } 514 resourcesOutput = append(resourcesOutput, fmt.Sprintf("%v MHz|%v|%v|%v", 515 cpuUsage, 516 memUsage, 517 humanize.IBytes(uint64(*alloc.Resources.DiskMB*bytesPerMegabyte)), 518 firstAddr)) 519 for i := 1; i < len(addr); i++ { 520 resourcesOutput = append(resourcesOutput, fmt.Sprintf("||||%v", addr[i])) 521 } 522 c.Ui.Output(formatListWithSpaces(resourcesOutput)) 523 524 if len(deviceStats) > 0 { 525 c.Ui.Output("") 526 c.Ui.Output("Device Stats") 527 c.Ui.Output(formatList(getDeviceResources(deviceStats))) 528 } 529 530 if stats != nil { 531 if ru, ok := stats.Tasks[task]; ok && ru != nil && displayStats && ru.ResourceUsage != nil { 532 c.Ui.Output("") 533 c.outputVerboseResourceUsage(task, ru.ResourceUsage) 534 } 535 } 536 } 537 538 // outputVerboseResourceUsage outputs the verbose resource usage for the passed 539 // task 540 func (c *AllocStatusCommand) outputVerboseResourceUsage(task string, resourceUsage *api.ResourceUsage) { 541 memoryStats := resourceUsage.MemoryStats 542 cpuStats := resourceUsage.CpuStats 543 deviceStats := resourceUsage.DeviceStats 544 545 if memoryStats != nil && len(memoryStats.Measured) > 0 { 546 c.Ui.Output("Memory Stats") 547 548 // Sort the measured stats 549 sort.Strings(memoryStats.Measured) 550 551 var measuredStats []string 552 for _, measured := range memoryStats.Measured { 553 switch measured { 554 case "RSS": 555 measuredStats = append(measuredStats, humanize.IBytes(memoryStats.RSS)) 556 case "Cache": 557 measuredStats = append(measuredStats, humanize.IBytes(memoryStats.Cache)) 558 case "Swap": 559 measuredStats = append(measuredStats, humanize.IBytes(memoryStats.Swap)) 560 case "Usage": 561 measuredStats = append(measuredStats, humanize.IBytes(memoryStats.Usage)) 562 case "Max Usage": 563 measuredStats = append(measuredStats, humanize.IBytes(memoryStats.MaxUsage)) 564 case "Kernel Usage": 565 measuredStats = append(measuredStats, humanize.IBytes(memoryStats.KernelUsage)) 566 case "Kernel Max Usage": 567 measuredStats = append(measuredStats, humanize.IBytes(memoryStats.KernelMaxUsage)) 568 } 569 } 570 571 out := make([]string, 2) 572 out[0] = strings.Join(memoryStats.Measured, "|") 573 out[1] = strings.Join(measuredStats, "|") 574 c.Ui.Output(formatList(out)) 575 c.Ui.Output("") 576 } 577 578 if cpuStats != nil && len(cpuStats.Measured) > 0 { 579 c.Ui.Output("CPU Stats") 580 581 // Sort the measured stats 582 sort.Strings(cpuStats.Measured) 583 584 var measuredStats []string 585 for _, measured := range cpuStats.Measured { 586 switch measured { 587 case "Percent": 588 percent := strconv.FormatFloat(cpuStats.Percent, 'f', 2, 64) 589 measuredStats = append(measuredStats, fmt.Sprintf("%v%%", percent)) 590 case "Throttled Periods": 591 measuredStats = append(measuredStats, fmt.Sprintf("%v", cpuStats.ThrottledPeriods)) 592 case "Throttled Time": 593 measuredStats = append(measuredStats, fmt.Sprintf("%v", cpuStats.ThrottledTime)) 594 case "User Mode": 595 percent := strconv.FormatFloat(cpuStats.UserMode, 'f', 2, 64) 596 measuredStats = append(measuredStats, fmt.Sprintf("%v%%", percent)) 597 case "System Mode": 598 percent := strconv.FormatFloat(cpuStats.SystemMode, 'f', 2, 64) 599 measuredStats = append(measuredStats, fmt.Sprintf("%v%%", percent)) 600 } 601 } 602 603 out := make([]string, 2) 604 out[0] = strings.Join(cpuStats.Measured, "|") 605 out[1] = strings.Join(measuredStats, "|") 606 c.Ui.Output(formatList(out)) 607 } 608 609 if len(deviceStats) > 0 { 610 c.Ui.Output("") 611 c.Ui.Output("Device Stats") 612 613 printDeviceStats(c.Ui, deviceStats) 614 } 615 } 616 617 // shortTaskStatus prints out the current state of each task. 618 func (c *AllocStatusCommand) shortTaskStatus(alloc *api.Allocation) { 619 tasks := make([]string, 0, len(alloc.TaskStates)+1) 620 tasks = append(tasks, "Name|State|Last Event|Time") 621 for task := range c.sortedTaskStateIterator(alloc.TaskStates) { 622 state := alloc.TaskStates[task] 623 lastState := state.State 624 var lastEvent, lastTime string 625 626 l := len(state.Events) 627 if l != 0 { 628 last := state.Events[l-1] 629 lastEvent = last.Type 630 lastTime = formatUnixNanoTime(last.Time) 631 } 632 633 tasks = append(tasks, fmt.Sprintf("%s|%s|%s|%s", 634 task, lastState, lastEvent, lastTime)) 635 } 636 637 c.Ui.Output(c.Colorize().Color("\n[bold]Tasks[reset]")) 638 c.Ui.Output(formatList(tasks)) 639 } 640 641 // sortedTaskStateIterator is a helper that takes the task state map and returns a 642 // channel that returns the keys in a sorted order. 643 func (c *AllocStatusCommand) sortedTaskStateIterator(m map[string]*api.TaskState) <-chan string { 644 output := make(chan string, len(m)) 645 keys := make([]string, len(m)) 646 i := 0 647 for k := range m { 648 keys[i] = k 649 i++ 650 } 651 sort.Strings(keys) 652 653 for _, key := range keys { 654 output <- key 655 } 656 657 close(output) 658 return output 659 }