github.com/bigcommerce/nomad@v0.9.3-bc/command/alloc_status.go (about) 1 package command 2 3 import ( 4 "fmt" 5 "math" 6 "sort" 7 "strconv" 8 "strings" 9 "time" 10 11 humanize "github.com/dustin/go-humanize" 12 "github.com/hashicorp/nomad/api" 13 "github.com/hashicorp/nomad/api/contexts" 14 "github.com/hashicorp/nomad/client/allocrunner/taskrunner/restarts" 15 "github.com/posener/complete" 16 ) 17 18 type AllocStatusCommand struct { 19 Meta 20 } 21 22 func (c *AllocStatusCommand) Help() string { 23 helpText := ` 24 Usage: nomad alloc status [options] <allocation> 25 26 Display information about existing allocations and its tasks. This command can 27 be used to inspect the current status of an allocation, including its running 28 status, metadata, and verbose failure messages reported by internal 29 subsystems. 30 31 General Options: 32 33 ` + generalOptionsUsage() + ` 34 35 Alloc Status Options: 36 37 -short 38 Display short output. Shows only the most recent task event. 39 40 -stats 41 Display detailed resource usage statistics. 42 43 -verbose 44 Show full information. 45 46 -json 47 Output the allocation in its JSON format. 48 49 -t 50 Format and display allocation using a Go template. 51 ` 52 53 return strings.TrimSpace(helpText) 54 } 55 56 func (c *AllocStatusCommand) Synopsis() string { 57 return "Display allocation status information and metadata" 58 } 59 60 func (c *AllocStatusCommand) AutocompleteFlags() complete.Flags { 61 return mergeAutocompleteFlags(c.Meta.AutocompleteFlags(FlagSetClient), 62 complete.Flags{ 63 "-short": complete.PredictNothing, 64 "-verbose": complete.PredictNothing, 65 "-json": complete.PredictNothing, 66 "-t": complete.PredictAnything, 67 }) 68 } 69 70 func (c *AllocStatusCommand) AutocompleteArgs() complete.Predictor { 71 return complete.PredictFunc(func(a complete.Args) []string { 72 client, err := c.Meta.Client() 73 if err != nil { 74 return nil 75 } 76 77 resp, _, err := client.Search().PrefixSearch(a.Last, contexts.Allocs, nil) 78 if err != nil { 79 return []string{} 80 } 81 return resp.Matches[contexts.Allocs] 82 }) 83 } 84 85 func (c *AllocStatusCommand) Name() string { return "alloc status" } 86 87 func (c *AllocStatusCommand) Run(args []string) int { 88 var short, displayStats, verbose, json bool 89 var tmpl string 90 91 flags := c.Meta.FlagSet(c.Name(), FlagSetClient) 92 flags.Usage = func() { c.Ui.Output(c.Help()) } 93 flags.BoolVar(&short, "short", false, "") 94 flags.BoolVar(&verbose, "verbose", false, "") 95 flags.BoolVar(&displayStats, "stats", false, "") 96 flags.BoolVar(&json, "json", false, "") 97 flags.StringVar(&tmpl, "t", "", "") 98 99 if err := flags.Parse(args); err != nil { 100 return 1 101 } 102 103 // Check that we got exactly one allocation ID 104 args = flags.Args() 105 106 // Get the HTTP client 107 client, err := c.Meta.Client() 108 if err != nil { 109 c.Ui.Error(fmt.Sprintf("Error initializing client: %s", err)) 110 return 1 111 } 112 113 // If args not specified but output format is specified, format and output the allocations data list 114 if len(args) == 0 && json || len(tmpl) > 0 { 115 allocs, _, err := client.Allocations().List(nil) 116 if err != nil { 117 c.Ui.Error(fmt.Sprintf("Error querying allocations: %v", err)) 118 return 1 119 } 120 121 out, err := Format(json, tmpl, allocs) 122 if err != nil { 123 c.Ui.Error(err.Error()) 124 return 1 125 } 126 127 c.Ui.Output(out) 128 return 0 129 } 130 131 if len(args) != 1 { 132 c.Ui.Error("This command takes one of the following argument conditions:") 133 c.Ui.Error(" * A single <allocation>") 134 c.Ui.Error(" * No arguments, with output format specified") 135 c.Ui.Error(commandErrorText(c)) 136 return 1 137 } 138 allocID := args[0] 139 140 // Truncate the id unless full length is requested 141 length := shortId 142 if verbose { 143 length = fullId 144 } 145 146 // Query the allocation info 147 if len(allocID) == 1 { 148 c.Ui.Error(fmt.Sprintf("Identifier must contain at least two characters.")) 149 return 1 150 } 151 152 allocID = sanitizeUUIDPrefix(allocID) 153 allocs, _, err := client.Allocations().PrefixList(allocID) 154 if err != nil { 155 c.Ui.Error(fmt.Sprintf("Error querying allocation: %v", err)) 156 return 1 157 } 158 if len(allocs) == 0 { 159 c.Ui.Error(fmt.Sprintf("No allocation(s) with prefix or id %q found", allocID)) 160 return 1 161 } 162 if len(allocs) > 1 { 163 out := formatAllocListStubs(allocs, verbose, length) 164 c.Ui.Output(fmt.Sprintf("Prefix matched multiple allocations\n\n%s", out)) 165 return 0 166 } 167 // Prefix lookup matched a single allocation 168 alloc, _, err := client.Allocations().Info(allocs[0].ID, nil) 169 if err != nil { 170 c.Ui.Error(fmt.Sprintf("Error querying allocation: %s", err)) 171 return 1 172 } 173 174 // If output format is specified, format and output the data 175 if json || len(tmpl) > 0 { 176 out, err := Format(json, tmpl, alloc) 177 if err != nil { 178 c.Ui.Error(err.Error()) 179 return 1 180 } 181 182 c.Ui.Output(out) 183 return 0 184 } 185 186 // Format the allocation data 187 output, err := formatAllocBasicInfo(alloc, client, length, verbose) 188 if err != nil { 189 c.Ui.Error(err.Error()) 190 return 1 191 } 192 c.Ui.Output(output) 193 194 if short { 195 c.shortTaskStatus(alloc) 196 } else { 197 var statsErr error 198 var stats *api.AllocResourceUsage 199 stats, statsErr = client.Allocations().Stats(alloc, nil) 200 if statsErr != nil { 201 c.Ui.Output("") 202 if statsErr != api.NodeDownErr { 203 c.Ui.Error(fmt.Sprintf("Couldn't retrieve stats: %v", statsErr)) 204 } else { 205 c.Ui.Output("Omitting resource statistics since the node is down.") 206 } 207 } 208 c.outputTaskDetails(alloc, stats, displayStats) 209 } 210 211 // Format the detailed status 212 if verbose { 213 c.Ui.Output(c.Colorize().Color("\n[bold]Placement Metrics[reset]")) 214 c.Ui.Output(formatAllocMetrics(alloc.Metrics, true, " ")) 215 } 216 217 return 0 218 } 219 220 func formatAllocBasicInfo(alloc *api.Allocation, client *api.Client, uuidLength int, verbose bool) (string, error) { 221 var formattedCreateTime, formattedModifyTime string 222 223 if verbose { 224 formattedCreateTime = formatUnixNanoTime(alloc.CreateTime) 225 formattedModifyTime = formatUnixNanoTime(alloc.ModifyTime) 226 } else { 227 formattedCreateTime = prettyTimeDiff(time.Unix(0, alloc.CreateTime), time.Now()) 228 formattedModifyTime = prettyTimeDiff(time.Unix(0, alloc.ModifyTime), time.Now()) 229 } 230 231 basic := []string{ 232 fmt.Sprintf("ID|%s", limit(alloc.ID, uuidLength)), 233 fmt.Sprintf("Eval ID|%s", limit(alloc.EvalID, uuidLength)), 234 fmt.Sprintf("Name|%s", alloc.Name), 235 fmt.Sprintf("Node ID|%s", limit(alloc.NodeID, uuidLength)), 236 fmt.Sprintf("Node Name|%s", alloc.NodeName), 237 fmt.Sprintf("Job ID|%s", alloc.JobID), 238 fmt.Sprintf("Job Version|%d", getVersion(alloc.Job)), 239 fmt.Sprintf("Client Status|%s", alloc.ClientStatus), 240 fmt.Sprintf("Client Description|%s", alloc.ClientDescription), 241 fmt.Sprintf("Desired Status|%s", alloc.DesiredStatus), 242 fmt.Sprintf("Desired Description|%s", alloc.DesiredDescription), 243 fmt.Sprintf("Created|%s", formattedCreateTime), 244 fmt.Sprintf("Modified|%s", formattedModifyTime), 245 } 246 247 if alloc.DeploymentID != "" { 248 health := "unset" 249 canary := false 250 if alloc.DeploymentStatus != nil { 251 if alloc.DeploymentStatus.Healthy != nil { 252 if *alloc.DeploymentStatus.Healthy { 253 health = "healthy" 254 } else { 255 health = "unhealthy" 256 } 257 } 258 259 canary = alloc.DeploymentStatus.Canary 260 } 261 262 basic = append(basic, 263 fmt.Sprintf("Deployment ID|%s", limit(alloc.DeploymentID, uuidLength)), 264 fmt.Sprintf("Deployment Health|%s", health)) 265 if canary { 266 basic = append(basic, fmt.Sprintf("Canary|%v", true)) 267 } 268 } 269 270 if alloc.RescheduleTracker != nil && len(alloc.RescheduleTracker.Events) > 0 { 271 attempts, total := alloc.RescheduleInfo(time.Unix(0, alloc.ModifyTime)) 272 // Show this section only if the reschedule policy limits the number of attempts 273 if total > 0 { 274 reschedInfo := fmt.Sprintf("Reschedule Attempts|%d/%d", attempts, total) 275 basic = append(basic, reschedInfo) 276 } 277 } 278 if alloc.NextAllocation != "" { 279 basic = append(basic, 280 fmt.Sprintf("Replacement Alloc ID|%s", limit(alloc.NextAllocation, uuidLength))) 281 } 282 if alloc.FollowupEvalID != "" { 283 nextEvalTime := futureEvalTimePretty(alloc.FollowupEvalID, client) 284 if nextEvalTime != "" { 285 basic = append(basic, 286 fmt.Sprintf("Reschedule Eligibility|%s", nextEvalTime)) 287 } 288 } 289 290 if verbose { 291 basic = append(basic, 292 fmt.Sprintf("Evaluated Nodes|%d", alloc.Metrics.NodesEvaluated), 293 fmt.Sprintf("Filtered Nodes|%d", alloc.Metrics.NodesFiltered), 294 fmt.Sprintf("Exhausted Nodes|%d", alloc.Metrics.NodesExhausted), 295 fmt.Sprintf("Allocation Time|%s", alloc.Metrics.AllocationTime), 296 fmt.Sprintf("Failures|%d", alloc.Metrics.CoalescedFailures)) 297 } 298 299 return formatKV(basic), nil 300 } 301 302 // futureEvalTimePretty returns when the eval is eligible to reschedule 303 // relative to current time, based on the WaitUntil field 304 func futureEvalTimePretty(evalID string, client *api.Client) string { 305 evaluation, _, err := client.Evaluations().Info(evalID, nil) 306 // Eval time is not a critical output, 307 // don't return it on errors, if its not set or already in the past 308 if err != nil || evaluation.WaitUntil.IsZero() || time.Now().After(evaluation.WaitUntil) { 309 return "" 310 } 311 return prettyTimeDiff(evaluation.WaitUntil, time.Now()) 312 } 313 314 // outputTaskDetails prints task details for each task in the allocation, 315 // optionally printing verbose statistics if displayStats is set 316 func (c *AllocStatusCommand) outputTaskDetails(alloc *api.Allocation, stats *api.AllocResourceUsage, displayStats bool) { 317 for task := range c.sortedTaskStateIterator(alloc.TaskStates) { 318 state := alloc.TaskStates[task] 319 c.Ui.Output(c.Colorize().Color(fmt.Sprintf("\n[bold]Task %q is %q[reset]", task, state.State))) 320 c.outputTaskResources(alloc, task, stats, displayStats) 321 c.Ui.Output("") 322 c.outputTaskStatus(state) 323 } 324 } 325 326 func formatTaskTimes(t time.Time) string { 327 if t.IsZero() { 328 return "N/A" 329 } 330 331 return formatTime(t) 332 } 333 334 // outputTaskStatus prints out a list of the most recent events for the given 335 // task state. 336 func (c *AllocStatusCommand) outputTaskStatus(state *api.TaskState) { 337 basic := []string{ 338 fmt.Sprintf("Started At|%s", formatTaskTimes(state.StartedAt)), 339 fmt.Sprintf("Finished At|%s", formatTaskTimes(state.FinishedAt)), 340 fmt.Sprintf("Total Restarts|%d", state.Restarts), 341 fmt.Sprintf("Last Restart|%s", formatTaskTimes(state.LastRestart))} 342 343 c.Ui.Output("Task Events:") 344 c.Ui.Output(formatKV(basic)) 345 c.Ui.Output("") 346 347 c.Ui.Output("Recent Events:") 348 events := make([]string, len(state.Events)+1) 349 events[0] = "Time|Type|Description" 350 351 size := len(state.Events) 352 for i, event := range state.Events { 353 msg := event.DisplayMessage 354 if msg == "" { 355 msg = buildDisplayMessage(event) 356 } 357 formattedTime := formatUnixNanoTime(event.Time) 358 events[size-i] = fmt.Sprintf("%s|%s|%s", formattedTime, event.Type, msg) 359 // Reverse order so we are sorted by time 360 } 361 c.Ui.Output(formatList(events)) 362 } 363 364 func buildDisplayMessage(event *api.TaskEvent) string { 365 // Build up the description based on the event type. 366 var desc string 367 switch event.Type { 368 case api.TaskSetup: 369 desc = event.Message 370 case api.TaskStarted: 371 desc = "Task started by client" 372 case api.TaskReceived: 373 desc = "Task received by client" 374 case api.TaskFailedValidation: 375 if event.ValidationError != "" { 376 desc = event.ValidationError 377 } else { 378 desc = "Validation of task failed" 379 } 380 case api.TaskSetupFailure: 381 if event.SetupError != "" { 382 desc = event.SetupError 383 } else { 384 desc = "Task setup failed" 385 } 386 case api.TaskDriverFailure: 387 if event.DriverError != "" { 388 desc = event.DriverError 389 } else { 390 desc = "Failed to start task" 391 } 392 case api.TaskDownloadingArtifacts: 393 desc = "Client is downloading artifacts" 394 case api.TaskArtifactDownloadFailed: 395 if event.DownloadError != "" { 396 desc = event.DownloadError 397 } else { 398 desc = "Failed to download artifacts" 399 } 400 case api.TaskKilling: 401 if event.KillReason != "" { 402 desc = fmt.Sprintf("Killing task: %v", event.KillReason) 403 } else if event.KillTimeout != 0 { 404 desc = fmt.Sprintf("Sent interrupt. Waiting %v before force killing", event.KillTimeout) 405 } else { 406 desc = "Sent interrupt" 407 } 408 case api.TaskKilled: 409 if event.KillError != "" { 410 desc = event.KillError 411 } else { 412 desc = "Task successfully killed" 413 } 414 case api.TaskTerminated: 415 var parts []string 416 parts = append(parts, fmt.Sprintf("Exit Code: %d", event.ExitCode)) 417 418 if event.Signal != 0 { 419 parts = append(parts, fmt.Sprintf("Signal: %d", event.Signal)) 420 } 421 422 if event.Message != "" { 423 parts = append(parts, fmt.Sprintf("Exit Message: %q", event.Message)) 424 } 425 desc = strings.Join(parts, ", ") 426 case api.TaskRestarting: 427 in := fmt.Sprintf("Task restarting in %v", time.Duration(event.StartDelay)) 428 if event.RestartReason != "" && event.RestartReason != restarts.ReasonWithinPolicy { 429 desc = fmt.Sprintf("%s - %s", event.RestartReason, in) 430 } else { 431 desc = in 432 } 433 case api.TaskNotRestarting: 434 if event.RestartReason != "" { 435 desc = event.RestartReason 436 } else { 437 desc = "Task exceeded restart policy" 438 } 439 case api.TaskSiblingFailed: 440 if event.FailedSibling != "" { 441 desc = fmt.Sprintf("Task's sibling %q failed", event.FailedSibling) 442 } else { 443 desc = "Task's sibling failed" 444 } 445 case api.TaskSignaling: 446 sig := event.TaskSignal 447 reason := event.TaskSignalReason 448 449 if sig == "" && reason == "" { 450 desc = "Task being sent a signal" 451 } else if sig == "" { 452 desc = reason 453 } else if reason == "" { 454 desc = fmt.Sprintf("Task being sent signal %v", sig) 455 } else { 456 desc = fmt.Sprintf("Task being sent signal %v: %v", sig, reason) 457 } 458 case api.TaskRestartSignal: 459 if event.RestartReason != "" { 460 desc = event.RestartReason 461 } else { 462 desc = "Task signaled to restart" 463 } 464 case api.TaskDriverMessage: 465 desc = event.DriverMessage 466 case api.TaskLeaderDead: 467 desc = "Leader Task in Group dead" 468 default: 469 desc = event.Message 470 } 471 472 return desc 473 } 474 475 // outputTaskResources prints the task resources for the passed task and if 476 // displayStats is set, verbose resource usage statistics 477 func (c *AllocStatusCommand) outputTaskResources(alloc *api.Allocation, task string, stats *api.AllocResourceUsage, displayStats bool) { 478 resource, ok := alloc.TaskResources[task] 479 if !ok { 480 return 481 } 482 483 c.Ui.Output("Task Resources") 484 var addr []string 485 for _, nw := range resource.Networks { 486 ports := append(nw.DynamicPorts, nw.ReservedPorts...) 487 for _, port := range ports { 488 addr = append(addr, fmt.Sprintf("%v: %v:%v\n", port.Label, nw.IP, port.Value)) 489 } 490 } 491 492 var resourcesOutput []string 493 resourcesOutput = append(resourcesOutput, "CPU|Memory|Disk|Addresses") 494 firstAddr := "" 495 if len(addr) > 0 { 496 firstAddr = addr[0] 497 } 498 499 // Display the rolled up stats. If possible prefer the live statistics 500 cpuUsage := strconv.Itoa(*resource.CPU) 501 memUsage := humanize.IBytes(uint64(*resource.MemoryMB * bytesPerMegabyte)) 502 var deviceStats []*api.DeviceGroupStats 503 504 if stats != nil { 505 if ru, ok := stats.Tasks[task]; ok && ru != nil && ru.ResourceUsage != nil { 506 if cs := ru.ResourceUsage.CpuStats; cs != nil { 507 cpuUsage = fmt.Sprintf("%v/%v", math.Floor(cs.TotalTicks), cpuUsage) 508 } 509 if ms := ru.ResourceUsage.MemoryStats; ms != nil { 510 memUsage = fmt.Sprintf("%v/%v", humanize.IBytes(ms.RSS), memUsage) 511 } 512 deviceStats = ru.ResourceUsage.DeviceStats 513 } 514 } 515 resourcesOutput = append(resourcesOutput, fmt.Sprintf("%v MHz|%v|%v|%v", 516 cpuUsage, 517 memUsage, 518 humanize.IBytes(uint64(*alloc.Resources.DiskMB*bytesPerMegabyte)), 519 firstAddr)) 520 for i := 1; i < len(addr); i++ { 521 resourcesOutput = append(resourcesOutput, fmt.Sprintf("||||%v", addr[i])) 522 } 523 c.Ui.Output(formatListWithSpaces(resourcesOutput)) 524 525 if len(deviceStats) > 0 { 526 c.Ui.Output("") 527 c.Ui.Output("Device Stats") 528 c.Ui.Output(formatList(getDeviceResources(deviceStats))) 529 } 530 531 if stats != nil { 532 if ru, ok := stats.Tasks[task]; ok && ru != nil && displayStats && ru.ResourceUsage != nil { 533 c.Ui.Output("") 534 c.outputVerboseResourceUsage(task, ru.ResourceUsage) 535 } 536 } 537 } 538 539 // outputVerboseResourceUsage outputs the verbose resource usage for the passed 540 // task 541 func (c *AllocStatusCommand) outputVerboseResourceUsage(task string, resourceUsage *api.ResourceUsage) { 542 memoryStats := resourceUsage.MemoryStats 543 cpuStats := resourceUsage.CpuStats 544 deviceStats := resourceUsage.DeviceStats 545 546 if memoryStats != nil && len(memoryStats.Measured) > 0 { 547 c.Ui.Output("Memory Stats") 548 549 // Sort the measured stats 550 sort.Strings(memoryStats.Measured) 551 552 var measuredStats []string 553 for _, measured := range memoryStats.Measured { 554 switch measured { 555 case "RSS": 556 measuredStats = append(measuredStats, humanize.IBytes(memoryStats.RSS)) 557 case "Cache": 558 measuredStats = append(measuredStats, humanize.IBytes(memoryStats.Cache)) 559 case "Swap": 560 measuredStats = append(measuredStats, humanize.IBytes(memoryStats.Swap)) 561 case "Usage": 562 measuredStats = append(measuredStats, humanize.IBytes(memoryStats.Usage)) 563 case "Max Usage": 564 measuredStats = append(measuredStats, humanize.IBytes(memoryStats.MaxUsage)) 565 case "Kernel Usage": 566 measuredStats = append(measuredStats, humanize.IBytes(memoryStats.KernelUsage)) 567 case "Kernel Max Usage": 568 measuredStats = append(measuredStats, humanize.IBytes(memoryStats.KernelMaxUsage)) 569 } 570 } 571 572 out := make([]string, 2) 573 out[0] = strings.Join(memoryStats.Measured, "|") 574 out[1] = strings.Join(measuredStats, "|") 575 c.Ui.Output(formatList(out)) 576 c.Ui.Output("") 577 } 578 579 if cpuStats != nil && len(cpuStats.Measured) > 0 { 580 c.Ui.Output("CPU Stats") 581 582 // Sort the measured stats 583 sort.Strings(cpuStats.Measured) 584 585 var measuredStats []string 586 for _, measured := range cpuStats.Measured { 587 switch measured { 588 case "Percent": 589 percent := strconv.FormatFloat(cpuStats.Percent, 'f', 2, 64) 590 measuredStats = append(measuredStats, fmt.Sprintf("%v%%", percent)) 591 case "Throttled Periods": 592 measuredStats = append(measuredStats, fmt.Sprintf("%v", cpuStats.ThrottledPeriods)) 593 case "Throttled Time": 594 measuredStats = append(measuredStats, fmt.Sprintf("%v", cpuStats.ThrottledTime)) 595 case "User Mode": 596 percent := strconv.FormatFloat(cpuStats.UserMode, 'f', 2, 64) 597 measuredStats = append(measuredStats, fmt.Sprintf("%v%%", percent)) 598 case "System Mode": 599 percent := strconv.FormatFloat(cpuStats.SystemMode, 'f', 2, 64) 600 measuredStats = append(measuredStats, fmt.Sprintf("%v%%", percent)) 601 } 602 } 603 604 out := make([]string, 2) 605 out[0] = strings.Join(cpuStats.Measured, "|") 606 out[1] = strings.Join(measuredStats, "|") 607 c.Ui.Output(formatList(out)) 608 } 609 610 if len(deviceStats) > 0 { 611 c.Ui.Output("") 612 c.Ui.Output("Device Stats") 613 614 printDeviceStats(c.Ui, deviceStats) 615 } 616 } 617 618 // shortTaskStatus prints out the current state of each task. 619 func (c *AllocStatusCommand) shortTaskStatus(alloc *api.Allocation) { 620 tasks := make([]string, 0, len(alloc.TaskStates)+1) 621 tasks = append(tasks, "Name|State|Last Event|Time") 622 for task := range c.sortedTaskStateIterator(alloc.TaskStates) { 623 state := alloc.TaskStates[task] 624 lastState := state.State 625 var lastEvent, lastTime string 626 627 l := len(state.Events) 628 if l != 0 { 629 last := state.Events[l-1] 630 lastEvent = last.Type 631 lastTime = formatUnixNanoTime(last.Time) 632 } 633 634 tasks = append(tasks, fmt.Sprintf("%s|%s|%s|%s", 635 task, lastState, lastEvent, lastTime)) 636 } 637 638 c.Ui.Output(c.Colorize().Color("\n[bold]Tasks[reset]")) 639 c.Ui.Output(formatList(tasks)) 640 } 641 642 // sortedTaskStateIterator is a helper that takes the task state map and returns a 643 // channel that returns the keys in a sorted order. 644 func (c *AllocStatusCommand) sortedTaskStateIterator(m map[string]*api.TaskState) <-chan string { 645 output := make(chan string, len(m)) 646 keys := make([]string, len(m)) 647 i := 0 648 for k := range m { 649 keys[i] = k 650 i++ 651 } 652 sort.Strings(keys) 653 654 for _, key := range keys { 655 output <- key 656 } 657 658 close(output) 659 return output 660 }