github.com/inge4pres/terraform@v0.7.5-0.20160930053151-bd083f84f376/builtin/provisioners/chef/resource_provisioner.go (about) 1 package chef 2 3 import ( 4 "bytes" 5 "encoding/json" 6 "errors" 7 "fmt" 8 "io" 9 "log" 10 "os" 11 "path" 12 "regexp" 13 "strings" 14 "text/template" 15 "time" 16 17 "github.com/hashicorp/terraform/communicator" 18 "github.com/hashicorp/terraform/communicator/remote" 19 "github.com/hashicorp/terraform/terraform" 20 "github.com/mitchellh/go-homedir" 21 "github.com/mitchellh/go-linereader" 22 "github.com/mitchellh/mapstructure" 23 ) 24 25 const ( 26 clienrb = "client.rb" 27 defaultEnv = "_default" 28 firstBoot = "first-boot.json" 29 logfileDir = "logfiles" 30 linuxChefCmd = "chef-client" 31 linuxConfDir = "/etc/chef" 32 linuxNoOutput = "> /dev/null 2>&1" 33 linuxGemCmd = "/opt/chef/embedded/bin/gem" 34 linuxKnifeCmd = "knife" 35 secretKey = "encrypted_data_bag_secret" 36 windowsChefCmd = "cmd /c chef-client" 37 windowsConfDir = "C:/chef" 38 windowsNoOutput = "> nul 2>&1" 39 windowsGemCmd = "C:/opscode/chef/embedded/bin/gem" 40 windowsKnifeCmd = "cmd /c knife" 41 ) 42 43 const clientConf = ` 44 log_location STDOUT 45 chef_server_url "{{ .ServerURL }}" 46 node_name "{{ .NodeName }}" 47 {{ if .UsePolicyfile }} 48 use_policyfile true 49 policy_group "{{ .PolicyGroup }}" 50 policy_name "{{ .PolicyName }}" 51 {{ end -}} 52 53 {{ if .HTTPProxy }} 54 http_proxy "{{ .HTTPProxy }}" 55 ENV['http_proxy'] = "{{ .HTTPProxy }}" 56 ENV['HTTP_PROXY'] = "{{ .HTTPProxy }}" 57 {{ end -}} 58 59 {{ if .HTTPSProxy }} 60 https_proxy "{{ .HTTPSProxy }}" 61 ENV['https_proxy'] = "{{ .HTTPSProxy }}" 62 ENV['HTTPS_PROXY'] = "{{ .HTTPSProxy }}" 63 {{ end -}} 64 65 {{ if .NOProxy }} 66 no_proxy "{{ join .NOProxy "," }}" 67 ENV['no_proxy'] = "{{ join .NOProxy "," }}" 68 {{ end -}} 69 70 {{ if .SSLVerifyMode }} 71 ssl_verify_mode {{ .SSLVerifyMode }} 72 {{- end -}} 73 74 {{ if .DisableReporting }} 75 enable_reporting false 76 {{ end -}} 77 78 {{ if .ClientOptions }} 79 {{ join .ClientOptions "\n" }} 80 {{ end }} 81 ` 82 83 // Provisioner represents a Chef provisioner 84 type Provisioner struct { 85 AttributesJSON string `mapstructure:"attributes_json"` 86 ClientOptions []string `mapstructure:"client_options"` 87 DisableReporting bool `mapstructure:"disable_reporting"` 88 Environment string `mapstructure:"environment"` 89 FetchChefCertificates bool `mapstructure:"fetch_chef_certificates"` 90 LogToFile bool `mapstructure:"log_to_file"` 91 UsePolicyfile bool `mapstructure:"use_policyfile"` 92 PolicyGroup string `mapstructure:"policy_group"` 93 PolicyName string `mapstructure:"policy_name"` 94 HTTPProxy string `mapstructure:"http_proxy"` 95 HTTPSProxy string `mapstructure:"https_proxy"` 96 NOProxy []string `mapstructure:"no_proxy"` 97 NodeName string `mapstructure:"node_name"` 98 OhaiHints []string `mapstructure:"ohai_hints"` 99 OSType string `mapstructure:"os_type"` 100 RecreateClient bool `mapstructure:"recreate_client"` 101 PreventSudo bool `mapstructure:"prevent_sudo"` 102 RunList []string `mapstructure:"run_list"` 103 SecretKey string `mapstructure:"secret_key"` 104 ServerURL string `mapstructure:"server_url"` 105 SkipInstall bool `mapstructure:"skip_install"` 106 SSLVerifyMode string `mapstructure:"ssl_verify_mode"` 107 UserName string `mapstructure:"user_name"` 108 UserKey string `mapstructure:"user_key"` 109 VaultJSON string `mapstructure:"vault_json"` 110 Version string `mapstructure:"version"` 111 112 attributes map[string]interface{} 113 vaults map[string]string 114 115 cleanupUserKeyCmd string 116 createConfigFiles func(terraform.UIOutput, communicator.Communicator) error 117 installChefClient func(terraform.UIOutput, communicator.Communicator) error 118 fetchChefCertificates func(terraform.UIOutput, communicator.Communicator) error 119 generateClientKey func(terraform.UIOutput, communicator.Communicator) error 120 configureVaults func(terraform.UIOutput, communicator.Communicator) error 121 runChefClient func(terraform.UIOutput, communicator.Communicator) error 122 useSudo bool 123 124 // Deprecated Fields 125 ValidationClientName string `mapstructure:"validation_client_name"` 126 ValidationKey string `mapstructure:"validation_key"` 127 } 128 129 // ResourceProvisioner represents a generic chef provisioner 130 type ResourceProvisioner struct{} 131 132 // Apply executes the file provisioner 133 func (r *ResourceProvisioner) Apply( 134 o terraform.UIOutput, 135 s *terraform.InstanceState, 136 c *terraform.ResourceConfig) error { 137 // Decode the raw config for this provisioner 138 p, err := r.decodeConfig(c) 139 if err != nil { 140 return err 141 } 142 143 if p.OSType == "" { 144 switch s.Ephemeral.ConnInfo["type"] { 145 case "ssh", "": // The default connection type is ssh, so if the type is empty assume ssh 146 p.OSType = "linux" 147 case "winrm": 148 p.OSType = "windows" 149 default: 150 return fmt.Errorf("Unsupported connection type: %s", s.Ephemeral.ConnInfo["type"]) 151 } 152 } 153 154 // Set some values based on the targeted OS 155 switch p.OSType { 156 case "linux": 157 p.cleanupUserKeyCmd = fmt.Sprintf("rm -f %s", path.Join(linuxConfDir, p.UserName+".pem")) 158 p.createConfigFiles = p.linuxCreateConfigFiles 159 p.installChefClient = p.linuxInstallChefClient 160 p.fetchChefCertificates = p.fetchChefCertificatesFunc(linuxKnifeCmd, linuxConfDir) 161 p.generateClientKey = p.generateClientKeyFunc(linuxKnifeCmd, linuxConfDir, linuxNoOutput) 162 p.configureVaults = p.configureVaultsFunc(linuxGemCmd, linuxKnifeCmd, linuxConfDir) 163 p.runChefClient = p.runChefClientFunc(linuxChefCmd, linuxConfDir) 164 p.useSudo = !p.PreventSudo && s.Ephemeral.ConnInfo["user"] != "root" 165 case "windows": 166 p.cleanupUserKeyCmd = fmt.Sprintf("cd %s && del /F /Q %s", windowsConfDir, p.UserName+".pem") 167 p.createConfigFiles = p.windowsCreateConfigFiles 168 p.installChefClient = p.windowsInstallChefClient 169 p.fetchChefCertificates = p.fetchChefCertificatesFunc(windowsKnifeCmd, windowsConfDir) 170 p.generateClientKey = p.generateClientKeyFunc(windowsKnifeCmd, windowsConfDir, windowsNoOutput) 171 p.configureVaults = p.configureVaultsFunc(windowsGemCmd, windowsKnifeCmd, windowsConfDir) 172 p.runChefClient = p.runChefClientFunc(windowsChefCmd, windowsConfDir) 173 p.useSudo = false 174 default: 175 return fmt.Errorf("Unsupported os type: %s", p.OSType) 176 } 177 178 // Get a new communicator 179 comm, err := communicator.New(s) 180 if err != nil { 181 return err 182 } 183 184 // Wait and retry until we establish the connection 185 err = retryFunc(comm.Timeout(), func() error { 186 err := comm.Connect(o) 187 return err 188 }) 189 if err != nil { 190 return err 191 } 192 defer comm.Disconnect() 193 194 // Make sure we always delete the user key from the new node! 195 defer func() { 196 o.Output("Cleanup user key...") 197 if err := p.runCommand(o, comm, p.cleanupUserKeyCmd); err != nil { 198 o.Output("WARNING: Failed to cleanup user key on new node: " + err.Error()) 199 } 200 }() 201 202 if !p.SkipInstall { 203 if err := p.installChefClient(o, comm); err != nil { 204 return err 205 } 206 } 207 208 o.Output("Creating configuration files...") 209 if err := p.createConfigFiles(o, comm); err != nil { 210 return err 211 } 212 213 if p.FetchChefCertificates { 214 o.Output("Fetch Chef certificates...") 215 if err := p.fetchChefCertificates(o, comm); err != nil { 216 return err 217 } 218 } 219 220 o.Output("Generate the private key...") 221 if err := p.generateClientKey(o, comm); err != nil { 222 return err 223 } 224 225 if p.VaultJSON != "" { 226 o.Output("Configure Chef vaults...") 227 if err := p.configureVaults(o, comm); err != nil { 228 return err 229 } 230 } 231 232 o.Output("Starting initial Chef-Client run...") 233 if err := p.runChefClient(o, comm); err != nil { 234 return err 235 } 236 237 return nil 238 } 239 240 // Validate checks if the required arguments are configured 241 func (r *ResourceProvisioner) Validate(c *terraform.ResourceConfig) (ws []string, es []error) { 242 p, err := r.decodeConfig(c) 243 if err != nil { 244 es = append(es, err) 245 return ws, es 246 } 247 248 if p.NodeName == "" { 249 es = append(es, errors.New("Key not found: node_name")) 250 } 251 if !p.UsePolicyfile && p.RunList == nil { 252 es = append(es, errors.New("Key not found: run_list")) 253 } 254 if p.ServerURL == "" { 255 es = append(es, errors.New("Key not found: server_url")) 256 } 257 if p.UsePolicyfile && p.PolicyName == "" { 258 es = append(es, errors.New("Policyfile enabled but key not found: policy_name")) 259 } 260 if p.UsePolicyfile && p.PolicyGroup == "" { 261 es = append(es, errors.New("Policyfile enabled but key not found: policy_group")) 262 } 263 if p.UserName == "" && p.ValidationClientName == "" { 264 es = append(es, errors.New( 265 "One of user_name or the deprecated validation_client_name must be provided")) 266 } 267 if p.UserKey == "" && p.ValidationKey == "" { 268 es = append(es, errors.New( 269 "One of user_key or the deprecated validation_key must be provided")) 270 } 271 if p.ValidationClientName != "" { 272 ws = append(ws, "validation_client_name is deprecated, please use user_name instead") 273 } 274 if p.ValidationKey != "" { 275 ws = append(ws, "validation_key is deprecated, please use user_key instead") 276 277 if p.RecreateClient { 278 es = append(es, errors.New( 279 "Cannot use recreate_client=true with the deprecated validation_key, please provide a user_key")) 280 } 281 if p.VaultJSON != "" { 282 es = append(es, errors.New( 283 "Cannot configure chef vaults using the deprecated validation_key, please provide a user_key")) 284 } 285 } 286 287 return ws, es 288 } 289 290 func (r *ResourceProvisioner) decodeConfig(c *terraform.ResourceConfig) (*Provisioner, error) { 291 p := new(Provisioner) 292 293 decConf := &mapstructure.DecoderConfig{ 294 ErrorUnused: true, 295 WeaklyTypedInput: true, 296 Result: p, 297 } 298 dec, err := mapstructure.NewDecoder(decConf) 299 if err != nil { 300 return nil, err 301 } 302 303 // We need to merge both configs into a single map first. Order is 304 // important as we need to make sure interpolated values are used 305 // over raw values. This makes sure that all values are there even 306 // if some still need to be interpolated later on. Without this 307 // the validation will fail when using a variable for a required 308 // parameter (the node_name for example). 309 m := make(map[string]interface{}) 310 311 for k, v := range c.Raw { 312 m[k] = v 313 } 314 315 for k, v := range c.Config { 316 m[k] = v 317 } 318 319 if err := dec.Decode(m); err != nil { 320 return nil, err 321 } 322 323 // Make sure the supplied URL has a trailing slash 324 p.ServerURL = strings.TrimSuffix(p.ServerURL, "/") + "/" 325 326 if p.Environment == "" { 327 p.Environment = defaultEnv 328 } 329 330 for i, hint := range p.OhaiHints { 331 hintPath, err := homedir.Expand(hint) 332 if err != nil { 333 return nil, fmt.Errorf("Error expanding the path %s: %v", hint, err) 334 } 335 p.OhaiHints[i] = hintPath 336 } 337 338 if p.UserName == "" && p.ValidationClientName != "" { 339 p.UserName = p.ValidationClientName 340 } 341 342 if p.UserKey == "" && p.ValidationKey != "" { 343 p.UserKey = p.ValidationKey 344 } 345 346 if attrs, ok := c.Config["attributes_json"].(string); ok { 347 var m map[string]interface{} 348 if err := json.Unmarshal([]byte(attrs), &m); err != nil { 349 return nil, fmt.Errorf("Error parsing attributes_json: %v", err) 350 } 351 p.attributes = m 352 } 353 354 if vaults, ok := c.Config["vault_json"].(string); ok { 355 var m map[string]string 356 if err := json.Unmarshal([]byte(vaults), &m); err != nil { 357 return nil, fmt.Errorf("Error parsing vault_json: %v", err) 358 } 359 p.vaults = m 360 } 361 362 return p, nil 363 } 364 365 func (p *Provisioner) deployConfigFiles( 366 o terraform.UIOutput, 367 comm communicator.Communicator, 368 confDir string) error { 369 // Copy the user key to the new instance 370 pk := strings.NewReader(p.UserKey) 371 if err := comm.Upload(path.Join(confDir, p.UserName+".pem"), pk); err != nil { 372 return fmt.Errorf("Uploading user key failed: %v", err) 373 } 374 375 if p.SecretKey != "" { 376 // Copy the secret key to the new instance 377 s := strings.NewReader(p.SecretKey) 378 if err := comm.Upload(path.Join(confDir, secretKey), s); err != nil { 379 return fmt.Errorf("Uploading %s failed: %v", secretKey, err) 380 } 381 } 382 383 // Make sure the SSLVerifyMode value is written as a symbol 384 if p.SSLVerifyMode != "" && !strings.HasPrefix(p.SSLVerifyMode, ":") { 385 p.SSLVerifyMode = fmt.Sprintf(":%s", p.SSLVerifyMode) 386 } 387 388 // Make strings.Join available for use within the template 389 funcMap := template.FuncMap{ 390 "join": strings.Join, 391 } 392 393 // Create a new template and parse the client config into it 394 t := template.Must(template.New(clienrb).Funcs(funcMap).Parse(clientConf)) 395 396 var buf bytes.Buffer 397 err := t.Execute(&buf, p) 398 if err != nil { 399 return fmt.Errorf("Error executing %s template: %s", clienrb, err) 400 } 401 402 // Copy the client config to the new instance 403 if err := comm.Upload(path.Join(confDir, clienrb), &buf); err != nil { 404 return fmt.Errorf("Uploading %s failed: %v", clienrb, err) 405 } 406 407 // Create a map with first boot settings 408 fb := make(map[string]interface{}) 409 if p.attributes != nil { 410 fb = p.attributes 411 } 412 413 // Check if the run_list was also in the attributes and if so log a warning 414 // that it will be overwritten with the value of the run_list argument. 415 if _, found := fb["run_list"]; found { 416 log.Printf("[WARNING] Found a 'run_list' specified in the configured attributes! " + 417 "This value will be overwritten by the value of the `run_list` argument!") 418 } 419 420 // Add the initial runlist to the first boot settings 421 if !p.UsePolicyfile { 422 fb["run_list"] = p.RunList 423 } 424 425 // Marshal the first boot settings to JSON 426 d, err := json.Marshal(fb) 427 if err != nil { 428 return fmt.Errorf("Failed to create %s data: %s", firstBoot, err) 429 } 430 431 // Copy the first-boot.json to the new instance 432 if err := comm.Upload(path.Join(confDir, firstBoot), bytes.NewReader(d)); err != nil { 433 return fmt.Errorf("Uploading %s failed: %v", firstBoot, err) 434 } 435 436 return nil 437 } 438 439 func (p *Provisioner) deployOhaiHints( 440 o terraform.UIOutput, 441 comm communicator.Communicator, 442 hintDir string) error { 443 for _, hint := range p.OhaiHints { 444 // Open the hint file 445 f, err := os.Open(hint) 446 if err != nil { 447 return err 448 } 449 defer f.Close() 450 451 // Copy the hint to the new instance 452 if err := comm.Upload(path.Join(hintDir, path.Base(hint)), f); err != nil { 453 return fmt.Errorf("Uploading %s failed: %v", path.Base(hint), err) 454 } 455 } 456 457 return nil 458 } 459 460 func (p *Provisioner) fetchChefCertificatesFunc( 461 knifeCmd string, 462 confDir string) func(terraform.UIOutput, communicator.Communicator) error { 463 return func(o terraform.UIOutput, comm communicator.Communicator) error { 464 clientrb := path.Join(confDir, clienrb) 465 cmd := fmt.Sprintf("%s ssl fetch -c %s", knifeCmd, clientrb) 466 467 return p.runCommand(o, comm, cmd) 468 } 469 } 470 471 func (p *Provisioner) generateClientKeyFunc( 472 knifeCmd string, 473 confDir string, 474 noOutput string) func(terraform.UIOutput, communicator.Communicator) error { 475 return func(o terraform.UIOutput, comm communicator.Communicator) error { 476 options := fmt.Sprintf("-c %s -u %s --key %s", 477 path.Join(confDir, clienrb), 478 p.UserName, 479 path.Join(confDir, p.UserName+".pem"), 480 ) 481 482 // See if we already have a node object 483 getNodeCmd := fmt.Sprintf("%s node show %s %s %s", knifeCmd, p.NodeName, options, noOutput) 484 node := p.runCommand(o, comm, getNodeCmd) == nil 485 486 // See if we already have a client object 487 getClientCmd := fmt.Sprintf("%s client show %s %s %s", knifeCmd, p.NodeName, options, noOutput) 488 client := p.runCommand(o, comm, getClientCmd) == nil 489 490 // If we have a client, we can only continue if we are to recreate the client 491 if client && !p.RecreateClient { 492 return fmt.Errorf( 493 "Chef client %q already exists, set recreate_client=true to automatically recreate the client", p.NodeName) 494 } 495 496 // If the node exists, try to delete it 497 if node { 498 deleteNodeCmd := fmt.Sprintf("%s node delete %s -y %s", 499 knifeCmd, 500 p.NodeName, 501 options, 502 ) 503 if err := p.runCommand(o, comm, deleteNodeCmd); err != nil { 504 return err 505 } 506 } 507 508 // If the client exists, try to delete it 509 if client { 510 deleteClientCmd := fmt.Sprintf("%s client delete %s -y %s", 511 knifeCmd, 512 p.NodeName, 513 options, 514 ) 515 if err := p.runCommand(o, comm, deleteClientCmd); err != nil { 516 return err 517 } 518 } 519 520 // Create the new client object 521 createClientCmd := fmt.Sprintf("%s client create %s -d -f %s %s", 522 knifeCmd, 523 p.NodeName, 524 path.Join(confDir, "client.pem"), 525 options, 526 ) 527 528 return p.runCommand(o, comm, createClientCmd) 529 } 530 } 531 532 func (p *Provisioner) configureVaultsFunc( 533 gemCmd string, 534 knifeCmd string, 535 confDir string) func(terraform.UIOutput, communicator.Communicator) error { 536 return func(o terraform.UIOutput, comm communicator.Communicator) error { 537 if err := p.runCommand(o, comm, fmt.Sprintf("%s install chef-vault", gemCmd)); err != nil { 538 return err 539 } 540 541 options := fmt.Sprintf("-c %s -u %s --key %s", 542 path.Join(confDir, clienrb), 543 p.UserName, 544 path.Join(confDir, p.UserName+".pem"), 545 ) 546 547 for vault, item := range p.vaults { 548 updateCmd := fmt.Sprintf("%s vault update %s %s -A %s -M client %s", 549 knifeCmd, 550 vault, 551 item, 552 p.NodeName, 553 options, 554 ) 555 if err := p.runCommand(o, comm, updateCmd); err != nil { 556 return err 557 } 558 } 559 560 return nil 561 } 562 } 563 564 func (p *Provisioner) runChefClientFunc( 565 chefCmd string, 566 confDir string) func(terraform.UIOutput, communicator.Communicator) error { 567 return func(o terraform.UIOutput, comm communicator.Communicator) error { 568 fb := path.Join(confDir, firstBoot) 569 var cmd string 570 571 // Policyfiles do not support chef environments, so don't pass the `-E` flag. 572 if p.UsePolicyfile { 573 cmd = fmt.Sprintf("%s -j %q", chefCmd, fb) 574 } else { 575 cmd = fmt.Sprintf("%s -j %q -E %q", chefCmd, fb, p.Environment) 576 } 577 578 if p.LogToFile { 579 if err := os.MkdirAll(logfileDir, 0755); err != nil { 580 return fmt.Errorf("Error creating logfile directory %s: %v", logfileDir, err) 581 } 582 583 logFile := path.Join(logfileDir, p.NodeName) 584 f, err := os.Create(path.Join(logFile)) 585 if err != nil { 586 return fmt.Errorf("Error creating logfile %s: %v", logFile, err) 587 } 588 f.Close() 589 590 o.Output("Writing Chef Client output to " + logFile) 591 o = p 592 } 593 594 return p.runCommand(o, comm, cmd) 595 } 596 } 597 598 // Output implementation of terraform.UIOutput interface 599 func (p *Provisioner) Output(output string) { 600 logFile := path.Join(logfileDir, p.NodeName) 601 f, err := os.OpenFile(logFile, os.O_APPEND|os.O_WRONLY, 0666) 602 if err != nil { 603 log.Printf("Error creating logfile %s: %v", logFile, err) 604 return 605 } 606 defer f.Close() 607 608 // These steps are needed to remove any ANSI escape codes used to colorize 609 // the output and to make sure we have proper line endings before writing 610 // the string to the logfile. 611 re := regexp.MustCompile(`\x1b\[[0-9;]+m`) 612 output = re.ReplaceAllString(output, "") 613 output = strings.Replace(output, "\r", "\n", -1) 614 615 if _, err := f.WriteString(output); err != nil { 616 log.Printf("Error writing output to logfile %s: %v", logFile, err) 617 } 618 619 if err := f.Sync(); err != nil { 620 log.Printf("Error saving logfile %s to disk: %v", logFile, err) 621 } 622 } 623 624 // runCommand is used to run already prepared commands 625 func (p *Provisioner) runCommand( 626 o terraform.UIOutput, 627 comm communicator.Communicator, 628 command string) error { 629 // Unless prevented, prefix the command with sudo 630 if p.useSudo { 631 command = "sudo " + command 632 } 633 634 outR, outW := io.Pipe() 635 errR, errW := io.Pipe() 636 outDoneCh := make(chan struct{}) 637 errDoneCh := make(chan struct{}) 638 go p.copyOutput(o, outR, outDoneCh) 639 go p.copyOutput(o, errR, errDoneCh) 640 641 cmd := &remote.Cmd{ 642 Command: command, 643 Stdout: outW, 644 Stderr: errW, 645 } 646 647 err := comm.Start(cmd) 648 if err != nil { 649 return fmt.Errorf("Error executing command %q: %v", cmd.Command, err) 650 } 651 652 cmd.Wait() 653 if cmd.ExitStatus != 0 { 654 err = fmt.Errorf( 655 "Command %q exited with non-zero exit status: %d", cmd.Command, cmd.ExitStatus) 656 } 657 658 // Wait for output to clean up 659 outW.Close() 660 errW.Close() 661 <-outDoneCh 662 <-errDoneCh 663 664 return err 665 } 666 667 func (p *Provisioner) copyOutput(o terraform.UIOutput, r io.Reader, doneCh chan<- struct{}) { 668 defer close(doneCh) 669 lr := linereader.New(r) 670 for line := range lr.Ch { 671 o.Output(line) 672 } 673 } 674 675 // retryFunc is used to retry a function for a given duration 676 func retryFunc(timeout time.Duration, f func() error) error { 677 finish := time.After(timeout) 678 for { 679 err := f() 680 if err == nil { 681 return nil 682 } 683 log.Printf("Retryable error: %v", err) 684 685 select { 686 case <-finish: 687 return err 688 case <-time.After(3 * time.Second): 689 } 690 } 691 }