github.com/djenriquez/nomad-1@v0.8.1/client/consul_template.go (about) 1 package client 2 3 import ( 4 "fmt" 5 "math/rand" 6 "os" 7 "path/filepath" 8 "sort" 9 "strconv" 10 "strings" 11 "sync" 12 "time" 13 14 ctconf "github.com/hashicorp/consul-template/config" 15 "github.com/hashicorp/consul-template/manager" 16 "github.com/hashicorp/consul-template/signals" 17 envparse "github.com/hashicorp/go-envparse" 18 multierror "github.com/hashicorp/go-multierror" 19 "github.com/hashicorp/nomad/client/config" 20 "github.com/hashicorp/nomad/client/driver/env" 21 "github.com/hashicorp/nomad/helper" 22 "github.com/hashicorp/nomad/nomad/structs" 23 ) 24 25 const ( 26 // consulTemplateSourceName is the source name when using the TaskHooks. 27 consulTemplateSourceName = "Template" 28 29 // hostSrcOption is the Client option that determines whether the template 30 // source may be from the host 31 hostSrcOption = "template.allow_host_source" 32 33 // missingDepEventLimit is the number of missing dependencies that will be 34 // logged before we switch to showing just the number of missing 35 // dependencies. 36 missingDepEventLimit = 3 37 38 // DefaultMaxTemplateEventRate is the default maximum rate at which a 39 // template event should be fired. 40 DefaultMaxTemplateEventRate = 3 * time.Second 41 ) 42 43 // TaskHooks is an interface which provides hooks into the tasks life-cycle 44 type TaskHooks interface { 45 // Restart is used to restart the task 46 Restart(source, reason string, failure bool) 47 48 // Signal is used to signal the task 49 Signal(source, reason string, s os.Signal) error 50 51 // UnblockStart is used to unblock the starting of the task. This should be 52 // called after prestart work is completed 53 UnblockStart(source string) 54 55 // Kill is used to kill the task because of the passed error. If fail is set 56 // to true, the task is marked as failed 57 Kill(source, reason string, fail bool) 58 59 // EmitEvent is used to emit an event to be stored in the tasks events. 60 EmitEvent(source, message string) 61 } 62 63 // TaskTemplateManager is used to run a set of templates for a given task 64 type TaskTemplateManager struct { 65 // config holds the template managers configuration 66 config *TaskTemplateManagerConfig 67 68 // lookup allows looking up the set of Nomad templates by their consul-template ID 69 lookup map[string][]*structs.Template 70 71 // runner is the consul-template runner 72 runner *manager.Runner 73 74 // signals is a lookup map from the string representation of a signal to its 75 // actual signal 76 signals map[string]os.Signal 77 78 // shutdownCh is used to signal and started goroutine to shutdown 79 shutdownCh chan struct{} 80 81 // shutdown marks whether the manager has been shutdown 82 shutdown bool 83 shutdownLock sync.Mutex 84 } 85 86 // TaskTemplateManagerConfig is used to configure an instance of the 87 // TaskTemplateManager 88 type TaskTemplateManagerConfig struct { 89 // Hooks is used to interact with the task the template manager is being run 90 // for 91 Hooks TaskHooks 92 93 // Templates is the set of templates we are managing 94 Templates []*structs.Template 95 96 // ClientConfig is the Nomad Client configuration 97 ClientConfig *config.Config 98 99 // VaultToken is the Vault token for the task. 100 VaultToken string 101 102 // TaskDir is the task's directory 103 TaskDir string 104 105 // EnvBuilder is the environment variable builder for the task. 106 EnvBuilder *env.Builder 107 108 // MaxTemplateEventRate is the maximum rate at which we should emit events. 109 MaxTemplateEventRate time.Duration 110 111 // retryRate is only used for testing and is used to increase the retry rate 112 retryRate time.Duration 113 } 114 115 // Validate validates the configuration. 116 func (c *TaskTemplateManagerConfig) Validate() error { 117 if c == nil { 118 return fmt.Errorf("Nil config passed") 119 } else if c.Hooks == nil { 120 return fmt.Errorf("Invalid task hooks given") 121 } else if c.ClientConfig == nil { 122 return fmt.Errorf("Invalid client config given") 123 } else if c.TaskDir == "" { 124 return fmt.Errorf("Invalid task directory given") 125 } else if c.EnvBuilder == nil { 126 return fmt.Errorf("Invalid task environment given") 127 } else if c.MaxTemplateEventRate == 0 { 128 return fmt.Errorf("Invalid max template event rate given") 129 } 130 131 return nil 132 } 133 134 func NewTaskTemplateManager(config *TaskTemplateManagerConfig) (*TaskTemplateManager, error) { 135 // Check pre-conditions 136 if err := config.Validate(); err != nil { 137 return nil, err 138 } 139 140 tm := &TaskTemplateManager{ 141 config: config, 142 shutdownCh: make(chan struct{}), 143 } 144 145 // Parse the signals that we need 146 for _, tmpl := range config.Templates { 147 if tmpl.ChangeSignal == "" { 148 continue 149 } 150 151 sig, err := signals.Parse(tmpl.ChangeSignal) 152 if err != nil { 153 return nil, fmt.Errorf("Failed to parse signal %q", tmpl.ChangeSignal) 154 } 155 156 if tm.signals == nil { 157 tm.signals = make(map[string]os.Signal) 158 } 159 160 tm.signals[tmpl.ChangeSignal] = sig 161 } 162 163 // Build the consul-template runner 164 runner, lookup, err := templateRunner(config) 165 if err != nil { 166 return nil, err 167 } 168 tm.runner = runner 169 tm.lookup = lookup 170 171 go tm.run() 172 return tm, nil 173 } 174 175 // Stop is used to stop the consul-template runner 176 func (tm *TaskTemplateManager) Stop() { 177 tm.shutdownLock.Lock() 178 defer tm.shutdownLock.Unlock() 179 180 if tm.shutdown { 181 return 182 } 183 184 close(tm.shutdownCh) 185 tm.shutdown = true 186 187 // Stop the consul-template runner 188 if tm.runner != nil { 189 tm.runner.Stop() 190 } 191 } 192 193 // run is the long lived loop that handles errors and templates being rendered 194 func (tm *TaskTemplateManager) run() { 195 // Runner is nil if there is no templates 196 if tm.runner == nil { 197 // Unblock the start if there is nothing to do 198 tm.config.Hooks.UnblockStart(consulTemplateSourceName) 199 return 200 } 201 202 // Start the runner 203 go tm.runner.Start() 204 205 // Block till all the templates have been rendered 206 tm.handleFirstRender() 207 208 // Detect if there was a shutdown. 209 select { 210 case <-tm.shutdownCh: 211 return 212 default: 213 } 214 215 // Read environment variables from env templates before we unblock 216 envMap, err := loadTemplateEnv(tm.config.Templates, tm.config.TaskDir) 217 if err != nil { 218 tm.config.Hooks.Kill(consulTemplateSourceName, err.Error(), true) 219 return 220 } 221 tm.config.EnvBuilder.SetTemplateEnv(envMap) 222 223 // Unblock the task 224 tm.config.Hooks.UnblockStart(consulTemplateSourceName) 225 226 // If all our templates are change mode no-op, then we can exit here 227 if tm.allTemplatesNoop() { 228 return 229 } 230 231 // handle all subsequent render events. 232 tm.handleTemplateRerenders(time.Now()) 233 } 234 235 // handleFirstRender blocks till all templates have been rendered 236 func (tm *TaskTemplateManager) handleFirstRender() { 237 // missingDependencies is the set of missing dependencies. 238 var missingDependencies map[string]struct{} 239 240 // eventTimer is used to trigger the firing of an event showing the missing 241 // dependencies. 242 eventTimer := time.NewTimer(tm.config.MaxTemplateEventRate) 243 if !eventTimer.Stop() { 244 <-eventTimer.C 245 } 246 247 // outstandingEvent tracks whether there is an outstanding event that should 248 // be fired. 249 outstandingEvent := false 250 251 // Wait till all the templates have been rendered 252 WAIT: 253 for { 254 select { 255 case <-tm.shutdownCh: 256 return 257 case err, ok := <-tm.runner.ErrCh: 258 if !ok { 259 continue 260 } 261 262 tm.config.Hooks.Kill(consulTemplateSourceName, err.Error(), true) 263 case <-tm.runner.TemplateRenderedCh(): 264 // A template has been rendered, figure out what to do 265 events := tm.runner.RenderEvents() 266 267 // Not all templates have been rendered yet 268 if len(events) < len(tm.lookup) { 269 continue 270 } 271 272 for _, event := range events { 273 // This template hasn't been rendered 274 if event.LastWouldRender.IsZero() { 275 continue WAIT 276 } 277 } 278 279 break WAIT 280 case <-tm.runner.RenderEventCh(): 281 events := tm.runner.RenderEvents() 282 joinedSet := make(map[string]struct{}) 283 for _, event := range events { 284 missing := event.MissingDeps 285 if missing == nil { 286 continue 287 } 288 289 for _, dep := range missing.List() { 290 joinedSet[dep.String()] = struct{}{} 291 } 292 } 293 294 // Check to see if the new joined set is the same as the old 295 different := len(joinedSet) != len(missingDependencies) 296 if !different { 297 for k := range joinedSet { 298 if _, ok := missingDependencies[k]; !ok { 299 different = true 300 break 301 } 302 } 303 } 304 305 // Nothing to do 306 if !different { 307 continue 308 } 309 310 // Update the missing set 311 missingDependencies = joinedSet 312 313 // Update the event timer channel 314 if !outstandingEvent { 315 // We got new data so reset 316 outstandingEvent = true 317 eventTimer.Reset(tm.config.MaxTemplateEventRate) 318 } 319 case <-eventTimer.C: 320 if missingDependencies == nil { 321 continue 322 } 323 324 // Clear the outstanding event 325 outstandingEvent = false 326 327 // Build the missing set 328 missingSlice := make([]string, 0, len(missingDependencies)) 329 for k := range missingDependencies { 330 missingSlice = append(missingSlice, k) 331 } 332 sort.Strings(missingSlice) 333 334 if l := len(missingSlice); l > missingDepEventLimit { 335 missingSlice[missingDepEventLimit] = fmt.Sprintf("and %d more", l-missingDepEventLimit) 336 missingSlice = missingSlice[:missingDepEventLimit+1] 337 } 338 339 missingStr := strings.Join(missingSlice, ", ") 340 tm.config.Hooks.EmitEvent(consulTemplateSourceName, fmt.Sprintf("Missing: %s", missingStr)) 341 } 342 } 343 } 344 345 // handleTemplateRerenders is used to handle template render events after they 346 // have all rendered. It takes action based on which set of templates re-render. 347 // The passed allRenderedTime is the time at which all templates have rendered. 348 // This is used to avoid signaling the task for any render event before hand. 349 func (tm *TaskTemplateManager) handleTemplateRerenders(allRenderedTime time.Time) { 350 // A lookup for the last time the template was handled 351 handledRenders := make(map[string]time.Time, len(tm.config.Templates)) 352 353 for { 354 select { 355 case <-tm.shutdownCh: 356 return 357 case err, ok := <-tm.runner.ErrCh: 358 if !ok { 359 continue 360 } 361 362 tm.config.Hooks.Kill(consulTemplateSourceName, err.Error(), true) 363 case <-tm.runner.TemplateRenderedCh(): 364 // A template has been rendered, figure out what to do 365 var handling []string 366 signals := make(map[string]struct{}) 367 restart := false 368 var splay time.Duration 369 370 events := tm.runner.RenderEvents() 371 for id, event := range events { 372 373 // First time through 374 if allRenderedTime.After(event.LastDidRender) || allRenderedTime.Equal(event.LastDidRender) { 375 handledRenders[id] = allRenderedTime 376 continue 377 } 378 379 // We have already handled this one 380 if htime := handledRenders[id]; htime.After(event.LastDidRender) || htime.Equal(event.LastDidRender) { 381 continue 382 } 383 384 // Lookup the template and determine what to do 385 tmpls, ok := tm.lookup[id] 386 if !ok { 387 tm.config.Hooks.Kill(consulTemplateSourceName, fmt.Sprintf("template runner returned unknown template id %q", id), true) 388 return 389 } 390 391 // Read environment variables from templates 392 envMap, err := loadTemplateEnv(tm.config.Templates, tm.config.TaskDir) 393 if err != nil { 394 tm.config.Hooks.Kill(consulTemplateSourceName, err.Error(), true) 395 return 396 } 397 tm.config.EnvBuilder.SetTemplateEnv(envMap) 398 399 for _, tmpl := range tmpls { 400 switch tmpl.ChangeMode { 401 case structs.TemplateChangeModeSignal: 402 signals[tmpl.ChangeSignal] = struct{}{} 403 case structs.TemplateChangeModeRestart: 404 restart = true 405 case structs.TemplateChangeModeNoop: 406 continue 407 } 408 409 if tmpl.Splay > splay { 410 splay = tmpl.Splay 411 } 412 } 413 414 handling = append(handling, id) 415 } 416 417 if restart || len(signals) != 0 { 418 if splay != 0 { 419 ns := splay.Nanoseconds() 420 offset := rand.Int63n(ns) 421 t := time.Duration(offset) 422 423 select { 424 case <-time.After(t): 425 case <-tm.shutdownCh: 426 return 427 } 428 } 429 430 // Update handle time 431 for _, id := range handling { 432 handledRenders[id] = events[id].LastDidRender 433 } 434 435 if restart { 436 const failure = false 437 tm.config.Hooks.Restart(consulTemplateSourceName, "template with change_mode restart re-rendered", failure) 438 } else if len(signals) != 0 { 439 var mErr multierror.Error 440 for signal := range signals { 441 err := tm.config.Hooks.Signal(consulTemplateSourceName, "template re-rendered", tm.signals[signal]) 442 if err != nil { 443 multierror.Append(&mErr, err) 444 } 445 } 446 447 if err := mErr.ErrorOrNil(); err != nil { 448 flat := make([]os.Signal, 0, len(signals)) 449 for signal := range signals { 450 flat = append(flat, tm.signals[signal]) 451 } 452 tm.config.Hooks.Kill(consulTemplateSourceName, fmt.Sprintf("Sending signals %v failed: %v", flat, err), true) 453 } 454 } 455 } 456 } 457 } 458 } 459 460 // allTemplatesNoop returns whether all the managed templates have change mode noop. 461 func (tm *TaskTemplateManager) allTemplatesNoop() bool { 462 for _, tmpl := range tm.config.Templates { 463 if tmpl.ChangeMode != structs.TemplateChangeModeNoop { 464 return false 465 } 466 } 467 468 return true 469 } 470 471 // templateRunner returns a consul-template runner for the given templates and a 472 // lookup by destination to the template. If no templates are in the config, a 473 // nil template runner and lookup is returned. 474 func templateRunner(config *TaskTemplateManagerConfig) ( 475 *manager.Runner, map[string][]*structs.Template, error) { 476 477 if len(config.Templates) == 0 { 478 return nil, nil, nil 479 } 480 481 // Parse the templates 482 ctmplMapping, err := parseTemplateConfigs(config) 483 if err != nil { 484 return nil, nil, err 485 } 486 487 // Create the runner configuration. 488 runnerConfig, err := newRunnerConfig(config, ctmplMapping) 489 if err != nil { 490 return nil, nil, err 491 } 492 493 runner, err := manager.NewRunner(runnerConfig, false, false) 494 if err != nil { 495 return nil, nil, err 496 } 497 498 // Set Nomad's environment variables 499 runner.Env = config.EnvBuilder.Build().All() 500 501 // Build the lookup 502 idMap := runner.TemplateConfigMapping() 503 lookup := make(map[string][]*structs.Template, len(idMap)) 504 for id, ctmpls := range idMap { 505 for _, ctmpl := range ctmpls { 506 templates := lookup[id] 507 templates = append(templates, ctmplMapping[ctmpl]) 508 lookup[id] = templates 509 } 510 } 511 512 return runner, lookup, nil 513 } 514 515 // parseTemplateConfigs converts the tasks templates in the config into 516 // consul-templates 517 func parseTemplateConfigs(config *TaskTemplateManagerConfig) (map[ctconf.TemplateConfig]*structs.Template, error) { 518 allowAbs := config.ClientConfig.ReadBoolDefault(hostSrcOption, true) 519 taskEnv := config.EnvBuilder.Build() 520 521 ctmpls := make(map[ctconf.TemplateConfig]*structs.Template, len(config.Templates)) 522 for _, tmpl := range config.Templates { 523 var src, dest string 524 if tmpl.SourcePath != "" { 525 if filepath.IsAbs(tmpl.SourcePath) { 526 if !allowAbs { 527 return nil, fmt.Errorf("Specifying absolute template paths disallowed by client config: %q", tmpl.SourcePath) 528 } 529 530 src = tmpl.SourcePath 531 } else { 532 src = filepath.Join(config.TaskDir, taskEnv.ReplaceEnv(tmpl.SourcePath)) 533 } 534 } 535 if tmpl.DestPath != "" { 536 dest = filepath.Join(config.TaskDir, taskEnv.ReplaceEnv(tmpl.DestPath)) 537 } 538 539 ct := ctconf.DefaultTemplateConfig() 540 ct.Source = &src 541 ct.Destination = &dest 542 ct.Contents = &tmpl.EmbeddedTmpl 543 ct.LeftDelim = &tmpl.LeftDelim 544 ct.RightDelim = &tmpl.RightDelim 545 546 // Set the permissions 547 if tmpl.Perms != "" { 548 v, err := strconv.ParseUint(tmpl.Perms, 8, 12) 549 if err != nil { 550 return nil, fmt.Errorf("Failed to parse %q as octal: %v", tmpl.Perms, err) 551 } 552 m := os.FileMode(v) 553 ct.Perms = &m 554 } 555 ct.Finalize() 556 557 ctmpls[*ct] = tmpl 558 } 559 560 return ctmpls, nil 561 } 562 563 // newRunnerConfig returns a consul-template runner configuration, setting the 564 // Vault and Consul configurations based on the clients configs. 565 func newRunnerConfig(config *TaskTemplateManagerConfig, 566 templateMapping map[ctconf.TemplateConfig]*structs.Template) (*ctconf.Config, error) { 567 568 cc := config.ClientConfig 569 conf := ctconf.DefaultConfig() 570 571 // Gather the consul-template templates 572 flat := ctconf.TemplateConfigs(make([]*ctconf.TemplateConfig, 0, len(templateMapping))) 573 for ctmpl := range templateMapping { 574 local := ctmpl 575 flat = append(flat, &local) 576 } 577 conf.Templates = &flat 578 579 // Go through the templates and determine the minimum Vault grace 580 vaultGrace := time.Duration(-1) 581 for _, tmpl := range templateMapping { 582 // Initial condition 583 if vaultGrace < 0 { 584 vaultGrace = tmpl.VaultGrace 585 } else if tmpl.VaultGrace < vaultGrace { 586 vaultGrace = tmpl.VaultGrace 587 } 588 } 589 590 // Force faster retries 591 if config.retryRate != 0 { 592 rate := config.retryRate 593 conf.Consul.Retry.Backoff = &rate 594 } 595 596 // Setup the Consul config 597 if cc.ConsulConfig != nil { 598 conf.Consul.Address = &cc.ConsulConfig.Addr 599 conf.Consul.Token = &cc.ConsulConfig.Token 600 601 if cc.ConsulConfig.EnableSSL != nil && *cc.ConsulConfig.EnableSSL { 602 verify := cc.ConsulConfig.VerifySSL != nil && *cc.ConsulConfig.VerifySSL 603 conf.Consul.SSL = &ctconf.SSLConfig{ 604 Enabled: helper.BoolToPtr(true), 605 Verify: &verify, 606 Cert: &cc.ConsulConfig.CertFile, 607 Key: &cc.ConsulConfig.KeyFile, 608 CaCert: &cc.ConsulConfig.CAFile, 609 } 610 } 611 612 if cc.ConsulConfig.Auth != "" { 613 parts := strings.SplitN(cc.ConsulConfig.Auth, ":", 2) 614 if len(parts) != 2 { 615 return nil, fmt.Errorf("Failed to parse Consul Auth config") 616 } 617 618 conf.Consul.Auth = &ctconf.AuthConfig{ 619 Enabled: helper.BoolToPtr(true), 620 Username: &parts[0], 621 Password: &parts[1], 622 } 623 } 624 } 625 626 // Setup the Vault config 627 // Always set these to ensure nothing is picked up from the environment 628 emptyStr := "" 629 conf.Vault.RenewToken = helper.BoolToPtr(false) 630 conf.Vault.Token = &emptyStr 631 if cc.VaultConfig != nil && cc.VaultConfig.IsEnabled() { 632 conf.Vault.Address = &cc.VaultConfig.Addr 633 conf.Vault.Token = &config.VaultToken 634 conf.Vault.Grace = helper.TimeToPtr(vaultGrace) 635 636 if strings.HasPrefix(cc.VaultConfig.Addr, "https") || cc.VaultConfig.TLSCertFile != "" { 637 skipVerify := cc.VaultConfig.TLSSkipVerify != nil && *cc.VaultConfig.TLSSkipVerify 638 verify := !skipVerify 639 conf.Vault.SSL = &ctconf.SSLConfig{ 640 Enabled: helper.BoolToPtr(true), 641 Verify: &verify, 642 Cert: &cc.VaultConfig.TLSCertFile, 643 Key: &cc.VaultConfig.TLSKeyFile, 644 CaCert: &cc.VaultConfig.TLSCaFile, 645 CaPath: &cc.VaultConfig.TLSCaPath, 646 ServerName: &cc.VaultConfig.TLSServerName, 647 } 648 } else { 649 conf.Vault.SSL = &ctconf.SSLConfig{ 650 Enabled: helper.BoolToPtr(false), 651 Verify: helper.BoolToPtr(false), 652 Cert: &emptyStr, 653 Key: &emptyStr, 654 CaCert: &emptyStr, 655 CaPath: &emptyStr, 656 ServerName: &emptyStr, 657 } 658 } 659 } 660 661 conf.Finalize() 662 return conf, nil 663 } 664 665 // loadTemplateEnv loads task environment variables from all templates. 666 func loadTemplateEnv(tmpls []*structs.Template, taskDir string) (map[string]string, error) { 667 all := make(map[string]string, 50) 668 for _, t := range tmpls { 669 if !t.Envvars { 670 continue 671 } 672 f, err := os.Open(filepath.Join(taskDir, t.DestPath)) 673 if err != nil { 674 return nil, fmt.Errorf("error opening env template: %v", err) 675 } 676 defer f.Close() 677 678 // Parse environment fil 679 vars, err := envparse.Parse(f) 680 if err != nil { 681 return nil, fmt.Errorf("error parsing env template %q: %v", t.DestPath, err) 682 } 683 for k, v := range vars { 684 all[k] = v 685 } 686 } 687 return all, nil 688 }