github.com/maier/nomad@v0.4.1-0.20161110003312-a9e3d0b8549d/client/consul_template.go (about) 1 package client 2 3 import ( 4 "fmt" 5 "os" 6 "path/filepath" 7 "strings" 8 "sync" 9 "time" 10 11 ctconf "github.com/hashicorp/consul-template/config" 12 "github.com/hashicorp/consul-template/manager" 13 "github.com/hashicorp/consul-template/signals" 14 "github.com/hashicorp/consul-template/watch" 15 multierror "github.com/hashicorp/go-multierror" 16 "github.com/hashicorp/nomad/client/config" 17 "github.com/hashicorp/nomad/client/driver/env" 18 "github.com/hashicorp/nomad/nomad/structs" 19 ) 20 21 const ( 22 // hostSrcOption is the Client option that determines whether the template 23 // source may be from the host 24 hostSrcOption = "template.allow_host_source" 25 ) 26 27 var ( 28 // testRetryRate is used to speed up tests by setting consul-templates retry 29 // rate to something low 30 testRetryRate time.Duration = 0 31 ) 32 33 // TaskHooks is an interface which provides hooks into the tasks life-cycle 34 type TaskHooks interface { 35 // Restart is used to restart the task 36 Restart(source, reason string) 37 38 // Signal is used to signal the task 39 Signal(source, reason string, s os.Signal) error 40 41 // UnblockStart is used to unblock the starting of the task. This should be 42 // called after prestart work is completed 43 UnblockStart(source string) 44 45 // Kill is used to kill the task because of the passed error. If fail is set 46 // to true, the task is marked as failed 47 Kill(source, reason string, fail bool) 48 } 49 50 // TaskTemplateManager is used to run a set of templates for a given task 51 type TaskTemplateManager struct { 52 // templates is the set of templates we are managing 53 templates []*structs.Template 54 55 // lookup allows looking up the set of Nomad templates by their consul-template ID 56 lookup map[string][]*structs.Template 57 58 // hooks is used to signal/restart the task as templates are rendered 59 hook TaskHooks 60 61 // runner is the consul-template runner 62 runner *manager.Runner 63 64 // signals is a lookup map from the string representation of a signal to its 65 // actual signal 66 signals map[string]os.Signal 67 68 // shutdownCh is used to signal and started goroutine to shutdown 69 shutdownCh chan struct{} 70 71 // shutdown marks whether the manager has been shutdown 72 shutdown bool 73 shutdownLock sync.Mutex 74 } 75 76 func NewTaskTemplateManager(hook TaskHooks, tmpls []*structs.Template, 77 config *config.Config, vaultToken, taskDir string, 78 taskEnv *env.TaskEnvironment) (*TaskTemplateManager, error) { 79 80 // Check pre-conditions 81 if hook == nil { 82 return nil, fmt.Errorf("Invalid task hook given") 83 } else if config == nil { 84 return nil, fmt.Errorf("Invalid config given") 85 } else if taskDir == "" { 86 return nil, fmt.Errorf("Invalid task directory given") 87 } else if taskEnv == nil { 88 return nil, fmt.Errorf("Invalid task environment given") 89 } 90 91 tm := &TaskTemplateManager{ 92 templates: tmpls, 93 hook: hook, 94 shutdownCh: make(chan struct{}), 95 } 96 97 // Parse the signals that we need 98 for _, tmpl := range tmpls { 99 if tmpl.ChangeSignal == "" { 100 continue 101 } 102 103 sig, err := signals.Parse(tmpl.ChangeSignal) 104 if err != nil { 105 return nil, fmt.Errorf("Failed to parse signal %q", tmpl.ChangeSignal) 106 } 107 108 if tm.signals == nil { 109 tm.signals = make(map[string]os.Signal) 110 } 111 112 tm.signals[tmpl.ChangeSignal] = sig 113 } 114 115 // Build the consul-template runner 116 runner, lookup, err := templateRunner(tmpls, config, vaultToken, taskDir, taskEnv) 117 if err != nil { 118 return nil, err 119 } 120 tm.runner = runner 121 tm.lookup = lookup 122 123 go tm.run() 124 return tm, nil 125 } 126 127 // Stop is used to stop the consul-template runner 128 func (tm *TaskTemplateManager) Stop() { 129 tm.shutdownLock.Lock() 130 defer tm.shutdownLock.Unlock() 131 132 if tm.shutdown { 133 return 134 } 135 136 close(tm.shutdownCh) 137 tm.shutdown = true 138 139 // Stop the consul-template runner 140 if tm.runner != nil { 141 tm.runner.Stop() 142 } 143 } 144 145 // run is the long lived loop that handles errors and templates being rendered 146 func (tm *TaskTemplateManager) run() { 147 // Runner is nil if there is no templates 148 if tm.runner == nil { 149 // Unblock the start if there is nothing to do 150 tm.hook.UnblockStart("consul-template") 151 return 152 } 153 154 // Start the runner 155 go tm.runner.Start() 156 157 // Track when they have all been rendered so we don't signal the task for 158 // any render event before hand 159 var allRenderedTime time.Time 160 161 // Handle the first rendering 162 // Wait till all the templates have been rendered 163 WAIT: 164 for { 165 select { 166 case <-tm.shutdownCh: 167 return 168 case err, ok := <-tm.runner.ErrCh: 169 if !ok { 170 continue 171 } 172 173 tm.hook.Kill("consul-template", err.Error(), true) 174 case <-tm.runner.TemplateRenderedCh(): 175 // A template has been rendered, figure out what to do 176 events := tm.runner.RenderEvents() 177 178 // Not all templates have been rendered yet 179 if len(events) < len(tm.lookup) { 180 continue 181 } 182 183 for _, event := range events { 184 // This template hasn't been rendered 185 if event.LastWouldRender.IsZero() { 186 continue WAIT 187 } 188 } 189 190 break WAIT 191 } 192 } 193 194 allRenderedTime = time.Now() 195 tm.hook.UnblockStart("consul-template") 196 197 // If all our templates are change mode no-op, then we can exit here 198 if tm.allTemplatesNoop() { 199 return 200 } 201 202 // A lookup for the last time the template was handled 203 numTemplates := len(tm.templates) 204 handledRenders := make(map[string]time.Time, numTemplates) 205 206 for { 207 select { 208 case <-tm.shutdownCh: 209 return 210 case err, ok := <-tm.runner.ErrCh: 211 if !ok { 212 continue 213 } 214 215 tm.hook.Kill("consul-template", err.Error(), true) 216 case <-tm.runner.TemplateRenderedCh(): 217 // A template has been rendered, figure out what to do 218 var handling []string 219 signals := make(map[string]struct{}) 220 restart := false 221 var splay time.Duration 222 223 events := tm.runner.RenderEvents() 224 for id, event := range events { 225 226 // First time through 227 if allRenderedTime.After(event.LastDidRender) || allRenderedTime.Equal(event.LastDidRender) { 228 handledRenders[id] = allRenderedTime 229 continue 230 } 231 232 // We have already handled this one 233 if htime := handledRenders[id]; htime.After(event.LastDidRender) || htime.Equal(event.LastDidRender) { 234 continue 235 } 236 237 // Lookup the template and determine what to do 238 tmpls, ok := tm.lookup[id] 239 if !ok { 240 tm.hook.Kill("consul-template", fmt.Sprintf("consul-template runner returned unknown template id %q", id), true) 241 return 242 } 243 244 for _, tmpl := range tmpls { 245 switch tmpl.ChangeMode { 246 case structs.TemplateChangeModeSignal: 247 signals[tmpl.ChangeSignal] = struct{}{} 248 case structs.TemplateChangeModeRestart: 249 restart = true 250 case structs.TemplateChangeModeNoop: 251 continue 252 } 253 254 if tmpl.Splay > splay { 255 splay = tmpl.Splay 256 } 257 } 258 259 handling = append(handling, id) 260 } 261 262 if restart || len(signals) != 0 { 263 if splay != 0 { 264 select { 265 case <-time.After(time.Duration(splay)): 266 case <-tm.shutdownCh: 267 return 268 } 269 } 270 271 // Update handle time 272 for _, id := range handling { 273 handledRenders[id] = events[id].LastDidRender 274 } 275 276 if restart { 277 tm.hook.Restart("consul-template", "template with change_mode restart re-rendered") 278 } else if len(signals) != 0 { 279 var mErr multierror.Error 280 for signal := range signals { 281 err := tm.hook.Signal("consul-template", "template re-rendered", tm.signals[signal]) 282 if err != nil { 283 multierror.Append(&mErr, err) 284 } 285 } 286 287 if err := mErr.ErrorOrNil(); err != nil { 288 flat := make([]os.Signal, 0, len(signals)) 289 for signal := range signals { 290 flat = append(flat, tm.signals[signal]) 291 } 292 tm.hook.Kill("consul-template", fmt.Sprintf("Sending signals %v failed: %v", flat, err), true) 293 } 294 } 295 } 296 } 297 } 298 } 299 300 // allTemplatesNoop returns whether all the managed templates have change mode noop. 301 func (tm *TaskTemplateManager) allTemplatesNoop() bool { 302 for _, tmpl := range tm.templates { 303 if tmpl.ChangeMode != structs.TemplateChangeModeNoop { 304 return false 305 } 306 } 307 308 return true 309 } 310 311 // templateRunner returns a consul-template runner for the given templates and a 312 // lookup by destination to the template. If no templates are given, a nil 313 // template runner and lookup is returned. 314 func templateRunner(tmpls []*structs.Template, config *config.Config, 315 vaultToken, taskDir string, taskEnv *env.TaskEnvironment) ( 316 *manager.Runner, map[string][]*structs.Template, error) { 317 318 if len(tmpls) == 0 { 319 return nil, nil, nil 320 } 321 322 runnerConfig, err := runnerConfig(config, vaultToken) 323 if err != nil { 324 return nil, nil, err 325 } 326 327 // Parse the templates 328 allowAbs := config.ReadBoolDefault(hostSrcOption, true) 329 ctmplMapping, err := parseTemplateConfigs(tmpls, taskDir, taskEnv, allowAbs) 330 if err != nil { 331 return nil, nil, err 332 } 333 334 // Set the config 335 flat := make([]*ctconf.ConfigTemplate, 0, len(ctmplMapping)) 336 for ctmpl := range ctmplMapping { 337 local := ctmpl 338 flat = append(flat, &local) 339 } 340 runnerConfig.ConfigTemplates = flat 341 342 runner, err := manager.NewRunner(runnerConfig, false, false) 343 if err != nil { 344 return nil, nil, err 345 } 346 347 // Build the lookup 348 idMap := runner.ConfigTemplateMapping() 349 lookup := make(map[string][]*structs.Template, len(idMap)) 350 for id, ctmpls := range idMap { 351 for _, ctmpl := range ctmpls { 352 templates := lookup[id] 353 templates = append(templates, ctmplMapping[ctmpl]) 354 lookup[id] = templates 355 } 356 } 357 358 return runner, lookup, nil 359 } 360 361 // parseTemplateConfigs converts the tasks templates into consul-templates 362 func parseTemplateConfigs(tmpls []*structs.Template, taskDir string, 363 taskEnv *env.TaskEnvironment, allowAbs bool) (map[ctconf.ConfigTemplate]*structs.Template, error) { 364 // Build the task environment 365 // TODO Should be able to inject the Nomad env vars into Consul-template for 366 // rendering 367 taskEnv.Build() 368 369 ctmpls := make(map[ctconf.ConfigTemplate]*structs.Template, len(tmpls)) 370 for _, tmpl := range tmpls { 371 var src, dest string 372 if tmpl.SourcePath != "" { 373 if filepath.IsAbs(tmpl.SourcePath) { 374 if !allowAbs { 375 return nil, fmt.Errorf("Specifying absolute template paths disallowed by client config: %q", tmpl.SourcePath) 376 } 377 378 src = tmpl.SourcePath 379 } else { 380 src = filepath.Join(taskDir, taskEnv.ReplaceEnv(tmpl.SourcePath)) 381 } 382 } 383 if tmpl.DestPath != "" { 384 dest = filepath.Join(taskDir, taskEnv.ReplaceEnv(tmpl.DestPath)) 385 } 386 387 ct := ctconf.ConfigTemplate{ 388 Source: src, 389 Destination: dest, 390 EmbeddedTemplate: tmpl.EmbeddedTmpl, 391 Perms: ctconf.DefaultFilePerms, 392 Wait: &watch.Wait{}, 393 } 394 395 ctmpls[ct] = tmpl 396 } 397 398 return ctmpls, nil 399 } 400 401 // runnerConfig returns a consul-template runner configuration, setting the 402 // Vault and Consul configurations based on the clients configs. 403 func runnerConfig(config *config.Config, vaultToken string) (*ctconf.Config, error) { 404 conf := &ctconf.Config{} 405 406 set := func(keys []string) { 407 for _, k := range keys { 408 conf.Set(k) 409 } 410 } 411 412 if testRetryRate != 0 { 413 conf.Retry = testRetryRate 414 conf.Set("retry") 415 } 416 417 // Setup the Consul config 418 if config.ConsulConfig != nil { 419 conf.Consul = config.ConsulConfig.Addr 420 conf.Token = config.ConsulConfig.Token 421 set([]string{"consul", "token"}) 422 423 if config.ConsulConfig.EnableSSL { 424 conf.SSL = &ctconf.SSLConfig{ 425 Enabled: true, 426 Verify: config.ConsulConfig.VerifySSL, 427 Cert: config.ConsulConfig.CertFile, 428 Key: config.ConsulConfig.KeyFile, 429 CaCert: config.ConsulConfig.CAFile, 430 } 431 set([]string{"ssl", "ssl.enabled", "ssl.verify", "ssl.cert", "ssl.key", "ssl.ca_cert"}) 432 } 433 434 if config.ConsulConfig.Auth != "" { 435 parts := strings.SplitN(config.ConsulConfig.Auth, ":", 2) 436 if len(parts) != 2 { 437 return nil, fmt.Errorf("Failed to parse Consul Auth config") 438 } 439 440 conf.Auth = &ctconf.AuthConfig{ 441 Enabled: true, 442 Username: parts[0], 443 Password: parts[1], 444 } 445 446 set([]string{"auth", "auth.username", "auth.password", "auth.enabled"}) 447 } 448 } 449 450 // Setup the Vault config 451 if config.VaultConfig != nil && config.VaultConfig.IsEnabled() { 452 conf.Vault = &ctconf.VaultConfig{ 453 Address: config.VaultConfig.Addr, 454 Token: vaultToken, 455 RenewToken: false, 456 } 457 set([]string{"vault", "vault.address", "vault.token", "vault.renew_token"}) 458 459 if strings.HasPrefix(config.VaultConfig.Addr, "https") || config.VaultConfig.TLSCertFile != "" { 460 verify := config.VaultConfig.TLSSkipVerify == nil || !*config.VaultConfig.TLSSkipVerify 461 conf.Vault.SSL = &ctconf.SSLConfig{ 462 Enabled: true, 463 Verify: !verify, 464 Cert: config.VaultConfig.TLSCertFile, 465 Key: config.VaultConfig.TLSKeyFile, 466 CaCert: config.VaultConfig.TLSCaFile, 467 CaPath: config.VaultConfig.TLSCaPath, 468 } 469 470 set([]string{"vault.ssl", "vault.ssl.enabled", "vault.ssl.verify", 471 "vault.ssl.cert", "vault.ssl.key", "vault.ssl.ca_cert"}) 472 } 473 } 474 475 return conf, nil 476 }