github.com/manicqin/nomad@v0.9.5/client/allocrunner/alloc_runner_hooks.go (about) 1 package allocrunner 2 3 import ( 4 "fmt" 5 "time" 6 7 multierror "github.com/hashicorp/go-multierror" 8 "github.com/hashicorp/nomad/client/allocrunner/interfaces" 9 clientconfig "github.com/hashicorp/nomad/client/config" 10 "github.com/hashicorp/nomad/client/taskenv" 11 "github.com/hashicorp/nomad/nomad/structs" 12 "github.com/hashicorp/nomad/plugins/drivers" 13 ) 14 15 type networkIsolationSetter interface { 16 SetNetworkIsolation(*drivers.NetworkIsolationSpec) 17 } 18 19 // allocNetworkIsolationSetter is a shim to allow the alloc network hook to 20 // set the alloc network isolation configuration without full access 21 // to the alloc runner 22 type allocNetworkIsolationSetter struct { 23 ar *allocRunner 24 } 25 26 func (a *allocNetworkIsolationSetter) SetNetworkIsolation(n *drivers.NetworkIsolationSpec) { 27 for _, tr := range a.ar.tasks { 28 tr.SetNetworkIsolation(n) 29 } 30 } 31 32 // allocHealthSetter is a shim to allow the alloc health watcher hook to set 33 // and clear the alloc health without full access to the alloc runner state 34 type allocHealthSetter struct { 35 ar *allocRunner 36 } 37 38 // HasHealth returns true if a deployment status is already set. 39 func (a *allocHealthSetter) HasHealth() bool { 40 a.ar.stateLock.Lock() 41 defer a.ar.stateLock.Unlock() 42 return a.ar.state.DeploymentStatus.HasHealth() 43 } 44 45 // ClearHealth allows the health watcher hook to clear the alloc's deployment 46 // health if the deployment id changes. It does not update the server as the 47 // status is only cleared when already receiving an update from the server. 48 // 49 // Only for use by health hook. 50 func (a *allocHealthSetter) ClearHealth() { 51 a.ar.stateLock.Lock() 52 a.ar.state.ClearDeploymentStatus() 53 a.ar.persistDeploymentStatus(nil) 54 a.ar.stateLock.Unlock() 55 } 56 57 // SetHealth allows the health watcher hook to set the alloc's 58 // deployment/migration health and emit task events. 59 // 60 // Only for use by health hook. 61 func (a *allocHealthSetter) SetHealth(healthy, isDeploy bool, trackerTaskEvents map[string]*structs.TaskEvent) { 62 // Updating alloc deployment state is tricky because it may be nil, but 63 // if it's not then we need to maintain the values of Canary and 64 // ModifyIndex as they're only mutated by the server. 65 a.ar.stateLock.Lock() 66 a.ar.state.SetDeploymentStatus(time.Now(), healthy) 67 a.ar.persistDeploymentStatus(a.ar.state.DeploymentStatus) 68 terminalDesiredState := a.ar.Alloc().ServerTerminalStatus() 69 a.ar.stateLock.Unlock() 70 71 // If deployment is unhealthy emit task events explaining why 72 if !healthy && isDeploy && !terminalDesiredState { 73 for task, event := range trackerTaskEvents { 74 if tr, ok := a.ar.tasks[task]; ok { 75 // Append but don't emit event since the server 76 // will be updated below 77 tr.AppendEvent(event) 78 } 79 } 80 } 81 82 // Gather the state of the other tasks 83 states := make(map[string]*structs.TaskState, len(a.ar.tasks)) 84 for name, tr := range a.ar.tasks { 85 states[name] = tr.TaskState() 86 } 87 88 // Build the client allocation 89 calloc := a.ar.clientAlloc(states) 90 91 // Update the server 92 a.ar.stateUpdater.AllocStateUpdated(calloc) 93 94 // Broadcast client alloc to listeners 95 a.ar.allocBroadcaster.Send(calloc) 96 } 97 98 // initRunnerHooks intializes the runners hooks. 99 func (ar *allocRunner) initRunnerHooks(config *clientconfig.Config) error { 100 hookLogger := ar.logger.Named("runner_hook") 101 102 // create health setting shim 103 hs := &allocHealthSetter{ar} 104 105 // create network isolation setting shim 106 ns := &allocNetworkIsolationSetter{ar: ar} 107 108 // build the network manager 109 nm, err := newNetworkManager(ar.Alloc(), ar.driverManager) 110 if err != nil { 111 return fmt.Errorf("failed to configure network manager: %v", err) 112 } 113 114 // create network configurator 115 nc, err := newNetworkConfigurator(hookLogger, ar.Alloc(), config) 116 if err != nil { 117 return fmt.Errorf("failed to initialize network configurator: %v", err) 118 } 119 120 // Create the alloc directory hook. This is run first to ensure the 121 // directory path exists for other hooks. 122 alloc := ar.Alloc() 123 ar.runnerHooks = []interfaces.RunnerHook{ 124 newAllocDirHook(hookLogger, ar.allocDir), 125 newUpstreamAllocsHook(hookLogger, ar.prevAllocWatcher), 126 newDiskMigrationHook(hookLogger, ar.prevAllocMigrator, ar.allocDir), 127 newAllocHealthWatcherHook(hookLogger, alloc, hs, ar.Listener(), ar.consulClient), 128 newNetworkHook(hookLogger, ns, alloc, nm, nc), 129 newGroupServiceHook(groupServiceHookConfig{ 130 alloc: alloc, 131 consul: ar.consulClient, 132 restarter: ar, 133 taskEnvBuilder: taskenv.NewBuilder(config.Node, ar.Alloc(), nil, config.Region).SetAllocDir(ar.allocDir.AllocDir), 134 logger: hookLogger, 135 }), 136 newConsulSockHook(hookLogger, alloc, ar.allocDir, config.ConsulConfig), 137 } 138 139 return nil 140 } 141 142 // prerun is used to run the runners prerun hooks. 143 func (ar *allocRunner) prerun() error { 144 if ar.logger.IsTrace() { 145 start := time.Now() 146 ar.logger.Trace("running pre-run hooks", "start", start) 147 defer func() { 148 end := time.Now() 149 ar.logger.Trace("finished pre-run hooks", "end", end, "duration", end.Sub(start)) 150 }() 151 } 152 153 for _, hook := range ar.runnerHooks { 154 pre, ok := hook.(interfaces.RunnerPrerunHook) 155 if !ok { 156 continue 157 } 158 159 name := pre.Name() 160 var start time.Time 161 if ar.logger.IsTrace() { 162 start = time.Now() 163 ar.logger.Trace("running pre-run hook", "name", name, "start", start) 164 } 165 166 if err := pre.Prerun(); err != nil { 167 return fmt.Errorf("pre-run hook %q failed: %v", name, err) 168 } 169 170 if ar.logger.IsTrace() { 171 end := time.Now() 172 ar.logger.Trace("finished pre-run hook", "name", name, "end", end, "duration", end.Sub(start)) 173 } 174 } 175 176 return nil 177 } 178 179 // update runs the alloc runner update hooks. Update hooks are run 180 // asynchronously with all other alloc runner operations. 181 func (ar *allocRunner) update(update *structs.Allocation) error { 182 if ar.logger.IsTrace() { 183 start := time.Now() 184 ar.logger.Trace("running update hooks", "start", start) 185 defer func() { 186 end := time.Now() 187 ar.logger.Trace("finished update hooks", "end", end, "duration", end.Sub(start)) 188 }() 189 } 190 191 req := &interfaces.RunnerUpdateRequest{ 192 Alloc: update, 193 } 194 195 var merr multierror.Error 196 for _, hook := range ar.runnerHooks { 197 h, ok := hook.(interfaces.RunnerUpdateHook) 198 if !ok { 199 continue 200 } 201 202 name := h.Name() 203 var start time.Time 204 if ar.logger.IsTrace() { 205 start = time.Now() 206 ar.logger.Trace("running update hook", "name", name, "start", start) 207 } 208 209 if err := h.Update(req); err != nil { 210 merr.Errors = append(merr.Errors, fmt.Errorf("update hook %q failed: %v", name, err)) 211 } 212 213 if ar.logger.IsTrace() { 214 end := time.Now() 215 ar.logger.Trace("finished update hooks", "name", name, "end", end, "duration", end.Sub(start)) 216 } 217 } 218 219 return merr.ErrorOrNil() 220 } 221 222 // postrun is used to run the runners postrun hooks. 223 func (ar *allocRunner) postrun() error { 224 if ar.logger.IsTrace() { 225 start := time.Now() 226 ar.logger.Trace("running post-run hooks", "start", start) 227 defer func() { 228 end := time.Now() 229 ar.logger.Trace("finished post-run hooks", "end", end, "duration", end.Sub(start)) 230 }() 231 } 232 233 for _, hook := range ar.runnerHooks { 234 post, ok := hook.(interfaces.RunnerPostrunHook) 235 if !ok { 236 continue 237 } 238 239 name := post.Name() 240 var start time.Time 241 if ar.logger.IsTrace() { 242 start = time.Now() 243 ar.logger.Trace("running post-run hook", "name", name, "start", start) 244 } 245 246 if err := post.Postrun(); err != nil { 247 return fmt.Errorf("hook %q failed: %v", name, err) 248 } 249 250 if ar.logger.IsTrace() { 251 end := time.Now() 252 ar.logger.Trace("finished post-run hooks", "name", name, "end", end, "duration", end.Sub(start)) 253 } 254 } 255 256 return nil 257 } 258 259 // destroy is used to run the runners destroy hooks. All hooks are run and 260 // errors are returned as a multierror. 261 func (ar *allocRunner) destroy() error { 262 if ar.logger.IsTrace() { 263 start := time.Now() 264 ar.logger.Trace("running destroy hooks", "start", start) 265 defer func() { 266 end := time.Now() 267 ar.logger.Trace("finished destroy hooks", "end", end, "duration", end.Sub(start)) 268 }() 269 } 270 271 var merr multierror.Error 272 for _, hook := range ar.runnerHooks { 273 h, ok := hook.(interfaces.RunnerDestroyHook) 274 if !ok { 275 continue 276 } 277 278 name := h.Name() 279 var start time.Time 280 if ar.logger.IsTrace() { 281 start = time.Now() 282 ar.logger.Trace("running destroy hook", "name", name, "start", start) 283 } 284 285 if err := h.Destroy(); err != nil { 286 merr.Errors = append(merr.Errors, fmt.Errorf("destroy hook %q failed: %v", name, err)) 287 } 288 289 if ar.logger.IsTrace() { 290 end := time.Now() 291 ar.logger.Trace("finished destroy hooks", "name", name, "end", end, "duration", end.Sub(start)) 292 } 293 } 294 295 return merr.ErrorOrNil() 296 } 297 298 func (ar *allocRunner) preKillHooks() { 299 for _, hook := range ar.runnerHooks { 300 pre, ok := hook.(interfaces.RunnerPreKillHook) 301 if !ok { 302 continue 303 } 304 305 name := pre.Name() 306 var start time.Time 307 if ar.logger.IsTrace() { 308 start = time.Now() 309 ar.logger.Trace("running alloc pre shutdown hook", "name", name, "start", start) 310 } 311 312 pre.PreKill() 313 314 if ar.logger.IsTrace() { 315 end := time.Now() 316 ar.logger.Trace("finished alloc pre shutdown hook", "name", name, "end", end, "duration", end.Sub(start)) 317 } 318 } 319 } 320 321 // shutdownHooks calls graceful shutdown hooks for when the agent is exiting. 322 func (ar *allocRunner) shutdownHooks() { 323 for _, hook := range ar.runnerHooks { 324 sh, ok := hook.(interfaces.ShutdownHook) 325 if !ok { 326 continue 327 } 328 329 name := sh.Name() 330 var start time.Time 331 if ar.logger.IsTrace() { 332 start = time.Now() 333 ar.logger.Trace("running shutdown hook", "name", name, "start", start) 334 } 335 336 sh.Shutdown() 337 338 if ar.logger.IsTrace() { 339 end := time.Now() 340 ar.logger.Trace("finished shutdown hooks", "name", name, "end", end, "duration", end.Sub(start)) 341 } 342 } 343 }