github.com/manicqin/nomad@v0.9.5/client/allocrunner/alloc_runner_hooks.go (about)

     1  package allocrunner
     2  
     3  import (
     4  	"fmt"
     5  	"time"
     6  
     7  	multierror "github.com/hashicorp/go-multierror"
     8  	"github.com/hashicorp/nomad/client/allocrunner/interfaces"
     9  	clientconfig "github.com/hashicorp/nomad/client/config"
    10  	"github.com/hashicorp/nomad/client/taskenv"
    11  	"github.com/hashicorp/nomad/nomad/structs"
    12  	"github.com/hashicorp/nomad/plugins/drivers"
    13  )
    14  
    15  type networkIsolationSetter interface {
    16  	SetNetworkIsolation(*drivers.NetworkIsolationSpec)
    17  }
    18  
    19  // allocNetworkIsolationSetter is a shim to allow the alloc network hook to
    20  // set the alloc network isolation configuration without full access
    21  // to the alloc runner
    22  type allocNetworkIsolationSetter struct {
    23  	ar *allocRunner
    24  }
    25  
    26  func (a *allocNetworkIsolationSetter) SetNetworkIsolation(n *drivers.NetworkIsolationSpec) {
    27  	for _, tr := range a.ar.tasks {
    28  		tr.SetNetworkIsolation(n)
    29  	}
    30  }
    31  
    32  // allocHealthSetter is a shim to allow the alloc health watcher hook to set
    33  // and clear the alloc health without full access to the alloc runner state
    34  type allocHealthSetter struct {
    35  	ar *allocRunner
    36  }
    37  
    38  // HasHealth returns true if a deployment status is already set.
    39  func (a *allocHealthSetter) HasHealth() bool {
    40  	a.ar.stateLock.Lock()
    41  	defer a.ar.stateLock.Unlock()
    42  	return a.ar.state.DeploymentStatus.HasHealth()
    43  }
    44  
    45  // ClearHealth allows the health watcher hook to clear the alloc's deployment
    46  // health if the deployment id changes. It does not update the server as the
    47  // status is only cleared when already receiving an update from the server.
    48  //
    49  // Only for use by health hook.
    50  func (a *allocHealthSetter) ClearHealth() {
    51  	a.ar.stateLock.Lock()
    52  	a.ar.state.ClearDeploymentStatus()
    53  	a.ar.persistDeploymentStatus(nil)
    54  	a.ar.stateLock.Unlock()
    55  }
    56  
    57  // SetHealth allows the health watcher hook to set the alloc's
    58  // deployment/migration health and emit task events.
    59  //
    60  // Only for use by health hook.
    61  func (a *allocHealthSetter) SetHealth(healthy, isDeploy bool, trackerTaskEvents map[string]*structs.TaskEvent) {
    62  	// Updating alloc deployment state is tricky because it may be nil, but
    63  	// if it's not then we need to maintain the values of Canary and
    64  	// ModifyIndex as they're only mutated by the server.
    65  	a.ar.stateLock.Lock()
    66  	a.ar.state.SetDeploymentStatus(time.Now(), healthy)
    67  	a.ar.persistDeploymentStatus(a.ar.state.DeploymentStatus)
    68  	terminalDesiredState := a.ar.Alloc().ServerTerminalStatus()
    69  	a.ar.stateLock.Unlock()
    70  
    71  	// If deployment is unhealthy emit task events explaining why
    72  	if !healthy && isDeploy && !terminalDesiredState {
    73  		for task, event := range trackerTaskEvents {
    74  			if tr, ok := a.ar.tasks[task]; ok {
    75  				// Append but don't emit event since the server
    76  				// will be updated below
    77  				tr.AppendEvent(event)
    78  			}
    79  		}
    80  	}
    81  
    82  	// Gather the state of the other tasks
    83  	states := make(map[string]*structs.TaskState, len(a.ar.tasks))
    84  	for name, tr := range a.ar.tasks {
    85  		states[name] = tr.TaskState()
    86  	}
    87  
    88  	// Build the client allocation
    89  	calloc := a.ar.clientAlloc(states)
    90  
    91  	// Update the server
    92  	a.ar.stateUpdater.AllocStateUpdated(calloc)
    93  
    94  	// Broadcast client alloc to listeners
    95  	a.ar.allocBroadcaster.Send(calloc)
    96  }
    97  
    98  // initRunnerHooks intializes the runners hooks.
    99  func (ar *allocRunner) initRunnerHooks(config *clientconfig.Config) error {
   100  	hookLogger := ar.logger.Named("runner_hook")
   101  
   102  	// create health setting shim
   103  	hs := &allocHealthSetter{ar}
   104  
   105  	// create network isolation setting shim
   106  	ns := &allocNetworkIsolationSetter{ar: ar}
   107  
   108  	// build the network manager
   109  	nm, err := newNetworkManager(ar.Alloc(), ar.driverManager)
   110  	if err != nil {
   111  		return fmt.Errorf("failed to configure network manager: %v", err)
   112  	}
   113  
   114  	// create network configurator
   115  	nc, err := newNetworkConfigurator(hookLogger, ar.Alloc(), config)
   116  	if err != nil {
   117  		return fmt.Errorf("failed to initialize network configurator: %v", err)
   118  	}
   119  
   120  	// Create the alloc directory hook. This is run first to ensure the
   121  	// directory path exists for other hooks.
   122  	alloc := ar.Alloc()
   123  	ar.runnerHooks = []interfaces.RunnerHook{
   124  		newAllocDirHook(hookLogger, ar.allocDir),
   125  		newUpstreamAllocsHook(hookLogger, ar.prevAllocWatcher),
   126  		newDiskMigrationHook(hookLogger, ar.prevAllocMigrator, ar.allocDir),
   127  		newAllocHealthWatcherHook(hookLogger, alloc, hs, ar.Listener(), ar.consulClient),
   128  		newNetworkHook(hookLogger, ns, alloc, nm, nc),
   129  		newGroupServiceHook(groupServiceHookConfig{
   130  			alloc:          alloc,
   131  			consul:         ar.consulClient,
   132  			restarter:      ar,
   133  			taskEnvBuilder: taskenv.NewBuilder(config.Node, ar.Alloc(), nil, config.Region).SetAllocDir(ar.allocDir.AllocDir),
   134  			logger:         hookLogger,
   135  		}),
   136  		newConsulSockHook(hookLogger, alloc, ar.allocDir, config.ConsulConfig),
   137  	}
   138  
   139  	return nil
   140  }
   141  
   142  // prerun is used to run the runners prerun hooks.
   143  func (ar *allocRunner) prerun() error {
   144  	if ar.logger.IsTrace() {
   145  		start := time.Now()
   146  		ar.logger.Trace("running pre-run hooks", "start", start)
   147  		defer func() {
   148  			end := time.Now()
   149  			ar.logger.Trace("finished pre-run hooks", "end", end, "duration", end.Sub(start))
   150  		}()
   151  	}
   152  
   153  	for _, hook := range ar.runnerHooks {
   154  		pre, ok := hook.(interfaces.RunnerPrerunHook)
   155  		if !ok {
   156  			continue
   157  		}
   158  
   159  		name := pre.Name()
   160  		var start time.Time
   161  		if ar.logger.IsTrace() {
   162  			start = time.Now()
   163  			ar.logger.Trace("running pre-run hook", "name", name, "start", start)
   164  		}
   165  
   166  		if err := pre.Prerun(); err != nil {
   167  			return fmt.Errorf("pre-run hook %q failed: %v", name, err)
   168  		}
   169  
   170  		if ar.logger.IsTrace() {
   171  			end := time.Now()
   172  			ar.logger.Trace("finished pre-run hook", "name", name, "end", end, "duration", end.Sub(start))
   173  		}
   174  	}
   175  
   176  	return nil
   177  }
   178  
   179  // update runs the alloc runner update hooks. Update hooks are run
   180  // asynchronously with all other alloc runner operations.
   181  func (ar *allocRunner) update(update *structs.Allocation) error {
   182  	if ar.logger.IsTrace() {
   183  		start := time.Now()
   184  		ar.logger.Trace("running update hooks", "start", start)
   185  		defer func() {
   186  			end := time.Now()
   187  			ar.logger.Trace("finished update hooks", "end", end, "duration", end.Sub(start))
   188  		}()
   189  	}
   190  
   191  	req := &interfaces.RunnerUpdateRequest{
   192  		Alloc: update,
   193  	}
   194  
   195  	var merr multierror.Error
   196  	for _, hook := range ar.runnerHooks {
   197  		h, ok := hook.(interfaces.RunnerUpdateHook)
   198  		if !ok {
   199  			continue
   200  		}
   201  
   202  		name := h.Name()
   203  		var start time.Time
   204  		if ar.logger.IsTrace() {
   205  			start = time.Now()
   206  			ar.logger.Trace("running update hook", "name", name, "start", start)
   207  		}
   208  
   209  		if err := h.Update(req); err != nil {
   210  			merr.Errors = append(merr.Errors, fmt.Errorf("update hook %q failed: %v", name, err))
   211  		}
   212  
   213  		if ar.logger.IsTrace() {
   214  			end := time.Now()
   215  			ar.logger.Trace("finished update hooks", "name", name, "end", end, "duration", end.Sub(start))
   216  		}
   217  	}
   218  
   219  	return merr.ErrorOrNil()
   220  }
   221  
   222  // postrun is used to run the runners postrun hooks.
   223  func (ar *allocRunner) postrun() error {
   224  	if ar.logger.IsTrace() {
   225  		start := time.Now()
   226  		ar.logger.Trace("running post-run hooks", "start", start)
   227  		defer func() {
   228  			end := time.Now()
   229  			ar.logger.Trace("finished post-run hooks", "end", end, "duration", end.Sub(start))
   230  		}()
   231  	}
   232  
   233  	for _, hook := range ar.runnerHooks {
   234  		post, ok := hook.(interfaces.RunnerPostrunHook)
   235  		if !ok {
   236  			continue
   237  		}
   238  
   239  		name := post.Name()
   240  		var start time.Time
   241  		if ar.logger.IsTrace() {
   242  			start = time.Now()
   243  			ar.logger.Trace("running post-run hook", "name", name, "start", start)
   244  		}
   245  
   246  		if err := post.Postrun(); err != nil {
   247  			return fmt.Errorf("hook %q failed: %v", name, err)
   248  		}
   249  
   250  		if ar.logger.IsTrace() {
   251  			end := time.Now()
   252  			ar.logger.Trace("finished post-run hooks", "name", name, "end", end, "duration", end.Sub(start))
   253  		}
   254  	}
   255  
   256  	return nil
   257  }
   258  
   259  // destroy is used to run the runners destroy hooks. All hooks are run and
   260  // errors are returned as a multierror.
   261  func (ar *allocRunner) destroy() error {
   262  	if ar.logger.IsTrace() {
   263  		start := time.Now()
   264  		ar.logger.Trace("running destroy hooks", "start", start)
   265  		defer func() {
   266  			end := time.Now()
   267  			ar.logger.Trace("finished destroy hooks", "end", end, "duration", end.Sub(start))
   268  		}()
   269  	}
   270  
   271  	var merr multierror.Error
   272  	for _, hook := range ar.runnerHooks {
   273  		h, ok := hook.(interfaces.RunnerDestroyHook)
   274  		if !ok {
   275  			continue
   276  		}
   277  
   278  		name := h.Name()
   279  		var start time.Time
   280  		if ar.logger.IsTrace() {
   281  			start = time.Now()
   282  			ar.logger.Trace("running destroy hook", "name", name, "start", start)
   283  		}
   284  
   285  		if err := h.Destroy(); err != nil {
   286  			merr.Errors = append(merr.Errors, fmt.Errorf("destroy hook %q failed: %v", name, err))
   287  		}
   288  
   289  		if ar.logger.IsTrace() {
   290  			end := time.Now()
   291  			ar.logger.Trace("finished destroy hooks", "name", name, "end", end, "duration", end.Sub(start))
   292  		}
   293  	}
   294  
   295  	return merr.ErrorOrNil()
   296  }
   297  
   298  func (ar *allocRunner) preKillHooks() {
   299  	for _, hook := range ar.runnerHooks {
   300  		pre, ok := hook.(interfaces.RunnerPreKillHook)
   301  		if !ok {
   302  			continue
   303  		}
   304  
   305  		name := pre.Name()
   306  		var start time.Time
   307  		if ar.logger.IsTrace() {
   308  			start = time.Now()
   309  			ar.logger.Trace("running alloc pre shutdown hook", "name", name, "start", start)
   310  		}
   311  
   312  		pre.PreKill()
   313  
   314  		if ar.logger.IsTrace() {
   315  			end := time.Now()
   316  			ar.logger.Trace("finished alloc pre shutdown hook", "name", name, "end", end, "duration", end.Sub(start))
   317  		}
   318  	}
   319  }
   320  
   321  // shutdownHooks calls graceful shutdown hooks for when the agent is exiting.
   322  func (ar *allocRunner) shutdownHooks() {
   323  	for _, hook := range ar.runnerHooks {
   324  		sh, ok := hook.(interfaces.ShutdownHook)
   325  		if !ok {
   326  			continue
   327  		}
   328  
   329  		name := sh.Name()
   330  		var start time.Time
   331  		if ar.logger.IsTrace() {
   332  			start = time.Now()
   333  			ar.logger.Trace("running shutdown hook", "name", name, "start", start)
   334  		}
   335  
   336  		sh.Shutdown()
   337  
   338  		if ar.logger.IsTrace() {
   339  			end := time.Now()
   340  			ar.logger.Trace("finished shutdown hooks", "name", name, "end", end, "duration", end.Sub(start))
   341  		}
   342  	}
   343  }