github.com/dkerwin/nomad@v0.3.3-0.20160525181927-74554135514b/client/driver/exec.go (about)

     1  package driver
     2  
     3  import (
     4  	"encoding/json"
     5  	"fmt"
     6  	"log"
     7  	"os/exec"
     8  	"path/filepath"
     9  	"strings"
    10  	"time"
    11  
    12  	"github.com/hashicorp/go-multierror"
    13  	"github.com/hashicorp/go-plugin"
    14  	"github.com/hashicorp/nomad/client/allocdir"
    15  	"github.com/hashicorp/nomad/client/config"
    16  	"github.com/hashicorp/nomad/client/driver/executor"
    17  	cstructs "github.com/hashicorp/nomad/client/driver/structs"
    18  	"github.com/hashicorp/nomad/helper/discover"
    19  	"github.com/hashicorp/nomad/helper/fields"
    20  	"github.com/hashicorp/nomad/nomad/structs"
    21  	"github.com/mitchellh/mapstructure"
    22  )
    23  
    24  const (
    25  	// The key populated in Node Attributes to indicate the presence of the Exec
    26  	// driver
    27  	execDriverAttr = "driver.exec"
    28  )
    29  
    30  // ExecDriver fork/execs tasks using as many of the underlying OS's isolation
    31  // features.
    32  type ExecDriver struct {
    33  	DriverContext
    34  }
    35  
    36  type ExecDriverConfig struct {
    37  	Command string   `mapstructure:"command"`
    38  	Args    []string `mapstructure:"args"`
    39  }
    40  
    41  // execHandle is returned from Start/Open as a handle to the PID
    42  type execHandle struct {
    43  	pluginClient    *plugin.Client
    44  	executor        executor.Executor
    45  	isolationConfig *cstructs.IsolationConfig
    46  	userPid         int
    47  	allocDir        *allocdir.AllocDir
    48  	killTimeout     time.Duration
    49  	maxKillTimeout  time.Duration
    50  	logger          *log.Logger
    51  	waitCh          chan *cstructs.WaitResult
    52  	doneCh          chan struct{}
    53  	version         string
    54  }
    55  
    56  // NewExecDriver is used to create a new exec driver
    57  func NewExecDriver(ctx *DriverContext) Driver {
    58  	return &ExecDriver{DriverContext: *ctx}
    59  }
    60  
    61  // Validate is used to validate the driver configuration
    62  func (d *ExecDriver) Validate(config map[string]interface{}) error {
    63  	fd := &fields.FieldData{
    64  		Raw: config,
    65  		Schema: map[string]*fields.FieldSchema{
    66  			"command": &fields.FieldSchema{
    67  				Type:     fields.TypeString,
    68  				Required: true,
    69  			},
    70  			"args": &fields.FieldSchema{
    71  				Type: fields.TypeArray,
    72  			},
    73  		},
    74  	}
    75  
    76  	if err := fd.Validate(); err != nil {
    77  		return err
    78  	}
    79  
    80  	return nil
    81  }
    82  
    83  func (d *ExecDriver) Periodic() (bool, time.Duration) {
    84  	return true, 15 * time.Second
    85  }
    86  
    87  func (d *ExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, error) {
    88  	var driverConfig ExecDriverConfig
    89  	if err := mapstructure.WeakDecode(task.Config, &driverConfig); err != nil {
    90  		return nil, err
    91  	}
    92  
    93  	// Get the command to be ran
    94  	command := driverConfig.Command
    95  	if err := validateCommand(command, "args"); err != nil {
    96  		return nil, err
    97  	}
    98  
    99  	// Set the host environment variables.
   100  	filter := strings.Split(d.config.ReadDefault("env.blacklist", config.DefaultEnvBlacklist), ",")
   101  	d.taskEnv.AppendHostEnvvars(filter)
   102  
   103  	// Get the task directory for storing the executor logs.
   104  	taskDir, ok := ctx.AllocDir.TaskDirs[d.DriverContext.taskName]
   105  	if !ok {
   106  		return nil, fmt.Errorf("Could not find task directory for task: %v", d.DriverContext.taskName)
   107  	}
   108  
   109  	bin, err := discover.NomadExecutable()
   110  	if err != nil {
   111  		return nil, fmt.Errorf("unable to find the nomad binary: %v", err)
   112  	}
   113  	pluginLogFile := filepath.Join(taskDir, fmt.Sprintf("%s-executor.out", task.Name))
   114  	pluginConfig := &plugin.ClientConfig{
   115  		Cmd: exec.Command(bin, "executor", pluginLogFile),
   116  	}
   117  
   118  	exec, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput, d.config)
   119  	if err != nil {
   120  		return nil, err
   121  	}
   122  	executorCtx := &executor.ExecutorContext{
   123  		TaskEnv:  d.taskEnv,
   124  		Driver:   "exec",
   125  		AllocDir: ctx.AllocDir,
   126  		AllocID:  ctx.AllocID,
   127  		Task:     task,
   128  	}
   129  
   130  	ps, err := exec.LaunchCmd(&executor.ExecCommand{
   131  		Cmd:            command,
   132  		Args:           driverConfig.Args,
   133  		FSIsolation:    true,
   134  		ResourceLimits: true,
   135  		User:           getExecutorUser(task),
   136  	}, executorCtx)
   137  	if err != nil {
   138  		pluginClient.Kill()
   139  		return nil, err
   140  	}
   141  	d.logger.Printf("[DEBUG] driver.exec: started process via plugin with pid: %v", ps.Pid)
   142  
   143  	// Return a driver handle
   144  	maxKill := d.DriverContext.config.MaxKillTimeout
   145  	h := &execHandle{
   146  		pluginClient:    pluginClient,
   147  		userPid:         ps.Pid,
   148  		executor:        exec,
   149  		allocDir:        ctx.AllocDir,
   150  		isolationConfig: ps.IsolationConfig,
   151  		killTimeout:     GetKillTimeout(task.KillTimeout, maxKill),
   152  		maxKillTimeout:  maxKill,
   153  		logger:          d.logger,
   154  		version:         d.config.Version,
   155  		doneCh:          make(chan struct{}),
   156  		waitCh:          make(chan *cstructs.WaitResult, 1),
   157  	}
   158  	if err := exec.SyncServices(consulContext(d.config, "")); err != nil {
   159  		d.logger.Printf("[ERR] driver.exec: error registering services with consul for task: %q: %v", task.Name, err)
   160  	}
   161  	go h.run()
   162  	return h, nil
   163  }
   164  
   165  type execId struct {
   166  	Version         string
   167  	KillTimeout     time.Duration
   168  	MaxKillTimeout  time.Duration
   169  	UserPid         int
   170  	TaskDir         string
   171  	AllocDir        *allocdir.AllocDir
   172  	IsolationConfig *cstructs.IsolationConfig
   173  	PluginConfig    *PluginReattachConfig
   174  }
   175  
   176  func (d *ExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error) {
   177  	id := &execId{}
   178  	if err := json.Unmarshal([]byte(handleID), id); err != nil {
   179  		return nil, fmt.Errorf("Failed to parse handle '%s': %v", handleID, err)
   180  	}
   181  
   182  	pluginConfig := &plugin.ClientConfig{
   183  		Reattach: id.PluginConfig.PluginConfig(),
   184  	}
   185  	exec, client, err := createExecutor(pluginConfig, d.config.LogOutput, d.config)
   186  	if err != nil {
   187  		merrs := new(multierror.Error)
   188  		merrs.Errors = append(merrs.Errors, err)
   189  		d.logger.Println("[ERR] driver.exec: error connecting to plugin so destroying plugin pid and user pid")
   190  		if e := destroyPlugin(id.PluginConfig.Pid, id.UserPid); e != nil {
   191  			merrs.Errors = append(merrs.Errors, fmt.Errorf("error destroying plugin and userpid: %v", e))
   192  		}
   193  		if id.IsolationConfig != nil {
   194  			isoConf := id.IsolationConfig
   195  			ePid := pluginConfig.Reattach.Pid
   196  			if e := executor.DestroyCgroup(isoConf.Cgroup, isoConf.CgroupPaths, ePid); e != nil {
   197  				merrs.Errors = append(merrs.Errors, fmt.Errorf("destroying cgroup failed: %v", e))
   198  			}
   199  		}
   200  		if e := ctx.AllocDir.UnmountAll(); e != nil {
   201  			merrs.Errors = append(merrs.Errors, e)
   202  		}
   203  		return nil, fmt.Errorf("error connecting to plugin: %v", merrs.ErrorOrNil())
   204  	}
   205  
   206  	ver, _ := exec.Version()
   207  	d.logger.Printf("[DEBUG] driver.exec : version of executor: %v", ver.Version)
   208  	// Return a driver handle
   209  	h := &execHandle{
   210  		pluginClient:    client,
   211  		executor:        exec,
   212  		userPid:         id.UserPid,
   213  		allocDir:        id.AllocDir,
   214  		isolationConfig: id.IsolationConfig,
   215  		logger:          d.logger,
   216  		version:         id.Version,
   217  		killTimeout:     id.KillTimeout,
   218  		maxKillTimeout:  id.MaxKillTimeout,
   219  		doneCh:          make(chan struct{}),
   220  		waitCh:          make(chan *cstructs.WaitResult, 1),
   221  	}
   222  	if err := exec.SyncServices(consulContext(d.config, "")); err != nil {
   223  		d.logger.Printf("[ERR] driver.exec: error registering services with consul: %v", err)
   224  	}
   225  	go h.run()
   226  	return h, nil
   227  }
   228  
   229  func (h *execHandle) ID() string {
   230  	id := execId{
   231  		Version:         h.version,
   232  		KillTimeout:     h.killTimeout,
   233  		MaxKillTimeout:  h.maxKillTimeout,
   234  		PluginConfig:    NewPluginReattachConfig(h.pluginClient.ReattachConfig()),
   235  		UserPid:         h.userPid,
   236  		AllocDir:        h.allocDir,
   237  		IsolationConfig: h.isolationConfig,
   238  	}
   239  
   240  	data, err := json.Marshal(id)
   241  	if err != nil {
   242  		h.logger.Printf("[ERR] driver.exec: failed to marshal ID to JSON: %s", err)
   243  	}
   244  	return string(data)
   245  }
   246  
   247  func (h *execHandle) WaitCh() chan *cstructs.WaitResult {
   248  	return h.waitCh
   249  }
   250  
   251  func (h *execHandle) Update(task *structs.Task) error {
   252  	// Store the updated kill timeout.
   253  	h.killTimeout = GetKillTimeout(task.KillTimeout, h.maxKillTimeout)
   254  	h.executor.UpdateTask(task)
   255  
   256  	// Update is not possible
   257  	return nil
   258  }
   259  
   260  func (h *execHandle) Kill() error {
   261  	if err := h.executor.ShutDown(); err != nil {
   262  		if h.pluginClient.Exited() {
   263  			return nil
   264  		}
   265  		return fmt.Errorf("executor Shutdown failed: %v", err)
   266  	}
   267  
   268  	select {
   269  	case <-h.doneCh:
   270  		return nil
   271  	case <-time.After(h.killTimeout):
   272  		if h.pluginClient.Exited() {
   273  			return nil
   274  		}
   275  		if err := h.executor.Exit(); err != nil {
   276  			return fmt.Errorf("executor Exit failed: %v", err)
   277  		}
   278  
   279  		return nil
   280  	}
   281  }
   282  
   283  func (h *execHandle) run() {
   284  	ps, err := h.executor.Wait()
   285  	close(h.doneCh)
   286  
   287  	// If the exitcode is 0 and we had an error that means the plugin didn't
   288  	// connect and doesn't know the state of the user process so we are killing
   289  	// the user process so that when we create a new executor on restarting the
   290  	// new user process doesn't have collisions with resources that the older
   291  	// user pid might be holding onto.
   292  	if ps.ExitCode == 0 && err != nil {
   293  		if h.isolationConfig != nil {
   294  			isoConf := h.isolationConfig
   295  			ePid := h.pluginClient.ReattachConfig().Pid
   296  			if e := executor.DestroyCgroup(isoConf.Cgroup, isoConf.CgroupPaths, ePid); e != nil {
   297  				h.logger.Printf("[ERR] driver.exec: destroying cgroup failed while killing cgroup: %v", e)
   298  			}
   299  		}
   300  		if e := h.allocDir.UnmountAll(); e != nil {
   301  			h.logger.Printf("[ERR] driver.exec: unmounting dev,proc and alloc dirs failed: %v", e)
   302  		}
   303  	}
   304  	h.waitCh <- cstructs.NewWaitResult(ps.ExitCode, ps.Signal, err)
   305  	close(h.waitCh)
   306  	// Remove services
   307  	if err := h.executor.DeregisterServices(); err != nil {
   308  		h.logger.Printf("[ERR] driver.exec: failed to deregister services: %v", err)
   309  	}
   310  
   311  	if err := h.executor.Exit(); err != nil {
   312  		h.logger.Printf("[ERR] driver.exec: error destroying executor: %v", err)
   313  	}
   314  	h.pluginClient.Kill()
   315  }