github.com/dkerwin/nomad@v0.3.3-0.20160525181927-74554135514b/client/driver/exec.go (about) 1 package driver 2 3 import ( 4 "encoding/json" 5 "fmt" 6 "log" 7 "os/exec" 8 "path/filepath" 9 "strings" 10 "time" 11 12 "github.com/hashicorp/go-multierror" 13 "github.com/hashicorp/go-plugin" 14 "github.com/hashicorp/nomad/client/allocdir" 15 "github.com/hashicorp/nomad/client/config" 16 "github.com/hashicorp/nomad/client/driver/executor" 17 cstructs "github.com/hashicorp/nomad/client/driver/structs" 18 "github.com/hashicorp/nomad/helper/discover" 19 "github.com/hashicorp/nomad/helper/fields" 20 "github.com/hashicorp/nomad/nomad/structs" 21 "github.com/mitchellh/mapstructure" 22 ) 23 24 const ( 25 // The key populated in Node Attributes to indicate the presence of the Exec 26 // driver 27 execDriverAttr = "driver.exec" 28 ) 29 30 // ExecDriver fork/execs tasks using as many of the underlying OS's isolation 31 // features. 32 type ExecDriver struct { 33 DriverContext 34 } 35 36 type ExecDriverConfig struct { 37 Command string `mapstructure:"command"` 38 Args []string `mapstructure:"args"` 39 } 40 41 // execHandle is returned from Start/Open as a handle to the PID 42 type execHandle struct { 43 pluginClient *plugin.Client 44 executor executor.Executor 45 isolationConfig *cstructs.IsolationConfig 46 userPid int 47 allocDir *allocdir.AllocDir 48 killTimeout time.Duration 49 maxKillTimeout time.Duration 50 logger *log.Logger 51 waitCh chan *cstructs.WaitResult 52 doneCh chan struct{} 53 version string 54 } 55 56 // NewExecDriver is used to create a new exec driver 57 func NewExecDriver(ctx *DriverContext) Driver { 58 return &ExecDriver{DriverContext: *ctx} 59 } 60 61 // Validate is used to validate the driver configuration 62 func (d *ExecDriver) Validate(config map[string]interface{}) error { 63 fd := &fields.FieldData{ 64 Raw: config, 65 Schema: map[string]*fields.FieldSchema{ 66 "command": &fields.FieldSchema{ 67 Type: fields.TypeString, 68 Required: true, 69 }, 70 "args": &fields.FieldSchema{ 71 Type: fields.TypeArray, 72 }, 73 }, 74 } 75 76 if err := fd.Validate(); err != nil { 77 return err 78 } 79 80 return nil 81 } 82 83 func (d *ExecDriver) Periodic() (bool, time.Duration) { 84 return true, 15 * time.Second 85 } 86 87 func (d *ExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, error) { 88 var driverConfig ExecDriverConfig 89 if err := mapstructure.WeakDecode(task.Config, &driverConfig); err != nil { 90 return nil, err 91 } 92 93 // Get the command to be ran 94 command := driverConfig.Command 95 if err := validateCommand(command, "args"); err != nil { 96 return nil, err 97 } 98 99 // Set the host environment variables. 100 filter := strings.Split(d.config.ReadDefault("env.blacklist", config.DefaultEnvBlacklist), ",") 101 d.taskEnv.AppendHostEnvvars(filter) 102 103 // Get the task directory for storing the executor logs. 104 taskDir, ok := ctx.AllocDir.TaskDirs[d.DriverContext.taskName] 105 if !ok { 106 return nil, fmt.Errorf("Could not find task directory for task: %v", d.DriverContext.taskName) 107 } 108 109 bin, err := discover.NomadExecutable() 110 if err != nil { 111 return nil, fmt.Errorf("unable to find the nomad binary: %v", err) 112 } 113 pluginLogFile := filepath.Join(taskDir, fmt.Sprintf("%s-executor.out", task.Name)) 114 pluginConfig := &plugin.ClientConfig{ 115 Cmd: exec.Command(bin, "executor", pluginLogFile), 116 } 117 118 exec, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput, d.config) 119 if err != nil { 120 return nil, err 121 } 122 executorCtx := &executor.ExecutorContext{ 123 TaskEnv: d.taskEnv, 124 Driver: "exec", 125 AllocDir: ctx.AllocDir, 126 AllocID: ctx.AllocID, 127 Task: task, 128 } 129 130 ps, err := exec.LaunchCmd(&executor.ExecCommand{ 131 Cmd: command, 132 Args: driverConfig.Args, 133 FSIsolation: true, 134 ResourceLimits: true, 135 User: getExecutorUser(task), 136 }, executorCtx) 137 if err != nil { 138 pluginClient.Kill() 139 return nil, err 140 } 141 d.logger.Printf("[DEBUG] driver.exec: started process via plugin with pid: %v", ps.Pid) 142 143 // Return a driver handle 144 maxKill := d.DriverContext.config.MaxKillTimeout 145 h := &execHandle{ 146 pluginClient: pluginClient, 147 userPid: ps.Pid, 148 executor: exec, 149 allocDir: ctx.AllocDir, 150 isolationConfig: ps.IsolationConfig, 151 killTimeout: GetKillTimeout(task.KillTimeout, maxKill), 152 maxKillTimeout: maxKill, 153 logger: d.logger, 154 version: d.config.Version, 155 doneCh: make(chan struct{}), 156 waitCh: make(chan *cstructs.WaitResult, 1), 157 } 158 if err := exec.SyncServices(consulContext(d.config, "")); err != nil { 159 d.logger.Printf("[ERR] driver.exec: error registering services with consul for task: %q: %v", task.Name, err) 160 } 161 go h.run() 162 return h, nil 163 } 164 165 type execId struct { 166 Version string 167 KillTimeout time.Duration 168 MaxKillTimeout time.Duration 169 UserPid int 170 TaskDir string 171 AllocDir *allocdir.AllocDir 172 IsolationConfig *cstructs.IsolationConfig 173 PluginConfig *PluginReattachConfig 174 } 175 176 func (d *ExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error) { 177 id := &execId{} 178 if err := json.Unmarshal([]byte(handleID), id); err != nil { 179 return nil, fmt.Errorf("Failed to parse handle '%s': %v", handleID, err) 180 } 181 182 pluginConfig := &plugin.ClientConfig{ 183 Reattach: id.PluginConfig.PluginConfig(), 184 } 185 exec, client, err := createExecutor(pluginConfig, d.config.LogOutput, d.config) 186 if err != nil { 187 merrs := new(multierror.Error) 188 merrs.Errors = append(merrs.Errors, err) 189 d.logger.Println("[ERR] driver.exec: error connecting to plugin so destroying plugin pid and user pid") 190 if e := destroyPlugin(id.PluginConfig.Pid, id.UserPid); e != nil { 191 merrs.Errors = append(merrs.Errors, fmt.Errorf("error destroying plugin and userpid: %v", e)) 192 } 193 if id.IsolationConfig != nil { 194 isoConf := id.IsolationConfig 195 ePid := pluginConfig.Reattach.Pid 196 if e := executor.DestroyCgroup(isoConf.Cgroup, isoConf.CgroupPaths, ePid); e != nil { 197 merrs.Errors = append(merrs.Errors, fmt.Errorf("destroying cgroup failed: %v", e)) 198 } 199 } 200 if e := ctx.AllocDir.UnmountAll(); e != nil { 201 merrs.Errors = append(merrs.Errors, e) 202 } 203 return nil, fmt.Errorf("error connecting to plugin: %v", merrs.ErrorOrNil()) 204 } 205 206 ver, _ := exec.Version() 207 d.logger.Printf("[DEBUG] driver.exec : version of executor: %v", ver.Version) 208 // Return a driver handle 209 h := &execHandle{ 210 pluginClient: client, 211 executor: exec, 212 userPid: id.UserPid, 213 allocDir: id.AllocDir, 214 isolationConfig: id.IsolationConfig, 215 logger: d.logger, 216 version: id.Version, 217 killTimeout: id.KillTimeout, 218 maxKillTimeout: id.MaxKillTimeout, 219 doneCh: make(chan struct{}), 220 waitCh: make(chan *cstructs.WaitResult, 1), 221 } 222 if err := exec.SyncServices(consulContext(d.config, "")); err != nil { 223 d.logger.Printf("[ERR] driver.exec: error registering services with consul: %v", err) 224 } 225 go h.run() 226 return h, nil 227 } 228 229 func (h *execHandle) ID() string { 230 id := execId{ 231 Version: h.version, 232 KillTimeout: h.killTimeout, 233 MaxKillTimeout: h.maxKillTimeout, 234 PluginConfig: NewPluginReattachConfig(h.pluginClient.ReattachConfig()), 235 UserPid: h.userPid, 236 AllocDir: h.allocDir, 237 IsolationConfig: h.isolationConfig, 238 } 239 240 data, err := json.Marshal(id) 241 if err != nil { 242 h.logger.Printf("[ERR] driver.exec: failed to marshal ID to JSON: %s", err) 243 } 244 return string(data) 245 } 246 247 func (h *execHandle) WaitCh() chan *cstructs.WaitResult { 248 return h.waitCh 249 } 250 251 func (h *execHandle) Update(task *structs.Task) error { 252 // Store the updated kill timeout. 253 h.killTimeout = GetKillTimeout(task.KillTimeout, h.maxKillTimeout) 254 h.executor.UpdateTask(task) 255 256 // Update is not possible 257 return nil 258 } 259 260 func (h *execHandle) Kill() error { 261 if err := h.executor.ShutDown(); err != nil { 262 if h.pluginClient.Exited() { 263 return nil 264 } 265 return fmt.Errorf("executor Shutdown failed: %v", err) 266 } 267 268 select { 269 case <-h.doneCh: 270 return nil 271 case <-time.After(h.killTimeout): 272 if h.pluginClient.Exited() { 273 return nil 274 } 275 if err := h.executor.Exit(); err != nil { 276 return fmt.Errorf("executor Exit failed: %v", err) 277 } 278 279 return nil 280 } 281 } 282 283 func (h *execHandle) run() { 284 ps, err := h.executor.Wait() 285 close(h.doneCh) 286 287 // If the exitcode is 0 and we had an error that means the plugin didn't 288 // connect and doesn't know the state of the user process so we are killing 289 // the user process so that when we create a new executor on restarting the 290 // new user process doesn't have collisions with resources that the older 291 // user pid might be holding onto. 292 if ps.ExitCode == 0 && err != nil { 293 if h.isolationConfig != nil { 294 isoConf := h.isolationConfig 295 ePid := h.pluginClient.ReattachConfig().Pid 296 if e := executor.DestroyCgroup(isoConf.Cgroup, isoConf.CgroupPaths, ePid); e != nil { 297 h.logger.Printf("[ERR] driver.exec: destroying cgroup failed while killing cgroup: %v", e) 298 } 299 } 300 if e := h.allocDir.UnmountAll(); e != nil { 301 h.logger.Printf("[ERR] driver.exec: unmounting dev,proc and alloc dirs failed: %v", e) 302 } 303 } 304 h.waitCh <- cstructs.NewWaitResult(ps.ExitCode, ps.Signal, err) 305 close(h.waitCh) 306 // Remove services 307 if err := h.executor.DeregisterServices(); err != nil { 308 h.logger.Printf("[ERR] driver.exec: failed to deregister services: %v", err) 309 } 310 311 if err := h.executor.Exit(); err != nil { 312 h.logger.Printf("[ERR] driver.exec: error destroying executor: %v", err) 313 } 314 h.pluginClient.Kill() 315 }