github.com/dkerwin/nomad@v0.3.3-0.20160525181927-74554135514b/client/driver/qemu.go (about) 1 package driver 2 3 import ( 4 "encoding/json" 5 "fmt" 6 "log" 7 "os/exec" 8 "path/filepath" 9 "regexp" 10 "runtime" 11 "strings" 12 "time" 13 14 "github.com/hashicorp/go-plugin" 15 "github.com/hashicorp/nomad/client/allocdir" 16 "github.com/hashicorp/nomad/client/config" 17 "github.com/hashicorp/nomad/client/driver/executor" 18 cstructs "github.com/hashicorp/nomad/client/driver/structs" 19 "github.com/hashicorp/nomad/client/fingerprint" 20 "github.com/hashicorp/nomad/helper/discover" 21 "github.com/hashicorp/nomad/helper/fields" 22 "github.com/hashicorp/nomad/nomad/structs" 23 "github.com/mitchellh/mapstructure" 24 ) 25 26 var ( 27 reQemuVersion = regexp.MustCompile(`version (\d[\.\d+]+)`) 28 ) 29 30 const ( 31 // The key populated in Node Attributes to indicate presence of the Qemu 32 // driver 33 qemuDriverAttr = "driver.qemu" 34 ) 35 36 // QemuDriver is a driver for running images via Qemu 37 // We attempt to chose sane defaults for now, with more configuration available 38 // planned in the future 39 type QemuDriver struct { 40 DriverContext 41 fingerprint.StaticFingerprinter 42 } 43 44 type QemuDriverConfig struct { 45 ImagePath string `mapstructure:"image_path"` 46 Accelerator string `mapstructure:"accelerator"` 47 PortMap []map[string]int `mapstructure:"port_map"` // A map of host port labels and to guest ports. 48 } 49 50 // qemuHandle is returned from Start/Open as a handle to the PID 51 type qemuHandle struct { 52 pluginClient *plugin.Client 53 userPid int 54 executor executor.Executor 55 allocDir *allocdir.AllocDir 56 killTimeout time.Duration 57 maxKillTimeout time.Duration 58 logger *log.Logger 59 version string 60 waitCh chan *cstructs.WaitResult 61 doneCh chan struct{} 62 } 63 64 // NewQemuDriver is used to create a new exec driver 65 func NewQemuDriver(ctx *DriverContext) Driver { 66 return &QemuDriver{DriverContext: *ctx} 67 } 68 69 // Validate is used to validate the driver configuration 70 func (d *QemuDriver) Validate(config map[string]interface{}) error { 71 fd := &fields.FieldData{ 72 Raw: config, 73 Schema: map[string]*fields.FieldSchema{ 74 "image_path": &fields.FieldSchema{ 75 Type: fields.TypeString, 76 Required: true, 77 }, 78 "accelerator": &fields.FieldSchema{ 79 Type: fields.TypeString, 80 }, 81 "port_map": &fields.FieldSchema{ 82 Type: fields.TypeArray, 83 }, 84 }, 85 } 86 87 if err := fd.Validate(); err != nil { 88 return err 89 } 90 91 return nil 92 } 93 94 func (d *QemuDriver) Fingerprint(cfg *config.Config, node *structs.Node) (bool, error) { 95 // Get the current status so that we can log any debug messages only if the 96 // state changes 97 _, currentlyEnabled := node.Attributes[qemuDriverAttr] 98 99 bin := "qemu-system-x86_64" 100 if runtime.GOOS == "windows" { 101 // On windows, the "qemu-system-x86_64" command does not respond to the 102 // version flag. 103 bin = "qemu-img" 104 } 105 outBytes, err := exec.Command(bin, "--version").Output() 106 if err != nil { 107 delete(node.Attributes, qemuDriverAttr) 108 return false, nil 109 } 110 out := strings.TrimSpace(string(outBytes)) 111 112 matches := reQemuVersion.FindStringSubmatch(out) 113 if len(matches) != 2 { 114 delete(node.Attributes, qemuDriverAttr) 115 return false, fmt.Errorf("Unable to parse Qemu version string: %#v", matches) 116 } 117 118 if !currentlyEnabled { 119 d.logger.Printf("[DEBUG] driver.qemu: enabling driver") 120 } 121 node.Attributes[qemuDriverAttr] = "1" 122 node.Attributes["driver.qemu.version"] = matches[1] 123 return true, nil 124 } 125 126 // Run an existing Qemu image. Start() will pull down an existing, valid Qemu 127 // image and save it to the Drivers Allocation Dir 128 func (d *QemuDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, error) { 129 var driverConfig QemuDriverConfig 130 if err := mapstructure.WeakDecode(task.Config, &driverConfig); err != nil { 131 return nil, err 132 } 133 134 if len(driverConfig.PortMap) > 1 { 135 return nil, fmt.Errorf("Only one port_map block is allowed in the qemu driver config") 136 } 137 138 // Get the image source 139 vmPath := driverConfig.ImagePath 140 if vmPath == "" { 141 return nil, fmt.Errorf("image_path must be set") 142 } 143 vmID := filepath.Base(vmPath) 144 145 // Get the tasks local directory. 146 taskDir, ok := ctx.AllocDir.TaskDirs[d.DriverContext.taskName] 147 if !ok { 148 return nil, fmt.Errorf("Could not find task directory for task: %v", d.DriverContext.taskName) 149 } 150 151 // Parse configuration arguments 152 // Create the base arguments 153 accelerator := "tcg" 154 if driverConfig.Accelerator != "" { 155 accelerator = driverConfig.Accelerator 156 } 157 // TODO: Check a lower bounds, e.g. the default 128 of Qemu 158 mem := fmt.Sprintf("%dM", task.Resources.MemoryMB) 159 160 absPath, err := GetAbsolutePath("qemu-system-x86_64") 161 if err != nil { 162 return nil, err 163 } 164 165 args := []string{ 166 absPath, 167 "-machine", "type=pc,accel=" + accelerator, 168 "-name", vmID, 169 "-m", mem, 170 "-drive", "file=" + vmPath, 171 "-nodefconfig", 172 "-nodefaults", 173 "-nographic", 174 } 175 176 // Check the Resources required Networks to add port mappings. If no resources 177 // are required, we assume the VM is a purely compute job and does not require 178 // the outside world to be able to reach it. VMs ran without port mappings can 179 // still reach out to the world, but without port mappings it is effectively 180 // firewalled 181 protocols := []string{"udp", "tcp"} 182 if len(task.Resources.Networks) > 0 && len(driverConfig.PortMap) == 1 { 183 // Loop through the port map and construct the hostfwd string, to map 184 // reserved ports to the ports listenting in the VM 185 // Ex: hostfwd=tcp::22000-:22,hostfwd=tcp::80-:8080 186 var forwarding []string 187 taskPorts := task.Resources.Networks[0].MapLabelToValues(nil) 188 for label, guest := range driverConfig.PortMap[0] { 189 host, ok := taskPorts[label] 190 if !ok { 191 return nil, fmt.Errorf("Unknown port label %q", label) 192 } 193 194 for _, p := range protocols { 195 forwarding = append(forwarding, fmt.Sprintf("hostfwd=%s::%d-:%d", p, host, guest)) 196 } 197 } 198 199 if len(forwarding) != 0 { 200 args = append(args, 201 "-netdev", 202 fmt.Sprintf("user,id=user.0,%s", strings.Join(forwarding, ",")), 203 "-device", "virtio-net,netdev=user.0", 204 ) 205 } 206 } 207 208 // If using KVM, add optimization args 209 if accelerator == "kvm" { 210 args = append(args, 211 "-enable-kvm", 212 "-cpu", "host", 213 // Do we have cores information available to the Driver? 214 // "-smp", fmt.Sprintf("%d", cores), 215 ) 216 } 217 218 d.logger.Printf("[DEBUG] Starting QemuVM command: %q", strings.Join(args, " ")) 219 bin, err := discover.NomadExecutable() 220 if err != nil { 221 return nil, fmt.Errorf("unable to find the nomad binary: %v", err) 222 } 223 224 pluginLogFile := filepath.Join(taskDir, fmt.Sprintf("%s-executor.out", task.Name)) 225 pluginConfig := &plugin.ClientConfig{ 226 Cmd: exec.Command(bin, "executor", pluginLogFile), 227 } 228 229 exec, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput, d.config) 230 if err != nil { 231 return nil, err 232 } 233 executorCtx := &executor.ExecutorContext{ 234 TaskEnv: d.taskEnv, 235 Driver: "qemu", 236 AllocDir: ctx.AllocDir, 237 AllocID: ctx.AllocID, 238 Task: task, 239 } 240 ps, err := exec.LaunchCmd(&executor.ExecCommand{ 241 Cmd: args[0], 242 Args: args[1:], 243 User: task.User, 244 }, executorCtx) 245 if err != nil { 246 pluginClient.Kill() 247 return nil, err 248 } 249 d.logger.Printf("[INFO] Started new QemuVM: %s", vmID) 250 251 // Create and Return Handle 252 maxKill := d.DriverContext.config.MaxKillTimeout 253 h := &qemuHandle{ 254 pluginClient: pluginClient, 255 executor: exec, 256 userPid: ps.Pid, 257 allocDir: ctx.AllocDir, 258 killTimeout: GetKillTimeout(task.KillTimeout, maxKill), 259 maxKillTimeout: maxKill, 260 version: d.config.Version, 261 logger: d.logger, 262 doneCh: make(chan struct{}), 263 waitCh: make(chan *cstructs.WaitResult, 1), 264 } 265 266 if err := h.executor.SyncServices(consulContext(d.config, "")); err != nil { 267 h.logger.Printf("[ERR] driver.qemu: error registering services for task: %q: %v", task.Name, err) 268 } 269 go h.run() 270 return h, nil 271 } 272 273 type qemuId struct { 274 Version string 275 KillTimeout time.Duration 276 MaxKillTimeout time.Duration 277 UserPid int 278 PluginConfig *PluginReattachConfig 279 AllocDir *allocdir.AllocDir 280 } 281 282 func (d *QemuDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error) { 283 id := &qemuId{} 284 if err := json.Unmarshal([]byte(handleID), id); err != nil { 285 return nil, fmt.Errorf("Failed to parse handle '%s': %v", handleID, err) 286 } 287 288 pluginConfig := &plugin.ClientConfig{ 289 Reattach: id.PluginConfig.PluginConfig(), 290 } 291 292 exec, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput, d.config) 293 if err != nil { 294 d.logger.Println("[ERR] driver.qemu: error connecting to plugin so destroying plugin pid and user pid") 295 if e := destroyPlugin(id.PluginConfig.Pid, id.UserPid); e != nil { 296 d.logger.Printf("[ERR] driver.qemu: error destroying plugin and userpid: %v", e) 297 } 298 return nil, fmt.Errorf("error connecting to plugin: %v", err) 299 } 300 301 ver, _ := exec.Version() 302 d.logger.Printf("[DEBUG] driver.qemu: version of executor: %v", ver.Version) 303 // Return a driver handle 304 h := &qemuHandle{ 305 pluginClient: pluginClient, 306 executor: exec, 307 userPid: id.UserPid, 308 allocDir: id.AllocDir, 309 logger: d.logger, 310 killTimeout: id.KillTimeout, 311 maxKillTimeout: id.MaxKillTimeout, 312 version: id.Version, 313 doneCh: make(chan struct{}), 314 waitCh: make(chan *cstructs.WaitResult, 1), 315 } 316 if err := h.executor.SyncServices(consulContext(d.config, "")); err != nil { 317 h.logger.Printf("[ERR] driver.qemu: error registering services: %v", err) 318 } 319 go h.run() 320 return h, nil 321 } 322 323 func (h *qemuHandle) ID() string { 324 id := qemuId{ 325 Version: h.version, 326 KillTimeout: h.killTimeout, 327 MaxKillTimeout: h.maxKillTimeout, 328 PluginConfig: NewPluginReattachConfig(h.pluginClient.ReattachConfig()), 329 UserPid: h.userPid, 330 AllocDir: h.allocDir, 331 } 332 333 data, err := json.Marshal(id) 334 if err != nil { 335 h.logger.Printf("[ERR] driver.qemu: failed to marshal ID to JSON: %s", err) 336 } 337 return string(data) 338 } 339 340 func (h *qemuHandle) WaitCh() chan *cstructs.WaitResult { 341 return h.waitCh 342 } 343 344 func (h *qemuHandle) Update(task *structs.Task) error { 345 // Store the updated kill timeout. 346 h.killTimeout = GetKillTimeout(task.KillTimeout, h.maxKillTimeout) 347 h.executor.UpdateTask(task) 348 349 // Update is not possible 350 return nil 351 } 352 353 // TODO: allow a 'shutdown_command' that can be executed over a ssh connection 354 // to the VM 355 func (h *qemuHandle) Kill() error { 356 if err := h.executor.ShutDown(); err != nil { 357 if h.pluginClient.Exited() { 358 return nil 359 } 360 return fmt.Errorf("executor Shutdown failed: %v", err) 361 } 362 363 select { 364 case <-h.doneCh: 365 return nil 366 case <-time.After(h.killTimeout): 367 if h.pluginClient.Exited() { 368 return nil 369 } 370 if err := h.executor.Exit(); err != nil { 371 return fmt.Errorf("executor Exit failed: %v", err) 372 } 373 374 return nil 375 } 376 } 377 378 func (h *qemuHandle) run() { 379 ps, err := h.executor.Wait() 380 if ps.ExitCode == 0 && err != nil { 381 if e := killProcess(h.userPid); e != nil { 382 h.logger.Printf("[ERR] driver.qemu: error killing user process: %v", e) 383 } 384 if e := h.allocDir.UnmountAll(); e != nil { 385 h.logger.Printf("[ERR] driver.qemu: unmounting dev,proc and alloc dirs failed: %v", e) 386 } 387 } 388 close(h.doneCh) 389 h.waitCh <- &cstructs.WaitResult{ExitCode: ps.ExitCode, Signal: ps.Signal, Err: err} 390 close(h.waitCh) 391 // Remove services 392 if err := h.executor.DeregisterServices(); err != nil { 393 h.logger.Printf("[ERR] driver.qemu: failed to deregister services: %v", err) 394 } 395 396 h.executor.Exit() 397 h.pluginClient.Kill() 398 }