github.com/mattyr/nomad@v0.3.3-0.20160919021406-3485a065154a/client/driver/qemu.go (about) 1 package driver 2 3 import ( 4 "encoding/json" 5 "fmt" 6 "log" 7 "os/exec" 8 "path/filepath" 9 "regexp" 10 "runtime" 11 "strings" 12 "time" 13 14 "github.com/hashicorp/go-plugin" 15 "github.com/hashicorp/nomad/client/allocdir" 16 "github.com/hashicorp/nomad/client/config" 17 "github.com/hashicorp/nomad/client/driver/executor" 18 dstructs "github.com/hashicorp/nomad/client/driver/structs" 19 "github.com/hashicorp/nomad/client/fingerprint" 20 cstructs "github.com/hashicorp/nomad/client/structs" 21 "github.com/hashicorp/nomad/helper/discover" 22 "github.com/hashicorp/nomad/helper/fields" 23 "github.com/hashicorp/nomad/nomad/structs" 24 "github.com/mitchellh/mapstructure" 25 ) 26 27 var ( 28 reQemuVersion = regexp.MustCompile(`version (\d[\.\d+]+)`) 29 ) 30 31 const ( 32 // The key populated in Node Attributes to indicate presence of the Qemu 33 // driver 34 qemuDriverAttr = "driver.qemu" 35 ) 36 37 // QemuDriver is a driver for running images via Qemu 38 // We attempt to chose sane defaults for now, with more configuration available 39 // planned in the future 40 type QemuDriver struct { 41 DriverContext 42 fingerprint.StaticFingerprinter 43 } 44 45 type QemuDriverConfig struct { 46 ImagePath string `mapstructure:"image_path"` 47 Accelerator string `mapstructure:"accelerator"` 48 PortMap []map[string]int `mapstructure:"port_map"` // A map of host port labels and to guest ports. 49 Args []string `mapstructure:"args"` // extra arguments to qemu executable 50 } 51 52 // qemuHandle is returned from Start/Open as a handle to the PID 53 type qemuHandle struct { 54 pluginClient *plugin.Client 55 userPid int 56 executor executor.Executor 57 allocDir *allocdir.AllocDir 58 killTimeout time.Duration 59 maxKillTimeout time.Duration 60 logger *log.Logger 61 version string 62 waitCh chan *dstructs.WaitResult 63 doneCh chan struct{} 64 } 65 66 // NewQemuDriver is used to create a new exec driver 67 func NewQemuDriver(ctx *DriverContext) Driver { 68 return &QemuDriver{DriverContext: *ctx} 69 } 70 71 // Validate is used to validate the driver configuration 72 func (d *QemuDriver) Validate(config map[string]interface{}) error { 73 fd := &fields.FieldData{ 74 Raw: config, 75 Schema: map[string]*fields.FieldSchema{ 76 "image_path": &fields.FieldSchema{ 77 Type: fields.TypeString, 78 Required: true, 79 }, 80 "accelerator": &fields.FieldSchema{ 81 Type: fields.TypeString, 82 }, 83 "port_map": &fields.FieldSchema{ 84 Type: fields.TypeArray, 85 }, 86 "args": &fields.FieldSchema{ 87 Type: fields.TypeArray, 88 }, 89 }, 90 } 91 92 if err := fd.Validate(); err != nil { 93 return err 94 } 95 96 return nil 97 } 98 99 func (d *QemuDriver) Fingerprint(cfg *config.Config, node *structs.Node) (bool, error) { 100 // Get the current status so that we can log any debug messages only if the 101 // state changes 102 _, currentlyEnabled := node.Attributes[qemuDriverAttr] 103 104 bin := "qemu-system-x86_64" 105 if runtime.GOOS == "windows" { 106 // On windows, the "qemu-system-x86_64" command does not respond to the 107 // version flag. 108 bin = "qemu-img" 109 } 110 outBytes, err := exec.Command(bin, "--version").Output() 111 if err != nil { 112 delete(node.Attributes, qemuDriverAttr) 113 return false, nil 114 } 115 out := strings.TrimSpace(string(outBytes)) 116 117 matches := reQemuVersion.FindStringSubmatch(out) 118 if len(matches) != 2 { 119 delete(node.Attributes, qemuDriverAttr) 120 return false, fmt.Errorf("Unable to parse Qemu version string: %#v", matches) 121 } 122 123 if !currentlyEnabled { 124 d.logger.Printf("[DEBUG] driver.qemu: enabling driver") 125 } 126 node.Attributes[qemuDriverAttr] = "1" 127 node.Attributes["driver.qemu.version"] = matches[1] 128 return true, nil 129 } 130 131 // Run an existing Qemu image. Start() will pull down an existing, valid Qemu 132 // image and save it to the Drivers Allocation Dir 133 func (d *QemuDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, error) { 134 var driverConfig QemuDriverConfig 135 if err := mapstructure.WeakDecode(task.Config, &driverConfig); err != nil { 136 return nil, err 137 } 138 139 if len(driverConfig.PortMap) > 1 { 140 return nil, fmt.Errorf("Only one port_map block is allowed in the qemu driver config") 141 } 142 143 // Get the image source 144 vmPath := driverConfig.ImagePath 145 if vmPath == "" { 146 return nil, fmt.Errorf("image_path must be set") 147 } 148 vmID := filepath.Base(vmPath) 149 150 // Get the tasks local directory. 151 taskDir, ok := ctx.AllocDir.TaskDirs[d.DriverContext.taskName] 152 if !ok { 153 return nil, fmt.Errorf("Could not find task directory for task: %v", d.DriverContext.taskName) 154 } 155 156 // Parse configuration arguments 157 // Create the base arguments 158 accelerator := "tcg" 159 if driverConfig.Accelerator != "" { 160 accelerator = driverConfig.Accelerator 161 } 162 // TODO: Check a lower bounds, e.g. the default 128 of Qemu 163 mem := fmt.Sprintf("%dM", task.Resources.MemoryMB) 164 165 absPath, err := GetAbsolutePath("qemu-system-x86_64") 166 if err != nil { 167 return nil, err 168 } 169 170 args := []string{ 171 absPath, 172 "-machine", "type=pc,accel=" + accelerator, 173 "-name", vmID, 174 "-m", mem, 175 "-drive", "file=" + vmPath, 176 "-nographic", 177 } 178 179 // Add pass through arguments to qemu executable. A user can specify 180 // these arguments in driver task configuration. These arguments are 181 // passed directly to the qemu driver as command line options. 182 // For example, args = [ "-nodefconfig", "-nodefaults" ] 183 // This will allow a VM with embedded configuration to boot successfully. 184 args = append(args, driverConfig.Args...) 185 186 // Check the Resources required Networks to add port mappings. If no resources 187 // are required, we assume the VM is a purely compute job and does not require 188 // the outside world to be able to reach it. VMs ran without port mappings can 189 // still reach out to the world, but without port mappings it is effectively 190 // firewalled 191 protocols := []string{"udp", "tcp"} 192 if len(task.Resources.Networks) > 0 && len(driverConfig.PortMap) == 1 { 193 // Loop through the port map and construct the hostfwd string, to map 194 // reserved ports to the ports listenting in the VM 195 // Ex: hostfwd=tcp::22000-:22,hostfwd=tcp::80-:8080 196 var forwarding []string 197 taskPorts := task.Resources.Networks[0].MapLabelToValues(nil) 198 for label, guest := range driverConfig.PortMap[0] { 199 host, ok := taskPorts[label] 200 if !ok { 201 return nil, fmt.Errorf("Unknown port label %q", label) 202 } 203 204 for _, p := range protocols { 205 forwarding = append(forwarding, fmt.Sprintf("hostfwd=%s::%d-:%d", p, host, guest)) 206 } 207 } 208 209 if len(forwarding) != 0 { 210 args = append(args, 211 "-netdev", 212 fmt.Sprintf("user,id=user.0,%s", strings.Join(forwarding, ",")), 213 "-device", "virtio-net,netdev=user.0", 214 ) 215 } 216 } 217 218 // If using KVM, add optimization args 219 if accelerator == "kvm" { 220 args = append(args, 221 "-enable-kvm", 222 "-cpu", "host", 223 // Do we have cores information available to the Driver? 224 // "-smp", fmt.Sprintf("%d", cores), 225 ) 226 } 227 228 d.logger.Printf("[DEBUG] Starting QemuVM command: %q", strings.Join(args, " ")) 229 bin, err := discover.NomadExecutable() 230 if err != nil { 231 return nil, fmt.Errorf("unable to find the nomad binary: %v", err) 232 } 233 234 pluginLogFile := filepath.Join(taskDir, fmt.Sprintf("%s-executor.out", task.Name)) 235 pluginConfig := &plugin.ClientConfig{ 236 Cmd: exec.Command(bin, "executor", pluginLogFile), 237 } 238 239 exec, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput, d.config) 240 if err != nil { 241 return nil, err 242 } 243 executorCtx := &executor.ExecutorContext{ 244 TaskEnv: d.taskEnv, 245 Driver: "qemu", 246 AllocDir: ctx.AllocDir, 247 AllocID: ctx.AllocID, 248 Task: task, 249 } 250 ps, err := exec.LaunchCmd(&executor.ExecCommand{ 251 Cmd: args[0], 252 Args: args[1:], 253 User: task.User, 254 }, executorCtx) 255 if err != nil { 256 pluginClient.Kill() 257 return nil, err 258 } 259 d.logger.Printf("[INFO] Started new QemuVM: %s", vmID) 260 261 // Create and Return Handle 262 maxKill := d.DriverContext.config.MaxKillTimeout 263 h := &qemuHandle{ 264 pluginClient: pluginClient, 265 executor: exec, 266 userPid: ps.Pid, 267 allocDir: ctx.AllocDir, 268 killTimeout: GetKillTimeout(task.KillTimeout, maxKill), 269 maxKillTimeout: maxKill, 270 version: d.config.Version, 271 logger: d.logger, 272 doneCh: make(chan struct{}), 273 waitCh: make(chan *dstructs.WaitResult, 1), 274 } 275 276 if err := h.executor.SyncServices(consulContext(d.config, "")); err != nil { 277 h.logger.Printf("[ERR] driver.qemu: error registering services for task: %q: %v", task.Name, err) 278 } 279 go h.run() 280 return h, nil 281 } 282 283 type qemuId struct { 284 Version string 285 KillTimeout time.Duration 286 MaxKillTimeout time.Duration 287 UserPid int 288 PluginConfig *PluginReattachConfig 289 AllocDir *allocdir.AllocDir 290 } 291 292 func (d *QemuDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error) { 293 id := &qemuId{} 294 if err := json.Unmarshal([]byte(handleID), id); err != nil { 295 return nil, fmt.Errorf("Failed to parse handle '%s': %v", handleID, err) 296 } 297 298 pluginConfig := &plugin.ClientConfig{ 299 Reattach: id.PluginConfig.PluginConfig(), 300 } 301 302 exec, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput, d.config) 303 if err != nil { 304 d.logger.Println("[ERR] driver.qemu: error connecting to plugin so destroying plugin pid and user pid") 305 if e := destroyPlugin(id.PluginConfig.Pid, id.UserPid); e != nil { 306 d.logger.Printf("[ERR] driver.qemu: error destroying plugin and userpid: %v", e) 307 } 308 return nil, fmt.Errorf("error connecting to plugin: %v", err) 309 } 310 311 ver, _ := exec.Version() 312 d.logger.Printf("[DEBUG] driver.qemu: version of executor: %v", ver.Version) 313 // Return a driver handle 314 h := &qemuHandle{ 315 pluginClient: pluginClient, 316 executor: exec, 317 userPid: id.UserPid, 318 allocDir: id.AllocDir, 319 logger: d.logger, 320 killTimeout: id.KillTimeout, 321 maxKillTimeout: id.MaxKillTimeout, 322 version: id.Version, 323 doneCh: make(chan struct{}), 324 waitCh: make(chan *dstructs.WaitResult, 1), 325 } 326 if err := h.executor.SyncServices(consulContext(d.config, "")); err != nil { 327 h.logger.Printf("[ERR] driver.qemu: error registering services: %v", err) 328 } 329 go h.run() 330 return h, nil 331 } 332 333 func (h *qemuHandle) ID() string { 334 id := qemuId{ 335 Version: h.version, 336 KillTimeout: h.killTimeout, 337 MaxKillTimeout: h.maxKillTimeout, 338 PluginConfig: NewPluginReattachConfig(h.pluginClient.ReattachConfig()), 339 UserPid: h.userPid, 340 AllocDir: h.allocDir, 341 } 342 343 data, err := json.Marshal(id) 344 if err != nil { 345 h.logger.Printf("[ERR] driver.qemu: failed to marshal ID to JSON: %s", err) 346 } 347 return string(data) 348 } 349 350 func (h *qemuHandle) WaitCh() chan *dstructs.WaitResult { 351 return h.waitCh 352 } 353 354 func (h *qemuHandle) Update(task *structs.Task) error { 355 // Store the updated kill timeout. 356 h.killTimeout = GetKillTimeout(task.KillTimeout, h.maxKillTimeout) 357 h.executor.UpdateTask(task) 358 359 // Update is not possible 360 return nil 361 } 362 363 // TODO: allow a 'shutdown_command' that can be executed over a ssh connection 364 // to the VM 365 func (h *qemuHandle) Kill() error { 366 if err := h.executor.ShutDown(); err != nil { 367 if h.pluginClient.Exited() { 368 return nil 369 } 370 return fmt.Errorf("executor Shutdown failed: %v", err) 371 } 372 373 select { 374 case <-h.doneCh: 375 return nil 376 case <-time.After(h.killTimeout): 377 if h.pluginClient.Exited() { 378 return nil 379 } 380 if err := h.executor.Exit(); err != nil { 381 return fmt.Errorf("executor Exit failed: %v", err) 382 } 383 384 return nil 385 } 386 } 387 388 func (h *qemuHandle) Stats() (*cstructs.TaskResourceUsage, error) { 389 return h.executor.Stats() 390 } 391 392 func (h *qemuHandle) run() { 393 ps, err := h.executor.Wait() 394 if ps.ExitCode == 0 && err != nil { 395 if e := killProcess(h.userPid); e != nil { 396 h.logger.Printf("[ERR] driver.qemu: error killing user process: %v", e) 397 } 398 if e := h.allocDir.UnmountAll(); e != nil { 399 h.logger.Printf("[ERR] driver.qemu: unmounting dev,proc and alloc dirs failed: %v", e) 400 } 401 } 402 close(h.doneCh) 403 h.waitCh <- &dstructs.WaitResult{ExitCode: ps.ExitCode, Signal: ps.Signal, Err: err} 404 close(h.waitCh) 405 // Remove services 406 if err := h.executor.DeregisterServices(); err != nil { 407 h.logger.Printf("[ERR] driver.qemu: failed to deregister services: %v", err) 408 } 409 410 h.executor.Exit() 411 h.pluginClient.Kill() 412 }