github.com/mattyr/nomad@v0.3.3-0.20160919021406-3485a065154a/client/driver/qemu.go (about)

     1  package driver
     2  
     3  import (
     4  	"encoding/json"
     5  	"fmt"
     6  	"log"
     7  	"os/exec"
     8  	"path/filepath"
     9  	"regexp"
    10  	"runtime"
    11  	"strings"
    12  	"time"
    13  
    14  	"github.com/hashicorp/go-plugin"
    15  	"github.com/hashicorp/nomad/client/allocdir"
    16  	"github.com/hashicorp/nomad/client/config"
    17  	"github.com/hashicorp/nomad/client/driver/executor"
    18  	dstructs "github.com/hashicorp/nomad/client/driver/structs"
    19  	"github.com/hashicorp/nomad/client/fingerprint"
    20  	cstructs "github.com/hashicorp/nomad/client/structs"
    21  	"github.com/hashicorp/nomad/helper/discover"
    22  	"github.com/hashicorp/nomad/helper/fields"
    23  	"github.com/hashicorp/nomad/nomad/structs"
    24  	"github.com/mitchellh/mapstructure"
    25  )
    26  
    27  var (
    28  	reQemuVersion = regexp.MustCompile(`version (\d[\.\d+]+)`)
    29  )
    30  
    31  const (
    32  	// The key populated in Node Attributes to indicate presence of the Qemu
    33  	// driver
    34  	qemuDriverAttr = "driver.qemu"
    35  )
    36  
    37  // QemuDriver is a driver for running images via Qemu
    38  // We attempt to chose sane defaults for now, with more configuration available
    39  // planned in the future
    40  type QemuDriver struct {
    41  	DriverContext
    42  	fingerprint.StaticFingerprinter
    43  }
    44  
    45  type QemuDriverConfig struct {
    46  	ImagePath   string           `mapstructure:"image_path"`
    47  	Accelerator string           `mapstructure:"accelerator"`
    48  	PortMap     []map[string]int `mapstructure:"port_map"` // A map of host port labels and to guest ports.
    49  	Args        []string         `mapstructure:"args"`     // extra arguments to qemu executable
    50  }
    51  
    52  // qemuHandle is returned from Start/Open as a handle to the PID
    53  type qemuHandle struct {
    54  	pluginClient   *plugin.Client
    55  	userPid        int
    56  	executor       executor.Executor
    57  	allocDir       *allocdir.AllocDir
    58  	killTimeout    time.Duration
    59  	maxKillTimeout time.Duration
    60  	logger         *log.Logger
    61  	version        string
    62  	waitCh         chan *dstructs.WaitResult
    63  	doneCh         chan struct{}
    64  }
    65  
    66  // NewQemuDriver is used to create a new exec driver
    67  func NewQemuDriver(ctx *DriverContext) Driver {
    68  	return &QemuDriver{DriverContext: *ctx}
    69  }
    70  
    71  // Validate is used to validate the driver configuration
    72  func (d *QemuDriver) Validate(config map[string]interface{}) error {
    73  	fd := &fields.FieldData{
    74  		Raw: config,
    75  		Schema: map[string]*fields.FieldSchema{
    76  			"image_path": &fields.FieldSchema{
    77  				Type:     fields.TypeString,
    78  				Required: true,
    79  			},
    80  			"accelerator": &fields.FieldSchema{
    81  				Type: fields.TypeString,
    82  			},
    83  			"port_map": &fields.FieldSchema{
    84  				Type: fields.TypeArray,
    85  			},
    86  			"args": &fields.FieldSchema{
    87  				Type: fields.TypeArray,
    88  			},
    89  		},
    90  	}
    91  
    92  	if err := fd.Validate(); err != nil {
    93  		return err
    94  	}
    95  
    96  	return nil
    97  }
    98  
    99  func (d *QemuDriver) Fingerprint(cfg *config.Config, node *structs.Node) (bool, error) {
   100  	// Get the current status so that we can log any debug messages only if the
   101  	// state changes
   102  	_, currentlyEnabled := node.Attributes[qemuDriverAttr]
   103  
   104  	bin := "qemu-system-x86_64"
   105  	if runtime.GOOS == "windows" {
   106  		// On windows, the "qemu-system-x86_64" command does not respond to the
   107  		// version flag.
   108  		bin = "qemu-img"
   109  	}
   110  	outBytes, err := exec.Command(bin, "--version").Output()
   111  	if err != nil {
   112  		delete(node.Attributes, qemuDriverAttr)
   113  		return false, nil
   114  	}
   115  	out := strings.TrimSpace(string(outBytes))
   116  
   117  	matches := reQemuVersion.FindStringSubmatch(out)
   118  	if len(matches) != 2 {
   119  		delete(node.Attributes, qemuDriverAttr)
   120  		return false, fmt.Errorf("Unable to parse Qemu version string: %#v", matches)
   121  	}
   122  
   123  	if !currentlyEnabled {
   124  		d.logger.Printf("[DEBUG] driver.qemu: enabling driver")
   125  	}
   126  	node.Attributes[qemuDriverAttr] = "1"
   127  	node.Attributes["driver.qemu.version"] = matches[1]
   128  	return true, nil
   129  }
   130  
   131  // Run an existing Qemu image. Start() will pull down an existing, valid Qemu
   132  // image and save it to the Drivers Allocation Dir
   133  func (d *QemuDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, error) {
   134  	var driverConfig QemuDriverConfig
   135  	if err := mapstructure.WeakDecode(task.Config, &driverConfig); err != nil {
   136  		return nil, err
   137  	}
   138  
   139  	if len(driverConfig.PortMap) > 1 {
   140  		return nil, fmt.Errorf("Only one port_map block is allowed in the qemu driver config")
   141  	}
   142  
   143  	// Get the image source
   144  	vmPath := driverConfig.ImagePath
   145  	if vmPath == "" {
   146  		return nil, fmt.Errorf("image_path must be set")
   147  	}
   148  	vmID := filepath.Base(vmPath)
   149  
   150  	// Get the tasks local directory.
   151  	taskDir, ok := ctx.AllocDir.TaskDirs[d.DriverContext.taskName]
   152  	if !ok {
   153  		return nil, fmt.Errorf("Could not find task directory for task: %v", d.DriverContext.taskName)
   154  	}
   155  
   156  	// Parse configuration arguments
   157  	// Create the base arguments
   158  	accelerator := "tcg"
   159  	if driverConfig.Accelerator != "" {
   160  		accelerator = driverConfig.Accelerator
   161  	}
   162  	// TODO: Check a lower bounds, e.g. the default 128 of Qemu
   163  	mem := fmt.Sprintf("%dM", task.Resources.MemoryMB)
   164  
   165  	absPath, err := GetAbsolutePath("qemu-system-x86_64")
   166  	if err != nil {
   167  		return nil, err
   168  	}
   169  
   170  	args := []string{
   171  		absPath,
   172  		"-machine", "type=pc,accel=" + accelerator,
   173  		"-name", vmID,
   174  		"-m", mem,
   175  		"-drive", "file=" + vmPath,
   176  		"-nographic",
   177  	}
   178  
   179  	// Add pass through arguments to qemu executable. A user can specify
   180  	// these arguments in driver task configuration. These arguments are
   181  	// passed directly to the qemu driver as command line options.
   182  	// For example, args = [ "-nodefconfig", "-nodefaults" ]
   183  	// This will allow a VM with embedded configuration to boot successfully.
   184  	args = append(args, driverConfig.Args...)
   185  
   186  	// Check the Resources required Networks to add port mappings. If no resources
   187  	// are required, we assume the VM is a purely compute job and does not require
   188  	// the outside world to be able to reach it. VMs ran without port mappings can
   189  	// still reach out to the world, but without port mappings it is effectively
   190  	// firewalled
   191  	protocols := []string{"udp", "tcp"}
   192  	if len(task.Resources.Networks) > 0 && len(driverConfig.PortMap) == 1 {
   193  		// Loop through the port map and construct the hostfwd string, to map
   194  		// reserved ports to the ports listenting in the VM
   195  		// Ex: hostfwd=tcp::22000-:22,hostfwd=tcp::80-:8080
   196  		var forwarding []string
   197  		taskPorts := task.Resources.Networks[0].MapLabelToValues(nil)
   198  		for label, guest := range driverConfig.PortMap[0] {
   199  			host, ok := taskPorts[label]
   200  			if !ok {
   201  				return nil, fmt.Errorf("Unknown port label %q", label)
   202  			}
   203  
   204  			for _, p := range protocols {
   205  				forwarding = append(forwarding, fmt.Sprintf("hostfwd=%s::%d-:%d", p, host, guest))
   206  			}
   207  		}
   208  
   209  		if len(forwarding) != 0 {
   210  			args = append(args,
   211  				"-netdev",
   212  				fmt.Sprintf("user,id=user.0,%s", strings.Join(forwarding, ",")),
   213  				"-device", "virtio-net,netdev=user.0",
   214  			)
   215  		}
   216  	}
   217  
   218  	// If using KVM, add optimization args
   219  	if accelerator == "kvm" {
   220  		args = append(args,
   221  			"-enable-kvm",
   222  			"-cpu", "host",
   223  			// Do we have cores information available to the Driver?
   224  			// "-smp", fmt.Sprintf("%d", cores),
   225  		)
   226  	}
   227  
   228  	d.logger.Printf("[DEBUG] Starting QemuVM command: %q", strings.Join(args, " "))
   229  	bin, err := discover.NomadExecutable()
   230  	if err != nil {
   231  		return nil, fmt.Errorf("unable to find the nomad binary: %v", err)
   232  	}
   233  
   234  	pluginLogFile := filepath.Join(taskDir, fmt.Sprintf("%s-executor.out", task.Name))
   235  	pluginConfig := &plugin.ClientConfig{
   236  		Cmd: exec.Command(bin, "executor", pluginLogFile),
   237  	}
   238  
   239  	exec, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput, d.config)
   240  	if err != nil {
   241  		return nil, err
   242  	}
   243  	executorCtx := &executor.ExecutorContext{
   244  		TaskEnv:  d.taskEnv,
   245  		Driver:   "qemu",
   246  		AllocDir: ctx.AllocDir,
   247  		AllocID:  ctx.AllocID,
   248  		Task:     task,
   249  	}
   250  	ps, err := exec.LaunchCmd(&executor.ExecCommand{
   251  		Cmd:  args[0],
   252  		Args: args[1:],
   253  		User: task.User,
   254  	}, executorCtx)
   255  	if err != nil {
   256  		pluginClient.Kill()
   257  		return nil, err
   258  	}
   259  	d.logger.Printf("[INFO] Started new QemuVM: %s", vmID)
   260  
   261  	// Create and Return Handle
   262  	maxKill := d.DriverContext.config.MaxKillTimeout
   263  	h := &qemuHandle{
   264  		pluginClient:   pluginClient,
   265  		executor:       exec,
   266  		userPid:        ps.Pid,
   267  		allocDir:       ctx.AllocDir,
   268  		killTimeout:    GetKillTimeout(task.KillTimeout, maxKill),
   269  		maxKillTimeout: maxKill,
   270  		version:        d.config.Version,
   271  		logger:         d.logger,
   272  		doneCh:         make(chan struct{}),
   273  		waitCh:         make(chan *dstructs.WaitResult, 1),
   274  	}
   275  
   276  	if err := h.executor.SyncServices(consulContext(d.config, "")); err != nil {
   277  		h.logger.Printf("[ERR] driver.qemu: error registering services for task: %q: %v", task.Name, err)
   278  	}
   279  	go h.run()
   280  	return h, nil
   281  }
   282  
   283  type qemuId struct {
   284  	Version        string
   285  	KillTimeout    time.Duration
   286  	MaxKillTimeout time.Duration
   287  	UserPid        int
   288  	PluginConfig   *PluginReattachConfig
   289  	AllocDir       *allocdir.AllocDir
   290  }
   291  
   292  func (d *QemuDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error) {
   293  	id := &qemuId{}
   294  	if err := json.Unmarshal([]byte(handleID), id); err != nil {
   295  		return nil, fmt.Errorf("Failed to parse handle '%s': %v", handleID, err)
   296  	}
   297  
   298  	pluginConfig := &plugin.ClientConfig{
   299  		Reattach: id.PluginConfig.PluginConfig(),
   300  	}
   301  
   302  	exec, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput, d.config)
   303  	if err != nil {
   304  		d.logger.Println("[ERR] driver.qemu: error connecting to plugin so destroying plugin pid and user pid")
   305  		if e := destroyPlugin(id.PluginConfig.Pid, id.UserPid); e != nil {
   306  			d.logger.Printf("[ERR] driver.qemu: error destroying plugin and userpid: %v", e)
   307  		}
   308  		return nil, fmt.Errorf("error connecting to plugin: %v", err)
   309  	}
   310  
   311  	ver, _ := exec.Version()
   312  	d.logger.Printf("[DEBUG] driver.qemu: version of executor: %v", ver.Version)
   313  	// Return a driver handle
   314  	h := &qemuHandle{
   315  		pluginClient:   pluginClient,
   316  		executor:       exec,
   317  		userPid:        id.UserPid,
   318  		allocDir:       id.AllocDir,
   319  		logger:         d.logger,
   320  		killTimeout:    id.KillTimeout,
   321  		maxKillTimeout: id.MaxKillTimeout,
   322  		version:        id.Version,
   323  		doneCh:         make(chan struct{}),
   324  		waitCh:         make(chan *dstructs.WaitResult, 1),
   325  	}
   326  	if err := h.executor.SyncServices(consulContext(d.config, "")); err != nil {
   327  		h.logger.Printf("[ERR] driver.qemu: error registering services: %v", err)
   328  	}
   329  	go h.run()
   330  	return h, nil
   331  }
   332  
   333  func (h *qemuHandle) ID() string {
   334  	id := qemuId{
   335  		Version:        h.version,
   336  		KillTimeout:    h.killTimeout,
   337  		MaxKillTimeout: h.maxKillTimeout,
   338  		PluginConfig:   NewPluginReattachConfig(h.pluginClient.ReattachConfig()),
   339  		UserPid:        h.userPid,
   340  		AllocDir:       h.allocDir,
   341  	}
   342  
   343  	data, err := json.Marshal(id)
   344  	if err != nil {
   345  		h.logger.Printf("[ERR] driver.qemu: failed to marshal ID to JSON: %s", err)
   346  	}
   347  	return string(data)
   348  }
   349  
   350  func (h *qemuHandle) WaitCh() chan *dstructs.WaitResult {
   351  	return h.waitCh
   352  }
   353  
   354  func (h *qemuHandle) Update(task *structs.Task) error {
   355  	// Store the updated kill timeout.
   356  	h.killTimeout = GetKillTimeout(task.KillTimeout, h.maxKillTimeout)
   357  	h.executor.UpdateTask(task)
   358  
   359  	// Update is not possible
   360  	return nil
   361  }
   362  
   363  // TODO: allow a 'shutdown_command' that can be executed over a ssh connection
   364  // to the VM
   365  func (h *qemuHandle) Kill() error {
   366  	if err := h.executor.ShutDown(); err != nil {
   367  		if h.pluginClient.Exited() {
   368  			return nil
   369  		}
   370  		return fmt.Errorf("executor Shutdown failed: %v", err)
   371  	}
   372  
   373  	select {
   374  	case <-h.doneCh:
   375  		return nil
   376  	case <-time.After(h.killTimeout):
   377  		if h.pluginClient.Exited() {
   378  			return nil
   379  		}
   380  		if err := h.executor.Exit(); err != nil {
   381  			return fmt.Errorf("executor Exit failed: %v", err)
   382  		}
   383  
   384  		return nil
   385  	}
   386  }
   387  
   388  func (h *qemuHandle) Stats() (*cstructs.TaskResourceUsage, error) {
   389  	return h.executor.Stats()
   390  }
   391  
   392  func (h *qemuHandle) run() {
   393  	ps, err := h.executor.Wait()
   394  	if ps.ExitCode == 0 && err != nil {
   395  		if e := killProcess(h.userPid); e != nil {
   396  			h.logger.Printf("[ERR] driver.qemu: error killing user process: %v", e)
   397  		}
   398  		if e := h.allocDir.UnmountAll(); e != nil {
   399  			h.logger.Printf("[ERR] driver.qemu: unmounting dev,proc and alloc dirs failed: %v", e)
   400  		}
   401  	}
   402  	close(h.doneCh)
   403  	h.waitCh <- &dstructs.WaitResult{ExitCode: ps.ExitCode, Signal: ps.Signal, Err: err}
   404  	close(h.waitCh)
   405  	// Remove services
   406  	if err := h.executor.DeregisterServices(); err != nil {
   407  		h.logger.Printf("[ERR] driver.qemu: failed to deregister services: %v", err)
   408  	}
   409  
   410  	h.executor.Exit()
   411  	h.pluginClient.Kill()
   412  }