github.com/dkerwin/nomad@v0.3.3-0.20160525181927-74554135514b/client/driver/qemu.go (about)

     1  package driver
     2  
     3  import (
     4  	"encoding/json"
     5  	"fmt"
     6  	"log"
     7  	"os/exec"
     8  	"path/filepath"
     9  	"regexp"
    10  	"runtime"
    11  	"strings"
    12  	"time"
    13  
    14  	"github.com/hashicorp/go-plugin"
    15  	"github.com/hashicorp/nomad/client/allocdir"
    16  	"github.com/hashicorp/nomad/client/config"
    17  	"github.com/hashicorp/nomad/client/driver/executor"
    18  	cstructs "github.com/hashicorp/nomad/client/driver/structs"
    19  	"github.com/hashicorp/nomad/client/fingerprint"
    20  	"github.com/hashicorp/nomad/helper/discover"
    21  	"github.com/hashicorp/nomad/helper/fields"
    22  	"github.com/hashicorp/nomad/nomad/structs"
    23  	"github.com/mitchellh/mapstructure"
    24  )
    25  
    26  var (
    27  	reQemuVersion = regexp.MustCompile(`version (\d[\.\d+]+)`)
    28  )
    29  
    30  const (
    31  	// The key populated in Node Attributes to indicate presence of the Qemu
    32  	// driver
    33  	qemuDriverAttr = "driver.qemu"
    34  )
    35  
    36  // QemuDriver is a driver for running images via Qemu
    37  // We attempt to chose sane defaults for now, with more configuration available
    38  // planned in the future
    39  type QemuDriver struct {
    40  	DriverContext
    41  	fingerprint.StaticFingerprinter
    42  }
    43  
    44  type QemuDriverConfig struct {
    45  	ImagePath   string           `mapstructure:"image_path"`
    46  	Accelerator string           `mapstructure:"accelerator"`
    47  	PortMap     []map[string]int `mapstructure:"port_map"` // A map of host port labels and to guest ports.
    48  }
    49  
    50  // qemuHandle is returned from Start/Open as a handle to the PID
    51  type qemuHandle struct {
    52  	pluginClient   *plugin.Client
    53  	userPid        int
    54  	executor       executor.Executor
    55  	allocDir       *allocdir.AllocDir
    56  	killTimeout    time.Duration
    57  	maxKillTimeout time.Duration
    58  	logger         *log.Logger
    59  	version        string
    60  	waitCh         chan *cstructs.WaitResult
    61  	doneCh         chan struct{}
    62  }
    63  
    64  // NewQemuDriver is used to create a new exec driver
    65  func NewQemuDriver(ctx *DriverContext) Driver {
    66  	return &QemuDriver{DriverContext: *ctx}
    67  }
    68  
    69  // Validate is used to validate the driver configuration
    70  func (d *QemuDriver) Validate(config map[string]interface{}) error {
    71  	fd := &fields.FieldData{
    72  		Raw: config,
    73  		Schema: map[string]*fields.FieldSchema{
    74  			"image_path": &fields.FieldSchema{
    75  				Type:     fields.TypeString,
    76  				Required: true,
    77  			},
    78  			"accelerator": &fields.FieldSchema{
    79  				Type: fields.TypeString,
    80  			},
    81  			"port_map": &fields.FieldSchema{
    82  				Type: fields.TypeArray,
    83  			},
    84  		},
    85  	}
    86  
    87  	if err := fd.Validate(); err != nil {
    88  		return err
    89  	}
    90  
    91  	return nil
    92  }
    93  
    94  func (d *QemuDriver) Fingerprint(cfg *config.Config, node *structs.Node) (bool, error) {
    95  	// Get the current status so that we can log any debug messages only if the
    96  	// state changes
    97  	_, currentlyEnabled := node.Attributes[qemuDriverAttr]
    98  
    99  	bin := "qemu-system-x86_64"
   100  	if runtime.GOOS == "windows" {
   101  		// On windows, the "qemu-system-x86_64" command does not respond to the
   102  		// version flag.
   103  		bin = "qemu-img"
   104  	}
   105  	outBytes, err := exec.Command(bin, "--version").Output()
   106  	if err != nil {
   107  		delete(node.Attributes, qemuDriverAttr)
   108  		return false, nil
   109  	}
   110  	out := strings.TrimSpace(string(outBytes))
   111  
   112  	matches := reQemuVersion.FindStringSubmatch(out)
   113  	if len(matches) != 2 {
   114  		delete(node.Attributes, qemuDriverAttr)
   115  		return false, fmt.Errorf("Unable to parse Qemu version string: %#v", matches)
   116  	}
   117  
   118  	if !currentlyEnabled {
   119  		d.logger.Printf("[DEBUG] driver.qemu: enabling driver")
   120  	}
   121  	node.Attributes[qemuDriverAttr] = "1"
   122  	node.Attributes["driver.qemu.version"] = matches[1]
   123  	return true, nil
   124  }
   125  
   126  // Run an existing Qemu image. Start() will pull down an existing, valid Qemu
   127  // image and save it to the Drivers Allocation Dir
   128  func (d *QemuDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, error) {
   129  	var driverConfig QemuDriverConfig
   130  	if err := mapstructure.WeakDecode(task.Config, &driverConfig); err != nil {
   131  		return nil, err
   132  	}
   133  
   134  	if len(driverConfig.PortMap) > 1 {
   135  		return nil, fmt.Errorf("Only one port_map block is allowed in the qemu driver config")
   136  	}
   137  
   138  	// Get the image source
   139  	vmPath := driverConfig.ImagePath
   140  	if vmPath == "" {
   141  		return nil, fmt.Errorf("image_path must be set")
   142  	}
   143  	vmID := filepath.Base(vmPath)
   144  
   145  	// Get the tasks local directory.
   146  	taskDir, ok := ctx.AllocDir.TaskDirs[d.DriverContext.taskName]
   147  	if !ok {
   148  		return nil, fmt.Errorf("Could not find task directory for task: %v", d.DriverContext.taskName)
   149  	}
   150  
   151  	// Parse configuration arguments
   152  	// Create the base arguments
   153  	accelerator := "tcg"
   154  	if driverConfig.Accelerator != "" {
   155  		accelerator = driverConfig.Accelerator
   156  	}
   157  	// TODO: Check a lower bounds, e.g. the default 128 of Qemu
   158  	mem := fmt.Sprintf("%dM", task.Resources.MemoryMB)
   159  
   160  	absPath, err := GetAbsolutePath("qemu-system-x86_64")
   161  	if err != nil {
   162  		return nil, err
   163  	}
   164  
   165  	args := []string{
   166  		absPath,
   167  		"-machine", "type=pc,accel=" + accelerator,
   168  		"-name", vmID,
   169  		"-m", mem,
   170  		"-drive", "file=" + vmPath,
   171  		"-nodefconfig",
   172  		"-nodefaults",
   173  		"-nographic",
   174  	}
   175  
   176  	// Check the Resources required Networks to add port mappings. If no resources
   177  	// are required, we assume the VM is a purely compute job and does not require
   178  	// the outside world to be able to reach it. VMs ran without port mappings can
   179  	// still reach out to the world, but without port mappings it is effectively
   180  	// firewalled
   181  	protocols := []string{"udp", "tcp"}
   182  	if len(task.Resources.Networks) > 0 && len(driverConfig.PortMap) == 1 {
   183  		// Loop through the port map and construct the hostfwd string, to map
   184  		// reserved ports to the ports listenting in the VM
   185  		// Ex: hostfwd=tcp::22000-:22,hostfwd=tcp::80-:8080
   186  		var forwarding []string
   187  		taskPorts := task.Resources.Networks[0].MapLabelToValues(nil)
   188  		for label, guest := range driverConfig.PortMap[0] {
   189  			host, ok := taskPorts[label]
   190  			if !ok {
   191  				return nil, fmt.Errorf("Unknown port label %q", label)
   192  			}
   193  
   194  			for _, p := range protocols {
   195  				forwarding = append(forwarding, fmt.Sprintf("hostfwd=%s::%d-:%d", p, host, guest))
   196  			}
   197  		}
   198  
   199  		if len(forwarding) != 0 {
   200  			args = append(args,
   201  				"-netdev",
   202  				fmt.Sprintf("user,id=user.0,%s", strings.Join(forwarding, ",")),
   203  				"-device", "virtio-net,netdev=user.0",
   204  			)
   205  		}
   206  	}
   207  
   208  	// If using KVM, add optimization args
   209  	if accelerator == "kvm" {
   210  		args = append(args,
   211  			"-enable-kvm",
   212  			"-cpu", "host",
   213  			// Do we have cores information available to the Driver?
   214  			// "-smp", fmt.Sprintf("%d", cores),
   215  		)
   216  	}
   217  
   218  	d.logger.Printf("[DEBUG] Starting QemuVM command: %q", strings.Join(args, " "))
   219  	bin, err := discover.NomadExecutable()
   220  	if err != nil {
   221  		return nil, fmt.Errorf("unable to find the nomad binary: %v", err)
   222  	}
   223  
   224  	pluginLogFile := filepath.Join(taskDir, fmt.Sprintf("%s-executor.out", task.Name))
   225  	pluginConfig := &plugin.ClientConfig{
   226  		Cmd: exec.Command(bin, "executor", pluginLogFile),
   227  	}
   228  
   229  	exec, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput, d.config)
   230  	if err != nil {
   231  		return nil, err
   232  	}
   233  	executorCtx := &executor.ExecutorContext{
   234  		TaskEnv:  d.taskEnv,
   235  		Driver:   "qemu",
   236  		AllocDir: ctx.AllocDir,
   237  		AllocID:  ctx.AllocID,
   238  		Task:     task,
   239  	}
   240  	ps, err := exec.LaunchCmd(&executor.ExecCommand{
   241  		Cmd:  args[0],
   242  		Args: args[1:],
   243  		User: task.User,
   244  	}, executorCtx)
   245  	if err != nil {
   246  		pluginClient.Kill()
   247  		return nil, err
   248  	}
   249  	d.logger.Printf("[INFO] Started new QemuVM: %s", vmID)
   250  
   251  	// Create and Return Handle
   252  	maxKill := d.DriverContext.config.MaxKillTimeout
   253  	h := &qemuHandle{
   254  		pluginClient:   pluginClient,
   255  		executor:       exec,
   256  		userPid:        ps.Pid,
   257  		allocDir:       ctx.AllocDir,
   258  		killTimeout:    GetKillTimeout(task.KillTimeout, maxKill),
   259  		maxKillTimeout: maxKill,
   260  		version:        d.config.Version,
   261  		logger:         d.logger,
   262  		doneCh:         make(chan struct{}),
   263  		waitCh:         make(chan *cstructs.WaitResult, 1),
   264  	}
   265  
   266  	if err := h.executor.SyncServices(consulContext(d.config, "")); err != nil {
   267  		h.logger.Printf("[ERR] driver.qemu: error registering services for task: %q: %v", task.Name, err)
   268  	}
   269  	go h.run()
   270  	return h, nil
   271  }
   272  
   273  type qemuId struct {
   274  	Version        string
   275  	KillTimeout    time.Duration
   276  	MaxKillTimeout time.Duration
   277  	UserPid        int
   278  	PluginConfig   *PluginReattachConfig
   279  	AllocDir       *allocdir.AllocDir
   280  }
   281  
   282  func (d *QemuDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error) {
   283  	id := &qemuId{}
   284  	if err := json.Unmarshal([]byte(handleID), id); err != nil {
   285  		return nil, fmt.Errorf("Failed to parse handle '%s': %v", handleID, err)
   286  	}
   287  
   288  	pluginConfig := &plugin.ClientConfig{
   289  		Reattach: id.PluginConfig.PluginConfig(),
   290  	}
   291  
   292  	exec, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput, d.config)
   293  	if err != nil {
   294  		d.logger.Println("[ERR] driver.qemu: error connecting to plugin so destroying plugin pid and user pid")
   295  		if e := destroyPlugin(id.PluginConfig.Pid, id.UserPid); e != nil {
   296  			d.logger.Printf("[ERR] driver.qemu: error destroying plugin and userpid: %v", e)
   297  		}
   298  		return nil, fmt.Errorf("error connecting to plugin: %v", err)
   299  	}
   300  
   301  	ver, _ := exec.Version()
   302  	d.logger.Printf("[DEBUG] driver.qemu: version of executor: %v", ver.Version)
   303  	// Return a driver handle
   304  	h := &qemuHandle{
   305  		pluginClient:   pluginClient,
   306  		executor:       exec,
   307  		userPid:        id.UserPid,
   308  		allocDir:       id.AllocDir,
   309  		logger:         d.logger,
   310  		killTimeout:    id.KillTimeout,
   311  		maxKillTimeout: id.MaxKillTimeout,
   312  		version:        id.Version,
   313  		doneCh:         make(chan struct{}),
   314  		waitCh:         make(chan *cstructs.WaitResult, 1),
   315  	}
   316  	if err := h.executor.SyncServices(consulContext(d.config, "")); err != nil {
   317  		h.logger.Printf("[ERR] driver.qemu: error registering services: %v", err)
   318  	}
   319  	go h.run()
   320  	return h, nil
   321  }
   322  
   323  func (h *qemuHandle) ID() string {
   324  	id := qemuId{
   325  		Version:        h.version,
   326  		KillTimeout:    h.killTimeout,
   327  		MaxKillTimeout: h.maxKillTimeout,
   328  		PluginConfig:   NewPluginReattachConfig(h.pluginClient.ReattachConfig()),
   329  		UserPid:        h.userPid,
   330  		AllocDir:       h.allocDir,
   331  	}
   332  
   333  	data, err := json.Marshal(id)
   334  	if err != nil {
   335  		h.logger.Printf("[ERR] driver.qemu: failed to marshal ID to JSON: %s", err)
   336  	}
   337  	return string(data)
   338  }
   339  
   340  func (h *qemuHandle) WaitCh() chan *cstructs.WaitResult {
   341  	return h.waitCh
   342  }
   343  
   344  func (h *qemuHandle) Update(task *structs.Task) error {
   345  	// Store the updated kill timeout.
   346  	h.killTimeout = GetKillTimeout(task.KillTimeout, h.maxKillTimeout)
   347  	h.executor.UpdateTask(task)
   348  
   349  	// Update is not possible
   350  	return nil
   351  }
   352  
   353  // TODO: allow a 'shutdown_command' that can be executed over a ssh connection
   354  // to the VM
   355  func (h *qemuHandle) Kill() error {
   356  	if err := h.executor.ShutDown(); err != nil {
   357  		if h.pluginClient.Exited() {
   358  			return nil
   359  		}
   360  		return fmt.Errorf("executor Shutdown failed: %v", err)
   361  	}
   362  
   363  	select {
   364  	case <-h.doneCh:
   365  		return nil
   366  	case <-time.After(h.killTimeout):
   367  		if h.pluginClient.Exited() {
   368  			return nil
   369  		}
   370  		if err := h.executor.Exit(); err != nil {
   371  			return fmt.Errorf("executor Exit failed: %v", err)
   372  		}
   373  
   374  		return nil
   375  	}
   376  }
   377  
   378  func (h *qemuHandle) run() {
   379  	ps, err := h.executor.Wait()
   380  	if ps.ExitCode == 0 && err != nil {
   381  		if e := killProcess(h.userPid); e != nil {
   382  			h.logger.Printf("[ERR] driver.qemu: error killing user process: %v", e)
   383  		}
   384  		if e := h.allocDir.UnmountAll(); e != nil {
   385  			h.logger.Printf("[ERR] driver.qemu: unmounting dev,proc and alloc dirs failed: %v", e)
   386  		}
   387  	}
   388  	close(h.doneCh)
   389  	h.waitCh <- &cstructs.WaitResult{ExitCode: ps.ExitCode, Signal: ps.Signal, Err: err}
   390  	close(h.waitCh)
   391  	// Remove services
   392  	if err := h.executor.DeregisterServices(); err != nil {
   393  		h.logger.Printf("[ERR] driver.qemu: failed to deregister services: %v", err)
   394  	}
   395  
   396  	h.executor.Exit()
   397  	h.pluginClient.Kill()
   398  }