github.com/ranjib/nomad@v0.1.1-0.20160225204057-97751b02f70b/client/driver/qemu.go (about)

     1  package driver
     2  
     3  import (
     4  	"encoding/json"
     5  	"fmt"
     6  	"log"
     7  	"os/exec"
     8  	"path/filepath"
     9  	"regexp"
    10  	"runtime"
    11  	"strings"
    12  	"time"
    13  
    14  	"github.com/hashicorp/go-plugin"
    15  	"github.com/hashicorp/nomad/client/allocdir"
    16  	"github.com/hashicorp/nomad/client/config"
    17  	"github.com/hashicorp/nomad/client/driver/executor"
    18  	cstructs "github.com/hashicorp/nomad/client/driver/structs"
    19  	"github.com/hashicorp/nomad/client/fingerprint"
    20  	"github.com/hashicorp/nomad/client/getter"
    21  	"github.com/hashicorp/nomad/helper/discover"
    22  	"github.com/hashicorp/nomad/nomad/structs"
    23  	"github.com/mitchellh/mapstructure"
    24  )
    25  
    26  var (
    27  	reQemuVersion = regexp.MustCompile(`version (\d[\.\d+]+)`)
    28  )
    29  
    30  // QemuDriver is a driver for running images via Qemu
    31  // We attempt to chose sane defaults for now, with more configuration available
    32  // planned in the future
    33  type QemuDriver struct {
    34  	DriverContext
    35  	fingerprint.StaticFingerprinter
    36  }
    37  
    38  type QemuDriverConfig struct {
    39  	ArtifactSource string           `mapstructure:"artifact_source"`
    40  	Checksum       string           `mapstructure:"checksum"`
    41  	Accelerator    string           `mapstructure:"accelerator"`
    42  	PortMap        []map[string]int `mapstructure:"port_map"` // A map of host port labels and to guest ports.
    43  }
    44  
    45  // qemuHandle is returned from Start/Open as a handle to the PID
    46  type qemuHandle struct {
    47  	pluginClient *plugin.Client
    48  	userPid      int
    49  	executor     executor.Executor
    50  	allocDir     *allocdir.AllocDir
    51  	killTimeout  time.Duration
    52  	logger       *log.Logger
    53  	version      string
    54  	waitCh       chan *cstructs.WaitResult
    55  	doneCh       chan struct{}
    56  }
    57  
    58  // NewQemuDriver is used to create a new exec driver
    59  func NewQemuDriver(ctx *DriverContext) Driver {
    60  	return &QemuDriver{DriverContext: *ctx}
    61  }
    62  
    63  func (d *QemuDriver) Fingerprint(cfg *config.Config, node *structs.Node) (bool, error) {
    64  	bin := "qemu-system-x86_64"
    65  	if runtime.GOOS == "windows" {
    66  		// On windows, the "qemu-system-x86_64" command does not respond to the
    67  		// version flag.
    68  		bin = "qemu-img"
    69  	}
    70  	outBytes, err := exec.Command(bin, "--version").Output()
    71  	if err != nil {
    72  		return false, nil
    73  	}
    74  	out := strings.TrimSpace(string(outBytes))
    75  
    76  	matches := reQemuVersion.FindStringSubmatch(out)
    77  	if len(matches) != 2 {
    78  		return false, fmt.Errorf("Unable to parse Qemu version string: %#v", matches)
    79  	}
    80  
    81  	node.Attributes["driver.qemu"] = "1"
    82  	node.Attributes["driver.qemu.version"] = matches[1]
    83  
    84  	return true, nil
    85  }
    86  
    87  // Run an existing Qemu image. Start() will pull down an existing, valid Qemu
    88  // image and save it to the Drivers Allocation Dir
    89  func (d *QemuDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, error) {
    90  	var driverConfig QemuDriverConfig
    91  	if err := mapstructure.WeakDecode(task.Config, &driverConfig); err != nil {
    92  		return nil, err
    93  	}
    94  
    95  	if len(driverConfig.PortMap) > 1 {
    96  		return nil, fmt.Errorf("Only one port_map block is allowed in the qemu driver config")
    97  	}
    98  
    99  	// Get the image source
   100  	source, ok := task.Config["artifact_source"]
   101  	if !ok || source == "" {
   102  		return nil, fmt.Errorf("Missing source image Qemu driver")
   103  	}
   104  
   105  	// Qemu defaults to 128M of RAM for a given VM. Instead, we force users to
   106  	// supply a memory size in the tasks resources
   107  	if task.Resources == nil || task.Resources.MemoryMB == 0 {
   108  		return nil, fmt.Errorf("Missing required Task Resource: Memory")
   109  	}
   110  
   111  	// Get the tasks local directory.
   112  	taskDir, ok := ctx.AllocDir.TaskDirs[d.DriverContext.taskName]
   113  	if !ok {
   114  		return nil, fmt.Errorf("Could not find task directory for task: %v", d.DriverContext.taskName)
   115  	}
   116  
   117  	// Proceed to download an artifact to be executed.
   118  	vmPath, err := getter.GetArtifact(
   119  		taskDir,
   120  		driverConfig.ArtifactSource,
   121  		driverConfig.Checksum,
   122  		d.logger,
   123  	)
   124  	if err != nil {
   125  		return nil, err
   126  	}
   127  
   128  	vmID := filepath.Base(vmPath)
   129  
   130  	// Parse configuration arguments
   131  	// Create the base arguments
   132  	accelerator := "tcg"
   133  	if driverConfig.Accelerator != "" {
   134  		accelerator = driverConfig.Accelerator
   135  	}
   136  	// TODO: Check a lower bounds, e.g. the default 128 of Qemu
   137  	mem := fmt.Sprintf("%dM", task.Resources.MemoryMB)
   138  
   139  	args := []string{
   140  		"qemu-system-x86_64",
   141  		"-machine", "type=pc,accel=" + accelerator,
   142  		"-name", vmID,
   143  		"-m", mem,
   144  		"-drive", "file=" + vmPath,
   145  		"-nodefconfig",
   146  		"-nodefaults",
   147  		"-nographic",
   148  	}
   149  
   150  	// Check the Resources required Networks to add port mappings. If no resources
   151  	// are required, we assume the VM is a purely compute job and does not require
   152  	// the outside world to be able to reach it. VMs ran without port mappings can
   153  	// still reach out to the world, but without port mappings it is effectively
   154  	// firewalled
   155  	protocols := []string{"udp", "tcp"}
   156  	if len(task.Resources.Networks) > 0 && len(driverConfig.PortMap) == 1 {
   157  		// Loop through the port map and construct the hostfwd string, to map
   158  		// reserved ports to the ports listenting in the VM
   159  		// Ex: hostfwd=tcp::22000-:22,hostfwd=tcp::80-:8080
   160  		var forwarding []string
   161  		taskPorts := task.Resources.Networks[0].MapLabelToValues(nil)
   162  		for label, guest := range driverConfig.PortMap[0] {
   163  			host, ok := taskPorts[label]
   164  			if !ok {
   165  				return nil, fmt.Errorf("Unknown port label %q", label)
   166  			}
   167  
   168  			for _, p := range protocols {
   169  				forwarding = append(forwarding, fmt.Sprintf("hostfwd=%s::%d-:%d", p, host, guest))
   170  			}
   171  		}
   172  
   173  		if len(forwarding) != 0 {
   174  			args = append(args,
   175  				"-netdev",
   176  				fmt.Sprintf("user,id=user.0,%s", strings.Join(forwarding, ",")),
   177  				"-device", "virtio-net,netdev=user.0",
   178  			)
   179  		}
   180  	}
   181  
   182  	// If using KVM, add optimization args
   183  	if accelerator == "kvm" {
   184  		args = append(args,
   185  			"-enable-kvm",
   186  			"-cpu", "host",
   187  			// Do we have cores information available to the Driver?
   188  			// "-smp", fmt.Sprintf("%d", cores),
   189  		)
   190  	}
   191  
   192  	d.logger.Printf("[DEBUG] Starting QemuVM command: %q", strings.Join(args, " "))
   193  	bin, err := discover.NomadExecutable()
   194  	if err != nil {
   195  		return nil, fmt.Errorf("unable to find the nomad binary: %v", err)
   196  	}
   197  
   198  	pluginLogFile := filepath.Join(taskDir, fmt.Sprintf("%s-executor.out", task.Name))
   199  	pluginConfig := &plugin.ClientConfig{
   200  		Cmd: exec.Command(bin, "executor", pluginLogFile),
   201  	}
   202  
   203  	exec, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput, d.config)
   204  	if err != nil {
   205  		return nil, err
   206  	}
   207  	executorCtx := &executor.ExecutorContext{
   208  		TaskEnv:       d.taskEnv,
   209  		AllocDir:      ctx.AllocDir,
   210  		TaskName:      task.Name,
   211  		TaskResources: task.Resources,
   212  		LogConfig:     task.LogConfig,
   213  	}
   214  	ps, err := exec.LaunchCmd(&executor.ExecCommand{Cmd: args[0], Args: args[1:]}, executorCtx)
   215  	if err != nil {
   216  		pluginClient.Kill()
   217  		return nil, fmt.Errorf("error starting process via the plugin: %v", err)
   218  	}
   219  	d.logger.Printf("[INFO] Started new QemuVM: %s", vmID)
   220  
   221  	// Create and Return Handle
   222  	h := &qemuHandle{
   223  		pluginClient: pluginClient,
   224  		executor:     exec,
   225  		userPid:      ps.Pid,
   226  		allocDir:     ctx.AllocDir,
   227  		killTimeout:  d.DriverContext.KillTimeout(task),
   228  		version:      d.config.Version,
   229  		logger:       d.logger,
   230  		doneCh:       make(chan struct{}),
   231  		waitCh:       make(chan *cstructs.WaitResult, 1),
   232  	}
   233  
   234  	go h.run()
   235  	return h, nil
   236  }
   237  
   238  type qemuId struct {
   239  	Version      string
   240  	KillTimeout  time.Duration
   241  	UserPid      int
   242  	PluginConfig *PluginReattachConfig
   243  	AllocDir     *allocdir.AllocDir
   244  }
   245  
   246  func (d *QemuDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error) {
   247  	id := &qemuId{}
   248  	if err := json.Unmarshal([]byte(handleID), id); err != nil {
   249  		return nil, fmt.Errorf("Failed to parse handle '%s': %v", handleID, err)
   250  	}
   251  
   252  	pluginConfig := &plugin.ClientConfig{
   253  		Reattach: id.PluginConfig.PluginConfig(),
   254  	}
   255  
   256  	executor, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput, d.config)
   257  	if err != nil {
   258  		d.logger.Println("[ERR] driver.qemu: error connecting to plugin so destroying plugin pid and user pid")
   259  		if e := destroyPlugin(id.PluginConfig.Pid, id.UserPid); e != nil {
   260  			d.logger.Printf("[ERR] driver.qemu: error destroying plugin and userpid: %v", e)
   261  		}
   262  		return nil, fmt.Errorf("error connecting to plugin: %v", err)
   263  	}
   264  
   265  	// Return a driver handle
   266  	h := &qemuHandle{
   267  		pluginClient: pluginClient,
   268  		executor:     executor,
   269  		userPid:      id.UserPid,
   270  		allocDir:     id.AllocDir,
   271  		logger:       d.logger,
   272  		killTimeout:  id.KillTimeout,
   273  		version:      id.Version,
   274  		doneCh:       make(chan struct{}),
   275  		waitCh:       make(chan *cstructs.WaitResult, 1),
   276  	}
   277  	go h.run()
   278  	return h, nil
   279  }
   280  
   281  func (h *qemuHandle) ID() string {
   282  	id := qemuId{
   283  		Version:      h.version,
   284  		KillTimeout:  h.killTimeout,
   285  		PluginConfig: NewPluginReattachConfig(h.pluginClient.ReattachConfig()),
   286  		UserPid:      h.userPid,
   287  		AllocDir:     h.allocDir,
   288  	}
   289  
   290  	data, err := json.Marshal(id)
   291  	if err != nil {
   292  		h.logger.Printf("[ERR] driver.qemu: failed to marshal ID to JSON: %s", err)
   293  	}
   294  	return string(data)
   295  }
   296  
   297  func (h *qemuHandle) WaitCh() chan *cstructs.WaitResult {
   298  	return h.waitCh
   299  }
   300  
   301  func (h *qemuHandle) Update(task *structs.Task) error {
   302  	// Store the updated kill timeout.
   303  	h.killTimeout = task.KillTimeout
   304  	h.executor.UpdateLogConfig(task.LogConfig)
   305  
   306  	// Update is not possible
   307  	return nil
   308  }
   309  
   310  // TODO: allow a 'shutdown_command' that can be executed over a ssh connection
   311  // to the VM
   312  func (h *qemuHandle) Kill() error {
   313  	if err := h.executor.ShutDown(); err != nil {
   314  		if h.pluginClient.Exited() {
   315  			return nil
   316  		}
   317  		return fmt.Errorf("executor Shutdown failed: %v", err)
   318  	}
   319  
   320  	select {
   321  	case <-h.doneCh:
   322  		return nil
   323  	case <-time.After(h.killTimeout):
   324  		if h.pluginClient.Exited() {
   325  			return nil
   326  		}
   327  		if err := h.executor.Exit(); err != nil {
   328  			return fmt.Errorf("executor Exit failed: %v", err)
   329  		}
   330  
   331  		return nil
   332  	}
   333  }
   334  
   335  func (h *qemuHandle) run() {
   336  	ps, err := h.executor.Wait()
   337  	if ps.ExitCode == 0 && err != nil {
   338  		if e := killProcess(h.userPid); e != nil {
   339  			h.logger.Printf("[ERR] driver.qemu: error killing user process: %v", e)
   340  		}
   341  		if e := h.allocDir.UnmountAll(); e != nil {
   342  			h.logger.Printf("[ERR] driver.qemu: unmounting dev,proc and alloc dirs failed: %v", e)
   343  		}
   344  	}
   345  	close(h.doneCh)
   346  	h.waitCh <- &cstructs.WaitResult{ExitCode: ps.ExitCode, Signal: 0, Err: err}
   347  	close(h.waitCh)
   348  	h.pluginClient.Kill()
   349  }