github.com/anth0d/nomad@v0.0.0-20221214183521-ae3a0a2cad06/drivers/qemu/driver.go (about)

     1  package qemu
     2  
     3  import (
     4  	"context"
     5  	"errors"
     6  	"fmt"
     7  	"net"
     8  	"os"
     9  	"os/exec"
    10  	"path/filepath"
    11  	"regexp"
    12  	"runtime"
    13  	"strings"
    14  	"time"
    15  
    16  	hclog "github.com/hashicorp/go-hclog"
    17  	"github.com/hashicorp/nomad/client/taskenv"
    18  	"github.com/hashicorp/nomad/drivers/shared/eventer"
    19  	"github.com/hashicorp/nomad/drivers/shared/executor"
    20  	"github.com/hashicorp/nomad/helper/pluginutils/hclutils"
    21  	"github.com/hashicorp/nomad/helper/pluginutils/loader"
    22  	"github.com/hashicorp/nomad/plugins/base"
    23  	"github.com/hashicorp/nomad/plugins/drivers"
    24  	"github.com/hashicorp/nomad/plugins/shared/hclspec"
    25  	pstructs "github.com/hashicorp/nomad/plugins/shared/structs"
    26  )
    27  
    28  const (
    29  	// pluginName is the name of the plugin
    30  	pluginName = "qemu"
    31  
    32  	// fingerprintPeriod is the interval at which the driver will send fingerprint responses
    33  	fingerprintPeriod = 30 * time.Second
    34  
    35  	// The key populated in Node Attributes to indicate presence of the Qemu driver
    36  	driverAttr        = "driver.qemu"
    37  	driverVersionAttr = "driver.qemu.version"
    38  
    39  	// Represents an ACPI shutdown request to the VM (emulates pressing a physical power button)
    40  	// Reference: https://en.wikibooks.org/wiki/QEMU/Monitor
    41  	// Use a short file name since socket paths have a maximum length.
    42  	qemuGracefulShutdownMsg = "system_powerdown\n"
    43  	qemuMonitorSocketName   = "qm.sock"
    44  
    45  	// Socket file enabling communication with the Qemu Guest Agent (if enabled and running)
    46  	// Use a short file name since socket paths have a maximum length.
    47  	qemuGuestAgentSocketName = "qa.sock"
    48  
    49  	// taskHandleVersion is the version of task handle which this driver sets
    50  	// and understands how to decode driver state
    51  	taskHandleVersion = 1
    52  )
    53  
    54  var (
    55  	// PluginID is the qemu plugin metadata registered in the plugin
    56  	// catalog.
    57  	PluginID = loader.PluginID{
    58  		Name:       pluginName,
    59  		PluginType: base.PluginTypeDriver,
    60  	}
    61  
    62  	// PluginConfig is the qemu driver factory function registered in the
    63  	// plugin catalog.
    64  	PluginConfig = &loader.InternalPluginConfig{
    65  		Config:  map[string]interface{}{},
    66  		Factory: func(ctx context.Context, l hclog.Logger) interface{} { return NewQemuDriver(ctx, l) },
    67  	}
    68  
    69  	versionRegex = regexp.MustCompile(`version (\d[\.\d+]+)`)
    70  
    71  	// pluginInfo is the response returned for the PluginInfo RPC
    72  	pluginInfo = &base.PluginInfoResponse{
    73  		Type:              base.PluginTypeDriver,
    74  		PluginApiVersions: []string{drivers.ApiVersion010},
    75  		PluginVersion:     "0.1.0",
    76  		Name:              pluginName,
    77  	}
    78  
    79  	// configSpec is the hcl specification returned by the ConfigSchema RPC
    80  	configSpec = hclspec.NewObject(map[string]*hclspec.Spec{
    81  		"image_paths":    hclspec.NewAttr("image_paths", "list(string)", false),
    82  		"args_allowlist": hclspec.NewAttr("args_allowlist", "list(string)", false),
    83  	})
    84  
    85  	// taskConfigSpec is the hcl specification for the driver config section of
    86  	// a taskConfig within a job. It is returned in the TaskConfigSchema RPC
    87  	taskConfigSpec = hclspec.NewObject(map[string]*hclspec.Spec{
    88  		"image_path":        hclspec.NewAttr("image_path", "string", true),
    89  		"drive_interface":   hclspec.NewAttr("drive_interface", "string", false),
    90  		"accelerator":       hclspec.NewAttr("accelerator", "string", false),
    91  		"graceful_shutdown": hclspec.NewAttr("graceful_shutdown", "bool", false),
    92  		"guest_agent":       hclspec.NewAttr("guest_agent", "bool", false),
    93  		"args":              hclspec.NewAttr("args", "list(string)", false),
    94  		"port_map":          hclspec.NewAttr("port_map", "list(map(number))", false),
    95  	})
    96  
    97  	// capabilities is returned by the Capabilities RPC and indicates what
    98  	// optional features this driver supports
    99  	capabilities = &drivers.Capabilities{
   100  		SendSignals: false,
   101  		Exec:        false,
   102  		FSIsolation: drivers.FSIsolationImage,
   103  		NetIsolationModes: []drivers.NetIsolationMode{
   104  			drivers.NetIsolationModeHost,
   105  			drivers.NetIsolationModeGroup,
   106  		},
   107  		MountConfigs: drivers.MountConfigSupportNone,
   108  	}
   109  
   110  	_ drivers.DriverPlugin = (*Driver)(nil)
   111  )
   112  
   113  // TaskConfig is the driver configuration of a taskConfig within a job
   114  type TaskConfig struct {
   115  	ImagePath        string             `codec:"image_path"`
   116  	Accelerator      string             `codec:"accelerator"`
   117  	Args             []string           `codec:"args"`     // extra arguments to qemu executable
   118  	PortMap          hclutils.MapStrInt `codec:"port_map"` // A map of host port and the port name defined in the image manifest file
   119  	GracefulShutdown bool               `codec:"graceful_shutdown"`
   120  	DriveInterface   string             `codec:"drive_interface"` // Use interface for image
   121  	GuestAgent       bool               `codec:"guest_agent"`
   122  }
   123  
   124  // TaskState is the state which is encoded in the handle returned in StartTask.
   125  // This information is needed to rebuild the taskConfig state and handler
   126  // during recovery.
   127  type TaskState struct {
   128  	ReattachConfig *pstructs.ReattachConfig
   129  	TaskConfig     *drivers.TaskConfig
   130  	Pid            int
   131  	StartedAt      time.Time
   132  }
   133  
   134  // Config is the driver configuration set by SetConfig RPC call
   135  type Config struct {
   136  	// ImagePaths is an allow-list of paths qemu is allowed to load an image from
   137  	ImagePaths []string `codec:"image_paths"`
   138  
   139  	// ArgsAllowList is an allow-list of arguments the jobspec can
   140  	// include in arguments to qemu, so that cluster operators can can
   141  	// prevent access to devices
   142  	ArgsAllowList []string `codec:"args_allowlist"`
   143  }
   144  
   145  // Driver is a driver for running images via Qemu
   146  type Driver struct {
   147  	// eventer is used to handle multiplexing of TaskEvents calls such that an
   148  	// event can be broadcast to all callers
   149  	eventer *eventer.Eventer
   150  
   151  	// config is the driver configuration set by the SetConfig RPC
   152  	config Config
   153  
   154  	// tasks is the in memory datastore mapping taskIDs to qemuTaskHandle
   155  	tasks *taskStore
   156  
   157  	// ctx is the context for the driver. It is passed to other subsystems to
   158  	// coordinate shutdown
   159  	ctx context.Context
   160  
   161  	// nomadConf is the client agent's configuration
   162  	nomadConfig *base.ClientDriverConfig
   163  
   164  	// logger will log to the Nomad agent
   165  	logger hclog.Logger
   166  }
   167  
   168  func NewQemuDriver(ctx context.Context, logger hclog.Logger) drivers.DriverPlugin {
   169  	logger = logger.Named(pluginName)
   170  	return &Driver{
   171  		eventer: eventer.NewEventer(ctx, logger),
   172  		tasks:   newTaskStore(),
   173  		ctx:     ctx,
   174  		logger:  logger,
   175  	}
   176  }
   177  
   178  func (d *Driver) PluginInfo() (*base.PluginInfoResponse, error) {
   179  	return pluginInfo, nil
   180  }
   181  
   182  func (d *Driver) ConfigSchema() (*hclspec.Spec, error) {
   183  	return configSpec, nil
   184  }
   185  
   186  func (d *Driver) SetConfig(cfg *base.Config) error {
   187  	var config Config
   188  	if len(cfg.PluginConfig) != 0 {
   189  		if err := base.MsgPackDecode(cfg.PluginConfig, &config); err != nil {
   190  			return err
   191  		}
   192  	}
   193  
   194  	d.config = config
   195  	if cfg.AgentConfig != nil {
   196  		d.nomadConfig = cfg.AgentConfig.Driver
   197  	}
   198  	return nil
   199  }
   200  
   201  func (d *Driver) TaskConfigSchema() (*hclspec.Spec, error) {
   202  	return taskConfigSpec, nil
   203  }
   204  
   205  func (d *Driver) Capabilities() (*drivers.Capabilities, error) {
   206  	return capabilities, nil
   207  }
   208  
   209  func (d *Driver) Fingerprint(ctx context.Context) (<-chan *drivers.Fingerprint, error) {
   210  	ch := make(chan *drivers.Fingerprint)
   211  	go d.handleFingerprint(ctx, ch)
   212  	return ch, nil
   213  }
   214  
   215  func (d *Driver) handleFingerprint(ctx context.Context, ch chan *drivers.Fingerprint) {
   216  	ticker := time.NewTimer(0)
   217  	for {
   218  		select {
   219  		case <-ctx.Done():
   220  			return
   221  		case <-d.ctx.Done():
   222  			return
   223  		case <-ticker.C:
   224  			ticker.Reset(fingerprintPeriod)
   225  			ch <- d.buildFingerprint()
   226  		}
   227  	}
   228  }
   229  
   230  func (d *Driver) buildFingerprint() *drivers.Fingerprint {
   231  	fingerprint := &drivers.Fingerprint{
   232  		Attributes:        map[string]*pstructs.Attribute{},
   233  		Health:            drivers.HealthStateHealthy,
   234  		HealthDescription: drivers.DriverHealthy,
   235  	}
   236  
   237  	bin := "qemu-system-x86_64"
   238  	if runtime.GOOS == "windows" {
   239  		// On windows, the "qemu-system-x86_64" command does not respond to the
   240  		// version flag.
   241  		bin = "qemu-img"
   242  	}
   243  	outBytes, err := exec.Command(bin, "--version").Output()
   244  	if err != nil {
   245  		// return no error, as it isn't an error to not find qemu, it just means we
   246  		// can't use it.
   247  		fingerprint.Health = drivers.HealthStateUndetected
   248  		fingerprint.HealthDescription = ""
   249  		return fingerprint
   250  	}
   251  	out := strings.TrimSpace(string(outBytes))
   252  
   253  	matches := versionRegex.FindStringSubmatch(out)
   254  	if len(matches) != 2 {
   255  		fingerprint.Health = drivers.HealthStateUndetected
   256  		fingerprint.HealthDescription = fmt.Sprintf("Failed to parse qemu version from %v", out)
   257  		return fingerprint
   258  	}
   259  	currentQemuVersion := matches[1]
   260  	fingerprint.Attributes[driverAttr] = pstructs.NewBoolAttribute(true)
   261  	fingerprint.Attributes[driverVersionAttr] = pstructs.NewStringAttribute(currentQemuVersion)
   262  	return fingerprint
   263  }
   264  
   265  func (d *Driver) RecoverTask(handle *drivers.TaskHandle) error {
   266  	if handle == nil {
   267  		return fmt.Errorf("error: handle cannot be nil")
   268  	}
   269  
   270  	// If already attached to handle there's nothing to recover.
   271  	if _, ok := d.tasks.Get(handle.Config.ID); ok {
   272  		d.logger.Trace("nothing to recover; task already exists",
   273  			"task_id", handle.Config.ID,
   274  			"task_name", handle.Config.Name,
   275  		)
   276  		return nil
   277  	}
   278  
   279  	var taskState TaskState
   280  	if err := handle.GetDriverState(&taskState); err != nil {
   281  		d.logger.Error("failed to decode taskConfig state from handle", "error", err, "task_id", handle.Config.ID)
   282  		return fmt.Errorf("failed to decode taskConfig state from handle: %v", err)
   283  	}
   284  
   285  	plugRC, err := pstructs.ReattachConfigToGoPlugin(taskState.ReattachConfig)
   286  	if err != nil {
   287  		d.logger.Error("failed to build ReattachConfig from taskConfig state", "error", err, "task_id", handle.Config.ID)
   288  		return fmt.Errorf("failed to build ReattachConfig from taskConfig state: %v", err)
   289  	}
   290  
   291  	execImpl, pluginClient, err := executor.ReattachToExecutor(plugRC,
   292  		d.logger.With("task_name", handle.Config.Name, "alloc_id", handle.Config.AllocID))
   293  	if err != nil {
   294  		d.logger.Error("failed to reattach to executor", "error", err, "task_id", handle.Config.ID)
   295  		return fmt.Errorf("failed to reattach to executor: %v", err)
   296  	}
   297  
   298  	// Try to restore monitor socket path.
   299  	taskDir := filepath.Join(handle.Config.AllocDir, handle.Config.Name)
   300  	possiblePaths := []string{
   301  		filepath.Join(taskDir, qemuMonitorSocketName),
   302  		// Support restoring tasks that used the old socket name.
   303  		filepath.Join(taskDir, "qemu-monitor.sock"),
   304  	}
   305  
   306  	var monitorPath string
   307  	for _, path := range possiblePaths {
   308  		if _, err := os.Stat(path); err == nil {
   309  			monitorPath = path
   310  			d.logger.Debug("found existing monitor socket", "monitor", monitorPath)
   311  			break
   312  		}
   313  	}
   314  
   315  	h := &taskHandle{
   316  		exec:         execImpl,
   317  		pid:          taskState.Pid,
   318  		monitorPath:  monitorPath,
   319  		pluginClient: pluginClient,
   320  		taskConfig:   taskState.TaskConfig,
   321  		procState:    drivers.TaskStateRunning,
   322  		startedAt:    taskState.StartedAt,
   323  		exitResult:   &drivers.ExitResult{},
   324  		logger:       d.logger,
   325  	}
   326  
   327  	d.tasks.Set(taskState.TaskConfig.ID, h)
   328  
   329  	go h.run()
   330  	return nil
   331  }
   332  
   333  func isAllowedImagePath(allowedPaths []string, allocDir, imagePath string) bool {
   334  	if !filepath.IsAbs(imagePath) {
   335  		imagePath = filepath.Join(allocDir, imagePath)
   336  	}
   337  
   338  	isParent := func(parent, path string) bool {
   339  		rel, err := filepath.Rel(parent, path)
   340  		return err == nil && !strings.HasPrefix(rel, "..")
   341  	}
   342  
   343  	// check if path is under alloc dir
   344  	if isParent(allocDir, imagePath) {
   345  		return true
   346  	}
   347  
   348  	// check allowed paths
   349  	for _, ap := range allowedPaths {
   350  		if isParent(ap, imagePath) {
   351  			return true
   352  		}
   353  	}
   354  
   355  	return false
   356  }
   357  
   358  // hardcoded list of drive interfaces, Qemu currently supports
   359  var allowedDriveInterfaces = []string{"ide", "scsi", "sd", "mtd", "floppy", "pflash", "virtio", "none"}
   360  
   361  func isAllowedDriveInterface(driveInterface string) bool {
   362  	for _, ai := range allowedDriveInterfaces {
   363  		if driveInterface == ai {
   364  			return true
   365  		}
   366  	}
   367  
   368  	return false
   369  }
   370  
   371  // validateArgs ensures that all QEMU command line params are in the
   372  // allowlist. This function must be called after all interpolation has
   373  // taken place.
   374  func validateArgs(pluginConfigAllowList, args []string) error {
   375  	if len(pluginConfigAllowList) > 0 {
   376  		allowed := map[string]struct{}{}
   377  		for _, arg := range pluginConfigAllowList {
   378  			allowed[arg] = struct{}{}
   379  		}
   380  		for _, arg := range args {
   381  			if strings.HasPrefix(strings.TrimSpace(arg), "-") {
   382  				if _, ok := allowed[arg]; !ok {
   383  					return fmt.Errorf("%q is not in args_allowlist", arg)
   384  				}
   385  			}
   386  		}
   387  	}
   388  	return nil
   389  }
   390  
   391  func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drivers.DriverNetwork, error) {
   392  	if _, ok := d.tasks.Get(cfg.ID); ok {
   393  		return nil, nil, fmt.Errorf("taskConfig with ID '%s' already started", cfg.ID)
   394  	}
   395  
   396  	var driverConfig TaskConfig
   397  
   398  	if err := cfg.DecodeDriverConfig(&driverConfig); err != nil {
   399  		return nil, nil, fmt.Errorf("failed to decode driver config: %v", err)
   400  	}
   401  
   402  	// ensure that PortMap variables are populated early on
   403  	cfg.Env = taskenv.SetPortMapEnvs(cfg.Env, driverConfig.PortMap)
   404  
   405  	handle := drivers.NewTaskHandle(taskHandleVersion)
   406  	handle.Config = cfg
   407  
   408  	if err := validateArgs(d.config.ArgsAllowList, driverConfig.Args); err != nil {
   409  		return nil, nil, err
   410  	}
   411  
   412  	// Get the image source
   413  	vmPath := driverConfig.ImagePath
   414  	if vmPath == "" {
   415  		return nil, nil, fmt.Errorf("image_path must be set")
   416  	}
   417  	vmID := filepath.Base(vmPath)
   418  
   419  	if !isAllowedImagePath(d.config.ImagePaths, cfg.AllocDir, vmPath) {
   420  		return nil, nil, fmt.Errorf("image_path is not in the allowed paths")
   421  	}
   422  
   423  	// Parse configuration arguments
   424  	// Create the base arguments
   425  	accelerator := "tcg"
   426  	if driverConfig.Accelerator != "" {
   427  		accelerator = driverConfig.Accelerator
   428  	}
   429  
   430  	mb := cfg.Resources.NomadResources.Memory.MemoryMB
   431  	if mb < 128 || mb > 4000000 {
   432  		return nil, nil, fmt.Errorf("Qemu memory assignment out of bounds")
   433  	}
   434  	mem := fmt.Sprintf("%dM", mb)
   435  
   436  	absPath, err := GetAbsolutePath("qemu-system-x86_64")
   437  	if err != nil {
   438  		return nil, nil, err
   439  	}
   440  
   441  	driveInterface := "ide"
   442  	if driverConfig.DriveInterface != "" {
   443  		driveInterface = driverConfig.DriveInterface
   444  	}
   445  	if !isAllowedDriveInterface(driveInterface) {
   446  		return nil, nil, fmt.Errorf("Unsupported drive_interface")
   447  	}
   448  
   449  	args := []string{
   450  		absPath,
   451  		"-machine", "type=pc,accel=" + accelerator,
   452  		"-name", vmID,
   453  		"-m", mem,
   454  		"-drive", "file=" + vmPath + ",if=" + driveInterface,
   455  		"-nographic",
   456  	}
   457  
   458  	var netdevArgs []string
   459  	if cfg.DNS != nil {
   460  		if len(cfg.DNS.Servers) > 0 {
   461  			netdevArgs = append(netdevArgs, "dns="+cfg.DNS.Servers[0])
   462  		}
   463  
   464  		for _, s := range cfg.DNS.Searches {
   465  			netdevArgs = append(netdevArgs, "dnssearch="+s)
   466  		}
   467  	}
   468  
   469  	taskDir := filepath.Join(cfg.AllocDir, cfg.Name)
   470  
   471  	var monitorPath string
   472  	if driverConfig.GracefulShutdown {
   473  		if runtime.GOOS == "windows" {
   474  			return nil, nil, errors.New("QEMU graceful shutdown is unsupported on the Windows platform")
   475  		}
   476  		// This socket will be used to manage the virtual machine (for example,
   477  		// to perform graceful shutdowns)
   478  		monitorPath = filepath.Join(taskDir, qemuMonitorSocketName)
   479  		if err := validateSocketPath(monitorPath); err != nil {
   480  			return nil, nil, err
   481  		}
   482  		d.logger.Debug("got monitor path", "monitorPath", monitorPath)
   483  		args = append(args, "-monitor", fmt.Sprintf("unix:%s,server,nowait", monitorPath))
   484  	}
   485  
   486  	if driverConfig.GuestAgent {
   487  		if runtime.GOOS == "windows" {
   488  			return nil, nil, errors.New("QEMU Guest Agent socket is unsupported on the Windows platform")
   489  		}
   490  		// This socket will be used to communicate with the Guest Agent (if it's running)
   491  		agentSocketPath := filepath.Join(taskDir, qemuGuestAgentSocketName)
   492  		if err := validateSocketPath(agentSocketPath); err != nil {
   493  			return nil, nil, err
   494  		}
   495  
   496  		args = append(args, "-chardev", fmt.Sprintf("socket,path=%s,server,nowait,id=qga0", agentSocketPath))
   497  		args = append(args, "-device", "virtio-serial")
   498  		args = append(args, "-device", "virtserialport,chardev=qga0,name=org.qemu.guest_agent.0")
   499  	}
   500  
   501  	// Add pass through arguments to qemu executable. A user can specify
   502  	// these arguments in driver task configuration. These arguments are
   503  	// passed directly to the qemu driver as command line options.
   504  	// For example, args = [ "-nodefconfig", "-nodefaults" ]
   505  	// This will allow a VM with embedded configuration to boot successfully.
   506  	args = append(args, driverConfig.Args...)
   507  
   508  	// Check the Resources required Networks to add port mappings. If no resources
   509  	// are required, we assume the VM is a purely compute job and does not require
   510  	// the outside world to be able to reach it. VMs ran without port mappings can
   511  	// still reach out to the world, but without port mappings it is effectively
   512  	// firewalled
   513  	protocols := []string{"udp", "tcp"}
   514  	if len(cfg.Resources.NomadResources.Networks) > 0 {
   515  		// Loop through the port map and construct the hostfwd string, to map
   516  		// reserved ports to the ports listenting in the VM
   517  		// Ex: hostfwd=tcp::22000-:22,hostfwd=tcp::80-:8080
   518  		taskPorts := cfg.Resources.NomadResources.Networks[0].PortLabels()
   519  		for label, guest := range driverConfig.PortMap {
   520  			host, ok := taskPorts[label]
   521  			if !ok {
   522  				return nil, nil, fmt.Errorf("Unknown port label %q", label)
   523  			}
   524  
   525  			for _, p := range protocols {
   526  				netdevArgs = append(netdevArgs, fmt.Sprintf("hostfwd=%s::%d-:%d", p, host, guest))
   527  			}
   528  		}
   529  
   530  		if len(netdevArgs) != 0 {
   531  			args = append(args,
   532  				"-netdev",
   533  				fmt.Sprintf("user,id=user.0,%s", strings.Join(netdevArgs, ",")),
   534  				"-device", "virtio-net,netdev=user.0",
   535  			)
   536  		}
   537  	}
   538  
   539  	// If using KVM, add optimization args
   540  	if accelerator == "kvm" {
   541  		if runtime.GOOS == "windows" {
   542  			return nil, nil, errors.New("KVM accelerator is unsupported on the Windows platform")
   543  		}
   544  		args = append(args,
   545  			"-enable-kvm",
   546  			"-cpu", "host",
   547  		)
   548  
   549  		if cfg.Resources.LinuxResources != nil && cfg.Resources.LinuxResources.CpusetCpus != "" {
   550  			cores := strings.Split(cfg.Resources.LinuxResources.CpusetCpus, ",")
   551  			args = append(args,
   552  				"-smp", fmt.Sprintf("%d", len(cores)),
   553  			)
   554  		}
   555  	}
   556  	d.logger.Debug("starting QemuVM command ", "args", strings.Join(args, " "))
   557  
   558  	pluginLogFile := filepath.Join(cfg.TaskDir().Dir, fmt.Sprintf("%s-executor.out", cfg.Name))
   559  	executorConfig := &executor.ExecutorConfig{
   560  		LogFile:  pluginLogFile,
   561  		LogLevel: "debug",
   562  	}
   563  
   564  	execImpl, pluginClient, err := executor.CreateExecutor(
   565  		d.logger.With("task_name", handle.Config.Name, "alloc_id", handle.Config.AllocID),
   566  		d.nomadConfig, executorConfig)
   567  	if err != nil {
   568  		return nil, nil, err
   569  	}
   570  
   571  	execCmd := &executor.ExecCommand{
   572  		Cmd:              args[0],
   573  		Args:             args[1:],
   574  		Env:              cfg.EnvList(),
   575  		User:             cfg.User,
   576  		TaskDir:          cfg.TaskDir().Dir,
   577  		StdoutPath:       cfg.StdoutPath,
   578  		StderrPath:       cfg.StderrPath,
   579  		NetworkIsolation: cfg.NetworkIsolation,
   580  	}
   581  	ps, err := execImpl.Launch(execCmd)
   582  	if err != nil {
   583  		pluginClient.Kill()
   584  		return nil, nil, err
   585  	}
   586  	d.logger.Debug("started new QemuVM", "ID", vmID)
   587  
   588  	h := &taskHandle{
   589  		exec:         execImpl,
   590  		pid:          ps.Pid,
   591  		monitorPath:  monitorPath,
   592  		pluginClient: pluginClient,
   593  		taskConfig:   cfg,
   594  		procState:    drivers.TaskStateRunning,
   595  		startedAt:    time.Now().Round(time.Millisecond),
   596  		logger:       d.logger,
   597  	}
   598  
   599  	qemuDriverState := TaskState{
   600  		ReattachConfig: pstructs.ReattachConfigFromGoPlugin(pluginClient.ReattachConfig()),
   601  		Pid:            ps.Pid,
   602  		TaskConfig:     cfg,
   603  		StartedAt:      h.startedAt,
   604  	}
   605  
   606  	if err := handle.SetDriverState(&qemuDriverState); err != nil {
   607  		d.logger.Error("failed to start task, error setting driver state", "error", err)
   608  		execImpl.Shutdown("", 0)
   609  		pluginClient.Kill()
   610  		return nil, nil, fmt.Errorf("failed to set driver state: %v", err)
   611  	}
   612  
   613  	d.tasks.Set(cfg.ID, h)
   614  	go h.run()
   615  
   616  	var driverNetwork *drivers.DriverNetwork
   617  	if len(driverConfig.PortMap) == 1 {
   618  		driverNetwork = &drivers.DriverNetwork{
   619  			PortMap: driverConfig.PortMap,
   620  		}
   621  	}
   622  	return handle, driverNetwork, nil
   623  }
   624  
   625  func (d *Driver) WaitTask(ctx context.Context, taskID string) (<-chan *drivers.ExitResult, error) {
   626  	handle, ok := d.tasks.Get(taskID)
   627  	if !ok {
   628  		return nil, drivers.ErrTaskNotFound
   629  	}
   630  
   631  	ch := make(chan *drivers.ExitResult)
   632  	go d.handleWait(ctx, handle, ch)
   633  
   634  	return ch, nil
   635  }
   636  
   637  func (d *Driver) StopTask(taskID string, timeout time.Duration, signal string) error {
   638  	handle, ok := d.tasks.Get(taskID)
   639  	if !ok {
   640  		return drivers.ErrTaskNotFound
   641  	}
   642  
   643  	// Attempt a graceful shutdown only if it was configured in the job
   644  	if handle.monitorPath != "" {
   645  		if err := sendQemuShutdown(d.logger, handle.monitorPath, handle.pid); err != nil {
   646  			d.logger.Debug("error sending graceful shutdown ", "pid", handle.pid, "error", err)
   647  		}
   648  	} else {
   649  		d.logger.Debug("monitor socket is empty, forcing shutdown")
   650  	}
   651  
   652  	// TODO(preetha) we are calling shutdown on the executor here
   653  	// after attempting a graceful qemu shutdown, qemu process may
   654  	// not be around when we call exec.shutdown
   655  	if err := handle.exec.Shutdown(signal, timeout); err != nil {
   656  		if handle.pluginClient.Exited() {
   657  			return nil
   658  		}
   659  		return fmt.Errorf("executor Shutdown failed: %v", err)
   660  	}
   661  
   662  	return nil
   663  }
   664  
   665  func (d *Driver) DestroyTask(taskID string, force bool) error {
   666  	handle, ok := d.tasks.Get(taskID)
   667  	if !ok {
   668  		return drivers.ErrTaskNotFound
   669  	}
   670  
   671  	if handle.IsRunning() && !force {
   672  		return fmt.Errorf("cannot destroy running task")
   673  	}
   674  
   675  	if !handle.pluginClient.Exited() {
   676  		if err := handle.exec.Shutdown("", 0); err != nil {
   677  			handle.logger.Error("destroying executor failed", "error", err)
   678  		}
   679  
   680  		handle.pluginClient.Kill()
   681  	}
   682  
   683  	d.tasks.Delete(taskID)
   684  	return nil
   685  }
   686  
   687  func (d *Driver) InspectTask(taskID string) (*drivers.TaskStatus, error) {
   688  	handle, ok := d.tasks.Get(taskID)
   689  	if !ok {
   690  		return nil, drivers.ErrTaskNotFound
   691  	}
   692  
   693  	return handle.TaskStatus(), nil
   694  }
   695  
   696  func (d *Driver) TaskStats(ctx context.Context, taskID string, interval time.Duration) (<-chan *drivers.TaskResourceUsage, error) {
   697  	handle, ok := d.tasks.Get(taskID)
   698  	if !ok {
   699  		return nil, drivers.ErrTaskNotFound
   700  	}
   701  
   702  	return handle.exec.Stats(ctx, interval)
   703  }
   704  
   705  func (d *Driver) TaskEvents(ctx context.Context) (<-chan *drivers.TaskEvent, error) {
   706  	return d.eventer.TaskEvents(ctx)
   707  }
   708  
   709  func (d *Driver) SignalTask(taskID string, signal string) error {
   710  	return fmt.Errorf("Qemu driver can't signal commands")
   711  }
   712  
   713  func (d *Driver) ExecTask(taskID string, cmdArgs []string, timeout time.Duration) (*drivers.ExecTaskResult, error) {
   714  	return nil, fmt.Errorf("Qemu driver can't execute commands")
   715  
   716  }
   717  
   718  // GetAbsolutePath returns the absolute path of the passed binary by resolving
   719  // it in the path and following symlinks.
   720  func GetAbsolutePath(bin string) (string, error) {
   721  	lp, err := exec.LookPath(bin)
   722  	if err != nil {
   723  		return "", fmt.Errorf("failed to resolve path to %q executable: %v", bin, err)
   724  	}
   725  
   726  	return filepath.EvalSymlinks(lp)
   727  }
   728  
   729  func (d *Driver) handleWait(ctx context.Context, handle *taskHandle, ch chan *drivers.ExitResult) {
   730  	defer close(ch)
   731  	var result *drivers.ExitResult
   732  	ps, err := handle.exec.Wait(ctx)
   733  	if err != nil {
   734  		result = &drivers.ExitResult{
   735  			Err: fmt.Errorf("executor: error waiting on process: %v", err),
   736  		}
   737  	} else {
   738  		result = &drivers.ExitResult{
   739  			ExitCode: ps.ExitCode,
   740  			Signal:   ps.Signal,
   741  		}
   742  	}
   743  
   744  	select {
   745  	case <-ctx.Done():
   746  	case <-d.ctx.Done():
   747  	case ch <- result:
   748  	}
   749  }
   750  
   751  // validateSocketPath provides best effort validation of socket paths since
   752  // some rules may be platform-dependant.
   753  func validateSocketPath(path string) error {
   754  	if maxSocketPathLen > 0 && len(path) > maxSocketPathLen {
   755  		return fmt.Errorf(
   756  			"socket path %s is longer than the maximum length allowed (%d), try to reduce the task name or Nomad's data_dir if possible.",
   757  			path, maxSocketPathLen)
   758  	}
   759  
   760  	return nil
   761  }
   762  
   763  // sendQemuShutdown attempts to issue an ACPI power-off command via the qemu
   764  // monitor
   765  func sendQemuShutdown(logger hclog.Logger, monitorPath string, userPid int) error {
   766  	if monitorPath == "" {
   767  		return errors.New("monitorPath not set")
   768  	}
   769  	monitorSocket, err := net.Dial("unix", monitorPath)
   770  	if err != nil {
   771  		logger.Warn("could not connect to qemu monitor", "pid", userPid, "monitorPath", monitorPath, "error", err)
   772  		return err
   773  	}
   774  	defer monitorSocket.Close()
   775  	logger.Debug("sending graceful shutdown command to qemu monitor socket", "monitor_path", monitorPath, "pid", userPid)
   776  	_, err = monitorSocket.Write([]byte(qemuGracefulShutdownMsg))
   777  	if err != nil {
   778  		logger.Warn("failed to send shutdown message", "shutdown message", qemuGracefulShutdownMsg, "monitorPath", monitorPath, "userPid", userPid, "error", err)
   779  	}
   780  	return err
   781  }