github.com/anth0d/nomad@v0.0.0-20221214183521-ae3a0a2cad06/drivers/qemu/driver.go (about) 1 package qemu 2 3 import ( 4 "context" 5 "errors" 6 "fmt" 7 "net" 8 "os" 9 "os/exec" 10 "path/filepath" 11 "regexp" 12 "runtime" 13 "strings" 14 "time" 15 16 hclog "github.com/hashicorp/go-hclog" 17 "github.com/hashicorp/nomad/client/taskenv" 18 "github.com/hashicorp/nomad/drivers/shared/eventer" 19 "github.com/hashicorp/nomad/drivers/shared/executor" 20 "github.com/hashicorp/nomad/helper/pluginutils/hclutils" 21 "github.com/hashicorp/nomad/helper/pluginutils/loader" 22 "github.com/hashicorp/nomad/plugins/base" 23 "github.com/hashicorp/nomad/plugins/drivers" 24 "github.com/hashicorp/nomad/plugins/shared/hclspec" 25 pstructs "github.com/hashicorp/nomad/plugins/shared/structs" 26 ) 27 28 const ( 29 // pluginName is the name of the plugin 30 pluginName = "qemu" 31 32 // fingerprintPeriod is the interval at which the driver will send fingerprint responses 33 fingerprintPeriod = 30 * time.Second 34 35 // The key populated in Node Attributes to indicate presence of the Qemu driver 36 driverAttr = "driver.qemu" 37 driverVersionAttr = "driver.qemu.version" 38 39 // Represents an ACPI shutdown request to the VM (emulates pressing a physical power button) 40 // Reference: https://en.wikibooks.org/wiki/QEMU/Monitor 41 // Use a short file name since socket paths have a maximum length. 42 qemuGracefulShutdownMsg = "system_powerdown\n" 43 qemuMonitorSocketName = "qm.sock" 44 45 // Socket file enabling communication with the Qemu Guest Agent (if enabled and running) 46 // Use a short file name since socket paths have a maximum length. 47 qemuGuestAgentSocketName = "qa.sock" 48 49 // taskHandleVersion is the version of task handle which this driver sets 50 // and understands how to decode driver state 51 taskHandleVersion = 1 52 ) 53 54 var ( 55 // PluginID is the qemu plugin metadata registered in the plugin 56 // catalog. 57 PluginID = loader.PluginID{ 58 Name: pluginName, 59 PluginType: base.PluginTypeDriver, 60 } 61 62 // PluginConfig is the qemu driver factory function registered in the 63 // plugin catalog. 64 PluginConfig = &loader.InternalPluginConfig{ 65 Config: map[string]interface{}{}, 66 Factory: func(ctx context.Context, l hclog.Logger) interface{} { return NewQemuDriver(ctx, l) }, 67 } 68 69 versionRegex = regexp.MustCompile(`version (\d[\.\d+]+)`) 70 71 // pluginInfo is the response returned for the PluginInfo RPC 72 pluginInfo = &base.PluginInfoResponse{ 73 Type: base.PluginTypeDriver, 74 PluginApiVersions: []string{drivers.ApiVersion010}, 75 PluginVersion: "0.1.0", 76 Name: pluginName, 77 } 78 79 // configSpec is the hcl specification returned by the ConfigSchema RPC 80 configSpec = hclspec.NewObject(map[string]*hclspec.Spec{ 81 "image_paths": hclspec.NewAttr("image_paths", "list(string)", false), 82 "args_allowlist": hclspec.NewAttr("args_allowlist", "list(string)", false), 83 }) 84 85 // taskConfigSpec is the hcl specification for the driver config section of 86 // a taskConfig within a job. It is returned in the TaskConfigSchema RPC 87 taskConfigSpec = hclspec.NewObject(map[string]*hclspec.Spec{ 88 "image_path": hclspec.NewAttr("image_path", "string", true), 89 "drive_interface": hclspec.NewAttr("drive_interface", "string", false), 90 "accelerator": hclspec.NewAttr("accelerator", "string", false), 91 "graceful_shutdown": hclspec.NewAttr("graceful_shutdown", "bool", false), 92 "guest_agent": hclspec.NewAttr("guest_agent", "bool", false), 93 "args": hclspec.NewAttr("args", "list(string)", false), 94 "port_map": hclspec.NewAttr("port_map", "list(map(number))", false), 95 }) 96 97 // capabilities is returned by the Capabilities RPC and indicates what 98 // optional features this driver supports 99 capabilities = &drivers.Capabilities{ 100 SendSignals: false, 101 Exec: false, 102 FSIsolation: drivers.FSIsolationImage, 103 NetIsolationModes: []drivers.NetIsolationMode{ 104 drivers.NetIsolationModeHost, 105 drivers.NetIsolationModeGroup, 106 }, 107 MountConfigs: drivers.MountConfigSupportNone, 108 } 109 110 _ drivers.DriverPlugin = (*Driver)(nil) 111 ) 112 113 // TaskConfig is the driver configuration of a taskConfig within a job 114 type TaskConfig struct { 115 ImagePath string `codec:"image_path"` 116 Accelerator string `codec:"accelerator"` 117 Args []string `codec:"args"` // extra arguments to qemu executable 118 PortMap hclutils.MapStrInt `codec:"port_map"` // A map of host port and the port name defined in the image manifest file 119 GracefulShutdown bool `codec:"graceful_shutdown"` 120 DriveInterface string `codec:"drive_interface"` // Use interface for image 121 GuestAgent bool `codec:"guest_agent"` 122 } 123 124 // TaskState is the state which is encoded in the handle returned in StartTask. 125 // This information is needed to rebuild the taskConfig state and handler 126 // during recovery. 127 type TaskState struct { 128 ReattachConfig *pstructs.ReattachConfig 129 TaskConfig *drivers.TaskConfig 130 Pid int 131 StartedAt time.Time 132 } 133 134 // Config is the driver configuration set by SetConfig RPC call 135 type Config struct { 136 // ImagePaths is an allow-list of paths qemu is allowed to load an image from 137 ImagePaths []string `codec:"image_paths"` 138 139 // ArgsAllowList is an allow-list of arguments the jobspec can 140 // include in arguments to qemu, so that cluster operators can can 141 // prevent access to devices 142 ArgsAllowList []string `codec:"args_allowlist"` 143 } 144 145 // Driver is a driver for running images via Qemu 146 type Driver struct { 147 // eventer is used to handle multiplexing of TaskEvents calls such that an 148 // event can be broadcast to all callers 149 eventer *eventer.Eventer 150 151 // config is the driver configuration set by the SetConfig RPC 152 config Config 153 154 // tasks is the in memory datastore mapping taskIDs to qemuTaskHandle 155 tasks *taskStore 156 157 // ctx is the context for the driver. It is passed to other subsystems to 158 // coordinate shutdown 159 ctx context.Context 160 161 // nomadConf is the client agent's configuration 162 nomadConfig *base.ClientDriverConfig 163 164 // logger will log to the Nomad agent 165 logger hclog.Logger 166 } 167 168 func NewQemuDriver(ctx context.Context, logger hclog.Logger) drivers.DriverPlugin { 169 logger = logger.Named(pluginName) 170 return &Driver{ 171 eventer: eventer.NewEventer(ctx, logger), 172 tasks: newTaskStore(), 173 ctx: ctx, 174 logger: logger, 175 } 176 } 177 178 func (d *Driver) PluginInfo() (*base.PluginInfoResponse, error) { 179 return pluginInfo, nil 180 } 181 182 func (d *Driver) ConfigSchema() (*hclspec.Spec, error) { 183 return configSpec, nil 184 } 185 186 func (d *Driver) SetConfig(cfg *base.Config) error { 187 var config Config 188 if len(cfg.PluginConfig) != 0 { 189 if err := base.MsgPackDecode(cfg.PluginConfig, &config); err != nil { 190 return err 191 } 192 } 193 194 d.config = config 195 if cfg.AgentConfig != nil { 196 d.nomadConfig = cfg.AgentConfig.Driver 197 } 198 return nil 199 } 200 201 func (d *Driver) TaskConfigSchema() (*hclspec.Spec, error) { 202 return taskConfigSpec, nil 203 } 204 205 func (d *Driver) Capabilities() (*drivers.Capabilities, error) { 206 return capabilities, nil 207 } 208 209 func (d *Driver) Fingerprint(ctx context.Context) (<-chan *drivers.Fingerprint, error) { 210 ch := make(chan *drivers.Fingerprint) 211 go d.handleFingerprint(ctx, ch) 212 return ch, nil 213 } 214 215 func (d *Driver) handleFingerprint(ctx context.Context, ch chan *drivers.Fingerprint) { 216 ticker := time.NewTimer(0) 217 for { 218 select { 219 case <-ctx.Done(): 220 return 221 case <-d.ctx.Done(): 222 return 223 case <-ticker.C: 224 ticker.Reset(fingerprintPeriod) 225 ch <- d.buildFingerprint() 226 } 227 } 228 } 229 230 func (d *Driver) buildFingerprint() *drivers.Fingerprint { 231 fingerprint := &drivers.Fingerprint{ 232 Attributes: map[string]*pstructs.Attribute{}, 233 Health: drivers.HealthStateHealthy, 234 HealthDescription: drivers.DriverHealthy, 235 } 236 237 bin := "qemu-system-x86_64" 238 if runtime.GOOS == "windows" { 239 // On windows, the "qemu-system-x86_64" command does not respond to the 240 // version flag. 241 bin = "qemu-img" 242 } 243 outBytes, err := exec.Command(bin, "--version").Output() 244 if err != nil { 245 // return no error, as it isn't an error to not find qemu, it just means we 246 // can't use it. 247 fingerprint.Health = drivers.HealthStateUndetected 248 fingerprint.HealthDescription = "" 249 return fingerprint 250 } 251 out := strings.TrimSpace(string(outBytes)) 252 253 matches := versionRegex.FindStringSubmatch(out) 254 if len(matches) != 2 { 255 fingerprint.Health = drivers.HealthStateUndetected 256 fingerprint.HealthDescription = fmt.Sprintf("Failed to parse qemu version from %v", out) 257 return fingerprint 258 } 259 currentQemuVersion := matches[1] 260 fingerprint.Attributes[driverAttr] = pstructs.NewBoolAttribute(true) 261 fingerprint.Attributes[driverVersionAttr] = pstructs.NewStringAttribute(currentQemuVersion) 262 return fingerprint 263 } 264 265 func (d *Driver) RecoverTask(handle *drivers.TaskHandle) error { 266 if handle == nil { 267 return fmt.Errorf("error: handle cannot be nil") 268 } 269 270 // If already attached to handle there's nothing to recover. 271 if _, ok := d.tasks.Get(handle.Config.ID); ok { 272 d.logger.Trace("nothing to recover; task already exists", 273 "task_id", handle.Config.ID, 274 "task_name", handle.Config.Name, 275 ) 276 return nil 277 } 278 279 var taskState TaskState 280 if err := handle.GetDriverState(&taskState); err != nil { 281 d.logger.Error("failed to decode taskConfig state from handle", "error", err, "task_id", handle.Config.ID) 282 return fmt.Errorf("failed to decode taskConfig state from handle: %v", err) 283 } 284 285 plugRC, err := pstructs.ReattachConfigToGoPlugin(taskState.ReattachConfig) 286 if err != nil { 287 d.logger.Error("failed to build ReattachConfig from taskConfig state", "error", err, "task_id", handle.Config.ID) 288 return fmt.Errorf("failed to build ReattachConfig from taskConfig state: %v", err) 289 } 290 291 execImpl, pluginClient, err := executor.ReattachToExecutor(plugRC, 292 d.logger.With("task_name", handle.Config.Name, "alloc_id", handle.Config.AllocID)) 293 if err != nil { 294 d.logger.Error("failed to reattach to executor", "error", err, "task_id", handle.Config.ID) 295 return fmt.Errorf("failed to reattach to executor: %v", err) 296 } 297 298 // Try to restore monitor socket path. 299 taskDir := filepath.Join(handle.Config.AllocDir, handle.Config.Name) 300 possiblePaths := []string{ 301 filepath.Join(taskDir, qemuMonitorSocketName), 302 // Support restoring tasks that used the old socket name. 303 filepath.Join(taskDir, "qemu-monitor.sock"), 304 } 305 306 var monitorPath string 307 for _, path := range possiblePaths { 308 if _, err := os.Stat(path); err == nil { 309 monitorPath = path 310 d.logger.Debug("found existing monitor socket", "monitor", monitorPath) 311 break 312 } 313 } 314 315 h := &taskHandle{ 316 exec: execImpl, 317 pid: taskState.Pid, 318 monitorPath: monitorPath, 319 pluginClient: pluginClient, 320 taskConfig: taskState.TaskConfig, 321 procState: drivers.TaskStateRunning, 322 startedAt: taskState.StartedAt, 323 exitResult: &drivers.ExitResult{}, 324 logger: d.logger, 325 } 326 327 d.tasks.Set(taskState.TaskConfig.ID, h) 328 329 go h.run() 330 return nil 331 } 332 333 func isAllowedImagePath(allowedPaths []string, allocDir, imagePath string) bool { 334 if !filepath.IsAbs(imagePath) { 335 imagePath = filepath.Join(allocDir, imagePath) 336 } 337 338 isParent := func(parent, path string) bool { 339 rel, err := filepath.Rel(parent, path) 340 return err == nil && !strings.HasPrefix(rel, "..") 341 } 342 343 // check if path is under alloc dir 344 if isParent(allocDir, imagePath) { 345 return true 346 } 347 348 // check allowed paths 349 for _, ap := range allowedPaths { 350 if isParent(ap, imagePath) { 351 return true 352 } 353 } 354 355 return false 356 } 357 358 // hardcoded list of drive interfaces, Qemu currently supports 359 var allowedDriveInterfaces = []string{"ide", "scsi", "sd", "mtd", "floppy", "pflash", "virtio", "none"} 360 361 func isAllowedDriveInterface(driveInterface string) bool { 362 for _, ai := range allowedDriveInterfaces { 363 if driveInterface == ai { 364 return true 365 } 366 } 367 368 return false 369 } 370 371 // validateArgs ensures that all QEMU command line params are in the 372 // allowlist. This function must be called after all interpolation has 373 // taken place. 374 func validateArgs(pluginConfigAllowList, args []string) error { 375 if len(pluginConfigAllowList) > 0 { 376 allowed := map[string]struct{}{} 377 for _, arg := range pluginConfigAllowList { 378 allowed[arg] = struct{}{} 379 } 380 for _, arg := range args { 381 if strings.HasPrefix(strings.TrimSpace(arg), "-") { 382 if _, ok := allowed[arg]; !ok { 383 return fmt.Errorf("%q is not in args_allowlist", arg) 384 } 385 } 386 } 387 } 388 return nil 389 } 390 391 func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drivers.DriverNetwork, error) { 392 if _, ok := d.tasks.Get(cfg.ID); ok { 393 return nil, nil, fmt.Errorf("taskConfig with ID '%s' already started", cfg.ID) 394 } 395 396 var driverConfig TaskConfig 397 398 if err := cfg.DecodeDriverConfig(&driverConfig); err != nil { 399 return nil, nil, fmt.Errorf("failed to decode driver config: %v", err) 400 } 401 402 // ensure that PortMap variables are populated early on 403 cfg.Env = taskenv.SetPortMapEnvs(cfg.Env, driverConfig.PortMap) 404 405 handle := drivers.NewTaskHandle(taskHandleVersion) 406 handle.Config = cfg 407 408 if err := validateArgs(d.config.ArgsAllowList, driverConfig.Args); err != nil { 409 return nil, nil, err 410 } 411 412 // Get the image source 413 vmPath := driverConfig.ImagePath 414 if vmPath == "" { 415 return nil, nil, fmt.Errorf("image_path must be set") 416 } 417 vmID := filepath.Base(vmPath) 418 419 if !isAllowedImagePath(d.config.ImagePaths, cfg.AllocDir, vmPath) { 420 return nil, nil, fmt.Errorf("image_path is not in the allowed paths") 421 } 422 423 // Parse configuration arguments 424 // Create the base arguments 425 accelerator := "tcg" 426 if driverConfig.Accelerator != "" { 427 accelerator = driverConfig.Accelerator 428 } 429 430 mb := cfg.Resources.NomadResources.Memory.MemoryMB 431 if mb < 128 || mb > 4000000 { 432 return nil, nil, fmt.Errorf("Qemu memory assignment out of bounds") 433 } 434 mem := fmt.Sprintf("%dM", mb) 435 436 absPath, err := GetAbsolutePath("qemu-system-x86_64") 437 if err != nil { 438 return nil, nil, err 439 } 440 441 driveInterface := "ide" 442 if driverConfig.DriveInterface != "" { 443 driveInterface = driverConfig.DriveInterface 444 } 445 if !isAllowedDriveInterface(driveInterface) { 446 return nil, nil, fmt.Errorf("Unsupported drive_interface") 447 } 448 449 args := []string{ 450 absPath, 451 "-machine", "type=pc,accel=" + accelerator, 452 "-name", vmID, 453 "-m", mem, 454 "-drive", "file=" + vmPath + ",if=" + driveInterface, 455 "-nographic", 456 } 457 458 var netdevArgs []string 459 if cfg.DNS != nil { 460 if len(cfg.DNS.Servers) > 0 { 461 netdevArgs = append(netdevArgs, "dns="+cfg.DNS.Servers[0]) 462 } 463 464 for _, s := range cfg.DNS.Searches { 465 netdevArgs = append(netdevArgs, "dnssearch="+s) 466 } 467 } 468 469 taskDir := filepath.Join(cfg.AllocDir, cfg.Name) 470 471 var monitorPath string 472 if driverConfig.GracefulShutdown { 473 if runtime.GOOS == "windows" { 474 return nil, nil, errors.New("QEMU graceful shutdown is unsupported on the Windows platform") 475 } 476 // This socket will be used to manage the virtual machine (for example, 477 // to perform graceful shutdowns) 478 monitorPath = filepath.Join(taskDir, qemuMonitorSocketName) 479 if err := validateSocketPath(monitorPath); err != nil { 480 return nil, nil, err 481 } 482 d.logger.Debug("got monitor path", "monitorPath", monitorPath) 483 args = append(args, "-monitor", fmt.Sprintf("unix:%s,server,nowait", monitorPath)) 484 } 485 486 if driverConfig.GuestAgent { 487 if runtime.GOOS == "windows" { 488 return nil, nil, errors.New("QEMU Guest Agent socket is unsupported on the Windows platform") 489 } 490 // This socket will be used to communicate with the Guest Agent (if it's running) 491 agentSocketPath := filepath.Join(taskDir, qemuGuestAgentSocketName) 492 if err := validateSocketPath(agentSocketPath); err != nil { 493 return nil, nil, err 494 } 495 496 args = append(args, "-chardev", fmt.Sprintf("socket,path=%s,server,nowait,id=qga0", agentSocketPath)) 497 args = append(args, "-device", "virtio-serial") 498 args = append(args, "-device", "virtserialport,chardev=qga0,name=org.qemu.guest_agent.0") 499 } 500 501 // Add pass through arguments to qemu executable. A user can specify 502 // these arguments in driver task configuration. These arguments are 503 // passed directly to the qemu driver as command line options. 504 // For example, args = [ "-nodefconfig", "-nodefaults" ] 505 // This will allow a VM with embedded configuration to boot successfully. 506 args = append(args, driverConfig.Args...) 507 508 // Check the Resources required Networks to add port mappings. If no resources 509 // are required, we assume the VM is a purely compute job and does not require 510 // the outside world to be able to reach it. VMs ran without port mappings can 511 // still reach out to the world, but without port mappings it is effectively 512 // firewalled 513 protocols := []string{"udp", "tcp"} 514 if len(cfg.Resources.NomadResources.Networks) > 0 { 515 // Loop through the port map and construct the hostfwd string, to map 516 // reserved ports to the ports listenting in the VM 517 // Ex: hostfwd=tcp::22000-:22,hostfwd=tcp::80-:8080 518 taskPorts := cfg.Resources.NomadResources.Networks[0].PortLabels() 519 for label, guest := range driverConfig.PortMap { 520 host, ok := taskPorts[label] 521 if !ok { 522 return nil, nil, fmt.Errorf("Unknown port label %q", label) 523 } 524 525 for _, p := range protocols { 526 netdevArgs = append(netdevArgs, fmt.Sprintf("hostfwd=%s::%d-:%d", p, host, guest)) 527 } 528 } 529 530 if len(netdevArgs) != 0 { 531 args = append(args, 532 "-netdev", 533 fmt.Sprintf("user,id=user.0,%s", strings.Join(netdevArgs, ",")), 534 "-device", "virtio-net,netdev=user.0", 535 ) 536 } 537 } 538 539 // If using KVM, add optimization args 540 if accelerator == "kvm" { 541 if runtime.GOOS == "windows" { 542 return nil, nil, errors.New("KVM accelerator is unsupported on the Windows platform") 543 } 544 args = append(args, 545 "-enable-kvm", 546 "-cpu", "host", 547 ) 548 549 if cfg.Resources.LinuxResources != nil && cfg.Resources.LinuxResources.CpusetCpus != "" { 550 cores := strings.Split(cfg.Resources.LinuxResources.CpusetCpus, ",") 551 args = append(args, 552 "-smp", fmt.Sprintf("%d", len(cores)), 553 ) 554 } 555 } 556 d.logger.Debug("starting QemuVM command ", "args", strings.Join(args, " ")) 557 558 pluginLogFile := filepath.Join(cfg.TaskDir().Dir, fmt.Sprintf("%s-executor.out", cfg.Name)) 559 executorConfig := &executor.ExecutorConfig{ 560 LogFile: pluginLogFile, 561 LogLevel: "debug", 562 } 563 564 execImpl, pluginClient, err := executor.CreateExecutor( 565 d.logger.With("task_name", handle.Config.Name, "alloc_id", handle.Config.AllocID), 566 d.nomadConfig, executorConfig) 567 if err != nil { 568 return nil, nil, err 569 } 570 571 execCmd := &executor.ExecCommand{ 572 Cmd: args[0], 573 Args: args[1:], 574 Env: cfg.EnvList(), 575 User: cfg.User, 576 TaskDir: cfg.TaskDir().Dir, 577 StdoutPath: cfg.StdoutPath, 578 StderrPath: cfg.StderrPath, 579 NetworkIsolation: cfg.NetworkIsolation, 580 } 581 ps, err := execImpl.Launch(execCmd) 582 if err != nil { 583 pluginClient.Kill() 584 return nil, nil, err 585 } 586 d.logger.Debug("started new QemuVM", "ID", vmID) 587 588 h := &taskHandle{ 589 exec: execImpl, 590 pid: ps.Pid, 591 monitorPath: monitorPath, 592 pluginClient: pluginClient, 593 taskConfig: cfg, 594 procState: drivers.TaskStateRunning, 595 startedAt: time.Now().Round(time.Millisecond), 596 logger: d.logger, 597 } 598 599 qemuDriverState := TaskState{ 600 ReattachConfig: pstructs.ReattachConfigFromGoPlugin(pluginClient.ReattachConfig()), 601 Pid: ps.Pid, 602 TaskConfig: cfg, 603 StartedAt: h.startedAt, 604 } 605 606 if err := handle.SetDriverState(&qemuDriverState); err != nil { 607 d.logger.Error("failed to start task, error setting driver state", "error", err) 608 execImpl.Shutdown("", 0) 609 pluginClient.Kill() 610 return nil, nil, fmt.Errorf("failed to set driver state: %v", err) 611 } 612 613 d.tasks.Set(cfg.ID, h) 614 go h.run() 615 616 var driverNetwork *drivers.DriverNetwork 617 if len(driverConfig.PortMap) == 1 { 618 driverNetwork = &drivers.DriverNetwork{ 619 PortMap: driverConfig.PortMap, 620 } 621 } 622 return handle, driverNetwork, nil 623 } 624 625 func (d *Driver) WaitTask(ctx context.Context, taskID string) (<-chan *drivers.ExitResult, error) { 626 handle, ok := d.tasks.Get(taskID) 627 if !ok { 628 return nil, drivers.ErrTaskNotFound 629 } 630 631 ch := make(chan *drivers.ExitResult) 632 go d.handleWait(ctx, handle, ch) 633 634 return ch, nil 635 } 636 637 func (d *Driver) StopTask(taskID string, timeout time.Duration, signal string) error { 638 handle, ok := d.tasks.Get(taskID) 639 if !ok { 640 return drivers.ErrTaskNotFound 641 } 642 643 // Attempt a graceful shutdown only if it was configured in the job 644 if handle.monitorPath != "" { 645 if err := sendQemuShutdown(d.logger, handle.monitorPath, handle.pid); err != nil { 646 d.logger.Debug("error sending graceful shutdown ", "pid", handle.pid, "error", err) 647 } 648 } else { 649 d.logger.Debug("monitor socket is empty, forcing shutdown") 650 } 651 652 // TODO(preetha) we are calling shutdown on the executor here 653 // after attempting a graceful qemu shutdown, qemu process may 654 // not be around when we call exec.shutdown 655 if err := handle.exec.Shutdown(signal, timeout); err != nil { 656 if handle.pluginClient.Exited() { 657 return nil 658 } 659 return fmt.Errorf("executor Shutdown failed: %v", err) 660 } 661 662 return nil 663 } 664 665 func (d *Driver) DestroyTask(taskID string, force bool) error { 666 handle, ok := d.tasks.Get(taskID) 667 if !ok { 668 return drivers.ErrTaskNotFound 669 } 670 671 if handle.IsRunning() && !force { 672 return fmt.Errorf("cannot destroy running task") 673 } 674 675 if !handle.pluginClient.Exited() { 676 if err := handle.exec.Shutdown("", 0); err != nil { 677 handle.logger.Error("destroying executor failed", "error", err) 678 } 679 680 handle.pluginClient.Kill() 681 } 682 683 d.tasks.Delete(taskID) 684 return nil 685 } 686 687 func (d *Driver) InspectTask(taskID string) (*drivers.TaskStatus, error) { 688 handle, ok := d.tasks.Get(taskID) 689 if !ok { 690 return nil, drivers.ErrTaskNotFound 691 } 692 693 return handle.TaskStatus(), nil 694 } 695 696 func (d *Driver) TaskStats(ctx context.Context, taskID string, interval time.Duration) (<-chan *drivers.TaskResourceUsage, error) { 697 handle, ok := d.tasks.Get(taskID) 698 if !ok { 699 return nil, drivers.ErrTaskNotFound 700 } 701 702 return handle.exec.Stats(ctx, interval) 703 } 704 705 func (d *Driver) TaskEvents(ctx context.Context) (<-chan *drivers.TaskEvent, error) { 706 return d.eventer.TaskEvents(ctx) 707 } 708 709 func (d *Driver) SignalTask(taskID string, signal string) error { 710 return fmt.Errorf("Qemu driver can't signal commands") 711 } 712 713 func (d *Driver) ExecTask(taskID string, cmdArgs []string, timeout time.Duration) (*drivers.ExecTaskResult, error) { 714 return nil, fmt.Errorf("Qemu driver can't execute commands") 715 716 } 717 718 // GetAbsolutePath returns the absolute path of the passed binary by resolving 719 // it in the path and following symlinks. 720 func GetAbsolutePath(bin string) (string, error) { 721 lp, err := exec.LookPath(bin) 722 if err != nil { 723 return "", fmt.Errorf("failed to resolve path to %q executable: %v", bin, err) 724 } 725 726 return filepath.EvalSymlinks(lp) 727 } 728 729 func (d *Driver) handleWait(ctx context.Context, handle *taskHandle, ch chan *drivers.ExitResult) { 730 defer close(ch) 731 var result *drivers.ExitResult 732 ps, err := handle.exec.Wait(ctx) 733 if err != nil { 734 result = &drivers.ExitResult{ 735 Err: fmt.Errorf("executor: error waiting on process: %v", err), 736 } 737 } else { 738 result = &drivers.ExitResult{ 739 ExitCode: ps.ExitCode, 740 Signal: ps.Signal, 741 } 742 } 743 744 select { 745 case <-ctx.Done(): 746 case <-d.ctx.Done(): 747 case ch <- result: 748 } 749 } 750 751 // validateSocketPath provides best effort validation of socket paths since 752 // some rules may be platform-dependant. 753 func validateSocketPath(path string) error { 754 if maxSocketPathLen > 0 && len(path) > maxSocketPathLen { 755 return fmt.Errorf( 756 "socket path %s is longer than the maximum length allowed (%d), try to reduce the task name or Nomad's data_dir if possible.", 757 path, maxSocketPathLen) 758 } 759 760 return nil 761 } 762 763 // sendQemuShutdown attempts to issue an ACPI power-off command via the qemu 764 // monitor 765 func sendQemuShutdown(logger hclog.Logger, monitorPath string, userPid int) error { 766 if monitorPath == "" { 767 return errors.New("monitorPath not set") 768 } 769 monitorSocket, err := net.Dial("unix", monitorPath) 770 if err != nil { 771 logger.Warn("could not connect to qemu monitor", "pid", userPid, "monitorPath", monitorPath, "error", err) 772 return err 773 } 774 defer monitorSocket.Close() 775 logger.Debug("sending graceful shutdown command to qemu monitor socket", "monitor_path", monitorPath, "pid", userPid) 776 _, err = monitorSocket.Write([]byte(qemuGracefulShutdownMsg)) 777 if err != nil { 778 logger.Warn("failed to send shutdown message", "shutdown message", qemuGracefulShutdownMsg, "monitorPath", monitorPath, "userPid", userPid, "error", err) 779 } 780 return err 781 }