github.com/bigcommerce/nomad@v0.9.3-bc/drivers/shared/executor/executor.go (about) 1 package executor 2 3 import ( 4 "context" 5 "fmt" 6 "io" 7 "io/ioutil" 8 "os" 9 "os/exec" 10 "path/filepath" 11 "runtime" 12 "strings" 13 "syscall" 14 "time" 15 16 "github.com/armon/circbuf" 17 "github.com/hashicorp/consul-template/signals" 18 hclog "github.com/hashicorp/go-hclog" 19 multierror "github.com/hashicorp/go-multierror" 20 "github.com/hashicorp/nomad/client/allocdir" 21 "github.com/hashicorp/nomad/client/lib/fifo" 22 "github.com/hashicorp/nomad/client/stats" 23 cstructs "github.com/hashicorp/nomad/client/structs" 24 "github.com/hashicorp/nomad/plugins/drivers" 25 "github.com/kr/pty" 26 27 shelpers "github.com/hashicorp/nomad/helper/stats" 28 ) 29 30 const ( 31 // ExecutorVersionLatest is the current and latest version of the executor 32 ExecutorVersionLatest = "2.0.0" 33 34 // ExecutorVersionPre0_9 is the version of executor use prior to the release 35 // of 0.9.x 36 ExecutorVersionPre0_9 = "1.1.0" 37 ) 38 39 var ( 40 // The statistics the basic executor exposes 41 ExecutorBasicMeasuredMemStats = []string{"RSS", "Swap"} 42 ExecutorBasicMeasuredCpuStats = []string{"System Mode", "User Mode", "Percent"} 43 ) 44 45 // Executor is the interface which allows a driver to launch and supervise 46 // a process 47 type Executor interface { 48 // Launch a user process configured by the given ExecCommand 49 Launch(launchCmd *ExecCommand) (*ProcessState, error) 50 51 // Wait blocks until the process exits or an error occures 52 Wait(ctx context.Context) (*ProcessState, error) 53 54 // Shutdown will shutdown the executor by stopping the user process, 55 // cleaning up and resources created by the executor. The shutdown sequence 56 // will first send the given signal to the process. This defaults to "SIGINT" 57 // if not specified. The executor will then wait for the process to exit 58 // before cleaning up other resources. If the executor waits longer than the 59 // given grace period, the process is forcefully killed. 60 // 61 // To force kill the user process, gracePeriod can be set to 0. 62 Shutdown(signal string, gracePeriod time.Duration) error 63 64 // UpdateResources updates any resource isolation enforcement with new 65 // constraints if supported. 66 UpdateResources(*drivers.Resources) error 67 68 // Version returns the executor API version 69 Version() (*ExecutorVersion, error) 70 71 // Returns a channel of stats. Stats are collected and 72 // pushed to the channel on the given interval 73 Stats(context.Context, time.Duration) (<-chan *cstructs.TaskResourceUsage, error) 74 75 // Signal sends the given signal to the user process 76 Signal(os.Signal) error 77 78 // Exec executes the given command and args inside the executor context 79 // and returns the output and exit code. 80 Exec(deadline time.Time, cmd string, args []string) ([]byte, int, error) 81 82 ExecStreaming(ctx context.Context, cmd []string, tty bool, 83 stream drivers.ExecTaskStream) error 84 } 85 86 // ExecCommand holds the user command, args, and other isolation related 87 // settings. 88 type ExecCommand struct { 89 // Cmd is the command that the user wants to run. 90 Cmd string 91 92 // Args is the args of the command that the user wants to run. 93 Args []string 94 95 // Resources defined by the task 96 Resources *drivers.Resources 97 98 // StdoutPath is the path the process stdout should be written to 99 StdoutPath string 100 stdout io.WriteCloser 101 102 // StderrPath is the path the process stderr should be written to 103 StderrPath string 104 stderr io.WriteCloser 105 106 // Env is the list of KEY=val pairs of environment variables to be set 107 Env []string 108 109 // User is the user which the executor uses to run the command. 110 User string 111 112 // TaskDir is the directory path on the host where for the task 113 TaskDir string 114 115 // ResourceLimits determines whether resource limits are enforced by the 116 // executor. 117 ResourceLimits bool 118 119 // Cgroup marks whether we put the process in a cgroup. Setting this field 120 // doesn't enforce resource limits. To enforce limits, set ResourceLimits. 121 // Using the cgroup does allow more precise cleanup of processes. 122 BasicProcessCgroup bool 123 124 // Mounts are the host paths to be be made available inside rootfs 125 Mounts []*drivers.MountConfig 126 127 // Devices are the the device nodes to be created in isolation environment 128 Devices []*drivers.DeviceConfig 129 } 130 131 // SetWriters sets the writer for the process stdout and stderr. This should 132 // not be used if writing to a file path such as a fifo file. SetStdoutWriter 133 // is mainly used for unit testing purposes. 134 func (c *ExecCommand) SetWriters(out io.WriteCloser, err io.WriteCloser) { 135 c.stdout = out 136 c.stderr = err 137 } 138 139 // GetWriters returns the unexported io.WriteCloser for the stdout and stderr 140 // handles. This is mainly used for unit testing purposes. 141 func (c *ExecCommand) GetWriters() (stdout io.WriteCloser, stderr io.WriteCloser) { 142 return c.stdout, c.stderr 143 } 144 145 type nopCloser struct { 146 io.Writer 147 } 148 149 func (nopCloser) Close() error { return nil } 150 151 // Stdout returns a writer for the configured file descriptor 152 func (c *ExecCommand) Stdout() (io.WriteCloser, error) { 153 if c.stdout == nil { 154 if c.StdoutPath != "" { 155 f, err := fifo.OpenWriter(c.StdoutPath) 156 if err != nil { 157 return nil, fmt.Errorf("failed to create stdout: %v", err) 158 } 159 c.stdout = f 160 } else { 161 c.stdout = nopCloser{ioutil.Discard} 162 } 163 } 164 return c.stdout, nil 165 } 166 167 // Stderr returns a writer for the configured file descriptor 168 func (c *ExecCommand) Stderr() (io.WriteCloser, error) { 169 if c.stderr == nil { 170 if c.StderrPath != "" { 171 f, err := fifo.OpenWriter(c.StderrPath) 172 if err != nil { 173 return nil, fmt.Errorf("failed to create stderr: %v", err) 174 } 175 c.stderr = f 176 } else { 177 c.stderr = nopCloser{ioutil.Discard} 178 } 179 } 180 return c.stderr, nil 181 } 182 183 func (c *ExecCommand) Close() { 184 if c.stdout != nil { 185 c.stdout.Close() 186 } 187 if c.stderr != nil { 188 c.stderr.Close() 189 } 190 } 191 192 // ProcessState holds information about the state of a user process. 193 type ProcessState struct { 194 Pid int 195 ExitCode int 196 Signal int 197 Time time.Time 198 } 199 200 // ExecutorVersion is the version of the executor 201 type ExecutorVersion struct { 202 Version string 203 } 204 205 func (v *ExecutorVersion) GoString() string { 206 return v.Version 207 } 208 209 // UniversalExecutor is an implementation of the Executor which launches and 210 // supervises processes. In addition to process supervision it provides resource 211 // and file system isolation 212 type UniversalExecutor struct { 213 childCmd exec.Cmd 214 commandCfg *ExecCommand 215 216 exitState *ProcessState 217 processExited chan interface{} 218 219 // resConCtx is used to track and cleanup additional resources created by 220 // the executor. Currently this is only used for cgroups. 221 resConCtx resourceContainerContext 222 223 totalCpuStats *stats.CpuStats 224 userCpuStats *stats.CpuStats 225 systemCpuStats *stats.CpuStats 226 pidCollector *pidCollector 227 228 logger hclog.Logger 229 } 230 231 // NewExecutor returns an Executor 232 func NewExecutor(logger hclog.Logger) Executor { 233 logger = logger.Named("executor") 234 if err := shelpers.Init(); err != nil { 235 logger.Error("unable to initialize stats", "error", err) 236 } 237 return &UniversalExecutor{ 238 logger: logger, 239 processExited: make(chan interface{}), 240 totalCpuStats: stats.NewCpuStats(), 241 userCpuStats: stats.NewCpuStats(), 242 systemCpuStats: stats.NewCpuStats(), 243 pidCollector: newPidCollector(logger), 244 } 245 } 246 247 // Version returns the api version of the executor 248 func (e *UniversalExecutor) Version() (*ExecutorVersion, error) { 249 return &ExecutorVersion{Version: ExecutorVersionLatest}, nil 250 } 251 252 // Launch launches the main process and returns its state. It also 253 // configures an applies isolation on certain platforms. 254 func (e *UniversalExecutor) Launch(command *ExecCommand) (*ProcessState, error) { 255 e.logger.Trace("preparing to launch command", "command", command.Cmd, "args", strings.Join(command.Args, " ")) 256 257 e.commandCfg = command 258 259 // setting the user of the process 260 if command.User != "" { 261 e.logger.Debug("running command as user", "user", command.User) 262 if err := e.runAs(command.User); err != nil { 263 return nil, err 264 } 265 } 266 267 // set the task dir as the working directory for the command 268 e.childCmd.Dir = e.commandCfg.TaskDir 269 270 // start command in separate process group 271 if err := e.setNewProcessGroup(); err != nil { 272 return nil, err 273 } 274 275 // Setup cgroups on linux 276 if err := e.configureResourceContainer(os.Getpid()); err != nil { 277 return nil, err 278 } 279 280 stdout, err := e.commandCfg.Stdout() 281 if err != nil { 282 return nil, err 283 } 284 stderr, err := e.commandCfg.Stderr() 285 if err != nil { 286 return nil, err 287 } 288 289 e.childCmd.Stdout = stdout 290 e.childCmd.Stderr = stderr 291 292 // Look up the binary path and make it executable 293 absPath, err := lookupBin(command.TaskDir, command.Cmd) 294 if err != nil { 295 return nil, err 296 } 297 298 if err := makeExecutable(absPath); err != nil { 299 return nil, err 300 } 301 302 path := absPath 303 304 // Set the commands arguments 305 e.childCmd.Path = path 306 e.childCmd.Args = append([]string{e.childCmd.Path}, command.Args...) 307 e.childCmd.Env = e.commandCfg.Env 308 309 // Start the process 310 e.logger.Debug("launching", "command", command.Cmd, "args", strings.Join(command.Args, " ")) 311 if err := e.childCmd.Start(); err != nil { 312 return nil, fmt.Errorf("failed to start command path=%q --- args=%q: %v", path, e.childCmd.Args, err) 313 } 314 315 go e.pidCollector.collectPids(e.processExited, getAllPids) 316 go e.wait() 317 return &ProcessState{Pid: e.childCmd.Process.Pid, ExitCode: -1, Time: time.Now()}, nil 318 } 319 320 // Exec a command inside a container for exec and java drivers. 321 func (e *UniversalExecutor) Exec(deadline time.Time, name string, args []string) ([]byte, int, error) { 322 ctx, cancel := context.WithDeadline(context.Background(), deadline) 323 defer cancel() 324 return ExecScript(ctx, e.childCmd.Dir, e.commandCfg.Env, e.childCmd.SysProcAttr, name, args) 325 } 326 327 // ExecScript executes cmd with args and returns the output, exit code, and 328 // error. Output is truncated to drivers/shared/structs.CheckBufSize 329 func ExecScript(ctx context.Context, dir string, env []string, attrs *syscall.SysProcAttr, 330 name string, args []string) ([]byte, int, error) { 331 cmd := exec.CommandContext(ctx, name, args...) 332 333 // Copy runtime environment from the main command 334 cmd.SysProcAttr = attrs 335 cmd.Dir = dir 336 cmd.Env = env 337 338 // Capture output 339 buf, _ := circbuf.NewBuffer(int64(drivers.CheckBufSize)) 340 cmd.Stdout = buf 341 cmd.Stderr = buf 342 343 if err := cmd.Run(); err != nil { 344 exitErr, ok := err.(*exec.ExitError) 345 if !ok { 346 // Non-exit error, return it and let the caller treat 347 // it as a critical failure 348 return nil, 0, err 349 } 350 351 // Some kind of error happened; default to critical 352 exitCode := 2 353 if status, ok := exitErr.Sys().(syscall.WaitStatus); ok { 354 exitCode = status.ExitStatus() 355 } 356 357 // Don't return the exitError as the caller only needs the 358 // output and code. 359 return buf.Bytes(), exitCode, nil 360 } 361 return buf.Bytes(), 0, nil 362 } 363 364 func (e *UniversalExecutor) ExecStreaming(ctx context.Context, command []string, tty bool, 365 stream drivers.ExecTaskStream) error { 366 367 if len(command) == 0 { 368 return fmt.Errorf("command is required") 369 } 370 371 cmd := exec.CommandContext(ctx, command[0], command[1:]...) 372 373 cmd.Dir = "/" 374 cmd.Env = e.childCmd.Env 375 376 execHelper := &execHelper{ 377 logger: e.logger, 378 379 newTerminal: func() (func() (*os.File, error), *os.File, error) { 380 pty, tty, err := pty.Open() 381 if err != nil { 382 return nil, nil, err 383 } 384 385 return func() (*os.File, error) { return pty, nil }, tty, err 386 }, 387 setTTY: func(tty *os.File) error { 388 cmd.SysProcAttr = sessionCmdAttr(tty) 389 390 cmd.Stdin = tty 391 cmd.Stdout = tty 392 cmd.Stderr = tty 393 return nil 394 }, 395 setIO: func(stdin io.Reader, stdout, stderr io.Writer) error { 396 cmd.Stdin = stdin 397 cmd.Stdout = stdout 398 cmd.Stderr = stderr 399 return nil 400 }, 401 processStart: cmd.Start, 402 processWait: func() (*os.ProcessState, error) { 403 err := cmd.Wait() 404 return cmd.ProcessState, err 405 }, 406 } 407 408 return execHelper.run(ctx, tty, stream) 409 } 410 411 // Wait waits until a process has exited and returns it's exitcode and errors 412 func (e *UniversalExecutor) Wait(ctx context.Context) (*ProcessState, error) { 413 select { 414 case <-ctx.Done(): 415 return nil, ctx.Err() 416 case <-e.processExited: 417 return e.exitState, nil 418 } 419 } 420 421 func (e *UniversalExecutor) UpdateResources(resources *drivers.Resources) error { 422 return nil 423 } 424 425 func (e *UniversalExecutor) wait() { 426 defer close(e.processExited) 427 defer e.commandCfg.Close() 428 pid := e.childCmd.Process.Pid 429 err := e.childCmd.Wait() 430 if err == nil { 431 e.exitState = &ProcessState{Pid: pid, ExitCode: 0, Time: time.Now()} 432 return 433 } 434 435 exitCode := 1 436 var signal int 437 if exitErr, ok := err.(*exec.ExitError); ok { 438 if status, ok := exitErr.Sys().(syscall.WaitStatus); ok { 439 exitCode = status.ExitStatus() 440 if status.Signaled() { 441 // bash(1) uses the lower 7 bits of a uint8 442 // to indicate normal program failure (see 443 // <sysexits.h>). If a process terminates due 444 // to a signal, encode the signal number to 445 // indicate which signal caused the process 446 // to terminate. Mirror this exit code 447 // encoding scheme. 448 const exitSignalBase = 128 449 signal = int(status.Signal()) 450 exitCode = exitSignalBase + signal 451 } 452 } 453 } else { 454 e.logger.Warn("unexpected Cmd.Wait() error type", "error", err) 455 } 456 457 e.exitState = &ProcessState{Pid: pid, ExitCode: exitCode, Signal: signal, Time: time.Now()} 458 } 459 460 var ( 461 // finishedErr is the error message received when trying to kill and already 462 // exited process. 463 finishedErr = "os: process already finished" 464 465 // noSuchProcessErr is the error message received when trying to kill a non 466 // existing process (e.g. when killing a process group). 467 noSuchProcessErr = "no such process" 468 ) 469 470 // Exit cleans up the alloc directory, destroys resource container and kills the 471 // user process 472 func (e *UniversalExecutor) Shutdown(signal string, grace time.Duration) error { 473 e.logger.Debug("shutdown requested", "signal", signal, "grace_period_ms", grace.Round(time.Millisecond)) 474 var merr multierror.Error 475 476 // If the executor did not launch a process, return. 477 if e.commandCfg == nil { 478 return nil 479 } 480 481 // If there is no process we can't shutdown 482 if e.childCmd.Process == nil { 483 e.logger.Warn("failed to shutdown", "error", "no process found") 484 return fmt.Errorf("executor failed to shutdown error: no process found") 485 } 486 487 proc, err := os.FindProcess(e.childCmd.Process.Pid) 488 if err != nil { 489 err = fmt.Errorf("executor failed to find process: %v", err) 490 e.logger.Warn("failed to shutdown", "error", err) 491 return err 492 } 493 494 // If grace is 0 then skip shutdown logic 495 if grace > 0 { 496 // Default signal to SIGINT if not set 497 if signal == "" { 498 signal = "SIGINT" 499 } 500 501 sig, ok := signals.SignalLookup[signal] 502 if !ok { 503 err = fmt.Errorf("error unknown signal given for shutdown: %s", signal) 504 e.logger.Warn("failed to shutdown", "error", err) 505 return err 506 } 507 508 if err := e.shutdownProcess(sig, proc); err != nil { 509 e.logger.Warn("failed to shutdown", "error", err) 510 return err 511 } 512 513 select { 514 case <-e.processExited: 515 case <-time.After(grace): 516 proc.Kill() 517 } 518 } else { 519 proc.Kill() 520 } 521 522 // Wait for process to exit 523 select { 524 case <-e.processExited: 525 case <-time.After(time.Second * 15): 526 e.logger.Warn("process did not exit after 15 seconds") 527 merr.Errors = append(merr.Errors, fmt.Errorf("process did not exit after 15 seconds")) 528 } 529 530 // Prefer killing the process via the resource container. 531 if !(e.commandCfg.ResourceLimits || e.commandCfg.BasicProcessCgroup) { 532 if err := e.cleanupChildProcesses(proc); err != nil && err.Error() != finishedErr { 533 merr.Errors = append(merr.Errors, 534 fmt.Errorf("can't kill process with pid %d: %v", e.childCmd.Process.Pid, err)) 535 } 536 } 537 538 if e.commandCfg.ResourceLimits || e.commandCfg.BasicProcessCgroup { 539 if err := e.resConCtx.executorCleanup(); err != nil { 540 merr.Errors = append(merr.Errors, err) 541 } 542 } 543 544 if err := merr.ErrorOrNil(); err != nil { 545 e.logger.Warn("failed to shutdown", "error", err) 546 return err 547 } 548 549 return nil 550 } 551 552 // Signal sends the passed signal to the task 553 func (e *UniversalExecutor) Signal(s os.Signal) error { 554 if e.childCmd.Process == nil { 555 return fmt.Errorf("Task not yet run") 556 } 557 558 e.logger.Debug("sending signal to PID", "signal", s, "pid", e.childCmd.Process.Pid) 559 err := e.childCmd.Process.Signal(s) 560 if err != nil { 561 e.logger.Error("sending signal failed", "signal", s, "error", err) 562 return err 563 } 564 565 return nil 566 } 567 568 func (e *UniversalExecutor) Stats(ctx context.Context, interval time.Duration) (<-chan *cstructs.TaskResourceUsage, error) { 569 ch := make(chan *cstructs.TaskResourceUsage) 570 go e.handleStats(ch, ctx, interval) 571 return ch, nil 572 } 573 574 func (e *UniversalExecutor) handleStats(ch chan *cstructs.TaskResourceUsage, ctx context.Context, interval time.Duration) { 575 defer close(ch) 576 timer := time.NewTimer(0) 577 for { 578 select { 579 case <-ctx.Done(): 580 return 581 582 case <-timer.C: 583 timer.Reset(interval) 584 } 585 586 pidStats, err := e.pidCollector.pidStats() 587 if err != nil { 588 e.logger.Warn("error collecting stats", "error", err) 589 return 590 } 591 592 select { 593 case <-ctx.Done(): 594 return 595 case ch <- aggregatedResourceUsage(e.systemCpuStats, pidStats): 596 } 597 } 598 } 599 600 // lookupBin looks for path to the binary to run by looking for the binary in 601 // the following locations, in-order: 602 // task/local/, task/, on the host file system, in host $PATH 603 // The return path is absolute. 604 func lookupBin(taskDir string, bin string) (string, error) { 605 // Check in the local directory 606 local := filepath.Join(taskDir, allocdir.TaskLocal, bin) 607 if _, err := os.Stat(local); err == nil { 608 return local, nil 609 } 610 611 // Check at the root of the task's directory 612 root := filepath.Join(taskDir, bin) 613 if _, err := os.Stat(root); err == nil { 614 return root, nil 615 } 616 617 // when checking host paths, check with Stat first if path is absolute 618 // as exec.LookPath only considers files already marked as executable 619 // and only consider this for absolute paths to avoid depending on 620 // current directory of nomad which may cause unexpected behavior 621 if _, err := os.Stat(bin); err == nil && filepath.IsAbs(bin) { 622 return bin, nil 623 } 624 625 // Check the $PATH 626 if host, err := exec.LookPath(bin); err == nil { 627 return host, nil 628 } 629 630 return "", fmt.Errorf("binary %q could not be found", bin) 631 } 632 633 // makeExecutable makes the given file executable for root,group,others. 634 func makeExecutable(binPath string) error { 635 if runtime.GOOS == "windows" { 636 return nil 637 } 638 639 fi, err := os.Stat(binPath) 640 if err != nil { 641 if os.IsNotExist(err) { 642 return fmt.Errorf("binary %q does not exist", binPath) 643 } 644 return fmt.Errorf("specified binary is invalid: %v", err) 645 } 646 647 // If it is not executable, make it so. 648 perm := fi.Mode().Perm() 649 req := os.FileMode(0555) 650 if perm&req != req { 651 if err := os.Chmod(binPath, perm|req); err != nil { 652 return fmt.Errorf("error making %q executable: %s", binPath, err) 653 } 654 } 655 return nil 656 }