github.com/dkerwin/nomad@v0.3.3-0.20160525181927-74554135514b/client/driver/executor/executor_linux.go (about)

     1  package executor
     2  
     3  import (
     4  	"fmt"
     5  	"os"
     6  	"os/user"
     7  	"path/filepath"
     8  	"strconv"
     9  	"strings"
    10  	"syscall"
    11  
    12  	"github.com/hashicorp/go-multierror"
    13  	"github.com/opencontainers/runc/libcontainer/cgroups"
    14  	cgroupFs "github.com/opencontainers/runc/libcontainer/cgroups/fs"
    15  	cgroupConfig "github.com/opencontainers/runc/libcontainer/configs"
    16  
    17  	"github.com/hashicorp/nomad/client/allocdir"
    18  	"github.com/hashicorp/nomad/nomad/structs"
    19  )
    20  
    21  var (
    22  	// A mapping of directories on the host OS to attempt to embed inside each
    23  	// task's chroot.
    24  	chrootEnv = map[string]string{
    25  		"/bin":            "/bin",
    26  		"/etc":            "/etc",
    27  		"/lib":            "/lib",
    28  		"/lib32":          "/lib32",
    29  		"/lib64":          "/lib64",
    30  		"/run/resolvconf": "/run/resolvconf",
    31  		"/sbin":           "/sbin",
    32  		"/usr":            "/usr",
    33  	}
    34  )
    35  
    36  // configureIsolation configures chroot and creates cgroups
    37  func (e *UniversalExecutor) configureIsolation() error {
    38  	if e.command.FSIsolation {
    39  		if err := e.configureChroot(); err != nil {
    40  			return err
    41  		}
    42  	}
    43  
    44  	if e.command.ResourceLimits {
    45  		if err := e.configureCgroups(e.ctx.Task.Resources); err != nil {
    46  			return fmt.Errorf("error creating cgroups: %v", err)
    47  		}
    48  	}
    49  	return nil
    50  }
    51  
    52  // applyLimits puts a process in a pre-configured cgroup
    53  func (e *UniversalExecutor) applyLimits(pid int) error {
    54  	if !e.command.ResourceLimits {
    55  		return nil
    56  	}
    57  
    58  	// Entering the process in the cgroup
    59  	manager := getCgroupManager(e.groups, nil)
    60  	if err := manager.Apply(pid); err != nil {
    61  		e.logger.Printf("[ERR] executor: error applying pid to cgroup: %v", err)
    62  		if er := e.removeChrootMounts(); er != nil {
    63  			e.logger.Printf("[ERR] executor: error removing chroot: %v", er)
    64  		}
    65  		return err
    66  	}
    67  	e.cgPaths = manager.GetPaths()
    68  	cgConfig := cgroupConfig.Config{Cgroups: e.groups}
    69  	if err := manager.Set(&cgConfig); err != nil {
    70  		e.logger.Printf("[ERR] executor: error setting cgroup config: %v", err)
    71  		if er := DestroyCgroup(e.groups, e.cgPaths, os.Getpid()); er != nil {
    72  			e.logger.Printf("[ERR] executor: error destroying cgroup: %v", er)
    73  		}
    74  		if er := e.removeChrootMounts(); er != nil {
    75  			e.logger.Printf("[ERR] executor: error removing chroot: %v", er)
    76  		}
    77  		return err
    78  	}
    79  	return nil
    80  }
    81  
    82  // configureCgroups converts a Nomad Resources specification into the equivalent
    83  // cgroup configuration. It returns an error if the resources are invalid.
    84  func (e *UniversalExecutor) configureCgroups(resources *structs.Resources) error {
    85  	e.groups = &cgroupConfig.Cgroup{}
    86  	e.groups.Resources = &cgroupConfig.Resources{}
    87  	cgroupName := structs.GenerateUUID()
    88  	e.groups.Path = filepath.Join("/nomad", cgroupName)
    89  
    90  	// TODO: verify this is needed for things like network access
    91  	e.groups.Resources.AllowAllDevices = true
    92  
    93  	if resources.MemoryMB > 0 {
    94  		// Total amount of memory allowed to consume
    95  		e.groups.Resources.Memory = int64(resources.MemoryMB * 1024 * 1024)
    96  		// Disable swap to avoid issues on the machine
    97  		e.groups.Resources.MemorySwap = int64(-1)
    98  	}
    99  
   100  	if resources.CPU < 2 {
   101  		return fmt.Errorf("resources.CPU must be equal to or greater than 2: %v", resources.CPU)
   102  	}
   103  
   104  	// Set the relative CPU shares for this cgroup.
   105  	e.groups.Resources.CpuShares = int64(resources.CPU)
   106  
   107  	if resources.IOPS != 0 {
   108  		// Validate it is in an acceptable range.
   109  		if resources.IOPS < 10 || resources.IOPS > 1000 {
   110  			return fmt.Errorf("resources.IOPS must be between 10 and 1000: %d", resources.IOPS)
   111  		}
   112  
   113  		e.groups.Resources.BlkioWeight = uint16(resources.IOPS)
   114  	}
   115  
   116  	return nil
   117  }
   118  
   119  // runAs takes a user id as a string and looks up the user, and sets the command
   120  // to execute as that user.
   121  func (e *UniversalExecutor) runAs(userid string) error {
   122  	u, err := user.Lookup(userid)
   123  	if err != nil {
   124  		return fmt.Errorf("Failed to identify user %v: %v", userid, err)
   125  	}
   126  
   127  	// Convert the uid and gid
   128  	uid, err := strconv.ParseUint(u.Uid, 10, 32)
   129  	if err != nil {
   130  		return fmt.Errorf("Unable to convert userid to uint32: %s", err)
   131  	}
   132  	gid, err := strconv.ParseUint(u.Gid, 10, 32)
   133  	if err != nil {
   134  		return fmt.Errorf("Unable to convert groupid to uint32: %s", err)
   135  	}
   136  
   137  	// Set the command to run as that user and group.
   138  	if e.cmd.SysProcAttr == nil {
   139  		e.cmd.SysProcAttr = &syscall.SysProcAttr{}
   140  	}
   141  	if e.cmd.SysProcAttr.Credential == nil {
   142  		e.cmd.SysProcAttr.Credential = &syscall.Credential{}
   143  	}
   144  	e.cmd.SysProcAttr.Credential.Uid = uint32(uid)
   145  	e.cmd.SysProcAttr.Credential.Gid = uint32(gid)
   146  
   147  	return nil
   148  }
   149  
   150  // configureChroot configures a chroot
   151  func (e *UniversalExecutor) configureChroot() error {
   152  	allocDir := e.ctx.AllocDir
   153  	if err := allocDir.MountSharedDir(e.ctx.Task.Name); err != nil {
   154  		return err
   155  	}
   156  
   157  	if err := allocDir.Embed(e.ctx.Task.Name, chrootEnv); err != nil {
   158  		return err
   159  	}
   160  
   161  	// Set the tasks AllocDir environment variable.
   162  	e.ctx.TaskEnv.
   163  		SetAllocDir(filepath.Join("/", allocdir.SharedAllocName)).
   164  		SetTaskLocalDir(filepath.Join("/", allocdir.TaskLocal)).
   165  		Build()
   166  
   167  	if e.cmd.SysProcAttr == nil {
   168  		e.cmd.SysProcAttr = &syscall.SysProcAttr{}
   169  	}
   170  	e.cmd.SysProcAttr.Chroot = e.taskDir
   171  	e.cmd.Dir = "/"
   172  
   173  	if err := allocDir.MountSpecialDirs(e.taskDir); err != nil {
   174  		return err
   175  	}
   176  
   177  	return nil
   178  }
   179  
   180  // cleanTaskDir is an idempotent operation to clean the task directory and
   181  // should be called when tearing down the task.
   182  func (e *UniversalExecutor) removeChrootMounts() error {
   183  	// Prevent a race between Wait/ForceStop
   184  	e.cgLock.Lock()
   185  	defer e.cgLock.Unlock()
   186  	return e.ctx.AllocDir.UnmountAll()
   187  }
   188  
   189  // destroyCgroup kills all processes in the cgroup and removes the cgroup
   190  // configuration from the host. This function is idempotent.
   191  func DestroyCgroup(groups *cgroupConfig.Cgroup, cgPaths map[string]string, executorPid int) error {
   192  	mErrs := new(multierror.Error)
   193  	if groups == nil {
   194  		return fmt.Errorf("Can't destroy: cgroup configuration empty")
   195  	}
   196  
   197  	// Move the executor into the global cgroup so that the task specific
   198  	// cgroup can be destroyed.
   199  	nilGroup := &cgroupConfig.Cgroup{}
   200  	nilGroup.Path = "/"
   201  	nilGroup.Resources = groups.Resources
   202  	nilManager := getCgroupManager(nilGroup, nil)
   203  	err := nilManager.Apply(executorPid)
   204  	if err != nil && !strings.Contains(err.Error(), "no such process") {
   205  		return fmt.Errorf("failed to remove executor pid %d: %v", executorPid, err)
   206  	}
   207  
   208  	// Freeze the Cgroup so that it can not continue to fork/exec.
   209  	manager := getCgroupManager(groups, cgPaths)
   210  	err = manager.Freeze(cgroupConfig.Frozen)
   211  	if err != nil && !strings.Contains(err.Error(), "no such file or directory") {
   212  		return fmt.Errorf("failed to freeze cgroup: %v", err)
   213  	}
   214  
   215  	var procs []*os.Process
   216  	pids, err := manager.GetAllPids()
   217  	if err != nil {
   218  		multierror.Append(mErrs, fmt.Errorf("error getting pids: %v", err))
   219  
   220  		// Unfreeze the cgroup.
   221  		err = manager.Freeze(cgroupConfig.Thawed)
   222  		if err != nil && !strings.Contains(err.Error(), "no such file or directory") {
   223  			multierror.Append(mErrs, fmt.Errorf("failed to unfreeze cgroup: %v", err))
   224  		}
   225  		return mErrs.ErrorOrNil()
   226  	}
   227  
   228  	// Kill the processes in the cgroup
   229  	for _, pid := range pids {
   230  		proc, err := os.FindProcess(pid)
   231  		if err != nil {
   232  			multierror.Append(mErrs, fmt.Errorf("error finding process %v: %v", pid, err))
   233  			continue
   234  		}
   235  
   236  		procs = append(procs, proc)
   237  		if e := proc.Kill(); e != nil {
   238  			multierror.Append(mErrs, fmt.Errorf("error killing process %v: %v", pid, e))
   239  		}
   240  	}
   241  
   242  	// Unfreeze the cgroug so we can wait.
   243  	err = manager.Freeze(cgroupConfig.Thawed)
   244  	if err != nil && !strings.Contains(err.Error(), "no such file or directory") {
   245  		multierror.Append(mErrs, fmt.Errorf("failed to unfreeze cgroup: %v", err))
   246  	}
   247  
   248  	// Wait on the killed processes to ensure they are cleaned up.
   249  	for _, proc := range procs {
   250  		// Don't capture the error because we expect this to fail for
   251  		// processes we didn't fork.
   252  		proc.Wait()
   253  	}
   254  
   255  	// Remove the cgroup.
   256  	if err := manager.Destroy(); err != nil {
   257  		multierror.Append(mErrs, fmt.Errorf("failed to delete the cgroup directories: %v", err))
   258  	}
   259  	return mErrs.ErrorOrNil()
   260  }
   261  
   262  // getCgroupManager returns the correct libcontainer cgroup manager.
   263  func getCgroupManager(groups *cgroupConfig.Cgroup, paths map[string]string) cgroups.Manager {
   264  	return &cgroupFs.Manager{Cgroups: groups, Paths: paths}
   265  }