github.com/dkerwin/nomad@v0.3.3-0.20160525181927-74554135514b/client/driver/executor/executor_linux.go (about) 1 package executor 2 3 import ( 4 "fmt" 5 "os" 6 "os/user" 7 "path/filepath" 8 "strconv" 9 "strings" 10 "syscall" 11 12 "github.com/hashicorp/go-multierror" 13 "github.com/opencontainers/runc/libcontainer/cgroups" 14 cgroupFs "github.com/opencontainers/runc/libcontainer/cgroups/fs" 15 cgroupConfig "github.com/opencontainers/runc/libcontainer/configs" 16 17 "github.com/hashicorp/nomad/client/allocdir" 18 "github.com/hashicorp/nomad/nomad/structs" 19 ) 20 21 var ( 22 // A mapping of directories on the host OS to attempt to embed inside each 23 // task's chroot. 24 chrootEnv = map[string]string{ 25 "/bin": "/bin", 26 "/etc": "/etc", 27 "/lib": "/lib", 28 "/lib32": "/lib32", 29 "/lib64": "/lib64", 30 "/run/resolvconf": "/run/resolvconf", 31 "/sbin": "/sbin", 32 "/usr": "/usr", 33 } 34 ) 35 36 // configureIsolation configures chroot and creates cgroups 37 func (e *UniversalExecutor) configureIsolation() error { 38 if e.command.FSIsolation { 39 if err := e.configureChroot(); err != nil { 40 return err 41 } 42 } 43 44 if e.command.ResourceLimits { 45 if err := e.configureCgroups(e.ctx.Task.Resources); err != nil { 46 return fmt.Errorf("error creating cgroups: %v", err) 47 } 48 } 49 return nil 50 } 51 52 // applyLimits puts a process in a pre-configured cgroup 53 func (e *UniversalExecutor) applyLimits(pid int) error { 54 if !e.command.ResourceLimits { 55 return nil 56 } 57 58 // Entering the process in the cgroup 59 manager := getCgroupManager(e.groups, nil) 60 if err := manager.Apply(pid); err != nil { 61 e.logger.Printf("[ERR] executor: error applying pid to cgroup: %v", err) 62 if er := e.removeChrootMounts(); er != nil { 63 e.logger.Printf("[ERR] executor: error removing chroot: %v", er) 64 } 65 return err 66 } 67 e.cgPaths = manager.GetPaths() 68 cgConfig := cgroupConfig.Config{Cgroups: e.groups} 69 if err := manager.Set(&cgConfig); err != nil { 70 e.logger.Printf("[ERR] executor: error setting cgroup config: %v", err) 71 if er := DestroyCgroup(e.groups, e.cgPaths, os.Getpid()); er != nil { 72 e.logger.Printf("[ERR] executor: error destroying cgroup: %v", er) 73 } 74 if er := e.removeChrootMounts(); er != nil { 75 e.logger.Printf("[ERR] executor: error removing chroot: %v", er) 76 } 77 return err 78 } 79 return nil 80 } 81 82 // configureCgroups converts a Nomad Resources specification into the equivalent 83 // cgroup configuration. It returns an error if the resources are invalid. 84 func (e *UniversalExecutor) configureCgroups(resources *structs.Resources) error { 85 e.groups = &cgroupConfig.Cgroup{} 86 e.groups.Resources = &cgroupConfig.Resources{} 87 cgroupName := structs.GenerateUUID() 88 e.groups.Path = filepath.Join("/nomad", cgroupName) 89 90 // TODO: verify this is needed for things like network access 91 e.groups.Resources.AllowAllDevices = true 92 93 if resources.MemoryMB > 0 { 94 // Total amount of memory allowed to consume 95 e.groups.Resources.Memory = int64(resources.MemoryMB * 1024 * 1024) 96 // Disable swap to avoid issues on the machine 97 e.groups.Resources.MemorySwap = int64(-1) 98 } 99 100 if resources.CPU < 2 { 101 return fmt.Errorf("resources.CPU must be equal to or greater than 2: %v", resources.CPU) 102 } 103 104 // Set the relative CPU shares for this cgroup. 105 e.groups.Resources.CpuShares = int64(resources.CPU) 106 107 if resources.IOPS != 0 { 108 // Validate it is in an acceptable range. 109 if resources.IOPS < 10 || resources.IOPS > 1000 { 110 return fmt.Errorf("resources.IOPS must be between 10 and 1000: %d", resources.IOPS) 111 } 112 113 e.groups.Resources.BlkioWeight = uint16(resources.IOPS) 114 } 115 116 return nil 117 } 118 119 // runAs takes a user id as a string and looks up the user, and sets the command 120 // to execute as that user. 121 func (e *UniversalExecutor) runAs(userid string) error { 122 u, err := user.Lookup(userid) 123 if err != nil { 124 return fmt.Errorf("Failed to identify user %v: %v", userid, err) 125 } 126 127 // Convert the uid and gid 128 uid, err := strconv.ParseUint(u.Uid, 10, 32) 129 if err != nil { 130 return fmt.Errorf("Unable to convert userid to uint32: %s", err) 131 } 132 gid, err := strconv.ParseUint(u.Gid, 10, 32) 133 if err != nil { 134 return fmt.Errorf("Unable to convert groupid to uint32: %s", err) 135 } 136 137 // Set the command to run as that user and group. 138 if e.cmd.SysProcAttr == nil { 139 e.cmd.SysProcAttr = &syscall.SysProcAttr{} 140 } 141 if e.cmd.SysProcAttr.Credential == nil { 142 e.cmd.SysProcAttr.Credential = &syscall.Credential{} 143 } 144 e.cmd.SysProcAttr.Credential.Uid = uint32(uid) 145 e.cmd.SysProcAttr.Credential.Gid = uint32(gid) 146 147 return nil 148 } 149 150 // configureChroot configures a chroot 151 func (e *UniversalExecutor) configureChroot() error { 152 allocDir := e.ctx.AllocDir 153 if err := allocDir.MountSharedDir(e.ctx.Task.Name); err != nil { 154 return err 155 } 156 157 if err := allocDir.Embed(e.ctx.Task.Name, chrootEnv); err != nil { 158 return err 159 } 160 161 // Set the tasks AllocDir environment variable. 162 e.ctx.TaskEnv. 163 SetAllocDir(filepath.Join("/", allocdir.SharedAllocName)). 164 SetTaskLocalDir(filepath.Join("/", allocdir.TaskLocal)). 165 Build() 166 167 if e.cmd.SysProcAttr == nil { 168 e.cmd.SysProcAttr = &syscall.SysProcAttr{} 169 } 170 e.cmd.SysProcAttr.Chroot = e.taskDir 171 e.cmd.Dir = "/" 172 173 if err := allocDir.MountSpecialDirs(e.taskDir); err != nil { 174 return err 175 } 176 177 return nil 178 } 179 180 // cleanTaskDir is an idempotent operation to clean the task directory and 181 // should be called when tearing down the task. 182 func (e *UniversalExecutor) removeChrootMounts() error { 183 // Prevent a race between Wait/ForceStop 184 e.cgLock.Lock() 185 defer e.cgLock.Unlock() 186 return e.ctx.AllocDir.UnmountAll() 187 } 188 189 // destroyCgroup kills all processes in the cgroup and removes the cgroup 190 // configuration from the host. This function is idempotent. 191 func DestroyCgroup(groups *cgroupConfig.Cgroup, cgPaths map[string]string, executorPid int) error { 192 mErrs := new(multierror.Error) 193 if groups == nil { 194 return fmt.Errorf("Can't destroy: cgroup configuration empty") 195 } 196 197 // Move the executor into the global cgroup so that the task specific 198 // cgroup can be destroyed. 199 nilGroup := &cgroupConfig.Cgroup{} 200 nilGroup.Path = "/" 201 nilGroup.Resources = groups.Resources 202 nilManager := getCgroupManager(nilGroup, nil) 203 err := nilManager.Apply(executorPid) 204 if err != nil && !strings.Contains(err.Error(), "no such process") { 205 return fmt.Errorf("failed to remove executor pid %d: %v", executorPid, err) 206 } 207 208 // Freeze the Cgroup so that it can not continue to fork/exec. 209 manager := getCgroupManager(groups, cgPaths) 210 err = manager.Freeze(cgroupConfig.Frozen) 211 if err != nil && !strings.Contains(err.Error(), "no such file or directory") { 212 return fmt.Errorf("failed to freeze cgroup: %v", err) 213 } 214 215 var procs []*os.Process 216 pids, err := manager.GetAllPids() 217 if err != nil { 218 multierror.Append(mErrs, fmt.Errorf("error getting pids: %v", err)) 219 220 // Unfreeze the cgroup. 221 err = manager.Freeze(cgroupConfig.Thawed) 222 if err != nil && !strings.Contains(err.Error(), "no such file or directory") { 223 multierror.Append(mErrs, fmt.Errorf("failed to unfreeze cgroup: %v", err)) 224 } 225 return mErrs.ErrorOrNil() 226 } 227 228 // Kill the processes in the cgroup 229 for _, pid := range pids { 230 proc, err := os.FindProcess(pid) 231 if err != nil { 232 multierror.Append(mErrs, fmt.Errorf("error finding process %v: %v", pid, err)) 233 continue 234 } 235 236 procs = append(procs, proc) 237 if e := proc.Kill(); e != nil { 238 multierror.Append(mErrs, fmt.Errorf("error killing process %v: %v", pid, e)) 239 } 240 } 241 242 // Unfreeze the cgroug so we can wait. 243 err = manager.Freeze(cgroupConfig.Thawed) 244 if err != nil && !strings.Contains(err.Error(), "no such file or directory") { 245 multierror.Append(mErrs, fmt.Errorf("failed to unfreeze cgroup: %v", err)) 246 } 247 248 // Wait on the killed processes to ensure they are cleaned up. 249 for _, proc := range procs { 250 // Don't capture the error because we expect this to fail for 251 // processes we didn't fork. 252 proc.Wait() 253 } 254 255 // Remove the cgroup. 256 if err := manager.Destroy(); err != nil { 257 multierror.Append(mErrs, fmt.Errorf("failed to delete the cgroup directories: %v", err)) 258 } 259 return mErrs.ErrorOrNil() 260 } 261 262 // getCgroupManager returns the correct libcontainer cgroup manager. 263 func getCgroupManager(groups *cgroupConfig.Cgroup, paths map[string]string) cgroups.Manager { 264 return &cgroupFs.Manager{Cgroups: groups, Paths: paths} 265 }