github.com/ferranbt/nomad@v0.9.3-0.20190607002617-85c449b7667c/drivers/shared/executor/executor_linux_test.go (about) 1 package executor 2 3 import ( 4 "context" 5 "fmt" 6 "io/ioutil" 7 "os" 8 "path/filepath" 9 "regexp" 10 "strconv" 11 "strings" 12 "testing" 13 "time" 14 15 "github.com/hashicorp/nomad/client/allocdir" 16 "github.com/hashicorp/nomad/client/taskenv" 17 "github.com/hashicorp/nomad/client/testutil" 18 "github.com/hashicorp/nomad/helper/testlog" 19 "github.com/hashicorp/nomad/nomad/mock" 20 "github.com/hashicorp/nomad/plugins/drivers" 21 tu "github.com/hashicorp/nomad/testutil" 22 lconfigs "github.com/opencontainers/runc/libcontainer/configs" 23 "github.com/stretchr/testify/require" 24 "golang.org/x/sys/unix" 25 ) 26 27 func init() { 28 executorFactories["LibcontainerExecutor"] = libcontainerFactory 29 } 30 31 var libcontainerFactory = executorFactory{ 32 new: NewExecutorWithIsolation, 33 configureExecCmd: func(t *testing.T, cmd *ExecCommand) { 34 cmd.ResourceLimits = true 35 setupRootfs(t, cmd.TaskDir) 36 }, 37 } 38 39 // testExecutorContextWithChroot returns an ExecutorContext and AllocDir with 40 // chroot. Use testExecutorContext if you don't need a chroot. 41 // 42 // The caller is responsible for calling AllocDir.Destroy() to cleanup. 43 func testExecutorCommandWithChroot(t *testing.T) *testExecCmd { 44 chrootEnv := map[string]string{ 45 "/etc/ld.so.cache": "/etc/ld.so.cache", 46 "/etc/ld.so.conf": "/etc/ld.so.conf", 47 "/etc/ld.so.conf.d": "/etc/ld.so.conf.d", 48 "/etc/passwd": "/etc/passwd", 49 "/lib": "/lib", 50 "/lib64": "/lib64", 51 "/usr/lib": "/usr/lib", 52 "/bin/ls": "/bin/ls", 53 "/bin/cat": "/bin/cat", 54 "/bin/echo": "/bin/echo", 55 "/bin/bash": "/bin/bash", 56 "/bin/sleep": "/bin/sleep", 57 "/foobar": "/does/not/exist", 58 } 59 60 alloc := mock.Alloc() 61 task := alloc.Job.TaskGroups[0].Tasks[0] 62 taskEnv := taskenv.NewBuilder(mock.Node(), alloc, task, "global").Build() 63 64 allocDir := allocdir.NewAllocDir(testlog.HCLogger(t), filepath.Join(os.TempDir(), alloc.ID)) 65 if err := allocDir.Build(); err != nil { 66 t.Fatalf("AllocDir.Build() failed: %v", err) 67 } 68 if err := allocDir.NewTaskDir(task.Name).Build(true, chrootEnv); err != nil { 69 allocDir.Destroy() 70 t.Fatalf("allocDir.NewTaskDir(%q) failed: %v", task.Name, err) 71 } 72 td := allocDir.TaskDirs[task.Name] 73 cmd := &ExecCommand{ 74 Env: taskEnv.List(), 75 TaskDir: td.Dir, 76 Resources: &drivers.Resources{ 77 NomadResources: alloc.AllocatedResources.Tasks[task.Name], 78 }, 79 } 80 81 testCmd := &testExecCmd{ 82 command: cmd, 83 allocDir: allocDir, 84 } 85 configureTLogging(t, testCmd) 86 return testCmd 87 } 88 89 func TestExecutor_IsolationAndConstraints(t *testing.T) { 90 t.Parallel() 91 require := require.New(t) 92 testutil.ExecCompatible(t) 93 94 testExecCmd := testExecutorCommandWithChroot(t) 95 execCmd, allocDir := testExecCmd.command, testExecCmd.allocDir 96 execCmd.Cmd = "/bin/ls" 97 execCmd.Args = []string{"-F", "/", "/etc/"} 98 defer allocDir.Destroy() 99 100 execCmd.ResourceLimits = true 101 102 executor := NewExecutorWithIsolation(testlog.HCLogger(t)) 103 defer executor.Shutdown("SIGKILL", 0) 104 105 ps, err := executor.Launch(execCmd) 106 require.NoError(err) 107 require.NotZero(ps.Pid) 108 109 state, err := executor.Wait(context.Background()) 110 require.NoError(err) 111 require.Zero(state.ExitCode) 112 113 // Check if the resource constraints were applied 114 if lexec, ok := executor.(*LibcontainerExecutor); ok { 115 state, err := lexec.container.State() 116 require.NoError(err) 117 118 memLimits := filepath.Join(state.CgroupPaths["memory"], "memory.limit_in_bytes") 119 data, err := ioutil.ReadFile(memLimits) 120 require.NoError(err) 121 122 expectedMemLim := strconv.Itoa(int(execCmd.Resources.NomadResources.Memory.MemoryMB * 1024 * 1024)) 123 actualMemLim := strings.TrimSpace(string(data)) 124 require.Equal(actualMemLim, expectedMemLim) 125 require.NoError(executor.Shutdown("", 0)) 126 executor.Wait(context.Background()) 127 128 // Check if Nomad has actually removed the cgroups 129 tu.WaitForResult(func() (bool, error) { 130 _, err = os.Stat(memLimits) 131 if err == nil { 132 return false, fmt.Errorf("expected an error from os.Stat %s", memLimits) 133 } 134 return true, nil 135 }, func(err error) { t.Error(err) }) 136 137 } 138 expected := `/: 139 alloc/ 140 bin/ 141 dev/ 142 etc/ 143 lib/ 144 lib64/ 145 local/ 146 proc/ 147 secrets/ 148 sys/ 149 tmp/ 150 usr/ 151 152 /etc/: 153 ld.so.cache 154 ld.so.conf 155 ld.so.conf.d/ 156 passwd` 157 tu.WaitForResult(func() (bool, error) { 158 output := testExecCmd.stdout.String() 159 act := strings.TrimSpace(string(output)) 160 if act != expected { 161 return false, fmt.Errorf("Command output incorrectly: want %v; got %v", expected, act) 162 } 163 return true, nil 164 }, func(err error) { t.Error(err) }) 165 } 166 167 func TestUniversalExecutor_LookupTaskBin(t *testing.T) { 168 t.Parallel() 169 require := require.New(t) 170 171 // Create a temp dir 172 tmpDir, err := ioutil.TempDir("", "") 173 require.Nil(err) 174 defer os.Remove(tmpDir) 175 176 // Create the command 177 cmd := &ExecCommand{Env: []string{"PATH=/bin"}, TaskDir: tmpDir} 178 179 // Make a foo subdir 180 os.MkdirAll(filepath.Join(tmpDir, "foo"), 0700) 181 182 // Write a file under foo 183 filePath := filepath.Join(tmpDir, "foo", "tmp.txt") 184 err = ioutil.WriteFile(filePath, []byte{1, 2}, os.ModeAppend) 185 require.NoError(err) 186 187 // Lookout with an absolute path to the binary 188 cmd.Cmd = "/foo/tmp.txt" 189 _, err = lookupTaskBin(cmd) 190 require.NoError(err) 191 192 // Write a file under local subdir 193 os.MkdirAll(filepath.Join(tmpDir, "local"), 0700) 194 filePath2 := filepath.Join(tmpDir, "local", "tmp.txt") 195 ioutil.WriteFile(filePath2, []byte{1, 2}, os.ModeAppend) 196 197 // Lookup with file name, should find the one we wrote above 198 cmd.Cmd = "tmp.txt" 199 _, err = lookupTaskBin(cmd) 200 require.NoError(err) 201 202 // Lookup a host absolute path 203 cmd.Cmd = "/bin/sh" 204 _, err = lookupTaskBin(cmd) 205 require.Error(err) 206 } 207 208 // Exec Launch looks for the binary only inside the chroot 209 func TestExecutor_EscapeContainer(t *testing.T) { 210 t.Parallel() 211 require := require.New(t) 212 testutil.ExecCompatible(t) 213 214 testExecCmd := testExecutorCommandWithChroot(t) 215 execCmd, allocDir := testExecCmd.command, testExecCmd.allocDir 216 execCmd.Cmd = "/bin/kill" // missing from the chroot container 217 defer allocDir.Destroy() 218 219 execCmd.ResourceLimits = true 220 221 executor := NewExecutorWithIsolation(testlog.HCLogger(t)) 222 defer executor.Shutdown("SIGKILL", 0) 223 224 _, err := executor.Launch(execCmd) 225 require.Error(err) 226 require.Regexp("^file /bin/kill not found under path", err) 227 228 // Bare files are looked up using the system path, inside the container 229 allocDir.Destroy() 230 testExecCmd = testExecutorCommandWithChroot(t) 231 execCmd, allocDir = testExecCmd.command, testExecCmd.allocDir 232 execCmd.Cmd = "kill" 233 _, err = executor.Launch(execCmd) 234 require.Error(err) 235 require.Regexp("^file kill not found under path", err) 236 237 allocDir.Destroy() 238 testExecCmd = testExecutorCommandWithChroot(t) 239 execCmd, allocDir = testExecCmd.command, testExecCmd.allocDir 240 execCmd.Cmd = "echo" 241 _, err = executor.Launch(execCmd) 242 require.NoError(err) 243 } 244 245 func TestExecutor_Capabilities(t *testing.T) { 246 t.Parallel() 247 testutil.ExecCompatible(t) 248 249 cases := []struct { 250 user string 251 caps string 252 }{ 253 { 254 user: "nobody", 255 caps: ` 256 CapInh: 0000000000000000 257 CapPrm: 0000000000000000 258 CapEff: 0000000000000000 259 CapBnd: 0000003fffffffff 260 CapAmb: 0000000000000000`, 261 }, 262 { 263 user: "root", 264 caps: ` 265 CapInh: 0000000000000000 266 CapPrm: 0000003fffffffff 267 CapEff: 0000003fffffffff 268 CapBnd: 0000003fffffffff 269 CapAmb: 0000000000000000`, 270 }, 271 } 272 273 for _, c := range cases { 274 t.Run(c.user, func(t *testing.T) { 275 require := require.New(t) 276 277 testExecCmd := testExecutorCommandWithChroot(t) 278 execCmd, allocDir := testExecCmd.command, testExecCmd.allocDir 279 defer allocDir.Destroy() 280 281 execCmd.User = c.user 282 execCmd.ResourceLimits = true 283 execCmd.Cmd = "/bin/bash" 284 execCmd.Args = []string{"-c", "cat /proc/$$/status"} 285 286 executor := NewExecutorWithIsolation(testlog.HCLogger(t)) 287 defer executor.Shutdown("SIGKILL", 0) 288 289 _, err := executor.Launch(execCmd) 290 require.NoError(err) 291 292 ch := make(chan interface{}) 293 go func() { 294 executor.Wait(context.Background()) 295 close(ch) 296 }() 297 298 select { 299 case <-ch: 300 // all good 301 case <-time.After(5 * time.Second): 302 require.Fail("timeout waiting for exec to shutdown") 303 } 304 305 canonical := func(s string) string { 306 s = strings.TrimSpace(s) 307 s = regexp.MustCompile("[ \t]+").ReplaceAllString(s, " ") 308 s = regexp.MustCompile("[\n\r]+").ReplaceAllString(s, "\n") 309 return s 310 } 311 312 expected := canonical(c.caps) 313 tu.WaitForResult(func() (bool, error) { 314 output := canonical(testExecCmd.stdout.String()) 315 if !strings.Contains(output, expected) { 316 return false, fmt.Errorf("capabilities didn't match: want\n%v\n; got:\n%v\n", expected, output) 317 } 318 return true, nil 319 }, func(err error) { require.NoError(err) }) 320 }) 321 } 322 323 } 324 325 func TestExecutor_ClientCleanup(t *testing.T) { 326 t.Parallel() 327 testutil.ExecCompatible(t) 328 require := require.New(t) 329 330 testExecCmd := testExecutorCommandWithChroot(t) 331 execCmd, allocDir := testExecCmd.command, testExecCmd.allocDir 332 defer allocDir.Destroy() 333 334 executor := NewExecutorWithIsolation(testlog.HCLogger(t)) 335 defer executor.Shutdown("", 0) 336 337 // Need to run a command which will produce continuous output but not 338 // too quickly to ensure executor.Exit() stops the process. 339 execCmd.Cmd = "/bin/bash" 340 execCmd.Args = []string{"-c", "while true; do /bin/echo X; /bin/sleep 1; done"} 341 execCmd.ResourceLimits = true 342 343 ps, err := executor.Launch(execCmd) 344 345 require.NoError(err) 346 require.NotZero(ps.Pid) 347 time.Sleep(500 * time.Millisecond) 348 require.NoError(executor.Shutdown("SIGINT", 100*time.Millisecond)) 349 350 ch := make(chan interface{}) 351 go func() { 352 executor.Wait(context.Background()) 353 close(ch) 354 }() 355 356 select { 357 case <-ch: 358 // all good 359 case <-time.After(5 * time.Second): 360 require.Fail("timeout waiting for exec to shutdown") 361 } 362 363 output := testExecCmd.stdout.String() 364 require.NotZero(len(output)) 365 time.Sleep(2 * time.Second) 366 output1 := testExecCmd.stdout.String() 367 require.Equal(len(output), len(output1)) 368 } 369 370 func TestExecutor_cmdDevices(t *testing.T) { 371 input := []*drivers.DeviceConfig{ 372 { 373 HostPath: "/dev/null", 374 TaskPath: "/task/dev/null", 375 Permissions: "rwm", 376 }, 377 } 378 379 expected := &lconfigs.Device{ 380 Path: "/task/dev/null", 381 Type: 99, 382 Major: 1, 383 Minor: 3, 384 Permissions: "rwm", 385 } 386 387 found, err := cmdDevices(input) 388 require.NoError(t, err) 389 require.Len(t, found, 1) 390 391 // ignore file permission and ownership 392 // as they are host specific potentially 393 d := found[0] 394 d.FileMode = 0 395 d.Uid = 0 396 d.Gid = 0 397 398 require.EqualValues(t, expected, d) 399 } 400 401 func TestExecutor_cmdMounts(t *testing.T) { 402 input := []*drivers.MountConfig{ 403 { 404 HostPath: "/host/path-ro", 405 TaskPath: "/task/path-ro", 406 Readonly: true, 407 }, 408 { 409 HostPath: "/host/path-rw", 410 TaskPath: "/task/path-rw", 411 Readonly: false, 412 }, 413 } 414 415 expected := []*lconfigs.Mount{ 416 { 417 Source: "/host/path-ro", 418 Destination: "/task/path-ro", 419 Flags: unix.MS_BIND | unix.MS_RDONLY, 420 Device: "bind", 421 }, 422 { 423 Source: "/host/path-rw", 424 Destination: "/task/path-rw", 425 Flags: unix.MS_BIND, 426 Device: "bind", 427 }, 428 } 429 430 require.EqualValues(t, expected, cmdMounts(input)) 431 }