github.com/bigcommerce/nomad@v0.9.3-bc/drivers/shared/executor/executor_linux_test.go (about) 1 package executor 2 3 import ( 4 "context" 5 "fmt" 6 "io/ioutil" 7 "os" 8 "path/filepath" 9 "regexp" 10 "strconv" 11 "strings" 12 "testing" 13 "time" 14 15 "github.com/hashicorp/nomad/client/allocdir" 16 "github.com/hashicorp/nomad/client/taskenv" 17 "github.com/hashicorp/nomad/client/testutil" 18 "github.com/hashicorp/nomad/helper/testlog" 19 "github.com/hashicorp/nomad/nomad/mock" 20 "github.com/hashicorp/nomad/plugins/drivers" 21 tu "github.com/hashicorp/nomad/testutil" 22 lconfigs "github.com/opencontainers/runc/libcontainer/configs" 23 "github.com/stretchr/testify/require" 24 "golang.org/x/sys/unix" 25 ) 26 27 func init() { 28 executorFactories["LibcontainerExecutor"] = libcontainerFactory 29 } 30 31 var libcontainerFactory = executorFactory{ 32 new: NewExecutorWithIsolation, 33 configureExecCmd: func(t *testing.T, cmd *ExecCommand) { 34 cmd.ResourceLimits = true 35 setupRootfs(t, cmd.TaskDir) 36 }, 37 } 38 39 // testExecutorContextWithChroot returns an ExecutorContext and AllocDir with 40 // chroot. Use testExecutorContext if you don't need a chroot. 41 // 42 // The caller is responsible for calling AllocDir.Destroy() to cleanup. 43 func testExecutorCommandWithChroot(t *testing.T) *testExecCmd { 44 chrootEnv := map[string]string{ 45 "/etc/ld.so.cache": "/etc/ld.so.cache", 46 "/etc/ld.so.conf": "/etc/ld.so.conf", 47 "/etc/ld.so.conf.d": "/etc/ld.so.conf.d", 48 "/etc/passwd": "/etc/passwd", 49 "/lib": "/lib", 50 "/lib64": "/lib64", 51 "/usr/lib": "/usr/lib", 52 "/bin/ls": "/bin/ls", 53 "/bin/cat": "/bin/cat", 54 "/bin/echo": "/bin/echo", 55 "/bin/bash": "/bin/bash", 56 "/bin/sleep": "/bin/sleep", 57 "/foobar": "/does/not/exist", 58 } 59 60 alloc := mock.Alloc() 61 task := alloc.Job.TaskGroups[0].Tasks[0] 62 taskEnv := taskenv.NewBuilder(mock.Node(), alloc, task, "global").Build() 63 64 allocDir := allocdir.NewAllocDir(testlog.HCLogger(t), filepath.Join(os.TempDir(), alloc.ID)) 65 if err := allocDir.Build(); err != nil { 66 t.Fatalf("AllocDir.Build() failed: %v", err) 67 } 68 if err := allocDir.NewTaskDir(task.Name).Build(true, chrootEnv); err != nil { 69 allocDir.Destroy() 70 t.Fatalf("allocDir.NewTaskDir(%q) failed: %v", task.Name, err) 71 } 72 td := allocDir.TaskDirs[task.Name] 73 cmd := &ExecCommand{ 74 Env: taskEnv.List(), 75 TaskDir: td.Dir, 76 Resources: &drivers.Resources{ 77 NomadResources: alloc.AllocatedResources.Tasks[task.Name], 78 }, 79 } 80 81 testCmd := &testExecCmd{ 82 command: cmd, 83 allocDir: allocDir, 84 } 85 configureTLogging(t, testCmd) 86 return testCmd 87 } 88 89 func TestExecutor_IsolationAndConstraints(t *testing.T) { 90 t.Parallel() 91 require := require.New(t) 92 testutil.ExecCompatible(t) 93 94 testExecCmd := testExecutorCommandWithChroot(t) 95 execCmd, allocDir := testExecCmd.command, testExecCmd.allocDir 96 execCmd.Cmd = "/bin/ls" 97 execCmd.Args = []string{"-F", "/", "/etc/"} 98 defer allocDir.Destroy() 99 100 execCmd.ResourceLimits = true 101 102 executor := NewExecutorWithIsolation(testlog.HCLogger(t)) 103 defer executor.Shutdown("SIGKILL", 0) 104 105 ps, err := executor.Launch(execCmd) 106 require.NoError(err) 107 require.NotZero(ps.Pid) 108 109 state, err := executor.Wait(context.Background()) 110 require.NoError(err) 111 require.Zero(state.ExitCode) 112 113 // Check if the resource constraints were applied 114 if lexec, ok := executor.(*LibcontainerExecutor); ok { 115 state, err := lexec.container.State() 116 require.NoError(err) 117 118 memLimits := filepath.Join(state.CgroupPaths["memory"], "memory.limit_in_bytes") 119 data, err := ioutil.ReadFile(memLimits) 120 require.NoError(err) 121 122 expectedMemLim := strconv.Itoa(int(execCmd.Resources.NomadResources.Memory.MemoryMB * 1024 * 1024)) 123 actualMemLim := strings.TrimSpace(string(data)) 124 require.Equal(actualMemLim, expectedMemLim) 125 require.NoError(executor.Shutdown("", 0)) 126 executor.Wait(context.Background()) 127 128 // Check if Nomad has actually removed the cgroups 129 tu.WaitForResult(func() (bool, error) { 130 _, err = os.Stat(memLimits) 131 if err == nil { 132 return false, fmt.Errorf("expected an error from os.Stat %s", memLimits) 133 } 134 return true, nil 135 }, func(err error) { t.Error(err) }) 136 137 } 138 expected := `/: 139 alloc/ 140 bin/ 141 dev/ 142 etc/ 143 lib/ 144 lib64/ 145 local/ 146 proc/ 147 secrets/ 148 sys/ 149 tmp/ 150 usr/ 151 152 /etc/: 153 ld.so.cache 154 ld.so.conf 155 ld.so.conf.d/ 156 passwd` 157 tu.WaitForResult(func() (bool, error) { 158 output := testExecCmd.stdout.String() 159 act := strings.TrimSpace(string(output)) 160 if act != expected { 161 return false, fmt.Errorf("Command output incorrectly: want %v; got %v", expected, act) 162 } 163 return true, nil 164 }, func(err error) { t.Error(err) }) 165 } 166 167 // TestExecutor_CgroupPaths asserts that process starts with independent cgroups 168 // hierarchy created for this process 169 func TestExecutor_CgroupPaths(t *testing.T) { 170 t.Parallel() 171 require := require.New(t) 172 testutil.ExecCompatible(t) 173 174 testExecCmd := testExecutorCommandWithChroot(t) 175 execCmd, allocDir := testExecCmd.command, testExecCmd.allocDir 176 execCmd.Cmd = "/bin/bash" 177 execCmd.Args = []string{"-c", "sleep 0.2; cat /proc/self/cgroup"} 178 defer allocDir.Destroy() 179 180 execCmd.ResourceLimits = true 181 182 executor := NewExecutorWithIsolation(testlog.HCLogger(t)) 183 defer executor.Shutdown("SIGKILL", 0) 184 185 ps, err := executor.Launch(execCmd) 186 require.NoError(err) 187 require.NotZero(ps.Pid) 188 189 state, err := executor.Wait(context.Background()) 190 require.NoError(err) 191 require.Zero(state.ExitCode) 192 193 tu.WaitForResult(func() (bool, error) { 194 output := strings.TrimSpace(testExecCmd.stdout.String()) 195 // sanity check that we got some cgroups 196 if !strings.Contains(output, ":devices:") { 197 return false, fmt.Errorf("was expected cgroup files but found:\n%v", output) 198 } 199 lines := strings.Split(output, "\n") 200 for _, line := range lines { 201 // Every cgroup entry should be /nomad/$ALLOC_ID 202 if line == "" { 203 continue 204 } 205 206 // Skip rdma subsystem; rdma was added in most recent kernels and libcontainer/docker 207 // don't isolate it by default. 208 if strings.Contains(line, ":rdma:") { 209 continue 210 } 211 212 if !strings.Contains(line, ":/nomad/") { 213 return false, fmt.Errorf("Not a member of the alloc's cgroup: expected=...:/nomad/... -- found=%q", line) 214 } 215 } 216 return true, nil 217 }, func(err error) { t.Error(err) }) 218 } 219 220 func TestUniversalExecutor_LookupTaskBin(t *testing.T) { 221 t.Parallel() 222 require := require.New(t) 223 224 // Create a temp dir 225 tmpDir, err := ioutil.TempDir("", "") 226 require.Nil(err) 227 defer os.Remove(tmpDir) 228 229 // Create the command 230 cmd := &ExecCommand{Env: []string{"PATH=/bin"}, TaskDir: tmpDir} 231 232 // Make a foo subdir 233 os.MkdirAll(filepath.Join(tmpDir, "foo"), 0700) 234 235 // Write a file under foo 236 filePath := filepath.Join(tmpDir, "foo", "tmp.txt") 237 err = ioutil.WriteFile(filePath, []byte{1, 2}, os.ModeAppend) 238 require.NoError(err) 239 240 // Lookout with an absolute path to the binary 241 cmd.Cmd = "/foo/tmp.txt" 242 _, err = lookupTaskBin(cmd) 243 require.NoError(err) 244 245 // Write a file under local subdir 246 os.MkdirAll(filepath.Join(tmpDir, "local"), 0700) 247 filePath2 := filepath.Join(tmpDir, "local", "tmp.txt") 248 ioutil.WriteFile(filePath2, []byte{1, 2}, os.ModeAppend) 249 250 // Lookup with file name, should find the one we wrote above 251 cmd.Cmd = "tmp.txt" 252 _, err = lookupTaskBin(cmd) 253 require.NoError(err) 254 255 // Lookup a host absolute path 256 cmd.Cmd = "/bin/sh" 257 _, err = lookupTaskBin(cmd) 258 require.Error(err) 259 } 260 261 // Exec Launch looks for the binary only inside the chroot 262 func TestExecutor_EscapeContainer(t *testing.T) { 263 t.Parallel() 264 require := require.New(t) 265 testutil.ExecCompatible(t) 266 267 testExecCmd := testExecutorCommandWithChroot(t) 268 execCmd, allocDir := testExecCmd.command, testExecCmd.allocDir 269 execCmd.Cmd = "/bin/kill" // missing from the chroot container 270 defer allocDir.Destroy() 271 272 execCmd.ResourceLimits = true 273 274 executor := NewExecutorWithIsolation(testlog.HCLogger(t)) 275 defer executor.Shutdown("SIGKILL", 0) 276 277 _, err := executor.Launch(execCmd) 278 require.Error(err) 279 require.Regexp("^file /bin/kill not found under path", err) 280 281 // Bare files are looked up using the system path, inside the container 282 allocDir.Destroy() 283 testExecCmd = testExecutorCommandWithChroot(t) 284 execCmd, allocDir = testExecCmd.command, testExecCmd.allocDir 285 execCmd.Cmd = "kill" 286 _, err = executor.Launch(execCmd) 287 require.Error(err) 288 require.Regexp("^file kill not found under path", err) 289 290 allocDir.Destroy() 291 testExecCmd = testExecutorCommandWithChroot(t) 292 execCmd, allocDir = testExecCmd.command, testExecCmd.allocDir 293 execCmd.Cmd = "echo" 294 _, err = executor.Launch(execCmd) 295 require.NoError(err) 296 } 297 298 func TestExecutor_Capabilities(t *testing.T) { 299 t.Parallel() 300 testutil.ExecCompatible(t) 301 302 cases := []struct { 303 user string 304 caps string 305 }{ 306 { 307 user: "nobody", 308 caps: ` 309 CapInh: 0000000000000000 310 CapPrm: 0000000000000000 311 CapEff: 0000000000000000 312 CapBnd: 0000003fffffffff 313 CapAmb: 0000000000000000`, 314 }, 315 { 316 user: "root", 317 caps: ` 318 CapInh: 0000000000000000 319 CapPrm: 0000003fffffffff 320 CapEff: 0000003fffffffff 321 CapBnd: 0000003fffffffff 322 CapAmb: 0000000000000000`, 323 }, 324 } 325 326 for _, c := range cases { 327 t.Run(c.user, func(t *testing.T) { 328 require := require.New(t) 329 330 testExecCmd := testExecutorCommandWithChroot(t) 331 execCmd, allocDir := testExecCmd.command, testExecCmd.allocDir 332 defer allocDir.Destroy() 333 334 execCmd.User = c.user 335 execCmd.ResourceLimits = true 336 execCmd.Cmd = "/bin/bash" 337 execCmd.Args = []string{"-c", "cat /proc/$$/status"} 338 339 executor := NewExecutorWithIsolation(testlog.HCLogger(t)) 340 defer executor.Shutdown("SIGKILL", 0) 341 342 _, err := executor.Launch(execCmd) 343 require.NoError(err) 344 345 ch := make(chan interface{}) 346 go func() { 347 executor.Wait(context.Background()) 348 close(ch) 349 }() 350 351 select { 352 case <-ch: 353 // all good 354 case <-time.After(5 * time.Second): 355 require.Fail("timeout waiting for exec to shutdown") 356 } 357 358 canonical := func(s string) string { 359 s = strings.TrimSpace(s) 360 s = regexp.MustCompile("[ \t]+").ReplaceAllString(s, " ") 361 s = regexp.MustCompile("[\n\r]+").ReplaceAllString(s, "\n") 362 return s 363 } 364 365 expected := canonical(c.caps) 366 tu.WaitForResult(func() (bool, error) { 367 output := canonical(testExecCmd.stdout.String()) 368 if !strings.Contains(output, expected) { 369 return false, fmt.Errorf("capabilities didn't match: want\n%v\n; got:\n%v\n", expected, output) 370 } 371 return true, nil 372 }, func(err error) { require.NoError(err) }) 373 }) 374 } 375 376 } 377 378 func TestExecutor_ClientCleanup(t *testing.T) { 379 t.Parallel() 380 testutil.ExecCompatible(t) 381 require := require.New(t) 382 383 testExecCmd := testExecutorCommandWithChroot(t) 384 execCmd, allocDir := testExecCmd.command, testExecCmd.allocDir 385 defer allocDir.Destroy() 386 387 executor := NewExecutorWithIsolation(testlog.HCLogger(t)) 388 defer executor.Shutdown("", 0) 389 390 // Need to run a command which will produce continuous output but not 391 // too quickly to ensure executor.Exit() stops the process. 392 execCmd.Cmd = "/bin/bash" 393 execCmd.Args = []string{"-c", "while true; do /bin/echo X; /bin/sleep 1; done"} 394 execCmd.ResourceLimits = true 395 396 ps, err := executor.Launch(execCmd) 397 398 require.NoError(err) 399 require.NotZero(ps.Pid) 400 time.Sleep(500 * time.Millisecond) 401 require.NoError(executor.Shutdown("SIGINT", 100*time.Millisecond)) 402 403 ch := make(chan interface{}) 404 go func() { 405 executor.Wait(context.Background()) 406 close(ch) 407 }() 408 409 select { 410 case <-ch: 411 // all good 412 case <-time.After(5 * time.Second): 413 require.Fail("timeout waiting for exec to shutdown") 414 } 415 416 output := testExecCmd.stdout.String() 417 require.NotZero(len(output)) 418 time.Sleep(2 * time.Second) 419 output1 := testExecCmd.stdout.String() 420 require.Equal(len(output), len(output1)) 421 } 422 423 func TestExecutor_cmdDevices(t *testing.T) { 424 input := []*drivers.DeviceConfig{ 425 { 426 HostPath: "/dev/null", 427 TaskPath: "/task/dev/null", 428 Permissions: "rwm", 429 }, 430 } 431 432 expected := &lconfigs.Device{ 433 Path: "/task/dev/null", 434 Type: 99, 435 Major: 1, 436 Minor: 3, 437 Permissions: "rwm", 438 } 439 440 found, err := cmdDevices(input) 441 require.NoError(t, err) 442 require.Len(t, found, 1) 443 444 // ignore file permission and ownership 445 // as they are host specific potentially 446 d := found[0] 447 d.FileMode = 0 448 d.Uid = 0 449 d.Gid = 0 450 451 require.EqualValues(t, expected, d) 452 } 453 454 func TestExecutor_cmdMounts(t *testing.T) { 455 input := []*drivers.MountConfig{ 456 { 457 HostPath: "/host/path-ro", 458 TaskPath: "/task/path-ro", 459 Readonly: true, 460 }, 461 { 462 HostPath: "/host/path-rw", 463 TaskPath: "/task/path-rw", 464 Readonly: false, 465 }, 466 } 467 468 expected := []*lconfigs.Mount{ 469 { 470 Source: "/host/path-ro", 471 Destination: "/task/path-ro", 472 Flags: unix.MS_BIND | unix.MS_RDONLY, 473 Device: "bind", 474 }, 475 { 476 Source: "/host/path-rw", 477 Destination: "/task/path-rw", 478 Flags: unix.MS_BIND, 479 Device: "bind", 480 }, 481 } 482 483 require.EqualValues(t, expected, cmdMounts(input)) 484 }