github.com/anth0d/nomad@v0.0.0-20221214183521-ae3a0a2cad06/drivers/exec/driver_test.go (about) 1 package exec 2 3 import ( 4 "bytes" 5 "context" 6 "errors" 7 "fmt" 8 "io/ioutil" 9 "os" 10 "path/filepath" 11 "runtime" 12 "strconv" 13 "strings" 14 "sync" 15 "syscall" 16 "testing" 17 "time" 18 19 "github.com/hashicorp/nomad/ci" 20 "github.com/hashicorp/nomad/client/lib/cgutil" 21 ctestutils "github.com/hashicorp/nomad/client/testutil" 22 "github.com/hashicorp/nomad/drivers/shared/executor" 23 "github.com/hashicorp/nomad/helper/pluginutils/hclutils" 24 "github.com/hashicorp/nomad/helper/testlog" 25 "github.com/hashicorp/nomad/helper/testtask" 26 "github.com/hashicorp/nomad/helper/uuid" 27 "github.com/hashicorp/nomad/nomad/structs" 28 basePlug "github.com/hashicorp/nomad/plugins/base" 29 "github.com/hashicorp/nomad/plugins/drivers" 30 dtestutil "github.com/hashicorp/nomad/plugins/drivers/testutils" 31 "github.com/hashicorp/nomad/testutil" 32 "github.com/stretchr/testify/require" 33 ) 34 35 const ( 36 cgroupParent = "testing.slice" 37 ) 38 39 func TestMain(m *testing.M) { 40 if !testtask.Run() { 41 os.Exit(m.Run()) 42 } 43 } 44 45 func testResources(allocID, task string) *drivers.Resources { 46 if allocID == "" || task == "" { 47 panic("must be set") 48 } 49 50 r := &drivers.Resources{ 51 NomadResources: &structs.AllocatedTaskResources{ 52 Memory: structs.AllocatedMemoryResources{ 53 MemoryMB: 128, 54 }, 55 Cpu: structs.AllocatedCpuResources{ 56 CpuShares: 100, 57 }, 58 }, 59 LinuxResources: &drivers.LinuxResources{ 60 MemoryLimitBytes: 134217728, 61 CPUShares: 100, 62 }, 63 } 64 65 if cgutil.UseV2 { 66 r.LinuxResources.CpusetCgroupPath = filepath.Join(cgutil.CgroupRoot, cgroupParent, cgutil.CgroupScope(allocID, task)) 67 } 68 69 return r 70 } 71 72 func TestExecDriver_Fingerprint_NonLinux(t *testing.T) { 73 ci.Parallel(t) 74 require := require.New(t) 75 if runtime.GOOS == "linux" { 76 t.Skip("Test only available not on Linux") 77 } 78 79 ctx, cancel := context.WithCancel(context.Background()) 80 defer cancel() 81 82 d := NewExecDriver(ctx, testlog.HCLogger(t)) 83 harness := dtestutil.NewDriverHarness(t, d) 84 85 fingerCh, err := harness.Fingerprint(context.Background()) 86 require.NoError(err) 87 select { 88 case finger := <-fingerCh: 89 require.Equal(drivers.HealthStateUndetected, finger.Health) 90 case <-time.After(time.Duration(testutil.TestMultiplier()*5) * time.Second): 91 require.Fail("timeout receiving fingerprint") 92 } 93 } 94 95 func TestExecDriver_Fingerprint(t *testing.T) { 96 ci.Parallel(t) 97 require := require.New(t) 98 99 ctestutils.ExecCompatible(t) 100 101 ctx, cancel := context.WithCancel(context.Background()) 102 defer cancel() 103 104 d := NewExecDriver(ctx, testlog.HCLogger(t)) 105 harness := dtestutil.NewDriverHarness(t, d) 106 107 fingerCh, err := harness.Fingerprint(context.Background()) 108 require.NoError(err) 109 select { 110 case finger := <-fingerCh: 111 require.Equal(drivers.HealthStateHealthy, finger.Health) 112 require.True(finger.Attributes["driver.exec"].GetBool()) 113 case <-time.After(time.Duration(testutil.TestMultiplier()*5) * time.Second): 114 require.Fail("timeout receiving fingerprint") 115 } 116 } 117 118 func TestExecDriver_StartWait(t *testing.T) { 119 ci.Parallel(t) 120 ctestutils.ExecCompatible(t) 121 122 ctx, cancel := context.WithCancel(context.Background()) 123 defer cancel() 124 125 logger := testlog.HCLogger(t) 126 127 d := NewExecDriver(ctx, logger) 128 harness := dtestutil.NewDriverHarness(t, d) 129 allocID := uuid.Generate() 130 task := &drivers.TaskConfig{ 131 AllocID: allocID, 132 ID: uuid.Generate(), 133 Name: "test", 134 Resources: testResources(allocID, "test"), 135 } 136 137 tc := &TaskConfig{ 138 Command: "cat", 139 Args: []string{"/proc/self/cgroup"}, 140 } 141 require.NoError(t, task.EncodeConcreteDriverConfig(&tc)) 142 143 cleanup := harness.MkAllocDir(task, false) 144 defer cleanup() 145 146 handle, _, err := harness.StartTask(task) 147 require.NoError(t, err) 148 149 ch, err := harness.WaitTask(context.Background(), handle.Config.ID) 150 require.NoError(t, err) 151 result := <-ch 152 require.Zero(t, result.ExitCode) 153 require.NoError(t, harness.DestroyTask(task.ID, true)) 154 } 155 156 func TestExecDriver_StartWaitStopKill(t *testing.T) { 157 ci.Parallel(t) 158 ctestutils.ExecCompatible(t) 159 160 ctx, cancel := context.WithCancel(context.Background()) 161 defer cancel() 162 163 d := NewExecDriver(ctx, testlog.HCLogger(t)) 164 harness := dtestutil.NewDriverHarness(t, d) 165 allocID := uuid.Generate() 166 task := &drivers.TaskConfig{ 167 AllocID: allocID, 168 ID: uuid.Generate(), 169 Name: "test", 170 Resources: testResources(allocID, "test"), 171 } 172 173 tc := &TaskConfig{ 174 Command: "/bin/bash", 175 Args: []string{"-c", "echo hi; sleep 600"}, 176 } 177 require.NoError(t, task.EncodeConcreteDriverConfig(&tc)) 178 179 cleanup := harness.MkAllocDir(task, false) 180 defer cleanup() 181 182 handle, _, err := harness.StartTask(task) 183 require.NoError(t, err) 184 defer harness.DestroyTask(task.ID, true) 185 186 ch, err := harness.WaitTask(context.Background(), handle.Config.ID) 187 require.NoError(t, err) 188 189 require.NoError(t, harness.WaitUntilStarted(task.ID, 1*time.Second)) 190 191 go func() { 192 harness.StopTask(task.ID, 2*time.Second, "SIGINT") 193 }() 194 195 select { 196 case result := <-ch: 197 require.False(t, result.Successful()) 198 case <-time.After(10 * time.Second): 199 require.Fail(t, "timeout waiting for task to shutdown") 200 } 201 202 // Ensure that the task is marked as dead, but account 203 // for WaitTask() closing channel before internal state is updated 204 testutil.WaitForResult(func() (bool, error) { 205 status, err := harness.InspectTask(task.ID) 206 if err != nil { 207 return false, fmt.Errorf("inspecting task failed: %v", err) 208 } 209 if status.State != drivers.TaskStateExited { 210 return false, fmt.Errorf("task hasn't exited yet; status: %v", status.State) 211 } 212 213 return true, nil 214 }, func(err error) { 215 require.NoError(t, err) 216 }) 217 218 require.NoError(t, harness.DestroyTask(task.ID, true)) 219 } 220 221 func TestExecDriver_StartWaitRecover(t *testing.T) { 222 ci.Parallel(t) 223 ctestutils.ExecCompatible(t) 224 225 dCtx, dCancel := context.WithCancel(context.Background()) 226 defer dCancel() 227 228 d := NewExecDriver(dCtx, testlog.HCLogger(t)) 229 harness := dtestutil.NewDriverHarness(t, d) 230 allocID := uuid.Generate() 231 task := &drivers.TaskConfig{ 232 AllocID: allocID, 233 ID: uuid.Generate(), 234 Name: "test", 235 Resources: testResources(allocID, "test"), 236 } 237 238 tc := &TaskConfig{ 239 Command: "/bin/sleep", 240 Args: []string{"5"}, 241 } 242 require.NoError(t, task.EncodeConcreteDriverConfig(&tc)) 243 244 cleanup := harness.MkAllocDir(task, false) 245 defer cleanup() 246 247 handle, _, err := harness.StartTask(task) 248 require.NoError(t, err) 249 250 ctx, cancel := context.WithCancel(context.Background()) 251 252 ch, err := harness.WaitTask(ctx, handle.Config.ID) 253 require.NoError(t, err) 254 255 var wg sync.WaitGroup 256 wg.Add(1) 257 go func() { 258 defer wg.Done() 259 result := <-ch 260 require.Error(t, result.Err) 261 }() 262 263 require.NoError(t, harness.WaitUntilStarted(task.ID, 1*time.Second)) 264 cancel() 265 266 waitCh := make(chan struct{}) 267 go func() { 268 defer close(waitCh) 269 wg.Wait() 270 }() 271 272 select { 273 case <-waitCh: 274 status, err := harness.InspectTask(task.ID) 275 require.NoError(t, err) 276 require.Equal(t, drivers.TaskStateRunning, status.State) 277 case <-time.After(1 * time.Second): 278 require.Fail(t, "timeout waiting for task wait to cancel") 279 } 280 281 // Loose task 282 d.(*Driver).tasks.Delete(task.ID) 283 _, err = harness.InspectTask(task.ID) 284 require.Error(t, err) 285 286 require.NoError(t, harness.RecoverTask(handle)) 287 status, err := harness.InspectTask(task.ID) 288 require.NoError(t, err) 289 require.Equal(t, drivers.TaskStateRunning, status.State) 290 291 require.NoError(t, harness.StopTask(task.ID, 0, "")) 292 require.NoError(t, harness.DestroyTask(task.ID, true)) 293 } 294 295 // TestExecDriver_NoOrphans asserts that when the main 296 // task dies, the orphans in the PID namespaces are killed by the kernel 297 func TestExecDriver_NoOrphans(t *testing.T) { 298 ci.Parallel(t) 299 ctestutils.ExecCompatible(t) 300 301 ctx, cancel := context.WithCancel(context.Background()) 302 defer cancel() 303 304 d := NewExecDriver(ctx, testlog.HCLogger(t)) 305 harness := dtestutil.NewDriverHarness(t, d) 306 defer harness.Kill() 307 308 config := &Config{ 309 NoPivotRoot: false, 310 DefaultModePID: executor.IsolationModePrivate, 311 DefaultModeIPC: executor.IsolationModePrivate, 312 } 313 314 var data []byte 315 require.NoError(t, basePlug.MsgPackEncode(&data, config)) 316 baseConfig := &basePlug.Config{PluginConfig: data} 317 require.NoError(t, harness.SetConfig(baseConfig)) 318 319 allocID := uuid.Generate() 320 task := &drivers.TaskConfig{ 321 AllocID: allocID, 322 ID: uuid.Generate(), 323 Name: "test", 324 } 325 326 if cgutil.UseV2 { 327 task.Resources = testResources(allocID, "test") 328 } 329 330 cleanup := harness.MkAllocDir(task, true) 331 defer cleanup() 332 333 taskConfig := map[string]interface{}{} 334 taskConfig["command"] = "/bin/sh" 335 // print the child PID in the task PID namespace, then sleep for 5 seconds to give us a chance to examine processes 336 taskConfig["args"] = []string{"-c", fmt.Sprintf(`sleep 3600 & sleep 20`)} 337 require.NoError(t, task.EncodeConcreteDriverConfig(&taskConfig)) 338 339 handle, _, err := harness.StartTask(task) 340 require.NoError(t, err) 341 defer harness.DestroyTask(task.ID, true) 342 343 waitCh, err := harness.WaitTask(context.Background(), handle.Config.ID) 344 require.NoError(t, err) 345 346 require.NoError(t, harness.WaitUntilStarted(task.ID, 1*time.Second)) 347 348 var childPids []int 349 taskState := TaskState{} 350 testutil.WaitForResult(func() (bool, error) { 351 require.NoError(t, handle.GetDriverState(&taskState)) 352 if taskState.Pid == 0 { 353 return false, fmt.Errorf("task PID is zero") 354 } 355 356 children, err := ioutil.ReadFile(fmt.Sprintf("/proc/%d/task/%d/children", taskState.Pid, taskState.Pid)) 357 if err != nil { 358 return false, fmt.Errorf("error reading /proc for children: %v", err) 359 } 360 pids := strings.Fields(string(children)) 361 if len(pids) < 2 { 362 return false, fmt.Errorf("error waiting for two children, currently %d", len(pids)) 363 } 364 for _, cpid := range pids { 365 p, err := strconv.Atoi(cpid) 366 if err != nil { 367 return false, fmt.Errorf("error parsing child pids from /proc: %s", cpid) 368 } 369 childPids = append(childPids, p) 370 } 371 return true, nil 372 }, func(err error) { 373 require.NoError(t, err) 374 }) 375 376 select { 377 case result := <-waitCh: 378 require.True(t, result.Successful(), "command failed: %#v", result) 379 case <-time.After(30 * time.Second): 380 require.Fail(t, "timeout waiting for task to shutdown") 381 } 382 383 // isProcessRunning returns an error if process is not running 384 isProcessRunning := func(pid int) error { 385 process, err := os.FindProcess(pid) 386 if err != nil { 387 return fmt.Errorf("failed to find process: %s", err) 388 } 389 390 err = process.Signal(syscall.Signal(0)) 391 if err != nil { 392 return fmt.Errorf("failed to signal process: %s", err) 393 } 394 395 return nil 396 } 397 398 // task should be dead 399 require.Error(t, isProcessRunning(taskState.Pid)) 400 401 // all children should eventually be killed by OS 402 testutil.WaitForResult(func() (bool, error) { 403 for _, cpid := range childPids { 404 err := isProcessRunning(cpid) 405 if err == nil { 406 return false, fmt.Errorf("child process %d is still running", cpid) 407 } 408 if !strings.Contains(err.Error(), "failed to signal process") { 409 return false, fmt.Errorf("unexpected error: %v", err) 410 } 411 } 412 return true, nil 413 }, func(err error) { 414 require.NoError(t, err) 415 }) 416 } 417 418 func TestExecDriver_Stats(t *testing.T) { 419 ci.Parallel(t) 420 ctestutils.ExecCompatible(t) 421 422 dctx, dcancel := context.WithCancel(context.Background()) 423 defer dcancel() 424 425 d := NewExecDriver(dctx, testlog.HCLogger(t)) 426 harness := dtestutil.NewDriverHarness(t, d) 427 428 allocID := uuid.Generate() 429 task := &drivers.TaskConfig{ 430 AllocID: allocID, 431 ID: uuid.Generate(), 432 Name: "test", 433 Resources: testResources(allocID, "test"), 434 } 435 436 tc := &TaskConfig{ 437 Command: "/bin/sleep", 438 Args: []string{"5"}, 439 } 440 require.NoError(t, task.EncodeConcreteDriverConfig(&tc)) 441 442 cleanup := harness.MkAllocDir(task, false) 443 defer cleanup() 444 445 handle, _, err := harness.StartTask(task) 446 require.NoError(t, err) 447 require.NotNil(t, handle) 448 449 require.NoError(t, harness.WaitUntilStarted(task.ID, 1*time.Second)) 450 ctx, cancel := context.WithCancel(context.Background()) 451 defer cancel() 452 statsCh, err := harness.TaskStats(ctx, task.ID, time.Second*10) 453 require.NoError(t, err) 454 select { 455 case stats := <-statsCh: 456 require.NotEmpty(t, stats.ResourceUsage.MemoryStats.Measured) 457 require.NotZero(t, stats.Timestamp) 458 require.WithinDuration(t, time.Now(), time.Unix(0, stats.Timestamp), time.Second) 459 case <-time.After(time.Second): 460 require.Fail(t, "timeout receiving from channel") 461 } 462 463 require.NoError(t, harness.DestroyTask(task.ID, true)) 464 } 465 466 func TestExecDriver_Start_Wait_AllocDir(t *testing.T) { 467 ci.Parallel(t) 468 ctestutils.ExecCompatible(t) 469 470 ctx, cancel := context.WithCancel(context.Background()) 471 defer cancel() 472 473 d := NewExecDriver(ctx, testlog.HCLogger(t)) 474 harness := dtestutil.NewDriverHarness(t, d) 475 allocID := uuid.Generate() 476 task := &drivers.TaskConfig{ 477 AllocID: allocID, 478 ID: uuid.Generate(), 479 Name: "sleep", 480 Resources: testResources(allocID, "test"), 481 } 482 cleanup := harness.MkAllocDir(task, false) 483 defer cleanup() 484 485 exp := []byte{'w', 'i', 'n'} 486 file := "output.txt" 487 tc := &TaskConfig{ 488 Command: "/bin/bash", 489 Args: []string{ 490 "-c", 491 fmt.Sprintf(`sleep 1; echo -n %s > /alloc/%s`, string(exp), file), 492 }, 493 } 494 require.NoError(t, task.EncodeConcreteDriverConfig(&tc)) 495 496 handle, _, err := harness.StartTask(task) 497 require.NoError(t, err) 498 require.NotNil(t, handle) 499 500 // Task should terminate quickly 501 waitCh, err := harness.WaitTask(context.Background(), task.ID) 502 require.NoError(t, err) 503 select { 504 case res := <-waitCh: 505 require.True(t, res.Successful(), "task should have exited successfully: %v", res) 506 case <-time.After(time.Duration(testutil.TestMultiplier()*5) * time.Second): 507 require.Fail(t, "timeout waiting for task") 508 } 509 510 // Check that data was written to the shared alloc directory. 511 outputFile := filepath.Join(task.TaskDir().SharedAllocDir, file) 512 act, err := ioutil.ReadFile(outputFile) 513 require.NoError(t, err) 514 require.Exactly(t, exp, act) 515 516 require.NoError(t, harness.DestroyTask(task.ID, true)) 517 } 518 519 func TestExecDriver_User(t *testing.T) { 520 ci.Parallel(t) 521 ctestutils.ExecCompatible(t) 522 523 ctx, cancel := context.WithCancel(context.Background()) 524 defer cancel() 525 526 d := NewExecDriver(ctx, testlog.HCLogger(t)) 527 harness := dtestutil.NewDriverHarness(t, d) 528 allocID := uuid.Generate() 529 task := &drivers.TaskConfig{ 530 AllocID: allocID, 531 ID: uuid.Generate(), 532 Name: "sleep", 533 User: "alice", 534 Resources: testResources(allocID, "sleep"), 535 } 536 cleanup := harness.MkAllocDir(task, false) 537 defer cleanup() 538 539 tc := &TaskConfig{ 540 Command: "/bin/sleep", 541 Args: []string{"100"}, 542 } 543 require.NoError(t, task.EncodeConcreteDriverConfig(&tc)) 544 545 handle, _, err := harness.StartTask(task) 546 require.Error(t, err) 547 require.Nil(t, handle) 548 549 msg := "user alice" 550 if !strings.Contains(err.Error(), msg) { 551 t.Fatalf("Expecting '%v' in '%v'", msg, err) 552 } 553 } 554 555 // TestExecDriver_HandlerExec ensures the exec driver's handle properly 556 // executes commands inside the container. 557 func TestExecDriver_HandlerExec(t *testing.T) { 558 ci.Parallel(t) 559 ctestutils.ExecCompatible(t) 560 561 ctx, cancel := context.WithCancel(context.Background()) 562 defer cancel() 563 564 d := NewExecDriver(ctx, testlog.HCLogger(t)) 565 harness := dtestutil.NewDriverHarness(t, d) 566 allocID := uuid.Generate() 567 task := &drivers.TaskConfig{ 568 AllocID: allocID, 569 ID: uuid.Generate(), 570 Name: "sleep", 571 Resources: testResources(allocID, "sleep"), 572 } 573 cleanup := harness.MkAllocDir(task, false) 574 defer cleanup() 575 576 tc := &TaskConfig{ 577 Command: "/bin/sleep", 578 Args: []string{"9000"}, 579 } 580 require.NoError(t, task.EncodeConcreteDriverConfig(&tc)) 581 582 handle, _, err := harness.StartTask(task) 583 require.NoError(t, err) 584 require.NotNil(t, handle) 585 586 // Assert cgroup membership 587 res, err := harness.ExecTask(task.ID, []string{"/bin/cat", "/proc/self/cgroup"}, time.Second) 588 require.NoError(t, err) 589 require.True(t, res.ExitResult.Successful()) 590 stdout := strings.TrimSpace(string(res.Stdout)) 591 if !cgutil.UseV2 { 592 for _, line := range strings.Split(stdout, "\n") { 593 // skip empty lines 594 if line == "" { 595 continue 596 } 597 // skip rdma & misc subsystems 598 if strings.Contains(line, ":rdma:") || strings.Contains(line, ":misc:") || strings.Contains(line, "::") { 599 continue 600 } 601 // assert we are in a nomad cgroup 602 if !strings.Contains(line, ":/nomad/") { 603 t.Fatalf("not a member of the allocs nomad cgroup: %q", line) 604 } 605 } 606 } else { 607 require.True(t, strings.HasSuffix(stdout, ".scope"), "actual stdout %q", stdout) 608 } 609 610 // Exec a command that should fail 611 res, err = harness.ExecTask(task.ID, []string{"/usr/bin/stat", "lkjhdsaflkjshowaisxmcvnlia"}, time.Second) 612 require.NoError(t, err) 613 require.False(t, res.ExitResult.Successful()) 614 if expected := "No such file or directory"; !bytes.Contains(res.Stdout, []byte(expected)) { 615 t.Fatalf("expected output to contain %q but found: %q", expected, res.Stdout) 616 } 617 618 require.NoError(t, harness.DestroyTask(task.ID, true)) 619 } 620 621 func TestExecDriver_DevicesAndMounts(t *testing.T) { 622 ci.Parallel(t) 623 ctestutils.ExecCompatible(t) 624 625 tmpDir := t.TempDir() 626 627 err := ioutil.WriteFile(filepath.Join(tmpDir, "testfile"), []byte("from-host"), 600) 628 require.NoError(t, err) 629 630 ctx, cancel := context.WithCancel(context.Background()) 631 defer cancel() 632 633 d := NewExecDriver(ctx, testlog.HCLogger(t)) 634 harness := dtestutil.NewDriverHarness(t, d) 635 allocID := uuid.Generate() 636 task := &drivers.TaskConfig{ 637 ID: uuid.Generate(), 638 Name: "test", 639 User: "root", // need permission to read mounts paths 640 Resources: testResources(allocID, "test"), 641 StdoutPath: filepath.Join(tmpDir, "task-stdout"), 642 StderrPath: filepath.Join(tmpDir, "task-stderr"), 643 Devices: []*drivers.DeviceConfig{ 644 { 645 TaskPath: "/dev/inserted-random", 646 HostPath: "/dev/random", 647 Permissions: "rw", 648 }, 649 }, 650 Mounts: []*drivers.MountConfig{ 651 { 652 TaskPath: "/tmp/task-path-rw", 653 HostPath: tmpDir, 654 Readonly: false, 655 }, 656 { 657 TaskPath: "/tmp/task-path-ro", 658 HostPath: tmpDir, 659 Readonly: true, 660 }, 661 }, 662 } 663 664 require.NoError(t, ioutil.WriteFile(task.StdoutPath, []byte{}, 660)) 665 require.NoError(t, ioutil.WriteFile(task.StderrPath, []byte{}, 660)) 666 667 tc := &TaskConfig{ 668 Command: "/bin/bash", 669 Args: []string{"-c", ` 670 export LANG=en.UTF-8 671 echo "mounted device /inserted-random: $(stat -c '%t:%T' /dev/inserted-random)" 672 echo "reading from ro path: $(cat /tmp/task-path-ro/testfile)" 673 echo "reading from rw path: $(cat /tmp/task-path-rw/testfile)" 674 touch /tmp/task-path-rw/testfile && echo 'overwriting file in rw succeeded' 675 touch /tmp/task-path-rw/testfile-from-rw && echo from-exec > /tmp/task-path-rw/testfile-from-rw && echo 'writing new file in rw succeeded' 676 touch /tmp/task-path-ro/testfile && echo 'overwriting file in ro succeeded' 677 touch /tmp/task-path-ro/testfile-from-ro && echo from-exec > /tmp/task-path-ro/testfile-from-ro && echo 'writing new file in ro succeeded' 678 exit 0 679 `}, 680 } 681 require.NoError(t, task.EncodeConcreteDriverConfig(&tc)) 682 683 cleanup := harness.MkAllocDir(task, false) 684 defer cleanup() 685 686 handle, _, err := harness.StartTask(task) 687 require.NoError(t, err) 688 689 ch, err := harness.WaitTask(context.Background(), handle.Config.ID) 690 require.NoError(t, err) 691 result := <-ch 692 require.NoError(t, harness.DestroyTask(task.ID, true)) 693 694 stdout, err := ioutil.ReadFile(task.StdoutPath) 695 require.NoError(t, err) 696 require.Equal(t, `mounted device /inserted-random: 1:8 697 reading from ro path: from-host 698 reading from rw path: from-host 699 overwriting file in rw succeeded 700 writing new file in rw succeeded`, strings.TrimSpace(string(stdout))) 701 702 stderr, err := ioutil.ReadFile(task.StderrPath) 703 require.NoError(t, err) 704 require.Equal(t, `touch: cannot touch '/tmp/task-path-ro/testfile': Read-only file system 705 touch: cannot touch '/tmp/task-path-ro/testfile-from-ro': Read-only file system`, strings.TrimSpace(string(stderr))) 706 707 // testing exit code last so we can inspect output first 708 require.Zero(t, result.ExitCode) 709 710 fromRWContent, err := ioutil.ReadFile(filepath.Join(tmpDir, "testfile-from-rw")) 711 require.NoError(t, err) 712 require.Equal(t, "from-exec", strings.TrimSpace(string(fromRWContent))) 713 } 714 715 func TestConfig_ParseAllHCL(t *testing.T) { 716 ci.Parallel(t) 717 718 cfgStr := ` 719 config { 720 command = "/bin/bash" 721 args = ["-c", "echo hello"] 722 }` 723 724 expected := &TaskConfig{ 725 Command: "/bin/bash", 726 Args: []string{"-c", "echo hello"}, 727 } 728 729 var tc *TaskConfig 730 hclutils.NewConfigParser(taskConfigSpec).ParseHCL(t, cfgStr, &tc) 731 require.EqualValues(t, expected, tc) 732 } 733 734 func TestExecDriver_NoPivotRoot(t *testing.T) { 735 ci.Parallel(t) 736 ctestutils.ExecCompatible(t) 737 738 ctx, cancel := context.WithCancel(context.Background()) 739 defer cancel() 740 741 d := NewExecDriver(ctx, testlog.HCLogger(t)) 742 harness := dtestutil.NewDriverHarness(t, d) 743 744 config := &Config{ 745 NoPivotRoot: true, 746 DefaultModePID: executor.IsolationModePrivate, 747 DefaultModeIPC: executor.IsolationModePrivate, 748 } 749 750 var data []byte 751 require.NoError(t, basePlug.MsgPackEncode(&data, config)) 752 bconfig := &basePlug.Config{PluginConfig: data} 753 require.NoError(t, harness.SetConfig(bconfig)) 754 755 allocID := uuid.Generate() 756 task := &drivers.TaskConfig{ 757 AllocID: allocID, 758 ID: uuid.Generate(), 759 Name: "sleep", 760 Resources: testResources(allocID, "sleep"), 761 } 762 cleanup := harness.MkAllocDir(task, false) 763 defer cleanup() 764 765 tc := &TaskConfig{ 766 Command: "/bin/sleep", 767 Args: []string{"100"}, 768 } 769 require.NoError(t, task.EncodeConcreteDriverConfig(&tc)) 770 771 handle, _, err := harness.StartTask(task) 772 require.NoError(t, err) 773 require.NotNil(t, handle) 774 require.NoError(t, harness.DestroyTask(task.ID, true)) 775 } 776 777 func TestDriver_Config_validate(t *testing.T) { 778 ci.Parallel(t) 779 t.Run("pid/ipc", func(t *testing.T) { 780 for _, tc := range []struct { 781 pidMode, ipcMode string 782 exp error 783 }{ 784 {pidMode: "host", ipcMode: "host", exp: nil}, 785 {pidMode: "private", ipcMode: "host", exp: nil}, 786 {pidMode: "host", ipcMode: "private", exp: nil}, 787 {pidMode: "private", ipcMode: "private", exp: nil}, 788 {pidMode: "other", ipcMode: "private", exp: errors.New(`default_pid_mode must be "private" or "host", got "other"`)}, 789 {pidMode: "private", ipcMode: "other", exp: errors.New(`default_ipc_mode must be "private" or "host", got "other"`)}, 790 } { 791 require.Equal(t, tc.exp, (&Config{ 792 DefaultModePID: tc.pidMode, 793 DefaultModeIPC: tc.ipcMode, 794 }).validate()) 795 } 796 }) 797 798 t.Run("allow_caps", func(t *testing.T) { 799 for _, tc := range []struct { 800 ac []string 801 exp error 802 }{ 803 {ac: []string{}, exp: nil}, 804 {ac: []string{"all"}, exp: nil}, 805 {ac: []string{"chown", "sys_time"}, exp: nil}, 806 {ac: []string{"CAP_CHOWN", "cap_sys_time"}, exp: nil}, 807 {ac: []string{"chown", "not_valid", "sys_time"}, exp: errors.New("allow_caps configured with capabilities not supported by system: not_valid")}, 808 } { 809 require.Equal(t, tc.exp, (&Config{ 810 DefaultModePID: "private", 811 DefaultModeIPC: "private", 812 AllowCaps: tc.ac, 813 }).validate()) 814 } 815 }) 816 } 817 818 func TestDriver_TaskConfig_validate(t *testing.T) { 819 ci.Parallel(t) 820 t.Run("pid/ipc", func(t *testing.T) { 821 for _, tc := range []struct { 822 pidMode, ipcMode string 823 exp error 824 }{ 825 {pidMode: "host", ipcMode: "host", exp: nil}, 826 {pidMode: "host", ipcMode: "private", exp: nil}, 827 {pidMode: "host", ipcMode: "", exp: nil}, 828 {pidMode: "host", ipcMode: "other", exp: errors.New(`ipc_mode must be "private" or "host", got "other"`)}, 829 830 {pidMode: "host", ipcMode: "host", exp: nil}, 831 {pidMode: "private", ipcMode: "host", exp: nil}, 832 {pidMode: "", ipcMode: "host", exp: nil}, 833 {pidMode: "other", ipcMode: "host", exp: errors.New(`pid_mode must be "private" or "host", got "other"`)}, 834 } { 835 require.Equal(t, tc.exp, (&TaskConfig{ 836 ModePID: tc.pidMode, 837 ModeIPC: tc.ipcMode, 838 }).validate()) 839 } 840 }) 841 842 t.Run("cap_add", func(t *testing.T) { 843 for _, tc := range []struct { 844 adds []string 845 exp error 846 }{ 847 {adds: nil, exp: nil}, 848 {adds: []string{"chown"}, exp: nil}, 849 {adds: []string{"CAP_CHOWN"}, exp: nil}, 850 {adds: []string{"chown", "sys_time"}, exp: nil}, 851 {adds: []string{"chown", "not_valid", "sys_time"}, exp: errors.New("cap_add configured with capabilities not supported by system: not_valid")}, 852 } { 853 require.Equal(t, tc.exp, (&TaskConfig{ 854 CapAdd: tc.adds, 855 }).validate()) 856 } 857 }) 858 859 t.Run("cap_drop", func(t *testing.T) { 860 for _, tc := range []struct { 861 drops []string 862 exp error 863 }{ 864 {drops: nil, exp: nil}, 865 {drops: []string{"chown"}, exp: nil}, 866 {drops: []string{"CAP_CHOWN"}, exp: nil}, 867 {drops: []string{"chown", "sys_time"}, exp: nil}, 868 {drops: []string{"chown", "not_valid", "sys_time"}, exp: errors.New("cap_drop configured with capabilities not supported by system: not_valid")}, 869 } { 870 require.Equal(t, tc.exp, (&TaskConfig{ 871 CapDrop: tc.drops, 872 }).validate()) 873 } 874 }) 875 }