github.com/anth0d/nomad@v0.0.0-20221214183521-ae3a0a2cad06/drivers/exec/driver_test.go (about)

     1  package exec
     2  
     3  import (
     4  	"bytes"
     5  	"context"
     6  	"errors"
     7  	"fmt"
     8  	"io/ioutil"
     9  	"os"
    10  	"path/filepath"
    11  	"runtime"
    12  	"strconv"
    13  	"strings"
    14  	"sync"
    15  	"syscall"
    16  	"testing"
    17  	"time"
    18  
    19  	"github.com/hashicorp/nomad/ci"
    20  	"github.com/hashicorp/nomad/client/lib/cgutil"
    21  	ctestutils "github.com/hashicorp/nomad/client/testutil"
    22  	"github.com/hashicorp/nomad/drivers/shared/executor"
    23  	"github.com/hashicorp/nomad/helper/pluginutils/hclutils"
    24  	"github.com/hashicorp/nomad/helper/testlog"
    25  	"github.com/hashicorp/nomad/helper/testtask"
    26  	"github.com/hashicorp/nomad/helper/uuid"
    27  	"github.com/hashicorp/nomad/nomad/structs"
    28  	basePlug "github.com/hashicorp/nomad/plugins/base"
    29  	"github.com/hashicorp/nomad/plugins/drivers"
    30  	dtestutil "github.com/hashicorp/nomad/plugins/drivers/testutils"
    31  	"github.com/hashicorp/nomad/testutil"
    32  	"github.com/stretchr/testify/require"
    33  )
    34  
    35  const (
    36  	cgroupParent = "testing.slice"
    37  )
    38  
    39  func TestMain(m *testing.M) {
    40  	if !testtask.Run() {
    41  		os.Exit(m.Run())
    42  	}
    43  }
    44  
    45  func testResources(allocID, task string) *drivers.Resources {
    46  	if allocID == "" || task == "" {
    47  		panic("must be set")
    48  	}
    49  
    50  	r := &drivers.Resources{
    51  		NomadResources: &structs.AllocatedTaskResources{
    52  			Memory: structs.AllocatedMemoryResources{
    53  				MemoryMB: 128,
    54  			},
    55  			Cpu: structs.AllocatedCpuResources{
    56  				CpuShares: 100,
    57  			},
    58  		},
    59  		LinuxResources: &drivers.LinuxResources{
    60  			MemoryLimitBytes: 134217728,
    61  			CPUShares:        100,
    62  		},
    63  	}
    64  
    65  	if cgutil.UseV2 {
    66  		r.LinuxResources.CpusetCgroupPath = filepath.Join(cgutil.CgroupRoot, cgroupParent, cgutil.CgroupScope(allocID, task))
    67  	}
    68  
    69  	return r
    70  }
    71  
    72  func TestExecDriver_Fingerprint_NonLinux(t *testing.T) {
    73  	ci.Parallel(t)
    74  	require := require.New(t)
    75  	if runtime.GOOS == "linux" {
    76  		t.Skip("Test only available not on Linux")
    77  	}
    78  
    79  	ctx, cancel := context.WithCancel(context.Background())
    80  	defer cancel()
    81  
    82  	d := NewExecDriver(ctx, testlog.HCLogger(t))
    83  	harness := dtestutil.NewDriverHarness(t, d)
    84  
    85  	fingerCh, err := harness.Fingerprint(context.Background())
    86  	require.NoError(err)
    87  	select {
    88  	case finger := <-fingerCh:
    89  		require.Equal(drivers.HealthStateUndetected, finger.Health)
    90  	case <-time.After(time.Duration(testutil.TestMultiplier()*5) * time.Second):
    91  		require.Fail("timeout receiving fingerprint")
    92  	}
    93  }
    94  
    95  func TestExecDriver_Fingerprint(t *testing.T) {
    96  	ci.Parallel(t)
    97  	require := require.New(t)
    98  
    99  	ctestutils.ExecCompatible(t)
   100  
   101  	ctx, cancel := context.WithCancel(context.Background())
   102  	defer cancel()
   103  
   104  	d := NewExecDriver(ctx, testlog.HCLogger(t))
   105  	harness := dtestutil.NewDriverHarness(t, d)
   106  
   107  	fingerCh, err := harness.Fingerprint(context.Background())
   108  	require.NoError(err)
   109  	select {
   110  	case finger := <-fingerCh:
   111  		require.Equal(drivers.HealthStateHealthy, finger.Health)
   112  		require.True(finger.Attributes["driver.exec"].GetBool())
   113  	case <-time.After(time.Duration(testutil.TestMultiplier()*5) * time.Second):
   114  		require.Fail("timeout receiving fingerprint")
   115  	}
   116  }
   117  
   118  func TestExecDriver_StartWait(t *testing.T) {
   119  	ci.Parallel(t)
   120  	ctestutils.ExecCompatible(t)
   121  
   122  	ctx, cancel := context.WithCancel(context.Background())
   123  	defer cancel()
   124  
   125  	logger := testlog.HCLogger(t)
   126  
   127  	d := NewExecDriver(ctx, logger)
   128  	harness := dtestutil.NewDriverHarness(t, d)
   129  	allocID := uuid.Generate()
   130  	task := &drivers.TaskConfig{
   131  		AllocID:   allocID,
   132  		ID:        uuid.Generate(),
   133  		Name:      "test",
   134  		Resources: testResources(allocID, "test"),
   135  	}
   136  
   137  	tc := &TaskConfig{
   138  		Command: "cat",
   139  		Args:    []string{"/proc/self/cgroup"},
   140  	}
   141  	require.NoError(t, task.EncodeConcreteDriverConfig(&tc))
   142  
   143  	cleanup := harness.MkAllocDir(task, false)
   144  	defer cleanup()
   145  
   146  	handle, _, err := harness.StartTask(task)
   147  	require.NoError(t, err)
   148  
   149  	ch, err := harness.WaitTask(context.Background(), handle.Config.ID)
   150  	require.NoError(t, err)
   151  	result := <-ch
   152  	require.Zero(t, result.ExitCode)
   153  	require.NoError(t, harness.DestroyTask(task.ID, true))
   154  }
   155  
   156  func TestExecDriver_StartWaitStopKill(t *testing.T) {
   157  	ci.Parallel(t)
   158  	ctestutils.ExecCompatible(t)
   159  
   160  	ctx, cancel := context.WithCancel(context.Background())
   161  	defer cancel()
   162  
   163  	d := NewExecDriver(ctx, testlog.HCLogger(t))
   164  	harness := dtestutil.NewDriverHarness(t, d)
   165  	allocID := uuid.Generate()
   166  	task := &drivers.TaskConfig{
   167  		AllocID:   allocID,
   168  		ID:        uuid.Generate(),
   169  		Name:      "test",
   170  		Resources: testResources(allocID, "test"),
   171  	}
   172  
   173  	tc := &TaskConfig{
   174  		Command: "/bin/bash",
   175  		Args:    []string{"-c", "echo hi; sleep 600"},
   176  	}
   177  	require.NoError(t, task.EncodeConcreteDriverConfig(&tc))
   178  
   179  	cleanup := harness.MkAllocDir(task, false)
   180  	defer cleanup()
   181  
   182  	handle, _, err := harness.StartTask(task)
   183  	require.NoError(t, err)
   184  	defer harness.DestroyTask(task.ID, true)
   185  
   186  	ch, err := harness.WaitTask(context.Background(), handle.Config.ID)
   187  	require.NoError(t, err)
   188  
   189  	require.NoError(t, harness.WaitUntilStarted(task.ID, 1*time.Second))
   190  
   191  	go func() {
   192  		harness.StopTask(task.ID, 2*time.Second, "SIGINT")
   193  	}()
   194  
   195  	select {
   196  	case result := <-ch:
   197  		require.False(t, result.Successful())
   198  	case <-time.After(10 * time.Second):
   199  		require.Fail(t, "timeout waiting for task to shutdown")
   200  	}
   201  
   202  	// Ensure that the task is marked as dead, but account
   203  	// for WaitTask() closing channel before internal state is updated
   204  	testutil.WaitForResult(func() (bool, error) {
   205  		status, err := harness.InspectTask(task.ID)
   206  		if err != nil {
   207  			return false, fmt.Errorf("inspecting task failed: %v", err)
   208  		}
   209  		if status.State != drivers.TaskStateExited {
   210  			return false, fmt.Errorf("task hasn't exited yet; status: %v", status.State)
   211  		}
   212  
   213  		return true, nil
   214  	}, func(err error) {
   215  		require.NoError(t, err)
   216  	})
   217  
   218  	require.NoError(t, harness.DestroyTask(task.ID, true))
   219  }
   220  
   221  func TestExecDriver_StartWaitRecover(t *testing.T) {
   222  	ci.Parallel(t)
   223  	ctestutils.ExecCompatible(t)
   224  
   225  	dCtx, dCancel := context.WithCancel(context.Background())
   226  	defer dCancel()
   227  
   228  	d := NewExecDriver(dCtx, testlog.HCLogger(t))
   229  	harness := dtestutil.NewDriverHarness(t, d)
   230  	allocID := uuid.Generate()
   231  	task := &drivers.TaskConfig{
   232  		AllocID:   allocID,
   233  		ID:        uuid.Generate(),
   234  		Name:      "test",
   235  		Resources: testResources(allocID, "test"),
   236  	}
   237  
   238  	tc := &TaskConfig{
   239  		Command: "/bin/sleep",
   240  		Args:    []string{"5"},
   241  	}
   242  	require.NoError(t, task.EncodeConcreteDriverConfig(&tc))
   243  
   244  	cleanup := harness.MkAllocDir(task, false)
   245  	defer cleanup()
   246  
   247  	handle, _, err := harness.StartTask(task)
   248  	require.NoError(t, err)
   249  
   250  	ctx, cancel := context.WithCancel(context.Background())
   251  
   252  	ch, err := harness.WaitTask(ctx, handle.Config.ID)
   253  	require.NoError(t, err)
   254  
   255  	var wg sync.WaitGroup
   256  	wg.Add(1)
   257  	go func() {
   258  		defer wg.Done()
   259  		result := <-ch
   260  		require.Error(t, result.Err)
   261  	}()
   262  
   263  	require.NoError(t, harness.WaitUntilStarted(task.ID, 1*time.Second))
   264  	cancel()
   265  
   266  	waitCh := make(chan struct{})
   267  	go func() {
   268  		defer close(waitCh)
   269  		wg.Wait()
   270  	}()
   271  
   272  	select {
   273  	case <-waitCh:
   274  		status, err := harness.InspectTask(task.ID)
   275  		require.NoError(t, err)
   276  		require.Equal(t, drivers.TaskStateRunning, status.State)
   277  	case <-time.After(1 * time.Second):
   278  		require.Fail(t, "timeout waiting for task wait to cancel")
   279  	}
   280  
   281  	// Loose task
   282  	d.(*Driver).tasks.Delete(task.ID)
   283  	_, err = harness.InspectTask(task.ID)
   284  	require.Error(t, err)
   285  
   286  	require.NoError(t, harness.RecoverTask(handle))
   287  	status, err := harness.InspectTask(task.ID)
   288  	require.NoError(t, err)
   289  	require.Equal(t, drivers.TaskStateRunning, status.State)
   290  
   291  	require.NoError(t, harness.StopTask(task.ID, 0, ""))
   292  	require.NoError(t, harness.DestroyTask(task.ID, true))
   293  }
   294  
   295  // TestExecDriver_NoOrphans asserts that when the main
   296  // task dies, the orphans in the PID namespaces are killed by the kernel
   297  func TestExecDriver_NoOrphans(t *testing.T) {
   298  	ci.Parallel(t)
   299  	ctestutils.ExecCompatible(t)
   300  
   301  	ctx, cancel := context.WithCancel(context.Background())
   302  	defer cancel()
   303  
   304  	d := NewExecDriver(ctx, testlog.HCLogger(t))
   305  	harness := dtestutil.NewDriverHarness(t, d)
   306  	defer harness.Kill()
   307  
   308  	config := &Config{
   309  		NoPivotRoot:    false,
   310  		DefaultModePID: executor.IsolationModePrivate,
   311  		DefaultModeIPC: executor.IsolationModePrivate,
   312  	}
   313  
   314  	var data []byte
   315  	require.NoError(t, basePlug.MsgPackEncode(&data, config))
   316  	baseConfig := &basePlug.Config{PluginConfig: data}
   317  	require.NoError(t, harness.SetConfig(baseConfig))
   318  
   319  	allocID := uuid.Generate()
   320  	task := &drivers.TaskConfig{
   321  		AllocID: allocID,
   322  		ID:      uuid.Generate(),
   323  		Name:    "test",
   324  	}
   325  
   326  	if cgutil.UseV2 {
   327  		task.Resources = testResources(allocID, "test")
   328  	}
   329  
   330  	cleanup := harness.MkAllocDir(task, true)
   331  	defer cleanup()
   332  
   333  	taskConfig := map[string]interface{}{}
   334  	taskConfig["command"] = "/bin/sh"
   335  	// print the child PID in the task PID namespace, then sleep for 5 seconds to give us a chance to examine processes
   336  	taskConfig["args"] = []string{"-c", fmt.Sprintf(`sleep 3600 & sleep 20`)}
   337  	require.NoError(t, task.EncodeConcreteDriverConfig(&taskConfig))
   338  
   339  	handle, _, err := harness.StartTask(task)
   340  	require.NoError(t, err)
   341  	defer harness.DestroyTask(task.ID, true)
   342  
   343  	waitCh, err := harness.WaitTask(context.Background(), handle.Config.ID)
   344  	require.NoError(t, err)
   345  
   346  	require.NoError(t, harness.WaitUntilStarted(task.ID, 1*time.Second))
   347  
   348  	var childPids []int
   349  	taskState := TaskState{}
   350  	testutil.WaitForResult(func() (bool, error) {
   351  		require.NoError(t, handle.GetDriverState(&taskState))
   352  		if taskState.Pid == 0 {
   353  			return false, fmt.Errorf("task PID is zero")
   354  		}
   355  
   356  		children, err := ioutil.ReadFile(fmt.Sprintf("/proc/%d/task/%d/children", taskState.Pid, taskState.Pid))
   357  		if err != nil {
   358  			return false, fmt.Errorf("error reading /proc for children: %v", err)
   359  		}
   360  		pids := strings.Fields(string(children))
   361  		if len(pids) < 2 {
   362  			return false, fmt.Errorf("error waiting for two children, currently %d", len(pids))
   363  		}
   364  		for _, cpid := range pids {
   365  			p, err := strconv.Atoi(cpid)
   366  			if err != nil {
   367  				return false, fmt.Errorf("error parsing child pids from /proc: %s", cpid)
   368  			}
   369  			childPids = append(childPids, p)
   370  		}
   371  		return true, nil
   372  	}, func(err error) {
   373  		require.NoError(t, err)
   374  	})
   375  
   376  	select {
   377  	case result := <-waitCh:
   378  		require.True(t, result.Successful(), "command failed: %#v", result)
   379  	case <-time.After(30 * time.Second):
   380  		require.Fail(t, "timeout waiting for task to shutdown")
   381  	}
   382  
   383  	// isProcessRunning returns an error if process is not running
   384  	isProcessRunning := func(pid int) error {
   385  		process, err := os.FindProcess(pid)
   386  		if err != nil {
   387  			return fmt.Errorf("failed to find process: %s", err)
   388  		}
   389  
   390  		err = process.Signal(syscall.Signal(0))
   391  		if err != nil {
   392  			return fmt.Errorf("failed to signal process: %s", err)
   393  		}
   394  
   395  		return nil
   396  	}
   397  
   398  	// task should be dead
   399  	require.Error(t, isProcessRunning(taskState.Pid))
   400  
   401  	// all children should eventually be killed by OS
   402  	testutil.WaitForResult(func() (bool, error) {
   403  		for _, cpid := range childPids {
   404  			err := isProcessRunning(cpid)
   405  			if err == nil {
   406  				return false, fmt.Errorf("child process %d is still running", cpid)
   407  			}
   408  			if !strings.Contains(err.Error(), "failed to signal process") {
   409  				return false, fmt.Errorf("unexpected error: %v", err)
   410  			}
   411  		}
   412  		return true, nil
   413  	}, func(err error) {
   414  		require.NoError(t, err)
   415  	})
   416  }
   417  
   418  func TestExecDriver_Stats(t *testing.T) {
   419  	ci.Parallel(t)
   420  	ctestutils.ExecCompatible(t)
   421  
   422  	dctx, dcancel := context.WithCancel(context.Background())
   423  	defer dcancel()
   424  
   425  	d := NewExecDriver(dctx, testlog.HCLogger(t))
   426  	harness := dtestutil.NewDriverHarness(t, d)
   427  
   428  	allocID := uuid.Generate()
   429  	task := &drivers.TaskConfig{
   430  		AllocID:   allocID,
   431  		ID:        uuid.Generate(),
   432  		Name:      "test",
   433  		Resources: testResources(allocID, "test"),
   434  	}
   435  
   436  	tc := &TaskConfig{
   437  		Command: "/bin/sleep",
   438  		Args:    []string{"5"},
   439  	}
   440  	require.NoError(t, task.EncodeConcreteDriverConfig(&tc))
   441  
   442  	cleanup := harness.MkAllocDir(task, false)
   443  	defer cleanup()
   444  
   445  	handle, _, err := harness.StartTask(task)
   446  	require.NoError(t, err)
   447  	require.NotNil(t, handle)
   448  
   449  	require.NoError(t, harness.WaitUntilStarted(task.ID, 1*time.Second))
   450  	ctx, cancel := context.WithCancel(context.Background())
   451  	defer cancel()
   452  	statsCh, err := harness.TaskStats(ctx, task.ID, time.Second*10)
   453  	require.NoError(t, err)
   454  	select {
   455  	case stats := <-statsCh:
   456  		require.NotEmpty(t, stats.ResourceUsage.MemoryStats.Measured)
   457  		require.NotZero(t, stats.Timestamp)
   458  		require.WithinDuration(t, time.Now(), time.Unix(0, stats.Timestamp), time.Second)
   459  	case <-time.After(time.Second):
   460  		require.Fail(t, "timeout receiving from channel")
   461  	}
   462  
   463  	require.NoError(t, harness.DestroyTask(task.ID, true))
   464  }
   465  
   466  func TestExecDriver_Start_Wait_AllocDir(t *testing.T) {
   467  	ci.Parallel(t)
   468  	ctestutils.ExecCompatible(t)
   469  
   470  	ctx, cancel := context.WithCancel(context.Background())
   471  	defer cancel()
   472  
   473  	d := NewExecDriver(ctx, testlog.HCLogger(t))
   474  	harness := dtestutil.NewDriverHarness(t, d)
   475  	allocID := uuid.Generate()
   476  	task := &drivers.TaskConfig{
   477  		AllocID:   allocID,
   478  		ID:        uuid.Generate(),
   479  		Name:      "sleep",
   480  		Resources: testResources(allocID, "test"),
   481  	}
   482  	cleanup := harness.MkAllocDir(task, false)
   483  	defer cleanup()
   484  
   485  	exp := []byte{'w', 'i', 'n'}
   486  	file := "output.txt"
   487  	tc := &TaskConfig{
   488  		Command: "/bin/bash",
   489  		Args: []string{
   490  			"-c",
   491  			fmt.Sprintf(`sleep 1; echo -n %s > /alloc/%s`, string(exp), file),
   492  		},
   493  	}
   494  	require.NoError(t, task.EncodeConcreteDriverConfig(&tc))
   495  
   496  	handle, _, err := harness.StartTask(task)
   497  	require.NoError(t, err)
   498  	require.NotNil(t, handle)
   499  
   500  	// Task should terminate quickly
   501  	waitCh, err := harness.WaitTask(context.Background(), task.ID)
   502  	require.NoError(t, err)
   503  	select {
   504  	case res := <-waitCh:
   505  		require.True(t, res.Successful(), "task should have exited successfully: %v", res)
   506  	case <-time.After(time.Duration(testutil.TestMultiplier()*5) * time.Second):
   507  		require.Fail(t, "timeout waiting for task")
   508  	}
   509  
   510  	// Check that data was written to the shared alloc directory.
   511  	outputFile := filepath.Join(task.TaskDir().SharedAllocDir, file)
   512  	act, err := ioutil.ReadFile(outputFile)
   513  	require.NoError(t, err)
   514  	require.Exactly(t, exp, act)
   515  
   516  	require.NoError(t, harness.DestroyTask(task.ID, true))
   517  }
   518  
   519  func TestExecDriver_User(t *testing.T) {
   520  	ci.Parallel(t)
   521  	ctestutils.ExecCompatible(t)
   522  
   523  	ctx, cancel := context.WithCancel(context.Background())
   524  	defer cancel()
   525  
   526  	d := NewExecDriver(ctx, testlog.HCLogger(t))
   527  	harness := dtestutil.NewDriverHarness(t, d)
   528  	allocID := uuid.Generate()
   529  	task := &drivers.TaskConfig{
   530  		AllocID:   allocID,
   531  		ID:        uuid.Generate(),
   532  		Name:      "sleep",
   533  		User:      "alice",
   534  		Resources: testResources(allocID, "sleep"),
   535  	}
   536  	cleanup := harness.MkAllocDir(task, false)
   537  	defer cleanup()
   538  
   539  	tc := &TaskConfig{
   540  		Command: "/bin/sleep",
   541  		Args:    []string{"100"},
   542  	}
   543  	require.NoError(t, task.EncodeConcreteDriverConfig(&tc))
   544  
   545  	handle, _, err := harness.StartTask(task)
   546  	require.Error(t, err)
   547  	require.Nil(t, handle)
   548  
   549  	msg := "user alice"
   550  	if !strings.Contains(err.Error(), msg) {
   551  		t.Fatalf("Expecting '%v' in '%v'", msg, err)
   552  	}
   553  }
   554  
   555  // TestExecDriver_HandlerExec ensures the exec driver's handle properly
   556  // executes commands inside the container.
   557  func TestExecDriver_HandlerExec(t *testing.T) {
   558  	ci.Parallel(t)
   559  	ctestutils.ExecCompatible(t)
   560  
   561  	ctx, cancel := context.WithCancel(context.Background())
   562  	defer cancel()
   563  
   564  	d := NewExecDriver(ctx, testlog.HCLogger(t))
   565  	harness := dtestutil.NewDriverHarness(t, d)
   566  	allocID := uuid.Generate()
   567  	task := &drivers.TaskConfig{
   568  		AllocID:   allocID,
   569  		ID:        uuid.Generate(),
   570  		Name:      "sleep",
   571  		Resources: testResources(allocID, "sleep"),
   572  	}
   573  	cleanup := harness.MkAllocDir(task, false)
   574  	defer cleanup()
   575  
   576  	tc := &TaskConfig{
   577  		Command: "/bin/sleep",
   578  		Args:    []string{"9000"},
   579  	}
   580  	require.NoError(t, task.EncodeConcreteDriverConfig(&tc))
   581  
   582  	handle, _, err := harness.StartTask(task)
   583  	require.NoError(t, err)
   584  	require.NotNil(t, handle)
   585  
   586  	// Assert cgroup membership
   587  	res, err := harness.ExecTask(task.ID, []string{"/bin/cat", "/proc/self/cgroup"}, time.Second)
   588  	require.NoError(t, err)
   589  	require.True(t, res.ExitResult.Successful())
   590  	stdout := strings.TrimSpace(string(res.Stdout))
   591  	if !cgutil.UseV2 {
   592  		for _, line := range strings.Split(stdout, "\n") {
   593  			// skip empty lines
   594  			if line == "" {
   595  				continue
   596  			}
   597  			// skip rdma & misc subsystems
   598  			if strings.Contains(line, ":rdma:") || strings.Contains(line, ":misc:") || strings.Contains(line, "::") {
   599  				continue
   600  			}
   601  			// assert we are in a nomad cgroup
   602  			if !strings.Contains(line, ":/nomad/") {
   603  				t.Fatalf("not a member of the allocs nomad cgroup: %q", line)
   604  			}
   605  		}
   606  	} else {
   607  		require.True(t, strings.HasSuffix(stdout, ".scope"), "actual stdout %q", stdout)
   608  	}
   609  
   610  	// Exec a command that should fail
   611  	res, err = harness.ExecTask(task.ID, []string{"/usr/bin/stat", "lkjhdsaflkjshowaisxmcvnlia"}, time.Second)
   612  	require.NoError(t, err)
   613  	require.False(t, res.ExitResult.Successful())
   614  	if expected := "No such file or directory"; !bytes.Contains(res.Stdout, []byte(expected)) {
   615  		t.Fatalf("expected output to contain %q but found: %q", expected, res.Stdout)
   616  	}
   617  
   618  	require.NoError(t, harness.DestroyTask(task.ID, true))
   619  }
   620  
   621  func TestExecDriver_DevicesAndMounts(t *testing.T) {
   622  	ci.Parallel(t)
   623  	ctestutils.ExecCompatible(t)
   624  
   625  	tmpDir := t.TempDir()
   626  
   627  	err := ioutil.WriteFile(filepath.Join(tmpDir, "testfile"), []byte("from-host"), 600)
   628  	require.NoError(t, err)
   629  
   630  	ctx, cancel := context.WithCancel(context.Background())
   631  	defer cancel()
   632  
   633  	d := NewExecDriver(ctx, testlog.HCLogger(t))
   634  	harness := dtestutil.NewDriverHarness(t, d)
   635  	allocID := uuid.Generate()
   636  	task := &drivers.TaskConfig{
   637  		ID:         uuid.Generate(),
   638  		Name:       "test",
   639  		User:       "root", // need permission to read mounts paths
   640  		Resources:  testResources(allocID, "test"),
   641  		StdoutPath: filepath.Join(tmpDir, "task-stdout"),
   642  		StderrPath: filepath.Join(tmpDir, "task-stderr"),
   643  		Devices: []*drivers.DeviceConfig{
   644  			{
   645  				TaskPath:    "/dev/inserted-random",
   646  				HostPath:    "/dev/random",
   647  				Permissions: "rw",
   648  			},
   649  		},
   650  		Mounts: []*drivers.MountConfig{
   651  			{
   652  				TaskPath: "/tmp/task-path-rw",
   653  				HostPath: tmpDir,
   654  				Readonly: false,
   655  			},
   656  			{
   657  				TaskPath: "/tmp/task-path-ro",
   658  				HostPath: tmpDir,
   659  				Readonly: true,
   660  			},
   661  		},
   662  	}
   663  
   664  	require.NoError(t, ioutil.WriteFile(task.StdoutPath, []byte{}, 660))
   665  	require.NoError(t, ioutil.WriteFile(task.StderrPath, []byte{}, 660))
   666  
   667  	tc := &TaskConfig{
   668  		Command: "/bin/bash",
   669  		Args: []string{"-c", `
   670  export LANG=en.UTF-8
   671  echo "mounted device /inserted-random: $(stat -c '%t:%T' /dev/inserted-random)"
   672  echo "reading from ro path: $(cat /tmp/task-path-ro/testfile)"
   673  echo "reading from rw path: $(cat /tmp/task-path-rw/testfile)"
   674  touch /tmp/task-path-rw/testfile && echo 'overwriting file in rw succeeded'
   675  touch /tmp/task-path-rw/testfile-from-rw && echo from-exec >  /tmp/task-path-rw/testfile-from-rw && echo 'writing new file in rw succeeded'
   676  touch /tmp/task-path-ro/testfile && echo 'overwriting file in ro succeeded'
   677  touch /tmp/task-path-ro/testfile-from-ro && echo from-exec >  /tmp/task-path-ro/testfile-from-ro && echo 'writing new file in ro succeeded'
   678  exit 0
   679  `},
   680  	}
   681  	require.NoError(t, task.EncodeConcreteDriverConfig(&tc))
   682  
   683  	cleanup := harness.MkAllocDir(task, false)
   684  	defer cleanup()
   685  
   686  	handle, _, err := harness.StartTask(task)
   687  	require.NoError(t, err)
   688  
   689  	ch, err := harness.WaitTask(context.Background(), handle.Config.ID)
   690  	require.NoError(t, err)
   691  	result := <-ch
   692  	require.NoError(t, harness.DestroyTask(task.ID, true))
   693  
   694  	stdout, err := ioutil.ReadFile(task.StdoutPath)
   695  	require.NoError(t, err)
   696  	require.Equal(t, `mounted device /inserted-random: 1:8
   697  reading from ro path: from-host
   698  reading from rw path: from-host
   699  overwriting file in rw succeeded
   700  writing new file in rw succeeded`, strings.TrimSpace(string(stdout)))
   701  
   702  	stderr, err := ioutil.ReadFile(task.StderrPath)
   703  	require.NoError(t, err)
   704  	require.Equal(t, `touch: cannot touch '/tmp/task-path-ro/testfile': Read-only file system
   705  touch: cannot touch '/tmp/task-path-ro/testfile-from-ro': Read-only file system`, strings.TrimSpace(string(stderr)))
   706  
   707  	// testing exit code last so we can inspect output first
   708  	require.Zero(t, result.ExitCode)
   709  
   710  	fromRWContent, err := ioutil.ReadFile(filepath.Join(tmpDir, "testfile-from-rw"))
   711  	require.NoError(t, err)
   712  	require.Equal(t, "from-exec", strings.TrimSpace(string(fromRWContent)))
   713  }
   714  
   715  func TestConfig_ParseAllHCL(t *testing.T) {
   716  	ci.Parallel(t)
   717  
   718  	cfgStr := `
   719  config {
   720    command = "/bin/bash"
   721    args = ["-c", "echo hello"]
   722  }`
   723  
   724  	expected := &TaskConfig{
   725  		Command: "/bin/bash",
   726  		Args:    []string{"-c", "echo hello"},
   727  	}
   728  
   729  	var tc *TaskConfig
   730  	hclutils.NewConfigParser(taskConfigSpec).ParseHCL(t, cfgStr, &tc)
   731  	require.EqualValues(t, expected, tc)
   732  }
   733  
   734  func TestExecDriver_NoPivotRoot(t *testing.T) {
   735  	ci.Parallel(t)
   736  	ctestutils.ExecCompatible(t)
   737  
   738  	ctx, cancel := context.WithCancel(context.Background())
   739  	defer cancel()
   740  
   741  	d := NewExecDriver(ctx, testlog.HCLogger(t))
   742  	harness := dtestutil.NewDriverHarness(t, d)
   743  
   744  	config := &Config{
   745  		NoPivotRoot:    true,
   746  		DefaultModePID: executor.IsolationModePrivate,
   747  		DefaultModeIPC: executor.IsolationModePrivate,
   748  	}
   749  
   750  	var data []byte
   751  	require.NoError(t, basePlug.MsgPackEncode(&data, config))
   752  	bconfig := &basePlug.Config{PluginConfig: data}
   753  	require.NoError(t, harness.SetConfig(bconfig))
   754  
   755  	allocID := uuid.Generate()
   756  	task := &drivers.TaskConfig{
   757  		AllocID:   allocID,
   758  		ID:        uuid.Generate(),
   759  		Name:      "sleep",
   760  		Resources: testResources(allocID, "sleep"),
   761  	}
   762  	cleanup := harness.MkAllocDir(task, false)
   763  	defer cleanup()
   764  
   765  	tc := &TaskConfig{
   766  		Command: "/bin/sleep",
   767  		Args:    []string{"100"},
   768  	}
   769  	require.NoError(t, task.EncodeConcreteDriverConfig(&tc))
   770  
   771  	handle, _, err := harness.StartTask(task)
   772  	require.NoError(t, err)
   773  	require.NotNil(t, handle)
   774  	require.NoError(t, harness.DestroyTask(task.ID, true))
   775  }
   776  
   777  func TestDriver_Config_validate(t *testing.T) {
   778  	ci.Parallel(t)
   779  	t.Run("pid/ipc", func(t *testing.T) {
   780  		for _, tc := range []struct {
   781  			pidMode, ipcMode string
   782  			exp              error
   783  		}{
   784  			{pidMode: "host", ipcMode: "host", exp: nil},
   785  			{pidMode: "private", ipcMode: "host", exp: nil},
   786  			{pidMode: "host", ipcMode: "private", exp: nil},
   787  			{pidMode: "private", ipcMode: "private", exp: nil},
   788  			{pidMode: "other", ipcMode: "private", exp: errors.New(`default_pid_mode must be "private" or "host", got "other"`)},
   789  			{pidMode: "private", ipcMode: "other", exp: errors.New(`default_ipc_mode must be "private" or "host", got "other"`)},
   790  		} {
   791  			require.Equal(t, tc.exp, (&Config{
   792  				DefaultModePID: tc.pidMode,
   793  				DefaultModeIPC: tc.ipcMode,
   794  			}).validate())
   795  		}
   796  	})
   797  
   798  	t.Run("allow_caps", func(t *testing.T) {
   799  		for _, tc := range []struct {
   800  			ac  []string
   801  			exp error
   802  		}{
   803  			{ac: []string{}, exp: nil},
   804  			{ac: []string{"all"}, exp: nil},
   805  			{ac: []string{"chown", "sys_time"}, exp: nil},
   806  			{ac: []string{"CAP_CHOWN", "cap_sys_time"}, exp: nil},
   807  			{ac: []string{"chown", "not_valid", "sys_time"}, exp: errors.New("allow_caps configured with capabilities not supported by system: not_valid")},
   808  		} {
   809  			require.Equal(t, tc.exp, (&Config{
   810  				DefaultModePID: "private",
   811  				DefaultModeIPC: "private",
   812  				AllowCaps:      tc.ac,
   813  			}).validate())
   814  		}
   815  	})
   816  }
   817  
   818  func TestDriver_TaskConfig_validate(t *testing.T) {
   819  	ci.Parallel(t)
   820  	t.Run("pid/ipc", func(t *testing.T) {
   821  		for _, tc := range []struct {
   822  			pidMode, ipcMode string
   823  			exp              error
   824  		}{
   825  			{pidMode: "host", ipcMode: "host", exp: nil},
   826  			{pidMode: "host", ipcMode: "private", exp: nil},
   827  			{pidMode: "host", ipcMode: "", exp: nil},
   828  			{pidMode: "host", ipcMode: "other", exp: errors.New(`ipc_mode must be "private" or "host", got "other"`)},
   829  
   830  			{pidMode: "host", ipcMode: "host", exp: nil},
   831  			{pidMode: "private", ipcMode: "host", exp: nil},
   832  			{pidMode: "", ipcMode: "host", exp: nil},
   833  			{pidMode: "other", ipcMode: "host", exp: errors.New(`pid_mode must be "private" or "host", got "other"`)},
   834  		} {
   835  			require.Equal(t, tc.exp, (&TaskConfig{
   836  				ModePID: tc.pidMode,
   837  				ModeIPC: tc.ipcMode,
   838  			}).validate())
   839  		}
   840  	})
   841  
   842  	t.Run("cap_add", func(t *testing.T) {
   843  		for _, tc := range []struct {
   844  			adds []string
   845  			exp  error
   846  		}{
   847  			{adds: nil, exp: nil},
   848  			{adds: []string{"chown"}, exp: nil},
   849  			{adds: []string{"CAP_CHOWN"}, exp: nil},
   850  			{adds: []string{"chown", "sys_time"}, exp: nil},
   851  			{adds: []string{"chown", "not_valid", "sys_time"}, exp: errors.New("cap_add configured with capabilities not supported by system: not_valid")},
   852  		} {
   853  			require.Equal(t, tc.exp, (&TaskConfig{
   854  				CapAdd: tc.adds,
   855  			}).validate())
   856  		}
   857  	})
   858  
   859  	t.Run("cap_drop", func(t *testing.T) {
   860  		for _, tc := range []struct {
   861  			drops []string
   862  			exp   error
   863  		}{
   864  			{drops: nil, exp: nil},
   865  			{drops: []string{"chown"}, exp: nil},
   866  			{drops: []string{"CAP_CHOWN"}, exp: nil},
   867  			{drops: []string{"chown", "sys_time"}, exp: nil},
   868  			{drops: []string{"chown", "not_valid", "sys_time"}, exp: errors.New("cap_drop configured with capabilities not supported by system: not_valid")},
   869  		} {
   870  			require.Equal(t, tc.exp, (&TaskConfig{
   871  				CapDrop: tc.drops,
   872  			}).validate())
   873  		}
   874  	})
   875  }