github.com/bigcommerce/nomad@v0.9.3-bc/drivers/shared/executor/executor_linux_test.go (about)

     1  package executor
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  	"io/ioutil"
     7  	"os"
     8  	"path/filepath"
     9  	"regexp"
    10  	"strconv"
    11  	"strings"
    12  	"testing"
    13  	"time"
    14  
    15  	"github.com/hashicorp/nomad/client/allocdir"
    16  	"github.com/hashicorp/nomad/client/taskenv"
    17  	"github.com/hashicorp/nomad/client/testutil"
    18  	"github.com/hashicorp/nomad/helper/testlog"
    19  	"github.com/hashicorp/nomad/nomad/mock"
    20  	"github.com/hashicorp/nomad/plugins/drivers"
    21  	tu "github.com/hashicorp/nomad/testutil"
    22  	lconfigs "github.com/opencontainers/runc/libcontainer/configs"
    23  	"github.com/stretchr/testify/require"
    24  	"golang.org/x/sys/unix"
    25  )
    26  
    27  func init() {
    28  	executorFactories["LibcontainerExecutor"] = libcontainerFactory
    29  }
    30  
    31  var libcontainerFactory = executorFactory{
    32  	new: NewExecutorWithIsolation,
    33  	configureExecCmd: func(t *testing.T, cmd *ExecCommand) {
    34  		cmd.ResourceLimits = true
    35  		setupRootfs(t, cmd.TaskDir)
    36  	},
    37  }
    38  
    39  // testExecutorContextWithChroot returns an ExecutorContext and AllocDir with
    40  // chroot. Use testExecutorContext if you don't need a chroot.
    41  //
    42  // The caller is responsible for calling AllocDir.Destroy() to cleanup.
    43  func testExecutorCommandWithChroot(t *testing.T) *testExecCmd {
    44  	chrootEnv := map[string]string{
    45  		"/etc/ld.so.cache":  "/etc/ld.so.cache",
    46  		"/etc/ld.so.conf":   "/etc/ld.so.conf",
    47  		"/etc/ld.so.conf.d": "/etc/ld.so.conf.d",
    48  		"/etc/passwd":       "/etc/passwd",
    49  		"/lib":              "/lib",
    50  		"/lib64":            "/lib64",
    51  		"/usr/lib":          "/usr/lib",
    52  		"/bin/ls":           "/bin/ls",
    53  		"/bin/cat":          "/bin/cat",
    54  		"/bin/echo":         "/bin/echo",
    55  		"/bin/bash":         "/bin/bash",
    56  		"/bin/sleep":        "/bin/sleep",
    57  		"/foobar":           "/does/not/exist",
    58  	}
    59  
    60  	alloc := mock.Alloc()
    61  	task := alloc.Job.TaskGroups[0].Tasks[0]
    62  	taskEnv := taskenv.NewBuilder(mock.Node(), alloc, task, "global").Build()
    63  
    64  	allocDir := allocdir.NewAllocDir(testlog.HCLogger(t), filepath.Join(os.TempDir(), alloc.ID))
    65  	if err := allocDir.Build(); err != nil {
    66  		t.Fatalf("AllocDir.Build() failed: %v", err)
    67  	}
    68  	if err := allocDir.NewTaskDir(task.Name).Build(true, chrootEnv); err != nil {
    69  		allocDir.Destroy()
    70  		t.Fatalf("allocDir.NewTaskDir(%q) failed: %v", task.Name, err)
    71  	}
    72  	td := allocDir.TaskDirs[task.Name]
    73  	cmd := &ExecCommand{
    74  		Env:     taskEnv.List(),
    75  		TaskDir: td.Dir,
    76  		Resources: &drivers.Resources{
    77  			NomadResources: alloc.AllocatedResources.Tasks[task.Name],
    78  		},
    79  	}
    80  
    81  	testCmd := &testExecCmd{
    82  		command:  cmd,
    83  		allocDir: allocDir,
    84  	}
    85  	configureTLogging(t, testCmd)
    86  	return testCmd
    87  }
    88  
    89  func TestExecutor_IsolationAndConstraints(t *testing.T) {
    90  	t.Parallel()
    91  	require := require.New(t)
    92  	testutil.ExecCompatible(t)
    93  
    94  	testExecCmd := testExecutorCommandWithChroot(t)
    95  	execCmd, allocDir := testExecCmd.command, testExecCmd.allocDir
    96  	execCmd.Cmd = "/bin/ls"
    97  	execCmd.Args = []string{"-F", "/", "/etc/"}
    98  	defer allocDir.Destroy()
    99  
   100  	execCmd.ResourceLimits = true
   101  
   102  	executor := NewExecutorWithIsolation(testlog.HCLogger(t))
   103  	defer executor.Shutdown("SIGKILL", 0)
   104  
   105  	ps, err := executor.Launch(execCmd)
   106  	require.NoError(err)
   107  	require.NotZero(ps.Pid)
   108  
   109  	state, err := executor.Wait(context.Background())
   110  	require.NoError(err)
   111  	require.Zero(state.ExitCode)
   112  
   113  	// Check if the resource constraints were applied
   114  	if lexec, ok := executor.(*LibcontainerExecutor); ok {
   115  		state, err := lexec.container.State()
   116  		require.NoError(err)
   117  
   118  		memLimits := filepath.Join(state.CgroupPaths["memory"], "memory.limit_in_bytes")
   119  		data, err := ioutil.ReadFile(memLimits)
   120  		require.NoError(err)
   121  
   122  		expectedMemLim := strconv.Itoa(int(execCmd.Resources.NomadResources.Memory.MemoryMB * 1024 * 1024))
   123  		actualMemLim := strings.TrimSpace(string(data))
   124  		require.Equal(actualMemLim, expectedMemLim)
   125  		require.NoError(executor.Shutdown("", 0))
   126  		executor.Wait(context.Background())
   127  
   128  		// Check if Nomad has actually removed the cgroups
   129  		tu.WaitForResult(func() (bool, error) {
   130  			_, err = os.Stat(memLimits)
   131  			if err == nil {
   132  				return false, fmt.Errorf("expected an error from os.Stat %s", memLimits)
   133  			}
   134  			return true, nil
   135  		}, func(err error) { t.Error(err) })
   136  
   137  	}
   138  	expected := `/:
   139  alloc/
   140  bin/
   141  dev/
   142  etc/
   143  lib/
   144  lib64/
   145  local/
   146  proc/
   147  secrets/
   148  sys/
   149  tmp/
   150  usr/
   151  
   152  /etc/:
   153  ld.so.cache
   154  ld.so.conf
   155  ld.so.conf.d/
   156  passwd`
   157  	tu.WaitForResult(func() (bool, error) {
   158  		output := testExecCmd.stdout.String()
   159  		act := strings.TrimSpace(string(output))
   160  		if act != expected {
   161  			return false, fmt.Errorf("Command output incorrectly: want %v; got %v", expected, act)
   162  		}
   163  		return true, nil
   164  	}, func(err error) { t.Error(err) })
   165  }
   166  
   167  // TestExecutor_CgroupPaths asserts that process starts with independent cgroups
   168  // hierarchy created for this process
   169  func TestExecutor_CgroupPaths(t *testing.T) {
   170  	t.Parallel()
   171  	require := require.New(t)
   172  	testutil.ExecCompatible(t)
   173  
   174  	testExecCmd := testExecutorCommandWithChroot(t)
   175  	execCmd, allocDir := testExecCmd.command, testExecCmd.allocDir
   176  	execCmd.Cmd = "/bin/bash"
   177  	execCmd.Args = []string{"-c", "sleep 0.2; cat /proc/self/cgroup"}
   178  	defer allocDir.Destroy()
   179  
   180  	execCmd.ResourceLimits = true
   181  
   182  	executor := NewExecutorWithIsolation(testlog.HCLogger(t))
   183  	defer executor.Shutdown("SIGKILL", 0)
   184  
   185  	ps, err := executor.Launch(execCmd)
   186  	require.NoError(err)
   187  	require.NotZero(ps.Pid)
   188  
   189  	state, err := executor.Wait(context.Background())
   190  	require.NoError(err)
   191  	require.Zero(state.ExitCode)
   192  
   193  	tu.WaitForResult(func() (bool, error) {
   194  		output := strings.TrimSpace(testExecCmd.stdout.String())
   195  		// sanity check that we got some cgroups
   196  		if !strings.Contains(output, ":devices:") {
   197  			return false, fmt.Errorf("was expected cgroup files but found:\n%v", output)
   198  		}
   199  		lines := strings.Split(output, "\n")
   200  		for _, line := range lines {
   201  			// Every cgroup entry should be /nomad/$ALLOC_ID
   202  			if line == "" {
   203  				continue
   204  			}
   205  
   206  			// Skip rdma subsystem; rdma was added in most recent kernels and libcontainer/docker
   207  			// don't isolate it by default.
   208  			if strings.Contains(line, ":rdma:") {
   209  				continue
   210  			}
   211  
   212  			if !strings.Contains(line, ":/nomad/") {
   213  				return false, fmt.Errorf("Not a member of the alloc's cgroup: expected=...:/nomad/... -- found=%q", line)
   214  			}
   215  		}
   216  		return true, nil
   217  	}, func(err error) { t.Error(err) })
   218  }
   219  
   220  func TestUniversalExecutor_LookupTaskBin(t *testing.T) {
   221  	t.Parallel()
   222  	require := require.New(t)
   223  
   224  	// Create a temp dir
   225  	tmpDir, err := ioutil.TempDir("", "")
   226  	require.Nil(err)
   227  	defer os.Remove(tmpDir)
   228  
   229  	// Create the command
   230  	cmd := &ExecCommand{Env: []string{"PATH=/bin"}, TaskDir: tmpDir}
   231  
   232  	// Make a foo subdir
   233  	os.MkdirAll(filepath.Join(tmpDir, "foo"), 0700)
   234  
   235  	// Write a file under foo
   236  	filePath := filepath.Join(tmpDir, "foo", "tmp.txt")
   237  	err = ioutil.WriteFile(filePath, []byte{1, 2}, os.ModeAppend)
   238  	require.NoError(err)
   239  
   240  	// Lookout with an absolute path to the binary
   241  	cmd.Cmd = "/foo/tmp.txt"
   242  	_, err = lookupTaskBin(cmd)
   243  	require.NoError(err)
   244  
   245  	// Write a file under local subdir
   246  	os.MkdirAll(filepath.Join(tmpDir, "local"), 0700)
   247  	filePath2 := filepath.Join(tmpDir, "local", "tmp.txt")
   248  	ioutil.WriteFile(filePath2, []byte{1, 2}, os.ModeAppend)
   249  
   250  	// Lookup with file name, should find the one we wrote above
   251  	cmd.Cmd = "tmp.txt"
   252  	_, err = lookupTaskBin(cmd)
   253  	require.NoError(err)
   254  
   255  	// Lookup a host absolute path
   256  	cmd.Cmd = "/bin/sh"
   257  	_, err = lookupTaskBin(cmd)
   258  	require.Error(err)
   259  }
   260  
   261  // Exec Launch looks for the binary only inside the chroot
   262  func TestExecutor_EscapeContainer(t *testing.T) {
   263  	t.Parallel()
   264  	require := require.New(t)
   265  	testutil.ExecCompatible(t)
   266  
   267  	testExecCmd := testExecutorCommandWithChroot(t)
   268  	execCmd, allocDir := testExecCmd.command, testExecCmd.allocDir
   269  	execCmd.Cmd = "/bin/kill" // missing from the chroot container
   270  	defer allocDir.Destroy()
   271  
   272  	execCmd.ResourceLimits = true
   273  
   274  	executor := NewExecutorWithIsolation(testlog.HCLogger(t))
   275  	defer executor.Shutdown("SIGKILL", 0)
   276  
   277  	_, err := executor.Launch(execCmd)
   278  	require.Error(err)
   279  	require.Regexp("^file /bin/kill not found under path", err)
   280  
   281  	// Bare files are looked up using the system path, inside the container
   282  	allocDir.Destroy()
   283  	testExecCmd = testExecutorCommandWithChroot(t)
   284  	execCmd, allocDir = testExecCmd.command, testExecCmd.allocDir
   285  	execCmd.Cmd = "kill"
   286  	_, err = executor.Launch(execCmd)
   287  	require.Error(err)
   288  	require.Regexp("^file kill not found under path", err)
   289  
   290  	allocDir.Destroy()
   291  	testExecCmd = testExecutorCommandWithChroot(t)
   292  	execCmd, allocDir = testExecCmd.command, testExecCmd.allocDir
   293  	execCmd.Cmd = "echo"
   294  	_, err = executor.Launch(execCmd)
   295  	require.NoError(err)
   296  }
   297  
   298  func TestExecutor_Capabilities(t *testing.T) {
   299  	t.Parallel()
   300  	testutil.ExecCompatible(t)
   301  
   302  	cases := []struct {
   303  		user string
   304  		caps string
   305  	}{
   306  		{
   307  			user: "nobody",
   308  			caps: `
   309  CapInh: 0000000000000000
   310  CapPrm: 0000000000000000
   311  CapEff: 0000000000000000
   312  CapBnd: 0000003fffffffff
   313  CapAmb: 0000000000000000`,
   314  		},
   315  		{
   316  			user: "root",
   317  			caps: `
   318  CapInh: 0000000000000000
   319  CapPrm: 0000003fffffffff
   320  CapEff: 0000003fffffffff
   321  CapBnd: 0000003fffffffff
   322  CapAmb: 0000000000000000`,
   323  		},
   324  	}
   325  
   326  	for _, c := range cases {
   327  		t.Run(c.user, func(t *testing.T) {
   328  			require := require.New(t)
   329  
   330  			testExecCmd := testExecutorCommandWithChroot(t)
   331  			execCmd, allocDir := testExecCmd.command, testExecCmd.allocDir
   332  			defer allocDir.Destroy()
   333  
   334  			execCmd.User = c.user
   335  			execCmd.ResourceLimits = true
   336  			execCmd.Cmd = "/bin/bash"
   337  			execCmd.Args = []string{"-c", "cat /proc/$$/status"}
   338  
   339  			executor := NewExecutorWithIsolation(testlog.HCLogger(t))
   340  			defer executor.Shutdown("SIGKILL", 0)
   341  
   342  			_, err := executor.Launch(execCmd)
   343  			require.NoError(err)
   344  
   345  			ch := make(chan interface{})
   346  			go func() {
   347  				executor.Wait(context.Background())
   348  				close(ch)
   349  			}()
   350  
   351  			select {
   352  			case <-ch:
   353  				// all good
   354  			case <-time.After(5 * time.Second):
   355  				require.Fail("timeout waiting for exec to shutdown")
   356  			}
   357  
   358  			canonical := func(s string) string {
   359  				s = strings.TrimSpace(s)
   360  				s = regexp.MustCompile("[ \t]+").ReplaceAllString(s, " ")
   361  				s = regexp.MustCompile("[\n\r]+").ReplaceAllString(s, "\n")
   362  				return s
   363  			}
   364  
   365  			expected := canonical(c.caps)
   366  			tu.WaitForResult(func() (bool, error) {
   367  				output := canonical(testExecCmd.stdout.String())
   368  				if !strings.Contains(output, expected) {
   369  					return false, fmt.Errorf("capabilities didn't match: want\n%v\n; got:\n%v\n", expected, output)
   370  				}
   371  				return true, nil
   372  			}, func(err error) { require.NoError(err) })
   373  		})
   374  	}
   375  
   376  }
   377  
   378  func TestExecutor_ClientCleanup(t *testing.T) {
   379  	t.Parallel()
   380  	testutil.ExecCompatible(t)
   381  	require := require.New(t)
   382  
   383  	testExecCmd := testExecutorCommandWithChroot(t)
   384  	execCmd, allocDir := testExecCmd.command, testExecCmd.allocDir
   385  	defer allocDir.Destroy()
   386  
   387  	executor := NewExecutorWithIsolation(testlog.HCLogger(t))
   388  	defer executor.Shutdown("", 0)
   389  
   390  	// Need to run a command which will produce continuous output but not
   391  	// too quickly to ensure executor.Exit() stops the process.
   392  	execCmd.Cmd = "/bin/bash"
   393  	execCmd.Args = []string{"-c", "while true; do /bin/echo X; /bin/sleep 1; done"}
   394  	execCmd.ResourceLimits = true
   395  
   396  	ps, err := executor.Launch(execCmd)
   397  
   398  	require.NoError(err)
   399  	require.NotZero(ps.Pid)
   400  	time.Sleep(500 * time.Millisecond)
   401  	require.NoError(executor.Shutdown("SIGINT", 100*time.Millisecond))
   402  
   403  	ch := make(chan interface{})
   404  	go func() {
   405  		executor.Wait(context.Background())
   406  		close(ch)
   407  	}()
   408  
   409  	select {
   410  	case <-ch:
   411  		// all good
   412  	case <-time.After(5 * time.Second):
   413  		require.Fail("timeout waiting for exec to shutdown")
   414  	}
   415  
   416  	output := testExecCmd.stdout.String()
   417  	require.NotZero(len(output))
   418  	time.Sleep(2 * time.Second)
   419  	output1 := testExecCmd.stdout.String()
   420  	require.Equal(len(output), len(output1))
   421  }
   422  
   423  func TestExecutor_cmdDevices(t *testing.T) {
   424  	input := []*drivers.DeviceConfig{
   425  		{
   426  			HostPath:    "/dev/null",
   427  			TaskPath:    "/task/dev/null",
   428  			Permissions: "rwm",
   429  		},
   430  	}
   431  
   432  	expected := &lconfigs.Device{
   433  		Path:        "/task/dev/null",
   434  		Type:        99,
   435  		Major:       1,
   436  		Minor:       3,
   437  		Permissions: "rwm",
   438  	}
   439  
   440  	found, err := cmdDevices(input)
   441  	require.NoError(t, err)
   442  	require.Len(t, found, 1)
   443  
   444  	// ignore file permission and ownership
   445  	// as they are host specific potentially
   446  	d := found[0]
   447  	d.FileMode = 0
   448  	d.Uid = 0
   449  	d.Gid = 0
   450  
   451  	require.EqualValues(t, expected, d)
   452  }
   453  
   454  func TestExecutor_cmdMounts(t *testing.T) {
   455  	input := []*drivers.MountConfig{
   456  		{
   457  			HostPath: "/host/path-ro",
   458  			TaskPath: "/task/path-ro",
   459  			Readonly: true,
   460  		},
   461  		{
   462  			HostPath: "/host/path-rw",
   463  			TaskPath: "/task/path-rw",
   464  			Readonly: false,
   465  		},
   466  	}
   467  
   468  	expected := []*lconfigs.Mount{
   469  		{
   470  			Source:      "/host/path-ro",
   471  			Destination: "/task/path-ro",
   472  			Flags:       unix.MS_BIND | unix.MS_RDONLY,
   473  			Device:      "bind",
   474  		},
   475  		{
   476  			Source:      "/host/path-rw",
   477  			Destination: "/task/path-rw",
   478  			Flags:       unix.MS_BIND,
   479  			Device:      "bind",
   480  		},
   481  	}
   482  
   483  	require.EqualValues(t, expected, cmdMounts(input))
   484  }