github.com/ferranbt/nomad@v0.9.3-0.20190607002617-85c449b7667c/drivers/shared/executor/executor_linux_test.go (about)

     1  package executor
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  	"io/ioutil"
     7  	"os"
     8  	"path/filepath"
     9  	"regexp"
    10  	"strconv"
    11  	"strings"
    12  	"testing"
    13  	"time"
    14  
    15  	"github.com/hashicorp/nomad/client/allocdir"
    16  	"github.com/hashicorp/nomad/client/taskenv"
    17  	"github.com/hashicorp/nomad/client/testutil"
    18  	"github.com/hashicorp/nomad/helper/testlog"
    19  	"github.com/hashicorp/nomad/nomad/mock"
    20  	"github.com/hashicorp/nomad/plugins/drivers"
    21  	tu "github.com/hashicorp/nomad/testutil"
    22  	lconfigs "github.com/opencontainers/runc/libcontainer/configs"
    23  	"github.com/stretchr/testify/require"
    24  	"golang.org/x/sys/unix"
    25  )
    26  
    27  func init() {
    28  	executorFactories["LibcontainerExecutor"] = libcontainerFactory
    29  }
    30  
    31  var libcontainerFactory = executorFactory{
    32  	new: NewExecutorWithIsolation,
    33  	configureExecCmd: func(t *testing.T, cmd *ExecCommand) {
    34  		cmd.ResourceLimits = true
    35  		setupRootfs(t, cmd.TaskDir)
    36  	},
    37  }
    38  
    39  // testExecutorContextWithChroot returns an ExecutorContext and AllocDir with
    40  // chroot. Use testExecutorContext if you don't need a chroot.
    41  //
    42  // The caller is responsible for calling AllocDir.Destroy() to cleanup.
    43  func testExecutorCommandWithChroot(t *testing.T) *testExecCmd {
    44  	chrootEnv := map[string]string{
    45  		"/etc/ld.so.cache":  "/etc/ld.so.cache",
    46  		"/etc/ld.so.conf":   "/etc/ld.so.conf",
    47  		"/etc/ld.so.conf.d": "/etc/ld.so.conf.d",
    48  		"/etc/passwd":       "/etc/passwd",
    49  		"/lib":              "/lib",
    50  		"/lib64":            "/lib64",
    51  		"/usr/lib":          "/usr/lib",
    52  		"/bin/ls":           "/bin/ls",
    53  		"/bin/cat":          "/bin/cat",
    54  		"/bin/echo":         "/bin/echo",
    55  		"/bin/bash":         "/bin/bash",
    56  		"/bin/sleep":        "/bin/sleep",
    57  		"/foobar":           "/does/not/exist",
    58  	}
    59  
    60  	alloc := mock.Alloc()
    61  	task := alloc.Job.TaskGroups[0].Tasks[0]
    62  	taskEnv := taskenv.NewBuilder(mock.Node(), alloc, task, "global").Build()
    63  
    64  	allocDir := allocdir.NewAllocDir(testlog.HCLogger(t), filepath.Join(os.TempDir(), alloc.ID))
    65  	if err := allocDir.Build(); err != nil {
    66  		t.Fatalf("AllocDir.Build() failed: %v", err)
    67  	}
    68  	if err := allocDir.NewTaskDir(task.Name).Build(true, chrootEnv); err != nil {
    69  		allocDir.Destroy()
    70  		t.Fatalf("allocDir.NewTaskDir(%q) failed: %v", task.Name, err)
    71  	}
    72  	td := allocDir.TaskDirs[task.Name]
    73  	cmd := &ExecCommand{
    74  		Env:     taskEnv.List(),
    75  		TaskDir: td.Dir,
    76  		Resources: &drivers.Resources{
    77  			NomadResources: alloc.AllocatedResources.Tasks[task.Name],
    78  		},
    79  	}
    80  
    81  	testCmd := &testExecCmd{
    82  		command:  cmd,
    83  		allocDir: allocDir,
    84  	}
    85  	configureTLogging(t, testCmd)
    86  	return testCmd
    87  }
    88  
    89  func TestExecutor_IsolationAndConstraints(t *testing.T) {
    90  	t.Parallel()
    91  	require := require.New(t)
    92  	testutil.ExecCompatible(t)
    93  
    94  	testExecCmd := testExecutorCommandWithChroot(t)
    95  	execCmd, allocDir := testExecCmd.command, testExecCmd.allocDir
    96  	execCmd.Cmd = "/bin/ls"
    97  	execCmd.Args = []string{"-F", "/", "/etc/"}
    98  	defer allocDir.Destroy()
    99  
   100  	execCmd.ResourceLimits = true
   101  
   102  	executor := NewExecutorWithIsolation(testlog.HCLogger(t))
   103  	defer executor.Shutdown("SIGKILL", 0)
   104  
   105  	ps, err := executor.Launch(execCmd)
   106  	require.NoError(err)
   107  	require.NotZero(ps.Pid)
   108  
   109  	state, err := executor.Wait(context.Background())
   110  	require.NoError(err)
   111  	require.Zero(state.ExitCode)
   112  
   113  	// Check if the resource constraints were applied
   114  	if lexec, ok := executor.(*LibcontainerExecutor); ok {
   115  		state, err := lexec.container.State()
   116  		require.NoError(err)
   117  
   118  		memLimits := filepath.Join(state.CgroupPaths["memory"], "memory.limit_in_bytes")
   119  		data, err := ioutil.ReadFile(memLimits)
   120  		require.NoError(err)
   121  
   122  		expectedMemLim := strconv.Itoa(int(execCmd.Resources.NomadResources.Memory.MemoryMB * 1024 * 1024))
   123  		actualMemLim := strings.TrimSpace(string(data))
   124  		require.Equal(actualMemLim, expectedMemLim)
   125  		require.NoError(executor.Shutdown("", 0))
   126  		executor.Wait(context.Background())
   127  
   128  		// Check if Nomad has actually removed the cgroups
   129  		tu.WaitForResult(func() (bool, error) {
   130  			_, err = os.Stat(memLimits)
   131  			if err == nil {
   132  				return false, fmt.Errorf("expected an error from os.Stat %s", memLimits)
   133  			}
   134  			return true, nil
   135  		}, func(err error) { t.Error(err) })
   136  
   137  	}
   138  	expected := `/:
   139  alloc/
   140  bin/
   141  dev/
   142  etc/
   143  lib/
   144  lib64/
   145  local/
   146  proc/
   147  secrets/
   148  sys/
   149  tmp/
   150  usr/
   151  
   152  /etc/:
   153  ld.so.cache
   154  ld.so.conf
   155  ld.so.conf.d/
   156  passwd`
   157  	tu.WaitForResult(func() (bool, error) {
   158  		output := testExecCmd.stdout.String()
   159  		act := strings.TrimSpace(string(output))
   160  		if act != expected {
   161  			return false, fmt.Errorf("Command output incorrectly: want %v; got %v", expected, act)
   162  		}
   163  		return true, nil
   164  	}, func(err error) { t.Error(err) })
   165  }
   166  
   167  func TestUniversalExecutor_LookupTaskBin(t *testing.T) {
   168  	t.Parallel()
   169  	require := require.New(t)
   170  
   171  	// Create a temp dir
   172  	tmpDir, err := ioutil.TempDir("", "")
   173  	require.Nil(err)
   174  	defer os.Remove(tmpDir)
   175  
   176  	// Create the command
   177  	cmd := &ExecCommand{Env: []string{"PATH=/bin"}, TaskDir: tmpDir}
   178  
   179  	// Make a foo subdir
   180  	os.MkdirAll(filepath.Join(tmpDir, "foo"), 0700)
   181  
   182  	// Write a file under foo
   183  	filePath := filepath.Join(tmpDir, "foo", "tmp.txt")
   184  	err = ioutil.WriteFile(filePath, []byte{1, 2}, os.ModeAppend)
   185  	require.NoError(err)
   186  
   187  	// Lookout with an absolute path to the binary
   188  	cmd.Cmd = "/foo/tmp.txt"
   189  	_, err = lookupTaskBin(cmd)
   190  	require.NoError(err)
   191  
   192  	// Write a file under local subdir
   193  	os.MkdirAll(filepath.Join(tmpDir, "local"), 0700)
   194  	filePath2 := filepath.Join(tmpDir, "local", "tmp.txt")
   195  	ioutil.WriteFile(filePath2, []byte{1, 2}, os.ModeAppend)
   196  
   197  	// Lookup with file name, should find the one we wrote above
   198  	cmd.Cmd = "tmp.txt"
   199  	_, err = lookupTaskBin(cmd)
   200  	require.NoError(err)
   201  
   202  	// Lookup a host absolute path
   203  	cmd.Cmd = "/bin/sh"
   204  	_, err = lookupTaskBin(cmd)
   205  	require.Error(err)
   206  }
   207  
   208  // Exec Launch looks for the binary only inside the chroot
   209  func TestExecutor_EscapeContainer(t *testing.T) {
   210  	t.Parallel()
   211  	require := require.New(t)
   212  	testutil.ExecCompatible(t)
   213  
   214  	testExecCmd := testExecutorCommandWithChroot(t)
   215  	execCmd, allocDir := testExecCmd.command, testExecCmd.allocDir
   216  	execCmd.Cmd = "/bin/kill" // missing from the chroot container
   217  	defer allocDir.Destroy()
   218  
   219  	execCmd.ResourceLimits = true
   220  
   221  	executor := NewExecutorWithIsolation(testlog.HCLogger(t))
   222  	defer executor.Shutdown("SIGKILL", 0)
   223  
   224  	_, err := executor.Launch(execCmd)
   225  	require.Error(err)
   226  	require.Regexp("^file /bin/kill not found under path", err)
   227  
   228  	// Bare files are looked up using the system path, inside the container
   229  	allocDir.Destroy()
   230  	testExecCmd = testExecutorCommandWithChroot(t)
   231  	execCmd, allocDir = testExecCmd.command, testExecCmd.allocDir
   232  	execCmd.Cmd = "kill"
   233  	_, err = executor.Launch(execCmd)
   234  	require.Error(err)
   235  	require.Regexp("^file kill not found under path", err)
   236  
   237  	allocDir.Destroy()
   238  	testExecCmd = testExecutorCommandWithChroot(t)
   239  	execCmd, allocDir = testExecCmd.command, testExecCmd.allocDir
   240  	execCmd.Cmd = "echo"
   241  	_, err = executor.Launch(execCmd)
   242  	require.NoError(err)
   243  }
   244  
   245  func TestExecutor_Capabilities(t *testing.T) {
   246  	t.Parallel()
   247  	testutil.ExecCompatible(t)
   248  
   249  	cases := []struct {
   250  		user string
   251  		caps string
   252  	}{
   253  		{
   254  			user: "nobody",
   255  			caps: `
   256  CapInh: 0000000000000000
   257  CapPrm: 0000000000000000
   258  CapEff: 0000000000000000
   259  CapBnd: 0000003fffffffff
   260  CapAmb: 0000000000000000`,
   261  		},
   262  		{
   263  			user: "root",
   264  			caps: `
   265  CapInh: 0000000000000000
   266  CapPrm: 0000003fffffffff
   267  CapEff: 0000003fffffffff
   268  CapBnd: 0000003fffffffff
   269  CapAmb: 0000000000000000`,
   270  		},
   271  	}
   272  
   273  	for _, c := range cases {
   274  		t.Run(c.user, func(t *testing.T) {
   275  			require := require.New(t)
   276  
   277  			testExecCmd := testExecutorCommandWithChroot(t)
   278  			execCmd, allocDir := testExecCmd.command, testExecCmd.allocDir
   279  			defer allocDir.Destroy()
   280  
   281  			execCmd.User = c.user
   282  			execCmd.ResourceLimits = true
   283  			execCmd.Cmd = "/bin/bash"
   284  			execCmd.Args = []string{"-c", "cat /proc/$$/status"}
   285  
   286  			executor := NewExecutorWithIsolation(testlog.HCLogger(t))
   287  			defer executor.Shutdown("SIGKILL", 0)
   288  
   289  			_, err := executor.Launch(execCmd)
   290  			require.NoError(err)
   291  
   292  			ch := make(chan interface{})
   293  			go func() {
   294  				executor.Wait(context.Background())
   295  				close(ch)
   296  			}()
   297  
   298  			select {
   299  			case <-ch:
   300  				// all good
   301  			case <-time.After(5 * time.Second):
   302  				require.Fail("timeout waiting for exec to shutdown")
   303  			}
   304  
   305  			canonical := func(s string) string {
   306  				s = strings.TrimSpace(s)
   307  				s = regexp.MustCompile("[ \t]+").ReplaceAllString(s, " ")
   308  				s = regexp.MustCompile("[\n\r]+").ReplaceAllString(s, "\n")
   309  				return s
   310  			}
   311  
   312  			expected := canonical(c.caps)
   313  			tu.WaitForResult(func() (bool, error) {
   314  				output := canonical(testExecCmd.stdout.String())
   315  				if !strings.Contains(output, expected) {
   316  					return false, fmt.Errorf("capabilities didn't match: want\n%v\n; got:\n%v\n", expected, output)
   317  				}
   318  				return true, nil
   319  			}, func(err error) { require.NoError(err) })
   320  		})
   321  	}
   322  
   323  }
   324  
   325  func TestExecutor_ClientCleanup(t *testing.T) {
   326  	t.Parallel()
   327  	testutil.ExecCompatible(t)
   328  	require := require.New(t)
   329  
   330  	testExecCmd := testExecutorCommandWithChroot(t)
   331  	execCmd, allocDir := testExecCmd.command, testExecCmd.allocDir
   332  	defer allocDir.Destroy()
   333  
   334  	executor := NewExecutorWithIsolation(testlog.HCLogger(t))
   335  	defer executor.Shutdown("", 0)
   336  
   337  	// Need to run a command which will produce continuous output but not
   338  	// too quickly to ensure executor.Exit() stops the process.
   339  	execCmd.Cmd = "/bin/bash"
   340  	execCmd.Args = []string{"-c", "while true; do /bin/echo X; /bin/sleep 1; done"}
   341  	execCmd.ResourceLimits = true
   342  
   343  	ps, err := executor.Launch(execCmd)
   344  
   345  	require.NoError(err)
   346  	require.NotZero(ps.Pid)
   347  	time.Sleep(500 * time.Millisecond)
   348  	require.NoError(executor.Shutdown("SIGINT", 100*time.Millisecond))
   349  
   350  	ch := make(chan interface{})
   351  	go func() {
   352  		executor.Wait(context.Background())
   353  		close(ch)
   354  	}()
   355  
   356  	select {
   357  	case <-ch:
   358  		// all good
   359  	case <-time.After(5 * time.Second):
   360  		require.Fail("timeout waiting for exec to shutdown")
   361  	}
   362  
   363  	output := testExecCmd.stdout.String()
   364  	require.NotZero(len(output))
   365  	time.Sleep(2 * time.Second)
   366  	output1 := testExecCmd.stdout.String()
   367  	require.Equal(len(output), len(output1))
   368  }
   369  
   370  func TestExecutor_cmdDevices(t *testing.T) {
   371  	input := []*drivers.DeviceConfig{
   372  		{
   373  			HostPath:    "/dev/null",
   374  			TaskPath:    "/task/dev/null",
   375  			Permissions: "rwm",
   376  		},
   377  	}
   378  
   379  	expected := &lconfigs.Device{
   380  		Path:        "/task/dev/null",
   381  		Type:        99,
   382  		Major:       1,
   383  		Minor:       3,
   384  		Permissions: "rwm",
   385  	}
   386  
   387  	found, err := cmdDevices(input)
   388  	require.NoError(t, err)
   389  	require.Len(t, found, 1)
   390  
   391  	// ignore file permission and ownership
   392  	// as they are host specific potentially
   393  	d := found[0]
   394  	d.FileMode = 0
   395  	d.Uid = 0
   396  	d.Gid = 0
   397  
   398  	require.EqualValues(t, expected, d)
   399  }
   400  
   401  func TestExecutor_cmdMounts(t *testing.T) {
   402  	input := []*drivers.MountConfig{
   403  		{
   404  			HostPath: "/host/path-ro",
   405  			TaskPath: "/task/path-ro",
   406  			Readonly: true,
   407  		},
   408  		{
   409  			HostPath: "/host/path-rw",
   410  			TaskPath: "/task/path-rw",
   411  			Readonly: false,
   412  		},
   413  	}
   414  
   415  	expected := []*lconfigs.Mount{
   416  		{
   417  			Source:      "/host/path-ro",
   418  			Destination: "/task/path-ro",
   419  			Flags:       unix.MS_BIND | unix.MS_RDONLY,
   420  			Device:      "bind",
   421  		},
   422  		{
   423  			Source:      "/host/path-rw",
   424  			Destination: "/task/path-rw",
   425  			Flags:       unix.MS_BIND,
   426  			Device:      "bind",
   427  		},
   428  	}
   429  
   430  	require.EqualValues(t, expected, cmdMounts(input))
   431  }