github.com/iqoqo/nomad@v0.11.3-0.20200911112621-d7021c74d101/drivers/rawexec/driver_test.go (about)

     1  package rawexec
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  	"io/ioutil"
     7  	"os"
     8  	"path/filepath"
     9  	"runtime"
    10  	"strconv"
    11  	"sync"
    12  	"syscall"
    13  	"testing"
    14  	"time"
    15  
    16  	ctestutil "github.com/hashicorp/nomad/client/testutil"
    17  	"github.com/hashicorp/nomad/helper/pluginutils/hclutils"
    18  	"github.com/hashicorp/nomad/helper/testlog"
    19  	"github.com/hashicorp/nomad/helper/testtask"
    20  	"github.com/hashicorp/nomad/helper/uuid"
    21  	basePlug "github.com/hashicorp/nomad/plugins/base"
    22  	"github.com/hashicorp/nomad/plugins/drivers"
    23  	dtestutil "github.com/hashicorp/nomad/plugins/drivers/testutils"
    24  	pstructs "github.com/hashicorp/nomad/plugins/shared/structs"
    25  	"github.com/hashicorp/nomad/testutil"
    26  	"github.com/stretchr/testify/require"
    27  )
    28  
    29  func TestMain(m *testing.M) {
    30  	if !testtask.Run() {
    31  		os.Exit(m.Run())
    32  	}
    33  }
    34  
    35  func newEnabledRawExecDriver(t *testing.T) *Driver {
    36  	d := NewRawExecDriver(testlog.HCLogger(t)).(*Driver)
    37  	d.config.Enabled = true
    38  	return d
    39  }
    40  
    41  func TestRawExecDriver_SetConfig(t *testing.T) {
    42  	t.Parallel()
    43  	require := require.New(t)
    44  
    45  	d := NewRawExecDriver(testlog.HCLogger(t))
    46  	harness := dtestutil.NewDriverHarness(t, d)
    47  	defer harness.Kill()
    48  
    49  	bconfig := &basePlug.Config{}
    50  
    51  	// Disable raw exec.
    52  	config := &Config{}
    53  
    54  	var data []byte
    55  	require.NoError(basePlug.MsgPackEncode(&data, config))
    56  	bconfig.PluginConfig = data
    57  	require.NoError(harness.SetConfig(bconfig))
    58  	require.Exactly(config, d.(*Driver).config)
    59  
    60  	config.Enabled = true
    61  	config.NoCgroups = true
    62  	data = []byte{}
    63  	require.NoError(basePlug.MsgPackEncode(&data, config))
    64  	bconfig.PluginConfig = data
    65  	require.NoError(harness.SetConfig(bconfig))
    66  	require.Exactly(config, d.(*Driver).config)
    67  
    68  	config.NoCgroups = false
    69  	data = []byte{}
    70  	require.NoError(basePlug.MsgPackEncode(&data, config))
    71  	bconfig.PluginConfig = data
    72  	require.NoError(harness.SetConfig(bconfig))
    73  	require.Exactly(config, d.(*Driver).config)
    74  }
    75  
    76  func TestRawExecDriver_Fingerprint(t *testing.T) {
    77  	t.Parallel()
    78  
    79  	fingerprintTest := func(config *Config, expected *drivers.Fingerprint) func(t *testing.T) {
    80  		return func(t *testing.T) {
    81  			require := require.New(t)
    82  			d := newEnabledRawExecDriver(t)
    83  			harness := dtestutil.NewDriverHarness(t, d)
    84  			defer harness.Kill()
    85  
    86  			var data []byte
    87  			require.NoError(basePlug.MsgPackEncode(&data, config))
    88  			bconfig := &basePlug.Config{
    89  				PluginConfig: data,
    90  			}
    91  			require.NoError(harness.SetConfig(bconfig))
    92  
    93  			fingerCh, err := harness.Fingerprint(context.Background())
    94  			require.NoError(err)
    95  			select {
    96  			case result := <-fingerCh:
    97  				require.Equal(expected, result)
    98  			case <-time.After(time.Duration(testutil.TestMultiplier()) * time.Second):
    99  				require.Fail("timeout receiving fingerprint")
   100  			}
   101  		}
   102  	}
   103  
   104  	cases := []struct {
   105  		Name     string
   106  		Conf     Config
   107  		Expected drivers.Fingerprint
   108  	}{
   109  		{
   110  			Name: "Disabled",
   111  			Conf: Config{
   112  				Enabled: false,
   113  			},
   114  			Expected: drivers.Fingerprint{
   115  				Attributes:        nil,
   116  				Health:            drivers.HealthStateUndetected,
   117  				HealthDescription: "disabled",
   118  			},
   119  		},
   120  		{
   121  			Name: "Enabled",
   122  			Conf: Config{
   123  				Enabled: true,
   124  			},
   125  			Expected: drivers.Fingerprint{
   126  				Attributes:        map[string]*pstructs.Attribute{"driver.raw_exec": pstructs.NewBoolAttribute(true)},
   127  				Health:            drivers.HealthStateHealthy,
   128  				HealthDescription: drivers.DriverHealthy,
   129  			},
   130  		},
   131  	}
   132  
   133  	for _, tc := range cases {
   134  		t.Run(tc.Name, fingerprintTest(&tc.Conf, &tc.Expected))
   135  	}
   136  }
   137  
   138  func TestRawExecDriver_StartWait(t *testing.T) {
   139  	t.Parallel()
   140  	require := require.New(t)
   141  
   142  	d := newEnabledRawExecDriver(t)
   143  	harness := dtestutil.NewDriverHarness(t, d)
   144  	defer harness.Kill()
   145  	task := &drivers.TaskConfig{
   146  		ID:   uuid.Generate(),
   147  		Name: "test",
   148  	}
   149  
   150  	tc := &TaskConfig{
   151  		Command: testtask.Path(),
   152  		Args:    []string{"sleep", "10ms"},
   153  	}
   154  	require.NoError(task.EncodeConcreteDriverConfig(&tc))
   155  	testtask.SetTaskConfigEnv(task)
   156  
   157  	cleanup := harness.MkAllocDir(task, false)
   158  	defer cleanup()
   159  
   160  	handle, _, err := harness.StartTask(task)
   161  	require.NoError(err)
   162  
   163  	ch, err := harness.WaitTask(context.Background(), handle.Config.ID)
   164  	require.NoError(err)
   165  
   166  	var result *drivers.ExitResult
   167  	select {
   168  	case result = <-ch:
   169  	case <-time.After(5 * time.Second):
   170  		t.Fatal("timed out")
   171  	}
   172  
   173  	require.Zero(result.ExitCode)
   174  	require.Zero(result.Signal)
   175  	require.False(result.OOMKilled)
   176  	require.NoError(result.Err)
   177  	require.NoError(harness.DestroyTask(task.ID, true))
   178  }
   179  
   180  func TestRawExecDriver_StartWaitRecoverWaitStop(t *testing.T) {
   181  	t.Parallel()
   182  	require := require.New(t)
   183  
   184  	d := newEnabledRawExecDriver(t)
   185  	harness := dtestutil.NewDriverHarness(t, d)
   186  	defer harness.Kill()
   187  
   188  	// Disable cgroups so test works without root
   189  	config := &Config{NoCgroups: true, Enabled: true}
   190  	var data []byte
   191  	require.NoError(basePlug.MsgPackEncode(&data, config))
   192  	bconfig := &basePlug.Config{PluginConfig: data}
   193  	require.NoError(harness.SetConfig(bconfig))
   194  
   195  	task := &drivers.TaskConfig{
   196  		ID:   uuid.Generate(),
   197  		Name: "sleep",
   198  	}
   199  	tc := &TaskConfig{
   200  		Command: testtask.Path(),
   201  		Args:    []string{"sleep", "100s"},
   202  	}
   203  	require.NoError(task.EncodeConcreteDriverConfig(&tc))
   204  
   205  	testtask.SetTaskConfigEnv(task)
   206  	cleanup := harness.MkAllocDir(task, false)
   207  	defer cleanup()
   208  
   209  	handle, _, err := harness.StartTask(task)
   210  	require.NoError(err)
   211  
   212  	ch, err := harness.WaitTask(context.Background(), task.ID)
   213  	require.NoError(err)
   214  
   215  	var waitDone bool
   216  	var wg sync.WaitGroup
   217  	wg.Add(1)
   218  	go func() {
   219  		defer wg.Done()
   220  		result := <-ch
   221  		require.Error(result.Err)
   222  		waitDone = true
   223  	}()
   224  
   225  	originalStatus, err := d.InspectTask(task.ID)
   226  	require.NoError(err)
   227  
   228  	d.tasks.Delete(task.ID)
   229  
   230  	wg.Wait()
   231  	require.True(waitDone)
   232  	_, err = d.InspectTask(task.ID)
   233  	require.Equal(drivers.ErrTaskNotFound, err)
   234  
   235  	err = d.RecoverTask(handle)
   236  	require.NoError(err)
   237  
   238  	status, err := d.InspectTask(task.ID)
   239  	require.NoError(err)
   240  	require.Exactly(originalStatus, status)
   241  
   242  	ch, err = harness.WaitTask(context.Background(), task.ID)
   243  	require.NoError(err)
   244  
   245  	wg.Add(1)
   246  	waitDone = false
   247  	go func() {
   248  		defer wg.Done()
   249  		result := <-ch
   250  		require.NoError(result.Err)
   251  		require.NotZero(result.ExitCode)
   252  		require.Equal(9, result.Signal)
   253  		waitDone = true
   254  	}()
   255  
   256  	time.Sleep(300 * time.Millisecond)
   257  	require.NoError(d.StopTask(task.ID, 0, "SIGKILL"))
   258  	wg.Wait()
   259  	require.NoError(d.DestroyTask(task.ID, false))
   260  	require.True(waitDone)
   261  }
   262  
   263  func TestRawExecDriver_Start_Wait_AllocDir(t *testing.T) {
   264  	t.Parallel()
   265  	require := require.New(t)
   266  
   267  	d := newEnabledRawExecDriver(t)
   268  	harness := dtestutil.NewDriverHarness(t, d)
   269  	defer harness.Kill()
   270  
   271  	task := &drivers.TaskConfig{
   272  		ID:   uuid.Generate(),
   273  		Name: "sleep",
   274  	}
   275  
   276  	cleanup := harness.MkAllocDir(task, false)
   277  	defer cleanup()
   278  
   279  	exp := []byte("win")
   280  	file := "output.txt"
   281  	outPath := fmt.Sprintf(`%s/%s`, task.TaskDir().SharedAllocDir, file)
   282  
   283  	tc := &TaskConfig{
   284  		Command: testtask.Path(),
   285  		Args:    []string{"sleep", "1s", "write", string(exp), outPath},
   286  	}
   287  	require.NoError(task.EncodeConcreteDriverConfig(&tc))
   288  	testtask.SetTaskConfigEnv(task)
   289  
   290  	_, _, err := harness.StartTask(task)
   291  	require.NoError(err)
   292  
   293  	// Task should terminate quickly
   294  	waitCh, err := harness.WaitTask(context.Background(), task.ID)
   295  	require.NoError(err)
   296  
   297  	select {
   298  	case res := <-waitCh:
   299  		require.NoError(res.Err)
   300  		require.True(res.Successful())
   301  	case <-time.After(time.Duration(testutil.TestMultiplier()*5) * time.Second):
   302  		require.Fail("WaitTask timeout")
   303  	}
   304  
   305  	// Check that data was written to the shared alloc directory.
   306  	outputFile := filepath.Join(task.TaskDir().SharedAllocDir, file)
   307  	act, err := ioutil.ReadFile(outputFile)
   308  	require.NoError(err)
   309  	require.Exactly(exp, act)
   310  	require.NoError(harness.DestroyTask(task.ID, true))
   311  }
   312  
   313  // This test creates a process tree such that without cgroups tracking the
   314  // processes cleanup of the children would not be possible. Thus the test
   315  // asserts that the processes get killed properly when using cgroups.
   316  func TestRawExecDriver_Start_Kill_Wait_Cgroup(t *testing.T) {
   317  	ctestutil.ExecCompatible(t)
   318  	t.Parallel()
   319  	require := require.New(t)
   320  	pidFile := "pid"
   321  
   322  	d := newEnabledRawExecDriver(t)
   323  	harness := dtestutil.NewDriverHarness(t, d)
   324  	defer harness.Kill()
   325  
   326  	task := &drivers.TaskConfig{
   327  		ID:   uuid.Generate(),
   328  		Name: "sleep",
   329  		User: "root",
   330  	}
   331  
   332  	cleanup := harness.MkAllocDir(task, false)
   333  	defer cleanup()
   334  
   335  	tc := &TaskConfig{
   336  		Command: testtask.Path(),
   337  		Args:    []string{"fork/exec", pidFile, "pgrp", "0", "sleep", "20s"},
   338  	}
   339  	require.NoError(task.EncodeConcreteDriverConfig(&tc))
   340  	testtask.SetTaskConfigEnv(task)
   341  
   342  	_, _, err := harness.StartTask(task)
   343  	require.NoError(err)
   344  
   345  	// Find the process
   346  	var pidData []byte
   347  	testutil.WaitForResult(func() (bool, error) {
   348  		var err error
   349  		pidData, err = ioutil.ReadFile(filepath.Join(task.TaskDir().Dir, pidFile))
   350  		if err != nil {
   351  			return false, err
   352  		}
   353  
   354  		if len(pidData) == 0 {
   355  			return false, fmt.Errorf("pidFile empty")
   356  		}
   357  
   358  		return true, nil
   359  	}, func(err error) {
   360  		require.NoError(err)
   361  	})
   362  
   363  	pid, err := strconv.Atoi(string(pidData))
   364  	require.NoError(err, "failed to read pidData: %s", string(pidData))
   365  
   366  	// Check the pid is up
   367  	process, err := os.FindProcess(pid)
   368  	require.NoError(err)
   369  	require.NoError(process.Signal(syscall.Signal(0)))
   370  
   371  	var wg sync.WaitGroup
   372  	wg.Add(1)
   373  	go func() {
   374  		defer wg.Done()
   375  		time.Sleep(1 * time.Second)
   376  		err := harness.StopTask(task.ID, 0, "")
   377  
   378  		// Can't rely on the ordering between wait and kill on CI/travis...
   379  		if !testutil.IsCI() {
   380  			require.NoError(err)
   381  		}
   382  	}()
   383  
   384  	// Task should terminate quickly
   385  	waitCh, err := harness.WaitTask(context.Background(), task.ID)
   386  	require.NoError(err)
   387  	select {
   388  	case res := <-waitCh:
   389  		require.False(res.Successful())
   390  	case <-time.After(time.Duration(testutil.TestMultiplier()*5) * time.Second):
   391  		require.Fail("WaitTask timeout")
   392  	}
   393  
   394  	testutil.WaitForResult(func() (bool, error) {
   395  		if err := process.Signal(syscall.Signal(0)); err == nil {
   396  			return false, fmt.Errorf("process should not exist: %v", pid)
   397  		}
   398  
   399  		return true, nil
   400  	}, func(err error) {
   401  		require.NoError(err)
   402  	})
   403  
   404  	wg.Wait()
   405  	require.NoError(harness.DestroyTask(task.ID, true))
   406  }
   407  
   408  func TestRawExecDriver_Exec(t *testing.T) {
   409  	t.Parallel()
   410  	require := require.New(t)
   411  
   412  	d := newEnabledRawExecDriver(t)
   413  	harness := dtestutil.NewDriverHarness(t, d)
   414  	defer harness.Kill()
   415  
   416  	task := &drivers.TaskConfig{
   417  		ID:   uuid.Generate(),
   418  		Name: "sleep",
   419  	}
   420  
   421  	cleanup := harness.MkAllocDir(task, false)
   422  	defer cleanup()
   423  
   424  	tc := &TaskConfig{
   425  		Command: testtask.Path(),
   426  		Args:    []string{"sleep", "9000s"},
   427  	}
   428  	require.NoError(task.EncodeConcreteDriverConfig(&tc))
   429  	testtask.SetTaskConfigEnv(task)
   430  
   431  	_, _, err := harness.StartTask(task)
   432  	require.NoError(err)
   433  
   434  	if runtime.GOOS == "windows" {
   435  		// Exec a command that should work
   436  		res, err := harness.ExecTask(task.ID, []string{"cmd.exe", "/c", "echo", "hello"}, 1*time.Second)
   437  		require.NoError(err)
   438  		require.True(res.ExitResult.Successful())
   439  		require.Equal(string(res.Stdout), "hello\r\n")
   440  
   441  		// Exec a command that should fail
   442  		res, err = harness.ExecTask(task.ID, []string{"cmd.exe", "/c", "stat", "notarealfile123abc"}, 1*time.Second)
   443  		require.NoError(err)
   444  		require.False(res.ExitResult.Successful())
   445  		require.Contains(string(res.Stdout), "not recognized")
   446  	} else {
   447  		// Exec a command that should work
   448  		res, err := harness.ExecTask(task.ID, []string{"/usr/bin/stat", "/tmp"}, 1*time.Second)
   449  		require.NoError(err)
   450  		require.True(res.ExitResult.Successful())
   451  		require.True(len(res.Stdout) > 100)
   452  
   453  		// Exec a command that should fail
   454  		res, err = harness.ExecTask(task.ID, []string{"/usr/bin/stat", "notarealfile123abc"}, 1*time.Second)
   455  		require.NoError(err)
   456  		require.False(res.ExitResult.Successful())
   457  		require.Contains(string(res.Stdout), "No such file or directory")
   458  	}
   459  
   460  	require.NoError(harness.DestroyTask(task.ID, true))
   461  }
   462  
   463  func TestConfig_ParseAllHCL(t *testing.T) {
   464  	cfgStr := `
   465  config {
   466    command = "/bin/bash"
   467    args = ["-c", "echo hello"]
   468  }`
   469  
   470  	expected := &TaskConfig{
   471  		Command: "/bin/bash",
   472  		Args:    []string{"-c", "echo hello"},
   473  	}
   474  
   475  	var tc *TaskConfig
   476  	hclutils.NewConfigParser(taskConfigSpec).ParseHCL(t, cfgStr, &tc)
   477  
   478  	require.EqualValues(t, expected, tc)
   479  }
   480  
   481  func TestRawExecDriver_Disabled(t *testing.T) {
   482  	t.Parallel()
   483  	require := require.New(t)
   484  
   485  	d := newEnabledRawExecDriver(t)
   486  	d.config.Enabled = false
   487  
   488  	harness := dtestutil.NewDriverHarness(t, d)
   489  	defer harness.Kill()
   490  	task := &drivers.TaskConfig{
   491  		ID:   uuid.Generate(),
   492  		Name: "test",
   493  	}
   494  
   495  	handle, _, err := harness.StartTask(task)
   496  	require.Error(err)
   497  	require.Contains(err.Error(), errDisabledDriver.Error())
   498  	require.Nil(handle)
   499  }