github.com/Ilhicas/nomad@v1.0.4-0.20210304152020-e86851182bc3/e2e/isolation/isolation.go (about)

     1  package isolation
     2  
     3  import (
     4  	"bytes"
     5  	"context"
     6  	"fmt"
     7  	"strings"
     8  	"time"
     9  
    10  	"github.com/hashicorp/nomad/api"
    11  	"github.com/hashicorp/nomad/e2e/e2eutil"
    12  	"github.com/hashicorp/nomad/e2e/framework"
    13  	"github.com/hashicorp/nomad/helper/uuid"
    14  	"github.com/stretchr/testify/require"
    15  )
    16  
    17  type IsolationTest struct {
    18  	framework.TC
    19  
    20  	jobIDs []string
    21  }
    22  
    23  func init() {
    24  	framework.AddSuites(&framework.TestSuite{
    25  		Component:   "Isolation",
    26  		CanRunLocal: true,
    27  		Cases: []framework.TestCase{
    28  			new(IsolationTest),
    29  		},
    30  	})
    31  }
    32  
    33  func (tc *IsolationTest) BeforeAll(f *framework.F) {
    34  	t := f.T()
    35  	e2eutil.WaitForLeader(t, tc.Nomad())
    36  	e2eutil.WaitForNodesReady(t, tc.Nomad(), 1)
    37  }
    38  
    39  func (tc *IsolationTest) TestIsolation_ExecDriver_PIDNamespacing(f *framework.F) {
    40  	t := f.T()
    41  
    42  	clientNodes, err := e2eutil.ListLinuxClientNodes(tc.Nomad())
    43  	require.Nil(t, err)
    44  
    45  	if len(clientNodes) == 0 {
    46  		t.Skip("no Linux clients")
    47  	}
    48  
    49  	jobID := "isolation-pid-namespace-" + uuid.Short()
    50  	file := "isolation/input/exec.nomad"
    51  	allocs := e2eutil.RegisterAndWaitForAllocs(t, tc.Nomad(), file, jobID, "")
    52  	require.Equal(t, len(allocs), 1, fmt.Sprintf("failed to register %s", jobID))
    53  
    54  	tc.jobIDs = append(tc.jobIDs, jobID)
    55  	defer func() {
    56  		_, _, err = tc.Nomad().Jobs().Deregister(jobID, true, nil)
    57  		require.NoError(t, err)
    58  	}()
    59  
    60  	allocID := allocs[0].ID
    61  	e2eutil.WaitForAllocStopped(t, tc.Nomad(), allocID)
    62  
    63  	out, err := e2eutil.AllocLogs(allocID, e2eutil.LogsStdOut)
    64  	require.NoError(t, err, fmt.Sprintf("could not get logs for alloc %s", allocID))
    65  
    66  	require.Contains(t, out, "my pid is 1\n")
    67  }
    68  
    69  func (tc *IsolationTest) TestIsolation_ExecDriver_PIDNamespacing_host(f *framework.F) {
    70  	t := f.T()
    71  
    72  	clientNodes, err := e2eutil.ListLinuxClientNodes(tc.Nomad())
    73  	require.Nil(t, err)
    74  
    75  	if len(clientNodes) == 0 {
    76  		t.Skip("no Linux clients")
    77  	}
    78  
    79  	jobID := "isolation-pid-namespace-" + uuid.Short()
    80  	file := "isolation/input/exec_host.nomad"
    81  	allocs := e2eutil.RegisterAndWaitForAllocs(t, tc.Nomad(), file, jobID, "")
    82  	require.Equal(t, len(allocs), 1, fmt.Sprintf("failed to register %s", jobID))
    83  
    84  	tc.jobIDs = append(tc.jobIDs, jobID)
    85  	defer func() {
    86  		_, _, err = tc.Nomad().Jobs().Deregister(jobID, true, nil)
    87  		require.NoError(t, err)
    88  	}()
    89  
    90  	allocID := allocs[0].ID
    91  	e2eutil.WaitForAllocStopped(t, tc.Nomad(), allocID)
    92  
    93  	out, err := e2eutil.AllocLogs(allocID, e2eutil.LogsStdOut)
    94  	require.NoError(t, err, fmt.Sprintf("could not get logs for alloc %s", allocID))
    95  
    96  	require.NotContains(t, out, "my pid is 1\n")
    97  }
    98  
    99  func (tc *IsolationTest) TestIsolation_ExecDriver_PIDNamespacing_AllocExec(f *framework.F) {
   100  	t := f.T()
   101  
   102  	clientNodes, err := e2eutil.ListLinuxClientNodes(tc.Nomad())
   103  	require.Nil(t, err)
   104  
   105  	if len(clientNodes) == 0 {
   106  		t.Skip("no Linux clients")
   107  	}
   108  
   109  	jobID := "isolation-pid-namespace-" + uuid.Short()
   110  	file := "isolation/input/alloc_exec.nomad"
   111  	allocs := e2eutil.RegisterAndWaitForAllocs(t, tc.Nomad(), file, jobID, "")
   112  	require.Equal(t, len(allocs), 1, fmt.Sprintf("failed to register %s", jobID))
   113  
   114  	defer func() {
   115  		_, _, err = tc.Nomad().Jobs().Deregister(jobID, true, nil)
   116  		require.NoError(t, err)
   117  	}()
   118  
   119  	allocID := allocs[0].ID
   120  	e2eutil.WaitForAllocRunning(t, tc.Nomad(), allocID)
   121  
   122  	alloc, _, err := tc.Nomad().Allocations().Info(allocID, nil)
   123  	require.NoError(t, err)
   124  	require.NotNil(t, alloc)
   125  
   126  	resizeCh := make(chan api.TerminalSize)
   127  	var tty bool
   128  
   129  	ctx, cancelFn := context.WithTimeout(context.Background(), 15*time.Second)
   130  	defer cancelFn()
   131  
   132  	var stdout, stderr bytes.Buffer
   133  
   134  	exitCode, err := tc.Nomad().Allocations().Exec(
   135  		ctx,
   136  		alloc,
   137  		"main",
   138  		tty,
   139  		[]string{"ps", "ax"},
   140  		bytes.NewReader([]byte("")),
   141  		&stdout,
   142  		&stderr,
   143  		resizeCh,
   144  		nil,
   145  	)
   146  	require.NoError(t, err)
   147  	require.Equal(t, 0, exitCode)
   148  
   149  	lines := strings.Split(strings.TrimSpace(stdout.String()), "\n")
   150  	// header, sleep process, ps ax process are the only output lines expected
   151  	require.Len(t, lines, 3)
   152  }
   153  
   154  func (tc *IsolationTest) TestIsolation_JavaDriver_PIDNamespacing(f *framework.F) {
   155  	t := f.T()
   156  
   157  	clientNodes, err := e2eutil.ListLinuxClientNodes(tc.Nomad())
   158  	require.Nil(t, err)
   159  
   160  	if len(clientNodes) == 0 {
   161  		t.Skip("no Linux clients")
   162  	}
   163  
   164  	jobID := "isolation-pid-namespace-" + uuid.Short()
   165  	file := "isolation/input/java.nomad"
   166  	allocs := e2eutil.RegisterAndWaitForAllocs(t, tc.Nomad(), file, jobID, "")
   167  	require.Equal(t, len(allocs), 1, fmt.Sprintf("failed to register %s", jobID))
   168  
   169  	tc.jobIDs = append(tc.jobIDs, jobID)
   170  	defer func() {
   171  		_, _, err = tc.Nomad().Jobs().Deregister(jobID, true, nil)
   172  		require.NoError(t, err)
   173  	}()
   174  
   175  	allocID := allocs[0].ID
   176  	e2eutil.WaitForAllocStopped(t, tc.Nomad(), allocID)
   177  
   178  	out, err := e2eutil.AllocTaskLogs(allocID, "pid", e2eutil.LogsStdOut)
   179  	require.NoError(t, err, fmt.Sprintf("could not get logs for alloc %s", allocID))
   180  
   181  	require.Contains(t, out, "my pid is 1\n")
   182  }
   183  
   184  func (tc *IsolationTest) TestIsolation_JavaDriver_PIDNamespacing_host(f *framework.F) {
   185  	t := f.T()
   186  
   187  	clientNodes, err := e2eutil.ListLinuxClientNodes(tc.Nomad())
   188  	require.Nil(t, err)
   189  
   190  	if len(clientNodes) == 0 {
   191  		t.Skip("no Linux clients")
   192  	}
   193  
   194  	jobID := "isolation-pid-namespace-" + uuid.Short()
   195  	file := "isolation/input/java_host.nomad"
   196  	allocs := e2eutil.RegisterAndWaitForAllocs(t, tc.Nomad(), file, jobID, "")
   197  	require.Equal(t, len(allocs), 1, fmt.Sprintf("failed to register %s", jobID))
   198  
   199  	tc.jobIDs = append(tc.jobIDs, jobID)
   200  	defer func() {
   201  		_, _, err = tc.Nomad().Jobs().Deregister(jobID, true, nil)
   202  		require.NoError(t, err)
   203  	}()
   204  
   205  	allocID := allocs[0].ID
   206  	e2eutil.WaitForAllocStopped(t, tc.Nomad(), allocID)
   207  
   208  	out, err := e2eutil.AllocTaskLogs(allocID, "pid", e2eutil.LogsStdOut)
   209  	require.NoError(t, err, fmt.Sprintf("could not get logs for alloc %s", allocID))
   210  
   211  	require.NotContains(t, out, "my pid is 1\n")
   212  }
   213  
   214  func (tc *IsolationTest) TestIsolation_JavaDriver_PIDNamespacing_AllocExec(f *framework.F) {
   215  	t := f.T()
   216  
   217  	clientNodes, err := e2eutil.ListLinuxClientNodes(tc.Nomad())
   218  	require.Nil(t, err)
   219  
   220  	if len(clientNodes) == 0 {
   221  		t.Skip("no Linux clients")
   222  	}
   223  
   224  	jobID := "isolation-pid-namespace-" + uuid.Short()
   225  	file := "isolation/input/alloc_exec_java.nomad"
   226  	allocs := e2eutil.RegisterAndWaitForAllocs(t, tc.Nomad(), file, jobID, "")
   227  	require.Equal(t, len(allocs), 1, fmt.Sprintf("failed to register %s", jobID))
   228  
   229  	defer func() {
   230  		_, _, err = tc.Nomad().Jobs().Deregister(jobID, true, nil)
   231  		require.NoError(t, err)
   232  	}()
   233  
   234  	allocID := allocs[0].ID
   235  	e2eutil.WaitForAllocTaskRunning(t, tc.Nomad(), allocID, "sleep")
   236  
   237  	alloc, _, err := tc.Nomad().Allocations().Info(allocID, nil)
   238  	require.NoError(t, err)
   239  	require.NotNil(t, alloc)
   240  
   241  	resizeCh := make(chan api.TerminalSize)
   242  	var tty bool
   243  
   244  	ctx, cancelFn := context.WithTimeout(context.Background(), 15*time.Second)
   245  	defer cancelFn()
   246  
   247  	var stdout, stderr bytes.Buffer
   248  
   249  	exitCode, err := tc.Nomad().Allocations().Exec(
   250  		ctx,
   251  		alloc,
   252  		"sleep",
   253  		tty,
   254  		[]string{"ps", "ax"},
   255  		bytes.NewReader([]byte("")),
   256  		&stdout,
   257  		&stderr,
   258  		resizeCh,
   259  		nil,
   260  	)
   261  	require.NoError(t, err)
   262  	require.Equal(t, 0, exitCode)
   263  
   264  	lines := strings.Split(strings.TrimSpace(stdout.String()), "\n")
   265  	// header, sleep process, ps ax process are the only output lines expected
   266  	require.Len(t, lines, 3)
   267  }
   268  
   269  func (tc *IsolationTest) TestIsolation_RawExecDriver_NoPIDNamespacing(f *framework.F) {
   270  	t := f.T()
   271  
   272  	clientNodes, err := e2eutil.ListLinuxClientNodes(tc.Nomad())
   273  	require.Nil(t, err)
   274  
   275  	if len(clientNodes) == 0 {
   276  		t.Skip("no Linux clients")
   277  	}
   278  
   279  	jobID := "isolation-pid-namespace-" + uuid.Short()
   280  	file := "isolation/input/raw_exec.nomad"
   281  
   282  	allocs := e2eutil.RegisterAndWaitForAllocs(t, tc.Nomad(), file, jobID, "")
   283  	require.Equal(t, len(allocs), 1, fmt.Sprintf("failed to register %s", jobID))
   284  
   285  	defer func() {
   286  		_, _, err = tc.Nomad().Jobs().Deregister(jobID, true, nil)
   287  		require.NoError(t, err)
   288  	}()
   289  
   290  	allocID := allocs[0].ID
   291  	e2eutil.WaitForAllocStopped(t, tc.Nomad(), allocID)
   292  
   293  	out, err := e2eutil.AllocLogs(allocID, e2eutil.LogsStdOut)
   294  	require.NoError(t, err, fmt.Sprintf("could not get logs for alloc %s", allocID))
   295  
   296  	var pid uint64
   297  	_, err = fmt.Sscanf(out, "my pid is %d", &pid)
   298  	require.NoError(t, err)
   299  
   300  	require.Greater(t, pid, uint64(1))
   301  }