github.com/anth0d/nomad@v0.0.0-20221214183521-ae3a0a2cad06/e2e/consul/namespaces_oss.go (about)

     1  //go:build !ent
     2  // +build !ent
     3  
     4  // Nomad OSS ignores Consul Namespace configuration in jobs, these e2e tests
     5  // verify everything still works and is registered into the "default" namespace,
     6  // since e2e always uses Consul Enterprise. With Consul OSS, there  are no namespaces.
     7  // and these tests will not work.
     8  
     9  package consul
    10  
    11  import (
    12  	"os"
    13  	"sort"
    14  
    15  	capi "github.com/hashicorp/consul/api"
    16  	"github.com/hashicorp/nomad/e2e/e2eutil"
    17  	"github.com/hashicorp/nomad/e2e/framework"
    18  	"github.com/stretchr/testify/require"
    19  )
    20  
    21  func (tc *ConsulNamespacesE2ETest) AfterEach(f *framework.F) {
    22  	if os.Getenv("NOMAD_TEST_SKIPCLEANUP") == "1" {
    23  		return
    24  	}
    25  
    26  	// cleanup jobs
    27  	for _, id := range tc.jobIDs {
    28  		_, _, err := tc.Nomad().Jobs().Deregister(id, true, nil)
    29  		f.NoError(err)
    30  	}
    31  
    32  	// do garbage collection
    33  	err := tc.Nomad().System().GarbageCollect()
    34  	f.NoError(err)
    35  
    36  	// reset accumulators
    37  	tc.tokenIDs = make(map[string][]string)
    38  	tc.policyIDs = make(map[string][]string)
    39  }
    40  
    41  func (tc *ConsulNamespacesE2ETest) TestConsulRegisterGroupServices(f *framework.F) {
    42  	nomadClient := tc.Nomad()
    43  	jobID := "cns-group-services"
    44  	tc.jobIDs = append(tc.jobIDs, jobID)
    45  
    46  	// Run job and wait for allocs
    47  	allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, cnsJobGroupServices, jobID, "")
    48  	require.Len(f.T(), allocations, 3)
    49  	allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations)
    50  	e2eutil.WaitForAllocsRunning(f.T(), tc.Nomad(), allocIDs)
    51  
    52  	r := f.Assertions
    53  	c := tc.Consul()
    54  	namespace := consulNamespace
    55  
    56  	// Verify our services were registered into "default"
    57  	e2eutil.RequireConsulRegistered(r, c, namespace, "b1", 1)
    58  	e2eutil.RequireConsulRegistered(r, c, namespace, "b2", 1)
    59  	e2eutil.RequireConsulRegistered(r, c, namespace, "c1", 1)
    60  	e2eutil.RequireConsulRegistered(r, c, namespace, "c2", 1)
    61  	e2eutil.RequireConsulRegistered(r, c, namespace, "z1", 1)
    62  	e2eutil.RequireConsulRegistered(r, c, namespace, "z2", 1)
    63  
    64  	// Verify our services are all healthy
    65  	e2eutil.RequireConsulStatus(r, c, namespace, "b1", "passing")
    66  	e2eutil.RequireConsulStatus(r, c, namespace, "b2", "passing")
    67  	e2eutil.RequireConsulStatus(r, c, namespace, "c1", "passing")
    68  	e2eutil.RequireConsulStatus(r, c, namespace, "c2", "passing")
    69  	e2eutil.RequireConsulStatus(r, c, namespace, "z1", "passing")
    70  	e2eutil.RequireConsulStatus(r, c, namespace, "z2", "passing")
    71  
    72  	// Verify our services were NOT registered into specified consul namespaces
    73  	e2eutil.RequireConsulRegistered(r, c, "banana", "b1", 0)
    74  	e2eutil.RequireConsulRegistered(r, c, "banana", "b2", 0)
    75  	e2eutil.RequireConsulRegistered(r, c, "cherry", "c1", 0)
    76  	e2eutil.RequireConsulRegistered(r, c, "cherry", "c2", 0)
    77  
    78  	// Stop the job
    79  	e2eutil.WaitForJobStopped(f.T(), nomadClient, jobID)
    80  
    81  	// Verify that services were de-registered in Consul
    82  	e2eutil.RequireConsulDeregistered(r, c, namespace, "b1")
    83  	e2eutil.RequireConsulDeregistered(r, c, namespace, "b2")
    84  	e2eutil.RequireConsulDeregistered(r, c, namespace, "c1")
    85  	e2eutil.RequireConsulDeregistered(r, c, namespace, "c2")
    86  	e2eutil.RequireConsulDeregistered(r, c, namespace, "z1")
    87  	e2eutil.RequireConsulDeregistered(r, c, namespace, "z2")
    88  }
    89  
    90  func (tc *ConsulNamespacesE2ETest) TestConsulRegisterTaskServices(f *framework.F) {
    91  	nomadClient := tc.Nomad()
    92  	jobID := "cns-task-services"
    93  	tc.jobIDs = append(tc.jobIDs, jobID)
    94  
    95  	// Run job and wait for allocs
    96  	allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, cnsJobTaskServices, jobID, "")
    97  	require.Len(f.T(), allocations, 3)
    98  	allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations)
    99  	e2eutil.WaitForAllocsRunning(f.T(), tc.Nomad(), allocIDs)
   100  
   101  	r := f.Assertions
   102  	c := tc.Consul()
   103  	namespace := consulNamespace
   104  
   105  	// Verify our services were registered into "default"
   106  	e2eutil.RequireConsulRegistered(r, c, namespace, "b1", 1)
   107  	e2eutil.RequireConsulRegistered(r, c, namespace, "b2", 1)
   108  	e2eutil.RequireConsulRegistered(r, c, namespace, "c1", 1)
   109  	e2eutil.RequireConsulRegistered(r, c, namespace, "c2", 1)
   110  	e2eutil.RequireConsulRegistered(r, c, namespace, "z1", 1)
   111  	e2eutil.RequireConsulRegistered(r, c, namespace, "z2", 1)
   112  
   113  	// Verify our services are all healthy
   114  	e2eutil.RequireConsulStatus(r, c, namespace, "b1", "passing")
   115  	e2eutil.RequireConsulStatus(r, c, namespace, "b2", "passing")
   116  	e2eutil.RequireConsulStatus(r, c, namespace, "c1", "passing")
   117  	e2eutil.RequireConsulStatus(r, c, namespace, "c2", "passing")
   118  	e2eutil.RequireConsulStatus(r, c, namespace, "z1", "passing")
   119  	e2eutil.RequireConsulStatus(r, c, namespace, "z2", "passing")
   120  
   121  	// Verify our services were NOT registered into specified consul namespaces
   122  	e2eutil.RequireConsulRegistered(r, c, "banana", "b1", 0)
   123  	e2eutil.RequireConsulRegistered(r, c, "banana", "b2", 0)
   124  	e2eutil.RequireConsulRegistered(r, c, "cherry", "c1", 0)
   125  	e2eutil.RequireConsulRegistered(r, c, "cherry", "c2", 0)
   126  	e2eutil.RequireConsulRegistered(r, c, "cherry", "z1", 0)
   127  	e2eutil.RequireConsulRegistered(r, c, "cherry", "z2", 0)
   128  
   129  	// Stop the job
   130  	e2eutil.WaitForJobStopped(f.T(), nomadClient, jobID)
   131  
   132  	// Verify that services were de-registered from Consul
   133  	e2eutil.RequireConsulDeregistered(r, c, namespace, "b1")
   134  	e2eutil.RequireConsulDeregistered(r, c, namespace, "b2")
   135  	e2eutil.RequireConsulDeregistered(r, c, namespace, "c1")
   136  	e2eutil.RequireConsulDeregistered(r, c, namespace, "b2")
   137  	e2eutil.RequireConsulDeregistered(r, c, namespace, "z1")
   138  	e2eutil.RequireConsulDeregistered(r, c, namespace, "z2")
   139  }
   140  
   141  func (tc *ConsulNamespacesE2ETest) TestConsulTemplateKV(f *framework.F) {
   142  	t := f.T()
   143  	nomadClient := tc.Nomad()
   144  	jobID := "cns-template-kv"
   145  	tc.jobIDs = append(tc.jobIDs, jobID)
   146  
   147  	// Run job and wait for allocs to complete
   148  	allocations := e2eutil.RegisterAndWaitForAllocs(t, nomadClient, cnsJobTemplateKV, jobID, "")
   149  	require.Len(t, allocations, 2)
   150  	allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations)
   151  	e2eutil.WaitForAllocsStopped(f.T(), tc.Nomad(), allocIDs)
   152  
   153  	// Sort allocs by name
   154  	sort.Sort(e2eutil.AllocsByName(allocations))
   155  
   156  	// Check template read from default namespace even if namespace set
   157  	textB, err := e2eutil.AllocTaskLogs(allocations[0].ID, "task-b", e2eutil.LogsStdOut)
   158  	require.NoError(t, err)
   159  	require.Equal(t, "value: ns_default", textB)
   160  
   161  	// Check template read from default namespace if no namespace set
   162  	textZ, err := e2eutil.AllocTaskLogs(allocations[1].ID, "task-z", e2eutil.LogsStdOut)
   163  	require.NoError(t, err)
   164  	require.Equal(t, "value: ns_default", textZ)
   165  
   166  	//  Stop the job
   167  	e2eutil.WaitForJobStopped(t, nomadClient, jobID)
   168  }
   169  
   170  func (tc *ConsulNamespacesE2ETest) TestConsulConnectSidecars(f *framework.F) {
   171  	nomadClient := tc.Nomad()
   172  	jobID := "cns-connect-sidecars"
   173  	tc.jobIDs = append(tc.jobIDs, jobID)
   174  
   175  	// Run job and wait for allocs
   176  	allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, cnsJobConnectSidecars, jobID, "")
   177  	require.Len(f.T(), allocations, 4)
   178  	allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations)
   179  	e2eutil.WaitForAllocsRunning(f.T(), tc.Nomad(), allocIDs)
   180  
   181  	r := f.Assertions
   182  	c := tc.Consul()
   183  	namespace := consulNamespace
   184  
   185  	// Verify services with cns set were registered into "default"
   186  	e2eutil.RequireConsulRegistered(r, c, namespace, "count-api", 1)
   187  	e2eutil.RequireConsulRegistered(r, c, namespace, "count-api-sidecar-proxy", 1)
   188  	e2eutil.RequireConsulRegistered(r, c, namespace, "count-dashboard", 1)
   189  	e2eutil.RequireConsulRegistered(r, c, namespace, "count-dashboard-sidecar-proxy", 1)
   190  
   191  	// Verify services without cns set were registered into "default"
   192  	e2eutil.RequireConsulRegistered(r, c, namespace, "count-api-z", 1)
   193  	e2eutil.RequireConsulRegistered(r, c, namespace, "count-api-z-sidecar-proxy", 1)
   194  	e2eutil.RequireConsulRegistered(r, c, namespace, "count-dashboard-z", 1)
   195  	e2eutil.RequireConsulRegistered(r, c, namespace, "count-dashboard-z-sidecar-proxy", 1)
   196  
   197  	// Verify our services were NOT registered into specified consul namespaces
   198  	e2eutil.RequireConsulRegistered(r, c, "apple", "count-api", 0)
   199  	e2eutil.RequireConsulRegistered(r, c, "apple", "count-api-sidecar-proxy", 0)
   200  	e2eutil.RequireConsulRegistered(r, c, "apple", "count-dashboard", 0)
   201  	e2eutil.RequireConsulRegistered(r, c, "apple", "count-dashb0ard-sidecar-proxy", 0)
   202  
   203  	// Stop the job
   204  	e2eutil.WaitForJobStopped(f.T(), nomadClient, jobID)
   205  
   206  	// Verify that services were de-registered from Consul
   207  	e2eutil.RequireConsulDeregistered(r, c, namespace, "count-api")
   208  	e2eutil.RequireConsulDeregistered(r, c, namespace, "count-api-sidecar-proxy")
   209  	e2eutil.RequireConsulDeregistered(r, c, namespace, "count-dashboard")
   210  	e2eutil.RequireConsulDeregistered(r, c, namespace, "count-dashboard-sidecar-proxy")
   211  	e2eutil.RequireConsulDeregistered(r, c, namespace, "count-api-z")
   212  	e2eutil.RequireConsulDeregistered(r, c, namespace, "count-api-z-sidecar-proxy")
   213  	e2eutil.RequireConsulDeregistered(r, c, namespace, "count-dashboard-z")
   214  	e2eutil.RequireConsulDeregistered(r, c, namespace, "count-dashboard-z-sidecar-proxy")
   215  }
   216  
   217  func (tc *ConsulNamespacesE2ETest) TestConsulConnectIngressGateway(f *framework.F) {
   218  	nomadClient := tc.Nomad()
   219  	jobID := "cns-connect-ingress"
   220  	tc.jobIDs = append(tc.jobIDs, jobID)
   221  
   222  	// Run job and wait for allocs
   223  	allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, cnsJobConnectIngress, jobID, "")
   224  	require.Len(f.T(), allocations, 4) // 2 x (1 service + 1 gateway)
   225  	allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations)
   226  	e2eutil.WaitForAllocsRunning(f.T(), tc.Nomad(), allocIDs)
   227  
   228  	r := f.Assertions
   229  	c := tc.Consul()
   230  	namespace := consulNamespace
   231  
   232  	// Verify services with cns set were registered into "default"
   233  	e2eutil.RequireConsulRegistered(r, c, namespace, "my-ingress-service", 1)
   234  	e2eutil.RequireConsulRegistered(r, c, namespace, "uuid-api", 1)
   235  
   236  	// Verify services without cns set were registered into "default"
   237  	e2eutil.RequireConsulRegistered(r, c, namespace, "my-ingress-service-z", 1)
   238  	e2eutil.RequireConsulRegistered(r, c, namespace, "uuid-api-z", 1)
   239  
   240  	// Verify services with cns set were NOT registered into specified consul namespaces
   241  	e2eutil.RequireConsulRegistered(r, c, "apple", "my-ingress-service", 0)
   242  	e2eutil.RequireConsulRegistered(r, c, "apple", "uuid-api", 0)
   243  
   244  	// Read the config entry of gateway with cns set, checking it exists in "default' namespace
   245  	ce := e2eutil.ReadConsulConfigEntry(f.T(), c, namespace, "ingress-gateway", "my-ingress-service")
   246  	require.Equal(f.T(), namespace, ce.GetNamespace())
   247  
   248  	// Read the config entry of gateway without cns set, checking it exists in "default' namespace
   249  	ceZ := e2eutil.ReadConsulConfigEntry(f.T(), c, namespace, "ingress-gateway", "my-ingress-service-z")
   250  	require.Equal(f.T(), namespace, ceZ.GetNamespace())
   251  
   252  	// Stop the job
   253  	e2eutil.WaitForJobStopped(f.T(), nomadClient, jobID)
   254  
   255  	// Remove the config entries
   256  	e2eutil.DeleteConsulConfigEntry(f.T(), c, namespace, "ingress-gateway", "my-ingress-service")
   257  	e2eutil.DeleteConsulConfigEntry(f.T(), c, namespace, "ingress-gateway", "my-ingress-service-z")
   258  }
   259  
   260  func (tc *ConsulNamespacesE2ETest) TestConsulConnectTerminatingGateway(f *framework.F) {
   261  	nomadClient := tc.Nomad()
   262  	jobID := "cns-connect-terminating"
   263  	tc.jobIDs = append(tc.jobIDs, jobID)
   264  
   265  	// Run job and wait for allocs
   266  	allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, cnsJobConnectTerminating, jobID, "")
   267  	require.Len(f.T(), allocations, 6) // 2 x (2 services + 1 gateway)
   268  	allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations)
   269  	e2eutil.WaitForAllocsRunning(f.T(), tc.Nomad(), allocIDs)
   270  
   271  	r := f.Assertions
   272  	c := tc.Consul()
   273  	namespace := consulNamespace
   274  
   275  	// Verify services with cns set were registered into "default" Consul namespace
   276  	e2eutil.RequireConsulRegistered(r, c, namespace, "api-gateway", 1)
   277  	e2eutil.RequireConsulRegistered(r, c, namespace, "count-api", 1)
   278  	e2eutil.RequireConsulRegistered(r, c, namespace, "count-dashboard", 1)
   279  
   280  	// Verify services without cns set were registered into "default" Consul namespace
   281  	e2eutil.RequireConsulRegistered(r, c, namespace, "api-gateway-z", 1)
   282  	e2eutil.RequireConsulRegistered(r, c, namespace, "count-api-z", 1)
   283  	e2eutil.RequireConsulRegistered(r, c, namespace, "count-dashboard-z", 1)
   284  
   285  	// Verify services with cns set were NOT registered into specified consul namespaces
   286  	e2eutil.RequireConsulRegistered(r, c, "apple", "api-gateway", 0)
   287  	e2eutil.RequireConsulRegistered(r, c, "apple", "count-api", 0)
   288  	e2eutil.RequireConsulRegistered(r, c, "apple", "count-dashboard", 0)
   289  
   290  	// Read the config entry of gateway with cns set, checking it exists in "default' namespace
   291  	ce := e2eutil.ReadConsulConfigEntry(f.T(), c, namespace, "terminating-gateway", "api-gateway")
   292  	require.Equal(f.T(), namespace, ce.GetNamespace())
   293  
   294  	// Read the config entry of gateway without cns set, checking it exists in "default' namespace
   295  	ceZ := e2eutil.ReadConsulConfigEntry(f.T(), c, namespace, "terminating-gateway", "api-gateway-z")
   296  	require.Equal(f.T(), namespace, ceZ.GetNamespace())
   297  
   298  	// Stop the job
   299  	e2eutil.WaitForJobStopped(f.T(), nomadClient, jobID)
   300  
   301  	// Remove the config entries
   302  	e2eutil.DeleteConsulConfigEntry(f.T(), c, namespace, "terminating-gateway", "api-gateway")
   303  	e2eutil.DeleteConsulConfigEntry(f.T(), c, namespace, "terminating-gateway", "api-gateway-z")
   304  }
   305  
   306  func (tc *ConsulNamespacesE2ETest) TestConsulScriptChecksTask(f *framework.F) {
   307  	nomadClient := tc.Nomad()
   308  	jobID := "cns-script-checks-task"
   309  	tc.jobIDs = append(tc.jobIDs, jobID)
   310  
   311  	// Run job and wait for allocs
   312  	allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, cnsJobScriptChecksTask, jobID, "")
   313  	require.Len(f.T(), allocations, 2)
   314  	allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations)
   315  	e2eutil.WaitForAllocsRunning(f.T(), tc.Nomad(), allocIDs)
   316  
   317  	r := f.Assertions
   318  	c := tc.Consul()
   319  	namespace := consulNamespace
   320  
   321  	sort.Sort(e2eutil.AllocsByName(allocations))
   322  	allocsWithSetNamespace := allocations[0:1]
   323  	allocsWithNoNamespace := allocations[1:2]
   324  
   325  	// Verify checks were registered into "default" Consul namespace
   326  	e2eutil.RequireConsulStatus(r, c, namespace, "service-1a", capi.HealthPassing)
   327  	e2eutil.RequireConsulStatus(r, c, namespace, "service-2a", capi.HealthWarning)
   328  	e2eutil.RequireConsulStatus(r, c, namespace, "service-3a", capi.HealthCritical)
   329  
   330  	// Check in warning state becomes healthy after check passes for the service
   331  	// with specified Consul namespace
   332  	//
   333  	// (ensures UpdateTTL is respecting namespace)
   334  	_, _, err := exec(nomadClient, allocsWithSetNamespace,
   335  		[]string{"/bin/sh", "-c", "touch ${NOMAD_TASK_DIR}/alive-2ab"})
   336  	r.NoError(err)
   337  	e2eutil.RequireConsulStatus(r, c, namespace, "service-2a", capi.HealthPassing)
   338  
   339  	// Verify checks were registered into "default" Consul namespace when no
   340  	// namespace was specified.
   341  	e2eutil.RequireConsulStatus(r, c, namespace, "service-1z", capi.HealthPassing)
   342  	e2eutil.RequireConsulStatus(r, c, namespace, "service-2z", capi.HealthWarning)
   343  	e2eutil.RequireConsulStatus(r, c, namespace, "service-3z", capi.HealthCritical)
   344  
   345  	// Check in warning state becomes healthy after check passes for the service
   346  	// with specified Consul namespace
   347  	//
   348  	// (ensures UpdateTTL is respecting namespace)
   349  	_, _, errZ := exec(nomadClient, allocsWithNoNamespace,
   350  		[]string{"/bin/sh", "-c", "touch ${NOMAD_TASK_DIR}/alive-2zb"})
   351  	r.NoError(errZ)
   352  	e2eutil.RequireConsulStatus(r, c, namespace, "service-2z", capi.HealthPassing)
   353  
   354  	// Stop the job
   355  	e2eutil.WaitForJobStopped(f.T(), nomadClient, jobID)
   356  }
   357  
   358  func (tc *ConsulNamespacesE2ETest) TestConsulScriptChecksGroup(f *framework.F) {
   359  	nomadClient := tc.Nomad()
   360  	jobID := "cns-script-checks-group"
   361  	tc.jobIDs = append(tc.jobIDs, jobID)
   362  
   363  	// Run job and wait for allocs
   364  	allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, cnsJobScriptChecksGroup, jobID, "")
   365  	require.Len(f.T(), allocations, 2)
   366  	allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations)
   367  	e2eutil.WaitForAllocsRunning(f.T(), tc.Nomad(), allocIDs)
   368  
   369  	r := f.Assertions
   370  	c := tc.Consul()
   371  	namespace := consulNamespace
   372  
   373  	sort.Sort(e2eutil.AllocsByName(allocations))
   374  	allocsWithSetNamespace := allocations[0:1]
   375  	allocsWithNoNamespace := allocations[1:2]
   376  
   377  	// Verify checks were registered into "default" Consul namespace
   378  	e2eutil.RequireConsulStatus(r, c, namespace, "service-1a", capi.HealthPassing)
   379  	e2eutil.RequireConsulStatus(r, c, namespace, "service-2a", capi.HealthWarning)
   380  	e2eutil.RequireConsulStatus(r, c, namespace, "service-3a", capi.HealthCritical)
   381  
   382  	// Check in warning state becomes healthy after check passes for the service
   383  	// with specified Consul namespace
   384  	//
   385  	// (ensures UpdateTTL is respecting namespace)
   386  	_, _, err := exec(nomadClient, allocsWithSetNamespace,
   387  		[]string{"/bin/sh", "-c", "touch /tmp/${NOMAD_ALLOC_ID}-alive-2ab"})
   388  	r.NoError(err)
   389  	e2eutil.RequireConsulStatus(r, c, namespace, "service-2a", capi.HealthPassing)
   390  
   391  	// Verify checks were registered into "default" Consul namespace when no
   392  	// namespace was specified.
   393  	e2eutil.RequireConsulStatus(r, c, namespace, "service-1z", capi.HealthPassing)
   394  	e2eutil.RequireConsulStatus(r, c, namespace, "service-2z", capi.HealthWarning)
   395  	e2eutil.RequireConsulStatus(r, c, namespace, "service-3z", capi.HealthCritical)
   396  
   397  	// Check in warning state becomes healthy after check passes for the service
   398  	// with specified Consul namespace
   399  	//
   400  	// (ensures UpdateTTL is respecting namespace)
   401  	_, _, errZ := exec(nomadClient, allocsWithNoNamespace,
   402  		[]string{"/bin/sh", "-c", "touch /tmp/${NOMAD_ALLOC_ID}-alive-2zb"})
   403  	r.NoError(errZ)
   404  	e2eutil.RequireConsulStatus(r, c, namespace, "service-2z", capi.HealthPassing)
   405  
   406  	// Stop the job
   407  	e2eutil.WaitForJobStopped(f.T(), nomadClient, jobID)
   408  }