github.com/anth0d/nomad@v0.0.0-20221214183521-ae3a0a2cad06/e2e/consul/namespaces.go (about)

     1  package consul
     2  
     3  import (
     4  	"fmt"
     5  	"os"
     6  	"sort"
     7  
     8  	capi "github.com/hashicorp/consul/api"
     9  	"github.com/hashicorp/nomad/e2e/e2eutil"
    10  	"github.com/hashicorp/nomad/e2e/framework"
    11  	"github.com/hashicorp/nomad/helper"
    12  	"github.com/stretchr/testify/require"
    13  )
    14  
    15  // Job files used to test Consul Namespaces. Each job should run on Nomad OSS
    16  // and Nomad ENT with expectations set accordingly.
    17  //
    18  // All tests require Consul Enterprise.
    19  const (
    20  	cnsJobGroupServices      = "consul/input/namespaces/services_group.nomad"
    21  	cnsJobTaskServices       = "consul/input/namespaces/services_task.nomad"
    22  	cnsJobTemplateKV         = "consul/input/namespaces/template_kv.nomad"
    23  	cnsJobConnectSidecars    = "consul/input/namespaces/connect_sidecars.nomad"
    24  	cnsJobConnectIngress     = "consul/input/namespaces/connect_ingress.nomad"
    25  	cnsJobConnectTerminating = "consul/input/namespaces/connect_terminating.nomad"
    26  	cnsJobScriptChecksTask   = "consul/input/namespaces/script_checks_task.nomad"
    27  	cnsJobScriptChecksGroup  = "consul/input/namespaces/script_checks_group.nomad"
    28  )
    29  
    30  var (
    31  	// consulNamespaces represents the custom consul namespaces we create and
    32  	// can make use of in tests, but usefully so only in Nomad Enterprise
    33  	consulNamespaces = []string{"apple", "banana", "cherry"}
    34  
    35  	// allConsulNamespaces represents all namespaces we expect in consul after
    36  	// creating consulNamespaces, which then includes "default", which is the
    37  	// only namespace accessed by Nomad OSS (outside of agent configuration)
    38  	allConsulNamespaces = append(consulNamespaces, "default")
    39  )
    40  
    41  func init() {
    42  	framework.AddSuites(&framework.TestSuite{
    43  		Component:   "ConsulNamespaces",
    44  		CanRunLocal: true,
    45  		Consul:      true,
    46  		Cases: []framework.TestCase{
    47  			new(ConsulNamespacesE2ETest),
    48  		},
    49  	})
    50  }
    51  
    52  type ConsulNamespacesE2ETest struct {
    53  	framework.TC
    54  
    55  	jobIDs []string
    56  
    57  	// cToken contains the Consul global-management token
    58  	cToken string
    59  
    60  	// created policy and token IDs should be set here so they can be cleaned
    61  	// up after each test case, organized by namespace
    62  	policyIDs map[string][]string
    63  	tokenIDs  map[string][]string
    64  }
    65  
    66  func (tc *ConsulNamespacesE2ETest) BeforeAll(f *framework.F) {
    67  	tc.policyIDs = make(map[string][]string)
    68  	tc.tokenIDs = make(map[string][]string)
    69  
    70  	e2eutil.WaitForLeader(f.T(), tc.Nomad())
    71  	e2eutil.WaitForNodesReady(f.T(), tc.Nomad(), 1)
    72  
    73  	tc.cToken = os.Getenv("CONSUL_HTTP_TOKEN")
    74  
    75  	// create a set of consul namespaces in which to register services
    76  	e2eutil.CreateConsulNamespaces(f.T(), tc.Consul(), consulNamespaces)
    77  
    78  	// insert a key of the same name into KV for each namespace, where the value
    79  	// contains the namespace name making it easy to determine which namespace
    80  	// consul template actually accessed
    81  	for _, namespace := range allConsulNamespaces {
    82  		value := fmt.Sprintf("ns_%s", namespace)
    83  		e2eutil.PutConsulKey(f.T(), tc.Consul(), namespace, "ns-kv-example", value)
    84  	}
    85  }
    86  
    87  func (tc *ConsulNamespacesE2ETest) AfterAll(f *framework.F) {
    88  	e2eutil.DeleteConsulNamespaces(f.T(), tc.Consul(), consulNamespaces)
    89  }
    90  
    91  func (tc *ConsulNamespacesE2ETest) TestNamespacesExist(f *framework.F) {
    92  	// make sure our namespaces exist + default
    93  	namespaces := e2eutil.ListConsulNamespaces(f.T(), tc.Consul())
    94  	require.True(f.T(), helper.SliceSetEq(namespaces, append(consulNamespaces, "default")))
    95  }
    96  
    97  func (tc *ConsulNamespacesE2ETest) testConsulRegisterGroupServices(f *framework.F, token, nsA, nsB, nsC, nsZ string) {
    98  	nomadClient := tc.Nomad()
    99  	jobID := "cns-group-services"
   100  	tc.jobIDs = append(tc.jobIDs, jobID)
   101  
   102  	// Run job and wait for allocs
   103  	allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, cnsJobGroupServices, jobID, token)
   104  	require.Len(f.T(), allocations, 3)
   105  	allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations)
   106  	e2eutil.WaitForAllocsRunning(f.T(), tc.Nomad(), allocIDs)
   107  
   108  	r := f.Assertions
   109  	c := tc.Consul()
   110  
   111  	// Verify services with namespace set are registered into expected namespaces
   112  	e2eutil.RequireConsulRegistered(r, c, nsB, "b1", 1)
   113  	e2eutil.RequireConsulRegistered(r, c, nsB, "b2", 1)
   114  	e2eutil.RequireConsulRegistered(r, c, nsC, "c1", 1)
   115  	e2eutil.RequireConsulRegistered(r, c, nsC, "c2", 1)
   116  
   117  	// Verify services without namespace set are registered into default
   118  	e2eutil.RequireConsulRegistered(r, c, nsZ, "z1", 1)
   119  	e2eutil.RequireConsulRegistered(r, c, nsZ, "z2", 1)
   120  
   121  	// Verify our services are all healthy
   122  	e2eutil.RequireConsulStatus(r, c, nsB, "b1", "passing")
   123  	e2eutil.RequireConsulStatus(r, c, nsB, "b2", "passing")
   124  	e2eutil.RequireConsulStatus(r, c, nsC, "c1", "passing")
   125  	e2eutil.RequireConsulStatus(r, c, nsC, "c2", "passing")
   126  	e2eutil.RequireConsulStatus(r, c, nsZ, "z1", "passing")
   127  	e2eutil.RequireConsulStatus(r, c, nsZ, "z2", "passing")
   128  
   129  	// Stop the job
   130  	e2eutil.WaitForJobStopped(f.T(), nomadClient, jobID)
   131  
   132  	// Verify that services were de-registered from Consul
   133  	e2eutil.RequireConsulDeregistered(r, c, nsB, "b1")
   134  	e2eutil.RequireConsulDeregistered(r, c, nsB, "b2")
   135  	e2eutil.RequireConsulDeregistered(r, c, nsC, "c1")
   136  	e2eutil.RequireConsulDeregistered(r, c, nsC, "c2")
   137  	e2eutil.RequireConsulDeregistered(r, c, nsZ, "z1")
   138  	e2eutil.RequireConsulDeregistered(r, c, nsZ, "z2")
   139  }
   140  
   141  func (tc *ConsulNamespacesE2ETest) testConsulRegisterTaskServices(f *framework.F, token, nsA, nsB, nsC, nsZ string) {
   142  	nomadClient := tc.Nomad()
   143  	jobID := "cns-task-services"
   144  	tc.jobIDs = append(tc.jobIDs, jobID)
   145  
   146  	// Run job and wait for allocs
   147  	allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, cnsJobTaskServices, jobID, token)
   148  	require.Len(f.T(), allocations, 3)
   149  	allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations)
   150  	e2eutil.WaitForAllocsRunning(f.T(), tc.Nomad(), allocIDs)
   151  
   152  	r := f.Assertions
   153  	c := tc.Consul()
   154  
   155  	// Verify our services were registered into expected namespaces
   156  	e2eutil.RequireConsulRegistered(r, c, nsB, "b1", 1)
   157  	e2eutil.RequireConsulRegistered(r, c, nsB, "b2", 1)
   158  	e2eutil.RequireConsulRegistered(r, c, nsC, "c1", 1)
   159  	e2eutil.RequireConsulRegistered(r, c, nsC, "c2", 1)
   160  	e2eutil.RequireConsulRegistered(r, c, nsZ, "z1", 1)
   161  	e2eutil.RequireConsulRegistered(r, c, nsZ, "z2", 1)
   162  
   163  	// Verify our services are all healthy
   164  	e2eutil.RequireConsulStatus(r, c, nsB, "b1", "passing")
   165  	e2eutil.RequireConsulStatus(r, c, nsB, "b2", "passing")
   166  	e2eutil.RequireConsulStatus(r, c, nsC, "c1", "passing")
   167  	e2eutil.RequireConsulStatus(r, c, nsC, "c2", "passing")
   168  	e2eutil.RequireConsulStatus(r, c, nsZ, "z1", "passing")
   169  	e2eutil.RequireConsulStatus(r, c, nsZ, "z2", "passing")
   170  
   171  	// Stop the job
   172  	e2eutil.WaitForJobStopped(f.T(), nomadClient, jobID)
   173  
   174  	// Verify that services were de-registered from Consul
   175  	e2eutil.RequireConsulDeregistered(r, c, nsB, "b1")
   176  	e2eutil.RequireConsulDeregistered(r, c, nsB, "b2")
   177  	e2eutil.RequireConsulDeregistered(r, c, nsC, "c1")
   178  	e2eutil.RequireConsulDeregistered(r, c, nsC, "c2")
   179  	e2eutil.RequireConsulDeregistered(r, c, nsZ, "z1")
   180  	e2eutil.RequireConsulDeregistered(r, c, nsZ, "z2")
   181  }
   182  
   183  func (tc *ConsulNamespacesE2ETest) testConsulTemplateKV(f *framework.F, token, expB, expZ string) {
   184  	t := f.T()
   185  	nomadClient := tc.Nomad()
   186  	jobID := "cns-template-kv"
   187  	tc.jobIDs = append(tc.jobIDs, jobID)
   188  
   189  	// Run job and wait for allocs to complete
   190  	allocations := e2eutil.RegisterAndWaitForAllocs(t, nomadClient, cnsJobTemplateKV, jobID, token)
   191  	require.Len(t, allocations, 2)
   192  	allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations)
   193  	e2eutil.WaitForAllocsStopped(f.T(), tc.Nomad(), allocIDs)
   194  
   195  	// Sort allocs by name
   196  	sort.Sort(e2eutil.AllocsByName(allocations))
   197  
   198  	// Check template read from expected namespace when namespace set
   199  	textB, err := e2eutil.AllocTaskLogs(allocations[0].ID, "task-b", e2eutil.LogsStdOut)
   200  	require.NoError(t, err)
   201  	require.Equal(t, expB, textB)
   202  
   203  	// Check template read from default namespace if no namespace set
   204  	textZ, err := e2eutil.AllocTaskLogs(allocations[1].ID, "task-z", e2eutil.LogsStdOut)
   205  	require.NoError(t, err)
   206  	require.Equal(t, expZ, textZ)
   207  
   208  	//  Stop the job
   209  	e2eutil.WaitForJobStopped(t, nomadClient, jobID)
   210  }
   211  
   212  func (tc *ConsulNamespacesE2ETest) testConsulConnectSidecars(f *framework.F, token, nsA, nsZ string) {
   213  	nomadClient := tc.Nomad()
   214  	jobID := "cns-connect-sidecars"
   215  	tc.jobIDs = append(tc.jobIDs, jobID)
   216  
   217  	// Run job and wait for allocs
   218  	allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, cnsJobConnectSidecars, jobID, token)
   219  	require.Len(f.T(), allocations, 4)
   220  	allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations)
   221  	e2eutil.WaitForAllocsRunning(f.T(), tc.Nomad(), allocIDs)
   222  
   223  	r := f.Assertions
   224  	c := tc.Consul()
   225  
   226  	// Verify services with cns set were registered into expected namespace
   227  	e2eutil.RequireConsulRegistered(r, c, nsA, "count-api", 1)
   228  	e2eutil.RequireConsulRegistered(r, c, nsA, "count-api-sidecar-proxy", 1)
   229  	e2eutil.RequireConsulRegistered(r, c, nsA, "count-dashboard", 1)
   230  	e2eutil.RequireConsulRegistered(r, c, nsA, "count-dashboard-sidecar-proxy", 1)
   231  
   232  	// Verify services without cns set were registered into default
   233  	e2eutil.RequireConsulRegistered(r, c, nsZ, "count-api-z", 1)
   234  	e2eutil.RequireConsulRegistered(r, c, nsZ, "count-api-z-sidecar-proxy", 1)
   235  	e2eutil.RequireConsulRegistered(r, c, nsZ, "count-dashboard-z", 1)
   236  	e2eutil.RequireConsulRegistered(r, c, nsZ, "count-dashboard-z-sidecar-proxy", 1)
   237  
   238  	// Stop the job
   239  	e2eutil.WaitForJobStopped(f.T(), nomadClient, jobID)
   240  
   241  	// Verify that services were de-registered from Consul
   242  	e2eutil.RequireConsulDeregistered(r, c, nsA, "count-api")
   243  	e2eutil.RequireConsulDeregistered(r, c, nsA, "count-api-sidecar-proxy")
   244  	e2eutil.RequireConsulDeregistered(r, c, nsA, "count-dashboard")
   245  	e2eutil.RequireConsulDeregistered(r, c, nsA, "count-dashboard-sidecar-proxy")
   246  	e2eutil.RequireConsulDeregistered(r, c, nsZ, "count-api-z")
   247  	e2eutil.RequireConsulDeregistered(r, c, nsZ, "count-api-z-sidecar-proxy")
   248  	e2eutil.RequireConsulDeregistered(r, c, nsZ, "count-dashboard-z")
   249  	e2eutil.RequireConsulDeregistered(r, c, nsZ, "count-dashboard-z-sidecar-proxy")
   250  }
   251  
   252  func (tc *ConsulNamespacesE2ETest) testConsulConnectIngressGateway(f *framework.F, token, nsA, nsZ string) {
   253  	nomadClient := tc.Nomad()
   254  	jobID := "cns-connect-ingress"
   255  	tc.jobIDs = append(tc.jobIDs, jobID)
   256  
   257  	// Run job and wait for allocs
   258  	allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, cnsJobConnectIngress, jobID, token)
   259  	require.Len(f.T(), allocations, 4) // 2 x (1 service + 1 gateway)
   260  	allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations)
   261  	e2eutil.WaitForAllocsRunning(f.T(), tc.Nomad(), allocIDs)
   262  
   263  	r := f.Assertions
   264  	c := tc.Consul()
   265  
   266  	// Verify services with cns set were registered into expected namespace
   267  	e2eutil.RequireConsulRegistered(r, c, nsA, "my-ingress-service", 1)
   268  	e2eutil.RequireConsulRegistered(r, c, nsA, "uuid-api", 1)
   269  
   270  	// Verify services without cns set were registered into default
   271  	e2eutil.RequireConsulRegistered(r, c, nsZ, "my-ingress-service-z", 1)
   272  	e2eutil.RequireConsulRegistered(r, c, nsZ, "uuid-api-z", 1)
   273  
   274  	// Read the config entry of gateway with cns set, checking it exists in expected namespace
   275  	ce := e2eutil.ReadConsulConfigEntry(f.T(), c, nsA, "ingress-gateway", "my-ingress-service")
   276  	require.Equal(f.T(), nsA, ce.GetNamespace())
   277  
   278  	// Read the config entry of gateway without cns set, checking it exists in default namespace
   279  	ceZ := e2eutil.ReadConsulConfigEntry(f.T(), c, nsZ, "ingress-gateway", "my-ingress-service-z")
   280  	require.Equal(f.T(), nsZ, ceZ.GetNamespace())
   281  
   282  	// Stop the job
   283  	e2eutil.WaitForJobStopped(f.T(), nomadClient, jobID)
   284  
   285  	// Remove the config entries
   286  	e2eutil.DeleteConsulConfigEntry(f.T(), c, nsA, "ingress-gateway", "my-ingress-service")
   287  	e2eutil.DeleteConsulConfigEntry(f.T(), c, nsZ, "ingress-gateway", "my-ingress-service-z")
   288  }
   289  
   290  func (tc *ConsulNamespacesE2ETest) testConsulConnectTerminatingGateway(f *framework.F, token, nsA, nsZ string) {
   291  	nomadClient := tc.Nomad()
   292  	jobID := "cns-connect-terminating"
   293  	tc.jobIDs = append(tc.jobIDs, jobID)
   294  
   295  	// Run job and wait for allocs
   296  	allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, cnsJobConnectTerminating, jobID, token)
   297  	require.Len(f.T(), allocations, 6) // 2 x (2 services + 1 gateway)
   298  	allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations)
   299  	e2eutil.WaitForAllocsRunning(f.T(), tc.Nomad(), allocIDs)
   300  
   301  	r := f.Assertions
   302  	c := tc.Consul()
   303  
   304  	// Verify services with cns set were registered into "default" Consul namespace
   305  	e2eutil.RequireConsulRegistered(r, c, nsA, "api-gateway", 1)
   306  	e2eutil.RequireConsulRegistered(r, c, nsA, "count-api", 1)
   307  	e2eutil.RequireConsulRegistered(r, c, nsA, "count-dashboard", 1)
   308  
   309  	// Verify services without cns set were registered into "default" Consul namespace
   310  	e2eutil.RequireConsulRegistered(r, c, nsZ, "api-gateway-z", 1)
   311  	e2eutil.RequireConsulRegistered(r, c, nsZ, "count-api-z", 1)
   312  	e2eutil.RequireConsulRegistered(r, c, nsZ, "count-dashboard-z", 1)
   313  
   314  	// Read the config entry of gateway with cns set, checking it exists in "default' namespace
   315  	ce := e2eutil.ReadConsulConfigEntry(f.T(), c, nsA, "terminating-gateway", "api-gateway")
   316  	require.Equal(f.T(), nsA, ce.GetNamespace())
   317  
   318  	// Read the config entry of gateway without cns set, checking it exists in "default' namespace
   319  	ceZ := e2eutil.ReadConsulConfigEntry(f.T(), c, nsZ, "terminating-gateway", "api-gateway-z")
   320  	require.Equal(f.T(), nsZ, ceZ.GetNamespace())
   321  
   322  	// Stop the job
   323  	e2eutil.WaitForJobStopped(f.T(), nomadClient, jobID)
   324  
   325  	// Remove the config entries
   326  	e2eutil.DeleteConsulConfigEntry(f.T(), c, nsA, "terminating-gateway", "api-gateway")
   327  	e2eutil.DeleteConsulConfigEntry(f.T(), c, nsZ, "terminating-gateway", "api-gateway-z")
   328  }
   329  
   330  func (tc *ConsulNamespacesE2ETest) testConsulScriptChecksTask(f *framework.F, token, nsA, nsZ string) {
   331  	nomadClient := tc.Nomad()
   332  	jobID := "cns-script-checks-task"
   333  	tc.jobIDs = append(tc.jobIDs, jobID)
   334  
   335  	// Run job and wait for allocs
   336  	allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, cnsJobScriptChecksTask, jobID, token)
   337  	require.Len(f.T(), allocations, 2)
   338  	allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations)
   339  	e2eutil.WaitForAllocsRunning(f.T(), tc.Nomad(), allocIDs)
   340  
   341  	r := f.Assertions
   342  	c := tc.Consul()
   343  
   344  	sort.Sort(e2eutil.AllocsByName(allocations))
   345  	allocsWithSetNamespace := allocations[0:1]
   346  	allocsWithNoNamespace := allocations[1:2]
   347  
   348  	// Verify checks with namespace set are set into expected namespace
   349  	e2eutil.RequireConsulStatus(r, c, nsA, "service-1a", capi.HealthPassing)
   350  	e2eutil.RequireConsulStatus(r, c, nsA, "service-2a", capi.HealthWarning)
   351  	e2eutil.RequireConsulStatus(r, c, nsA, "service-3a", capi.HealthCritical)
   352  
   353  	// Check in warning state becomes healthy after check passes for the service
   354  	// with specified Consul namespace
   355  	//
   356  	// (ensures UpdateTTL is respecting namespace)
   357  	_, _, err := exec(nomadClient, allocsWithSetNamespace,
   358  		[]string{"/bin/sh", "-c", "touch ${NOMAD_TASK_DIR}/alive-2ab"})
   359  	r.NoError(err)
   360  	e2eutil.RequireConsulStatus(r, c, nsA, "service-2a", capi.HealthPassing)
   361  
   362  	// Verify checks without namespace are set in default namespace
   363  	e2eutil.RequireConsulStatus(r, c, nsZ, "service-1z", capi.HealthPassing)
   364  	e2eutil.RequireConsulStatus(r, c, nsZ, "service-2z", capi.HealthWarning)
   365  	e2eutil.RequireConsulStatus(r, c, nsZ, "service-3z", capi.HealthCritical)
   366  
   367  	// Check in warning state becomes healthy after check passes for the service
   368  	// with specified Consul namespace
   369  	//
   370  	// (ensures UpdateTTL is respecting namespace)
   371  	_, _, errZ := exec(nomadClient, allocsWithNoNamespace,
   372  		[]string{"/bin/sh", "-c", "touch ${NOMAD_TASK_DIR}/alive-2zb"})
   373  	r.NoError(errZ)
   374  	e2eutil.RequireConsulStatus(r, c, nsZ, "service-2z", capi.HealthPassing)
   375  
   376  	// Stop the job
   377  	e2eutil.WaitForJobStopped(f.T(), nomadClient, jobID)
   378  }
   379  
   380  func (tc *ConsulNamespacesE2ETest) testConsulScriptChecksGroup(f *framework.F, token, nsA, nsZ string) {
   381  	nomadClient := tc.Nomad()
   382  	jobID := "cns-script-checks-group"
   383  	tc.jobIDs = append(tc.jobIDs, jobID)
   384  
   385  	// Run job and wait for allocs
   386  	allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, cnsJobScriptChecksGroup, jobID, token)
   387  	require.Len(f.T(), allocations, 2)
   388  	allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations)
   389  	e2eutil.WaitForAllocsRunning(f.T(), tc.Nomad(), allocIDs)
   390  
   391  	r := f.Assertions
   392  	c := tc.Consul()
   393  
   394  	sort.Sort(e2eutil.AllocsByName(allocations))
   395  	allocsWithSetNamespace := allocations[0:1]
   396  	allocsWithNoNamespace := allocations[1:2]
   397  
   398  	// Verify checks were registered into "default" Consul namespace
   399  	e2eutil.RequireConsulStatus(r, c, nsA, "service-1a", capi.HealthPassing)
   400  	e2eutil.RequireConsulStatus(r, c, nsA, "service-2a", capi.HealthWarning)
   401  	e2eutil.RequireConsulStatus(r, c, nsA, "service-3a", capi.HealthCritical)
   402  
   403  	// Check in warning state becomes healthy after check passes for the service
   404  	// with specified Consul namespace
   405  	//
   406  	// (ensures UpdateTTL is respecting namespace)
   407  	_, _, err := exec(nomadClient, allocsWithSetNamespace,
   408  		[]string{"/bin/sh", "-c", "touch /tmp/${NOMAD_ALLOC_ID}-alive-2ab"})
   409  	r.NoError(err)
   410  	e2eutil.RequireConsulStatus(r, c, nsA, "service-2a", capi.HealthPassing)
   411  
   412  	// Verify checks were registered into "default" Consul namespace when no
   413  	// namespace was specified.
   414  	e2eutil.RequireConsulStatus(r, c, nsZ, "service-1z", capi.HealthPassing)
   415  	e2eutil.RequireConsulStatus(r, c, nsZ, "service-2z", capi.HealthWarning)
   416  	e2eutil.RequireConsulStatus(r, c, nsZ, "service-3z", capi.HealthCritical)
   417  
   418  	// Check in warning state becomes healthy after check passes for the service
   419  	// with specified Consul namespace
   420  	//
   421  	// (ensures UpdateTTL is respecting namespace)
   422  	_, _, errZ := exec(nomadClient, allocsWithNoNamespace,
   423  		[]string{"/bin/sh", "-c", "touch /tmp/${NOMAD_ALLOC_ID}-alive-2zb"})
   424  	r.NoError(errZ)
   425  	e2eutil.RequireConsulStatus(r, c, nsZ, "service-2z", capi.HealthPassing)
   426  
   427  	// Stop the job
   428  	e2eutil.WaitForJobStopped(f.T(), nomadClient, jobID)
   429  }