github.com/anth0d/nomad@v0.0.0-20221214183521-ae3a0a2cad06/e2e/consul/namespaces_oss.go (about) 1 //go:build !ent 2 // +build !ent 3 4 // Nomad OSS ignores Consul Namespace configuration in jobs, these e2e tests 5 // verify everything still works and is registered into the "default" namespace, 6 // since e2e always uses Consul Enterprise. With Consul OSS, there are no namespaces. 7 // and these tests will not work. 8 9 package consul 10 11 import ( 12 "os" 13 "sort" 14 15 capi "github.com/hashicorp/consul/api" 16 "github.com/hashicorp/nomad/e2e/e2eutil" 17 "github.com/hashicorp/nomad/e2e/framework" 18 "github.com/stretchr/testify/require" 19 ) 20 21 func (tc *ConsulNamespacesE2ETest) AfterEach(f *framework.F) { 22 if os.Getenv("NOMAD_TEST_SKIPCLEANUP") == "1" { 23 return 24 } 25 26 // cleanup jobs 27 for _, id := range tc.jobIDs { 28 _, _, err := tc.Nomad().Jobs().Deregister(id, true, nil) 29 f.NoError(err) 30 } 31 32 // do garbage collection 33 err := tc.Nomad().System().GarbageCollect() 34 f.NoError(err) 35 36 // reset accumulators 37 tc.tokenIDs = make(map[string][]string) 38 tc.policyIDs = make(map[string][]string) 39 } 40 41 func (tc *ConsulNamespacesE2ETest) TestConsulRegisterGroupServices(f *framework.F) { 42 nomadClient := tc.Nomad() 43 jobID := "cns-group-services" 44 tc.jobIDs = append(tc.jobIDs, jobID) 45 46 // Run job and wait for allocs 47 allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, cnsJobGroupServices, jobID, "") 48 require.Len(f.T(), allocations, 3) 49 allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations) 50 e2eutil.WaitForAllocsRunning(f.T(), tc.Nomad(), allocIDs) 51 52 r := f.Assertions 53 c := tc.Consul() 54 namespace := consulNamespace 55 56 // Verify our services were registered into "default" 57 e2eutil.RequireConsulRegistered(r, c, namespace, "b1", 1) 58 e2eutil.RequireConsulRegistered(r, c, namespace, "b2", 1) 59 e2eutil.RequireConsulRegistered(r, c, namespace, "c1", 1) 60 e2eutil.RequireConsulRegistered(r, c, namespace, "c2", 1) 61 e2eutil.RequireConsulRegistered(r, c, namespace, "z1", 1) 62 e2eutil.RequireConsulRegistered(r, c, namespace, "z2", 1) 63 64 // Verify our services are all healthy 65 e2eutil.RequireConsulStatus(r, c, namespace, "b1", "passing") 66 e2eutil.RequireConsulStatus(r, c, namespace, "b2", "passing") 67 e2eutil.RequireConsulStatus(r, c, namespace, "c1", "passing") 68 e2eutil.RequireConsulStatus(r, c, namespace, "c2", "passing") 69 e2eutil.RequireConsulStatus(r, c, namespace, "z1", "passing") 70 e2eutil.RequireConsulStatus(r, c, namespace, "z2", "passing") 71 72 // Verify our services were NOT registered into specified consul namespaces 73 e2eutil.RequireConsulRegistered(r, c, "banana", "b1", 0) 74 e2eutil.RequireConsulRegistered(r, c, "banana", "b2", 0) 75 e2eutil.RequireConsulRegistered(r, c, "cherry", "c1", 0) 76 e2eutil.RequireConsulRegistered(r, c, "cherry", "c2", 0) 77 78 // Stop the job 79 e2eutil.WaitForJobStopped(f.T(), nomadClient, jobID) 80 81 // Verify that services were de-registered in Consul 82 e2eutil.RequireConsulDeregistered(r, c, namespace, "b1") 83 e2eutil.RequireConsulDeregistered(r, c, namespace, "b2") 84 e2eutil.RequireConsulDeregistered(r, c, namespace, "c1") 85 e2eutil.RequireConsulDeregistered(r, c, namespace, "c2") 86 e2eutil.RequireConsulDeregistered(r, c, namespace, "z1") 87 e2eutil.RequireConsulDeregistered(r, c, namespace, "z2") 88 } 89 90 func (tc *ConsulNamespacesE2ETest) TestConsulRegisterTaskServices(f *framework.F) { 91 nomadClient := tc.Nomad() 92 jobID := "cns-task-services" 93 tc.jobIDs = append(tc.jobIDs, jobID) 94 95 // Run job and wait for allocs 96 allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, cnsJobTaskServices, jobID, "") 97 require.Len(f.T(), allocations, 3) 98 allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations) 99 e2eutil.WaitForAllocsRunning(f.T(), tc.Nomad(), allocIDs) 100 101 r := f.Assertions 102 c := tc.Consul() 103 namespace := consulNamespace 104 105 // Verify our services were registered into "default" 106 e2eutil.RequireConsulRegistered(r, c, namespace, "b1", 1) 107 e2eutil.RequireConsulRegistered(r, c, namespace, "b2", 1) 108 e2eutil.RequireConsulRegistered(r, c, namespace, "c1", 1) 109 e2eutil.RequireConsulRegistered(r, c, namespace, "c2", 1) 110 e2eutil.RequireConsulRegistered(r, c, namespace, "z1", 1) 111 e2eutil.RequireConsulRegistered(r, c, namespace, "z2", 1) 112 113 // Verify our services are all healthy 114 e2eutil.RequireConsulStatus(r, c, namespace, "b1", "passing") 115 e2eutil.RequireConsulStatus(r, c, namespace, "b2", "passing") 116 e2eutil.RequireConsulStatus(r, c, namespace, "c1", "passing") 117 e2eutil.RequireConsulStatus(r, c, namespace, "c2", "passing") 118 e2eutil.RequireConsulStatus(r, c, namespace, "z1", "passing") 119 e2eutil.RequireConsulStatus(r, c, namespace, "z2", "passing") 120 121 // Verify our services were NOT registered into specified consul namespaces 122 e2eutil.RequireConsulRegistered(r, c, "banana", "b1", 0) 123 e2eutil.RequireConsulRegistered(r, c, "banana", "b2", 0) 124 e2eutil.RequireConsulRegistered(r, c, "cherry", "c1", 0) 125 e2eutil.RequireConsulRegistered(r, c, "cherry", "c2", 0) 126 e2eutil.RequireConsulRegistered(r, c, "cherry", "z1", 0) 127 e2eutil.RequireConsulRegistered(r, c, "cherry", "z2", 0) 128 129 // Stop the job 130 e2eutil.WaitForJobStopped(f.T(), nomadClient, jobID) 131 132 // Verify that services were de-registered from Consul 133 e2eutil.RequireConsulDeregistered(r, c, namespace, "b1") 134 e2eutil.RequireConsulDeregistered(r, c, namespace, "b2") 135 e2eutil.RequireConsulDeregistered(r, c, namespace, "c1") 136 e2eutil.RequireConsulDeregistered(r, c, namespace, "b2") 137 e2eutil.RequireConsulDeregistered(r, c, namespace, "z1") 138 e2eutil.RequireConsulDeregistered(r, c, namespace, "z2") 139 } 140 141 func (tc *ConsulNamespacesE2ETest) TestConsulTemplateKV(f *framework.F) { 142 t := f.T() 143 nomadClient := tc.Nomad() 144 jobID := "cns-template-kv" 145 tc.jobIDs = append(tc.jobIDs, jobID) 146 147 // Run job and wait for allocs to complete 148 allocations := e2eutil.RegisterAndWaitForAllocs(t, nomadClient, cnsJobTemplateKV, jobID, "") 149 require.Len(t, allocations, 2) 150 allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations) 151 e2eutil.WaitForAllocsStopped(f.T(), tc.Nomad(), allocIDs) 152 153 // Sort allocs by name 154 sort.Sort(e2eutil.AllocsByName(allocations)) 155 156 // Check template read from default namespace even if namespace set 157 textB, err := e2eutil.AllocTaskLogs(allocations[0].ID, "task-b", e2eutil.LogsStdOut) 158 require.NoError(t, err) 159 require.Equal(t, "value: ns_default", textB) 160 161 // Check template read from default namespace if no namespace set 162 textZ, err := e2eutil.AllocTaskLogs(allocations[1].ID, "task-z", e2eutil.LogsStdOut) 163 require.NoError(t, err) 164 require.Equal(t, "value: ns_default", textZ) 165 166 // Stop the job 167 e2eutil.WaitForJobStopped(t, nomadClient, jobID) 168 } 169 170 func (tc *ConsulNamespacesE2ETest) TestConsulConnectSidecars(f *framework.F) { 171 nomadClient := tc.Nomad() 172 jobID := "cns-connect-sidecars" 173 tc.jobIDs = append(tc.jobIDs, jobID) 174 175 // Run job and wait for allocs 176 allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, cnsJobConnectSidecars, jobID, "") 177 require.Len(f.T(), allocations, 4) 178 allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations) 179 e2eutil.WaitForAllocsRunning(f.T(), tc.Nomad(), allocIDs) 180 181 r := f.Assertions 182 c := tc.Consul() 183 namespace := consulNamespace 184 185 // Verify services with cns set were registered into "default" 186 e2eutil.RequireConsulRegistered(r, c, namespace, "count-api", 1) 187 e2eutil.RequireConsulRegistered(r, c, namespace, "count-api-sidecar-proxy", 1) 188 e2eutil.RequireConsulRegistered(r, c, namespace, "count-dashboard", 1) 189 e2eutil.RequireConsulRegistered(r, c, namespace, "count-dashboard-sidecar-proxy", 1) 190 191 // Verify services without cns set were registered into "default" 192 e2eutil.RequireConsulRegistered(r, c, namespace, "count-api-z", 1) 193 e2eutil.RequireConsulRegistered(r, c, namespace, "count-api-z-sidecar-proxy", 1) 194 e2eutil.RequireConsulRegistered(r, c, namespace, "count-dashboard-z", 1) 195 e2eutil.RequireConsulRegistered(r, c, namespace, "count-dashboard-z-sidecar-proxy", 1) 196 197 // Verify our services were NOT registered into specified consul namespaces 198 e2eutil.RequireConsulRegistered(r, c, "apple", "count-api", 0) 199 e2eutil.RequireConsulRegistered(r, c, "apple", "count-api-sidecar-proxy", 0) 200 e2eutil.RequireConsulRegistered(r, c, "apple", "count-dashboard", 0) 201 e2eutil.RequireConsulRegistered(r, c, "apple", "count-dashb0ard-sidecar-proxy", 0) 202 203 // Stop the job 204 e2eutil.WaitForJobStopped(f.T(), nomadClient, jobID) 205 206 // Verify that services were de-registered from Consul 207 e2eutil.RequireConsulDeregistered(r, c, namespace, "count-api") 208 e2eutil.RequireConsulDeregistered(r, c, namespace, "count-api-sidecar-proxy") 209 e2eutil.RequireConsulDeregistered(r, c, namespace, "count-dashboard") 210 e2eutil.RequireConsulDeregistered(r, c, namespace, "count-dashboard-sidecar-proxy") 211 e2eutil.RequireConsulDeregistered(r, c, namespace, "count-api-z") 212 e2eutil.RequireConsulDeregistered(r, c, namespace, "count-api-z-sidecar-proxy") 213 e2eutil.RequireConsulDeregistered(r, c, namespace, "count-dashboard-z") 214 e2eutil.RequireConsulDeregistered(r, c, namespace, "count-dashboard-z-sidecar-proxy") 215 } 216 217 func (tc *ConsulNamespacesE2ETest) TestConsulConnectIngressGateway(f *framework.F) { 218 nomadClient := tc.Nomad() 219 jobID := "cns-connect-ingress" 220 tc.jobIDs = append(tc.jobIDs, jobID) 221 222 // Run job and wait for allocs 223 allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, cnsJobConnectIngress, jobID, "") 224 require.Len(f.T(), allocations, 4) // 2 x (1 service + 1 gateway) 225 allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations) 226 e2eutil.WaitForAllocsRunning(f.T(), tc.Nomad(), allocIDs) 227 228 r := f.Assertions 229 c := tc.Consul() 230 namespace := consulNamespace 231 232 // Verify services with cns set were registered into "default" 233 e2eutil.RequireConsulRegistered(r, c, namespace, "my-ingress-service", 1) 234 e2eutil.RequireConsulRegistered(r, c, namespace, "uuid-api", 1) 235 236 // Verify services without cns set were registered into "default" 237 e2eutil.RequireConsulRegistered(r, c, namespace, "my-ingress-service-z", 1) 238 e2eutil.RequireConsulRegistered(r, c, namespace, "uuid-api-z", 1) 239 240 // Verify services with cns set were NOT registered into specified consul namespaces 241 e2eutil.RequireConsulRegistered(r, c, "apple", "my-ingress-service", 0) 242 e2eutil.RequireConsulRegistered(r, c, "apple", "uuid-api", 0) 243 244 // Read the config entry of gateway with cns set, checking it exists in "default' namespace 245 ce := e2eutil.ReadConsulConfigEntry(f.T(), c, namespace, "ingress-gateway", "my-ingress-service") 246 require.Equal(f.T(), namespace, ce.GetNamespace()) 247 248 // Read the config entry of gateway without cns set, checking it exists in "default' namespace 249 ceZ := e2eutil.ReadConsulConfigEntry(f.T(), c, namespace, "ingress-gateway", "my-ingress-service-z") 250 require.Equal(f.T(), namespace, ceZ.GetNamespace()) 251 252 // Stop the job 253 e2eutil.WaitForJobStopped(f.T(), nomadClient, jobID) 254 255 // Remove the config entries 256 e2eutil.DeleteConsulConfigEntry(f.T(), c, namespace, "ingress-gateway", "my-ingress-service") 257 e2eutil.DeleteConsulConfigEntry(f.T(), c, namespace, "ingress-gateway", "my-ingress-service-z") 258 } 259 260 func (tc *ConsulNamespacesE2ETest) TestConsulConnectTerminatingGateway(f *framework.F) { 261 nomadClient := tc.Nomad() 262 jobID := "cns-connect-terminating" 263 tc.jobIDs = append(tc.jobIDs, jobID) 264 265 // Run job and wait for allocs 266 allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, cnsJobConnectTerminating, jobID, "") 267 require.Len(f.T(), allocations, 6) // 2 x (2 services + 1 gateway) 268 allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations) 269 e2eutil.WaitForAllocsRunning(f.T(), tc.Nomad(), allocIDs) 270 271 r := f.Assertions 272 c := tc.Consul() 273 namespace := consulNamespace 274 275 // Verify services with cns set were registered into "default" Consul namespace 276 e2eutil.RequireConsulRegistered(r, c, namespace, "api-gateway", 1) 277 e2eutil.RequireConsulRegistered(r, c, namespace, "count-api", 1) 278 e2eutil.RequireConsulRegistered(r, c, namespace, "count-dashboard", 1) 279 280 // Verify services without cns set were registered into "default" Consul namespace 281 e2eutil.RequireConsulRegistered(r, c, namespace, "api-gateway-z", 1) 282 e2eutil.RequireConsulRegistered(r, c, namespace, "count-api-z", 1) 283 e2eutil.RequireConsulRegistered(r, c, namespace, "count-dashboard-z", 1) 284 285 // Verify services with cns set were NOT registered into specified consul namespaces 286 e2eutil.RequireConsulRegistered(r, c, "apple", "api-gateway", 0) 287 e2eutil.RequireConsulRegistered(r, c, "apple", "count-api", 0) 288 e2eutil.RequireConsulRegistered(r, c, "apple", "count-dashboard", 0) 289 290 // Read the config entry of gateway with cns set, checking it exists in "default' namespace 291 ce := e2eutil.ReadConsulConfigEntry(f.T(), c, namespace, "terminating-gateway", "api-gateway") 292 require.Equal(f.T(), namespace, ce.GetNamespace()) 293 294 // Read the config entry of gateway without cns set, checking it exists in "default' namespace 295 ceZ := e2eutil.ReadConsulConfigEntry(f.T(), c, namespace, "terminating-gateway", "api-gateway-z") 296 require.Equal(f.T(), namespace, ceZ.GetNamespace()) 297 298 // Stop the job 299 e2eutil.WaitForJobStopped(f.T(), nomadClient, jobID) 300 301 // Remove the config entries 302 e2eutil.DeleteConsulConfigEntry(f.T(), c, namespace, "terminating-gateway", "api-gateway") 303 e2eutil.DeleteConsulConfigEntry(f.T(), c, namespace, "terminating-gateway", "api-gateway-z") 304 } 305 306 func (tc *ConsulNamespacesE2ETest) TestConsulScriptChecksTask(f *framework.F) { 307 nomadClient := tc.Nomad() 308 jobID := "cns-script-checks-task" 309 tc.jobIDs = append(tc.jobIDs, jobID) 310 311 // Run job and wait for allocs 312 allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, cnsJobScriptChecksTask, jobID, "") 313 require.Len(f.T(), allocations, 2) 314 allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations) 315 e2eutil.WaitForAllocsRunning(f.T(), tc.Nomad(), allocIDs) 316 317 r := f.Assertions 318 c := tc.Consul() 319 namespace := consulNamespace 320 321 sort.Sort(e2eutil.AllocsByName(allocations)) 322 allocsWithSetNamespace := allocations[0:1] 323 allocsWithNoNamespace := allocations[1:2] 324 325 // Verify checks were registered into "default" Consul namespace 326 e2eutil.RequireConsulStatus(r, c, namespace, "service-1a", capi.HealthPassing) 327 e2eutil.RequireConsulStatus(r, c, namespace, "service-2a", capi.HealthWarning) 328 e2eutil.RequireConsulStatus(r, c, namespace, "service-3a", capi.HealthCritical) 329 330 // Check in warning state becomes healthy after check passes for the service 331 // with specified Consul namespace 332 // 333 // (ensures UpdateTTL is respecting namespace) 334 _, _, err := exec(nomadClient, allocsWithSetNamespace, 335 []string{"/bin/sh", "-c", "touch ${NOMAD_TASK_DIR}/alive-2ab"}) 336 r.NoError(err) 337 e2eutil.RequireConsulStatus(r, c, namespace, "service-2a", capi.HealthPassing) 338 339 // Verify checks were registered into "default" Consul namespace when no 340 // namespace was specified. 341 e2eutil.RequireConsulStatus(r, c, namespace, "service-1z", capi.HealthPassing) 342 e2eutil.RequireConsulStatus(r, c, namespace, "service-2z", capi.HealthWarning) 343 e2eutil.RequireConsulStatus(r, c, namespace, "service-3z", capi.HealthCritical) 344 345 // Check in warning state becomes healthy after check passes for the service 346 // with specified Consul namespace 347 // 348 // (ensures UpdateTTL is respecting namespace) 349 _, _, errZ := exec(nomadClient, allocsWithNoNamespace, 350 []string{"/bin/sh", "-c", "touch ${NOMAD_TASK_DIR}/alive-2zb"}) 351 r.NoError(errZ) 352 e2eutil.RequireConsulStatus(r, c, namespace, "service-2z", capi.HealthPassing) 353 354 // Stop the job 355 e2eutil.WaitForJobStopped(f.T(), nomadClient, jobID) 356 } 357 358 func (tc *ConsulNamespacesE2ETest) TestConsulScriptChecksGroup(f *framework.F) { 359 nomadClient := tc.Nomad() 360 jobID := "cns-script-checks-group" 361 tc.jobIDs = append(tc.jobIDs, jobID) 362 363 // Run job and wait for allocs 364 allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, cnsJobScriptChecksGroup, jobID, "") 365 require.Len(f.T(), allocations, 2) 366 allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations) 367 e2eutil.WaitForAllocsRunning(f.T(), tc.Nomad(), allocIDs) 368 369 r := f.Assertions 370 c := tc.Consul() 371 namespace := consulNamespace 372 373 sort.Sort(e2eutil.AllocsByName(allocations)) 374 allocsWithSetNamespace := allocations[0:1] 375 allocsWithNoNamespace := allocations[1:2] 376 377 // Verify checks were registered into "default" Consul namespace 378 e2eutil.RequireConsulStatus(r, c, namespace, "service-1a", capi.HealthPassing) 379 e2eutil.RequireConsulStatus(r, c, namespace, "service-2a", capi.HealthWarning) 380 e2eutil.RequireConsulStatus(r, c, namespace, "service-3a", capi.HealthCritical) 381 382 // Check in warning state becomes healthy after check passes for the service 383 // with specified Consul namespace 384 // 385 // (ensures UpdateTTL is respecting namespace) 386 _, _, err := exec(nomadClient, allocsWithSetNamespace, 387 []string{"/bin/sh", "-c", "touch /tmp/${NOMAD_ALLOC_ID}-alive-2ab"}) 388 r.NoError(err) 389 e2eutil.RequireConsulStatus(r, c, namespace, "service-2a", capi.HealthPassing) 390 391 // Verify checks were registered into "default" Consul namespace when no 392 // namespace was specified. 393 e2eutil.RequireConsulStatus(r, c, namespace, "service-1z", capi.HealthPassing) 394 e2eutil.RequireConsulStatus(r, c, namespace, "service-2z", capi.HealthWarning) 395 e2eutil.RequireConsulStatus(r, c, namespace, "service-3z", capi.HealthCritical) 396 397 // Check in warning state becomes healthy after check passes for the service 398 // with specified Consul namespace 399 // 400 // (ensures UpdateTTL is respecting namespace) 401 _, _, errZ := exec(nomadClient, allocsWithNoNamespace, 402 []string{"/bin/sh", "-c", "touch /tmp/${NOMAD_ALLOC_ID}-alive-2zb"}) 403 r.NoError(errZ) 404 e2eutil.RequireConsulStatus(r, c, namespace, "service-2z", capi.HealthPassing) 405 406 // Stop the job 407 e2eutil.WaitForJobStopped(f.T(), nomadClient, jobID) 408 }