github.com/anth0d/nomad@v0.0.0-20221214183521-ae3a0a2cad06/e2e/consul/namespaces.go (about) 1 package consul 2 3 import ( 4 "fmt" 5 "os" 6 "sort" 7 8 capi "github.com/hashicorp/consul/api" 9 "github.com/hashicorp/nomad/e2e/e2eutil" 10 "github.com/hashicorp/nomad/e2e/framework" 11 "github.com/hashicorp/nomad/helper" 12 "github.com/stretchr/testify/require" 13 ) 14 15 // Job files used to test Consul Namespaces. Each job should run on Nomad OSS 16 // and Nomad ENT with expectations set accordingly. 17 // 18 // All tests require Consul Enterprise. 19 const ( 20 cnsJobGroupServices = "consul/input/namespaces/services_group.nomad" 21 cnsJobTaskServices = "consul/input/namespaces/services_task.nomad" 22 cnsJobTemplateKV = "consul/input/namespaces/template_kv.nomad" 23 cnsJobConnectSidecars = "consul/input/namespaces/connect_sidecars.nomad" 24 cnsJobConnectIngress = "consul/input/namespaces/connect_ingress.nomad" 25 cnsJobConnectTerminating = "consul/input/namespaces/connect_terminating.nomad" 26 cnsJobScriptChecksTask = "consul/input/namespaces/script_checks_task.nomad" 27 cnsJobScriptChecksGroup = "consul/input/namespaces/script_checks_group.nomad" 28 ) 29 30 var ( 31 // consulNamespaces represents the custom consul namespaces we create and 32 // can make use of in tests, but usefully so only in Nomad Enterprise 33 consulNamespaces = []string{"apple", "banana", "cherry"} 34 35 // allConsulNamespaces represents all namespaces we expect in consul after 36 // creating consulNamespaces, which then includes "default", which is the 37 // only namespace accessed by Nomad OSS (outside of agent configuration) 38 allConsulNamespaces = append(consulNamespaces, "default") 39 ) 40 41 func init() { 42 framework.AddSuites(&framework.TestSuite{ 43 Component: "ConsulNamespaces", 44 CanRunLocal: true, 45 Consul: true, 46 Cases: []framework.TestCase{ 47 new(ConsulNamespacesE2ETest), 48 }, 49 }) 50 } 51 52 type ConsulNamespacesE2ETest struct { 53 framework.TC 54 55 jobIDs []string 56 57 // cToken contains the Consul global-management token 58 cToken string 59 60 // created policy and token IDs should be set here so they can be cleaned 61 // up after each test case, organized by namespace 62 policyIDs map[string][]string 63 tokenIDs map[string][]string 64 } 65 66 func (tc *ConsulNamespacesE2ETest) BeforeAll(f *framework.F) { 67 tc.policyIDs = make(map[string][]string) 68 tc.tokenIDs = make(map[string][]string) 69 70 e2eutil.WaitForLeader(f.T(), tc.Nomad()) 71 e2eutil.WaitForNodesReady(f.T(), tc.Nomad(), 1) 72 73 tc.cToken = os.Getenv("CONSUL_HTTP_TOKEN") 74 75 // create a set of consul namespaces in which to register services 76 e2eutil.CreateConsulNamespaces(f.T(), tc.Consul(), consulNamespaces) 77 78 // insert a key of the same name into KV for each namespace, where the value 79 // contains the namespace name making it easy to determine which namespace 80 // consul template actually accessed 81 for _, namespace := range allConsulNamespaces { 82 value := fmt.Sprintf("ns_%s", namespace) 83 e2eutil.PutConsulKey(f.T(), tc.Consul(), namespace, "ns-kv-example", value) 84 } 85 } 86 87 func (tc *ConsulNamespacesE2ETest) AfterAll(f *framework.F) { 88 e2eutil.DeleteConsulNamespaces(f.T(), tc.Consul(), consulNamespaces) 89 } 90 91 func (tc *ConsulNamespacesE2ETest) TestNamespacesExist(f *framework.F) { 92 // make sure our namespaces exist + default 93 namespaces := e2eutil.ListConsulNamespaces(f.T(), tc.Consul()) 94 require.True(f.T(), helper.SliceSetEq(namespaces, append(consulNamespaces, "default"))) 95 } 96 97 func (tc *ConsulNamespacesE2ETest) testConsulRegisterGroupServices(f *framework.F, token, nsA, nsB, nsC, nsZ string) { 98 nomadClient := tc.Nomad() 99 jobID := "cns-group-services" 100 tc.jobIDs = append(tc.jobIDs, jobID) 101 102 // Run job and wait for allocs 103 allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, cnsJobGroupServices, jobID, token) 104 require.Len(f.T(), allocations, 3) 105 allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations) 106 e2eutil.WaitForAllocsRunning(f.T(), tc.Nomad(), allocIDs) 107 108 r := f.Assertions 109 c := tc.Consul() 110 111 // Verify services with namespace set are registered into expected namespaces 112 e2eutil.RequireConsulRegistered(r, c, nsB, "b1", 1) 113 e2eutil.RequireConsulRegistered(r, c, nsB, "b2", 1) 114 e2eutil.RequireConsulRegistered(r, c, nsC, "c1", 1) 115 e2eutil.RequireConsulRegistered(r, c, nsC, "c2", 1) 116 117 // Verify services without namespace set are registered into default 118 e2eutil.RequireConsulRegistered(r, c, nsZ, "z1", 1) 119 e2eutil.RequireConsulRegistered(r, c, nsZ, "z2", 1) 120 121 // Verify our services are all healthy 122 e2eutil.RequireConsulStatus(r, c, nsB, "b1", "passing") 123 e2eutil.RequireConsulStatus(r, c, nsB, "b2", "passing") 124 e2eutil.RequireConsulStatus(r, c, nsC, "c1", "passing") 125 e2eutil.RequireConsulStatus(r, c, nsC, "c2", "passing") 126 e2eutil.RequireConsulStatus(r, c, nsZ, "z1", "passing") 127 e2eutil.RequireConsulStatus(r, c, nsZ, "z2", "passing") 128 129 // Stop the job 130 e2eutil.WaitForJobStopped(f.T(), nomadClient, jobID) 131 132 // Verify that services were de-registered from Consul 133 e2eutil.RequireConsulDeregistered(r, c, nsB, "b1") 134 e2eutil.RequireConsulDeregistered(r, c, nsB, "b2") 135 e2eutil.RequireConsulDeregistered(r, c, nsC, "c1") 136 e2eutil.RequireConsulDeregistered(r, c, nsC, "c2") 137 e2eutil.RequireConsulDeregistered(r, c, nsZ, "z1") 138 e2eutil.RequireConsulDeregistered(r, c, nsZ, "z2") 139 } 140 141 func (tc *ConsulNamespacesE2ETest) testConsulRegisterTaskServices(f *framework.F, token, nsA, nsB, nsC, nsZ string) { 142 nomadClient := tc.Nomad() 143 jobID := "cns-task-services" 144 tc.jobIDs = append(tc.jobIDs, jobID) 145 146 // Run job and wait for allocs 147 allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, cnsJobTaskServices, jobID, token) 148 require.Len(f.T(), allocations, 3) 149 allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations) 150 e2eutil.WaitForAllocsRunning(f.T(), tc.Nomad(), allocIDs) 151 152 r := f.Assertions 153 c := tc.Consul() 154 155 // Verify our services were registered into expected namespaces 156 e2eutil.RequireConsulRegistered(r, c, nsB, "b1", 1) 157 e2eutil.RequireConsulRegistered(r, c, nsB, "b2", 1) 158 e2eutil.RequireConsulRegistered(r, c, nsC, "c1", 1) 159 e2eutil.RequireConsulRegistered(r, c, nsC, "c2", 1) 160 e2eutil.RequireConsulRegistered(r, c, nsZ, "z1", 1) 161 e2eutil.RequireConsulRegistered(r, c, nsZ, "z2", 1) 162 163 // Verify our services are all healthy 164 e2eutil.RequireConsulStatus(r, c, nsB, "b1", "passing") 165 e2eutil.RequireConsulStatus(r, c, nsB, "b2", "passing") 166 e2eutil.RequireConsulStatus(r, c, nsC, "c1", "passing") 167 e2eutil.RequireConsulStatus(r, c, nsC, "c2", "passing") 168 e2eutil.RequireConsulStatus(r, c, nsZ, "z1", "passing") 169 e2eutil.RequireConsulStatus(r, c, nsZ, "z2", "passing") 170 171 // Stop the job 172 e2eutil.WaitForJobStopped(f.T(), nomadClient, jobID) 173 174 // Verify that services were de-registered from Consul 175 e2eutil.RequireConsulDeregistered(r, c, nsB, "b1") 176 e2eutil.RequireConsulDeregistered(r, c, nsB, "b2") 177 e2eutil.RequireConsulDeregistered(r, c, nsC, "c1") 178 e2eutil.RequireConsulDeregistered(r, c, nsC, "c2") 179 e2eutil.RequireConsulDeregistered(r, c, nsZ, "z1") 180 e2eutil.RequireConsulDeregistered(r, c, nsZ, "z2") 181 } 182 183 func (tc *ConsulNamespacesE2ETest) testConsulTemplateKV(f *framework.F, token, expB, expZ string) { 184 t := f.T() 185 nomadClient := tc.Nomad() 186 jobID := "cns-template-kv" 187 tc.jobIDs = append(tc.jobIDs, jobID) 188 189 // Run job and wait for allocs to complete 190 allocations := e2eutil.RegisterAndWaitForAllocs(t, nomadClient, cnsJobTemplateKV, jobID, token) 191 require.Len(t, allocations, 2) 192 allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations) 193 e2eutil.WaitForAllocsStopped(f.T(), tc.Nomad(), allocIDs) 194 195 // Sort allocs by name 196 sort.Sort(e2eutil.AllocsByName(allocations)) 197 198 // Check template read from expected namespace when namespace set 199 textB, err := e2eutil.AllocTaskLogs(allocations[0].ID, "task-b", e2eutil.LogsStdOut) 200 require.NoError(t, err) 201 require.Equal(t, expB, textB) 202 203 // Check template read from default namespace if no namespace set 204 textZ, err := e2eutil.AllocTaskLogs(allocations[1].ID, "task-z", e2eutil.LogsStdOut) 205 require.NoError(t, err) 206 require.Equal(t, expZ, textZ) 207 208 // Stop the job 209 e2eutil.WaitForJobStopped(t, nomadClient, jobID) 210 } 211 212 func (tc *ConsulNamespacesE2ETest) testConsulConnectSidecars(f *framework.F, token, nsA, nsZ string) { 213 nomadClient := tc.Nomad() 214 jobID := "cns-connect-sidecars" 215 tc.jobIDs = append(tc.jobIDs, jobID) 216 217 // Run job and wait for allocs 218 allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, cnsJobConnectSidecars, jobID, token) 219 require.Len(f.T(), allocations, 4) 220 allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations) 221 e2eutil.WaitForAllocsRunning(f.T(), tc.Nomad(), allocIDs) 222 223 r := f.Assertions 224 c := tc.Consul() 225 226 // Verify services with cns set were registered into expected namespace 227 e2eutil.RequireConsulRegistered(r, c, nsA, "count-api", 1) 228 e2eutil.RequireConsulRegistered(r, c, nsA, "count-api-sidecar-proxy", 1) 229 e2eutil.RequireConsulRegistered(r, c, nsA, "count-dashboard", 1) 230 e2eutil.RequireConsulRegistered(r, c, nsA, "count-dashboard-sidecar-proxy", 1) 231 232 // Verify services without cns set were registered into default 233 e2eutil.RequireConsulRegistered(r, c, nsZ, "count-api-z", 1) 234 e2eutil.RequireConsulRegistered(r, c, nsZ, "count-api-z-sidecar-proxy", 1) 235 e2eutil.RequireConsulRegistered(r, c, nsZ, "count-dashboard-z", 1) 236 e2eutil.RequireConsulRegistered(r, c, nsZ, "count-dashboard-z-sidecar-proxy", 1) 237 238 // Stop the job 239 e2eutil.WaitForJobStopped(f.T(), nomadClient, jobID) 240 241 // Verify that services were de-registered from Consul 242 e2eutil.RequireConsulDeregistered(r, c, nsA, "count-api") 243 e2eutil.RequireConsulDeregistered(r, c, nsA, "count-api-sidecar-proxy") 244 e2eutil.RequireConsulDeregistered(r, c, nsA, "count-dashboard") 245 e2eutil.RequireConsulDeregistered(r, c, nsA, "count-dashboard-sidecar-proxy") 246 e2eutil.RequireConsulDeregistered(r, c, nsZ, "count-api-z") 247 e2eutil.RequireConsulDeregistered(r, c, nsZ, "count-api-z-sidecar-proxy") 248 e2eutil.RequireConsulDeregistered(r, c, nsZ, "count-dashboard-z") 249 e2eutil.RequireConsulDeregistered(r, c, nsZ, "count-dashboard-z-sidecar-proxy") 250 } 251 252 func (tc *ConsulNamespacesE2ETest) testConsulConnectIngressGateway(f *framework.F, token, nsA, nsZ string) { 253 nomadClient := tc.Nomad() 254 jobID := "cns-connect-ingress" 255 tc.jobIDs = append(tc.jobIDs, jobID) 256 257 // Run job and wait for allocs 258 allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, cnsJobConnectIngress, jobID, token) 259 require.Len(f.T(), allocations, 4) // 2 x (1 service + 1 gateway) 260 allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations) 261 e2eutil.WaitForAllocsRunning(f.T(), tc.Nomad(), allocIDs) 262 263 r := f.Assertions 264 c := tc.Consul() 265 266 // Verify services with cns set were registered into expected namespace 267 e2eutil.RequireConsulRegistered(r, c, nsA, "my-ingress-service", 1) 268 e2eutil.RequireConsulRegistered(r, c, nsA, "uuid-api", 1) 269 270 // Verify services without cns set were registered into default 271 e2eutil.RequireConsulRegistered(r, c, nsZ, "my-ingress-service-z", 1) 272 e2eutil.RequireConsulRegistered(r, c, nsZ, "uuid-api-z", 1) 273 274 // Read the config entry of gateway with cns set, checking it exists in expected namespace 275 ce := e2eutil.ReadConsulConfigEntry(f.T(), c, nsA, "ingress-gateway", "my-ingress-service") 276 require.Equal(f.T(), nsA, ce.GetNamespace()) 277 278 // Read the config entry of gateway without cns set, checking it exists in default namespace 279 ceZ := e2eutil.ReadConsulConfigEntry(f.T(), c, nsZ, "ingress-gateway", "my-ingress-service-z") 280 require.Equal(f.T(), nsZ, ceZ.GetNamespace()) 281 282 // Stop the job 283 e2eutil.WaitForJobStopped(f.T(), nomadClient, jobID) 284 285 // Remove the config entries 286 e2eutil.DeleteConsulConfigEntry(f.T(), c, nsA, "ingress-gateway", "my-ingress-service") 287 e2eutil.DeleteConsulConfigEntry(f.T(), c, nsZ, "ingress-gateway", "my-ingress-service-z") 288 } 289 290 func (tc *ConsulNamespacesE2ETest) testConsulConnectTerminatingGateway(f *framework.F, token, nsA, nsZ string) { 291 nomadClient := tc.Nomad() 292 jobID := "cns-connect-terminating" 293 tc.jobIDs = append(tc.jobIDs, jobID) 294 295 // Run job and wait for allocs 296 allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, cnsJobConnectTerminating, jobID, token) 297 require.Len(f.T(), allocations, 6) // 2 x (2 services + 1 gateway) 298 allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations) 299 e2eutil.WaitForAllocsRunning(f.T(), tc.Nomad(), allocIDs) 300 301 r := f.Assertions 302 c := tc.Consul() 303 304 // Verify services with cns set were registered into "default" Consul namespace 305 e2eutil.RequireConsulRegistered(r, c, nsA, "api-gateway", 1) 306 e2eutil.RequireConsulRegistered(r, c, nsA, "count-api", 1) 307 e2eutil.RequireConsulRegistered(r, c, nsA, "count-dashboard", 1) 308 309 // Verify services without cns set were registered into "default" Consul namespace 310 e2eutil.RequireConsulRegistered(r, c, nsZ, "api-gateway-z", 1) 311 e2eutil.RequireConsulRegistered(r, c, nsZ, "count-api-z", 1) 312 e2eutil.RequireConsulRegistered(r, c, nsZ, "count-dashboard-z", 1) 313 314 // Read the config entry of gateway with cns set, checking it exists in "default' namespace 315 ce := e2eutil.ReadConsulConfigEntry(f.T(), c, nsA, "terminating-gateway", "api-gateway") 316 require.Equal(f.T(), nsA, ce.GetNamespace()) 317 318 // Read the config entry of gateway without cns set, checking it exists in "default' namespace 319 ceZ := e2eutil.ReadConsulConfigEntry(f.T(), c, nsZ, "terminating-gateway", "api-gateway-z") 320 require.Equal(f.T(), nsZ, ceZ.GetNamespace()) 321 322 // Stop the job 323 e2eutil.WaitForJobStopped(f.T(), nomadClient, jobID) 324 325 // Remove the config entries 326 e2eutil.DeleteConsulConfigEntry(f.T(), c, nsA, "terminating-gateway", "api-gateway") 327 e2eutil.DeleteConsulConfigEntry(f.T(), c, nsZ, "terminating-gateway", "api-gateway-z") 328 } 329 330 func (tc *ConsulNamespacesE2ETest) testConsulScriptChecksTask(f *framework.F, token, nsA, nsZ string) { 331 nomadClient := tc.Nomad() 332 jobID := "cns-script-checks-task" 333 tc.jobIDs = append(tc.jobIDs, jobID) 334 335 // Run job and wait for allocs 336 allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, cnsJobScriptChecksTask, jobID, token) 337 require.Len(f.T(), allocations, 2) 338 allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations) 339 e2eutil.WaitForAllocsRunning(f.T(), tc.Nomad(), allocIDs) 340 341 r := f.Assertions 342 c := tc.Consul() 343 344 sort.Sort(e2eutil.AllocsByName(allocations)) 345 allocsWithSetNamespace := allocations[0:1] 346 allocsWithNoNamespace := allocations[1:2] 347 348 // Verify checks with namespace set are set into expected namespace 349 e2eutil.RequireConsulStatus(r, c, nsA, "service-1a", capi.HealthPassing) 350 e2eutil.RequireConsulStatus(r, c, nsA, "service-2a", capi.HealthWarning) 351 e2eutil.RequireConsulStatus(r, c, nsA, "service-3a", capi.HealthCritical) 352 353 // Check in warning state becomes healthy after check passes for the service 354 // with specified Consul namespace 355 // 356 // (ensures UpdateTTL is respecting namespace) 357 _, _, err := exec(nomadClient, allocsWithSetNamespace, 358 []string{"/bin/sh", "-c", "touch ${NOMAD_TASK_DIR}/alive-2ab"}) 359 r.NoError(err) 360 e2eutil.RequireConsulStatus(r, c, nsA, "service-2a", capi.HealthPassing) 361 362 // Verify checks without namespace are set in default namespace 363 e2eutil.RequireConsulStatus(r, c, nsZ, "service-1z", capi.HealthPassing) 364 e2eutil.RequireConsulStatus(r, c, nsZ, "service-2z", capi.HealthWarning) 365 e2eutil.RequireConsulStatus(r, c, nsZ, "service-3z", capi.HealthCritical) 366 367 // Check in warning state becomes healthy after check passes for the service 368 // with specified Consul namespace 369 // 370 // (ensures UpdateTTL is respecting namespace) 371 _, _, errZ := exec(nomadClient, allocsWithNoNamespace, 372 []string{"/bin/sh", "-c", "touch ${NOMAD_TASK_DIR}/alive-2zb"}) 373 r.NoError(errZ) 374 e2eutil.RequireConsulStatus(r, c, nsZ, "service-2z", capi.HealthPassing) 375 376 // Stop the job 377 e2eutil.WaitForJobStopped(f.T(), nomadClient, jobID) 378 } 379 380 func (tc *ConsulNamespacesE2ETest) testConsulScriptChecksGroup(f *framework.F, token, nsA, nsZ string) { 381 nomadClient := tc.Nomad() 382 jobID := "cns-script-checks-group" 383 tc.jobIDs = append(tc.jobIDs, jobID) 384 385 // Run job and wait for allocs 386 allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, cnsJobScriptChecksGroup, jobID, token) 387 require.Len(f.T(), allocations, 2) 388 allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations) 389 e2eutil.WaitForAllocsRunning(f.T(), tc.Nomad(), allocIDs) 390 391 r := f.Assertions 392 c := tc.Consul() 393 394 sort.Sort(e2eutil.AllocsByName(allocations)) 395 allocsWithSetNamespace := allocations[0:1] 396 allocsWithNoNamespace := allocations[1:2] 397 398 // Verify checks were registered into "default" Consul namespace 399 e2eutil.RequireConsulStatus(r, c, nsA, "service-1a", capi.HealthPassing) 400 e2eutil.RequireConsulStatus(r, c, nsA, "service-2a", capi.HealthWarning) 401 e2eutil.RequireConsulStatus(r, c, nsA, "service-3a", capi.HealthCritical) 402 403 // Check in warning state becomes healthy after check passes for the service 404 // with specified Consul namespace 405 // 406 // (ensures UpdateTTL is respecting namespace) 407 _, _, err := exec(nomadClient, allocsWithSetNamespace, 408 []string{"/bin/sh", "-c", "touch /tmp/${NOMAD_ALLOC_ID}-alive-2ab"}) 409 r.NoError(err) 410 e2eutil.RequireConsulStatus(r, c, nsA, "service-2a", capi.HealthPassing) 411 412 // Verify checks were registered into "default" Consul namespace when no 413 // namespace was specified. 414 e2eutil.RequireConsulStatus(r, c, nsZ, "service-1z", capi.HealthPassing) 415 e2eutil.RequireConsulStatus(r, c, nsZ, "service-2z", capi.HealthWarning) 416 e2eutil.RequireConsulStatus(r, c, nsZ, "service-3z", capi.HealthCritical) 417 418 // Check in warning state becomes healthy after check passes for the service 419 // with specified Consul namespace 420 // 421 // (ensures UpdateTTL is respecting namespace) 422 _, _, errZ := exec(nomadClient, allocsWithNoNamespace, 423 []string{"/bin/sh", "-c", "touch /tmp/${NOMAD_ALLOC_ID}-alive-2zb"}) 424 r.NoError(errZ) 425 e2eutil.RequireConsulStatus(r, c, nsZ, "service-2z", capi.HealthPassing) 426 427 // Stop the job 428 e2eutil.WaitForJobStopped(f.T(), nomadClient, jobID) 429 }