github.com/anth0d/nomad@v0.0.0-20221214183521-ae3a0a2cad06/e2e/connect/acls.go (about) 1 package connect 2 3 import ( 4 "os" 5 "regexp" 6 "testing" 7 "time" 8 9 consulapi "github.com/hashicorp/consul/api" 10 uuidparse "github.com/hashicorp/go-uuid" 11 nomadapi "github.com/hashicorp/nomad/api" 12 "github.com/hashicorp/nomad/e2e/e2eutil" 13 "github.com/hashicorp/nomad/e2e/framework" 14 "github.com/hashicorp/nomad/helper/uuid" 15 "github.com/hashicorp/nomad/jobspec" 16 "github.com/stretchr/testify/require" 17 ) 18 19 type ConnectACLsE2ETest struct { 20 framework.TC 21 22 // used to store the root token so we can reset the client back to 23 // it as needed 24 consulManagementToken string 25 26 // things to cleanup after each test case 27 jobIDs []string 28 consulPolicyIDs []string 29 consulTokenIDs []string 30 } 31 32 func (tc *ConnectACLsE2ETest) BeforeAll(f *framework.F) { 33 // Wait for Nomad to be ready before doing anything. 34 e2eutil.WaitForLeader(f.T(), tc.Nomad()) 35 e2eutil.WaitForNodesReady(f.T(), tc.Nomad(), 2) 36 37 // Validate the consul root token exists, otherwise tests are just 38 // going to be a train wreck. 39 tc.consulManagementToken = os.Getenv(envConsulToken) 40 41 _, err := uuidparse.ParseUUID(tc.consulManagementToken) 42 f.NoError(err, "CONSUL_HTTP_TOKEN not set") 43 44 // ensure SI tokens from previous test cases were removed 45 f.Eventually(func() bool { 46 siTokens := tc.countSITokens(f.T()) 47 f.T().Log("cleanup: checking for remaining SI tokens:", siTokens) 48 return len(siTokens) == 0 49 }, 2*time.Minute, 2*time.Second, "SI tokens did not get removed") 50 } 51 52 // AfterEach does cleanup of Consul ACL objects that were created during each 53 // test case. Each test case may assume it is starting from a "fresh" state - 54 // as if the consul ACL bootstrap process had just taken place. 55 func (tc *ConnectACLsE2ETest) AfterEach(f *framework.F) { 56 if os.Getenv("NOMAD_TEST_SKIPCLEANUP") == "1" { 57 return 58 } 59 60 t := f.T() 61 62 // cleanup jobs 63 for _, id := range tc.jobIDs { 64 t.Log("cleanup: deregister nomad job id:", id) 65 _, _, err := tc.Nomad().Jobs().Deregister(id, true, nil) 66 f.NoError(err) 67 } 68 69 // cleanup consul tokens 70 for _, id := range tc.consulTokenIDs { 71 t.Log("cleanup: delete consul token id:", id) 72 _, err := tc.Consul().ACL().TokenDelete(id, &consulapi.WriteOptions{Token: tc.consulManagementToken}) 73 f.NoError(err) 74 } 75 76 // cleanup consul policies 77 for _, id := range tc.consulPolicyIDs { 78 t.Log("cleanup: delete consul policy id:", id) 79 _, err := tc.Consul().ACL().PolicyDelete(id, &consulapi.WriteOptions{Token: tc.consulManagementToken}) 80 f.NoError(err) 81 } 82 83 // do garbage collection 84 err := tc.Nomad().System().GarbageCollect() 85 f.NoError(err) 86 87 // assert there are no leftover SI tokens, which may take a minute to be 88 // cleaned up 89 f.Eventually(func() bool { 90 siTokens := tc.countSITokens(t) 91 t.Log("cleanup: checking for remaining SI tokens:", siTokens) 92 return len(siTokens) == 0 93 }, 2*time.Minute, 2*time.Second, "SI tokens did not get removed") 94 95 tc.jobIDs = []string{} 96 tc.consulTokenIDs = []string{} 97 tc.consulPolicyIDs = []string{} 98 } 99 100 // todo(shoenig): follow up refactor with e2eutil.ConsulPolicy 101 type consulPolicy struct { 102 Name string // e.g. nomad-operator 103 Rules string // e.g. service "" { policy="write" } 104 } 105 106 // todo(shoenig): follow up refactor with e2eutil.ConsulPolicy 107 func (tc *ConnectACLsE2ETest) createConsulPolicy(p consulPolicy, f *framework.F) string { 108 result, _, err := tc.Consul().ACL().PolicyCreate(&consulapi.ACLPolicy{ 109 Name: p.Name, 110 Description: "test policy " + p.Name, 111 Rules: p.Rules, 112 }, &consulapi.WriteOptions{Token: tc.consulManagementToken}) 113 f.NoError(err, "failed to create consul policy") 114 tc.consulPolicyIDs = append(tc.consulPolicyIDs, result.ID) 115 return result.ID 116 } 117 118 // todo(shoenig): follow up refactor with e2eutil.ConsulPolicy 119 func (tc *ConnectACLsE2ETest) createOperatorToken(policyID string, f *framework.F) string { 120 token, _, err := tc.Consul().ACL().TokenCreate(&consulapi.ACLToken{ 121 Description: "operator token", 122 Policies: []*consulapi.ACLTokenPolicyLink{{ID: policyID}}, 123 }, &consulapi.WriteOptions{Token: tc.consulManagementToken}) 124 f.NoError(err, "failed to create operator token") 125 tc.consulTokenIDs = append(tc.consulTokenIDs, token.AccessorID) 126 return token.SecretID 127 } 128 129 func (tc *ConnectACLsE2ETest) TestConnectACLsRegisterMasterToken(f *framework.F) { 130 t := f.T() 131 132 t.Log("test register Connect job w/ ACLs enabled w/ master token") 133 134 jobID := "connect" + uuid.Generate()[0:8] 135 tc.jobIDs = append(tc.jobIDs, jobID) 136 137 jobAPI := tc.Nomad().Jobs() 138 139 job, err := jobspec.ParseFile(demoConnectJob) 140 f.NoError(err) 141 142 // Set the job file to use the consul master token. 143 // One should never do this in practice, but, it should work. 144 // https://www.consul.io/docs/acl/acl-system.html#builtin-tokens 145 job.ConsulToken = &tc.consulManagementToken 146 job.ID = &jobID 147 148 // Avoid using Register here, because that would actually create and run the 149 // Job which runs the task, creates the SI token, which all needs to be 150 // given time to settle and cleaned up. That is all covered in the big slow 151 // test at the bottom. 152 resp, _, err := jobAPI.Plan(job, false, nil) 153 f.NoError(err) 154 f.NotNil(resp) 155 } 156 157 func (tc *ConnectACLsE2ETest) TestConnectACLsRegisterMissingOperatorToken(f *framework.F) { 158 t := f.T() 159 160 t.Skip("we don't have consul.allow_unauthenticated=false set because it would required updating every E2E test to pass a Consul token") 161 162 t.Log("test register Connect job w/ ACLs enabled w/o operator token") 163 164 jobID := "connect" + uuid.Short() 165 tc.jobIDs = append(tc.jobIDs, jobID) // need to clean up if the test fails 166 167 job, err := jobspec.ParseFile(demoConnectJob) 168 f.NoError(err) 169 jobAPI := tc.Nomad().Jobs() 170 171 // Explicitly show the ConsulToken is not set 172 job.ConsulToken = nil 173 job.ID = &jobID 174 175 _, _, err = jobAPI.Register(job, nil) 176 f.Error(err) 177 178 t.Log("job correctly rejected, with error:", err) 179 } 180 181 func (tc *ConnectACLsE2ETest) TestConnectACLsRegisterFakeOperatorToken(f *framework.F) { 182 t := f.T() 183 184 t.Skip("we don't have consul.allow_unauthenticated=false set because it would required updating every E2E test to pass a Consul token") 185 186 t.Log("test register Connect job w/ ACLs enabled w/ operator token") 187 188 policyID := tc.createConsulPolicy(consulPolicy{ 189 Name: "nomad-operator-policy-" + uuid.Short(), 190 Rules: `service "count-api" { policy = "write" } service "count-dashboard" { policy = "write" }`, 191 }, f) 192 t.Log("created operator policy:", policyID) 193 194 // generate a fake consul token token 195 fakeToken := uuid.Generate() 196 197 jobID := "connect" + uuid.Short() 198 tc.jobIDs = append(tc.jobIDs, jobID) // need to clean up if the test fails 199 200 job := tc.parseJobSpecFile(t, demoConnectJob) 201 202 jobAPI := tc.Nomad().Jobs() 203 204 // deliberately set the fake Consul token 205 job.ConsulToken = &fakeToken 206 job.ID = &jobID 207 208 // should fail, because the token is fake 209 _, _, err := jobAPI.Register(job, nil) 210 f.Error(err) 211 t.Log("job correctly rejected, with error:", err) 212 } 213 214 func (tc *ConnectACLsE2ETest) TestConnectACLsConnectDemo(f *framework.F) { 215 t := f.T() 216 217 t.Log("test register Connect job w/ ACLs enabled w/ operator token") 218 219 // === Setup ACL policy and mint Operator token === 220 221 // create a policy allowing writes of services "count-api" and "count-dashboard" 222 policyID := tc.createConsulPolicy(consulPolicy{ 223 Name: "nomad-operator-policy-" + uuid.Short(), 224 Rules: `service "count-api" { policy = "write" } service "count-dashboard" { policy = "write" }`, 225 }, f) 226 t.Log("created operator policy:", policyID) 227 228 // create a Consul "operator token" blessed with the above policy 229 operatorToken := tc.createOperatorToken(policyID, f) 230 t.Log("created operator token:", operatorToken) 231 232 jobID := connectJobID() 233 tc.jobIDs = append(tc.jobIDs, jobID) 234 235 allocs := e2eutil.RegisterAndWaitForAllocs(t, tc.Nomad(), demoConnectJob, jobID, operatorToken) 236 f.Equal(2, len(allocs), "expected 2 allocs for connect demo", allocs) 237 allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocs) 238 f.Equal(2, len(allocIDs), "expected 2 allocIDs for connect demo", allocIDs) 239 e2eutil.WaitForAllocsRunning(t, tc.Nomad(), allocIDs) 240 241 // === Check Consul SI tokens were generated for sidecars === 242 foundSITokens := tc.countSITokens(t) 243 f.Equal(2, len(foundSITokens), "expected 2 SI tokens total: %v", foundSITokens) 244 f.Equal(1, foundSITokens["connect-proxy-count-api"], "expected 1 SI token for connect-proxy-count-api: %v", foundSITokens) 245 f.Equal(1, foundSITokens["connect-proxy-count-dashboard"], "expected 1 SI token for connect-proxy-count-dashboard: %v", foundSITokens) 246 247 t.Log("connect legacy job with ACLs enable finished") 248 } 249 250 func (tc *ConnectACLsE2ETest) TestConnectACLsConnectNativeDemo(f *framework.F) { 251 t := f.T() 252 253 t.Log("test register Connect job w/ ACLs enabled w/ operator token") 254 255 // === Setup ACL policy and mint Operator token === 256 257 // create a policy allowing writes of services "uuid-fe" and "uuid-api" 258 policyID := tc.createConsulPolicy(consulPolicy{ 259 Name: "nomad-operator-policy-" + uuid.Short(), 260 Rules: `service "uuid-fe" { policy = "write" } service "uuid-api" { policy = "write" }`, 261 }, f) 262 t.Log("created operator policy:", policyID) 263 264 // create a Consul "operator token" blessed with the above policy 265 operatorToken := tc.createOperatorToken(policyID, f) 266 t.Log("created operator token:", operatorToken) 267 268 jobID := connectJobID() 269 tc.jobIDs = append(tc.jobIDs, jobID) 270 271 allocs := e2eutil.RegisterAndWaitForAllocs(t, tc.Nomad(), demoConnectNativeJob, jobID, operatorToken) 272 allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocs) 273 e2eutil.WaitForAllocsRunning(t, tc.Nomad(), allocIDs) 274 275 // === Check Consul SI tokens were generated for native tasks === 276 foundSITokens := tc.countSITokens(t) 277 f.Equal(2, len(foundSITokens), "expected 2 SI tokens total: %v", foundSITokens) 278 f.Equal(1, foundSITokens["frontend"], "expected 1 SI token for frontend: %v", foundSITokens) 279 f.Equal(1, foundSITokens["generate"], "expected 1 SI token for generate: %v", foundSITokens) 280 281 t.Log("connect native job with ACLs enabled finished") 282 } 283 284 func (tc *ConnectACLsE2ETest) TestConnectACLsConnectIngressGatewayDemo(f *framework.F) { 285 t := f.T() 286 287 t.Log("test register Connect Ingress Gateway job w/ ACLs enabled") 288 289 // setup ACL policy and mint operator token 290 291 policyID := tc.createConsulPolicy(consulPolicy{ 292 Name: "nomad-operator-policy-" + uuid.Short(), 293 Rules: `service "my-ingress-service" { policy = "write" } service "uuid-api" { policy = "write" }`, 294 }, f) 295 operatorToken := tc.createOperatorToken(policyID, f) 296 t.Log("created operator token:", operatorToken) 297 298 jobID := connectJobID() 299 tc.jobIDs = append(tc.jobIDs, jobID) 300 301 allocs := e2eutil.RegisterAndWaitForAllocs(t, tc.Nomad(), demoConnectIngressGateway, jobID, operatorToken) 302 allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocs) 303 e2eutil.WaitForAllocsRunning(t, tc.Nomad(), allocIDs) 304 305 foundSITokens := tc.countSITokens(t) 306 f.Equal(2, len(foundSITokens), "expected 2 SI tokens total: %v", foundSITokens) 307 f.Equal(1, foundSITokens["connect-ingress-my-ingress-service"], "expected 1 SI token for connect-ingress-my-ingress-service: %v", foundSITokens) 308 f.Equal(1, foundSITokens["generate"], "expected 1 SI token for generate: %v", foundSITokens) 309 310 t.Log("connect ingress gateway job with ACLs enabled finished") 311 } 312 313 func (tc *ConnectACLsE2ETest) TestConnectACLsConnectTerminatingGatewayDemo(f *framework.F) { 314 t := f.T() 315 316 t.Log("test register Connect Terminating Gateway job w/ ACLs enabled") 317 318 // setup ACL policy and mint operator token 319 320 policyID := tc.createConsulPolicy(consulPolicy{ 321 Name: "nomad-operator-policy-" + uuid.Short(), 322 Rules: `service "api-gateway" { policy = "write" } service "count-dashboard" { policy = "write" }`, 323 }, f) 324 operatorToken := tc.createOperatorToken(policyID, f) 325 t.Log("created operator token:", operatorToken) 326 327 jobID := connectJobID() 328 tc.jobIDs = append(tc.jobIDs, jobID) 329 330 allocs := e2eutil.RegisterAndWaitForAllocs(t, tc.Nomad(), demoConnectTerminatingGateway, jobID, operatorToken) 331 allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocs) 332 e2eutil.WaitForAllocsRunning(t, tc.Nomad(), allocIDs) 333 334 foundSITokens := tc.countSITokens(t) 335 f.Equal(2, len(foundSITokens), "expected 2 SI tokens total: %v", foundSITokens) 336 f.Equal(1, foundSITokens["connect-terminating-api-gateway"], "expected 1 SI token for connect-terminating-api-gateway: %v", foundSITokens) 337 f.Equal(1, foundSITokens["connect-proxy-count-dashboard"], "expected 1 SI token for count-dashboard: %v", foundSITokens) 338 339 t.Log("connect terminating gateway job with ACLs enabled finished") 340 } 341 342 var ( 343 siTokenRe = regexp.MustCompile(`_nomad_si \[[\w-]{36}] \[[\w-]{36}] \[([\S]+)]`) 344 ) 345 346 func (tc *ConnectACLsE2ETest) serviceofSIToken(description string) string { 347 if m := siTokenRe.FindStringSubmatch(description); len(m) == 2 { 348 return m[1] 349 } 350 return "" 351 } 352 353 func (tc *ConnectACLsE2ETest) countSITokens(t *testing.T) map[string]int { 354 aclAPI := tc.Consul().ACL() 355 tokens, _, err := aclAPI.TokenList(&consulapi.QueryOptions{ 356 Token: tc.consulManagementToken, 357 }) 358 require.NoError(t, err) 359 360 // count the number of SI tokens matching each service name 361 foundSITokens := make(map[string]int) 362 for _, token := range tokens { 363 if service := tc.serviceofSIToken(token.Description); service != "" { 364 foundSITokens[service]++ 365 } 366 } 367 368 return foundSITokens 369 } 370 371 func (tc *ConnectACLsE2ETest) parseJobSpecFile(t *testing.T, filename string) *nomadapi.Job { 372 job, err := jobspec.ParseFile(filename) 373 require.NoError(t, err) 374 return job 375 }