github.com/hernad/nomad@v1.6.112/e2e/auth/auth_test.go (about) 1 // Copyright (c) HashiCorp, Inc. 2 // SPDX-License-Identifier: MPL-2.0 3 4 package auth 5 6 import ( 7 "fmt" 8 "os" 9 "testing" 10 "time" 11 12 "github.com/hernad/nomad/api" 13 "github.com/hernad/nomad/e2e/e2eutil" 14 "github.com/hernad/nomad/helper/uuid" 15 "github.com/shoenig/test/must" 16 ) 17 18 var validPolicySpec = ` 19 namespace "%s" { 20 policy = "read" 21 variables { 22 path "test/*" { 23 capabilities = [ "write", "destroy" ] 24 } 25 } 26 } 27 28 node { 29 policy = "write" 30 } 31 ` 32 33 // TestAuth verifies that we're correctly enforcing ACLs with different 34 // combinations of tokens, policies, API types, and topologies. 35 func TestAuth(t *testing.T) { 36 37 // Wait until we have a usable cluster before running the tests. 38 nomadClient := e2eutil.NomadClient(t) 39 e2eutil.WaitForLeader(t, nomadClient) 40 e2eutil.WaitForNodesReady(t, nomadClient, 1) 41 42 nodes, _, err := nomadClient.Nodes().List(nil) 43 must.NoError(t, err, must.Sprint("expected no error from root client")) 44 must.Greater(t, 0, len(nodes)) 45 node, _, err := nomadClient.Nodes().Info(nodes[0].ID, nil) 46 47 ns := uuid.Generate() 48 validPolicyName := uuid.Generate() 49 invalidPolicyName := uuid.Generate() 50 51 setupAuthTest(t, nomadClient, ns, validPolicyName, invalidPolicyName) 52 53 // Test cases that exercise requests directly to the server 54 t.Run("AnonServerRequests", testAnonServerRequests(node, ns)) 55 t.Run("BogusServerRequests", testBogusServerRequests(nomadClient, node, ns)) 56 t.Run("InvalidPermissionsServerRequests", 57 testInvalidPermissionsServerRequests(nomadClient, node, ns, invalidPolicyName)) 58 t.Run("ValidPermissionsServerRequests", 59 testValidPermissionsServerRequests(nomadClient, node, ns, validPolicyName)) 60 61 // Test cases that exercise requests forwarded from the client 62 t.Run("AnonClientRequests", testAnonClientRequests(node, ns)) 63 t.Run("BogusClientRequests", testBogusClientRequests(nomadClient, node, ns)) 64 t.Run("InvalidPermissionsClientRequests", 65 testInvalidPermissionsClientRequests(nomadClient, node, ns, invalidPolicyName)) 66 t.Run("ValidPermissionsClientRequests", 67 testValidPermissionsClientRequests(nomadClient, node, ns, validPolicyName)) 68 } 69 70 func testAnonServerRequests(node *api.Node, ns string) func(t *testing.T) { 71 return func(t *testing.T) { 72 nomadClient := e2eutil.NomadClient(t) 73 nomadClient.SetSecretID("") 74 75 testReadNamespaceAPI(t, nomadClient, ns, "", true) 76 testNodeAPI(t, nomadClient, node.ID, "", true) 77 testVariablesAPI(t, nomadClient, ns, "", true, true) 78 } 79 } 80 81 func testBogusServerRequests(nomadClient *api.Client, 82 node *api.Node, ns string) func(t *testing.T) { 83 return func(t *testing.T) { 84 authToken := uuid.Generate() 85 86 testReadNamespaceAPI(t, nomadClient, ns, authToken, true) 87 testNodeAPI(t, nomadClient, node.ID, authToken, true) 88 testVariablesAPI(t, nomadClient, ns, authToken, true, true) 89 } 90 } 91 92 func testInvalidPermissionsServerRequests(nomadClient *api.Client, 93 node *api.Node, ns, policyName string) func(t *testing.T) { 94 return func(t *testing.T) { 95 token, _, err := nomadClient.ACLTokens().Create(&api.ACLToken{ 96 Name: policyName, 97 Type: "client", 98 Policies: []string{policyName}, 99 ExpirationTTL: time.Minute, 100 }, nil) 101 must.NoError(t, err) 102 authToken := token.SecretID 103 104 testReadNamespaceAPI(t, nomadClient, ns, authToken, true) 105 testNodeAPI(t, nomadClient, node.ID, authToken, true) 106 testVariablesAPI(t, nomadClient, ns, authToken, true, true) 107 } 108 } 109 110 func testValidPermissionsServerRequests(nomadClient *api.Client, 111 node *api.Node, ns, policyName string) func(t *testing.T) { 112 return func(t *testing.T) { 113 token, _, err := nomadClient.ACLTokens().Create(&api.ACLToken{ 114 Name: policyName, 115 Type: "client", 116 Policies: []string{policyName}, 117 ExpirationTTL: time.Minute, 118 }, nil) 119 must.NoError(t, err) 120 authToken := token.SecretID 121 122 testReadNamespaceAPI(t, nomadClient, ns, authToken, false) 123 testNodeAPI(t, nomadClient, node.ID, authToken, false) 124 testVariablesAPI(t, nomadClient, ns, authToken, false, true) 125 } 126 } 127 128 func testAnonClientRequests(node *api.Node, ns string) func(t *testing.T) { 129 return func(t *testing.T) { 130 config := api.DefaultConfig() 131 config.Address = addressForNode(node) 132 nomadClient, err := api.NewClient(config) 133 nomadClient.SetSecretID("") 134 must.NoError(t, err) 135 136 testReadNamespaceAPI(t, nomadClient, ns, "", true) 137 testNodeAPI(t, nomadClient, node.ID, "", true) 138 testVariablesAPI(t, nomadClient, ns, "", true, true) 139 } 140 } 141 142 func testBogusClientRequests(rootClient *api.Client, 143 node *api.Node, ns string) func(t *testing.T) { 144 return func(t *testing.T) { 145 config := api.DefaultConfig() 146 config.Address = addressForNode(node) 147 nomadClient, err := api.NewClient(config) 148 must.NoError(t, err) 149 150 authToken := uuid.Generate() 151 152 testReadNamespaceAPI(t, nomadClient, ns, authToken, true) 153 testNodeAPI(t, nomadClient, node.ID, authToken, true) 154 testVariablesAPI(t, nomadClient, ns, authToken, true, true) 155 } 156 } 157 158 func testInvalidPermissionsClientRequests(rootClient *api.Client, 159 node *api.Node, ns, policyName string) func(t *testing.T) { 160 return func(t *testing.T) { 161 token, _, err := rootClient.ACLTokens().Create(&api.ACLToken{ 162 Name: policyName, 163 Type: "client", 164 Policies: []string{policyName}, 165 ExpirationTTL: time.Minute, 166 }, nil) 167 must.NoError(t, err) 168 169 config := api.DefaultConfig() 170 config.Address = addressForNode(node) 171 nomadClient, err := api.NewClient(config) 172 must.NoError(t, err) 173 174 authToken := token.SecretID 175 176 testReadNamespaceAPI(t, nomadClient, ns, authToken, true) 177 testNodeAPI(t, nomadClient, node.ID, authToken, true) 178 testVariablesAPI(t, nomadClient, ns, authToken, true, true) 179 } 180 } 181 182 func testValidPermissionsClientRequests(rootClient *api.Client, 183 node *api.Node, ns, policyName string) func(t *testing.T) { 184 return func(t *testing.T) { 185 token, _, err := rootClient.ACLTokens().Create(&api.ACLToken{ 186 Name: policyName, 187 Type: "client", 188 Policies: []string{policyName}, 189 ExpirationTTL: time.Minute, 190 }, nil) 191 must.NoError(t, err) 192 193 config := api.DefaultConfig() 194 config.Address = addressForNode(node) 195 nomadClient, err := api.NewClient(config) 196 must.NoError(t, err) 197 198 authToken := token.SecretID 199 200 testReadNamespaceAPI(t, nomadClient, ns, authToken, false) 201 testNodeAPI(t, nomadClient, node.ID, authToken, false) 202 testVariablesAPI(t, nomadClient, ns, authToken, false, true) 203 } 204 } 205 206 // testReadNamespaceAPI exercises an API that requires any namespace capability 207 func testReadNamespaceAPI(t *testing.T, nomadClient *api.Client, ns, authToken string, expectErr bool) { 208 t.Helper() 209 opts := &api.QueryOptions{AuthToken: authToken} 210 _, _, err := nomadClient.Namespaces().Info(ns, opts) 211 if expectErr { 212 must.Error(t, err, must.Sprint("expected error when reading namespace")) 213 } else { 214 must.NoError(t, err, must.Sprint("expected no error reading namespace")) 215 } 216 } 217 218 // testNodeAPI exercises an API that requires the node:write permission 219 func testNodeAPI(t *testing.T, nomadClient *api.Client, nodeID, authToken string, expectErr bool) { 220 t.Helper() 221 opts := &api.WriteOptions{AuthToken: authToken} 222 _, _, err := nomadClient.Nodes().ForceEvaluate(nodeID, opts) 223 if expectErr { 224 must.Error(t, err, must.Sprint("expected error when force-evaluating node")) 225 } else { 226 must.NoError(t, err, must.Sprint("expected no error force-evaluating node")) 227 } 228 } 229 230 // testVariablesAPI exercises an API that requires namespace capabilities for 231 // variables 232 func testVariablesAPI(t *testing.T, nomadClient *api.Client, ns, authToken string, expectErrTestPath, expectErrOutsidePath bool) { 233 t.Helper() 234 opts := &api.WriteOptions{Namespace: ns, AuthToken: authToken} 235 236 _, _, err := nomadClient.Variables().Create(&api.Variable{ 237 Namespace: ns, 238 Path: "test/" + t.Name(), 239 Items: map[string]string{"foo": t.Name()}, 240 }, opts) 241 242 if expectErrTestPath { 243 must.Error(t, err, must.Sprint("expected error when writing variable")) 244 } else { 245 must.NoError(t, err, must.Sprint("expected no error writing variable")) 246 } 247 t.Cleanup(func() { 248 _, err := nomadClient.Variables().Delete("test/"+t.Name(), opts) 249 if !expectErrTestPath { 250 must.NoError(t, err, must.Sprint("expected no error cleaning up variable")) 251 } 252 }) 253 254 _, _, err = nomadClient.Variables().Create(&api.Variable{ 255 Namespace: ns, 256 Path: "other/" + t.Name(), 257 Items: map[string]string{"foo": t.Name()}, 258 }, opts) 259 260 if expectErrOutsidePath { 261 must.Error(t, err, must.Sprint("expected error when writing variable")) 262 } else { 263 must.NoError(t, err, must.Sprint("expected no error writing variable")) 264 } 265 t.Cleanup(func() { 266 // no test should ever write this variable, so we don't expect delete to 267 // work either but need it for cleanup just in case we did write it 268 nomadClient.Variables().Delete("other/"+t.Name(), opts) 269 }) 270 271 } 272 273 func setupAuthTest(t *testing.T, nomadClient *api.Client, 274 ns, validPolicyName, invalidPolicyName string) { 275 t.Helper() 276 277 _, err := nomadClient.Namespaces().Register(&api.Namespace{Name: ns}, nil) 278 must.NoError(t, err, must.Sprint("expected no error when registering namespace")) 279 280 t.Cleanup(func() { 281 _, err := nomadClient.Namespaces().Delete(ns, nil) 282 must.NoError(t, err, must.Sprint("expected no error cleaning up namespace")) 283 }) 284 285 // Create a valid and useful policy 286 _, err = nomadClient.ACLPolicies().Upsert(&api.ACLPolicy{ 287 Name: validPolicyName, 288 Rules: fmt.Sprintf(validPolicySpec, ns), 289 }, nil) 290 must.NoError(t, err, must.Sprint("expected no error when registering policy")) 291 292 t.Cleanup(func() { 293 _, err := nomadClient.ACLPolicies().Delete(validPolicyName, nil) 294 must.NoError(t, err, must.Sprint("expected no error cleaning up ACL policy")) 295 }) 296 297 // Create a useless policy 298 _, err = nomadClient.ACLPolicies().Upsert(&api.ACLPolicy{ 299 Name: invalidPolicyName, 300 Rules: `plugin { policy = "read" }`, 301 }, nil) 302 must.NoError(t, err, must.Sprint("expected no error when registering policy")) 303 304 t.Cleanup(func() { 305 _, err := nomadClient.ACLPolicies().Delete(invalidPolicyName, nil) 306 must.NoError(t, err, must.Sprint("expected no error cleaning up ACL policy")) 307 }) 308 } 309 310 // addressForNode is a hacky way of getting the address with or without 311 // mTLS. The test code can't read the api.Client's internals to see if we're in 312 // mTLS mode, so we assume if the environment is set up for mTLS that we're 313 // using it. We also need to make sure we're using the AWS public IP address for 314 // machines running in the nightly E2E environment, and that address isn't the 315 // advertised address 316 func addressForNode(node *api.Node) string { 317 if publicIP, ok := node.Attributes["unique.platform.aws.public-ipv4"]; ok { 318 if v := os.Getenv("NOMAD_CACERT"); v != "" { 319 return fmt.Sprintf("https://%s:4646", publicIP) 320 } else { 321 return fmt.Sprintf("http://%s:4646", publicIP) 322 } 323 } 324 325 if v := os.Getenv("NOMAD_CACERT"); v != "" { 326 return fmt.Sprintf("https://%s", node.HTTPAddr) 327 } 328 return fmt.Sprintf("http://%s", node.HTTPAddr) 329 }