github.com/hernad/nomad@v1.6.112/e2e/auth/auth_test.go (about)

     1  // Copyright (c) HashiCorp, Inc.
     2  // SPDX-License-Identifier: MPL-2.0
     3  
     4  package auth
     5  
     6  import (
     7  	"fmt"
     8  	"os"
     9  	"testing"
    10  	"time"
    11  
    12  	"github.com/hernad/nomad/api"
    13  	"github.com/hernad/nomad/e2e/e2eutil"
    14  	"github.com/hernad/nomad/helper/uuid"
    15  	"github.com/shoenig/test/must"
    16  )
    17  
    18  var validPolicySpec = `
    19  namespace "%s" {
    20    policy = "read"
    21    variables {
    22      path "test/*" {
    23        capabilities = [ "write", "destroy" ]
    24  	}
    25    }
    26  }
    27  
    28  node {
    29    policy = "write"
    30  }
    31  `
    32  
    33  // TestAuth verifies that we're correctly enforcing ACLs with different
    34  // combinations of tokens, policies, API types, and topologies.
    35  func TestAuth(t *testing.T) {
    36  
    37  	// Wait until we have a usable cluster before running the tests.
    38  	nomadClient := e2eutil.NomadClient(t)
    39  	e2eutil.WaitForLeader(t, nomadClient)
    40  	e2eutil.WaitForNodesReady(t, nomadClient, 1)
    41  
    42  	nodes, _, err := nomadClient.Nodes().List(nil)
    43  	must.NoError(t, err, must.Sprint("expected no error from root client"))
    44  	must.Greater(t, 0, len(nodes))
    45  	node, _, err := nomadClient.Nodes().Info(nodes[0].ID, nil)
    46  
    47  	ns := uuid.Generate()
    48  	validPolicyName := uuid.Generate()
    49  	invalidPolicyName := uuid.Generate()
    50  
    51  	setupAuthTest(t, nomadClient, ns, validPolicyName, invalidPolicyName)
    52  
    53  	// Test cases that exercise requests directly to the server
    54  	t.Run("AnonServerRequests", testAnonServerRequests(node, ns))
    55  	t.Run("BogusServerRequests", testBogusServerRequests(nomadClient, node, ns))
    56  	t.Run("InvalidPermissionsServerRequests",
    57  		testInvalidPermissionsServerRequests(nomadClient, node, ns, invalidPolicyName))
    58  	t.Run("ValidPermissionsServerRequests",
    59  		testValidPermissionsServerRequests(nomadClient, node, ns, validPolicyName))
    60  
    61  	// Test cases that exercise requests forwarded from the client
    62  	t.Run("AnonClientRequests", testAnonClientRequests(node, ns))
    63  	t.Run("BogusClientRequests", testBogusClientRequests(nomadClient, node, ns))
    64  	t.Run("InvalidPermissionsClientRequests",
    65  		testInvalidPermissionsClientRequests(nomadClient, node, ns, invalidPolicyName))
    66  	t.Run("ValidPermissionsClientRequests",
    67  		testValidPermissionsClientRequests(nomadClient, node, ns, validPolicyName))
    68  }
    69  
    70  func testAnonServerRequests(node *api.Node, ns string) func(t *testing.T) {
    71  	return func(t *testing.T) {
    72  		nomadClient := e2eutil.NomadClient(t)
    73  		nomadClient.SetSecretID("")
    74  
    75  		testReadNamespaceAPI(t, nomadClient, ns, "", true)
    76  		testNodeAPI(t, nomadClient, node.ID, "", true)
    77  		testVariablesAPI(t, nomadClient, ns, "", true, true)
    78  	}
    79  }
    80  
    81  func testBogusServerRequests(nomadClient *api.Client,
    82  	node *api.Node, ns string) func(t *testing.T) {
    83  	return func(t *testing.T) {
    84  		authToken := uuid.Generate()
    85  
    86  		testReadNamespaceAPI(t, nomadClient, ns, authToken, true)
    87  		testNodeAPI(t, nomadClient, node.ID, authToken, true)
    88  		testVariablesAPI(t, nomadClient, ns, authToken, true, true)
    89  	}
    90  }
    91  
    92  func testInvalidPermissionsServerRequests(nomadClient *api.Client,
    93  	node *api.Node, ns, policyName string) func(t *testing.T) {
    94  	return func(t *testing.T) {
    95  		token, _, err := nomadClient.ACLTokens().Create(&api.ACLToken{
    96  			Name:          policyName,
    97  			Type:          "client",
    98  			Policies:      []string{policyName},
    99  			ExpirationTTL: time.Minute,
   100  		}, nil)
   101  		must.NoError(t, err)
   102  		authToken := token.SecretID
   103  
   104  		testReadNamespaceAPI(t, nomadClient, ns, authToken, true)
   105  		testNodeAPI(t, nomadClient, node.ID, authToken, true)
   106  		testVariablesAPI(t, nomadClient, ns, authToken, true, true)
   107  	}
   108  }
   109  
   110  func testValidPermissionsServerRequests(nomadClient *api.Client,
   111  	node *api.Node, ns, policyName string) func(t *testing.T) {
   112  	return func(t *testing.T) {
   113  		token, _, err := nomadClient.ACLTokens().Create(&api.ACLToken{
   114  			Name:          policyName,
   115  			Type:          "client",
   116  			Policies:      []string{policyName},
   117  			ExpirationTTL: time.Minute,
   118  		}, nil)
   119  		must.NoError(t, err)
   120  		authToken := token.SecretID
   121  
   122  		testReadNamespaceAPI(t, nomadClient, ns, authToken, false)
   123  		testNodeAPI(t, nomadClient, node.ID, authToken, false)
   124  		testVariablesAPI(t, nomadClient, ns, authToken, false, true)
   125  	}
   126  }
   127  
   128  func testAnonClientRequests(node *api.Node, ns string) func(t *testing.T) {
   129  	return func(t *testing.T) {
   130  		config := api.DefaultConfig()
   131  		config.Address = addressForNode(node)
   132  		nomadClient, err := api.NewClient(config)
   133  		nomadClient.SetSecretID("")
   134  		must.NoError(t, err)
   135  
   136  		testReadNamespaceAPI(t, nomadClient, ns, "", true)
   137  		testNodeAPI(t, nomadClient, node.ID, "", true)
   138  		testVariablesAPI(t, nomadClient, ns, "", true, true)
   139  	}
   140  }
   141  
   142  func testBogusClientRequests(rootClient *api.Client,
   143  	node *api.Node, ns string) func(t *testing.T) {
   144  	return func(t *testing.T) {
   145  		config := api.DefaultConfig()
   146  		config.Address = addressForNode(node)
   147  		nomadClient, err := api.NewClient(config)
   148  		must.NoError(t, err)
   149  
   150  		authToken := uuid.Generate()
   151  
   152  		testReadNamespaceAPI(t, nomadClient, ns, authToken, true)
   153  		testNodeAPI(t, nomadClient, node.ID, authToken, true)
   154  		testVariablesAPI(t, nomadClient, ns, authToken, true, true)
   155  	}
   156  }
   157  
   158  func testInvalidPermissionsClientRequests(rootClient *api.Client,
   159  	node *api.Node, ns, policyName string) func(t *testing.T) {
   160  	return func(t *testing.T) {
   161  		token, _, err := rootClient.ACLTokens().Create(&api.ACLToken{
   162  			Name:          policyName,
   163  			Type:          "client",
   164  			Policies:      []string{policyName},
   165  			ExpirationTTL: time.Minute,
   166  		}, nil)
   167  		must.NoError(t, err)
   168  
   169  		config := api.DefaultConfig()
   170  		config.Address = addressForNode(node)
   171  		nomadClient, err := api.NewClient(config)
   172  		must.NoError(t, err)
   173  
   174  		authToken := token.SecretID
   175  
   176  		testReadNamespaceAPI(t, nomadClient, ns, authToken, true)
   177  		testNodeAPI(t, nomadClient, node.ID, authToken, true)
   178  		testVariablesAPI(t, nomadClient, ns, authToken, true, true)
   179  	}
   180  }
   181  
   182  func testValidPermissionsClientRequests(rootClient *api.Client,
   183  	node *api.Node, ns, policyName string) func(t *testing.T) {
   184  	return func(t *testing.T) {
   185  		token, _, err := rootClient.ACLTokens().Create(&api.ACLToken{
   186  			Name:          policyName,
   187  			Type:          "client",
   188  			Policies:      []string{policyName},
   189  			ExpirationTTL: time.Minute,
   190  		}, nil)
   191  		must.NoError(t, err)
   192  
   193  		config := api.DefaultConfig()
   194  		config.Address = addressForNode(node)
   195  		nomadClient, err := api.NewClient(config)
   196  		must.NoError(t, err)
   197  
   198  		authToken := token.SecretID
   199  
   200  		testReadNamespaceAPI(t, nomadClient, ns, authToken, false)
   201  		testNodeAPI(t, nomadClient, node.ID, authToken, false)
   202  		testVariablesAPI(t, nomadClient, ns, authToken, false, true)
   203  	}
   204  }
   205  
   206  // testReadNamespaceAPI exercises an API that requires any namespace capability
   207  func testReadNamespaceAPI(t *testing.T, nomadClient *api.Client, ns, authToken string, expectErr bool) {
   208  	t.Helper()
   209  	opts := &api.QueryOptions{AuthToken: authToken}
   210  	_, _, err := nomadClient.Namespaces().Info(ns, opts)
   211  	if expectErr {
   212  		must.Error(t, err, must.Sprint("expected error when reading namespace"))
   213  	} else {
   214  		must.NoError(t, err, must.Sprint("expected no error reading namespace"))
   215  	}
   216  }
   217  
   218  // testNodeAPI exercises an API that requires the node:write permission
   219  func testNodeAPI(t *testing.T, nomadClient *api.Client, nodeID, authToken string, expectErr bool) {
   220  	t.Helper()
   221  	opts := &api.WriteOptions{AuthToken: authToken}
   222  	_, _, err := nomadClient.Nodes().ForceEvaluate(nodeID, opts)
   223  	if expectErr {
   224  		must.Error(t, err, must.Sprint("expected error when force-evaluating node"))
   225  	} else {
   226  		must.NoError(t, err, must.Sprint("expected no error force-evaluating node"))
   227  	}
   228  }
   229  
   230  // testVariablesAPI exercises an API that requires namespace capabilities for
   231  // variables
   232  func testVariablesAPI(t *testing.T, nomadClient *api.Client, ns, authToken string, expectErrTestPath, expectErrOutsidePath bool) {
   233  	t.Helper()
   234  	opts := &api.WriteOptions{Namespace: ns, AuthToken: authToken}
   235  
   236  	_, _, err := nomadClient.Variables().Create(&api.Variable{
   237  		Namespace: ns,
   238  		Path:      "test/" + t.Name(),
   239  		Items:     map[string]string{"foo": t.Name()},
   240  	}, opts)
   241  
   242  	if expectErrTestPath {
   243  		must.Error(t, err, must.Sprint("expected error when writing variable"))
   244  	} else {
   245  		must.NoError(t, err, must.Sprint("expected no error writing variable"))
   246  	}
   247  	t.Cleanup(func() {
   248  		_, err := nomadClient.Variables().Delete("test/"+t.Name(), opts)
   249  		if !expectErrTestPath {
   250  			must.NoError(t, err, must.Sprint("expected no error cleaning up variable"))
   251  		}
   252  	})
   253  
   254  	_, _, err = nomadClient.Variables().Create(&api.Variable{
   255  		Namespace: ns,
   256  		Path:      "other/" + t.Name(),
   257  		Items:     map[string]string{"foo": t.Name()},
   258  	}, opts)
   259  
   260  	if expectErrOutsidePath {
   261  		must.Error(t, err, must.Sprint("expected error when writing variable"))
   262  	} else {
   263  		must.NoError(t, err, must.Sprint("expected no error writing variable"))
   264  	}
   265  	t.Cleanup(func() {
   266  		// no test should ever write this variable, so we don't expect delete to
   267  		// work either but need it for cleanup just in case we did write it
   268  		nomadClient.Variables().Delete("other/"+t.Name(), opts)
   269  	})
   270  
   271  }
   272  
   273  func setupAuthTest(t *testing.T, nomadClient *api.Client,
   274  	ns, validPolicyName, invalidPolicyName string) {
   275  	t.Helper()
   276  
   277  	_, err := nomadClient.Namespaces().Register(&api.Namespace{Name: ns}, nil)
   278  	must.NoError(t, err, must.Sprint("expected no error when registering namespace"))
   279  
   280  	t.Cleanup(func() {
   281  		_, err := nomadClient.Namespaces().Delete(ns, nil)
   282  		must.NoError(t, err, must.Sprint("expected no error cleaning up namespace"))
   283  	})
   284  
   285  	// Create a valid and useful policy
   286  	_, err = nomadClient.ACLPolicies().Upsert(&api.ACLPolicy{
   287  		Name:  validPolicyName,
   288  		Rules: fmt.Sprintf(validPolicySpec, ns),
   289  	}, nil)
   290  	must.NoError(t, err, must.Sprint("expected no error when registering policy"))
   291  
   292  	t.Cleanup(func() {
   293  		_, err := nomadClient.ACLPolicies().Delete(validPolicyName, nil)
   294  		must.NoError(t, err, must.Sprint("expected no error cleaning up ACL policy"))
   295  	})
   296  
   297  	// Create a useless policy
   298  	_, err = nomadClient.ACLPolicies().Upsert(&api.ACLPolicy{
   299  		Name:  invalidPolicyName,
   300  		Rules: `plugin { policy = "read" }`,
   301  	}, nil)
   302  	must.NoError(t, err, must.Sprint("expected no error when registering policy"))
   303  
   304  	t.Cleanup(func() {
   305  		_, err := nomadClient.ACLPolicies().Delete(invalidPolicyName, nil)
   306  		must.NoError(t, err, must.Sprint("expected no error cleaning up ACL policy"))
   307  	})
   308  }
   309  
   310  // addressForNode is a hacky way of getting the address with or without
   311  // mTLS. The test code can't read the api.Client's internals to see if we're in
   312  // mTLS mode, so we assume if the environment is set up for mTLS that we're
   313  // using it. We also need to make sure we're using the AWS public IP address for
   314  // machines running in the nightly E2E environment, and that address isn't the
   315  // advertised address
   316  func addressForNode(node *api.Node) string {
   317  	if publicIP, ok := node.Attributes["unique.platform.aws.public-ipv4"]; ok {
   318  		if v := os.Getenv("NOMAD_CACERT"); v != "" {
   319  			return fmt.Sprintf("https://%s:4646", publicIP)
   320  		} else {
   321  			return fmt.Sprintf("http://%s:4646", publicIP)
   322  		}
   323  	}
   324  
   325  	if v := os.Getenv("NOMAD_CACERT"); v != "" {
   326  		return fmt.Sprintf("https://%s", node.HTTPAddr)
   327  	}
   328  	return fmt.Sprintf("http://%s", node.HTTPAddr)
   329  }