github.com/anth0d/nomad@v0.0.0-20221214183521-ae3a0a2cad06/e2e/connect/acls.go (about)

     1  package connect
     2  
     3  import (
     4  	"os"
     5  	"regexp"
     6  	"testing"
     7  	"time"
     8  
     9  	consulapi "github.com/hashicorp/consul/api"
    10  	uuidparse "github.com/hashicorp/go-uuid"
    11  	nomadapi "github.com/hashicorp/nomad/api"
    12  	"github.com/hashicorp/nomad/e2e/e2eutil"
    13  	"github.com/hashicorp/nomad/e2e/framework"
    14  	"github.com/hashicorp/nomad/helper/uuid"
    15  	"github.com/hashicorp/nomad/jobspec"
    16  	"github.com/stretchr/testify/require"
    17  )
    18  
    19  type ConnectACLsE2ETest struct {
    20  	framework.TC
    21  
    22  	// used to store the root token so we can reset the client back to
    23  	// it as needed
    24  	consulManagementToken string
    25  
    26  	// things to cleanup after each test case
    27  	jobIDs          []string
    28  	consulPolicyIDs []string
    29  	consulTokenIDs  []string
    30  }
    31  
    32  func (tc *ConnectACLsE2ETest) BeforeAll(f *framework.F) {
    33  	// Wait for Nomad to be ready before doing anything.
    34  	e2eutil.WaitForLeader(f.T(), tc.Nomad())
    35  	e2eutil.WaitForNodesReady(f.T(), tc.Nomad(), 2)
    36  
    37  	// Validate the consul root token exists, otherwise tests are just
    38  	// going to be a train wreck.
    39  	tc.consulManagementToken = os.Getenv(envConsulToken)
    40  
    41  	_, err := uuidparse.ParseUUID(tc.consulManagementToken)
    42  	f.NoError(err, "CONSUL_HTTP_TOKEN not set")
    43  
    44  	// ensure SI tokens from previous test cases were removed
    45  	f.Eventually(func() bool {
    46  		siTokens := tc.countSITokens(f.T())
    47  		f.T().Log("cleanup: checking for remaining SI tokens:", siTokens)
    48  		return len(siTokens) == 0
    49  	}, 2*time.Minute, 2*time.Second, "SI tokens did not get removed")
    50  }
    51  
    52  // AfterEach does cleanup of Consul ACL objects that were created during each
    53  // test case. Each test case may assume it is starting from a "fresh" state -
    54  // as if the consul ACL bootstrap process had just taken place.
    55  func (tc *ConnectACLsE2ETest) AfterEach(f *framework.F) {
    56  	if os.Getenv("NOMAD_TEST_SKIPCLEANUP") == "1" {
    57  		return
    58  	}
    59  
    60  	t := f.T()
    61  
    62  	// cleanup jobs
    63  	for _, id := range tc.jobIDs {
    64  		t.Log("cleanup: deregister nomad job id:", id)
    65  		_, _, err := tc.Nomad().Jobs().Deregister(id, true, nil)
    66  		f.NoError(err)
    67  	}
    68  
    69  	// cleanup consul tokens
    70  	for _, id := range tc.consulTokenIDs {
    71  		t.Log("cleanup: delete consul token id:", id)
    72  		_, err := tc.Consul().ACL().TokenDelete(id, &consulapi.WriteOptions{Token: tc.consulManagementToken})
    73  		f.NoError(err)
    74  	}
    75  
    76  	// cleanup consul policies
    77  	for _, id := range tc.consulPolicyIDs {
    78  		t.Log("cleanup: delete consul policy id:", id)
    79  		_, err := tc.Consul().ACL().PolicyDelete(id, &consulapi.WriteOptions{Token: tc.consulManagementToken})
    80  		f.NoError(err)
    81  	}
    82  
    83  	// do garbage collection
    84  	err := tc.Nomad().System().GarbageCollect()
    85  	f.NoError(err)
    86  
    87  	// assert there are no leftover SI tokens, which may take a minute to be
    88  	// cleaned up
    89  	f.Eventually(func() bool {
    90  		siTokens := tc.countSITokens(t)
    91  		t.Log("cleanup: checking for remaining SI tokens:", siTokens)
    92  		return len(siTokens) == 0
    93  	}, 2*time.Minute, 2*time.Second, "SI tokens did not get removed")
    94  
    95  	tc.jobIDs = []string{}
    96  	tc.consulTokenIDs = []string{}
    97  	tc.consulPolicyIDs = []string{}
    98  }
    99  
   100  // todo(shoenig): follow up refactor with e2eutil.ConsulPolicy
   101  type consulPolicy struct {
   102  	Name  string // e.g. nomad-operator
   103  	Rules string // e.g. service "" { policy="write" }
   104  }
   105  
   106  // todo(shoenig): follow up refactor with e2eutil.ConsulPolicy
   107  func (tc *ConnectACLsE2ETest) createConsulPolicy(p consulPolicy, f *framework.F) string {
   108  	result, _, err := tc.Consul().ACL().PolicyCreate(&consulapi.ACLPolicy{
   109  		Name:        p.Name,
   110  		Description: "test policy " + p.Name,
   111  		Rules:       p.Rules,
   112  	}, &consulapi.WriteOptions{Token: tc.consulManagementToken})
   113  	f.NoError(err, "failed to create consul policy")
   114  	tc.consulPolicyIDs = append(tc.consulPolicyIDs, result.ID)
   115  	return result.ID
   116  }
   117  
   118  // todo(shoenig): follow up refactor with e2eutil.ConsulPolicy
   119  func (tc *ConnectACLsE2ETest) createOperatorToken(policyID string, f *framework.F) string {
   120  	token, _, err := tc.Consul().ACL().TokenCreate(&consulapi.ACLToken{
   121  		Description: "operator token",
   122  		Policies:    []*consulapi.ACLTokenPolicyLink{{ID: policyID}},
   123  	}, &consulapi.WriteOptions{Token: tc.consulManagementToken})
   124  	f.NoError(err, "failed to create operator token")
   125  	tc.consulTokenIDs = append(tc.consulTokenIDs, token.AccessorID)
   126  	return token.SecretID
   127  }
   128  
   129  func (tc *ConnectACLsE2ETest) TestConnectACLsRegisterMasterToken(f *framework.F) {
   130  	t := f.T()
   131  
   132  	t.Log("test register Connect job w/ ACLs enabled w/ master token")
   133  
   134  	jobID := "connect" + uuid.Generate()[0:8]
   135  	tc.jobIDs = append(tc.jobIDs, jobID)
   136  
   137  	jobAPI := tc.Nomad().Jobs()
   138  
   139  	job, err := jobspec.ParseFile(demoConnectJob)
   140  	f.NoError(err)
   141  
   142  	// Set the job file to use the consul master token.
   143  	// One should never do this in practice, but, it should work.
   144  	// https://www.consul.io/docs/acl/acl-system.html#builtin-tokens
   145  	job.ConsulToken = &tc.consulManagementToken
   146  	job.ID = &jobID
   147  
   148  	// Avoid using Register here, because that would actually create and run the
   149  	// Job which runs the task, creates the SI token, which all needs to be
   150  	// given time to settle and cleaned up. That is all covered in the big slow
   151  	// test at the bottom.
   152  	resp, _, err := jobAPI.Plan(job, false, nil)
   153  	f.NoError(err)
   154  	f.NotNil(resp)
   155  }
   156  
   157  func (tc *ConnectACLsE2ETest) TestConnectACLsRegisterMissingOperatorToken(f *framework.F) {
   158  	t := f.T()
   159  
   160  	t.Skip("we don't have consul.allow_unauthenticated=false set because it would required updating every E2E test to pass a Consul token")
   161  
   162  	t.Log("test register Connect job w/ ACLs enabled w/o operator token")
   163  
   164  	jobID := "connect" + uuid.Short()
   165  	tc.jobIDs = append(tc.jobIDs, jobID) // need to clean up if the test fails
   166  
   167  	job, err := jobspec.ParseFile(demoConnectJob)
   168  	f.NoError(err)
   169  	jobAPI := tc.Nomad().Jobs()
   170  
   171  	// Explicitly show the ConsulToken is not set
   172  	job.ConsulToken = nil
   173  	job.ID = &jobID
   174  
   175  	_, _, err = jobAPI.Register(job, nil)
   176  	f.Error(err)
   177  
   178  	t.Log("job correctly rejected, with error:", err)
   179  }
   180  
   181  func (tc *ConnectACLsE2ETest) TestConnectACLsRegisterFakeOperatorToken(f *framework.F) {
   182  	t := f.T()
   183  
   184  	t.Skip("we don't have consul.allow_unauthenticated=false set because it would required updating every E2E test to pass a Consul token")
   185  
   186  	t.Log("test register Connect job w/ ACLs enabled w/ operator token")
   187  
   188  	policyID := tc.createConsulPolicy(consulPolicy{
   189  		Name:  "nomad-operator-policy-" + uuid.Short(),
   190  		Rules: `service "count-api" { policy = "write" } service "count-dashboard" { policy = "write" }`,
   191  	}, f)
   192  	t.Log("created operator policy:", policyID)
   193  
   194  	// generate a fake consul token token
   195  	fakeToken := uuid.Generate()
   196  
   197  	jobID := "connect" + uuid.Short()
   198  	tc.jobIDs = append(tc.jobIDs, jobID) // need to clean up if the test fails
   199  
   200  	job := tc.parseJobSpecFile(t, demoConnectJob)
   201  
   202  	jobAPI := tc.Nomad().Jobs()
   203  
   204  	// deliberately set the fake Consul token
   205  	job.ConsulToken = &fakeToken
   206  	job.ID = &jobID
   207  
   208  	// should fail, because the token is fake
   209  	_, _, err := jobAPI.Register(job, nil)
   210  	f.Error(err)
   211  	t.Log("job correctly rejected, with error:", err)
   212  }
   213  
   214  func (tc *ConnectACLsE2ETest) TestConnectACLsConnectDemo(f *framework.F) {
   215  	t := f.T()
   216  
   217  	t.Log("test register Connect job w/ ACLs enabled w/ operator token")
   218  
   219  	// === Setup ACL policy and mint Operator token ===
   220  
   221  	// create a policy allowing writes of services "count-api" and "count-dashboard"
   222  	policyID := tc.createConsulPolicy(consulPolicy{
   223  		Name:  "nomad-operator-policy-" + uuid.Short(),
   224  		Rules: `service "count-api" { policy = "write" } service "count-dashboard" { policy = "write" }`,
   225  	}, f)
   226  	t.Log("created operator policy:", policyID)
   227  
   228  	// create a Consul "operator token" blessed with the above policy
   229  	operatorToken := tc.createOperatorToken(policyID, f)
   230  	t.Log("created operator token:", operatorToken)
   231  
   232  	jobID := connectJobID()
   233  	tc.jobIDs = append(tc.jobIDs, jobID)
   234  
   235  	allocs := e2eutil.RegisterAndWaitForAllocs(t, tc.Nomad(), demoConnectJob, jobID, operatorToken)
   236  	f.Equal(2, len(allocs), "expected 2 allocs for connect demo", allocs)
   237  	allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocs)
   238  	f.Equal(2, len(allocIDs), "expected 2 allocIDs for connect demo", allocIDs)
   239  	e2eutil.WaitForAllocsRunning(t, tc.Nomad(), allocIDs)
   240  
   241  	// === Check Consul SI tokens were generated for sidecars ===
   242  	foundSITokens := tc.countSITokens(t)
   243  	f.Equal(2, len(foundSITokens), "expected 2 SI tokens total: %v", foundSITokens)
   244  	f.Equal(1, foundSITokens["connect-proxy-count-api"], "expected 1 SI token for connect-proxy-count-api: %v", foundSITokens)
   245  	f.Equal(1, foundSITokens["connect-proxy-count-dashboard"], "expected 1 SI token for connect-proxy-count-dashboard: %v", foundSITokens)
   246  
   247  	t.Log("connect legacy job with ACLs enable finished")
   248  }
   249  
   250  func (tc *ConnectACLsE2ETest) TestConnectACLsConnectNativeDemo(f *framework.F) {
   251  	t := f.T()
   252  
   253  	t.Log("test register Connect job w/ ACLs enabled w/ operator token")
   254  
   255  	// === Setup ACL policy and mint Operator token ===
   256  
   257  	// create a policy allowing writes of services "uuid-fe" and "uuid-api"
   258  	policyID := tc.createConsulPolicy(consulPolicy{
   259  		Name:  "nomad-operator-policy-" + uuid.Short(),
   260  		Rules: `service "uuid-fe" { policy = "write" } service "uuid-api" { policy = "write" }`,
   261  	}, f)
   262  	t.Log("created operator policy:", policyID)
   263  
   264  	// create a Consul "operator token" blessed with the above policy
   265  	operatorToken := tc.createOperatorToken(policyID, f)
   266  	t.Log("created operator token:", operatorToken)
   267  
   268  	jobID := connectJobID()
   269  	tc.jobIDs = append(tc.jobIDs, jobID)
   270  
   271  	allocs := e2eutil.RegisterAndWaitForAllocs(t, tc.Nomad(), demoConnectNativeJob, jobID, operatorToken)
   272  	allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocs)
   273  	e2eutil.WaitForAllocsRunning(t, tc.Nomad(), allocIDs)
   274  
   275  	// === Check Consul SI tokens were generated for native tasks ===
   276  	foundSITokens := tc.countSITokens(t)
   277  	f.Equal(2, len(foundSITokens), "expected 2 SI tokens total: %v", foundSITokens)
   278  	f.Equal(1, foundSITokens["frontend"], "expected 1 SI token for frontend: %v", foundSITokens)
   279  	f.Equal(1, foundSITokens["generate"], "expected 1 SI token for generate: %v", foundSITokens)
   280  
   281  	t.Log("connect native job with ACLs enabled finished")
   282  }
   283  
   284  func (tc *ConnectACLsE2ETest) TestConnectACLsConnectIngressGatewayDemo(f *framework.F) {
   285  	t := f.T()
   286  
   287  	t.Log("test register Connect Ingress Gateway job w/ ACLs enabled")
   288  
   289  	// setup ACL policy and mint operator token
   290  
   291  	policyID := tc.createConsulPolicy(consulPolicy{
   292  		Name:  "nomad-operator-policy-" + uuid.Short(),
   293  		Rules: `service "my-ingress-service" { policy = "write" } service "uuid-api" { policy = "write" }`,
   294  	}, f)
   295  	operatorToken := tc.createOperatorToken(policyID, f)
   296  	t.Log("created operator token:", operatorToken)
   297  
   298  	jobID := connectJobID()
   299  	tc.jobIDs = append(tc.jobIDs, jobID)
   300  
   301  	allocs := e2eutil.RegisterAndWaitForAllocs(t, tc.Nomad(), demoConnectIngressGateway, jobID, operatorToken)
   302  	allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocs)
   303  	e2eutil.WaitForAllocsRunning(t, tc.Nomad(), allocIDs)
   304  
   305  	foundSITokens := tc.countSITokens(t)
   306  	f.Equal(2, len(foundSITokens), "expected 2 SI tokens total: %v", foundSITokens)
   307  	f.Equal(1, foundSITokens["connect-ingress-my-ingress-service"], "expected 1 SI token for connect-ingress-my-ingress-service: %v", foundSITokens)
   308  	f.Equal(1, foundSITokens["generate"], "expected 1 SI token for generate: %v", foundSITokens)
   309  
   310  	t.Log("connect ingress gateway job with ACLs enabled finished")
   311  }
   312  
   313  func (tc *ConnectACLsE2ETest) TestConnectACLsConnectTerminatingGatewayDemo(f *framework.F) {
   314  	t := f.T()
   315  
   316  	t.Log("test register Connect Terminating Gateway job w/ ACLs enabled")
   317  
   318  	// setup ACL policy and mint operator token
   319  
   320  	policyID := tc.createConsulPolicy(consulPolicy{
   321  		Name:  "nomad-operator-policy-" + uuid.Short(),
   322  		Rules: `service "api-gateway" { policy = "write" } service "count-dashboard" { policy = "write" }`,
   323  	}, f)
   324  	operatorToken := tc.createOperatorToken(policyID, f)
   325  	t.Log("created operator token:", operatorToken)
   326  
   327  	jobID := connectJobID()
   328  	tc.jobIDs = append(tc.jobIDs, jobID)
   329  
   330  	allocs := e2eutil.RegisterAndWaitForAllocs(t, tc.Nomad(), demoConnectTerminatingGateway, jobID, operatorToken)
   331  	allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocs)
   332  	e2eutil.WaitForAllocsRunning(t, tc.Nomad(), allocIDs)
   333  
   334  	foundSITokens := tc.countSITokens(t)
   335  	f.Equal(2, len(foundSITokens), "expected 2 SI tokens total: %v", foundSITokens)
   336  	f.Equal(1, foundSITokens["connect-terminating-api-gateway"], "expected 1 SI token for connect-terminating-api-gateway: %v", foundSITokens)
   337  	f.Equal(1, foundSITokens["connect-proxy-count-dashboard"], "expected 1 SI token for count-dashboard: %v", foundSITokens)
   338  
   339  	t.Log("connect terminating gateway job with ACLs enabled finished")
   340  }
   341  
   342  var (
   343  	siTokenRe = regexp.MustCompile(`_nomad_si \[[\w-]{36}] \[[\w-]{36}] \[([\S]+)]`)
   344  )
   345  
   346  func (tc *ConnectACLsE2ETest) serviceofSIToken(description string) string {
   347  	if m := siTokenRe.FindStringSubmatch(description); len(m) == 2 {
   348  		return m[1]
   349  	}
   350  	return ""
   351  }
   352  
   353  func (tc *ConnectACLsE2ETest) countSITokens(t *testing.T) map[string]int {
   354  	aclAPI := tc.Consul().ACL()
   355  	tokens, _, err := aclAPI.TokenList(&consulapi.QueryOptions{
   356  		Token: tc.consulManagementToken,
   357  	})
   358  	require.NoError(t, err)
   359  
   360  	// count the number of SI tokens matching each service name
   361  	foundSITokens := make(map[string]int)
   362  	for _, token := range tokens {
   363  		if service := tc.serviceofSIToken(token.Description); service != "" {
   364  			foundSITokens[service]++
   365  		}
   366  	}
   367  
   368  	return foundSITokens
   369  }
   370  
   371  func (tc *ConnectACLsE2ETest) parseJobSpecFile(t *testing.T, filename string) *nomadapi.Job {
   372  	job, err := jobspec.ParseFile(filename)
   373  	require.NoError(t, err)
   374  	return job
   375  }