github.com/hernad/nomad@v1.6.112/e2e/vaultsecrets/vaultsecrets.go (about)

     1  // Copyright (c) HashiCorp, Inc.
     2  // SPDX-License-Identifier: MPL-2.0
     3  
     4  package vaultsecrets
     5  
     6  import (
     7  	"context"
     8  	"fmt"
     9  	"io"
    10  	"os"
    11  	"os/exec"
    12  	"regexp"
    13  	"strings"
    14  	"time"
    15  
    16  	e2e "github.com/hernad/nomad/e2e/e2eutil"
    17  	"github.com/hernad/nomad/e2e/framework"
    18  	"github.com/hernad/nomad/helper/uuid"
    19  	"github.com/hernad/nomad/testutil"
    20  )
    21  
    22  const ns = ""
    23  
    24  type VaultSecretsTest struct {
    25  	framework.TC
    26  	secretsPath string
    27  	pkiPath     string
    28  	jobIDs      []string
    29  	policies    []string
    30  }
    31  
    32  func init() {
    33  	framework.AddSuites(&framework.TestSuite{
    34  		Component:   "VaultSecrets",
    35  		CanRunLocal: true,
    36  		Consul:      true,
    37  		Vault:       true,
    38  		Cases: []framework.TestCase{
    39  			new(VaultSecretsTest),
    40  		},
    41  	})
    42  }
    43  
    44  func (tc *VaultSecretsTest) BeforeAll(f *framework.F) {
    45  	e2e.WaitForLeader(f.T(), tc.Nomad())
    46  	e2e.WaitForNodesReady(f.T(), tc.Nomad(), 1)
    47  }
    48  
    49  func (tc *VaultSecretsTest) AfterEach(f *framework.F) {
    50  	if os.Getenv("NOMAD_TEST_SKIPCLEANUP") == "1" {
    51  		return
    52  	}
    53  
    54  	for _, id := range tc.jobIDs {
    55  		_, err := e2e.Command("nomad", "job", "stop", "-purge", id)
    56  		f.Assert().NoError(err, "could not clean up job", id)
    57  	}
    58  	tc.jobIDs = []string{}
    59  
    60  	for _, policy := range tc.policies {
    61  		_, err := e2e.Command("vault", "policy", "delete", policy)
    62  		f.Assert().NoError(err, "could not clean up vault policy", policy)
    63  	}
    64  	tc.policies = []string{}
    65  
    66  	// disabling the secrets engines will wipe all the secrets as well
    67  	_, err := e2e.Command("vault", "secrets", "disable", tc.secretsPath)
    68  	f.Assert().NoError(err)
    69  	_, err = e2e.Command("vault", "secrets", "disable", tc.pkiPath)
    70  	f.Assert().NoError(err)
    71  
    72  	_, err = e2e.Command("nomad", "system", "gc")
    73  	f.NoError(err)
    74  }
    75  
    76  func (tc *VaultSecretsTest) TestVaultSecrets(f *framework.F) {
    77  
    78  	// use a random suffix to encapsulate test keys, polices, etc.
    79  	// for cleanup from vault
    80  	testID := uuid.Generate()[0:8]
    81  	jobID := "test-vault-secrets-" + testID
    82  	tc.secretsPath = "secrets-" + testID
    83  	tc.pkiPath = "pki-" + testID
    84  	secretValue := uuid.Generate()
    85  	secretKey := tc.secretsPath + "/data/myapp"
    86  	pkiCertIssue := tc.pkiPath + "/issue/nomad"
    87  	policyID := "access-secrets-" + testID
    88  	index := 0
    89  	wc := &e2e.WaitConfig{Retries: 500}
    90  	interval, retries := wc.OrDefault()
    91  
    92  	setupCmds := []string{
    93  
    94  		// configure KV secrets engine
    95  		// Note: the secret key is written to 'secret-###/myapp' but the kv2 API
    96  		// for Vault implicitly turns that into 'secret-###/data/myapp' so we
    97  		// need to use the longer path for everything other than kv put/get
    98  		fmt.Sprintf("vault secrets enable -path=%s kv-v2", tc.secretsPath),
    99  		fmt.Sprintf("vault kv put %s/myapp key=%s", tc.secretsPath, secretValue),
   100  		fmt.Sprintf("vault secrets tune -max-lease-ttl=1m %s", tc.secretsPath),
   101  
   102  		// configure PKI secrets engine
   103  		fmt.Sprintf("vault secrets enable -path=%s pki", tc.pkiPath),
   104  		fmt.Sprintf("vault write %s/root/generate/internal "+
   105  			"common_name=service.consul ttl=1h", tc.pkiPath),
   106  		fmt.Sprintf("vault write %s/roles/nomad "+
   107  			"allowed_domains=service.consul "+
   108  			"allow_subdomains=true "+
   109  			"generate_lease=true "+
   110  			"max_ttl=1m", tc.pkiPath),
   111  		fmt.Sprintf("vault secrets tune -max-lease-ttl=1m %s", tc.pkiPath),
   112  	}
   113  
   114  	for _, setupCmd := range setupCmds {
   115  		cmd := strings.Split(setupCmd, " ")
   116  		out, err := e2e.Command(cmd[0], cmd[1:]...)
   117  		f.NoError(err, fmt.Sprintf("error for %q:\n%s", setupCmd, out))
   118  	}
   119  
   120  	// we can't set an empty policy in our job, so write a bogus policy that
   121  	// doesn't have access to any of the paths we're using
   122  	out, err := writePolicy(policyID, "./vaultsecrets/input/policy-bad.hcl", testID)
   123  	f.NoError(err, out)
   124  	tc.policies = append(tc.policies, policyID)
   125  
   126  	index++
   127  	err = runJob(jobID, testID, index)
   128  	f.NoError(err, "could not register job")
   129  	tc.jobIDs = append(tc.jobIDs, jobID)
   130  
   131  	// job doesn't have access to secrets, so they can't start
   132  	err = e2e.WaitForAllocStatusExpected(jobID, ns, []string{"pending"})
   133  	f.NoError(err, "expected pending allocation")
   134  
   135  	// we should get a task event about why they can't start
   136  	expect := fmt.Sprintf("Missing: vault.read(%s), vault.write(%s", secretKey, pkiCertIssue)
   137  
   138  	allocID, err := latestAllocID(jobID)
   139  	f.NoError(err)
   140  
   141  	testutil.WaitForResultRetries(retries, func() (bool, error) {
   142  		time.Sleep(interval)
   143  		out, err := e2e.Command("nomad", "alloc", "status", allocID)
   144  		f.NoError(err, "could not get allocation status")
   145  		return strings.Contains(out, expect),
   146  			fmt.Errorf("expected '%s', got\n%v", expect, out)
   147  	}, func(e error) {
   148  		f.NoError(e)
   149  	})
   150  
   151  	// write a working policy and redeploy
   152  	out, err = writePolicy(policyID, "./vaultsecrets/input/policy-good.hcl", testID)
   153  	f.NoError(err, out)
   154  	index++
   155  	err = runJob(jobID, testID, index)
   156  	f.NoError(err, "could not register job")
   157  
   158  	// record the rough start of vault token TTL window, so that we don't have
   159  	// to wait excessively later on
   160  	ttlStart := time.Now()
   161  
   162  	// job should be now unblocked
   163  	err = e2e.WaitForAllocStatusExpected(jobID, ns, []string{"running", "complete"})
   164  	f.NoError(err, "expected running allocation")
   165  
   166  	allocID, err = latestAllocID(jobID)
   167  	f.NoError(err)
   168  
   169  	renderedCert, err := waitForAllocSecret(allocID, "task", "/secrets/certificate.crt",
   170  		func(out string) bool {
   171  			return strings.Contains(out, "BEGIN CERTIFICATE")
   172  		}, wc)
   173  	f.NoError(err)
   174  
   175  	_, err = waitForAllocSecret(allocID, "task", "/secrets/access.key",
   176  		func(out string) bool {
   177  			return strings.Contains(out, secretValue)
   178  		}, wc)
   179  	f.NoError(err)
   180  
   181  	var re = regexp.MustCompile(`VAULT_TOKEN=(.*)`)
   182  
   183  	// check vault token was written and save it for later comparison
   184  	out, err = e2e.AllocExec(allocID, "task", "env", ns, nil)
   185  	f.NoError(err)
   186  	match := re.FindStringSubmatch(out)
   187  	f.NotNil(match, fmt.Errorf("could not find VAULT_TOKEN, got:%v\n", out))
   188  	taskToken := match[1]
   189  
   190  	// Update secret
   191  	out, err = e2e.Command("vault", "kv", "put",
   192  		fmt.Sprintf("%s/myapp", tc.secretsPath), "key=UPDATED")
   193  	f.NoError(err, out)
   194  
   195  	elapsed := time.Since(ttlStart)
   196  	time.Sleep((time.Second * 60) - elapsed)
   197  
   198  	// tokens will not be updated
   199  	out, err = e2e.AllocExec(allocID, "task", "env", ns, nil)
   200  	f.NoError(err)
   201  	match = re.FindStringSubmatch(out)
   202  	f.NotNil(match, fmt.Errorf("could not find VAULT_TOKEN, got:%v\n", out))
   203  	f.Equal(taskToken, match[1])
   204  
   205  	// cert will be renewed
   206  	_, err = waitForAllocSecret(allocID, "task", "/secrets/certificate.crt",
   207  		func(out string) bool {
   208  			return strings.Contains(out, "BEGIN CERTIFICATE") &&
   209  				out != renderedCert
   210  		}, wc)
   211  	f.NoError(err)
   212  
   213  	// secret will *not* be renewed because it doesn't have a lease to expire
   214  	_, err = waitForAllocSecret(allocID, "task", "/secrets/access.key",
   215  		func(out string) bool {
   216  			return strings.Contains(out, secretValue)
   217  		}, wc)
   218  	f.NoError(err)
   219  
   220  }
   221  
   222  // We need to namespace the keys in the policy, so read it in and replace the
   223  // values of the policy names
   224  func writePolicy(policyID, policyPath, testID string) (string, error) {
   225  	raw, err := os.ReadFile(policyPath)
   226  	if err != nil {
   227  		return "", err
   228  	}
   229  	policyDoc := string(raw)
   230  	policyDoc = strings.ReplaceAll(policyDoc, "TESTID", testID)
   231  
   232  	ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
   233  	defer cancel()
   234  	cmd := exec.CommandContext(ctx, "vault", "policy", "write", policyID, "-")
   235  	stdin, err := cmd.StdinPipe()
   236  	if err != nil {
   237  		return "", err
   238  	}
   239  
   240  	go func() {
   241  		defer stdin.Close()
   242  		io.WriteString(stdin, policyDoc)
   243  	}()
   244  
   245  	out, err := cmd.CombinedOutput()
   246  	return string(out), err
   247  }
   248  
   249  // We need to namespace the vault paths in the job, so parse it
   250  // and replace the values of the template and vault fields
   251  func runJob(jobID, testID string, index int) error {
   252  
   253  	raw, err := os.ReadFile("./vaultsecrets/input/secrets.nomad")
   254  	if err != nil {
   255  		return err
   256  	}
   257  	jobspec := string(raw)
   258  	jobspec = strings.ReplaceAll(jobspec, "TESTID", testID)
   259  	jobspec = strings.ReplaceAll(jobspec, "DEPLOYNUMBER", string(rune(index)))
   260  
   261  	return e2e.RegisterFromJobspec(jobID, jobspec)
   262  }
   263  
   264  // waitForAllocSecret is similar to e2e.WaitForAllocFile but uses `alloc exec`
   265  // to be able to read the secrets dir, which is not available to `alloc fs`
   266  func waitForAllocSecret(allocID, taskID, path string, test func(string) bool, wc *e2e.WaitConfig) (string, error) {
   267  	var err error
   268  	var out string
   269  	interval, retries := wc.OrDefault()
   270  
   271  	testutil.WaitForResultRetries(retries, func() (bool, error) {
   272  		time.Sleep(interval)
   273  		out, err = e2e.Command("nomad", "alloc", "exec", "-task", taskID, allocID, "cat", path)
   274  		if err != nil {
   275  			return false, fmt.Errorf("could not get file %q from allocation %q: %v",
   276  				path, allocID, err)
   277  		}
   278  		return test(out),
   279  			fmt.Errorf("test for file content failed: got\n%#v", out)
   280  	}, func(e error) {
   281  		err = e
   282  	})
   283  	return out, err
   284  }
   285  
   286  // this will always be sorted
   287  func latestAllocID(jobID string) (string, error) {
   288  	allocs, err := e2e.AllocsForJob(jobID, ns)
   289  	if err != nil {
   290  		return "", err
   291  	}
   292  	return allocs[0]["ID"], nil
   293  }