github.com/anth0d/nomad@v0.0.0-20221214183521-ae3a0a2cad06/e2e/consul/consul.go (about)

     1  package consul
     2  
     3  import (
     4  	"fmt"
     5  	"os"
     6  
     7  	api "github.com/hashicorp/nomad/api"
     8  	"github.com/hashicorp/nomad/e2e/e2eutil"
     9  	"github.com/hashicorp/nomad/e2e/framework"
    10  	"github.com/hashicorp/nomad/helper"
    11  	"github.com/hashicorp/nomad/helper/uuid"
    12  	"github.com/hashicorp/nomad/nomad/structs"
    13  	"github.com/hashicorp/nomad/testutil"
    14  	"github.com/stretchr/testify/require"
    15  )
    16  
    17  const (
    18  	consulJobBasic      = "consul/input/consul_example.nomad"
    19  	consulJobCanaryTags = "consul/input/canary_tags.nomad"
    20  
    21  	consulJobRegisterOnUpdatePart1 = "consul/input/services_empty.nomad"
    22  	consulJobRegisterOnUpdatePart2 = "consul/input/services_present.nomad"
    23  )
    24  
    25  const (
    26  	// unless otherwise set, tests should just use the default consul namespace
    27  	consulNamespace = "default"
    28  )
    29  
    30  type ConsulE2ETest struct {
    31  	framework.TC
    32  	jobIds []string
    33  }
    34  
    35  func init() {
    36  	framework.AddSuites(&framework.TestSuite{
    37  		Component:   "Consul",
    38  		CanRunLocal: true,
    39  		Consul:      true,
    40  		Cases: []framework.TestCase{
    41  			new(ConsulE2ETest),
    42  			new(ScriptChecksE2ETest),
    43  			new(CheckRestartE2ETest),
    44  			new(OnUpdateChecksTest),
    45  		},
    46  	})
    47  }
    48  
    49  func (tc *ConsulE2ETest) BeforeAll(f *framework.F) {
    50  	e2eutil.WaitForLeader(f.T(), tc.Nomad())
    51  	e2eutil.WaitForNodesReady(f.T(), tc.Nomad(), 1)
    52  }
    53  
    54  func (tc *ConsulE2ETest) AfterEach(f *framework.F) {
    55  	if os.Getenv("NOMAD_TEST_SKIPCLEANUP") == "1" {
    56  		return
    57  	}
    58  
    59  	for _, id := range tc.jobIds {
    60  		_, _, err := tc.Nomad().Jobs().Deregister(id, true, nil)
    61  		require.NoError(f.T(), err)
    62  	}
    63  	tc.jobIds = []string{}
    64  	require.NoError(f.T(), tc.Nomad().System().GarbageCollect())
    65  }
    66  
    67  // TestConsulRegistration asserts that a job registers services with tags in Consul.
    68  func (tc *ConsulE2ETest) TestConsulRegistration(f *framework.F) {
    69  	t := f.T()
    70  	r := require.New(t)
    71  
    72  	nomadClient := tc.Nomad()
    73  	jobId := "consul" + uuid.Short()
    74  	tc.jobIds = append(tc.jobIds, jobId)
    75  
    76  	allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, consulJobBasic, jobId, "")
    77  	require.Equal(t, 3, len(allocations))
    78  	allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations)
    79  	e2eutil.WaitForAllocsRunning(t, tc.Nomad(), allocIDs)
    80  
    81  	expectedTags := []string{
    82  		"cache",
    83  		"global",
    84  	}
    85  
    86  	// Assert services get registered
    87  	e2eutil.RequireConsulRegistered(r, tc.Consul(), consulNamespace, "consul-example", 3)
    88  	services, _, err := tc.Consul().Catalog().Service("consul-example", "", nil)
    89  	require.NoError(t, err)
    90  	for _, s := range services {
    91  		// If we've made it this far the tags should *always* match
    92  		require.ElementsMatch(t, expectedTags, s.ServiceTags)
    93  	}
    94  
    95  	// Stop the job
    96  	e2eutil.WaitForJobStopped(t, nomadClient, jobId)
    97  
    98  	// Verify that services were de-registered in Consul
    99  	e2eutil.RequireConsulDeregistered(r, tc.Consul(), consulNamespace, "consul-example")
   100  }
   101  
   102  func (tc *ConsulE2ETest) TestConsulRegisterOnUpdate(f *framework.F) {
   103  	t := f.T()
   104  	r := require.New(t)
   105  
   106  	nomadClient := tc.Nomad()
   107  	catalog := tc.Consul().Catalog()
   108  	jobID := "consul" + uuid.Short()
   109  	tc.jobIds = append(tc.jobIds, jobID)
   110  
   111  	// Initial job has no services for task.
   112  	allocations := e2eutil.RegisterAndWaitForAllocs(t, nomadClient, consulJobRegisterOnUpdatePart1, jobID, "")
   113  	require.Equal(t, 1, len(allocations))
   114  	allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations)
   115  	e2eutil.WaitForAllocsRunning(t, tc.Nomad(), allocIDs)
   116  
   117  	// Assert service not yet registered.
   118  	results, _, err := catalog.Service("nc-service", "", nil)
   119  	require.NoError(t, err)
   120  	require.Empty(t, results)
   121  
   122  	// On update, add services for task.
   123  	allocations = e2eutil.RegisterAndWaitForAllocs(t, nomadClient, consulJobRegisterOnUpdatePart2, jobID, "")
   124  	require.Equal(t, 1, len(allocations))
   125  	allocIDs = e2eutil.AllocIDsFromAllocationListStubs(allocations)
   126  	e2eutil.WaitForAllocsRunning(t, tc.Nomad(), allocIDs)
   127  
   128  	// Assert service is now registered.
   129  	e2eutil.RequireConsulRegistered(r, tc.Consul(), consulNamespace, "nc-service", 1)
   130  }
   131  
   132  // TestCanaryInplaceUpgrades verifies setting and unsetting canary tags
   133  func (tc *ConsulE2ETest) TestCanaryInplaceUpgrades(f *framework.F) {
   134  	t := f.T()
   135  
   136  	// TODO(shoenig) https://github.com/hashicorp/nomad/issues/9627
   137  	t.Skip("THIS TEST IS BROKEN (#9627)")
   138  
   139  	nomadClient := tc.Nomad()
   140  	consulClient := tc.Consul()
   141  	jobId := "consul" + uuid.Generate()[0:8]
   142  	tc.jobIds = append(tc.jobIds, jobId)
   143  
   144  	allocs := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, consulJobCanaryTags, jobId, "")
   145  	require.Equal(t, 2, len(allocs))
   146  
   147  	allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocs)
   148  	e2eutil.WaitForAllocsRunning(t, nomadClient, allocIDs)
   149  
   150  	// Start a deployment
   151  	job, _, err := nomadClient.Jobs().Info(jobId, nil)
   152  	require.NoError(t, err)
   153  	job.Meta = map[string]string{"version": "2"}
   154  	resp, _, err := nomadClient.Jobs().Register(job, nil)
   155  	require.NoError(t, err)
   156  	require.NotEmpty(t, resp.EvalID)
   157  
   158  	// Eventually have a canary
   159  	var activeDeploy *api.Deployment
   160  	testutil.WaitForResult(func() (bool, error) {
   161  		deploys, _, err := nomadClient.Jobs().Deployments(jobId, false, nil)
   162  		if err != nil {
   163  			return false, err
   164  		}
   165  		if expected := 2; len(deploys) != expected {
   166  			return false, fmt.Errorf("expected 2 deploys but found %v", deploys)
   167  		}
   168  
   169  		for _, d := range deploys {
   170  			if d.Status == structs.DeploymentStatusRunning {
   171  				activeDeploy = d
   172  				break
   173  			}
   174  		}
   175  		if activeDeploy == nil {
   176  			return false, fmt.Errorf("no running deployments: %v", deploys)
   177  		}
   178  		if expected := 1; len(activeDeploy.TaskGroups["consul_canary_test"].PlacedCanaries) != expected {
   179  			return false, fmt.Errorf("expected %d placed canaries but found %#v",
   180  				expected, activeDeploy.TaskGroups["consul_canary_test"])
   181  		}
   182  
   183  		return true, nil
   184  	}, func(err error) {
   185  		f.NoError(err, "error while waiting for deploys")
   186  	})
   187  
   188  	allocID := activeDeploy.TaskGroups["consul_canary_test"].PlacedCanaries[0]
   189  	testutil.WaitForResult(func() (bool, error) {
   190  		alloc, _, err := nomadClient.Allocations().Info(allocID, nil)
   191  		if err != nil {
   192  			return false, err
   193  		}
   194  
   195  		if alloc.DeploymentStatus == nil {
   196  			return false, fmt.Errorf("canary alloc %s has no deployment status", allocID)
   197  		}
   198  		if alloc.DeploymentStatus.Healthy == nil {
   199  			return false, fmt.Errorf("canary alloc %s has no deployment health: %#v",
   200  				allocID, alloc.DeploymentStatus)
   201  		}
   202  		return *alloc.DeploymentStatus.Healthy, fmt.Errorf("expected healthy canary but found: %#v",
   203  			alloc.DeploymentStatus)
   204  	}, func(err error) {
   205  		f.NoError(err, "error waiting for canary to be healthy")
   206  	})
   207  
   208  	// Check Consul for canary tags
   209  	testutil.WaitForResult(func() (bool, error) {
   210  		consulServices, _, err := consulClient.Catalog().Service("canarytest", "", nil)
   211  		if err != nil {
   212  			return false, err
   213  		}
   214  		for _, s := range consulServices {
   215  			if helper.SliceSetEq([]string{"canary", "foo"}, s.ServiceTags) {
   216  				return true, nil
   217  			}
   218  		}
   219  		return false, fmt.Errorf(`could not find service tags {"canary", "foo"}: %#v`, consulServices)
   220  	}, func(err error) {
   221  		f.NoError(err, "error waiting for canary tags")
   222  	})
   223  
   224  	// Promote canary
   225  	{
   226  		resp, _, err := nomadClient.Deployments().PromoteAll(activeDeploy.ID, nil)
   227  		require.NoError(t, err)
   228  		require.NotEmpty(t, resp.EvalID)
   229  	}
   230  
   231  	// Eventually canary is promoted
   232  	testutil.WaitForResult(func() (bool, error) {
   233  		alloc, _, err := nomadClient.Allocations().Info(allocID, nil)
   234  		if err != nil {
   235  			return false, err
   236  		}
   237  		return !alloc.DeploymentStatus.Canary, fmt.Errorf("still a canary")
   238  	}, func(err error) {
   239  		require.NoError(t, err, "error waiting for canary to be promoted")
   240  	})
   241  
   242  	// Verify that no instances have canary tags
   243  	expected := []string{"foo", "bar"}
   244  	testutil.WaitForResult(func() (bool, error) {
   245  		consulServices, _, err := consulClient.Catalog().Service("canarytest", "", nil)
   246  		if err != nil {
   247  			return false, err
   248  		}
   249  		for _, s := range consulServices {
   250  			if !helper.SliceSetEq(expected, s.ServiceTags) {
   251  				return false, fmt.Errorf("expected %#v Consul tags but found %#v",
   252  					expected, s.ServiceTags)
   253  			}
   254  		}
   255  		return true, nil
   256  	}, func(err error) {
   257  		require.NoError(t, err, "error waiting for non-canary tags")
   258  	})
   259  
   260  }