github.com/anth0d/nomad@v0.0.0-20221214183521-ae3a0a2cad06/e2e/consul/consul.go (about) 1 package consul 2 3 import ( 4 "fmt" 5 "os" 6 7 api "github.com/hashicorp/nomad/api" 8 "github.com/hashicorp/nomad/e2e/e2eutil" 9 "github.com/hashicorp/nomad/e2e/framework" 10 "github.com/hashicorp/nomad/helper" 11 "github.com/hashicorp/nomad/helper/uuid" 12 "github.com/hashicorp/nomad/nomad/structs" 13 "github.com/hashicorp/nomad/testutil" 14 "github.com/stretchr/testify/require" 15 ) 16 17 const ( 18 consulJobBasic = "consul/input/consul_example.nomad" 19 consulJobCanaryTags = "consul/input/canary_tags.nomad" 20 21 consulJobRegisterOnUpdatePart1 = "consul/input/services_empty.nomad" 22 consulJobRegisterOnUpdatePart2 = "consul/input/services_present.nomad" 23 ) 24 25 const ( 26 // unless otherwise set, tests should just use the default consul namespace 27 consulNamespace = "default" 28 ) 29 30 type ConsulE2ETest struct { 31 framework.TC 32 jobIds []string 33 } 34 35 func init() { 36 framework.AddSuites(&framework.TestSuite{ 37 Component: "Consul", 38 CanRunLocal: true, 39 Consul: true, 40 Cases: []framework.TestCase{ 41 new(ConsulE2ETest), 42 new(ScriptChecksE2ETest), 43 new(CheckRestartE2ETest), 44 new(OnUpdateChecksTest), 45 }, 46 }) 47 } 48 49 func (tc *ConsulE2ETest) BeforeAll(f *framework.F) { 50 e2eutil.WaitForLeader(f.T(), tc.Nomad()) 51 e2eutil.WaitForNodesReady(f.T(), tc.Nomad(), 1) 52 } 53 54 func (tc *ConsulE2ETest) AfterEach(f *framework.F) { 55 if os.Getenv("NOMAD_TEST_SKIPCLEANUP") == "1" { 56 return 57 } 58 59 for _, id := range tc.jobIds { 60 _, _, err := tc.Nomad().Jobs().Deregister(id, true, nil) 61 require.NoError(f.T(), err) 62 } 63 tc.jobIds = []string{} 64 require.NoError(f.T(), tc.Nomad().System().GarbageCollect()) 65 } 66 67 // TestConsulRegistration asserts that a job registers services with tags in Consul. 68 func (tc *ConsulE2ETest) TestConsulRegistration(f *framework.F) { 69 t := f.T() 70 r := require.New(t) 71 72 nomadClient := tc.Nomad() 73 jobId := "consul" + uuid.Short() 74 tc.jobIds = append(tc.jobIds, jobId) 75 76 allocations := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, consulJobBasic, jobId, "") 77 require.Equal(t, 3, len(allocations)) 78 allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations) 79 e2eutil.WaitForAllocsRunning(t, tc.Nomad(), allocIDs) 80 81 expectedTags := []string{ 82 "cache", 83 "global", 84 } 85 86 // Assert services get registered 87 e2eutil.RequireConsulRegistered(r, tc.Consul(), consulNamespace, "consul-example", 3) 88 services, _, err := tc.Consul().Catalog().Service("consul-example", "", nil) 89 require.NoError(t, err) 90 for _, s := range services { 91 // If we've made it this far the tags should *always* match 92 require.ElementsMatch(t, expectedTags, s.ServiceTags) 93 } 94 95 // Stop the job 96 e2eutil.WaitForJobStopped(t, nomadClient, jobId) 97 98 // Verify that services were de-registered in Consul 99 e2eutil.RequireConsulDeregistered(r, tc.Consul(), consulNamespace, "consul-example") 100 } 101 102 func (tc *ConsulE2ETest) TestConsulRegisterOnUpdate(f *framework.F) { 103 t := f.T() 104 r := require.New(t) 105 106 nomadClient := tc.Nomad() 107 catalog := tc.Consul().Catalog() 108 jobID := "consul" + uuid.Short() 109 tc.jobIds = append(tc.jobIds, jobID) 110 111 // Initial job has no services for task. 112 allocations := e2eutil.RegisterAndWaitForAllocs(t, nomadClient, consulJobRegisterOnUpdatePart1, jobID, "") 113 require.Equal(t, 1, len(allocations)) 114 allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocations) 115 e2eutil.WaitForAllocsRunning(t, tc.Nomad(), allocIDs) 116 117 // Assert service not yet registered. 118 results, _, err := catalog.Service("nc-service", "", nil) 119 require.NoError(t, err) 120 require.Empty(t, results) 121 122 // On update, add services for task. 123 allocations = e2eutil.RegisterAndWaitForAllocs(t, nomadClient, consulJobRegisterOnUpdatePart2, jobID, "") 124 require.Equal(t, 1, len(allocations)) 125 allocIDs = e2eutil.AllocIDsFromAllocationListStubs(allocations) 126 e2eutil.WaitForAllocsRunning(t, tc.Nomad(), allocIDs) 127 128 // Assert service is now registered. 129 e2eutil.RequireConsulRegistered(r, tc.Consul(), consulNamespace, "nc-service", 1) 130 } 131 132 // TestCanaryInplaceUpgrades verifies setting and unsetting canary tags 133 func (tc *ConsulE2ETest) TestCanaryInplaceUpgrades(f *framework.F) { 134 t := f.T() 135 136 // TODO(shoenig) https://github.com/hashicorp/nomad/issues/9627 137 t.Skip("THIS TEST IS BROKEN (#9627)") 138 139 nomadClient := tc.Nomad() 140 consulClient := tc.Consul() 141 jobId := "consul" + uuid.Generate()[0:8] 142 tc.jobIds = append(tc.jobIds, jobId) 143 144 allocs := e2eutil.RegisterAndWaitForAllocs(f.T(), nomadClient, consulJobCanaryTags, jobId, "") 145 require.Equal(t, 2, len(allocs)) 146 147 allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocs) 148 e2eutil.WaitForAllocsRunning(t, nomadClient, allocIDs) 149 150 // Start a deployment 151 job, _, err := nomadClient.Jobs().Info(jobId, nil) 152 require.NoError(t, err) 153 job.Meta = map[string]string{"version": "2"} 154 resp, _, err := nomadClient.Jobs().Register(job, nil) 155 require.NoError(t, err) 156 require.NotEmpty(t, resp.EvalID) 157 158 // Eventually have a canary 159 var activeDeploy *api.Deployment 160 testutil.WaitForResult(func() (bool, error) { 161 deploys, _, err := nomadClient.Jobs().Deployments(jobId, false, nil) 162 if err != nil { 163 return false, err 164 } 165 if expected := 2; len(deploys) != expected { 166 return false, fmt.Errorf("expected 2 deploys but found %v", deploys) 167 } 168 169 for _, d := range deploys { 170 if d.Status == structs.DeploymentStatusRunning { 171 activeDeploy = d 172 break 173 } 174 } 175 if activeDeploy == nil { 176 return false, fmt.Errorf("no running deployments: %v", deploys) 177 } 178 if expected := 1; len(activeDeploy.TaskGroups["consul_canary_test"].PlacedCanaries) != expected { 179 return false, fmt.Errorf("expected %d placed canaries but found %#v", 180 expected, activeDeploy.TaskGroups["consul_canary_test"]) 181 } 182 183 return true, nil 184 }, func(err error) { 185 f.NoError(err, "error while waiting for deploys") 186 }) 187 188 allocID := activeDeploy.TaskGroups["consul_canary_test"].PlacedCanaries[0] 189 testutil.WaitForResult(func() (bool, error) { 190 alloc, _, err := nomadClient.Allocations().Info(allocID, nil) 191 if err != nil { 192 return false, err 193 } 194 195 if alloc.DeploymentStatus == nil { 196 return false, fmt.Errorf("canary alloc %s has no deployment status", allocID) 197 } 198 if alloc.DeploymentStatus.Healthy == nil { 199 return false, fmt.Errorf("canary alloc %s has no deployment health: %#v", 200 allocID, alloc.DeploymentStatus) 201 } 202 return *alloc.DeploymentStatus.Healthy, fmt.Errorf("expected healthy canary but found: %#v", 203 alloc.DeploymentStatus) 204 }, func(err error) { 205 f.NoError(err, "error waiting for canary to be healthy") 206 }) 207 208 // Check Consul for canary tags 209 testutil.WaitForResult(func() (bool, error) { 210 consulServices, _, err := consulClient.Catalog().Service("canarytest", "", nil) 211 if err != nil { 212 return false, err 213 } 214 for _, s := range consulServices { 215 if helper.SliceSetEq([]string{"canary", "foo"}, s.ServiceTags) { 216 return true, nil 217 } 218 } 219 return false, fmt.Errorf(`could not find service tags {"canary", "foo"}: %#v`, consulServices) 220 }, func(err error) { 221 f.NoError(err, "error waiting for canary tags") 222 }) 223 224 // Promote canary 225 { 226 resp, _, err := nomadClient.Deployments().PromoteAll(activeDeploy.ID, nil) 227 require.NoError(t, err) 228 require.NotEmpty(t, resp.EvalID) 229 } 230 231 // Eventually canary is promoted 232 testutil.WaitForResult(func() (bool, error) { 233 alloc, _, err := nomadClient.Allocations().Info(allocID, nil) 234 if err != nil { 235 return false, err 236 } 237 return !alloc.DeploymentStatus.Canary, fmt.Errorf("still a canary") 238 }, func(err error) { 239 require.NoError(t, err, "error waiting for canary to be promoted") 240 }) 241 242 // Verify that no instances have canary tags 243 expected := []string{"foo", "bar"} 244 testutil.WaitForResult(func() (bool, error) { 245 consulServices, _, err := consulClient.Catalog().Service("canarytest", "", nil) 246 if err != nil { 247 return false, err 248 } 249 for _, s := range consulServices { 250 if !helper.SliceSetEq(expected, s.ServiceTags) { 251 return false, fmt.Errorf("expected %#v Consul tags but found %#v", 252 expected, s.ServiceTags) 253 } 254 } 255 return true, nil 256 }, func(err error) { 257 require.NoError(t, err, "error waiting for non-canary tags") 258 }) 259 260 }