github.com/hernad/nomad@v1.6.112/e2e/consul/on_update.go (about) 1 // Copyright (c) HashiCorp, Inc. 2 // SPDX-License-Identifier: MPL-2.0 3 4 package consul 5 6 import ( 7 "fmt" 8 "time" 9 10 "github.com/hernad/nomad/e2e/e2eutil" 11 "github.com/hernad/nomad/e2e/framework" 12 "github.com/hernad/nomad/helper/uuid" 13 "github.com/hernad/nomad/testutil" 14 ) 15 16 type OnUpdateChecksTest struct { 17 framework.TC 18 jobIDs []string 19 } 20 21 func (tc *OnUpdateChecksTest) BeforeAll(f *framework.F) { 22 // Ensure cluster has leader before running tests 23 e2eutil.WaitForLeader(f.T(), tc.Nomad()) 24 // Ensure that we have at least 1 client node in ready state 25 e2eutil.WaitForNodesReady(f.T(), tc.Nomad(), 1) 26 } 27 28 func (tc *OnUpdateChecksTest) AfterEach(f *framework.F) { 29 nomadClient := tc.Nomad() 30 j := nomadClient.Jobs() 31 32 for _, id := range tc.jobIDs { 33 j.Deregister(id, true, nil) 34 } 35 _, err := e2eutil.Command("nomad", "system", "gc") 36 f.NoError(err) 37 } 38 39 // TestOnUpdateCheck_IgnoreWarning_IgnoreErrors ensures that deployments 40 // complete successfully with service checks that warn and error when on_update 41 // is specified to ignore either. 42 func (tc *OnUpdateChecksTest) TestOnUpdateCheck_IgnoreWarning_IgnoreErrors(f *framework.F) { 43 uuid := uuid.Generate() 44 jobID := fmt.Sprintf("on-update-%s", uuid[0:8]) 45 tc.jobIDs = append(tc.jobIDs, jobID) 46 47 f.NoError( 48 e2eutil.Register(jobID, "consul/input/on_update.nomad"), 49 "should have registered successfully", 50 ) 51 52 wc := &e2eutil.WaitConfig{ 53 Interval: 1 * time.Second, 54 Retries: 60, 55 } 56 f.NoError( 57 e2eutil.WaitForLastDeploymentStatus(jobID, "", "successful", wc), 58 "deployment should have completed successfully", 59 ) 60 61 // register update with on_update = ignore 62 // this check errors, deployment should still be successful 63 f.NoError( 64 e2eutil.Register(jobID, "consul/input/on_update_2.nomad"), 65 "should have registered successfully", 66 ) 67 68 f.NoError( 69 e2eutil.WaitForLastDeploymentStatus(jobID, "", "successful", wc), 70 "deployment should have completed successfully", 71 ) 72 } 73 74 // TestOnUpdate_CheckRestart ensures that a service check set to ignore 75 // warnings still follows the check_restart block if the task becomes 76 // unhealthy after a deployment is successful. on_update_check_restart has a 77 // script check that should report as a warning status for the deployment to 78 // become healthy. The script check then reports unhealthy and the 79 // check_restart policy should restart the task 80 func (tc *OnUpdateChecksTest) TestOnUpdate_CheckRestart(f *framework.F) { 81 uuid := uuid.Generate() 82 jobID := fmt.Sprintf("on-update-restart-%s", uuid[0:8]) 83 tc.jobIDs = append(tc.jobIDs, jobID) 84 85 f.NoError( 86 e2eutil.Register(jobID, "consul/input/on_update_check_restart.nomad"), 87 "should have registered successfully", 88 ) 89 90 wc := &e2eutil.WaitConfig{ 91 Interval: 1 * time.Second, 92 Retries: 60, 93 } 94 f.NoError( 95 e2eutil.WaitForLastDeploymentStatus(jobID, "", "successful", wc), 96 "deployment should have completed successfully", 97 ) 98 99 // Wait for and ensure that allocation restarted 100 testutil.WaitForResultRetries(wc.Retries, func() (bool, error) { 101 time.Sleep(wc.Interval) 102 allocs, err := e2eutil.AllocTaskEventsForJob(jobID, "") 103 if err != nil { 104 return false, err 105 } 106 107 for allocID, allocEvents := range allocs { 108 var allocRestarted bool 109 var eventTypes []string 110 for _, events := range allocEvents { 111 eventTypes = append(eventTypes, events["Type"]) 112 if events["Type"] == "Restart Signaled" { 113 allocRestarted = true 114 } 115 } 116 if allocRestarted { 117 return true, nil 118 } 119 return false, fmt.Errorf("alloc %s expected to restart got %v", allocID, eventTypes) 120 } 121 122 return true, nil 123 }, func(err error) { 124 f.NoError(err) 125 }) 126 }