github.com/anth0d/nomad@v0.0.0-20221214183521-ae3a0a2cad06/e2e/consul/check_restart.go (about) 1 package consul 2 3 import ( 4 "fmt" 5 "os" 6 "reflect" 7 "regexp" 8 "strings" 9 "time" 10 11 e2e "github.com/hashicorp/nomad/e2e/e2eutil" 12 "github.com/hashicorp/nomad/e2e/framework" 13 "github.com/hashicorp/nomad/helper/uuid" 14 ) 15 16 const ns = "" 17 18 type CheckRestartE2ETest struct { 19 framework.TC 20 jobIds []string 21 } 22 23 func (tc *CheckRestartE2ETest) BeforeAll(f *framework.F) { 24 e2e.WaitForLeader(f.T(), tc.Nomad()) 25 e2e.WaitForNodesReady(f.T(), tc.Nomad(), 1) 26 } 27 28 func (tc *CheckRestartE2ETest) AfterEach(f *framework.F) { 29 if os.Getenv("NOMAD_TEST_SKIPCLEANUP") == "1" { 30 return 31 } 32 33 for _, id := range tc.jobIds { 34 err := e2e.StopJob(id, "-purge") 35 f.Assert().NoError(err) 36 } 37 tc.jobIds = []string{} 38 _, err := e2e.Command("nomad", "system", "gc") 39 f.Assert().NoError(err) 40 } 41 42 // TestGroupCheckRestart runs a job with a group service that will never 43 // become healthy. Both tasks should be restarted up to the 'restart' limit. 44 func (tc *CheckRestartE2ETest) TestGroupCheckRestart(f *framework.F) { 45 46 jobID := "test-group-check-restart-" + uuid.Short() 47 f.NoError(e2e.Register(jobID, "consul/input/checks_group_restart.nomad")) 48 tc.jobIds = append(tc.jobIds, jobID) 49 50 f.NoError( 51 e2e.WaitForAllocStatusComparison( 52 func() ([]string, error) { return e2e.AllocStatuses(jobID, ns) }, 53 func(got []string) bool { return reflect.DeepEqual(got, []string{"failed"}) }, 54 &e2e.WaitConfig{Interval: time.Second * 10, Retries: 30}, 55 )) 56 57 allocs, err := e2e.AllocsForJob(jobID, ns) 58 f.NoError(err) 59 f.Len(allocs, 1) 60 61 allocID := allocs[0]["ID"] 62 expected := "Exceeded allowed attempts 2 in interval 5m0s and mode is \"fail\"" 63 64 out, err := e2e.Command("nomad", "alloc", "status", allocID) 65 f.NoError(err, "could not get allocation status") 66 f.Contains(out, expected, 67 fmt.Errorf("expected '%s', got\n%v", expected, out)) 68 69 re := regexp.MustCompile(`Total Restarts += (.*)\n`) 70 match := re.FindAllStringSubmatch(out, -1) 71 for _, m := range match { 72 f.Equal("2", strings.TrimSpace(m[1]), 73 fmt.Errorf("expected exactly 2 restarts for both tasks, got:\n%v", out)) 74 } 75 } 76 77 // TestTaskCheckRestart runs a job with a task service that will never become 78 // healthy. Only the failed task should be restarted up to the 'restart' 79 // limit. 80 func (tc *CheckRestartE2ETest) TestTaskCheckRestart(f *framework.F) { 81 82 jobID := "test-task-check-restart-" + uuid.Short() 83 f.NoError(e2e.Register(jobID, "consul/input/checks_task_restart.nomad")) 84 tc.jobIds = append(tc.jobIds, jobID) 85 86 var allocID string 87 88 f.NoError( 89 e2e.WaitForAllocStatusComparison( 90 func() ([]string, error) { return e2e.AllocStatuses(jobID, ns) }, 91 func(got []string) bool { return reflect.DeepEqual(got, []string{"failed"}) }, 92 &e2e.WaitConfig{Interval: time.Second * 10, Retries: 30}, 93 )) 94 95 expected := "Exceeded allowed attempts 2 in interval 5m0s and mode is \"fail\"" 96 97 out, err := e2e.Command("nomad", "alloc", "status", allocID) 98 f.NoError(err, "could not get allocation status") 99 f.Contains(out, expected, 100 fmt.Errorf("expected '%s', got\n%v", expected, out)) 101 102 re := regexp.MustCompile(`Total Restarts += (.*)\n`) 103 match := re.FindAllStringSubmatch(out, -1) 104 f.Equal("2", strings.TrimSpace(match[0][1]), 105 fmt.Errorf("expected exactly 2 restarts for failed task, got:\n%v", out)) 106 107 f.Equal("0", strings.TrimSpace(match[1][1]), 108 fmt.Errorf("expected exactly no restarts for healthy task, got:\n%v", out)) 109 }