github.com/anth0d/nomad@v0.0.0-20221214183521-ae3a0a2cad06/e2e/consul/check_restart.go (about)

     1  package consul
     2  
     3  import (
     4  	"fmt"
     5  	"os"
     6  	"reflect"
     7  	"regexp"
     8  	"strings"
     9  	"time"
    10  
    11  	e2e "github.com/hashicorp/nomad/e2e/e2eutil"
    12  	"github.com/hashicorp/nomad/e2e/framework"
    13  	"github.com/hashicorp/nomad/helper/uuid"
    14  )
    15  
    16  const ns = ""
    17  
    18  type CheckRestartE2ETest struct {
    19  	framework.TC
    20  	jobIds []string
    21  }
    22  
    23  func (tc *CheckRestartE2ETest) BeforeAll(f *framework.F) {
    24  	e2e.WaitForLeader(f.T(), tc.Nomad())
    25  	e2e.WaitForNodesReady(f.T(), tc.Nomad(), 1)
    26  }
    27  
    28  func (tc *CheckRestartE2ETest) AfterEach(f *framework.F) {
    29  	if os.Getenv("NOMAD_TEST_SKIPCLEANUP") == "1" {
    30  		return
    31  	}
    32  
    33  	for _, id := range tc.jobIds {
    34  		err := e2e.StopJob(id, "-purge")
    35  		f.Assert().NoError(err)
    36  	}
    37  	tc.jobIds = []string{}
    38  	_, err := e2e.Command("nomad", "system", "gc")
    39  	f.Assert().NoError(err)
    40  }
    41  
    42  // TestGroupCheckRestart runs a job with a group service that will never
    43  // become healthy. Both tasks should be restarted up to the 'restart' limit.
    44  func (tc *CheckRestartE2ETest) TestGroupCheckRestart(f *framework.F) {
    45  
    46  	jobID := "test-group-check-restart-" + uuid.Short()
    47  	f.NoError(e2e.Register(jobID, "consul/input/checks_group_restart.nomad"))
    48  	tc.jobIds = append(tc.jobIds, jobID)
    49  
    50  	f.NoError(
    51  		e2e.WaitForAllocStatusComparison(
    52  			func() ([]string, error) { return e2e.AllocStatuses(jobID, ns) },
    53  			func(got []string) bool { return reflect.DeepEqual(got, []string{"failed"}) },
    54  			&e2e.WaitConfig{Interval: time.Second * 10, Retries: 30},
    55  		))
    56  
    57  	allocs, err := e2e.AllocsForJob(jobID, ns)
    58  	f.NoError(err)
    59  	f.Len(allocs, 1)
    60  
    61  	allocID := allocs[0]["ID"]
    62  	expected := "Exceeded allowed attempts 2 in interval 5m0s and mode is \"fail\""
    63  
    64  	out, err := e2e.Command("nomad", "alloc", "status", allocID)
    65  	f.NoError(err, "could not get allocation status")
    66  	f.Contains(out, expected,
    67  		fmt.Errorf("expected '%s', got\n%v", expected, out))
    68  
    69  	re := regexp.MustCompile(`Total Restarts += (.*)\n`)
    70  	match := re.FindAllStringSubmatch(out, -1)
    71  	for _, m := range match {
    72  		f.Equal("2", strings.TrimSpace(m[1]),
    73  			fmt.Errorf("expected exactly 2 restarts for both tasks, got:\n%v", out))
    74  	}
    75  }
    76  
    77  // TestTaskCheckRestart runs a job with a task service that will never become
    78  // healthy. Only the failed task should be restarted up to the 'restart'
    79  // limit.
    80  func (tc *CheckRestartE2ETest) TestTaskCheckRestart(f *framework.F) {
    81  
    82  	jobID := "test-task-check-restart-" + uuid.Short()
    83  	f.NoError(e2e.Register(jobID, "consul/input/checks_task_restart.nomad"))
    84  	tc.jobIds = append(tc.jobIds, jobID)
    85  
    86  	var allocID string
    87  
    88  	f.NoError(
    89  		e2e.WaitForAllocStatusComparison(
    90  			func() ([]string, error) { return e2e.AllocStatuses(jobID, ns) },
    91  			func(got []string) bool { return reflect.DeepEqual(got, []string{"failed"}) },
    92  			&e2e.WaitConfig{Interval: time.Second * 10, Retries: 30},
    93  		))
    94  
    95  	expected := "Exceeded allowed attempts 2 in interval 5m0s and mode is \"fail\""
    96  
    97  	out, err := e2e.Command("nomad", "alloc", "status", allocID)
    98  	f.NoError(err, "could not get allocation status")
    99  	f.Contains(out, expected,
   100  		fmt.Errorf("expected '%s', got\n%v", expected, out))
   101  
   102  	re := regexp.MustCompile(`Total Restarts += (.*)\n`)
   103  	match := re.FindAllStringSubmatch(out, -1)
   104  	f.Equal("2", strings.TrimSpace(match[0][1]),
   105  		fmt.Errorf("expected exactly 2 restarts for failed task, got:\n%v", out))
   106  
   107  	f.Equal("0", strings.TrimSpace(match[1][1]),
   108  		fmt.Errorf("expected exactly no restarts for healthy task, got:\n%v", out))
   109  }