github.com/anth0d/nomad@v0.0.0-20221214183521-ae3a0a2cad06/e2e/e2eutil/utils.go (about)

     1  package e2eutil
     2  
     3  import (
     4  	"bytes"
     5  	"fmt"
     6  	"os"
     7  	"testing"
     8  	"text/template"
     9  	"time"
    10  
    11  	api "github.com/hashicorp/nomad/api"
    12  	"github.com/hashicorp/nomad/helper/pointer"
    13  	"github.com/hashicorp/nomad/jobspec2"
    14  	"github.com/hashicorp/nomad/nomad/structs"
    15  	"github.com/hashicorp/nomad/testutil"
    16  	"github.com/kr/pretty"
    17  	"github.com/stretchr/testify/require"
    18  )
    19  
    20  // retries is used to control how many times to retry checking if the cluster has a leader yet
    21  const retries = 500
    22  
    23  func WaitForLeader(t *testing.T, nomadClient *api.Client) {
    24  	statusAPI := nomadClient.Status()
    25  
    26  	testutil.WaitForResultRetries(retries, func() (bool, error) {
    27  		leader, err := statusAPI.Leader()
    28  		return leader != "", err
    29  	}, func(err error) {
    30  		require.NoError(t, err, "failed to find leader")
    31  	})
    32  }
    33  
    34  // WaitForNodesReady waits until at least `nodes` number of nodes are ready or
    35  // fails the test.
    36  func WaitForNodesReady(t *testing.T, nomadClient *api.Client, nodes int) {
    37  	nodesAPI := nomadClient.Nodes()
    38  
    39  	testutil.WaitForResultRetries(retries, func() (bool, error) {
    40  		defer time.Sleep(time.Millisecond * 100)
    41  		nodesList, _, err := nodesAPI.List(nil)
    42  		if err != nil {
    43  			return false, fmt.Errorf("error listing nodes: %v", err)
    44  		}
    45  
    46  		eligibleNodes := 0
    47  		for _, node := range nodesList {
    48  			if node.Status == "ready" {
    49  				eligibleNodes++
    50  			}
    51  		}
    52  
    53  		return eligibleNodes >= nodes, fmt.Errorf("only %d nodes ready (wanted at least %d)", eligibleNodes, nodes)
    54  	}, func(err error) {
    55  		require.NoError(t, err, "failed to get enough ready nodes")
    56  	})
    57  }
    58  
    59  func stringToPtrOrNil(s string) *string {
    60  	if s == "" {
    61  		return nil
    62  	}
    63  	return pointer.Of(s)
    64  }
    65  
    66  func Parse2(t *testing.T, jobFile string) (*api.Job, error) {
    67  	f, err := os.Open(jobFile)
    68  	require.NoError(t, err)
    69  	return jobspec2.Parse(jobFile, f)
    70  }
    71  
    72  func RegisterAllocs(t *testing.T, nomadClient *api.Client, jobFile, jobID, cToken string) []*api.AllocationListStub {
    73  
    74  	// Parse job
    75  	job, err := Parse2(t, jobFile)
    76  	require.NoError(t, err)
    77  
    78  	// Set custom job ID (distinguish among tests)
    79  	job.ID = pointer.Of(jobID)
    80  
    81  	// Set a Consul "operator" token for the job, if provided.
    82  	job.ConsulToken = stringToPtrOrNil(cToken)
    83  
    84  	// Register job
    85  	var idx uint64
    86  	jobs := nomadClient.Jobs()
    87  	testutil.WaitForResult(func() (bool, error) {
    88  		resp, meta, err := jobs.Register(job, nil)
    89  		if err != nil {
    90  			return false, err
    91  		}
    92  		idx = meta.LastIndex
    93  		return resp.EvalID != "", fmt.Errorf("expected EvalID:%s", pretty.Sprint(resp))
    94  	}, func(err error) {
    95  		require.NoError(t, err)
    96  	})
    97  
    98  	allocs, _, err := jobs.Allocations(jobID, false, &api.QueryOptions{WaitIndex: idx})
    99  	require.NoError(t, err)
   100  	return allocs
   101  }
   102  
   103  // RegisterAndWaitForAllocs wraps RegisterAllocs but blocks until Evals
   104  // successfully create Allocs.
   105  func RegisterAndWaitForAllocs(t *testing.T, nomadClient *api.Client, jobFile, jobID, cToken string) []*api.AllocationListStub {
   106  	jobs := nomadClient.Jobs()
   107  
   108  	// Start allocations
   109  	RegisterAllocs(t, nomadClient, jobFile, jobID, cToken)
   110  
   111  	var err error
   112  	allocs := []*api.AllocationListStub{}
   113  	evals := []*api.Evaluation{}
   114  
   115  	// Wrap in retry to wait until placement
   116  	testutil.WaitForResultRetries(retries, func() (bool, error) {
   117  		time.Sleep(time.Second)
   118  
   119  		allocs, _, err = jobs.Allocations(jobID, false, nil)
   120  		if len(allocs) == 0 {
   121  			evals, _, err = nomadClient.Jobs().Evaluations(jobID, nil)
   122  			return false, fmt.Errorf("no allocations for job %v", jobID)
   123  		}
   124  
   125  		return true, nil
   126  	}, func(e error) {
   127  		msg := fmt.Sprintf("allocations not placed for %s", jobID)
   128  		for _, eval := range evals {
   129  			msg += fmt.Sprintf("\n  %s - %s", eval.Status, eval.StatusDescription)
   130  		}
   131  
   132  		require.Fail(t, msg, "full evals: %v", pretty.Sprint(evals))
   133  	})
   134  
   135  	require.NoError(t, err) // we only care about the last error
   136  
   137  	return allocs
   138  }
   139  
   140  func WaitForAllocRunning(t *testing.T, nomadClient *api.Client, allocID string) {
   141  	t.Helper()
   142  
   143  	testutil.WaitForResultRetries(retries, func() (bool, error) {
   144  		time.Sleep(time.Millisecond * 100)
   145  		alloc, _, err := nomadClient.Allocations().Info(allocID, nil)
   146  		if err != nil {
   147  			return false, err
   148  		}
   149  
   150  		return alloc.ClientStatus == structs.AllocClientStatusRunning, fmt.Errorf("expected status running, but was: %s\n%v", alloc.ClientStatus, pretty.Sprint(alloc))
   151  	}, func(err error) {
   152  		require.NoError(t, err, "failed to wait on alloc")
   153  	})
   154  }
   155  
   156  func WaitForAllocTaskRunning(t *testing.T, nomadClient *api.Client, allocID, task string) {
   157  	WaitForAllocTaskState(t, nomadClient, allocID, task, structs.TaskStateRunning)
   158  }
   159  
   160  func WaitForAllocTaskComplete(t *testing.T, nomadClient *api.Client, allocID, task string) {
   161  	WaitForAllocTaskState(t, nomadClient, allocID, task, structs.TaskStateDead)
   162  }
   163  
   164  func WaitForAllocTaskState(t *testing.T, nomadClient *api.Client, allocID, task, state string) {
   165  	testutil.WaitForResultRetries(retries, func() (bool, error) {
   166  		time.Sleep(time.Millisecond * 500)
   167  		alloc, _, err := nomadClient.Allocations().Info(allocID, nil)
   168  		if err != nil {
   169  			return false, err
   170  		}
   171  		currentState := "n/a"
   172  		if taskState := alloc.TaskStates[task]; taskState != nil {
   173  			currentState = taskState.State
   174  		}
   175  		return currentState == state, fmt.Errorf("expected status %s, but was: %s", state, currentState)
   176  	}, func(err error) {
   177  		t.Fatalf("failed to wait on alloc task: %v", err)
   178  	})
   179  }
   180  
   181  func WaitForAllocsRunning(t *testing.T, nomadClient *api.Client, allocIDs []string) {
   182  	for _, allocID := range allocIDs {
   183  		WaitForAllocRunning(t, nomadClient, allocID)
   184  	}
   185  }
   186  
   187  func WaitForAllocsNotPending(t *testing.T, nomadClient *api.Client, allocIDs []string) {
   188  	for _, allocID := range allocIDs {
   189  		WaitForAllocNotPending(t, nomadClient, allocID)
   190  	}
   191  }
   192  
   193  func WaitForAllocNotPending(t *testing.T, nomadClient *api.Client, allocID string) {
   194  	testutil.WaitForResultRetries(retries, func() (bool, error) {
   195  		time.Sleep(time.Millisecond * 100)
   196  		alloc, _, err := nomadClient.Allocations().Info(allocID, nil)
   197  		if err != nil {
   198  			return false, err
   199  		}
   200  
   201  		return alloc.ClientStatus != structs.AllocClientStatusPending, fmt.Errorf("expected status not pending, but was: %s", alloc.ClientStatus)
   202  	}, func(err error) {
   203  		require.NoError(t, err, "failed to wait on alloc")
   204  	})
   205  }
   206  
   207  // WaitForJobStopped stops a job and waits for all of its allocs to terminate.
   208  func WaitForJobStopped(t *testing.T, nomadClient *api.Client, job string) {
   209  	allocs, _, err := nomadClient.Jobs().Allocations(job, true, nil)
   210  	require.NoError(t, err, "error getting allocations for job %q", job)
   211  	ids := AllocIDsFromAllocationListStubs(allocs)
   212  	_, _, err = nomadClient.Jobs().Deregister(job, true, nil)
   213  	require.NoError(t, err, "error deregistering job %q", job)
   214  	for _, id := range ids {
   215  		WaitForAllocStopped(t, nomadClient, id)
   216  	}
   217  }
   218  
   219  func WaitForAllocsStopped(t *testing.T, nomadClient *api.Client, allocIDs []string) {
   220  	for _, allocID := range allocIDs {
   221  		WaitForAllocStopped(t, nomadClient, allocID)
   222  	}
   223  }
   224  
   225  func WaitForAllocStopped(t *testing.T, nomadClient *api.Client, allocID string) {
   226  	testutil.WaitForResultRetries(retries, func() (bool, error) {
   227  		time.Sleep(time.Millisecond * 100)
   228  		alloc, _, err := nomadClient.Allocations().Info(allocID, nil)
   229  		if err != nil {
   230  			return false, err
   231  		}
   232  		switch alloc.ClientStatus {
   233  		case structs.AllocClientStatusComplete:
   234  			return true, nil
   235  		case structs.AllocClientStatusFailed:
   236  			return true, nil
   237  		case structs.AllocClientStatusLost:
   238  			return true, nil
   239  		default:
   240  			return false, fmt.Errorf("expected stopped alloc, but was: %s",
   241  				alloc.ClientStatus)
   242  		}
   243  	}, func(err error) {
   244  		require.NoError(t, err, "failed to wait on alloc")
   245  	})
   246  }
   247  
   248  func WaitForAllocStatus(t *testing.T, nomadClient *api.Client, allocID string, status string) {
   249  	testutil.WaitForResultRetries(retries, func() (bool, error) {
   250  		time.Sleep(time.Millisecond * 100)
   251  		alloc, _, err := nomadClient.Allocations().Info(allocID, nil)
   252  		if err != nil {
   253  			return false, err
   254  		}
   255  		switch alloc.ClientStatus {
   256  		case status:
   257  			return true, nil
   258  		default:
   259  			return false, fmt.Errorf("expected %s alloc, but was: %s", status, alloc.ClientStatus)
   260  		}
   261  	}, func(err error) {
   262  		t.Fatalf("failed to wait on alloc: %v", err)
   263  	})
   264  }
   265  
   266  func WaitForAllocsStatus(t *testing.T, nomadClient *api.Client, allocIDs []string, status string) {
   267  	for _, allocID := range allocIDs {
   268  		WaitForAllocStatus(t, nomadClient, allocID, status)
   269  	}
   270  }
   271  
   272  func AllocIDsFromAllocationListStubs(allocs []*api.AllocationListStub) []string {
   273  	allocIDs := make([]string, 0, len(allocs))
   274  	for _, alloc := range allocs {
   275  		allocIDs = append(allocIDs, alloc.ID)
   276  	}
   277  	return allocIDs
   278  }
   279  
   280  func DeploymentsForJob(t *testing.T, nomadClient *api.Client, jobID string) []*api.Deployment {
   281  	ds, _, err := nomadClient.Deployments().List(nil)
   282  	require.NoError(t, err)
   283  
   284  	out := []*api.Deployment{}
   285  	for _, d := range ds {
   286  		if d.JobID == jobID {
   287  			out = append(out, d)
   288  		}
   289  	}
   290  
   291  	return out
   292  }
   293  
   294  func WaitForDeployment(t *testing.T, nomadClient *api.Client, deployID string, status string, statusDesc string) {
   295  	testutil.WaitForResultRetries(retries, func() (bool, error) {
   296  		time.Sleep(time.Millisecond * 100)
   297  		deploy, _, err := nomadClient.Deployments().Info(deployID, nil)
   298  		if err != nil {
   299  			return false, err
   300  		}
   301  
   302  		if deploy.Status == status && deploy.StatusDescription == statusDesc {
   303  			return true, nil
   304  		}
   305  		return false, fmt.Errorf("expected status %s \"%s\", but got: %s \"%s\"",
   306  			status,
   307  			statusDesc,
   308  			deploy.Status,
   309  			deploy.StatusDescription,
   310  		)
   311  
   312  	}, func(err error) {
   313  		require.NoError(t, err, "failed to wait on deployment")
   314  	})
   315  }
   316  
   317  // DumpEvals for a job. This is intended to be used during test development or
   318  // prior to exiting a test after an assertion failed.
   319  func DumpEvals(c *api.Client, jobID string) string {
   320  	evals, _, err := c.Jobs().Evaluations(jobID, nil)
   321  	if err != nil {
   322  		return fmt.Sprintf("error retrieving evals for job %q: %s", jobID, err)
   323  	}
   324  	if len(evals) == 0 {
   325  		return fmt.Sprintf("no evals found for job %q", jobID)
   326  	}
   327  	buf := bytes.NewBuffer(nil)
   328  	for i, e := range evals {
   329  		err := EvalTemplate.Execute(buf, map[string]interface{}{
   330  			"Index": i + 1,
   331  			"Total": len(evals),
   332  			"Eval":  e,
   333  		})
   334  		if err != nil {
   335  			fmt.Fprintf(buf, "error rendering eval: %s\n", err)
   336  		}
   337  	}
   338  	return buf.String()
   339  }
   340  
   341  var EvalTemplate = template.Must(template.New("dump_eval").Parse(
   342  	`{{.Index}}/{{.Total}} Job {{.Eval.JobID}} Eval {{.Eval.ID}}
   343    Type:         {{.Eval.Type}}
   344    TriggeredBy:  {{.Eval.TriggeredBy}}
   345    {{- if .Eval.DeploymentID}}
   346    Deployment:   {{.Eval.DeploymentID}}
   347    {{- end}}
   348    Status:       {{.Eval.Status}} {{if .Eval.StatusDescription}}({{.Eval.StatusDescription}}){{end}}
   349    {{- if .Eval.Wait}}
   350    Wait:         {{.Eval.Wait}} <- DEPRECATED
   351    {{- end}}
   352    {{- if not .Eval.WaitUntil.IsZero}}
   353    WaitUntil:    {{.Eval.WaitUntil}}
   354    {{- end}}
   355    {{- if .Eval.NextEval}}
   356    NextEval:     {{.Eval.NextEval}}
   357    {{- end}}
   358    {{- if .Eval.PreviousEval}}
   359    PrevEval:     {{.Eval.PreviousEval}}
   360    {{- end}}
   361    {{- if .Eval.BlockedEval}}
   362    BlockedEval:  {{.Eval.BlockedEval}}
   363    {{- end}}
   364    {{- if .Eval.FailedTGAllocs }}
   365    Failed Allocs:
   366    {{- end}}
   367    {{- range $k, $v := .Eval.FailedTGAllocs}}
   368      Failed Group: {{$k}}
   369        NodesEvaluated: {{$v.NodesEvaluated}}
   370        NodesFiltered:  {{$v.NodesFiltered}}
   371        NodesAvailable: {{range $dc, $n := $v.NodesAvailable}}{{$dc}}:{{$n}} {{end}}
   372        NodesExhausted: {{$v.NodesExhausted}}
   373        ClassFiltered:  {{len $v.ClassFiltered}}
   374        ConstraintFilt: {{len $v.ConstraintFiltered}}
   375        DimensionExhst: {{range $d, $n := $v.DimensionExhausted}}{{$d}}:{{$n}} {{end}}
   376        ResourcesExhst: {{range $r, $n := $v.ResourcesExhausted}}{{$r}}:{{$n}} {{end}}
   377        QuotaExhausted: {{range $i, $q := $v.QuotaExhausted}}{{$q}} {{end}}
   378        CoalescedFail:  {{$v.CoalescedFailures}}
   379        ScoreMetaData:  {{len $v.ScoreMetaData}}
   380        AllocationTime: {{$v.AllocationTime}}
   381    {{- end}}
   382    {{- if .Eval.QueuedAllocations}}
   383    QueuedAllocs: {{range $k, $n := .Eval.QueuedAllocations}}{{$k}}:{{$n}} {{end}}
   384    {{- end}}
   385    SnapshotIdx:  {{.Eval.SnapshotIndex}}
   386    CreateIndex:  {{.Eval.CreateIndex}}
   387    ModifyIndex:  {{.Eval.ModifyIndex}}
   388  `))