github.com/anth0d/nomad@v0.0.0-20221214183521-ae3a0a2cad06/e2e/overlap/overlap_test.go (about)

     1  package overlap
     2  
     3  import (
     4  	"fmt"
     5  	"testing"
     6  	"time"
     7  
     8  	"github.com/hashicorp/nomad/api"
     9  	"github.com/hashicorp/nomad/e2e/e2eutil"
    10  	"github.com/hashicorp/nomad/helper/uuid"
    11  	"github.com/hashicorp/nomad/testutil"
    12  	"github.com/shoenig/test/must"
    13  )
    14  
    15  // TestOverlap asserts that the resources used by an allocation are not
    16  // considered free until their ClientStatus is terminal.
    17  //
    18  // See: https://github.com/hashicorp/nomad/issues/10440
    19  func TestOverlap(t *testing.T) {
    20  	nomadClient := e2eutil.NomadClient(t)
    21  	e2eutil.WaitForLeader(t, nomadClient)
    22  
    23  	getJob := func() (*api.Job, string) {
    24  		job, err := e2eutil.Parse2(t, "testdata/overlap.nomad")
    25  		must.NoError(t, err)
    26  		jobID := *job.ID + uuid.Short()
    27  		job.ID = &jobID
    28  		return job, *job.ID
    29  	}
    30  	job1, jobID1 := getJob()
    31  
    32  	// Register initial job that should block subsequent job's placement until
    33  	// its shutdown_delay is up.
    34  	_, _, err := nomadClient.Jobs().Register(job1, nil)
    35  	must.NoError(t, err)
    36  	defer e2eutil.WaitForJobStopped(t, nomadClient, jobID1)
    37  
    38  	var origAlloc *api.AllocationListStub
    39  	testutil.Wait(t, func() (bool, error) {
    40  		time.Sleep(500 * time.Millisecond)
    41  
    42  		a, _, err := nomadClient.Jobs().Allocations(jobID1, false, nil)
    43  		must.NoError(t, err)
    44  		if n := len(a); n == 0 {
    45  			evalOut := e2eutil.DumpEvals(nomadClient, jobID1)
    46  			return false, fmt.Errorf("timed out before an allocation was found for %s. Evals:\n%s", jobID1, evalOut)
    47  		}
    48  		must.Len(t, 1, a)
    49  
    50  		origAlloc = a[0]
    51  		return origAlloc.ClientStatus == "running", fmt.Errorf("timed out before alloc %s for %s was running: %s",
    52  			origAlloc.ID, jobID1, origAlloc.ClientStatus)
    53  	})
    54  
    55  	// Stop job but don't wait for ClientStatus terminal
    56  	_, _, err = nomadClient.Jobs().Deregister(jobID1, false, nil)
    57  	must.NoError(t, err)
    58  	minStopTime := time.Now().Add(job1.TaskGroups[0].Tasks[0].ShutdownDelay)
    59  
    60  	testutil.Wait(t, func() (bool, error) {
    61  		a, _, err := nomadClient.Allocations().Info(origAlloc.ID, nil)
    62  		must.NoError(t, err)
    63  		ds, cs := a.DesiredStatus, a.ClientStatus
    64  		return ds == "stop" && cs == "running", fmt.Errorf("expected alloc %s to be stop|running but found %s|%s",
    65  			a.ID, ds, cs)
    66  	})
    67  
    68  	// Start replacement job on same node and assert it is blocked because the
    69  	// static port is already in use.
    70  	job2, jobID2 := getJob()
    71  	job2.Constraints = append(job2.Constraints, api.NewConstraint("${node.unique.id}", "=", origAlloc.NodeID))
    72  	job2.TaskGroups[0].Tasks[0].ShutdownDelay = 0 // no need on the followup
    73  
    74  	resp, _, err := nomadClient.Jobs().Register(job2, nil)
    75  	must.NoError(t, err)
    76  	defer e2eutil.WaitForJobStopped(t, nomadClient, jobID2)
    77  
    78  	testutil.Wait(t, func() (bool, error) {
    79  		e, _, err := nomadClient.Evaluations().Info(resp.EvalID, nil)
    80  		must.NoError(t, err)
    81  		if e == nil {
    82  			return false, fmt.Errorf("eval %s does not exist yet", resp.EvalID)
    83  		}
    84  		return e.BlockedEval != "", fmt.Errorf("expected a blocked eval to be created but found: %#v", *e)
    85  	})
    86  
    87  	// Wait for job1's ShutdownDelay for origAlloc.ClientStatus to go terminal
    88  	sleepyTime := minStopTime.Sub(time.Now())
    89  	if sleepyTime > 0 {
    90  		t.Logf("Followup job %s blocked. Sleeping for the rest of %s's shutdown_delay (%.3s/%s)",
    91  			*job2.ID, *job1.ID, sleepyTime, job1.TaskGroups[0].Tasks[0].ShutdownDelay)
    92  		time.Sleep(sleepyTime)
    93  	}
    94  
    95  	testutil.Wait(t, func() (bool, error) {
    96  		a, _, err := nomadClient.Allocations().Info(origAlloc.ID, nil)
    97  		must.NoError(t, err)
    98  		return a.ClientStatus == "complete", fmt.Errorf("expected original alloc %s to be complete but is %s",
    99  			a.ID, a.ClientStatus)
   100  	})
   101  
   102  	// Assert replacement job unblocked and running
   103  	testutil.Wait(t, func() (bool, error) {
   104  		time.Sleep(500 * time.Millisecond)
   105  
   106  		a, _, err := nomadClient.Jobs().Allocations(jobID2, true, nil)
   107  		must.NoError(t, err)
   108  		if n := len(a); n == 0 {
   109  			evalOut := e2eutil.DumpEvals(nomadClient, jobID2)
   110  			return false, fmt.Errorf("timed out before an allocation was found for %s; Evals:\n%s", jobID2, evalOut)
   111  		}
   112  		must.Len(t, 1, a)
   113  
   114  		return a[0].ClientStatus == "running", fmt.Errorf("timed out before alloc %s for %s was running: %s",
   115  			a[0].ID, jobID2, a[0].ClientStatus)
   116  	})
   117  }