github.com/anth0d/nomad@v0.0.0-20221214183521-ae3a0a2cad06/e2e/overlap/overlap_test.go (about) 1 package overlap 2 3 import ( 4 "fmt" 5 "testing" 6 "time" 7 8 "github.com/hashicorp/nomad/api" 9 "github.com/hashicorp/nomad/e2e/e2eutil" 10 "github.com/hashicorp/nomad/helper/uuid" 11 "github.com/hashicorp/nomad/testutil" 12 "github.com/shoenig/test/must" 13 ) 14 15 // TestOverlap asserts that the resources used by an allocation are not 16 // considered free until their ClientStatus is terminal. 17 // 18 // See: https://github.com/hashicorp/nomad/issues/10440 19 func TestOverlap(t *testing.T) { 20 nomadClient := e2eutil.NomadClient(t) 21 e2eutil.WaitForLeader(t, nomadClient) 22 23 getJob := func() (*api.Job, string) { 24 job, err := e2eutil.Parse2(t, "testdata/overlap.nomad") 25 must.NoError(t, err) 26 jobID := *job.ID + uuid.Short() 27 job.ID = &jobID 28 return job, *job.ID 29 } 30 job1, jobID1 := getJob() 31 32 // Register initial job that should block subsequent job's placement until 33 // its shutdown_delay is up. 34 _, _, err := nomadClient.Jobs().Register(job1, nil) 35 must.NoError(t, err) 36 defer e2eutil.WaitForJobStopped(t, nomadClient, jobID1) 37 38 var origAlloc *api.AllocationListStub 39 testutil.Wait(t, func() (bool, error) { 40 time.Sleep(500 * time.Millisecond) 41 42 a, _, err := nomadClient.Jobs().Allocations(jobID1, false, nil) 43 must.NoError(t, err) 44 if n := len(a); n == 0 { 45 evalOut := e2eutil.DumpEvals(nomadClient, jobID1) 46 return false, fmt.Errorf("timed out before an allocation was found for %s. Evals:\n%s", jobID1, evalOut) 47 } 48 must.Len(t, 1, a) 49 50 origAlloc = a[0] 51 return origAlloc.ClientStatus == "running", fmt.Errorf("timed out before alloc %s for %s was running: %s", 52 origAlloc.ID, jobID1, origAlloc.ClientStatus) 53 }) 54 55 // Stop job but don't wait for ClientStatus terminal 56 _, _, err = nomadClient.Jobs().Deregister(jobID1, false, nil) 57 must.NoError(t, err) 58 minStopTime := time.Now().Add(job1.TaskGroups[0].Tasks[0].ShutdownDelay) 59 60 testutil.Wait(t, func() (bool, error) { 61 a, _, err := nomadClient.Allocations().Info(origAlloc.ID, nil) 62 must.NoError(t, err) 63 ds, cs := a.DesiredStatus, a.ClientStatus 64 return ds == "stop" && cs == "running", fmt.Errorf("expected alloc %s to be stop|running but found %s|%s", 65 a.ID, ds, cs) 66 }) 67 68 // Start replacement job on same node and assert it is blocked because the 69 // static port is already in use. 70 job2, jobID2 := getJob() 71 job2.Constraints = append(job2.Constraints, api.NewConstraint("${node.unique.id}", "=", origAlloc.NodeID)) 72 job2.TaskGroups[0].Tasks[0].ShutdownDelay = 0 // no need on the followup 73 74 resp, _, err := nomadClient.Jobs().Register(job2, nil) 75 must.NoError(t, err) 76 defer e2eutil.WaitForJobStopped(t, nomadClient, jobID2) 77 78 testutil.Wait(t, func() (bool, error) { 79 e, _, err := nomadClient.Evaluations().Info(resp.EvalID, nil) 80 must.NoError(t, err) 81 if e == nil { 82 return false, fmt.Errorf("eval %s does not exist yet", resp.EvalID) 83 } 84 return e.BlockedEval != "", fmt.Errorf("expected a blocked eval to be created but found: %#v", *e) 85 }) 86 87 // Wait for job1's ShutdownDelay for origAlloc.ClientStatus to go terminal 88 sleepyTime := minStopTime.Sub(time.Now()) 89 if sleepyTime > 0 { 90 t.Logf("Followup job %s blocked. Sleeping for the rest of %s's shutdown_delay (%.3s/%s)", 91 *job2.ID, *job1.ID, sleepyTime, job1.TaskGroups[0].Tasks[0].ShutdownDelay) 92 time.Sleep(sleepyTime) 93 } 94 95 testutil.Wait(t, func() (bool, error) { 96 a, _, err := nomadClient.Allocations().Info(origAlloc.ID, nil) 97 must.NoError(t, err) 98 return a.ClientStatus == "complete", fmt.Errorf("expected original alloc %s to be complete but is %s", 99 a.ID, a.ClientStatus) 100 }) 101 102 // Assert replacement job unblocked and running 103 testutil.Wait(t, func() (bool, error) { 104 time.Sleep(500 * time.Millisecond) 105 106 a, _, err := nomadClient.Jobs().Allocations(jobID2, true, nil) 107 must.NoError(t, err) 108 if n := len(a); n == 0 { 109 evalOut := e2eutil.DumpEvals(nomadClient, jobID2) 110 return false, fmt.Errorf("timed out before an allocation was found for %s; Evals:\n%s", jobID2, evalOut) 111 } 112 must.Len(t, 1, a) 113 114 return a[0].ClientStatus == "running", fmt.Errorf("timed out before alloc %s for %s was running: %s", 115 a[0].ID, jobID2, a[0].ClientStatus) 116 }) 117 }