github.com/hernad/nomad@v1.6.112/e2e/overlap/overlap_test.go (about)

     1  // Copyright (c) HashiCorp, Inc.
     2  // SPDX-License-Identifier: MPL-2.0
     3  
     4  package overlap
     5  
     6  import (
     7  	"fmt"
     8  	"testing"
     9  	"time"
    10  
    11  	"github.com/hernad/nomad/api"
    12  	"github.com/hernad/nomad/e2e/e2eutil"
    13  	"github.com/hernad/nomad/helper/uuid"
    14  	"github.com/hernad/nomad/testutil"
    15  	"github.com/shoenig/test/must"
    16  )
    17  
    18  // TestOverlap asserts that the resources used by an allocation are not
    19  // considered free until their ClientStatus is terminal.
    20  //
    21  // See: https://github.com/hernad/nomad/issues/10440
    22  func TestOverlap(t *testing.T) {
    23  	nomadClient := e2eutil.NomadClient(t)
    24  	e2eutil.WaitForLeader(t, nomadClient)
    25  
    26  	getJob := func() (*api.Job, string) {
    27  		job, err := e2eutil.Parse2(t, "testdata/overlap.nomad")
    28  		must.NoError(t, err)
    29  		jobID := *job.ID + uuid.Short()
    30  		job.ID = &jobID
    31  		return job, *job.ID
    32  	}
    33  	job1, jobID1 := getJob()
    34  
    35  	// Register initial job that should block subsequent job's placement until
    36  	// its shutdown_delay is up.
    37  	_, _, err := nomadClient.Jobs().Register(job1, nil)
    38  	must.NoError(t, err)
    39  	defer e2eutil.WaitForJobStopped(t, nomadClient, jobID1)
    40  
    41  	var origAlloc *api.AllocationListStub
    42  	testutil.Wait(t, func() (bool, error) {
    43  		time.Sleep(500 * time.Millisecond)
    44  
    45  		a, _, err := nomadClient.Jobs().Allocations(jobID1, false, nil)
    46  		must.NoError(t, err)
    47  		if n := len(a); n == 0 {
    48  			evalOut := e2eutil.DumpEvals(nomadClient, jobID1)
    49  			return false, fmt.Errorf("timed out before an allocation was found for %s. Evals:\n%s", jobID1, evalOut)
    50  		}
    51  		must.Len(t, 1, a)
    52  
    53  		origAlloc = a[0]
    54  		return origAlloc.ClientStatus == "running", fmt.Errorf("timed out before alloc %s for %s was running: %s",
    55  			origAlloc.ID, jobID1, origAlloc.ClientStatus)
    56  	})
    57  
    58  	// Stop job but don't wait for ClientStatus terminal
    59  	_, _, err = nomadClient.Jobs().Deregister(jobID1, false, nil)
    60  	must.NoError(t, err)
    61  	minStopTime := time.Now().Add(job1.TaskGroups[0].Tasks[0].ShutdownDelay)
    62  
    63  	testutil.Wait(t, func() (bool, error) {
    64  		a, _, err := nomadClient.Allocations().Info(origAlloc.ID, nil)
    65  		must.NoError(t, err)
    66  		ds, cs := a.DesiredStatus, a.ClientStatus
    67  		return ds == "stop" && cs == "running", fmt.Errorf("expected alloc %s to be stop|running but found %s|%s",
    68  			a.ID, ds, cs)
    69  	})
    70  
    71  	// Start replacement job on same node and assert it is blocked because the
    72  	// static port is already in use.
    73  	job2, jobID2 := getJob()
    74  	job2.Constraints = append(job2.Constraints, api.NewConstraint("${node.unique.id}", "=", origAlloc.NodeID))
    75  	job2.TaskGroups[0].Tasks[0].ShutdownDelay = 0 // no need on the followup
    76  
    77  	resp, _, err := nomadClient.Jobs().Register(job2, nil)
    78  	must.NoError(t, err)
    79  	defer e2eutil.WaitForJobStopped(t, nomadClient, jobID2)
    80  
    81  	testutil.Wait(t, func() (bool, error) {
    82  		e, _, err := nomadClient.Evaluations().Info(resp.EvalID, nil)
    83  		must.NoError(t, err)
    84  		if e == nil {
    85  			return false, fmt.Errorf("eval %s does not exist yet", resp.EvalID)
    86  		}
    87  		return e.BlockedEval != "", fmt.Errorf("expected a blocked eval to be created but found: %#v", *e)
    88  	})
    89  
    90  	// Wait for job1's ShutdownDelay for origAlloc.ClientStatus to go terminal
    91  	sleepyTime := minStopTime.Sub(time.Now())
    92  	if sleepyTime > 0 {
    93  		t.Logf("Followup job %s blocked. Sleeping for the rest of %s's shutdown_delay (%.3s/%s)",
    94  			*job2.ID, *job1.ID, sleepyTime, job1.TaskGroups[0].Tasks[0].ShutdownDelay)
    95  		time.Sleep(sleepyTime)
    96  	}
    97  
    98  	testutil.Wait(t, func() (bool, error) {
    99  		a, _, err := nomadClient.Allocations().Info(origAlloc.ID, nil)
   100  		must.NoError(t, err)
   101  		return a.ClientStatus == "complete", fmt.Errorf("expected original alloc %s to be complete but is %s",
   102  			a.ID, a.ClientStatus)
   103  	})
   104  
   105  	// Assert replacement job unblocked and running
   106  	testutil.Wait(t, func() (bool, error) {
   107  		time.Sleep(500 * time.Millisecond)
   108  
   109  		a, _, err := nomadClient.Jobs().Allocations(jobID2, true, nil)
   110  		must.NoError(t, err)
   111  		if n := len(a); n == 0 {
   112  			evalOut := e2eutil.DumpEvals(nomadClient, jobID2)
   113  			return false, fmt.Errorf("timed out before an allocation was found for %s; Evals:\n%s", jobID2, evalOut)
   114  		}
   115  		must.Len(t, 1, a)
   116  
   117  		return a[0].ClientStatus == "running", fmt.Errorf("timed out before alloc %s for %s was running: %s",
   118  			a[0].ID, jobID2, a[0].ClientStatus)
   119  	})
   120  }