github.com/iqoqo/nomad@v0.11.3-0.20200911112621-d7021c74d101/e2e/systemsched/systemsched.go (about)

     1  package systemsched
     2  
     3  import (
     4  	"github.com/hashicorp/nomad/api"
     5  	"github.com/hashicorp/nomad/e2e/e2eutil"
     6  	"github.com/hashicorp/nomad/e2e/framework"
     7  	"github.com/hashicorp/nomad/nomad/structs"
     8  	"github.com/stretchr/testify/require"
     9  )
    10  
    11  type SystemSchedTest struct {
    12  	framework.TC
    13  	jobIDs []string
    14  }
    15  
    16  func init() {
    17  	framework.AddSuites(&framework.TestSuite{
    18  		Component:   "SystemScheduler",
    19  		CanRunLocal: true,
    20  		Cases: []framework.TestCase{
    21  			new(SystemSchedTest),
    22  		},
    23  	})
    24  }
    25  
    26  func (tc *SystemSchedTest) BeforeAll(f *framework.F) {
    27  	// Ensure cluster has leader before running tests
    28  	e2eutil.WaitForLeader(f.T(), tc.Nomad())
    29  	e2eutil.WaitForNodesReady(f.T(), tc.Nomad(), 4)
    30  }
    31  
    32  func (tc *SystemSchedTest) TestJobUpdateOnIneligbleNode(f *framework.F) {
    33  	t := f.T()
    34  	nomadClient := tc.Nomad()
    35  
    36  	jobID := "system_deployment"
    37  	tc.jobIDs = append(tc.jobIDs, jobID)
    38  	e2eutil.RegisterAndWaitForAllocs(t, nomadClient, "systemsched/input/system_job0.nomad", jobID, "")
    39  
    40  	jobs := nomadClient.Jobs()
    41  	allocs, _, err := jobs.Allocations(jobID, true, nil)
    42  	require.NoError(t, err)
    43  
    44  	var allocIDs []string
    45  	for _, alloc := range allocs {
    46  		allocIDs = append(allocIDs, alloc.ID)
    47  	}
    48  
    49  	// Wait for allocations to get past initial pending state
    50  	e2eutil.WaitForAllocsNotPending(t, nomadClient, allocIDs)
    51  
    52  	// Mark one node as ineligible
    53  	nodesAPI := tc.Nomad().Nodes()
    54  	disabledNodeID := allocs[0].NodeID
    55  	_, err = nodesAPI.ToggleEligibility(disabledNodeID, false, nil)
    56  	require.NoError(t, err)
    57  
    58  	// Assert all jobs still running
    59  	jobs = nomadClient.Jobs()
    60  	allocs, _, err = jobs.Allocations(jobID, true, nil)
    61  
    62  	allocIDs = nil
    63  	for _, alloc := range allocs {
    64  		allocIDs = append(allocIDs, alloc.ID)
    65  	}
    66  
    67  	require.NoError(t, err)
    68  	allocForDisabledNode := make(map[string]*api.AllocationListStub)
    69  
    70  	// Wait for allocs to run and collect allocs on ineligible node
    71  	// Allocation could have failed, ensure there is one thats running
    72  	// and that it is the correct version (0)
    73  	e2eutil.WaitForAllocsNotPending(t, nomadClient, allocIDs)
    74  	for _, alloc := range allocs {
    75  		if alloc.NodeID == disabledNodeID {
    76  			allocForDisabledNode[alloc.ID] = alloc
    77  		}
    78  	}
    79  
    80  	// Filter down to only our latest running alloc
    81  	for _, alloc := range allocForDisabledNode {
    82  		require.Equal(t, uint64(0), alloc.JobVersion)
    83  		if alloc.ClientStatus == structs.AllocClientStatusComplete {
    84  			// remove the old complete alloc from map
    85  			delete(allocForDisabledNode, alloc.ID)
    86  		}
    87  	}
    88  	require.NotEmpty(t, allocForDisabledNode)
    89  	require.Len(t, allocForDisabledNode, 1)
    90  
    91  	// Update job
    92  	e2eutil.RegisterAndWaitForAllocs(t, nomadClient, "systemsched/input/system_job1.nomad", jobID, "")
    93  
    94  	// Get updated allocations
    95  	jobs = nomadClient.Jobs()
    96  	allocs, _, err = jobs.Allocations(jobID, false, nil)
    97  	require.NoError(t, err)
    98  
    99  	allocIDs = nil
   100  	for _, alloc := range allocs {
   101  		allocIDs = append(allocIDs, alloc.ID)
   102  	}
   103  
   104  	// Wait for allocs to start
   105  	e2eutil.WaitForAllocsNotPending(t, nomadClient, allocIDs)
   106  
   107  	// Get latest alloc status now that they are no longer pending
   108  	allocs, _, err = jobs.Allocations(jobID, false, nil)
   109  	require.NoError(t, err)
   110  
   111  	var foundPreviousAlloc bool
   112  	for _, dAlloc := range allocForDisabledNode {
   113  		for _, alloc := range allocs {
   114  			if alloc.ID == dAlloc.ID {
   115  				foundPreviousAlloc = true
   116  				require.Equal(t, uint64(0), alloc.JobVersion)
   117  			} else {
   118  				// Ensure allocs running on non disabled node are
   119  				// newer version
   120  				if alloc.ClientStatus == structs.AllocClientStatusRunning {
   121  					require.Equal(t, uint64(1), alloc.JobVersion)
   122  				}
   123  			}
   124  		}
   125  	}
   126  	require.True(t, foundPreviousAlloc, "unable to find previous alloc for ineligible node")
   127  }
   128  
   129  func (tc *SystemSchedTest) AfterEach(f *framework.F) {
   130  	nomadClient := tc.Nomad()
   131  
   132  	// Mark all nodes eligible
   133  	nodesAPI := tc.Nomad().Nodes()
   134  	nodes, _, _ := nodesAPI.List(nil)
   135  	for _, node := range nodes {
   136  		nodesAPI.ToggleEligibility(node.ID, true, nil)
   137  	}
   138  
   139  	jobs := nomadClient.Jobs()
   140  	// Stop all jobs in test
   141  	for _, id := range tc.jobIDs {
   142  		jobs.Deregister(id, true, nil)
   143  	}
   144  	tc.jobIDs = []string{}
   145  	// Garbage collect
   146  	nomadClient.System().GarbageCollect()
   147  }