github.com/anth0d/nomad@v0.0.0-20221214183521-ae3a0a2cad06/e2e/spread/spread_test.go (about)

     1  package spread
     2  
     3  import (
     4  	"fmt"
     5  	"strings"
     6  	"testing"
     7  	"time"
     8  
     9  	"github.com/hashicorp/nomad/api"
    10  	"github.com/hashicorp/nomad/e2e/e2eutil"
    11  	"github.com/hashicorp/nomad/helper/uuid"
    12  	"github.com/shoenig/test/must"
    13  	"github.com/stretchr/testify/assert"
    14  )
    15  
    16  const (
    17  	evenJobFilePath     = "./input/even_spread.nomad"
    18  	multipleJobFilePath = "./input/multiple_spread.nomad"
    19  )
    20  
    21  func TestSpread(t *testing.T) {
    22  
    23  	nomadClient := e2eutil.NomadClient(t)
    24  	e2eutil.WaitForLeader(t, nomadClient)
    25  	e2eutil.WaitForNodesReady(t, nomadClient, 4)
    26  
    27  	// Run our test cases.
    28  	t.Run("TestSpread_Even", testSpreadEven)
    29  	t.Run("TestSpread_Multiple", testSpreadMultiple)
    30  }
    31  
    32  func testSpreadEven(t *testing.T) {
    33  
    34  	nomadClient := e2eutil.NomadClient(t)
    35  
    36  	// Generate a job ID and register the test job.
    37  	jobID := "spread-" + uuid.Short()
    38  	allocs := e2eutil.RegisterAndWaitForAllocs(t, nomadClient, evenJobFilePath, jobID, "")
    39  
    40  	// Ensure the test cleans its own job and allocations fully, so it does not
    41  	// impact other spread tests.
    42  	t.Cleanup(func() { cleanupJob(t, nomadClient, jobID, allocs) })
    43  
    44  	dcToAllocs := make(map[string]int)
    45  
    46  	for _, allocStub := range allocs {
    47  		alloc, _, err := nomadClient.Allocations().Info(allocStub.ID, nil)
    48  		must.NoError(t, err)
    49  		must.Greater(t, 0, len(alloc.Metrics.ScoreMetaData))
    50  		node, _, err := nomadClient.Nodes().Info(alloc.NodeID, nil)
    51  		must.NoError(t, err)
    52  		dcToAllocs[node.Datacenter]++
    53  	}
    54  
    55  	must.Eq(t, map[string]int{"dc1": 3, "dc2": 3}, dcToAllocs)
    56  }
    57  
    58  func testSpreadMultiple(t *testing.T) {
    59  
    60  	nomadClient := e2eutil.NomadClient(t)
    61  
    62  	// Generate a job ID and register the test job.
    63  	jobID := "spread-" + uuid.Short()
    64  	allocs := e2eutil.RegisterAndWaitForAllocs(t, nomadClient, multipleJobFilePath, jobID, "")
    65  
    66  	// Ensure the test cleans its own job and allocations fully, so it does not
    67  	// impact other spread tests.
    68  	t.Cleanup(func() { cleanupJob(t, nomadClient, jobID, allocs) })
    69  
    70  	// Verify spread score and alloc distribution
    71  	dcToAllocs := make(map[string]int)
    72  	rackToAllocs := make(map[string]int)
    73  	allocMetrics := make(map[string]*api.AllocationMetric)
    74  
    75  	for _, allocStub := range allocs {
    76  		alloc, _, err := nomadClient.Allocations().Info(allocStub.ID, nil)
    77  		must.NoError(t, err)
    78  		must.Greater(t, 0, len(alloc.Metrics.ScoreMetaData))
    79  		allocMetrics[allocStub.ID] = alloc.Metrics
    80  
    81  		node, _, err := nomadClient.Nodes().Info(alloc.NodeID, nil)
    82  		must.NoError(t, err)
    83  		dcToAllocs[node.Datacenter]++
    84  
    85  		if rack := node.Meta["rack"]; rack != "" {
    86  			rackToAllocs[rack]++
    87  		}
    88  	}
    89  
    90  	failureReport := report(allocMetrics)
    91  
    92  	must.Eq(t, map[string]int{"dc1": 5, "dc2": 5}, dcToAllocs, failureReport)
    93  	must.Eq(t, map[string]int{"r1": 7, "r2": 3}, rackToAllocs, failureReport)
    94  }
    95  
    96  func cleanupJob(t *testing.T, nomadClient *api.Client, jobID string, allocs []*api.AllocationListStub) {
    97  
    98  	_, _, err := nomadClient.Jobs().Deregister(jobID, true, nil)
    99  	assert.NoError(t, err)
   100  
   101  	// Ensure that all allocations have been removed from state. This is an
   102  	// important aspect of the cleaning required which allows the spread
   103  	// test to run successfully.
   104  	assert.Eventually(t, func() bool {
   105  
   106  		// Run the garbage collector to remove all terminal allocations.
   107  		must.NoError(t, nomadClient.System().GarbageCollect())
   108  
   109  		for _, allocStub := range allocs {
   110  			_, _, err := nomadClient.Allocations().Info(allocStub.ID, nil)
   111  			if err == nil {
   112  				return false
   113  			} else {
   114  				if !strings.Contains(err.Error(), "alloc not found") {
   115  					return false
   116  				}
   117  			}
   118  		}
   119  		return true
   120  	}, 10*time.Second, 200*time.Millisecond)
   121  }
   122  
   123  func report(metrics map[string]*api.AllocationMetric) must.PostScript {
   124  	var s strings.Builder
   125  	for allocID, m := range metrics {
   126  		s.WriteString("Alloc ID: " + allocID + "\n")
   127  		s.WriteString(fmt.Sprintf("  NodesEvaluated: %d\n", m.NodesEvaluated))
   128  		s.WriteString(fmt.Sprintf("  NodesAvailable: %#v\n", m.NodesAvailable))
   129  		s.WriteString(fmt.Sprintf("  ClassFiltered: %#v\n", m.ClassFiltered))
   130  		s.WriteString(fmt.Sprintf("  ConstraintFiltered: %#v\n", m.ConstraintFiltered))
   131  		s.WriteString(fmt.Sprintf("  NodesExhausted: %d\n", m.NodesExhausted))
   132  		s.WriteString(fmt.Sprintf("  ClassExhausted: %#v\n", m.ClassExhausted))
   133  		s.WriteString(fmt.Sprintf("  DimensionExhausted: %#v\n", m.DimensionExhausted))
   134  		s.WriteString(fmt.Sprintf("  QuotaExhausted: %#v\n", m.QuotaExhausted))
   135  		for _, nodeMeta := range m.ScoreMetaData {
   136  			s.WriteString(fmt.Sprintf("    NodeID: %s, NormScore: %f, Scores: %#v\n",
   137  				nodeMeta.NodeID, nodeMeta.NormScore, nodeMeta.Scores))
   138  		}
   139  	}
   140  	return must.Sprint(s.String())
   141  }