github.com/adityamillind98/nomad@v0.11.8/command/agent/metrics_endpoint_test.go (about)

     1  package agent
     2  
     3  import (
     4  	"net/http"
     5  	"net/http/httptest"
     6  	"strings"
     7  	"testing"
     8  	"time"
     9  
    10  	"github.com/armon/go-metrics"
    11  	"github.com/hashicorp/nomad/nomad/mock"
    12  	"github.com/hashicorp/nomad/nomad/structs"
    13  	"github.com/hashicorp/nomad/testutil"
    14  	"github.com/stretchr/testify/assert"
    15  	"github.com/stretchr/testify/require"
    16  )
    17  
    18  func TestHTTP_MetricsWithIllegalMethod(t *testing.T) {
    19  	assert := assert.New(t)
    20  
    21  	t.Parallel()
    22  	httpTest(t, nil, func(s *TestAgent) {
    23  		req, err := http.NewRequest("DELETE", "/v1/metrics", nil)
    24  		assert.Nil(err)
    25  		respW := httptest.NewRecorder()
    26  
    27  		_, err = s.Server.MetricsRequest(respW, req)
    28  		assert.NotNil(err, "HTTP DELETE should not be accepted for this endpoint")
    29  	})
    30  }
    31  
    32  func TestHTTP_Metrics(t *testing.T) {
    33  	assert := assert.New(t)
    34  
    35  	t.Parallel()
    36  	httpTest(t, nil, func(s *TestAgent) {
    37  		// make a separate HTTP request first, to ensure Nomad has written metrics
    38  		// and prevent a race condition
    39  		req, err := http.NewRequest("GET", "/v1/agent/self", nil)
    40  		assert.Nil(err)
    41  		respW := httptest.NewRecorder()
    42  		s.Server.AgentSelfRequest(respW, req)
    43  
    44  		// now make a metrics endpoint request, which should be already initialized
    45  		// and written to
    46  		req, err = http.NewRequest("GET", "/v1/metrics", nil)
    47  		assert.Nil(err)
    48  		respW = httptest.NewRecorder()
    49  
    50  		testutil.WaitForResult(func() (bool, error) {
    51  			resp, err := s.Server.MetricsRequest(respW, req)
    52  			if err != nil {
    53  				return false, err
    54  			}
    55  			respW.Flush()
    56  
    57  			res := resp.(metrics.MetricsSummary)
    58  			return len(res.Gauges) != 0, nil
    59  		}, func(err error) {
    60  			t.Fatalf("should have metrics: %v", err)
    61  		})
    62  	})
    63  }
    64  
    65  // When emitting metrics, the client should use the local copy of the allocs with
    66  // updated task states (not the copy submitted by the server).
    67  //
    68  // **Cannot** be run in parallel as metrics are global.
    69  func TestHTTP_FreshClientAllocMetrics(t *testing.T) {
    70  	require := require.New(t)
    71  	numTasks := 10
    72  
    73  	httpTest(t, func(c *Config) {
    74  		c.Telemetry.PublishAllocationMetrics = true
    75  		c.Telemetry.PublishNodeMetrics = true
    76  		c.Telemetry.BackwardsCompatibleMetrics = false
    77  		c.Telemetry.DisableTaggedMetrics = false
    78  	}, func(s *TestAgent) {
    79  		// Create the job, wait for it to finish
    80  		job := mock.BatchJob()
    81  		job.TaskGroups[0].Count = numTasks
    82  		testutil.RegisterJob(t, s.RPC, job)
    83  		testutil.WaitForResult(func() (bool, error) {
    84  			time.Sleep(200 * time.Millisecond)
    85  			args := &structs.JobSpecificRequest{}
    86  			args.JobID = job.ID
    87  			args.QueryOptions.Region = "global"
    88  			var resp structs.SingleJobResponse
    89  			err := s.RPC("Job.GetJob", args, &resp)
    90  			return err == nil && resp.Job.Status == "dead", err
    91  		}, func(err error) {
    92  			require.Fail("timed-out waiting for job to complete")
    93  		})
    94  
    95  		nodeID := s.client.NodeID()
    96  
    97  		// wait for metrics to converge
    98  		var pending, running, terminal float32 = -1.0, -1.0, -1.0
    99  		testutil.WaitForResultRetries(100, func() (bool, error) {
   100  			time.Sleep(100 * time.Millisecond)
   101  			req, err := http.NewRequest("GET", "/v1/metrics", nil)
   102  			require.NoError(err)
   103  			respW := httptest.NewRecorder()
   104  
   105  			obj, err := s.Server.MetricsRequest(respW, req)
   106  			if err != nil {
   107  				return false, err
   108  			}
   109  
   110  			metrics := obj.(metrics.MetricsSummary)
   111  			for _, g := range metrics.Gauges {
   112  
   113  				// ignore client metrics belonging to other test nodes
   114  				// from other tests that contaminate go-metrics reporting
   115  				if g.DisplayLabels["node_id"] != nodeID {
   116  					continue
   117  				}
   118  
   119  				if strings.HasSuffix(g.Name, "client.allocations.pending") {
   120  					pending = g.Value
   121  				}
   122  				if strings.HasSuffix(g.Name, "client.allocations.running") {
   123  					running = g.Value
   124  				}
   125  				if strings.HasSuffix(g.Name, "client.allocations.terminal") {
   126  					terminal = g.Value
   127  				}
   128  			}
   129  			// client alloc metrics should reflect that there is numTasks terminal allocs and no other allocs
   130  			return pending == float32(0) && running == float32(0) &&
   131  				terminal == float32(numTasks), nil
   132  		}, func(err error) {
   133  			require.Fail("timed out waiting for metrics to converge",
   134  				"expected: (pending: 0, running: 0, terminal: %v), got: (pending: %v, running: %v, terminal: %v)", numTasks, pending, running, terminal)
   135  		})
   136  	})
   137  }