github.com/prestonp/nomad@v0.10.4/command/agent/metrics_endpoint_test.go (about)

     1  package agent
     2  
     3  import (
     4  	"net/http"
     5  	"net/http/httptest"
     6  	"strings"
     7  	"testing"
     8  	"time"
     9  
    10  	"github.com/armon/go-metrics"
    11  	"github.com/hashicorp/nomad/nomad/mock"
    12  	"github.com/hashicorp/nomad/nomad/structs"
    13  	"github.com/hashicorp/nomad/testutil"
    14  	"github.com/stretchr/testify/assert"
    15  	"github.com/stretchr/testify/require"
    16  )
    17  
    18  func TestHTTP_MetricsWithIllegalMethod(t *testing.T) {
    19  	assert := assert.New(t)
    20  
    21  	t.Parallel()
    22  	httpTest(t, nil, func(s *TestAgent) {
    23  		req, err := http.NewRequest("DELETE", "/v1/metrics", nil)
    24  		assert.Nil(err)
    25  		respW := httptest.NewRecorder()
    26  
    27  		_, err = s.Server.MetricsRequest(respW, req)
    28  		assert.NotNil(err, "HTTP DELETE should not be accepted for this endpoint")
    29  	})
    30  }
    31  
    32  func TestHTTP_Metrics(t *testing.T) {
    33  	assert := assert.New(t)
    34  
    35  	t.Parallel()
    36  	httpTest(t, nil, func(s *TestAgent) {
    37  		// make a separate HTTP request first, to ensure Nomad has written metrics
    38  		// and prevent a race condition
    39  		req, err := http.NewRequest("GET", "/v1/agent/self", nil)
    40  		assert.Nil(err)
    41  		respW := httptest.NewRecorder()
    42  		s.Server.AgentSelfRequest(respW, req)
    43  
    44  		// now make a metrics endpoint request, which should be already initialized
    45  		// and written to
    46  		req, err = http.NewRequest("GET", "/v1/metrics", nil)
    47  		assert.Nil(err)
    48  		respW = httptest.NewRecorder()
    49  
    50  		testutil.WaitForResult(func() (bool, error) {
    51  			resp, err := s.Server.MetricsRequest(respW, req)
    52  			if err != nil {
    53  				return false, err
    54  			}
    55  			respW.Flush()
    56  
    57  			res := resp.(metrics.MetricsSummary)
    58  			return len(res.Gauges) != 0, nil
    59  		}, func(err error) {
    60  			t.Fatalf("should have metrics: %v", err)
    61  		})
    62  	})
    63  }
    64  
    65  // When emitting metrics, the client should use the local copy of the allocs with
    66  // updated task states (not the copy submitted by the server).
    67  func TestHTTP_FreshClientAllocMetrics(t *testing.T) {
    68  	t.Parallel()
    69  	require := require.New(t)
    70  	numTasks := 10
    71  
    72  	httpTest(t, func(c *Config) {
    73  		c.Telemetry.PublishAllocationMetrics = true
    74  		c.Telemetry.PublishNodeMetrics = true
    75  		c.Telemetry.BackwardsCompatibleMetrics = false
    76  		c.Telemetry.DisableTaggedMetrics = false
    77  	}, func(s *TestAgent) {
    78  		// Create the job, wait for it to finish
    79  		job := mock.BatchJob()
    80  		job.TaskGroups[0].Count = numTasks
    81  		testutil.RegisterJob(t, s.RPC, job)
    82  		testutil.WaitForResult(func() (bool, error) {
    83  			time.Sleep(200 * time.Millisecond)
    84  			args := &structs.JobSpecificRequest{}
    85  			args.JobID = job.ID
    86  			args.QueryOptions.Region = "global"
    87  			var resp structs.SingleJobResponse
    88  			err := s.RPC("Job.GetJob", args, &resp)
    89  			return err == nil && resp.Job.Status == "dead", err
    90  		}, func(err error) {
    91  			require.Fail("timed-out waiting for job to complete")
    92  		})
    93  
    94  		nodeID := s.client.NodeID()
    95  
    96  		// wait for metrics to converge
    97  		var pending, running, terminal float32 = -1.0, -1.0, -1.0
    98  		testutil.WaitForResultRetries(100, func() (bool, error) {
    99  			time.Sleep(100 * time.Millisecond)
   100  			req, err := http.NewRequest("GET", "/v1/metrics", nil)
   101  			require.NoError(err)
   102  			respW := httptest.NewRecorder()
   103  
   104  			obj, err := s.Server.MetricsRequest(respW, req)
   105  			if err != nil {
   106  				return false, err
   107  			}
   108  
   109  			metrics := obj.(metrics.MetricsSummary)
   110  			for _, g := range metrics.Gauges {
   111  
   112  				// ignore client metrics belonging to other test nodes
   113  				// from other tests that contaminate go-metrics reporting
   114  				if g.DisplayLabels["node_id"] != nodeID {
   115  					continue
   116  				}
   117  
   118  				if strings.HasSuffix(g.Name, "client.allocations.pending") {
   119  					pending = g.Value
   120  				}
   121  				if strings.HasSuffix(g.Name, "client.allocations.running") {
   122  					running = g.Value
   123  				}
   124  				if strings.HasSuffix(g.Name, "client.allocations.terminal") {
   125  					terminal = g.Value
   126  				}
   127  			}
   128  			// client alloc metrics should reflect that there is numTasks terminal allocs and no other allocs
   129  			return pending == float32(0) && running == float32(0) &&
   130  				terminal == float32(numTasks), nil
   131  		}, func(err error) {
   132  			require.Fail("timed out waiting for metrics to converge",
   133  				"expected: (pending: 0, running: 0, terminal: %v), got: (pending: %v, running: %v, terminal: %v)", numTasks, pending, running, terminal)
   134  		})
   135  	})
   136  }