github.com/karlem/nomad@v0.10.2-rc1/command/agent/metrics_endpoint_test.go (about)

     1  package agent
     2  
     3  import (
     4  	"net/http"
     5  	"net/http/httptest"
     6  	"strings"
     7  	"testing"
     8  	"time"
     9  
    10  	"github.com/armon/go-metrics"
    11  	"github.com/hashicorp/nomad/nomad/mock"
    12  	"github.com/hashicorp/nomad/nomad/structs"
    13  	"github.com/hashicorp/nomad/testutil"
    14  	"github.com/stretchr/testify/assert"
    15  	"github.com/stretchr/testify/require"
    16  )
    17  
    18  func TestHTTP_MetricsWithIllegalMethod(t *testing.T) {
    19  	assert := assert.New(t)
    20  
    21  	t.Parallel()
    22  	httpTest(t, nil, func(s *TestAgent) {
    23  		req, err := http.NewRequest("DELETE", "/v1/metrics", nil)
    24  		assert.Nil(err)
    25  		respW := httptest.NewRecorder()
    26  
    27  		_, err = s.Server.MetricsRequest(respW, req)
    28  		assert.NotNil(err, "HTTP DELETE should not be accepted for this endpoint")
    29  	})
    30  }
    31  
    32  func TestHTTP_Metrics(t *testing.T) {
    33  	assert := assert.New(t)
    34  
    35  	t.Parallel()
    36  	httpTest(t, nil, func(s *TestAgent) {
    37  		// make a separate HTTP request first, to ensure Nomad has written metrics
    38  		// and prevent a race condition
    39  		req, err := http.NewRequest("GET", "/v1/agent/self", nil)
    40  		assert.Nil(err)
    41  		respW := httptest.NewRecorder()
    42  		s.Server.AgentSelfRequest(respW, req)
    43  
    44  		// now make a metrics endpoint request, which should be already initialized
    45  		// and written to
    46  		req, err = http.NewRequest("GET", "/v1/metrics", nil)
    47  		assert.Nil(err)
    48  		respW = httptest.NewRecorder()
    49  
    50  		testutil.WaitForResult(func() (bool, error) {
    51  			resp, err := s.Server.MetricsRequest(respW, req)
    52  			if err != nil {
    53  				return false, err
    54  			}
    55  			respW.Flush()
    56  
    57  			res := resp.(metrics.MetricsSummary)
    58  			return len(res.Gauges) != 0, nil
    59  		}, func(err error) {
    60  			t.Fatalf("should have metrics: %v", err)
    61  		})
    62  	})
    63  }
    64  
    65  // When emitting metrics, the client should use the local copy of the allocs with
    66  // updated task states (not the copy submitted by the server).
    67  func TestHTTP_FreshClientAllocMetrics(t *testing.T) {
    68  	t.Parallel()
    69  	require := require.New(t)
    70  	numTasks := 10
    71  
    72  	httpTest(t, func(c *Config) {
    73  		c.Telemetry.PublishAllocationMetrics = true
    74  		c.Telemetry.PublishNodeMetrics = true
    75  		c.Telemetry.BackwardsCompatibleMetrics = false
    76  		c.Telemetry.DisableTaggedMetrics = false
    77  	}, func(s *TestAgent) {
    78  		// Create the job, wait for it to finish
    79  		job := mock.BatchJob()
    80  		job.TaskGroups[0].Count = numTasks
    81  		testutil.RegisterJob(t, s.RPC, job)
    82  		testutil.WaitForResult(func() (bool, error) {
    83  			time.Sleep(200 * time.Millisecond)
    84  			args := &structs.JobSpecificRequest{}
    85  			args.JobID = job.ID
    86  			args.QueryOptions.Region = "global"
    87  			var resp structs.SingleJobResponse
    88  			err := s.RPC("Job.GetJob", args, &resp)
    89  			return err == nil && resp.Job.Status == "dead", err
    90  		}, func(err error) {
    91  			require.Fail("timed-out waiting for job to complete")
    92  		})
    93  
    94  		// wait for metrics to converge
    95  		var pending, running, terminal float32 = -1.0, -1.0, -1.0
    96  		testutil.WaitForResultRetries(100, func() (bool, error) {
    97  			time.Sleep(100 * time.Millisecond)
    98  			req, err := http.NewRequest("GET", "/v1/metrics", nil)
    99  			require.NoError(err)
   100  			respW := httptest.NewRecorder()
   101  
   102  			obj, err := s.Server.MetricsRequest(respW, req)
   103  			if err != nil {
   104  				return false, err
   105  			}
   106  
   107  			metrics := obj.(metrics.MetricsSummary)
   108  			for _, g := range metrics.Gauges {
   109  				if strings.HasSuffix(g.Name, "client.allocations.pending") {
   110  					pending = g.Value
   111  				}
   112  				if strings.HasSuffix(g.Name, "client.allocations.running") {
   113  					running = g.Value
   114  				}
   115  				if strings.HasSuffix(g.Name, "client.allocations.terminal") {
   116  					terminal = g.Value
   117  				}
   118  			}
   119  			// client alloc metrics should reflect that there is numTasks terminal allocs and no other allocs
   120  			return pending == float32(0) && running == float32(0) &&
   121  				terminal == float32(numTasks), nil
   122  		}, func(err error) {
   123  			require.Fail("timed out waiting for metrics to converge",
   124  				"expected: (pending: 0, running: 0, terminal: %v), got: (pending: %v, running: %v, terminal: %v)", numTasks, pending, running, terminal)
   125  		})
   126  	})
   127  }