github.com/adityamillind98/nomad@v0.11.8/command/agent/metrics_endpoint_test.go (about) 1 package agent 2 3 import ( 4 "net/http" 5 "net/http/httptest" 6 "strings" 7 "testing" 8 "time" 9 10 "github.com/armon/go-metrics" 11 "github.com/hashicorp/nomad/nomad/mock" 12 "github.com/hashicorp/nomad/nomad/structs" 13 "github.com/hashicorp/nomad/testutil" 14 "github.com/stretchr/testify/assert" 15 "github.com/stretchr/testify/require" 16 ) 17 18 func TestHTTP_MetricsWithIllegalMethod(t *testing.T) { 19 assert := assert.New(t) 20 21 t.Parallel() 22 httpTest(t, nil, func(s *TestAgent) { 23 req, err := http.NewRequest("DELETE", "/v1/metrics", nil) 24 assert.Nil(err) 25 respW := httptest.NewRecorder() 26 27 _, err = s.Server.MetricsRequest(respW, req) 28 assert.NotNil(err, "HTTP DELETE should not be accepted for this endpoint") 29 }) 30 } 31 32 func TestHTTP_Metrics(t *testing.T) { 33 assert := assert.New(t) 34 35 t.Parallel() 36 httpTest(t, nil, func(s *TestAgent) { 37 // make a separate HTTP request first, to ensure Nomad has written metrics 38 // and prevent a race condition 39 req, err := http.NewRequest("GET", "/v1/agent/self", nil) 40 assert.Nil(err) 41 respW := httptest.NewRecorder() 42 s.Server.AgentSelfRequest(respW, req) 43 44 // now make a metrics endpoint request, which should be already initialized 45 // and written to 46 req, err = http.NewRequest("GET", "/v1/metrics", nil) 47 assert.Nil(err) 48 respW = httptest.NewRecorder() 49 50 testutil.WaitForResult(func() (bool, error) { 51 resp, err := s.Server.MetricsRequest(respW, req) 52 if err != nil { 53 return false, err 54 } 55 respW.Flush() 56 57 res := resp.(metrics.MetricsSummary) 58 return len(res.Gauges) != 0, nil 59 }, func(err error) { 60 t.Fatalf("should have metrics: %v", err) 61 }) 62 }) 63 } 64 65 // When emitting metrics, the client should use the local copy of the allocs with 66 // updated task states (not the copy submitted by the server). 67 // 68 // **Cannot** be run in parallel as metrics are global. 69 func TestHTTP_FreshClientAllocMetrics(t *testing.T) { 70 require := require.New(t) 71 numTasks := 10 72 73 httpTest(t, func(c *Config) { 74 c.Telemetry.PublishAllocationMetrics = true 75 c.Telemetry.PublishNodeMetrics = true 76 c.Telemetry.BackwardsCompatibleMetrics = false 77 c.Telemetry.DisableTaggedMetrics = false 78 }, func(s *TestAgent) { 79 // Create the job, wait for it to finish 80 job := mock.BatchJob() 81 job.TaskGroups[0].Count = numTasks 82 testutil.RegisterJob(t, s.RPC, job) 83 testutil.WaitForResult(func() (bool, error) { 84 time.Sleep(200 * time.Millisecond) 85 args := &structs.JobSpecificRequest{} 86 args.JobID = job.ID 87 args.QueryOptions.Region = "global" 88 var resp structs.SingleJobResponse 89 err := s.RPC("Job.GetJob", args, &resp) 90 return err == nil && resp.Job.Status == "dead", err 91 }, func(err error) { 92 require.Fail("timed-out waiting for job to complete") 93 }) 94 95 nodeID := s.client.NodeID() 96 97 // wait for metrics to converge 98 var pending, running, terminal float32 = -1.0, -1.0, -1.0 99 testutil.WaitForResultRetries(100, func() (bool, error) { 100 time.Sleep(100 * time.Millisecond) 101 req, err := http.NewRequest("GET", "/v1/metrics", nil) 102 require.NoError(err) 103 respW := httptest.NewRecorder() 104 105 obj, err := s.Server.MetricsRequest(respW, req) 106 if err != nil { 107 return false, err 108 } 109 110 metrics := obj.(metrics.MetricsSummary) 111 for _, g := range metrics.Gauges { 112 113 // ignore client metrics belonging to other test nodes 114 // from other tests that contaminate go-metrics reporting 115 if g.DisplayLabels["node_id"] != nodeID { 116 continue 117 } 118 119 if strings.HasSuffix(g.Name, "client.allocations.pending") { 120 pending = g.Value 121 } 122 if strings.HasSuffix(g.Name, "client.allocations.running") { 123 running = g.Value 124 } 125 if strings.HasSuffix(g.Name, "client.allocations.terminal") { 126 terminal = g.Value 127 } 128 } 129 // client alloc metrics should reflect that there is numTasks terminal allocs and no other allocs 130 return pending == float32(0) && running == float32(0) && 131 terminal == float32(numTasks), nil 132 }, func(err error) { 133 require.Fail("timed out waiting for metrics to converge", 134 "expected: (pending: 0, running: 0, terminal: %v), got: (pending: %v, running: %v, terminal: %v)", numTasks, pending, running, terminal) 135 }) 136 }) 137 }