github.com/bigcommerce/nomad@v0.9.3-bc/command/agent/operator_endpoint_test.go (about) 1 package agent 2 3 import ( 4 "bytes" 5 "fmt" 6 "net/http" 7 "net/http/httptest" 8 "strings" 9 "testing" 10 "time" 11 12 "github.com/hashicorp/consul/testutil/retry" 13 "github.com/hashicorp/nomad/api" 14 "github.com/hashicorp/nomad/nomad/structs" 15 "github.com/stretchr/testify/assert" 16 "github.com/stretchr/testify/require" 17 ) 18 19 func TestHTTP_OperatorRaftConfiguration(t *testing.T) { 20 t.Parallel() 21 httpTest(t, nil, func(s *TestAgent) { 22 body := bytes.NewBuffer(nil) 23 req, err := http.NewRequest("GET", "/v1/operator/raft/configuration", body) 24 if err != nil { 25 t.Fatalf("err: %v", err) 26 } 27 28 resp := httptest.NewRecorder() 29 obj, err := s.Server.OperatorRaftConfiguration(resp, req) 30 if err != nil { 31 t.Fatalf("err: %v", err) 32 } 33 if resp.Code != 200 { 34 t.Fatalf("bad code: %d", resp.Code) 35 } 36 out, ok := obj.(structs.RaftConfigurationResponse) 37 if !ok { 38 t.Fatalf("unexpected: %T", obj) 39 } 40 if len(out.Servers) != 1 || 41 !out.Servers[0].Leader || 42 !out.Servers[0].Voter { 43 t.Fatalf("bad: %v", out) 44 } 45 }) 46 } 47 48 func TestHTTP_OperatorRaftPeer(t *testing.T) { 49 assert := assert.New(t) 50 t.Parallel() 51 httpTest(t, nil, func(s *TestAgent) { 52 body := bytes.NewBuffer(nil) 53 req, err := http.NewRequest("DELETE", "/v1/operator/raft/peer?address=nope", body) 54 assert.Nil(err) 55 56 // If we get this error, it proves we sent the address all the 57 // way through. 58 resp := httptest.NewRecorder() 59 _, err = s.Server.OperatorRaftPeer(resp, req) 60 if err == nil || !strings.Contains(err.Error(), 61 "address \"nope\" was not found in the Raft configuration") { 62 t.Fatalf("err: %v", err) 63 } 64 }) 65 66 httpTest(t, nil, func(s *TestAgent) { 67 body := bytes.NewBuffer(nil) 68 req, err := http.NewRequest("DELETE", "/v1/operator/raft/peer?id=nope", body) 69 assert.Nil(err) 70 71 // If we get this error, it proves we sent the address all the 72 // way through. 73 resp := httptest.NewRecorder() 74 _, err = s.Server.OperatorRaftPeer(resp, req) 75 if err == nil || !strings.Contains(err.Error(), 76 "id \"nope\" was not found in the Raft configuration") { 77 t.Fatalf("err: %v", err) 78 } 79 }) 80 } 81 82 func TestOperator_AutopilotGetConfiguration(t *testing.T) { 83 t.Parallel() 84 httpTest(t, nil, func(s *TestAgent) { 85 body := bytes.NewBuffer(nil) 86 req, _ := http.NewRequest("GET", "/v1/operator/autopilot/configuration", body) 87 resp := httptest.NewRecorder() 88 obj, err := s.Server.OperatorAutopilotConfiguration(resp, req) 89 if err != nil { 90 t.Fatalf("err: %v", err) 91 } 92 if resp.Code != 200 { 93 t.Fatalf("bad code: %d", resp.Code) 94 } 95 out, ok := obj.(api.AutopilotConfiguration) 96 if !ok { 97 t.Fatalf("unexpected: %T", obj) 98 } 99 if !out.CleanupDeadServers { 100 t.Fatalf("bad: %#v", out) 101 } 102 }) 103 } 104 105 func TestOperator_AutopilotSetConfiguration(t *testing.T) { 106 t.Parallel() 107 httpTest(t, nil, func(s *TestAgent) { 108 body := bytes.NewBuffer([]byte(`{"CleanupDeadServers": false}`)) 109 req, _ := http.NewRequest("PUT", "/v1/operator/autopilot/configuration", body) 110 resp := httptest.NewRecorder() 111 if _, err := s.Server.OperatorAutopilotConfiguration(resp, req); err != nil { 112 t.Fatalf("err: %v", err) 113 } 114 if resp.Code != 200 { 115 t.Fatalf("bad code: %d, %q", resp.Code, resp.Body.String()) 116 } 117 118 args := structs.GenericRequest{ 119 QueryOptions: structs.QueryOptions{ 120 Region: s.Config.Region, 121 }, 122 } 123 124 var reply structs.AutopilotConfig 125 if err := s.RPC("Operator.AutopilotGetConfiguration", &args, &reply); err != nil { 126 t.Fatalf("err: %v", err) 127 } 128 if reply.CleanupDeadServers { 129 t.Fatalf("bad: %#v", reply) 130 } 131 }) 132 } 133 134 func TestOperator_AutopilotCASConfiguration(t *testing.T) { 135 t.Parallel() 136 httpTest(t, nil, func(s *TestAgent) { 137 body := bytes.NewBuffer([]byte(`{"CleanupDeadServers": false}`)) 138 req, _ := http.NewRequest("PUT", "/v1/operator/autopilot/configuration", body) 139 resp := httptest.NewRecorder() 140 if _, err := s.Server.OperatorAutopilotConfiguration(resp, req); err != nil { 141 t.Fatalf("err: %v", err) 142 } 143 if resp.Code != 200 { 144 t.Fatalf("bad code: %d", resp.Code) 145 } 146 147 args := structs.GenericRequest{ 148 QueryOptions: structs.QueryOptions{ 149 Region: s.Config.Region, 150 }, 151 } 152 153 var reply structs.AutopilotConfig 154 if err := s.RPC("Operator.AutopilotGetConfiguration", &args, &reply); err != nil { 155 t.Fatalf("err: %v", err) 156 } 157 158 if reply.CleanupDeadServers { 159 t.Fatalf("bad: %#v", reply) 160 } 161 162 // Create a CAS request, bad index 163 { 164 buf := bytes.NewBuffer([]byte(`{"CleanupDeadServers": true}`)) 165 req, _ := http.NewRequest("PUT", fmt.Sprintf("/v1/operator/autopilot/configuration?cas=%d", reply.ModifyIndex-1), buf) 166 resp := httptest.NewRecorder() 167 obj, err := s.Server.OperatorAutopilotConfiguration(resp, req) 168 if err != nil { 169 t.Fatalf("err: %v", err) 170 } 171 172 if res := obj.(bool); res { 173 t.Fatalf("should NOT work") 174 } 175 } 176 177 // Create a CAS request, good index 178 { 179 buf := bytes.NewBuffer([]byte(`{"CleanupDeadServers": true}`)) 180 req, _ := http.NewRequest("PUT", fmt.Sprintf("/v1/operator/autopilot/configuration?cas=%d", reply.ModifyIndex), buf) 181 resp := httptest.NewRecorder() 182 obj, err := s.Server.OperatorAutopilotConfiguration(resp, req) 183 if err != nil { 184 t.Fatalf("err: %v", err) 185 } 186 187 if res := obj.(bool); !res { 188 t.Fatalf("should work") 189 } 190 } 191 192 // Verify the update 193 if err := s.RPC("Operator.AutopilotGetConfiguration", &args, &reply); err != nil { 194 t.Fatalf("err: %v", err) 195 } 196 if !reply.CleanupDeadServers { 197 t.Fatalf("bad: %#v", reply) 198 } 199 }) 200 } 201 202 func TestOperator_ServerHealth(t *testing.T) { 203 httpTest(t, func(c *Config) { 204 c.Server.RaftProtocol = 3 205 }, func(s *TestAgent) { 206 body := bytes.NewBuffer(nil) 207 req, _ := http.NewRequest("GET", "/v1/operator/autopilot/health", body) 208 retry.Run(t, func(r *retry.R) { 209 resp := httptest.NewRecorder() 210 obj, err := s.Server.OperatorServerHealth(resp, req) 211 if err != nil { 212 r.Fatalf("err: %v", err) 213 } 214 if resp.Code != 200 { 215 r.Fatalf("bad code: %d", resp.Code) 216 } 217 out, ok := obj.(*api.OperatorHealthReply) 218 if !ok { 219 r.Fatalf("unexpected: %T", obj) 220 } 221 if len(out.Servers) != 1 || 222 !out.Servers[0].Healthy || 223 out.Servers[0].Name != s.server.LocalMember().Name || 224 out.Servers[0].SerfStatus != "alive" || 225 out.FailureTolerance != 0 { 226 r.Fatalf("bad: %v, %q", out, s.server.LocalMember().Name) 227 } 228 }) 229 }) 230 } 231 232 func TestOperator_ServerHealth_Unhealthy(t *testing.T) { 233 t.Parallel() 234 httpTest(t, func(c *Config) { 235 c.Server.RaftProtocol = 3 236 c.Autopilot.LastContactThreshold = -1 * time.Second 237 }, func(s *TestAgent) { 238 body := bytes.NewBuffer(nil) 239 req, _ := http.NewRequest("GET", "/v1/operator/autopilot/health", body) 240 retry.Run(t, func(r *retry.R) { 241 resp := httptest.NewRecorder() 242 obj, err := s.Server.OperatorServerHealth(resp, req) 243 if err != nil { 244 r.Fatalf("err: %v", err) 245 } 246 if resp.Code != 429 { 247 r.Fatalf("bad code: %d, %v", resp.Code, obj.(*api.OperatorHealthReply)) 248 } 249 out, ok := obj.(*api.OperatorHealthReply) 250 if !ok { 251 r.Fatalf("unexpected: %T", obj) 252 } 253 if len(out.Servers) != 1 || 254 out.Healthy || 255 out.Servers[0].Name != s.server.LocalMember().Name { 256 r.Fatalf("bad: %#v", out.Servers) 257 } 258 }) 259 }) 260 } 261 262 func TestOperator_SchedulerGetConfiguration(t *testing.T) { 263 t.Parallel() 264 httpTest(t, nil, func(s *TestAgent) { 265 require := require.New(t) 266 body := bytes.NewBuffer(nil) 267 req, _ := http.NewRequest("GET", "/v1/operator/scheduler/configuration", body) 268 resp := httptest.NewRecorder() 269 obj, err := s.Server.OperatorSchedulerConfiguration(resp, req) 270 require.Nil(err) 271 require.Equal(200, resp.Code) 272 out, ok := obj.(structs.SchedulerConfigurationResponse) 273 require.True(ok) 274 275 // Only system jobs can preempt other jobs by default. 276 require.True(out.SchedulerConfig.PreemptionConfig.SystemSchedulerEnabled) 277 require.False(out.SchedulerConfig.PreemptionConfig.BatchSchedulerEnabled) 278 require.False(out.SchedulerConfig.PreemptionConfig.ServiceSchedulerEnabled) 279 }) 280 } 281 282 func TestOperator_SchedulerSetConfiguration(t *testing.T) { 283 t.Parallel() 284 httpTest(t, nil, func(s *TestAgent) { 285 require := require.New(t) 286 body := bytes.NewBuffer([]byte(`{"PreemptionConfig": { 287 "SystemSchedulerEnabled": true, 288 "ServiceSchedulerEnabled": true 289 }}`)) 290 req, _ := http.NewRequest("PUT", "/v1/operator/scheduler/configuration", body) 291 resp := httptest.NewRecorder() 292 setResp, err := s.Server.OperatorSchedulerConfiguration(resp, req) 293 require.Nil(err) 294 require.Equal(200, resp.Code) 295 schedSetResp, ok := setResp.(structs.SchedulerSetConfigurationResponse) 296 require.True(ok) 297 require.NotZero(schedSetResp.Index) 298 299 args := structs.GenericRequest{ 300 QueryOptions: structs.QueryOptions{ 301 Region: s.Config.Region, 302 }, 303 } 304 305 var reply structs.SchedulerConfigurationResponse 306 err = s.RPC("Operator.SchedulerGetConfiguration", &args, &reply) 307 require.Nil(err) 308 require.True(reply.SchedulerConfig.PreemptionConfig.SystemSchedulerEnabled) 309 require.True(reply.SchedulerConfig.PreemptionConfig.ServiceSchedulerEnabled) 310 }) 311 } 312 313 func TestOperator_SchedulerCASConfiguration(t *testing.T) { 314 t.Parallel() 315 httpTest(t, nil, func(s *TestAgent) { 316 require := require.New(t) 317 body := bytes.NewBuffer([]byte(`{"PreemptionConfig": { 318 "SystemSchedulerEnabled": true, 319 "BatchSchedulerEnabled":true 320 }}`)) 321 req, _ := http.NewRequest("PUT", "/v1/operator/scheduler/configuration", body) 322 resp := httptest.NewRecorder() 323 setResp, err := s.Server.OperatorSchedulerConfiguration(resp, req) 324 require.Nil(err) 325 require.Equal(200, resp.Code) 326 schedSetResp, ok := setResp.(structs.SchedulerSetConfigurationResponse) 327 require.True(ok) 328 require.NotZero(schedSetResp.Index) 329 330 args := structs.GenericRequest{ 331 QueryOptions: structs.QueryOptions{ 332 Region: s.Config.Region, 333 }, 334 } 335 336 var reply structs.SchedulerConfigurationResponse 337 if err := s.RPC("Operator.SchedulerGetConfiguration", &args, &reply); err != nil { 338 t.Fatalf("err: %v", err) 339 } 340 require.True(reply.SchedulerConfig.PreemptionConfig.SystemSchedulerEnabled) 341 require.True(reply.SchedulerConfig.PreemptionConfig.BatchSchedulerEnabled) 342 343 // Create a CAS request, bad index 344 { 345 buf := bytes.NewBuffer([]byte(`{"PreemptionConfig": { 346 "SystemSchedulerEnabled": false, 347 "BatchSchedulerEnabled":true 348 }}`)) 349 req, _ := http.NewRequest("PUT", fmt.Sprintf("/v1/operator/scheduler/configuration?cas=%d", reply.QueryMeta.Index-1), buf) 350 resp := httptest.NewRecorder() 351 setResp, err := s.Server.OperatorSchedulerConfiguration(resp, req) 352 require.Nil(err) 353 // Verify that the response has Updated=false 354 schedSetResp, ok := setResp.(structs.SchedulerSetConfigurationResponse) 355 require.True(ok) 356 require.NotZero(schedSetResp.Index) 357 require.False(schedSetResp.Updated) 358 } 359 360 // Create a CAS request, good index 361 { 362 buf := bytes.NewBuffer([]byte(`{"PreemptionConfig": { 363 "SystemSchedulerEnabled": false, 364 "BatchSchedulerEnabled":false 365 }}`)) 366 req, _ := http.NewRequest("PUT", fmt.Sprintf("/v1/operator/scheduler/configuration?cas=%d", reply.QueryMeta.Index), buf) 367 resp := httptest.NewRecorder() 368 setResp, err := s.Server.OperatorSchedulerConfiguration(resp, req) 369 require.Nil(err) 370 // Verify that the response has Updated=true 371 schedSetResp, ok := setResp.(structs.SchedulerSetConfigurationResponse) 372 require.True(ok) 373 require.NotZero(schedSetResp.Index) 374 require.True(schedSetResp.Updated) 375 } 376 377 // Verify the update 378 if err := s.RPC("Operator.SchedulerGetConfiguration", &args, &reply); err != nil { 379 t.Fatalf("err: %v", err) 380 } 381 require.False(reply.SchedulerConfig.PreemptionConfig.SystemSchedulerEnabled) 382 require.False(reply.SchedulerConfig.PreemptionConfig.BatchSchedulerEnabled) 383 }) 384 }