github.com/justinjmoses/evergreen@v0.0.0-20170530173719-1d50e381ff0d/cli/http.go (about) 1 package cli 2 3 import ( 4 "bytes" 5 "encoding/json" 6 "fmt" 7 "io" 8 "io/ioutil" 9 "net/http" 10 "net/url" 11 "strings" 12 13 "github.com/evergreen-ci/evergreen" 14 "github.com/evergreen-ci/evergreen/model" 15 "github.com/evergreen-ci/evergreen/model/patch" 16 "github.com/evergreen-ci/evergreen/model/version" 17 "github.com/evergreen-ci/evergreen/service" 18 "github.com/evergreen-ci/evergreen/util" 19 "github.com/evergreen-ci/evergreen/validator" 20 "github.com/mongodb/grip" 21 "github.com/pkg/errors" 22 ) 23 24 // APIClient manages requests to the API server endpoints, and unmarshaling the results into 25 // usable structures. 26 type APIClient struct { 27 APIRoot string 28 httpClient http.Client 29 User string 30 APIKey string 31 } 32 33 // APIError is an implementation of error for reporting unexpected results from API calls. 34 type APIError struct { 35 body string 36 status string 37 code int 38 } 39 40 func (ae APIError) Error() string { 41 return fmt.Sprintf("Unexpected reply from server (%v): %v", ae.status, ae.body) 42 } 43 44 // NewAPIError creates an APIError by reading the body of the response and its status code. 45 func NewAPIError(resp *http.Response) APIError { 46 defer resp.Body.Close() 47 bodyBytes, _ := ioutil.ReadAll(resp.Body) // ignore error, request has already failed anyway 48 bodyStr := string(bodyBytes) 49 return APIError{bodyStr, resp.Status, resp.StatusCode} 50 } 51 52 // getAPIClients loads and returns user settings along with two APIClients: one configured for the API 53 // server endpoints, and another for the REST api. 54 func getAPIClients(o *Options) (*APIClient, *APIClient, *model.CLISettings, error) { 55 settings, err := LoadSettings(o) 56 if err != nil { 57 return nil, nil, nil, err 58 } 59 60 // create a client for the regular API server 61 ac := &APIClient{APIRoot: settings.APIServerHost, User: settings.User, APIKey: settings.APIKey} 62 63 // create client for the REST api 64 apiUrl, err := url.Parse(settings.APIServerHost) 65 if err != nil { 66 return nil, nil, nil, errors.Errorf("Settings file contains an invalid url: %v", err) 67 } 68 69 rc := &APIClient{ 70 APIRoot: apiUrl.Scheme + "://" + apiUrl.Host + "/rest/v1", 71 User: settings.User, 72 APIKey: settings.APIKey, 73 } 74 return ac, rc, settings, nil 75 } 76 77 // doReq performs a request of the given method type against path. 78 // If body is not nil, also includes it as a request body as url-encoded data with the 79 // appropriate header 80 func (ac *APIClient) doReq(method, path string, body io.Reader) (*http.Response, error) { 81 req, err := http.NewRequest(method, fmt.Sprintf("%s/%s", ac.APIRoot, path), body) 82 if err != nil { 83 return nil, err 84 } 85 86 req.Header.Add("Api-Key", ac.APIKey) 87 req.Header.Add("Api-User", ac.User) 88 resp, err := ac.httpClient.Do(req) 89 if err != nil { 90 return nil, err 91 } 92 if resp == nil { 93 return nil, errors.New("empty response from server") 94 } 95 return resp, nil 96 } 97 98 func (ac *APIClient) get(path string, body io.Reader) (*http.Response, error) { 99 return ac.doReq("GET", path, body) 100 } 101 102 func (ac *APIClient) delete(path string, body io.Reader) (*http.Response, error) { 103 return ac.doReq("DELETE", path, body) 104 } 105 106 func (ac *APIClient) put(path string, body io.Reader) (*http.Response, error) { 107 return ac.doReq("PUT", path, body) 108 } 109 110 func (ac *APIClient) post(path string, body io.Reader) (*http.Response, error) { 111 return ac.doReq("POST", path, body) 112 } 113 114 func (ac *APIClient) modifyExisting(patchId, action string) error { 115 data := struct { 116 PatchId string `json:"patch_id"` 117 Action string `json:"action"` 118 }{patchId, action} 119 120 rPipe, wPipe := io.Pipe() 121 encoder := json.NewEncoder(wPipe) 122 go func() { 123 grip.Warning(encoder.Encode(data)) 124 grip.Warning(wPipe.Close()) 125 }() 126 defer rPipe.Close() 127 128 resp, err := ac.post(fmt.Sprintf("patches/%s", patchId), rPipe) 129 if err != nil { 130 return err 131 } 132 if resp.StatusCode != http.StatusOK { 133 return NewAPIError(resp) 134 } 135 return nil 136 } 137 138 // ValidateLocalConfig validates the local project config with the server 139 func (ac *APIClient) ValidateLocalConfig(data []byte) ([]validator.ValidationError, error) { 140 resp, err := ac.post("validate", bytes.NewBuffer(data)) 141 if err != nil { 142 return nil, err 143 } 144 if resp.StatusCode == http.StatusBadRequest { 145 errors := []validator.ValidationError{} 146 err = util.ReadJSONInto(resp.Body, &errors) 147 if err != nil { 148 return nil, NewAPIError(resp) 149 } 150 return errors, nil 151 } else if resp.StatusCode != http.StatusOK { 152 return nil, NewAPIError(resp) 153 } 154 return nil, nil 155 } 156 157 func (ac *APIClient) CancelPatch(patchId string) error { 158 return ac.modifyExisting(patchId, "cancel") 159 } 160 161 func (ac *APIClient) FinalizePatch(patchId string) error { 162 return ac.modifyExisting(patchId, "finalize") 163 } 164 165 // GetPatches requests a list of the user's patches from the API and returns them as a list 166 func (ac *APIClient) GetPatches(n int) ([]patch.Patch, error) { 167 resp, err := ac.get(fmt.Sprintf("patches/mine?n=%v", n), nil) 168 if err != nil { 169 return nil, err 170 } 171 if resp.StatusCode != http.StatusOK { 172 return nil, NewAPIError(resp) 173 } 174 patches := []patch.Patch{} 175 if err := util.ReadJSONInto(resp.Body, &patches); err != nil { 176 return nil, err 177 } 178 return patches, nil 179 } 180 181 // GetProjectRef requests project details from the API server for a given project ID. 182 func (ac *APIClient) GetProjectRef(projectId string) (*model.ProjectRef, error) { 183 resp, err := ac.get(fmt.Sprintf("/ref/%s", projectId), nil) 184 if err != nil { 185 return nil, err 186 } 187 if resp.StatusCode != http.StatusOK { 188 return nil, NewAPIError(resp) 189 } 190 ref := &model.ProjectRef{} 191 if err := util.ReadJSONInto(resp.Body, ref); err != nil { 192 return nil, err 193 } 194 return ref, nil 195 } 196 197 // GetPatch gets a patch from the server given a patch id. 198 func (ac *APIClient) GetPatch(patchId string) (*service.RestPatch, error) { 199 resp, err := ac.get(fmt.Sprintf("patches/%v", patchId), nil) 200 if err != nil { 201 return nil, err 202 } 203 if resp.StatusCode != http.StatusOK { 204 return nil, NewAPIError(resp) 205 } 206 ref := &service.RestPatch{} 207 if err := util.ReadJSONInto(resp.Body, ref); err != nil { 208 return nil, err 209 } 210 return ref, nil 211 } 212 213 // GetPatchedConfig takes in patch id and returns the patched project config. 214 func (ac *APIClient) GetPatchedConfig(patchId string) (*model.Project, error) { 215 resp, err := ac.get(fmt.Sprintf("patches/%v/config", patchId), nil) 216 if err != nil { 217 return nil, err 218 } 219 if resp.StatusCode != http.StatusOK { 220 return nil, NewAPIError(resp) 221 } 222 ref := &model.Project{} 223 if err := util.ReadYAMLInto(resp.Body, ref); err != nil { 224 return nil, err 225 } 226 return ref, nil 227 } 228 229 // GetVersionConfig fetches the config requests project details from the API server for a given project ID. 230 func (ac *APIClient) GetConfig(versionId string) (*model.Project, error) { 231 resp, err := ac.get(fmt.Sprintf("versions/%v/config", versionId), nil) 232 if err != nil { 233 return nil, err 234 } 235 if resp.StatusCode != http.StatusOK { 236 return nil, NewAPIError(resp) 237 } 238 ref := &model.Project{} 239 if err := util.ReadYAMLInto(resp.Body, ref); err != nil { 240 return nil, err 241 } 242 return ref, nil 243 } 244 245 // GetLastGreen returns the most recent successful version for the given project and variants. 246 func (ac *APIClient) GetLastGreen(project string, variants []string) (*version.Version, error) { 247 qs := []string{} 248 for _, v := range variants { 249 qs = append(qs, url.QueryEscape(v)) 250 } 251 q := strings.Join(qs, "&") 252 resp, err := ac.get(fmt.Sprintf("projects/%v/last_green?%v", project, q), nil) 253 if err != nil { 254 return nil, err 255 } 256 if resp.StatusCode != http.StatusOK { 257 return nil, NewAPIError(resp) 258 } 259 v := &version.Version{} 260 if err := util.ReadJSONInto(resp.Body, v); err != nil { 261 return nil, err 262 } 263 return v, nil 264 } 265 266 // DeletePatchModule makes a request to the API server to delete the given module from a patch 267 func (ac *APIClient) DeletePatchModule(patchId, module string) error { 268 resp, err := ac.delete(fmt.Sprintf("patches/%s/modules?module=%v", patchId, url.QueryEscape(module)), nil) 269 if err != nil { 270 return err 271 } 272 if resp.StatusCode != http.StatusOK { 273 return NewAPIError(resp) 274 } 275 return nil 276 } 277 278 // UpdatePatchModule makes a request to the API server to set a module patch on the given patch ID. 279 func (ac *APIClient) UpdatePatchModule(patchId, module, patch, base string) error { 280 data := struct { 281 Module string `json:"module"` 282 Patch string `json:"patch"` 283 Githash string `json:"githash"` 284 }{module, patch, base} 285 286 rPipe, wPipe := io.Pipe() 287 encoder := json.NewEncoder(wPipe) 288 go func() { 289 grip.Warning(encoder.Encode(data)) 290 grip.Warning(wPipe.Close()) 291 }() 292 defer rPipe.Close() 293 294 resp, err := ac.post(fmt.Sprintf("patches/%s/modules", patchId), rPipe) 295 if err != nil { 296 return err 297 } 298 if resp.StatusCode != http.StatusOK { 299 return NewAPIError(resp) 300 } 301 return nil 302 } 303 304 func (ac *APIClient) ListProjects() ([]model.ProjectRef, error) { 305 resp, err := ac.get("projects", nil) 306 if err != nil { 307 return nil, err 308 } 309 if resp.StatusCode != http.StatusOK { 310 return nil, NewAPIError(resp) 311 } 312 projs := []model.ProjectRef{} 313 if err := util.ReadJSONInto(resp.Body, &projs); err != nil { 314 return nil, err 315 } 316 return projs, nil 317 } 318 319 func (ac *APIClient) ListTasks(project string) ([]model.ProjectTask, error) { 320 resp, err := ac.get(fmt.Sprintf("tasks/%v", project), nil) 321 if err != nil { 322 return nil, err 323 } 324 if resp.StatusCode != http.StatusOK { 325 return nil, NewAPIError(resp) 326 } 327 tasks := []model.ProjectTask{} 328 if err := util.ReadJSONInto(resp.Body, &tasks); err != nil { 329 return nil, err 330 } 331 return tasks, nil 332 } 333 334 func (ac *APIClient) ListVariants(project string) ([]model.BuildVariant, error) { 335 resp, err := ac.get(fmt.Sprintf("variants/%v", project), nil) 336 if err != nil { 337 return nil, err 338 } 339 if resp.StatusCode != http.StatusOK { 340 return nil, NewAPIError(resp) 341 } 342 variants := []model.BuildVariant{} 343 if err := util.ReadJSONInto(resp.Body, &variants); err != nil { 344 return nil, err 345 } 346 return variants, nil 347 } 348 349 // PutPatch submits a new patch for the given project to the API server. If successful, returns 350 // the patch object itself. 351 func (ac *APIClient) PutPatch(incomingPatch patchSubmission) (*patch.Patch, error) { 352 data := struct { 353 Description string `json:"desc"` 354 Project string `json:"project"` 355 Patch string `json:"patch"` 356 Githash string `json:"githash"` 357 Variants string `json:"buildvariants"` //TODO make this an array 358 Tasks []string `json:"tasks"` 359 Finalize bool `json:"finalize"` 360 }{ 361 incomingPatch.description, 362 incomingPatch.projectId, 363 incomingPatch.patchData, 364 incomingPatch.base, 365 incomingPatch.variants, 366 incomingPatch.tasks, 367 incomingPatch.finalize, 368 } 369 370 rPipe, wPipe := io.Pipe() 371 encoder := json.NewEncoder(wPipe) 372 go func() { 373 grip.Warning(encoder.Encode(data)) 374 grip.Warning(wPipe.Close()) 375 }() 376 defer rPipe.Close() 377 378 resp, err := ac.put("patches/", rPipe) 379 if err != nil { 380 return nil, err 381 } 382 383 if resp.StatusCode != http.StatusCreated { 384 return nil, NewAPIError(resp) 385 } 386 387 reply := struct { 388 Patch *patch.Patch `json:"patch"` 389 }{} 390 391 if err := util.ReadJSONInto(resp.Body, &reply); err != nil { 392 return nil, err 393 } 394 395 return reply.Patch, nil 396 } 397 398 // CheckUpdates fetches information about available updates to client binaries from the server. 399 func (ac *APIClient) CheckUpdates() (*evergreen.ClientConfig, error) { 400 resp, err := ac.get("update", nil) 401 if err != nil { 402 return nil, err 403 } 404 405 if resp.StatusCode != http.StatusOK { 406 return nil, NewAPIError(resp) 407 } 408 409 reply := evergreen.ClientConfig{} 410 if err := util.ReadJSONInto(resp.Body, &reply); err != nil { 411 return nil, err 412 } 413 return &reply, nil 414 } 415 416 func (ac *APIClient) GetTask(taskId string) (*service.RestTask, error) { 417 resp, err := ac.get("tasks/"+taskId, nil) 418 if err != nil { 419 return nil, err 420 } 421 if resp.StatusCode == http.StatusNotFound { 422 return nil, nil 423 } 424 425 if resp.StatusCode != http.StatusOK { 426 return nil, NewAPIError(resp) 427 } 428 429 reply := service.RestTask{} 430 if err := util.ReadJSONInto(resp.Body, &reply); err != nil { 431 return nil, err 432 } 433 return &reply, nil 434 } 435 436 // GetHostUtilizationStats takes in an integer granularity, which is in seconds, and the number of days back and makes a 437 // REST API call to get host utilization statistics. 438 func (ac *APIClient) GetHostUtilizationStats(granularity, daysBack int, csv bool) (io.ReadCloser, error) { 439 resp, err := ac.get(fmt.Sprintf("scheduler/host_utilization?granularity=%v&numberDays=%v&csv=%v", 440 granularity, daysBack, csv), nil) 441 if err != nil { 442 return nil, err 443 } 444 if resp.StatusCode == http.StatusNotFound { 445 return nil, errors.New("not found") 446 } 447 448 if resp.StatusCode != http.StatusOK { 449 return nil, NewAPIError(resp) 450 } 451 452 return resp.Body, nil 453 } 454 455 // GetAverageSchedulerStats takes in an integer granularity, which is in seconds, the number of days back, and a distro id 456 // and makes a REST API call to get host utilization statistics. 457 func (ac *APIClient) GetAverageSchedulerStats(granularity, daysBack int, distroId string, csv bool) (io.ReadCloser, error) { 458 resp, err := ac.get(fmt.Sprintf("scheduler/distro/%v/stats?granularity=%v&numberDays=%v&csv=%v", 459 distroId, granularity, daysBack, csv), nil) 460 if err != nil { 461 return nil, err 462 } 463 if resp.StatusCode == http.StatusNotFound { 464 return nil, errors.New("not found") 465 } 466 467 if resp.StatusCode != http.StatusOK { 468 return nil, NewAPIError(resp) 469 } 470 471 return resp.Body, nil 472 } 473 474 // GetOptimalMakespan takes in an integer granularity, which is in seconds, and the number of days back and makes a 475 // REST API call to get the optimal and actual makespan for builds going back however many days. 476 func (ac *APIClient) GetOptimalMakespans(numberBuilds int, csv bool) (io.ReadCloser, error) { 477 resp, err := ac.get(fmt.Sprintf("scheduler/makespans?number=%v&csv=%v", numberBuilds, csv), nil) 478 if err != nil { 479 return nil, err 480 } 481 if resp.StatusCode == http.StatusNotFound { 482 return nil, errors.New("not found") 483 } 484 485 if resp.StatusCode != http.StatusOK { 486 return nil, NewAPIError(resp) 487 } 488 489 return resp.Body, nil 490 } 491 492 // GetTestHistory takes in a project identifier, the url query parameter string, and a csv flag and 493 // returns the body of the response of the test_history api endpoint. 494 func (ac *APIClient) GetTestHistory(project, queryParams string, isCSV bool) (io.ReadCloser, error) { 495 if isCSV { 496 queryParams += "&csv=true" 497 } 498 resp, err := ac.get(fmt.Sprintf("projects/%v/test_history?%v", project, queryParams), nil) 499 if err != nil { 500 return nil, err 501 } 502 if resp.StatusCode == http.StatusNotFound { 503 return nil, errors.New("not found") 504 } 505 506 if resp.StatusCode != http.StatusOK { 507 return nil, NewAPIError(resp) 508 } 509 510 return resp.Body, nil 511 } 512 513 // GetPatchModules retrieves a list of modules available for a given patch. 514 func (ac *APIClient) GetPatchModules(patchId, projectId string) ([]string, error) { 515 var out []string 516 517 resp, err := ac.get(fmt.Sprintf("patches/%s/%s/modules", patchId, projectId), nil) 518 if err != nil { 519 return out, err 520 } 521 522 if resp.StatusCode != http.StatusOK { 523 return out, NewAPIError(resp) 524 } 525 526 data := struct { 527 Project string `json:"project"` 528 Modules []string `json:"modules"` 529 }{} 530 531 err = util.ReadJSONInto(resp.Body, &data) 532 if err != nil { 533 return out, err 534 } 535 out = data.Modules 536 537 return out, nil 538 }