github.com/justinjmoses/evergreen@v0.0.0-20170530173719-1d50e381ff0d/cli/http.go (about)

     1  package cli
     2  
     3  import (
     4  	"bytes"
     5  	"encoding/json"
     6  	"fmt"
     7  	"io"
     8  	"io/ioutil"
     9  	"net/http"
    10  	"net/url"
    11  	"strings"
    12  
    13  	"github.com/evergreen-ci/evergreen"
    14  	"github.com/evergreen-ci/evergreen/model"
    15  	"github.com/evergreen-ci/evergreen/model/patch"
    16  	"github.com/evergreen-ci/evergreen/model/version"
    17  	"github.com/evergreen-ci/evergreen/service"
    18  	"github.com/evergreen-ci/evergreen/util"
    19  	"github.com/evergreen-ci/evergreen/validator"
    20  	"github.com/mongodb/grip"
    21  	"github.com/pkg/errors"
    22  )
    23  
    24  // APIClient manages requests to the API server endpoints, and unmarshaling the results into
    25  // usable structures.
    26  type APIClient struct {
    27  	APIRoot    string
    28  	httpClient http.Client
    29  	User       string
    30  	APIKey     string
    31  }
    32  
    33  // APIError is an implementation of error for reporting unexpected results from API calls.
    34  type APIError struct {
    35  	body   string
    36  	status string
    37  	code   int
    38  }
    39  
    40  func (ae APIError) Error() string {
    41  	return fmt.Sprintf("Unexpected reply from server (%v): %v", ae.status, ae.body)
    42  }
    43  
    44  // NewAPIError creates an APIError by reading the body of the response and its status code.
    45  func NewAPIError(resp *http.Response) APIError {
    46  	defer resp.Body.Close()
    47  	bodyBytes, _ := ioutil.ReadAll(resp.Body) // ignore error, request has already failed anyway
    48  	bodyStr := string(bodyBytes)
    49  	return APIError{bodyStr, resp.Status, resp.StatusCode}
    50  }
    51  
    52  // getAPIClients loads and returns user settings along with two APIClients: one configured for the API
    53  // server endpoints, and another for the REST api.
    54  func getAPIClients(o *Options) (*APIClient, *APIClient, *model.CLISettings, error) {
    55  	settings, err := LoadSettings(o)
    56  	if err != nil {
    57  		return nil, nil, nil, err
    58  	}
    59  
    60  	// create a client for the regular API server
    61  	ac := &APIClient{APIRoot: settings.APIServerHost, User: settings.User, APIKey: settings.APIKey}
    62  
    63  	// create client for the REST api
    64  	apiUrl, err := url.Parse(settings.APIServerHost)
    65  	if err != nil {
    66  		return nil, nil, nil, errors.Errorf("Settings file contains an invalid url: %v", err)
    67  	}
    68  
    69  	rc := &APIClient{
    70  		APIRoot: apiUrl.Scheme + "://" + apiUrl.Host + "/rest/v1",
    71  		User:    settings.User,
    72  		APIKey:  settings.APIKey,
    73  	}
    74  	return ac, rc, settings, nil
    75  }
    76  
    77  // doReq performs a request of the given method type against path.
    78  // If body is not nil, also includes it as a request body as url-encoded data with the
    79  // appropriate header
    80  func (ac *APIClient) doReq(method, path string, body io.Reader) (*http.Response, error) {
    81  	req, err := http.NewRequest(method, fmt.Sprintf("%s/%s", ac.APIRoot, path), body)
    82  	if err != nil {
    83  		return nil, err
    84  	}
    85  
    86  	req.Header.Add("Api-Key", ac.APIKey)
    87  	req.Header.Add("Api-User", ac.User)
    88  	resp, err := ac.httpClient.Do(req)
    89  	if err != nil {
    90  		return nil, err
    91  	}
    92  	if resp == nil {
    93  		return nil, errors.New("empty response from server")
    94  	}
    95  	return resp, nil
    96  }
    97  
    98  func (ac *APIClient) get(path string, body io.Reader) (*http.Response, error) {
    99  	return ac.doReq("GET", path, body)
   100  }
   101  
   102  func (ac *APIClient) delete(path string, body io.Reader) (*http.Response, error) {
   103  	return ac.doReq("DELETE", path, body)
   104  }
   105  
   106  func (ac *APIClient) put(path string, body io.Reader) (*http.Response, error) {
   107  	return ac.doReq("PUT", path, body)
   108  }
   109  
   110  func (ac *APIClient) post(path string, body io.Reader) (*http.Response, error) {
   111  	return ac.doReq("POST", path, body)
   112  }
   113  
   114  func (ac *APIClient) modifyExisting(patchId, action string) error {
   115  	data := struct {
   116  		PatchId string `json:"patch_id"`
   117  		Action  string `json:"action"`
   118  	}{patchId, action}
   119  
   120  	rPipe, wPipe := io.Pipe()
   121  	encoder := json.NewEncoder(wPipe)
   122  	go func() {
   123  		grip.Warning(encoder.Encode(data))
   124  		grip.Warning(wPipe.Close())
   125  	}()
   126  	defer rPipe.Close()
   127  
   128  	resp, err := ac.post(fmt.Sprintf("patches/%s", patchId), rPipe)
   129  	if err != nil {
   130  		return err
   131  	}
   132  	if resp.StatusCode != http.StatusOK {
   133  		return NewAPIError(resp)
   134  	}
   135  	return nil
   136  }
   137  
   138  // ValidateLocalConfig validates the local project config with the server
   139  func (ac *APIClient) ValidateLocalConfig(data []byte) ([]validator.ValidationError, error) {
   140  	resp, err := ac.post("validate", bytes.NewBuffer(data))
   141  	if err != nil {
   142  		return nil, err
   143  	}
   144  	if resp.StatusCode == http.StatusBadRequest {
   145  		errors := []validator.ValidationError{}
   146  		err = util.ReadJSONInto(resp.Body, &errors)
   147  		if err != nil {
   148  			return nil, NewAPIError(resp)
   149  		}
   150  		return errors, nil
   151  	} else if resp.StatusCode != http.StatusOK {
   152  		return nil, NewAPIError(resp)
   153  	}
   154  	return nil, nil
   155  }
   156  
   157  func (ac *APIClient) CancelPatch(patchId string) error {
   158  	return ac.modifyExisting(patchId, "cancel")
   159  }
   160  
   161  func (ac *APIClient) FinalizePatch(patchId string) error {
   162  	return ac.modifyExisting(patchId, "finalize")
   163  }
   164  
   165  // GetPatches requests a list of the user's patches from the API and returns them as a list
   166  func (ac *APIClient) GetPatches(n int) ([]patch.Patch, error) {
   167  	resp, err := ac.get(fmt.Sprintf("patches/mine?n=%v", n), nil)
   168  	if err != nil {
   169  		return nil, err
   170  	}
   171  	if resp.StatusCode != http.StatusOK {
   172  		return nil, NewAPIError(resp)
   173  	}
   174  	patches := []patch.Patch{}
   175  	if err := util.ReadJSONInto(resp.Body, &patches); err != nil {
   176  		return nil, err
   177  	}
   178  	return patches, nil
   179  }
   180  
   181  // GetProjectRef requests project details from the API server for a given project ID.
   182  func (ac *APIClient) GetProjectRef(projectId string) (*model.ProjectRef, error) {
   183  	resp, err := ac.get(fmt.Sprintf("/ref/%s", projectId), nil)
   184  	if err != nil {
   185  		return nil, err
   186  	}
   187  	if resp.StatusCode != http.StatusOK {
   188  		return nil, NewAPIError(resp)
   189  	}
   190  	ref := &model.ProjectRef{}
   191  	if err := util.ReadJSONInto(resp.Body, ref); err != nil {
   192  		return nil, err
   193  	}
   194  	return ref, nil
   195  }
   196  
   197  // GetPatch gets a patch from the server given a patch id.
   198  func (ac *APIClient) GetPatch(patchId string) (*service.RestPatch, error) {
   199  	resp, err := ac.get(fmt.Sprintf("patches/%v", patchId), nil)
   200  	if err != nil {
   201  		return nil, err
   202  	}
   203  	if resp.StatusCode != http.StatusOK {
   204  		return nil, NewAPIError(resp)
   205  	}
   206  	ref := &service.RestPatch{}
   207  	if err := util.ReadJSONInto(resp.Body, ref); err != nil {
   208  		return nil, err
   209  	}
   210  	return ref, nil
   211  }
   212  
   213  // GetPatchedConfig takes in patch id and returns the patched project config.
   214  func (ac *APIClient) GetPatchedConfig(patchId string) (*model.Project, error) {
   215  	resp, err := ac.get(fmt.Sprintf("patches/%v/config", patchId), nil)
   216  	if err != nil {
   217  		return nil, err
   218  	}
   219  	if resp.StatusCode != http.StatusOK {
   220  		return nil, NewAPIError(resp)
   221  	}
   222  	ref := &model.Project{}
   223  	if err := util.ReadYAMLInto(resp.Body, ref); err != nil {
   224  		return nil, err
   225  	}
   226  	return ref, nil
   227  }
   228  
   229  // GetVersionConfig fetches the config requests project details from the API server for a given project ID.
   230  func (ac *APIClient) GetConfig(versionId string) (*model.Project, error) {
   231  	resp, err := ac.get(fmt.Sprintf("versions/%v/config", versionId), nil)
   232  	if err != nil {
   233  		return nil, err
   234  	}
   235  	if resp.StatusCode != http.StatusOK {
   236  		return nil, NewAPIError(resp)
   237  	}
   238  	ref := &model.Project{}
   239  	if err := util.ReadYAMLInto(resp.Body, ref); err != nil {
   240  		return nil, err
   241  	}
   242  	return ref, nil
   243  }
   244  
   245  // GetLastGreen returns the most recent successful version for the given project and variants.
   246  func (ac *APIClient) GetLastGreen(project string, variants []string) (*version.Version, error) {
   247  	qs := []string{}
   248  	for _, v := range variants {
   249  		qs = append(qs, url.QueryEscape(v))
   250  	}
   251  	q := strings.Join(qs, "&")
   252  	resp, err := ac.get(fmt.Sprintf("projects/%v/last_green?%v", project, q), nil)
   253  	if err != nil {
   254  		return nil, err
   255  	}
   256  	if resp.StatusCode != http.StatusOK {
   257  		return nil, NewAPIError(resp)
   258  	}
   259  	v := &version.Version{}
   260  	if err := util.ReadJSONInto(resp.Body, v); err != nil {
   261  		return nil, err
   262  	}
   263  	return v, nil
   264  }
   265  
   266  // DeletePatchModule makes a request to the API server to delete the given module from a patch
   267  func (ac *APIClient) DeletePatchModule(patchId, module string) error {
   268  	resp, err := ac.delete(fmt.Sprintf("patches/%s/modules?module=%v", patchId, url.QueryEscape(module)), nil)
   269  	if err != nil {
   270  		return err
   271  	}
   272  	if resp.StatusCode != http.StatusOK {
   273  		return NewAPIError(resp)
   274  	}
   275  	return nil
   276  }
   277  
   278  // UpdatePatchModule makes a request to the API server to set a module patch on the given patch ID.
   279  func (ac *APIClient) UpdatePatchModule(patchId, module, patch, base string) error {
   280  	data := struct {
   281  		Module  string `json:"module"`
   282  		Patch   string `json:"patch"`
   283  		Githash string `json:"githash"`
   284  	}{module, patch, base}
   285  
   286  	rPipe, wPipe := io.Pipe()
   287  	encoder := json.NewEncoder(wPipe)
   288  	go func() {
   289  		grip.Warning(encoder.Encode(data))
   290  		grip.Warning(wPipe.Close())
   291  	}()
   292  	defer rPipe.Close()
   293  
   294  	resp, err := ac.post(fmt.Sprintf("patches/%s/modules", patchId), rPipe)
   295  	if err != nil {
   296  		return err
   297  	}
   298  	if resp.StatusCode != http.StatusOK {
   299  		return NewAPIError(resp)
   300  	}
   301  	return nil
   302  }
   303  
   304  func (ac *APIClient) ListProjects() ([]model.ProjectRef, error) {
   305  	resp, err := ac.get("projects", nil)
   306  	if err != nil {
   307  		return nil, err
   308  	}
   309  	if resp.StatusCode != http.StatusOK {
   310  		return nil, NewAPIError(resp)
   311  	}
   312  	projs := []model.ProjectRef{}
   313  	if err := util.ReadJSONInto(resp.Body, &projs); err != nil {
   314  		return nil, err
   315  	}
   316  	return projs, nil
   317  }
   318  
   319  func (ac *APIClient) ListTasks(project string) ([]model.ProjectTask, error) {
   320  	resp, err := ac.get(fmt.Sprintf("tasks/%v", project), nil)
   321  	if err != nil {
   322  		return nil, err
   323  	}
   324  	if resp.StatusCode != http.StatusOK {
   325  		return nil, NewAPIError(resp)
   326  	}
   327  	tasks := []model.ProjectTask{}
   328  	if err := util.ReadJSONInto(resp.Body, &tasks); err != nil {
   329  		return nil, err
   330  	}
   331  	return tasks, nil
   332  }
   333  
   334  func (ac *APIClient) ListVariants(project string) ([]model.BuildVariant, error) {
   335  	resp, err := ac.get(fmt.Sprintf("variants/%v", project), nil)
   336  	if err != nil {
   337  		return nil, err
   338  	}
   339  	if resp.StatusCode != http.StatusOK {
   340  		return nil, NewAPIError(resp)
   341  	}
   342  	variants := []model.BuildVariant{}
   343  	if err := util.ReadJSONInto(resp.Body, &variants); err != nil {
   344  		return nil, err
   345  	}
   346  	return variants, nil
   347  }
   348  
   349  // PutPatch submits a new patch for the given project to the API server. If successful, returns
   350  // the patch object itself.
   351  func (ac *APIClient) PutPatch(incomingPatch patchSubmission) (*patch.Patch, error) {
   352  	data := struct {
   353  		Description string   `json:"desc"`
   354  		Project     string   `json:"project"`
   355  		Patch       string   `json:"patch"`
   356  		Githash     string   `json:"githash"`
   357  		Variants    string   `json:"buildvariants"` //TODO make this an array
   358  		Tasks       []string `json:"tasks"`
   359  		Finalize    bool     `json:"finalize"`
   360  	}{
   361  		incomingPatch.description,
   362  		incomingPatch.projectId,
   363  		incomingPatch.patchData,
   364  		incomingPatch.base,
   365  		incomingPatch.variants,
   366  		incomingPatch.tasks,
   367  		incomingPatch.finalize,
   368  	}
   369  
   370  	rPipe, wPipe := io.Pipe()
   371  	encoder := json.NewEncoder(wPipe)
   372  	go func() {
   373  		grip.Warning(encoder.Encode(data))
   374  		grip.Warning(wPipe.Close())
   375  	}()
   376  	defer rPipe.Close()
   377  
   378  	resp, err := ac.put("patches/", rPipe)
   379  	if err != nil {
   380  		return nil, err
   381  	}
   382  
   383  	if resp.StatusCode != http.StatusCreated {
   384  		return nil, NewAPIError(resp)
   385  	}
   386  
   387  	reply := struct {
   388  		Patch *patch.Patch `json:"patch"`
   389  	}{}
   390  
   391  	if err := util.ReadJSONInto(resp.Body, &reply); err != nil {
   392  		return nil, err
   393  	}
   394  
   395  	return reply.Patch, nil
   396  }
   397  
   398  // CheckUpdates fetches information about available updates to client binaries from the server.
   399  func (ac *APIClient) CheckUpdates() (*evergreen.ClientConfig, error) {
   400  	resp, err := ac.get("update", nil)
   401  	if err != nil {
   402  		return nil, err
   403  	}
   404  
   405  	if resp.StatusCode != http.StatusOK {
   406  		return nil, NewAPIError(resp)
   407  	}
   408  
   409  	reply := evergreen.ClientConfig{}
   410  	if err := util.ReadJSONInto(resp.Body, &reply); err != nil {
   411  		return nil, err
   412  	}
   413  	return &reply, nil
   414  }
   415  
   416  func (ac *APIClient) GetTask(taskId string) (*service.RestTask, error) {
   417  	resp, err := ac.get("tasks/"+taskId, nil)
   418  	if err != nil {
   419  		return nil, err
   420  	}
   421  	if resp.StatusCode == http.StatusNotFound {
   422  		return nil, nil
   423  	}
   424  
   425  	if resp.StatusCode != http.StatusOK {
   426  		return nil, NewAPIError(resp)
   427  	}
   428  
   429  	reply := service.RestTask{}
   430  	if err := util.ReadJSONInto(resp.Body, &reply); err != nil {
   431  		return nil, err
   432  	}
   433  	return &reply, nil
   434  }
   435  
   436  // GetHostUtilizationStats takes in an integer granularity, which is in seconds, and the number of days back and makes a
   437  // REST API call to get host utilization statistics.
   438  func (ac *APIClient) GetHostUtilizationStats(granularity, daysBack int, csv bool) (io.ReadCloser, error) {
   439  	resp, err := ac.get(fmt.Sprintf("scheduler/host_utilization?granularity=%v&numberDays=%v&csv=%v",
   440  		granularity, daysBack, csv), nil)
   441  	if err != nil {
   442  		return nil, err
   443  	}
   444  	if resp.StatusCode == http.StatusNotFound {
   445  		return nil, errors.New("not found")
   446  	}
   447  
   448  	if resp.StatusCode != http.StatusOK {
   449  		return nil, NewAPIError(resp)
   450  	}
   451  
   452  	return resp.Body, nil
   453  }
   454  
   455  // GetAverageSchedulerStats takes in an integer granularity, which is in seconds, the number of days back, and a distro id
   456  // and makes a REST API call to get host utilization statistics.
   457  func (ac *APIClient) GetAverageSchedulerStats(granularity, daysBack int, distroId string, csv bool) (io.ReadCloser, error) {
   458  	resp, err := ac.get(fmt.Sprintf("scheduler/distro/%v/stats?granularity=%v&numberDays=%v&csv=%v",
   459  		distroId, granularity, daysBack, csv), nil)
   460  	if err != nil {
   461  		return nil, err
   462  	}
   463  	if resp.StatusCode == http.StatusNotFound {
   464  		return nil, errors.New("not found")
   465  	}
   466  
   467  	if resp.StatusCode != http.StatusOK {
   468  		return nil, NewAPIError(resp)
   469  	}
   470  
   471  	return resp.Body, nil
   472  }
   473  
   474  // GetOptimalMakespan takes in an integer granularity, which is in seconds, and the number of days back and makes a
   475  // REST API call to get the optimal and actual makespan for builds going back however many days.
   476  func (ac *APIClient) GetOptimalMakespans(numberBuilds int, csv bool) (io.ReadCloser, error) {
   477  	resp, err := ac.get(fmt.Sprintf("scheduler/makespans?number=%v&csv=%v", numberBuilds, csv), nil)
   478  	if err != nil {
   479  		return nil, err
   480  	}
   481  	if resp.StatusCode == http.StatusNotFound {
   482  		return nil, errors.New("not found")
   483  	}
   484  
   485  	if resp.StatusCode != http.StatusOK {
   486  		return nil, NewAPIError(resp)
   487  	}
   488  
   489  	return resp.Body, nil
   490  }
   491  
   492  // GetTestHistory takes in a project identifier, the url query parameter string, and a csv flag and
   493  // returns the body of the response of the test_history api endpoint.
   494  func (ac *APIClient) GetTestHistory(project, queryParams string, isCSV bool) (io.ReadCloser, error) {
   495  	if isCSV {
   496  		queryParams += "&csv=true"
   497  	}
   498  	resp, err := ac.get(fmt.Sprintf("projects/%v/test_history?%v", project, queryParams), nil)
   499  	if err != nil {
   500  		return nil, err
   501  	}
   502  	if resp.StatusCode == http.StatusNotFound {
   503  		return nil, errors.New("not found")
   504  	}
   505  
   506  	if resp.StatusCode != http.StatusOK {
   507  		return nil, NewAPIError(resp)
   508  	}
   509  
   510  	return resp.Body, nil
   511  }
   512  
   513  // GetPatchModules retrieves a list of modules available for a given patch.
   514  func (ac *APIClient) GetPatchModules(patchId, projectId string) ([]string, error) {
   515  	var out []string
   516  
   517  	resp, err := ac.get(fmt.Sprintf("patches/%s/%s/modules", patchId, projectId), nil)
   518  	if err != nil {
   519  		return out, err
   520  	}
   521  
   522  	if resp.StatusCode != http.StatusOK {
   523  		return out, NewAPIError(resp)
   524  	}
   525  
   526  	data := struct {
   527  		Project string   `json:"project"`
   528  		Modules []string `json:"modules"`
   529  	}{}
   530  
   531  	err = util.ReadJSONInto(resp.Body, &data)
   532  	if err != nil {
   533  		return out, err
   534  	}
   535  	out = data.Modules
   536  
   537  	return out, nil
   538  }