github.com/justinjmoses/evergreen@v0.0.0-20170530173719-1d50e381ff0d/service/api_patch.go (about)

     1  package service
     2  
     3  import (
     4  	"fmt"
     5  	"net/http"
     6  	"strings"
     7  	"time"
     8  
     9  	"github.com/evergreen-ci/evergreen"
    10  	"github.com/evergreen-ci/evergreen/db"
    11  	"github.com/evergreen-ci/evergreen/model"
    12  	"github.com/evergreen-ci/evergreen/model/patch"
    13  	"github.com/evergreen-ci/evergreen/model/user"
    14  	"github.com/evergreen-ci/evergreen/thirdparty"
    15  	"github.com/evergreen-ci/evergreen/util"
    16  	"github.com/evergreen-ci/evergreen/validator"
    17  	"github.com/gorilla/mux"
    18  	"github.com/pkg/errors"
    19  	"gopkg.in/mgo.v2/bson"
    20  	"gopkg.in/yaml.v2"
    21  )
    22  
    23  const formMimeType = "application/x-www-form-urlencoded"
    24  
    25  // PatchAPIResponse is returned by all patch-related API calls
    26  type PatchAPIResponse struct {
    27  	Message string       `json:"message"`
    28  	Action  string       `json:"action"`
    29  	Patch   *patch.Patch `json:"patch"`
    30  }
    31  
    32  // PatchAPIRequest in the input struct with which we process patch requests
    33  type PatchAPIRequest struct {
    34  	ProjectId     string
    35  	ModuleName    string
    36  	Githash       string
    37  	PatchContent  string
    38  	BuildVariants []string
    39  	Tasks         []string
    40  	Description   string
    41  }
    42  
    43  func getSummaries(patchContent string) ([]patch.Summary, error) {
    44  	summaries := []patch.Summary{}
    45  	if patchContent != "" {
    46  		gitOutput, err := thirdparty.GitApplyNumstat(patchContent)
    47  		if err != nil {
    48  			return nil, errors.Wrap(err, "couldn't validate patch")
    49  		}
    50  		if gitOutput == nil {
    51  			return nil, errors.New("couldn't validate patch: git apply --numstat returned empty")
    52  		}
    53  
    54  		summaries, err = thirdparty.ParseGitSummary(gitOutput)
    55  		if err != nil {
    56  			return nil, errors.Wrap(err, "couldn't validate patch")
    57  		}
    58  	}
    59  	return summaries, nil
    60  }
    61  
    62  // CreatePatch checks an API request to see if it is safe and sane.
    63  // Returns the relevant patch metadata, the patch document, and any errors that occur.
    64  func (pr *PatchAPIRequest) CreatePatch(finalize bool, oauthToken string,
    65  	dbUser *user.DBUser, settings *evergreen.Settings) (*model.Project, *patch.Patch, error) {
    66  	var repoOwner, repo string
    67  	var module *model.Module
    68  
    69  	projectRef, err := model.FindOneProjectRef(pr.ProjectId)
    70  	if err != nil {
    71  		return nil, nil, errors.Wrapf(err, "Could not find project ref %v", pr.ProjectId)
    72  	}
    73  
    74  	repoOwner = projectRef.Owner
    75  	repo = projectRef.Repo
    76  
    77  	if !projectRef.Enabled {
    78  		return nil, nil, errors.Wrapf(err, "project %v is disabled", projectRef.Identifier)
    79  	}
    80  
    81  	if len(pr.Githash) != 40 {
    82  		return nil, nil, errors.New("invalid githash")
    83  	}
    84  
    85  	gitCommit, err := thirdparty.GetCommitEvent(oauthToken, repoOwner, repo, pr.Githash)
    86  	if err != nil {
    87  		return nil, nil, errors.Errorf("could not find base revision %v for project %v: %v",
    88  			pr.Githash, projectRef.Identifier, err)
    89  
    90  	}
    91  	if gitCommit == nil {
    92  		return nil, nil, errors.Errorf("commit hash %v doesn't seem to exist", pr.Githash)
    93  	}
    94  
    95  	summaries, err := getSummaries(pr.PatchContent)
    96  	if err != nil {
    97  		return nil, nil, err
    98  	}
    99  
   100  	if finalize && (len(pr.BuildVariants) == 0 || pr.BuildVariants[0] == "") {
   101  		return nil, nil, errors.New("no buildvariants specified")
   102  	}
   103  
   104  	createTime := time.Now()
   105  
   106  	// create a new object ID to use as reference for the patch data
   107  	patchFileId := bson.NewObjectId().Hex()
   108  	patchDoc := &patch.Patch{
   109  		Id:            bson.NewObjectId(),
   110  		Description:   pr.Description,
   111  		Author:        dbUser.Id,
   112  		Project:       pr.ProjectId,
   113  		Githash:       pr.Githash,
   114  		CreateTime:    createTime,
   115  		Status:        evergreen.PatchCreated,
   116  		BuildVariants: pr.BuildVariants,
   117  		Tasks:         pr.Tasks,
   118  		Patches: []patch.ModulePatch{
   119  			{
   120  				ModuleName: "",
   121  				Githash:    pr.Githash,
   122  				PatchSet: patch.PatchSet{
   123  					Patch:       pr.PatchContent,
   124  					PatchFileId: patchFileId,
   125  					Summary:     summaries,
   126  				},
   127  			},
   128  		},
   129  	}
   130  
   131  	// Get and validate patched config and add it to the patch document
   132  	project, err := validator.GetPatchedProject(patchDoc, settings)
   133  	if err != nil {
   134  		return nil, nil, errors.Wrap(err, "invalid patched config")
   135  	}
   136  
   137  	if pr.ModuleName != "" {
   138  		// is there a module? validate it.
   139  		module, err = project.GetModuleByName(pr.ModuleName)
   140  		if err != nil {
   141  			return nil, nil, errors.Wrapf(err, "could not find module %v", pr.ModuleName)
   142  		}
   143  		if module == nil {
   144  			return nil, nil, errors.Errorf("no module named %v", pr.ModuleName)
   145  		}
   146  	}
   147  
   148  	// verify that all variants exists
   149  	for _, buildVariant := range pr.BuildVariants {
   150  		if buildVariant == "all" || buildVariant == "" {
   151  			continue
   152  		}
   153  		bv := project.FindBuildVariant(buildVariant)
   154  		if bv == nil {
   155  			return nil, nil, errors.Errorf("No such buildvariant: %v", buildVariant)
   156  		}
   157  	}
   158  
   159  	// write the patch content into a GridFS file under a new ObjectId after validating.
   160  	err = db.WriteGridFile(patch.GridFSPrefix, patchFileId, strings.NewReader(pr.PatchContent))
   161  	if err != nil {
   162  		return nil, nil, errors.Wrap(err, "failed to write patch file to db")
   163  	}
   164  
   165  	// add the project config
   166  	projectYamlBytes, err := yaml.Marshal(project)
   167  	if err != nil {
   168  		return nil, nil, errors.Wrap(err, "error marshaling patched config")
   169  	}
   170  
   171  	// set the patch number based on patch author
   172  	patchDoc.PatchNumber, err = dbUser.IncPatchNumber()
   173  	if err != nil {
   174  		return nil, nil, errors.Wrap(err, "error computing patch num")
   175  	}
   176  	patchDoc.PatchedConfig = string(projectYamlBytes)
   177  
   178  	patchDoc.ClearPatchData()
   179  
   180  	return project, patchDoc, nil
   181  }
   182  
   183  // submitPatch creates the Patch document, adds the patched project config to it,
   184  // and saves the patches to GridFS to be retrieved
   185  func (as *APIServer) submitPatch(w http.ResponseWriter, r *http.Request) {
   186  	dbUser := MustHaveUser(r)
   187  	var apiRequest PatchAPIRequest
   188  	var finalize bool
   189  	if r.Header.Get("Content-Type") == formMimeType {
   190  		patchContent := r.FormValue("patch")
   191  		if patchContent == "" {
   192  			as.LoggedError(w, r, http.StatusBadRequest, errors.New("Error: Patch must not be empty"))
   193  			return
   194  		}
   195  		apiRequest = PatchAPIRequest{
   196  			ProjectId:     r.FormValue("project"),
   197  			ModuleName:    r.FormValue("module"),
   198  			Githash:       r.FormValue("githash"),
   199  			PatchContent:  r.FormValue("patch"),
   200  			BuildVariants: strings.Split(r.FormValue("buildvariants"), ","),
   201  			Description:   r.FormValue("desc"),
   202  		}
   203  		finalize = strings.ToLower(r.FormValue("finalize")) == "true"
   204  	} else {
   205  		data := struct {
   206  			Description string   `json:"desc"`
   207  			Project     string   `json:"project"`
   208  			Patch       string   `json:"patch"`
   209  			Githash     string   `json:"githash"`
   210  			Variants    string   `json:"buildvariants"`
   211  			Tasks       []string `json:"tasks"`
   212  			Finalize    bool     `json:"finalize"`
   213  		}{}
   214  		if err := util.ReadJSONInto(util.NewRequestReader(r), &data); err != nil {
   215  			as.LoggedError(w, r, http.StatusBadRequest, err)
   216  			return
   217  		}
   218  		if len(data.Patch) > patch.SizeLimit {
   219  			as.LoggedError(w, r, http.StatusBadRequest, errors.New("Patch is too large."))
   220  		}
   221  		finalize = data.Finalize
   222  
   223  		apiRequest = PatchAPIRequest{
   224  			ProjectId:     data.Project,
   225  			ModuleName:    r.FormValue("module"),
   226  			Githash:       data.Githash,
   227  			PatchContent:  data.Patch,
   228  			BuildVariants: strings.Split(data.Variants, ","),
   229  			Tasks:         data.Tasks,
   230  			Description:   data.Description,
   231  		}
   232  	}
   233  
   234  	project, patchDoc, err := apiRequest.CreatePatch(
   235  		finalize, as.Settings.Credentials["github"], dbUser, &as.Settings)
   236  	if err != nil {
   237  		as.LoggedError(w, r, http.StatusBadRequest, errors.Wrap(err, "Invalid patch"))
   238  		return
   239  	}
   240  
   241  	//expand tasks and build variants and include dependencies
   242  	if len(patchDoc.BuildVariants) == 1 && patchDoc.BuildVariants[0] == "all" {
   243  		patchDoc.BuildVariants = []string{}
   244  		for _, buildVariant := range project.BuildVariants {
   245  			if buildVariant.Disabled {
   246  				continue
   247  			}
   248  			patchDoc.BuildVariants = append(patchDoc.BuildVariants, buildVariant.Name)
   249  		}
   250  	}
   251  
   252  	if len(patchDoc.Tasks) == 1 && patchDoc.Tasks[0] == "all" {
   253  		patchDoc.Tasks = []string{}
   254  		for _, t := range project.Tasks {
   255  			if t.Patchable != nil && !(*t.Patchable) {
   256  				continue
   257  			}
   258  			patchDoc.Tasks = append(patchDoc.Tasks, t.Name)
   259  		}
   260  	}
   261  
   262  	var pairs []model.TVPair
   263  	for _, v := range patchDoc.BuildVariants {
   264  		for _, t := range patchDoc.Tasks {
   265  			if project.FindTaskForVariant(t, v) != nil {
   266  				pairs = append(pairs, model.TVPair{v, t})
   267  			}
   268  		}
   269  	}
   270  
   271  	// update variant and tasks to include dependencies
   272  	pairs = model.IncludePatchDependencies(project, pairs)
   273  
   274  	patchDoc.SyncVariantsTasks(model.TVPairsToVariantTasks(pairs))
   275  
   276  	if err = patchDoc.Insert(); err != nil {
   277  		as.LoggedError(w, r, http.StatusInternalServerError, errors.Wrap(err, "error inserting patch"))
   278  		return
   279  	}
   280  
   281  	if finalize {
   282  		if _, err = model.FinalizePatch(patchDoc, &as.Settings); err != nil {
   283  			as.LoggedError(w, r, http.StatusInternalServerError, err)
   284  			return
   285  		}
   286  	}
   287  
   288  	as.WriteJSON(w, http.StatusCreated, PatchAPIResponse{Patch: patchDoc})
   289  }
   290  
   291  // Get the patch with the specified request it
   292  func getPatchFromRequest(r *http.Request) (*patch.Patch, error) {
   293  	// get id and secret from the request.
   294  	vars := mux.Vars(r)
   295  	patchIdStr := vars["patchId"]
   296  	if len(patchIdStr) == 0 {
   297  		return nil, errors.New("no patch id supplied")
   298  	}
   299  	if !patch.IsValidId(patchIdStr) {
   300  		return nil, errors.Errorf("patch id '%v' is not valid object id", patchIdStr)
   301  	}
   302  
   303  	// find the patch
   304  	existingPatch, err := patch.FindOne(patch.ById(patch.NewId(patchIdStr)))
   305  	if err != nil {
   306  		return nil, err
   307  	}
   308  	if existingPatch == nil {
   309  		return nil, errors.Errorf("no existing request with id: %v", patchIdStr)
   310  	}
   311  
   312  	return existingPatch, nil
   313  }
   314  
   315  func (as *APIServer) updatePatchModule(w http.ResponseWriter, r *http.Request) {
   316  	p, err := getPatchFromRequest(r)
   317  	if err != nil {
   318  		as.WriteJSON(w, http.StatusBadRequest, err.Error())
   319  		return
   320  	}
   321  
   322  	var moduleName, patchContent, githash string
   323  
   324  	if r.Header.Get("Content-Type") == formMimeType {
   325  		moduleName, patchContent, githash = r.FormValue("module"), r.FormValue("patch"), r.FormValue("githash")
   326  	} else {
   327  		data := struct {
   328  			Module  string `json:"module"`
   329  			Patch   string `json:"patch"`
   330  			Githash string `json:"githash"`
   331  		}{}
   332  		if err := util.ReadJSONInto(util.NewRequestReader(r), &data); err != nil {
   333  			as.LoggedError(w, r, http.StatusBadRequest, err)
   334  			return
   335  		}
   336  		moduleName, patchContent, githash = data.Module, data.Patch, data.Githash
   337  	}
   338  
   339  	projectRef, err := model.FindOneProjectRef(p.Project)
   340  	if err != nil {
   341  		as.LoggedError(w, r, http.StatusInternalServerError, errors.Wrapf(err, "Error getting project ref with id %v", p.Project))
   342  		return
   343  	}
   344  	project, err := model.FindProject("", projectRef)
   345  	if err != nil {
   346  		as.LoggedError(w, r, http.StatusInternalServerError, errors.Wrap(err, "Error getting patch"))
   347  		return
   348  	}
   349  	if project == nil {
   350  		as.LoggedError(w, r, http.StatusNotFound, errors.Errorf("can't find project: %v", p.Project))
   351  		return
   352  	}
   353  
   354  	module, err := project.GetModuleByName(moduleName)
   355  	if err != nil || module == nil {
   356  		as.LoggedError(w, r, http.StatusBadRequest, errors.Errorf("No such module: %s", moduleName))
   357  		return
   358  	}
   359  
   360  	summaries, err := getSummaries(patchContent)
   361  	if err != nil {
   362  		as.LoggedError(w, r, http.StatusInternalServerError, err)
   363  	}
   364  	repoOwner, repo := module.GetRepoOwnerAndName()
   365  
   366  	commitInfo, err := thirdparty.GetCommitEvent(as.Settings.Credentials["github"], repoOwner, repo, githash)
   367  	if err != nil {
   368  		as.LoggedError(w, r, http.StatusInternalServerError, err)
   369  		return
   370  	}
   371  
   372  	if commitInfo == nil {
   373  		as.WriteJSON(w, http.StatusBadRequest, errors.New("commit hash doesn't seem to exist"))
   374  		return
   375  	}
   376  
   377  	// write the patch content into a GridFS file under a new ObjectId.
   378  	patchFileId := bson.NewObjectId().Hex()
   379  	err = db.WriteGridFile(patch.GridFSPrefix, patchFileId, strings.NewReader(patchContent))
   380  	if err != nil {
   381  		as.LoggedError(w, r, http.StatusInternalServerError, errors.Wrap(err, "failed to write patch file to db"))
   382  		return
   383  	}
   384  
   385  	modulePatch := patch.ModulePatch{
   386  		ModuleName: moduleName,
   387  		Githash:    githash,
   388  		PatchSet: patch.PatchSet{
   389  			PatchFileId: patchFileId,
   390  			Summary:     summaries,
   391  		},
   392  	}
   393  
   394  	if err = p.UpdateModulePatch(modulePatch); err != nil {
   395  		as.LoggedError(w, r, http.StatusInternalServerError, err)
   396  		return
   397  	}
   398  
   399  	as.WriteJSON(w, http.StatusOK, "Patch module updated")
   400  	return
   401  }
   402  
   403  // listPatches returns a user's "n" most recent patches.
   404  func (as *APIServer) listPatches(w http.ResponseWriter, r *http.Request) {
   405  	dbUser := MustHaveUser(r)
   406  	n, err := util.GetIntValue(r, "n", 0)
   407  	if err != nil {
   408  		as.LoggedError(w, r, http.StatusBadRequest, errors.Wrap(err, "cannot read value n"))
   409  		return
   410  	}
   411  	query := patch.ByUser(dbUser.Id).Sort([]string{"-" + patch.CreateTimeKey})
   412  	if n > 0 {
   413  		query = query.Limit(n)
   414  	}
   415  	patches, err := patch.Find(query)
   416  	if err != nil {
   417  		as.LoggedError(w, r, http.StatusInternalServerError,
   418  			errors.Wrapf(err, "error finding patches for user %s", dbUser.Id))
   419  		return
   420  	}
   421  	as.WriteJSON(w, http.StatusOK, patches)
   422  }
   423  
   424  func (as *APIServer) existingPatchRequest(w http.ResponseWriter, r *http.Request) {
   425  	dbUser := MustHaveUser(r)
   426  
   427  	p, err := getPatchFromRequest(r)
   428  	if err != nil {
   429  		http.Error(w, err.Error(), http.StatusNotFound)
   430  		return
   431  	}
   432  
   433  	if !getGlobalLock(r.RemoteAddr, p.Id.String(), PatchLockTitle) {
   434  		as.LoggedError(w, r, http.StatusInternalServerError, ErrLockTimeout)
   435  		return
   436  	}
   437  	defer releaseGlobalLock(r.RemoteAddr, p.Id.String(), PatchLockTitle)
   438  
   439  	var action, desc string
   440  	if r.Header.Get("Content-Type") == formMimeType {
   441  		action = r.FormValue("action")
   442  	} else {
   443  		data := struct {
   444  			PatchId     string `json:"patch_id"`
   445  			Action      string `json:"action"`
   446  			Description string `json:"description"`
   447  		}{}
   448  		if err := util.ReadJSONInto(util.NewRequestReader(r), &data); err != nil {
   449  			as.LoggedError(w, r, http.StatusBadRequest, err)
   450  			return
   451  		}
   452  		action, desc = data.Action, data.Description
   453  	}
   454  
   455  	// dispatch to handlers based on specified action
   456  	switch action {
   457  	case "update":
   458  		err := p.SetDescription(desc)
   459  		if err != nil {
   460  			as.LoggedError(w, r, http.StatusInternalServerError, err)
   461  			return
   462  		}
   463  		as.WriteJSON(w, http.StatusOK, "patch updated")
   464  	case "finalize":
   465  		if p.Activated {
   466  			http.Error(w, "patch is already finalized", http.StatusBadRequest)
   467  			return
   468  		}
   469  		patchedProject, err := validator.GetPatchedProject(p, &as.Settings)
   470  		if err != nil {
   471  			as.LoggedError(w, r, http.StatusInternalServerError, err)
   472  			return
   473  		}
   474  		projectYamlBytes, err := yaml.Marshal(patchedProject)
   475  		if err != nil {
   476  			as.LoggedError(w, r, http.StatusInternalServerError, errors.Wrap(err, "error marshaling patched config"))
   477  			return
   478  		}
   479  		p.PatchedConfig = string(projectYamlBytes)
   480  		_, err = model.FinalizePatch(p, &as.Settings)
   481  		if err != nil {
   482  			as.LoggedError(w, r, http.StatusInternalServerError, err)
   483  			return
   484  		}
   485  
   486  		as.WriteJSON(w, http.StatusOK, "patch finalized")
   487  	case "cancel":
   488  		err = model.CancelPatch(p, dbUser.Id)
   489  		if err != nil {
   490  			as.LoggedError(w, r, http.StatusInternalServerError, err)
   491  			return
   492  		}
   493  		as.WriteJSON(w, http.StatusOK, "patch deleted")
   494  	default:
   495  		http.Error(w, fmt.Sprintf("Unrecognized action: %v", action), http.StatusBadRequest)
   496  	}
   497  }
   498  
   499  func (as *APIServer) summarizePatch(w http.ResponseWriter, r *http.Request) {
   500  	p, err := getPatchFromRequest(r)
   501  	if err != nil {
   502  		http.Error(w, "not found", http.StatusNotFound)
   503  		return
   504  	}
   505  	as.WriteJSON(w, http.StatusOK, PatchAPIResponse{Patch: p})
   506  }
   507  
   508  func (as *APIServer) listPatchModules(w http.ResponseWriter, r *http.Request) {
   509  	_, project := MustHaveProject(r)
   510  
   511  	p, err := getPatchFromRequest(r)
   512  	if err != nil {
   513  		http.Error(w, "not found", http.StatusNotFound)
   514  		return
   515  	}
   516  
   517  	data := struct {
   518  		Project string   `json:"project"`
   519  		Modules []string `json:"modules"`
   520  	}{
   521  		Project: project.Identifier,
   522  	}
   523  
   524  	mods := map[string]struct{}{}
   525  
   526  	for _, m := range project.Modules {
   527  		if m.Name == "" {
   528  			continue
   529  		}
   530  		mods[m.Name] = struct{}{}
   531  	}
   532  
   533  	for _, m := range p.Patches {
   534  		mods[m.ModuleName] = struct{}{}
   535  	}
   536  
   537  	for m := range mods {
   538  		data.Modules = append(data.Modules, m)
   539  	}
   540  
   541  	as.WriteJSON(w, http.StatusOK, &data)
   542  }
   543  
   544  func (as *APIServer) deletePatchModule(w http.ResponseWriter, r *http.Request) {
   545  	p, err := getPatchFromRequest(r)
   546  	if err != nil {
   547  		as.LoggedError(w, r, http.StatusBadRequest, err)
   548  		return
   549  	}
   550  	moduleName := r.FormValue("module")
   551  	if moduleName == "" {
   552  		as.WriteJSON(w, http.StatusBadRequest, "You must specify a module to delete")
   553  		return
   554  	}
   555  
   556  	// don't mess with already finalized requests
   557  	if p.Activated {
   558  		response := fmt.Sprintf("Can't delete module - path already finalized")
   559  		as.WriteJSON(w, http.StatusBadRequest, response)
   560  		return
   561  	}
   562  
   563  	err = p.RemoveModulePatch(moduleName)
   564  	if err != nil {
   565  		as.LoggedError(w, r, http.StatusInternalServerError, err)
   566  		return
   567  	}
   568  
   569  	as.WriteJSON(w, http.StatusOK, PatchAPIResponse{Message: "module removed from patch."})
   570  }