github.com/justinjmoses/evergreen@v0.0.0-20170530173719-1d50e381ff0d/service/api_patch.go (about) 1 package service 2 3 import ( 4 "fmt" 5 "net/http" 6 "strings" 7 "time" 8 9 "github.com/evergreen-ci/evergreen" 10 "github.com/evergreen-ci/evergreen/db" 11 "github.com/evergreen-ci/evergreen/model" 12 "github.com/evergreen-ci/evergreen/model/patch" 13 "github.com/evergreen-ci/evergreen/model/user" 14 "github.com/evergreen-ci/evergreen/thirdparty" 15 "github.com/evergreen-ci/evergreen/util" 16 "github.com/evergreen-ci/evergreen/validator" 17 "github.com/gorilla/mux" 18 "github.com/pkg/errors" 19 "gopkg.in/mgo.v2/bson" 20 "gopkg.in/yaml.v2" 21 ) 22 23 const formMimeType = "application/x-www-form-urlencoded" 24 25 // PatchAPIResponse is returned by all patch-related API calls 26 type PatchAPIResponse struct { 27 Message string `json:"message"` 28 Action string `json:"action"` 29 Patch *patch.Patch `json:"patch"` 30 } 31 32 // PatchAPIRequest in the input struct with which we process patch requests 33 type PatchAPIRequest struct { 34 ProjectId string 35 ModuleName string 36 Githash string 37 PatchContent string 38 BuildVariants []string 39 Tasks []string 40 Description string 41 } 42 43 func getSummaries(patchContent string) ([]patch.Summary, error) { 44 summaries := []patch.Summary{} 45 if patchContent != "" { 46 gitOutput, err := thirdparty.GitApplyNumstat(patchContent) 47 if err != nil { 48 return nil, errors.Wrap(err, "couldn't validate patch") 49 } 50 if gitOutput == nil { 51 return nil, errors.New("couldn't validate patch: git apply --numstat returned empty") 52 } 53 54 summaries, err = thirdparty.ParseGitSummary(gitOutput) 55 if err != nil { 56 return nil, errors.Wrap(err, "couldn't validate patch") 57 } 58 } 59 return summaries, nil 60 } 61 62 // CreatePatch checks an API request to see if it is safe and sane. 63 // Returns the relevant patch metadata, the patch document, and any errors that occur. 64 func (pr *PatchAPIRequest) CreatePatch(finalize bool, oauthToken string, 65 dbUser *user.DBUser, settings *evergreen.Settings) (*model.Project, *patch.Patch, error) { 66 var repoOwner, repo string 67 var module *model.Module 68 69 projectRef, err := model.FindOneProjectRef(pr.ProjectId) 70 if err != nil { 71 return nil, nil, errors.Wrapf(err, "Could not find project ref %v", pr.ProjectId) 72 } 73 74 repoOwner = projectRef.Owner 75 repo = projectRef.Repo 76 77 if !projectRef.Enabled { 78 return nil, nil, errors.Wrapf(err, "project %v is disabled", projectRef.Identifier) 79 } 80 81 if len(pr.Githash) != 40 { 82 return nil, nil, errors.New("invalid githash") 83 } 84 85 gitCommit, err := thirdparty.GetCommitEvent(oauthToken, repoOwner, repo, pr.Githash) 86 if err != nil { 87 return nil, nil, errors.Errorf("could not find base revision %v for project %v: %v", 88 pr.Githash, projectRef.Identifier, err) 89 90 } 91 if gitCommit == nil { 92 return nil, nil, errors.Errorf("commit hash %v doesn't seem to exist", pr.Githash) 93 } 94 95 summaries, err := getSummaries(pr.PatchContent) 96 if err != nil { 97 return nil, nil, err 98 } 99 100 if finalize && (len(pr.BuildVariants) == 0 || pr.BuildVariants[0] == "") { 101 return nil, nil, errors.New("no buildvariants specified") 102 } 103 104 createTime := time.Now() 105 106 // create a new object ID to use as reference for the patch data 107 patchFileId := bson.NewObjectId().Hex() 108 patchDoc := &patch.Patch{ 109 Id: bson.NewObjectId(), 110 Description: pr.Description, 111 Author: dbUser.Id, 112 Project: pr.ProjectId, 113 Githash: pr.Githash, 114 CreateTime: createTime, 115 Status: evergreen.PatchCreated, 116 BuildVariants: pr.BuildVariants, 117 Tasks: pr.Tasks, 118 Patches: []patch.ModulePatch{ 119 { 120 ModuleName: "", 121 Githash: pr.Githash, 122 PatchSet: patch.PatchSet{ 123 Patch: pr.PatchContent, 124 PatchFileId: patchFileId, 125 Summary: summaries, 126 }, 127 }, 128 }, 129 } 130 131 // Get and validate patched config and add it to the patch document 132 project, err := validator.GetPatchedProject(patchDoc, settings) 133 if err != nil { 134 return nil, nil, errors.Wrap(err, "invalid patched config") 135 } 136 137 if pr.ModuleName != "" { 138 // is there a module? validate it. 139 module, err = project.GetModuleByName(pr.ModuleName) 140 if err != nil { 141 return nil, nil, errors.Wrapf(err, "could not find module %v", pr.ModuleName) 142 } 143 if module == nil { 144 return nil, nil, errors.Errorf("no module named %v", pr.ModuleName) 145 } 146 } 147 148 // verify that all variants exists 149 for _, buildVariant := range pr.BuildVariants { 150 if buildVariant == "all" || buildVariant == "" { 151 continue 152 } 153 bv := project.FindBuildVariant(buildVariant) 154 if bv == nil { 155 return nil, nil, errors.Errorf("No such buildvariant: %v", buildVariant) 156 } 157 } 158 159 // write the patch content into a GridFS file under a new ObjectId after validating. 160 err = db.WriteGridFile(patch.GridFSPrefix, patchFileId, strings.NewReader(pr.PatchContent)) 161 if err != nil { 162 return nil, nil, errors.Wrap(err, "failed to write patch file to db") 163 } 164 165 // add the project config 166 projectYamlBytes, err := yaml.Marshal(project) 167 if err != nil { 168 return nil, nil, errors.Wrap(err, "error marshaling patched config") 169 } 170 171 // set the patch number based on patch author 172 patchDoc.PatchNumber, err = dbUser.IncPatchNumber() 173 if err != nil { 174 return nil, nil, errors.Wrap(err, "error computing patch num") 175 } 176 patchDoc.PatchedConfig = string(projectYamlBytes) 177 178 patchDoc.ClearPatchData() 179 180 return project, patchDoc, nil 181 } 182 183 // submitPatch creates the Patch document, adds the patched project config to it, 184 // and saves the patches to GridFS to be retrieved 185 func (as *APIServer) submitPatch(w http.ResponseWriter, r *http.Request) { 186 dbUser := MustHaveUser(r) 187 var apiRequest PatchAPIRequest 188 var finalize bool 189 if r.Header.Get("Content-Type") == formMimeType { 190 patchContent := r.FormValue("patch") 191 if patchContent == "" { 192 as.LoggedError(w, r, http.StatusBadRequest, errors.New("Error: Patch must not be empty")) 193 return 194 } 195 apiRequest = PatchAPIRequest{ 196 ProjectId: r.FormValue("project"), 197 ModuleName: r.FormValue("module"), 198 Githash: r.FormValue("githash"), 199 PatchContent: r.FormValue("patch"), 200 BuildVariants: strings.Split(r.FormValue("buildvariants"), ","), 201 Description: r.FormValue("desc"), 202 } 203 finalize = strings.ToLower(r.FormValue("finalize")) == "true" 204 } else { 205 data := struct { 206 Description string `json:"desc"` 207 Project string `json:"project"` 208 Patch string `json:"patch"` 209 Githash string `json:"githash"` 210 Variants string `json:"buildvariants"` 211 Tasks []string `json:"tasks"` 212 Finalize bool `json:"finalize"` 213 }{} 214 if err := util.ReadJSONInto(util.NewRequestReader(r), &data); err != nil { 215 as.LoggedError(w, r, http.StatusBadRequest, err) 216 return 217 } 218 if len(data.Patch) > patch.SizeLimit { 219 as.LoggedError(w, r, http.StatusBadRequest, errors.New("Patch is too large.")) 220 } 221 finalize = data.Finalize 222 223 apiRequest = PatchAPIRequest{ 224 ProjectId: data.Project, 225 ModuleName: r.FormValue("module"), 226 Githash: data.Githash, 227 PatchContent: data.Patch, 228 BuildVariants: strings.Split(data.Variants, ","), 229 Tasks: data.Tasks, 230 Description: data.Description, 231 } 232 } 233 234 project, patchDoc, err := apiRequest.CreatePatch( 235 finalize, as.Settings.Credentials["github"], dbUser, &as.Settings) 236 if err != nil { 237 as.LoggedError(w, r, http.StatusBadRequest, errors.Wrap(err, "Invalid patch")) 238 return 239 } 240 241 //expand tasks and build variants and include dependencies 242 if len(patchDoc.BuildVariants) == 1 && patchDoc.BuildVariants[0] == "all" { 243 patchDoc.BuildVariants = []string{} 244 for _, buildVariant := range project.BuildVariants { 245 if buildVariant.Disabled { 246 continue 247 } 248 patchDoc.BuildVariants = append(patchDoc.BuildVariants, buildVariant.Name) 249 } 250 } 251 252 if len(patchDoc.Tasks) == 1 && patchDoc.Tasks[0] == "all" { 253 patchDoc.Tasks = []string{} 254 for _, t := range project.Tasks { 255 if t.Patchable != nil && !(*t.Patchable) { 256 continue 257 } 258 patchDoc.Tasks = append(patchDoc.Tasks, t.Name) 259 } 260 } 261 262 var pairs []model.TVPair 263 for _, v := range patchDoc.BuildVariants { 264 for _, t := range patchDoc.Tasks { 265 if project.FindTaskForVariant(t, v) != nil { 266 pairs = append(pairs, model.TVPair{v, t}) 267 } 268 } 269 } 270 271 // update variant and tasks to include dependencies 272 pairs = model.IncludePatchDependencies(project, pairs) 273 274 patchDoc.SyncVariantsTasks(model.TVPairsToVariantTasks(pairs)) 275 276 if err = patchDoc.Insert(); err != nil { 277 as.LoggedError(w, r, http.StatusInternalServerError, errors.Wrap(err, "error inserting patch")) 278 return 279 } 280 281 if finalize { 282 if _, err = model.FinalizePatch(patchDoc, &as.Settings); err != nil { 283 as.LoggedError(w, r, http.StatusInternalServerError, err) 284 return 285 } 286 } 287 288 as.WriteJSON(w, http.StatusCreated, PatchAPIResponse{Patch: patchDoc}) 289 } 290 291 // Get the patch with the specified request it 292 func getPatchFromRequest(r *http.Request) (*patch.Patch, error) { 293 // get id and secret from the request. 294 vars := mux.Vars(r) 295 patchIdStr := vars["patchId"] 296 if len(patchIdStr) == 0 { 297 return nil, errors.New("no patch id supplied") 298 } 299 if !patch.IsValidId(patchIdStr) { 300 return nil, errors.Errorf("patch id '%v' is not valid object id", patchIdStr) 301 } 302 303 // find the patch 304 existingPatch, err := patch.FindOne(patch.ById(patch.NewId(patchIdStr))) 305 if err != nil { 306 return nil, err 307 } 308 if existingPatch == nil { 309 return nil, errors.Errorf("no existing request with id: %v", patchIdStr) 310 } 311 312 return existingPatch, nil 313 } 314 315 func (as *APIServer) updatePatchModule(w http.ResponseWriter, r *http.Request) { 316 p, err := getPatchFromRequest(r) 317 if err != nil { 318 as.WriteJSON(w, http.StatusBadRequest, err.Error()) 319 return 320 } 321 322 var moduleName, patchContent, githash string 323 324 if r.Header.Get("Content-Type") == formMimeType { 325 moduleName, patchContent, githash = r.FormValue("module"), r.FormValue("patch"), r.FormValue("githash") 326 } else { 327 data := struct { 328 Module string `json:"module"` 329 Patch string `json:"patch"` 330 Githash string `json:"githash"` 331 }{} 332 if err := util.ReadJSONInto(util.NewRequestReader(r), &data); err != nil { 333 as.LoggedError(w, r, http.StatusBadRequest, err) 334 return 335 } 336 moduleName, patchContent, githash = data.Module, data.Patch, data.Githash 337 } 338 339 projectRef, err := model.FindOneProjectRef(p.Project) 340 if err != nil { 341 as.LoggedError(w, r, http.StatusInternalServerError, errors.Wrapf(err, "Error getting project ref with id %v", p.Project)) 342 return 343 } 344 project, err := model.FindProject("", projectRef) 345 if err != nil { 346 as.LoggedError(w, r, http.StatusInternalServerError, errors.Wrap(err, "Error getting patch")) 347 return 348 } 349 if project == nil { 350 as.LoggedError(w, r, http.StatusNotFound, errors.Errorf("can't find project: %v", p.Project)) 351 return 352 } 353 354 module, err := project.GetModuleByName(moduleName) 355 if err != nil || module == nil { 356 as.LoggedError(w, r, http.StatusBadRequest, errors.Errorf("No such module: %s", moduleName)) 357 return 358 } 359 360 summaries, err := getSummaries(patchContent) 361 if err != nil { 362 as.LoggedError(w, r, http.StatusInternalServerError, err) 363 } 364 repoOwner, repo := module.GetRepoOwnerAndName() 365 366 commitInfo, err := thirdparty.GetCommitEvent(as.Settings.Credentials["github"], repoOwner, repo, githash) 367 if err != nil { 368 as.LoggedError(w, r, http.StatusInternalServerError, err) 369 return 370 } 371 372 if commitInfo == nil { 373 as.WriteJSON(w, http.StatusBadRequest, errors.New("commit hash doesn't seem to exist")) 374 return 375 } 376 377 // write the patch content into a GridFS file under a new ObjectId. 378 patchFileId := bson.NewObjectId().Hex() 379 err = db.WriteGridFile(patch.GridFSPrefix, patchFileId, strings.NewReader(patchContent)) 380 if err != nil { 381 as.LoggedError(w, r, http.StatusInternalServerError, errors.Wrap(err, "failed to write patch file to db")) 382 return 383 } 384 385 modulePatch := patch.ModulePatch{ 386 ModuleName: moduleName, 387 Githash: githash, 388 PatchSet: patch.PatchSet{ 389 PatchFileId: patchFileId, 390 Summary: summaries, 391 }, 392 } 393 394 if err = p.UpdateModulePatch(modulePatch); err != nil { 395 as.LoggedError(w, r, http.StatusInternalServerError, err) 396 return 397 } 398 399 as.WriteJSON(w, http.StatusOK, "Patch module updated") 400 return 401 } 402 403 // listPatches returns a user's "n" most recent patches. 404 func (as *APIServer) listPatches(w http.ResponseWriter, r *http.Request) { 405 dbUser := MustHaveUser(r) 406 n, err := util.GetIntValue(r, "n", 0) 407 if err != nil { 408 as.LoggedError(w, r, http.StatusBadRequest, errors.Wrap(err, "cannot read value n")) 409 return 410 } 411 query := patch.ByUser(dbUser.Id).Sort([]string{"-" + patch.CreateTimeKey}) 412 if n > 0 { 413 query = query.Limit(n) 414 } 415 patches, err := patch.Find(query) 416 if err != nil { 417 as.LoggedError(w, r, http.StatusInternalServerError, 418 errors.Wrapf(err, "error finding patches for user %s", dbUser.Id)) 419 return 420 } 421 as.WriteJSON(w, http.StatusOK, patches) 422 } 423 424 func (as *APIServer) existingPatchRequest(w http.ResponseWriter, r *http.Request) { 425 dbUser := MustHaveUser(r) 426 427 p, err := getPatchFromRequest(r) 428 if err != nil { 429 http.Error(w, err.Error(), http.StatusNotFound) 430 return 431 } 432 433 if !getGlobalLock(r.RemoteAddr, p.Id.String(), PatchLockTitle) { 434 as.LoggedError(w, r, http.StatusInternalServerError, ErrLockTimeout) 435 return 436 } 437 defer releaseGlobalLock(r.RemoteAddr, p.Id.String(), PatchLockTitle) 438 439 var action, desc string 440 if r.Header.Get("Content-Type") == formMimeType { 441 action = r.FormValue("action") 442 } else { 443 data := struct { 444 PatchId string `json:"patch_id"` 445 Action string `json:"action"` 446 Description string `json:"description"` 447 }{} 448 if err := util.ReadJSONInto(util.NewRequestReader(r), &data); err != nil { 449 as.LoggedError(w, r, http.StatusBadRequest, err) 450 return 451 } 452 action, desc = data.Action, data.Description 453 } 454 455 // dispatch to handlers based on specified action 456 switch action { 457 case "update": 458 err := p.SetDescription(desc) 459 if err != nil { 460 as.LoggedError(w, r, http.StatusInternalServerError, err) 461 return 462 } 463 as.WriteJSON(w, http.StatusOK, "patch updated") 464 case "finalize": 465 if p.Activated { 466 http.Error(w, "patch is already finalized", http.StatusBadRequest) 467 return 468 } 469 patchedProject, err := validator.GetPatchedProject(p, &as.Settings) 470 if err != nil { 471 as.LoggedError(w, r, http.StatusInternalServerError, err) 472 return 473 } 474 projectYamlBytes, err := yaml.Marshal(patchedProject) 475 if err != nil { 476 as.LoggedError(w, r, http.StatusInternalServerError, errors.Wrap(err, "error marshaling patched config")) 477 return 478 } 479 p.PatchedConfig = string(projectYamlBytes) 480 _, err = model.FinalizePatch(p, &as.Settings) 481 if err != nil { 482 as.LoggedError(w, r, http.StatusInternalServerError, err) 483 return 484 } 485 486 as.WriteJSON(w, http.StatusOK, "patch finalized") 487 case "cancel": 488 err = model.CancelPatch(p, dbUser.Id) 489 if err != nil { 490 as.LoggedError(w, r, http.StatusInternalServerError, err) 491 return 492 } 493 as.WriteJSON(w, http.StatusOK, "patch deleted") 494 default: 495 http.Error(w, fmt.Sprintf("Unrecognized action: %v", action), http.StatusBadRequest) 496 } 497 } 498 499 func (as *APIServer) summarizePatch(w http.ResponseWriter, r *http.Request) { 500 p, err := getPatchFromRequest(r) 501 if err != nil { 502 http.Error(w, "not found", http.StatusNotFound) 503 return 504 } 505 as.WriteJSON(w, http.StatusOK, PatchAPIResponse{Patch: p}) 506 } 507 508 func (as *APIServer) listPatchModules(w http.ResponseWriter, r *http.Request) { 509 _, project := MustHaveProject(r) 510 511 p, err := getPatchFromRequest(r) 512 if err != nil { 513 http.Error(w, "not found", http.StatusNotFound) 514 return 515 } 516 517 data := struct { 518 Project string `json:"project"` 519 Modules []string `json:"modules"` 520 }{ 521 Project: project.Identifier, 522 } 523 524 mods := map[string]struct{}{} 525 526 for _, m := range project.Modules { 527 if m.Name == "" { 528 continue 529 } 530 mods[m.Name] = struct{}{} 531 } 532 533 for _, m := range p.Patches { 534 mods[m.ModuleName] = struct{}{} 535 } 536 537 for m := range mods { 538 data.Modules = append(data.Modules, m) 539 } 540 541 as.WriteJSON(w, http.StatusOK, &data) 542 } 543 544 func (as *APIServer) deletePatchModule(w http.ResponseWriter, r *http.Request) { 545 p, err := getPatchFromRequest(r) 546 if err != nil { 547 as.LoggedError(w, r, http.StatusBadRequest, err) 548 return 549 } 550 moduleName := r.FormValue("module") 551 if moduleName == "" { 552 as.WriteJSON(w, http.StatusBadRequest, "You must specify a module to delete") 553 return 554 } 555 556 // don't mess with already finalized requests 557 if p.Activated { 558 response := fmt.Sprintf("Can't delete module - path already finalized") 559 as.WriteJSON(w, http.StatusBadRequest, response) 560 return 561 } 562 563 err = p.RemoveModulePatch(moduleName) 564 if err != nil { 565 as.LoggedError(w, r, http.StatusInternalServerError, err) 566 return 567 } 568 569 as.WriteJSON(w, http.StatusOK, PatchAPIResponse{Message: "module removed from patch."}) 570 }