github.com/justinjmoses/evergreen@v0.0.0-20170530173719-1d50e381ff0d/service/rest_version.go (about) 1 package service 2 3 import ( 4 "encoding/json" 5 "fmt" 6 "net/http" 7 "strings" 8 "time" 9 10 "github.com/evergreen-ci/evergreen" 11 "github.com/evergreen-ci/evergreen/db" 12 "github.com/evergreen-ci/evergreen/model" 13 "github.com/evergreen-ci/evergreen/model/build" 14 "github.com/evergreen-ci/evergreen/model/version" 15 "github.com/evergreen-ci/evergreen/util" 16 "github.com/gorilla/mux" 17 "github.com/mongodb/grip" 18 "github.com/pkg/errors" 19 "gopkg.in/mgo.v2/bson" 20 ) 21 22 const NumRecentVersions = 10 23 24 type recentVersionsContent struct { 25 Project string `json:"project"` 26 Versions []versionLessInfo `json:"versions"` 27 } 28 29 type versionStatusByTaskContent struct { 30 Id string `json:"version_id"` 31 Tasks map[string]versionStatusByTask `json:"tasks"` 32 } 33 34 type versionStatusByBuildContent struct { 35 Id string `json:"version_id"` 36 Builds map[string]versionStatusByBuild `json:"builds"` 37 } 38 39 type restVersion struct { 40 Id string `json:"id"` 41 CreateTime time.Time `json:"create_time"` 42 StartTime time.Time `json:"start_time"` 43 FinishTime time.Time `json:"finish_time"` 44 Project string `json:"project"` 45 Revision string `json:"revision"` 46 Author string `json:"author"` 47 AuthorEmail string `json:"author_email"` 48 Message string `json:"message"` 49 Status string `json:"status"` 50 Activated bool `json:"activated"` 51 BuildIds []string `json:"builds"` 52 BuildVariants []string `json:"build_variants"` 53 RevisionOrderNumber int `json:"order"` 54 Owner string `json:"owner_name"` 55 Repo string `json:"repo_name"` 56 Branch string `json:"branch_name"` 57 RepoKind string `json:"repo_kind"` 58 BatchTime int `json:"batch_time"` 59 Identifier string `json:"identifier"` 60 Remote bool `json:"remote"` 61 RemotePath string `json:"remote_path"` 62 Requester string `json:"requester"` 63 Config string `json:"config,omitempty"` 64 } 65 66 type versionLessInfo struct { 67 Id string `json:"version_id"` 68 Author string `json:"author"` 69 Revision string `json:"revision"` 70 Message string `json:"message"` 71 Builds versionByBuild `json:"builds"` 72 } 73 74 type versionStatus struct { 75 Id string `json:"task_id"` 76 Status string `json:"status"` 77 TimeTaken time.Duration `json:"time_taken"` 78 } 79 80 type versionByBuild map[string]versionBuildInfo 81 82 type versionBuildInfo struct { 83 Id string `json:"build_id"` 84 Name string `json:"name"` 85 Tasks versionByBuildByTask `json:"tasks"` 86 } 87 88 type versionByBuildByTask map[string]versionStatus 89 90 type versionStatusByTask map[string]versionStatus 91 92 type versionStatusByBuild map[string]versionStatus 93 94 // copyVersion copies the fields of a Version struct into a restVersion struct 95 func copyVersion(srcVersion *version.Version, destVersion *restVersion) { 96 destVersion.Id = srcVersion.Id 97 destVersion.CreateTime = srcVersion.CreateTime 98 destVersion.StartTime = srcVersion.StartTime 99 destVersion.FinishTime = srcVersion.FinishTime 100 destVersion.Project = srcVersion.Identifier 101 destVersion.Revision = srcVersion.Revision 102 destVersion.Author = srcVersion.Author 103 destVersion.AuthorEmail = srcVersion.AuthorEmail 104 destVersion.Message = srcVersion.Message 105 destVersion.Status = srcVersion.Status 106 destVersion.BuildIds = srcVersion.BuildIds 107 destVersion.RevisionOrderNumber = srcVersion.RevisionOrderNumber 108 destVersion.Owner = srcVersion.Owner 109 destVersion.Repo = srcVersion.Repo 110 destVersion.Branch = srcVersion.Branch 111 destVersion.RepoKind = srcVersion.RepoKind 112 destVersion.Identifier = srcVersion.Identifier 113 destVersion.Remote = srcVersion.Remote 114 destVersion.RemotePath = srcVersion.RemotePath 115 destVersion.Requester = srcVersion.Requester 116 destVersion.Config = srcVersion.Config 117 } 118 119 // Returns a JSON response of an array with the NumRecentVersions 120 // most recent versions (sorted on commit order number descending). 121 func (restapi restAPI) getRecentVersions(w http.ResponseWriter, r *http.Request) { 122 projectId := mux.Vars(r)["project_id"] 123 124 versions, err := version.Find(version.ByMostRecentForRequester(projectId, evergreen.RepotrackerVersionRequester).Limit(10)) 125 if err != nil { 126 msg := fmt.Sprintf("Error finding recent versions of project '%v'", projectId) 127 grip.Errorf("%v: %+v", msg, err) 128 restapi.WriteJSON(w, http.StatusInternalServerError, responseError{Message: msg}) 129 return 130 } 131 132 // Create a slice of version ids to find all relevant builds 133 versionIds := make([]string, 0, len(versions)) 134 135 // Cache the order of versions in a map for lookup by their id 136 versionIdx := make(map[string]int, len(versions)) 137 138 for i, version := range versions { 139 versionIds = append(versionIds, version.Id) 140 versionIdx[version.Id] = i 141 } 142 143 // Find all builds corresponding the set of version ids 144 builds, err := build.Find( 145 build.ByVersions(versionIds). 146 WithFields(build.BuildVariantKey, build.DisplayNameKey, build.TasksKey, build.VersionKey)) 147 if err != nil { 148 msg := fmt.Sprintf("Error finding recent versions of project '%v'", projectId) 149 grip.Errorf("%v: %+v", msg, err) 150 restapi.WriteJSON(w, http.StatusInternalServerError, responseError{Message: msg}) 151 return 152 } 153 154 result := recentVersionsContent{ 155 Project: projectId, 156 Versions: make([]versionLessInfo, 0, len(versions)), 157 } 158 159 for _, version := range versions { 160 versionInfo := versionLessInfo{ 161 Id: version.Id, 162 Author: version.Author, 163 Revision: version.Revision, 164 Message: version.Message, 165 Builds: make(versionByBuild), 166 } 167 168 result.Versions = append(result.Versions, versionInfo) 169 } 170 171 for _, build := range builds { 172 buildInfo := versionBuildInfo{ 173 Id: build.Id, 174 Name: build.DisplayName, 175 Tasks: make(versionByBuildByTask, len(build.Tasks)), 176 } 177 178 for _, task := range build.Tasks { 179 buildInfo.Tasks[task.DisplayName] = versionStatus{ 180 Id: task.Id, 181 Status: task.Status, 182 TimeTaken: task.TimeTaken, 183 } 184 } 185 186 versionInfo := result.Versions[versionIdx[build.Version]] 187 versionInfo.Builds[build.BuildVariant] = buildInfo 188 } 189 190 restapi.WriteJSON(w, http.StatusOK, result) 191 return 192 } 193 194 // Returns a JSON response with the marshaled output of the version 195 // specified in the request. 196 func (restapi restAPI) getVersionInfo(w http.ResponseWriter, r *http.Request) { 197 projCtx := MustHaveRESTContext(r) 198 srcVersion := projCtx.Version 199 if srcVersion == nil { 200 restapi.WriteJSON(w, http.StatusNotFound, responseError{Message: "error finding version"}) 201 return 202 } 203 204 destVersion := &restVersion{} 205 copyVersion(srcVersion, destVersion) 206 for _, buildStatus := range srcVersion.BuildVariants { 207 destVersion.BuildVariants = append(destVersion.BuildVariants, buildStatus.BuildVariant) 208 grip.Errorln("adding BuildVariant", buildStatus.BuildVariant) 209 } 210 211 restapi.WriteJSON(w, http.StatusOK, destVersion) 212 return 213 } 214 215 // Returns a JSON response with the marshaled output of the version 216 // specified in the request. 217 func (restapi restAPI) getVersionConfig(w http.ResponseWriter, r *http.Request) { 218 projCtx := MustHaveRESTContext(r) 219 srcVersion := projCtx.Version 220 if srcVersion == nil { 221 restapi.WriteJSON(w, http.StatusNotFound, responseError{Message: "version not found"}) 222 return 223 } 224 w.Header().Set("Content-Type", "application/x-yaml; charset=utf-8") 225 w.WriteHeader(http.StatusOK) 226 _, err := w.Write([]byte(projCtx.Version.Config)) 227 grip.Warning(errors.Wrap(err, "problem writing response")) 228 return 229 } 230 231 // Returns a JSON response with the marshaled output of the version 232 // specified by its revision and project name in the request. 233 func (restapi restAPI) getVersionInfoViaRevision(w http.ResponseWriter, r *http.Request) { 234 vars := mux.Vars(r) 235 projectId := vars["project_id"] 236 revision := vars["revision"] 237 238 srcVersion, err := version.FindOne(version.ByProjectIdAndRevision(projectId, revision)) 239 if err != nil || srcVersion == nil { 240 msg := fmt.Sprintf("Error finding revision '%v' for project '%v'", revision, projectId) 241 statusCode := http.StatusNotFound 242 243 if err != nil { 244 grip.Errorf("%v: %+v", msg, err) 245 statusCode = http.StatusInternalServerError 246 } 247 248 restapi.WriteJSON(w, statusCode, responseError{Message: msg}) 249 return 250 } 251 252 destVersion := &restVersion{} 253 copyVersion(srcVersion, destVersion) 254 255 for _, buildStatus := range srcVersion.BuildVariants { 256 destVersion.BuildVariants = append(destVersion.BuildVariants, buildStatus.BuildVariant) 257 grip.Errorln("adding BuildVariant", buildStatus.BuildVariant) 258 } 259 260 restapi.WriteJSON(w, http.StatusOK, destVersion) 261 return 262 263 } 264 265 // Modifies part of the version specified in the request, and returns a 266 // JSON response with the marshaled output of its new state. 267 func (restapi restAPI) modifyVersionInfo(w http.ResponseWriter, r *http.Request) { 268 projCtx := MustHaveRESTContext(r) 269 user := MustHaveUser(r) 270 v := projCtx.Version 271 if v == nil { 272 restapi.WriteJSON(w, http.StatusNotFound, responseError{Message: "error finding version"}) 273 return 274 } 275 276 input := struct { 277 Activated *bool `json:"activated"` 278 }{} 279 280 body := util.NewRequestReader(r) 281 defer body.Close() 282 283 if err := json.NewDecoder(body).Decode(&input); err != nil { 284 http.Error(w, fmt.Sprintf("problem parsing input: %v", err.Error()), 285 http.StatusInternalServerError) 286 } 287 288 if input.Activated != nil { 289 if err := model.SetVersionActivation(v.Id, *input.Activated, user.Id); err != nil { 290 state := "inactive" 291 if *input.Activated { 292 state = "active" 293 } 294 295 msg := fmt.Sprintf("Error marking version '%v' as %v", v.Id, state) 296 restapi.WriteJSON(w, http.StatusInternalServerError, responseError{Message: msg}) 297 return 298 } 299 } 300 301 restapi.getVersionInfo(w, r) 302 } 303 304 // Returns a JSON response with the status of the specified version 305 // either grouped by the task names or the build variant names depending 306 // on the "groupby" query parameter. 307 func (restapi *restAPI) getVersionStatus(w http.ResponseWriter, r *http.Request) { 308 versionId := mux.Vars(r)["version_id"] 309 groupBy := r.FormValue("groupby") 310 311 switch groupBy { 312 case "": // default to group by tasks 313 fallthrough 314 case "tasks": 315 restapi.getVersionStatusByTask(versionId, w) 316 return 317 case "builds": 318 restapi.getVersionStatusByBuild(versionId, w) 319 return 320 default: 321 msg := fmt.Sprintf("Invalid groupby parameter '%v'", groupBy) 322 restapi.WriteJSON(w, http.StatusBadRequest, responseError{Message: msg}) 323 return 324 } 325 } 326 327 // Returns a JSON response with the status of the specified version 328 // grouped on the tasks. The keys of the object are the task names, 329 // with each key in the nested object representing a particular build 330 // variant. 331 func (restapi *restAPI) getVersionStatusByTask(versionId string, w http.ResponseWriter) { 332 id := "_id" 333 taskName := "task_name" 334 statuses := "statuses" 335 336 pipeline := []bson.M{ 337 // 1. Find only builds corresponding to the specified version 338 { 339 "$match": bson.M{ 340 build.VersionKey: versionId, 341 }, 342 }, 343 // 2. Loop through each task run on a particular build variant 344 { 345 "$unwind": fmt.Sprintf("$%v", build.TasksKey), 346 }, 347 // 3. Group on the task name and construct a new document containing 348 // all of the relevant info about the task status 349 { 350 "$group": bson.M{ 351 id: fmt.Sprintf("$%v.%v", build.TasksKey, build.TaskCacheDisplayNameKey), 352 statuses: bson.M{ 353 "$push": bson.M{ 354 build.BuildVariantKey: fmt.Sprintf("$%v", build.BuildVariantKey), 355 build.TaskCacheIdKey: fmt.Sprintf("$%v.%v", build.TasksKey, build.TaskCacheIdKey), 356 build.TaskCacheStatusKey: fmt.Sprintf("$%v.%v", build.TasksKey, build.TaskCacheStatusKey), 357 build.TaskCacheStartTimeKey: fmt.Sprintf("$%v.%v", build.TasksKey, build.TaskCacheStartTimeKey), 358 build.TaskCacheTimeTakenKey: fmt.Sprintf("$%v.%v", build.TasksKey, build.TaskCacheTimeTakenKey), 359 build.TaskCacheActivatedKey: fmt.Sprintf("$%v.%v", build.TasksKey, build.TaskCacheActivatedKey), 360 }, 361 }, 362 }, 363 }, 364 // 4. Rename the "_id" field to "task_name" 365 { 366 "$project": bson.M{ 367 id: 0, 368 taskName: fmt.Sprintf("$%v", id), 369 statuses: 1, 370 }, 371 }, 372 } 373 374 // Anonymous struct used to unmarshal output from the aggregation pipeline 375 var tasks []struct { 376 DisplayName string `bson:"task_name"` 377 Statuses []struct { 378 BuildVariant string `bson:"build_variant"` 379 // Use an anonyous field to make the semantics of inlining 380 build.TaskCache `bson:",inline"` 381 } `bson:"statuses"` 382 } 383 384 err := db.Aggregate(build.Collection, pipeline, &tasks) 385 if err != nil { 386 msg := fmt.Sprintf("Error finding status for version '%v'", versionId) 387 grip.Errorf("%v: %+v", msg, err) 388 restapi.WriteJSON(w, http.StatusInternalServerError, responseError{Message: msg}) 389 return 390 } 391 392 result := versionStatusByTaskContent{ 393 Id: versionId, 394 Tasks: make(map[string]versionStatusByTask, len(tasks)), 395 } 396 397 for _, task := range tasks { 398 statuses := make(versionStatusByTask, len(task.Statuses)) 399 for _, task := range task.Statuses { 400 status := versionStatus{ 401 Id: task.Id, 402 Status: task.Status, 403 TimeTaken: task.TimeTaken, 404 } 405 statuses[task.BuildVariant] = status 406 } 407 result.Tasks[task.DisplayName] = statuses 408 } 409 410 restapi.WriteJSON(w, http.StatusOK, result) 411 return 412 413 } 414 415 // Returns a JSON response with the status of the specified version 416 // grouped on the build variants. The keys of the object are the build 417 // variant name, with each key in the nested object representing a 418 // particular task. 419 func (restapi restAPI) getVersionStatusByBuild(versionId string, w http.ResponseWriter) { 420 // Get all of the builds corresponding to this version 421 builds, err := build.Find( 422 build.ByVersion(versionId).WithFields(build.BuildVariantKey, build.TasksKey), 423 ) 424 if err != nil { 425 msg := fmt.Sprintf("Error finding status for version '%v'", versionId) 426 grip.Errorf("%v: %+v", msg, err) 427 restapi.WriteJSON(w, http.StatusInternalServerError, responseError{Message: msg}) 428 return 429 } 430 431 result := versionStatusByBuildContent{ 432 Id: versionId, 433 Builds: make(map[string]versionStatusByBuild, len(builds)), 434 } 435 436 for _, build := range builds { 437 statuses := make(versionStatusByBuild, len(build.Tasks)) 438 for _, task := range build.Tasks { 439 status := versionStatus{ 440 Id: task.Id, 441 Status: task.Status, 442 TimeTaken: task.TimeTaken, 443 } 444 statuses[task.DisplayName] = status 445 } 446 result.Builds[build.BuildVariant] = statuses 447 } 448 449 restapi.WriteJSON(w, http.StatusOK, result) 450 return 451 452 } 453 454 // lastGreen returns the most recent version for which the supplied variants completely pass. 455 func (ra *restAPI) lastGreen(w http.ResponseWriter, r *http.Request) { 456 projCtx := MustHaveRESTContext(r) 457 p := projCtx.Project 458 if p == nil { 459 http.Error(w, "project not found", http.StatusNotFound) 460 return 461 } 462 463 // queryParams should list build variants, example: 464 // GET /rest/v1/projects/mongodb-mongo-master/last_green?linux-64=1&windows-64=1 465 queryParams := r.URL.Query() 466 467 // Make sure all query params are valid variants and put them in an array 468 var bvs []string 469 for key := range queryParams { 470 if p.FindBuildVariant(key) != nil { 471 bvs = append(bvs, key) 472 } else { 473 msg := fmt.Sprintf("build variant '%v' does not exist", key) 474 http.Error(w, msg, http.StatusNotFound) 475 return 476 } 477 } 478 479 // Get latest version for which all the given build variants passed. 480 version, err := model.FindLastPassingVersionForBuildVariants(p, bvs) 481 if err != nil { 482 ra.LoggedError(w, r, http.StatusInternalServerError, err) 483 return 484 } 485 if version == nil { 486 msg := fmt.Sprintf("Couldn't find latest green version for %v", strings.Join(bvs, ", ")) 487 http.Error(w, msg, http.StatusNotFound) 488 return 489 } 490 491 ra.WriteJSON(w, http.StatusOK, version) 492 }