github.com/billybanfield/evergreen@v0.0.0-20170525200750-eeee692790f7/service/models.go (about) 1 package service 2 3 import ( 4 "fmt" 5 "html/template" 6 "time" 7 8 "github.com/evergreen-ci/evergreen" 9 "github.com/evergreen-ci/evergreen/apimodels" 10 "github.com/evergreen-ci/evergreen/db" 11 "github.com/evergreen-ci/evergreen/model" 12 "github.com/evergreen-ci/evergreen/model/build" 13 "github.com/evergreen-ci/evergreen/model/distro" 14 "github.com/evergreen-ci/evergreen/model/host" 15 "github.com/evergreen-ci/evergreen/model/patch" 16 "github.com/evergreen-ci/evergreen/model/task" 17 "github.com/evergreen-ci/evergreen/model/version" 18 "github.com/evergreen-ci/evergreen/plugin" 19 "github.com/mongodb/grip" 20 "github.com/pkg/errors" 21 "gopkg.in/mgo.v2/bson" 22 ) 23 24 type timelineData struct { 25 TotalVersions int 26 Versions []uiVersion 27 } 28 29 type hostsData struct { 30 Hosts []uiHost 31 } 32 33 type pluginData struct { 34 Includes []template.HTML 35 Panels plugin.PanelLayout 36 Data map[string]interface{} 37 } 38 39 type uiVersion struct { 40 Version version.Version 41 Builds []uiBuild 42 PatchInfo *uiPatch `json:",omitempty"` 43 ActiveTasks int 44 RepoOwner string `json:"repo_owner"` 45 Repo string `json:"repo_name"` 46 } 47 48 type uiPatch struct { 49 Patch patch.Patch 50 StatusDiffs interface{} 51 52 // only used on task pages 53 BaseTimeTaken time.Duration `json:"base_time_taken"` 54 55 // for linking to other pages 56 BaseVersionId string 57 BaseBuildId string 58 BaseTaskId string 59 } 60 61 type uiHost struct { 62 Host host.Host 63 RunningTask *task.Task 64 } 65 66 type uiBuild struct { 67 Build build.Build 68 Version version.Version 69 PatchInfo *uiPatch `json:",omitempty"` 70 Tasks []uiTask 71 Elapsed time.Duration 72 CurrentTime int64 73 RepoOwner string `json:"repo_owner"` 74 Repo string `json:"repo_name"` 75 TaskStatusCount taskStatusCount `json:"taskStatusCount"` 76 } 77 78 // taskStatusCount holds all the counts for task statuses for a given build. 79 type taskStatusCount struct { 80 Succeeded int `json:"succeeded"` 81 Failed int `json:"failed"` 82 Started int `json:"started"` 83 Undispatched int `json:"undispatched"` 84 Inactive int `json:"inactive"` 85 Dispatched int `json:"dispatched"` 86 TimedOut int `json:"timed_out"` 87 } 88 89 func (tsc *taskStatusCount) incrementStatus(status string, statusDetails apimodels.TaskEndDetail) { 90 switch status { 91 case evergreen.TaskSucceeded: 92 tsc.Succeeded++ 93 case evergreen.TaskFailed: 94 if statusDetails.TimedOut && statusDetails.Description == "heartbeat" { 95 tsc.TimedOut++ 96 } else { 97 tsc.Failed++ 98 } 99 case evergreen.TaskStarted, evergreen.TaskDispatched: 100 tsc.Started++ 101 case evergreen.TaskUndispatched: 102 tsc.Undispatched++ 103 case evergreen.TaskInactive: 104 tsc.Inactive++ 105 } 106 } 107 108 type uiTask struct { 109 Task task.Task 110 Gitspec string 111 BuildDisplay string 112 TaskLog []model.LogMessage 113 NextTasks []task.Task 114 PreviousTasks []task.Task 115 Elapsed time.Duration 116 StartTime int64 117 FailedTestNames []string `json:"failed_test_names"` 118 } 119 120 func PopulateUIVersion(version *version.Version) (*uiVersion, error) { 121 buildIds := version.BuildIds 122 dbBuilds, err := build.Find(build.ByIds(buildIds)) 123 if err != nil { 124 return nil, err 125 } 126 127 buildsMap := make(map[string]build.Build) 128 for _, dbBuild := range dbBuilds { 129 buildsMap[dbBuild.Id] = dbBuild 130 } 131 132 uiBuilds := make([]uiBuild, len(dbBuilds)) 133 for buildIdx, buildId := range buildIds { 134 build := buildsMap[buildId] 135 buildAsUI := uiBuild{Build: build} 136 137 //Use the build's task cache, instead of querying for each individual task. 138 uiTasks := make([]uiTask, len(build.Tasks)) 139 for i, t := range build.Tasks { 140 uiTasks[i] = uiTask{ 141 Task: task.Task{ 142 Id: t.Id, 143 Status: t.Status, 144 Details: t.StatusDetails, 145 DisplayName: t.DisplayName, 146 }, 147 } 148 } 149 150 buildAsUI.Tasks = uiTasks 151 uiBuilds[buildIdx] = buildAsUI 152 } 153 return &uiVersion{Version: (*version), Builds: uiBuilds}, nil 154 } 155 156 /////////////////////////////////////////////////////////////////////////// 157 //// Functions to create and populate the models 158 /////////////////////////////////////////////////////////////////////////// 159 160 func getTimelineData(projectName, requester string, versionsToSkip, versionsPerPage int) (*timelineData, error) { 161 data := &timelineData{} 162 163 // get the total number of versions in the database (used for pagination) 164 totalVersions, err := version.Count(version.ByProjectId(projectName)) 165 if err != nil { 166 return nil, err 167 } 168 data.TotalVersions = totalVersions 169 170 q := version.ByMostRecentForRequester(projectName, requester).WithoutFields(version.ConfigKey). 171 Skip(versionsToSkip * versionsPerPage).Limit(versionsPerPage) 172 173 // get the most recent versions, to display in their entirety on the page 174 versionsFromDB, err := version.Find(q) 175 if err != nil { 176 return nil, err 177 } 178 179 // create the necessary uiVersion struct for each version 180 uiVersions := make([]uiVersion, len(versionsFromDB)) 181 for versionIdx, version := range versionsFromDB { 182 versionAsUI := uiVersion{Version: version} 183 uiVersions[versionIdx] = versionAsUI 184 185 buildIds := version.BuildIds 186 dbBuilds, err := build.Find(build.ByIds(buildIds)) 187 grip.ErrorWhenln(err != nil, "Ids:", buildIds) 188 189 buildsMap := make(map[string]build.Build) 190 for _, dbBuild := range dbBuilds { 191 buildsMap[dbBuild.Id] = dbBuild 192 } 193 194 uiBuilds := make([]uiBuild, len(dbBuilds)) 195 for buildIdx, buildId := range buildIds { 196 build := buildsMap[buildId] 197 buildAsUI := uiBuild{Build: build} 198 uiBuilds[buildIdx] = buildAsUI 199 } 200 versionAsUI.Builds = uiBuilds 201 uiVersions[versionIdx] = versionAsUI 202 } 203 204 data.Versions = uiVersions 205 return data, nil 206 } 207 208 // getBuildVariantHistory returns a slice of builds that surround a given build. 209 // As many as 'before' builds (less recent builds) plus as many as 'after' builds 210 // (more recent builds) are returned. 211 func getBuildVariantHistory(buildId string, before int, after int) ([]build.Build, error) { 212 b, err := build.FindOne(build.ById(buildId)) 213 if err != nil { 214 return nil, errors.WithStack(err) 215 } 216 if b == nil { 217 return nil, errors.Errorf("no build with id %v", buildId) 218 } 219 220 lessRecentBuilds, err := build.Find( 221 build.ByBeforeRevision(b.Project, b.BuildVariant, b.RevisionOrderNumber). 222 WithFields(build.IdKey, build.TasksKey, build.StatusKey, build.VersionKey, build.ActivatedKey). 223 Limit(before)) 224 if err != nil { 225 return nil, errors.WithStack(err) 226 } 227 228 moreRecentBuilds, err := build.Find( 229 build.ByAfterRevision(b.Project, b.BuildVariant, b.RevisionOrderNumber). 230 WithFields(build.IdKey, build.TasksKey, build.StatusKey, build.VersionKey, build.ActivatedKey). 231 Limit(after)) 232 if err != nil { 233 return nil, errors.WithStack(err) 234 } 235 236 builds := make([]build.Build, 0, len(lessRecentBuilds)+len(moreRecentBuilds)) 237 for i := len(moreRecentBuilds); i > 0; i-- { 238 builds = append(builds, moreRecentBuilds[i-1]) 239 } 240 builds = append(builds, lessRecentBuilds...) 241 return builds, nil 242 } 243 244 // Given build id, get last successful build before this one 245 func getBuildVariantHistoryLastSuccess(buildId string) (*build.Build, error) { 246 b, err := build.FindOne(build.ById(buildId)) 247 if err != nil { 248 return nil, errors.WithStack(err) 249 } 250 if b.Status == evergreen.BuildSucceeded { 251 return b, nil 252 } 253 b, err = b.PreviousSuccessful() 254 return b, errors.WithStack(err) 255 } 256 257 func getVersionHistory(versionId string, N int) ([]version.Version, error) { 258 v, err := version.FindOne(version.ById(versionId)) 259 if err != nil { 260 return nil, errors.WithStack(err) 261 } else if v == nil { 262 return nil, errors.Errorf("Version '%v' not found", versionId) 263 } 264 265 // Versions in the same push event, assuming that no two push events happen at the exact same time 266 // Never want more than 2N+1 versions, so make sure we add a limit 267 268 siblingVersions, err := version.Find(db.Query( 269 bson.M{ 270 version.RevisionOrderNumberKey: v.RevisionOrderNumber, 271 version.RequesterKey: evergreen.RepotrackerVersionRequester, 272 version.IdentifierKey: v.Identifier, 273 }).WithoutFields(version.ConfigKey).Sort([]string{version.RevisionOrderNumberKey}).Limit(2*N + 1)) 274 if err != nil { 275 return nil, errors.WithStack(err) 276 } 277 278 versionIndex := -1 279 for i := 0; i < len(siblingVersions); i++ { 280 if siblingVersions[i].Id == v.Id { 281 versionIndex = i 282 } 283 } 284 285 numSiblings := len(siblingVersions) - 1 286 versions := siblingVersions 287 288 if versionIndex < N { 289 // There are less than N later versions from the same push event 290 // N subsequent versions plus the specified one 291 subsequentVersions, err := version.Find( 292 //TODO encapsulate this query in version pkg 293 db.Query(bson.M{ 294 version.RevisionOrderNumberKey: bson.M{"$gt": v.RevisionOrderNumber}, 295 version.RequesterKey: evergreen.RepotrackerVersionRequester, 296 version.IdentifierKey: v.Identifier, 297 }).WithoutFields(version.ConfigKey).Sort([]string{version.RevisionOrderNumberKey}).Limit(N - versionIndex)) 298 if err != nil { 299 return nil, errors.WithStack(err) 300 } 301 302 // Reverse the second array so we have the versions ordered "newest one first" 303 for i := 0; i < len(subsequentVersions)/2; i++ { 304 subsequentVersions[i], subsequentVersions[len(subsequentVersions)-1-i] = subsequentVersions[len(subsequentVersions)-1-i], subsequentVersions[i] 305 } 306 307 versions = append(subsequentVersions, versions...) 308 } 309 310 if numSiblings-versionIndex < N { 311 previousVersions, err := version.Find(db.Query(bson.M{ 312 version.RevisionOrderNumberKey: bson.M{"$lt": v.RevisionOrderNumber}, 313 version.RequesterKey: evergreen.RepotrackerVersionRequester, 314 version.IdentifierKey: v.Identifier, 315 }).WithoutFields(version.ConfigKey).Sort([]string{fmt.Sprintf("-%v", version.RevisionOrderNumberKey)}).Limit(N)) 316 if err != nil { 317 return nil, errors.WithStack(err) 318 } 319 versions = append(versions, previousVersions...) 320 } 321 322 return versions, nil 323 } 324 325 func getHostsData(includeSpawnedHosts bool) (*hostsData, error) { 326 data := &hostsData{} 327 328 // get all of the hosts 329 var dbHosts []host.Host 330 var err error 331 if includeSpawnedHosts { 332 dbHosts, err = host.Find(host.IsRunning) 333 } else { 334 dbHosts, err = host.Find(host.ByUserWithRunningStatus(evergreen.User)) 335 } 336 337 if err != nil { 338 return nil, errors.WithStack(err) 339 } 340 341 // convert the hosts to the ui models 342 uiHosts := make([]uiHost, len(dbHosts)) 343 for idx, dbHost := range dbHosts { 344 // we only need the distro id for the hosts page 345 dbHost.Distro = distro.Distro{Id: dbHost.Distro.Id} 346 host := uiHost{ 347 Host: dbHost, 348 RunningTask: nil, 349 } 350 351 uiHosts[idx] = host 352 // get the task running on this host 353 if dbHost.RunningTask != "" { 354 task, err := task.FindOne(task.ById(dbHost.RunningTask)) 355 if err != nil { 356 return nil, errors.WithStack(err) 357 } 358 grip.ErrorWhenf(task == nil, "Hosts page could not find task %s for host %s", 359 dbHost.RunningTask, dbHost.Id) 360 uiHosts[idx].RunningTask = task 361 } 362 } 363 data.Hosts = uiHosts 364 return data, nil 365 } 366 367 // getPluginDataAndHTML returns all data needed to properly render plugins 368 // for a page. It logs errors but does not return them, as plugin errors 369 // cannot stop the rendering of the rest of the page 370 func getPluginDataAndHTML(pluginManager plugin.PanelManager, page plugin.PageScope, ctx plugin.UIContext) pluginData { 371 includes, err := pluginManager.Includes(page) 372 if err != nil { 373 grip.Errorf("error getting include html from plugin manager on %v page: %v", 374 page, err) 375 } 376 377 panels, err := pluginManager.Panels(page) 378 if err != nil { 379 grip.Errorf("error getting panel html from plugin manager on %v page: %v", 380 page, err) 381 } 382 383 data, err := pluginManager.UIData(ctx, page) 384 if err != nil { 385 grip.Errorf("error getting plugin data on %v page: %+v", page, err) 386 } 387 388 return pluginData{includes, panels, data} 389 }