github.com/billybanfield/evergreen@v0.0.0-20170525200750-eeee692790f7/model/task_lifecycle.go (about) 1 package model 2 3 import ( 4 "fmt" 5 "time" 6 7 "github.com/evergreen-ci/evergreen" 8 "github.com/evergreen-ci/evergreen/apimodels" 9 "github.com/evergreen-ci/evergreen/model/build" 10 "github.com/evergreen-ci/evergreen/model/event" 11 "github.com/evergreen-ci/evergreen/model/patch" 12 "github.com/evergreen-ci/evergreen/model/task" 13 "github.com/evergreen-ci/evergreen/util" 14 "github.com/mongodb/grip" 15 "github.com/pkg/errors" 16 ) 17 18 func SetActiveState(taskId string, caller string, active bool) error { 19 t, err := task.FindOne(task.ById(taskId)) 20 if err != nil { 21 return err 22 } 23 if active { 24 // if the task is being activated, make sure to activate all of the task's 25 // dependencies as well 26 for _, dep := range t.DependsOn { 27 if err = SetActiveState(dep.TaskId, caller, true); err != nil { 28 return errors.Wrapf(err, "error activating dependency for %v with id %v", 29 taskId, dep.TaskId) 30 } 31 } 32 33 if t.DispatchTime != util.ZeroTime && t.Status == evergreen.TaskUndispatched { 34 err = resetTask(t.Id) 35 if err != nil { 36 return errors.Wrap(err, "error resetting task") 37 } 38 } else { 39 err = t.ActivateTask(caller) 40 if err != nil { 41 return errors.Wrap(err, "error while activating task") 42 } 43 } 44 // If the task was not activated by step back, and either the caller is not evergreen 45 // or the task was originally activated by evergreen, deactivate the task 46 } else if !evergreen.IsSystemActivator(caller) || evergreen.IsSystemActivator(t.ActivatedBy) { 47 // We are trying to deactivate this task 48 // So we check if the person trying to deactivate is evergreen. 49 // If it is not, then we can deactivate it. 50 // Otherwise, if it was originally activated by evergreen, anything can 51 // decativate it. 52 53 err = t.DeactivateTask(caller) 54 if err != nil { 55 return errors.Wrap(err, "error deactivating task") 56 } 57 } else { 58 return nil 59 } 60 61 if active { 62 event.LogTaskActivated(taskId, caller) 63 } else { 64 event.LogTaskDeactivated(taskId, caller) 65 } 66 return errors.WithStack(build.SetCachedTaskActivated(t.BuildId, taskId, active)) 67 } 68 69 // ActivatePreviousTask will set the Active state for the first task with a 70 // revision order number less than the current task's revision order number. 71 func ActivatePreviousTask(taskId, caller string) error { 72 // find the task first 73 t, err := task.FindOne(task.ById(taskId)) 74 if err != nil { 75 return errors.WithStack(err) 76 } 77 78 // find previous task limiting to just the last one 79 prevTask, err := task.FindOne(task.ByBeforeRevision(t.RevisionOrderNumber, t.BuildVariant, t.DisplayName, t.Project, t.Requester)) 80 if err != nil { 81 return errors.Wrap(err, "Error finding previous task") 82 } 83 84 // if this is the first time we're running the task, or it's finished or it is blacklisted 85 if prevTask == nil || task.IsFinished(*prevTask) || prevTask.Priority < 0 { 86 return nil 87 } 88 89 // activate the task 90 return errors.WithStack(SetActiveState(prevTask.Id, caller, true)) 91 } 92 93 // reset task finds a task, attempts to archive it, and resets the task and resets the TaskCache in the build as well. 94 func resetTask(taskId string) error { 95 t, err := task.FindOne(task.ById(taskId)) 96 if err != nil { 97 return errors.WithStack(err) 98 } 99 100 if err = t.Archive(); err != nil { 101 return errors.Wrap(err, "can't restart task because it can't be archived") 102 } 103 104 if err = t.Reset(); err != nil { 105 return errors.WithStack(err) 106 } 107 108 // update the cached version of the task, in its build document 109 if err = build.ResetCachedTask(t.BuildId, t.Id); err != nil { 110 return errors.WithStack(err) 111 } 112 113 return errors.WithStack(UpdateBuildAndVersionStatusForTask(t.Id)) 114 } 115 116 // TryResetTask resets a task 117 func TryResetTask(taskId, user, origin string, p *Project, detail *apimodels.TaskEndDetail) error { 118 t, err := task.FindOne(task.ById(taskId)) 119 if err != nil { 120 return errors.WithStack(err) 121 } 122 // if we've reached the max number of executions for this task, mark it as finished and failed 123 if t.Execution >= evergreen.MaxTaskExecution { 124 // restarting from the UI bypasses the restart cap 125 message := fmt.Sprintf("Task '%v' reached max execution (%v):", t.Id, evergreen.MaxTaskExecution) 126 if origin == evergreen.UIPackage || origin == evergreen.RESTV2Package { 127 grip.Debugln(message, "allowing exception for", user) 128 } else { 129 grip.Debugln(message, "marking as failed") 130 if detail != nil { 131 return errors.WithStack(MarkEnd(t.Id, origin, time.Now(), detail, p, false)) 132 } else { 133 panic(fmt.Sprintf("TryResetTask called with nil TaskEndDetail by %s", origin)) 134 } 135 } 136 } 137 138 // only allow re-execution for failed or successful tasks 139 if !task.IsFinished(*t) { 140 // this is to disallow terminating running tasks via the UI 141 if origin == evergreen.UIPackage || origin == evergreen.RESTV2Package { 142 grip.Debugf("Unsatisfiable '%s' reset request on '%s' (status: '%s')", 143 user, t.Id, t.Status) 144 return errors.Errorf("Task '%v' is currently '%v' - cannot reset task in this status", 145 t.Id, t.Status) 146 } 147 } 148 149 if detail != nil { 150 if err = t.MarkEnd(time.Now(), detail); err != nil { 151 return errors.Wrap(err, "Error marking task as ended") 152 } 153 } 154 155 if err = resetTask(t.Id); err == nil { 156 if origin == evergreen.UIPackage || origin == evergreen.RESTV2Package { 157 event.LogTaskRestarted(t.Id, user) 158 } else { 159 event.LogTaskRestarted(t.Id, origin) 160 } 161 } 162 return errors.WithStack(err) 163 } 164 165 func AbortTask(taskId, caller string) error { 166 t, err := task.FindOne(task.ById(taskId)) 167 if err != nil { 168 return err 169 } 170 171 if !task.IsAbortable(*t) { 172 return errors.Errorf("Task '%v' is currently '%v' - cannot abort task"+ 173 " in this status", t.Id, t.Status) 174 } 175 176 grip.Debugln("Aborting task", t.Id) 177 // set the active state and then set the abort 178 if err = SetActiveState(t.Id, caller, false); err != nil { 179 return err 180 } 181 event.LogTaskAbortRequest(t.Id, caller) 182 return t.SetAborted() 183 } 184 185 // Deactivate any previously activated but undispatched 186 // tasks for the same build variant + display name + project combination 187 // as the task. 188 func DeactivatePreviousTasks(taskId, caller string) error { 189 t, err := task.FindOne(task.ById(taskId)) 190 if err != nil { 191 return err 192 } 193 statuses := []string{evergreen.TaskUndispatched} 194 allTasks, err := task.Find(task.ByActivatedBeforeRevisionWithStatuses(t.RevisionOrderNumber, statuses, t.BuildVariant, 195 t.DisplayName, t.Project)) 196 if err != nil { 197 return err 198 } 199 for _, t := range allTasks { 200 err = SetActiveState(t.Id, caller, false) 201 if err != nil { 202 return err 203 } 204 event.LogTaskDeactivated(t.Id, caller) 205 // update the cached version of the task, in its build document to be deactivated 206 if err = build.SetCachedTaskActivated(t.BuildId, t.Id, false); err != nil { 207 return err 208 } 209 } 210 211 return nil 212 } 213 214 // Returns true if the task should stepback upon failure, and false 215 // otherwise. Note that the setting is obtained from the top-level 216 // project, if not explicitly set on the task. 217 func getStepback(taskId string, project *Project) (bool, error) { 218 t, err := task.FindOne(task.ById(taskId)) 219 if err != nil { 220 return false, err 221 } 222 223 projectTask := project.FindProjectTask(t.DisplayName) 224 225 // Check if the task overrides the stepback policy specified by the project 226 if projectTask != nil && projectTask.Stepback != nil { 227 return *projectTask.Stepback, nil 228 } 229 230 // Check if the build variant overrides the stepback policy specified by the project 231 for _, buildVariant := range project.BuildVariants { 232 if t.BuildVariant == buildVariant.Name { 233 if buildVariant.Stepback != nil { 234 return *buildVariant.Stepback, nil 235 } 236 break 237 } 238 } 239 240 return project.Stepback, nil 241 } 242 243 // doStepBack performs a stepback on the task if there is a previous task and if not it returns nothing. 244 func doStepback(t *task.Task) error { 245 //See if there is a prior success for this particular task. 246 //If there isn't, we should not activate the previous task because 247 //it could trigger stepping backwards ad infinitum. 248 prevTask, err := t.PreviousCompletedTask(t.Project, []string{evergreen.TaskSucceeded}) 249 if prevTask == nil { 250 return nil 251 } 252 if err != nil { 253 return errors.Wrap(err, "Error locating previous successful task") 254 } 255 256 // activate the previous task to pinpoint regression 257 return errors.WithStack(ActivatePreviousTask(t.Id, evergreen.StepbackTaskActivator)) 258 } 259 260 // MarkEnd updates the task as being finished, performs a stepback if necessary, and updates the build status 261 func MarkEnd(taskId, caller string, finishTime time.Time, detail *apimodels.TaskEndDetail, 262 p *Project, deactivatePrevious bool) error { 263 264 t, err := task.FindOne(task.ById(taskId)) 265 if err != nil { 266 return err 267 } 268 if t == nil { 269 return errors.Errorf("Task not found for taskId: %s", taskId) 270 } 271 272 for _, result := range t.TestResults { 273 if result.Status == evergreen.TestFailedStatus { 274 detail.Status = evergreen.TaskFailed 275 break 276 } 277 } 278 279 t.Details = *detail 280 281 if t.Status == detail.Status { 282 grip.Warningf("Tried to mark task %s as finished twice", t.Id) 283 return nil 284 } 285 286 err = t.MarkEnd(finishTime, detail) 287 if err != nil { 288 return err 289 } 290 status := t.UIStatus() 291 event.LogTaskFinished(t.Id, t.HostId, status) 292 293 // update the cached version of the task, in its build document 294 err = build.SetCachedTaskFinished(t.BuildId, t.Id, detail, t.TimeTaken) 295 if err != nil { 296 return errors.Wrap(err, "error updating build") 297 } 298 299 // no need to activate/deactivate other task if this is a patch request's task 300 if t.Requester == evergreen.PatchVersionRequester { 301 return errors.Wrap(UpdateBuildAndVersionStatusForTask(t.Id), 302 "Error updating build status (1)") 303 } 304 if detail.Status == evergreen.TaskFailed { 305 shouldStepBack, err := getStepback(t.Id, p) 306 if err != nil { 307 return errors.WithStack(err) 308 } 309 if shouldStepBack { 310 if err = doStepback(t); err != nil { 311 return errors.Wrap(err, "Error during step back") 312 } 313 } else { 314 grip.Debugln("Not stepping backwards on task failure:", t.Id) 315 } 316 } else if deactivatePrevious { 317 // if the task was successful, ignore running previous 318 // activated tasks for this buildvariant 319 320 if err = DeactivatePreviousTasks(t.Id, caller); err != nil { 321 return errors.Wrap(err, "Error deactivating previous task") 322 } 323 } 324 325 // update the build 326 if err := UpdateBuildAndVersionStatusForTask(t.Id); err != nil { 327 return errors.Wrap(err, "Error updating build status (2)") 328 } 329 330 return nil 331 } 332 333 // updateMakespans 334 func updateMakespans(b *build.Build) error { 335 // find all tasks associated with the build 336 tasks, err := task.Find(task.ByBuildId(b.Id)) 337 if err != nil { 338 return errors.WithStack(err) 339 } 340 341 depPath := FindPredictedMakespan(tasks) 342 return errors.WithStack(b.UpdateMakespans(depPath.TotalTime, CalculateActualMakespan(tasks))) 343 } 344 345 // UpdateBuildStatusForTask finds all the builds for a task and updates the 346 // status of the build based on the task's status. 347 func UpdateBuildAndVersionStatusForTask(taskId string) error { 348 // retrieve the task by the task id 349 t, err := task.FindOne(task.ById(taskId)) 350 if err != nil { 351 return errors.WithStack(err) 352 } 353 354 finishTime := time.Now() 355 // get all of the tasks in the same build 356 b, err := build.FindOne(build.ById(t.BuildId)) 357 if err != nil { 358 return errors.WithStack(err) 359 } 360 361 buildTasks, err := task.Find(task.ByBuildId(b.Id)) 362 if err != nil { 363 return errors.WithStack(err) 364 } 365 366 pushTaskExists := false 367 for _, t := range buildTasks { 368 if t.DisplayName == evergreen.PushStage { 369 pushTaskExists = true 370 } 371 } 372 373 failedTask := false 374 pushSuccess := true 375 pushCompleted := false 376 finishedTasks := 0 377 378 // update the build's status based on tasks for this build 379 for _, t := range buildTasks { 380 if task.IsFinished(t) { 381 finishedTasks++ 382 383 // if it was a compile task, mark the build status accordingly 384 if t.DisplayName == evergreen.CompileStage { 385 if t.Status != evergreen.TaskSucceeded { 386 failedTask = true 387 finishedTasks = -1 388 err = b.MarkFinished(evergreen.BuildFailed, finishTime) 389 if err != nil { 390 err = errors.Wrap(err, "Error marking build as finished") 391 grip.Error(err) 392 return err 393 } 394 break 395 } 396 } else if t.DisplayName == evergreen.PushStage { 397 pushCompleted = true 398 // if it's a finished push, check if it was successful 399 if t.Status != evergreen.TaskSucceeded { 400 err = b.UpdateStatus(evergreen.BuildFailed) 401 if err != nil { 402 err = errors.Wrap(err, "Error updating build status") 403 grip.Error(err) 404 return err 405 } 406 pushSuccess = false 407 } 408 } else { 409 // update the build's status when a test task isn't successful 410 if t.Status != evergreen.TaskSucceeded { 411 err = b.UpdateStatus(evergreen.BuildFailed) 412 if err != nil { 413 err = errors.Wrap(err, "Error updating build status") 414 grip.Error(err) 415 return err 416 } 417 failedTask = true 418 } 419 420 // update the cached version of the task, in its build document 421 err = build.SetCachedTaskFinished(t.BuildId, t.Id, &t.Details, t.TimeTaken) 422 if err != nil { 423 return fmt.Errorf("error updating build: %v", err.Error()) 424 } 425 426 } 427 } 428 } 429 430 // if there are no failed tasks, mark the build as started 431 if !failedTask { 432 if err = b.UpdateStatus(evergreen.BuildStarted); err != nil { 433 err = errors.Wrap(err, "Error updating build status") 434 grip.Error(err) 435 return err 436 } 437 } 438 439 // if a compile task didn't fail, then the 440 // build is only finished when both the compile 441 // and test tasks are completed or when those are 442 // both completed in addition to a push (a push 443 // does not occur if there's a failed task) 444 if finishedTasks >= len(buildTasks)-1 { 445 446 if !failedTask { 447 if pushTaskExists { // this build has a push task associated with it. 448 if pushCompleted && pushSuccess { // the push succeeded, so mark the build as succeeded. 449 err = b.MarkFinished(evergreen.BuildSucceeded, finishTime) 450 if err != nil { 451 err = errors.Wrap(err, "Error marking build as finished") 452 grip.Error(err) 453 return err 454 } 455 } else if pushCompleted && !pushSuccess { // the push failed, mark build failed. 456 err = b.MarkFinished(evergreen.BuildFailed, finishTime) 457 if err != nil { 458 err = errors.Wrap(err, "Error marking build as finished") 459 grip.Error(err) 460 return err 461 } 462 } 463 464 // Otherwise, this build does have a "push" task, but it hasn't finished yet 465 // So do nothing, since we don't know the status yet. 466 467 if err = MarkVersionCompleted(b.Version, finishTime); err != nil { 468 err = errors.Wrap(err, "Error marking version as finished") 469 grip.Error(err) 470 return err 471 } 472 } else { // this build has no push task. so go ahead and mark it success/failure. 473 if err = b.MarkFinished(evergreen.BuildSucceeded, finishTime); err != nil { 474 err = errors.Wrap(err, "Error marking build as finished") 475 grip.Error(err) 476 return err 477 } 478 if b.Requester == evergreen.PatchVersionRequester { 479 if err = TryMarkPatchBuildFinished(b, finishTime); err != nil { 480 err = errors.Wrap(err, "Error marking patch as finished") 481 grip.Error(err) 482 return err 483 } 484 } 485 if err = MarkVersionCompleted(b.Version, finishTime); err != nil { 486 err = errors.Wrap(err, "Error marking version as finished") 487 grip.Error(err) 488 return err 489 } 490 } 491 } else { 492 // some task failed 493 if err = b.MarkFinished(evergreen.BuildFailed, finishTime); err != nil { 494 err = errors.Wrap(err, "Error marking build as finished") 495 grip.Error(err) 496 return err 497 } 498 if b.Requester == evergreen.PatchVersionRequester { 499 if err = TryMarkPatchBuildFinished(b, finishTime); err != nil { 500 err = errors.Wrap(err, "Error marking patch as finished") 501 grip.Error(err) 502 return err 503 } 504 } 505 if err = MarkVersionCompleted(b.Version, finishTime); err != nil { 506 err = errors.Wrap(err, "Error marking version as finished") 507 grip.Error(err) 508 return err 509 } 510 } 511 512 // update the build's makespan information if the task has finished 513 if err = updateMakespans(b); err != nil { 514 err = errors.Wrap(err, "Error updating makespan information") 515 grip.Error(err) 516 return err 517 } 518 } 519 520 // this is helpful for when we restart a compile task 521 if finishedTasks == 0 { 522 err = b.UpdateStatus(evergreen.BuildCreated) 523 if err != nil { 524 err = errors.Wrap(err, "Error updating build status") 525 grip.Error(err) 526 return err 527 } 528 } 529 530 return nil 531 } 532 533 // MarkStart updates the task, build, version and if necessary, patch documents with the task start time 534 func MarkStart(taskId string) error { 535 t, err := task.FindOne(task.ById(taskId)) 536 if err != nil { 537 return errors.WithStack(err) 538 } 539 startTime := time.Now() 540 if err = t.MarkStart(startTime); err != nil { 541 return errors.WithStack(err) 542 } 543 event.LogTaskStarted(t.Id) 544 545 // ensure the appropriate build is marked as started if necessary 546 if err = build.TryMarkStarted(t.BuildId, startTime); err != nil { 547 return errors.WithStack(err) 548 } 549 550 // ensure the appropriate version is marked as started if necessary 551 if err = MarkVersionStarted(t.Version, startTime); err != nil { 552 return errors.WithStack(err) 553 } 554 555 // if it's a patch, mark the patch as started if necessary 556 if t.Requester == evergreen.PatchVersionRequester { 557 if err = patch.TryMarkStarted(t.Version, startTime); err != nil { 558 return errors.WithStack(err) 559 } 560 } 561 562 // update the cached version of the task, in its build document 563 return build.SetCachedTaskStarted(t.BuildId, t.Id, startTime) 564 } 565 566 func MarkTaskUndispatched(t *task.Task) error { 567 // record that the task as undispatched on the host 568 if err := t.MarkAsUndispatched(); err != nil { 569 return errors.WithStack(err) 570 } 571 // the task was successfully dispatched, log the event 572 event.LogTaskUndispatched(t.Id, t.HostId) 573 574 // update the cached version of the task in its related build document 575 if err := build.SetCachedTaskUndispatched(t.BuildId, t.Id); err != nil { 576 return errors.WithStack(err) 577 } 578 return nil 579 } 580 581 func MarkTaskDispatched(t *task.Task, hostId, distroId string) error { 582 // record that the task was dispatched on the host 583 if err := t.MarkAsDispatched(hostId, distroId, time.Now()); err != nil { 584 return errors.Wrapf(err, "error marking task %s as dispatched "+ 585 "on host %s", t.Id, hostId) 586 } 587 // the task was successfully dispatched, log the event 588 event.LogTaskDispatched(t.Id, hostId) 589 590 // update the cached version of the task in its related build document 591 if err := build.SetCachedTaskDispatched(t.BuildId, t.Id); err != nil { 592 return errors.Wrapf(err, "error updating task cache in build %s", t.BuildId) 593 } 594 return nil 595 }