github.com/billybanfield/evergreen@v0.0.0-20170525200750-eeee692790f7/model/task_lifecycle.go (about)

     1  package model
     2  
     3  import (
     4  	"fmt"
     5  	"time"
     6  
     7  	"github.com/evergreen-ci/evergreen"
     8  	"github.com/evergreen-ci/evergreen/apimodels"
     9  	"github.com/evergreen-ci/evergreen/model/build"
    10  	"github.com/evergreen-ci/evergreen/model/event"
    11  	"github.com/evergreen-ci/evergreen/model/patch"
    12  	"github.com/evergreen-ci/evergreen/model/task"
    13  	"github.com/evergreen-ci/evergreen/util"
    14  	"github.com/mongodb/grip"
    15  	"github.com/pkg/errors"
    16  )
    17  
    18  func SetActiveState(taskId string, caller string, active bool) error {
    19  	t, err := task.FindOne(task.ById(taskId))
    20  	if err != nil {
    21  		return err
    22  	}
    23  	if active {
    24  		// if the task is being activated, make sure to activate all of the task's
    25  		// dependencies as well
    26  		for _, dep := range t.DependsOn {
    27  			if err = SetActiveState(dep.TaskId, caller, true); err != nil {
    28  				return errors.Wrapf(err, "error activating dependency for %v with id %v",
    29  					taskId, dep.TaskId)
    30  			}
    31  		}
    32  
    33  		if t.DispatchTime != util.ZeroTime && t.Status == evergreen.TaskUndispatched {
    34  			err = resetTask(t.Id)
    35  			if err != nil {
    36  				return errors.Wrap(err, "error resetting task")
    37  			}
    38  		} else {
    39  			err = t.ActivateTask(caller)
    40  			if err != nil {
    41  				return errors.Wrap(err, "error while activating task")
    42  			}
    43  		}
    44  		// If the task was not activated by step back, and either the caller is not evergreen
    45  		// or the task was originally activated by evergreen, deactivate the task
    46  	} else if !evergreen.IsSystemActivator(caller) || evergreen.IsSystemActivator(t.ActivatedBy) {
    47  		// We are trying to deactivate this task
    48  		// So we check if the person trying to deactivate is evergreen.
    49  		// If it is not, then we can deactivate it.
    50  		// Otherwise, if it was originally activated by evergreen, anything can
    51  		// decativate it.
    52  
    53  		err = t.DeactivateTask(caller)
    54  		if err != nil {
    55  			return errors.Wrap(err, "error deactivating task")
    56  		}
    57  	} else {
    58  		return nil
    59  	}
    60  
    61  	if active {
    62  		event.LogTaskActivated(taskId, caller)
    63  	} else {
    64  		event.LogTaskDeactivated(taskId, caller)
    65  	}
    66  	return errors.WithStack(build.SetCachedTaskActivated(t.BuildId, taskId, active))
    67  }
    68  
    69  // ActivatePreviousTask will set the Active state for the first task with a
    70  // revision order number less than the current task's revision order number.
    71  func ActivatePreviousTask(taskId, caller string) error {
    72  	// find the task first
    73  	t, err := task.FindOne(task.ById(taskId))
    74  	if err != nil {
    75  		return errors.WithStack(err)
    76  	}
    77  
    78  	// find previous task limiting to just the last one
    79  	prevTask, err := task.FindOne(task.ByBeforeRevision(t.RevisionOrderNumber, t.BuildVariant, t.DisplayName, t.Project, t.Requester))
    80  	if err != nil {
    81  		return errors.Wrap(err, "Error finding previous task")
    82  	}
    83  
    84  	// if this is the first time we're running the task, or it's finished or it is blacklisted
    85  	if prevTask == nil || task.IsFinished(*prevTask) || prevTask.Priority < 0 {
    86  		return nil
    87  	}
    88  
    89  	// activate the task
    90  	return errors.WithStack(SetActiveState(prevTask.Id, caller, true))
    91  }
    92  
    93  // reset task finds a task, attempts to archive it, and resets the task and resets the TaskCache in the build as well.
    94  func resetTask(taskId string) error {
    95  	t, err := task.FindOne(task.ById(taskId))
    96  	if err != nil {
    97  		return errors.WithStack(err)
    98  	}
    99  
   100  	if err = t.Archive(); err != nil {
   101  		return errors.Wrap(err, "can't restart task because it can't be archived")
   102  	}
   103  
   104  	if err = t.Reset(); err != nil {
   105  		return errors.WithStack(err)
   106  	}
   107  
   108  	// update the cached version of the task, in its build document
   109  	if err = build.ResetCachedTask(t.BuildId, t.Id); err != nil {
   110  		return errors.WithStack(err)
   111  	}
   112  
   113  	return errors.WithStack(UpdateBuildAndVersionStatusForTask(t.Id))
   114  }
   115  
   116  // TryResetTask resets a task
   117  func TryResetTask(taskId, user, origin string, p *Project, detail *apimodels.TaskEndDetail) error {
   118  	t, err := task.FindOne(task.ById(taskId))
   119  	if err != nil {
   120  		return errors.WithStack(err)
   121  	}
   122  	// if we've reached the max number of executions for this task, mark it as finished and failed
   123  	if t.Execution >= evergreen.MaxTaskExecution {
   124  		// restarting from the UI bypasses the restart cap
   125  		message := fmt.Sprintf("Task '%v' reached max execution (%v):", t.Id, evergreen.MaxTaskExecution)
   126  		if origin == evergreen.UIPackage || origin == evergreen.RESTV2Package {
   127  			grip.Debugln(message, "allowing exception for", user)
   128  		} else {
   129  			grip.Debugln(message, "marking as failed")
   130  			if detail != nil {
   131  				return errors.WithStack(MarkEnd(t.Id, origin, time.Now(), detail, p, false))
   132  			} else {
   133  				panic(fmt.Sprintf("TryResetTask called with nil TaskEndDetail by %s", origin))
   134  			}
   135  		}
   136  	}
   137  
   138  	// only allow re-execution for failed or successful tasks
   139  	if !task.IsFinished(*t) {
   140  		// this is to disallow terminating running tasks via the UI
   141  		if origin == evergreen.UIPackage || origin == evergreen.RESTV2Package {
   142  			grip.Debugf("Unsatisfiable '%s' reset request on '%s' (status: '%s')",
   143  				user, t.Id, t.Status)
   144  			return errors.Errorf("Task '%v' is currently '%v' - cannot reset task in this status",
   145  				t.Id, t.Status)
   146  		}
   147  	}
   148  
   149  	if detail != nil {
   150  		if err = t.MarkEnd(time.Now(), detail); err != nil {
   151  			return errors.Wrap(err, "Error marking task as ended")
   152  		}
   153  	}
   154  
   155  	if err = resetTask(t.Id); err == nil {
   156  		if origin == evergreen.UIPackage || origin == evergreen.RESTV2Package {
   157  			event.LogTaskRestarted(t.Id, user)
   158  		} else {
   159  			event.LogTaskRestarted(t.Id, origin)
   160  		}
   161  	}
   162  	return errors.WithStack(err)
   163  }
   164  
   165  func AbortTask(taskId, caller string) error {
   166  	t, err := task.FindOne(task.ById(taskId))
   167  	if err != nil {
   168  		return err
   169  	}
   170  
   171  	if !task.IsAbortable(*t) {
   172  		return errors.Errorf("Task '%v' is currently '%v' - cannot abort task"+
   173  			" in this status", t.Id, t.Status)
   174  	}
   175  
   176  	grip.Debugln("Aborting task", t.Id)
   177  	// set the active state and then set the abort
   178  	if err = SetActiveState(t.Id, caller, false); err != nil {
   179  		return err
   180  	}
   181  	event.LogTaskAbortRequest(t.Id, caller)
   182  	return t.SetAborted()
   183  }
   184  
   185  // Deactivate any previously activated but undispatched
   186  // tasks for the same build variant + display name + project combination
   187  // as the task.
   188  func DeactivatePreviousTasks(taskId, caller string) error {
   189  	t, err := task.FindOne(task.ById(taskId))
   190  	if err != nil {
   191  		return err
   192  	}
   193  	statuses := []string{evergreen.TaskUndispatched}
   194  	allTasks, err := task.Find(task.ByActivatedBeforeRevisionWithStatuses(t.RevisionOrderNumber, statuses, t.BuildVariant,
   195  		t.DisplayName, t.Project))
   196  	if err != nil {
   197  		return err
   198  	}
   199  	for _, t := range allTasks {
   200  		err = SetActiveState(t.Id, caller, false)
   201  		if err != nil {
   202  			return err
   203  		}
   204  		event.LogTaskDeactivated(t.Id, caller)
   205  		// update the cached version of the task, in its build document to be deactivated
   206  		if err = build.SetCachedTaskActivated(t.BuildId, t.Id, false); err != nil {
   207  			return err
   208  		}
   209  	}
   210  
   211  	return nil
   212  }
   213  
   214  // Returns true if the task should stepback upon failure, and false
   215  // otherwise. Note that the setting is obtained from the top-level
   216  // project, if not explicitly set on the task.
   217  func getStepback(taskId string, project *Project) (bool, error) {
   218  	t, err := task.FindOne(task.ById(taskId))
   219  	if err != nil {
   220  		return false, err
   221  	}
   222  
   223  	projectTask := project.FindProjectTask(t.DisplayName)
   224  
   225  	// Check if the task overrides the stepback policy specified by the project
   226  	if projectTask != nil && projectTask.Stepback != nil {
   227  		return *projectTask.Stepback, nil
   228  	}
   229  
   230  	// Check if the build variant overrides the stepback policy specified by the project
   231  	for _, buildVariant := range project.BuildVariants {
   232  		if t.BuildVariant == buildVariant.Name {
   233  			if buildVariant.Stepback != nil {
   234  				return *buildVariant.Stepback, nil
   235  			}
   236  			break
   237  		}
   238  	}
   239  
   240  	return project.Stepback, nil
   241  }
   242  
   243  // doStepBack performs a stepback on the task if there is a previous task and if not it returns nothing.
   244  func doStepback(t *task.Task) error {
   245  	//See if there is a prior success for this particular task.
   246  	//If there isn't, we should not activate the previous task because
   247  	//it could trigger stepping backwards ad infinitum.
   248  	prevTask, err := t.PreviousCompletedTask(t.Project, []string{evergreen.TaskSucceeded})
   249  	if prevTask == nil {
   250  		return nil
   251  	}
   252  	if err != nil {
   253  		return errors.Wrap(err, "Error locating previous successful task")
   254  	}
   255  
   256  	// activate the previous task to pinpoint regression
   257  	return errors.WithStack(ActivatePreviousTask(t.Id, evergreen.StepbackTaskActivator))
   258  }
   259  
   260  // MarkEnd updates the task as being finished, performs a stepback if necessary, and updates the build status
   261  func MarkEnd(taskId, caller string, finishTime time.Time, detail *apimodels.TaskEndDetail,
   262  	p *Project, deactivatePrevious bool) error {
   263  
   264  	t, err := task.FindOne(task.ById(taskId))
   265  	if err != nil {
   266  		return err
   267  	}
   268  	if t == nil {
   269  		return errors.Errorf("Task not found for taskId: %s", taskId)
   270  	}
   271  
   272  	for _, result := range t.TestResults {
   273  		if result.Status == evergreen.TestFailedStatus {
   274  			detail.Status = evergreen.TaskFailed
   275  			break
   276  		}
   277  	}
   278  
   279  	t.Details = *detail
   280  
   281  	if t.Status == detail.Status {
   282  		grip.Warningf("Tried to mark task %s as finished twice", t.Id)
   283  		return nil
   284  	}
   285  
   286  	err = t.MarkEnd(finishTime, detail)
   287  	if err != nil {
   288  		return err
   289  	}
   290  	status := t.UIStatus()
   291  	event.LogTaskFinished(t.Id, t.HostId, status)
   292  
   293  	// update the cached version of the task, in its build document
   294  	err = build.SetCachedTaskFinished(t.BuildId, t.Id, detail, t.TimeTaken)
   295  	if err != nil {
   296  		return errors.Wrap(err, "error updating build")
   297  	}
   298  
   299  	// no need to activate/deactivate other task if this is a patch request's task
   300  	if t.Requester == evergreen.PatchVersionRequester {
   301  		return errors.Wrap(UpdateBuildAndVersionStatusForTask(t.Id),
   302  			"Error updating build status (1)")
   303  	}
   304  	if detail.Status == evergreen.TaskFailed {
   305  		shouldStepBack, err := getStepback(t.Id, p)
   306  		if err != nil {
   307  			return errors.WithStack(err)
   308  		}
   309  		if shouldStepBack {
   310  			if err = doStepback(t); err != nil {
   311  				return errors.Wrap(err, "Error during step back")
   312  			}
   313  		} else {
   314  			grip.Debugln("Not stepping backwards on task failure:", t.Id)
   315  		}
   316  	} else if deactivatePrevious {
   317  		// if the task was successful, ignore running previous
   318  		// activated tasks for this buildvariant
   319  
   320  		if err = DeactivatePreviousTasks(t.Id, caller); err != nil {
   321  			return errors.Wrap(err, "Error deactivating previous task")
   322  		}
   323  	}
   324  
   325  	// update the build
   326  	if err := UpdateBuildAndVersionStatusForTask(t.Id); err != nil {
   327  		return errors.Wrap(err, "Error updating build status (2)")
   328  	}
   329  
   330  	return nil
   331  }
   332  
   333  // updateMakespans
   334  func updateMakespans(b *build.Build) error {
   335  	// find all tasks associated with the build
   336  	tasks, err := task.Find(task.ByBuildId(b.Id))
   337  	if err != nil {
   338  		return errors.WithStack(err)
   339  	}
   340  
   341  	depPath := FindPredictedMakespan(tasks)
   342  	return errors.WithStack(b.UpdateMakespans(depPath.TotalTime, CalculateActualMakespan(tasks)))
   343  }
   344  
   345  // UpdateBuildStatusForTask finds all the builds for a task and updates the
   346  // status of the build based on the task's status.
   347  func UpdateBuildAndVersionStatusForTask(taskId string) error {
   348  	// retrieve the task by the task id
   349  	t, err := task.FindOne(task.ById(taskId))
   350  	if err != nil {
   351  		return errors.WithStack(err)
   352  	}
   353  
   354  	finishTime := time.Now()
   355  	// get all of the tasks in the same build
   356  	b, err := build.FindOne(build.ById(t.BuildId))
   357  	if err != nil {
   358  		return errors.WithStack(err)
   359  	}
   360  
   361  	buildTasks, err := task.Find(task.ByBuildId(b.Id))
   362  	if err != nil {
   363  		return errors.WithStack(err)
   364  	}
   365  
   366  	pushTaskExists := false
   367  	for _, t := range buildTasks {
   368  		if t.DisplayName == evergreen.PushStage {
   369  			pushTaskExists = true
   370  		}
   371  	}
   372  
   373  	failedTask := false
   374  	pushSuccess := true
   375  	pushCompleted := false
   376  	finishedTasks := 0
   377  
   378  	// update the build's status based on tasks for this build
   379  	for _, t := range buildTasks {
   380  		if task.IsFinished(t) {
   381  			finishedTasks++
   382  
   383  			// if it was a compile task, mark the build status accordingly
   384  			if t.DisplayName == evergreen.CompileStage {
   385  				if t.Status != evergreen.TaskSucceeded {
   386  					failedTask = true
   387  					finishedTasks = -1
   388  					err = b.MarkFinished(evergreen.BuildFailed, finishTime)
   389  					if err != nil {
   390  						err = errors.Wrap(err, "Error marking build as finished")
   391  						grip.Error(err)
   392  						return err
   393  					}
   394  					break
   395  				}
   396  			} else if t.DisplayName == evergreen.PushStage {
   397  				pushCompleted = true
   398  				// if it's a finished push, check if it was successful
   399  				if t.Status != evergreen.TaskSucceeded {
   400  					err = b.UpdateStatus(evergreen.BuildFailed)
   401  					if err != nil {
   402  						err = errors.Wrap(err, "Error updating build status")
   403  						grip.Error(err)
   404  						return err
   405  					}
   406  					pushSuccess = false
   407  				}
   408  			} else {
   409  				// update the build's status when a test task isn't successful
   410  				if t.Status != evergreen.TaskSucceeded {
   411  					err = b.UpdateStatus(evergreen.BuildFailed)
   412  					if err != nil {
   413  						err = errors.Wrap(err, "Error updating build status")
   414  						grip.Error(err)
   415  						return err
   416  					}
   417  					failedTask = true
   418  				}
   419  
   420  				// update the cached version of the task, in its build document
   421  				err = build.SetCachedTaskFinished(t.BuildId, t.Id, &t.Details, t.TimeTaken)
   422  				if err != nil {
   423  					return fmt.Errorf("error updating build: %v", err.Error())
   424  				}
   425  
   426  			}
   427  		}
   428  	}
   429  
   430  	// if there are no failed tasks, mark the build as started
   431  	if !failedTask {
   432  		if err = b.UpdateStatus(evergreen.BuildStarted); err != nil {
   433  			err = errors.Wrap(err, "Error updating build status")
   434  			grip.Error(err)
   435  			return err
   436  		}
   437  	}
   438  
   439  	// if a compile task didn't fail, then the
   440  	// build is only finished when both the compile
   441  	// and test tasks are completed or when those are
   442  	// both completed in addition to a push (a push
   443  	// does not occur if there's a failed task)
   444  	if finishedTasks >= len(buildTasks)-1 {
   445  
   446  		if !failedTask {
   447  			if pushTaskExists { // this build has a push task associated with it.
   448  				if pushCompleted && pushSuccess { // the push succeeded, so mark the build as succeeded.
   449  					err = b.MarkFinished(evergreen.BuildSucceeded, finishTime)
   450  					if err != nil {
   451  						err = errors.Wrap(err, "Error marking build as finished")
   452  						grip.Error(err)
   453  						return err
   454  					}
   455  				} else if pushCompleted && !pushSuccess { // the push failed, mark build failed.
   456  					err = b.MarkFinished(evergreen.BuildFailed, finishTime)
   457  					if err != nil {
   458  						err = errors.Wrap(err, "Error marking build as finished")
   459  						grip.Error(err)
   460  						return err
   461  					}
   462  				}
   463  
   464  				// Otherwise, this build does have a "push" task, but it hasn't finished yet
   465  				// So do nothing, since we don't know the status yet.
   466  
   467  				if err = MarkVersionCompleted(b.Version, finishTime); err != nil {
   468  					err = errors.Wrap(err, "Error marking version as finished")
   469  					grip.Error(err)
   470  					return err
   471  				}
   472  			} else { // this build has no push task. so go ahead and mark it success/failure.
   473  				if err = b.MarkFinished(evergreen.BuildSucceeded, finishTime); err != nil {
   474  					err = errors.Wrap(err, "Error marking build as finished")
   475  					grip.Error(err)
   476  					return err
   477  				}
   478  				if b.Requester == evergreen.PatchVersionRequester {
   479  					if err = TryMarkPatchBuildFinished(b, finishTime); err != nil {
   480  						err = errors.Wrap(err, "Error marking patch as finished")
   481  						grip.Error(err)
   482  						return err
   483  					}
   484  				}
   485  				if err = MarkVersionCompleted(b.Version, finishTime); err != nil {
   486  					err = errors.Wrap(err, "Error marking version as finished")
   487  					grip.Error(err)
   488  					return err
   489  				}
   490  			}
   491  		} else {
   492  			// some task failed
   493  			if err = b.MarkFinished(evergreen.BuildFailed, finishTime); err != nil {
   494  				err = errors.Wrap(err, "Error marking build as finished")
   495  				grip.Error(err)
   496  				return err
   497  			}
   498  			if b.Requester == evergreen.PatchVersionRequester {
   499  				if err = TryMarkPatchBuildFinished(b, finishTime); err != nil {
   500  					err = errors.Wrap(err, "Error marking patch as finished")
   501  					grip.Error(err)
   502  					return err
   503  				}
   504  			}
   505  			if err = MarkVersionCompleted(b.Version, finishTime); err != nil {
   506  				err = errors.Wrap(err, "Error marking version as finished")
   507  				grip.Error(err)
   508  				return err
   509  			}
   510  		}
   511  
   512  		// update the build's makespan information if the task has finished
   513  		if err = updateMakespans(b); err != nil {
   514  			err = errors.Wrap(err, "Error updating makespan information")
   515  			grip.Error(err)
   516  			return err
   517  		}
   518  	}
   519  
   520  	// this is helpful for when we restart a compile task
   521  	if finishedTasks == 0 {
   522  		err = b.UpdateStatus(evergreen.BuildCreated)
   523  		if err != nil {
   524  			err = errors.Wrap(err, "Error updating build status")
   525  			grip.Error(err)
   526  			return err
   527  		}
   528  	}
   529  
   530  	return nil
   531  }
   532  
   533  // MarkStart updates the task, build, version and if necessary, patch documents with the task start time
   534  func MarkStart(taskId string) error {
   535  	t, err := task.FindOne(task.ById(taskId))
   536  	if err != nil {
   537  		return errors.WithStack(err)
   538  	}
   539  	startTime := time.Now()
   540  	if err = t.MarkStart(startTime); err != nil {
   541  		return errors.WithStack(err)
   542  	}
   543  	event.LogTaskStarted(t.Id)
   544  
   545  	// ensure the appropriate build is marked as started if necessary
   546  	if err = build.TryMarkStarted(t.BuildId, startTime); err != nil {
   547  		return errors.WithStack(err)
   548  	}
   549  
   550  	// ensure the appropriate version is marked as started if necessary
   551  	if err = MarkVersionStarted(t.Version, startTime); err != nil {
   552  		return errors.WithStack(err)
   553  	}
   554  
   555  	// if it's a patch, mark the patch as started if necessary
   556  	if t.Requester == evergreen.PatchVersionRequester {
   557  		if err = patch.TryMarkStarted(t.Version, startTime); err != nil {
   558  			return errors.WithStack(err)
   559  		}
   560  	}
   561  
   562  	// update the cached version of the task, in its build document
   563  	return build.SetCachedTaskStarted(t.BuildId, t.Id, startTime)
   564  }
   565  
   566  func MarkTaskUndispatched(t *task.Task) error {
   567  	// record that the task as undispatched on the host
   568  	if err := t.MarkAsUndispatched(); err != nil {
   569  		return errors.WithStack(err)
   570  	}
   571  	// the task was successfully dispatched, log the event
   572  	event.LogTaskUndispatched(t.Id, t.HostId)
   573  
   574  	// update the cached version of the task in its related build document
   575  	if err := build.SetCachedTaskUndispatched(t.BuildId, t.Id); err != nil {
   576  		return errors.WithStack(err)
   577  	}
   578  	return nil
   579  }
   580  
   581  func MarkTaskDispatched(t *task.Task, hostId, distroId string) error {
   582  	// record that the task was dispatched on the host
   583  	if err := t.MarkAsDispatched(hostId, distroId, time.Now()); err != nil {
   584  		return errors.Wrapf(err, "error marking task %s as dispatched "+
   585  			"on host %s", t.Id, hostId)
   586  	}
   587  	// the task was successfully dispatched, log the event
   588  	event.LogTaskDispatched(t.Id, hostId)
   589  
   590  	// update the cached version of the task in its related build document
   591  	if err := build.SetCachedTaskDispatched(t.BuildId, t.Id); err != nil {
   592  		return errors.Wrapf(err, "error updating task cache in build %s", t.BuildId)
   593  	}
   594  	return nil
   595  }