github.com/billybanfield/evergreen@v0.0.0-20170525200750-eeee692790f7/model/project.go (about)

     1  package model
     2  
     3  import (
     4  	"fmt"
     5  	"strings"
     6  
     7  	"github.com/evergreen-ci/evergreen"
     8  	"github.com/evergreen-ci/evergreen/command"
     9  	"github.com/evergreen-ci/evergreen/db/bsonutil"
    10  	"github.com/evergreen-ci/evergreen/model/build"
    11  	"github.com/evergreen-ci/evergreen/model/distro"
    12  	"github.com/evergreen-ci/evergreen/model/task"
    13  	"github.com/evergreen-ci/evergreen/model/version"
    14  	"github.com/evergreen-ci/evergreen/util"
    15  	"github.com/pkg/errors"
    16  	ignore "github.com/sabhiram/go-git-ignore"
    17  )
    18  
    19  const (
    20  	TestCommandType   = "test"
    21  	SystemCommandType = "system"
    22  )
    23  
    24  const (
    25  	// DefaultCommandType is a system configuration option that is used to
    26  	// differentiate between setup related commands and actual testing commands.
    27  	DefaultCommandType = TestCommandType
    28  )
    29  
    30  type Project struct {
    31  	Enabled         bool                       `yaml:"enabled,omitempty" bson:"enabled"`
    32  	Stepback        bool                       `yaml:"stepback,omitempty" bson:"stepback"`
    33  	BatchTime       int                        `yaml:"batchtime,omitempty" bson:"batch_time"`
    34  	Owner           string                     `yaml:"owner,omitempty" bson:"owner_name"`
    35  	Repo            string                     `yaml:"repo,omitempty" bson:"repo_name"`
    36  	RemotePath      string                     `yaml:"remote_path,omitempty" bson:"remote_path"`
    37  	RepoKind        string                     `yaml:"repokind,omitempty" bson:"repo_kind"`
    38  	Branch          string                     `yaml:"branch,omitempty" bson:"branch_name"`
    39  	Identifier      string                     `yaml:"identifier,omitempty" bson:"identifier"`
    40  	DisplayName     string                     `yaml:"display_name,omitempty" bson:"display_name"`
    41  	CommandType     string                     `yaml:"command_type,omitempty" bson:"command_type"`
    42  	Ignore          []string                   `yaml:"ignore,omitempty" bson:"ignore"`
    43  	Pre             *YAMLCommandSet            `yaml:"pre,omitempty" bson:"pre"`
    44  	Post            *YAMLCommandSet            `yaml:"post,omitempty" bson:"post"`
    45  	Timeout         *YAMLCommandSet            `yaml:"timeout,omitempty" bson:"timeout"`
    46  	CallbackTimeout int                        `yaml:"callback_timeout_secs,omitempty" bson:"callback_timeout_secs"`
    47  	Modules         []Module                   `yaml:"modules,omitempty" bson:"modules"`
    48  	BuildVariants   []BuildVariant             `yaml:"buildvariants,omitempty" bson:"build_variants"`
    49  	Functions       map[string]*YAMLCommandSet `yaml:"functions,omitempty" bson:"functions"`
    50  	Tasks           []ProjectTask              `yaml:"tasks,omitempty" bson:"tasks"`
    51  	ExecTimeoutSecs int                        `yaml:"exec_timeout_secs,omitempty" bson:"exec_timeout_secs"`
    52  
    53  	// Flag that indicates a project as requiring user authentication
    54  	Private bool `yaml:"private,omitempty" bson:"private"`
    55  }
    56  
    57  // Unmarshalled from the "tasks" list in an individual build variant
    58  type BuildVariantTask struct {
    59  	// Name has to match the name field of one of the tasks specified at
    60  	// the project level, or an error will be thrown
    61  	Name string `yaml:"name,omitempty" bson:"name"`
    62  
    63  	// fields to overwrite ProjectTask settings
    64  	Patchable *bool             `yaml:"patchable,omitempty" bson:"patchable,omitempty"`
    65  	Priority  int64             `yaml:"priority,omitempty" bson:"priority"`
    66  	DependsOn []TaskDependency  `yaml:"depends_on,omitempty" bson:"depends_on"`
    67  	Requires  []TaskRequirement `yaml:"requires,omitempty" bson:"requires"`
    68  
    69  	// currently unsupported (TODO EVG-578)
    70  	ExecTimeoutSecs int   `yaml:"exec_timeout_secs,omitempty" bson:"exec_timeout_secs"`
    71  	Stepback        *bool `yaml:"stepback,omitempty" bson:"stepback,omitempty"`
    72  
    73  	// the distros that the task can be run on
    74  	Distros []string `yaml:"distros,omitempty" bson:"distros"`
    75  }
    76  
    77  // Populate updates the base fields of the BuildVariantTask with
    78  // fields from the project task definition.
    79  func (bvt *BuildVariantTask) Populate(pt ProjectTask) {
    80  	// We never update "Name" or "Commands"
    81  	if len(bvt.DependsOn) == 0 {
    82  		bvt.DependsOn = pt.DependsOn
    83  	}
    84  	if len(bvt.Requires) == 0 {
    85  		bvt.Requires = pt.Requires
    86  	}
    87  	if bvt.Priority == 0 {
    88  		bvt.Priority = pt.Priority
    89  	}
    90  	if bvt.Patchable == nil {
    91  		bvt.Patchable = pt.Patchable
    92  	}
    93  	// TODO these are copied but unused until EVG-578 is completed
    94  	if bvt.ExecTimeoutSecs == 0 {
    95  		bvt.ExecTimeoutSecs = pt.ExecTimeoutSecs
    96  	}
    97  	if bvt.Stepback == nil {
    98  		bvt.Stepback = pt.Stepback
    99  	}
   100  }
   101  
   102  // UnmarshalYAML allows tasks to be referenced as single selector strings.
   103  // This works by first attempting to unmarshal the YAML into a string
   104  // and then falling back to the BuildVariantTask struct.
   105  func (bvt *BuildVariantTask) UnmarshalYAML(unmarshal func(interface{}) error) error {
   106  	// first, attempt to unmarshal just a selector string
   107  	var onlySelector string
   108  	if err := unmarshal(&onlySelector); err == nil {
   109  		bvt.Name = onlySelector
   110  		return nil
   111  	}
   112  	// we define a new type so that we can grab the yaml struct tags without the struct methods,
   113  	// preventing infinte recursion on the UnmarshalYAML() method.
   114  	type bvtCopyType BuildVariantTask
   115  	var bvtc bvtCopyType
   116  	err := unmarshal(&bvtc)
   117  	if err != nil {
   118  		return err
   119  	}
   120  	*bvt = BuildVariantTask(bvtc)
   121  	return nil
   122  }
   123  
   124  type BuildVariant struct {
   125  	Name        string            `yaml:"name,omitempty" bson:"name"`
   126  	DisplayName string            `yaml:"display_name,omitempty" bson:"display_name"`
   127  	Expansions  map[string]string `yaml:"expansions,omitempty" bson:"expansions"`
   128  	Modules     []string          `yaml:"modules,omitempty" bson:"modules"`
   129  	Disabled    bool              `yaml:"disabled,omitempty" bson:"disabled"`
   130  	Tags        []string          `yaml:"tags,omitempty" bson:"tags"`
   131  	Push        bool              `yaml:"push,omitempty" bson:"push"`
   132  
   133  	// Use a *int for 2 possible states
   134  	// nil - not overriding the project setting
   135  	// non-nil - overriding the project setting with this BatchTime
   136  	BatchTime *int `yaml:"batchtime,omitempty" bson:"batchtime,omitempty"`
   137  
   138  	// Use a *bool so that there are 3 possible states:
   139  	//   1. nil   = not overriding the project setting (default)
   140  	//   2. true  = overriding the project setting with true
   141  	//   3. false = overriding the project setting with false
   142  	Stepback *bool `yaml:"stepback,omitempty" bson:"stepback,omitempty"`
   143  
   144  	// the default distros.  will be used to run a task if no distro field is
   145  	// provided for the task
   146  	RunOn []string `yaml:"run_on,omitempty" bson:"run_on"`
   147  
   148  	// all of the tasks to be run on the build variant, compile through tests.
   149  	Tasks []BuildVariantTask `yaml:"tasks,omitempty" bson:"tasks"`
   150  }
   151  
   152  type Module struct {
   153  	Name   string `yaml:"name,omitempty" bson:"name"`
   154  	Branch string `yaml:"branch,omitempty" bson:"branch"`
   155  	Repo   string `yaml:"repo,omitempty" bson:"repo"`
   156  	Prefix string `yaml:"prefix,omitempty" bson:"prefix"`
   157  }
   158  
   159  type TestSuite struct {
   160  	Name  string `yaml:"name,omitempty"`
   161  	Phase string `yaml:"phase,omitempty"`
   162  }
   163  
   164  type PluginCommandConf struct {
   165  	Function string `yaml:"func,omitempty" bson:"func"`
   166  	// Type is used to differentiate between setup related commands and actual
   167  	// testing commands.
   168  	Type string `yaml:"type,omitempty" bson:"type"`
   169  
   170  	// DisplayName is a human readable description of the function of a given
   171  	// command.
   172  	DisplayName string `yaml:"display_name,omitempty" bson:"display_name"`
   173  
   174  	// Command is a unique identifier for the command configuration. It consists of a
   175  	// plugin name and a command name.
   176  	Command string `yaml:"command,omitempty" bson:"command"`
   177  
   178  	// Variants is used to enumerate the particular sets of buildvariants to run
   179  	// this command configuration on. If it is empty, it is run on all defined
   180  	// variants.
   181  	Variants []string `yaml:"variants,omitempty" bson:"variants"`
   182  
   183  	// TimeoutSecs indicates the maximum duration the command is allowed to run
   184  	// for. If undefined, it is unbounded.
   185  	TimeoutSecs int `yaml:"timeout_secs,omitempty" bson:"timeout_secs"`
   186  
   187  	// Params are used to supply configuratiion specific information.
   188  	Params map[string]interface{} `yaml:"params,omitempty" bson:"params"`
   189  
   190  	// Vars defines variables that can be used within commands.
   191  	Vars map[string]string `yaml:"vars,omitempty" bson:"vars"`
   192  }
   193  
   194  type ArtifactInstructions struct {
   195  	Include      []string `yaml:"include,omitempty" bson:"include"`
   196  	ExcludeFiles []string `yaml:"excludefiles,omitempty" bson:"exclude_files"`
   197  }
   198  
   199  type YAMLCommandSet struct {
   200  	SingleCommand *PluginCommandConf
   201  	MultiCommand  []PluginCommandConf
   202  }
   203  
   204  func (c *YAMLCommandSet) List() []PluginCommandConf {
   205  	if len(c.MultiCommand) > 0 {
   206  		return c.MultiCommand
   207  	}
   208  	if c.SingleCommand != nil && (c.SingleCommand.Command != "" || c.SingleCommand.Function != "") {
   209  		return []PluginCommandConf{*c.SingleCommand}
   210  	}
   211  	return []PluginCommandConf{}
   212  }
   213  
   214  func (c *YAMLCommandSet) MarshalYAML() (interface{}, error) {
   215  	if c == nil {
   216  		return nil, nil
   217  	}
   218  	return c.List(), nil
   219  }
   220  
   221  func (c *YAMLCommandSet) UnmarshalYAML(unmarshal func(interface{}) error) error {
   222  	err1 := unmarshal(&(c.MultiCommand))
   223  	err2 := unmarshal(&(c.SingleCommand))
   224  	if err1 == nil || err2 == nil {
   225  		return nil
   226  	}
   227  	return err1
   228  }
   229  
   230  // TaskDependency holds configuration information about a task that must finish before
   231  // the task that contains the dependency can run.
   232  type TaskDependency struct {
   233  	Name          string `yaml:"name,omitempty" bson:"name"`
   234  	Variant       string `yaml:"variant,omitempty" bson:"variant,omitempty"`
   235  	Status        string `yaml:"status,omitempty" bson:"status,omitempty"`
   236  	PatchOptional bool   `yaml:"patch_optional,omitempty" bson:"patch_optional,omitempty"`
   237  }
   238  
   239  // TaskRequirement represents tasks that must exist along with
   240  // the requirement's holder. This is only used when configuring patches.
   241  type TaskRequirement struct {
   242  	Name    string `yaml:"name,omitempty" bson:"name"`
   243  	Variant string `yaml:"variant,omitempty" bson:"variant,omitempty"`
   244  }
   245  
   246  // UnmarshalYAML allows tasks to be referenced as single selector strings.
   247  // This works by first attempting to unmarshal the YAML into a string
   248  // and then falling back to the TaskDependency struct.
   249  func (td *TaskDependency) UnmarshalYAML(unmarshal func(interface{}) error) error {
   250  	// first, attempt to unmarshal just a selector string
   251  	var onlySelector string
   252  	if err := unmarshal(&onlySelector); err == nil {
   253  		td.Name = onlySelector
   254  		return nil
   255  	}
   256  	// we define a new type so that we can grab the yaml struct tags without the struct methods,
   257  	// preventing infinte recursion on the UnmarshalYAML() method.
   258  	type tdCopyType TaskDependency
   259  	var tdc tdCopyType
   260  	err := unmarshal(&tdc)
   261  	if err != nil {
   262  		return err
   263  	}
   264  	*td = TaskDependency(tdc)
   265  	return nil
   266  }
   267  
   268  // Unmarshalled from the "tasks" list in the project file
   269  type ProjectTask struct {
   270  	Name            string              `yaml:"name,omitempty" bson:"name"`
   271  	Priority        int64               `yaml:"priority,omitempty" bson:"priority"`
   272  	ExecTimeoutSecs int                 `yaml:"exec_timeout_secs,omitempty" bson:"exec_timeout_secs"`
   273  	DependsOn       []TaskDependency    `yaml:"depends_on,omitempty" bson:"depends_on"`
   274  	Requires        []TaskRequirement   `yaml:"requires,omitempty" bson:"requires"`
   275  	Commands        []PluginCommandConf `yaml:"commands,omitempty" bson:"commands"`
   276  	Tags            []string            `yaml:"tags,omitempty" bson:"tags"`
   277  
   278  	// Use a *bool so that there are 3 possible states:
   279  	//   1. nil   = not overriding the project setting (default)
   280  	//   2. true  = overriding the project setting with true
   281  	//   3. false = overriding the project setting with false
   282  	Patchable *bool `yaml:"patchable,omitempty" bson:"patchable,omitempty"`
   283  	Stepback  *bool `yaml:"stepback,omitempty" bson:"stepback,omitempty"`
   284  }
   285  
   286  type TaskConfig struct {
   287  	Distro       *distro.Distro
   288  	Version      *version.Version
   289  	ProjectRef   *ProjectRef
   290  	Project      *Project
   291  	Task         *task.Task
   292  	BuildVariant *BuildVariant
   293  	Expansions   *command.Expansions
   294  	WorkDir      string
   295  }
   296  
   297  // TaskIdTable is a map of [variant, task display name]->[task id].
   298  type TaskIdTable map[TVPair]string
   299  
   300  // TVPair is a helper type for mapping bv/task pairs to ids.
   301  type TVPair struct {
   302  	Variant  string
   303  	TaskName string
   304  }
   305  
   306  type TVPairSet []TVPair
   307  
   308  // ByVariant returns a list of TVPairs filtered to include only those
   309  // for the given variant
   310  func (tvps TVPairSet) ByVariant(variant string) TVPairSet {
   311  	p := []TVPair{}
   312  	for _, pair := range tvps {
   313  		if pair.Variant != variant {
   314  			continue
   315  		}
   316  		p = append(p, pair)
   317  	}
   318  	return TVPairSet(p)
   319  }
   320  
   321  // TaskNames extracts the unique set of task names for a given variant in the set of task/variant pairs.
   322  func (tvps TVPairSet) TaskNames(variant string) []string {
   323  	taskSet := map[string]bool{}
   324  	taskNames := []string{}
   325  	for _, pair := range tvps {
   326  		// skip over any pairs that aren't for the given variant
   327  		if pair.Variant != variant {
   328  			continue
   329  		}
   330  		// skip over tasks we already picked up
   331  		if _, ok := taskSet[pair.TaskName]; ok {
   332  			continue
   333  		}
   334  		taskSet[pair.TaskName] = true
   335  		taskNames = append(taskNames, pair.TaskName)
   336  	}
   337  	return taskNames
   338  }
   339  
   340  // String returns the pair's name in a readable form.
   341  func (p TVPair) String() string {
   342  	return fmt.Sprintf("%v/%v", p.Variant, p.TaskName)
   343  }
   344  
   345  // AddId adds the Id for the task/variant combination to the table.
   346  func (tt TaskIdTable) AddId(variant, taskName, id string) {
   347  	tt[TVPair{variant, taskName}] = id
   348  }
   349  
   350  // GetId returns the Id for the given task on the given variant.
   351  // Returns the empty string if the task/variant does not exist.
   352  func (tt TaskIdTable) GetId(variant, taskName string) string {
   353  	return tt[TVPair{variant, taskName}]
   354  }
   355  
   356  // GetIdsForAllVariants returns all task Ids for taskName on all variants.
   357  func (tt TaskIdTable) GetIdsForAllVariants(taskName string) []string {
   358  	return tt.GetIdsForAllVariantsExcluding(taskName, TVPair{})
   359  }
   360  
   361  // GetIdsForAllVariants returns all task Ids for taskName on all variants, excluding
   362  // the specific task denoted by the task/variant pair.
   363  func (tt TaskIdTable) GetIdsForAllVariantsExcluding(taskName string, exclude TVPair) []string {
   364  	ids := []string{}
   365  	for pair := range tt {
   366  		if pair.TaskName == taskName && pair != exclude {
   367  			if id := tt[pair]; id != "" {
   368  				ids = append(ids, id)
   369  			}
   370  		}
   371  	}
   372  	return ids
   373  }
   374  
   375  // GetIdsForTasks returns all task Ids for tasks on all variants != the current task.
   376  // The current variant and task must be passed in to avoid cycle generation.
   377  func (tt TaskIdTable) GetIdsForAllTasks(currentVariant, taskName string) []string {
   378  	ids := []string{}
   379  	for pair := range tt {
   380  		if !(pair.TaskName == taskName && pair.Variant == currentVariant) {
   381  			if id := tt[pair]; id != "" {
   382  				ids = append(ids, id)
   383  			}
   384  		}
   385  	}
   386  	return ids
   387  }
   388  
   389  // TaskIdTable builds a TaskIdTable for the given version and project
   390  func NewTaskIdTable(p *Project, v *version.Version) TaskIdTable {
   391  	// init the variant map
   392  	table := TaskIdTable{}
   393  	for _, bv := range p.BuildVariants {
   394  		for _, t := range bv.Tasks {
   395  			// create a unique Id for each task
   396  			taskId := util.CleanName(
   397  				fmt.Sprintf("%v_%v_%v_%v_%v",
   398  					p.Identifier, bv.Name, t.Name, v.Revision, v.CreateTime.Format(build.IdTimeLayout)))
   399  			table[TVPair{bv.Name, t.Name}] = taskId
   400  		}
   401  	}
   402  	return table
   403  }
   404  
   405  // NewPatchTaskIdTable constructs a new TaskIdTable (map of [variant, task display name]->[task id])
   406  func NewPatchTaskIdTable(proj *Project, v *version.Version, patchConfig TVPairSet) TaskIdTable {
   407  	table := TaskIdTable{}
   408  	processedVariants := map[string]bool{}
   409  	for _, vt := range patchConfig {
   410  		// don't hit the same variant more than once
   411  		if _, ok := processedVariants[vt.Variant]; ok {
   412  			continue
   413  		}
   414  		processedVariants[vt.Variant] = true
   415  		// we must track the project's variants definitions as well,
   416  		// so that we don't create Ids for variants that don't exist.
   417  		projBV := proj.FindBuildVariant(vt.Variant)
   418  		taskNamesForVariant := patchConfig.TaskNames(vt.Variant)
   419  		for _, t := range projBV.Tasks {
   420  			// create Ids for each task that can run on the variant and is requested by the patch.
   421  			if util.SliceContains(taskNamesForVariant, t.Name) {
   422  				taskId := util.CleanName(
   423  					fmt.Sprintf("%v_%v_%v_%v_%v",
   424  						proj.Identifier, projBV.Name, t.Name, v.Revision,
   425  						v.CreateTime.Format(build.IdTimeLayout)))
   426  				table[TVPair{vt.Variant, t.Name}] = taskId
   427  			}
   428  		}
   429  	}
   430  	return table
   431  }
   432  
   433  var (
   434  	// bson fields for the project struct
   435  	ProjectIdentifierKey    = bsonutil.MustHaveTag(Project{}, "Identifier")
   436  	ProjectPreKey           = bsonutil.MustHaveTag(Project{}, "Pre")
   437  	ProjectPostKey          = bsonutil.MustHaveTag(Project{}, "Post")
   438  	ProjectModulesKey       = bsonutil.MustHaveTag(Project{}, "Modules")
   439  	ProjectBuildVariantsKey = bsonutil.MustHaveTag(Project{}, "BuildVariants")
   440  	ProjectFunctionsKey     = bsonutil.MustHaveTag(Project{}, "Functions")
   441  	ProjectStepbackKey      = bsonutil.MustHaveTag(Project{}, "Stepback")
   442  	ProjectTasksKey         = bsonutil.MustHaveTag(Project{}, "Tasks")
   443  )
   444  
   445  func NewTaskConfig(d *distro.Distro, v *version.Version, p *Project, t *task.Task, r *ProjectRef) (*TaskConfig, error) {
   446  	// do a check on if the project is empty
   447  	if p == nil {
   448  		return nil, errors.Errorf("project for task with branch %v is empty", t.Project)
   449  	}
   450  
   451  	// check on if the project ref is empty
   452  	if r == nil {
   453  		return nil, errors.Errorf("Project ref with identifier: %v was empty", p.Identifier)
   454  	}
   455  
   456  	bv := p.FindBuildVariant(t.BuildVariant)
   457  	if bv == nil {
   458  		return nil, errors.Errorf("couldn't find buildvariant: '%v'", t.BuildVariant)
   459  	}
   460  
   461  	e := populateExpansions(d, v, bv, t)
   462  	return &TaskConfig{d, v, r, p, t, bv, e, d.WorkDir}, nil
   463  }
   464  
   465  func populateExpansions(d *distro.Distro, v *version.Version, bv *BuildVariant, t *task.Task) *command.Expansions {
   466  	expansions := command.NewExpansions(map[string]string{})
   467  	expansions.Put("execution", fmt.Sprintf("%v", t.Execution))
   468  	expansions.Put("version_id", t.Version)
   469  	expansions.Put("task_id", t.Id)
   470  	expansions.Put("task_name", t.DisplayName)
   471  	expansions.Put("build_id", t.BuildId)
   472  	expansions.Put("build_variant", t.BuildVariant)
   473  	expansions.Put("workdir", d.WorkDir)
   474  	expansions.Put("revision", t.Revision)
   475  	expansions.Put("project", t.Project)
   476  	expansions.Put("branch_name", v.Branch)
   477  	expansions.Put("author", v.Author)
   478  	expansions.Put("distro_id", d.Id)
   479  	if t.Requester == evergreen.PatchVersionRequester {
   480  		expansions.Put("is_patch", "true")
   481  	}
   482  	for _, e := range d.Expansions {
   483  		expansions.Put(e.Key, e.Value)
   484  	}
   485  	expansions.Update(bv.Expansions)
   486  	return expansions
   487  }
   488  
   489  // GetSpecForTask returns a ProjectTask spec for the given name.
   490  // Returns an empty ProjectTask if none exists.
   491  func (p Project) GetSpecForTask(name string) ProjectTask {
   492  	for _, pt := range p.Tasks {
   493  		if pt.Name == name {
   494  			return pt
   495  		}
   496  	}
   497  	return ProjectTask{}
   498  }
   499  
   500  func (p *Project) GetVariantMappings() map[string]string {
   501  	mappings := make(map[string]string)
   502  	for _, buildVariant := range p.BuildVariants {
   503  		mappings[buildVariant.Name] = buildVariant.DisplayName
   504  	}
   505  	return mappings
   506  }
   507  
   508  func (p *Project) GetVariantsWithTask(taskName string) []string {
   509  	var variantsList []string
   510  	for _, buildVariant := range p.BuildVariants {
   511  		for _, task := range buildVariant.Tasks {
   512  			if task.Name == taskName {
   513  				variantsList = append(variantsList, buildVariant.Name)
   514  			}
   515  		}
   516  	}
   517  	return variantsList
   518  }
   519  
   520  // RunOnVariant returns true if the plugin command should run on variant; returns false otherwise
   521  func (p PluginCommandConf) RunOnVariant(variant string) bool {
   522  	return len(p.Variants) == 0 || util.SliceContains(p.Variants, variant)
   523  }
   524  
   525  // GetDisplayName returns the  display name of the plugin command. If none is
   526  // defined, it returns the command's identifier.
   527  func (p PluginCommandConf) GetDisplayName() string {
   528  	if p.DisplayName != "" {
   529  		return p.DisplayName
   530  	}
   531  	return p.Command
   532  }
   533  
   534  // GetType returns the type of this command if one is explicitly specified. If
   535  // no type is specified, it checks the default command type of the project. If
   536  // one is specified, it returns that, if not, it returns the DefaultCommandType.
   537  func (p PluginCommandConf) GetType(prj *Project) string {
   538  	if p.Type != "" {
   539  		return p.Type
   540  	}
   541  	if prj.CommandType != "" {
   542  		return prj.CommandType
   543  	}
   544  	return DefaultCommandType
   545  }
   546  
   547  func (m *Module) GetRepoOwnerAndName() (string, string) {
   548  	parts := strings.Split(m.Repo, ":")
   549  	basename := parts[len(parts)-1]
   550  	ownerAndName := strings.TrimSuffix(basename, ".git")
   551  	ownersplit := strings.Split(ownerAndName, "/")
   552  	if len(ownersplit) != 2 {
   553  		return "", ""
   554  	} else {
   555  		return ownersplit[0], ownersplit[1]
   556  	}
   557  }
   558  
   559  func FindProject(revision string, projectRef *ProjectRef) (*Project, error) {
   560  	if projectRef == nil {
   561  		return nil, errors.New("projectRef given is nil")
   562  	}
   563  	if projectRef.Identifier == "" {
   564  		return nil, errors.New("Invalid project with blank identifier")
   565  	}
   566  
   567  	project := &Project{}
   568  	project.Identifier = projectRef.Identifier
   569  	// when the revision is empty we find the last known good configuration from the versions
   570  	// If the last known good configuration does not exist,
   571  	// load the configuration from the local config in the project ref.
   572  	if revision == "" {
   573  		lastGoodVersion, err := version.FindOne(version.ByLastKnownGoodConfig(projectRef.Identifier))
   574  		if err != nil {
   575  			return nil, errors.Wrapf(err, "Error finding recent valid version for %v: %v", projectRef.Identifier)
   576  		}
   577  		if lastGoodVersion != nil {
   578  			// for new repositories, we don't want to error out when we don't have
   579  			// any versions stored in the database so we default to the skeletal
   580  			// information we already have from the project file on disk
   581  			err = LoadProjectInto([]byte(lastGoodVersion.Config), projectRef.Identifier, project)
   582  			if err != nil {
   583  				return nil, errors.Wrapf(err, "Error loading project from "+
   584  					"last good version for project, %v", lastGoodVersion.Identifier)
   585  			}
   586  		} else {
   587  			// Check to see if there is a local configuration in the project ref
   588  			if projectRef.LocalConfig != "" {
   589  				err = LoadProjectInto([]byte(projectRef.LocalConfig), projectRef.Identifier, project)
   590  				if err != nil {
   591  					return nil, errors.Wrapf(err, "Error loading local config for project ref, %v", projectRef.Identifier)
   592  				}
   593  			}
   594  		}
   595  	}
   596  
   597  	if revision != "" {
   598  		// we immediately return an error if the repotracker version isn't found
   599  		// for the given project at the given revision
   600  		version, err := version.FindOne(version.ByProjectIdAndRevision(projectRef.Identifier, revision))
   601  		if err != nil {
   602  			return nil, errors.Wrapf(err, "error fetching version for project %v revision %v", projectRef.Identifier, revision)
   603  		}
   604  		if version == nil {
   605  			// fall back to the skeletal project
   606  			return project, nil
   607  		}
   608  
   609  		project = &Project{}
   610  		if err = LoadProjectInto([]byte(version.Config), projectRef.Identifier, project); err != nil {
   611  			return nil, errors.Wrap(err, "Error loading project from version")
   612  		}
   613  	}
   614  	return project, nil
   615  }
   616  
   617  func (p *Project) FindTaskForVariant(task, variant string) *BuildVariantTask {
   618  	bv := p.FindBuildVariant(variant)
   619  	if bv == nil {
   620  		return nil
   621  	}
   622  	for _, bvt := range bv.Tasks {
   623  		if bvt.Name == task {
   624  			bvt.Populate(*p.FindProjectTask(task))
   625  			return &bvt
   626  		}
   627  	}
   628  	return nil
   629  }
   630  
   631  func (p *Project) FindBuildVariant(build string) *BuildVariant {
   632  	for _, b := range p.BuildVariants {
   633  		if b.Name == build {
   634  			return &b
   635  		}
   636  	}
   637  	return nil
   638  }
   639  
   640  func (p *Project) FindProjectTask(name string) *ProjectTask {
   641  	for _, t := range p.Tasks {
   642  		if t.Name == name {
   643  			return &t
   644  		}
   645  	}
   646  	return nil
   647  }
   648  
   649  func (p *Project) GetModuleByName(name string) (*Module, error) {
   650  	for _, v := range p.Modules {
   651  		if v.Name == name {
   652  			return &v, nil
   653  		}
   654  	}
   655  	return nil, errors.New("No such module on this project.")
   656  }
   657  
   658  func (p *Project) FindTasksForVariant(build string) []string {
   659  	for _, b := range p.BuildVariants {
   660  		if b.Name == build {
   661  			tasks := make([]string, 0, len(b.Tasks))
   662  			for _, task := range b.Tasks {
   663  				tasks = append(tasks, task.Name)
   664  			}
   665  			return tasks
   666  		}
   667  	}
   668  	return nil
   669  }
   670  
   671  func (p *Project) FindAllVariants() []string {
   672  	variants := make([]string, 0, len(p.BuildVariants))
   673  	for _, b := range p.BuildVariants {
   674  		variants = append(variants, b.Name)
   675  	}
   676  	return variants
   677  }
   678  
   679  // FindAllBuildVariantTasks returns every BuildVariantTask, fully populated,
   680  // for all variants of a project.
   681  func (p *Project) FindAllBuildVariantTasks() []BuildVariantTask {
   682  	tasksByName := map[string]*ProjectTask{}
   683  	for i, t := range p.Tasks {
   684  		tasksByName[t.Name] = &p.Tasks[i]
   685  	}
   686  	allBVTs := []BuildVariantTask{}
   687  	for _, b := range p.BuildVariants {
   688  		for _, t := range b.Tasks {
   689  			if pTask := tasksByName[t.Name]; pTask != nil {
   690  				t.Populate(*pTask)
   691  				allBVTs = append(allBVTs, t)
   692  			}
   693  		}
   694  	}
   695  	return allBVTs
   696  }
   697  
   698  // FindVariantsWithTask returns the name of each variant containing
   699  // the given task name.
   700  func (p *Project) FindVariantsWithTask(task string) []string {
   701  	variants := make([]string, 0, len(p.BuildVariants))
   702  	for _, b := range p.BuildVariants {
   703  		for _, t := range b.Tasks {
   704  			if t.Name == task {
   705  				variants = append(variants, b.Name)
   706  			}
   707  		}
   708  	}
   709  	return variants
   710  }
   711  
   712  // IgnoresAllFiles takes in a slice of filepaths and checks to see if
   713  // all files are matched by the project's Ignore regular expressions.
   714  func (p *Project) IgnoresAllFiles(files []string) bool {
   715  	if len(p.Ignore) == 0 || len(files) == 0 {
   716  		return false
   717  	}
   718  	// CompileIgnoreLines has a silly API: it always returns a nil error.
   719  	ignorer, _ := ignore.CompileIgnoreLines(p.Ignore...)
   720  	for _, f := range files {
   721  		if !ignorer.MatchesPath(f) {
   722  			return false
   723  		}
   724  	}
   725  	return true
   726  }