sigs.k8s.io/prow@v0.0.0-20240503223140-c5e374dc7eb1/pkg/config/jobs.go (about)

     1  /*
     2  Copyright 2017 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package config
    18  
    19  import (
    20  	"errors"
    21  	"fmt"
    22  	"net/url"
    23  	"regexp"
    24  	"strings"
    25  	"time"
    26  
    27  	pipelinev1beta1 "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1beta1"
    28  
    29  	v1 "k8s.io/api/core/v1"
    30  	"k8s.io/apimachinery/pkg/util/sets"
    31  	prowapi "sigs.k8s.io/prow/pkg/apis/prowjobs/v1"
    32  	"sigs.k8s.io/prow/pkg/github"
    33  )
    34  
    35  const (
    36  	schemeHTTP  = "http"
    37  	schemeHTTPS = "https"
    38  )
    39  
    40  // +k8s:deepcopy-gen=true
    41  
    42  // Presets can be used to re-use settings across multiple jobs.
    43  type Preset struct {
    44  	Labels       map[string]string `json:"labels"`
    45  	Env          []v1.EnvVar       `json:"env"`
    46  	Volumes      []v1.Volume       `json:"volumes"`
    47  	VolumeMounts []v1.VolumeMount  `json:"volumeMounts"`
    48  }
    49  
    50  func mergePreset(preset Preset, labels map[string]string, containers []v1.Container, volumes *[]v1.Volume) error {
    51  	for l, v := range preset.Labels {
    52  		if v2, ok := labels[l]; !ok || v2 != v {
    53  			return nil
    54  		}
    55  	}
    56  	for _, e1 := range preset.Env {
    57  		for i := range containers {
    58  			for _, e2 := range containers[i].Env {
    59  				if e1.Name == e2.Name {
    60  					return fmt.Errorf("env var duplicated in pod spec: %s", e1.Name)
    61  				}
    62  			}
    63  			containers[i].Env = append(containers[i].Env, e1)
    64  		}
    65  	}
    66  	for _, v1 := range preset.Volumes {
    67  		for _, v2 := range *volumes {
    68  			if v1.Name == v2.Name {
    69  				return fmt.Errorf("volume duplicated in pod spec: %s", v1.Name)
    70  			}
    71  		}
    72  		*volumes = append(*volumes, v1)
    73  	}
    74  	for _, vm1 := range preset.VolumeMounts {
    75  		for i := range containers {
    76  			for _, vm2 := range containers[i].VolumeMounts {
    77  				if vm1.Name == vm2.Name {
    78  					return fmt.Errorf("volume mount duplicated in pod spec: %s", vm1.Name)
    79  				}
    80  			}
    81  			containers[i].VolumeMounts = append(containers[i].VolumeMounts, vm1)
    82  		}
    83  	}
    84  	return nil
    85  }
    86  
    87  // +k8s:deepcopy-gen=true
    88  
    89  // JobBase contains attributes common to all job types
    90  type JobBase struct {
    91  	// The name of the job. Must match regex [A-Za-z0-9-._]+
    92  	// e.g. pull-test-infra-bazel-build
    93  	Name string `json:"name"`
    94  	// Labels are added to prowjobs and pods created for this job.
    95  	Labels map[string]string `json:"labels,omitempty"`
    96  	// MaximumConcurrency of this job, 0 implies no limit.
    97  	MaxConcurrency int `json:"max_concurrency,omitempty"`
    98  	// Agent that will take care of running this job. Defaults to "kubernetes"
    99  	Agent string `json:"agent,omitempty"`
   100  	// Cluster is the alias of the cluster to run this job in.
   101  	// (Default: kube.DefaultClusterAlias)
   102  	Cluster string `json:"cluster,omitempty"`
   103  	// Namespace is the namespace in which pods schedule.
   104  	//   nil: results in config.PodNamespace (aka pod default)
   105  	//   empty: results in config.ProwJobNamespace (aka same as prowjob)
   106  	Namespace *string `json:"namespace,omitempty"`
   107  	// ErrorOnEviction indicates that the ProwJob should be completed and given
   108  	// the ErrorState status if the pod that is executing the job is evicted.
   109  	// If this field is unspecified or false, a new pod will be created to replace
   110  	// the evicted one.
   111  	ErrorOnEviction bool `json:"error_on_eviction,omitempty"`
   112  	// SourcePath contains the path where this job is defined
   113  	SourcePath string `json:"-"`
   114  	// Spec is the Kubernetes pod spec used if Agent is kubernetes.
   115  	Spec *v1.PodSpec `json:"spec,omitempty"`
   116  	// PipelineRunSpec is the tekton pipeline spec used if Agent is tekton-pipeline.
   117  	PipelineRunSpec *pipelinev1beta1.PipelineRunSpec `json:"pipeline_run_spec,omitempty"`
   118  	// TektonPipelineRunSpec is the versioned tekton pipeline spec used if Agent is tekton-pipeline.
   119  	TektonPipelineRunSpec *prowapi.TektonPipelineRunSpec `json:"tekton_pipeline_run_spec,omitempty"`
   120  	// Annotations are unused by prow itself, but provide a space to configure other automation.
   121  	Annotations map[string]string `json:"annotations,omitempty"`
   122  	// ReporterConfig provides the option to configure reporting on job level
   123  	ReporterConfig *prowapi.ReporterConfig `json:"reporter_config,omitempty"`
   124  	// RerunAuthConfig specifies who can rerun the job
   125  	RerunAuthConfig *prowapi.RerunAuthConfig `json:"rerun_auth_config,omitempty"`
   126  	// Hidden defines if the job is hidden. If set to `true`, only Deck instances
   127  	// that have the flag `--hiddenOnly=true or `--show-hidden=true` set will show it.
   128  	// Presubmits and Postsubmits can also be set to hidden by
   129  	// adding their repository in Decks `hidden_repo` setting.
   130  	Hidden bool `json:"hidden,omitempty"`
   131  	// ProwJobDefault holds configuration options provided as defaults
   132  	// in the Prow config
   133  	ProwJobDefault *prowapi.ProwJobDefault `json:"prowjob_defaults,omitempty"`
   134  	// Name of the job queue specifying maximum concurrency, omission implies no limit.
   135  	// Works in parallel with MaxConcurrency and the limit is selected from the
   136  	// minimal setting of those two fields.
   137  	JobQueueName string `json:"job_queue_name,omitempty"`
   138  
   139  	UtilityConfig
   140  }
   141  
   142  func (jb JobBase) GetName() string {
   143  	return jb.Name
   144  }
   145  
   146  func (jb JobBase) GetLabels() map[string]string {
   147  	return jb.Labels
   148  }
   149  
   150  func (jb JobBase) GetAnnotations() map[string]string {
   151  	return jb.Annotations
   152  }
   153  
   154  func (jb JobBase) HasPipelineRunSpec() bool {
   155  	if jb.TektonPipelineRunSpec != nil && jb.TektonPipelineRunSpec.V1Beta1 != nil {
   156  		return true
   157  	}
   158  	if jb.PipelineRunSpec != nil {
   159  		return true
   160  	}
   161  	return false
   162  }
   163  
   164  func (jb JobBase) GetPipelineRunSpec() (*pipelinev1beta1.PipelineRunSpec, error) {
   165  	var found *pipelinev1beta1.PipelineRunSpec
   166  	if jb.TektonPipelineRunSpec != nil {
   167  		found = jb.TektonPipelineRunSpec.V1Beta1
   168  	}
   169  	if found == nil && jb.PipelineRunSpec != nil {
   170  		found = jb.PipelineRunSpec
   171  	}
   172  	if found == nil {
   173  		return nil, errors.New("pipeline run spec not found")
   174  	}
   175  	return found, nil
   176  }
   177  
   178  // +k8s:deepcopy-gen=true
   179  
   180  // Presubmit runs on PRs.
   181  type Presubmit struct {
   182  	JobBase
   183  
   184  	// AlwaysRun automatically for every PR, or only when a comment triggers it.
   185  	AlwaysRun bool `json:"always_run"`
   186  
   187  	// Optional indicates that the job's status context should not be required for merge.
   188  	Optional bool `json:"optional,omitempty"`
   189  
   190  	// Trigger is the regular expression to trigger the job.
   191  	// e.g. `@k8s-bot e2e test this`
   192  	// RerunCommand must also be specified if this field is specified.
   193  	// (Default: `(?m)^/test (?:.*? )?<job name>(?: .*?)?$`)
   194  	Trigger string `json:"trigger,omitempty"`
   195  
   196  	// The RerunCommand to give users. Must match Trigger.
   197  	// Trigger must also be specified if this field is specified.
   198  	// (Default: `/test <job name>`)
   199  	RerunCommand string `json:"rerun_command,omitempty"`
   200  
   201  	// RunBeforeMerge indicates that a job should always run by Tide as long as
   202  	// Brancher matches.
   203  	// This is used when a prowjob is so expensive that it's not ideal to run on
   204  	// every single push from all PRs.
   205  	RunBeforeMerge bool `json:"run_before_merge,omitempty"`
   206  
   207  	Brancher
   208  
   209  	RegexpChangeMatcher
   210  
   211  	Reporter
   212  
   213  	JenkinsSpec *JenkinsSpec `json:"jenkins_spec,omitempty"`
   214  
   215  	// We'll set these when we load it.
   216  	re *CopyableRegexp // from Trigger.
   217  }
   218  
   219  // +k8s:deepcopy-gen=true
   220  
   221  // CopyableRegexp wraps around regexp.Regexp. It's sole purpose is to allow us to
   222  // create a manual DeepCopyInto() method for it, because the standard library's
   223  // regexp package does not define one for us (making it impossible to generate
   224  // DeepCopy() methods for any type that uses the regexp.Regexp type directly).
   225  type CopyableRegexp struct {
   226  	*regexp.Regexp
   227  }
   228  
   229  func (in *CopyableRegexp) DeepCopyInto(out *CopyableRegexp) {
   230  	// We use the deprecated Regexp.Copy() function here, because it's better to
   231  	// defer to the package's own Copy() method instead of creating our own.
   232  	//
   233  	// Unfortunately there is no way to tell golangci-lint (our linter) to only
   234  	// ignore the check SA1019 (Using a deprecated function, variable, constant
   235  	// or field), and we have to disable the entire staticcheck linter for this
   236  	// one line of code.
   237  	//
   238  	// nolint:staticcheck
   239  	*out = CopyableRegexp{in.Copy()}
   240  }
   241  
   242  // +k8s:deepcopy-gen=true
   243  
   244  // Postsubmit runs on push events.
   245  type Postsubmit struct {
   246  	JobBase
   247  
   248  	// AlwaysRun determines whether we should try to run this job it (or not run
   249  	// it). The key difference with the AlwaysRun field for Presubmits is that
   250  	// here, we essentially treat "true" as the default value as Postsubmits by
   251  	// default run unless there is some falsifying condition.
   252  	//
   253  	// The use of a pointer allows us to check if the field was or was not
   254  	// provided by the user. This is required because otherwise when we
   255  	// Unmarshal() the bytes into this struct, we'll get a default "false" value
   256  	// if this field is not provided, which is the opposite of what we want.
   257  	AlwaysRun *bool `json:"always_run,omitempty"`
   258  
   259  	RegexpChangeMatcher
   260  
   261  	Brancher
   262  
   263  	// TODO(krzyzacy): Move existing `Report` into `Skip_Report` once this is deployed
   264  	Reporter
   265  
   266  	JenkinsSpec *JenkinsSpec `json:"jenkins_spec,omitempty"`
   267  }
   268  
   269  // Periodic runs on a timer.
   270  type Periodic struct {
   271  	JobBase
   272  
   273  	// (deprecated)Interval to wait between two runs of the job.
   274  	// Consecutive jobs are run at `interval` duration apart, provided the
   275  	// previous job has completed.
   276  	Interval string `json:"interval,omitempty"`
   277  	// MinimumInterval to wait between two runs of the job.
   278  	// Consecutive jobs are run at `interval` + `duration of previous job` apart.
   279  	MinimumInterval string `json:"minimum_interval,omitempty"`
   280  	// Cron representation of job trigger time
   281  	Cron string `json:"cron,omitempty"`
   282  	// Tags for config entries
   283  	Tags []string `json:"tags,omitempty"`
   284  
   285  	interval         time.Duration
   286  	minimum_interval time.Duration
   287  }
   288  
   289  // JenkinsSpec holds optional Jenkins job config
   290  type JenkinsSpec struct {
   291  	// Job is managed by the GH branch source plugin
   292  	// and requires a specific path
   293  	GitHubBranchSourceJob bool `json:"github_branch_source_job,omitempty"`
   294  }
   295  
   296  // SetInterval updates interval, the frequency duration it runs.
   297  func (p *Periodic) SetInterval(d time.Duration) {
   298  	p.interval = d
   299  }
   300  
   301  // GetInterval returns interval, the frequency duration it runs.
   302  func (p *Periodic) GetInterval() time.Duration {
   303  	return p.interval
   304  }
   305  
   306  // SetMinimumInterval updates minimum_interval, the minimum frequency duration it runs.
   307  func (p *Periodic) SetMinimumInterval(d time.Duration) {
   308  	p.minimum_interval = d
   309  }
   310  
   311  // GetMinimumInterval returns minimum_interval, the minimum frequency duration it runs.
   312  func (p *Periodic) GetMinimumInterval() time.Duration {
   313  	return p.minimum_interval
   314  }
   315  
   316  // +k8s:deepcopy-gen=true
   317  
   318  // Brancher is for shared code between jobs that only run against certain
   319  // branches. An empty brancher runs against all branches.
   320  type Brancher struct {
   321  	// Do not run against these branches. Default is no branches.
   322  	SkipBranches []string `json:"skip_branches,omitempty"`
   323  	// Only run against these branches. Default is all branches.
   324  	Branches []string `json:"branches,omitempty"`
   325  
   326  	// We'll set these when we load it.
   327  	re     *CopyableRegexp
   328  	reSkip *CopyableRegexp
   329  }
   330  
   331  // +k8s:deepcopy-gen=true
   332  
   333  // RegexpChangeMatcher is for code shared between jobs that run only when certain files are changed.
   334  type RegexpChangeMatcher struct {
   335  	// RunIfChanged defines a regex used to select which subset of file changes should trigger this job.
   336  	// If any file in the changeset matches this regex, the job will be triggered
   337  	// Additionally AlwaysRun is mutually exclusive with RunIfChanged.
   338  	RunIfChanged string `json:"run_if_changed,omitempty"`
   339  	// SkipIfOnlyChanged defines a regex used to select which subset of file changes should trigger this job.
   340  	// If all files in the changeset match this regex, the job will be skipped.
   341  	// In other words, this is the negation of RunIfChanged.
   342  	// Additionally AlwaysRun is mutually exclusive with SkipIfOnlyChanged.
   343  	SkipIfOnlyChanged string          `json:"skip_if_only_changed,omitempty"`
   344  	reChanges         *CopyableRegexp // from RunIfChanged xor SkipIfOnlyChanged
   345  }
   346  
   347  type Reporter struct {
   348  	// Context is the name of the GitHub status context for the job.
   349  	// Defaults: the same as the name of the job.
   350  	Context string `json:"context,omitempty"`
   351  	// SkipReport skips commenting and setting status on GitHub.
   352  	SkipReport bool `json:"skip_report,omitempty"`
   353  }
   354  
   355  // RunsAgainstAllBranch returns true if there are both branches and skip_branches are unset
   356  func (br Brancher) RunsAgainstAllBranch() bool {
   357  	return len(br.SkipBranches) == 0 && len(br.Branches) == 0
   358  }
   359  
   360  // ShouldRun returns true if the input branch matches, given the allow/deny list.
   361  func (br Brancher) ShouldRun(branch string) bool {
   362  	if br.RunsAgainstAllBranch() {
   363  		return true
   364  	}
   365  
   366  	// Favor SkipBranches over Branches
   367  	if len(br.SkipBranches) != 0 && br.reSkip.MatchString(branch) {
   368  		return false
   369  	}
   370  	if len(br.Branches) == 0 || br.re.MatchString(branch) {
   371  		return true
   372  	}
   373  	return false
   374  }
   375  
   376  // Intersects checks if other Brancher would trigger for the same branch.
   377  func (br Brancher) Intersects(other Brancher) bool {
   378  	if br.RunsAgainstAllBranch() || other.RunsAgainstAllBranch() {
   379  		return true
   380  	}
   381  	if len(br.Branches) > 0 {
   382  		baseBranches := sets.New[string](br.Branches...)
   383  		if len(other.Branches) > 0 {
   384  			otherBranches := sets.New[string](other.Branches...)
   385  			return baseBranches.Intersection(otherBranches).Len() > 0
   386  		}
   387  
   388  		// Actually test our branches against the other brancher - if there are regex skip lists, simple comparison
   389  		// is insufficient.
   390  		for _, b := range sets.List(baseBranches) {
   391  			if other.ShouldRun(b) {
   392  				return true
   393  			}
   394  		}
   395  		return false
   396  	}
   397  	if len(other.Branches) == 0 {
   398  		// There can only be one Brancher with skip_branches.
   399  		return true
   400  	}
   401  	return other.Intersects(br)
   402  }
   403  
   404  // CouldRun determines if its possible for a set of changes to trigger this condition
   405  func (cm RegexpChangeMatcher) CouldRun() bool {
   406  	return cm.RunIfChanged != "" || cm.SkipIfOnlyChanged != ""
   407  }
   408  
   409  // ShouldRun determines if we can know for certain that the job should run. We can either
   410  // know for certain that the job should or should not run based on the matcher, or we can
   411  // not be able to determine that fact at all.
   412  func (cm RegexpChangeMatcher) ShouldRun(changes ChangedFilesProvider) (determined bool, shouldRun bool, err error) {
   413  	if cm.CouldRun() {
   414  		changeList, err := changes()
   415  		if err != nil {
   416  			return true, false, err
   417  		}
   418  		return true, cm.RunsAgainstChanges(changeList), nil
   419  	}
   420  	return false, false, nil
   421  }
   422  
   423  // RunsAgainstChanges returns true if any of the changed input paths match the run_if_changed regex;
   424  // OR if any of the changed input paths *don't* match the skip_if_only_changed regex.
   425  func (cm RegexpChangeMatcher) RunsAgainstChanges(changes []string) bool {
   426  	for _, change := range changes {
   427  		// RunIfChanged triggers the run if *any* change matches the supplied regex.
   428  		if cm.RunIfChanged != "" && cm.reChanges.MatchString(change) {
   429  			return true
   430  			// SkipIfOnlyChanged triggers the run if any change *doesn't* match the supplied regex.
   431  		} else if cm.SkipIfOnlyChanged != "" && !cm.reChanges.MatchString(change) {
   432  			return true
   433  		}
   434  	}
   435  	return false
   436  }
   437  
   438  // CouldRun determines if the postsubmit could run against a specific
   439  // base ref
   440  func (ps Postsubmit) CouldRun(baseRef string) bool {
   441  	return ps.Brancher.ShouldRun(baseRef)
   442  }
   443  
   444  // ShouldRun determines if the postsubmit should run in response to a
   445  // set of changes. This is evaluated lazily, if necessary.
   446  func (ps Postsubmit) ShouldRun(baseRef string, changes ChangedFilesProvider) (bool, error) {
   447  	if !ps.CouldRun(baseRef) {
   448  		return false, nil
   449  	}
   450  
   451  	// Consider `run_if_changed` or `skip_if_only_changed` rules.
   452  	if determined, shouldRun, err := ps.RegexpChangeMatcher.ShouldRun(changes); err != nil {
   453  		return false, err
   454  	} else if determined {
   455  		return shouldRun, nil
   456  	}
   457  
   458  	// At this point neither `run_if_changed` nor `skip_if_only_changed` were
   459  	// set. We're left with 2 cases: (1) `always_run: ...` was provided
   460  	// explicitly, or (2) this field was not defined in the job at all. In the
   461  	// second case, we default to "true".
   462  
   463  	// If the `always_run` field was explicitly set, return it.
   464  	if ps.AlwaysRun != nil {
   465  		return *ps.AlwaysRun, nil
   466  	}
   467  
   468  	// Postsubmits default to always run. This is the case if `always_run` was
   469  	// not explicitly set.
   470  	return true, nil
   471  }
   472  
   473  // CouldRun determines if the presubmit could run against a specific
   474  // base ref
   475  func (ps Presubmit) CouldRun(baseRef string) bool {
   476  	return ps.Brancher.ShouldRun(baseRef)
   477  }
   478  
   479  // ShouldRun determines if the presubmit should run against a specific
   480  // base ref, or in response to a set of changes. The latter mechanism
   481  // is evaluated lazily, if necessary.
   482  func (ps Presubmit) ShouldRun(baseRef string, changes ChangedFilesProvider, forced, defaults bool) (bool, error) {
   483  	if !ps.CouldRun(baseRef) {
   484  		return false, nil
   485  	}
   486  	if ps.AlwaysRun {
   487  		return true, nil
   488  	}
   489  	if forced {
   490  		return true, nil
   491  	}
   492  	determined, shouldRun, err := ps.RegexpChangeMatcher.ShouldRun(changes)
   493  	return (determined && shouldRun) || defaults, err
   494  }
   495  
   496  // TriggersConditionally determines if the presubmit triggers conditionally (if it may or may not trigger).
   497  func (ps Presubmit) TriggersConditionally() bool {
   498  	return ps.NeedsExplicitTrigger() || ps.RegexpChangeMatcher.CouldRun()
   499  }
   500  
   501  // NeedsExplicitTrigger determines if the presubmit requires a human action to trigger it or not.
   502  func (ps Presubmit) NeedsExplicitTrigger() bool {
   503  	return !ps.AlwaysRun && !ps.RegexpChangeMatcher.CouldRun()
   504  }
   505  
   506  // TriggerMatches returns true if the comment body should trigger this presubmit.
   507  //
   508  // This is usually a /test foo string.
   509  func (ps Presubmit) TriggerMatches(body string) bool {
   510  	return ps.Trigger != "" && ps.re.MatchString(body)
   511  }
   512  
   513  // ContextRequired checks whether a context is required from github points of view (required check).
   514  func (ps Presubmit) ContextRequired() bool {
   515  	return !ps.Optional && !ps.SkipReport
   516  }
   517  
   518  // ChangedFilesProvider returns a slice of modified files.
   519  type ChangedFilesProvider func() ([]string, error)
   520  
   521  type githubClient interface {
   522  	GetPullRequestChanges(org, repo string, number int) ([]github.PullRequestChange, error)
   523  }
   524  
   525  // NewGitHubDeferredChangedFilesProvider uses a closure to lazily retrieve the file changes only if they are needed.
   526  // We only have to fetch the changes if there is at least one RunIfChanged/SkipIfOnlyChanged job that is not being
   527  // force run (due to a `/retest` after a failure or because it is explicitly triggered with `/test foo`).
   528  func NewGitHubDeferredChangedFilesProvider(client githubClient, org, repo string, num int) ChangedFilesProvider {
   529  	var changedFiles []string
   530  	return func() ([]string, error) {
   531  		// Fetch the changed files from github at most once.
   532  		if changedFiles == nil {
   533  			changes, err := client.GetPullRequestChanges(org, repo, num)
   534  			if err != nil {
   535  				return nil, fmt.Errorf("error getting pull request changes: %w", err)
   536  			}
   537  			for _, change := range changes {
   538  				changedFiles = append(changedFiles, change.Filename)
   539  			}
   540  		}
   541  		return changedFiles, nil
   542  	}
   543  }
   544  
   545  // +k8s:deepcopy-gen=true
   546  
   547  // UtilityConfig holds decoration metadata, such as how to clone and additional containers/etc
   548  type UtilityConfig struct {
   549  	// Decorate determines if we decorate the PodSpec or not
   550  	Decorate *bool `json:"decorate,omitempty"`
   551  
   552  	// PathAlias is the location under <root-dir>/src
   553  	// where the repository under test is cloned. If this
   554  	// is not set, <root-dir>/src/github.com/org/repo will
   555  	// be used as the default.
   556  	PathAlias string `json:"path_alias,omitempty"`
   557  	// CloneURI is the URI that is used to clone the
   558  	// repository. If unset, will default to
   559  	// `https://github.com/org/repo.git`.
   560  	CloneURI string `json:"clone_uri,omitempty"`
   561  	// SkipSubmodules determines if submodules should be
   562  	// cloned when the job is run. Defaults to false.
   563  	SkipSubmodules bool `json:"skip_submodules,omitempty"`
   564  	// CloneDepth is the depth of the clone that will be used.
   565  	// A depth of zero will do a full clone.
   566  	CloneDepth int `json:"clone_depth,omitempty"`
   567  	// SkipFetchHead tells prow to avoid a git fetch <remote> call.
   568  	// The git fetch <remote> <BaseRef> call occurs regardless.
   569  	SkipFetchHead bool `json:"skip_fetch_head,omitempty"`
   570  
   571  	// ExtraRefs are auxiliary repositories that
   572  	// need to be cloned, determined from config
   573  	ExtraRefs []prowapi.Refs `json:"extra_refs,omitempty"`
   574  
   575  	// DecorationConfig holds configuration options for
   576  	// decorating PodSpecs that users provide
   577  	DecorationConfig *prowapi.DecorationConfig `json:"decoration_config,omitempty"`
   578  }
   579  
   580  // Validate ensures all the values set in the UtilityConfig are valid.
   581  func (u *UtilityConfig) Validate() error {
   582  	cloneURIValidate := func(cloneURI string) error {
   583  		// Trim user from uri if exists.
   584  		cloneURI = cloneURI[strings.Index(cloneURI, "@")+1:]
   585  
   586  		if len(u.CloneURI) != 0 {
   587  			uri, err := url.Parse(cloneURI)
   588  			if err != nil {
   589  				return fmt.Errorf("couldn't parse uri from clone_uri: %w", err)
   590  			}
   591  
   592  			if u.DecorationConfig != nil && u.DecorationConfig.OauthTokenSecret != nil {
   593  				if uri.Scheme != schemeHTTP && uri.Scheme != schemeHTTPS {
   594  					return fmt.Errorf("scheme must be http or https when OAuth secret is specified: %s", cloneURI)
   595  				}
   596  			}
   597  		}
   598  
   599  		return nil
   600  	}
   601  
   602  	if err := cloneURIValidate(u.CloneURI); err != nil {
   603  		return err
   604  	}
   605  
   606  	for i, ref := range u.ExtraRefs {
   607  		if err := cloneURIValidate(ref.CloneURI); err != nil {
   608  			return fmt.Errorf("extra_ref[%d]: %w", i, err)
   609  		}
   610  	}
   611  
   612  	return nil
   613  }
   614  
   615  // SetPresubmits updates c.PresubmitStatic to jobs, after compiling and validating their regexes.
   616  func (c *JobConfig) SetPresubmits(jobs map[string][]Presubmit) error {
   617  	nj := map[string][]Presubmit{}
   618  	for k, v := range jobs {
   619  		nj[k] = make([]Presubmit, len(v))
   620  		copy(nj[k], v)
   621  		if err := SetPresubmitRegexes(nj[k]); err != nil {
   622  			return err
   623  		}
   624  	}
   625  	c.PresubmitsStatic = nj
   626  	return nil
   627  }
   628  
   629  // SetPostsubmits updates c.Postsubmits to jobs, after compiling and validating their regexes.
   630  func (c *JobConfig) SetPostsubmits(jobs map[string][]Postsubmit) error {
   631  	nj := map[string][]Postsubmit{}
   632  	for k, v := range jobs {
   633  		nj[k] = make([]Postsubmit, len(v))
   634  		copy(nj[k], v)
   635  		if err := SetPostsubmitRegexes(nj[k]); err != nil {
   636  			return err
   637  		}
   638  	}
   639  	c.PostsubmitsStatic = nj
   640  	return nil
   641  }
   642  
   643  // AllStaticPresubmits returns all static prow presubmit jobs in repos.
   644  // if repos is empty, return all presubmits.
   645  // Be aware that this does not return Presubmits that are versioned inside
   646  // the repo via the `inrepoconfig` feature and hence this list may be
   647  // incomplete.
   648  func (c *JobConfig) AllStaticPresubmits(repos []string) []Presubmit {
   649  	var res []Presubmit
   650  
   651  	for repo, v := range c.PresubmitsStatic {
   652  		if len(repos) == 0 {
   653  			res = append(res, v...)
   654  		} else {
   655  			for _, r := range repos {
   656  				if r == repo {
   657  					res = append(res, v...)
   658  					break
   659  				}
   660  			}
   661  		}
   662  	}
   663  
   664  	return res
   665  }
   666  
   667  // AllPostsubmits returns all prow postsubmit jobs in repos.
   668  // if repos is empty, return all postsubmits.
   669  // Be aware that this does not return Postsubmits that are versioned inside
   670  // the repo via the `inrepoconfig` feature and hence this list may be
   671  // incomplete.
   672  func (c *JobConfig) AllStaticPostsubmits(repos []string) []Postsubmit {
   673  	var res []Postsubmit
   674  
   675  	for repo, v := range c.PostsubmitsStatic {
   676  		if len(repos) == 0 {
   677  			res = append(res, v...)
   678  		} else {
   679  			for _, r := range repos {
   680  				if r == repo {
   681  					res = append(res, v...)
   682  					break
   683  				}
   684  			}
   685  		}
   686  	}
   687  
   688  	return res
   689  }
   690  
   691  // AllPeriodics returns all prow periodic jobs.
   692  func (c *JobConfig) AllPeriodics() []Periodic {
   693  	listPeriodic := func(ps []Periodic) []Periodic {
   694  		var res []Periodic
   695  		res = append(res, ps...)
   696  		return res
   697  	}
   698  
   699  	return listPeriodic(c.Periodics)
   700  }
   701  
   702  // ClearCompiledRegexes removes compiled regexes from the presubmits,
   703  // useful for testing when deep equality is needed between presubmits
   704  func ClearCompiledRegexes(presubmits []Presubmit) {
   705  	for i := range presubmits {
   706  		presubmits[i].re = nil
   707  		presubmits[i].Brancher.re = nil
   708  		presubmits[i].Brancher.reSkip = nil
   709  		presubmits[i].RegexpChangeMatcher.reChanges = nil
   710  	}
   711  }