github.com/munnerz/test-infra@v0.0.0-20190108210205-ce3d181dc989/prow/apis/prowjobs/v1/types.go (about)

     1  /*
     2  Copyright 2018 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package v1
    18  
    19  import (
    20  	"errors"
    21  	"fmt"
    22  	"strings"
    23  	"time"
    24  
    25  	buildv1alpha1 "github.com/knative/build/pkg/apis/build/v1alpha1"
    26  	corev1 "k8s.io/api/core/v1"
    27  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    28  )
    29  
    30  // ProwJobType specifies how the job is triggered.
    31  type ProwJobType string
    32  
    33  // Various job types.
    34  const (
    35  	// PresubmitJob means it runs on unmerged PRs.
    36  	PresubmitJob ProwJobType = "presubmit"
    37  	// PostsubmitJob means it runs on each new commit.
    38  	PostsubmitJob = "postsubmit"
    39  	// Periodic job means it runs on a time-basis, unrelated to git changes.
    40  	PeriodicJob = "periodic"
    41  	// BatchJob tests multiple unmerged PRs at the same time.
    42  	BatchJob = "batch"
    43  )
    44  
    45  // ProwJobState specifies whether the job is running
    46  type ProwJobState string
    47  
    48  // Various job states.
    49  const (
    50  	// TriggeredState means the job has been created but not yet scheduled.
    51  	TriggeredState ProwJobState = "triggered"
    52  	// PendingState means the job is scheduled but not yet running.
    53  	PendingState = "pending"
    54  	// SuccessState means the job completed without error (exit 0)
    55  	SuccessState = "success"
    56  	// FailureState means the job completed with errors (exit non-zero)
    57  	FailureState = "failure"
    58  	// AbortedState means prow killed the job early (new commit pushed, perhaps).
    59  	AbortedState = "aborted"
    60  	// ErrorState means the job could not schedule (bad config, perhaps).
    61  	ErrorState = "error"
    62  )
    63  
    64  // ProwJobAgent specifies the controller (such as plank or jenkins-agent) that runs the job.
    65  type ProwJobAgent string
    66  
    67  const (
    68  	// KubernetesAgent means prow will create a pod to run this job.
    69  	KubernetesAgent ProwJobAgent = "kubernetes"
    70  	// JenkinsAgent means prow will schedule the job on jenkins.
    71  	JenkinsAgent = "jenkins"
    72  	// KnativeBuildAgent means prow will schedule the job via a build-crd resource.
    73  	KnativeBuildAgent = "knative-build"
    74  )
    75  
    76  const (
    77  	// DefaultClusterAlias specifies the default cluster key to schedule jobs.
    78  	DefaultClusterAlias = "default"
    79  )
    80  
    81  // +genclient
    82  // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
    83  
    84  // ProwJob contains the spec as well as runtime metadata.
    85  type ProwJob struct {
    86  	metav1.TypeMeta   `json:",inline"`
    87  	metav1.ObjectMeta `json:"metadata,omitempty"`
    88  
    89  	Spec   ProwJobSpec   `json:"spec,omitempty"`
    90  	Status ProwJobStatus `json:"status,omitempty"`
    91  }
    92  
    93  // ProwJobSpec configures the details of the prow job.
    94  //
    95  // Details include the podspec, code to clone, the cluster it runs
    96  // any child jobs, concurrency limitations, etc.
    97  type ProwJobSpec struct {
    98  	// Type is the type of job and informs how
    99  	// the jobs is triggered
   100  	Type ProwJobType `json:"type,omitempty"`
   101  	// Agent determines which controller fulfills
   102  	// this specific ProwJobSpec and runs the job
   103  	Agent ProwJobAgent `json:"agent,omitempty"`
   104  	// Cluster is which Kubernetes cluster is used
   105  	// to run the job, only applicable for that
   106  	// specific agent
   107  	Cluster string `json:"cluster,omitempty"`
   108  	// Namespace defines where to create pods/resources.
   109  	Namespace string `json:"namespace,omitempty"`
   110  	// Job is the name of the job
   111  	Job string `json:"job,omitempty"`
   112  	// Refs is the code under test, determined at
   113  	// runtime by Prow itself
   114  	Refs *Refs `json:"refs,omitempty"`
   115  	// ExtraRefs are auxiliary repositories that
   116  	// need to be cloned, determined from config
   117  	ExtraRefs []Refs `json:"extra_refs,omitempty"`
   118  
   119  	// Report determines if the result of this job should
   120  	// be posted as a status on GitHub
   121  	Report bool `json:"report,omitempty"`
   122  	// Context is the name of the status context used to
   123  	// report back to GitHub
   124  	Context string `json:"context,omitempty"`
   125  	// RerunCommand is the command a user would write to
   126  	// trigger this job on their pull request
   127  	RerunCommand string `json:"rerun_command,omitempty"`
   128  	// MaxConcurrency restricts the total number of instances
   129  	// of this job that can run in parallel at once
   130  	MaxConcurrency int `json:"max_concurrency,omitempty"`
   131  	// ErrorOnEviction indicates that the ProwJob should be completed and given
   132  	// the ErrorState status if the pod that is executing the job is evicted.
   133  	// If this field is unspecified or false, a new pod will be created to replace
   134  	// the evicted one.
   135  	ErrorOnEviction bool `json:"error_on_eviction,omitempty"`
   136  
   137  	// PodSpec provides the basis for running the test under
   138  	// a Kubernetes agent
   139  	PodSpec *corev1.PodSpec `json:"pod_spec,omitempty"`
   140  
   141  	// BuildSpec provides the basis for running the test as
   142  	// a build-crd resource
   143  	// https://github.com/knative/build
   144  	BuildSpec *buildv1alpha1.BuildSpec `json:"build_spec,omitempty"`
   145  
   146  	// DecorationConfig holds configuration options for
   147  	// decorating PodSpecs that users provide
   148  	DecorationConfig *DecorationConfig `json:"decoration_config,omitempty"`
   149  
   150  	// RunAfterSuccess are jobs that should be triggered if
   151  	// this job runs and does not fail
   152  	RunAfterSuccess []ProwJobSpec `json:"run_after_success,omitempty"`
   153  }
   154  
   155  // DecorationConfig specifies how to augment pods.
   156  //
   157  // This is primarily used to provide automatic integration with gubernator
   158  // and testgrid.
   159  type DecorationConfig struct {
   160  	// Timeout is how long the pod utilities will wait
   161  	// before aborting a job with SIGINT.
   162  	Timeout time.Duration `json:"timeout,omitempty"`
   163  	// GracePeriod is how long the pod utilities will wait
   164  	// after sending SIGINT to send SIGKILL when aborting
   165  	// a job. Only applicable if decorating the PodSpec.
   166  	GracePeriod time.Duration `json:"grace_period,omitempty"`
   167  	// UtilityImages holds pull specs for utility container
   168  	// images used to decorate a PodSpec.
   169  	UtilityImages *UtilityImages `json:"utility_images,omitempty"`
   170  	// GCSConfiguration holds options for pushing logs and
   171  	// artifacts to GCS from a job.
   172  	GCSConfiguration *GCSConfiguration `json:"gcs_configuration,omitempty"`
   173  	// GCSCredentialsSecret is the name of the Kubernetes secret
   174  	// that holds GCS push credentials.
   175  	GCSCredentialsSecret string `json:"gcs_credentials_secret,omitempty"`
   176  	// SSHKeySecrets are the names of Kubernetes secrets that contain
   177  	// SSK keys which should be used during the cloning process.
   178  	SSHKeySecrets []string `json:"ssh_key_secrets,omitempty"`
   179  	// SSHHostFingerprints are the fingerprints of known SSH hosts
   180  	// that the cloning process can trust.
   181  	// Create with ssh-keyscan [-t rsa] host
   182  	SSHHostFingerprints []string `json:"ssh_host_fingerprints,omitempty"`
   183  	// SkipCloning determines if we should clone source code in the
   184  	// initcontainers for jobs that specify refs
   185  	SkipCloning *bool `json:"skip_cloning,omitempty"`
   186  	// CookieFileSecret is the name of a kubernetes secret that contains
   187  	// a git http.cookiefile, which should be used during the cloning process.
   188  	CookiefileSecret string `json:"cookiefile_secret,omitempty"`
   189  }
   190  
   191  // ApplyDefault applies the defaults for the ProwJob decoration. If a field has a zero value, it
   192  // replaces that with the value set in def.
   193  func (d *DecorationConfig) ApplyDefault(def *DecorationConfig) *DecorationConfig {
   194  	if d == nil && def == nil {
   195  		return nil
   196  	}
   197  	var merged DecorationConfig
   198  	if d != nil {
   199  		merged = *d
   200  	} else {
   201  		merged = *def
   202  	}
   203  	if d == nil || def == nil {
   204  		return &merged
   205  	}
   206  	merged.UtilityImages = merged.UtilityImages.ApplyDefault(def.UtilityImages)
   207  	merged.GCSConfiguration = merged.GCSConfiguration.ApplyDefault(def.GCSConfiguration)
   208  
   209  	if merged.Timeout == 0 {
   210  		merged.Timeout = def.Timeout
   211  	}
   212  	if merged.GracePeriod == 0 {
   213  		merged.GracePeriod = def.GracePeriod
   214  	}
   215  	if merged.GCSCredentialsSecret == "" {
   216  		merged.GCSCredentialsSecret = def.GCSCredentialsSecret
   217  	}
   218  	if len(merged.SSHKeySecrets) == 0 {
   219  		merged.SSHKeySecrets = def.SSHKeySecrets
   220  	}
   221  	if len(merged.SSHHostFingerprints) == 0 {
   222  		merged.SSHHostFingerprints = def.SSHHostFingerprints
   223  	}
   224  	if merged.SkipCloning == nil {
   225  		merged.SkipCloning = def.SkipCloning
   226  	}
   227  	if merged.CookiefileSecret == "" {
   228  		merged.CookiefileSecret = def.CookiefileSecret
   229  	}
   230  
   231  	return &merged
   232  }
   233  
   234  // Validate ensures all the values set in the DecorationConfig are valid.
   235  func (d *DecorationConfig) Validate() error {
   236  	if d.UtilityImages == nil {
   237  		return errors.New("utility image config is not specified")
   238  	}
   239  	var missing []string
   240  	if d.UtilityImages.CloneRefs == "" {
   241  		missing = append(missing, "clonerefs")
   242  	}
   243  	if d.UtilityImages.InitUpload == "" {
   244  		missing = append(missing, "initupload")
   245  	}
   246  	if d.UtilityImages.Entrypoint == "" {
   247  		missing = append(missing, "entrypoint")
   248  	}
   249  	if d.UtilityImages.Sidecar == "" {
   250  		missing = append(missing, "sidecar")
   251  	}
   252  	if len(missing) > 0 {
   253  		return fmt.Errorf("the following utility images are not specified: %q", missing)
   254  	}
   255  
   256  	if d.GCSConfiguration == nil {
   257  		return errors.New("GCS upload configuration is not specified")
   258  	}
   259  	if d.GCSCredentialsSecret == "" {
   260  		return errors.New("GCS upload credential secret is not specified")
   261  	}
   262  	if err := d.GCSConfiguration.Validate(); err != nil {
   263  		return fmt.Errorf("GCS configuration is invalid: %v", err)
   264  	}
   265  	return nil
   266  }
   267  
   268  // UtilityImages holds pull specs for the utility images
   269  // to be used for a job
   270  type UtilityImages struct {
   271  	// CloneRefs is the pull spec used for the clonerefs utility
   272  	CloneRefs string `json:"clonerefs,omitempty"`
   273  	// InitUpload is the pull spec used for the initupload utility
   274  	InitUpload string `json:"initupload,omitempty"`
   275  	// Entrypoint is the pull spec used for the entrypoint utility
   276  	Entrypoint string `json:"entrypoint,omitempty"`
   277  	// sidecar is the pull spec used for the sidecar utility
   278  	Sidecar string `json:"sidecar,omitempty"`
   279  }
   280  
   281  // ApplyDefault applies the defaults for the UtilityImages decorations. If a field has a zero value,
   282  // it replaces that with the value set in def.
   283  func (u *UtilityImages) ApplyDefault(def *UtilityImages) *UtilityImages {
   284  	if u == nil {
   285  		return def
   286  	} else if def == nil {
   287  		return u
   288  	}
   289  
   290  	merged := *u
   291  	if merged.CloneRefs == "" {
   292  		merged.CloneRefs = def.CloneRefs
   293  	}
   294  	if merged.InitUpload == "" {
   295  		merged.InitUpload = def.InitUpload
   296  	}
   297  	if merged.Entrypoint == "" {
   298  		merged.Entrypoint = def.Entrypoint
   299  	}
   300  	if merged.Sidecar == "" {
   301  		merged.Sidecar = def.Sidecar
   302  	}
   303  	return &merged
   304  }
   305  
   306  // PathStrategy specifies minutia about how to construct the url.
   307  // Usually consumed by gubernator/testgrid.
   308  const (
   309  	PathStrategyLegacy   = "legacy"
   310  	PathStrategySingle   = "single"
   311  	PathStrategyExplicit = "explicit"
   312  )
   313  
   314  // GCSConfiguration holds options for pushing logs and
   315  // artifacts to GCS from a job.
   316  type GCSConfiguration struct {
   317  	// Bucket is the GCS bucket to upload to
   318  	Bucket string `json:"bucket,omitempty"`
   319  	// PathPrefix is an optional path that follows the
   320  	// bucket name and comes before any structure
   321  	PathPrefix string `json:"path_prefix,omitempty"`
   322  	// PathStrategy dictates how the org and repo are used
   323  	// when calculating the full path to an artifact in GCS
   324  	PathStrategy string `json:"path_strategy,omitempty"`
   325  	// DefaultOrg is omitted from GCS paths when using the
   326  	// legacy or simple strategy
   327  	DefaultOrg string `json:"default_org,omitempty"`
   328  	// DefaultRepo is omitted from GCS paths when using the
   329  	// legacy or simple strategy
   330  	DefaultRepo string `json:"default_repo,omitempty"`
   331  }
   332  
   333  // ApplyDefault applies the defaults for GCSConfiguration decorations. If a field has a zero value,
   334  // it replaces that with the value set in def.
   335  func (g *GCSConfiguration) ApplyDefault(def *GCSConfiguration) *GCSConfiguration {
   336  	if g == nil && def == nil {
   337  		return nil
   338  	}
   339  	var merged GCSConfiguration
   340  	if g != nil {
   341  		merged = *g
   342  	} else {
   343  		merged = *def
   344  	}
   345  	if g == nil || def == nil {
   346  		return &merged
   347  	}
   348  
   349  	if merged.Bucket == "" {
   350  		merged.Bucket = def.Bucket
   351  	}
   352  	if merged.PathPrefix == "" {
   353  		merged.PathPrefix = def.PathPrefix
   354  	}
   355  	if merged.PathStrategy == "" {
   356  		merged.PathStrategy = def.PathStrategy
   357  	}
   358  	if merged.DefaultOrg == "" {
   359  		merged.DefaultOrg = def.DefaultOrg
   360  	}
   361  	if merged.DefaultRepo == "" {
   362  		merged.DefaultRepo = def.DefaultRepo
   363  	}
   364  	return &merged
   365  }
   366  
   367  // Validate ensures all the values set in the GCSConfiguration are valid.
   368  func (g *GCSConfiguration) Validate() error {
   369  	if g.PathStrategy != PathStrategyLegacy && g.PathStrategy != PathStrategyExplicit && g.PathStrategy != PathStrategySingle {
   370  		return fmt.Errorf("gcs_path_strategy must be one of %q, %q, or %q", PathStrategyLegacy, PathStrategyExplicit, PathStrategySingle)
   371  	}
   372  	if g.PathStrategy != PathStrategyExplicit && (g.DefaultOrg == "" || g.DefaultRepo == "") {
   373  		return fmt.Errorf("default org and repo must be provided for GCS strategy %q", g.PathStrategy)
   374  	}
   375  	return nil
   376  }
   377  
   378  // ProwJobStatus provides runtime metadata, such as when it finished, whether it is running, etc.
   379  type ProwJobStatus struct {
   380  	StartTime      metav1.Time  `json:"startTime,omitempty"`
   381  	CompletionTime *metav1.Time `json:"completionTime,omitempty"`
   382  	State          ProwJobState `json:"state,omitempty"`
   383  	Description    string       `json:"description,omitempty"`
   384  	URL            string       `json:"url,omitempty"`
   385  
   386  	// PodName applies only to ProwJobs fulfilled by
   387  	// plank. This field should always be the same as
   388  	// the ProwJob.ObjectMeta.Name field.
   389  	PodName string `json:"pod_name,omitempty"`
   390  
   391  	// BuildID is the build identifier vended either by tot
   392  	// or the snowflake library for this job and used as an
   393  	// identifier for grouping artifacts in GCS for views in
   394  	// TestGrid and Gubernator. Idenitifiers vended by tot
   395  	// are monotonically increasing whereas identifiers vended
   396  	// by the snowflake library are not.
   397  	BuildID string `json:"build_id,omitempty"`
   398  
   399  	// JenkinsBuildID applies only to ProwJobs fulfilled
   400  	// by the jenkins-operator. This field is the build
   401  	// identifier that Jenkins gave to the build for this
   402  	// ProwJob.
   403  	JenkinsBuildID string `json:"jenkins_build_id,omitempty"`
   404  
   405  	// PrevReportStates stores the previous reported prowjob state per reporter
   406  	// So crier won't make duplicated report attempt
   407  	PrevReportStates map[string]ProwJobState `json:"prev_report_states,omitempty"`
   408  }
   409  
   410  // Complete returns true if the prow job has finished
   411  func (j *ProwJob) Complete() bool {
   412  	// TODO(fejta): support a timeout?
   413  	return j.Status.CompletionTime != nil
   414  }
   415  
   416  // SetComplete marks the job as completed (at time now).
   417  func (j *ProwJob) SetComplete() {
   418  	j.Status.CompletionTime = new(metav1.Time)
   419  	*j.Status.CompletionTime = metav1.Now()
   420  }
   421  
   422  // ClusterAlias specifies the key in the clusters map to use.
   423  //
   424  // This allows scheduling a prow job somewhere aside from the default build cluster.
   425  func (j *ProwJob) ClusterAlias() string {
   426  	if j.Spec.Cluster == "" {
   427  		return DefaultClusterAlias
   428  	}
   429  	return j.Spec.Cluster
   430  }
   431  
   432  // Pull describes a pull request at a particular point in time.
   433  type Pull struct {
   434  	Number int    `json:"number,omitempty"`
   435  	Author string `json:"author,omitempty"`
   436  	SHA    string `json:"sha,omitempty"`
   437  
   438  	// Ref is git ref can be checked out for a change
   439  	// for example,
   440  	// github: pull/123/head
   441  	// gerrit: refs/changes/00/123/1
   442  	Ref string `json:"ref,omitempty"`
   443  }
   444  
   445  // Refs describes how the repo was constructed.
   446  type Refs struct {
   447  	// Org is something like kubernetes or k8s.io
   448  	Org string `json:"org,omitempty"`
   449  	// Repo is something like test-infra
   450  	Repo string `json:"repo,omitempty"`
   451  
   452  	BaseRef string `json:"base_ref,omitempty"`
   453  	BaseSHA string `json:"base_sha,omitempty"`
   454  
   455  	Pulls []Pull `json:"pulls,omitempty"`
   456  
   457  	// PathAlias is the location under <root-dir>/src
   458  	// where this repository is cloned. If this is not
   459  	// set, <root-dir>/src/github.com/org/repo will be
   460  	// used as the default.
   461  	PathAlias string `json:"path_alias,omitempty"`
   462  	// CloneURI is the URI that is used to clone the
   463  	// repository. If unset, will default to
   464  	// `https://github.com/org/repo.git`.
   465  	CloneURI string `json:"clone_uri,omitempty"`
   466  	// SkipSubmodules determines if submodules should be
   467  	// cloned when the job is run. Defaults to true.
   468  	SkipSubmodules bool `json:"skip_submodules,omitempty"`
   469  }
   470  
   471  func (r Refs) String() string {
   472  	rs := []string{fmt.Sprintf("%s:%s", r.BaseRef, r.BaseSHA)}
   473  	for _, pull := range r.Pulls {
   474  		ref := fmt.Sprintf("%d:%s", pull.Number, pull.SHA)
   475  
   476  		if pull.Ref != "" {
   477  			ref = fmt.Sprintf("%s:%s", ref, pull.Ref)
   478  		}
   479  
   480  		rs = append(rs, ref)
   481  	}
   482  	return strings.Join(rs, ",")
   483  }
   484  
   485  // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
   486  
   487  // ProwJobList is a list of ProwJob resources
   488  type ProwJobList struct {
   489  	metav1.TypeMeta `json:",inline"`
   490  	metav1.ListMeta `json:"metadata"`
   491  
   492  	Items []ProwJob `json:"items"`
   493  }