github.com/munnerz/test-infra@v0.0.0-20190108210205-ce3d181dc989/prow/apis/prowjobs/v1/types.go (about) 1 /* 2 Copyright 2018 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package v1 18 19 import ( 20 "errors" 21 "fmt" 22 "strings" 23 "time" 24 25 buildv1alpha1 "github.com/knative/build/pkg/apis/build/v1alpha1" 26 corev1 "k8s.io/api/core/v1" 27 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 28 ) 29 30 // ProwJobType specifies how the job is triggered. 31 type ProwJobType string 32 33 // Various job types. 34 const ( 35 // PresubmitJob means it runs on unmerged PRs. 36 PresubmitJob ProwJobType = "presubmit" 37 // PostsubmitJob means it runs on each new commit. 38 PostsubmitJob = "postsubmit" 39 // Periodic job means it runs on a time-basis, unrelated to git changes. 40 PeriodicJob = "periodic" 41 // BatchJob tests multiple unmerged PRs at the same time. 42 BatchJob = "batch" 43 ) 44 45 // ProwJobState specifies whether the job is running 46 type ProwJobState string 47 48 // Various job states. 49 const ( 50 // TriggeredState means the job has been created but not yet scheduled. 51 TriggeredState ProwJobState = "triggered" 52 // PendingState means the job is scheduled but not yet running. 53 PendingState = "pending" 54 // SuccessState means the job completed without error (exit 0) 55 SuccessState = "success" 56 // FailureState means the job completed with errors (exit non-zero) 57 FailureState = "failure" 58 // AbortedState means prow killed the job early (new commit pushed, perhaps). 59 AbortedState = "aborted" 60 // ErrorState means the job could not schedule (bad config, perhaps). 61 ErrorState = "error" 62 ) 63 64 // ProwJobAgent specifies the controller (such as plank or jenkins-agent) that runs the job. 65 type ProwJobAgent string 66 67 const ( 68 // KubernetesAgent means prow will create a pod to run this job. 69 KubernetesAgent ProwJobAgent = "kubernetes" 70 // JenkinsAgent means prow will schedule the job on jenkins. 71 JenkinsAgent = "jenkins" 72 // KnativeBuildAgent means prow will schedule the job via a build-crd resource. 73 KnativeBuildAgent = "knative-build" 74 ) 75 76 const ( 77 // DefaultClusterAlias specifies the default cluster key to schedule jobs. 78 DefaultClusterAlias = "default" 79 ) 80 81 // +genclient 82 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object 83 84 // ProwJob contains the spec as well as runtime metadata. 85 type ProwJob struct { 86 metav1.TypeMeta `json:",inline"` 87 metav1.ObjectMeta `json:"metadata,omitempty"` 88 89 Spec ProwJobSpec `json:"spec,omitempty"` 90 Status ProwJobStatus `json:"status,omitempty"` 91 } 92 93 // ProwJobSpec configures the details of the prow job. 94 // 95 // Details include the podspec, code to clone, the cluster it runs 96 // any child jobs, concurrency limitations, etc. 97 type ProwJobSpec struct { 98 // Type is the type of job and informs how 99 // the jobs is triggered 100 Type ProwJobType `json:"type,omitempty"` 101 // Agent determines which controller fulfills 102 // this specific ProwJobSpec and runs the job 103 Agent ProwJobAgent `json:"agent,omitempty"` 104 // Cluster is which Kubernetes cluster is used 105 // to run the job, only applicable for that 106 // specific agent 107 Cluster string `json:"cluster,omitempty"` 108 // Namespace defines where to create pods/resources. 109 Namespace string `json:"namespace,omitempty"` 110 // Job is the name of the job 111 Job string `json:"job,omitempty"` 112 // Refs is the code under test, determined at 113 // runtime by Prow itself 114 Refs *Refs `json:"refs,omitempty"` 115 // ExtraRefs are auxiliary repositories that 116 // need to be cloned, determined from config 117 ExtraRefs []Refs `json:"extra_refs,omitempty"` 118 119 // Report determines if the result of this job should 120 // be posted as a status on GitHub 121 Report bool `json:"report,omitempty"` 122 // Context is the name of the status context used to 123 // report back to GitHub 124 Context string `json:"context,omitempty"` 125 // RerunCommand is the command a user would write to 126 // trigger this job on their pull request 127 RerunCommand string `json:"rerun_command,omitempty"` 128 // MaxConcurrency restricts the total number of instances 129 // of this job that can run in parallel at once 130 MaxConcurrency int `json:"max_concurrency,omitempty"` 131 // ErrorOnEviction indicates that the ProwJob should be completed and given 132 // the ErrorState status if the pod that is executing the job is evicted. 133 // If this field is unspecified or false, a new pod will be created to replace 134 // the evicted one. 135 ErrorOnEviction bool `json:"error_on_eviction,omitempty"` 136 137 // PodSpec provides the basis for running the test under 138 // a Kubernetes agent 139 PodSpec *corev1.PodSpec `json:"pod_spec,omitempty"` 140 141 // BuildSpec provides the basis for running the test as 142 // a build-crd resource 143 // https://github.com/knative/build 144 BuildSpec *buildv1alpha1.BuildSpec `json:"build_spec,omitempty"` 145 146 // DecorationConfig holds configuration options for 147 // decorating PodSpecs that users provide 148 DecorationConfig *DecorationConfig `json:"decoration_config,omitempty"` 149 150 // RunAfterSuccess are jobs that should be triggered if 151 // this job runs and does not fail 152 RunAfterSuccess []ProwJobSpec `json:"run_after_success,omitempty"` 153 } 154 155 // DecorationConfig specifies how to augment pods. 156 // 157 // This is primarily used to provide automatic integration with gubernator 158 // and testgrid. 159 type DecorationConfig struct { 160 // Timeout is how long the pod utilities will wait 161 // before aborting a job with SIGINT. 162 Timeout time.Duration `json:"timeout,omitempty"` 163 // GracePeriod is how long the pod utilities will wait 164 // after sending SIGINT to send SIGKILL when aborting 165 // a job. Only applicable if decorating the PodSpec. 166 GracePeriod time.Duration `json:"grace_period,omitempty"` 167 // UtilityImages holds pull specs for utility container 168 // images used to decorate a PodSpec. 169 UtilityImages *UtilityImages `json:"utility_images,omitempty"` 170 // GCSConfiguration holds options for pushing logs and 171 // artifacts to GCS from a job. 172 GCSConfiguration *GCSConfiguration `json:"gcs_configuration,omitempty"` 173 // GCSCredentialsSecret is the name of the Kubernetes secret 174 // that holds GCS push credentials. 175 GCSCredentialsSecret string `json:"gcs_credentials_secret,omitempty"` 176 // SSHKeySecrets are the names of Kubernetes secrets that contain 177 // SSK keys which should be used during the cloning process. 178 SSHKeySecrets []string `json:"ssh_key_secrets,omitempty"` 179 // SSHHostFingerprints are the fingerprints of known SSH hosts 180 // that the cloning process can trust. 181 // Create with ssh-keyscan [-t rsa] host 182 SSHHostFingerprints []string `json:"ssh_host_fingerprints,omitempty"` 183 // SkipCloning determines if we should clone source code in the 184 // initcontainers for jobs that specify refs 185 SkipCloning *bool `json:"skip_cloning,omitempty"` 186 // CookieFileSecret is the name of a kubernetes secret that contains 187 // a git http.cookiefile, which should be used during the cloning process. 188 CookiefileSecret string `json:"cookiefile_secret,omitempty"` 189 } 190 191 // ApplyDefault applies the defaults for the ProwJob decoration. If a field has a zero value, it 192 // replaces that with the value set in def. 193 func (d *DecorationConfig) ApplyDefault(def *DecorationConfig) *DecorationConfig { 194 if d == nil && def == nil { 195 return nil 196 } 197 var merged DecorationConfig 198 if d != nil { 199 merged = *d 200 } else { 201 merged = *def 202 } 203 if d == nil || def == nil { 204 return &merged 205 } 206 merged.UtilityImages = merged.UtilityImages.ApplyDefault(def.UtilityImages) 207 merged.GCSConfiguration = merged.GCSConfiguration.ApplyDefault(def.GCSConfiguration) 208 209 if merged.Timeout == 0 { 210 merged.Timeout = def.Timeout 211 } 212 if merged.GracePeriod == 0 { 213 merged.GracePeriod = def.GracePeriod 214 } 215 if merged.GCSCredentialsSecret == "" { 216 merged.GCSCredentialsSecret = def.GCSCredentialsSecret 217 } 218 if len(merged.SSHKeySecrets) == 0 { 219 merged.SSHKeySecrets = def.SSHKeySecrets 220 } 221 if len(merged.SSHHostFingerprints) == 0 { 222 merged.SSHHostFingerprints = def.SSHHostFingerprints 223 } 224 if merged.SkipCloning == nil { 225 merged.SkipCloning = def.SkipCloning 226 } 227 if merged.CookiefileSecret == "" { 228 merged.CookiefileSecret = def.CookiefileSecret 229 } 230 231 return &merged 232 } 233 234 // Validate ensures all the values set in the DecorationConfig are valid. 235 func (d *DecorationConfig) Validate() error { 236 if d.UtilityImages == nil { 237 return errors.New("utility image config is not specified") 238 } 239 var missing []string 240 if d.UtilityImages.CloneRefs == "" { 241 missing = append(missing, "clonerefs") 242 } 243 if d.UtilityImages.InitUpload == "" { 244 missing = append(missing, "initupload") 245 } 246 if d.UtilityImages.Entrypoint == "" { 247 missing = append(missing, "entrypoint") 248 } 249 if d.UtilityImages.Sidecar == "" { 250 missing = append(missing, "sidecar") 251 } 252 if len(missing) > 0 { 253 return fmt.Errorf("the following utility images are not specified: %q", missing) 254 } 255 256 if d.GCSConfiguration == nil { 257 return errors.New("GCS upload configuration is not specified") 258 } 259 if d.GCSCredentialsSecret == "" { 260 return errors.New("GCS upload credential secret is not specified") 261 } 262 if err := d.GCSConfiguration.Validate(); err != nil { 263 return fmt.Errorf("GCS configuration is invalid: %v", err) 264 } 265 return nil 266 } 267 268 // UtilityImages holds pull specs for the utility images 269 // to be used for a job 270 type UtilityImages struct { 271 // CloneRefs is the pull spec used for the clonerefs utility 272 CloneRefs string `json:"clonerefs,omitempty"` 273 // InitUpload is the pull spec used for the initupload utility 274 InitUpload string `json:"initupload,omitempty"` 275 // Entrypoint is the pull spec used for the entrypoint utility 276 Entrypoint string `json:"entrypoint,omitempty"` 277 // sidecar is the pull spec used for the sidecar utility 278 Sidecar string `json:"sidecar,omitempty"` 279 } 280 281 // ApplyDefault applies the defaults for the UtilityImages decorations. If a field has a zero value, 282 // it replaces that with the value set in def. 283 func (u *UtilityImages) ApplyDefault(def *UtilityImages) *UtilityImages { 284 if u == nil { 285 return def 286 } else if def == nil { 287 return u 288 } 289 290 merged := *u 291 if merged.CloneRefs == "" { 292 merged.CloneRefs = def.CloneRefs 293 } 294 if merged.InitUpload == "" { 295 merged.InitUpload = def.InitUpload 296 } 297 if merged.Entrypoint == "" { 298 merged.Entrypoint = def.Entrypoint 299 } 300 if merged.Sidecar == "" { 301 merged.Sidecar = def.Sidecar 302 } 303 return &merged 304 } 305 306 // PathStrategy specifies minutia about how to construct the url. 307 // Usually consumed by gubernator/testgrid. 308 const ( 309 PathStrategyLegacy = "legacy" 310 PathStrategySingle = "single" 311 PathStrategyExplicit = "explicit" 312 ) 313 314 // GCSConfiguration holds options for pushing logs and 315 // artifacts to GCS from a job. 316 type GCSConfiguration struct { 317 // Bucket is the GCS bucket to upload to 318 Bucket string `json:"bucket,omitempty"` 319 // PathPrefix is an optional path that follows the 320 // bucket name and comes before any structure 321 PathPrefix string `json:"path_prefix,omitempty"` 322 // PathStrategy dictates how the org and repo are used 323 // when calculating the full path to an artifact in GCS 324 PathStrategy string `json:"path_strategy,omitempty"` 325 // DefaultOrg is omitted from GCS paths when using the 326 // legacy or simple strategy 327 DefaultOrg string `json:"default_org,omitempty"` 328 // DefaultRepo is omitted from GCS paths when using the 329 // legacy or simple strategy 330 DefaultRepo string `json:"default_repo,omitempty"` 331 } 332 333 // ApplyDefault applies the defaults for GCSConfiguration decorations. If a field has a zero value, 334 // it replaces that with the value set in def. 335 func (g *GCSConfiguration) ApplyDefault(def *GCSConfiguration) *GCSConfiguration { 336 if g == nil && def == nil { 337 return nil 338 } 339 var merged GCSConfiguration 340 if g != nil { 341 merged = *g 342 } else { 343 merged = *def 344 } 345 if g == nil || def == nil { 346 return &merged 347 } 348 349 if merged.Bucket == "" { 350 merged.Bucket = def.Bucket 351 } 352 if merged.PathPrefix == "" { 353 merged.PathPrefix = def.PathPrefix 354 } 355 if merged.PathStrategy == "" { 356 merged.PathStrategy = def.PathStrategy 357 } 358 if merged.DefaultOrg == "" { 359 merged.DefaultOrg = def.DefaultOrg 360 } 361 if merged.DefaultRepo == "" { 362 merged.DefaultRepo = def.DefaultRepo 363 } 364 return &merged 365 } 366 367 // Validate ensures all the values set in the GCSConfiguration are valid. 368 func (g *GCSConfiguration) Validate() error { 369 if g.PathStrategy != PathStrategyLegacy && g.PathStrategy != PathStrategyExplicit && g.PathStrategy != PathStrategySingle { 370 return fmt.Errorf("gcs_path_strategy must be one of %q, %q, or %q", PathStrategyLegacy, PathStrategyExplicit, PathStrategySingle) 371 } 372 if g.PathStrategy != PathStrategyExplicit && (g.DefaultOrg == "" || g.DefaultRepo == "") { 373 return fmt.Errorf("default org and repo must be provided for GCS strategy %q", g.PathStrategy) 374 } 375 return nil 376 } 377 378 // ProwJobStatus provides runtime metadata, such as when it finished, whether it is running, etc. 379 type ProwJobStatus struct { 380 StartTime metav1.Time `json:"startTime,omitempty"` 381 CompletionTime *metav1.Time `json:"completionTime,omitempty"` 382 State ProwJobState `json:"state,omitempty"` 383 Description string `json:"description,omitempty"` 384 URL string `json:"url,omitempty"` 385 386 // PodName applies only to ProwJobs fulfilled by 387 // plank. This field should always be the same as 388 // the ProwJob.ObjectMeta.Name field. 389 PodName string `json:"pod_name,omitempty"` 390 391 // BuildID is the build identifier vended either by tot 392 // or the snowflake library for this job and used as an 393 // identifier for grouping artifacts in GCS for views in 394 // TestGrid and Gubernator. Idenitifiers vended by tot 395 // are monotonically increasing whereas identifiers vended 396 // by the snowflake library are not. 397 BuildID string `json:"build_id,omitempty"` 398 399 // JenkinsBuildID applies only to ProwJobs fulfilled 400 // by the jenkins-operator. This field is the build 401 // identifier that Jenkins gave to the build for this 402 // ProwJob. 403 JenkinsBuildID string `json:"jenkins_build_id,omitempty"` 404 405 // PrevReportStates stores the previous reported prowjob state per reporter 406 // So crier won't make duplicated report attempt 407 PrevReportStates map[string]ProwJobState `json:"prev_report_states,omitempty"` 408 } 409 410 // Complete returns true if the prow job has finished 411 func (j *ProwJob) Complete() bool { 412 // TODO(fejta): support a timeout? 413 return j.Status.CompletionTime != nil 414 } 415 416 // SetComplete marks the job as completed (at time now). 417 func (j *ProwJob) SetComplete() { 418 j.Status.CompletionTime = new(metav1.Time) 419 *j.Status.CompletionTime = metav1.Now() 420 } 421 422 // ClusterAlias specifies the key in the clusters map to use. 423 // 424 // This allows scheduling a prow job somewhere aside from the default build cluster. 425 func (j *ProwJob) ClusterAlias() string { 426 if j.Spec.Cluster == "" { 427 return DefaultClusterAlias 428 } 429 return j.Spec.Cluster 430 } 431 432 // Pull describes a pull request at a particular point in time. 433 type Pull struct { 434 Number int `json:"number,omitempty"` 435 Author string `json:"author,omitempty"` 436 SHA string `json:"sha,omitempty"` 437 438 // Ref is git ref can be checked out for a change 439 // for example, 440 // github: pull/123/head 441 // gerrit: refs/changes/00/123/1 442 Ref string `json:"ref,omitempty"` 443 } 444 445 // Refs describes how the repo was constructed. 446 type Refs struct { 447 // Org is something like kubernetes or k8s.io 448 Org string `json:"org,omitempty"` 449 // Repo is something like test-infra 450 Repo string `json:"repo,omitempty"` 451 452 BaseRef string `json:"base_ref,omitempty"` 453 BaseSHA string `json:"base_sha,omitempty"` 454 455 Pulls []Pull `json:"pulls,omitempty"` 456 457 // PathAlias is the location under <root-dir>/src 458 // where this repository is cloned. If this is not 459 // set, <root-dir>/src/github.com/org/repo will be 460 // used as the default. 461 PathAlias string `json:"path_alias,omitempty"` 462 // CloneURI is the URI that is used to clone the 463 // repository. If unset, will default to 464 // `https://github.com/org/repo.git`. 465 CloneURI string `json:"clone_uri,omitempty"` 466 // SkipSubmodules determines if submodules should be 467 // cloned when the job is run. Defaults to true. 468 SkipSubmodules bool `json:"skip_submodules,omitempty"` 469 } 470 471 func (r Refs) String() string { 472 rs := []string{fmt.Sprintf("%s:%s", r.BaseRef, r.BaseSHA)} 473 for _, pull := range r.Pulls { 474 ref := fmt.Sprintf("%d:%s", pull.Number, pull.SHA) 475 476 if pull.Ref != "" { 477 ref = fmt.Sprintf("%s:%s", ref, pull.Ref) 478 } 479 480 rs = append(rs, ref) 481 } 482 return strings.Join(rs, ",") 483 } 484 485 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object 486 487 // ProwJobList is a list of ProwJob resources 488 type ProwJobList struct { 489 metav1.TypeMeta `json:",inline"` 490 metav1.ListMeta `json:"metadata"` 491 492 Items []ProwJob `json:"items"` 493 }