github.com/munnerz/test-infra@v0.0.0-20190108210205-ce3d181dc989/prow/config/jobs.go (about) 1 /* 2 Copyright 2017 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package config 18 19 import ( 20 "fmt" 21 "regexp" 22 "time" 23 24 buildv1alpha1 "github.com/knative/build/pkg/apis/build/v1alpha1" 25 26 "k8s.io/api/core/v1" 27 "k8s.io/apimachinery/pkg/util/sets" 28 "k8s.io/test-infra/prow/kube" 29 ) 30 31 // Preset is intended to match the k8s' PodPreset feature, and may be removed 32 // if that feature goes beta. 33 type Preset struct { 34 Labels map[string]string `json:"labels"` 35 Env []v1.EnvVar `json:"env"` 36 Volumes []v1.Volume `json:"volumes"` 37 VolumeMounts []v1.VolumeMount `json:"volumeMounts"` 38 } 39 40 func mergePreset(preset Preset, labels map[string]string, pod *v1.PodSpec) error { 41 if pod == nil { 42 return nil 43 } 44 for l, v := range preset.Labels { 45 if v2, ok := labels[l]; !ok || v2 != v { 46 return nil 47 } 48 } 49 for _, e1 := range preset.Env { 50 for i := range pod.Containers { 51 for _, e2 := range pod.Containers[i].Env { 52 if e1.Name == e2.Name { 53 return fmt.Errorf("env var duplicated in pod spec: %s", e1.Name) 54 } 55 } 56 pod.Containers[i].Env = append(pod.Containers[i].Env, e1) 57 } 58 } 59 for _, v1 := range preset.Volumes { 60 for _, v2 := range pod.Volumes { 61 if v1.Name == v2.Name { 62 return fmt.Errorf("volume duplicated in pod spec: %s", v1.Name) 63 } 64 } 65 pod.Volumes = append(pod.Volumes, v1) 66 } 67 for _, vm1 := range preset.VolumeMounts { 68 for i := range pod.Containers { 69 for _, vm2 := range pod.Containers[i].VolumeMounts { 70 if vm1.Name == vm2.Name { 71 return fmt.Errorf("volume mount duplicated in pod spec: %s", vm1.Name) 72 } 73 } 74 pod.Containers[i].VolumeMounts = append(pod.Containers[i].VolumeMounts, vm1) 75 } 76 } 77 return nil 78 } 79 80 // JobBase contains attributes common to all job types 81 type JobBase struct { 82 // The name of the job. Must match regex [A-Za-z0-9-._]+ 83 // e.g. pull-test-infra-bazel-build 84 Name string `json:"name"` 85 // Labels are added to prowjobs and pods created for this job. 86 Labels map[string]string `json:"labels,omitempty"` 87 // MaximumConcurrency of this job, 0 implies no limit. 88 MaxConcurrency int `json:"max_concurrency,omitempty"` 89 // Agent that will take care of running this job. 90 Agent string `json:"agent"` 91 // Cluster is the alias of the cluster to run this job in. 92 // (Default: kube.DefaultClusterAlias) 93 Cluster string `json:"cluster,omitempty"` 94 // Namespace is the namespace in which pods schedule. 95 // nil: results in config.PodNamespace (aka pod default) 96 // empty: results in config.ProwJobNamespace (aka same as prowjob) 97 Namespace *string `json:"namespace,omitempty"` 98 // ErrorOnEviction indicates that the ProwJob should be completed and given 99 // the ErrorState status if the pod that is executing the job is evicted. 100 // If this field is unspecified or false, a new pod will be created to replace 101 // the evicted one. 102 ErrorOnEviction bool `json:"error_on_eviction,omitempty"` 103 // SourcePath contains the path where this job is defined 104 SourcePath string `json:"-"` 105 // Spec is the Kubernetes pod spec used if Agent is kubernetes. 106 Spec *v1.PodSpec `json:"spec,omitempty"` 107 // BuildSpec is the Knative build spec used if Agent is knative-build. 108 BuildSpec *buildv1alpha1.BuildSpec `json:"build_spec,omitempty"` 109 110 UtilityConfig 111 } 112 113 // Presubmit runs on PRs. 114 type Presubmit struct { 115 JobBase 116 117 // AlwaysRun automatically for every PR, or only when a comment triggers it. 118 AlwaysRun bool `json:"always_run"` 119 120 // Context is the name of the GitHub status context for the job. 121 Context string `json:"context"` 122 // Optional indicates that the job's status context should not be required for merge. 123 Optional bool `json:"optional,omitempty"` 124 // SkipReport skips commenting and setting status on GitHub. 125 SkipReport bool `json:"skip_report,omitempty"` 126 127 // Trigger is the regular expression to trigger the job. 128 // e.g. `@k8s-bot e2e test this` 129 // RerunCommand must also be specified if this field is specified. 130 // (Default: `(?m)^/test (?:.*? )?<job name>(?: .*?)?$`) 131 Trigger string `json:"trigger"` 132 // The RerunCommand to give users. Must match Trigger. 133 // Trigger must also be specified if this field is specified. 134 // (Default: `/test <job name>`) 135 RerunCommand string `json:"rerun_command"` 136 137 // RunAfterSuccess is a list of jobs to run after successfully running this one. 138 RunAfterSuccess []Presubmit `json:"run_after_success,omitempty"` 139 140 Brancher 141 142 RegexpChangeMatcher 143 144 // We'll set these when we load it. 145 re *regexp.Regexp // from Trigger. 146 } 147 148 // Postsubmit runs on push events. 149 type Postsubmit struct { 150 JobBase 151 152 RegexpChangeMatcher 153 154 Brancher 155 156 // Run these jobs after successfully running this one. 157 RunAfterSuccess []Postsubmit `json:"run_after_success,omitempty"` 158 } 159 160 // Periodic runs on a timer. 161 type Periodic struct { 162 JobBase 163 164 // (deprecated)Interval to wait between two runs of the job. 165 Interval string `json:"interval"` 166 // Cron representation of job trigger time 167 Cron string `json:"cron"` 168 // Tags for config entries 169 Tags []string `json:"tags,omitempty"` 170 // Run these jobs after successfully running this one. 171 RunAfterSuccess []Periodic `json:"run_after_success,omitempty"` 172 173 interval time.Duration 174 } 175 176 // SetInterval updates interval, the frequency duration it runs. 177 func (p *Periodic) SetInterval(d time.Duration) { 178 p.interval = d 179 } 180 181 // GetInterval returns interval, the frequency duration it runs. 182 func (p *Periodic) GetInterval() time.Duration { 183 return p.interval 184 } 185 186 // Brancher is for shared code between jobs that only run against certain 187 // branches. An empty brancher runs against all branches. 188 type Brancher struct { 189 // Do not run against these branches. Default is no branches. 190 SkipBranches []string `json:"skip_branches,omitempty"` 191 // Only run against these branches. Default is all branches. 192 Branches []string `json:"branches,omitempty"` 193 194 // We'll set these when we load it. 195 re *regexp.Regexp 196 reSkip *regexp.Regexp 197 } 198 199 // RegexpChangeMatcher is for code shared between jobs that run only when certain files are changed. 200 type RegexpChangeMatcher struct { 201 // RunIfChanged defines a regex used to select which subset of file changes should trigger this job. 202 // If any file in the changeset matches this regex, the job will be triggered 203 RunIfChanged string `json:"run_if_changed,omitempty"` 204 reChanges *regexp.Regexp // from RunIfChanged 205 } 206 207 // RunsAgainstAllBranch returns true if there are both branches and skip_branches are unset 208 func (br Brancher) RunsAgainstAllBranch() bool { 209 return len(br.SkipBranches) == 0 && len(br.Branches) == 0 210 } 211 212 // RunsAgainstBranch returns true if the input branch matches, given the whitelist/blacklist. 213 func (br Brancher) RunsAgainstBranch(branch string) bool { 214 if br.RunsAgainstAllBranch() { 215 return true 216 } 217 218 // Favor SkipBranches over Branches 219 if len(br.SkipBranches) != 0 && br.reSkip.MatchString(branch) { 220 return false 221 } 222 if len(br.Branches) == 0 || br.re.MatchString(branch) { 223 return true 224 } 225 return false 226 } 227 228 // Intersects checks if other Brancher would trigger for the same branch. 229 func (br Brancher) Intersects(other Brancher) bool { 230 if br.RunsAgainstAllBranch() || other.RunsAgainstAllBranch() { 231 return true 232 } 233 if len(br.Branches) > 0 { 234 baseBranches := sets.NewString(br.Branches...) 235 if len(other.Branches) > 0 { 236 otherBranches := sets.NewString(other.Branches...) 237 if baseBranches.Intersection(otherBranches).Len() > 0 { 238 return true 239 } 240 return false 241 } 242 if !baseBranches.Intersection(sets.NewString(other.SkipBranches...)).Equal(baseBranches) { 243 return true 244 } 245 return false 246 } 247 if len(other.Branches) == 0 { 248 // There can only be one Brancher with skip_branches. 249 return true 250 } 251 return other.Intersects(br) 252 } 253 254 // RunsAgainstChanges returns true if any of the changed input paths match the run_if_changed regex. 255 func (cm RegexpChangeMatcher) RunsAgainstChanges(changes []string) bool { 256 if cm.RunIfChanged == "" { 257 return true 258 } 259 for _, change := range changes { 260 if cm.reChanges.MatchString(change) { 261 return true 262 } 263 } 264 return false 265 } 266 267 // TriggerMatches returns true if the comment body should trigger this presubmit. 268 // 269 // This is usually a /test foo string. 270 func (ps Presubmit) TriggerMatches(body string) bool { 271 return ps.re.MatchString(body) 272 } 273 274 // ContextRequired checks whether a context is required from github points of view (required check). 275 func (ps Presubmit) ContextRequired() bool { 276 if ps.Optional || ps.SkipReport { 277 return false 278 } 279 return true 280 } 281 282 // ChangedFilesProvider returns a slice of modified files. 283 type ChangedFilesProvider func() ([]string, error) 284 285 func matching(j Presubmit, body string, testAll bool) []Presubmit { 286 // When matching ignore whether the job runs for the branch or whether the job runs for the 287 // PR's changes. Even if the job doesn't run, it still matches the PR and may need to be marked 288 // as skipped on github. 289 var result []Presubmit 290 if (testAll && (j.AlwaysRun || j.RunIfChanged != "")) || j.TriggerMatches(body) { 291 result = append(result, j) 292 } 293 for _, child := range j.RunAfterSuccess { 294 result = append(result, matching(child, body, testAll)...) 295 } 296 return result 297 } 298 299 // MatchingPresubmits returns a slice of presubmits to trigger based on the repo and a comment text. 300 func (c *JobConfig) MatchingPresubmits(fullRepoName, body string, testAll bool) []Presubmit { 301 var result []Presubmit 302 if jobs, ok := c.Presubmits[fullRepoName]; ok { 303 for _, job := range jobs { 304 result = append(result, matching(job, body, testAll)...) 305 } 306 } 307 return result 308 } 309 310 // UtilityConfig holds decoration metadata, such as how to clone and additional containers/etc 311 type UtilityConfig struct { 312 // Decorate determines if we decorate the PodSpec or not 313 Decorate bool `json:"decorate,omitempty"` 314 315 // PathAlias is the location under <root-dir>/src 316 // where the repository under test is cloned. If this 317 // is not set, <root-dir>/src/github.com/org/repo will 318 // be used as the default. 319 PathAlias string `json:"path_alias,omitempty"` 320 // CloneURI is the URI that is used to clone the 321 // repository. If unset, will default to 322 // `https://github.com/org/repo.git`. 323 CloneURI string `json:"clone_uri,omitempty"` 324 // SkipSubmodules determines if submodules should be 325 // cloned when the job is run. Defaults to true. 326 SkipSubmodules bool `json:"skip_submodules,omitempty"` 327 328 // ExtraRefs are auxiliary repositories that 329 // need to be cloned, determined from config 330 ExtraRefs []kube.Refs `json:"extra_refs,omitempty"` 331 332 // DecorationConfig holds configuration options for 333 // decorating PodSpecs that users provide 334 DecorationConfig *kube.DecorationConfig `json:"decoration_config,omitempty"` 335 } 336 337 // RetestPresubmits returns all presubmits that should be run given a /retest command. 338 // This is the set of all presubmits intersected with ((alwaysRun + runContexts) - skipContexts) 339 func (c *JobConfig) RetestPresubmits(fullRepoName string, skipContexts, runContexts map[string]bool) []Presubmit { 340 var result []Presubmit 341 if jobs, ok := c.Presubmits[fullRepoName]; ok { 342 for _, job := range jobs { 343 if skipContexts[job.Context] { 344 continue 345 } 346 if job.AlwaysRun || job.RunIfChanged != "" || runContexts[job.Context] { 347 result = append(result, job) 348 } 349 } 350 } 351 return result 352 } 353 354 // GetPresubmit returns the presubmit job for the provided repo and job name. 355 func (c *JobConfig) GetPresubmit(repo, jobName string) *Presubmit { 356 presubmits := c.AllPresubmits([]string{repo}) 357 for i := range presubmits { 358 ps := presubmits[i] 359 if ps.Name == jobName { 360 return &ps 361 } 362 } 363 return nil 364 } 365 366 // SetPresubmits updates c.Presubmits to jobs, after compiling and validating their regexes. 367 func (c *JobConfig) SetPresubmits(jobs map[string][]Presubmit) error { 368 nj := map[string][]Presubmit{} 369 for k, v := range jobs { 370 nj[k] = make([]Presubmit, len(v)) 371 copy(nj[k], v) 372 if err := SetPresubmitRegexes(nj[k]); err != nil { 373 return err 374 } 375 } 376 c.Presubmits = nj 377 return nil 378 } 379 380 // SetPostsubmits updates c.Postsubmits to jobs, after compiling and validating their regexes. 381 func (c *JobConfig) SetPostsubmits(jobs map[string][]Postsubmit) error { 382 nj := map[string][]Postsubmit{} 383 for k, v := range jobs { 384 nj[k] = make([]Postsubmit, len(v)) 385 copy(nj[k], v) 386 if err := SetPostsubmitRegexes(nj[k]); err != nil { 387 return err 388 } 389 } 390 c.Postsubmits = nj 391 return nil 392 } 393 394 // listPresubmits list all the presubmit for a given repo including the run after success jobs. 395 func listPresubmits(ps []Presubmit) []Presubmit { 396 var res []Presubmit 397 for _, p := range ps { 398 res = append(res, p) 399 res = append(res, listPresubmits(p.RunAfterSuccess)...) 400 } 401 return res 402 } 403 404 // AllPresubmits returns all prow presubmit jobs in repos. 405 // if repos is empty, return all presubmits. 406 func (c *JobConfig) AllPresubmits(repos []string) []Presubmit { 407 var res []Presubmit 408 409 for repo, v := range c.Presubmits { 410 if len(repos) == 0 { 411 res = append(res, listPresubmits(v)...) 412 } else { 413 for _, r := range repos { 414 if r == repo { 415 res = append(res, listPresubmits(v)...) 416 break 417 } 418 } 419 } 420 } 421 422 return res 423 } 424 425 // listPostsubmits list all the postsubmits for a given repo including the run after success jobs. 426 func listPostsubmits(ps []Postsubmit) []Postsubmit { 427 var res []Postsubmit 428 for _, p := range ps { 429 res = append(res, p) 430 res = append(res, listPostsubmits(p.RunAfterSuccess)...) 431 } 432 return res 433 } 434 435 // AllPostsubmits returns all prow postsubmit jobs in repos. 436 // if repos is empty, return all postsubmits. 437 func (c *JobConfig) AllPostsubmits(repos []string) []Postsubmit { 438 var res []Postsubmit 439 440 for repo, v := range c.Postsubmits { 441 if len(repos) == 0 { 442 res = append(res, listPostsubmits(v)...) 443 } else { 444 for _, r := range repos { 445 if r == repo { 446 res = append(res, listPostsubmits(v)...) 447 break 448 } 449 } 450 } 451 } 452 453 return res 454 } 455 456 // AllPeriodics returns all prow periodic jobs. 457 func (c *JobConfig) AllPeriodics() []Periodic { 458 var listPeriodic func(ps []Periodic) []Periodic 459 listPeriodic = func(ps []Periodic) []Periodic { 460 var res []Periodic 461 for _, p := range ps { 462 res = append(res, p) 463 res = append(res, listPeriodic(p.RunAfterSuccess)...) 464 } 465 return res 466 } 467 468 return listPeriodic(c.Periodics) 469 }