github.com/abayer/test-infra@v0.0.5/mungegithub/features/repo-updates.go (about) 1 /* 2 Copyright 2016 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package features 18 19 import ( 20 "fmt" 21 "io/ioutil" 22 "os" 23 "os/exec" 24 "path" 25 "path/filepath" 26 "regexp" 27 "strings" 28 29 "k8s.io/apimachinery/pkg/util/sets" 30 "k8s.io/apimachinery/pkg/util/yaml" 31 "k8s.io/test-infra/mungegithub/github" 32 "k8s.io/test-infra/mungegithub/options" 33 34 parseYaml "github.com/ghodss/yaml" 35 "github.com/golang/glog" 36 ) 37 38 const ( 39 ownerFilename = "OWNERS" // file which contains approvers and reviewers 40 // RepoFeatureName is how mungers should indicate this is required 41 RepoFeatureName = "gitrepos" 42 // Github's api uses "" (empty) string as basedir by convention but it's clearer to use "/" 43 baseDirConvention = "" 44 ) 45 46 type assignmentConfig struct { 47 Approvers []string `json:"approvers" yaml:"approvers"` 48 Reviewers []string `json:"reviewers" yaml:"reviewers"` 49 Labels []string `json:"labels" yaml:"labels"` 50 } 51 52 type aliasData struct { 53 // Contains the mapping between aliases and lists of members. 54 AliasMap map[string][]string `json:"aliases"` 55 } 56 57 type aliasReader interface { 58 read() ([]byte, error) 59 } 60 61 func (o *RepoInfo) read() ([]byte, error) { 62 return ioutil.ReadFile(o.aliasFile) 63 } 64 65 // RepoInfo provides information about users in OWNERS files in a git repo 66 type RepoInfo struct { 67 baseDir string 68 enableMdYaml bool 69 useReviewers bool 70 71 aliasFile string 72 aliasData *aliasData 73 aliasReader aliasReader 74 75 projectDir string 76 approvers map[string]sets.String 77 reviewers map[string]sets.String 78 labels map[string]sets.String 79 allPossibleLabels sets.String 80 config *github.Config 81 } 82 83 func init() { 84 RegisterFeature(&RepoInfo{}) 85 } 86 87 // Name is just going to return the name mungers use to request this feature 88 func (o *RepoInfo) Name() string { 89 return RepoFeatureName 90 } 91 92 // by default, github's api doesn't root the project directory at "/" and instead uses the empty string for the base dir 93 // of the project. And the built-in dir function returns "." for empty strings, so for consistency, we use this 94 // canonicalize to get the directories of files in a consistent format with NO "/" at the root (a/b/c/ -> a/b/c) 95 func canonicalize(path string) string { 96 if path == "." { 97 return baseDirConvention 98 } 99 return strings.TrimSuffix(path, "/") 100 } 101 102 func (o *RepoInfo) walkFunc(path string, info os.FileInfo, err error) error { 103 if err != nil { 104 glog.Errorf("%v", err) 105 return nil 106 } 107 filename := filepath.Base(path) 108 if info.Mode().IsDir() { 109 switch filename { 110 case ".git": 111 return filepath.SkipDir 112 case "_output": 113 return filepath.SkipDir 114 } 115 } 116 if !info.Mode().IsRegular() { 117 return nil 118 } 119 120 c := &assignmentConfig{} 121 122 // '.md' files may contain assignees at the top of the file in a yaml header 123 // Flag guarded because this is only enabled in some repos 124 if o.enableMdYaml && filename != ownerFilename && strings.HasSuffix(filename, "md") { 125 // Parse the yaml header from the file if it exists and marshal into the config 126 if err := decodeAssignmentConfig(path, c); err != nil { 127 glog.Errorf("%v", err) 128 return err 129 } 130 131 // Set assignees for this file using the relative path if they were found 132 path, err = filepath.Rel(o.projectDir, path) 133 if err != nil { 134 glog.Errorf("Unable to find relative path between %q and %q: %v", o.projectDir, path, err) 135 return err 136 } 137 c.normalizeUsers() 138 o.approvers[path] = sets.NewString(c.Approvers...) 139 o.reviewers[path] = sets.NewString(c.Reviewers...) 140 return nil 141 } 142 143 if filename != ownerFilename { 144 return nil 145 } 146 147 file, err := os.Open(path) 148 if err != nil { 149 glog.Errorf("Could not open %q: %v", path, err) 150 return nil 151 } 152 defer file.Close() 153 154 if err := yaml.NewYAMLToJSONDecoder(file).Decode(c); err != nil { 155 glog.Errorf("Could not decode %q: %v", path, err) 156 return nil 157 } 158 159 path, err = filepath.Rel(o.projectDir, path) 160 path = filepath.Dir(path) 161 if err != nil { 162 glog.Errorf("Unable to find relative path between %q and %q: %v", o.projectDir, path, err) 163 return err 164 } 165 path = canonicalize(path) 166 c.normalizeUsers() 167 o.approvers[path] = sets.NewString(c.Approvers...) 168 o.reviewers[path] = sets.NewString(c.Reviewers...) 169 if len(c.Labels) > 0 { 170 o.labels[path] = sets.NewString(c.Labels...) 171 o.allPossibleLabels.Insert(c.Labels...) 172 } 173 return nil 174 } 175 176 // caseNormalizeAll normalizes all entries in 177 // the assignment configuration so that we can do case- 178 // insensitive operations on the entries 179 func (c *assignmentConfig) normalizeUsers() { 180 c.Approvers = caseNormalizeAll(c.Approvers) 181 c.Reviewers = caseNormalizeAll(c.Reviewers) 182 } 183 184 func caseNormalizeAll(users []string) []string { 185 normalizedUsers := users[:0] 186 for _, user := range users { 187 normalizedUsers = append(normalizedUsers, caseNormalize(user)) 188 } 189 return normalizedUsers 190 } 191 192 func caseNormalize(user string) string { 193 return strings.ToLower(user) 194 } 195 196 // decodeAssignmentConfig will parse the yaml header if it exists and unmarshal it into an assignmentConfig. 197 // If no yaml header is found, do nothing 198 // Returns an error if the file cannot be read or the yaml header is found but cannot be unmarshalled 199 var mdStructuredHeaderRegex = regexp.MustCompile("^---\n(.|\n)*\n---") 200 201 func decodeAssignmentConfig(path string, config *assignmentConfig) error { 202 fileBytes, err := ioutil.ReadFile(path) 203 if err != nil { 204 return err 205 } 206 // Parse the yaml header from the top of the file. Will return an empty string if regex does not match. 207 meta := mdStructuredHeaderRegex.FindString(string(fileBytes)) 208 209 // Unmarshal the yaml header into the config 210 return parseYaml.Unmarshal([]byte(meta), &config) 211 } 212 213 func (o *RepoInfo) updateRepoAliases() error { 214 if len(o.aliasFile) == 0 { 215 return nil 216 } 217 218 // read and check the alias-file. 219 fileContents, err := o.aliasReader.read() 220 if os.IsNotExist(err) { 221 glog.Infof("Missing alias-file (%s), using empty alias structure.", o.aliasFile) 222 o.aliasData = &aliasData{} 223 return nil 224 } 225 if err != nil { 226 return fmt.Errorf("Unable to read alias file: %v", err) 227 } 228 229 var data aliasData 230 if err := parseYaml.Unmarshal(fileContents, &data); err != nil { 231 return fmt.Errorf("Failed to decode the alias file: %v", err) 232 } 233 234 o.aliasData = normalizeAliases(data) 235 return nil 236 } 237 238 // normalizeAliases normalizes all entries in the alias data 239 // so that we can do case-insensitive resolution of aliases 240 func normalizeAliases(rawData aliasData) *aliasData { 241 normalizedData := aliasData{AliasMap: map[string][]string{}} 242 for alias, users := range rawData.AliasMap { 243 normalizedData.AliasMap[caseNormalize(alias)] = caseNormalizeAll(users) 244 } 245 return &normalizedData 246 } 247 248 // expandAllAliases expands all of the aliases in the 249 // lists of approvers and reviewers for paths we track 250 func (o *RepoInfo) expandAllAliases() { 251 for filePath := range o.approvers { 252 o.resolveAliases(o.approvers[filePath]) 253 } 254 255 for filePath := range o.reviewers { 256 o.resolveAliases(o.reviewers[filePath]) 257 } 258 } 259 260 // resolveAliases returns the members that need to 261 // be added and removed from a set to yield a set with 262 // all aliases expanded 263 func (o *RepoInfo) resolveAliases(users sets.String) { 264 for _, owner := range users.List() { 265 if aliases := o.resolveAlias(owner); len(aliases) > 0 { 266 users.Delete(owner) 267 users.Insert(aliases...) 268 } 269 } 270 } 271 272 // resolveAlias resolves the identity or identities for 273 // an alias. If we have no record of the alias, an empty 274 // slice is returned 275 func (o *RepoInfo) resolveAlias(owner string) []string { 276 if val, ok := o.aliasData.AliasMap[owner]; ok { 277 return val 278 } 279 return []string{} 280 } 281 282 func (o *RepoInfo) updateRepoUsers() error { 283 out, err := o.GitCommand([]string{"rev-parse", "HEAD"}) 284 if err != nil { 285 glog.Errorf("Unable get sha of HEAD:\n%s\n%v", string(out), err) 286 return err 287 } 288 sha := out 289 290 o.approvers = map[string]sets.String{} 291 o.reviewers = map[string]sets.String{} 292 o.labels = map[string]sets.String{} 293 o.allPossibleLabels = sets.String{} 294 err = filepath.Walk(o.projectDir, o.walkFunc) 295 if err != nil { 296 glog.Errorf("Got error %v", err) 297 } 298 o.expandAllAliases() 299 if err := o.pruneRepoUsers(); err != nil { 300 return err 301 } 302 glog.Infof("Loaded config from %s:%s", o.projectDir, sha) 303 glog.V(5).Infof("approvers: %v", o.approvers) 304 glog.V(5).Infof("reviewers: %v", o.reviewers) 305 return nil 306 } 307 308 // Initialize will initialize the munger 309 func (o *RepoInfo) Initialize(config *github.Config) error { 310 o.config = config 311 o.projectDir = path.Join(o.baseDir, o.config.Project) 312 313 if len(o.baseDir) == 0 { 314 glog.Fatalf("--repo-dir is required with selected munger(s)") 315 } 316 finfo, err := os.Stat(o.baseDir) 317 if err != nil { 318 return fmt.Errorf("Unable to stat --repo-dir: %v", err) 319 } 320 if !finfo.IsDir() { 321 return fmt.Errorf("--repo-dir is not a directory") 322 } 323 324 if o.fsckRepo() != nil { 325 if err := o.cleanUp(o.projectDir); err != nil { 326 return fmt.Errorf("Unable to remove old clone directory at %v: %v", o.projectDir, err) 327 } 328 } 329 if !o.exists() { 330 if cloneUrl, err := o.cloneRepo(); err != nil { 331 return fmt.Errorf("Unable to clone %v: %v", cloneUrl, err) 332 } 333 } 334 335 o.aliasData = &aliasData{} 336 o.aliasReader = o 337 if err := o.updateRepoAliases(); err != nil { 338 return err 339 } 340 return o.updateRepoUsers() 341 } 342 343 func (o *RepoInfo) cleanUp(path string) error { 344 return os.RemoveAll(path) 345 } 346 347 func (o *RepoInfo) exists() bool { 348 _, err := os.Stat(o.projectDir) 349 return !os.IsNotExist(err) 350 } 351 352 func (o *RepoInfo) fsckRepo() error { 353 if !o.exists() { 354 return fmt.Errorf("fsck repo failed: %q is missing", o.projectDir) 355 } 356 output, err := o.gitCommandDir([]string{"fsck"}, o.projectDir) 357 if err != nil { 358 glog.Errorf("fsck repo failed: %s", output) 359 } 360 return err 361 } 362 363 func (o *RepoInfo) cloneRepo() (string, error) { 364 cloneUrl := fmt.Sprintf("https://github.com/%s/%s.git", o.config.Org, o.config.Project) 365 output, err := o.gitCommandDir([]string{"clone", cloneUrl, o.projectDir}, o.baseDir) 366 if err != nil { 367 glog.Errorf("Failed to clone github repo: %s", output) 368 } 369 return cloneUrl, err 370 } 371 372 func (o *RepoInfo) pruneRepoUsers() error { 373 whitelist, err := o.config.Collaborators() 374 if err != nil { 375 return err 376 } 377 378 for repo, users := range o.approvers { 379 o.approvers[repo] = whitelist.Intersection(users) 380 } 381 for repo, users := range o.reviewers { 382 o.reviewers[repo] = whitelist.Intersection(users) 383 } 384 return nil 385 } 386 387 // EachLoop is called at the start of every munge loop 388 func (o *RepoInfo) EachLoop() error { 389 if out, err := o.GitCommand([]string{"remote", "update"}); err != nil { 390 glog.Errorf("Unable to git remote update:\n%s\n%v", out, err) 391 } 392 if out, err := o.GitCommand([]string{"pull"}); err != nil { 393 glog.Errorf("Unable to run git pull:\n%s\n%v", string(out), err) 394 return err 395 } 396 397 if err := o.updateRepoAliases(); err != nil { 398 return err 399 } 400 return o.updateRepoUsers() 401 } 402 403 // RegisterOptions registers options for this feature; returns any that require a restart when changed. 404 func (o *RepoInfo) RegisterOptions(opts *options.Options) sets.String { 405 opts.RegisterString(&o.baseDir, "repo-dir", "", "Path to perform checkout of repository") 406 opts.RegisterBool(&o.enableMdYaml, "enable-md-yaml", false, "If true, look for assignees in md yaml headers.") 407 opts.RegisterBool(&o.useReviewers, "use-reviewers", false, "Use \"reviewers\" rather than \"approvers\" for review") 408 opts.RegisterString(&o.aliasFile, "alias-file", "", "File wherein team members and aliases exist.") 409 return sets.NewString("repo-dir") 410 } 411 412 // GitCommand will execute the git command with the `args` within the project directory. 413 func (o *RepoInfo) GitCommand(args []string) ([]byte, error) { 414 return o.gitCommandDir(args, o.projectDir) 415 } 416 417 // GitCommandDir will execute the git command with the `args` within the 'dir' directory. 418 func (o *RepoInfo) gitCommandDir(args []string, cmdDir string) ([]byte, error) { 419 cmd := exec.Command("git", args...) 420 cmd.Dir = cmdDir 421 return cmd.CombinedOutput() 422 } 423 424 // findOwnersForPath returns the OWNERS file path furthest down the tree for a specified file 425 // By default we use the reviewers section of owners flag but this can be configured by setting approvers to true 426 func findOwnersForPath(path string, ownerMap map[string]sets.String) string { 427 d := path 428 429 for { 430 n, ok := ownerMap[d] 431 if ok && len(n) != 0 { 432 return d 433 } 434 if d == baseDirConvention { 435 break 436 } 437 d = filepath.Dir(d) 438 d = canonicalize(d) 439 } 440 return "" 441 } 442 443 // FindApproversForPath returns the OWNERS file path furthest down the tree for a specified file 444 // that contains an approvers section 445 func (o *RepoInfo) FindApproverOwnersForPath(path string) string { 446 return findOwnersForPath(path, o.approvers) 447 } 448 449 // FindReviewersForPath returns the OWNERS file path furthest down the tree for a specified file 450 // that contains a reviewers section 451 func (o *RepoInfo) FindReviewersForPath(path string) string { 452 return findOwnersForPath(path, o.reviewers) 453 } 454 455 // AllPossibleOwnerLabels returns all labels found in any owners files 456 func (o *RepoInfo) AllPossibleOwnerLabels() sets.String { 457 return sets.NewString(o.allPossibleLabels.List()...) 458 } 459 460 // FindLabelsForPath returns a set of labels which should be applied to PRs 461 // modifying files under the given path. 462 func (o *RepoInfo) FindLabelsForPath(path string) sets.String { 463 s := sets.String{} 464 465 d := path 466 for { 467 l, ok := o.labels[d] 468 if ok && len(l) > 0 { 469 s = s.Union(l) 470 } 471 if d == baseDirConvention { 472 break 473 } 474 d = filepath.Dir(d) 475 d = canonicalize(d) 476 } 477 return s 478 } 479 480 // peopleForPath returns a set of users who are assignees to the 481 // requested file. The path variable should be a full path to a filename 482 // and not directory as the final directory will be discounted if enableMdYaml is true 483 // leafOnly indicates whether only the OWNERS deepest in the tree (closest to the file) 484 // should be returned or if all OWNERS in filepath should be returned 485 func peopleForPath(path string, people map[string]sets.String, leafOnly bool, enableMdYaml bool) sets.String { 486 d := path 487 if !enableMdYaml { 488 // if path is a directory, this will remove the leaf directory, and returns "." for topmost dir 489 d = filepath.Dir(d) 490 d = canonicalize(path) 491 } 492 493 out := sets.NewString() 494 for { 495 s, ok := people[d] 496 if ok { 497 out = out.Union(s) 498 if leafOnly && out.Len() > 0 { 499 break 500 } 501 } 502 if d == baseDirConvention { 503 break 504 } 505 d = filepath.Dir(d) 506 d = canonicalize(d) 507 } 508 return out 509 } 510 511 // LeafApprovers returns a set of users who are the closest approvers to the 512 // requested file. If pkg/OWNERS has user1 and pkg/util/OWNERS has user2 this 513 // will only return user2 for the path pkg/util/sets/file.go 514 func (o *RepoInfo) LeafApprovers(path string) sets.String { 515 return peopleForPath(path, o.approvers, true, o.enableMdYaml) 516 } 517 518 // Approvers returns ALL of the users who are approvers for the 519 // requested file (including approvers in parent dirs' OWNERS). 520 // If pkg/OWNERS has user1 and pkg/util/OWNERS has user2 this 521 // will return both user1 and user2 for the path pkg/util/sets/file.go 522 func (o *RepoInfo) Approvers(path string) sets.String { 523 return peopleForPath(path, o.approvers, false, o.enableMdYaml) 524 } 525 526 // LeafReviewers returns a set of users who are the closest reviewers to the 527 // requested file. If pkg/OWNERS has user1 and pkg/util/OWNERS has user2 this 528 // will only return user2 for the path pkg/util/sets/file.go 529 func (o *RepoInfo) LeafReviewers(path string) sets.String { 530 if !o.useReviewers { 531 return o.LeafApprovers(path) 532 } 533 return peopleForPath(path, o.reviewers, true, o.enableMdYaml) 534 } 535 536 // Reviewers returns ALL of the users who are reviewers for the 537 // requested file (including reviewers in parent dirs' OWNERS). 538 // If pkg/OWNERS has user1 and pkg/util/OWNERS has user2 this 539 // will return both user1 and user2 for the path pkg/util/sets/file.go 540 func (o *RepoInfo) Reviewers(path string) sets.String { 541 if !o.useReviewers { 542 return o.Approvers(path) 543 } 544 return peopleForPath(path, o.reviewers, false, o.enableMdYaml) 545 }