github.com/shashidharatd/test-infra@v0.0.0-20171006011030-71304e1ca560/mungegithub/features/repo-updates.go (about) 1 /* 2 Copyright 2016 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package features 18 19 import ( 20 "fmt" 21 "io/ioutil" 22 "os" 23 "os/exec" 24 "path" 25 "path/filepath" 26 "regexp" 27 "strings" 28 29 "k8s.io/kubernetes/pkg/util/sets" 30 "k8s.io/kubernetes/pkg/util/yaml" 31 "k8s.io/test-infra/mungegithub/github" 32 "k8s.io/test-infra/mungegithub/options" 33 34 parseYaml "github.com/ghodss/yaml" 35 "github.com/golang/glog" 36 ) 37 38 const ( 39 ownerFilename = "OWNERS" // file which contains approvers and reviewers 40 // RepoFeatureName is how mungers should indicate this is required 41 RepoFeatureName = "gitrepos" 42 // Github's api uses "" (empty) string as basedir by convention but it's clearer to use "/" 43 baseDirConvention = "" 44 ) 45 46 type assignmentConfig struct { 47 Assignees []string `json:"assignees" yaml:"assignees"` 48 Approvers []string `json:"approvers" yaml:"approvers"` 49 Reviewers []string `json:"reviewers" yaml:"reviewers"` 50 Labels []string `json:"labels" yaml:"labels"` 51 } 52 53 type aliasData struct { 54 // Contains the mapping between aliases and lists of members. 55 AliasMap map[string][]string `json:"aliases"` 56 } 57 58 type aliasReader interface { 59 read() ([]byte, error) 60 } 61 62 func (o *RepoInfo) read() ([]byte, error) { 63 return ioutil.ReadFile(o.aliasFile) 64 } 65 66 // RepoInfo provides information about users in OWNERS files in a git repo 67 type RepoInfo struct { 68 baseDir string 69 enableMdYaml bool 70 useReviewers bool 71 72 aliasFile string 73 aliasData *aliasData 74 aliasReader aliasReader 75 76 projectDir string 77 approvers map[string]sets.String 78 reviewers map[string]sets.String 79 labels map[string]sets.String 80 allPossibleLabels sets.String 81 config *github.Config 82 } 83 84 func init() { 85 RegisterFeature(&RepoInfo{}) 86 } 87 88 // Name is just going to return the name mungers use to request this feature 89 func (o *RepoInfo) Name() string { 90 return RepoFeatureName 91 } 92 93 // by default, github's api doesn't root the project directory at "/" and instead uses the empty string for the base dir 94 // of the project. And the built-in dir function returns "." for empty strings, so for consistency, we use this 95 // canonicalize to get the directories of files in a consistent format with NO "/" at the root (a/b/c/ -> a/b/c) 96 func canonicalize(path string) string { 97 if path == "." { 98 return baseDirConvention 99 } 100 return strings.TrimSuffix(path, "/") 101 } 102 103 func (o *RepoInfo) walkFunc(path string, info os.FileInfo, err error) error { 104 if err != nil { 105 glog.Errorf("%v", err) 106 return nil 107 } 108 filename := filepath.Base(path) 109 if info.Mode().IsDir() { 110 switch filename { 111 case ".git": 112 return filepath.SkipDir 113 case "_output": 114 return filepath.SkipDir 115 } 116 } 117 if !info.Mode().IsRegular() { 118 return nil 119 } 120 121 c := &assignmentConfig{} 122 123 // '.md' files may contain assignees at the top of the file in a yaml header 124 // Flag guarded because this is only enabled in some repos 125 if o.enableMdYaml && filename != ownerFilename && strings.HasSuffix(filename, "md") { 126 // Parse the yaml header from the file if it exists and marshal into the config 127 if err := decodeAssignmentConfig(path, c); err != nil { 128 glog.Errorf("%v", err) 129 return err 130 } 131 132 // Set assignees for this file using the relative path if they were found 133 path, err = filepath.Rel(o.projectDir, path) 134 if err != nil { 135 glog.Errorf("Unable to find relative path between %q and %q: %v", o.projectDir, path, err) 136 return err 137 } 138 c.normalizeUsers() 139 o.approvers[path] = sets.NewString(c.Approvers...) 140 o.approvers[path].Insert(c.Assignees...) 141 o.reviewers[path] = sets.NewString(c.Reviewers...) 142 return nil 143 } 144 145 if filename != ownerFilename { 146 return nil 147 } 148 149 file, err := os.Open(path) 150 if err != nil { 151 glog.Errorf("Could not open %q: %v", path, err) 152 return nil 153 } 154 defer file.Close() 155 156 if err := yaml.NewYAMLToJSONDecoder(file).Decode(c); err != nil { 157 glog.Errorf("Could not decode %q: %v", path, err) 158 return nil 159 } 160 161 path, err = filepath.Rel(o.projectDir, path) 162 path = filepath.Dir(path) 163 if err != nil { 164 glog.Errorf("Unable to find relative path between %q and %q: %v", o.projectDir, path, err) 165 return err 166 } 167 path = canonicalize(path) 168 c.normalizeUsers() 169 o.approvers[path] = sets.NewString(c.Approvers...) 170 o.approvers[path].Insert(c.Assignees...) 171 o.reviewers[path] = sets.NewString(c.Reviewers...) 172 if len(c.Labels) > 0 { 173 o.labels[path] = sets.NewString(c.Labels...) 174 o.allPossibleLabels.Insert(c.Labels...) 175 } 176 return nil 177 } 178 179 // caseNormalizeAll normalizes all entries in 180 // the assignment configuration so that we can do case- 181 // insensitive operations on the entries 182 func (c *assignmentConfig) normalizeUsers() { 183 c.Approvers = caseNormalizeAll(c.Approvers) 184 c.Assignees = caseNormalizeAll(c.Assignees) 185 c.Reviewers = caseNormalizeAll(c.Reviewers) 186 } 187 188 func caseNormalizeAll(users []string) []string { 189 normalizedUsers := users[:0] 190 for _, user := range users { 191 normalizedUsers = append(normalizedUsers, caseNormalize(user)) 192 } 193 return normalizedUsers 194 } 195 196 func caseNormalize(user string) string { 197 return strings.ToLower(user) 198 } 199 200 // decodeAssignmentConfig will parse the yaml header if it exists and unmarshal it into an assignmentConfig. 201 // If no yaml header is found, do nothing 202 // Returns an error if the file cannot be read or the yaml header is found but cannot be unmarshalled 203 var mdStructuredHeaderRegex = regexp.MustCompile("^---\n(.|\n)*\n---") 204 205 func decodeAssignmentConfig(path string, config *assignmentConfig) error { 206 fileBytes, err := ioutil.ReadFile(path) 207 if err != nil { 208 return err 209 } 210 // Parse the yaml header from the top of the file. Will return an empty string if regex does not match. 211 meta := mdStructuredHeaderRegex.FindString(string(fileBytes)) 212 213 // Unmarshal the yaml header into the config 214 return parseYaml.Unmarshal([]byte(meta), &config) 215 } 216 217 func (o *RepoInfo) updateRepoAliases() error { 218 if len(o.aliasFile) == 0 { 219 return nil 220 } 221 222 // read and check the alias-file. 223 fileContents, err := o.aliasReader.read() 224 if os.IsNotExist(err) { 225 glog.Infof("Missing alias-file (%s), using empty alias structure.", o.aliasFile) 226 o.aliasData = &aliasData{} 227 return nil 228 } 229 if err != nil { 230 return fmt.Errorf("Unable to read alias file: %v", err) 231 } 232 233 var data aliasData 234 if err := parseYaml.Unmarshal(fileContents, &data); err != nil { 235 return fmt.Errorf("Failed to decode the alias file: %v", err) 236 } 237 238 o.aliasData = normalizeAliases(data) 239 return nil 240 } 241 242 // normalizeAliases normalizes all entries in the alias data 243 // so that we can do case-insensitive resolution of aliases 244 func normalizeAliases(rawData aliasData) *aliasData { 245 normalizedData := aliasData{AliasMap: map[string][]string{}} 246 for alias, users := range rawData.AliasMap { 247 normalizedData.AliasMap[caseNormalize(alias)] = caseNormalizeAll(users) 248 } 249 return &normalizedData 250 } 251 252 // expandAllAliases expands all of the aliases in the 253 // lists of approvers and reviewers for paths we track 254 func (o *RepoInfo) expandAllAliases() { 255 for filePath := range o.approvers { 256 o.resolveAliases(o.approvers[filePath]) 257 } 258 259 for filePath := range o.reviewers { 260 o.resolveAliases(o.reviewers[filePath]) 261 } 262 } 263 264 // resolveAliases returns the members that need to 265 // be added and removed from a set to yield a set with 266 // all aliases expanded 267 func (o *RepoInfo) resolveAliases(users sets.String) { 268 for _, owner := range users.List() { 269 if aliases := o.resolveAlias(owner); len(aliases) > 0 { 270 users.Delete(owner) 271 users.Insert(aliases...) 272 } 273 } 274 } 275 276 // resolveAlias resolves the identity or identities for 277 // an alias. If we have no record of the alias, an empty 278 // slice is returned 279 func (o *RepoInfo) resolveAlias(owner string) []string { 280 if val, ok := o.aliasData.AliasMap[owner]; ok { 281 return val 282 } 283 return []string{} 284 } 285 286 func (o *RepoInfo) updateRepoUsers() error { 287 out, err := o.GitCommand([]string{"rev-parse", "HEAD"}) 288 if err != nil { 289 glog.Errorf("Unable get sha of HEAD:\n%s\n%v", string(out), err) 290 return err 291 } 292 sha := out 293 294 o.approvers = map[string]sets.String{} 295 o.reviewers = map[string]sets.String{} 296 o.labels = map[string]sets.String{} 297 o.allPossibleLabels = sets.String{} 298 err = filepath.Walk(o.projectDir, o.walkFunc) 299 if err != nil { 300 glog.Errorf("Got error %v", err) 301 } 302 o.expandAllAliases() 303 if err := o.pruneRepoUsers(); err != nil { 304 return err 305 } 306 glog.Infof("Loaded config from %s:%s", o.projectDir, sha) 307 glog.V(5).Infof("approvers: %v", o.approvers) 308 glog.V(5).Infof("reviewers: %v", o.reviewers) 309 return nil 310 } 311 312 // Initialize will initialize the munger 313 func (o *RepoInfo) Initialize(config *github.Config) error { 314 o.config = config 315 o.projectDir = path.Join(o.baseDir, o.config.Project) 316 317 if len(o.baseDir) == 0 { 318 glog.Fatalf("--repo-dir is required with selected munger(s)") 319 } 320 finfo, err := os.Stat(o.baseDir) 321 if err != nil { 322 return fmt.Errorf("Unable to stat --repo-dir: %v", err) 323 } 324 if !finfo.IsDir() { 325 return fmt.Errorf("--repo-dir is not a directory") 326 } 327 328 if o.fsckRepo() != nil { 329 if err := o.cleanUp(o.projectDir); err != nil { 330 return fmt.Errorf("Unable to remove old clone directory at %v: %v", o.projectDir, err) 331 } 332 } 333 if !o.exists() { 334 if cloneUrl, err := o.cloneRepo(); err != nil { 335 return fmt.Errorf("Unable to clone %v: %v", cloneUrl, err) 336 } 337 } 338 339 o.aliasData = &aliasData{} 340 o.aliasReader = o 341 if err := o.updateRepoAliases(); err != nil { 342 return err 343 } 344 return o.updateRepoUsers() 345 } 346 347 func (o *RepoInfo) cleanUp(path string) error { 348 return os.RemoveAll(path) 349 } 350 351 func (o *RepoInfo) exists() bool { 352 _, err := os.Stat(o.projectDir) 353 return !os.IsNotExist(err) 354 } 355 356 func (o *RepoInfo) fsckRepo() error { 357 if !o.exists() { 358 return fmt.Errorf("fsck repo failed: %q is missing", o.projectDir) 359 } 360 output, err := o.gitCommandDir([]string{"fsck"}, o.projectDir) 361 if err != nil { 362 glog.Errorf("fsck repo failed: %s", output) 363 } 364 return err 365 } 366 367 func (o *RepoInfo) cloneRepo() (string, error) { 368 cloneUrl := fmt.Sprintf("https://github.com/%s/%s.git", o.config.Org, o.config.Project) 369 output, err := o.gitCommandDir([]string{"clone", cloneUrl, o.projectDir}, o.baseDir) 370 if err != nil { 371 glog.Errorf("Failed to clone github repo: %s", output) 372 } 373 return cloneUrl, err 374 } 375 376 func (o *RepoInfo) pruneRepoUsers() error { 377 whitelist, err := o.config.Collaborators() 378 if err != nil { 379 return err 380 } 381 382 for repo, users := range o.approvers { 383 o.approvers[repo] = whitelist.Intersection(users) 384 } 385 for repo, users := range o.reviewers { 386 o.reviewers[repo] = whitelist.Intersection(users) 387 } 388 return nil 389 } 390 391 // EachLoop is called at the start of every munge loop 392 func (o *RepoInfo) EachLoop() error { 393 if out, err := o.GitCommand([]string{"remote", "update"}); err != nil { 394 glog.Errorf("Unable to git remote update:\n%s\n%v", out, err) 395 } 396 if out, err := o.GitCommand([]string{"pull"}); err != nil { 397 glog.Errorf("Unable to run git pull:\n%s\n%v", string(out), err) 398 return err 399 } 400 401 if err := o.updateRepoAliases(); err != nil { 402 return err 403 } 404 return o.updateRepoUsers() 405 } 406 407 // RegisterOptions registers options for this feature; returns any that require a restart when changed. 408 func (o *RepoInfo) RegisterOptions(opts *options.Options) sets.String { 409 opts.RegisterString(&o.baseDir, "repo-dir", "", "Path to perform checkout of repository") 410 opts.RegisterBool(&o.enableMdYaml, "enable-md-yaml", false, "If true, look for assignees in md yaml headers.") 411 opts.RegisterBool(&o.useReviewers, "use-reviewers", false, "Use \"reviewers\" rather than \"approvers\" for review") 412 opts.RegisterString(&o.aliasFile, "alias-file", "", "File wherein team members and aliases exist.") 413 return sets.NewString("repo-dir") 414 } 415 416 // GitCommand will execute the git command with the `args` within the project directory. 417 func (o *RepoInfo) GitCommand(args []string) ([]byte, error) { 418 return o.gitCommandDir(args, o.projectDir) 419 } 420 421 // GitCommandDir will execute the git command with the `args` within the 'dir' directory. 422 func (o *RepoInfo) gitCommandDir(args []string, cmdDir string) ([]byte, error) { 423 cmd := exec.Command("git", args...) 424 cmd.Dir = cmdDir 425 return cmd.CombinedOutput() 426 } 427 428 // findOwnersForPath returns the OWNERS file path furthest down the tree for a specified file 429 // By default we use the reviewers section of owners flag but this can be configured by setting approvers to true 430 func findOwnersForPath(path string, ownerMap map[string]sets.String) string { 431 d := path 432 433 for { 434 n, ok := ownerMap[d] 435 if ok && len(n) != 0 { 436 return d 437 } 438 if d == baseDirConvention { 439 break 440 } 441 d = filepath.Dir(d) 442 d = canonicalize(d) 443 } 444 return "" 445 } 446 447 // FindApproversForPath returns the OWNERS file path furthest down the tree for a specified file 448 // that contains an approvers section 449 func (o *RepoInfo) FindApproverOwnersForPath(path string) string { 450 return findOwnersForPath(path, o.approvers) 451 } 452 453 // FindReviewersForPath returns the OWNERS file path furthest down the tree for a specified file 454 // that contains a reviewers section 455 func (o *RepoInfo) FindReviewersForPath(path string) string { 456 return findOwnersForPath(path, o.reviewers) 457 } 458 459 // AllPossibleOwnerLabels returns all labels found in any owners files 460 func (o *RepoInfo) AllPossibleOwnerLabels() sets.String { 461 return sets.NewString(o.allPossibleLabels.List()...) 462 } 463 464 // FindLabelsForPath returns a set of labels which should be applied to PRs 465 // modifying files under the given path. 466 func (o *RepoInfo) FindLabelsForPath(path string) sets.String { 467 s := sets.String{} 468 469 d := path 470 for { 471 l, ok := o.labels[d] 472 if ok && len(l) > 0 { 473 s = s.Union(l) 474 } 475 if d == baseDirConvention { 476 break 477 } 478 d = filepath.Dir(d) 479 d = canonicalize(d) 480 } 481 return s 482 } 483 484 // peopleForPath returns a set of users who are assignees to the 485 // requested file. The path variable should be a full path to a filename 486 // and not directory as the final directory will be discounted if enableMdYaml is true 487 // leafOnly indicates whether only the OWNERS deepest in the tree (closest to the file) 488 // should be returned or if all OWNERS in filepath should be returned 489 func peopleForPath(path string, people map[string]sets.String, leafOnly bool, enableMdYaml bool) sets.String { 490 d := path 491 if !enableMdYaml { 492 // if path is a directory, this will remove the leaf directory, and returns "." for topmost dir 493 d = filepath.Dir(d) 494 d = canonicalize(path) 495 } 496 497 out := sets.NewString() 498 for { 499 s, ok := people[d] 500 if ok { 501 out = out.Union(s) 502 if leafOnly && out.Len() > 0 { 503 break 504 } 505 } 506 if d == baseDirConvention { 507 break 508 } 509 d = filepath.Dir(d) 510 d = canonicalize(d) 511 } 512 return out 513 } 514 515 // LeafApprovers returns a set of users who are the closest approvers to the 516 // requested file. If pkg/OWNERS has user1 and pkg/util/OWNERS has user2 this 517 // will only return user2 for the path pkg/util/sets/file.go 518 func (o *RepoInfo) LeafApprovers(path string) sets.String { 519 return peopleForPath(path, o.approvers, true, o.enableMdYaml) 520 } 521 522 // Approvers returns ALL of the users who are approvers for the 523 // requested file (including approvers in parent dirs' OWNERS). 524 // If pkg/OWNERS has user1 and pkg/util/OWNERS has user2 this 525 // will return both user1 and user2 for the path pkg/util/sets/file.go 526 func (o *RepoInfo) Approvers(path string) sets.String { 527 return peopleForPath(path, o.approvers, false, o.enableMdYaml) 528 } 529 530 // LeafReviewers returns a set of users who are the closest reviewers to the 531 // requested file. If pkg/OWNERS has user1 and pkg/util/OWNERS has user2 this 532 // will only return user2 for the path pkg/util/sets/file.go 533 func (o *RepoInfo) LeafReviewers(path string) sets.String { 534 if !o.useReviewers { 535 return o.LeafApprovers(path) 536 } 537 return peopleForPath(path, o.reviewers, true, o.enableMdYaml) 538 } 539 540 // Reviewers returns ALL of the users who are reviewers for the 541 // requested file (including reviewers in parent dirs' OWNERS). 542 // If pkg/OWNERS has user1 and pkg/util/OWNERS has user2 this 543 // will return both user1 and user2 for the path pkg/util/sets/file.go 544 func (o *RepoInfo) Reviewers(path string) sets.String { 545 if !o.useReviewers { 546 return o.Approvers(path) 547 } 548 return peopleForPath(path, o.reviewers, false, o.enableMdYaml) 549 }