github.com/shashidharatd/test-infra@v0.0.0-20171006011030-71304e1ca560/mungegithub/features/repo-updates.go (about)

     1  /*
     2  Copyright 2016 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package features
    18  
    19  import (
    20  	"fmt"
    21  	"io/ioutil"
    22  	"os"
    23  	"os/exec"
    24  	"path"
    25  	"path/filepath"
    26  	"regexp"
    27  	"strings"
    28  
    29  	"k8s.io/kubernetes/pkg/util/sets"
    30  	"k8s.io/kubernetes/pkg/util/yaml"
    31  	"k8s.io/test-infra/mungegithub/github"
    32  	"k8s.io/test-infra/mungegithub/options"
    33  
    34  	parseYaml "github.com/ghodss/yaml"
    35  	"github.com/golang/glog"
    36  )
    37  
    38  const (
    39  	ownerFilename = "OWNERS" // file which contains approvers and reviewers
    40  	// RepoFeatureName is how mungers should indicate this is required
    41  	RepoFeatureName = "gitrepos"
    42  	// Github's api uses "" (empty) string as basedir by convention but it's clearer to use "/"
    43  	baseDirConvention = ""
    44  )
    45  
    46  type assignmentConfig struct {
    47  	Assignees []string `json:"assignees" yaml:"assignees"`
    48  	Approvers []string `json:"approvers" yaml:"approvers"`
    49  	Reviewers []string `json:"reviewers" yaml:"reviewers"`
    50  	Labels    []string `json:"labels" yaml:"labels"`
    51  }
    52  
    53  type aliasData struct {
    54  	// Contains the mapping between aliases and lists of members.
    55  	AliasMap map[string][]string `json:"aliases"`
    56  }
    57  
    58  type aliasReader interface {
    59  	read() ([]byte, error)
    60  }
    61  
    62  func (o *RepoInfo) read() ([]byte, error) {
    63  	return ioutil.ReadFile(o.aliasFile)
    64  }
    65  
    66  // RepoInfo provides information about users in OWNERS files in a git repo
    67  type RepoInfo struct {
    68  	baseDir      string
    69  	enableMdYaml bool
    70  	useReviewers bool
    71  
    72  	aliasFile   string
    73  	aliasData   *aliasData
    74  	aliasReader aliasReader
    75  
    76  	projectDir        string
    77  	approvers         map[string]sets.String
    78  	reviewers         map[string]sets.String
    79  	labels            map[string]sets.String
    80  	allPossibleLabels sets.String
    81  	config            *github.Config
    82  }
    83  
    84  func init() {
    85  	RegisterFeature(&RepoInfo{})
    86  }
    87  
    88  // Name is just going to return the name mungers use to request this feature
    89  func (o *RepoInfo) Name() string {
    90  	return RepoFeatureName
    91  }
    92  
    93  // by default, github's api doesn't root the project directory at "/" and instead uses the empty string for the base dir
    94  // of the project. And the built-in dir function returns "." for empty strings, so for consistency, we use this
    95  // canonicalize to get the directories of files in a consistent format with NO "/" at the root (a/b/c/ -> a/b/c)
    96  func canonicalize(path string) string {
    97  	if path == "." {
    98  		return baseDirConvention
    99  	}
   100  	return strings.TrimSuffix(path, "/")
   101  }
   102  
   103  func (o *RepoInfo) walkFunc(path string, info os.FileInfo, err error) error {
   104  	if err != nil {
   105  		glog.Errorf("%v", err)
   106  		return nil
   107  	}
   108  	filename := filepath.Base(path)
   109  	if info.Mode().IsDir() {
   110  		switch filename {
   111  		case ".git":
   112  			return filepath.SkipDir
   113  		case "_output":
   114  			return filepath.SkipDir
   115  		}
   116  	}
   117  	if !info.Mode().IsRegular() {
   118  		return nil
   119  	}
   120  
   121  	c := &assignmentConfig{}
   122  
   123  	// '.md' files may contain assignees at the top of the file in a yaml header
   124  	// Flag guarded because this is only enabled in some repos
   125  	if o.enableMdYaml && filename != ownerFilename && strings.HasSuffix(filename, "md") {
   126  		// Parse the yaml header from the file if it exists and marshal into the config
   127  		if err := decodeAssignmentConfig(path, c); err != nil {
   128  			glog.Errorf("%v", err)
   129  			return err
   130  		}
   131  
   132  		// Set assignees for this file using the relative path if they were found
   133  		path, err = filepath.Rel(o.projectDir, path)
   134  		if err != nil {
   135  			glog.Errorf("Unable to find relative path between %q and %q: %v", o.projectDir, path, err)
   136  			return err
   137  		}
   138  		c.normalizeUsers()
   139  		o.approvers[path] = sets.NewString(c.Approvers...)
   140  		o.approvers[path].Insert(c.Assignees...)
   141  		o.reviewers[path] = sets.NewString(c.Reviewers...)
   142  		return nil
   143  	}
   144  
   145  	if filename != ownerFilename {
   146  		return nil
   147  	}
   148  
   149  	file, err := os.Open(path)
   150  	if err != nil {
   151  		glog.Errorf("Could not open %q: %v", path, err)
   152  		return nil
   153  	}
   154  	defer file.Close()
   155  
   156  	if err := yaml.NewYAMLToJSONDecoder(file).Decode(c); err != nil {
   157  		glog.Errorf("Could not decode %q: %v", path, err)
   158  		return nil
   159  	}
   160  
   161  	path, err = filepath.Rel(o.projectDir, path)
   162  	path = filepath.Dir(path)
   163  	if err != nil {
   164  		glog.Errorf("Unable to find relative path between %q and %q: %v", o.projectDir, path, err)
   165  		return err
   166  	}
   167  	path = canonicalize(path)
   168  	c.normalizeUsers()
   169  	o.approvers[path] = sets.NewString(c.Approvers...)
   170  	o.approvers[path].Insert(c.Assignees...)
   171  	o.reviewers[path] = sets.NewString(c.Reviewers...)
   172  	if len(c.Labels) > 0 {
   173  		o.labels[path] = sets.NewString(c.Labels...)
   174  		o.allPossibleLabels.Insert(c.Labels...)
   175  	}
   176  	return nil
   177  }
   178  
   179  // caseNormalizeAll normalizes all entries in
   180  // the assignment configuration so that we can do case-
   181  // insensitive operations on the entries
   182  func (c *assignmentConfig) normalizeUsers() {
   183  	c.Approvers = caseNormalizeAll(c.Approvers)
   184  	c.Assignees = caseNormalizeAll(c.Assignees)
   185  	c.Reviewers = caseNormalizeAll(c.Reviewers)
   186  }
   187  
   188  func caseNormalizeAll(users []string) []string {
   189  	normalizedUsers := users[:0]
   190  	for _, user := range users {
   191  		normalizedUsers = append(normalizedUsers, caseNormalize(user))
   192  	}
   193  	return normalizedUsers
   194  }
   195  
   196  func caseNormalize(user string) string {
   197  	return strings.ToLower(user)
   198  }
   199  
   200  // decodeAssignmentConfig will parse the yaml header if it exists and unmarshal it into an assignmentConfig.
   201  // If no yaml header is found, do nothing
   202  // Returns an error if the file cannot be read or the yaml header is found but cannot be unmarshalled
   203  var mdStructuredHeaderRegex = regexp.MustCompile("^---\n(.|\n)*\n---")
   204  
   205  func decodeAssignmentConfig(path string, config *assignmentConfig) error {
   206  	fileBytes, err := ioutil.ReadFile(path)
   207  	if err != nil {
   208  		return err
   209  	}
   210  	// Parse the yaml header from the top of the file.  Will return an empty string if regex does not match.
   211  	meta := mdStructuredHeaderRegex.FindString(string(fileBytes))
   212  
   213  	// Unmarshal the yaml header into the config
   214  	return parseYaml.Unmarshal([]byte(meta), &config)
   215  }
   216  
   217  func (o *RepoInfo) updateRepoAliases() error {
   218  	if len(o.aliasFile) == 0 {
   219  		return nil
   220  	}
   221  
   222  	// read and check the alias-file.
   223  	fileContents, err := o.aliasReader.read()
   224  	if os.IsNotExist(err) {
   225  		glog.Infof("Missing alias-file (%s), using empty alias structure.", o.aliasFile)
   226  		o.aliasData = &aliasData{}
   227  		return nil
   228  	}
   229  	if err != nil {
   230  		return fmt.Errorf("Unable to read alias file: %v", err)
   231  	}
   232  
   233  	var data aliasData
   234  	if err := parseYaml.Unmarshal(fileContents, &data); err != nil {
   235  		return fmt.Errorf("Failed to decode the alias file: %v", err)
   236  	}
   237  
   238  	o.aliasData = normalizeAliases(data)
   239  	return nil
   240  }
   241  
   242  // normalizeAliases normalizes all entries in the alias data
   243  // so that we can do case-insensitive resolution of aliases
   244  func normalizeAliases(rawData aliasData) *aliasData {
   245  	normalizedData := aliasData{AliasMap: map[string][]string{}}
   246  	for alias, users := range rawData.AliasMap {
   247  		normalizedData.AliasMap[caseNormalize(alias)] = caseNormalizeAll(users)
   248  	}
   249  	return &normalizedData
   250  }
   251  
   252  // expandAllAliases expands all of the aliases in the
   253  // lists of approvers and reviewers for paths we track
   254  func (o *RepoInfo) expandAllAliases() {
   255  	for filePath := range o.approvers {
   256  		o.resolveAliases(o.approvers[filePath])
   257  	}
   258  
   259  	for filePath := range o.reviewers {
   260  		o.resolveAliases(o.reviewers[filePath])
   261  	}
   262  }
   263  
   264  // resolveAliases returns the members that need to
   265  // be added and removed from a set to yield a set with
   266  // all aliases expanded
   267  func (o *RepoInfo) resolveAliases(users sets.String) {
   268  	for _, owner := range users.List() {
   269  		if aliases := o.resolveAlias(owner); len(aliases) > 0 {
   270  			users.Delete(owner)
   271  			users.Insert(aliases...)
   272  		}
   273  	}
   274  }
   275  
   276  // resolveAlias resolves the identity or identities for
   277  // an alias. If we have no record of the alias, an empty
   278  // slice is returned
   279  func (o *RepoInfo) resolveAlias(owner string) []string {
   280  	if val, ok := o.aliasData.AliasMap[owner]; ok {
   281  		return val
   282  	}
   283  	return []string{}
   284  }
   285  
   286  func (o *RepoInfo) updateRepoUsers() error {
   287  	out, err := o.GitCommand([]string{"rev-parse", "HEAD"})
   288  	if err != nil {
   289  		glog.Errorf("Unable get sha of HEAD:\n%s\n%v", string(out), err)
   290  		return err
   291  	}
   292  	sha := out
   293  
   294  	o.approvers = map[string]sets.String{}
   295  	o.reviewers = map[string]sets.String{}
   296  	o.labels = map[string]sets.String{}
   297  	o.allPossibleLabels = sets.String{}
   298  	err = filepath.Walk(o.projectDir, o.walkFunc)
   299  	if err != nil {
   300  		glog.Errorf("Got error %v", err)
   301  	}
   302  	o.expandAllAliases()
   303  	if err := o.pruneRepoUsers(); err != nil {
   304  		return err
   305  	}
   306  	glog.Infof("Loaded config from %s:%s", o.projectDir, sha)
   307  	glog.V(5).Infof("approvers: %v", o.approvers)
   308  	glog.V(5).Infof("reviewers: %v", o.reviewers)
   309  	return nil
   310  }
   311  
   312  // Initialize will initialize the munger
   313  func (o *RepoInfo) Initialize(config *github.Config) error {
   314  	o.config = config
   315  	o.projectDir = path.Join(o.baseDir, o.config.Project)
   316  
   317  	if len(o.baseDir) == 0 {
   318  		glog.Fatalf("--repo-dir is required with selected munger(s)")
   319  	}
   320  	finfo, err := os.Stat(o.baseDir)
   321  	if err != nil {
   322  		return fmt.Errorf("Unable to stat --repo-dir: %v", err)
   323  	}
   324  	if !finfo.IsDir() {
   325  		return fmt.Errorf("--repo-dir is not a directory")
   326  	}
   327  
   328  	if o.fsckRepo() != nil {
   329  		if err := o.cleanUp(o.projectDir); err != nil {
   330  			return fmt.Errorf("Unable to remove old clone directory at %v: %v", o.projectDir, err)
   331  		}
   332  	}
   333  	if !o.exists() {
   334  		if cloneUrl, err := o.cloneRepo(); err != nil {
   335  			return fmt.Errorf("Unable to clone %v: %v", cloneUrl, err)
   336  		}
   337  	}
   338  
   339  	o.aliasData = &aliasData{}
   340  	o.aliasReader = o
   341  	if err := o.updateRepoAliases(); err != nil {
   342  		return err
   343  	}
   344  	return o.updateRepoUsers()
   345  }
   346  
   347  func (o *RepoInfo) cleanUp(path string) error {
   348  	return os.RemoveAll(path)
   349  }
   350  
   351  func (o *RepoInfo) exists() bool {
   352  	_, err := os.Stat(o.projectDir)
   353  	return !os.IsNotExist(err)
   354  }
   355  
   356  func (o *RepoInfo) fsckRepo() error {
   357  	if !o.exists() {
   358  		return fmt.Errorf("fsck repo failed: %q is missing", o.projectDir)
   359  	}
   360  	output, err := o.gitCommandDir([]string{"fsck"}, o.projectDir)
   361  	if err != nil {
   362  		glog.Errorf("fsck repo failed: %s", output)
   363  	}
   364  	return err
   365  }
   366  
   367  func (o *RepoInfo) cloneRepo() (string, error) {
   368  	cloneUrl := fmt.Sprintf("https://github.com/%s/%s.git", o.config.Org, o.config.Project)
   369  	output, err := o.gitCommandDir([]string{"clone", cloneUrl, o.projectDir}, o.baseDir)
   370  	if err != nil {
   371  		glog.Errorf("Failed to clone github repo: %s", output)
   372  	}
   373  	return cloneUrl, err
   374  }
   375  
   376  func (o *RepoInfo) pruneRepoUsers() error {
   377  	whitelist, err := o.config.Collaborators()
   378  	if err != nil {
   379  		return err
   380  	}
   381  
   382  	for repo, users := range o.approvers {
   383  		o.approvers[repo] = whitelist.Intersection(users)
   384  	}
   385  	for repo, users := range o.reviewers {
   386  		o.reviewers[repo] = whitelist.Intersection(users)
   387  	}
   388  	return nil
   389  }
   390  
   391  // EachLoop is called at the start of every munge loop
   392  func (o *RepoInfo) EachLoop() error {
   393  	if out, err := o.GitCommand([]string{"remote", "update"}); err != nil {
   394  		glog.Errorf("Unable to git remote update:\n%s\n%v", out, err)
   395  	}
   396  	if out, err := o.GitCommand([]string{"pull"}); err != nil {
   397  		glog.Errorf("Unable to run git pull:\n%s\n%v", string(out), err)
   398  		return err
   399  	}
   400  
   401  	if err := o.updateRepoAliases(); err != nil {
   402  		return err
   403  	}
   404  	return o.updateRepoUsers()
   405  }
   406  
   407  // RegisterOptions registers options for this feature; returns any that require a restart when changed.
   408  func (o *RepoInfo) RegisterOptions(opts *options.Options) sets.String {
   409  	opts.RegisterString(&o.baseDir, "repo-dir", "", "Path to perform checkout of repository")
   410  	opts.RegisterBool(&o.enableMdYaml, "enable-md-yaml", false, "If true, look for assignees in md yaml headers.")
   411  	opts.RegisterBool(&o.useReviewers, "use-reviewers", false, "Use \"reviewers\" rather than \"approvers\" for review")
   412  	opts.RegisterString(&o.aliasFile, "alias-file", "", "File wherein team members and aliases exist.")
   413  	return sets.NewString("repo-dir")
   414  }
   415  
   416  // GitCommand will execute the git command with the `args` within the project directory.
   417  func (o *RepoInfo) GitCommand(args []string) ([]byte, error) {
   418  	return o.gitCommandDir(args, o.projectDir)
   419  }
   420  
   421  // GitCommandDir will execute the git command with the `args` within the 'dir' directory.
   422  func (o *RepoInfo) gitCommandDir(args []string, cmdDir string) ([]byte, error) {
   423  	cmd := exec.Command("git", args...)
   424  	cmd.Dir = cmdDir
   425  	return cmd.CombinedOutput()
   426  }
   427  
   428  // findOwnersForPath returns the OWNERS file path furthest down the tree for a specified file
   429  // By default we use the reviewers section of owners flag but this can be configured by setting approvers to true
   430  func findOwnersForPath(path string, ownerMap map[string]sets.String) string {
   431  	d := path
   432  
   433  	for {
   434  		n, ok := ownerMap[d]
   435  		if ok && len(n) != 0 {
   436  			return d
   437  		}
   438  		if d == baseDirConvention {
   439  			break
   440  		}
   441  		d = filepath.Dir(d)
   442  		d = canonicalize(d)
   443  	}
   444  	return ""
   445  }
   446  
   447  // FindApproversForPath returns the OWNERS file path furthest down the tree for a specified file
   448  // that contains an approvers section
   449  func (o *RepoInfo) FindApproverOwnersForPath(path string) string {
   450  	return findOwnersForPath(path, o.approvers)
   451  }
   452  
   453  // FindReviewersForPath returns the OWNERS file path furthest down the tree for a specified file
   454  // that contains a reviewers section
   455  func (o *RepoInfo) FindReviewersForPath(path string) string {
   456  	return findOwnersForPath(path, o.reviewers)
   457  }
   458  
   459  // AllPossibleOwnerLabels returns all labels found in any owners files
   460  func (o *RepoInfo) AllPossibleOwnerLabels() sets.String {
   461  	return sets.NewString(o.allPossibleLabels.List()...)
   462  }
   463  
   464  // FindLabelsForPath returns a set of labels which should be applied to PRs
   465  // modifying files under the given path.
   466  func (o *RepoInfo) FindLabelsForPath(path string) sets.String {
   467  	s := sets.String{}
   468  
   469  	d := path
   470  	for {
   471  		l, ok := o.labels[d]
   472  		if ok && len(l) > 0 {
   473  			s = s.Union(l)
   474  		}
   475  		if d == baseDirConvention {
   476  			break
   477  		}
   478  		d = filepath.Dir(d)
   479  		d = canonicalize(d)
   480  	}
   481  	return s
   482  }
   483  
   484  // peopleForPath returns a set of users who are assignees to the
   485  // requested file. The path variable should be a full path to a filename
   486  // and not directory as the final directory will be discounted if enableMdYaml is true
   487  // leafOnly indicates whether only the OWNERS deepest in the tree (closest to the file)
   488  // should be returned or if all OWNERS in filepath should be returned
   489  func peopleForPath(path string, people map[string]sets.String, leafOnly bool, enableMdYaml bool) sets.String {
   490  	d := path
   491  	if !enableMdYaml {
   492  		// if path is a directory, this will remove the leaf directory, and returns "." for topmost dir
   493  		d = filepath.Dir(d)
   494  		d = canonicalize(path)
   495  	}
   496  
   497  	out := sets.NewString()
   498  	for {
   499  		s, ok := people[d]
   500  		if ok {
   501  			out = out.Union(s)
   502  			if leafOnly && out.Len() > 0 {
   503  				break
   504  			}
   505  		}
   506  		if d == baseDirConvention {
   507  			break
   508  		}
   509  		d = filepath.Dir(d)
   510  		d = canonicalize(d)
   511  	}
   512  	return out
   513  }
   514  
   515  // LeafApprovers returns a set of users who are the closest approvers to the
   516  // requested file. If pkg/OWNERS has user1 and pkg/util/OWNERS has user2 this
   517  // will only return user2 for the path pkg/util/sets/file.go
   518  func (o *RepoInfo) LeafApprovers(path string) sets.String {
   519  	return peopleForPath(path, o.approvers, true, o.enableMdYaml)
   520  }
   521  
   522  // Approvers returns ALL of the users who are approvers for the
   523  // requested file (including approvers in parent dirs' OWNERS).
   524  // If pkg/OWNERS has user1 and pkg/util/OWNERS has user2 this
   525  // will return both user1 and user2 for the path pkg/util/sets/file.go
   526  func (o *RepoInfo) Approvers(path string) sets.String {
   527  	return peopleForPath(path, o.approvers, false, o.enableMdYaml)
   528  }
   529  
   530  // LeafReviewers returns a set of users who are the closest reviewers to the
   531  // requested file. If pkg/OWNERS has user1 and pkg/util/OWNERS has user2 this
   532  // will only return user2 for the path pkg/util/sets/file.go
   533  func (o *RepoInfo) LeafReviewers(path string) sets.String {
   534  	if !o.useReviewers {
   535  		return o.LeafApprovers(path)
   536  	}
   537  	return peopleForPath(path, o.reviewers, true, o.enableMdYaml)
   538  }
   539  
   540  // Reviewers returns ALL of the users who are reviewers for the
   541  // requested file (including reviewers in parent dirs' OWNERS).
   542  // If pkg/OWNERS has user1 and pkg/util/OWNERS has user2 this
   543  // will return both user1 and user2 for the path pkg/util/sets/file.go
   544  func (o *RepoInfo) Reviewers(path string) sets.String {
   545  	if !o.useReviewers {
   546  		return o.Approvers(path)
   547  	}
   548  	return peopleForPath(path, o.reviewers, false, o.enableMdYaml)
   549  }