github.com/abayer/test-infra@v0.0.5/prow/git/git.go (about) 1 /* 2 Copyright 2017 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 // Package git provides a client to plugins that can do git operations. 18 package git 19 20 import ( 21 "errors" 22 "fmt" 23 "io/ioutil" 24 "os" 25 "os/exec" 26 "path/filepath" 27 "strings" 28 "sync" 29 "time" 30 31 "github.com/sirupsen/logrus" 32 ) 33 34 const github = "github.com" 35 36 // Client can clone repos. It keeps a local cache, so successive clones of the 37 // same repo should be quick. Create with NewClient. Be sure to clean it up. 38 type Client struct { 39 // logger will be used to log git operations and must be set. 40 logger *logrus.Entry 41 42 credLock sync.RWMutex 43 // user is used when pushing or pulling code if specified. 44 user string 45 46 // needed to generate the token. 47 tokenGenerator func() []byte 48 49 // dir is the location of the git cache. 50 dir string 51 // git is the path to the git binary. 52 git string 53 // base is the base path for git clone calls. For users it will be set to 54 // GitHub, but for tests set it to a directory with git repos. 55 base string 56 57 // The mutex protects repoLocks which protect individual repos. This is 58 // necessary because Clone calls for the same repo are racy. Rather than 59 // one lock for all repos, use a lock per repo. 60 // Lock with Client.lockRepo, unlock with Client.unlockRepo. 61 rlm sync.Mutex 62 repoLocks map[string]*sync.Mutex 63 } 64 65 // Clean removes the local repo cache. The Client is unusable after calling. 66 func (c *Client) Clean() error { 67 return os.RemoveAll(c.dir) 68 } 69 70 // NewClient returns a client that talks to GitHub. It will fail if git is not 71 // in the PATH. 72 func NewClient() (*Client, error) { 73 g, err := exec.LookPath("git") 74 if err != nil { 75 return nil, err 76 } 77 t, err := ioutil.TempDir("", "git") 78 if err != nil { 79 return nil, err 80 } 81 return &Client{ 82 logger: logrus.WithField("client", "git"), 83 dir: t, 84 git: g, 85 base: fmt.Sprintf("https://%s", github), 86 repoLocks: make(map[string]*sync.Mutex), 87 }, nil 88 } 89 90 // SetRemote sets the remote for the client. This is not thread-safe, and is 91 // useful for testing. The client will clone from remote/org/repo, and Repo 92 // objects spun out of the client will also hit that path. 93 func (c *Client) SetRemote(remote string) { 94 c.base = remote 95 } 96 97 // SetCredentials sets credentials in the client to be used for pushing to 98 // or pulling from remote repositories. 99 func (c *Client) SetCredentials(user string, tokenGenerator func() []byte) { 100 c.credLock.Lock() 101 defer c.credLock.Unlock() 102 c.user = user 103 c.tokenGenerator = tokenGenerator 104 } 105 106 func (c *Client) getCredentials() (string, string) { 107 c.credLock.RLock() 108 defer c.credLock.RUnlock() 109 return c.user, string(c.tokenGenerator()) 110 } 111 112 func (c *Client) lockRepo(repo string) { 113 c.rlm.Lock() 114 if _, ok := c.repoLocks[repo]; !ok { 115 c.repoLocks[repo] = &sync.Mutex{} 116 } 117 m := c.repoLocks[repo] 118 c.rlm.Unlock() 119 m.Lock() 120 } 121 122 func (c *Client) unlockRepo(repo string) { 123 c.rlm.Lock() 124 defer c.rlm.Unlock() 125 c.repoLocks[repo].Unlock() 126 } 127 128 // Clone clones a repository. Pass the full repository name, such as 129 // "kubernetes/test-infra" as the repo. 130 // This function may take a long time if it is the first time cloning the repo. 131 // In that case, it must do a full git mirror clone. For large repos, this can 132 // take a while. Once that is done, it will do a git fetch instead of a clone, 133 // which will usually take at most a few seconds. 134 func (c *Client) Clone(repo string) (*Repo, error) { 135 c.lockRepo(repo) 136 defer c.unlockRepo(repo) 137 138 base := c.base 139 user, pass := c.getCredentials() 140 if user != "" && pass != "" { 141 base = fmt.Sprintf("https://%s:%s@%s", user, pass, github) 142 } 143 cache := filepath.Join(c.dir, repo) + ".git" 144 if _, err := os.Stat(cache); os.IsNotExist(err) { 145 // Cache miss, clone it now. 146 c.logger.Infof("Cloning %s for the first time.", repo) 147 if err := os.Mkdir(filepath.Dir(cache), os.ModePerm); err != nil && !os.IsExist(err) { 148 return nil, err 149 } 150 remote := fmt.Sprintf("%s/%s", base, repo) 151 if b, err := retryCmd(c.logger, "", c.git, "clone", "--mirror", remote, cache); err != nil { 152 return nil, fmt.Errorf("git cache clone error: %v. output: %s", err, string(b)) 153 } 154 } else if err != nil { 155 return nil, err 156 } else { 157 // Cache hit. Do a git fetch to keep updated. 158 c.logger.Infof("Fetching %s.", repo) 159 if b, err := retryCmd(c.logger, cache, c.git, "fetch"); err != nil { 160 return nil, fmt.Errorf("git fetch error: %v. output: %s", err, string(b)) 161 } 162 } 163 t, err := ioutil.TempDir("", "git") 164 if err != nil { 165 return nil, err 166 } 167 if b, err := exec.Command(c.git, "clone", cache, t).CombinedOutput(); err != nil { 168 return nil, fmt.Errorf("git repo clone error: %v. output: %s", err, string(b)) 169 } 170 return &Repo{ 171 Dir: t, 172 logger: c.logger, 173 git: c.git, 174 base: base, 175 repo: repo, 176 user: user, 177 pass: pass, 178 }, nil 179 } 180 181 // Repo is a clone of a git repository. Create with Client.Clone, and don't 182 // forget to clean it up after. 183 type Repo struct { 184 // Dir is the location of the git repo. 185 Dir string 186 187 // git is the path to the git binary. 188 git string 189 // base is the base path for remote git fetch calls. 190 base string 191 // repo is the full repo name: "org/repo". 192 repo string 193 // user is used for pushing to the remote repo. 194 user string 195 // pass is used for pushing to the remote repo. 196 pass string 197 198 logger *logrus.Entry 199 } 200 201 // Clean deletes the repo. It is unusable after calling. 202 func (r *Repo) Clean() error { 203 return os.RemoveAll(r.Dir) 204 } 205 206 func (r *Repo) gitCommand(arg ...string) *exec.Cmd { 207 cmd := exec.Command(r.git, arg...) 208 cmd.Dir = r.Dir 209 return cmd 210 } 211 212 // Checkout runs git checkout. 213 func (r *Repo) Checkout(commitlike string) error { 214 r.logger.Infof("Checkout %s.", commitlike) 215 co := r.gitCommand("checkout", commitlike) 216 if b, err := co.CombinedOutput(); err != nil { 217 return fmt.Errorf("error checking out %s: %v. output: %s", commitlike, err, string(b)) 218 } 219 return nil 220 } 221 222 // RevParse runs git rev-parse. 223 func (r *Repo) RevParse(commitlike string) (string, error) { 224 r.logger.Infof("RevParse %s.", commitlike) 225 b, err := r.gitCommand("rev-parse", commitlike).CombinedOutput() 226 if err != nil { 227 return "", fmt.Errorf("error rev-parsing %s: %v. output: %s", commitlike, err, string(b)) 228 } 229 return string(b), nil 230 } 231 232 // CheckoutNewBranch creates a new branch and checks it out. 233 func (r *Repo) CheckoutNewBranch(branch string) error { 234 r.logger.Infof("Create and checkout %s.", branch) 235 co := r.gitCommand("checkout", "-b", branch) 236 if b, err := co.CombinedOutput(); err != nil { 237 return fmt.Errorf("error checking out %s: %v. output: %s", branch, err, string(b)) 238 } 239 return nil 240 } 241 242 // Merge attempts to merge commitlike into the current branch. It returns true 243 // if the merge completes. It returns an error if the abort fails. 244 func (r *Repo) Merge(commitlike string) (bool, error) { 245 r.logger.Infof("Merging %s.", commitlike) 246 co := r.gitCommand("merge", "--no-ff", "--no-stat", "-m merge", commitlike) 247 if b, err := co.CombinedOutput(); err == nil { 248 return true, nil 249 } else { 250 r.logger.WithError(err).Warningf("Merge failed with output: %s", string(b)) 251 } 252 if b, err := r.gitCommand("merge", "--abort").CombinedOutput(); err != nil { 253 return false, fmt.Errorf("error aborting merge for commitlike %s: %v. output: %s", commitlike, err, string(b)) 254 } 255 return false, nil 256 } 257 258 // Am tries to apply the patch in the given path into the current branch 259 // by performing a three-way merge (similar to git cherry-pick). It returns 260 // an error if the patch cannot be applied. 261 func (r *Repo) Am(path string) error { 262 r.logger.Infof("Applying %s.", path) 263 co := r.gitCommand("am", "--3way", path) 264 b, err := co.CombinedOutput() 265 if err == nil { 266 return nil 267 } 268 output := string(b) 269 r.logger.WithError(err).Warningf("Patch apply failed with output: %s", output) 270 if b, abortErr := r.gitCommand("am", "--abort").CombinedOutput(); err != nil { 271 r.logger.WithError(abortErr).Warningf("Aborting patch apply failed with output: %s", string(b)) 272 } 273 applyMsg := "The copy of the patch that failed is found in: .git/rebase-apply/patch" 274 if strings.Contains(output, applyMsg) { 275 i := strings.Index(output, applyMsg) 276 err = fmt.Errorf("%s", output[:i]) 277 } 278 return err 279 } 280 281 // Push pushes over https to the provided owner/repo#branch using a password 282 // for basic auth. 283 func (r *Repo) Push(repo, branch string) error { 284 if r.user == "" || r.pass == "" { 285 return errors.New("cannot push without credentials - configure your git client") 286 } 287 r.logger.Infof("Pushing to '%s/%s (branch: %s)'.", r.user, repo, branch) 288 remote := fmt.Sprintf("https://%s:%s@%s/%s/%s", r.user, r.pass, github, r.user, repo) 289 co := r.gitCommand("push", remote, branch) 290 _, err := co.CombinedOutput() 291 return err 292 } 293 294 // CheckoutPullRequest does exactly that. 295 func (r *Repo) CheckoutPullRequest(number int) error { 296 r.logger.Infof("Fetching and checking out %s#%d.", r.repo, number) 297 if b, err := retryCmd(r.logger, r.Dir, r.git, "fetch", r.base+"/"+r.repo, fmt.Sprintf("pull/%d/head:pull%d", number, number)); err != nil { 298 return fmt.Errorf("git fetch failed for PR %d: %v. output: %s", number, err, string(b)) 299 } 300 co := r.gitCommand("checkout", fmt.Sprintf("pull%d", number)) 301 if b, err := co.CombinedOutput(); err != nil { 302 return fmt.Errorf("git checkout failed for PR %d: %v. output: %s", number, err, string(b)) 303 } 304 return nil 305 } 306 307 // Config runs git config. 308 func (r *Repo) Config(key, value string) error { 309 r.logger.Infof("Running git config %s %s", key, value) 310 if b, err := r.gitCommand("config", key, value).CombinedOutput(); err != nil { 311 return fmt.Errorf("git config %s %s failed: %v. output: %s", key, value, err, string(b)) 312 } 313 return nil 314 } 315 316 // retryCmd will retry the command a few times with backoff. Use this for any 317 // commands that will be talking to GitHub, such as clones or fetches. 318 func retryCmd(l *logrus.Entry, dir, cmd string, arg ...string) ([]byte, error) { 319 var b []byte 320 var err error 321 sleepyTime := time.Second 322 for i := 0; i < 3; i++ { 323 cmd := exec.Command(cmd, arg...) 324 cmd.Dir = dir 325 b, err = cmd.CombinedOutput() 326 if err != nil { 327 l.Warningf("Running %s %v returned error %v with output %s.", cmd, arg, err, string(b)) 328 time.Sleep(sleepyTime) 329 sleepyTime *= 2 330 continue 331 } 332 break 333 } 334 return b, err 335 }