github.com/n0needt0/glide@v0.0.0-20160325160517-844a77136d85/repo/vcs.go (about)

     1  package repo
     2  
     3  import (
     4  	"encoding/json"
     5  	"fmt"
     6  	"io/ioutil"
     7  	"net/http"
     8  	"net/url"
     9  	"os"
    10  	"os/exec"
    11  	"path/filepath"
    12  	"runtime"
    13  	"sort"
    14  	"strings"
    15  
    16  	"github.com/Masterminds/glide/cfg"
    17  	"github.com/Masterminds/glide/msg"
    18  	gpath "github.com/Masterminds/glide/path"
    19  	"github.com/Masterminds/semver"
    20  	v "github.com/Masterminds/vcs"
    21  )
    22  
    23  // VcsUpdate updates to a particular checkout based on the VCS setting.
    24  func VcsUpdate(dep *cfg.Dependency, dest, home string, cache, cacheGopath, useGopath, force, updateVendored bool, updated *UpdateTracker) error {
    25  
    26  	// If the dependency has already been pinned we can skip it. This is a
    27  	// faster path so we don't need to resolve it again.
    28  	if dep.Pin != "" {
    29  		msg.Debug("Dependency %s has already been pinned. Fetching updates skipped.", dep.Name)
    30  		return nil
    31  	}
    32  
    33  	if updated.Check(dep.Name) {
    34  		msg.Debug("%s was already updated, skipping.", dep.Name)
    35  		return nil
    36  	}
    37  	updated.Add(dep.Name)
    38  
    39  	msg.Info("Fetching updates for %s.\n", dep.Name)
    40  
    41  	if filterArchOs(dep) {
    42  		msg.Info("%s is not used for %s/%s.\n", dep.Name, runtime.GOOS, runtime.GOARCH)
    43  		return nil
    44  	}
    45  
    46  	// If destination doesn't exist we need to perform an initial checkout.
    47  	if _, err := os.Stat(dest); os.IsNotExist(err) {
    48  		if err = VcsGet(dep, dest, home, cache, cacheGopath, useGopath); err != nil {
    49  			msg.Warn("Unable to checkout %s\n", dep.Name)
    50  			return err
    51  		}
    52  	} else {
    53  		// At this point we have a directory for the package.
    54  
    55  		// When the directory is not empty and has no VCS directory it's
    56  		// a vendored files situation.
    57  		empty, err := gpath.IsDirectoryEmpty(dest)
    58  		if err != nil {
    59  			return err
    60  		}
    61  		_, err = v.DetectVcsFromFS(dest)
    62  		if updateVendored == false && empty == false && err == v.ErrCannotDetectVCS {
    63  			msg.Warn("%s appears to be a vendored package. Unable to update. Consider the '--update-vendored' flag.\n", dep.Name)
    64  		} else if updateVendored == false && empty == true && err == v.ErrCannotDetectVCS {
    65  			msg.Warn("%s is an empty directory. Fetching a new copy of the dependency.", dep.Name)
    66  			msg.Debug("Removing empty directory %s", dest)
    67  			err := os.RemoveAll(dest)
    68  			if err != nil {
    69  				return err
    70  			}
    71  			if err = VcsGet(dep, dest, home, cache, cacheGopath, useGopath); err != nil {
    72  				msg.Warn("Unable to checkout %s\n", dep.Name)
    73  				return err
    74  			}
    75  		} else {
    76  
    77  			if updateVendored == true && empty == false && err == v.ErrCannotDetectVCS {
    78  				// A vendored package, no repo, and updating the vendored packages
    79  				// has been opted into.
    80  				msg.Info("%s is a vendored package. Updating.", dep.Name)
    81  				err = os.RemoveAll(dest)
    82  				if err != nil {
    83  					msg.Err("Unable to update vendored dependency %s.\n", dep.Name)
    84  					return err
    85  				}
    86  				dep.UpdateAsVendored = true
    87  
    88  				if err = VcsGet(dep, dest, home, cache, cacheGopath, useGopath); err != nil {
    89  					msg.Warn("Unable to checkout %s\n", dep.Name)
    90  					return err
    91  				}
    92  
    93  				return nil
    94  			}
    95  
    96  			repo, err := dep.GetRepo(dest)
    97  
    98  			// Tried to checkout a repo to a path that does not work. Either the
    99  			// type or endpoint has changed. Force is being passed in so the old
   100  			// location can be removed and replaced with the new one.
   101  			// Warning, any changes in the old location will be deleted.
   102  			// TODO: Put dirty checking in on the existing local checkout.
   103  			if (err == v.ErrWrongVCS || err == v.ErrWrongRemote) && force == true {
   104  				var newRemote string
   105  				if len(dep.Repository) > 0 {
   106  					newRemote = dep.Repository
   107  				} else {
   108  					newRemote = "https://" + dep.Name
   109  				}
   110  
   111  				msg.Warn("Replacing %s with contents from %s\n", dep.Name, newRemote)
   112  				rerr := os.RemoveAll(dest)
   113  				if rerr != nil {
   114  					return rerr
   115  				}
   116  				if err = VcsGet(dep, dest, home, cache, cacheGopath, useGopath); err != nil {
   117  					msg.Warn("Unable to checkout %s\n", dep.Name)
   118  					return err
   119  				}
   120  
   121  				repo, err = dep.GetRepo(dest)
   122  				if err != nil {
   123  					return err
   124  				}
   125  			} else if err != nil {
   126  				return err
   127  			} else if repo.IsDirty() {
   128  				return fmt.Errorf("%s contains uncommitted changes. Skipping update", dep.Name)
   129  			}
   130  
   131  			// Check if the current version is a tag or commit id. If it is
   132  			// and that version is already checked out we can skip updating
   133  			// which is faster than going out to the Internet to perform
   134  			// an update.
   135  			if dep.Reference != "" {
   136  				version, err := repo.Version()
   137  				if err != nil {
   138  					return err
   139  				}
   140  				ib, err := isBranch(dep.Reference, repo)
   141  				if err != nil {
   142  					return err
   143  				}
   144  
   145  				// If the current version equals the ref and it's not a
   146  				// branch it's a tag or commit id so we can skip
   147  				// performing an update.
   148  				if version == dep.Reference && !ib {
   149  					msg.Debug("%s is already set to version %s. Skipping update.", dep.Name, dep.Reference)
   150  					return nil
   151  				}
   152  			}
   153  
   154  			if err := repo.Update(); err != nil {
   155  				msg.Warn("Download failed.\n")
   156  				return err
   157  			}
   158  		}
   159  	}
   160  
   161  	return nil
   162  }
   163  
   164  // VcsVersion set the VCS version for a checkout.
   165  func VcsVersion(dep *cfg.Dependency, vend string) error {
   166  
   167  	// If the dependency has already been pinned we can skip it. This is a
   168  	// faster path so we don't need to resolve it again.
   169  	if dep.Pin != "" {
   170  		msg.Debug("Dependency %s has already been pinned. Setting version skipped.", dep.Name)
   171  		return nil
   172  	}
   173  
   174  	cwd := filepath.Join(vend, dep.Name)
   175  
   176  	// If there is no reference configured there is nothing to set.
   177  	if dep.Reference == "" {
   178  		// Before exiting update the pinned version
   179  		repo, err := dep.GetRepo(cwd)
   180  		if err != nil {
   181  			return err
   182  		}
   183  		dep.Pin, err = repo.Version()
   184  		if err != nil {
   185  			return err
   186  		}
   187  		return nil
   188  	}
   189  
   190  	// When the directory is not empty and has no VCS directory it's
   191  	// a vendored files situation.
   192  	empty, err := gpath.IsDirectoryEmpty(cwd)
   193  	if err != nil {
   194  		return err
   195  	}
   196  	_, err = v.DetectVcsFromFS(cwd)
   197  	if empty == false && err == v.ErrCannotDetectVCS {
   198  		msg.Warn("%s appears to be a vendored package. Unable to set new version. Consider the '--update-vendored' flag.\n", dep.Name)
   199  	} else {
   200  		repo, err := dep.GetRepo(cwd)
   201  		if err != nil {
   202  			return err
   203  		}
   204  
   205  		ver := dep.Reference
   206  		// Referenes in Git can begin with a ^ which is similar to semver.
   207  		// If there is a ^ prefix we assume it's a semver constraint rather than
   208  		// part of the git/VCS commit id.
   209  		if repo.IsReference(ver) && !strings.HasPrefix(ver, "^") {
   210  			msg.Info("Setting version for %s to %s.\n", dep.Name, ver)
   211  		} else {
   212  
   213  			// Create the constraing first to make sure it's valid before
   214  			// working on the repo.
   215  			constraint, err := semver.NewConstraint(ver)
   216  
   217  			// Make sure the constriant is valid. At this point it's not a valid
   218  			// reference so if it's not a valid constrint we can exit early.
   219  			if err != nil {
   220  				msg.Warn("The reference '%s' is not valid\n", ver)
   221  				return err
   222  			}
   223  
   224  			// Get the tags and branches (in that order)
   225  			refs, err := getAllVcsRefs(repo)
   226  			if err != nil {
   227  				return err
   228  			}
   229  
   230  			// Convert and filter the list to semver.Version instances
   231  			semvers := getSemVers(refs)
   232  
   233  			// Sort semver list
   234  			sort.Sort(sort.Reverse(semver.Collection(semvers)))
   235  			found := false
   236  			for _, v := range semvers {
   237  				if constraint.Check(v) {
   238  					found = true
   239  					// If the constrint passes get the original reference
   240  					ver = v.Original()
   241  					break
   242  				}
   243  			}
   244  			if found {
   245  				msg.Info("Detected semantic version. Setting version for %s to %s.\n", dep.Name, ver)
   246  			} else {
   247  				msg.Warn("Unable to find semantic version for constraint %s %s\n", dep.Name, ver)
   248  			}
   249  		}
   250  		if err := repo.UpdateVersion(ver); err != nil {
   251  			return err
   252  		}
   253  		dep.Pin, err = repo.Version()
   254  		if err != nil {
   255  			return err
   256  		}
   257  	}
   258  
   259  	return nil
   260  }
   261  
   262  // VcsGet figures out how to fetch a dependency, and then gets it.
   263  //
   264  // VcsGet installs into the dest.
   265  func VcsGet(dep *cfg.Dependency, dest, home string, cache, cacheGopath, useGopath bool) error {
   266  	// When not skipping the $GOPATH look in it for a copy of the package
   267  	if useGopath {
   268  		// Check if the $GOPATH has a viable version to use and if so copy to vendor
   269  		gps := gpath.Gopaths()
   270  		for _, p := range gps {
   271  			d := filepath.Join(p, "src", dep.Name)
   272  			if _, err := os.Stat(d); err == nil {
   273  				empty, err := gpath.IsDirectoryEmpty(d)
   274  				if empty || err != nil {
   275  					continue
   276  				}
   277  
   278  				repo, err := dep.GetRepo(d)
   279  				if err != nil {
   280  					continue
   281  				}
   282  
   283  				// Dirty repos have uncommitted changes.
   284  				if repo.IsDirty() {
   285  					continue
   286  				}
   287  
   288  				// Having found a repo we copy it to vendor and update it.
   289  				msg.Info("Copying package %s from the GOPATH.", dep.Name)
   290  				msg.Debug("Found %s in GOPATH at %s. Copying to %s", dep.Name, d, dest)
   291  				err = gpath.CopyDir(d, dest)
   292  				if err != nil {
   293  					return err
   294  				}
   295  
   296  				// Update the repo in the vendor directory
   297  				msg.Debug("Updating %s, now in the vendor path at %s", dep.Name, dest)
   298  				repo, err = dep.GetRepo(dest)
   299  				if err != nil {
   300  					return err
   301  				}
   302  				err = repo.Update()
   303  				if err != nil {
   304  					return err
   305  				}
   306  
   307  				// If there is no reference set on the dep we try to checkout
   308  				// the default branch.
   309  				if dep.Reference == "" {
   310  					db := defaultBranch(repo, home)
   311  					if db != "" {
   312  						err = repo.UpdateVersion(db)
   313  						if err != nil && msg.Default.IsDebugging {
   314  							msg.Debug("Attempting to set the version on %s to %s failed. Error %s", dep.Name, db, err)
   315  						}
   316  					}
   317  				}
   318  				return nil
   319  			}
   320  		}
   321  	}
   322  
   323  	// When opting in to cache in the GOPATH attempt to do put a copy there.
   324  	if cacheGopath {
   325  
   326  		// Since we didn't find an existing copy in the GOPATHs try to clone there.
   327  		gp := gpath.Gopath()
   328  		if gp != "" {
   329  			d := filepath.Join(gp, "src", dep.Name)
   330  			if _, err := os.Stat(d); os.IsNotExist(err) {
   331  				// Empty directory so we checkout out the code here.
   332  				msg.Debug("Retrieving %s to %s before copying to vendor", dep.Name, d)
   333  				repo, err := dep.GetRepo(d)
   334  				if err != nil {
   335  					return err
   336  				}
   337  				repo.Get()
   338  
   339  				branch := findCurrentBranch(repo)
   340  				if branch != "" {
   341  					// we know the default branch so we can store it in the cache
   342  					var loc string
   343  					if dep.Repository != "" {
   344  						loc = dep.Repository
   345  					} else {
   346  						loc = "https://" + dep.Name
   347  					}
   348  					key, err := cacheCreateKey(loc)
   349  					if err == nil {
   350  						msg.Debug("Saving default branch for %s", repo.Remote())
   351  						c := cacheRepoInfo{DefaultBranch: branch}
   352  						err = saveCacheRepoData(key, c, home)
   353  						if msg.Default.IsDebugging && err == errCacheDisabled {
   354  							msg.Debug("Unable to cache default branch because caching is disabled")
   355  						}
   356  					}
   357  				}
   358  
   359  				msg.Debug("Copying %s from GOPATH at %s to %s", dep.Name, d, dest)
   360  				err = gpath.CopyDir(d, dest)
   361  				if err != nil {
   362  					return err
   363  				}
   364  
   365  				return nil
   366  			}
   367  		}
   368  	}
   369  
   370  	// If opting in to caching attempt to put it in the cache folder
   371  	if cache {
   372  		// Check if the cache has a viable version and try to use that.
   373  		var loc string
   374  		if dep.Repository != "" {
   375  			loc = dep.Repository
   376  		} else {
   377  			loc = "https://" + dep.Name
   378  		}
   379  		key, err := cacheCreateKey(loc)
   380  		if err == nil {
   381  			d := filepath.Join(home, "cache", "src", key)
   382  
   383  			repo, err := dep.GetRepo(d)
   384  			if err != nil {
   385  				return err
   386  			}
   387  			// If the directory does not exist this is a first cache.
   388  			if _, err = os.Stat(d); os.IsNotExist(err) {
   389  				msg.Debug("Adding %s to the cache for the first time", dep.Name)
   390  				err = repo.Get()
   391  				if err != nil {
   392  					return err
   393  				}
   394  				branch := findCurrentBranch(repo)
   395  				if branch != "" {
   396  					// we know the default branch so we can store it in the cache
   397  					var loc string
   398  					if dep.Repository != "" {
   399  						loc = dep.Repository
   400  					} else {
   401  						loc = "https://" + dep.Name
   402  					}
   403  					key, err := cacheCreateKey(loc)
   404  					if err == nil {
   405  						msg.Debug("Saving default branch for %s", repo.Remote())
   406  						c := cacheRepoInfo{DefaultBranch: branch}
   407  						err = saveCacheRepoData(key, c, home)
   408  						if err == errCacheDisabled {
   409  							msg.Debug("Unable to cache default branch because caching is disabled")
   410  						} else if err != nil {
   411  							msg.Debug("Error saving %s to cache. Error: %s", repo.Remote(), err)
   412  						}
   413  					}
   414  				}
   415  
   416  			} else {
   417  				msg.Debug("Updating %s in the cache", dep.Name)
   418  				err = repo.Update()
   419  				if err != nil {
   420  					return err
   421  				}
   422  			}
   423  
   424  			msg.Debug("Copying %s from the cache to %s", dep.Name, dest)
   425  			err = gpath.CopyDir(d, dest)
   426  			if err != nil {
   427  				return err
   428  			}
   429  
   430  			return nil
   431  		}
   432  
   433  		msg.Warn("Cache key generation error: %s", err)
   434  	}
   435  
   436  	// If unable to cache pull directly into the vendor/ directory.
   437  	repo, err := dep.GetRepo(dest)
   438  	if err != nil {
   439  		return err
   440  	}
   441  
   442  	gerr := repo.Get()
   443  
   444  	// Attempt to cache the default branch
   445  	if cache {
   446  		if branch := findCurrentBranch(repo); branch != "" {
   447  			// we know the default branch so we can store it in the cache
   448  			var loc string
   449  			if dep.Repository != "" {
   450  				loc = dep.Repository
   451  			} else {
   452  				loc = "https://" + dep.Name
   453  			}
   454  			key, err := cacheCreateKey(loc)
   455  			if err == nil {
   456  				msg.Debug("Saving default branch for %s", repo.Remote())
   457  				c := cacheRepoInfo{DefaultBranch: branch}
   458  				err = saveCacheRepoData(key, c, home)
   459  				if err == errCacheDisabled {
   460  					msg.Debug("Unable to cache default branch because caching is disabled")
   461  				} else if err != nil {
   462  					msg.Debug("Error saving %s to cache - Error: %s", repo.Remote(), err)
   463  				}
   464  			}
   465  		}
   466  	}
   467  
   468  	return gerr
   469  }
   470  
   471  // filterArchOs indicates a dependency should be filtered out because it is
   472  // the wrong GOOS or GOARCH.
   473  //
   474  // FIXME: Should this be moved to the dependency package?
   475  func filterArchOs(dep *cfg.Dependency) bool {
   476  	found := false
   477  	if len(dep.Arch) > 0 {
   478  		for _, a := range dep.Arch {
   479  			if a == runtime.GOARCH {
   480  				found = true
   481  			}
   482  		}
   483  		// If it's not found, it should be filtered out.
   484  		if !found {
   485  			return true
   486  		}
   487  	}
   488  
   489  	found = false
   490  	if len(dep.Os) > 0 {
   491  		for _, o := range dep.Os {
   492  			if o == runtime.GOOS {
   493  				found = true
   494  			}
   495  		}
   496  		if !found {
   497  			return true
   498  		}
   499  
   500  	}
   501  
   502  	return false
   503  }
   504  
   505  // isBranch returns true if the given string is a branch in VCS.
   506  func isBranch(branch string, repo v.Repo) (bool, error) {
   507  	branches, err := repo.Branches()
   508  	if err != nil {
   509  		return false, err
   510  	}
   511  	for _, b := range branches {
   512  		if b == branch {
   513  			return true, nil
   514  		}
   515  	}
   516  	return false, nil
   517  }
   518  
   519  // defaultBranch tries to ascertain the default branch for the given repo.
   520  // Some repos will have multiple branches in them (e.g. Git) while others
   521  // (e.g. Svn) will not.
   522  func defaultBranch(repo v.Repo, home string) string {
   523  
   524  	// Svn and Bzr use different locations (paths or entire locations)
   525  	// for branches so we won't have a default branch.
   526  	if repo.Vcs() == v.Svn || repo.Vcs() == v.Bzr {
   527  		return ""
   528  	}
   529  
   530  	// Check the cache for a value.
   531  	key, kerr := cacheCreateKey(repo.Remote())
   532  	var d cacheRepoInfo
   533  	if kerr == nil {
   534  		d, err := cacheRepoData(key, home)
   535  		if err == nil {
   536  			if d.DefaultBranch != "" {
   537  				return d.DefaultBranch
   538  			}
   539  		}
   540  	}
   541  
   542  	// If we don't have it in the store try some APIs
   543  	r := repo.Remote()
   544  	u, err := url.Parse(r)
   545  	if err != nil {
   546  		return ""
   547  	}
   548  	if u.Scheme == "" {
   549  		// Where there is no scheme we try urls like git@github.com:foo/bar
   550  		r = strings.Replace(r, ":", "/", -1)
   551  		r = "ssh://" + r
   552  		u, err = url.Parse(r)
   553  		if err != nil {
   554  			return ""
   555  		}
   556  		u.Scheme = ""
   557  	}
   558  	if u.Host == "github.com" {
   559  		parts := strings.Split(u.Path, "/")
   560  		if len(parts) != 2 {
   561  			return ""
   562  		}
   563  		api := fmt.Sprintf("https://api.github.com/repos/%s/%s", parts[0], parts[1])
   564  		resp, err := http.Get(api)
   565  		if err != nil {
   566  			return ""
   567  		}
   568  		defer resp.Body.Close()
   569  		if resp.StatusCode >= 300 || resp.StatusCode < 200 {
   570  			return ""
   571  		}
   572  		body, err := ioutil.ReadAll(resp.Body)
   573  		var data interface{}
   574  		err = json.Unmarshal(body, &data)
   575  		if err != nil {
   576  			return ""
   577  		}
   578  		gh := data.(map[string]interface{})
   579  		db := gh["default_branch"].(string)
   580  		if kerr == nil {
   581  			d.DefaultBranch = db
   582  			err := saveCacheRepoData(key, d, home)
   583  			if err == errCacheDisabled {
   584  				msg.Debug("Unable to cache default branch because caching is disabled")
   585  			} else if err != nil {
   586  				msg.Debug("Error saving %s to cache. Error: %s", repo.Remote(), err)
   587  			}
   588  		}
   589  		return db
   590  	}
   591  
   592  	if u.Host == "bitbucket.org" {
   593  		parts := strings.Split(u.Path, "/")
   594  		if len(parts) != 2 {
   595  			return ""
   596  		}
   597  		api := fmt.Sprintf("https://bitbucket.org/api/1.0/repositories/%s/%s/main-branch/", parts[0], parts[1])
   598  		resp, err := http.Get(api)
   599  		if err != nil {
   600  			return ""
   601  		}
   602  		defer resp.Body.Close()
   603  		if resp.StatusCode >= 300 || resp.StatusCode < 200 {
   604  			return ""
   605  		}
   606  		body, err := ioutil.ReadAll(resp.Body)
   607  		var data interface{}
   608  		err = json.Unmarshal(body, &data)
   609  		if err != nil {
   610  			return ""
   611  		}
   612  		bb := data.(map[string]interface{})
   613  		db := bb["name"].(string)
   614  		if kerr == nil {
   615  			d.DefaultBranch = db
   616  			err := saveCacheRepoData(key, d, home)
   617  			if err == errCacheDisabled {
   618  				msg.Debug("Unable to cache default branch because caching is disabled")
   619  			} else if err != nil {
   620  				msg.Debug("Error saving %s to cache. Error: %s", repo.Remote(), err)
   621  			}
   622  		}
   623  		return db
   624  	}
   625  
   626  	return ""
   627  }
   628  
   629  // From a local repo find out the current branch name if there is one.
   630  func findCurrentBranch(repo v.Repo) string {
   631  	msg.Debug("Attempting to find current branch for %s", repo.Remote())
   632  	// Svn and Bzr don't have default branches.
   633  	if repo.Vcs() == v.Svn || repo.Vcs() == v.Bzr {
   634  		return ""
   635  	}
   636  
   637  	if repo.Vcs() == v.Git {
   638  		c := exec.Command("git", "symbolic-ref", "--short", "HEAD")
   639  		c.Dir = repo.LocalPath()
   640  		c.Env = envForDir(c.Dir)
   641  		out, err := c.CombinedOutput()
   642  		if err != nil {
   643  			msg.Debug("Unable to find current branch for %s, error: %s", repo.Remote(), err)
   644  			return ""
   645  		}
   646  		return strings.TrimSpace(string(out))
   647  	}
   648  
   649  	if repo.Vcs() == v.Hg {
   650  		c := exec.Command("hg", "branch")
   651  		c.Dir = repo.LocalPath()
   652  		c.Env = envForDir(c.Dir)
   653  		out, err := c.CombinedOutput()
   654  		if err != nil {
   655  			msg.Debug("Unable to find current branch for %s, error: %s", repo.Remote(), err)
   656  			return ""
   657  		}
   658  		return strings.TrimSpace(string(out))
   659  	}
   660  
   661  	return ""
   662  }
   663  
   664  func envForDir(dir string) []string {
   665  	env := os.Environ()
   666  	return mergeEnvLists([]string{"PWD=" + dir}, env)
   667  }
   668  
   669  func mergeEnvLists(in, out []string) []string {
   670  NextVar:
   671  	for _, inkv := range in {
   672  		k := strings.SplitAfterN(inkv, "=", 2)[0]
   673  		for i, outkv := range out {
   674  			if strings.HasPrefix(outkv, k) {
   675  				out[i] = inkv
   676  				continue NextVar
   677  			}
   678  		}
   679  		out = append(out, inkv)
   680  	}
   681  	return out
   682  }