github.com/n0needt0/glide@v0.0.0-20160325160517-844a77136d85/repo/vcs.go (about) 1 package repo 2 3 import ( 4 "encoding/json" 5 "fmt" 6 "io/ioutil" 7 "net/http" 8 "net/url" 9 "os" 10 "os/exec" 11 "path/filepath" 12 "runtime" 13 "sort" 14 "strings" 15 16 "github.com/Masterminds/glide/cfg" 17 "github.com/Masterminds/glide/msg" 18 gpath "github.com/Masterminds/glide/path" 19 "github.com/Masterminds/semver" 20 v "github.com/Masterminds/vcs" 21 ) 22 23 // VcsUpdate updates to a particular checkout based on the VCS setting. 24 func VcsUpdate(dep *cfg.Dependency, dest, home string, cache, cacheGopath, useGopath, force, updateVendored bool, updated *UpdateTracker) error { 25 26 // If the dependency has already been pinned we can skip it. This is a 27 // faster path so we don't need to resolve it again. 28 if dep.Pin != "" { 29 msg.Debug("Dependency %s has already been pinned. Fetching updates skipped.", dep.Name) 30 return nil 31 } 32 33 if updated.Check(dep.Name) { 34 msg.Debug("%s was already updated, skipping.", dep.Name) 35 return nil 36 } 37 updated.Add(dep.Name) 38 39 msg.Info("Fetching updates for %s.\n", dep.Name) 40 41 if filterArchOs(dep) { 42 msg.Info("%s is not used for %s/%s.\n", dep.Name, runtime.GOOS, runtime.GOARCH) 43 return nil 44 } 45 46 // If destination doesn't exist we need to perform an initial checkout. 47 if _, err := os.Stat(dest); os.IsNotExist(err) { 48 if err = VcsGet(dep, dest, home, cache, cacheGopath, useGopath); err != nil { 49 msg.Warn("Unable to checkout %s\n", dep.Name) 50 return err 51 } 52 } else { 53 // At this point we have a directory for the package. 54 55 // When the directory is not empty and has no VCS directory it's 56 // a vendored files situation. 57 empty, err := gpath.IsDirectoryEmpty(dest) 58 if err != nil { 59 return err 60 } 61 _, err = v.DetectVcsFromFS(dest) 62 if updateVendored == false && empty == false && err == v.ErrCannotDetectVCS { 63 msg.Warn("%s appears to be a vendored package. Unable to update. Consider the '--update-vendored' flag.\n", dep.Name) 64 } else if updateVendored == false && empty == true && err == v.ErrCannotDetectVCS { 65 msg.Warn("%s is an empty directory. Fetching a new copy of the dependency.", dep.Name) 66 msg.Debug("Removing empty directory %s", dest) 67 err := os.RemoveAll(dest) 68 if err != nil { 69 return err 70 } 71 if err = VcsGet(dep, dest, home, cache, cacheGopath, useGopath); err != nil { 72 msg.Warn("Unable to checkout %s\n", dep.Name) 73 return err 74 } 75 } else { 76 77 if updateVendored == true && empty == false && err == v.ErrCannotDetectVCS { 78 // A vendored package, no repo, and updating the vendored packages 79 // has been opted into. 80 msg.Info("%s is a vendored package. Updating.", dep.Name) 81 err = os.RemoveAll(dest) 82 if err != nil { 83 msg.Err("Unable to update vendored dependency %s.\n", dep.Name) 84 return err 85 } 86 dep.UpdateAsVendored = true 87 88 if err = VcsGet(dep, dest, home, cache, cacheGopath, useGopath); err != nil { 89 msg.Warn("Unable to checkout %s\n", dep.Name) 90 return err 91 } 92 93 return nil 94 } 95 96 repo, err := dep.GetRepo(dest) 97 98 // Tried to checkout a repo to a path that does not work. Either the 99 // type or endpoint has changed. Force is being passed in so the old 100 // location can be removed and replaced with the new one. 101 // Warning, any changes in the old location will be deleted. 102 // TODO: Put dirty checking in on the existing local checkout. 103 if (err == v.ErrWrongVCS || err == v.ErrWrongRemote) && force == true { 104 var newRemote string 105 if len(dep.Repository) > 0 { 106 newRemote = dep.Repository 107 } else { 108 newRemote = "https://" + dep.Name 109 } 110 111 msg.Warn("Replacing %s with contents from %s\n", dep.Name, newRemote) 112 rerr := os.RemoveAll(dest) 113 if rerr != nil { 114 return rerr 115 } 116 if err = VcsGet(dep, dest, home, cache, cacheGopath, useGopath); err != nil { 117 msg.Warn("Unable to checkout %s\n", dep.Name) 118 return err 119 } 120 121 repo, err = dep.GetRepo(dest) 122 if err != nil { 123 return err 124 } 125 } else if err != nil { 126 return err 127 } else if repo.IsDirty() { 128 return fmt.Errorf("%s contains uncommitted changes. Skipping update", dep.Name) 129 } 130 131 // Check if the current version is a tag or commit id. If it is 132 // and that version is already checked out we can skip updating 133 // which is faster than going out to the Internet to perform 134 // an update. 135 if dep.Reference != "" { 136 version, err := repo.Version() 137 if err != nil { 138 return err 139 } 140 ib, err := isBranch(dep.Reference, repo) 141 if err != nil { 142 return err 143 } 144 145 // If the current version equals the ref and it's not a 146 // branch it's a tag or commit id so we can skip 147 // performing an update. 148 if version == dep.Reference && !ib { 149 msg.Debug("%s is already set to version %s. Skipping update.", dep.Name, dep.Reference) 150 return nil 151 } 152 } 153 154 if err := repo.Update(); err != nil { 155 msg.Warn("Download failed.\n") 156 return err 157 } 158 } 159 } 160 161 return nil 162 } 163 164 // VcsVersion set the VCS version for a checkout. 165 func VcsVersion(dep *cfg.Dependency, vend string) error { 166 167 // If the dependency has already been pinned we can skip it. This is a 168 // faster path so we don't need to resolve it again. 169 if dep.Pin != "" { 170 msg.Debug("Dependency %s has already been pinned. Setting version skipped.", dep.Name) 171 return nil 172 } 173 174 cwd := filepath.Join(vend, dep.Name) 175 176 // If there is no reference configured there is nothing to set. 177 if dep.Reference == "" { 178 // Before exiting update the pinned version 179 repo, err := dep.GetRepo(cwd) 180 if err != nil { 181 return err 182 } 183 dep.Pin, err = repo.Version() 184 if err != nil { 185 return err 186 } 187 return nil 188 } 189 190 // When the directory is not empty and has no VCS directory it's 191 // a vendored files situation. 192 empty, err := gpath.IsDirectoryEmpty(cwd) 193 if err != nil { 194 return err 195 } 196 _, err = v.DetectVcsFromFS(cwd) 197 if empty == false && err == v.ErrCannotDetectVCS { 198 msg.Warn("%s appears to be a vendored package. Unable to set new version. Consider the '--update-vendored' flag.\n", dep.Name) 199 } else { 200 repo, err := dep.GetRepo(cwd) 201 if err != nil { 202 return err 203 } 204 205 ver := dep.Reference 206 // Referenes in Git can begin with a ^ which is similar to semver. 207 // If there is a ^ prefix we assume it's a semver constraint rather than 208 // part of the git/VCS commit id. 209 if repo.IsReference(ver) && !strings.HasPrefix(ver, "^") { 210 msg.Info("Setting version for %s to %s.\n", dep.Name, ver) 211 } else { 212 213 // Create the constraing first to make sure it's valid before 214 // working on the repo. 215 constraint, err := semver.NewConstraint(ver) 216 217 // Make sure the constriant is valid. At this point it's not a valid 218 // reference so if it's not a valid constrint we can exit early. 219 if err != nil { 220 msg.Warn("The reference '%s' is not valid\n", ver) 221 return err 222 } 223 224 // Get the tags and branches (in that order) 225 refs, err := getAllVcsRefs(repo) 226 if err != nil { 227 return err 228 } 229 230 // Convert and filter the list to semver.Version instances 231 semvers := getSemVers(refs) 232 233 // Sort semver list 234 sort.Sort(sort.Reverse(semver.Collection(semvers))) 235 found := false 236 for _, v := range semvers { 237 if constraint.Check(v) { 238 found = true 239 // If the constrint passes get the original reference 240 ver = v.Original() 241 break 242 } 243 } 244 if found { 245 msg.Info("Detected semantic version. Setting version for %s to %s.\n", dep.Name, ver) 246 } else { 247 msg.Warn("Unable to find semantic version for constraint %s %s\n", dep.Name, ver) 248 } 249 } 250 if err := repo.UpdateVersion(ver); err != nil { 251 return err 252 } 253 dep.Pin, err = repo.Version() 254 if err != nil { 255 return err 256 } 257 } 258 259 return nil 260 } 261 262 // VcsGet figures out how to fetch a dependency, and then gets it. 263 // 264 // VcsGet installs into the dest. 265 func VcsGet(dep *cfg.Dependency, dest, home string, cache, cacheGopath, useGopath bool) error { 266 // When not skipping the $GOPATH look in it for a copy of the package 267 if useGopath { 268 // Check if the $GOPATH has a viable version to use and if so copy to vendor 269 gps := gpath.Gopaths() 270 for _, p := range gps { 271 d := filepath.Join(p, "src", dep.Name) 272 if _, err := os.Stat(d); err == nil { 273 empty, err := gpath.IsDirectoryEmpty(d) 274 if empty || err != nil { 275 continue 276 } 277 278 repo, err := dep.GetRepo(d) 279 if err != nil { 280 continue 281 } 282 283 // Dirty repos have uncommitted changes. 284 if repo.IsDirty() { 285 continue 286 } 287 288 // Having found a repo we copy it to vendor and update it. 289 msg.Info("Copying package %s from the GOPATH.", dep.Name) 290 msg.Debug("Found %s in GOPATH at %s. Copying to %s", dep.Name, d, dest) 291 err = gpath.CopyDir(d, dest) 292 if err != nil { 293 return err 294 } 295 296 // Update the repo in the vendor directory 297 msg.Debug("Updating %s, now in the vendor path at %s", dep.Name, dest) 298 repo, err = dep.GetRepo(dest) 299 if err != nil { 300 return err 301 } 302 err = repo.Update() 303 if err != nil { 304 return err 305 } 306 307 // If there is no reference set on the dep we try to checkout 308 // the default branch. 309 if dep.Reference == "" { 310 db := defaultBranch(repo, home) 311 if db != "" { 312 err = repo.UpdateVersion(db) 313 if err != nil && msg.Default.IsDebugging { 314 msg.Debug("Attempting to set the version on %s to %s failed. Error %s", dep.Name, db, err) 315 } 316 } 317 } 318 return nil 319 } 320 } 321 } 322 323 // When opting in to cache in the GOPATH attempt to do put a copy there. 324 if cacheGopath { 325 326 // Since we didn't find an existing copy in the GOPATHs try to clone there. 327 gp := gpath.Gopath() 328 if gp != "" { 329 d := filepath.Join(gp, "src", dep.Name) 330 if _, err := os.Stat(d); os.IsNotExist(err) { 331 // Empty directory so we checkout out the code here. 332 msg.Debug("Retrieving %s to %s before copying to vendor", dep.Name, d) 333 repo, err := dep.GetRepo(d) 334 if err != nil { 335 return err 336 } 337 repo.Get() 338 339 branch := findCurrentBranch(repo) 340 if branch != "" { 341 // we know the default branch so we can store it in the cache 342 var loc string 343 if dep.Repository != "" { 344 loc = dep.Repository 345 } else { 346 loc = "https://" + dep.Name 347 } 348 key, err := cacheCreateKey(loc) 349 if err == nil { 350 msg.Debug("Saving default branch for %s", repo.Remote()) 351 c := cacheRepoInfo{DefaultBranch: branch} 352 err = saveCacheRepoData(key, c, home) 353 if msg.Default.IsDebugging && err == errCacheDisabled { 354 msg.Debug("Unable to cache default branch because caching is disabled") 355 } 356 } 357 } 358 359 msg.Debug("Copying %s from GOPATH at %s to %s", dep.Name, d, dest) 360 err = gpath.CopyDir(d, dest) 361 if err != nil { 362 return err 363 } 364 365 return nil 366 } 367 } 368 } 369 370 // If opting in to caching attempt to put it in the cache folder 371 if cache { 372 // Check if the cache has a viable version and try to use that. 373 var loc string 374 if dep.Repository != "" { 375 loc = dep.Repository 376 } else { 377 loc = "https://" + dep.Name 378 } 379 key, err := cacheCreateKey(loc) 380 if err == nil { 381 d := filepath.Join(home, "cache", "src", key) 382 383 repo, err := dep.GetRepo(d) 384 if err != nil { 385 return err 386 } 387 // If the directory does not exist this is a first cache. 388 if _, err = os.Stat(d); os.IsNotExist(err) { 389 msg.Debug("Adding %s to the cache for the first time", dep.Name) 390 err = repo.Get() 391 if err != nil { 392 return err 393 } 394 branch := findCurrentBranch(repo) 395 if branch != "" { 396 // we know the default branch so we can store it in the cache 397 var loc string 398 if dep.Repository != "" { 399 loc = dep.Repository 400 } else { 401 loc = "https://" + dep.Name 402 } 403 key, err := cacheCreateKey(loc) 404 if err == nil { 405 msg.Debug("Saving default branch for %s", repo.Remote()) 406 c := cacheRepoInfo{DefaultBranch: branch} 407 err = saveCacheRepoData(key, c, home) 408 if err == errCacheDisabled { 409 msg.Debug("Unable to cache default branch because caching is disabled") 410 } else if err != nil { 411 msg.Debug("Error saving %s to cache. Error: %s", repo.Remote(), err) 412 } 413 } 414 } 415 416 } else { 417 msg.Debug("Updating %s in the cache", dep.Name) 418 err = repo.Update() 419 if err != nil { 420 return err 421 } 422 } 423 424 msg.Debug("Copying %s from the cache to %s", dep.Name, dest) 425 err = gpath.CopyDir(d, dest) 426 if err != nil { 427 return err 428 } 429 430 return nil 431 } 432 433 msg.Warn("Cache key generation error: %s", err) 434 } 435 436 // If unable to cache pull directly into the vendor/ directory. 437 repo, err := dep.GetRepo(dest) 438 if err != nil { 439 return err 440 } 441 442 gerr := repo.Get() 443 444 // Attempt to cache the default branch 445 if cache { 446 if branch := findCurrentBranch(repo); branch != "" { 447 // we know the default branch so we can store it in the cache 448 var loc string 449 if dep.Repository != "" { 450 loc = dep.Repository 451 } else { 452 loc = "https://" + dep.Name 453 } 454 key, err := cacheCreateKey(loc) 455 if err == nil { 456 msg.Debug("Saving default branch for %s", repo.Remote()) 457 c := cacheRepoInfo{DefaultBranch: branch} 458 err = saveCacheRepoData(key, c, home) 459 if err == errCacheDisabled { 460 msg.Debug("Unable to cache default branch because caching is disabled") 461 } else if err != nil { 462 msg.Debug("Error saving %s to cache - Error: %s", repo.Remote(), err) 463 } 464 } 465 } 466 } 467 468 return gerr 469 } 470 471 // filterArchOs indicates a dependency should be filtered out because it is 472 // the wrong GOOS or GOARCH. 473 // 474 // FIXME: Should this be moved to the dependency package? 475 func filterArchOs(dep *cfg.Dependency) bool { 476 found := false 477 if len(dep.Arch) > 0 { 478 for _, a := range dep.Arch { 479 if a == runtime.GOARCH { 480 found = true 481 } 482 } 483 // If it's not found, it should be filtered out. 484 if !found { 485 return true 486 } 487 } 488 489 found = false 490 if len(dep.Os) > 0 { 491 for _, o := range dep.Os { 492 if o == runtime.GOOS { 493 found = true 494 } 495 } 496 if !found { 497 return true 498 } 499 500 } 501 502 return false 503 } 504 505 // isBranch returns true if the given string is a branch in VCS. 506 func isBranch(branch string, repo v.Repo) (bool, error) { 507 branches, err := repo.Branches() 508 if err != nil { 509 return false, err 510 } 511 for _, b := range branches { 512 if b == branch { 513 return true, nil 514 } 515 } 516 return false, nil 517 } 518 519 // defaultBranch tries to ascertain the default branch for the given repo. 520 // Some repos will have multiple branches in them (e.g. Git) while others 521 // (e.g. Svn) will not. 522 func defaultBranch(repo v.Repo, home string) string { 523 524 // Svn and Bzr use different locations (paths or entire locations) 525 // for branches so we won't have a default branch. 526 if repo.Vcs() == v.Svn || repo.Vcs() == v.Bzr { 527 return "" 528 } 529 530 // Check the cache for a value. 531 key, kerr := cacheCreateKey(repo.Remote()) 532 var d cacheRepoInfo 533 if kerr == nil { 534 d, err := cacheRepoData(key, home) 535 if err == nil { 536 if d.DefaultBranch != "" { 537 return d.DefaultBranch 538 } 539 } 540 } 541 542 // If we don't have it in the store try some APIs 543 r := repo.Remote() 544 u, err := url.Parse(r) 545 if err != nil { 546 return "" 547 } 548 if u.Scheme == "" { 549 // Where there is no scheme we try urls like git@github.com:foo/bar 550 r = strings.Replace(r, ":", "/", -1) 551 r = "ssh://" + r 552 u, err = url.Parse(r) 553 if err != nil { 554 return "" 555 } 556 u.Scheme = "" 557 } 558 if u.Host == "github.com" { 559 parts := strings.Split(u.Path, "/") 560 if len(parts) != 2 { 561 return "" 562 } 563 api := fmt.Sprintf("https://api.github.com/repos/%s/%s", parts[0], parts[1]) 564 resp, err := http.Get(api) 565 if err != nil { 566 return "" 567 } 568 defer resp.Body.Close() 569 if resp.StatusCode >= 300 || resp.StatusCode < 200 { 570 return "" 571 } 572 body, err := ioutil.ReadAll(resp.Body) 573 var data interface{} 574 err = json.Unmarshal(body, &data) 575 if err != nil { 576 return "" 577 } 578 gh := data.(map[string]interface{}) 579 db := gh["default_branch"].(string) 580 if kerr == nil { 581 d.DefaultBranch = db 582 err := saveCacheRepoData(key, d, home) 583 if err == errCacheDisabled { 584 msg.Debug("Unable to cache default branch because caching is disabled") 585 } else if err != nil { 586 msg.Debug("Error saving %s to cache. Error: %s", repo.Remote(), err) 587 } 588 } 589 return db 590 } 591 592 if u.Host == "bitbucket.org" { 593 parts := strings.Split(u.Path, "/") 594 if len(parts) != 2 { 595 return "" 596 } 597 api := fmt.Sprintf("https://bitbucket.org/api/1.0/repositories/%s/%s/main-branch/", parts[0], parts[1]) 598 resp, err := http.Get(api) 599 if err != nil { 600 return "" 601 } 602 defer resp.Body.Close() 603 if resp.StatusCode >= 300 || resp.StatusCode < 200 { 604 return "" 605 } 606 body, err := ioutil.ReadAll(resp.Body) 607 var data interface{} 608 err = json.Unmarshal(body, &data) 609 if err != nil { 610 return "" 611 } 612 bb := data.(map[string]interface{}) 613 db := bb["name"].(string) 614 if kerr == nil { 615 d.DefaultBranch = db 616 err := saveCacheRepoData(key, d, home) 617 if err == errCacheDisabled { 618 msg.Debug("Unable to cache default branch because caching is disabled") 619 } else if err != nil { 620 msg.Debug("Error saving %s to cache. Error: %s", repo.Remote(), err) 621 } 622 } 623 return db 624 } 625 626 return "" 627 } 628 629 // From a local repo find out the current branch name if there is one. 630 func findCurrentBranch(repo v.Repo) string { 631 msg.Debug("Attempting to find current branch for %s", repo.Remote()) 632 // Svn and Bzr don't have default branches. 633 if repo.Vcs() == v.Svn || repo.Vcs() == v.Bzr { 634 return "" 635 } 636 637 if repo.Vcs() == v.Git { 638 c := exec.Command("git", "symbolic-ref", "--short", "HEAD") 639 c.Dir = repo.LocalPath() 640 c.Env = envForDir(c.Dir) 641 out, err := c.CombinedOutput() 642 if err != nil { 643 msg.Debug("Unable to find current branch for %s, error: %s", repo.Remote(), err) 644 return "" 645 } 646 return strings.TrimSpace(string(out)) 647 } 648 649 if repo.Vcs() == v.Hg { 650 c := exec.Command("hg", "branch") 651 c.Dir = repo.LocalPath() 652 c.Env = envForDir(c.Dir) 653 out, err := c.CombinedOutput() 654 if err != nil { 655 msg.Debug("Unable to find current branch for %s, error: %s", repo.Remote(), err) 656 return "" 657 } 658 return strings.TrimSpace(string(out)) 659 } 660 661 return "" 662 } 663 664 func envForDir(dir string) []string { 665 env := os.Environ() 666 return mergeEnvLists([]string{"PWD=" + dir}, env) 667 } 668 669 func mergeEnvLists(in, out []string) []string { 670 NextVar: 671 for _, inkv := range in { 672 k := strings.SplitAfterN(inkv, "=", 2)[0] 673 for i, outkv := range out { 674 if strings.HasPrefix(outkv, k) { 675 out[i] = inkv 676 continue NextVar 677 } 678 } 679 out = append(out, inkv) 680 } 681 return out 682 }