github.com/golang/dep@v0.5.4/gps/vcs_source.go (about) 1 // Copyright 2017 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package gps 6 7 import ( 8 "bytes" 9 "context" 10 "fmt" 11 "os" 12 "path/filepath" 13 "regexp" 14 "strings" 15 16 "github.com/Masterminds/semver" 17 "github.com/golang/dep/gps/pkgtree" 18 "github.com/golang/dep/internal/fs" 19 "github.com/pkg/errors" 20 ) 21 22 type baseVCSSource struct { 23 repo ctxRepo 24 } 25 26 func (bs *baseVCSSource) sourceType() string { 27 return string(bs.repo.Vcs()) 28 } 29 30 func (bs *baseVCSSource) existsLocally(ctx context.Context) bool { 31 return bs.repo.CheckLocal() 32 } 33 34 func (bs *baseVCSSource) existsUpstream(ctx context.Context) bool { 35 return bs.repo.Ping() 36 } 37 38 func (*baseVCSSource) existsCallsListVersions() bool { 39 return false 40 } 41 42 func (*baseVCSSource) listVersionsRequiresLocal() bool { 43 return false 44 } 45 46 func (bs *baseVCSSource) upstreamURL() string { 47 return bs.repo.Remote() 48 } 49 50 func (bs *baseVCSSource) disambiguateRevision(ctx context.Context, r Revision) (Revision, error) { 51 ci, err := bs.repo.CommitInfo(string(r)) 52 if err != nil { 53 return "", err 54 } 55 return Revision(ci.Commit), nil 56 } 57 58 func (bs *baseVCSSource) getManifestAndLock(ctx context.Context, pr ProjectRoot, r Revision, an ProjectAnalyzer) (Manifest, Lock, error) { 59 err := bs.repo.updateVersion(ctx, r.String()) 60 if err != nil { 61 return nil, nil, unwrapVcsErr(err) 62 } 63 64 m, l, err := an.DeriveManifestAndLock(bs.repo.LocalPath(), pr) 65 if err != nil { 66 return nil, nil, err 67 } 68 69 if l != nil && l != Lock(nil) { 70 l = prepLock(l) 71 } 72 73 return prepManifest(m), l, nil 74 } 75 76 func (bs *baseVCSSource) revisionPresentIn(r Revision) (bool, error) { 77 return bs.repo.IsReference(string(r)), nil 78 } 79 80 // initLocal clones/checks out the upstream repository to disk for the first 81 // time. 82 func (bs *baseVCSSource) initLocal(ctx context.Context) error { 83 err := bs.repo.get(ctx) 84 85 if err != nil { 86 return unwrapVcsErr(err) 87 } 88 return nil 89 } 90 91 // updateLocal ensures the local data (versions and code) we have about the 92 // source is fully up to date with that of the canonical upstream source. 93 func (bs *baseVCSSource) updateLocal(ctx context.Context) error { 94 err := bs.repo.fetch(ctx) 95 if err == nil { 96 return nil 97 } 98 99 ec, ok := bs.repo.(ensureCleaner) 100 if !ok { 101 return err 102 } 103 104 if err := ec.ensureClean(ctx); err != nil { 105 return unwrapVcsErr(err) 106 } 107 108 if err := bs.repo.fetch(ctx); err != nil { 109 return unwrapVcsErr(err) 110 } 111 return nil 112 } 113 114 func (bs *baseVCSSource) maybeClean(ctx context.Context) error { 115 ec, ok := bs.repo.(ensureCleaner) 116 if !ok { 117 return nil 118 } 119 120 if err := ec.ensureClean(ctx); err != nil { 121 return unwrapVcsErr(err) 122 } 123 return nil 124 } 125 126 func (bs *baseVCSSource) listPackages(ctx context.Context, pr ProjectRoot, r Revision) (ptree pkgtree.PackageTree, err error) { 127 err = bs.repo.updateVersion(ctx, r.String()) 128 129 if err != nil { 130 err = unwrapVcsErr(err) 131 } else { 132 ptree, err = pkgtree.ListPackages(bs.repo.LocalPath(), string(pr)) 133 } 134 135 return 136 } 137 138 func (bs *baseVCSSource) exportRevisionTo(ctx context.Context, r Revision, to string) error { 139 // Only make the parent dir, as CopyDir will balk on trying to write to an 140 // empty but existing dir. 141 if err := os.MkdirAll(filepath.Dir(to), 0777); err != nil { 142 return err 143 } 144 145 if err := bs.repo.updateVersion(ctx, r.String()); err != nil { 146 return unwrapVcsErr(err) 147 } 148 149 return fs.CopyDir(bs.repo.LocalPath(), to) 150 } 151 152 var ( 153 gitHashRE = regexp.MustCompile(`^[a-f0-9]{40}$`) 154 ) 155 156 // gitSource is a generic git repository implementation that should work with 157 // all standard git remotes. 158 type gitSource struct { 159 baseVCSSource 160 } 161 162 func (s *gitSource) exportRevisionTo(ctx context.Context, rev Revision, to string) error { 163 r := s.repo 164 165 if err := os.MkdirAll(to, 0777); err != nil { 166 return err 167 } 168 169 // Back up original index 170 idx, bak := filepath.Join(r.LocalPath(), ".git", "index"), filepath.Join(r.LocalPath(), ".git", "origindex") 171 err := fs.RenameWithFallback(idx, bak) 172 if err != nil { 173 return err 174 } 175 176 // could have an err here...but it's hard to imagine how? 177 defer fs.RenameWithFallback(bak, idx) 178 179 { 180 cmd := commandContext(ctx, "git", "read-tree", rev.String()) 181 cmd.SetDir(r.LocalPath()) 182 if out, err := cmd.CombinedOutput(); err != nil { 183 return errors.Wrap(err, string(out)) 184 } 185 } 186 187 // Ensure we have exactly one trailing slash 188 to = strings.TrimSuffix(to, string(os.PathSeparator)) + string(os.PathSeparator) 189 // Checkout from our temporary index to the desired target location on 190 // disk; now it's git's job to make it fast. 191 // 192 // Sadly, this approach *does* also write out vendor dirs. There doesn't 193 // appear to be a way to make checkout-index respect sparse checkout 194 // rules (-a supersedes it). The alternative is using plain checkout, 195 // though we have a bunch of housekeeping to do to set up, then tear 196 // down, the sparse checkout controls, as well as restore the original 197 // index and HEAD. 198 { 199 cmd := commandContext(ctx, "git", "checkout-index", "-a", "--prefix="+to) 200 cmd.SetDir(r.LocalPath()) 201 if out, err := cmd.CombinedOutput(); err != nil { 202 return errors.Wrap(err, string(out)) 203 } 204 } 205 206 return nil 207 } 208 209 func (s *gitSource) isValidHash(hash []byte) bool { 210 return gitHashRE.Match(hash) 211 } 212 213 func (*gitSource) existsCallsListVersions() bool { 214 return true 215 } 216 217 func (s *gitSource) listVersions(ctx context.Context) (vlist []PairedVersion, err error) { 218 r := s.repo 219 220 cmd := commandContext(ctx, "git", "ls-remote", r.Remote()) 221 // We want to invoke from a place where it's not possible for there to be a 222 // .git file instead of a .git directory, as git ls-remote will choke on the 223 // former and erroneously quit. However, we can't be sure that the repo 224 // exists on disk yet at this point; if it doesn't, then instead use the 225 // parent of the local path, as that's still likely a good bet. 226 if r.CheckLocal() { 227 cmd.SetDir(r.LocalPath()) 228 } else { 229 cmd.SetDir(filepath.Dir(r.LocalPath())) 230 } 231 // Ensure no prompting for PWs 232 cmd.SetEnv(append([]string{"GIT_ASKPASS=", "GIT_TERMINAL_PROMPT=0"}, os.Environ()...)) 233 out, err := cmd.CombinedOutput() 234 if err != nil { 235 return nil, errors.Wrap(err, string(out)) 236 } 237 238 all := bytes.Split(bytes.TrimSpace(out), []byte("\n")) 239 if len(all) == 1 && len(all[0]) == 0 { 240 return nil, fmt.Errorf("no data returned from ls-remote") 241 } 242 243 // Pull out the HEAD rev (it's always first) so we know what branches to 244 // mark as default. This is, perhaps, not the best way to glean this, but it 245 // was good enough for git itself until 1.8.5. Also, the alternative is 246 // sniffing data out of the pack protocol, which is a separate request, and 247 // also waaaay more than we want to do right now. 248 // 249 // The cost is that we could potentially have multiple branches marked as 250 // the default. If that does occur, a later check (again, emulating git 251 // <1.8.5 behavior) further narrows the failure mode by choosing master as 252 // the sole default branch if a) master exists and b) master is one of the 253 // branches marked as a default. 254 // 255 // This all reduces the failure mode to a very narrow range of 256 // circumstances. Nevertheless, if we do end up emitting multiple 257 // default branches, it is possible that a user could end up following a 258 // non-default branch, IF: 259 // 260 // * Multiple branches match the HEAD rev 261 // * None of them are master 262 // * The solver makes it into the branch list in the version queue 263 // * The user/tool has provided no constraint (so, anyConstraint) 264 // * A branch that is not actually the default, but happens to share the 265 // rev, is lexicographically less than the true default branch 266 // 267 // If all of those conditions are met, then the user would end up with an 268 // erroneous non-default branch in their lock file. 269 var headrev Revision 270 var onedef, multidef, defmaster bool 271 272 smap := make(map[string]int) 273 uniq := 0 274 vlist = make([]PairedVersion, len(all)) 275 for _, pair := range all { 276 var v PairedVersion 277 // Valid `git ls-remote` output should start with hash, be at least 278 // 45 chars long and 40th character should be '\t' 279 // 280 // See: https://github.com/golang/dep/pull/1160#issuecomment-328843519 281 if len(pair) < 45 || pair[40] != '\t' || !s.isValidHash(pair[:40]) { 282 continue 283 } 284 if string(pair[41:]) == "HEAD" { 285 // If HEAD is present, it's always first 286 headrev = Revision(pair[:40]) 287 } else if string(pair[46:51]) == "heads" { 288 rev := Revision(pair[:40]) 289 290 isdef := rev == headrev 291 n := string(pair[52:]) 292 if isdef { 293 if onedef { 294 multidef = true 295 } 296 onedef = true 297 if n == "master" { 298 defmaster = true 299 } 300 } 301 v = branchVersion{ 302 name: n, 303 isDefault: isdef, 304 }.Pair(rev).(PairedVersion) 305 306 vlist[uniq] = v 307 uniq++ 308 } else if string(pair[46:50]) == "tags" { 309 vstr := string(pair[51:]) 310 if strings.HasSuffix(vstr, "^{}") { 311 // If the suffix is there, then we *know* this is the rev of 312 // the underlying commit object that we actually want 313 vstr = strings.TrimSuffix(vstr, "^{}") 314 if i, ok := smap[vstr]; ok { 315 v = NewVersion(vstr).Pair(Revision(pair[:40])) 316 vlist[i] = v 317 continue 318 } 319 } else if _, ok := smap[vstr]; ok { 320 // Already saw the deref'd version of this tag, if one 321 // exists, so skip this. 322 continue 323 // Can only hit this branch if we somehow got the deref'd 324 // version first. Which should be impossible, but this 325 // covers us in case of weirdness, anyway. 326 } 327 v = NewVersion(vstr).Pair(Revision(pair[:40])) 328 smap[vstr] = uniq 329 vlist[uniq] = v 330 uniq++ 331 } 332 } 333 334 // Trim off excess from the slice 335 vlist = vlist[:uniq] 336 337 // There were multiple default branches, but one was master. So, go through 338 // and strip the default flag from all the non-master branches. 339 if multidef && defmaster { 340 for k, v := range vlist { 341 pv := v.(PairedVersion) 342 if bv, ok := pv.Unpair().(branchVersion); ok { 343 if bv.name != "master" && bv.isDefault { 344 bv.isDefault = false 345 vlist[k] = bv.Pair(pv.Revision()) 346 } 347 } 348 } 349 } 350 351 return 352 } 353 354 // gopkginSource is a specialized git source that performs additional filtering 355 // according to the input URL. 356 type gopkginSource struct { 357 gitSource 358 major uint64 359 unstable bool 360 // The aliased URL we report as being the one we talk to, even though we're 361 // actually talking directly to GitHub. 362 aliasURL string 363 } 364 365 func (s *gopkginSource) upstreamURL() string { 366 return s.aliasURL 367 } 368 369 func (s *gopkginSource) listVersions(ctx context.Context) ([]PairedVersion, error) { 370 ovlist, err := s.gitSource.listVersions(ctx) 371 if err != nil { 372 return nil, err 373 } 374 375 // Apply gopkg.in's filtering rules 376 vlist := make([]PairedVersion, len(ovlist)) 377 k := 0 378 var dbranch int // index of branch to be marked default 379 var bsv semver.Version 380 var defaultBranch PairedVersion 381 tryDefaultAsV0 := s.major == 0 382 for _, v := range ovlist { 383 // all git versions will always be paired 384 pv := v.(versionPair) 385 switch tv := pv.v.(type) { 386 case semVersion: 387 tryDefaultAsV0 = false 388 if tv.sv.Major() == s.major && !s.unstable { 389 vlist[k] = v 390 k++ 391 } 392 case branchVersion: 393 if tv.isDefault && defaultBranch == nil { 394 defaultBranch = pv 395 } 396 397 // The semver lib isn't exactly the same as gopkg.in's logic, but 398 // it's close enough that it's probably fine to use. We can be more 399 // exact if real problems crop up. 400 sv, err := semver.NewVersion(tv.name) 401 if err != nil { 402 continue 403 } 404 tryDefaultAsV0 = false 405 406 if sv.Major() != s.major { 407 // not the same major version as specified in the import path constraint 408 continue 409 } 410 411 // Gopkg.in has a special "-unstable" suffix which we need to handle 412 // separately. 413 if s.unstable != strings.HasSuffix(tv.name, gopkgUnstableSuffix) { 414 continue 415 } 416 417 // Turn off the default branch marker unconditionally; we can't know 418 // which one to mark as default until we've seen them all 419 tv.isDefault = false 420 // Figure out if this is the current leader for default branch 421 if bsv == (semver.Version{}) || bsv.LessThan(sv) { 422 bsv = sv 423 dbranch = k 424 } 425 pv.v = tv 426 vlist[k] = pv 427 k++ 428 } 429 // The switch skips plainVersions because they cannot possibly meet 430 // gopkg.in's requirements 431 } 432 433 vlist = vlist[:k] 434 if bsv != (semver.Version{}) { 435 dbv := vlist[dbranch].(versionPair) 436 vlist[dbranch] = branchVersion{ 437 name: dbv.v.(branchVersion).name, 438 isDefault: true, 439 }.Pair(dbv.r) 440 } 441 442 // Treat the default branch as v0 only when no other semver branches/tags exist 443 // See http://labix.org/gopkg.in#VersionZero 444 if tryDefaultAsV0 && defaultBranch != nil { 445 vlist = append(vlist, defaultBranch) 446 } 447 448 return vlist, nil 449 } 450 451 // bzrSource is a generic bzr repository implementation that should work with 452 // all standard bazaar remotes. 453 type bzrSource struct { 454 baseVCSSource 455 } 456 457 func (s *bzrSource) exportRevisionTo(ctx context.Context, rev Revision, to string) error { 458 if err := s.baseVCSSource.exportRevisionTo(ctx, rev, to); err != nil { 459 return err 460 } 461 462 return os.RemoveAll(filepath.Join(to, ".bzr")) 463 } 464 465 func (s *bzrSource) listVersionsRequiresLocal() bool { 466 return true 467 } 468 469 func (s *bzrSource) listVersions(ctx context.Context) ([]PairedVersion, error) { 470 r := s.repo 471 472 // Now, list all the tags 473 tagsCmd := commandContext(ctx, "bzr", "tags", "--show-ids", "-v") 474 tagsCmd.SetDir(r.LocalPath()) 475 out, err := tagsCmd.CombinedOutput() 476 if err != nil { 477 return nil, errors.Wrap(err, string(out)) 478 } 479 480 all := bytes.Split(bytes.TrimSpace(out), []byte("\n")) 481 482 viCmd := commandContext(ctx, "bzr", "version-info", "--custom", "--template={revision_id}", "--revision=branch:.") 483 viCmd.SetDir(r.LocalPath()) 484 branchrev, err := viCmd.CombinedOutput() 485 if err != nil { 486 return nil, errors.Wrap(err, string(branchrev)) 487 } 488 489 vlist := make([]PairedVersion, 0, len(all)+1) 490 491 // Now, all the tags. 492 for _, line := range all { 493 idx := bytes.IndexByte(line, 32) // space 494 v := NewVersion(string(line[:idx])) 495 r := Revision(bytes.TrimSpace(line[idx:])) 496 vlist = append(vlist, v.Pair(r)) 497 } 498 499 // Last, add the default branch, hardcoding the visual representation of it 500 // that bzr uses when operating in the workflow mode we're using. 501 v := newDefaultBranch("(default)") 502 vlist = append(vlist, v.Pair(Revision(string(branchrev)))) 503 504 return vlist, nil 505 } 506 507 func (s *bzrSource) disambiguateRevision(ctx context.Context, r Revision) (Revision, error) { 508 // If we used the default baseVCSSource behavior here, we would return the 509 // bazaar revision number, which is not a globally unique identifier - it is 510 // only unique within a branch. This is just the way that 511 // github.com/Masterminds/vcs chooses to handle bazaar. We want a 512 // disambiguated unique ID, though, so we need slightly different behavior: 513 // check whether r doesn't error when we try to look it up. If so, trust that 514 // it's a revision. 515 _, err := s.repo.CommitInfo(string(r)) 516 if err != nil { 517 return "", err 518 } 519 return r, nil 520 } 521 522 // hgSource is a generic hg repository implementation that should work with 523 // all standard mercurial servers. 524 type hgSource struct { 525 baseVCSSource 526 } 527 528 func (s *hgSource) exportRevisionTo(ctx context.Context, rev Revision, to string) error { 529 // TODO: use hg instead of the generic approach in 530 // baseVCSSource.exportRevisionTo to make it faster. 531 if err := s.baseVCSSource.exportRevisionTo(ctx, rev, to); err != nil { 532 return err 533 } 534 535 return os.RemoveAll(filepath.Join(to, ".hg")) 536 } 537 538 func (s *hgSource) listVersionsRequiresLocal() bool { 539 return true 540 } 541 542 func (s *hgSource) hgCmd(ctx context.Context, args ...string) cmd { 543 r := s.repo 544 cmd := commandContext(ctx, "hg", args...) 545 cmd.SetDir(r.LocalPath()) 546 // Let's make sure extensions don't interfere with our expectations 547 // regarding the output of commands. 548 cmd.Cmd.Env = append(cmd.Cmd.Env, "HGRCPATH=") 549 return cmd 550 } 551 552 func (s *hgSource) listVersions(ctx context.Context) ([]PairedVersion, error) { 553 var vlist []PairedVersion 554 555 // Now, list all the tags 556 tagsCmd := s.hgCmd(ctx, "tags", "--debug", "--verbose") 557 out, err := tagsCmd.CombinedOutput() 558 if err != nil { 559 return nil, errors.Wrap(err, string(out)) 560 } 561 562 all := bytes.Split(bytes.TrimSpace(out), []byte("\n")) 563 lbyt := []byte("local") 564 nulrev := []byte("0000000000000000000000000000000000000000") 565 for _, line := range all { 566 if bytes.Equal(lbyt, line[len(line)-len(lbyt):]) { 567 // Skip local tags 568 continue 569 } 570 571 // tip is magic, don't include it 572 if bytes.HasPrefix(line, []byte("tip")) { 573 continue 574 } 575 576 // Split on colon; this gets us the rev and the tag plus local revno 577 pair := bytes.Split(line, []byte(":")) 578 if bytes.Equal(nulrev, pair[1]) { 579 // null rev indicates this tag is marked for deletion 580 continue 581 } 582 583 idx := bytes.IndexByte(pair[0], 32) // space 584 v := NewVersion(string(pair[0][:idx])).Pair(Revision(pair[1])).(PairedVersion) 585 vlist = append(vlist, v) 586 } 587 588 // bookmarks next, because the presence of the magic @ bookmark has to 589 // determine how we handle the branches 590 var magicAt bool 591 bookmarksCmd := s.hgCmd(ctx, "bookmarks", "--debug") 592 out, err = bookmarksCmd.CombinedOutput() 593 if err != nil { 594 // better nothing than partial and misleading 595 return nil, errors.Wrap(err, string(out)) 596 } 597 598 out = bytes.TrimSpace(out) 599 if !bytes.Equal(out, []byte("no bookmarks set")) { 600 all = bytes.Split(out, []byte("\n")) 601 for _, line := range all { 602 // Trim leading spaces, and * marker if present 603 line = bytes.TrimLeft(line, " *") 604 pair := bytes.Split(line, []byte(":")) 605 // if this doesn't split exactly once, we have something weird 606 if len(pair) != 2 { 607 continue 608 } 609 610 // Split on colon; this gets us the rev and the branch plus local revno 611 idx := bytes.IndexByte(pair[0], 32) // space 612 // if it's the magic @ marker, make that the default branch 613 str := string(pair[0][:idx]) 614 var v PairedVersion 615 if str == "@" { 616 magicAt = true 617 v = newDefaultBranch(str).Pair(Revision(pair[1])).(PairedVersion) 618 } else { 619 v = NewBranch(str).Pair(Revision(pair[1])).(PairedVersion) 620 } 621 vlist = append(vlist, v) 622 } 623 } 624 625 cmd := s.hgCmd(ctx, "branches", "-c", "--debug") 626 out, err = cmd.CombinedOutput() 627 if err != nil { 628 // better nothing than partial and misleading 629 return nil, errors.Wrap(err, string(out)) 630 } 631 632 all = bytes.Split(bytes.TrimSpace(out), []byte("\n")) 633 for _, line := range all { 634 // Trim inactive and closed suffixes, if present; we represent these 635 // anyway 636 line = bytes.TrimSuffix(line, []byte(" (inactive)")) 637 line = bytes.TrimSuffix(line, []byte(" (closed)")) 638 639 // Split on colon; this gets us the rev and the branch plus local revno 640 pair := bytes.Split(line, []byte(":")) 641 idx := bytes.IndexByte(pair[0], 32) // space 642 str := string(pair[0][:idx]) 643 // if there was no magic @ bookmark, and this is mercurial's magic 644 // "default" branch, then mark it as default branch 645 var v PairedVersion 646 if !magicAt && str == "default" { 647 v = newDefaultBranch(str).Pair(Revision(pair[1])).(PairedVersion) 648 } else { 649 v = NewBranch(str).Pair(Revision(pair[1])).(PairedVersion) 650 } 651 vlist = append(vlist, v) 652 } 653 654 return vlist, nil 655 }