github.com/gophergala2016/cmd-go-js@v0.0.0-20160421080227-24a7748a7d62/cmd/go/vcs.go (about) 1 // Copyright 2012 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package main 6 7 import ( 8 "bytes" 9 "encoding/json" 10 "errors" 11 "fmt" 12 "log" 13 "net/url" 14 "os" 15 "os/exec" 16 "path/filepath" 17 "regexp" 18 "strings" 19 "sync" 20 21 "github.com/gophergala2016/cmd-go-js/internal/singleflight" 22 ) 23 24 // A vcsCmd describes how to use a version control system 25 // like Mercurial, Git, or Subversion. 26 type vcsCmd struct { 27 name string 28 cmd string // name of binary to invoke command 29 30 createCmd []string // commands to download a fresh copy of a repository 31 downloadCmd []string // commands to download updates into an existing repository 32 33 tagCmd []tagCmd // commands to list tags 34 tagLookupCmd []tagCmd // commands to lookup tags before running tagSyncCmd 35 tagSyncCmd []string // commands to sync to specific tag 36 tagSyncDefault []string // commands to sync to default tag 37 38 scheme []string 39 pingCmd string 40 41 remoteRepo func(v *vcsCmd, rootDir string) (remoteRepo string, err error) 42 resolveRepo func(v *vcsCmd, rootDir, remoteRepo string) (realRepo string, err error) 43 } 44 45 var isSecureScheme = map[string]bool{ 46 "https": true, 47 "git+ssh": true, 48 "bzr+ssh": true, 49 "svn+ssh": true, 50 "ssh": true, 51 } 52 53 func (v *vcsCmd) isSecure(repo string) bool { 54 u, err := url.Parse(repo) 55 if err != nil { 56 // If repo is not a URL, it's not secure. 57 return false 58 } 59 return isSecureScheme[u.Scheme] 60 } 61 62 // A tagCmd describes a command to list available tags 63 // that can be passed to tagSyncCmd. 64 type tagCmd struct { 65 cmd string // command to list tags 66 pattern string // regexp to extract tags from list 67 } 68 69 // vcsList lists the known version control systems 70 var vcsList = []*vcsCmd{ 71 vcsHg, 72 vcsGit, 73 vcsSvn, 74 vcsBzr, 75 } 76 77 // vcsByCmd returns the version control system for the given 78 // command name (hg, git, svn, bzr). 79 func vcsByCmd(cmd string) *vcsCmd { 80 for _, vcs := range vcsList { 81 if vcs.cmd == cmd { 82 return vcs 83 } 84 } 85 return nil 86 } 87 88 // vcsHg describes how to use Mercurial. 89 var vcsHg = &vcsCmd{ 90 name: "Mercurial", 91 cmd: "hg", 92 93 createCmd: []string{"clone -U {repo} {dir}"}, 94 downloadCmd: []string{"pull"}, 95 96 // We allow both tag and branch names as 'tags' 97 // for selecting a version. This lets people have 98 // a go.release.r60 branch and a go1 branch 99 // and make changes in both, without constantly 100 // editing .hgtags. 101 tagCmd: []tagCmd{ 102 {"tags", `^(\S+)`}, 103 {"branches", `^(\S+)`}, 104 }, 105 tagSyncCmd: []string{"update -r {tag}"}, 106 tagSyncDefault: []string{"update default"}, 107 108 scheme: []string{"https", "http", "ssh"}, 109 pingCmd: "identify {scheme}://{repo}", 110 remoteRepo: hgRemoteRepo, 111 } 112 113 func hgRemoteRepo(vcsHg *vcsCmd, rootDir string) (remoteRepo string, err error) { 114 out, err := vcsHg.runOutput(rootDir, "paths default") 115 if err != nil { 116 return "", err 117 } 118 return strings.TrimSpace(string(out)), nil 119 } 120 121 // vcsGit describes how to use Git. 122 var vcsGit = &vcsCmd{ 123 name: "Git", 124 cmd: "git", 125 126 createCmd: []string{"clone {repo} {dir}", "--git-dir={dir}/.git submodule update --init --recursive"}, 127 downloadCmd: []string{"pull --ff-only", "submodule update --init --recursive"}, 128 129 tagCmd: []tagCmd{ 130 // tags/xxx matches a git tag named xxx 131 // origin/xxx matches a git branch named xxx on the default remote repository 132 {"show-ref", `(?:tags|origin)/(\S+)$`}, 133 }, 134 tagLookupCmd: []tagCmd{ 135 {"show-ref tags/{tag} origin/{tag}", `((?:tags|origin)/\S+)$`}, 136 }, 137 tagSyncCmd: []string{"checkout {tag}", "submodule update --init --recursive"}, 138 // both createCmd and downloadCmd update the working dir. 139 // No need to do more here. We used to 'checkout master' 140 // but that doesn't work if the default branch is not named master. 141 // See golang.org/issue/9032. 142 tagSyncDefault: []string{"checkout master", "submodule update --init --recursive"}, 143 144 scheme: []string{"git", "https", "http", "git+ssh", "ssh"}, 145 pingCmd: "ls-remote {scheme}://{repo}", 146 remoteRepo: gitRemoteRepo, 147 } 148 149 // scpSyntaxRe matches the SCP-like addresses used by Git to access 150 // repositories by SSH. 151 var scpSyntaxRe = regexp.MustCompile(`^([a-zA-Z0-9_]+)@([a-zA-Z0-9._-]+):(.*)$`) 152 153 func gitRemoteRepo(vcsGit *vcsCmd, rootDir string) (remoteRepo string, err error) { 154 cmd := "config remote.origin.url" 155 errParse := errors.New("unable to parse output of git " + cmd) 156 errRemoteOriginNotFound := errors.New("remote origin not found") 157 outb, err := vcsGit.run1(rootDir, cmd, nil, false) 158 if err != nil { 159 // if it doesn't output any message, it means the config argument is correct, 160 // but the config value itself doesn't exist 161 if outb != nil && len(outb) == 0 { 162 return "", errRemoteOriginNotFound 163 } 164 return "", err 165 } 166 out := strings.TrimSpace(string(outb)) 167 168 var repoURL *url.URL 169 if m := scpSyntaxRe.FindStringSubmatch(out); m != nil { 170 // Match SCP-like syntax and convert it to a URL. 171 // Eg, "git@github.com:user/repo" becomes 172 // "ssh://git@github.com/user/repo". 173 repoURL = &url.URL{ 174 Scheme: "ssh", 175 User: url.User(m[1]), 176 Host: m[2], 177 RawPath: m[3], 178 } 179 } else { 180 repoURL, err = url.Parse(out) 181 if err != nil { 182 return "", err 183 } 184 } 185 186 // Iterate over insecure schemes too, because this function simply 187 // reports the state of the repo. If we can't see insecure schemes then 188 // we can't report the actual repo URL. 189 for _, s := range vcsGit.scheme { 190 if repoURL.Scheme == s { 191 return repoURL.String(), nil 192 } 193 } 194 return "", errParse 195 } 196 197 // vcsBzr describes how to use Bazaar. 198 var vcsBzr = &vcsCmd{ 199 name: "Bazaar", 200 cmd: "bzr", 201 202 createCmd: []string{"branch {repo} {dir}"}, 203 204 // Without --overwrite bzr will not pull tags that changed. 205 // Replace by --overwrite-tags after http://pad.lv/681792 goes in. 206 downloadCmd: []string{"pull --overwrite"}, 207 208 tagCmd: []tagCmd{{"tags", `^(\S+)`}}, 209 tagSyncCmd: []string{"update -r {tag}"}, 210 tagSyncDefault: []string{"update -r revno:-1"}, 211 212 scheme: []string{"https", "http", "bzr", "bzr+ssh"}, 213 pingCmd: "info {scheme}://{repo}", 214 remoteRepo: bzrRemoteRepo, 215 resolveRepo: bzrResolveRepo, 216 } 217 218 func bzrRemoteRepo(vcsBzr *vcsCmd, rootDir string) (remoteRepo string, err error) { 219 outb, err := vcsBzr.runOutput(rootDir, "config parent_location") 220 if err != nil { 221 return "", err 222 } 223 return strings.TrimSpace(string(outb)), nil 224 } 225 226 func bzrResolveRepo(vcsBzr *vcsCmd, rootDir, remoteRepo string) (realRepo string, err error) { 227 outb, err := vcsBzr.runOutput(rootDir, "info "+remoteRepo) 228 if err != nil { 229 return "", err 230 } 231 out := string(outb) 232 233 // Expect: 234 // ... 235 // (branch root|repository branch): <URL> 236 // ... 237 238 found := false 239 for _, prefix := range []string{"\n branch root: ", "\n repository branch: "} { 240 i := strings.Index(out, prefix) 241 if i >= 0 { 242 out = out[i+len(prefix):] 243 found = true 244 break 245 } 246 } 247 if !found { 248 return "", fmt.Errorf("unable to parse output of bzr info") 249 } 250 251 i := strings.Index(out, "\n") 252 if i < 0 { 253 return "", fmt.Errorf("unable to parse output of bzr info") 254 } 255 out = out[:i] 256 return strings.TrimSpace(string(out)), nil 257 } 258 259 // vcsSvn describes how to use Subversion. 260 var vcsSvn = &vcsCmd{ 261 name: "Subversion", 262 cmd: "svn", 263 264 createCmd: []string{"checkout {repo} {dir}"}, 265 downloadCmd: []string{"update"}, 266 267 // There is no tag command in subversion. 268 // The branch information is all in the path names. 269 270 scheme: []string{"https", "http", "svn", "svn+ssh"}, 271 pingCmd: "info {scheme}://{repo}", 272 remoteRepo: svnRemoteRepo, 273 } 274 275 func svnRemoteRepo(vcsSvn *vcsCmd, rootDir string) (remoteRepo string, err error) { 276 outb, err := vcsSvn.runOutput(rootDir, "info") 277 if err != nil { 278 return "", err 279 } 280 out := string(outb) 281 282 // Expect: 283 // ... 284 // Repository Root: <URL> 285 // ... 286 287 i := strings.Index(out, "\nRepository Root: ") 288 if i < 0 { 289 return "", fmt.Errorf("unable to parse output of svn info") 290 } 291 out = out[i+len("\nRepository Root: "):] 292 i = strings.Index(out, "\n") 293 if i < 0 { 294 return "", fmt.Errorf("unable to parse output of svn info") 295 } 296 out = out[:i] 297 return strings.TrimSpace(string(out)), nil 298 } 299 300 func (v *vcsCmd) String() string { 301 return v.name 302 } 303 304 // run runs the command line cmd in the given directory. 305 // keyval is a list of key, value pairs. run expands 306 // instances of {key} in cmd into value, but only after 307 // splitting cmd into individual arguments. 308 // If an error occurs, run prints the command line and the 309 // command's combined stdout+stderr to standard error. 310 // Otherwise run discards the command's output. 311 func (v *vcsCmd) run(dir string, cmd string, keyval ...string) error { 312 _, err := v.run1(dir, cmd, keyval, true) 313 return err 314 } 315 316 // runVerboseOnly is like run but only generates error output to standard error in verbose mode. 317 func (v *vcsCmd) runVerboseOnly(dir string, cmd string, keyval ...string) error { 318 _, err := v.run1(dir, cmd, keyval, false) 319 return err 320 } 321 322 // runOutput is like run but returns the output of the command. 323 func (v *vcsCmd) runOutput(dir string, cmd string, keyval ...string) ([]byte, error) { 324 return v.run1(dir, cmd, keyval, true) 325 } 326 327 // run1 is the generalized implementation of run and runOutput. 328 func (v *vcsCmd) run1(dir string, cmdline string, keyval []string, verbose bool) ([]byte, error) { 329 m := make(map[string]string) 330 for i := 0; i < len(keyval); i += 2 { 331 m[keyval[i]] = keyval[i+1] 332 } 333 args := strings.Fields(cmdline) 334 for i, arg := range args { 335 args[i] = expand(m, arg) 336 } 337 338 _, err := exec.LookPath(v.cmd) 339 if err != nil { 340 fmt.Fprintf(os.Stderr, 341 "go: missing %s command. See https://golang.org/s/gogetcmd\n", 342 v.name) 343 return nil, err 344 } 345 346 cmd := exec.Command(v.cmd, args...) 347 cmd.Dir = dir 348 cmd.Env = envForDir(cmd.Dir, os.Environ()) 349 if buildX { 350 fmt.Printf("cd %s\n", dir) 351 fmt.Printf("%s %s\n", v.cmd, strings.Join(args, " ")) 352 } 353 var buf bytes.Buffer 354 cmd.Stdout = &buf 355 cmd.Stderr = &buf 356 err = cmd.Run() 357 out := buf.Bytes() 358 if err != nil { 359 if verbose || buildV { 360 fmt.Fprintf(os.Stderr, "# cd %s; %s %s\n", dir, v.cmd, strings.Join(args, " ")) 361 os.Stderr.Write(out) 362 } 363 return out, err 364 } 365 return out, nil 366 } 367 368 // ping pings to determine scheme to use. 369 func (v *vcsCmd) ping(scheme, repo string) error { 370 return v.runVerboseOnly(".", v.pingCmd, "scheme", scheme, "repo", repo) 371 } 372 373 // create creates a new copy of repo in dir. 374 // The parent of dir must exist; dir must not. 375 func (v *vcsCmd) create(dir, repo string) error { 376 for _, cmd := range v.createCmd { 377 if !go15VendorExperiment && strings.Contains(cmd, "submodule") { 378 continue 379 } 380 if err := v.run(".", cmd, "dir", dir, "repo", repo); err != nil { 381 return err 382 } 383 } 384 return nil 385 } 386 387 // download downloads any new changes for the repo in dir. 388 func (v *vcsCmd) download(dir string) error { 389 if err := v.fixDetachedHead(dir); err != nil { 390 return err 391 } 392 for _, cmd := range v.downloadCmd { 393 if !go15VendorExperiment && strings.Contains(cmd, "submodule") { 394 continue 395 } 396 if err := v.run(dir, cmd); err != nil { 397 return err 398 } 399 } 400 return nil 401 } 402 403 // fixDetachedHead switches a Git repository in dir from a detached head to the master branch. 404 // Go versions before 1.2 downloaded Git repositories in an unfortunate way 405 // that resulted in the working tree state being on a detached head. 406 // That meant the repository was not usable for normal Git operations. 407 // Go 1.2 fixed that, but we can't pull into a detached head, so if this is 408 // a Git repository we check for being on a detached head and switch to the 409 // real branch, almost always called "master". 410 // TODO(dsymonds): Consider removing this for Go 1.3. 411 func (v *vcsCmd) fixDetachedHead(dir string) error { 412 if v != vcsGit { 413 return nil 414 } 415 416 // "git symbolic-ref HEAD" succeeds iff we are not on a detached head. 417 if err := v.runVerboseOnly(dir, "symbolic-ref HEAD"); err == nil { 418 // not on a detached head 419 return nil 420 } 421 if buildV { 422 log.Printf("%s on detached head; repairing", dir) 423 } 424 return v.run(dir, "checkout master") 425 } 426 427 // tags returns the list of available tags for the repo in dir. 428 func (v *vcsCmd) tags(dir string) ([]string, error) { 429 var tags []string 430 for _, tc := range v.tagCmd { 431 out, err := v.runOutput(dir, tc.cmd) 432 if err != nil { 433 return nil, err 434 } 435 re := regexp.MustCompile(`(?m-s)` + tc.pattern) 436 for _, m := range re.FindAllStringSubmatch(string(out), -1) { 437 tags = append(tags, m[1]) 438 } 439 } 440 return tags, nil 441 } 442 443 // tagSync syncs the repo in dir to the named tag, 444 // which either is a tag returned by tags or is v.tagDefault. 445 func (v *vcsCmd) tagSync(dir, tag string) error { 446 if v.tagSyncCmd == nil { 447 return nil 448 } 449 if tag != "" { 450 for _, tc := range v.tagLookupCmd { 451 out, err := v.runOutput(dir, tc.cmd, "tag", tag) 452 if err != nil { 453 return err 454 } 455 re := regexp.MustCompile(`(?m-s)` + tc.pattern) 456 m := re.FindStringSubmatch(string(out)) 457 if len(m) > 1 { 458 tag = m[1] 459 break 460 } 461 } 462 } 463 464 if tag == "" && v.tagSyncDefault != nil { 465 for _, cmd := range v.tagSyncDefault { 466 if !go15VendorExperiment && strings.Contains(cmd, "submodule") { 467 continue 468 } 469 if err := v.run(dir, cmd); err != nil { 470 return err 471 } 472 } 473 return nil 474 } 475 476 for _, cmd := range v.tagSyncCmd { 477 if !go15VendorExperiment && strings.Contains(cmd, "submodule") { 478 continue 479 } 480 if err := v.run(dir, cmd, "tag", tag); err != nil { 481 return err 482 } 483 } 484 return nil 485 } 486 487 // A vcsPath describes how to convert an import path into a 488 // version control system and repository name. 489 type vcsPath struct { 490 prefix string // prefix this description applies to 491 re string // pattern for import path 492 repo string // repository to use (expand with match of re) 493 vcs string // version control system to use (expand with match of re) 494 check func(match map[string]string) error // additional checks 495 ping bool // ping for scheme to use to download repo 496 497 regexp *regexp.Regexp // cached compiled form of re 498 } 499 500 // vcsForDir inspects dir and its parents to determine the 501 // version control system and code repository to use. 502 // On return, root is the import path 503 // corresponding to the root of the repository 504 // (thus root is a prefix of importPath). 505 func vcsForDir(p *Package) (vcs *vcsCmd, root string, err error) { 506 // Clean and double-check that dir is in (a subdirectory of) srcRoot. 507 dir := filepath.Clean(p.Dir) 508 srcRoot := filepath.Clean(p.build.SrcRoot) 509 if len(dir) <= len(srcRoot) || dir[len(srcRoot)] != filepath.Separator { 510 return nil, "", fmt.Errorf("directory %q is outside source root %q", dir, srcRoot) 511 } 512 513 origDir := dir 514 for len(dir) > len(srcRoot) { 515 for _, vcs := range vcsList { 516 if fi, err := os.Stat(filepath.Join(dir, "."+vcs.cmd)); err == nil && fi.IsDir() { 517 return vcs, dir[len(srcRoot)+1:], nil 518 } 519 } 520 521 // Move to parent. 522 ndir := filepath.Dir(dir) 523 if len(ndir) >= len(dir) { 524 // Shouldn't happen, but just in case, stop. 525 break 526 } 527 dir = ndir 528 } 529 530 return nil, "", fmt.Errorf("directory %q is not using a known version control system", origDir) 531 } 532 533 // repoRoot represents a version control system, a repo, and a root of 534 // where to put it on disk. 535 type repoRoot struct { 536 vcs *vcsCmd 537 538 // repo is the repository URL, including scheme 539 repo string 540 541 // root is the import path corresponding to the root of the 542 // repository 543 root string 544 } 545 546 var httpPrefixRE = regexp.MustCompile(`^https?:`) 547 548 // securityMode specifies whether a function should make network 549 // calls using insecure transports (eg, plain text HTTP). 550 // The zero value is "secure". 551 type securityMode int 552 553 const ( 554 secure securityMode = iota 555 insecure 556 ) 557 558 // repoRootForImportPath analyzes importPath to determine the 559 // version control system, and code repository to use. 560 func repoRootForImportPath(importPath string, security securityMode) (*repoRoot, error) { 561 rr, err := repoRootFromVCSPaths(importPath, "", security, vcsPaths) 562 if err == errUnknownSite { 563 // If there are wildcards, look up the thing before the wildcard, 564 // hoping it applies to the wildcarded parts too. 565 // This makes 'go get rsc.io/pdf/...' work in a fresh GOPATH. 566 lookup := strings.TrimSuffix(importPath, "/...") 567 if i := strings.Index(lookup, "/.../"); i >= 0 { 568 lookup = lookup[:i] 569 } 570 rr, err = repoRootForImportDynamic(lookup, security) 571 572 // repoRootForImportDynamic returns error detail 573 // that is irrelevant if the user didn't intend to use a 574 // dynamic import in the first place. 575 // Squelch it. 576 if err != nil { 577 if buildV { 578 log.Printf("import %q: %v", importPath, err) 579 } 580 err = fmt.Errorf("unrecognized import path %q", importPath) 581 } 582 } 583 if err != nil { 584 rr1, err1 := repoRootFromVCSPaths(importPath, "", security, vcsPathsAfterDynamic) 585 if err1 == nil { 586 rr = rr1 587 err = nil 588 } 589 } 590 591 if err == nil && strings.Contains(importPath, "...") && strings.Contains(rr.root, "...") { 592 // Do not allow wildcards in the repo root. 593 rr = nil 594 err = fmt.Errorf("cannot expand ... in %q", importPath) 595 } 596 return rr, err 597 } 598 599 var errUnknownSite = errors.New("dynamic lookup required to find mapping") 600 601 // repoRootFromVCSPaths attempts to map importPath to a repoRoot 602 // using the mappings defined in vcsPaths. 603 // If scheme is non-empty, that scheme is forced. 604 func repoRootFromVCSPaths(importPath, scheme string, security securityMode, vcsPaths []*vcsPath) (*repoRoot, error) { 605 // A common error is to use https://packagepath because that's what 606 // hg and git require. Diagnose this helpfully. 607 if loc := httpPrefixRE.FindStringIndex(importPath); loc != nil { 608 // The importPath has been cleaned, so has only one slash. The pattern 609 // ignores the slashes; the error message puts them back on the RHS at least. 610 return nil, fmt.Errorf("%q not allowed in import path", importPath[loc[0]:loc[1]]+"//") 611 } 612 for _, srv := range vcsPaths { 613 if !strings.HasPrefix(importPath, srv.prefix) { 614 continue 615 } 616 m := srv.regexp.FindStringSubmatch(importPath) 617 if m == nil { 618 if srv.prefix != "" { 619 return nil, fmt.Errorf("invalid %s import path %q", srv.prefix, importPath) 620 } 621 continue 622 } 623 624 // Build map of named subexpression matches for expand. 625 match := map[string]string{ 626 "prefix": srv.prefix, 627 "import": importPath, 628 } 629 for i, name := range srv.regexp.SubexpNames() { 630 if name != "" && match[name] == "" { 631 match[name] = m[i] 632 } 633 } 634 if srv.vcs != "" { 635 match["vcs"] = expand(match, srv.vcs) 636 } 637 if srv.repo != "" { 638 match["repo"] = expand(match, srv.repo) 639 } 640 if srv.check != nil { 641 if err := srv.check(match); err != nil { 642 return nil, err 643 } 644 } 645 vcs := vcsByCmd(match["vcs"]) 646 if vcs == nil { 647 return nil, fmt.Errorf("unknown version control system %q", match["vcs"]) 648 } 649 if srv.ping { 650 if scheme != "" { 651 match["repo"] = scheme + "://" + match["repo"] 652 } else { 653 for _, scheme := range vcs.scheme { 654 if security == secure && !isSecureScheme[scheme] { 655 continue 656 } 657 if vcs.ping(scheme, match["repo"]) == nil { 658 match["repo"] = scheme + "://" + match["repo"] 659 break 660 } 661 } 662 } 663 } 664 rr := &repoRoot{ 665 vcs: vcs, 666 repo: match["repo"], 667 root: match["root"], 668 } 669 return rr, nil 670 } 671 return nil, errUnknownSite 672 } 673 674 // repoRootForImportDynamic finds a *repoRoot for a custom domain that's not 675 // statically known by repoRootForImportPathStatic. 676 // 677 // This handles custom import paths like "name.tld/pkg/foo" or just "name.tld". 678 func repoRootForImportDynamic(importPath string, security securityMode) (*repoRoot, error) { 679 slash := strings.Index(importPath, "/") 680 if slash < 0 { 681 slash = len(importPath) 682 } 683 host := importPath[:slash] 684 if !strings.Contains(host, ".") { 685 return nil, errors.New("import path does not begin with hostname") 686 } 687 urlStr, body, err := httpsOrHTTP(importPath, security) 688 if err != nil { 689 msg := "https fetch: %v" 690 if security == insecure { 691 msg = "http/" + msg 692 } 693 return nil, fmt.Errorf(msg, err) 694 } 695 defer body.Close() 696 imports, err := parseMetaGoImports(body) 697 if err != nil { 698 return nil, fmt.Errorf("parsing %s: %v", importPath, err) 699 } 700 // Find the matched meta import. 701 mmi, err := matchGoImport(imports, importPath) 702 if err != nil { 703 if err != errNoMatch { 704 return nil, fmt.Errorf("parse %s: %v", urlStr, err) 705 } 706 return nil, fmt.Errorf("parse %s: no go-import meta tags", urlStr) 707 } 708 if buildV { 709 log.Printf("get %q: found meta tag %#v at %s", importPath, mmi, urlStr) 710 } 711 // If the import was "uni.edu/bob/project", which said the 712 // prefix was "uni.edu" and the RepoRoot was "evilroot.com", 713 // make sure we don't trust Bob and check out evilroot.com to 714 // "uni.edu" yet (possibly overwriting/preempting another 715 // non-evil student). Instead, first verify the root and see 716 // if it matches Bob's claim. 717 if mmi.Prefix != importPath { 718 if buildV { 719 log.Printf("get %q: verifying non-authoritative meta tag", importPath) 720 } 721 urlStr0 := urlStr 722 var imports []metaImport 723 urlStr, imports, err = metaImportsForPrefix(mmi.Prefix, security) 724 if err != nil { 725 return nil, err 726 } 727 metaImport2, err := matchGoImport(imports, importPath) 728 if err != nil || mmi != metaImport2 { 729 return nil, fmt.Errorf("%s and %s disagree about go-import for %s", urlStr0, urlStr, mmi.Prefix) 730 } 731 } 732 733 if !strings.Contains(mmi.RepoRoot, "://") { 734 return nil, fmt.Errorf("%s: invalid repo root %q; no scheme", urlStr, mmi.RepoRoot) 735 } 736 rr := &repoRoot{ 737 vcs: vcsByCmd(mmi.VCS), 738 repo: mmi.RepoRoot, 739 root: mmi.Prefix, 740 } 741 if rr.vcs == nil { 742 return nil, fmt.Errorf("%s: unknown vcs %q", urlStr, mmi.VCS) 743 } 744 return rr, nil 745 } 746 747 var fetchGroup singleflight.Group 748 var ( 749 fetchCacheMu sync.Mutex 750 fetchCache = map[string]fetchResult{} // key is metaImportsForPrefix's importPrefix 751 ) 752 753 // metaImportsForPrefix takes a package's root import path as declared in a <meta> tag 754 // and returns its HTML discovery URL and the parsed metaImport lines 755 // found on the page. 756 // 757 // The importPath is of the form "golang.org/x/tools". 758 // It is an error if no imports are found. 759 // urlStr will still be valid if err != nil. 760 // The returned urlStr will be of the form "https://golang.org/x/tools?go-get=1" 761 func metaImportsForPrefix(importPrefix string, security securityMode) (urlStr string, imports []metaImport, err error) { 762 setCache := func(res fetchResult) (fetchResult, error) { 763 fetchCacheMu.Lock() 764 defer fetchCacheMu.Unlock() 765 fetchCache[importPrefix] = res 766 return res, nil 767 } 768 769 resi, _, _ := fetchGroup.Do(importPrefix, func() (resi interface{}, err error) { 770 fetchCacheMu.Lock() 771 if res, ok := fetchCache[importPrefix]; ok { 772 fetchCacheMu.Unlock() 773 return res, nil 774 } 775 fetchCacheMu.Unlock() 776 777 urlStr, body, err := httpsOrHTTP(importPrefix, security) 778 if err != nil { 779 return setCache(fetchResult{urlStr: urlStr, err: fmt.Errorf("fetch %s: %v", urlStr, err)}) 780 } 781 imports, err := parseMetaGoImports(body) 782 if err != nil { 783 return setCache(fetchResult{urlStr: urlStr, err: fmt.Errorf("parsing %s: %v", urlStr, err)}) 784 } 785 if len(imports) == 0 { 786 err = fmt.Errorf("fetch %s: no go-import meta tag", urlStr) 787 } 788 return setCache(fetchResult{urlStr: urlStr, imports: imports, err: err}) 789 }) 790 res := resi.(fetchResult) 791 return res.urlStr, res.imports, res.err 792 } 793 794 type fetchResult struct { 795 urlStr string // e.g. "https://foo.com/x/bar?go-get=1" 796 imports []metaImport 797 err error 798 } 799 800 // metaImport represents the parsed <meta name="go-import" 801 // content="prefix vcs reporoot" /> tags from HTML files. 802 type metaImport struct { 803 Prefix, VCS, RepoRoot string 804 } 805 806 // errNoMatch is returned from matchGoImport when there's no applicable match. 807 var errNoMatch = errors.New("no import match") 808 809 // matchGoImport returns the metaImport from imports matching importPath. 810 // An error is returned if there are multiple matches. 811 // errNoMatch is returned if none match. 812 func matchGoImport(imports []metaImport, importPath string) (_ metaImport, err error) { 813 match := -1 814 for i, im := range imports { 815 if !strings.HasPrefix(importPath, im.Prefix) { 816 continue 817 } 818 if match != -1 { 819 err = fmt.Errorf("multiple meta tags match import path %q", importPath) 820 return 821 } 822 match = i 823 } 824 if match == -1 { 825 err = errNoMatch 826 return 827 } 828 return imports[match], nil 829 } 830 831 // expand rewrites s to replace {k} with match[k] for each key k in match. 832 func expand(match map[string]string, s string) string { 833 for k, v := range match { 834 s = strings.Replace(s, "{"+k+"}", v, -1) 835 } 836 return s 837 } 838 839 // vcsPaths defines the meaning of import paths referring to 840 // commonly-used VCS hosting sites (github.com/user/dir) 841 // and import paths referring to a fully-qualified importPath 842 // containing a VCS type (foo.com/repo.git/dir) 843 var vcsPaths = []*vcsPath{ 844 // Google Code - new syntax 845 { 846 prefix: "code.google.com/", 847 re: `^(?P<root>code\.google\.com/p/(?P<project>[a-z0-9\-]+)(\.(?P<subrepo>[a-z0-9\-]+))?)(/[A-Za-z0-9_.\-]+)*$`, 848 repo: "https://{root}", 849 check: googleCodeVCS, 850 }, 851 852 // Google Code - old syntax 853 { 854 re: `^(?P<project>[a-z0-9_\-.]+)\.googlecode\.com/(git|hg|svn)(?P<path>/.*)?$`, 855 check: oldGoogleCode, 856 }, 857 858 // Github 859 { 860 prefix: "github.com/", 861 re: `^(?P<root>github\.com/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(/[A-Za-z0-9_.\-]+)*$`, 862 vcs: "git", 863 repo: "https://{root}", 864 check: noVCSSuffix, 865 }, 866 867 // Bitbucket 868 { 869 prefix: "bitbucket.org/", 870 re: `^(?P<root>bitbucket\.org/(?P<bitname>[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+))(/[A-Za-z0-9_.\-]+)*$`, 871 repo: "https://{root}", 872 check: bitbucketVCS, 873 }, 874 875 // IBM DevOps Services (JazzHub) 876 { 877 prefix: "hub.jazz.net/git", 878 re: `^(?P<root>hub.jazz.net/git/[a-z0-9]+/[A-Za-z0-9_.\-]+)(/[A-Za-z0-9_.\-]+)*$`, 879 vcs: "git", 880 repo: "https://{root}", 881 check: noVCSSuffix, 882 }, 883 884 // Git at Apache 885 { 886 prefix: "git.apache.org", 887 re: `^(?P<root>git.apache.org/[a-z0-9_.\-]+\.git)(/[A-Za-z0-9_.\-]+)*$`, 888 vcs: "git", 889 repo: "https://{root}", 890 }, 891 892 // General syntax for any server. 893 // Must be last. 894 { 895 re: `^(?P<root>(?P<repo>([a-z0-9.\-]+\.)+[a-z0-9.\-]+(:[0-9]+)?/[A-Za-z0-9_.\-/]*?)\.(?P<vcs>bzr|git|hg|svn))(/[A-Za-z0-9_.\-]+)*$`, 896 ping: true, 897 }, 898 } 899 900 // vcsPathsAfterDynamic gives additional vcsPaths entries 901 // to try after the dynamic HTML check. 902 // This gives those sites a chance to introduce <meta> tags 903 // as part of a graceful transition away from the hard-coded logic. 904 var vcsPathsAfterDynamic = []*vcsPath{ 905 // Launchpad. See golang.org/issue/11436. 906 { 907 prefix: "launchpad.net/", 908 re: `^(?P<root>launchpad\.net/((?P<project>[A-Za-z0-9_.\-]+)(?P<series>/[A-Za-z0-9_.\-]+)?|~[A-Za-z0-9_.\-]+/(\+junk|[A-Za-z0-9_.\-]+)/[A-Za-z0-9_.\-]+))(/[A-Za-z0-9_.\-]+)*$`, 909 vcs: "bzr", 910 repo: "https://{root}", 911 check: launchpadVCS, 912 }, 913 } 914 915 func init() { 916 // fill in cached regexps. 917 // Doing this eagerly discovers invalid regexp syntax 918 // without having to run a command that needs that regexp. 919 for _, srv := range vcsPaths { 920 srv.regexp = regexp.MustCompile(srv.re) 921 } 922 for _, srv := range vcsPathsAfterDynamic { 923 srv.regexp = regexp.MustCompile(srv.re) 924 } 925 } 926 927 // noVCSSuffix checks that the repository name does not 928 // end in .foo for any version control system foo. 929 // The usual culprit is ".git". 930 func noVCSSuffix(match map[string]string) error { 931 repo := match["repo"] 932 for _, vcs := range vcsList { 933 if strings.HasSuffix(repo, "."+vcs.cmd) { 934 return fmt.Errorf("invalid version control suffix in %s path", match["prefix"]) 935 } 936 } 937 return nil 938 } 939 940 var googleCheckout = regexp.MustCompile(`id="checkoutcmd">(hg|git|svn)`) 941 942 // googleCodeVCS determines the version control system for 943 // a code.google.com repository, by scraping the project's 944 // /source/checkout page. 945 func googleCodeVCS(match map[string]string) error { 946 if err := noVCSSuffix(match); err != nil { 947 return err 948 } 949 data, err := httpGET(expand(match, "https://code.google.com/p/{project}/source/checkout?repo={subrepo}")) 950 if err != nil { 951 return err 952 } 953 954 if m := googleCheckout.FindSubmatch(data); m != nil { 955 if vcs := vcsByCmd(string(m[1])); vcs != nil { 956 // Subversion requires the old URLs. 957 // TODO: Test. 958 if vcs == vcsSvn { 959 if match["subrepo"] != "" { 960 return fmt.Errorf("sub-repositories not supported in Google Code Subversion projects") 961 } 962 match["repo"] = expand(match, "https://{project}.googlecode.com/svn") 963 } 964 match["vcs"] = vcs.cmd 965 return nil 966 } 967 } 968 969 return fmt.Errorf("unable to detect version control system for code.google.com/ path") 970 } 971 972 // oldGoogleCode is invoked for old-style foo.googlecode.com paths. 973 // It prints an error giving the equivalent new path. 974 func oldGoogleCode(match map[string]string) error { 975 return fmt.Errorf("invalid Google Code import path: use %s instead", 976 expand(match, "code.google.com/p/{project}{path}")) 977 } 978 979 // bitbucketVCS determines the version control system for a 980 // Bitbucket repository, by using the Bitbucket API. 981 func bitbucketVCS(match map[string]string) error { 982 if err := noVCSSuffix(match); err != nil { 983 return err 984 } 985 986 var resp struct { 987 SCM string `json:"scm"` 988 } 989 url := expand(match, "https://api.bitbucket.org/1.0/repositories/{bitname}") 990 data, err := httpGET(url) 991 if err != nil { 992 if httpErr, ok := err.(*httpError); ok && httpErr.statusCode == 403 { 993 // this may be a private repository. If so, attempt to determine which 994 // VCS it uses. See issue 5375. 995 root := match["root"] 996 for _, vcs := range []string{"git", "hg"} { 997 if vcsByCmd(vcs).ping("https", root) == nil { 998 resp.SCM = vcs 999 break 1000 } 1001 } 1002 } 1003 1004 if resp.SCM == "" { 1005 return err 1006 } 1007 } else { 1008 if err := json.Unmarshal(data, &resp); err != nil { 1009 return fmt.Errorf("decoding %s: %v", url, err) 1010 } 1011 } 1012 1013 if vcsByCmd(resp.SCM) != nil { 1014 match["vcs"] = resp.SCM 1015 if resp.SCM == "git" { 1016 match["repo"] += ".git" 1017 } 1018 return nil 1019 } 1020 1021 return fmt.Errorf("unable to detect version control system for bitbucket.org/ path") 1022 } 1023 1024 // launchpadVCS solves the ambiguity for "lp.net/project/foo". In this case, 1025 // "foo" could be a series name registered in Launchpad with its own branch, 1026 // and it could also be the name of a directory within the main project 1027 // branch one level up. 1028 func launchpadVCS(match map[string]string) error { 1029 if match["project"] == "" || match["series"] == "" { 1030 return nil 1031 } 1032 _, err := httpGET(expand(match, "https://code.launchpad.net/{project}{series}/.bzr/branch-format")) 1033 if err != nil { 1034 match["root"] = expand(match, "launchpad.net/{project}") 1035 match["repo"] = expand(match, "https://{root}") 1036 } 1037 return nil 1038 }