github.com/gocuntian/go@v0.0.0-20160610041250-fee02d270bf8/src/cmd/go/vcs.go (about) 1 // Copyright 2012 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package main 6 7 import ( 8 "bytes" 9 "encoding/json" 10 "errors" 11 "fmt" 12 "internal/singleflight" 13 "log" 14 "net/url" 15 "os" 16 "os/exec" 17 "path/filepath" 18 "regexp" 19 "strings" 20 "sync" 21 ) 22 23 // A vcsCmd describes how to use a version control system 24 // like Mercurial, Git, or Subversion. 25 type vcsCmd struct { 26 name string 27 cmd string // name of binary to invoke command 28 29 createCmd []string // commands to download a fresh copy of a repository 30 downloadCmd []string // commands to download updates into an existing repository 31 32 tagCmd []tagCmd // commands to list tags 33 tagLookupCmd []tagCmd // commands to lookup tags before running tagSyncCmd 34 tagSyncCmd []string // commands to sync to specific tag 35 tagSyncDefault []string // commands to sync to default tag 36 37 scheme []string 38 pingCmd string 39 40 remoteRepo func(v *vcsCmd, rootDir string) (remoteRepo string, err error) 41 resolveRepo func(v *vcsCmd, rootDir, remoteRepo string) (realRepo string, err error) 42 } 43 44 var isSecureScheme = map[string]bool{ 45 "https": true, 46 "git+ssh": true, 47 "bzr+ssh": true, 48 "svn+ssh": true, 49 "ssh": true, 50 } 51 52 func (v *vcsCmd) isSecure(repo string) bool { 53 u, err := url.Parse(repo) 54 if err != nil { 55 // If repo is not a URL, it's not secure. 56 return false 57 } 58 return isSecureScheme[u.Scheme] 59 } 60 61 // A tagCmd describes a command to list available tags 62 // that can be passed to tagSyncCmd. 63 type tagCmd struct { 64 cmd string // command to list tags 65 pattern string // regexp to extract tags from list 66 } 67 68 // vcsList lists the known version control systems 69 var vcsList = []*vcsCmd{ 70 vcsHg, 71 vcsGit, 72 vcsSvn, 73 vcsBzr, 74 } 75 76 // vcsByCmd returns the version control system for the given 77 // command name (hg, git, svn, bzr). 78 func vcsByCmd(cmd string) *vcsCmd { 79 for _, vcs := range vcsList { 80 if vcs.cmd == cmd { 81 return vcs 82 } 83 } 84 return nil 85 } 86 87 // vcsHg describes how to use Mercurial. 88 var vcsHg = &vcsCmd{ 89 name: "Mercurial", 90 cmd: "hg", 91 92 createCmd: []string{"clone -U {repo} {dir}"}, 93 downloadCmd: []string{"pull"}, 94 95 // We allow both tag and branch names as 'tags' 96 // for selecting a version. This lets people have 97 // a go.release.r60 branch and a go1 branch 98 // and make changes in both, without constantly 99 // editing .hgtags. 100 tagCmd: []tagCmd{ 101 {"tags", `^(\S+)`}, 102 {"branches", `^(\S+)`}, 103 }, 104 tagSyncCmd: []string{"update -r {tag}"}, 105 tagSyncDefault: []string{"update default"}, 106 107 scheme: []string{"https", "http", "ssh"}, 108 pingCmd: "identify {scheme}://{repo}", 109 remoteRepo: hgRemoteRepo, 110 } 111 112 func hgRemoteRepo(vcsHg *vcsCmd, rootDir string) (remoteRepo string, err error) { 113 out, err := vcsHg.runOutput(rootDir, "paths default") 114 if err != nil { 115 return "", err 116 } 117 return strings.TrimSpace(string(out)), nil 118 } 119 120 // vcsGit describes how to use Git. 121 var vcsGit = &vcsCmd{ 122 name: "Git", 123 cmd: "git", 124 125 createCmd: []string{"clone {repo} {dir}", "-go-internal-cd {dir} submodule update --init --recursive"}, 126 downloadCmd: []string{"pull --ff-only", "submodule update --init --recursive"}, 127 128 tagCmd: []tagCmd{ 129 // tags/xxx matches a git tag named xxx 130 // origin/xxx matches a git branch named xxx on the default remote repository 131 {"show-ref", `(?:tags|origin)/(\S+)$`}, 132 }, 133 tagLookupCmd: []tagCmd{ 134 {"show-ref tags/{tag} origin/{tag}", `((?:tags|origin)/\S+)$`}, 135 }, 136 tagSyncCmd: []string{"checkout {tag}", "submodule update --init --recursive"}, 137 // both createCmd and downloadCmd update the working dir. 138 // No need to do more here. We used to 'checkout master' 139 // but that doesn't work if the default branch is not named master. 140 // DO NOT add 'checkout master' here. 141 // See golang.org/issue/9032. 142 tagSyncDefault: []string{"submodule update --init --recursive"}, 143 144 scheme: []string{"git", "https", "http", "git+ssh", "ssh"}, 145 pingCmd: "ls-remote {scheme}://{repo}", 146 remoteRepo: gitRemoteRepo, 147 } 148 149 // scpSyntaxRe matches the SCP-like addresses used by Git to access 150 // repositories by SSH. 151 var scpSyntaxRe = regexp.MustCompile(`^([a-zA-Z0-9_]+)@([a-zA-Z0-9._-]+):(.*)$`) 152 153 func gitRemoteRepo(vcsGit *vcsCmd, rootDir string) (remoteRepo string, err error) { 154 cmd := "config remote.origin.url" 155 errParse := errors.New("unable to parse output of git " + cmd) 156 errRemoteOriginNotFound := errors.New("remote origin not found") 157 outb, err := vcsGit.run1(rootDir, cmd, nil, false) 158 if err != nil { 159 // if it doesn't output any message, it means the config argument is correct, 160 // but the config value itself doesn't exist 161 if outb != nil && len(outb) == 0 { 162 return "", errRemoteOriginNotFound 163 } 164 return "", err 165 } 166 out := strings.TrimSpace(string(outb)) 167 168 var repoURL *url.URL 169 if m := scpSyntaxRe.FindStringSubmatch(out); m != nil { 170 // Match SCP-like syntax and convert it to a URL. 171 // Eg, "git@github.com:user/repo" becomes 172 // "ssh://git@github.com/user/repo". 173 repoURL = &url.URL{ 174 Scheme: "ssh", 175 User: url.User(m[1]), 176 Host: m[2], 177 Path: m[3], 178 } 179 } else { 180 repoURL, err = url.Parse(out) 181 if err != nil { 182 return "", err 183 } 184 } 185 186 // Iterate over insecure schemes too, because this function simply 187 // reports the state of the repo. If we can't see insecure schemes then 188 // we can't report the actual repo URL. 189 for _, s := range vcsGit.scheme { 190 if repoURL.Scheme == s { 191 return repoURL.String(), nil 192 } 193 } 194 return "", errParse 195 } 196 197 // vcsBzr describes how to use Bazaar. 198 var vcsBzr = &vcsCmd{ 199 name: "Bazaar", 200 cmd: "bzr", 201 202 createCmd: []string{"branch {repo} {dir}"}, 203 204 // Without --overwrite bzr will not pull tags that changed. 205 // Replace by --overwrite-tags after http://pad.lv/681792 goes in. 206 downloadCmd: []string{"pull --overwrite"}, 207 208 tagCmd: []tagCmd{{"tags", `^(\S+)`}}, 209 tagSyncCmd: []string{"update -r {tag}"}, 210 tagSyncDefault: []string{"update -r revno:-1"}, 211 212 scheme: []string{"https", "http", "bzr", "bzr+ssh"}, 213 pingCmd: "info {scheme}://{repo}", 214 remoteRepo: bzrRemoteRepo, 215 resolveRepo: bzrResolveRepo, 216 } 217 218 func bzrRemoteRepo(vcsBzr *vcsCmd, rootDir string) (remoteRepo string, err error) { 219 outb, err := vcsBzr.runOutput(rootDir, "config parent_location") 220 if err != nil { 221 return "", err 222 } 223 return strings.TrimSpace(string(outb)), nil 224 } 225 226 func bzrResolveRepo(vcsBzr *vcsCmd, rootDir, remoteRepo string) (realRepo string, err error) { 227 outb, err := vcsBzr.runOutput(rootDir, "info "+remoteRepo) 228 if err != nil { 229 return "", err 230 } 231 out := string(outb) 232 233 // Expect: 234 // ... 235 // (branch root|repository branch): <URL> 236 // ... 237 238 found := false 239 for _, prefix := range []string{"\n branch root: ", "\n repository branch: "} { 240 i := strings.Index(out, prefix) 241 if i >= 0 { 242 out = out[i+len(prefix):] 243 found = true 244 break 245 } 246 } 247 if !found { 248 return "", fmt.Errorf("unable to parse output of bzr info") 249 } 250 251 i := strings.Index(out, "\n") 252 if i < 0 { 253 return "", fmt.Errorf("unable to parse output of bzr info") 254 } 255 out = out[:i] 256 return strings.TrimSpace(out), nil 257 } 258 259 // vcsSvn describes how to use Subversion. 260 var vcsSvn = &vcsCmd{ 261 name: "Subversion", 262 cmd: "svn", 263 264 createCmd: []string{"checkout {repo} {dir}"}, 265 downloadCmd: []string{"update"}, 266 267 // There is no tag command in subversion. 268 // The branch information is all in the path names. 269 270 scheme: []string{"https", "http", "svn", "svn+ssh"}, 271 pingCmd: "info {scheme}://{repo}", 272 remoteRepo: svnRemoteRepo, 273 } 274 275 func svnRemoteRepo(vcsSvn *vcsCmd, rootDir string) (remoteRepo string, err error) { 276 outb, err := vcsSvn.runOutput(rootDir, "info") 277 if err != nil { 278 return "", err 279 } 280 out := string(outb) 281 282 // Expect: 283 // ... 284 // Repository Root: <URL> 285 // ... 286 287 i := strings.Index(out, "\nRepository Root: ") 288 if i < 0 { 289 return "", fmt.Errorf("unable to parse output of svn info") 290 } 291 out = out[i+len("\nRepository Root: "):] 292 i = strings.Index(out, "\n") 293 if i < 0 { 294 return "", fmt.Errorf("unable to parse output of svn info") 295 } 296 out = out[:i] 297 return strings.TrimSpace(out), nil 298 } 299 300 func (v *vcsCmd) String() string { 301 return v.name 302 } 303 304 // run runs the command line cmd in the given directory. 305 // keyval is a list of key, value pairs. run expands 306 // instances of {key} in cmd into value, but only after 307 // splitting cmd into individual arguments. 308 // If an error occurs, run prints the command line and the 309 // command's combined stdout+stderr to standard error. 310 // Otherwise run discards the command's output. 311 func (v *vcsCmd) run(dir string, cmd string, keyval ...string) error { 312 _, err := v.run1(dir, cmd, keyval, true) 313 return err 314 } 315 316 // runVerboseOnly is like run but only generates error output to standard error in verbose mode. 317 func (v *vcsCmd) runVerboseOnly(dir string, cmd string, keyval ...string) error { 318 _, err := v.run1(dir, cmd, keyval, false) 319 return err 320 } 321 322 // runOutput is like run but returns the output of the command. 323 func (v *vcsCmd) runOutput(dir string, cmd string, keyval ...string) ([]byte, error) { 324 return v.run1(dir, cmd, keyval, true) 325 } 326 327 // run1 is the generalized implementation of run and runOutput. 328 func (v *vcsCmd) run1(dir string, cmdline string, keyval []string, verbose bool) ([]byte, error) { 329 m := make(map[string]string) 330 for i := 0; i < len(keyval); i += 2 { 331 m[keyval[i]] = keyval[i+1] 332 } 333 args := strings.Fields(cmdline) 334 for i, arg := range args { 335 args[i] = expand(m, arg) 336 } 337 338 if len(args) >= 2 && args[0] == "-go-internal-cd" { 339 if filepath.IsAbs(args[1]) { 340 dir = args[1] 341 } else { 342 dir = filepath.Join(dir, args[1]) 343 } 344 args = args[2:] 345 } 346 347 _, err := exec.LookPath(v.cmd) 348 if err != nil { 349 fmt.Fprintf(os.Stderr, 350 "go: missing %s command. See https://golang.org/s/gogetcmd\n", 351 v.name) 352 return nil, err 353 } 354 355 cmd := exec.Command(v.cmd, args...) 356 cmd.Dir = dir 357 cmd.Env = envForDir(cmd.Dir, os.Environ()) 358 if buildX { 359 fmt.Printf("cd %s\n", dir) 360 fmt.Printf("%s %s\n", v.cmd, strings.Join(args, " ")) 361 } 362 var buf bytes.Buffer 363 cmd.Stdout = &buf 364 cmd.Stderr = &buf 365 err = cmd.Run() 366 out := buf.Bytes() 367 if err != nil { 368 if verbose || buildV { 369 fmt.Fprintf(os.Stderr, "# cd %s; %s %s\n", dir, v.cmd, strings.Join(args, " ")) 370 os.Stderr.Write(out) 371 } 372 return out, err 373 } 374 return out, nil 375 } 376 377 // ping pings to determine scheme to use. 378 func (v *vcsCmd) ping(scheme, repo string) error { 379 return v.runVerboseOnly(".", v.pingCmd, "scheme", scheme, "repo", repo) 380 } 381 382 // create creates a new copy of repo in dir. 383 // The parent of dir must exist; dir must not. 384 func (v *vcsCmd) create(dir, repo string) error { 385 for _, cmd := range v.createCmd { 386 if strings.Contains(cmd, "submodule") { 387 continue 388 } 389 if err := v.run(".", cmd, "dir", dir, "repo", repo); err != nil { 390 return err 391 } 392 } 393 return nil 394 } 395 396 // download downloads any new changes for the repo in dir. 397 func (v *vcsCmd) download(dir string) error { 398 for _, cmd := range v.downloadCmd { 399 if strings.Contains(cmd, "submodule") { 400 continue 401 } 402 if err := v.run(dir, cmd); err != nil { 403 return err 404 } 405 } 406 return nil 407 } 408 409 // tags returns the list of available tags for the repo in dir. 410 func (v *vcsCmd) tags(dir string) ([]string, error) { 411 var tags []string 412 for _, tc := range v.tagCmd { 413 out, err := v.runOutput(dir, tc.cmd) 414 if err != nil { 415 return nil, err 416 } 417 re := regexp.MustCompile(`(?m-s)` + tc.pattern) 418 for _, m := range re.FindAllStringSubmatch(string(out), -1) { 419 tags = append(tags, m[1]) 420 } 421 } 422 return tags, nil 423 } 424 425 // tagSync syncs the repo in dir to the named tag, 426 // which either is a tag returned by tags or is v.tagDefault. 427 func (v *vcsCmd) tagSync(dir, tag string) error { 428 if v.tagSyncCmd == nil { 429 return nil 430 } 431 if tag != "" { 432 for _, tc := range v.tagLookupCmd { 433 out, err := v.runOutput(dir, tc.cmd, "tag", tag) 434 if err != nil { 435 return err 436 } 437 re := regexp.MustCompile(`(?m-s)` + tc.pattern) 438 m := re.FindStringSubmatch(string(out)) 439 if len(m) > 1 { 440 tag = m[1] 441 break 442 } 443 } 444 } 445 446 if tag == "" && v.tagSyncDefault != nil { 447 for _, cmd := range v.tagSyncDefault { 448 if strings.Contains(cmd, "submodule") { 449 continue 450 } 451 if err := v.run(dir, cmd); err != nil { 452 return err 453 } 454 } 455 return nil 456 } 457 458 for _, cmd := range v.tagSyncCmd { 459 if strings.Contains(cmd, "submodule") { 460 continue 461 } 462 if err := v.run(dir, cmd, "tag", tag); err != nil { 463 return err 464 } 465 } 466 return nil 467 } 468 469 // A vcsPath describes how to convert an import path into a 470 // version control system and repository name. 471 type vcsPath struct { 472 prefix string // prefix this description applies to 473 re string // pattern for import path 474 repo string // repository to use (expand with match of re) 475 vcs string // version control system to use (expand with match of re) 476 check func(match map[string]string) error // additional checks 477 ping bool // ping for scheme to use to download repo 478 479 regexp *regexp.Regexp // cached compiled form of re 480 } 481 482 // vcsFromDir inspects dir and its parents to determine the 483 // version control system and code repository to use. 484 // On return, root is the import path 485 // corresponding to the root of the repository. 486 func vcsFromDir(dir, srcRoot string) (vcs *vcsCmd, root string, err error) { 487 // Clean and double-check that dir is in (a subdirectory of) srcRoot. 488 dir = filepath.Clean(dir) 489 srcRoot = filepath.Clean(srcRoot) 490 if len(dir) <= len(srcRoot) || dir[len(srcRoot)] != filepath.Separator { 491 return nil, "", fmt.Errorf("directory %q is outside source root %q", dir, srcRoot) 492 } 493 494 origDir := dir 495 for len(dir) > len(srcRoot) { 496 for _, vcs := range vcsList { 497 if fi, err := os.Stat(filepath.Join(dir, "."+vcs.cmd)); err == nil && fi.IsDir() { 498 return vcs, filepath.ToSlash(dir[len(srcRoot)+1:]), nil 499 } 500 } 501 502 // Move to parent. 503 ndir := filepath.Dir(dir) 504 if len(ndir) >= len(dir) { 505 // Shouldn't happen, but just in case, stop. 506 break 507 } 508 dir = ndir 509 } 510 511 return nil, "", fmt.Errorf("directory %q is not using a known version control system", origDir) 512 } 513 514 // repoRoot represents a version control system, a repo, and a root of 515 // where to put it on disk. 516 type repoRoot struct { 517 vcs *vcsCmd 518 519 // repo is the repository URL, including scheme 520 repo string 521 522 // root is the import path corresponding to the root of the 523 // repository 524 root string 525 } 526 527 var httpPrefixRE = regexp.MustCompile(`^https?:`) 528 529 // securityMode specifies whether a function should make network 530 // calls using insecure transports (eg, plain text HTTP). 531 // The zero value is "secure". 532 type securityMode int 533 534 const ( 535 secure securityMode = iota 536 insecure 537 ) 538 539 // repoRootForImportPath analyzes importPath to determine the 540 // version control system, and code repository to use. 541 func repoRootForImportPath(importPath string, security securityMode) (*repoRoot, error) { 542 rr, err := repoRootFromVCSPaths(importPath, "", security, vcsPaths) 543 if err == errUnknownSite { 544 // If there are wildcards, look up the thing before the wildcard, 545 // hoping it applies to the wildcarded parts too. 546 // This makes 'go get rsc.io/pdf/...' work in a fresh GOPATH. 547 lookup := strings.TrimSuffix(importPath, "/...") 548 if i := strings.Index(lookup, "/.../"); i >= 0 { 549 lookup = lookup[:i] 550 } 551 rr, err = repoRootForImportDynamic(lookup, security) 552 if err != nil { 553 err = fmt.Errorf("unrecognized import path %q (%v)", importPath, err) 554 } 555 } 556 if err != nil { 557 rr1, err1 := repoRootFromVCSPaths(importPath, "", security, vcsPathsAfterDynamic) 558 if err1 == nil { 559 rr = rr1 560 err = nil 561 } 562 } 563 564 if err == nil && strings.Contains(importPath, "...") && strings.Contains(rr.root, "...") { 565 // Do not allow wildcards in the repo root. 566 rr = nil 567 err = fmt.Errorf("cannot expand ... in %q", importPath) 568 } 569 return rr, err 570 } 571 572 var errUnknownSite = errors.New("dynamic lookup required to find mapping") 573 574 // repoRootFromVCSPaths attempts to map importPath to a repoRoot 575 // using the mappings defined in vcsPaths. 576 // If scheme is non-empty, that scheme is forced. 577 func repoRootFromVCSPaths(importPath, scheme string, security securityMode, vcsPaths []*vcsPath) (*repoRoot, error) { 578 // A common error is to use https://packagepath because that's what 579 // hg and git require. Diagnose this helpfully. 580 if loc := httpPrefixRE.FindStringIndex(importPath); loc != nil { 581 // The importPath has been cleaned, so has only one slash. The pattern 582 // ignores the slashes; the error message puts them back on the RHS at least. 583 return nil, fmt.Errorf("%q not allowed in import path", importPath[loc[0]:loc[1]]+"//") 584 } 585 for _, srv := range vcsPaths { 586 if !strings.HasPrefix(importPath, srv.prefix) { 587 continue 588 } 589 m := srv.regexp.FindStringSubmatch(importPath) 590 if m == nil { 591 if srv.prefix != "" { 592 return nil, fmt.Errorf("invalid %s import path %q", srv.prefix, importPath) 593 } 594 continue 595 } 596 597 // Build map of named subexpression matches for expand. 598 match := map[string]string{ 599 "prefix": srv.prefix, 600 "import": importPath, 601 } 602 for i, name := range srv.regexp.SubexpNames() { 603 if name != "" && match[name] == "" { 604 match[name] = m[i] 605 } 606 } 607 if srv.vcs != "" { 608 match["vcs"] = expand(match, srv.vcs) 609 } 610 if srv.repo != "" { 611 match["repo"] = expand(match, srv.repo) 612 } 613 if srv.check != nil { 614 if err := srv.check(match); err != nil { 615 return nil, err 616 } 617 } 618 vcs := vcsByCmd(match["vcs"]) 619 if vcs == nil { 620 return nil, fmt.Errorf("unknown version control system %q", match["vcs"]) 621 } 622 if srv.ping { 623 if scheme != "" { 624 match["repo"] = scheme + "://" + match["repo"] 625 } else { 626 for _, scheme := range vcs.scheme { 627 if security == secure && !isSecureScheme[scheme] { 628 continue 629 } 630 if vcs.ping(scheme, match["repo"]) == nil { 631 match["repo"] = scheme + "://" + match["repo"] 632 break 633 } 634 } 635 } 636 } 637 rr := &repoRoot{ 638 vcs: vcs, 639 repo: match["repo"], 640 root: match["root"], 641 } 642 return rr, nil 643 } 644 return nil, errUnknownSite 645 } 646 647 // repoRootForImportDynamic finds a *repoRoot for a custom domain that's not 648 // statically known by repoRootForImportPathStatic. 649 // 650 // This handles custom import paths like "name.tld/pkg/foo" or just "name.tld". 651 func repoRootForImportDynamic(importPath string, security securityMode) (*repoRoot, error) { 652 slash := strings.Index(importPath, "/") 653 if slash < 0 { 654 slash = len(importPath) 655 } 656 host := importPath[:slash] 657 if !strings.Contains(host, ".") { 658 return nil, errors.New("import path does not begin with hostname") 659 } 660 urlStr, body, err := httpsOrHTTP(importPath, security) 661 if err != nil { 662 msg := "https fetch: %v" 663 if security == insecure { 664 msg = "http/" + msg 665 } 666 return nil, fmt.Errorf(msg, err) 667 } 668 defer body.Close() 669 imports, err := parseMetaGoImports(body) 670 if err != nil { 671 return nil, fmt.Errorf("parsing %s: %v", importPath, err) 672 } 673 // Find the matched meta import. 674 mmi, err := matchGoImport(imports, importPath) 675 if err != nil { 676 if err != errNoMatch { 677 return nil, fmt.Errorf("parse %s: %v", urlStr, err) 678 } 679 return nil, fmt.Errorf("parse %s: no go-import meta tags", urlStr) 680 } 681 if buildV { 682 log.Printf("get %q: found meta tag %#v at %s", importPath, mmi, urlStr) 683 } 684 // If the import was "uni.edu/bob/project", which said the 685 // prefix was "uni.edu" and the RepoRoot was "evilroot.com", 686 // make sure we don't trust Bob and check out evilroot.com to 687 // "uni.edu" yet (possibly overwriting/preempting another 688 // non-evil student). Instead, first verify the root and see 689 // if it matches Bob's claim. 690 if mmi.Prefix != importPath { 691 if buildV { 692 log.Printf("get %q: verifying non-authoritative meta tag", importPath) 693 } 694 urlStr0 := urlStr 695 var imports []metaImport 696 urlStr, imports, err = metaImportsForPrefix(mmi.Prefix, security) 697 if err != nil { 698 return nil, err 699 } 700 metaImport2, err := matchGoImport(imports, importPath) 701 if err != nil || mmi != metaImport2 { 702 return nil, fmt.Errorf("%s and %s disagree about go-import for %s", urlStr0, urlStr, mmi.Prefix) 703 } 704 } 705 706 if !strings.Contains(mmi.RepoRoot, "://") { 707 return nil, fmt.Errorf("%s: invalid repo root %q; no scheme", urlStr, mmi.RepoRoot) 708 } 709 rr := &repoRoot{ 710 vcs: vcsByCmd(mmi.VCS), 711 repo: mmi.RepoRoot, 712 root: mmi.Prefix, 713 } 714 if rr.vcs == nil { 715 return nil, fmt.Errorf("%s: unknown vcs %q", urlStr, mmi.VCS) 716 } 717 return rr, nil 718 } 719 720 var fetchGroup singleflight.Group 721 var ( 722 fetchCacheMu sync.Mutex 723 fetchCache = map[string]fetchResult{} // key is metaImportsForPrefix's importPrefix 724 ) 725 726 // metaImportsForPrefix takes a package's root import path as declared in a <meta> tag 727 // and returns its HTML discovery URL and the parsed metaImport lines 728 // found on the page. 729 // 730 // The importPath is of the form "golang.org/x/tools". 731 // It is an error if no imports are found. 732 // urlStr will still be valid if err != nil. 733 // The returned urlStr will be of the form "https://golang.org/x/tools?go-get=1" 734 func metaImportsForPrefix(importPrefix string, security securityMode) (urlStr string, imports []metaImport, err error) { 735 setCache := func(res fetchResult) (fetchResult, error) { 736 fetchCacheMu.Lock() 737 defer fetchCacheMu.Unlock() 738 fetchCache[importPrefix] = res 739 return res, nil 740 } 741 742 resi, _, _ := fetchGroup.Do(importPrefix, func() (resi interface{}, err error) { 743 fetchCacheMu.Lock() 744 if res, ok := fetchCache[importPrefix]; ok { 745 fetchCacheMu.Unlock() 746 return res, nil 747 } 748 fetchCacheMu.Unlock() 749 750 urlStr, body, err := httpsOrHTTP(importPrefix, security) 751 if err != nil { 752 return setCache(fetchResult{urlStr: urlStr, err: fmt.Errorf("fetch %s: %v", urlStr, err)}) 753 } 754 imports, err := parseMetaGoImports(body) 755 if err != nil { 756 return setCache(fetchResult{urlStr: urlStr, err: fmt.Errorf("parsing %s: %v", urlStr, err)}) 757 } 758 if len(imports) == 0 { 759 err = fmt.Errorf("fetch %s: no go-import meta tag", urlStr) 760 } 761 return setCache(fetchResult{urlStr: urlStr, imports: imports, err: err}) 762 }) 763 res := resi.(fetchResult) 764 return res.urlStr, res.imports, res.err 765 } 766 767 type fetchResult struct { 768 urlStr string // e.g. "https://foo.com/x/bar?go-get=1" 769 imports []metaImport 770 err error 771 } 772 773 // metaImport represents the parsed <meta name="go-import" 774 // content="prefix vcs reporoot" /> tags from HTML files. 775 type metaImport struct { 776 Prefix, VCS, RepoRoot string 777 } 778 779 // errNoMatch is returned from matchGoImport when there's no applicable match. 780 var errNoMatch = errors.New("no import match") 781 782 func splitPathHasPrefix(path, prefix []string) bool { 783 if len(path) < len(prefix) { 784 return false 785 } 786 for i, p := range prefix { 787 if path[i] != p { 788 return false 789 } 790 } 791 return true 792 } 793 794 // matchGoImport returns the metaImport from imports matching importPath. 795 // An error is returned if there are multiple matches. 796 // errNoMatch is returned if none match. 797 func matchGoImport(imports []metaImport, importPath string) (_ metaImport, err error) { 798 match := -1 799 imp := strings.Split(importPath, "/") 800 for i, im := range imports { 801 pre := strings.Split(im.Prefix, "/") 802 803 if !splitPathHasPrefix(imp, pre) { 804 continue 805 } 806 807 if match != -1 { 808 err = fmt.Errorf("multiple meta tags match import path %q", importPath) 809 return 810 } 811 match = i 812 } 813 if match == -1 { 814 err = errNoMatch 815 return 816 } 817 return imports[match], nil 818 } 819 820 // expand rewrites s to replace {k} with match[k] for each key k in match. 821 func expand(match map[string]string, s string) string { 822 for k, v := range match { 823 s = strings.Replace(s, "{"+k+"}", v, -1) 824 } 825 return s 826 } 827 828 // vcsPaths defines the meaning of import paths referring to 829 // commonly-used VCS hosting sites (github.com/user/dir) 830 // and import paths referring to a fully-qualified importPath 831 // containing a VCS type (foo.com/repo.git/dir) 832 var vcsPaths = []*vcsPath{ 833 // Github 834 { 835 prefix: "github.com/", 836 re: `^(?P<root>github\.com/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(/[A-Za-z0-9_.\-]+)*$`, 837 vcs: "git", 838 repo: "https://{root}", 839 check: noVCSSuffix, 840 }, 841 842 // Bitbucket 843 { 844 prefix: "bitbucket.org/", 845 re: `^(?P<root>bitbucket\.org/(?P<bitname>[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+))(/[A-Za-z0-9_.\-]+)*$`, 846 repo: "https://{root}", 847 check: bitbucketVCS, 848 }, 849 850 // IBM DevOps Services (JazzHub) 851 { 852 prefix: "hub.jazz.net/git", 853 re: `^(?P<root>hub.jazz.net/git/[a-z0-9]+/[A-Za-z0-9_.\-]+)(/[A-Za-z0-9_.\-]+)*$`, 854 vcs: "git", 855 repo: "https://{root}", 856 check: noVCSSuffix, 857 }, 858 859 // Git at Apache 860 { 861 prefix: "git.apache.org", 862 re: `^(?P<root>git.apache.org/[a-z0-9_.\-]+\.git)(/[A-Za-z0-9_.\-]+)*$`, 863 vcs: "git", 864 repo: "https://{root}", 865 }, 866 867 // Git at OpenStack 868 { 869 prefix: "git.openstack.org", 870 re: `^(?P<root>git\.openstack\.org/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(\.git)?(/[A-Za-z0-9_.\-]+)*$`, 871 vcs: "git", 872 repo: "https://{root}", 873 check: noVCSSuffix, 874 }, 875 876 // General syntax for any server. 877 // Must be last. 878 { 879 re: `^(?P<root>(?P<repo>([a-z0-9.\-]+\.)+[a-z0-9.\-]+(:[0-9]+)?(/~?[A-Za-z0-9_.\-]+)+?)\.(?P<vcs>bzr|git|hg|svn))(/~?[A-Za-z0-9_.\-]+)*$`, 880 ping: true, 881 }, 882 } 883 884 // vcsPathsAfterDynamic gives additional vcsPaths entries 885 // to try after the dynamic HTML check. 886 // This gives those sites a chance to introduce <meta> tags 887 // as part of a graceful transition away from the hard-coded logic. 888 var vcsPathsAfterDynamic = []*vcsPath{ 889 // Launchpad. See golang.org/issue/11436. 890 { 891 prefix: "launchpad.net/", 892 re: `^(?P<root>launchpad\.net/((?P<project>[A-Za-z0-9_.\-]+)(?P<series>/[A-Za-z0-9_.\-]+)?|~[A-Za-z0-9_.\-]+/(\+junk|[A-Za-z0-9_.\-]+)/[A-Za-z0-9_.\-]+))(/[A-Za-z0-9_.\-]+)*$`, 893 vcs: "bzr", 894 repo: "https://{root}", 895 check: launchpadVCS, 896 }, 897 } 898 899 func init() { 900 // fill in cached regexps. 901 // Doing this eagerly discovers invalid regexp syntax 902 // without having to run a command that needs that regexp. 903 for _, srv := range vcsPaths { 904 srv.regexp = regexp.MustCompile(srv.re) 905 } 906 for _, srv := range vcsPathsAfterDynamic { 907 srv.regexp = regexp.MustCompile(srv.re) 908 } 909 } 910 911 // noVCSSuffix checks that the repository name does not 912 // end in .foo for any version control system foo. 913 // The usual culprit is ".git". 914 func noVCSSuffix(match map[string]string) error { 915 repo := match["repo"] 916 for _, vcs := range vcsList { 917 if strings.HasSuffix(repo, "."+vcs.cmd) { 918 return fmt.Errorf("invalid version control suffix in %s path", match["prefix"]) 919 } 920 } 921 return nil 922 } 923 924 // bitbucketVCS determines the version control system for a 925 // Bitbucket repository, by using the Bitbucket API. 926 func bitbucketVCS(match map[string]string) error { 927 if err := noVCSSuffix(match); err != nil { 928 return err 929 } 930 931 var resp struct { 932 SCM string `json:"scm"` 933 } 934 url := expand(match, "https://api.bitbucket.org/1.0/repositories/{bitname}") 935 data, err := httpGET(url) 936 if err != nil { 937 if httpErr, ok := err.(*httpError); ok && httpErr.statusCode == 403 { 938 // this may be a private repository. If so, attempt to determine which 939 // VCS it uses. See issue 5375. 940 root := match["root"] 941 for _, vcs := range []string{"git", "hg"} { 942 if vcsByCmd(vcs).ping("https", root) == nil { 943 resp.SCM = vcs 944 break 945 } 946 } 947 } 948 949 if resp.SCM == "" { 950 return err 951 } 952 } else { 953 if err := json.Unmarshal(data, &resp); err != nil { 954 return fmt.Errorf("decoding %s: %v", url, err) 955 } 956 } 957 958 if vcsByCmd(resp.SCM) != nil { 959 match["vcs"] = resp.SCM 960 if resp.SCM == "git" { 961 match["repo"] += ".git" 962 } 963 return nil 964 } 965 966 return fmt.Errorf("unable to detect version control system for bitbucket.org/ path") 967 } 968 969 // launchpadVCS solves the ambiguity for "lp.net/project/foo". In this case, 970 // "foo" could be a series name registered in Launchpad with its own branch, 971 // and it could also be the name of a directory within the main project 972 // branch one level up. 973 func launchpadVCS(match map[string]string) error { 974 if match["project"] == "" || match["series"] == "" { 975 return nil 976 } 977 _, err := httpGET(expand(match, "https://code.launchpad.net/{project}{series}/.bzr/branch-format")) 978 if err != nil { 979 match["root"] = expand(match, "launchpad.net/{project}") 980 match["repo"] = expand(match, "https://{root}") 981 } 982 return nil 983 }