golang.org/toolchain@v0.0.1-go1.9rc2.windows-amd64/src/cmd/go/internal/get/vcs.go (about) 1 // Copyright 2012 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package get 6 7 import ( 8 "bytes" 9 "encoding/json" 10 "errors" 11 "fmt" 12 "internal/singleflight" 13 "log" 14 "net/url" 15 "os" 16 "os/exec" 17 "path/filepath" 18 "regexp" 19 "strings" 20 "sync" 21 22 "cmd/go/internal/base" 23 "cmd/go/internal/cfg" 24 "cmd/go/internal/web" 25 ) 26 27 // A vcsCmd describes how to use a version control system 28 // like Mercurial, Git, or Subversion. 29 type vcsCmd struct { 30 name string 31 cmd string // name of binary to invoke command 32 33 createCmd []string // commands to download a fresh copy of a repository 34 downloadCmd []string // commands to download updates into an existing repository 35 36 tagCmd []tagCmd // commands to list tags 37 tagLookupCmd []tagCmd // commands to lookup tags before running tagSyncCmd 38 tagSyncCmd []string // commands to sync to specific tag 39 tagSyncDefault []string // commands to sync to default tag 40 41 scheme []string 42 pingCmd string 43 44 remoteRepo func(v *vcsCmd, rootDir string) (remoteRepo string, err error) 45 resolveRepo func(v *vcsCmd, rootDir, remoteRepo string) (realRepo string, err error) 46 } 47 48 var defaultSecureScheme = map[string]bool{ 49 "https": true, 50 "git+ssh": true, 51 "bzr+ssh": true, 52 "svn+ssh": true, 53 "ssh": true, 54 } 55 56 func (v *vcsCmd) isSecure(repo string) bool { 57 u, err := url.Parse(repo) 58 if err != nil { 59 // If repo is not a URL, it's not secure. 60 return false 61 } 62 return v.isSecureScheme(u.Scheme) 63 } 64 65 func (v *vcsCmd) isSecureScheme(scheme string) bool { 66 switch v.cmd { 67 case "git": 68 // GIT_ALLOW_PROTOCOL is an environment variable defined by Git. It is a 69 // colon-separated list of schemes that are allowed to be used with git 70 // fetch/clone. Any scheme not mentioned will be considered insecure. 71 if allow := os.Getenv("GIT_ALLOW_PROTOCOL"); allow != "" { 72 for _, s := range strings.Split(allow, ":") { 73 if s == scheme { 74 return true 75 } 76 } 77 return false 78 } 79 } 80 return defaultSecureScheme[scheme] 81 } 82 83 // A tagCmd describes a command to list available tags 84 // that can be passed to tagSyncCmd. 85 type tagCmd struct { 86 cmd string // command to list tags 87 pattern string // regexp to extract tags from list 88 } 89 90 // vcsList lists the known version control systems 91 var vcsList = []*vcsCmd{ 92 vcsHg, 93 vcsGit, 94 vcsSvn, 95 vcsBzr, 96 } 97 98 // vcsByCmd returns the version control system for the given 99 // command name (hg, git, svn, bzr). 100 func vcsByCmd(cmd string) *vcsCmd { 101 for _, vcs := range vcsList { 102 if vcs.cmd == cmd { 103 return vcs 104 } 105 } 106 return nil 107 } 108 109 // vcsHg describes how to use Mercurial. 110 var vcsHg = &vcsCmd{ 111 name: "Mercurial", 112 cmd: "hg", 113 114 createCmd: []string{"clone -U {repo} {dir}"}, 115 downloadCmd: []string{"pull"}, 116 117 // We allow both tag and branch names as 'tags' 118 // for selecting a version. This lets people have 119 // a go.release.r60 branch and a go1 branch 120 // and make changes in both, without constantly 121 // editing .hgtags. 122 tagCmd: []tagCmd{ 123 {"tags", `^(\S+)`}, 124 {"branches", `^(\S+)`}, 125 }, 126 tagSyncCmd: []string{"update -r {tag}"}, 127 tagSyncDefault: []string{"update default"}, 128 129 scheme: []string{"https", "http", "ssh"}, 130 pingCmd: "identify {scheme}://{repo}", 131 remoteRepo: hgRemoteRepo, 132 } 133 134 func hgRemoteRepo(vcsHg *vcsCmd, rootDir string) (remoteRepo string, err error) { 135 out, err := vcsHg.runOutput(rootDir, "paths default") 136 if err != nil { 137 return "", err 138 } 139 return strings.TrimSpace(string(out)), nil 140 } 141 142 // vcsGit describes how to use Git. 143 var vcsGit = &vcsCmd{ 144 name: "Git", 145 cmd: "git", 146 147 createCmd: []string{"clone {repo} {dir}", "-go-internal-cd {dir} submodule update --init --recursive"}, 148 downloadCmd: []string{"pull --ff-only", "submodule update --init --recursive"}, 149 150 tagCmd: []tagCmd{ 151 // tags/xxx matches a git tag named xxx 152 // origin/xxx matches a git branch named xxx on the default remote repository 153 {"show-ref", `(?:tags|origin)/(\S+)$`}, 154 }, 155 tagLookupCmd: []tagCmd{ 156 {"show-ref tags/{tag} origin/{tag}", `((?:tags|origin)/\S+)$`}, 157 }, 158 tagSyncCmd: []string{"checkout {tag}", "submodule update --init --recursive"}, 159 // both createCmd and downloadCmd update the working dir. 160 // No need to do more here. We used to 'checkout master' 161 // but that doesn't work if the default branch is not named master. 162 // DO NOT add 'checkout master' here. 163 // See golang.org/issue/9032. 164 tagSyncDefault: []string{"submodule update --init --recursive"}, 165 166 scheme: []string{"git", "https", "http", "git+ssh", "ssh"}, 167 pingCmd: "ls-remote {scheme}://{repo}", 168 remoteRepo: gitRemoteRepo, 169 } 170 171 // scpSyntaxRe matches the SCP-like addresses used by Git to access 172 // repositories by SSH. 173 var scpSyntaxRe = regexp.MustCompile(`^([a-zA-Z0-9_]+)@([a-zA-Z0-9._-]+):(.*)$`) 174 175 func gitRemoteRepo(vcsGit *vcsCmd, rootDir string) (remoteRepo string, err error) { 176 cmd := "config remote.origin.url" 177 errParse := errors.New("unable to parse output of git " + cmd) 178 errRemoteOriginNotFound := errors.New("remote origin not found") 179 outb, err := vcsGit.run1(rootDir, cmd, nil, false) 180 if err != nil { 181 // if it doesn't output any message, it means the config argument is correct, 182 // but the config value itself doesn't exist 183 if outb != nil && len(outb) == 0 { 184 return "", errRemoteOriginNotFound 185 } 186 return "", err 187 } 188 out := strings.TrimSpace(string(outb)) 189 190 var repoURL *url.URL 191 if m := scpSyntaxRe.FindStringSubmatch(out); m != nil { 192 // Match SCP-like syntax and convert it to a URL. 193 // Eg, "git@github.com:user/repo" becomes 194 // "ssh://git@github.com/user/repo". 195 repoURL = &url.URL{ 196 Scheme: "ssh", 197 User: url.User(m[1]), 198 Host: m[2], 199 Path: m[3], 200 } 201 } else { 202 repoURL, err = url.Parse(out) 203 if err != nil { 204 return "", err 205 } 206 } 207 208 // Iterate over insecure schemes too, because this function simply 209 // reports the state of the repo. If we can't see insecure schemes then 210 // we can't report the actual repo URL. 211 for _, s := range vcsGit.scheme { 212 if repoURL.Scheme == s { 213 return repoURL.String(), nil 214 } 215 } 216 return "", errParse 217 } 218 219 // vcsBzr describes how to use Bazaar. 220 var vcsBzr = &vcsCmd{ 221 name: "Bazaar", 222 cmd: "bzr", 223 224 createCmd: []string{"branch {repo} {dir}"}, 225 226 // Without --overwrite bzr will not pull tags that changed. 227 // Replace by --overwrite-tags after http://pad.lv/681792 goes in. 228 downloadCmd: []string{"pull --overwrite"}, 229 230 tagCmd: []tagCmd{{"tags", `^(\S+)`}}, 231 tagSyncCmd: []string{"update -r {tag}"}, 232 tagSyncDefault: []string{"update -r revno:-1"}, 233 234 scheme: []string{"https", "http", "bzr", "bzr+ssh"}, 235 pingCmd: "info {scheme}://{repo}", 236 remoteRepo: bzrRemoteRepo, 237 resolveRepo: bzrResolveRepo, 238 } 239 240 func bzrRemoteRepo(vcsBzr *vcsCmd, rootDir string) (remoteRepo string, err error) { 241 outb, err := vcsBzr.runOutput(rootDir, "config parent_location") 242 if err != nil { 243 return "", err 244 } 245 return strings.TrimSpace(string(outb)), nil 246 } 247 248 func bzrResolveRepo(vcsBzr *vcsCmd, rootDir, remoteRepo string) (realRepo string, err error) { 249 outb, err := vcsBzr.runOutput(rootDir, "info "+remoteRepo) 250 if err != nil { 251 return "", err 252 } 253 out := string(outb) 254 255 // Expect: 256 // ... 257 // (branch root|repository branch): <URL> 258 // ... 259 260 found := false 261 for _, prefix := range []string{"\n branch root: ", "\n repository branch: "} { 262 i := strings.Index(out, prefix) 263 if i >= 0 { 264 out = out[i+len(prefix):] 265 found = true 266 break 267 } 268 } 269 if !found { 270 return "", fmt.Errorf("unable to parse output of bzr info") 271 } 272 273 i := strings.Index(out, "\n") 274 if i < 0 { 275 return "", fmt.Errorf("unable to parse output of bzr info") 276 } 277 out = out[:i] 278 return strings.TrimSpace(out), nil 279 } 280 281 // vcsSvn describes how to use Subversion. 282 var vcsSvn = &vcsCmd{ 283 name: "Subversion", 284 cmd: "svn", 285 286 createCmd: []string{"checkout {repo} {dir}"}, 287 downloadCmd: []string{"update"}, 288 289 // There is no tag command in subversion. 290 // The branch information is all in the path names. 291 292 scheme: []string{"https", "http", "svn", "svn+ssh"}, 293 pingCmd: "info {scheme}://{repo}", 294 remoteRepo: svnRemoteRepo, 295 } 296 297 func svnRemoteRepo(vcsSvn *vcsCmd, rootDir string) (remoteRepo string, err error) { 298 outb, err := vcsSvn.runOutput(rootDir, "info") 299 if err != nil { 300 return "", err 301 } 302 out := string(outb) 303 304 // Expect: 305 // 306 // ... 307 // URL: <URL> 308 // ... 309 // 310 // Note that we're not using the Repository Root line, 311 // because svn allows checking out subtrees. 312 // The URL will be the URL of the subtree (what we used with 'svn co') 313 // while the Repository Root may be a much higher parent. 314 i := strings.Index(out, "\nURL: ") 315 if i < 0 { 316 return "", fmt.Errorf("unable to parse output of svn info") 317 } 318 out = out[i+len("\nURL: "):] 319 i = strings.Index(out, "\n") 320 if i < 0 { 321 return "", fmt.Errorf("unable to parse output of svn info") 322 } 323 out = out[:i] 324 return strings.TrimSpace(out), nil 325 } 326 327 func (v *vcsCmd) String() string { 328 return v.name 329 } 330 331 // run runs the command line cmd in the given directory. 332 // keyval is a list of key, value pairs. run expands 333 // instances of {key} in cmd into value, but only after 334 // splitting cmd into individual arguments. 335 // If an error occurs, run prints the command line and the 336 // command's combined stdout+stderr to standard error. 337 // Otherwise run discards the command's output. 338 func (v *vcsCmd) run(dir string, cmd string, keyval ...string) error { 339 _, err := v.run1(dir, cmd, keyval, true) 340 return err 341 } 342 343 // runVerboseOnly is like run but only generates error output to standard error in verbose mode. 344 func (v *vcsCmd) runVerboseOnly(dir string, cmd string, keyval ...string) error { 345 _, err := v.run1(dir, cmd, keyval, false) 346 return err 347 } 348 349 // runOutput is like run but returns the output of the command. 350 func (v *vcsCmd) runOutput(dir string, cmd string, keyval ...string) ([]byte, error) { 351 return v.run1(dir, cmd, keyval, true) 352 } 353 354 // run1 is the generalized implementation of run and runOutput. 355 func (v *vcsCmd) run1(dir string, cmdline string, keyval []string, verbose bool) ([]byte, error) { 356 m := make(map[string]string) 357 for i := 0; i < len(keyval); i += 2 { 358 m[keyval[i]] = keyval[i+1] 359 } 360 args := strings.Fields(cmdline) 361 for i, arg := range args { 362 args[i] = expand(m, arg) 363 } 364 365 if len(args) >= 2 && args[0] == "-go-internal-cd" { 366 if filepath.IsAbs(args[1]) { 367 dir = args[1] 368 } else { 369 dir = filepath.Join(dir, args[1]) 370 } 371 args = args[2:] 372 } 373 374 _, err := exec.LookPath(v.cmd) 375 if err != nil { 376 fmt.Fprintf(os.Stderr, 377 "go: missing %s command. See https://golang.org/s/gogetcmd\n", 378 v.name) 379 return nil, err 380 } 381 382 cmd := exec.Command(v.cmd, args...) 383 cmd.Dir = dir 384 cmd.Env = base.EnvForDir(cmd.Dir, os.Environ()) 385 if cfg.BuildX { 386 fmt.Printf("cd %s\n", dir) 387 fmt.Printf("%s %s\n", v.cmd, strings.Join(args, " ")) 388 } 389 var buf bytes.Buffer 390 cmd.Stdout = &buf 391 cmd.Stderr = &buf 392 err = cmd.Run() 393 out := buf.Bytes() 394 if err != nil { 395 if verbose || cfg.BuildV { 396 fmt.Fprintf(os.Stderr, "# cd %s; %s %s\n", dir, v.cmd, strings.Join(args, " ")) 397 os.Stderr.Write(out) 398 } 399 return out, err 400 } 401 return out, nil 402 } 403 404 // ping pings to determine scheme to use. 405 func (v *vcsCmd) ping(scheme, repo string) error { 406 return v.runVerboseOnly(".", v.pingCmd, "scheme", scheme, "repo", repo) 407 } 408 409 // create creates a new copy of repo in dir. 410 // The parent of dir must exist; dir must not. 411 func (v *vcsCmd) create(dir, repo string) error { 412 for _, cmd := range v.createCmd { 413 if err := v.run(".", cmd, "dir", dir, "repo", repo); err != nil { 414 return err 415 } 416 } 417 return nil 418 } 419 420 // download downloads any new changes for the repo in dir. 421 func (v *vcsCmd) download(dir string) error { 422 for _, cmd := range v.downloadCmd { 423 if err := v.run(dir, cmd); err != nil { 424 return err 425 } 426 } 427 return nil 428 } 429 430 // tags returns the list of available tags for the repo in dir. 431 func (v *vcsCmd) tags(dir string) ([]string, error) { 432 var tags []string 433 for _, tc := range v.tagCmd { 434 out, err := v.runOutput(dir, tc.cmd) 435 if err != nil { 436 return nil, err 437 } 438 re := regexp.MustCompile(`(?m-s)` + tc.pattern) 439 for _, m := range re.FindAllStringSubmatch(string(out), -1) { 440 tags = append(tags, m[1]) 441 } 442 } 443 return tags, nil 444 } 445 446 // tagSync syncs the repo in dir to the named tag, 447 // which either is a tag returned by tags or is v.tagDefault. 448 func (v *vcsCmd) tagSync(dir, tag string) error { 449 if v.tagSyncCmd == nil { 450 return nil 451 } 452 if tag != "" { 453 for _, tc := range v.tagLookupCmd { 454 out, err := v.runOutput(dir, tc.cmd, "tag", tag) 455 if err != nil { 456 return err 457 } 458 re := regexp.MustCompile(`(?m-s)` + tc.pattern) 459 m := re.FindStringSubmatch(string(out)) 460 if len(m) > 1 { 461 tag = m[1] 462 break 463 } 464 } 465 } 466 467 if tag == "" && v.tagSyncDefault != nil { 468 for _, cmd := range v.tagSyncDefault { 469 if err := v.run(dir, cmd); err != nil { 470 return err 471 } 472 } 473 return nil 474 } 475 476 for _, cmd := range v.tagSyncCmd { 477 if err := v.run(dir, cmd, "tag", tag); err != nil { 478 return err 479 } 480 } 481 return nil 482 } 483 484 // A vcsPath describes how to convert an import path into a 485 // version control system and repository name. 486 type vcsPath struct { 487 prefix string // prefix this description applies to 488 re string // pattern for import path 489 repo string // repository to use (expand with match of re) 490 vcs string // version control system to use (expand with match of re) 491 check func(match map[string]string) error // additional checks 492 ping bool // ping for scheme to use to download repo 493 494 regexp *regexp.Regexp // cached compiled form of re 495 } 496 497 // vcsFromDir inspects dir and its parents to determine the 498 // version control system and code repository to use. 499 // On return, root is the import path 500 // corresponding to the root of the repository. 501 func vcsFromDir(dir, srcRoot string) (vcs *vcsCmd, root string, err error) { 502 // Clean and double-check that dir is in (a subdirectory of) srcRoot. 503 dir = filepath.Clean(dir) 504 srcRoot = filepath.Clean(srcRoot) 505 if len(dir) <= len(srcRoot) || dir[len(srcRoot)] != filepath.Separator { 506 return nil, "", fmt.Errorf("directory %q is outside source root %q", dir, srcRoot) 507 } 508 509 origDir := dir 510 for len(dir) > len(srcRoot) { 511 for _, vcs := range vcsList { 512 if _, err := os.Stat(filepath.Join(dir, "."+vcs.cmd)); err == nil { 513 return vcs, filepath.ToSlash(dir[len(srcRoot)+1:]), nil 514 } 515 } 516 517 // Move to parent. 518 ndir := filepath.Dir(dir) 519 if len(ndir) >= len(dir) { 520 // Shouldn't happen, but just in case, stop. 521 break 522 } 523 dir = ndir 524 } 525 526 return nil, "", fmt.Errorf("directory %q is not using a known version control system", origDir) 527 } 528 529 // repoRoot represents a version control system, a repo, and a root of 530 // where to put it on disk. 531 type repoRoot struct { 532 vcs *vcsCmd 533 534 // repo is the repository URL, including scheme 535 repo string 536 537 // root is the import path corresponding to the root of the 538 // repository 539 root string 540 541 // isCustom is true for custom import paths (those defined by HTML meta tags) 542 isCustom bool 543 } 544 545 var httpPrefixRE = regexp.MustCompile(`^https?:`) 546 547 // repoRootForImportPath analyzes importPath to determine the 548 // version control system, and code repository to use. 549 func repoRootForImportPath(importPath string, security web.SecurityMode) (*repoRoot, error) { 550 rr, err := repoRootFromVCSPaths(importPath, "", security, vcsPaths) 551 if err == errUnknownSite { 552 // If there are wildcards, look up the thing before the wildcard, 553 // hoping it applies to the wildcarded parts too. 554 // This makes 'go get rsc.io/pdf/...' work in a fresh GOPATH. 555 lookup := strings.TrimSuffix(importPath, "/...") 556 if i := strings.Index(lookup, "/.../"); i >= 0 { 557 lookup = lookup[:i] 558 } 559 rr, err = repoRootForImportDynamic(lookup, security) 560 if err != nil { 561 err = fmt.Errorf("unrecognized import path %q (%v)", importPath, err) 562 } 563 } 564 if err != nil { 565 rr1, err1 := repoRootFromVCSPaths(importPath, "", security, vcsPathsAfterDynamic) 566 if err1 == nil { 567 rr = rr1 568 err = nil 569 } 570 } 571 572 if err == nil && strings.Contains(importPath, "...") && strings.Contains(rr.root, "...") { 573 // Do not allow wildcards in the repo root. 574 rr = nil 575 err = fmt.Errorf("cannot expand ... in %q", importPath) 576 } 577 return rr, err 578 } 579 580 var errUnknownSite = errors.New("dynamic lookup required to find mapping") 581 582 // repoRootFromVCSPaths attempts to map importPath to a repoRoot 583 // using the mappings defined in vcsPaths. 584 // If scheme is non-empty, that scheme is forced. 585 func repoRootFromVCSPaths(importPath, scheme string, security web.SecurityMode, vcsPaths []*vcsPath) (*repoRoot, error) { 586 // A common error is to use https://packagepath because that's what 587 // hg and git require. Diagnose this helpfully. 588 if loc := httpPrefixRE.FindStringIndex(importPath); loc != nil { 589 // The importPath has been cleaned, so has only one slash. The pattern 590 // ignores the slashes; the error message puts them back on the RHS at least. 591 return nil, fmt.Errorf("%q not allowed in import path", importPath[loc[0]:loc[1]]+"//") 592 } 593 for _, srv := range vcsPaths { 594 if !strings.HasPrefix(importPath, srv.prefix) { 595 continue 596 } 597 m := srv.regexp.FindStringSubmatch(importPath) 598 if m == nil { 599 if srv.prefix != "" { 600 return nil, fmt.Errorf("invalid %s import path %q", srv.prefix, importPath) 601 } 602 continue 603 } 604 605 // Build map of named subexpression matches for expand. 606 match := map[string]string{ 607 "prefix": srv.prefix, 608 "import": importPath, 609 } 610 for i, name := range srv.regexp.SubexpNames() { 611 if name != "" && match[name] == "" { 612 match[name] = m[i] 613 } 614 } 615 if srv.vcs != "" { 616 match["vcs"] = expand(match, srv.vcs) 617 } 618 if srv.repo != "" { 619 match["repo"] = expand(match, srv.repo) 620 } 621 if srv.check != nil { 622 if err := srv.check(match); err != nil { 623 return nil, err 624 } 625 } 626 vcs := vcsByCmd(match["vcs"]) 627 if vcs == nil { 628 return nil, fmt.Errorf("unknown version control system %q", match["vcs"]) 629 } 630 if srv.ping { 631 if scheme != "" { 632 match["repo"] = scheme + "://" + match["repo"] 633 } else { 634 for _, scheme := range vcs.scheme { 635 if security == web.Secure && !vcs.isSecureScheme(scheme) { 636 continue 637 } 638 if vcs.ping(scheme, match["repo"]) == nil { 639 match["repo"] = scheme + "://" + match["repo"] 640 break 641 } 642 } 643 } 644 } 645 rr := &repoRoot{ 646 vcs: vcs, 647 repo: match["repo"], 648 root: match["root"], 649 } 650 return rr, nil 651 } 652 return nil, errUnknownSite 653 } 654 655 // repoRootForImportDynamic finds a *repoRoot for a custom domain that's not 656 // statically known by repoRootForImportPathStatic. 657 // 658 // This handles custom import paths like "name.tld/pkg/foo" or just "name.tld". 659 func repoRootForImportDynamic(importPath string, security web.SecurityMode) (*repoRoot, error) { 660 slash := strings.Index(importPath, "/") 661 if slash < 0 { 662 slash = len(importPath) 663 } 664 host := importPath[:slash] 665 if !strings.Contains(host, ".") { 666 return nil, errors.New("import path does not begin with hostname") 667 } 668 urlStr, body, err := web.GetMaybeInsecure(importPath, security) 669 if err != nil { 670 msg := "https fetch: %v" 671 if security == web.Insecure { 672 msg = "http/" + msg 673 } 674 return nil, fmt.Errorf(msg, err) 675 } 676 defer body.Close() 677 imports, err := parseMetaGoImports(body) 678 if err != nil { 679 return nil, fmt.Errorf("parsing %s: %v", importPath, err) 680 } 681 // Find the matched meta import. 682 mmi, err := matchGoImport(imports, importPath) 683 if err != nil { 684 if _, ok := err.(ImportMismatchError); !ok { 685 return nil, fmt.Errorf("parse %s: %v", urlStr, err) 686 } 687 return nil, fmt.Errorf("parse %s: no go-import meta tags (%s)", urlStr, err) 688 } 689 if cfg.BuildV { 690 log.Printf("get %q: found meta tag %#v at %s", importPath, mmi, urlStr) 691 } 692 // If the import was "uni.edu/bob/project", which said the 693 // prefix was "uni.edu" and the RepoRoot was "evilroot.com", 694 // make sure we don't trust Bob and check out evilroot.com to 695 // "uni.edu" yet (possibly overwriting/preempting another 696 // non-evil student). Instead, first verify the root and see 697 // if it matches Bob's claim. 698 if mmi.Prefix != importPath { 699 if cfg.BuildV { 700 log.Printf("get %q: verifying non-authoritative meta tag", importPath) 701 } 702 urlStr0 := urlStr 703 var imports []metaImport 704 urlStr, imports, err = metaImportsForPrefix(mmi.Prefix, security) 705 if err != nil { 706 return nil, err 707 } 708 metaImport2, err := matchGoImport(imports, importPath) 709 if err != nil || mmi != metaImport2 { 710 return nil, fmt.Errorf("%s and %s disagree about go-import for %s", urlStr0, urlStr, mmi.Prefix) 711 } 712 } 713 714 if !strings.Contains(mmi.RepoRoot, "://") { 715 return nil, fmt.Errorf("%s: invalid repo root %q; no scheme", urlStr, mmi.RepoRoot) 716 } 717 rr := &repoRoot{ 718 vcs: vcsByCmd(mmi.VCS), 719 repo: mmi.RepoRoot, 720 root: mmi.Prefix, 721 isCustom: true, 722 } 723 if rr.vcs == nil { 724 return nil, fmt.Errorf("%s: unknown vcs %q", urlStr, mmi.VCS) 725 } 726 return rr, nil 727 } 728 729 var fetchGroup singleflight.Group 730 var ( 731 fetchCacheMu sync.Mutex 732 fetchCache = map[string]fetchResult{} // key is metaImportsForPrefix's importPrefix 733 ) 734 735 // metaImportsForPrefix takes a package's root import path as declared in a <meta> tag 736 // and returns its HTML discovery URL and the parsed metaImport lines 737 // found on the page. 738 // 739 // The importPath is of the form "golang.org/x/tools". 740 // It is an error if no imports are found. 741 // urlStr will still be valid if err != nil. 742 // The returned urlStr will be of the form "https://golang.org/x/tools?go-get=1" 743 func metaImportsForPrefix(importPrefix string, security web.SecurityMode) (urlStr string, imports []metaImport, err error) { 744 setCache := func(res fetchResult) (fetchResult, error) { 745 fetchCacheMu.Lock() 746 defer fetchCacheMu.Unlock() 747 fetchCache[importPrefix] = res 748 return res, nil 749 } 750 751 resi, _, _ := fetchGroup.Do(importPrefix, func() (resi interface{}, err error) { 752 fetchCacheMu.Lock() 753 if res, ok := fetchCache[importPrefix]; ok { 754 fetchCacheMu.Unlock() 755 return res, nil 756 } 757 fetchCacheMu.Unlock() 758 759 urlStr, body, err := web.GetMaybeInsecure(importPrefix, security) 760 if err != nil { 761 return setCache(fetchResult{urlStr: urlStr, err: fmt.Errorf("fetch %s: %v", urlStr, err)}) 762 } 763 imports, err := parseMetaGoImports(body) 764 if err != nil { 765 return setCache(fetchResult{urlStr: urlStr, err: fmt.Errorf("parsing %s: %v", urlStr, err)}) 766 } 767 if len(imports) == 0 { 768 err = fmt.Errorf("fetch %s: no go-import meta tag", urlStr) 769 } 770 return setCache(fetchResult{urlStr: urlStr, imports: imports, err: err}) 771 }) 772 res := resi.(fetchResult) 773 return res.urlStr, res.imports, res.err 774 } 775 776 type fetchResult struct { 777 urlStr string // e.g. "https://foo.com/x/bar?go-get=1" 778 imports []metaImport 779 err error 780 } 781 782 // metaImport represents the parsed <meta name="go-import" 783 // content="prefix vcs reporoot" /> tags from HTML files. 784 type metaImport struct { 785 Prefix, VCS, RepoRoot string 786 } 787 788 func splitPathHasPrefix(path, prefix []string) bool { 789 if len(path) < len(prefix) { 790 return false 791 } 792 for i, p := range prefix { 793 if path[i] != p { 794 return false 795 } 796 } 797 return true 798 } 799 800 // A ImportMismatchError is returned where metaImport/s are present 801 // but none match our import path. 802 type ImportMismatchError struct { 803 importPath string 804 mismatches []string // the meta imports that were discarded for not matching our importPath 805 } 806 807 func (m ImportMismatchError) Error() string { 808 formattedStrings := make([]string, len(m.mismatches)) 809 for i, pre := range m.mismatches { 810 formattedStrings[i] = fmt.Sprintf("meta tag %s did not match import path %s", pre, m.importPath) 811 } 812 return strings.Join(formattedStrings, ", ") 813 } 814 815 // matchGoImport returns the metaImport from imports matching importPath. 816 // An error is returned if there are multiple matches. 817 // errNoMatch is returned if none match. 818 func matchGoImport(imports []metaImport, importPath string) (metaImport, error) { 819 match := -1 820 imp := strings.Split(importPath, "/") 821 822 errImportMismatch := ImportMismatchError{importPath: importPath} 823 for i, im := range imports { 824 pre := strings.Split(im.Prefix, "/") 825 826 if !splitPathHasPrefix(imp, pre) { 827 errImportMismatch.mismatches = append(errImportMismatch.mismatches, im.Prefix) 828 continue 829 } 830 831 if match != -1 { 832 return metaImport{}, fmt.Errorf("multiple meta tags match import path %q", importPath) 833 } 834 match = i 835 } 836 837 if match == -1 { 838 return metaImport{}, errImportMismatch 839 } 840 return imports[match], nil 841 } 842 843 // expand rewrites s to replace {k} with match[k] for each key k in match. 844 func expand(match map[string]string, s string) string { 845 for k, v := range match { 846 s = strings.Replace(s, "{"+k+"}", v, -1) 847 } 848 return s 849 } 850 851 // vcsPaths defines the meaning of import paths referring to 852 // commonly-used VCS hosting sites (github.com/user/dir) 853 // and import paths referring to a fully-qualified importPath 854 // containing a VCS type (foo.com/repo.git/dir) 855 var vcsPaths = []*vcsPath{ 856 // Github 857 { 858 prefix: "github.com/", 859 re: `^(?P<root>github\.com/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(/[\p{L}0-9_.\-]+)*$`, 860 vcs: "git", 861 repo: "https://{root}", 862 check: noVCSSuffix, 863 }, 864 865 // Bitbucket 866 { 867 prefix: "bitbucket.org/", 868 re: `^(?P<root>bitbucket\.org/(?P<bitname>[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+))(/[A-Za-z0-9_.\-]+)*$`, 869 repo: "https://{root}", 870 check: bitbucketVCS, 871 }, 872 873 // IBM DevOps Services (JazzHub) 874 { 875 prefix: "hub.jazz.net/git", 876 re: `^(?P<root>hub.jazz.net/git/[a-z0-9]+/[A-Za-z0-9_.\-]+)(/[A-Za-z0-9_.\-]+)*$`, 877 vcs: "git", 878 repo: "https://{root}", 879 check: noVCSSuffix, 880 }, 881 882 // Git at Apache 883 { 884 prefix: "git.apache.org", 885 re: `^(?P<root>git.apache.org/[a-z0-9_.\-]+\.git)(/[A-Za-z0-9_.\-]+)*$`, 886 vcs: "git", 887 repo: "https://{root}", 888 }, 889 890 // Git at OpenStack 891 { 892 prefix: "git.openstack.org", 893 re: `^(?P<root>git\.openstack\.org/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(\.git)?(/[A-Za-z0-9_.\-]+)*$`, 894 vcs: "git", 895 repo: "https://{root}", 896 }, 897 898 // General syntax for any server. 899 // Must be last. 900 { 901 re: `^(?P<root>(?P<repo>([a-z0-9.\-]+\.)+[a-z0-9.\-]+(:[0-9]+)?(/~?[A-Za-z0-9_.\-]+)+?)\.(?P<vcs>bzr|git|hg|svn))(/~?[A-Za-z0-9_.\-]+)*$`, 902 ping: true, 903 }, 904 } 905 906 // vcsPathsAfterDynamic gives additional vcsPaths entries 907 // to try after the dynamic HTML check. 908 // This gives those sites a chance to introduce <meta> tags 909 // as part of a graceful transition away from the hard-coded logic. 910 var vcsPathsAfterDynamic = []*vcsPath{ 911 // Launchpad. See golang.org/issue/11436. 912 { 913 prefix: "launchpad.net/", 914 re: `^(?P<root>launchpad\.net/((?P<project>[A-Za-z0-9_.\-]+)(?P<series>/[A-Za-z0-9_.\-]+)?|~[A-Za-z0-9_.\-]+/(\+junk|[A-Za-z0-9_.\-]+)/[A-Za-z0-9_.\-]+))(/[A-Za-z0-9_.\-]+)*$`, 915 vcs: "bzr", 916 repo: "https://{root}", 917 check: launchpadVCS, 918 }, 919 } 920 921 func init() { 922 // fill in cached regexps. 923 // Doing this eagerly discovers invalid regexp syntax 924 // without having to run a command that needs that regexp. 925 for _, srv := range vcsPaths { 926 srv.regexp = regexp.MustCompile(srv.re) 927 } 928 for _, srv := range vcsPathsAfterDynamic { 929 srv.regexp = regexp.MustCompile(srv.re) 930 } 931 } 932 933 // noVCSSuffix checks that the repository name does not 934 // end in .foo for any version control system foo. 935 // The usual culprit is ".git". 936 func noVCSSuffix(match map[string]string) error { 937 repo := match["repo"] 938 for _, vcs := range vcsList { 939 if strings.HasSuffix(repo, "."+vcs.cmd) { 940 return fmt.Errorf("invalid version control suffix in %s path", match["prefix"]) 941 } 942 } 943 return nil 944 } 945 946 // bitbucketVCS determines the version control system for a 947 // Bitbucket repository, by using the Bitbucket API. 948 func bitbucketVCS(match map[string]string) error { 949 if err := noVCSSuffix(match); err != nil { 950 return err 951 } 952 953 var resp struct { 954 SCM string `json:"scm"` 955 } 956 url := expand(match, "https://api.bitbucket.org/2.0/repositories/{bitname}?fields=scm") 957 data, err := web.Get(url) 958 if err != nil { 959 if httpErr, ok := err.(*web.HTTPError); ok && httpErr.StatusCode == 403 { 960 // this may be a private repository. If so, attempt to determine which 961 // VCS it uses. See issue 5375. 962 root := match["root"] 963 for _, vcs := range []string{"git", "hg"} { 964 if vcsByCmd(vcs).ping("https", root) == nil { 965 resp.SCM = vcs 966 break 967 } 968 } 969 } 970 971 if resp.SCM == "" { 972 return err 973 } 974 } else { 975 if err := json.Unmarshal(data, &resp); err != nil { 976 return fmt.Errorf("decoding %s: %v", url, err) 977 } 978 } 979 980 if vcsByCmd(resp.SCM) != nil { 981 match["vcs"] = resp.SCM 982 if resp.SCM == "git" { 983 match["repo"] += ".git" 984 } 985 return nil 986 } 987 988 return fmt.Errorf("unable to detect version control system for bitbucket.org/ path") 989 } 990 991 // launchpadVCS solves the ambiguity for "lp.net/project/foo". In this case, 992 // "foo" could be a series name registered in Launchpad with its own branch, 993 // and it could also be the name of a directory within the main project 994 // branch one level up. 995 func launchpadVCS(match map[string]string) error { 996 if match["project"] == "" || match["series"] == "" { 997 return nil 998 } 999 _, err := web.Get(expand(match, "https://code.launchpad.net/{project}{series}/.bzr/branch-format")) 1000 if err != nil { 1001 match["root"] = expand(match, "launchpad.net/{project}") 1002 match["repo"] = expand(match, "https://{root}") 1003 } 1004 return nil 1005 }