github.com/go-asm/go@v1.21.1-0.20240213172139-40c5ead50c48/cmd/go/vcs/vcs.go (about) 1 // Copyright 2012 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package vcs 6 7 import ( 8 "bytes" 9 "errors" 10 "fmt" 11 "io/fs" 12 "log" 13 urlpkg "net/url" 14 "os" 15 "os/exec" 16 "path/filepath" 17 "regexp" 18 "strconv" 19 "strings" 20 "sync" 21 "time" 22 23 "github.com/go-asm/go/lazyregexp" 24 "github.com/go-asm/go/singleflight" 25 26 "github.com/go-asm/go/cmd/go/base" 27 "github.com/go-asm/go/cmd/go/cfg" 28 "github.com/go-asm/go/cmd/go/search" 29 "github.com/go-asm/go/cmd/go/str" 30 "github.com/go-asm/go/cmd/go/web" 31 32 "golang.org/x/mod/module" 33 ) 34 35 // A Cmd describes how to use a version control system 36 // like Mercurial, Git, or Subversion. 37 type Cmd struct { 38 Name string 39 Cmd string // name of binary to invoke command 40 RootNames []rootName // filename and mode indicating the root of a checkout directory 41 42 CreateCmd []string // commands to download a fresh copy of a repository 43 DownloadCmd []string // commands to download updates into an existing repository 44 45 TagCmd []tagCmd // commands to list tags 46 TagLookupCmd []tagCmd // commands to lookup tags before running tagSyncCmd 47 TagSyncCmd []string // commands to sync to specific tag 48 TagSyncDefault []string // commands to sync to default tag 49 50 Scheme []string 51 PingCmd string 52 53 RemoteRepo func(v *Cmd, rootDir string) (remoteRepo string, err error) 54 ResolveRepo func(v *Cmd, rootDir, remoteRepo string) (realRepo string, err error) 55 Status func(v *Cmd, rootDir string) (Status, error) 56 } 57 58 // Status is the current state of a local repository. 59 type Status struct { 60 Revision string // Optional. 61 CommitTime time.Time // Optional. 62 Uncommitted bool // Required. 63 } 64 65 var ( 66 // VCSTestRepoURL is the URL of the HTTP server that serves the repos for 67 // vcs-test.golang.org. 68 // 69 // In tests, this is set to the URL of an httptest.Server hosting a 70 // github.com/go-asm/go/cmd/go/vcweb.Server. 71 VCSTestRepoURL string 72 73 // VCSTestHosts is the set of hosts supported by the vcs-test server. 74 VCSTestHosts []string 75 76 // VCSTestIsLocalHost reports whether the given URL refers to a local 77 // (loopback) host, such as "localhost" or "127.0.0.1:8080". 78 VCSTestIsLocalHost func(*urlpkg.URL) bool 79 ) 80 81 var defaultSecureScheme = map[string]bool{ 82 "https": true, 83 "git+ssh": true, 84 "bzr+ssh": true, 85 "svn+ssh": true, 86 "ssh": true, 87 } 88 89 func (v *Cmd) IsSecure(repo string) bool { 90 u, err := urlpkg.Parse(repo) 91 if err != nil { 92 // If repo is not a URL, it's not secure. 93 return false 94 } 95 if VCSTestRepoURL != "" && web.IsLocalHost(u) { 96 // If the vcstest server is in use, it may redirect to other local ports for 97 // other protocols (such as svn). Assume that all loopback addresses are 98 // secure during testing. 99 return true 100 } 101 return v.isSecureScheme(u.Scheme) 102 } 103 104 func (v *Cmd) isSecureScheme(scheme string) bool { 105 switch v.Cmd { 106 case "git": 107 // GIT_ALLOW_PROTOCOL is an environment variable defined by Git. It is a 108 // colon-separated list of schemes that are allowed to be used with git 109 // fetch/clone. Any scheme not mentioned will be considered insecure. 110 if allow := os.Getenv("GIT_ALLOW_PROTOCOL"); allow != "" { 111 for _, s := range strings.Split(allow, ":") { 112 if s == scheme { 113 return true 114 } 115 } 116 return false 117 } 118 } 119 return defaultSecureScheme[scheme] 120 } 121 122 // A tagCmd describes a command to list available tags 123 // that can be passed to tagSyncCmd. 124 type tagCmd struct { 125 cmd string // command to list tags 126 pattern string // regexp to extract tags from list 127 } 128 129 // vcsList lists the known version control systems 130 var vcsList = []*Cmd{ 131 vcsHg, 132 vcsGit, 133 vcsSvn, 134 vcsBzr, 135 vcsFossil, 136 } 137 138 // vcsMod is a stub for the "mod" scheme. It's returned by 139 // repoRootForImportPathDynamic, but is otherwise not treated as a VCS command. 140 var vcsMod = &Cmd{Name: "mod"} 141 142 // vcsByCmd returns the version control system for the given 143 // command name (hg, git, svn, bzr). 144 func vcsByCmd(cmd string) *Cmd { 145 for _, vcs := range vcsList { 146 if vcs.Cmd == cmd { 147 return vcs 148 } 149 } 150 return nil 151 } 152 153 // vcsHg describes how to use Mercurial. 154 var vcsHg = &Cmd{ 155 Name: "Mercurial", 156 Cmd: "hg", 157 RootNames: []rootName{ 158 {filename: ".hg", isDir: true}, 159 }, 160 161 CreateCmd: []string{"clone -U -- {repo} {dir}"}, 162 DownloadCmd: []string{"pull"}, 163 164 // We allow both tag and branch names as 'tags' 165 // for selecting a version. This lets people have 166 // a go.release.r60 branch and a go1 branch 167 // and make changes in both, without constantly 168 // editing .hgtags. 169 TagCmd: []tagCmd{ 170 {"tags", `^(\S+)`}, 171 {"branches", `^(\S+)`}, 172 }, 173 TagSyncCmd: []string{"update -r {tag}"}, 174 TagSyncDefault: []string{"update default"}, 175 176 Scheme: []string{"https", "http", "ssh"}, 177 PingCmd: "identify -- {scheme}://{repo}", 178 RemoteRepo: hgRemoteRepo, 179 Status: hgStatus, 180 } 181 182 func hgRemoteRepo(vcsHg *Cmd, rootDir string) (remoteRepo string, err error) { 183 out, err := vcsHg.runOutput(rootDir, "paths default") 184 if err != nil { 185 return "", err 186 } 187 return strings.TrimSpace(string(out)), nil 188 } 189 190 func hgStatus(vcsHg *Cmd, rootDir string) (Status, error) { 191 // Output changeset ID and seconds since epoch. 192 out, err := vcsHg.runOutputVerboseOnly(rootDir, `log -l1 -T {node}:{date|hgdate}`) 193 if err != nil { 194 return Status{}, err 195 } 196 197 // Successful execution without output indicates an empty repo (no commits). 198 var rev string 199 var commitTime time.Time 200 if len(out) > 0 { 201 // Strip trailing timezone offset. 202 if i := bytes.IndexByte(out, ' '); i > 0 { 203 out = out[:i] 204 } 205 rev, commitTime, err = parseRevTime(out) 206 if err != nil { 207 return Status{}, err 208 } 209 } 210 211 // Also look for untracked files. 212 out, err = vcsHg.runOutputVerboseOnly(rootDir, "status") 213 if err != nil { 214 return Status{}, err 215 } 216 uncommitted := len(out) > 0 217 218 return Status{ 219 Revision: rev, 220 CommitTime: commitTime, 221 Uncommitted: uncommitted, 222 }, nil 223 } 224 225 // parseRevTime parses commit details in "revision:seconds" format. 226 func parseRevTime(out []byte) (string, time.Time, error) { 227 buf := string(bytes.TrimSpace(out)) 228 229 i := strings.IndexByte(buf, ':') 230 if i < 1 { 231 return "", time.Time{}, errors.New("unrecognized VCS tool output") 232 } 233 rev := buf[:i] 234 235 secs, err := strconv.ParseInt(string(buf[i+1:]), 10, 64) 236 if err != nil { 237 return "", time.Time{}, fmt.Errorf("unrecognized VCS tool output: %v", err) 238 } 239 240 return rev, time.Unix(secs, 0), nil 241 } 242 243 // vcsGit describes how to use Git. 244 var vcsGit = &Cmd{ 245 Name: "Git", 246 Cmd: "git", 247 RootNames: []rootName{ 248 {filename: ".git", isDir: true}, 249 }, 250 251 CreateCmd: []string{"clone -- {repo} {dir}", "-go-internal-cd {dir} submodule update --init --recursive"}, 252 DownloadCmd: []string{"pull --ff-only", "submodule update --init --recursive"}, 253 254 TagCmd: []tagCmd{ 255 // tags/xxx matches a git tag named xxx 256 // origin/xxx matches a git branch named xxx on the default remote repository 257 {"show-ref", `(?:tags|origin)/(\S+)$`}, 258 }, 259 TagLookupCmd: []tagCmd{ 260 {"show-ref tags/{tag} origin/{tag}", `((?:tags|origin)/\S+)$`}, 261 }, 262 TagSyncCmd: []string{"checkout {tag}", "submodule update --init --recursive"}, 263 // both createCmd and downloadCmd update the working dir. 264 // No need to do more here. We used to 'checkout master' 265 // but that doesn't work if the default branch is not named master. 266 // DO NOT add 'checkout master' here. 267 // See golang.org/issue/9032. 268 TagSyncDefault: []string{"submodule update --init --recursive"}, 269 270 Scheme: []string{"git", "https", "http", "git+ssh", "ssh"}, 271 272 // Leave out the '--' separator in the ls-remote command: git 2.7.4 does not 273 // support such a separator for that command, and this use should be safe 274 // without it because the {scheme} value comes from the predefined list above. 275 // See golang.org/issue/33836. 276 PingCmd: "ls-remote {scheme}://{repo}", 277 278 RemoteRepo: gitRemoteRepo, 279 Status: gitStatus, 280 } 281 282 // scpSyntaxRe matches the SCP-like addresses used by Git to access 283 // repositories by SSH. 284 var scpSyntaxRe = lazyregexp.New(`^(\w+)@([\w.-]+):(.*)$`) 285 286 func gitRemoteRepo(vcsGit *Cmd, rootDir string) (remoteRepo string, err error) { 287 const cmd = "config remote.origin.url" 288 outb, err := vcsGit.run1(rootDir, cmd, nil, false) 289 if err != nil { 290 // if it doesn't output any message, it means the config argument is correct, 291 // but the config value itself doesn't exist 292 if outb != nil && len(outb) == 0 { 293 return "", errors.New("remote origin not found") 294 } 295 return "", err 296 } 297 out := strings.TrimSpace(string(outb)) 298 299 var repoURL *urlpkg.URL 300 if m := scpSyntaxRe.FindStringSubmatch(out); m != nil { 301 // Match SCP-like syntax and convert it to a URL. 302 // Eg, "git@github.com:user/repo" becomes 303 // "ssh://git@github.com/user/repo". 304 repoURL = &urlpkg.URL{ 305 Scheme: "ssh", 306 User: urlpkg.User(m[1]), 307 Host: m[2], 308 Path: m[3], 309 } 310 } else { 311 repoURL, err = urlpkg.Parse(out) 312 if err != nil { 313 return "", err 314 } 315 } 316 317 // Iterate over insecure schemes too, because this function simply 318 // reports the state of the repo. If we can't see insecure schemes then 319 // we can't report the actual repo URL. 320 for _, s := range vcsGit.Scheme { 321 if repoURL.Scheme == s { 322 return repoURL.String(), nil 323 } 324 } 325 return "", errors.New("unable to parse output of git " + cmd) 326 } 327 328 func gitStatus(vcsGit *Cmd, rootDir string) (Status, error) { 329 out, err := vcsGit.runOutputVerboseOnly(rootDir, "status --porcelain") 330 if err != nil { 331 return Status{}, err 332 } 333 uncommitted := len(out) > 0 334 335 // "git status" works for empty repositories, but "git show" does not. 336 // Assume there are no commits in the repo when "git show" fails with 337 // uncommitted files and skip tagging revision / committime. 338 var rev string 339 var commitTime time.Time 340 out, err = vcsGit.runOutputVerboseOnly(rootDir, "-c log.showsignature=false show -s --format=%H:%ct") 341 if err != nil && !uncommitted { 342 return Status{}, err 343 } else if err == nil { 344 rev, commitTime, err = parseRevTime(out) 345 if err != nil { 346 return Status{}, err 347 } 348 } 349 350 return Status{ 351 Revision: rev, 352 CommitTime: commitTime, 353 Uncommitted: uncommitted, 354 }, nil 355 } 356 357 // vcsBzr describes how to use Bazaar. 358 var vcsBzr = &Cmd{ 359 Name: "Bazaar", 360 Cmd: "bzr", 361 RootNames: []rootName{ 362 {filename: ".bzr", isDir: true}, 363 }, 364 365 CreateCmd: []string{"branch -- {repo} {dir}"}, 366 367 // Without --overwrite bzr will not pull tags that changed. 368 // Replace by --overwrite-tags after http://pad.lv/681792 goes in. 369 DownloadCmd: []string{"pull --overwrite"}, 370 371 TagCmd: []tagCmd{{"tags", `^(\S+)`}}, 372 TagSyncCmd: []string{"update -r {tag}"}, 373 TagSyncDefault: []string{"update -r revno:-1"}, 374 375 Scheme: []string{"https", "http", "bzr", "bzr+ssh"}, 376 PingCmd: "info -- {scheme}://{repo}", 377 RemoteRepo: bzrRemoteRepo, 378 ResolveRepo: bzrResolveRepo, 379 Status: bzrStatus, 380 } 381 382 func bzrRemoteRepo(vcsBzr *Cmd, rootDir string) (remoteRepo string, err error) { 383 outb, err := vcsBzr.runOutput(rootDir, "config parent_location") 384 if err != nil { 385 return "", err 386 } 387 return strings.TrimSpace(string(outb)), nil 388 } 389 390 func bzrResolveRepo(vcsBzr *Cmd, rootDir, remoteRepo string) (realRepo string, err error) { 391 outb, err := vcsBzr.runOutput(rootDir, "info "+remoteRepo) 392 if err != nil { 393 return "", err 394 } 395 out := string(outb) 396 397 // Expect: 398 // ... 399 // (branch root|repository branch): <URL> 400 // ... 401 402 found := false 403 for _, prefix := range []string{"\n branch root: ", "\n repository branch: "} { 404 i := strings.Index(out, prefix) 405 if i >= 0 { 406 out = out[i+len(prefix):] 407 found = true 408 break 409 } 410 } 411 if !found { 412 return "", fmt.Errorf("unable to parse output of bzr info") 413 } 414 415 i := strings.Index(out, "\n") 416 if i < 0 { 417 return "", fmt.Errorf("unable to parse output of bzr info") 418 } 419 out = out[:i] 420 return strings.TrimSpace(out), nil 421 } 422 423 func bzrStatus(vcsBzr *Cmd, rootDir string) (Status, error) { 424 outb, err := vcsBzr.runOutputVerboseOnly(rootDir, "version-info") 425 if err != nil { 426 return Status{}, err 427 } 428 out := string(outb) 429 430 // Expect (non-empty repositories only): 431 // 432 // revision-id: gopher@gopher.net-20211021072330-qshok76wfypw9lpm 433 // date: 2021-09-21 12:00:00 +1000 434 // ... 435 var rev string 436 var commitTime time.Time 437 438 for _, line := range strings.Split(out, "\n") { 439 i := strings.IndexByte(line, ':') 440 if i < 0 { 441 continue 442 } 443 key := line[:i] 444 value := strings.TrimSpace(line[i+1:]) 445 446 switch key { 447 case "revision-id": 448 rev = value 449 case "date": 450 var err error 451 commitTime, err = time.Parse("2006-01-02 15:04:05 -0700", value) 452 if err != nil { 453 return Status{}, errors.New("unable to parse output of bzr version-info") 454 } 455 } 456 } 457 458 outb, err = vcsBzr.runOutputVerboseOnly(rootDir, "status") 459 if err != nil { 460 return Status{}, err 461 } 462 463 // Skip warning when working directory is set to an older revision. 464 if bytes.HasPrefix(outb, []byte("working tree is out of date")) { 465 i := bytes.IndexByte(outb, '\n') 466 if i < 0 { 467 i = len(outb) 468 } 469 outb = outb[:i] 470 } 471 uncommitted := len(outb) > 0 472 473 return Status{ 474 Revision: rev, 475 CommitTime: commitTime, 476 Uncommitted: uncommitted, 477 }, nil 478 } 479 480 // vcsSvn describes how to use Subversion. 481 var vcsSvn = &Cmd{ 482 Name: "Subversion", 483 Cmd: "svn", 484 RootNames: []rootName{ 485 {filename: ".svn", isDir: true}, 486 }, 487 488 CreateCmd: []string{"checkout -- {repo} {dir}"}, 489 DownloadCmd: []string{"update"}, 490 491 // There is no tag command in subversion. 492 // The branch information is all in the path names. 493 494 Scheme: []string{"https", "http", "svn", "svn+ssh"}, 495 PingCmd: "info -- {scheme}://{repo}", 496 RemoteRepo: svnRemoteRepo, 497 } 498 499 func svnRemoteRepo(vcsSvn *Cmd, rootDir string) (remoteRepo string, err error) { 500 outb, err := vcsSvn.runOutput(rootDir, "info") 501 if err != nil { 502 return "", err 503 } 504 out := string(outb) 505 506 // Expect: 507 // 508 // ... 509 // URL: <URL> 510 // ... 511 // 512 // Note that we're not using the Repository Root line, 513 // because svn allows checking out subtrees. 514 // The URL will be the URL of the subtree (what we used with 'svn co') 515 // while the Repository Root may be a much higher parent. 516 i := strings.Index(out, "\nURL: ") 517 if i < 0 { 518 return "", fmt.Errorf("unable to parse output of svn info") 519 } 520 out = out[i+len("\nURL: "):] 521 i = strings.Index(out, "\n") 522 if i < 0 { 523 return "", fmt.Errorf("unable to parse output of svn info") 524 } 525 out = out[:i] 526 return strings.TrimSpace(out), nil 527 } 528 529 // fossilRepoName is the name go get associates with a fossil repository. In the 530 // real world the file can be named anything. 531 const fossilRepoName = ".fossil" 532 533 // vcsFossil describes how to use Fossil (fossil-scm.org) 534 var vcsFossil = &Cmd{ 535 Name: "Fossil", 536 Cmd: "fossil", 537 RootNames: []rootName{ 538 {filename: ".fslckout", isDir: false}, 539 {filename: "_FOSSIL_", isDir: false}, 540 }, 541 542 CreateCmd: []string{"-go-internal-mkdir {dir} clone -- {repo} " + filepath.Join("{dir}", fossilRepoName), "-go-internal-cd {dir} open .fossil"}, 543 DownloadCmd: []string{"up"}, 544 545 TagCmd: []tagCmd{{"tag ls", `(.*)`}}, 546 TagSyncCmd: []string{"up tag:{tag}"}, 547 TagSyncDefault: []string{"up trunk"}, 548 549 Scheme: []string{"https", "http"}, 550 RemoteRepo: fossilRemoteRepo, 551 Status: fossilStatus, 552 } 553 554 func fossilRemoteRepo(vcsFossil *Cmd, rootDir string) (remoteRepo string, err error) { 555 out, err := vcsFossil.runOutput(rootDir, "remote-url") 556 if err != nil { 557 return "", err 558 } 559 return strings.TrimSpace(string(out)), nil 560 } 561 562 var errFossilInfo = errors.New("unable to parse output of fossil info") 563 564 func fossilStatus(vcsFossil *Cmd, rootDir string) (Status, error) { 565 outb, err := vcsFossil.runOutputVerboseOnly(rootDir, "info") 566 if err != nil { 567 return Status{}, err 568 } 569 out := string(outb) 570 571 // Expect: 572 // ... 573 // checkout: 91ed71f22c77be0c3e250920f47bfd4e1f9024d2 2021-09-21 12:00:00 UTC 574 // ... 575 576 // Extract revision and commit time. 577 // Ensure line ends with UTC (known timezone offset). 578 const prefix = "\ncheckout:" 579 const suffix = " UTC" 580 i := strings.Index(out, prefix) 581 if i < 0 { 582 return Status{}, errFossilInfo 583 } 584 checkout := out[i+len(prefix):] 585 i = strings.Index(checkout, suffix) 586 if i < 0 { 587 return Status{}, errFossilInfo 588 } 589 checkout = strings.TrimSpace(checkout[:i]) 590 591 i = strings.IndexByte(checkout, ' ') 592 if i < 0 { 593 return Status{}, errFossilInfo 594 } 595 rev := checkout[:i] 596 597 commitTime, err := time.ParseInLocation(time.DateTime, checkout[i+1:], time.UTC) 598 if err != nil { 599 return Status{}, fmt.Errorf("%v: %v", errFossilInfo, err) 600 } 601 602 // Also look for untracked changes. 603 outb, err = vcsFossil.runOutputVerboseOnly(rootDir, "changes --differ") 604 if err != nil { 605 return Status{}, err 606 } 607 uncommitted := len(outb) > 0 608 609 return Status{ 610 Revision: rev, 611 CommitTime: commitTime, 612 Uncommitted: uncommitted, 613 }, nil 614 } 615 616 func (v *Cmd) String() string { 617 return v.Name 618 } 619 620 // run runs the command line cmd in the given directory. 621 // keyval is a list of key, value pairs. run expands 622 // instances of {key} in cmd into value, but only after 623 // splitting cmd into individual arguments. 624 // If an error occurs, run prints the command line and the 625 // command's combined stdout+stderr to standard error. 626 // Otherwise run discards the command's output. 627 func (v *Cmd) run(dir string, cmd string, keyval ...string) error { 628 _, err := v.run1(dir, cmd, keyval, true) 629 return err 630 } 631 632 // runVerboseOnly is like run but only generates error output to standard error in verbose mode. 633 func (v *Cmd) runVerboseOnly(dir string, cmd string, keyval ...string) error { 634 _, err := v.run1(dir, cmd, keyval, false) 635 return err 636 } 637 638 // runOutput is like run but returns the output of the command. 639 func (v *Cmd) runOutput(dir string, cmd string, keyval ...string) ([]byte, error) { 640 return v.run1(dir, cmd, keyval, true) 641 } 642 643 // runOutputVerboseOnly is like runOutput but only generates error output to 644 // standard error in verbose mode. 645 func (v *Cmd) runOutputVerboseOnly(dir string, cmd string, keyval ...string) ([]byte, error) { 646 return v.run1(dir, cmd, keyval, false) 647 } 648 649 // run1 is the generalized implementation of run and runOutput. 650 func (v *Cmd) run1(dir string, cmdline string, keyval []string, verbose bool) ([]byte, error) { 651 m := make(map[string]string) 652 for i := 0; i < len(keyval); i += 2 { 653 m[keyval[i]] = keyval[i+1] 654 } 655 args := strings.Fields(cmdline) 656 for i, arg := range args { 657 args[i] = expand(m, arg) 658 } 659 660 if len(args) >= 2 && args[0] == "-go-internal-mkdir" { 661 var err error 662 if filepath.IsAbs(args[1]) { 663 err = os.Mkdir(args[1], fs.ModePerm) 664 } else { 665 err = os.Mkdir(filepath.Join(dir, args[1]), fs.ModePerm) 666 } 667 if err != nil { 668 return nil, err 669 } 670 args = args[2:] 671 } 672 673 if len(args) >= 2 && args[0] == "-go-internal-cd" { 674 if filepath.IsAbs(args[1]) { 675 dir = args[1] 676 } else { 677 dir = filepath.Join(dir, args[1]) 678 } 679 args = args[2:] 680 } 681 682 _, err := cfg.LookPath(v.Cmd) 683 if err != nil { 684 fmt.Fprintf(os.Stderr, 685 "go: missing %s command. See https://golang.org/s/gogetcmd\n", 686 v.Name) 687 return nil, err 688 } 689 690 cmd := exec.Command(v.Cmd, args...) 691 cmd.Dir = dir 692 if cfg.BuildX { 693 fmt.Fprintf(os.Stderr, "cd %s\n", dir) 694 fmt.Fprintf(os.Stderr, "%s %s\n", v.Cmd, strings.Join(args, " ")) 695 } 696 out, err := cmd.Output() 697 if err != nil { 698 if verbose || cfg.BuildV { 699 fmt.Fprintf(os.Stderr, "# cd %s; %s %s\n", dir, v.Cmd, strings.Join(args, " ")) 700 if ee, ok := err.(*exec.ExitError); ok && len(ee.Stderr) > 0 { 701 os.Stderr.Write(ee.Stderr) 702 } else { 703 fmt.Fprintln(os.Stderr, err.Error()) 704 } 705 } 706 } 707 return out, err 708 } 709 710 // Ping pings to determine scheme to use. 711 func (v *Cmd) Ping(scheme, repo string) error { 712 // Run the ping command in an arbitrary working directory, 713 // but don't let the current working directory pollute the results. 714 // In module mode, we expect GOMODCACHE to exist and be a safe place for 715 // commands; in GOPATH mode, we expect that to be true of GOPATH/src. 716 dir := cfg.GOMODCACHE 717 if !cfg.ModulesEnabled { 718 dir = filepath.Join(cfg.BuildContext.GOPATH, "src") 719 } 720 os.MkdirAll(dir, 0777) // Ignore errors — if unsuccessful, the command will likely fail. 721 722 release, err := base.AcquireNet() 723 if err != nil { 724 return err 725 } 726 defer release() 727 728 return v.runVerboseOnly(dir, v.PingCmd, "scheme", scheme, "repo", repo) 729 } 730 731 // Create creates a new copy of repo in dir. 732 // The parent of dir must exist; dir must not. 733 func (v *Cmd) Create(dir, repo string) error { 734 release, err := base.AcquireNet() 735 if err != nil { 736 return err 737 } 738 defer release() 739 740 for _, cmd := range v.CreateCmd { 741 if err := v.run(filepath.Dir(dir), cmd, "dir", dir, "repo", repo); err != nil { 742 return err 743 } 744 } 745 return nil 746 } 747 748 // Download downloads any new changes for the repo in dir. 749 func (v *Cmd) Download(dir string) error { 750 release, err := base.AcquireNet() 751 if err != nil { 752 return err 753 } 754 defer release() 755 756 for _, cmd := range v.DownloadCmd { 757 if err := v.run(dir, cmd); err != nil { 758 return err 759 } 760 } 761 return nil 762 } 763 764 // Tags returns the list of available tags for the repo in dir. 765 func (v *Cmd) Tags(dir string) ([]string, error) { 766 var tags []string 767 for _, tc := range v.TagCmd { 768 out, err := v.runOutput(dir, tc.cmd) 769 if err != nil { 770 return nil, err 771 } 772 re := regexp.MustCompile(`(?m-s)` + tc.pattern) 773 for _, m := range re.FindAllStringSubmatch(string(out), -1) { 774 tags = append(tags, m[1]) 775 } 776 } 777 return tags, nil 778 } 779 780 // TagSync syncs the repo in dir to the named tag, 781 // which either is a tag returned by tags or is v.tagDefault. 782 func (v *Cmd) TagSync(dir, tag string) error { 783 if v.TagSyncCmd == nil { 784 return nil 785 } 786 if tag != "" { 787 for _, tc := range v.TagLookupCmd { 788 out, err := v.runOutput(dir, tc.cmd, "tag", tag) 789 if err != nil { 790 return err 791 } 792 re := regexp.MustCompile(`(?m-s)` + tc.pattern) 793 m := re.FindStringSubmatch(string(out)) 794 if len(m) > 1 { 795 tag = m[1] 796 break 797 } 798 } 799 } 800 801 release, err := base.AcquireNet() 802 if err != nil { 803 return err 804 } 805 defer release() 806 807 if tag == "" && v.TagSyncDefault != nil { 808 for _, cmd := range v.TagSyncDefault { 809 if err := v.run(dir, cmd); err != nil { 810 return err 811 } 812 } 813 return nil 814 } 815 816 for _, cmd := range v.TagSyncCmd { 817 if err := v.run(dir, cmd, "tag", tag); err != nil { 818 return err 819 } 820 } 821 return nil 822 } 823 824 // A vcsPath describes how to convert an import path into a 825 // version control system and repository name. 826 type vcsPath struct { 827 pathPrefix string // prefix this description applies to 828 regexp *lazyregexp.Regexp // compiled pattern for import path 829 repo string // repository to use (expand with match of re) 830 vcs string // version control system to use (expand with match of re) 831 check func(match map[string]string) error // additional checks 832 schemelessRepo bool // if true, the repo pattern lacks a scheme 833 } 834 835 // FromDir inspects dir and its parents to determine the 836 // version control system and code repository to use. 837 // If no repository is found, FromDir returns an error 838 // equivalent to os.ErrNotExist. 839 func FromDir(dir, srcRoot string, allowNesting bool) (repoDir string, vcsCmd *Cmd, err error) { 840 // Clean and double-check that dir is in (a subdirectory of) srcRoot. 841 dir = filepath.Clean(dir) 842 if srcRoot != "" { 843 srcRoot = filepath.Clean(srcRoot) 844 if len(dir) <= len(srcRoot) || dir[len(srcRoot)] != filepath.Separator { 845 return "", nil, fmt.Errorf("directory %q is outside source root %q", dir, srcRoot) 846 } 847 } 848 849 origDir := dir 850 for len(dir) > len(srcRoot) { 851 for _, vcs := range vcsList { 852 if isVCSRoot(dir, vcs.RootNames) { 853 // Record first VCS we find. 854 // If allowNesting is false (as it is in GOPATH), keep looking for 855 // repositories in parent directories and report an error if one is 856 // found to mitigate VCS injection attacks. 857 if vcsCmd == nil { 858 vcsCmd = vcs 859 repoDir = dir 860 if allowNesting { 861 return repoDir, vcsCmd, nil 862 } 863 continue 864 } 865 // Otherwise, we have one VCS inside a different VCS. 866 return "", nil, fmt.Errorf("directory %q uses %s, but parent %q uses %s", 867 repoDir, vcsCmd.Cmd, dir, vcs.Cmd) 868 } 869 } 870 871 // Move to parent. 872 ndir := filepath.Dir(dir) 873 if len(ndir) >= len(dir) { 874 break 875 } 876 dir = ndir 877 } 878 if vcsCmd == nil { 879 return "", nil, &vcsNotFoundError{dir: origDir} 880 } 881 return repoDir, vcsCmd, nil 882 } 883 884 // isVCSRoot identifies a VCS root by checking whether the directory contains 885 // any of the listed root names. 886 func isVCSRoot(dir string, rootNames []rootName) bool { 887 for _, root := range rootNames { 888 fi, err := os.Stat(filepath.Join(dir, root.filename)) 889 if err == nil && fi.IsDir() == root.isDir { 890 return true 891 } 892 } 893 894 return false 895 } 896 897 type rootName struct { 898 filename string 899 isDir bool 900 } 901 902 type vcsNotFoundError struct { 903 dir string 904 } 905 906 func (e *vcsNotFoundError) Error() string { 907 return fmt.Sprintf("directory %q is not using a known version control system", e.dir) 908 } 909 910 func (e *vcsNotFoundError) Is(err error) bool { 911 return err == os.ErrNotExist 912 } 913 914 // A govcsRule is a single GOVCS rule like private:hg|svn. 915 type govcsRule struct { 916 pattern string 917 allowed []string 918 } 919 920 // A govcsConfig is a full GOVCS configuration. 921 type govcsConfig []govcsRule 922 923 func parseGOVCS(s string) (govcsConfig, error) { 924 s = strings.TrimSpace(s) 925 if s == "" { 926 return nil, nil 927 } 928 var cfg govcsConfig 929 have := make(map[string]string) 930 for _, item := range strings.Split(s, ",") { 931 item = strings.TrimSpace(item) 932 if item == "" { 933 return nil, fmt.Errorf("empty entry in GOVCS") 934 } 935 pattern, list, found := strings.Cut(item, ":") 936 if !found { 937 return nil, fmt.Errorf("malformed entry in GOVCS (missing colon): %q", item) 938 } 939 pattern, list = strings.TrimSpace(pattern), strings.TrimSpace(list) 940 if pattern == "" { 941 return nil, fmt.Errorf("empty pattern in GOVCS: %q", item) 942 } 943 if list == "" { 944 return nil, fmt.Errorf("empty VCS list in GOVCS: %q", item) 945 } 946 if search.IsRelativePath(pattern) { 947 return nil, fmt.Errorf("relative pattern not allowed in GOVCS: %q", pattern) 948 } 949 if old := have[pattern]; old != "" { 950 return nil, fmt.Errorf("unreachable pattern in GOVCS: %q after %q", item, old) 951 } 952 have[pattern] = item 953 allowed := strings.Split(list, "|") 954 for i, a := range allowed { 955 a = strings.TrimSpace(a) 956 if a == "" { 957 return nil, fmt.Errorf("empty VCS name in GOVCS: %q", item) 958 } 959 allowed[i] = a 960 } 961 cfg = append(cfg, govcsRule{pattern, allowed}) 962 } 963 return cfg, nil 964 } 965 966 func (c *govcsConfig) allow(path string, private bool, vcs string) bool { 967 for _, rule := range *c { 968 match := false 969 switch rule.pattern { 970 case "private": 971 match = private 972 case "public": 973 match = !private 974 default: 975 // Note: rule.pattern is known to be comma-free, 976 // so MatchPrefixPatterns is only matching a single pattern for us. 977 match = module.MatchPrefixPatterns(rule.pattern, path) 978 } 979 if !match { 980 continue 981 } 982 for _, allow := range rule.allowed { 983 if allow == vcs || allow == "all" { 984 return true 985 } 986 } 987 return false 988 } 989 990 // By default, nothing is allowed. 991 return false 992 } 993 994 var ( 995 govcs govcsConfig 996 govcsErr error 997 govcsOnce sync.Once 998 ) 999 1000 // defaultGOVCS is the default setting for GOVCS. 1001 // Setting GOVCS adds entries ahead of these but does not remove them. 1002 // (They are appended to the parsed GOVCS setting.) 1003 // 1004 // The rationale behind allowing only Git and Mercurial is that 1005 // these two systems have had the most attention to issues 1006 // of being run as clients of untrusted servers. In contrast, 1007 // Bazaar, Fossil, and Subversion have primarily been used 1008 // in trusted, authenticated environments and are not as well 1009 // scrutinized as attack surfaces. 1010 // 1011 // See golang.org/issue/41730 for details. 1012 var defaultGOVCS = govcsConfig{ 1013 {"private", []string{"all"}}, 1014 {"public", []string{"git", "hg"}}, 1015 } 1016 1017 // checkGOVCS checks whether the policy defined by the environment variable 1018 // GOVCS allows the given vcs command to be used with the given repository 1019 // root path. Note that root may not be a real package or module path; it's 1020 // the same as the root path in the go-import meta tag. 1021 func checkGOVCS(vcs *Cmd, root string) error { 1022 if vcs == vcsMod { 1023 // Direct module (proxy protocol) fetches don't 1024 // involve an external version control system 1025 // and are always allowed. 1026 return nil 1027 } 1028 1029 govcsOnce.Do(func() { 1030 govcs, govcsErr = parseGOVCS(os.Getenv("GOVCS")) 1031 govcs = append(govcs, defaultGOVCS...) 1032 }) 1033 if govcsErr != nil { 1034 return govcsErr 1035 } 1036 1037 private := module.MatchPrefixPatterns(cfg.GOPRIVATE, root) 1038 if !govcs.allow(root, private, vcs.Cmd) { 1039 what := "public" 1040 if private { 1041 what = "private" 1042 } 1043 return fmt.Errorf("GOVCS disallows using %s for %s %s; see 'go help vcs'", vcs.Cmd, what, root) 1044 } 1045 1046 return nil 1047 } 1048 1049 // RepoRoot describes the repository root for a tree of source code. 1050 type RepoRoot struct { 1051 Repo string // repository URL, including scheme 1052 Root string // import path corresponding to root of repo 1053 IsCustom bool // defined by served <meta> tags (as opposed to hard-coded pattern) 1054 VCS *Cmd 1055 } 1056 1057 func httpPrefix(s string) string { 1058 for _, prefix := range [...]string{"http:", "https:"} { 1059 if strings.HasPrefix(s, prefix) { 1060 return prefix 1061 } 1062 } 1063 return "" 1064 } 1065 1066 // ModuleMode specifies whether to prefer modules when looking up code sources. 1067 type ModuleMode int 1068 1069 const ( 1070 IgnoreMod ModuleMode = iota 1071 PreferMod 1072 ) 1073 1074 // RepoRootForImportPath analyzes importPath to determine the 1075 // version control system, and code repository to use. 1076 func RepoRootForImportPath(importPath string, mod ModuleMode, security web.SecurityMode) (*RepoRoot, error) { 1077 rr, err := repoRootFromVCSPaths(importPath, security, vcsPaths) 1078 if err == errUnknownSite { 1079 rr, err = repoRootForImportDynamic(importPath, mod, security) 1080 if err != nil { 1081 err = importErrorf(importPath, "unrecognized import path %q: %v", importPath, err) 1082 } 1083 } 1084 if err != nil { 1085 rr1, err1 := repoRootFromVCSPaths(importPath, security, vcsPathsAfterDynamic) 1086 if err1 == nil { 1087 rr = rr1 1088 err = nil 1089 } 1090 } 1091 1092 // Should have been taken care of above, but make sure. 1093 if err == nil && strings.Contains(importPath, "...") && strings.Contains(rr.Root, "...") { 1094 // Do not allow wildcards in the repo root. 1095 rr = nil 1096 err = importErrorf(importPath, "cannot expand ... in %q", importPath) 1097 } 1098 return rr, err 1099 } 1100 1101 var errUnknownSite = errors.New("dynamic lookup required to find mapping") 1102 1103 // repoRootFromVCSPaths attempts to map importPath to a repoRoot 1104 // using the mappings defined in vcsPaths. 1105 func repoRootFromVCSPaths(importPath string, security web.SecurityMode, vcsPaths []*vcsPath) (*RepoRoot, error) { 1106 if str.HasPathPrefix(importPath, "example.net") { 1107 // TODO(rsc): This should not be necessary, but it's required to keep 1108 // tests like ../../testdata/script/mod_get_extra.txt from using the network. 1109 // That script has everything it needs in the replacement set, but it is still 1110 // doing network calls. 1111 return nil, fmt.Errorf("no modules on example.net") 1112 } 1113 if importPath == "rsc.io" { 1114 // This special case allows tests like ../../testdata/script/govcs.txt 1115 // to avoid making any network calls. The module lookup for a path 1116 // like rsc.io/nonexist.svn/foo needs to not make a network call for 1117 // a lookup on rsc.io. 1118 return nil, fmt.Errorf("rsc.io is not a module") 1119 } 1120 // A common error is to use https://packagepath because that's what 1121 // hg and git require. Diagnose this helpfully. 1122 if prefix := httpPrefix(importPath); prefix != "" { 1123 // The importPath has been cleaned, so has only one slash. The pattern 1124 // ignores the slashes; the error message puts them back on the RHS at least. 1125 return nil, fmt.Errorf("%q not allowed in import path", prefix+"//") 1126 } 1127 for _, srv := range vcsPaths { 1128 if !str.HasPathPrefix(importPath, srv.pathPrefix) { 1129 continue 1130 } 1131 m := srv.regexp.FindStringSubmatch(importPath) 1132 if m == nil { 1133 if srv.pathPrefix != "" { 1134 return nil, importErrorf(importPath, "invalid %s import path %q", srv.pathPrefix, importPath) 1135 } 1136 continue 1137 } 1138 1139 // Build map of named subexpression matches for expand. 1140 match := map[string]string{ 1141 "prefix": srv.pathPrefix + "/", 1142 "import": importPath, 1143 } 1144 for i, name := range srv.regexp.SubexpNames() { 1145 if name != "" && match[name] == "" { 1146 match[name] = m[i] 1147 } 1148 } 1149 if srv.vcs != "" { 1150 match["vcs"] = expand(match, srv.vcs) 1151 } 1152 if srv.repo != "" { 1153 match["repo"] = expand(match, srv.repo) 1154 } 1155 if srv.check != nil { 1156 if err := srv.check(match); err != nil { 1157 return nil, err 1158 } 1159 } 1160 vcs := vcsByCmd(match["vcs"]) 1161 if vcs == nil { 1162 return nil, fmt.Errorf("unknown version control system %q", match["vcs"]) 1163 } 1164 if err := checkGOVCS(vcs, match["root"]); err != nil { 1165 return nil, err 1166 } 1167 var repoURL string 1168 if !srv.schemelessRepo { 1169 repoURL = match["repo"] 1170 } else { 1171 repo := match["repo"] 1172 var ok bool 1173 repoURL, ok = interceptVCSTest(repo, vcs, security) 1174 if !ok { 1175 scheme, err := func() (string, error) { 1176 for _, s := range vcs.Scheme { 1177 if security == web.SecureOnly && !vcs.isSecureScheme(s) { 1178 continue 1179 } 1180 1181 // If we know how to ping URL schemes for this VCS, 1182 // check that this repo works. 1183 // Otherwise, default to the first scheme 1184 // that meets the requested security level. 1185 if vcs.PingCmd == "" { 1186 return s, nil 1187 } 1188 if err := vcs.Ping(s, repo); err == nil { 1189 return s, nil 1190 } 1191 } 1192 securityFrag := "" 1193 if security == web.SecureOnly { 1194 securityFrag = "secure " 1195 } 1196 return "", fmt.Errorf("no %sprotocol found for repository", securityFrag) 1197 }() 1198 if err != nil { 1199 return nil, err 1200 } 1201 repoURL = scheme + "://" + repo 1202 } 1203 } 1204 rr := &RepoRoot{ 1205 Repo: repoURL, 1206 Root: match["root"], 1207 VCS: vcs, 1208 } 1209 return rr, nil 1210 } 1211 return nil, errUnknownSite 1212 } 1213 1214 func interceptVCSTest(repo string, vcs *Cmd, security web.SecurityMode) (repoURL string, ok bool) { 1215 if VCSTestRepoURL == "" { 1216 return "", false 1217 } 1218 if vcs == vcsMod { 1219 // Since the "mod" protocol is implemented internally, 1220 // requests will be intercepted at a lower level (in github.com/go-asm/go/cmd/go/web). 1221 return "", false 1222 } 1223 1224 if scheme, path, ok := strings.Cut(repo, "://"); ok { 1225 if security == web.SecureOnly && !vcs.isSecureScheme(scheme) { 1226 return "", false // Let the caller reject the original URL. 1227 } 1228 repo = path // Remove leading URL scheme if present. 1229 } 1230 for _, host := range VCSTestHosts { 1231 if !str.HasPathPrefix(repo, host) { 1232 continue 1233 } 1234 1235 httpURL := VCSTestRepoURL + strings.TrimPrefix(repo, host) 1236 1237 if vcs == vcsSvn { 1238 // Ping the vcweb HTTP server to tell it to initialize the SVN repository 1239 // and get the SVN server URL. 1240 u, err := urlpkg.Parse(httpURL + "?vcwebsvn=1") 1241 if err != nil { 1242 panic(fmt.Sprintf("invalid vcs-test repo URL: %v", err)) 1243 } 1244 svnURL, err := web.GetBytes(u) 1245 svnURL = bytes.TrimSpace(svnURL) 1246 if err == nil && len(svnURL) > 0 { 1247 return string(svnURL) + strings.TrimPrefix(repo, host), true 1248 } 1249 1250 // vcs-test doesn't have a svn handler for the given path, 1251 // so resolve the repo to HTTPS instead. 1252 } 1253 1254 return httpURL, true 1255 } 1256 return "", false 1257 } 1258 1259 // urlForImportPath returns a partially-populated URL for the given Go import path. 1260 // 1261 // The URL leaves the Scheme field blank so that web.Get will try any scheme 1262 // allowed by the selected security mode. 1263 func urlForImportPath(importPath string) (*urlpkg.URL, error) { 1264 slash := strings.Index(importPath, "/") 1265 if slash < 0 { 1266 slash = len(importPath) 1267 } 1268 host, path := importPath[:slash], importPath[slash:] 1269 if !strings.Contains(host, ".") { 1270 return nil, errors.New("import path does not begin with hostname") 1271 } 1272 if len(path) == 0 { 1273 path = "/" 1274 } 1275 return &urlpkg.URL{Host: host, Path: path, RawQuery: "go-get=1"}, nil 1276 } 1277 1278 // repoRootForImportDynamic finds a *RepoRoot for a custom domain that's not 1279 // statically known by repoRootFromVCSPaths. 1280 // 1281 // This handles custom import paths like "name.tld/pkg/foo" or just "name.tld". 1282 func repoRootForImportDynamic(importPath string, mod ModuleMode, security web.SecurityMode) (*RepoRoot, error) { 1283 url, err := urlForImportPath(importPath) 1284 if err != nil { 1285 return nil, err 1286 } 1287 resp, err := web.Get(security, url) 1288 if err != nil { 1289 msg := "https fetch: %v" 1290 if security == web.Insecure { 1291 msg = "http/" + msg 1292 } 1293 return nil, fmt.Errorf(msg, err) 1294 } 1295 body := resp.Body 1296 defer body.Close() 1297 imports, err := parseMetaGoImports(body, mod) 1298 if len(imports) == 0 { 1299 if respErr := resp.Err(); respErr != nil { 1300 // If the server's status was not OK, prefer to report that instead of 1301 // an XML parse error. 1302 return nil, respErr 1303 } 1304 } 1305 if err != nil { 1306 return nil, fmt.Errorf("parsing %s: %v", importPath, err) 1307 } 1308 // Find the matched meta import. 1309 mmi, err := matchGoImport(imports, importPath) 1310 if err != nil { 1311 if _, ok := err.(ImportMismatchError); !ok { 1312 return nil, fmt.Errorf("parse %s: %v", url, err) 1313 } 1314 return nil, fmt.Errorf("parse %s: no go-import meta tags (%s)", resp.URL, err) 1315 } 1316 if cfg.BuildV { 1317 log.Printf("get %q: found meta tag %#v at %s", importPath, mmi, url) 1318 } 1319 // If the import was "uni.edu/bob/project", which said the 1320 // prefix was "uni.edu" and the RepoRoot was "evilroot.com", 1321 // make sure we don't trust Bob and check out evilroot.com to 1322 // "uni.edu" yet (possibly overwriting/preempting another 1323 // non-evil student). Instead, first verify the root and see 1324 // if it matches Bob's claim. 1325 if mmi.Prefix != importPath { 1326 if cfg.BuildV { 1327 log.Printf("get %q: verifying non-authoritative meta tag", importPath) 1328 } 1329 var imports []metaImport 1330 url, imports, err = metaImportsForPrefix(mmi.Prefix, mod, security) 1331 if err != nil { 1332 return nil, err 1333 } 1334 metaImport2, err := matchGoImport(imports, importPath) 1335 if err != nil || mmi != metaImport2 { 1336 return nil, fmt.Errorf("%s and %s disagree about go-import for %s", resp.URL, url, mmi.Prefix) 1337 } 1338 } 1339 1340 if err := validateRepoRoot(mmi.RepoRoot); err != nil { 1341 return nil, fmt.Errorf("%s: invalid repo root %q: %v", resp.URL, mmi.RepoRoot, err) 1342 } 1343 var vcs *Cmd 1344 if mmi.VCS == "mod" { 1345 vcs = vcsMod 1346 } else { 1347 vcs = vcsByCmd(mmi.VCS) 1348 if vcs == nil { 1349 return nil, fmt.Errorf("%s: unknown vcs %q", resp.URL, mmi.VCS) 1350 } 1351 } 1352 1353 if err := checkGOVCS(vcs, mmi.Prefix); err != nil { 1354 return nil, err 1355 } 1356 1357 repoURL, ok := interceptVCSTest(mmi.RepoRoot, vcs, security) 1358 if !ok { 1359 repoURL = mmi.RepoRoot 1360 } 1361 rr := &RepoRoot{ 1362 Repo: repoURL, 1363 Root: mmi.Prefix, 1364 IsCustom: true, 1365 VCS: vcs, 1366 } 1367 return rr, nil 1368 } 1369 1370 // validateRepoRoot returns an error if repoRoot does not seem to be 1371 // a valid URL with scheme. 1372 func validateRepoRoot(repoRoot string) error { 1373 url, err := urlpkg.Parse(repoRoot) 1374 if err != nil { 1375 return err 1376 } 1377 if url.Scheme == "" { 1378 return errors.New("no scheme") 1379 } 1380 if url.Scheme == "file" { 1381 return errors.New("file scheme disallowed") 1382 } 1383 return nil 1384 } 1385 1386 var fetchGroup singleflight.Group 1387 var ( 1388 fetchCacheMu sync.Mutex 1389 fetchCache = map[string]fetchResult{} // key is metaImportsForPrefix's importPrefix 1390 ) 1391 1392 // metaImportsForPrefix takes a package's root import path as declared in a <meta> tag 1393 // and returns its HTML discovery URL and the parsed metaImport lines 1394 // found on the page. 1395 // 1396 // The importPath is of the form "golang.org/x/tools". 1397 // It is an error if no imports are found. 1398 // url will still be valid if err != nil. 1399 // The returned url will be of the form "https://golang.org/x/tools?go-get=1" 1400 func metaImportsForPrefix(importPrefix string, mod ModuleMode, security web.SecurityMode) (*urlpkg.URL, []metaImport, error) { 1401 setCache := func(res fetchResult) (fetchResult, error) { 1402 fetchCacheMu.Lock() 1403 defer fetchCacheMu.Unlock() 1404 fetchCache[importPrefix] = res 1405 return res, nil 1406 } 1407 1408 resi, _, _ := fetchGroup.Do(importPrefix, func() (resi any, err error) { 1409 fetchCacheMu.Lock() 1410 if res, ok := fetchCache[importPrefix]; ok { 1411 fetchCacheMu.Unlock() 1412 return res, nil 1413 } 1414 fetchCacheMu.Unlock() 1415 1416 url, err := urlForImportPath(importPrefix) 1417 if err != nil { 1418 return setCache(fetchResult{err: err}) 1419 } 1420 resp, err := web.Get(security, url) 1421 if err != nil { 1422 return setCache(fetchResult{url: url, err: fmt.Errorf("fetching %s: %v", importPrefix, err)}) 1423 } 1424 body := resp.Body 1425 defer body.Close() 1426 imports, err := parseMetaGoImports(body, mod) 1427 if len(imports) == 0 { 1428 if respErr := resp.Err(); respErr != nil { 1429 // If the server's status was not OK, prefer to report that instead of 1430 // an XML parse error. 1431 return setCache(fetchResult{url: url, err: respErr}) 1432 } 1433 } 1434 if err != nil { 1435 return setCache(fetchResult{url: url, err: fmt.Errorf("parsing %s: %v", resp.URL, err)}) 1436 } 1437 if len(imports) == 0 { 1438 err = fmt.Errorf("fetching %s: no go-import meta tag found in %s", importPrefix, resp.URL) 1439 } 1440 return setCache(fetchResult{url: url, imports: imports, err: err}) 1441 }) 1442 res := resi.(fetchResult) 1443 return res.url, res.imports, res.err 1444 } 1445 1446 type fetchResult struct { 1447 url *urlpkg.URL 1448 imports []metaImport 1449 err error 1450 } 1451 1452 // metaImport represents the parsed <meta name="go-import" 1453 // content="prefix vcs reporoot" /> tags from HTML files. 1454 type metaImport struct { 1455 Prefix, VCS, RepoRoot string 1456 } 1457 1458 // An ImportMismatchError is returned where metaImport/s are present 1459 // but none match our import path. 1460 type ImportMismatchError struct { 1461 importPath string 1462 mismatches []string // the meta imports that were discarded for not matching our importPath 1463 } 1464 1465 func (m ImportMismatchError) Error() string { 1466 formattedStrings := make([]string, len(m.mismatches)) 1467 for i, pre := range m.mismatches { 1468 formattedStrings[i] = fmt.Sprintf("meta tag %s did not match import path %s", pre, m.importPath) 1469 } 1470 return strings.Join(formattedStrings, ", ") 1471 } 1472 1473 // matchGoImport returns the metaImport from imports matching importPath. 1474 // An error is returned if there are multiple matches. 1475 // An ImportMismatchError is returned if none match. 1476 func matchGoImport(imports []metaImport, importPath string) (metaImport, error) { 1477 match := -1 1478 1479 errImportMismatch := ImportMismatchError{importPath: importPath} 1480 for i, im := range imports { 1481 if !str.HasPathPrefix(importPath, im.Prefix) { 1482 errImportMismatch.mismatches = append(errImportMismatch.mismatches, im.Prefix) 1483 continue 1484 } 1485 1486 if match >= 0 { 1487 if imports[match].VCS == "mod" && im.VCS != "mod" { 1488 // All the mod entries precede all the non-mod entries. 1489 // We have a mod entry and don't care about the rest, 1490 // matching or not. 1491 break 1492 } 1493 return metaImport{}, fmt.Errorf("multiple meta tags match import path %q", importPath) 1494 } 1495 match = i 1496 } 1497 1498 if match == -1 { 1499 return metaImport{}, errImportMismatch 1500 } 1501 return imports[match], nil 1502 } 1503 1504 // expand rewrites s to replace {k} with match[k] for each key k in match. 1505 func expand(match map[string]string, s string) string { 1506 // We want to replace each match exactly once, and the result of expansion 1507 // must not depend on the iteration order through the map. 1508 // A strings.Replacer has exactly the properties we're looking for. 1509 oldNew := make([]string, 0, 2*len(match)) 1510 for k, v := range match { 1511 oldNew = append(oldNew, "{"+k+"}", v) 1512 } 1513 return strings.NewReplacer(oldNew...).Replace(s) 1514 } 1515 1516 // vcsPaths defines the meaning of import paths referring to 1517 // commonly-used VCS hosting sites (github.com/user/dir) 1518 // and import paths referring to a fully-qualified importPath 1519 // containing a VCS type (foo.com/repo.git/dir) 1520 var vcsPaths = []*vcsPath{ 1521 // GitHub 1522 { 1523 pathPrefix: "github.com", 1524 regexp: lazyregexp.New(`^(?P<root>github\.com/[\w.\-]+/[\w.\-]+)(/[\w.\-]+)*$`), 1525 vcs: "git", 1526 repo: "https://{root}", 1527 check: noVCSSuffix, 1528 }, 1529 1530 // Bitbucket 1531 { 1532 pathPrefix: "bitbucket.org", 1533 regexp: lazyregexp.New(`^(?P<root>bitbucket\.org/(?P<bitname>[\w.\-]+/[\w.\-]+))(/[\w.\-]+)*$`), 1534 vcs: "git", 1535 repo: "https://{root}", 1536 check: noVCSSuffix, 1537 }, 1538 1539 // IBM DevOps Services (JazzHub) 1540 { 1541 pathPrefix: "hub.jazz.net/git", 1542 regexp: lazyregexp.New(`^(?P<root>hub\.jazz\.net/git/[a-z0-9]+/[\w.\-]+)(/[\w.\-]+)*$`), 1543 vcs: "git", 1544 repo: "https://{root}", 1545 check: noVCSSuffix, 1546 }, 1547 1548 // Git at Apache 1549 { 1550 pathPrefix: "git.apache.org", 1551 regexp: lazyregexp.New(`^(?P<root>git\.apache\.org/[a-z0-9_.\-]+\.git)(/[\w.\-]+)*$`), 1552 vcs: "git", 1553 repo: "https://{root}", 1554 }, 1555 1556 // Git at OpenStack 1557 { 1558 pathPrefix: "git.openstack.org", 1559 regexp: lazyregexp.New(`^(?P<root>git\.openstack\.org/[\w.\-]+/[\w.\-]+)(\.git)?(/[\w.\-]+)*$`), 1560 vcs: "git", 1561 repo: "https://{root}", 1562 }, 1563 1564 // chiselapp.com for fossil 1565 { 1566 pathPrefix: "chiselapp.com", 1567 regexp: lazyregexp.New(`^(?P<root>chiselapp\.com/user/[A-Za-z0-9]+/repository/[\w.\-]+)$`), 1568 vcs: "fossil", 1569 repo: "https://{root}", 1570 }, 1571 1572 // General syntax for any server. 1573 // Must be last. 1574 { 1575 regexp: lazyregexp.New(`(?P<root>(?P<repo>([a-z0-9.\-]+\.)+[a-z0-9.\-]+(:[0-9]+)?(/~?[\w.\-]+)+?)\.(?P<vcs>bzr|fossil|git|hg|svn))(/~?[\w.\-]+)*$`), 1576 schemelessRepo: true, 1577 }, 1578 } 1579 1580 // vcsPathsAfterDynamic gives additional vcsPaths entries 1581 // to try after the dynamic HTML check. 1582 // This gives those sites a chance to introduce <meta> tags 1583 // as part of a graceful transition away from the hard-coded logic. 1584 var vcsPathsAfterDynamic = []*vcsPath{ 1585 // Launchpad. See golang.org/issue/11436. 1586 { 1587 pathPrefix: "launchpad.net", 1588 regexp: lazyregexp.New(`^(?P<root>launchpad\.net/((?P<project>[\w.\-]+)(?P<series>/[\w.\-]+)?|~[\w.\-]+/(\+junk|[\w.\-]+)/[\w.\-]+))(/[\w.\-]+)*$`), 1589 vcs: "bzr", 1590 repo: "https://{root}", 1591 check: launchpadVCS, 1592 }, 1593 } 1594 1595 // noVCSSuffix checks that the repository name does not 1596 // end in .foo for any version control system foo. 1597 // The usual culprit is ".git". 1598 func noVCSSuffix(match map[string]string) error { 1599 repo := match["repo"] 1600 for _, vcs := range vcsList { 1601 if strings.HasSuffix(repo, "."+vcs.Cmd) { 1602 return fmt.Errorf("invalid version control suffix in %s path", match["prefix"]) 1603 } 1604 } 1605 return nil 1606 } 1607 1608 // launchpadVCS solves the ambiguity for "lp.net/project/foo". In this case, 1609 // "foo" could be a series name registered in Launchpad with its own branch, 1610 // and it could also be the name of a directory within the main project 1611 // branch one level up. 1612 func launchpadVCS(match map[string]string) error { 1613 if match["project"] == "" || match["series"] == "" { 1614 return nil 1615 } 1616 url := &urlpkg.URL{ 1617 Scheme: "https", 1618 Host: "code.launchpad.net", 1619 Path: expand(match, "/{project}{series}/.bzr/branch-format"), 1620 } 1621 _, err := web.GetBytes(url) 1622 if err != nil { 1623 match["root"] = expand(match, "launchpad.net/{project}") 1624 match["repo"] = expand(match, "https://{root}") 1625 } 1626 return nil 1627 } 1628 1629 // importError is a copy of load.importError, made to avoid a dependency cycle 1630 // on github.com/go-asm/go/cmd/go/load. It just needs to satisfy load.ImportPathError. 1631 type importError struct { 1632 importPath string 1633 err error 1634 } 1635 1636 func importErrorf(path, format string, args ...any) error { 1637 err := &importError{importPath: path, err: fmt.Errorf(format, args...)} 1638 if errStr := err.Error(); !strings.Contains(errStr, path) { 1639 panic(fmt.Sprintf("path %q not in error %q", path, errStr)) 1640 } 1641 return err 1642 } 1643 1644 func (e *importError) Error() string { 1645 return e.err.Error() 1646 } 1647 1648 func (e *importError) Unwrap() error { 1649 // Don't return e.err directly, since we're only wrapping an error if %w 1650 // was passed to ImportErrorf. 1651 return errors.Unwrap(e.err) 1652 } 1653 1654 func (e *importError) ImportPath() string { 1655 return e.importPath 1656 }