github.com/bir3/gocompiler@v0.3.205/src/cmd/gocmd/internal/vcs/vcs.go (about) 1 // Copyright 2012 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package vcs 6 7 import ( 8 "bytes" 9 "errors" 10 "fmt" 11 "github.com/bir3/gocompiler/src/internal/lazyregexp" 12 "github.com/bir3/gocompiler/src/internal/singleflight" 13 "io/fs" 14 "log" 15 urlpkg "net/url" 16 "os" 17 "os/exec" 18 "path/filepath" 19 "regexp" 20 "strconv" 21 "strings" 22 "sync" 23 "time" 24 25 "github.com/bir3/gocompiler/src/cmd/gocmd/internal/cfg" 26 "github.com/bir3/gocompiler/src/cmd/gocmd/internal/search" 27 "github.com/bir3/gocompiler/src/cmd/gocmd/internal/str" 28 "github.com/bir3/gocompiler/src/cmd/gocmd/internal/web" 29 30 "github.com/bir3/gocompiler/src/xvendor/golang.org/x/mod/module" 31 ) 32 33 // A Cmd describes how to use a version control system 34 // like Mercurial, Git, or Subversion. 35 type Cmd struct { 36 Name string 37 Cmd string // name of binary to invoke command 38 RootNames []rootName // filename and mode indicating the root of a checkout directory 39 40 CreateCmd []string // commands to download a fresh copy of a repository 41 DownloadCmd []string // commands to download updates into an existing repository 42 43 TagCmd []tagCmd // commands to list tags 44 TagLookupCmd []tagCmd // commands to lookup tags before running tagSyncCmd 45 TagSyncCmd []string // commands to sync to specific tag 46 TagSyncDefault []string // commands to sync to default tag 47 48 Scheme []string 49 PingCmd string 50 51 RemoteRepo func(v *Cmd, rootDir string) (remoteRepo string, err error) 52 ResolveRepo func(v *Cmd, rootDir, remoteRepo string) (realRepo string, err error) 53 Status func(v *Cmd, rootDir string) (Status, error) 54 } 55 56 // Status is the current state of a local repository. 57 type Status struct { 58 Revision string // Optional. 59 CommitTime time.Time // Optional. 60 Uncommitted bool // Required. 61 } 62 63 var ( 64 // VCSTestRepoURL is the URL of the HTTP server that serves the repos for 65 // vcs-test.golang.org. 66 // 67 // In tests, this is set to the URL of an httptest.Server hosting a 68 // cmd/go/internal/vcweb.Server. 69 VCSTestRepoURL string 70 71 // VCSTestHosts is the set of hosts supported by the vcs-test server. 72 VCSTestHosts []string 73 74 // VCSTestIsLocalHost reports whether the given URL refers to a local 75 // (loopback) host, such as "localhost" or "127.0.0.1:8080". 76 VCSTestIsLocalHost func(*urlpkg.URL) bool 77 ) 78 79 var defaultSecureScheme = map[string]bool{ 80 "https": true, 81 "git+ssh": true, 82 "bzr+ssh": true, 83 "svn+ssh": true, 84 "ssh": true, 85 } 86 87 func (v *Cmd) IsSecure(repo string) bool { 88 u, err := urlpkg.Parse(repo) 89 if err != nil { 90 // If repo is not a URL, it's not secure. 91 return false 92 } 93 if VCSTestRepoURL != "" && web.IsLocalHost(u) { 94 // If the vcstest server is in use, it may redirect to other local ports for 95 // other protocols (such as svn). Assume that all loopback addresses are 96 // secure during testing. 97 return true 98 } 99 return v.isSecureScheme(u.Scheme) 100 } 101 102 func (v *Cmd) isSecureScheme(scheme string) bool { 103 switch v.Cmd { 104 case "git": 105 // GIT_ALLOW_PROTOCOL is an environment variable defined by Git. It is a 106 // colon-separated list of schemes that are allowed to be used with git 107 // fetch/clone. Any scheme not mentioned will be considered insecure. 108 if allow := os.Getenv("GIT_ALLOW_PROTOCOL"); allow != "" { 109 for _, s := range strings.Split(allow, ":") { 110 if s == scheme { 111 return true 112 } 113 } 114 return false 115 } 116 } 117 return defaultSecureScheme[scheme] 118 } 119 120 // A tagCmd describes a command to list available tags 121 // that can be passed to tagSyncCmd. 122 type tagCmd struct { 123 cmd string // command to list tags 124 pattern string // regexp to extract tags from list 125 } 126 127 // vcsList lists the known version control systems 128 var vcsList = []*Cmd{ 129 vcsHg, 130 vcsGit, 131 vcsSvn, 132 vcsBzr, 133 vcsFossil, 134 } 135 136 // vcsMod is a stub for the "mod" scheme. It's returned by 137 // repoRootForImportPathDynamic, but is otherwise not treated as a VCS command. 138 var vcsMod = &Cmd{Name: "mod"} 139 140 // vcsByCmd returns the version control system for the given 141 // command name (hg, git, svn, bzr). 142 func vcsByCmd(cmd string) *Cmd { 143 for _, vcs := range vcsList { 144 if vcs.Cmd == cmd { 145 return vcs 146 } 147 } 148 return nil 149 } 150 151 // vcsHg describes how to use Mercurial. 152 var vcsHg = &Cmd{ 153 Name: "Mercurial", 154 Cmd: "hg", 155 RootNames: []rootName{ 156 {filename: ".hg", isDir: true}, 157 }, 158 159 CreateCmd: []string{"clone -U -- {repo} {dir}"}, 160 DownloadCmd: []string{"pull"}, 161 162 // We allow both tag and branch names as 'tags' 163 // for selecting a version. This lets people have 164 // a go.release.r60 branch and a go1 branch 165 // and make changes in both, without constantly 166 // editing .hgtags. 167 TagCmd: []tagCmd{ 168 {"tags", `^(\S+)`}, 169 {"branches", `^(\S+)`}, 170 }, 171 TagSyncCmd: []string{"update -r {tag}"}, 172 TagSyncDefault: []string{"update default"}, 173 174 Scheme: []string{"https", "http", "ssh"}, 175 PingCmd: "identify -- {scheme}://{repo}", 176 RemoteRepo: hgRemoteRepo, 177 Status: hgStatus, 178 } 179 180 func hgRemoteRepo(vcsHg *Cmd, rootDir string) (remoteRepo string, err error) { 181 out, err := vcsHg.runOutput(rootDir, "paths default") 182 if err != nil { 183 return "", err 184 } 185 return strings.TrimSpace(string(out)), nil 186 } 187 188 func hgStatus(vcsHg *Cmd, rootDir string) (Status, error) { 189 // Output changeset ID and seconds since epoch. 190 out, err := vcsHg.runOutputVerboseOnly(rootDir, `log -l1 -T {node}:{date|hgdate}`) 191 if err != nil { 192 return Status{}, err 193 } 194 195 // Successful execution without output indicates an empty repo (no commits). 196 var rev string 197 var commitTime time.Time 198 if len(out) > 0 { 199 // Strip trailing timezone offset. 200 if i := bytes.IndexByte(out, ' '); i > 0 { 201 out = out[:i] 202 } 203 rev, commitTime, err = parseRevTime(out) 204 if err != nil { 205 return Status{}, err 206 } 207 } 208 209 // Also look for untracked files. 210 out, err = vcsHg.runOutputVerboseOnly(rootDir, "status") 211 if err != nil { 212 return Status{}, err 213 } 214 uncommitted := len(out) > 0 215 216 return Status{ 217 Revision: rev, 218 CommitTime: commitTime, 219 Uncommitted: uncommitted, 220 }, nil 221 } 222 223 // parseRevTime parses commit details in "revision:seconds" format. 224 func parseRevTime(out []byte) (string, time.Time, error) { 225 buf := string(bytes.TrimSpace(out)) 226 227 i := strings.IndexByte(buf, ':') 228 if i < 1 { 229 return "", time.Time{}, errors.New("unrecognized VCS tool output") 230 } 231 rev := buf[:i] 232 233 secs, err := strconv.ParseInt(string(buf[i+1:]), 10, 64) 234 if err != nil { 235 return "", time.Time{}, fmt.Errorf("unrecognized VCS tool output: %v", err) 236 } 237 238 return rev, time.Unix(secs, 0), nil 239 } 240 241 // vcsGit describes how to use Git. 242 var vcsGit = &Cmd{ 243 Name: "Git", 244 Cmd: "git", 245 RootNames: []rootName{ 246 {filename: ".git", isDir: true}, 247 }, 248 249 CreateCmd: []string{"clone -- {repo} {dir}", "-go-internal-cd {dir} submodule update --init --recursive"}, 250 DownloadCmd: []string{"pull --ff-only", "submodule update --init --recursive"}, 251 252 TagCmd: []tagCmd{ 253 // tags/xxx matches a git tag named xxx 254 // origin/xxx matches a git branch named xxx on the default remote repository 255 {"show-ref", `(?:tags|origin)/(\S+)$`}, 256 }, 257 TagLookupCmd: []tagCmd{ 258 {"show-ref tags/{tag} origin/{tag}", `((?:tags|origin)/\S+)$`}, 259 }, 260 TagSyncCmd: []string{"checkout {tag}", "submodule update --init --recursive"}, 261 // both createCmd and downloadCmd update the working dir. 262 // No need to do more here. We used to 'checkout master' 263 // but that doesn't work if the default branch is not named master. 264 // DO NOT add 'checkout master' here. 265 // See golang.org/issue/9032. 266 TagSyncDefault: []string{"submodule update --init --recursive"}, 267 268 Scheme: []string{"git", "https", "http", "git+ssh", "ssh"}, 269 270 // Leave out the '--' separator in the ls-remote command: git 2.7.4 does not 271 // support such a separator for that command, and this use should be safe 272 // without it because the {scheme} value comes from the predefined list above. 273 // See golang.org/issue/33836. 274 PingCmd: "ls-remote {scheme}://{repo}", 275 276 RemoteRepo: gitRemoteRepo, 277 Status: gitStatus, 278 } 279 280 // scpSyntaxRe matches the SCP-like addresses used by Git to access 281 // repositories by SSH. 282 var scpSyntaxRe = lazyregexp.New(`^(\w+)@([\w.-]+):(.*)$`) 283 284 func gitRemoteRepo(vcsGit *Cmd, rootDir string) (remoteRepo string, err error) { 285 cmd := "config remote.origin.url" 286 errParse := errors.New("unable to parse output of git " + cmd) 287 errRemoteOriginNotFound := errors.New("remote origin not found") 288 outb, err := vcsGit.run1(rootDir, cmd, nil, false) 289 if err != nil { 290 // if it doesn't output any message, it means the config argument is correct, 291 // but the config value itself doesn't exist 292 if outb != nil && len(outb) == 0 { 293 return "", errRemoteOriginNotFound 294 } 295 return "", err 296 } 297 out := strings.TrimSpace(string(outb)) 298 299 var repoURL *urlpkg.URL 300 if m := scpSyntaxRe.FindStringSubmatch(out); m != nil { 301 // Match SCP-like syntax and convert it to a URL. 302 // Eg, "git@github.com:user/repo" becomes 303 // "ssh://git@github.com/user/repo". 304 repoURL = &urlpkg.URL{ 305 Scheme: "ssh", 306 User: urlpkg.User(m[1]), 307 Host: m[2], 308 Path: m[3], 309 } 310 } else { 311 repoURL, err = urlpkg.Parse(out) 312 if err != nil { 313 return "", err 314 } 315 } 316 317 // Iterate over insecure schemes too, because this function simply 318 // reports the state of the repo. If we can't see insecure schemes then 319 // we can't report the actual repo URL. 320 for _, s := range vcsGit.Scheme { 321 if repoURL.Scheme == s { 322 return repoURL.String(), nil 323 } 324 } 325 return "", errParse 326 } 327 328 func gitStatus(vcsGit *Cmd, rootDir string) (Status, error) { 329 out, err := vcsGit.runOutputVerboseOnly(rootDir, "status --porcelain") 330 if err != nil { 331 return Status{}, err 332 } 333 uncommitted := len(out) > 0 334 335 // "git status" works for empty repositories, but "git show" does not. 336 // Assume there are no commits in the repo when "git show" fails with 337 // uncommitted files and skip tagging revision / committime. 338 var rev string 339 var commitTime time.Time 340 out, err = vcsGit.runOutputVerboseOnly(rootDir, "-c log.showsignature=false show -s --format=%H:%ct") 341 if err != nil && !uncommitted { 342 return Status{}, err 343 } else if err == nil { 344 rev, commitTime, err = parseRevTime(out) 345 if err != nil { 346 return Status{}, err 347 } 348 } 349 350 return Status{ 351 Revision: rev, 352 CommitTime: commitTime, 353 Uncommitted: uncommitted, 354 }, nil 355 } 356 357 // vcsBzr describes how to use Bazaar. 358 var vcsBzr = &Cmd{ 359 Name: "Bazaar", 360 Cmd: "bzr", 361 RootNames: []rootName{ 362 {filename: ".bzr", isDir: true}, 363 }, 364 365 CreateCmd: []string{"branch -- {repo} {dir}"}, 366 367 // Without --overwrite bzr will not pull tags that changed. 368 // Replace by --overwrite-tags after http://pad.lv/681792 goes in. 369 DownloadCmd: []string{"pull --overwrite"}, 370 371 TagCmd: []tagCmd{{"tags", `^(\S+)`}}, 372 TagSyncCmd: []string{"update -r {tag}"}, 373 TagSyncDefault: []string{"update -r revno:-1"}, 374 375 Scheme: []string{"https", "http", "bzr", "bzr+ssh"}, 376 PingCmd: "info -- {scheme}://{repo}", 377 RemoteRepo: bzrRemoteRepo, 378 ResolveRepo: bzrResolveRepo, 379 Status: bzrStatus, 380 } 381 382 func bzrRemoteRepo(vcsBzr *Cmd, rootDir string) (remoteRepo string, err error) { 383 outb, err := vcsBzr.runOutput(rootDir, "config parent_location") 384 if err != nil { 385 return "", err 386 } 387 return strings.TrimSpace(string(outb)), nil 388 } 389 390 func bzrResolveRepo(vcsBzr *Cmd, rootDir, remoteRepo string) (realRepo string, err error) { 391 outb, err := vcsBzr.runOutput(rootDir, "info "+remoteRepo) 392 if err != nil { 393 return "", err 394 } 395 out := string(outb) 396 397 // Expect: 398 // ... 399 // (branch root|repository branch): <URL> 400 // ... 401 402 found := false 403 for _, prefix := range []string{"\n branch root: ", "\n repository branch: "} { 404 i := strings.Index(out, prefix) 405 if i >= 0 { 406 out = out[i+len(prefix):] 407 found = true 408 break 409 } 410 } 411 if !found { 412 return "", fmt.Errorf("unable to parse output of bzr info") 413 } 414 415 i := strings.Index(out, "\n") 416 if i < 0 { 417 return "", fmt.Errorf("unable to parse output of bzr info") 418 } 419 out = out[:i] 420 return strings.TrimSpace(out), nil 421 } 422 423 func bzrStatus(vcsBzr *Cmd, rootDir string) (Status, error) { 424 outb, err := vcsBzr.runOutputVerboseOnly(rootDir, "version-info") 425 if err != nil { 426 return Status{}, err 427 } 428 out := string(outb) 429 430 // Expect (non-empty repositories only): 431 // 432 // revision-id: gopher@gopher.net-20211021072330-qshok76wfypw9lpm 433 // date: 2021-09-21 12:00:00 +1000 434 // ... 435 var rev string 436 var commitTime time.Time 437 438 for _, line := range strings.Split(out, "\n") { 439 i := strings.IndexByte(line, ':') 440 if i < 0 { 441 continue 442 } 443 key := line[:i] 444 value := strings.TrimSpace(line[i+1:]) 445 446 switch key { 447 case "revision-id": 448 rev = value 449 case "date": 450 var err error 451 commitTime, err = time.Parse("2006-01-02 15:04:05 -0700", value) 452 if err != nil { 453 return Status{}, errors.New("unable to parse output of bzr version-info") 454 } 455 } 456 } 457 458 outb, err = vcsBzr.runOutputVerboseOnly(rootDir, "status") 459 if err != nil { 460 return Status{}, err 461 } 462 463 // Skip warning when working directory is set to an older revision. 464 if bytes.HasPrefix(outb, []byte("working tree is out of date")) { 465 i := bytes.IndexByte(outb, '\n') 466 if i < 0 { 467 i = len(outb) 468 } 469 outb = outb[:i] 470 } 471 uncommitted := len(outb) > 0 472 473 return Status{ 474 Revision: rev, 475 CommitTime: commitTime, 476 Uncommitted: uncommitted, 477 }, nil 478 } 479 480 // vcsSvn describes how to use Subversion. 481 var vcsSvn = &Cmd{ 482 Name: "Subversion", 483 Cmd: "svn", 484 RootNames: []rootName{ 485 {filename: ".svn", isDir: true}, 486 }, 487 488 CreateCmd: []string{"checkout -- {repo} {dir}"}, 489 DownloadCmd: []string{"update"}, 490 491 // There is no tag command in subversion. 492 // The branch information is all in the path names. 493 494 Scheme: []string{"https", "http", "svn", "svn+ssh"}, 495 PingCmd: "info -- {scheme}://{repo}", 496 RemoteRepo: svnRemoteRepo, 497 } 498 499 func svnRemoteRepo(vcsSvn *Cmd, rootDir string) (remoteRepo string, err error) { 500 outb, err := vcsSvn.runOutput(rootDir, "info") 501 if err != nil { 502 return "", err 503 } 504 out := string(outb) 505 506 // Expect: 507 // 508 // ... 509 // URL: <URL> 510 // ... 511 // 512 // Note that we're not using the Repository Root line, 513 // because svn allows checking out subtrees. 514 // The URL will be the URL of the subtree (what we used with 'svn co') 515 // while the Repository Root may be a much higher parent. 516 i := strings.Index(out, "\nURL: ") 517 if i < 0 { 518 return "", fmt.Errorf("unable to parse output of svn info") 519 } 520 out = out[i+len("\nURL: "):] 521 i = strings.Index(out, "\n") 522 if i < 0 { 523 return "", fmt.Errorf("unable to parse output of svn info") 524 } 525 out = out[:i] 526 return strings.TrimSpace(out), nil 527 } 528 529 // fossilRepoName is the name go get associates with a fossil repository. In the 530 // real world the file can be named anything. 531 const fossilRepoName = ".fossil" 532 533 // vcsFossil describes how to use Fossil (fossil-scm.org) 534 var vcsFossil = &Cmd{ 535 Name: "Fossil", 536 Cmd: "fossil", 537 RootNames: []rootName{ 538 {filename: ".fslckout", isDir: false}, 539 {filename: "_FOSSIL_", isDir: false}, 540 }, 541 542 CreateCmd: []string{"-go-internal-mkdir {dir} clone -- {repo} " + filepath.Join("{dir}", fossilRepoName), "-go-internal-cd {dir} open .fossil"}, 543 DownloadCmd: []string{"up"}, 544 545 TagCmd: []tagCmd{{"tag ls", `(.*)`}}, 546 TagSyncCmd: []string{"up tag:{tag}"}, 547 TagSyncDefault: []string{"up trunk"}, 548 549 Scheme: []string{"https", "http"}, 550 RemoteRepo: fossilRemoteRepo, 551 Status: fossilStatus, 552 } 553 554 func fossilRemoteRepo(vcsFossil *Cmd, rootDir string) (remoteRepo string, err error) { 555 out, err := vcsFossil.runOutput(rootDir, "remote-url") 556 if err != nil { 557 return "", err 558 } 559 return strings.TrimSpace(string(out)), nil 560 } 561 562 var errFossilInfo = errors.New("unable to parse output of fossil info") 563 564 func fossilStatus(vcsFossil *Cmd, rootDir string) (Status, error) { 565 outb, err := vcsFossil.runOutputVerboseOnly(rootDir, "info") 566 if err != nil { 567 return Status{}, err 568 } 569 out := string(outb) 570 571 // Expect: 572 // ... 573 // checkout: 91ed71f22c77be0c3e250920f47bfd4e1f9024d2 2021-09-21 12:00:00 UTC 574 // ... 575 576 // Extract revision and commit time. 577 // Ensure line ends with UTC (known timezone offset). 578 const prefix = "\ncheckout:" 579 const suffix = " UTC" 580 i := strings.Index(out, prefix) 581 if i < 0 { 582 return Status{}, errFossilInfo 583 } 584 checkout := out[i+len(prefix):] 585 i = strings.Index(checkout, suffix) 586 if i < 0 { 587 return Status{}, errFossilInfo 588 } 589 checkout = strings.TrimSpace(checkout[:i]) 590 591 i = strings.IndexByte(checkout, ' ') 592 if i < 0 { 593 return Status{}, errFossilInfo 594 } 595 rev := checkout[:i] 596 597 commitTime, err := time.ParseInLocation(time.DateTime, checkout[i+1:], time.UTC) 598 if err != nil { 599 return Status{}, fmt.Errorf("%v: %v", errFossilInfo, err) 600 } 601 602 // Also look for untracked changes. 603 outb, err = vcsFossil.runOutputVerboseOnly(rootDir, "changes --differ") 604 if err != nil { 605 return Status{}, err 606 } 607 uncommitted := len(outb) > 0 608 609 return Status{ 610 Revision: rev, 611 CommitTime: commitTime, 612 Uncommitted: uncommitted, 613 }, nil 614 } 615 616 func (v *Cmd) String() string { 617 return v.Name 618 } 619 620 // run runs the command line cmd in the given directory. 621 // keyval is a list of key, value pairs. run expands 622 // instances of {key} in cmd into value, but only after 623 // splitting cmd into individual arguments. 624 // If an error occurs, run prints the command line and the 625 // command's combined stdout+stderr to standard error. 626 // Otherwise run discards the command's output. 627 func (v *Cmd) run(dir string, cmd string, keyval ...string) error { 628 _, err := v.run1(dir, cmd, keyval, true) 629 return err 630 } 631 632 // runVerboseOnly is like run but only generates error output to standard error in verbose mode. 633 func (v *Cmd) runVerboseOnly(dir string, cmd string, keyval ...string) error { 634 _, err := v.run1(dir, cmd, keyval, false) 635 return err 636 } 637 638 // runOutput is like run but returns the output of the command. 639 func (v *Cmd) runOutput(dir string, cmd string, keyval ...string) ([]byte, error) { 640 return v.run1(dir, cmd, keyval, true) 641 } 642 643 // runOutputVerboseOnly is like runOutput but only generates error output to 644 // standard error in verbose mode. 645 func (v *Cmd) runOutputVerboseOnly(dir string, cmd string, keyval ...string) ([]byte, error) { 646 return v.run1(dir, cmd, keyval, false) 647 } 648 649 // run1 is the generalized implementation of run and runOutput. 650 func (v *Cmd) run1(dir string, cmdline string, keyval []string, verbose bool) ([]byte, error) { 651 m := make(map[string]string) 652 for i := 0; i < len(keyval); i += 2 { 653 m[keyval[i]] = keyval[i+1] 654 } 655 args := strings.Fields(cmdline) 656 for i, arg := range args { 657 args[i] = expand(m, arg) 658 } 659 660 if len(args) >= 2 && args[0] == "-go-internal-mkdir" { 661 var err error 662 if filepath.IsAbs(args[1]) { 663 err = os.Mkdir(args[1], fs.ModePerm) 664 } else { 665 err = os.Mkdir(filepath.Join(dir, args[1]), fs.ModePerm) 666 } 667 if err != nil { 668 return nil, err 669 } 670 args = args[2:] 671 } 672 673 if len(args) >= 2 && args[0] == "-go-internal-cd" { 674 if filepath.IsAbs(args[1]) { 675 dir = args[1] 676 } else { 677 dir = filepath.Join(dir, args[1]) 678 } 679 args = args[2:] 680 } 681 682 _, err := exec.LookPath(v.Cmd) 683 if err != nil { 684 fmt.Fprintf(os.Stderr, 685 "go: missing %s command. See https://golang.org/s/gogetcmd\n", 686 v.Name) 687 return nil, err 688 } 689 690 cmd := exec.Command(v.Cmd, args...) 691 cmd.Dir = dir 692 if cfg.BuildX { 693 fmt.Fprintf(os.Stderr, "cd %s\n", dir) 694 fmt.Fprintf(os.Stderr, "%s %s\n", v.Cmd, strings.Join(args, " ")) 695 } 696 out, err := cmd.Output() 697 if err != nil { 698 if verbose || cfg.BuildV { 699 fmt.Fprintf(os.Stderr, "# cd %s; %s %s\n", dir, v.Cmd, strings.Join(args, " ")) 700 if ee, ok := err.(*exec.ExitError); ok && len(ee.Stderr) > 0 { 701 os.Stderr.Write(ee.Stderr) 702 } else { 703 fmt.Fprintln(os.Stderr, err.Error()) 704 } 705 } 706 } 707 return out, err 708 } 709 710 // Ping pings to determine scheme to use. 711 func (v *Cmd) Ping(scheme, repo string) error { 712 // Run the ping command in an arbitrary working directory, 713 // but don't let the current working directory pollute the results. 714 // In module mode, we expect GOMODCACHE to exist and be a safe place for 715 // commands; in GOPATH mode, we expect that to be true of GOPATH/src. 716 dir := cfg.GOMODCACHE 717 if !cfg.ModulesEnabled { 718 dir = filepath.Join(cfg.BuildContext.GOPATH, "src") 719 } 720 os.MkdirAll(dir, 0777) // Ignore errors — if unsuccessful, the command will likely fail. 721 722 return v.runVerboseOnly(dir, v.PingCmd, "scheme", scheme, "repo", repo) 723 } 724 725 // Create creates a new copy of repo in dir. 726 // The parent of dir must exist; dir must not. 727 func (v *Cmd) Create(dir, repo string) error { 728 for _, cmd := range v.CreateCmd { 729 if err := v.run(filepath.Dir(dir), cmd, "dir", dir, "repo", repo); err != nil { 730 return err 731 } 732 } 733 return nil 734 } 735 736 // Download downloads any new changes for the repo in dir. 737 func (v *Cmd) Download(dir string) error { 738 for _, cmd := range v.DownloadCmd { 739 if err := v.run(dir, cmd); err != nil { 740 return err 741 } 742 } 743 return nil 744 } 745 746 // Tags returns the list of available tags for the repo in dir. 747 func (v *Cmd) Tags(dir string) ([]string, error) { 748 var tags []string 749 for _, tc := range v.TagCmd { 750 out, err := v.runOutput(dir, tc.cmd) 751 if err != nil { 752 return nil, err 753 } 754 re := regexp.MustCompile(`(?m-s)` + tc.pattern) 755 for _, m := range re.FindAllStringSubmatch(string(out), -1) { 756 tags = append(tags, m[1]) 757 } 758 } 759 return tags, nil 760 } 761 762 // tagSync syncs the repo in dir to the named tag, 763 // which either is a tag returned by tags or is v.tagDefault. 764 func (v *Cmd) TagSync(dir, tag string) error { 765 if v.TagSyncCmd == nil { 766 return nil 767 } 768 if tag != "" { 769 for _, tc := range v.TagLookupCmd { 770 out, err := v.runOutput(dir, tc.cmd, "tag", tag) 771 if err != nil { 772 return err 773 } 774 re := regexp.MustCompile(`(?m-s)` + tc.pattern) 775 m := re.FindStringSubmatch(string(out)) 776 if len(m) > 1 { 777 tag = m[1] 778 break 779 } 780 } 781 } 782 783 if tag == "" && v.TagSyncDefault != nil { 784 for _, cmd := range v.TagSyncDefault { 785 if err := v.run(dir, cmd); err != nil { 786 return err 787 } 788 } 789 return nil 790 } 791 792 for _, cmd := range v.TagSyncCmd { 793 if err := v.run(dir, cmd, "tag", tag); err != nil { 794 return err 795 } 796 } 797 return nil 798 } 799 800 // A vcsPath describes how to convert an import path into a 801 // version control system and repository name. 802 type vcsPath struct { 803 pathPrefix string // prefix this description applies to 804 regexp *lazyregexp.Regexp // compiled pattern for import path 805 repo string // repository to use (expand with match of re) 806 vcs string // version control system to use (expand with match of re) 807 check func(match map[string]string) error // additional checks 808 schemelessRepo bool // if true, the repo pattern lacks a scheme 809 } 810 811 // FromDir inspects dir and its parents to determine the 812 // version control system and code repository to use. 813 // If no repository is found, FromDir returns an error 814 // equivalent to os.ErrNotExist. 815 func FromDir(dir, srcRoot string, allowNesting bool) (repoDir string, vcsCmd *Cmd, err error) { 816 // Clean and double-check that dir is in (a subdirectory of) srcRoot. 817 dir = filepath.Clean(dir) 818 if srcRoot != "" { 819 srcRoot = filepath.Clean(srcRoot) 820 if len(dir) <= len(srcRoot) || dir[len(srcRoot)] != filepath.Separator { 821 return "", nil, fmt.Errorf("directory %q is outside source root %q", dir, srcRoot) 822 } 823 } 824 825 origDir := dir 826 for len(dir) > len(srcRoot) { 827 for _, vcs := range vcsList { 828 if isVCSRoot(dir, vcs.RootNames) { 829 // Record first VCS we find. 830 // If allowNesting is false (as it is in GOPATH), keep looking for 831 // repositories in parent directories and report an error if one is 832 // found to mitigate VCS injection attacks. 833 if vcsCmd == nil { 834 vcsCmd = vcs 835 repoDir = dir 836 if allowNesting { 837 return repoDir, vcsCmd, nil 838 } 839 continue 840 } 841 // Otherwise, we have one VCS inside a different VCS. 842 return "", nil, fmt.Errorf("directory %q uses %s, but parent %q uses %s", 843 repoDir, vcsCmd.Cmd, dir, vcs.Cmd) 844 } 845 } 846 847 // Move to parent. 848 ndir := filepath.Dir(dir) 849 if len(ndir) >= len(dir) { 850 break 851 } 852 dir = ndir 853 } 854 if vcsCmd == nil { 855 return "", nil, &vcsNotFoundError{dir: origDir} 856 } 857 return repoDir, vcsCmd, nil 858 } 859 860 // isVCSRoot identifies a VCS root by checking whether the directory contains 861 // any of the listed root names. 862 func isVCSRoot(dir string, rootNames []rootName) bool { 863 for _, root := range rootNames { 864 fi, err := os.Stat(filepath.Join(dir, root.filename)) 865 if err == nil && fi.IsDir() == root.isDir { 866 return true 867 } 868 } 869 870 return false 871 } 872 873 type rootName struct { 874 filename string 875 isDir bool 876 } 877 878 type vcsNotFoundError struct { 879 dir string 880 } 881 882 func (e *vcsNotFoundError) Error() string { 883 return fmt.Sprintf("directory %q is not using a known version control system", e.dir) 884 } 885 886 func (e *vcsNotFoundError) Is(err error) bool { 887 return err == os.ErrNotExist 888 } 889 890 // A govcsRule is a single GOVCS rule like private:hg|svn. 891 type govcsRule struct { 892 pattern string 893 allowed []string 894 } 895 896 // A govcsConfig is a full GOVCS configuration. 897 type govcsConfig []govcsRule 898 899 func parseGOVCS(s string) (govcsConfig, error) { 900 s = strings.TrimSpace(s) 901 if s == "" { 902 return nil, nil 903 } 904 var cfg govcsConfig 905 have := make(map[string]string) 906 for _, item := range strings.Split(s, ",") { 907 item = strings.TrimSpace(item) 908 if item == "" { 909 return nil, fmt.Errorf("empty entry in GOVCS") 910 } 911 pattern, list, found := strings.Cut(item, ":") 912 if !found { 913 return nil, fmt.Errorf("malformed entry in GOVCS (missing colon): %q", item) 914 } 915 pattern, list = strings.TrimSpace(pattern), strings.TrimSpace(list) 916 if pattern == "" { 917 return nil, fmt.Errorf("empty pattern in GOVCS: %q", item) 918 } 919 if list == "" { 920 return nil, fmt.Errorf("empty VCS list in GOVCS: %q", item) 921 } 922 if search.IsRelativePath(pattern) { 923 return nil, fmt.Errorf("relative pattern not allowed in GOVCS: %q", pattern) 924 } 925 if old := have[pattern]; old != "" { 926 return nil, fmt.Errorf("unreachable pattern in GOVCS: %q after %q", item, old) 927 } 928 have[pattern] = item 929 allowed := strings.Split(list, "|") 930 for i, a := range allowed { 931 a = strings.TrimSpace(a) 932 if a == "" { 933 return nil, fmt.Errorf("empty VCS name in GOVCS: %q", item) 934 } 935 allowed[i] = a 936 } 937 cfg = append(cfg, govcsRule{pattern, allowed}) 938 } 939 return cfg, nil 940 } 941 942 func (c *govcsConfig) allow(path string, private bool, vcs string) bool { 943 for _, rule := range *c { 944 match := false 945 switch rule.pattern { 946 case "private": 947 match = private 948 case "public": 949 match = !private 950 default: 951 // Note: rule.pattern is known to be comma-free, 952 // so MatchPrefixPatterns is only matching a single pattern for us. 953 match = module.MatchPrefixPatterns(rule.pattern, path) 954 } 955 if !match { 956 continue 957 } 958 for _, allow := range rule.allowed { 959 if allow == vcs || allow == "all" { 960 return true 961 } 962 } 963 return false 964 } 965 966 // By default, nothing is allowed. 967 return false 968 } 969 970 var ( 971 govcs govcsConfig 972 govcsErr error 973 govcsOnce sync.Once 974 ) 975 976 // defaultGOVCS is the default setting for GOVCS. 977 // Setting GOVCS adds entries ahead of these but does not remove them. 978 // (They are appended to the parsed GOVCS setting.) 979 // 980 // The rationale behind allowing only Git and Mercurial is that 981 // these two systems have had the most attention to issues 982 // of being run as clients of untrusted servers. In contrast, 983 // Bazaar, Fossil, and Subversion have primarily been used 984 // in trusted, authenticated environments and are not as well 985 // scrutinized as attack surfaces. 986 // 987 // See golang.org/issue/41730 for details. 988 var defaultGOVCS = govcsConfig{ 989 {"private", []string{"all"}}, 990 {"public", []string{"git", "hg"}}, 991 } 992 993 // CheckGOVCS checks whether the policy defined by the environment variable 994 // GOVCS allows the given vcs command to be used with the given repository 995 // root path. Note that root may not be a real package or module path; it's 996 // the same as the root path in the go-import meta tag. 997 func CheckGOVCS(vcs *Cmd, root string) error { 998 if vcs == vcsMod { 999 // Direct module (proxy protocol) fetches don't 1000 // involve an external version control system 1001 // and are always allowed. 1002 return nil 1003 } 1004 1005 govcsOnce.Do(func() { 1006 govcs, govcsErr = parseGOVCS(os.Getenv("GOVCS")) 1007 govcs = append(govcs, defaultGOVCS...) 1008 }) 1009 if govcsErr != nil { 1010 return govcsErr 1011 } 1012 1013 private := module.MatchPrefixPatterns(cfg.GOPRIVATE, root) 1014 if !govcs.allow(root, private, vcs.Cmd) { 1015 what := "public" 1016 if private { 1017 what = "private" 1018 } 1019 return fmt.Errorf("GOVCS disallows using %s for %s %s; see 'go help vcs'", vcs.Cmd, what, root) 1020 } 1021 1022 return nil 1023 } 1024 1025 // CheckNested checks for an incorrectly-nested VCS-inside-VCS 1026 // situation for dir, checking parents up until srcRoot. 1027 func CheckNested(vcs *Cmd, dir, srcRoot string) error { 1028 if len(dir) <= len(srcRoot) || dir[len(srcRoot)] != filepath.Separator { 1029 return fmt.Errorf("directory %q is outside source root %q", dir, srcRoot) 1030 } 1031 1032 otherDir := dir 1033 for len(otherDir) > len(srcRoot) { 1034 for _, otherVCS := range vcsList { 1035 if isVCSRoot(otherDir, otherVCS.RootNames) { 1036 // Allow expected vcs in original dir. 1037 if otherDir == dir && otherVCS == vcs { 1038 continue 1039 } 1040 // Otherwise, we have one VCS inside a different VCS. 1041 return fmt.Errorf("directory %q uses %s, but parent %q uses %s", dir, vcs.Cmd, otherDir, otherVCS.Cmd) 1042 } 1043 } 1044 // Move to parent. 1045 newDir := filepath.Dir(otherDir) 1046 if len(newDir) >= len(otherDir) { 1047 // Shouldn't happen, but just in case, stop. 1048 break 1049 } 1050 otherDir = newDir 1051 } 1052 1053 return nil 1054 } 1055 1056 // RepoRoot describes the repository root for a tree of source code. 1057 type RepoRoot struct { 1058 Repo string // repository URL, including scheme 1059 Root string // import path corresponding to root of repo 1060 IsCustom bool // defined by served <meta> tags (as opposed to hard-coded pattern) 1061 VCS *Cmd 1062 } 1063 1064 func httpPrefix(s string) string { 1065 for _, prefix := range [...]string{"http:", "https:"} { 1066 if strings.HasPrefix(s, prefix) { 1067 return prefix 1068 } 1069 } 1070 return "" 1071 } 1072 1073 // ModuleMode specifies whether to prefer modules when looking up code sources. 1074 type ModuleMode int 1075 1076 const ( 1077 IgnoreMod ModuleMode = iota 1078 PreferMod 1079 ) 1080 1081 // RepoRootForImportPath analyzes importPath to determine the 1082 // version control system, and code repository to use. 1083 func RepoRootForImportPath(importPath string, mod ModuleMode, security web.SecurityMode) (*RepoRoot, error) { 1084 rr, err := repoRootFromVCSPaths(importPath, security, vcsPaths) 1085 if err == errUnknownSite { 1086 rr, err = repoRootForImportDynamic(importPath, mod, security) 1087 if err != nil { 1088 err = importErrorf(importPath, "unrecognized import path %q: %v", importPath, err) 1089 } 1090 } 1091 if err != nil { 1092 rr1, err1 := repoRootFromVCSPaths(importPath, security, vcsPathsAfterDynamic) 1093 if err1 == nil { 1094 rr = rr1 1095 err = nil 1096 } 1097 } 1098 1099 // Should have been taken care of above, but make sure. 1100 if err == nil && strings.Contains(importPath, "...") && strings.Contains(rr.Root, "...") { 1101 // Do not allow wildcards in the repo root. 1102 rr = nil 1103 err = importErrorf(importPath, "cannot expand ... in %q", importPath) 1104 } 1105 return rr, err 1106 } 1107 1108 var errUnknownSite = errors.New("dynamic lookup required to find mapping") 1109 1110 // repoRootFromVCSPaths attempts to map importPath to a repoRoot 1111 // using the mappings defined in vcsPaths. 1112 func repoRootFromVCSPaths(importPath string, security web.SecurityMode, vcsPaths []*vcsPath) (*RepoRoot, error) { 1113 if str.HasPathPrefix(importPath, "example.net") { 1114 // TODO(rsc): This should not be necessary, but it's required to keep 1115 // tests like ../../testdata/script/mod_get_extra.txt from using the network. 1116 // That script has everything it needs in the replacement set, but it is still 1117 // doing network calls. 1118 return nil, fmt.Errorf("no modules on example.net") 1119 } 1120 if importPath == "rsc.io" { 1121 // This special case allows tests like ../../testdata/script/govcs.txt 1122 // to avoid making any network calls. The module lookup for a path 1123 // like rsc.io/nonexist.svn/foo needs to not make a network call for 1124 // a lookup on rsc.io. 1125 return nil, fmt.Errorf("rsc.io is not a module") 1126 } 1127 // A common error is to use https://packagepath because that's what 1128 // hg and git require. Diagnose this helpfully. 1129 if prefix := httpPrefix(importPath); prefix != "" { 1130 // The importPath has been cleaned, so has only one slash. The pattern 1131 // ignores the slashes; the error message puts them back on the RHS at least. 1132 return nil, fmt.Errorf("%q not allowed in import path", prefix+"//") 1133 } 1134 for _, srv := range vcsPaths { 1135 if !str.HasPathPrefix(importPath, srv.pathPrefix) { 1136 continue 1137 } 1138 m := srv.regexp.FindStringSubmatch(importPath) 1139 if m == nil { 1140 if srv.pathPrefix != "" { 1141 return nil, importErrorf(importPath, "invalid %s import path %q", srv.pathPrefix, importPath) 1142 } 1143 continue 1144 } 1145 1146 // Build map of named subexpression matches for expand. 1147 match := map[string]string{ 1148 "prefix": srv.pathPrefix + "/", 1149 "import": importPath, 1150 } 1151 for i, name := range srv.regexp.SubexpNames() { 1152 if name != "" && match[name] == "" { 1153 match[name] = m[i] 1154 } 1155 } 1156 if srv.vcs != "" { 1157 match["vcs"] = expand(match, srv.vcs) 1158 } 1159 if srv.repo != "" { 1160 match["repo"] = expand(match, srv.repo) 1161 } 1162 if srv.check != nil { 1163 if err := srv.check(match); err != nil { 1164 return nil, err 1165 } 1166 } 1167 vcs := vcsByCmd(match["vcs"]) 1168 if vcs == nil { 1169 return nil, fmt.Errorf("unknown version control system %q", match["vcs"]) 1170 } 1171 if err := CheckGOVCS(vcs, match["root"]); err != nil { 1172 return nil, err 1173 } 1174 var repoURL string 1175 if !srv.schemelessRepo { 1176 repoURL = match["repo"] 1177 } else { 1178 repo := match["repo"] 1179 var ok bool 1180 repoURL, ok = interceptVCSTest(repo, vcs, security) 1181 if !ok { 1182 scheme := vcs.Scheme[0] // default to first scheme 1183 if vcs.PingCmd != "" { 1184 // If we know how to test schemes, scan to find one. 1185 for _, s := range vcs.Scheme { 1186 if security == web.SecureOnly && !vcs.isSecureScheme(s) { 1187 continue 1188 } 1189 if vcs.Ping(s, repo) == nil { 1190 scheme = s 1191 break 1192 } 1193 } 1194 } 1195 repoURL = scheme + "://" + repo 1196 } 1197 } 1198 rr := &RepoRoot{ 1199 Repo: repoURL, 1200 Root: match["root"], 1201 VCS: vcs, 1202 } 1203 return rr, nil 1204 } 1205 return nil, errUnknownSite 1206 } 1207 1208 func interceptVCSTest(repo string, vcs *Cmd, security web.SecurityMode) (repoURL string, ok bool) { 1209 if VCSTestRepoURL == "" { 1210 return "", false 1211 } 1212 if vcs == vcsMod { 1213 // Since the "mod" protocol is implemented internally, 1214 // requests will be intercepted at a lower level (in cmd/go/internal/web). 1215 return "", false 1216 } 1217 1218 if scheme, path, ok := strings.Cut(repo, "://"); ok { 1219 if security == web.SecureOnly && !vcs.isSecureScheme(scheme) { 1220 return "", false // Let the caller reject the original URL. 1221 } 1222 repo = path // Remove leading URL scheme if present. 1223 } 1224 for _, host := range VCSTestHosts { 1225 if !str.HasPathPrefix(repo, host) { 1226 continue 1227 } 1228 1229 httpURL := VCSTestRepoURL + strings.TrimPrefix(repo, host) 1230 1231 if vcs == vcsSvn { 1232 // Ping the vcweb HTTP server to tell it to initialize the SVN repository 1233 // and get the SVN server URL. 1234 u, err := urlpkg.Parse(httpURL + "?vcwebsvn=1") 1235 if err != nil { 1236 panic(fmt.Sprintf("invalid vcs-test repo URL: %v", err)) 1237 } 1238 svnURL, err := web.GetBytes(u) 1239 svnURL = bytes.TrimSpace(svnURL) 1240 if err == nil && len(svnURL) > 0 { 1241 return string(svnURL) + strings.TrimPrefix(repo, host), true 1242 } 1243 1244 // vcs-test doesn't have a svn handler for the given path, 1245 // so resolve the repo to HTTPS instead. 1246 } 1247 1248 return httpURL, true 1249 } 1250 return "", false 1251 } 1252 1253 // urlForImportPath returns a partially-populated URL for the given Go import path. 1254 // 1255 // The URL leaves the Scheme field blank so that web.Get will try any scheme 1256 // allowed by the selected security mode. 1257 func urlForImportPath(importPath string) (*urlpkg.URL, error) { 1258 slash := strings.Index(importPath, "/") 1259 if slash < 0 { 1260 slash = len(importPath) 1261 } 1262 host, path := importPath[:slash], importPath[slash:] 1263 if !strings.Contains(host, ".") { 1264 return nil, errors.New("import path does not begin with hostname") 1265 } 1266 if len(path) == 0 { 1267 path = "/" 1268 } 1269 return &urlpkg.URL{Host: host, Path: path, RawQuery: "go-get=1"}, nil 1270 } 1271 1272 // repoRootForImportDynamic finds a *RepoRoot for a custom domain that's not 1273 // statically known by repoRootForImportPathStatic. 1274 // 1275 // This handles custom import paths like "name.tld/pkg/foo" or just "name.tld". 1276 func repoRootForImportDynamic(importPath string, mod ModuleMode, security web.SecurityMode) (*RepoRoot, error) { 1277 url, err := urlForImportPath(importPath) 1278 if err != nil { 1279 return nil, err 1280 } 1281 resp, err := web.Get(security, url) 1282 if err != nil { 1283 msg := "https fetch: %v" 1284 if security == web.Insecure { 1285 msg = "http/" + msg 1286 } 1287 return nil, fmt.Errorf(msg, err) 1288 } 1289 body := resp.Body 1290 defer body.Close() 1291 imports, err := parseMetaGoImports(body, mod) 1292 if len(imports) == 0 { 1293 if respErr := resp.Err(); respErr != nil { 1294 // If the server's status was not OK, prefer to report that instead of 1295 // an XML parse error. 1296 return nil, respErr 1297 } 1298 } 1299 if err != nil { 1300 return nil, fmt.Errorf("parsing %s: %v", importPath, err) 1301 } 1302 // Find the matched meta import. 1303 mmi, err := matchGoImport(imports, importPath) 1304 if err != nil { 1305 if _, ok := err.(ImportMismatchError); !ok { 1306 return nil, fmt.Errorf("parse %s: %v", url, err) 1307 } 1308 return nil, fmt.Errorf("parse %s: no go-import meta tags (%s)", resp.URL, err) 1309 } 1310 if cfg.BuildV { 1311 log.Printf("get %q: found meta tag %#v at %s", importPath, mmi, url) 1312 } 1313 // If the import was "uni.edu/bob/project", which said the 1314 // prefix was "uni.edu" and the RepoRoot was "evilroot.com", 1315 // make sure we don't trust Bob and check out evilroot.com to 1316 // "uni.edu" yet (possibly overwriting/preempting another 1317 // non-evil student). Instead, first verify the root and see 1318 // if it matches Bob's claim. 1319 if mmi.Prefix != importPath { 1320 if cfg.BuildV { 1321 log.Printf("get %q: verifying non-authoritative meta tag", importPath) 1322 } 1323 var imports []metaImport 1324 url, imports, err = metaImportsForPrefix(mmi.Prefix, mod, security) 1325 if err != nil { 1326 return nil, err 1327 } 1328 metaImport2, err := matchGoImport(imports, importPath) 1329 if err != nil || mmi != metaImport2 { 1330 return nil, fmt.Errorf("%s and %s disagree about go-import for %s", resp.URL, url, mmi.Prefix) 1331 } 1332 } 1333 1334 if err := validateRepoRoot(mmi.RepoRoot); err != nil { 1335 return nil, fmt.Errorf("%s: invalid repo root %q: %v", resp.URL, mmi.RepoRoot, err) 1336 } 1337 var vcs *Cmd 1338 if mmi.VCS == "mod" { 1339 vcs = vcsMod 1340 } else { 1341 vcs = vcsByCmd(mmi.VCS) 1342 if vcs == nil { 1343 return nil, fmt.Errorf("%s: unknown vcs %q", resp.URL, mmi.VCS) 1344 } 1345 } 1346 1347 if err := CheckGOVCS(vcs, mmi.Prefix); err != nil { 1348 return nil, err 1349 } 1350 1351 repoURL, ok := interceptVCSTest(mmi.RepoRoot, vcs, security) 1352 if !ok { 1353 repoURL = mmi.RepoRoot 1354 } 1355 rr := &RepoRoot{ 1356 Repo: repoURL, 1357 Root: mmi.Prefix, 1358 IsCustom: true, 1359 VCS: vcs, 1360 } 1361 return rr, nil 1362 } 1363 1364 // validateRepoRoot returns an error if repoRoot does not seem to be 1365 // a valid URL with scheme. 1366 func validateRepoRoot(repoRoot string) error { 1367 url, err := urlpkg.Parse(repoRoot) 1368 if err != nil { 1369 return err 1370 } 1371 if url.Scheme == "" { 1372 return errors.New("no scheme") 1373 } 1374 if url.Scheme == "file" { 1375 return errors.New("file scheme disallowed") 1376 } 1377 return nil 1378 } 1379 1380 var fetchGroup singleflight.Group 1381 var ( 1382 fetchCacheMu sync.Mutex 1383 fetchCache = map[string]fetchResult{} // key is metaImportsForPrefix's importPrefix 1384 ) 1385 1386 // metaImportsForPrefix takes a package's root import path as declared in a <meta> tag 1387 // and returns its HTML discovery URL and the parsed metaImport lines 1388 // found on the page. 1389 // 1390 // The importPath is of the form "golang.org/x/tools". 1391 // It is an error if no imports are found. 1392 // url will still be valid if err != nil. 1393 // The returned url will be of the form "https://golang.org/x/tools?go-get=1" 1394 func metaImportsForPrefix(importPrefix string, mod ModuleMode, security web.SecurityMode) (*urlpkg.URL, []metaImport, error) { 1395 setCache := func(res fetchResult) (fetchResult, error) { 1396 fetchCacheMu.Lock() 1397 defer fetchCacheMu.Unlock() 1398 fetchCache[importPrefix] = res 1399 return res, nil 1400 } 1401 1402 resi, _, _ := fetchGroup.Do(importPrefix, func() (resi any, err error) { 1403 fetchCacheMu.Lock() 1404 if res, ok := fetchCache[importPrefix]; ok { 1405 fetchCacheMu.Unlock() 1406 return res, nil 1407 } 1408 fetchCacheMu.Unlock() 1409 1410 url, err := urlForImportPath(importPrefix) 1411 if err != nil { 1412 return setCache(fetchResult{err: err}) 1413 } 1414 resp, err := web.Get(security, url) 1415 if err != nil { 1416 return setCache(fetchResult{url: url, err: fmt.Errorf("fetching %s: %v", importPrefix, err)}) 1417 } 1418 body := resp.Body 1419 defer body.Close() 1420 imports, err := parseMetaGoImports(body, mod) 1421 if len(imports) == 0 { 1422 if respErr := resp.Err(); respErr != nil { 1423 // If the server's status was not OK, prefer to report that instead of 1424 // an XML parse error. 1425 return setCache(fetchResult{url: url, err: respErr}) 1426 } 1427 } 1428 if err != nil { 1429 return setCache(fetchResult{url: url, err: fmt.Errorf("parsing %s: %v", resp.URL, err)}) 1430 } 1431 if len(imports) == 0 { 1432 err = fmt.Errorf("fetching %s: no go-import meta tag found in %s", importPrefix, resp.URL) 1433 } 1434 return setCache(fetchResult{url: url, imports: imports, err: err}) 1435 }) 1436 res := resi.(fetchResult) 1437 return res.url, res.imports, res.err 1438 } 1439 1440 type fetchResult struct { 1441 url *urlpkg.URL 1442 imports []metaImport 1443 err error 1444 } 1445 1446 // metaImport represents the parsed <meta name="go-import" 1447 // content="prefix vcs reporoot" /> tags from HTML files. 1448 type metaImport struct { 1449 Prefix, VCS, RepoRoot string 1450 } 1451 1452 // A ImportMismatchError is returned where metaImport/s are present 1453 // but none match our import path. 1454 type ImportMismatchError struct { 1455 importPath string 1456 mismatches []string // the meta imports that were discarded for not matching our importPath 1457 } 1458 1459 func (m ImportMismatchError) Error() string { 1460 formattedStrings := make([]string, len(m.mismatches)) 1461 for i, pre := range m.mismatches { 1462 formattedStrings[i] = fmt.Sprintf("meta tag %s did not match import path %s", pre, m.importPath) 1463 } 1464 return strings.Join(formattedStrings, ", ") 1465 } 1466 1467 // matchGoImport returns the metaImport from imports matching importPath. 1468 // An error is returned if there are multiple matches. 1469 // An ImportMismatchError is returned if none match. 1470 func matchGoImport(imports []metaImport, importPath string) (metaImport, error) { 1471 match := -1 1472 1473 errImportMismatch := ImportMismatchError{importPath: importPath} 1474 for i, im := range imports { 1475 if !str.HasPathPrefix(importPath, im.Prefix) { 1476 errImportMismatch.mismatches = append(errImportMismatch.mismatches, im.Prefix) 1477 continue 1478 } 1479 1480 if match >= 0 { 1481 if imports[match].VCS == "mod" && im.VCS != "mod" { 1482 // All the mod entries precede all the non-mod entries. 1483 // We have a mod entry and don't care about the rest, 1484 // matching or not. 1485 break 1486 } 1487 return metaImport{}, fmt.Errorf("multiple meta tags match import path %q", importPath) 1488 } 1489 match = i 1490 } 1491 1492 if match == -1 { 1493 return metaImport{}, errImportMismatch 1494 } 1495 return imports[match], nil 1496 } 1497 1498 // expand rewrites s to replace {k} with match[k] for each key k in match. 1499 func expand(match map[string]string, s string) string { 1500 // We want to replace each match exactly once, and the result of expansion 1501 // must not depend on the iteration order through the map. 1502 // A strings.Replacer has exactly the properties we're looking for. 1503 oldNew := make([]string, 0, 2*len(match)) 1504 for k, v := range match { 1505 oldNew = append(oldNew, "{"+k+"}", v) 1506 } 1507 return strings.NewReplacer(oldNew...).Replace(s) 1508 } 1509 1510 // vcsPaths defines the meaning of import paths referring to 1511 // commonly-used VCS hosting sites (github.com/user/dir) 1512 // and import paths referring to a fully-qualified importPath 1513 // containing a VCS type (foo.com/repo.git/dir) 1514 var vcsPaths = []*vcsPath{ 1515 // GitHub 1516 { 1517 pathPrefix: "github.com", 1518 regexp: lazyregexp.New(`^(?P<root>github\.com/[\w.\-]+/[\w.\-]+)(/[\w.\-]+)*$`), 1519 vcs: "git", 1520 repo: "https://{root}", 1521 check: noVCSSuffix, 1522 }, 1523 1524 // Bitbucket 1525 { 1526 pathPrefix: "bitbucket.org", 1527 regexp: lazyregexp.New(`^(?P<root>bitbucket\.org/(?P<bitname>[\w.\-]+/[\w.\-]+))(/[\w.\-]+)*$`), 1528 vcs: "git", 1529 repo: "https://{root}", 1530 check: noVCSSuffix, 1531 }, 1532 1533 // IBM DevOps Services (JazzHub) 1534 { 1535 pathPrefix: "hub.jazz.net/git", 1536 regexp: lazyregexp.New(`^(?P<root>hub\.jazz\.net/git/[a-z0-9]+/[\w.\-]+)(/[\w.\-]+)*$`), 1537 vcs: "git", 1538 repo: "https://{root}", 1539 check: noVCSSuffix, 1540 }, 1541 1542 // Git at Apache 1543 { 1544 pathPrefix: "git.apache.org", 1545 regexp: lazyregexp.New(`^(?P<root>git\.apache\.org/[a-z0-9_.\-]+\.git)(/[\w.\-]+)*$`), 1546 vcs: "git", 1547 repo: "https://{root}", 1548 }, 1549 1550 // Git at OpenStack 1551 { 1552 pathPrefix: "git.openstack.org", 1553 regexp: lazyregexp.New(`^(?P<root>git\.openstack\.org/[\w.\-]+/[\w.\-]+)(\.git)?(/[\w.\-]+)*$`), 1554 vcs: "git", 1555 repo: "https://{root}", 1556 }, 1557 1558 // chiselapp.com for fossil 1559 { 1560 pathPrefix: "chiselapp.com", 1561 regexp: lazyregexp.New(`^(?P<root>chiselapp\.com/user/[A-Za-z0-9]+/repository/[\w.\-]+)$`), 1562 vcs: "fossil", 1563 repo: "https://{root}", 1564 }, 1565 1566 // General syntax for any server. 1567 // Must be last. 1568 { 1569 regexp: lazyregexp.New(`(?P<root>(?P<repo>([a-z0-9.\-]+\.)+[a-z0-9.\-]+(:[0-9]+)?(/~?[\w.\-]+)+?)\.(?P<vcs>bzr|fossil|git|hg|svn))(/~?[\w.\-]+)*$`), 1570 schemelessRepo: true, 1571 }, 1572 } 1573 1574 // vcsPathsAfterDynamic gives additional vcsPaths entries 1575 // to try after the dynamic HTML check. 1576 // This gives those sites a chance to introduce <meta> tags 1577 // as part of a graceful transition away from the hard-coded logic. 1578 var vcsPathsAfterDynamic = []*vcsPath{ 1579 // Launchpad. See golang.org/issue/11436. 1580 { 1581 pathPrefix: "launchpad.net", 1582 regexp: lazyregexp.New(`^(?P<root>launchpad\.net/((?P<project>[\w.\-]+)(?P<series>/[\w.\-]+)?|~[\w.\-]+/(\+junk|[\w.\-]+)/[\w.\-]+))(/[\w.\-]+)*$`), 1583 vcs: "bzr", 1584 repo: "https://{root}", 1585 check: launchpadVCS, 1586 }, 1587 } 1588 1589 // noVCSSuffix checks that the repository name does not 1590 // end in .foo for any version control system foo. 1591 // The usual culprit is ".git". 1592 func noVCSSuffix(match map[string]string) error { 1593 repo := match["repo"] 1594 for _, vcs := range vcsList { 1595 if strings.HasSuffix(repo, "."+vcs.Cmd) { 1596 return fmt.Errorf("invalid version control suffix in %s path", match["prefix"]) 1597 } 1598 } 1599 return nil 1600 } 1601 1602 // launchpadVCS solves the ambiguity for "lp.net/project/foo". In this case, 1603 // "foo" could be a series name registered in Launchpad with its own branch, 1604 // and it could also be the name of a directory within the main project 1605 // branch one level up. 1606 func launchpadVCS(match map[string]string) error { 1607 if match["project"] == "" || match["series"] == "" { 1608 return nil 1609 } 1610 url := &urlpkg.URL{ 1611 Scheme: "https", 1612 Host: "code.launchpad.net", 1613 Path: expand(match, "/{project}{series}/.bzr/branch-format"), 1614 } 1615 _, err := web.GetBytes(url) 1616 if err != nil { 1617 match["root"] = expand(match, "launchpad.net/{project}") 1618 match["repo"] = expand(match, "https://{root}") 1619 } 1620 return nil 1621 } 1622 1623 // importError is a copy of load.importError, made to avoid a dependency cycle 1624 // on cmd/go/internal/load. It just needs to satisfy load.ImportPathError. 1625 type importError struct { 1626 importPath string 1627 err error 1628 } 1629 1630 func importErrorf(path, format string, args ...any) error { 1631 err := &importError{importPath: path, err: fmt.Errorf(format, args...)} 1632 if errStr := err.Error(); !strings.Contains(errStr, path) { 1633 panic(fmt.Sprintf("path %q not in error %q", path, errStr)) 1634 } 1635 return err 1636 } 1637 1638 func (e *importError) Error() string { 1639 return e.err.Error() 1640 } 1641 1642 func (e *importError) Unwrap() error { 1643 // Don't return e.err directly, since we're only wrapping an error if %w 1644 // was passed to ImportErrorf. 1645 return errors.Unwrap(e.err) 1646 } 1647 1648 func (e *importError) ImportPath() string { 1649 return e.importPath 1650 }