github.com/bir3/gocompiler@v0.3.205/src/cmd/gocmd/internal/modfetch/codehost/vcs.go (about) 1 // Copyright 2018 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package codehost 6 7 import ( 8 "errors" 9 "fmt" 10 "github.com/bir3/gocompiler/src/internal/lazyregexp" 11 "io" 12 "io/fs" 13 "os" 14 "path/filepath" 15 "sort" 16 "strconv" 17 "strings" 18 "sync" 19 "time" 20 21 "github.com/bir3/gocompiler/src/cmd/gocmd/internal/lockedfile" 22 "github.com/bir3/gocompiler/src/cmd/gocmd/internal/par" 23 "github.com/bir3/gocompiler/src/cmd/gocmd/internal/str" 24 ) 25 26 // A VCSError indicates an error using a version control system. 27 // The implication of a VCSError is that we know definitively where 28 // to get the code, but we can't access it due to the error. 29 // The caller should report this error instead of continuing to probe 30 // other possible module paths. 31 // 32 // TODO(golang.org/issue/31730): See if we can invert this. (Return a 33 // distinguished error for “repo not found” and treat everything else 34 // as terminal.) 35 type VCSError struct { 36 Err error 37 } 38 39 func (e *VCSError) Error() string { return e.Err.Error() } 40 41 func (e *VCSError) Unwrap() error { return e.Err } 42 43 func vcsErrorf(format string, a ...any) error { 44 return &VCSError{Err: fmt.Errorf(format, a...)} 45 } 46 47 func NewRepo(vcs, remote string) (Repo, error) { 48 type key struct { 49 vcs string 50 remote string 51 } 52 type cached struct { 53 repo Repo 54 err error 55 } 56 c := vcsRepoCache.Do(key{vcs, remote}, func() any { 57 repo, err := newVCSRepo(vcs, remote) 58 if err != nil { 59 err = &VCSError{err} 60 } 61 return cached{repo, err} 62 }).(cached) 63 64 return c.repo, c.err 65 } 66 67 var vcsRepoCache par.Cache 68 69 type vcsRepo struct { 70 mu lockedfile.Mutex // protects all commands, so we don't have to decide which are safe on a per-VCS basis 71 72 remote string 73 cmd *vcsCmd 74 dir string 75 76 tagsOnce sync.Once 77 tags map[string]bool 78 79 branchesOnce sync.Once 80 branches map[string]bool 81 82 fetchOnce sync.Once 83 fetchErr error 84 } 85 86 func newVCSRepo(vcs, remote string) (Repo, error) { 87 if vcs == "git" { 88 return newGitRepo(remote, false) 89 } 90 cmd := vcsCmds[vcs] 91 if cmd == nil { 92 return nil, fmt.Errorf("unknown vcs: %s %s", vcs, remote) 93 } 94 if !strings.Contains(remote, "://") { 95 return nil, fmt.Errorf("invalid vcs remote: %s %s", vcs, remote) 96 } 97 98 r := &vcsRepo{remote: remote, cmd: cmd} 99 var err error 100 r.dir, r.mu.Path, err = WorkDir(vcsWorkDirType+vcs, r.remote) 101 if err != nil { 102 return nil, err 103 } 104 105 if cmd.init == nil { 106 return r, nil 107 } 108 109 unlock, err := r.mu.Lock() 110 if err != nil { 111 return nil, err 112 } 113 defer unlock() 114 115 if _, err := os.Stat(filepath.Join(r.dir, "."+vcs)); err != nil { 116 if _, err := Run(r.dir, cmd.init(r.remote)); err != nil { 117 os.RemoveAll(r.dir) 118 return nil, err 119 } 120 } 121 return r, nil 122 } 123 124 const vcsWorkDirType = "vcs1." 125 126 type vcsCmd struct { 127 vcs string // vcs name "hg" 128 init func(remote string) []string // cmd to init repo to track remote 129 tags func(remote string) []string // cmd to list local tags 130 tagRE *lazyregexp.Regexp // regexp to extract tag names from output of tags cmd 131 branches func(remote string) []string // cmd to list local branches 132 branchRE *lazyregexp.Regexp // regexp to extract branch names from output of tags cmd 133 badLocalRevRE *lazyregexp.Regexp // regexp of names that must not be served out of local cache without doing fetch first 134 statLocal func(rev, remote string) []string // cmd to stat local rev 135 parseStat func(rev, out string) (*RevInfo, error) // cmd to parse output of statLocal 136 fetch []string // cmd to fetch everything from remote 137 latest string // name of latest commit on remote (tip, HEAD, etc) 138 readFile func(rev, file, remote string) []string // cmd to read rev's file 139 readZip func(rev, subdir, remote, target string) []string // cmd to read rev's subdir as zip file 140 doReadZip func(dst io.Writer, workDir, rev, subdir, remote string) error // arbitrary function to read rev's subdir as zip file 141 } 142 143 var re = lazyregexp.New 144 145 var vcsCmds = map[string]*vcsCmd{ 146 "hg": { 147 vcs: "hg", 148 init: func(remote string) []string { 149 return []string{"hg", "clone", "-U", "--", remote, "."} 150 }, 151 tags: func(remote string) []string { 152 return []string{"hg", "tags", "-q"} 153 }, 154 tagRE: re(`(?m)^[^\n]+$`), 155 branches: func(remote string) []string { 156 return []string{"hg", "branches", "-c", "-q"} 157 }, 158 branchRE: re(`(?m)^[^\n]+$`), 159 badLocalRevRE: re(`(?m)^(tip)$`), 160 statLocal: func(rev, remote string) []string { 161 return []string{"hg", "log", "-l1", "-r", rev, "--template", "{node} {date|hgdate} {tags}"} 162 }, 163 parseStat: hgParseStat, 164 fetch: []string{"hg", "pull", "-f"}, 165 latest: "tip", 166 readFile: func(rev, file, remote string) []string { 167 return []string{"hg", "cat", "-r", rev, file} 168 }, 169 readZip: func(rev, subdir, remote, target string) []string { 170 pattern := []string{} 171 if subdir != "" { 172 pattern = []string{"-I", subdir + "/**"} 173 } 174 return str.StringList("hg", "archive", "-t", "zip", "--no-decode", "-r", rev, "--prefix=prefix/", pattern, "--", target) 175 }, 176 }, 177 178 "svn": { 179 vcs: "svn", 180 init: nil, // no local checkout 181 tags: func(remote string) []string { 182 return []string{"svn", "list", "--", strings.TrimSuffix(remote, "/trunk") + "/tags"} 183 }, 184 tagRE: re(`(?m)^(.*?)/?$`), 185 statLocal: func(rev, remote string) []string { 186 suffix := "@" + rev 187 if rev == "latest" { 188 suffix = "" 189 } 190 return []string{"svn", "log", "-l1", "--xml", "--", remote + suffix} 191 }, 192 parseStat: svnParseStat, 193 latest: "latest", 194 readFile: func(rev, file, remote string) []string { 195 return []string{"svn", "cat", "--", remote + "/" + file + "@" + rev} 196 }, 197 doReadZip: svnReadZip, 198 }, 199 200 "bzr": { 201 vcs: "bzr", 202 init: func(remote string) []string { 203 return []string{"bzr", "branch", "--use-existing-dir", "--", remote, "."} 204 }, 205 fetch: []string{ 206 "bzr", "pull", "--overwrite-tags", 207 }, 208 tags: func(remote string) []string { 209 return []string{"bzr", "tags"} 210 }, 211 tagRE: re(`(?m)^\S+`), 212 badLocalRevRE: re(`^revno:-`), 213 statLocal: func(rev, remote string) []string { 214 return []string{"bzr", "log", "-l1", "--long", "--show-ids", "-r", rev} 215 }, 216 parseStat: bzrParseStat, 217 latest: "revno:-1", 218 readFile: func(rev, file, remote string) []string { 219 return []string{"bzr", "cat", "-r", rev, file} 220 }, 221 readZip: func(rev, subdir, remote, target string) []string { 222 extra := []string{} 223 if subdir != "" { 224 extra = []string{"./" + subdir} 225 } 226 return str.StringList("bzr", "export", "--format=zip", "-r", rev, "--root=prefix/", "--", target, extra) 227 }, 228 }, 229 230 "fossil": { 231 vcs: "fossil", 232 init: func(remote string) []string { 233 return []string{"fossil", "clone", "--", remote, ".fossil"} 234 }, 235 fetch: []string{"fossil", "pull", "-R", ".fossil"}, 236 tags: func(remote string) []string { 237 return []string{"fossil", "tag", "-R", ".fossil", "list"} 238 }, 239 tagRE: re(`XXXTODO`), 240 statLocal: func(rev, remote string) []string { 241 return []string{"fossil", "info", "-R", ".fossil", rev} 242 }, 243 parseStat: fossilParseStat, 244 latest: "trunk", 245 readFile: func(rev, file, remote string) []string { 246 return []string{"fossil", "cat", "-R", ".fossil", "-r", rev, file} 247 }, 248 readZip: func(rev, subdir, remote, target string) []string { 249 extra := []string{} 250 if subdir != "" && !strings.ContainsAny(subdir, "*?[],") { 251 extra = []string{"--include", subdir} 252 } 253 // Note that vcsRepo.ReadZip below rewrites this command 254 // to run in a different directory, to work around a fossil bug. 255 return str.StringList("fossil", "zip", "-R", ".fossil", "--name", "prefix", extra, "--", rev, target) 256 }, 257 }, 258 } 259 260 func (r *vcsRepo) loadTags() { 261 out, err := Run(r.dir, r.cmd.tags(r.remote)) 262 if err != nil { 263 return 264 } 265 266 // Run tag-listing command and extract tags. 267 r.tags = make(map[string]bool) 268 for _, tag := range r.cmd.tagRE.FindAllString(string(out), -1) { 269 if r.cmd.badLocalRevRE != nil && r.cmd.badLocalRevRE.MatchString(tag) { 270 continue 271 } 272 r.tags[tag] = true 273 } 274 } 275 276 func (r *vcsRepo) loadBranches() { 277 if r.cmd.branches == nil { 278 return 279 } 280 281 out, err := Run(r.dir, r.cmd.branches(r.remote)) 282 if err != nil { 283 return 284 } 285 286 r.branches = make(map[string]bool) 287 for _, branch := range r.cmd.branchRE.FindAllString(string(out), -1) { 288 if r.cmd.badLocalRevRE != nil && r.cmd.badLocalRevRE.MatchString(branch) { 289 continue 290 } 291 r.branches[branch] = true 292 } 293 } 294 295 func (r *vcsRepo) CheckReuse(old *Origin, subdir string) error { 296 return fmt.Errorf("vcs %s: CheckReuse: %w", r.cmd.vcs, ErrUnsupported) 297 } 298 299 func (r *vcsRepo) Tags(prefix string) (*Tags, error) { 300 unlock, err := r.mu.Lock() 301 if err != nil { 302 return nil, err 303 } 304 defer unlock() 305 306 r.tagsOnce.Do(r.loadTags) 307 tags := &Tags{ 308 // None of the other VCS provide a reasonable way to compute TagSum 309 // without downloading the whole repo, so we only include VCS and URL 310 // in the Origin. 311 Origin: &Origin{ 312 VCS: r.cmd.vcs, 313 URL: r.remote, 314 }, 315 List: []Tag{}, 316 } 317 for tag := range r.tags { 318 if strings.HasPrefix(tag, prefix) { 319 tags.List = append(tags.List, Tag{tag, ""}) 320 } 321 } 322 sort.Slice(tags.List, func(i, j int) bool { 323 return tags.List[i].Name < tags.List[j].Name 324 }) 325 return tags, nil 326 } 327 328 func (r *vcsRepo) Stat(rev string) (*RevInfo, error) { 329 unlock, err := r.mu.Lock() 330 if err != nil { 331 return nil, err 332 } 333 defer unlock() 334 335 if rev == "latest" { 336 rev = r.cmd.latest 337 } 338 r.branchesOnce.Do(r.loadBranches) 339 revOK := (r.cmd.badLocalRevRE == nil || !r.cmd.badLocalRevRE.MatchString(rev)) && !r.branches[rev] 340 if revOK { 341 if info, err := r.statLocal(rev); err == nil { 342 return info, nil 343 } 344 } 345 346 r.fetchOnce.Do(r.fetch) 347 if r.fetchErr != nil { 348 return nil, r.fetchErr 349 } 350 info, err := r.statLocal(rev) 351 if err != nil { 352 return nil, err 353 } 354 if !revOK { 355 info.Version = info.Name 356 } 357 return info, nil 358 } 359 360 func (r *vcsRepo) fetch() { 361 if len(r.cmd.fetch) > 0 { 362 _, r.fetchErr = Run(r.dir, r.cmd.fetch) 363 } 364 } 365 366 func (r *vcsRepo) statLocal(rev string) (*RevInfo, error) { 367 out, err := Run(r.dir, r.cmd.statLocal(rev, r.remote)) 368 if err != nil { 369 return nil, &UnknownRevisionError{Rev: rev} 370 } 371 info, err := r.cmd.parseStat(rev, string(out)) 372 if err != nil { 373 return nil, err 374 } 375 if info.Origin == nil { 376 info.Origin = new(Origin) 377 } 378 info.Origin.VCS = r.cmd.vcs 379 info.Origin.URL = r.remote 380 return info, nil 381 } 382 383 func (r *vcsRepo) Latest() (*RevInfo, error) { 384 return r.Stat("latest") 385 } 386 387 func (r *vcsRepo) ReadFile(rev, file string, maxSize int64) ([]byte, error) { 388 if rev == "latest" { 389 rev = r.cmd.latest 390 } 391 _, err := r.Stat(rev) // download rev into local repo 392 if err != nil { 393 return nil, err 394 } 395 396 // r.Stat acquires r.mu, so lock after that. 397 unlock, err := r.mu.Lock() 398 if err != nil { 399 return nil, err 400 } 401 defer unlock() 402 403 out, err := Run(r.dir, r.cmd.readFile(rev, file, r.remote)) 404 if err != nil { 405 return nil, fs.ErrNotExist 406 } 407 return out, nil 408 } 409 410 func (r *vcsRepo) RecentTag(rev, prefix string, allowed func(string) bool) (tag string, err error) { 411 // We don't technically need to lock here since we're returning an error 412 // uncondititonally, but doing so anyway will help to avoid baking in 413 // lock-inversion bugs. 414 unlock, err := r.mu.Lock() 415 if err != nil { 416 return "", err 417 } 418 defer unlock() 419 420 return "", vcsErrorf("vcs %s: RecentTag: %w", r.cmd.vcs, ErrUnsupported) 421 } 422 423 func (r *vcsRepo) DescendsFrom(rev, tag string) (bool, error) { 424 unlock, err := r.mu.Lock() 425 if err != nil { 426 return false, err 427 } 428 defer unlock() 429 430 return false, vcsErrorf("vcs %s: DescendsFrom: %w", r.cmd.vcs, ErrUnsupported) 431 } 432 433 func (r *vcsRepo) ReadZip(rev, subdir string, maxSize int64) (zip io.ReadCloser, err error) { 434 if r.cmd.readZip == nil && r.cmd.doReadZip == nil { 435 return nil, vcsErrorf("vcs %s: ReadZip: %w", r.cmd.vcs, ErrUnsupported) 436 } 437 438 unlock, err := r.mu.Lock() 439 if err != nil { 440 return nil, err 441 } 442 defer unlock() 443 444 if rev == "latest" { 445 rev = r.cmd.latest 446 } 447 f, err := os.CreateTemp("", "go-readzip-*.zip") 448 if err != nil { 449 return nil, err 450 } 451 if r.cmd.doReadZip != nil { 452 lw := &limitedWriter{ 453 W: f, 454 N: maxSize, 455 ErrLimitReached: errors.New("ReadZip: encoded file exceeds allowed size"), 456 } 457 err = r.cmd.doReadZip(lw, r.dir, rev, subdir, r.remote) 458 if err == nil { 459 _, err = f.Seek(0, io.SeekStart) 460 } 461 } else if r.cmd.vcs == "fossil" { 462 // If you run 463 // fossil zip -R .fossil --name prefix trunk /tmp/x.zip 464 // fossil fails with "unable to create directory /tmp" [sic]. 465 // Change the command to run in /tmp instead, 466 // replacing the -R argument with an absolute path. 467 args := r.cmd.readZip(rev, subdir, r.remote, filepath.Base(f.Name())) 468 for i := range args { 469 if args[i] == ".fossil" { 470 args[i] = filepath.Join(r.dir, ".fossil") 471 } 472 } 473 _, err = Run(filepath.Dir(f.Name()), args) 474 } else { 475 _, err = Run(r.dir, r.cmd.readZip(rev, subdir, r.remote, f.Name())) 476 } 477 if err != nil { 478 f.Close() 479 os.Remove(f.Name()) 480 return nil, err 481 } 482 return &deleteCloser{f}, nil 483 } 484 485 // deleteCloser is a file that gets deleted on Close. 486 type deleteCloser struct { 487 *os.File 488 } 489 490 func (d *deleteCloser) Close() error { 491 defer os.Remove(d.File.Name()) 492 return d.File.Close() 493 } 494 495 func hgParseStat(rev, out string) (*RevInfo, error) { 496 f := strings.Fields(string(out)) 497 if len(f) < 3 { 498 return nil, vcsErrorf("unexpected response from hg log: %q", out) 499 } 500 hash := f[0] 501 version := rev 502 if strings.HasPrefix(hash, version) { 503 version = hash // extend to full hash 504 } 505 t, err := strconv.ParseInt(f[1], 10, 64) 506 if err != nil { 507 return nil, vcsErrorf("invalid time from hg log: %q", out) 508 } 509 510 var tags []string 511 for _, tag := range f[3:] { 512 if tag != "tip" { 513 tags = append(tags, tag) 514 } 515 } 516 sort.Strings(tags) 517 518 info := &RevInfo{ 519 Origin: &Origin{ 520 Hash: hash, 521 }, 522 Name: hash, 523 Short: ShortenSHA1(hash), 524 Time: time.Unix(t, 0).UTC(), 525 Version: version, 526 Tags: tags, 527 } 528 return info, nil 529 } 530 531 func bzrParseStat(rev, out string) (*RevInfo, error) { 532 var revno int64 533 var tm time.Time 534 for _, line := range strings.Split(out, "\n") { 535 if line == "" || line[0] == ' ' || line[0] == '\t' { 536 // End of header, start of commit message. 537 break 538 } 539 if line[0] == '-' { 540 continue 541 } 542 before, after, found := strings.Cut(line, ":") 543 if !found { 544 // End of header, start of commit message. 545 break 546 } 547 key, val := before, strings.TrimSpace(after) 548 switch key { 549 case "revno": 550 if j := strings.Index(val, " "); j >= 0 { 551 val = val[:j] 552 } 553 i, err := strconv.ParseInt(val, 10, 64) 554 if err != nil { 555 return nil, vcsErrorf("unexpected revno from bzr log: %q", line) 556 } 557 revno = i 558 case "timestamp": 559 j := strings.Index(val, " ") 560 if j < 0 { 561 return nil, vcsErrorf("unexpected timestamp from bzr log: %q", line) 562 } 563 t, err := time.Parse("2006-01-02 15:04:05 -0700", val[j+1:]) 564 if err != nil { 565 return nil, vcsErrorf("unexpected timestamp from bzr log: %q", line) 566 } 567 tm = t.UTC() 568 } 569 } 570 if revno == 0 || tm.IsZero() { 571 return nil, vcsErrorf("unexpected response from bzr log: %q", out) 572 } 573 574 info := &RevInfo{ 575 Name: fmt.Sprintf("%d", revno), 576 Short: fmt.Sprintf("%012d", revno), 577 Time: tm, 578 Version: rev, 579 } 580 return info, nil 581 } 582 583 func fossilParseStat(rev, out string) (*RevInfo, error) { 584 for _, line := range strings.Split(out, "\n") { 585 if strings.HasPrefix(line, "uuid:") || strings.HasPrefix(line, "hash:") { 586 f := strings.Fields(line) 587 if len(f) != 5 || len(f[1]) != 40 || f[4] != "UTC" { 588 return nil, vcsErrorf("unexpected response from fossil info: %q", line) 589 } 590 t, err := time.Parse(time.DateTime, f[2]+" "+f[3]) 591 if err != nil { 592 return nil, vcsErrorf("unexpected response from fossil info: %q", line) 593 } 594 hash := f[1] 595 version := rev 596 if strings.HasPrefix(hash, version) { 597 version = hash // extend to full hash 598 } 599 info := &RevInfo{ 600 Origin: &Origin{ 601 Hash: hash, 602 }, 603 Name: hash, 604 Short: ShortenSHA1(hash), 605 Time: t, 606 Version: version, 607 } 608 return info, nil 609 } 610 } 611 return nil, vcsErrorf("unexpected response from fossil info: %q", out) 612 } 613 614 type limitedWriter struct { 615 W io.Writer 616 N int64 617 ErrLimitReached error 618 } 619 620 func (l *limitedWriter) Write(p []byte) (n int, err error) { 621 if l.N > 0 { 622 max := len(p) 623 if l.N < int64(max) { 624 max = int(l.N) 625 } 626 n, err = l.W.Write(p[:max]) 627 l.N -= int64(n) 628 if err != nil || n >= len(p) { 629 return n, err 630 } 631 } 632 633 return n, l.ErrLimitReached 634 }