github.com/bir3/gocompiler@v0.3.205/src/cmd/gocmd/internal/modfetch/codehost/vcs.go (about)

     1  // Copyright 2018 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package codehost
     6  
     7  import (
     8  	"errors"
     9  	"fmt"
    10  	"github.com/bir3/gocompiler/src/internal/lazyregexp"
    11  	"io"
    12  	"io/fs"
    13  	"os"
    14  	"path/filepath"
    15  	"sort"
    16  	"strconv"
    17  	"strings"
    18  	"sync"
    19  	"time"
    20  
    21  	"github.com/bir3/gocompiler/src/cmd/gocmd/internal/lockedfile"
    22  	"github.com/bir3/gocompiler/src/cmd/gocmd/internal/par"
    23  	"github.com/bir3/gocompiler/src/cmd/gocmd/internal/str"
    24  )
    25  
    26  // A VCSError indicates an error using a version control system.
    27  // The implication of a VCSError is that we know definitively where
    28  // to get the code, but we can't access it due to the error.
    29  // The caller should report this error instead of continuing to probe
    30  // other possible module paths.
    31  //
    32  // TODO(golang.org/issue/31730): See if we can invert this. (Return a
    33  // distinguished error for “repo not found” and treat everything else
    34  // as terminal.)
    35  type VCSError struct {
    36  	Err error
    37  }
    38  
    39  func (e *VCSError) Error() string { return e.Err.Error() }
    40  
    41  func (e *VCSError) Unwrap() error { return e.Err }
    42  
    43  func vcsErrorf(format string, a ...any) error {
    44  	return &VCSError{Err: fmt.Errorf(format, a...)}
    45  }
    46  
    47  func NewRepo(vcs, remote string) (Repo, error) {
    48  	type key struct {
    49  		vcs    string
    50  		remote string
    51  	}
    52  	type cached struct {
    53  		repo Repo
    54  		err  error
    55  	}
    56  	c := vcsRepoCache.Do(key{vcs, remote}, func() any {
    57  		repo, err := newVCSRepo(vcs, remote)
    58  		if err != nil {
    59  			err = &VCSError{err}
    60  		}
    61  		return cached{repo, err}
    62  	}).(cached)
    63  
    64  	return c.repo, c.err
    65  }
    66  
    67  var vcsRepoCache par.Cache
    68  
    69  type vcsRepo struct {
    70  	mu lockedfile.Mutex // protects all commands, so we don't have to decide which are safe on a per-VCS basis
    71  
    72  	remote string
    73  	cmd    *vcsCmd
    74  	dir    string
    75  
    76  	tagsOnce sync.Once
    77  	tags     map[string]bool
    78  
    79  	branchesOnce sync.Once
    80  	branches     map[string]bool
    81  
    82  	fetchOnce sync.Once
    83  	fetchErr  error
    84  }
    85  
    86  func newVCSRepo(vcs, remote string) (Repo, error) {
    87  	if vcs == "git" {
    88  		return newGitRepo(remote, false)
    89  	}
    90  	cmd := vcsCmds[vcs]
    91  	if cmd == nil {
    92  		return nil, fmt.Errorf("unknown vcs: %s %s", vcs, remote)
    93  	}
    94  	if !strings.Contains(remote, "://") {
    95  		return nil, fmt.Errorf("invalid vcs remote: %s %s", vcs, remote)
    96  	}
    97  
    98  	r := &vcsRepo{remote: remote, cmd: cmd}
    99  	var err error
   100  	r.dir, r.mu.Path, err = WorkDir(vcsWorkDirType+vcs, r.remote)
   101  	if err != nil {
   102  		return nil, err
   103  	}
   104  
   105  	if cmd.init == nil {
   106  		return r, nil
   107  	}
   108  
   109  	unlock, err := r.mu.Lock()
   110  	if err != nil {
   111  		return nil, err
   112  	}
   113  	defer unlock()
   114  
   115  	if _, err := os.Stat(filepath.Join(r.dir, "."+vcs)); err != nil {
   116  		if _, err := Run(r.dir, cmd.init(r.remote)); err != nil {
   117  			os.RemoveAll(r.dir)
   118  			return nil, err
   119  		}
   120  	}
   121  	return r, nil
   122  }
   123  
   124  const vcsWorkDirType = "vcs1."
   125  
   126  type vcsCmd struct {
   127  	vcs           string                                                         // vcs name "hg"
   128  	init          func(remote string) []string                                   // cmd to init repo to track remote
   129  	tags          func(remote string) []string                                   // cmd to list local tags
   130  	tagRE         *lazyregexp.Regexp                                             // regexp to extract tag names from output of tags cmd
   131  	branches      func(remote string) []string                                   // cmd to list local branches
   132  	branchRE      *lazyregexp.Regexp                                             // regexp to extract branch names from output of tags cmd
   133  	badLocalRevRE *lazyregexp.Regexp                                             // regexp of names that must not be served out of local cache without doing fetch first
   134  	statLocal     func(rev, remote string) []string                              // cmd to stat local rev
   135  	parseStat     func(rev, out string) (*RevInfo, error)                        // cmd to parse output of statLocal
   136  	fetch         []string                                                       // cmd to fetch everything from remote
   137  	latest        string                                                         // name of latest commit on remote (tip, HEAD, etc)
   138  	readFile      func(rev, file, remote string) []string                        // cmd to read rev's file
   139  	readZip       func(rev, subdir, remote, target string) []string              // cmd to read rev's subdir as zip file
   140  	doReadZip     func(dst io.Writer, workDir, rev, subdir, remote string) error // arbitrary function to read rev's subdir as zip file
   141  }
   142  
   143  var re = lazyregexp.New
   144  
   145  var vcsCmds = map[string]*vcsCmd{
   146  	"hg": {
   147  		vcs: "hg",
   148  		init: func(remote string) []string {
   149  			return []string{"hg", "clone", "-U", "--", remote, "."}
   150  		},
   151  		tags: func(remote string) []string {
   152  			return []string{"hg", "tags", "-q"}
   153  		},
   154  		tagRE: re(`(?m)^[^\n]+$`),
   155  		branches: func(remote string) []string {
   156  			return []string{"hg", "branches", "-c", "-q"}
   157  		},
   158  		branchRE:      re(`(?m)^[^\n]+$`),
   159  		badLocalRevRE: re(`(?m)^(tip)$`),
   160  		statLocal: func(rev, remote string) []string {
   161  			return []string{"hg", "log", "-l1", "-r", rev, "--template", "{node} {date|hgdate} {tags}"}
   162  		},
   163  		parseStat: hgParseStat,
   164  		fetch:     []string{"hg", "pull", "-f"},
   165  		latest:    "tip",
   166  		readFile: func(rev, file, remote string) []string {
   167  			return []string{"hg", "cat", "-r", rev, file}
   168  		},
   169  		readZip: func(rev, subdir, remote, target string) []string {
   170  			pattern := []string{}
   171  			if subdir != "" {
   172  				pattern = []string{"-I", subdir + "/**"}
   173  			}
   174  			return str.StringList("hg", "archive", "-t", "zip", "--no-decode", "-r", rev, "--prefix=prefix/", pattern, "--", target)
   175  		},
   176  	},
   177  
   178  	"svn": {
   179  		vcs:  "svn",
   180  		init: nil, // no local checkout
   181  		tags: func(remote string) []string {
   182  			return []string{"svn", "list", "--", strings.TrimSuffix(remote, "/trunk") + "/tags"}
   183  		},
   184  		tagRE: re(`(?m)^(.*?)/?$`),
   185  		statLocal: func(rev, remote string) []string {
   186  			suffix := "@" + rev
   187  			if rev == "latest" {
   188  				suffix = ""
   189  			}
   190  			return []string{"svn", "log", "-l1", "--xml", "--", remote + suffix}
   191  		},
   192  		parseStat: svnParseStat,
   193  		latest:    "latest",
   194  		readFile: func(rev, file, remote string) []string {
   195  			return []string{"svn", "cat", "--", remote + "/" + file + "@" + rev}
   196  		},
   197  		doReadZip: svnReadZip,
   198  	},
   199  
   200  	"bzr": {
   201  		vcs: "bzr",
   202  		init: func(remote string) []string {
   203  			return []string{"bzr", "branch", "--use-existing-dir", "--", remote, "."}
   204  		},
   205  		fetch: []string{
   206  			"bzr", "pull", "--overwrite-tags",
   207  		},
   208  		tags: func(remote string) []string {
   209  			return []string{"bzr", "tags"}
   210  		},
   211  		tagRE:         re(`(?m)^\S+`),
   212  		badLocalRevRE: re(`^revno:-`),
   213  		statLocal: func(rev, remote string) []string {
   214  			return []string{"bzr", "log", "-l1", "--long", "--show-ids", "-r", rev}
   215  		},
   216  		parseStat: bzrParseStat,
   217  		latest:    "revno:-1",
   218  		readFile: func(rev, file, remote string) []string {
   219  			return []string{"bzr", "cat", "-r", rev, file}
   220  		},
   221  		readZip: func(rev, subdir, remote, target string) []string {
   222  			extra := []string{}
   223  			if subdir != "" {
   224  				extra = []string{"./" + subdir}
   225  			}
   226  			return str.StringList("bzr", "export", "--format=zip", "-r", rev, "--root=prefix/", "--", target, extra)
   227  		},
   228  	},
   229  
   230  	"fossil": {
   231  		vcs: "fossil",
   232  		init: func(remote string) []string {
   233  			return []string{"fossil", "clone", "--", remote, ".fossil"}
   234  		},
   235  		fetch: []string{"fossil", "pull", "-R", ".fossil"},
   236  		tags: func(remote string) []string {
   237  			return []string{"fossil", "tag", "-R", ".fossil", "list"}
   238  		},
   239  		tagRE: re(`XXXTODO`),
   240  		statLocal: func(rev, remote string) []string {
   241  			return []string{"fossil", "info", "-R", ".fossil", rev}
   242  		},
   243  		parseStat: fossilParseStat,
   244  		latest:    "trunk",
   245  		readFile: func(rev, file, remote string) []string {
   246  			return []string{"fossil", "cat", "-R", ".fossil", "-r", rev, file}
   247  		},
   248  		readZip: func(rev, subdir, remote, target string) []string {
   249  			extra := []string{}
   250  			if subdir != "" && !strings.ContainsAny(subdir, "*?[],") {
   251  				extra = []string{"--include", subdir}
   252  			}
   253  			// Note that vcsRepo.ReadZip below rewrites this command
   254  			// to run in a different directory, to work around a fossil bug.
   255  			return str.StringList("fossil", "zip", "-R", ".fossil", "--name", "prefix", extra, "--", rev, target)
   256  		},
   257  	},
   258  }
   259  
   260  func (r *vcsRepo) loadTags() {
   261  	out, err := Run(r.dir, r.cmd.tags(r.remote))
   262  	if err != nil {
   263  		return
   264  	}
   265  
   266  	// Run tag-listing command and extract tags.
   267  	r.tags = make(map[string]bool)
   268  	for _, tag := range r.cmd.tagRE.FindAllString(string(out), -1) {
   269  		if r.cmd.badLocalRevRE != nil && r.cmd.badLocalRevRE.MatchString(tag) {
   270  			continue
   271  		}
   272  		r.tags[tag] = true
   273  	}
   274  }
   275  
   276  func (r *vcsRepo) loadBranches() {
   277  	if r.cmd.branches == nil {
   278  		return
   279  	}
   280  
   281  	out, err := Run(r.dir, r.cmd.branches(r.remote))
   282  	if err != nil {
   283  		return
   284  	}
   285  
   286  	r.branches = make(map[string]bool)
   287  	for _, branch := range r.cmd.branchRE.FindAllString(string(out), -1) {
   288  		if r.cmd.badLocalRevRE != nil && r.cmd.badLocalRevRE.MatchString(branch) {
   289  			continue
   290  		}
   291  		r.branches[branch] = true
   292  	}
   293  }
   294  
   295  func (r *vcsRepo) CheckReuse(old *Origin, subdir string) error {
   296  	return fmt.Errorf("vcs %s: CheckReuse: %w", r.cmd.vcs, ErrUnsupported)
   297  }
   298  
   299  func (r *vcsRepo) Tags(prefix string) (*Tags, error) {
   300  	unlock, err := r.mu.Lock()
   301  	if err != nil {
   302  		return nil, err
   303  	}
   304  	defer unlock()
   305  
   306  	r.tagsOnce.Do(r.loadTags)
   307  	tags := &Tags{
   308  		// None of the other VCS provide a reasonable way to compute TagSum
   309  		// without downloading the whole repo, so we only include VCS and URL
   310  		// in the Origin.
   311  		Origin: &Origin{
   312  			VCS: r.cmd.vcs,
   313  			URL: r.remote,
   314  		},
   315  		List: []Tag{},
   316  	}
   317  	for tag := range r.tags {
   318  		if strings.HasPrefix(tag, prefix) {
   319  			tags.List = append(tags.List, Tag{tag, ""})
   320  		}
   321  	}
   322  	sort.Slice(tags.List, func(i, j int) bool {
   323  		return tags.List[i].Name < tags.List[j].Name
   324  	})
   325  	return tags, nil
   326  }
   327  
   328  func (r *vcsRepo) Stat(rev string) (*RevInfo, error) {
   329  	unlock, err := r.mu.Lock()
   330  	if err != nil {
   331  		return nil, err
   332  	}
   333  	defer unlock()
   334  
   335  	if rev == "latest" {
   336  		rev = r.cmd.latest
   337  	}
   338  	r.branchesOnce.Do(r.loadBranches)
   339  	revOK := (r.cmd.badLocalRevRE == nil || !r.cmd.badLocalRevRE.MatchString(rev)) && !r.branches[rev]
   340  	if revOK {
   341  		if info, err := r.statLocal(rev); err == nil {
   342  			return info, nil
   343  		}
   344  	}
   345  
   346  	r.fetchOnce.Do(r.fetch)
   347  	if r.fetchErr != nil {
   348  		return nil, r.fetchErr
   349  	}
   350  	info, err := r.statLocal(rev)
   351  	if err != nil {
   352  		return nil, err
   353  	}
   354  	if !revOK {
   355  		info.Version = info.Name
   356  	}
   357  	return info, nil
   358  }
   359  
   360  func (r *vcsRepo) fetch() {
   361  	if len(r.cmd.fetch) > 0 {
   362  		_, r.fetchErr = Run(r.dir, r.cmd.fetch)
   363  	}
   364  }
   365  
   366  func (r *vcsRepo) statLocal(rev string) (*RevInfo, error) {
   367  	out, err := Run(r.dir, r.cmd.statLocal(rev, r.remote))
   368  	if err != nil {
   369  		return nil, &UnknownRevisionError{Rev: rev}
   370  	}
   371  	info, err := r.cmd.parseStat(rev, string(out))
   372  	if err != nil {
   373  		return nil, err
   374  	}
   375  	if info.Origin == nil {
   376  		info.Origin = new(Origin)
   377  	}
   378  	info.Origin.VCS = r.cmd.vcs
   379  	info.Origin.URL = r.remote
   380  	return info, nil
   381  }
   382  
   383  func (r *vcsRepo) Latest() (*RevInfo, error) {
   384  	return r.Stat("latest")
   385  }
   386  
   387  func (r *vcsRepo) ReadFile(rev, file string, maxSize int64) ([]byte, error) {
   388  	if rev == "latest" {
   389  		rev = r.cmd.latest
   390  	}
   391  	_, err := r.Stat(rev) // download rev into local repo
   392  	if err != nil {
   393  		return nil, err
   394  	}
   395  
   396  	// r.Stat acquires r.mu, so lock after that.
   397  	unlock, err := r.mu.Lock()
   398  	if err != nil {
   399  		return nil, err
   400  	}
   401  	defer unlock()
   402  
   403  	out, err := Run(r.dir, r.cmd.readFile(rev, file, r.remote))
   404  	if err != nil {
   405  		return nil, fs.ErrNotExist
   406  	}
   407  	return out, nil
   408  }
   409  
   410  func (r *vcsRepo) RecentTag(rev, prefix string, allowed func(string) bool) (tag string, err error) {
   411  	// We don't technically need to lock here since we're returning an error
   412  	// uncondititonally, but doing so anyway will help to avoid baking in
   413  	// lock-inversion bugs.
   414  	unlock, err := r.mu.Lock()
   415  	if err != nil {
   416  		return "", err
   417  	}
   418  	defer unlock()
   419  
   420  	return "", vcsErrorf("vcs %s: RecentTag: %w", r.cmd.vcs, ErrUnsupported)
   421  }
   422  
   423  func (r *vcsRepo) DescendsFrom(rev, tag string) (bool, error) {
   424  	unlock, err := r.mu.Lock()
   425  	if err != nil {
   426  		return false, err
   427  	}
   428  	defer unlock()
   429  
   430  	return false, vcsErrorf("vcs %s: DescendsFrom: %w", r.cmd.vcs, ErrUnsupported)
   431  }
   432  
   433  func (r *vcsRepo) ReadZip(rev, subdir string, maxSize int64) (zip io.ReadCloser, err error) {
   434  	if r.cmd.readZip == nil && r.cmd.doReadZip == nil {
   435  		return nil, vcsErrorf("vcs %s: ReadZip: %w", r.cmd.vcs, ErrUnsupported)
   436  	}
   437  
   438  	unlock, err := r.mu.Lock()
   439  	if err != nil {
   440  		return nil, err
   441  	}
   442  	defer unlock()
   443  
   444  	if rev == "latest" {
   445  		rev = r.cmd.latest
   446  	}
   447  	f, err := os.CreateTemp("", "go-readzip-*.zip")
   448  	if err != nil {
   449  		return nil, err
   450  	}
   451  	if r.cmd.doReadZip != nil {
   452  		lw := &limitedWriter{
   453  			W:               f,
   454  			N:               maxSize,
   455  			ErrLimitReached: errors.New("ReadZip: encoded file exceeds allowed size"),
   456  		}
   457  		err = r.cmd.doReadZip(lw, r.dir, rev, subdir, r.remote)
   458  		if err == nil {
   459  			_, err = f.Seek(0, io.SeekStart)
   460  		}
   461  	} else if r.cmd.vcs == "fossil" {
   462  		// If you run
   463  		//	fossil zip -R .fossil --name prefix trunk /tmp/x.zip
   464  		// fossil fails with "unable to create directory /tmp" [sic].
   465  		// Change the command to run in /tmp instead,
   466  		// replacing the -R argument with an absolute path.
   467  		args := r.cmd.readZip(rev, subdir, r.remote, filepath.Base(f.Name()))
   468  		for i := range args {
   469  			if args[i] == ".fossil" {
   470  				args[i] = filepath.Join(r.dir, ".fossil")
   471  			}
   472  		}
   473  		_, err = Run(filepath.Dir(f.Name()), args)
   474  	} else {
   475  		_, err = Run(r.dir, r.cmd.readZip(rev, subdir, r.remote, f.Name()))
   476  	}
   477  	if err != nil {
   478  		f.Close()
   479  		os.Remove(f.Name())
   480  		return nil, err
   481  	}
   482  	return &deleteCloser{f}, nil
   483  }
   484  
   485  // deleteCloser is a file that gets deleted on Close.
   486  type deleteCloser struct {
   487  	*os.File
   488  }
   489  
   490  func (d *deleteCloser) Close() error {
   491  	defer os.Remove(d.File.Name())
   492  	return d.File.Close()
   493  }
   494  
   495  func hgParseStat(rev, out string) (*RevInfo, error) {
   496  	f := strings.Fields(string(out))
   497  	if len(f) < 3 {
   498  		return nil, vcsErrorf("unexpected response from hg log: %q", out)
   499  	}
   500  	hash := f[0]
   501  	version := rev
   502  	if strings.HasPrefix(hash, version) {
   503  		version = hash // extend to full hash
   504  	}
   505  	t, err := strconv.ParseInt(f[1], 10, 64)
   506  	if err != nil {
   507  		return nil, vcsErrorf("invalid time from hg log: %q", out)
   508  	}
   509  
   510  	var tags []string
   511  	for _, tag := range f[3:] {
   512  		if tag != "tip" {
   513  			tags = append(tags, tag)
   514  		}
   515  	}
   516  	sort.Strings(tags)
   517  
   518  	info := &RevInfo{
   519  		Origin: &Origin{
   520  			Hash: hash,
   521  		},
   522  		Name:    hash,
   523  		Short:   ShortenSHA1(hash),
   524  		Time:    time.Unix(t, 0).UTC(),
   525  		Version: version,
   526  		Tags:    tags,
   527  	}
   528  	return info, nil
   529  }
   530  
   531  func bzrParseStat(rev, out string) (*RevInfo, error) {
   532  	var revno int64
   533  	var tm time.Time
   534  	for _, line := range strings.Split(out, "\n") {
   535  		if line == "" || line[0] == ' ' || line[0] == '\t' {
   536  			// End of header, start of commit message.
   537  			break
   538  		}
   539  		if line[0] == '-' {
   540  			continue
   541  		}
   542  		before, after, found := strings.Cut(line, ":")
   543  		if !found {
   544  			// End of header, start of commit message.
   545  			break
   546  		}
   547  		key, val := before, strings.TrimSpace(after)
   548  		switch key {
   549  		case "revno":
   550  			if j := strings.Index(val, " "); j >= 0 {
   551  				val = val[:j]
   552  			}
   553  			i, err := strconv.ParseInt(val, 10, 64)
   554  			if err != nil {
   555  				return nil, vcsErrorf("unexpected revno from bzr log: %q", line)
   556  			}
   557  			revno = i
   558  		case "timestamp":
   559  			j := strings.Index(val, " ")
   560  			if j < 0 {
   561  				return nil, vcsErrorf("unexpected timestamp from bzr log: %q", line)
   562  			}
   563  			t, err := time.Parse("2006-01-02 15:04:05 -0700", val[j+1:])
   564  			if err != nil {
   565  				return nil, vcsErrorf("unexpected timestamp from bzr log: %q", line)
   566  			}
   567  			tm = t.UTC()
   568  		}
   569  	}
   570  	if revno == 0 || tm.IsZero() {
   571  		return nil, vcsErrorf("unexpected response from bzr log: %q", out)
   572  	}
   573  
   574  	info := &RevInfo{
   575  		Name:    fmt.Sprintf("%d", revno),
   576  		Short:   fmt.Sprintf("%012d", revno),
   577  		Time:    tm,
   578  		Version: rev,
   579  	}
   580  	return info, nil
   581  }
   582  
   583  func fossilParseStat(rev, out string) (*RevInfo, error) {
   584  	for _, line := range strings.Split(out, "\n") {
   585  		if strings.HasPrefix(line, "uuid:") || strings.HasPrefix(line, "hash:") {
   586  			f := strings.Fields(line)
   587  			if len(f) != 5 || len(f[1]) != 40 || f[4] != "UTC" {
   588  				return nil, vcsErrorf("unexpected response from fossil info: %q", line)
   589  			}
   590  			t, err := time.Parse(time.DateTime, f[2]+" "+f[3])
   591  			if err != nil {
   592  				return nil, vcsErrorf("unexpected response from fossil info: %q", line)
   593  			}
   594  			hash := f[1]
   595  			version := rev
   596  			if strings.HasPrefix(hash, version) {
   597  				version = hash // extend to full hash
   598  			}
   599  			info := &RevInfo{
   600  				Origin: &Origin{
   601  					Hash: hash,
   602  				},
   603  				Name:    hash,
   604  				Short:   ShortenSHA1(hash),
   605  				Time:    t,
   606  				Version: version,
   607  			}
   608  			return info, nil
   609  		}
   610  	}
   611  	return nil, vcsErrorf("unexpected response from fossil info: %q", out)
   612  }
   613  
   614  type limitedWriter struct {
   615  	W               io.Writer
   616  	N               int64
   617  	ErrLimitReached error
   618  }
   619  
   620  func (l *limitedWriter) Write(p []byte) (n int, err error) {
   621  	if l.N > 0 {
   622  		max := len(p)
   623  		if l.N < int64(max) {
   624  			max = int(l.N)
   625  		}
   626  		n, err = l.W.Write(p[:max])
   627  		l.N -= int64(n)
   628  		if err != nil || n >= len(p) {
   629  			return n, err
   630  		}
   631  	}
   632  
   633  	return n, l.ErrLimitReached
   634  }