github.com/bir3/gocompiler@v0.3.205/src/cmd/gocmd/internal/modfetch/codehost/codehost.go (about)

     1  // Copyright 2018 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Package codehost defines the interface implemented by a code hosting source,
     6  // along with support code for use by implementations.
     7  package codehost
     8  
     9  import (
    10  	"bytes"
    11  	"crypto/sha256"
    12  	"fmt"
    13  	"io"
    14  	"io/fs"
    15  	"os"
    16  	"os/exec"
    17  	"path/filepath"
    18  	"strings"
    19  	"sync"
    20  	"time"
    21  
    22  	"github.com/bir3/gocompiler/src/cmd/gocmd/internal/cfg"
    23  	"github.com/bir3/gocompiler/src/cmd/gocmd/internal/lockedfile"
    24  	"github.com/bir3/gocompiler/src/cmd/gocmd/internal/str"
    25  
    26  	"github.com/bir3/gocompiler/src/xvendor/golang.org/x/mod/module"
    27  	"github.com/bir3/gocompiler/src/xvendor/golang.org/x/mod/semver"
    28  )
    29  
    30  // Downloaded size limits.
    31  const (
    32  	MaxGoMod   = 16 << 20  // maximum size of go.mod file
    33  	MaxLICENSE = 16 << 20  // maximum size of LICENSE file
    34  	MaxZipFile = 500 << 20 // maximum size of downloaded zip file
    35  )
    36  
    37  // A Repo represents a code hosting source.
    38  // Typical implementations include local version control repositories,
    39  // remote version control servers, and code hosting sites.
    40  //
    41  // A Repo must be safe for simultaneous use by multiple goroutines,
    42  // and callers must not modify returned values, which may be cached and shared.
    43  type Repo interface {
    44  	// CheckReuse checks whether the old origin information
    45  	// remains up to date. If so, whatever cached object it was
    46  	// taken from can be reused.
    47  	// The subdir gives subdirectory name where the module root is expected to be found,
    48  	// "" for the root or "sub/dir" for a subdirectory (no trailing slash).
    49  	CheckReuse(old *Origin, subdir string) error
    50  
    51  	// List lists all tags with the given prefix.
    52  	Tags(prefix string) (*Tags, error)
    53  
    54  	// Stat returns information about the revision rev.
    55  	// A revision can be any identifier known to the underlying service:
    56  	// commit hash, branch, tag, and so on.
    57  	Stat(rev string) (*RevInfo, error)
    58  
    59  	// Latest returns the latest revision on the default branch,
    60  	// whatever that means in the underlying implementation.
    61  	Latest() (*RevInfo, error)
    62  
    63  	// ReadFile reads the given file in the file tree corresponding to revision rev.
    64  	// It should refuse to read more than maxSize bytes.
    65  	//
    66  	// If the requested file does not exist it should return an error for which
    67  	// os.IsNotExist(err) returns true.
    68  	ReadFile(rev, file string, maxSize int64) (data []byte, err error)
    69  
    70  	// ReadZip downloads a zip file for the subdir subdirectory
    71  	// of the given revision to a new file in a given temporary directory.
    72  	// It should refuse to read more than maxSize bytes.
    73  	// It returns a ReadCloser for a streamed copy of the zip file.
    74  	// All files in the zip file are expected to be
    75  	// nested in a single top-level directory, whose name is not specified.
    76  	ReadZip(rev, subdir string, maxSize int64) (zip io.ReadCloser, err error)
    77  
    78  	// RecentTag returns the most recent tag on rev or one of its predecessors
    79  	// with the given prefix. allowed may be used to filter out unwanted versions.
    80  	RecentTag(rev, prefix string, allowed func(tag string) bool) (tag string, err error)
    81  
    82  	// DescendsFrom reports whether rev or any of its ancestors has the given tag.
    83  	//
    84  	// DescendsFrom must return true for any tag returned by RecentTag for the
    85  	// same revision.
    86  	DescendsFrom(rev, tag string) (bool, error)
    87  }
    88  
    89  // An Origin describes the provenance of a given repo method result.
    90  // It can be passed to CheckReuse (usually in a different go command invocation)
    91  // to see whether the result remains up-to-date.
    92  type Origin struct {
    93  	VCS    string `json:",omitempty"` // "git" etc
    94  	URL    string `json:",omitempty"` // URL of repository
    95  	Subdir string `json:",omitempty"` // subdirectory in repo
    96  
    97  	// If TagSum is non-empty, then the resolution of this module version
    98  	// depends on the set of tags present in the repo, specifically the tags
    99  	// of the form TagPrefix + a valid semver version.
   100  	// If the matching repo tags and their commit hashes still hash to TagSum,
   101  	// the Origin is still valid (at least as far as the tags are concerned).
   102  	// The exact checksum is up to the Repo implementation; see (*gitRepo).Tags.
   103  	TagPrefix string `json:",omitempty"`
   104  	TagSum    string `json:",omitempty"`
   105  
   106  	// If Ref is non-empty, then the resolution of this module version
   107  	// depends on Ref resolving to the revision identified by Hash.
   108  	// If Ref still resolves to Hash, the Origin is still valid (at least as far as Ref is concerned).
   109  	// For Git, the Ref is a full ref like "refs/heads/main" or "refs/tags/v1.2.3",
   110  	// and the Hash is the Git object hash the ref maps to.
   111  	// Other VCS might choose differently, but the idea is that Ref is the name
   112  	// with a mutable meaning while Hash is a name with an immutable meaning.
   113  	Ref  string `json:",omitempty"`
   114  	Hash string `json:",omitempty"`
   115  
   116  	// If RepoSum is non-empty, then the resolution of this module version
   117  	// failed due to the repo being available but the version not being present.
   118  	// This depends on the entire state of the repo, which RepoSum summarizes.
   119  	// For Git, this is a hash of all the refs and their hashes.
   120  	RepoSum string `json:",omitempty"`
   121  }
   122  
   123  // Checkable reports whether the Origin contains anything that can be checked.
   124  // If not, the Origin is purely informational and should fail a CheckReuse call.
   125  func (o *Origin) Checkable() bool {
   126  	return o.TagSum != "" || o.Ref != "" || o.Hash != "" || o.RepoSum != ""
   127  }
   128  
   129  // ClearCheckable clears the Origin enough to make Checkable return false.
   130  func (o *Origin) ClearCheckable() {
   131  	o.TagSum = ""
   132  	o.TagPrefix = ""
   133  	o.Ref = ""
   134  	o.Hash = ""
   135  	o.RepoSum = ""
   136  }
   137  
   138  // A Tags describes the available tags in a code repository.
   139  type Tags struct {
   140  	Origin *Origin
   141  	List   []Tag
   142  }
   143  
   144  // A Tag describes a single tag in a code repository.
   145  type Tag struct {
   146  	Name string
   147  	Hash string // content hash identifying tag's content, if available
   148  }
   149  
   150  // isOriginTag reports whether tag should be preserved
   151  // in the Tags method's Origin calculation.
   152  // We can safely ignore tags that are not look like pseudo-versions,
   153  // because ../coderepo.go's (*codeRepo).Versions ignores them too.
   154  // We can also ignore non-semver tags, but we have to include semver
   155  // tags with extra suffixes, because the pseudo-version base finder uses them.
   156  func isOriginTag(tag string) bool {
   157  	// modfetch.(*codeRepo).Versions uses Canonical == tag,
   158  	// but pseudo-version calculation has a weaker condition that
   159  	// the canonical is a prefix of the tag.
   160  	// Include those too, so that if any new one appears, we'll invalidate the cache entry.
   161  	// This will lead to spurious invalidation of version list results,
   162  	// but tags of this form being created should be fairly rare
   163  	// (and invalidate pseudo-version results anyway).
   164  	c := semver.Canonical(tag)
   165  	return c != "" && strings.HasPrefix(tag, c) && !module.IsPseudoVersion(tag)
   166  }
   167  
   168  // A RevInfo describes a single revision in a source code repository.
   169  type RevInfo struct {
   170  	Origin  *Origin
   171  	Name    string    // complete ID in underlying repository
   172  	Short   string    // shortened ID, for use in pseudo-version
   173  	Version string    // version used in lookup
   174  	Time    time.Time // commit time
   175  	Tags    []string  // known tags for commit
   176  }
   177  
   178  // UnknownRevisionError is an error equivalent to fs.ErrNotExist, but for a
   179  // revision rather than a file.
   180  type UnknownRevisionError struct {
   181  	Rev string
   182  }
   183  
   184  func (e *UnknownRevisionError) Error() string {
   185  	return "unknown revision " + e.Rev
   186  }
   187  func (UnknownRevisionError) Is(err error) bool {
   188  	return err == fs.ErrNotExist
   189  }
   190  
   191  // ErrNoCommits is an error equivalent to fs.ErrNotExist indicating that a given
   192  // repository or module contains no commits.
   193  var ErrNoCommits error = noCommitsError{}
   194  
   195  type noCommitsError struct{}
   196  
   197  func (noCommitsError) Error() string {
   198  	return "no commits"
   199  }
   200  func (noCommitsError) Is(err error) bool {
   201  	return err == fs.ErrNotExist
   202  }
   203  
   204  // ErrUnsupported indicates that a requested operation cannot be performed,
   205  // because it is unsupported. This error indicates that there is no alternative
   206  // way to perform the operation.
   207  //
   208  // TODO(#41198): Remove this declaration and use errors.ErrUnsupported instead.
   209  var ErrUnsupported = unsupportedOperationError{}
   210  
   211  type unsupportedOperationError struct{}
   212  
   213  func (unsupportedOperationError) Error() string {
   214  	return "unsupported operation"
   215  }
   216  
   217  // AllHex reports whether the revision rev is entirely lower-case hexadecimal digits.
   218  func AllHex(rev string) bool {
   219  	for i := 0; i < len(rev); i++ {
   220  		c := rev[i]
   221  		if '0' <= c && c <= '9' || 'a' <= c && c <= 'f' {
   222  			continue
   223  		}
   224  		return false
   225  	}
   226  	return true
   227  }
   228  
   229  // ShortenSHA1 shortens a SHA1 hash (40 hex digits) to the canonical length
   230  // used in pseudo-versions (12 hex digits).
   231  func ShortenSHA1(rev string) string {
   232  	if AllHex(rev) && len(rev) == 40 {
   233  		return rev[:12]
   234  	}
   235  	return rev
   236  }
   237  
   238  // WorkDir returns the name of the cached work directory to use for the
   239  // given repository type and name.
   240  func WorkDir(typ, name string) (dir, lockfile string, err error) {
   241  	if cfg.GOMODCACHE == "" {
   242  		return "", "", fmt.Errorf("neither GOPATH nor GOMODCACHE are set")
   243  	}
   244  
   245  	// We name the work directory for the SHA256 hash of the type and name.
   246  	// We intentionally avoid the actual name both because of possible
   247  	// conflicts with valid file system paths and because we want to ensure
   248  	// that one checkout is never nested inside another. That nesting has
   249  	// led to security problems in the past.
   250  	if strings.Contains(typ, ":") {
   251  		return "", "", fmt.Errorf("codehost.WorkDir: type cannot contain colon")
   252  	}
   253  	key := typ + ":" + name
   254  	dir = filepath.Join(cfg.GOMODCACHE, "cache/vcs", fmt.Sprintf("%x", sha256.Sum256([]byte(key))))
   255  
   256  	if cfg.BuildX {
   257  		fmt.Fprintf(os.Stderr, "mkdir -p %s # %s %s\n", filepath.Dir(dir), typ, name)
   258  	}
   259  	if err := os.MkdirAll(filepath.Dir(dir), 0777); err != nil {
   260  		return "", "", err
   261  	}
   262  
   263  	lockfile = dir + ".lock"
   264  	if cfg.BuildX {
   265  		fmt.Fprintf(os.Stderr, "# lock %s\n", lockfile)
   266  	}
   267  
   268  	unlock, err := lockedfile.MutexAt(lockfile).Lock()
   269  	if err != nil {
   270  		return "", "", fmt.Errorf("codehost.WorkDir: can't find or create lock file: %v", err)
   271  	}
   272  	defer unlock()
   273  
   274  	data, err := os.ReadFile(dir + ".info")
   275  	info, err2 := os.Stat(dir)
   276  	if err == nil && err2 == nil && info.IsDir() {
   277  		// Info file and directory both already exist: reuse.
   278  		have := strings.TrimSuffix(string(data), "\n")
   279  		if have != key {
   280  			return "", "", fmt.Errorf("%s exists with wrong content (have %q want %q)", dir+".info", have, key)
   281  		}
   282  		if cfg.BuildX {
   283  			fmt.Fprintf(os.Stderr, "# %s for %s %s\n", dir, typ, name)
   284  		}
   285  		return dir, lockfile, nil
   286  	}
   287  
   288  	// Info file or directory missing. Start from scratch.
   289  	if cfg.BuildX {
   290  		fmt.Fprintf(os.Stderr, "mkdir -p %s # %s %s\n", dir, typ, name)
   291  	}
   292  	os.RemoveAll(dir)
   293  	if err := os.MkdirAll(dir, 0777); err != nil {
   294  		return "", "", err
   295  	}
   296  	if err := os.WriteFile(dir+".info", []byte(key), 0666); err != nil {
   297  		os.RemoveAll(dir)
   298  		return "", "", err
   299  	}
   300  	return dir, lockfile, nil
   301  }
   302  
   303  type RunError struct {
   304  	Cmd      string
   305  	Err      error
   306  	Stderr   []byte
   307  	HelpText string
   308  }
   309  
   310  func (e *RunError) Error() string {
   311  	text := e.Cmd + ": " + e.Err.Error()
   312  	stderr := bytes.TrimRight(e.Stderr, "\n")
   313  	if len(stderr) > 0 {
   314  		text += ":\n\t" + strings.ReplaceAll(string(stderr), "\n", "\n\t")
   315  	}
   316  	if len(e.HelpText) > 0 {
   317  		text += "\n" + e.HelpText
   318  	}
   319  	return text
   320  }
   321  
   322  var dirLock sync.Map
   323  
   324  // Run runs the command line in the given directory
   325  // (an empty dir means the current directory).
   326  // It returns the standard output and, for a non-zero exit,
   327  // a *RunError indicating the command, exit status, and standard error.
   328  // Standard error is unavailable for commands that exit successfully.
   329  func Run(dir string, cmdline ...any) ([]byte, error) {
   330  	return RunWithStdin(dir, nil, cmdline...)
   331  }
   332  
   333  // bashQuoter escapes characters that have special meaning in double-quoted strings in the bash shell.
   334  // See https://www.gnu.org/software/bash/manual/html_node/Double-Quotes.html.
   335  var bashQuoter = strings.NewReplacer(`"`, `\"`, `$`, `\$`, "`", "\\`", `\`, `\\`)
   336  
   337  func RunWithStdin(dir string, stdin io.Reader, cmdline ...any) ([]byte, error) {
   338  	if dir != "" {
   339  		muIface, ok := dirLock.Load(dir)
   340  		if !ok {
   341  			muIface, _ = dirLock.LoadOrStore(dir, new(sync.Mutex))
   342  		}
   343  		mu := muIface.(*sync.Mutex)
   344  		mu.Lock()
   345  		defer mu.Unlock()
   346  	}
   347  
   348  	cmd := str.StringList(cmdline...)
   349  	if os.Getenv("TESTGOVCS") == "panic" {
   350  		panic(fmt.Sprintf("use of vcs: %v", cmd))
   351  	}
   352  	if cfg.BuildX {
   353  		text := new(strings.Builder)
   354  		if dir != "" {
   355  			text.WriteString("cd ")
   356  			text.WriteString(dir)
   357  			text.WriteString("; ")
   358  		}
   359  		for i, arg := range cmd {
   360  			if i > 0 {
   361  				text.WriteByte(' ')
   362  			}
   363  			switch {
   364  			case strings.ContainsAny(arg, "'"):
   365  				// Quote args that could be mistaken for quoted args.
   366  				text.WriteByte('"')
   367  				text.WriteString(bashQuoter.Replace(arg))
   368  				text.WriteByte('"')
   369  			case strings.ContainsAny(arg, "$`\\*?[\"\t\n\v\f\r \u0085\u00a0"):
   370  				// Quote args that contain special characters, glob patterns, or spaces.
   371  				text.WriteByte('\'')
   372  				text.WriteString(arg)
   373  				text.WriteByte('\'')
   374  			default:
   375  				text.WriteString(arg)
   376  			}
   377  		}
   378  		fmt.Fprintf(os.Stderr, "%s\n", text)
   379  		start := time.Now()
   380  		defer func() {
   381  			fmt.Fprintf(os.Stderr, "%.3fs # %s\n", time.Since(start).Seconds(), text)
   382  		}()
   383  	}
   384  	// TODO: Impose limits on command output size.
   385  	// TODO: Set environment to get English error messages.
   386  	var stderr bytes.Buffer
   387  	var stdout bytes.Buffer
   388  	c := exec.Command(cmd[0], cmd[1:]...)
   389  	c.Dir = dir
   390  	c.Stdin = stdin
   391  	c.Stderr = &stderr
   392  	c.Stdout = &stdout
   393  	err := c.Run()
   394  	if err != nil {
   395  		err = &RunError{Cmd: strings.Join(cmd, " ") + " in " + dir, Stderr: stderr.Bytes(), Err: err}
   396  	}
   397  	return stdout.Bytes(), err
   398  }