github.com/gagliardetto/golang-go@v0.0.0-20201020153340-53909ea70814/cmd/go/not-internal/modfetch/fetch.go (about)

     1  // Copyright 2018 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package modfetch
     6  
     7  import (
     8  	"archive/zip"
     9  	"bytes"
    10  	"errors"
    11  	"fmt"
    12  	"io"
    13  	"io/ioutil"
    14  	"os"
    15  	"path/filepath"
    16  	"sort"
    17  	"strings"
    18  	"sync"
    19  
    20  	"github.com/gagliardetto/golang-go/cmd/go/not-internal/base"
    21  	"github.com/gagliardetto/golang-go/cmd/go/not-internal/cfg"
    22  	"github.com/gagliardetto/golang-go/cmd/go/not-internal/lockedfile"
    23  	"github.com/gagliardetto/golang-go/cmd/go/not-internal/par"
    24  	"github.com/gagliardetto/golang-go/cmd/go/not-internal/renameio"
    25  	"github.com/gagliardetto/golang-go/cmd/go/not-internal/robustio"
    26  
    27  	"golang.org/x/mod/module"
    28  	"golang.org/x/mod/sumdb/dirhash"
    29  	modzip "golang.org/x/mod/zip"
    30  )
    31  
    32  var downloadCache par.Cache
    33  
    34  // Download downloads the specific module version to the
    35  // local download cache and returns the name of the directory
    36  // corresponding to the root of the module's file tree.
    37  func Download(mod module.Version) (dir string, err error) {
    38  	if PkgMod == "" {
    39  		// Do not download to current directory.
    40  		return "", fmt.Errorf("missing modfetch.PkgMod")
    41  	}
    42  
    43  	// The par.Cache here avoids duplicate work.
    44  	type cached struct {
    45  		dir string
    46  		err error
    47  	}
    48  	c := downloadCache.Do(mod, func() interface{} {
    49  		dir, err := download(mod)
    50  		if err != nil {
    51  			return cached{"", err}
    52  		}
    53  		checkMod(mod)
    54  		return cached{dir, nil}
    55  	}).(cached)
    56  	return c.dir, c.err
    57  }
    58  
    59  func download(mod module.Version) (dir string, err error) {
    60  	// If the directory exists, and no .partial file exists,
    61  	// the module has already been completely extracted.
    62  	// .partial files may be created when future versions of cmd/go
    63  	// extract module zip directories in place instead of extracting
    64  	// to a random temporary directory and renaming.
    65  	dir, err = DownloadDir(mod)
    66  	if err == nil {
    67  		return dir, nil
    68  	} else if dir == "" || !errors.Is(err, os.ErrNotExist) {
    69  		return "", err
    70  	}
    71  
    72  	// To avoid cluttering the cache with extraneous files,
    73  	// DownloadZip uses the same lockfile as Download.
    74  	// Invoke DownloadZip before locking the file.
    75  	zipfile, err := DownloadZip(mod)
    76  	if err != nil {
    77  		return "", err
    78  	}
    79  
    80  	unlock, err := lockVersion(mod)
    81  	if err != nil {
    82  		return "", err
    83  	}
    84  	defer unlock()
    85  
    86  	// Check whether the directory was populated while we were waiting on the lock.
    87  	_, dirErr := DownloadDir(mod)
    88  	if dirErr == nil {
    89  		return dir, nil
    90  	}
    91  	_, dirExists := dirErr.(*DownloadDirPartialError)
    92  
    93  	// Clean up any remaining temporary directories from previous runs, as well
    94  	// as partially extracted diectories created by future versions of cmd/go.
    95  	// This is only safe to do because the lock file ensures that their writers
    96  	// are no longer active.
    97  	parentDir := filepath.Dir(dir)
    98  	tmpPrefix := filepath.Base(dir) + ".tmp-"
    99  	if old, err := filepath.Glob(filepath.Join(parentDir, tmpPrefix+"*")); err == nil {
   100  		for _, path := range old {
   101  			RemoveAll(path) // best effort
   102  		}
   103  	}
   104  	if dirExists {
   105  		if err := RemoveAll(dir); err != nil {
   106  			return "", err
   107  		}
   108  	}
   109  
   110  	partialPath, err := CachePath(mod, "partial")
   111  	if err != nil {
   112  		return "", err
   113  	}
   114  	if err := os.Remove(partialPath); err != nil && !os.IsNotExist(err) {
   115  		return "", err
   116  	}
   117  
   118  	// Extract the zip file to a temporary directory, then rename it to the
   119  	// final path. That way, we can use the existence of the source directory to
   120  	// signal that it has been extracted successfully, and if someone deletes
   121  	// the entire directory (e.g. as an attempt to prune out file corruption)
   122  	// the module cache will still be left in a recoverable state.
   123  	if err := os.MkdirAll(parentDir, 0777); err != nil {
   124  		return "", err
   125  	}
   126  	tmpDir, err := ioutil.TempDir(parentDir, tmpPrefix)
   127  	if err != nil {
   128  		return "", err
   129  	}
   130  	defer func() {
   131  		if err != nil {
   132  			RemoveAll(tmpDir)
   133  		}
   134  	}()
   135  
   136  	if err := modzip.Unzip(tmpDir, mod, zipfile); err != nil {
   137  		fmt.Fprintf(os.Stderr, "-> %s\n", err)
   138  		return "", err
   139  	}
   140  
   141  	if err := robustio.Rename(tmpDir, dir); err != nil {
   142  		return "", err
   143  	}
   144  
   145  	if !cfg.ModCacheRW {
   146  		// Make dir read-only only *after* renaming it.
   147  		// os.Rename was observed to fail for read-only directories on macOS.
   148  		makeDirsReadOnly(dir)
   149  	}
   150  	return dir, nil
   151  }
   152  
   153  var downloadZipCache par.Cache
   154  
   155  // DownloadZip downloads the specific module version to the
   156  // local zip cache and returns the name of the zip file.
   157  func DownloadZip(mod module.Version) (zipfile string, err error) {
   158  	// The par.Cache here avoids duplicate work.
   159  	type cached struct {
   160  		zipfile string
   161  		err     error
   162  	}
   163  	c := downloadZipCache.Do(mod, func() interface{} {
   164  		zipfile, err := CachePath(mod, "zip")
   165  		if err != nil {
   166  			return cached{"", err}
   167  		}
   168  
   169  		// Skip locking if the zipfile already exists.
   170  		if _, err := os.Stat(zipfile); err == nil {
   171  			return cached{zipfile, nil}
   172  		}
   173  
   174  		// The zip file does not exist. Acquire the lock and create it.
   175  		if cfg.CmdName != "mod download" {
   176  			fmt.Fprintf(os.Stderr, "go: downloading %s %s\n", mod.Path, mod.Version)
   177  		}
   178  		unlock, err := lockVersion(mod)
   179  		if err != nil {
   180  			return cached{"", err}
   181  		}
   182  		defer unlock()
   183  
   184  		// Double-check that the zipfile was not created while we were waiting for
   185  		// the lock.
   186  		if _, err := os.Stat(zipfile); err == nil {
   187  			return cached{zipfile, nil}
   188  		}
   189  		if err := os.MkdirAll(filepath.Dir(zipfile), 0777); err != nil {
   190  			return cached{"", err}
   191  		}
   192  		if err := downloadZip(mod, zipfile); err != nil {
   193  			return cached{"", err}
   194  		}
   195  		return cached{zipfile, nil}
   196  	}).(cached)
   197  	return c.zipfile, c.err
   198  }
   199  
   200  func downloadZip(mod module.Version, zipfile string) (err error) {
   201  	// Clean up any remaining tempfiles from previous runs.
   202  	// This is only safe to do because the lock file ensures that their
   203  	// writers are no longer active.
   204  	for _, base := range []string{zipfile, zipfile + "hash"} {
   205  		if old, err := filepath.Glob(renameio.Pattern(base)); err == nil {
   206  			for _, path := range old {
   207  				os.Remove(path) // best effort
   208  			}
   209  		}
   210  	}
   211  
   212  	// From here to the os.Rename call below is functionally almost equivalent to
   213  	// renameio.WriteToFile, with one key difference: we want to validate the
   214  	// contents of the file (by hashing it) before we commit it. Because the file
   215  	// is zip-compressed, we need an actual file — or at least an io.ReaderAt — to
   216  	// validate it: we can't just tee the stream as we write it.
   217  	f, err := ioutil.TempFile(filepath.Dir(zipfile), filepath.Base(renameio.Pattern(zipfile)))
   218  	if err != nil {
   219  		return err
   220  	}
   221  	defer func() {
   222  		if err != nil {
   223  			f.Close()
   224  			os.Remove(f.Name())
   225  		}
   226  	}()
   227  
   228  	err = TryProxies(func(proxy string) error {
   229  		repo, err := Lookup(proxy, mod.Path)
   230  		if err != nil {
   231  			return err
   232  		}
   233  		return repo.Zip(f, mod.Version)
   234  	})
   235  	if err != nil {
   236  		return err
   237  	}
   238  
   239  	// Double-check that the paths within the zip file are well-formed.
   240  	//
   241  	// TODO(bcmills): There is a similar check within the Unzip function. Can we eliminate one?
   242  	fi, err := f.Stat()
   243  	if err != nil {
   244  		return err
   245  	}
   246  	z, err := zip.NewReader(f, fi.Size())
   247  	if err != nil {
   248  		return err
   249  	}
   250  	prefix := mod.Path + "@" + mod.Version + "/"
   251  	for _, f := range z.File {
   252  		if !strings.HasPrefix(f.Name, prefix) {
   253  			return fmt.Errorf("zip for %s has unexpected file %s", prefix[:len(prefix)-1], f.Name)
   254  		}
   255  	}
   256  
   257  	// Sync the file before renaming it: otherwise, after a crash the reader may
   258  	// observe a 0-length file instead of the actual contents.
   259  	// See https://golang.org/issue/22397#issuecomment-380831736.
   260  	if err := f.Sync(); err != nil {
   261  		return err
   262  	}
   263  	if err := f.Close(); err != nil {
   264  		return err
   265  	}
   266  
   267  	// Hash the zip file and check the sum before renaming to the final location.
   268  	hash, err := dirhash.HashZip(f.Name(), dirhash.DefaultHash)
   269  	if err != nil {
   270  		return err
   271  	}
   272  	if err := checkModSum(mod, hash); err != nil {
   273  		return err
   274  	}
   275  
   276  	if err := renameio.WriteFile(zipfile+"hash", []byte(hash), 0666); err != nil {
   277  		return err
   278  	}
   279  	if err := os.Rename(f.Name(), zipfile); err != nil {
   280  		return err
   281  	}
   282  
   283  	// TODO(bcmills): Should we make the .zip and .ziphash files read-only to discourage tampering?
   284  
   285  	return nil
   286  }
   287  
   288  // makeDirsReadOnly makes a best-effort attempt to remove write permissions for dir
   289  // and its transitive contents.
   290  func makeDirsReadOnly(dir string) {
   291  	type pathMode struct {
   292  		path string
   293  		mode os.FileMode
   294  	}
   295  	var dirs []pathMode // in lexical order
   296  	filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
   297  		if err == nil && info.Mode()&0222 != 0 {
   298  			if info.IsDir() {
   299  				dirs = append(dirs, pathMode{path, info.Mode()})
   300  			}
   301  		}
   302  		return nil
   303  	})
   304  
   305  	// Run over list backward to chmod children before parents.
   306  	for i := len(dirs) - 1; i >= 0; i-- {
   307  		os.Chmod(dirs[i].path, dirs[i].mode&^0222)
   308  	}
   309  }
   310  
   311  // RemoveAll removes a directory written by Download or Unzip, first applying
   312  // any permission changes needed to do so.
   313  func RemoveAll(dir string) error {
   314  	// Module cache has 0555 directories; make them writable in order to remove content.
   315  	filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
   316  		if err != nil {
   317  			return nil // ignore errors walking in file system
   318  		}
   319  		if info.IsDir() {
   320  			os.Chmod(path, 0777)
   321  		}
   322  		return nil
   323  	})
   324  	return os.RemoveAll(dir)
   325  }
   326  
   327  var GoSumFile string // path to go.sum; set by package modload
   328  
   329  type modSum struct {
   330  	mod module.Version
   331  	sum string
   332  }
   333  
   334  var goSum struct {
   335  	mu        sync.Mutex
   336  	m         map[module.Version][]string // content of go.sum file (+ go.modverify if present)
   337  	checked   map[modSum]bool             // sums actually checked during execution
   338  	dirty     bool                        // whether we added any new sums to m
   339  	overwrite bool                        // if true, overwrite go.sum without incorporating its contents
   340  	enabled   bool                        // whether to use go.sum at all
   341  	modverify string                      // path to go.modverify, to be deleted
   342  }
   343  
   344  // initGoSum initializes the go.sum data.
   345  // The boolean it returns reports whether the
   346  // use of go.sum is now enabled.
   347  // The goSum lock must be held.
   348  func initGoSum() (bool, error) {
   349  	if GoSumFile == "" {
   350  		return false, nil
   351  	}
   352  	if goSum.m != nil {
   353  		return true, nil
   354  	}
   355  
   356  	goSum.m = make(map[module.Version][]string)
   357  	goSum.checked = make(map[modSum]bool)
   358  	data, err := lockedfile.Read(GoSumFile)
   359  	if err != nil && !os.IsNotExist(err) {
   360  		return false, err
   361  	}
   362  	goSum.enabled = true
   363  	readGoSum(goSum.m, GoSumFile, data)
   364  
   365  	// Add old go.modverify file.
   366  	// We'll delete go.modverify in WriteGoSum.
   367  	alt := strings.TrimSuffix(GoSumFile, ".sum") + ".modverify"
   368  	if data, err := renameio.ReadFile(alt); err == nil {
   369  		migrate := make(map[module.Version][]string)
   370  		readGoSum(migrate, alt, data)
   371  		for mod, sums := range migrate {
   372  			for _, sum := range sums {
   373  				addModSumLocked(mod, sum)
   374  			}
   375  		}
   376  		goSum.modverify = alt
   377  	}
   378  	return true, nil
   379  }
   380  
   381  // emptyGoModHash is the hash of a 1-file tree containing a 0-length go.mod.
   382  // A bug caused us to write these into go.sum files for non-modules.
   383  // We detect and remove them.
   384  const emptyGoModHash = "h1:G7mAYYxgmS0lVkHyy2hEOLQCFB0DlQFTMLWggykrydY="
   385  
   386  // readGoSum parses data, which is the content of file,
   387  // and adds it to goSum.m. The goSum lock must be held.
   388  func readGoSum(dst map[module.Version][]string, file string, data []byte) error {
   389  	lineno := 0
   390  	for len(data) > 0 {
   391  		var line []byte
   392  		lineno++
   393  		i := bytes.IndexByte(data, '\n')
   394  		if i < 0 {
   395  			line, data = data, nil
   396  		} else {
   397  			line, data = data[:i], data[i+1:]
   398  		}
   399  		f := strings.Fields(string(line))
   400  		if len(f) == 0 {
   401  			// blank line; skip it
   402  			continue
   403  		}
   404  		if len(f) != 3 {
   405  			return fmt.Errorf("malformed go.sum:\n%s:%d: wrong number of fields %v", file, lineno, len(f))
   406  		}
   407  		if f[2] == emptyGoModHash {
   408  			// Old bug; drop it.
   409  			continue
   410  		}
   411  		mod := module.Version{Path: f[0], Version: f[1]}
   412  		dst[mod] = append(dst[mod], f[2])
   413  	}
   414  	return nil
   415  }
   416  
   417  // checkMod checks the given module's checksum.
   418  func checkMod(mod module.Version) {
   419  	if PkgMod == "" {
   420  		// Do not use current directory.
   421  		return
   422  	}
   423  
   424  	// Do the file I/O before acquiring the go.sum lock.
   425  	ziphash, err := CachePath(mod, "ziphash")
   426  	if err != nil {
   427  		base.Fatalf("verifying %v", module.VersionError(mod, err))
   428  	}
   429  	data, err := renameio.ReadFile(ziphash)
   430  	if err != nil {
   431  		if errors.Is(err, os.ErrNotExist) {
   432  			// This can happen if someone does rm -rf GOPATH/src/cache/download. So it goes.
   433  			return
   434  		}
   435  		base.Fatalf("verifying %v", module.VersionError(mod, err))
   436  	}
   437  	h := strings.TrimSpace(string(data))
   438  	if !strings.HasPrefix(h, "h1:") {
   439  		base.Fatalf("verifying %v", module.VersionError(mod, fmt.Errorf("unexpected ziphash: %q", h)))
   440  	}
   441  
   442  	if err := checkModSum(mod, h); err != nil {
   443  		base.Fatalf("%s", err)
   444  	}
   445  }
   446  
   447  // goModSum returns the checksum for the go.mod contents.
   448  func goModSum(data []byte) (string, error) {
   449  	return dirhash.Hash1([]string{"go.mod"}, func(string) (io.ReadCloser, error) {
   450  		return ioutil.NopCloser(bytes.NewReader(data)), nil
   451  	})
   452  }
   453  
   454  // checkGoMod checks the given module's go.mod checksum;
   455  // data is the go.mod content.
   456  func checkGoMod(path, version string, data []byte) error {
   457  	h, err := goModSum(data)
   458  	if err != nil {
   459  		return &module.ModuleError{Path: path, Version: version, Err: fmt.Errorf("verifying go.mod: %v", err)}
   460  	}
   461  
   462  	return checkModSum(module.Version{Path: path, Version: version + "/go.mod"}, h)
   463  }
   464  
   465  // checkModSum checks that the recorded checksum for mod is h.
   466  func checkModSum(mod module.Version, h string) error {
   467  	// We lock goSum when manipulating it,
   468  	// but we arrange to release the lock when calling checkSumDB,
   469  	// so that parallel calls to checkModHash can execute parallel calls
   470  	// to checkSumDB.
   471  
   472  	// Check whether mod+h is listed in go.sum already. If so, we're done.
   473  	goSum.mu.Lock()
   474  	inited, err := initGoSum()
   475  	if err != nil {
   476  		return err
   477  	}
   478  	done := inited && haveModSumLocked(mod, h)
   479  	goSum.mu.Unlock()
   480  
   481  	if done {
   482  		return nil
   483  	}
   484  
   485  	// Not listed, so we want to add them.
   486  	// Consult checksum database if appropriate.
   487  	if useSumDB(mod) {
   488  		// Calls base.Fatalf if mismatch detected.
   489  		if err := checkSumDB(mod, h); err != nil {
   490  			return err
   491  		}
   492  	}
   493  
   494  	// Add mod+h to go.sum, if it hasn't appeared already.
   495  	if inited {
   496  		goSum.mu.Lock()
   497  		addModSumLocked(mod, h)
   498  		goSum.mu.Unlock()
   499  	}
   500  	return nil
   501  }
   502  
   503  // haveModSumLocked reports whether the pair mod,h is already listed in go.sum.
   504  // If it finds a conflicting pair instead, it calls base.Fatalf.
   505  // goSum.mu must be locked.
   506  func haveModSumLocked(mod module.Version, h string) bool {
   507  	goSum.checked[modSum{mod, h}] = true
   508  	for _, vh := range goSum.m[mod] {
   509  		if h == vh {
   510  			return true
   511  		}
   512  		if strings.HasPrefix(vh, "h1:") {
   513  			base.Fatalf("verifying %s@%s: checksum mismatch\n\tdownloaded: %v\n\tgo.sum:     %v"+goSumMismatch, mod.Path, mod.Version, h, vh)
   514  		}
   515  	}
   516  	return false
   517  }
   518  
   519  // addModSumLocked adds the pair mod,h to go.sum.
   520  // goSum.mu must be locked.
   521  func addModSumLocked(mod module.Version, h string) {
   522  	if haveModSumLocked(mod, h) {
   523  		return
   524  	}
   525  	if len(goSum.m[mod]) > 0 {
   526  		fmt.Fprintf(os.Stderr, "warning: verifying %s@%s: unknown hashes in go.sum: %v; adding %v"+hashVersionMismatch, mod.Path, mod.Version, strings.Join(goSum.m[mod], ", "), h)
   527  	}
   528  	goSum.m[mod] = append(goSum.m[mod], h)
   529  	goSum.dirty = true
   530  }
   531  
   532  // checkSumDB checks the mod, h pair against the Go checksum database.
   533  // It calls base.Fatalf if the hash is to be rejected.
   534  func checkSumDB(mod module.Version, h string) error {
   535  	db, lines, err := lookupSumDB(mod)
   536  	if err != nil {
   537  		return module.VersionError(mod, fmt.Errorf("verifying module: %v", err))
   538  	}
   539  
   540  	have := mod.Path + " " + mod.Version + " " + h
   541  	prefix := mod.Path + " " + mod.Version + " h1:"
   542  	for _, line := range lines {
   543  		if line == have {
   544  			return nil
   545  		}
   546  		if strings.HasPrefix(line, prefix) {
   547  			return module.VersionError(mod, fmt.Errorf("verifying module: checksum mismatch\n\tdownloaded: %v\n\t%s: %v"+sumdbMismatch, h, db, line[len(prefix)-len("h1:"):]))
   548  		}
   549  	}
   550  	return nil
   551  }
   552  
   553  // Sum returns the checksum for the downloaded copy of the given module,
   554  // if present in the download cache.
   555  func Sum(mod module.Version) string {
   556  	if PkgMod == "" {
   557  		// Do not use current directory.
   558  		return ""
   559  	}
   560  
   561  	ziphash, err := CachePath(mod, "ziphash")
   562  	if err != nil {
   563  		return ""
   564  	}
   565  	data, err := renameio.ReadFile(ziphash)
   566  	if err != nil {
   567  		return ""
   568  	}
   569  	return strings.TrimSpace(string(data))
   570  }
   571  
   572  // WriteGoSum writes the go.sum file if it needs to be updated.
   573  func WriteGoSum() {
   574  	goSum.mu.Lock()
   575  	defer goSum.mu.Unlock()
   576  
   577  	if !goSum.enabled {
   578  		// If we haven't read the go.sum file yet, don't bother writing it: at best,
   579  		// we could rename the go.modverify file if it isn't empty, but we haven't
   580  		// needed to touch it so far — how important could it be?
   581  		return
   582  	}
   583  	if !goSum.dirty {
   584  		// Don't bother opening the go.sum file if we don't have anything to add.
   585  		return
   586  	}
   587  	if cfg.BuildMod == "readonly" {
   588  		base.Fatalf("go: updates to go.sum needed, disabled by -mod=readonly")
   589  	}
   590  
   591  	// Make a best-effort attempt to acquire the side lock, only to exclude
   592  	// previous versions of the 'go' command from making simultaneous edits.
   593  	if unlock, err := SideLock(); err == nil {
   594  		defer unlock()
   595  	}
   596  
   597  	err := lockedfile.Transform(GoSumFile, func(data []byte) ([]byte, error) {
   598  		if !goSum.overwrite {
   599  			// Incorporate any sums added by other processes in the meantime.
   600  			// Add only the sums that we actually checked: the user may have edited or
   601  			// truncated the file to remove erroneous hashes, and we shouldn't restore
   602  			// them without good reason.
   603  			goSum.m = make(map[module.Version][]string, len(goSum.m))
   604  			readGoSum(goSum.m, GoSumFile, data)
   605  			for ms := range goSum.checked {
   606  				addModSumLocked(ms.mod, ms.sum)
   607  				goSum.dirty = true
   608  			}
   609  		}
   610  
   611  		var mods []module.Version
   612  		for m := range goSum.m {
   613  			mods = append(mods, m)
   614  		}
   615  		module.Sort(mods)
   616  
   617  		var buf bytes.Buffer
   618  		for _, m := range mods {
   619  			list := goSum.m[m]
   620  			sort.Strings(list)
   621  			for _, h := range list {
   622  				fmt.Fprintf(&buf, "%s %s %s\n", m.Path, m.Version, h)
   623  			}
   624  		}
   625  		return buf.Bytes(), nil
   626  	})
   627  
   628  	if err != nil {
   629  		base.Fatalf("go: updating go.sum: %v", err)
   630  	}
   631  
   632  	goSum.checked = make(map[modSum]bool)
   633  	goSum.dirty = false
   634  	goSum.overwrite = false
   635  
   636  	if goSum.modverify != "" {
   637  		os.Remove(goSum.modverify) // best effort
   638  	}
   639  }
   640  
   641  // TrimGoSum trims go.sum to contain only the modules for which keep[m] is true.
   642  func TrimGoSum(keep map[module.Version]bool) {
   643  	goSum.mu.Lock()
   644  	defer goSum.mu.Unlock()
   645  	inited, err := initGoSum()
   646  	if err != nil {
   647  		base.Fatalf("%s", err)
   648  	}
   649  	if !inited {
   650  		return
   651  	}
   652  
   653  	for m := range goSum.m {
   654  		// If we're keeping x@v we also keep x@v/go.mod.
   655  		// Map x@v/go.mod back to x@v for the keep lookup.
   656  		noGoMod := module.Version{Path: m.Path, Version: strings.TrimSuffix(m.Version, "/go.mod")}
   657  		if !keep[m] && !keep[noGoMod] {
   658  			delete(goSum.m, m)
   659  			goSum.dirty = true
   660  			goSum.overwrite = true
   661  		}
   662  	}
   663  }
   664  
   665  const goSumMismatch = `
   666  
   667  SECURITY ERROR
   668  This download does NOT match an earlier download recorded in go.sum.
   669  The bits may have been replaced on the origin server, or an attacker may
   670  have intercepted the download attempt.
   671  
   672  For more information, see 'go help module-auth'.
   673  `
   674  
   675  const sumdbMismatch = `
   676  
   677  SECURITY ERROR
   678  This download does NOT match the one reported by the checksum server.
   679  The bits may have been replaced on the origin server, or an attacker may
   680  have intercepted the download attempt.
   681  
   682  For more information, see 'go help module-auth'.
   683  `
   684  
   685  const hashVersionMismatch = `
   686  
   687  SECURITY WARNING
   688  This download is listed in go.sum, but using an unknown hash algorithm.
   689  The download cannot be verified.
   690  
   691  For more information, see 'go help module-auth'.
   692  
   693  `
   694  
   695  var HelpModuleAuth = &base.Command{
   696  	UsageLine: "module-auth",
   697  	Short:     "module authentication using go.sum",
   698  	Long: `
   699  The go command tries to authenticate every downloaded module,
   700  checking that the bits downloaded for a specific module version today
   701  match bits downloaded yesterday. This ensures repeatable builds
   702  and detects introduction of unexpected changes, malicious or not.
   703  
   704  In each module's root, alongside go.mod, the go command maintains
   705  a file named go.sum containing the cryptographic checksums of the
   706  module's dependencies.
   707  
   708  The form of each line in go.sum is three fields:
   709  
   710  	<module> <version>[/go.mod] <hash>
   711  
   712  Each known module version results in two lines in the go.sum file.
   713  The first line gives the hash of the module version's file tree.
   714  The second line appends "/go.mod" to the version and gives the hash
   715  of only the module version's (possibly synthesized) go.mod file.
   716  The go.mod-only hash allows downloading and authenticating a
   717  module version's go.mod file, which is needed to compute the
   718  dependency graph, without also downloading all the module's source code.
   719  
   720  The hash begins with an algorithm prefix of the form "h<N>:".
   721  The only defined algorithm prefix is "h1:", which uses SHA-256.
   722  
   723  Module authentication failures
   724  
   725  The go command maintains a cache of downloaded packages and computes
   726  and records the cryptographic checksum of each package at download time.
   727  In normal operation, the go command checks the main module's go.sum file
   728  against these precomputed checksums instead of recomputing them on
   729  each command invocation. The 'go mod verify' command checks that
   730  the cached copies of module downloads still match both their recorded
   731  checksums and the entries in go.sum.
   732  
   733  In day-to-day development, the checksum of a given module version
   734  should never change. Each time a dependency is used by a given main
   735  module, the go command checks its local cached copy, freshly
   736  downloaded or not, against the main module's go.sum. If the checksums
   737  don't match, the go command reports the mismatch as a security error
   738  and refuses to run the build. When this happens, proceed with caution:
   739  code changing unexpectedly means today's build will not match
   740  yesterday's, and the unexpected change may not be beneficial.
   741  
   742  If the go command reports a mismatch in go.sum, the downloaded code
   743  for the reported module version does not match the one used in a
   744  previous build of the main module. It is important at that point
   745  to find out what the right checksum should be, to decide whether
   746  go.sum is wrong or the downloaded code is wrong. Usually go.sum is right:
   747  you want to use the same code you used yesterday.
   748  
   749  If a downloaded module is not yet included in go.sum and it is a publicly
   750  available module, the go command consults the Go checksum database to fetch
   751  the expected go.sum lines. If the downloaded code does not match those
   752  lines, the go command reports the mismatch and exits. Note that the
   753  database is not consulted for module versions already listed in go.sum.
   754  
   755  If a go.sum mismatch is reported, it is always worth investigating why
   756  the code downloaded today differs from what was downloaded yesterday.
   757  
   758  The GOSUMDB environment variable identifies the name of checksum database
   759  to use and optionally its public key and URL, as in:
   760  
   761  	GOSUMDB="sum.golang.org"
   762  	GOSUMDB="sum.golang.org+<publickey>"
   763  	GOSUMDB="sum.golang.org+<publickey> https://sum.golang.org"
   764  
   765  The go command knows the public key of sum.golang.org, and also that the name
   766  sum.golang.google.cn (available inside mainland China) connects to the
   767  sum.golang.org checksum database; use of any other database requires giving
   768  the public key explicitly.
   769  The URL defaults to "https://" followed by the database name.
   770  
   771  GOSUMDB defaults to "sum.golang.org", the Go checksum database run by Google.
   772  See https://sum.golang.org/privacy for the service's privacy policy.
   773  
   774  If GOSUMDB is set to "off", or if "go get" is invoked with the -insecure flag,
   775  the checksum database is not consulted, and all unrecognized modules are
   776  accepted, at the cost of giving up the security guarantee of verified repeatable
   777  downloads for all modules. A better way to bypass the checksum database
   778  for specific modules is to use the GOPRIVATE or GONOSUMDB environment
   779  variables. See 'go help module-private' for details.
   780  
   781  The 'go env -w' command (see 'go help env') can be used to set these variables
   782  for future go command invocations.
   783  `,
   784  }
   785  
   786  var HelpModulePrivate = &base.Command{
   787  	UsageLine: "module-private",
   788  	Short:     "module configuration for non-public modules",
   789  	Long: `
   790  The go command defaults to downloading modules from the public Go module
   791  mirror at proxy.golang.org. It also defaults to validating downloaded modules,
   792  regardless of source, against the public Go checksum database at sum.golang.org.
   793  These defaults work well for publicly available source code.
   794  
   795  The GOPRIVATE environment variable controls which modules the go command
   796  considers to be private (not available publicly) and should therefore not use the
   797  proxy or checksum database. The variable is a comma-separated list of
   798  glob patterns (in the syntax of Go's path.Match) of module path prefixes.
   799  For example,
   800  
   801  	GOPRIVATE=*.corp.example.com,rsc.io/private
   802  
   803  causes the go command to treat as private any module with a path prefix
   804  matching either pattern, including git.corp.example.com/xyzzy, rsc.io/private,
   805  and rsc.io/private/quux.
   806  
   807  The GOPRIVATE environment variable may be used by other tools as well to
   808  identify non-public modules. For example, an editor could use GOPRIVATE
   809  to decide whether to hyperlink a package import to a godoc.org page.
   810  
   811  For fine-grained control over module download and validation, the GONOPROXY
   812  and GONOSUMDB environment variables accept the same kind of glob list
   813  and override GOPRIVATE for the specific decision of whether to use the proxy
   814  and checksum database, respectively.
   815  
   816  For example, if a company ran a module proxy serving private modules,
   817  users would configure go using:
   818  
   819  	GOPRIVATE=*.corp.example.com
   820  	GOPROXY=proxy.example.com
   821  	GONOPROXY=none
   822  
   823  This would tell the go command and other tools that modules beginning with
   824  a corp.example.com subdomain are private but that the company proxy should
   825  be used for downloading both public and private modules, because
   826  GONOPROXY has been set to a pattern that won't match any modules,
   827  overriding GOPRIVATE.
   828  
   829  The 'go env -w' command (see 'go help env') can be used to set these variables
   830  for future go command invocations.
   831  `,
   832  }