github.com/golang/dep@v0.5.4/gps/source.go (about)

     1  // Copyright 2017 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package gps
     6  
     7  import (
     8  	"bytes"
     9  	"context"
    10  	"fmt"
    11  	"log"
    12  	"sync"
    13  
    14  	"github.com/golang/dep/gps/pkgtree"
    15  	"github.com/pkg/errors"
    16  )
    17  
    18  // sourceState represent the states that a source can be in, depending on how
    19  // much search and discovery work ahs been done by a source's managing gateway.
    20  //
    21  // These are basically used to achieve a cheap approximation of a FSM.
    22  type sourceState int32
    23  
    24  const (
    25  	// sourceExistsUpstream means the chosen source was verified upstream, during this execution.
    26  	sourceExistsUpstream sourceState = 1 << iota
    27  	// sourceExistsLocally means the repo was retrieved in the past.
    28  	sourceExistsLocally
    29  	// sourceHasLatestVersionList means the version list was refreshed within the cache window.
    30  	sourceHasLatestVersionList
    31  	// sourceHasLatestLocally means the repo was pulled fresh during this execution.
    32  	sourceHasLatestLocally
    33  )
    34  
    35  func (state sourceState) String() string {
    36  	var b bytes.Buffer
    37  	for _, s := range []struct {
    38  		sourceState
    39  		string
    40  	}{
    41  		{sourceExistsUpstream, "sourceExistsUpstream"},
    42  		{sourceExistsLocally, "sourceExistsLocally"},
    43  		{sourceHasLatestVersionList, "sourceHasLatestVersionList"},
    44  		{sourceHasLatestLocally, "sourceHasLatestLocally"},
    45  	} {
    46  		if state&s.sourceState > 0 {
    47  			if b.Len() > 0 {
    48  				b.WriteString("|")
    49  			}
    50  			b.WriteString(s.string)
    51  		}
    52  	}
    53  	return b.String()
    54  }
    55  
    56  type srcReturn struct {
    57  	*sourceGateway
    58  	error
    59  }
    60  
    61  type sourceCoordinator struct {
    62  	supervisor *supervisor
    63  	deducer    deducer
    64  	srcmut     sync.RWMutex // guards srcs and srcIdx
    65  	srcs       map[string]*sourceGateway
    66  	nameToURL  map[string]string
    67  	psrcmut    sync.Mutex // guards protoSrcs map
    68  	protoSrcs  map[string][]chan srcReturn
    69  	cachedir   string
    70  	cache      sourceCache
    71  	logger     *log.Logger
    72  }
    73  
    74  // newSourceCoordinator returns a new sourceCoordinator.
    75  // Passing a nil sourceCache defaults to an in-memory cache.
    76  func newSourceCoordinator(superv *supervisor, deducer deducer, cachedir string, cache sourceCache, logger *log.Logger) *sourceCoordinator {
    77  	if cache == nil {
    78  		cache = memoryCache{}
    79  	}
    80  	return &sourceCoordinator{
    81  		supervisor: superv,
    82  		deducer:    deducer,
    83  		cachedir:   cachedir,
    84  		cache:      cache,
    85  		logger:     logger,
    86  		srcs:       make(map[string]*sourceGateway),
    87  		nameToURL:  make(map[string]string),
    88  		protoSrcs:  make(map[string][]chan srcReturn),
    89  	}
    90  }
    91  
    92  func (sc *sourceCoordinator) close() {
    93  	if err := sc.cache.close(); err != nil {
    94  		sc.logger.Println(errors.Wrap(err, "failed to close the source cache"))
    95  	}
    96  }
    97  
    98  func (sc *sourceCoordinator) getSourceGatewayFor(ctx context.Context, id ProjectIdentifier) (*sourceGateway, error) {
    99  	if err := sc.supervisor.ctx.Err(); err != nil {
   100  		return nil, err
   101  	}
   102  
   103  	normalizedName := id.normalizedSource()
   104  
   105  	sc.srcmut.RLock()
   106  	if url, has := sc.nameToURL[normalizedName]; has {
   107  		srcGate, has := sc.srcs[url]
   108  		sc.srcmut.RUnlock()
   109  		if has {
   110  			return srcGate, nil
   111  		}
   112  		panic(fmt.Sprintf("%q was URL for %q in nameToURL, but no corresponding srcGate in srcs map", url, normalizedName))
   113  	}
   114  
   115  	// Without a direct match, we must fold the input name to a generally
   116  	// stable, caseless variant and primarily work from that. This ensures that
   117  	// on case-insensitive filesystems, we do not end up with multiple
   118  	// sourceGateways for paths that vary only by case. We perform folding
   119  	// unconditionally, independent of whether the underlying fs is
   120  	// case-sensitive, in order to ensure uniform behavior.
   121  	//
   122  	// This has significant implications. It is effectively deciding that the
   123  	// ProjectRoot portion of import paths are case-insensitive, which is by no
   124  	// means an invariant maintained by all hosting systems. If this presents a
   125  	// problem in practice, then we can explore expanding the deduction system
   126  	// to include case-sensitivity-for-roots metadata and treat it on a
   127  	// host-by-host basis. Such cases would still be rejected by the Go
   128  	// toolchain's compiler, though, and case-sensitivity in root names is
   129  	// likely to be at least frowned on if not disallowed by most hosting
   130  	// systems. So we follow this path, which is both a vastly simpler solution
   131  	// and one that seems quite likely to work in practice.
   132  	foldedNormalName := toFold(normalizedName)
   133  	notFolded := foldedNormalName != normalizedName
   134  	if notFolded {
   135  		// If the folded name differs from the input name, then there may
   136  		// already be an entry for it in the nameToURL map, so check again.
   137  		if url, has := sc.nameToURL[foldedNormalName]; has {
   138  			srcGate, has := sc.srcs[url]
   139  			// There was a match on the canonical folded variant. Upgrade to a
   140  			// write lock, so that future calls on this name don't need to
   141  			// burn cycles on folding.
   142  			sc.srcmut.RUnlock()
   143  			sc.srcmut.Lock()
   144  			// It may be possible that another goroutine could interleave
   145  			// between the unlock and re-lock. Even if they do, though, they'll
   146  			// only have recorded the same url value as we have here. In other
   147  			// words, these operations commute, so we can safely write here
   148  			// without checking again.
   149  			sc.nameToURL[normalizedName] = url
   150  			sc.srcmut.Unlock()
   151  			if has {
   152  				return srcGate, nil
   153  			}
   154  			panic(fmt.Sprintf("%q was URL for %q in nameToURL, but no corresponding srcGate in srcs map", url, normalizedName))
   155  		}
   156  	}
   157  	sc.srcmut.RUnlock()
   158  
   159  	// No gateway exists for this path yet; set up a proto, being careful to fold
   160  	// together simultaneous attempts on the same case-folded path.
   161  	sc.psrcmut.Lock()
   162  	if chans, has := sc.protoSrcs[foldedNormalName]; has {
   163  		// Another goroutine is already working on this normalizedName. Fold
   164  		// in with that work by attaching our return channels to the list.
   165  		rc := make(chan srcReturn, 1)
   166  		sc.protoSrcs[foldedNormalName] = append(chans, rc)
   167  		sc.psrcmut.Unlock()
   168  		ret := <-rc
   169  		return ret.sourceGateway, ret.error
   170  	}
   171  
   172  	sc.protoSrcs[foldedNormalName] = []chan srcReturn{}
   173  	sc.psrcmut.Unlock()
   174  
   175  	doReturn := func(sg *sourceGateway, err error) {
   176  		ret := srcReturn{sourceGateway: sg, error: err}
   177  		sc.psrcmut.Lock()
   178  		for _, rc := range sc.protoSrcs[foldedNormalName] {
   179  			rc <- ret
   180  		}
   181  		delete(sc.protoSrcs, foldedNormalName)
   182  		sc.psrcmut.Unlock()
   183  	}
   184  
   185  	pd, err := sc.deducer.deduceRootPath(ctx, normalizedName)
   186  	if err != nil {
   187  		// As in the deducer, don't cache errors so that externally-driven retry
   188  		// strategies can be constructed.
   189  		doReturn(nil, err)
   190  		return nil, err
   191  	}
   192  
   193  	// It'd be quite the feat - but not impossible - for a gateway
   194  	// corresponding to this normalizedName to have slid into the main
   195  	// sources map after the initial unlock, but before this goroutine got
   196  	// scheduled. Guard against that by checking the main sources map again
   197  	// and bailing out if we find an entry.
   198  	sc.srcmut.RLock()
   199  	if url, has := sc.nameToURL[foldedNormalName]; has {
   200  		if srcGate, has := sc.srcs[url]; has {
   201  			sc.srcmut.RUnlock()
   202  			doReturn(srcGate, nil)
   203  			return srcGate, nil
   204  		}
   205  		panic(fmt.Sprintf("%q was URL for %q in nameToURL, but no corresponding srcGate in srcs map", url, normalizedName))
   206  	}
   207  	sc.srcmut.RUnlock()
   208  
   209  	sc.srcmut.Lock()
   210  	defer sc.srcmut.Unlock()
   211  
   212  	// Get or create a sourceGateway.
   213  	var srcGate *sourceGateway
   214  	var url, unfoldedURL string
   215  	var errs errorSlice
   216  	for _, m := range pd.mb {
   217  		url = m.URL().String()
   218  		if notFolded {
   219  			// If the normalizedName and foldedNormalName differ, then we're pretty well
   220  			// guaranteed that returned URL will also need folding into canonical form.
   221  			unfoldedURL = url
   222  			url = toFold(url)
   223  		}
   224  		if sg, has := sc.srcs[url]; has {
   225  			srcGate = sg
   226  			break
   227  		}
   228  		src, err := m.try(ctx, sc.cachedir)
   229  		if err == nil {
   230  			cache := sc.cache.newSingleSourceCache(id)
   231  			srcGate, err = newSourceGateway(ctx, src, sc.supervisor, sc.cachedir, cache)
   232  			if err == nil {
   233  				sc.srcs[url] = srcGate
   234  				break
   235  			}
   236  		}
   237  		errs = append(errs, err)
   238  	}
   239  	if srcGate == nil {
   240  		doReturn(nil, errs)
   241  		return nil, errs
   242  	}
   243  
   244  	// Record the name -> URL mapping, making sure that we also get the
   245  	// self-mapping.
   246  	sc.nameToURL[foldedNormalName] = url
   247  	if url != foldedNormalName {
   248  		sc.nameToURL[url] = url
   249  	}
   250  
   251  	// Make sure we have both the folded and unfolded names and URLs recorded in
   252  	// the map, if the input needed folding.
   253  	if notFolded {
   254  		sc.nameToURL[normalizedName] = url
   255  		sc.nameToURL[unfoldedURL] = url
   256  	}
   257  
   258  	doReturn(srcGate, nil)
   259  	return srcGate, nil
   260  }
   261  
   262  // sourceGateways manage all incoming calls for data from sources, serializing
   263  // and caching them as needed.
   264  type sourceGateway struct {
   265  	cachedir string
   266  	srcState sourceState
   267  	src      source
   268  	cache    singleSourceCache
   269  	mu       sync.Mutex // global lock, serializes all behaviors
   270  	suprvsr  *supervisor
   271  }
   272  
   273  // newSourceGateway returns a new gateway for src. If the source exists locally,
   274  // the local state may be cleaned, otherwise we ping upstream.
   275  func newSourceGateway(ctx context.Context, src source, superv *supervisor, cachedir string, cache singleSourceCache) (*sourceGateway, error) {
   276  	var state sourceState
   277  	local := src.existsLocally(ctx)
   278  	if local {
   279  		state |= sourceExistsLocally
   280  		if err := superv.do(ctx, src.upstreamURL(), ctValidateLocal, src.maybeClean); err != nil {
   281  			return nil, err
   282  		}
   283  	}
   284  
   285  	sg := &sourceGateway{
   286  		srcState: state,
   287  		src:      src,
   288  		cachedir: cachedir,
   289  		cache:    cache,
   290  		suprvsr:  superv,
   291  	}
   292  
   293  	if !local {
   294  		if err := sg.require(ctx, sourceExistsUpstream); err != nil {
   295  			return nil, err
   296  		}
   297  	}
   298  
   299  	return sg, nil
   300  }
   301  
   302  func (sg *sourceGateway) syncLocal(ctx context.Context) error {
   303  	sg.mu.Lock()
   304  	err := sg.require(ctx, sourceExistsLocally|sourceHasLatestLocally)
   305  	sg.mu.Unlock()
   306  	return err
   307  }
   308  
   309  func (sg *sourceGateway) existsInCache(ctx context.Context) error {
   310  	sg.mu.Lock()
   311  	err := sg.require(ctx, sourceExistsLocally)
   312  	sg.mu.Unlock()
   313  	return err
   314  }
   315  
   316  func (sg *sourceGateway) existsUpstream(ctx context.Context) error {
   317  	sg.mu.Lock()
   318  	err := sg.require(ctx, sourceExistsUpstream)
   319  	sg.mu.Unlock()
   320  	return err
   321  }
   322  
   323  func (sg *sourceGateway) exportVersionTo(ctx context.Context, v Version, to string) error {
   324  	sg.mu.Lock()
   325  	defer sg.mu.Unlock()
   326  
   327  	err := sg.require(ctx, sourceExistsLocally)
   328  	if err != nil {
   329  		return err
   330  	}
   331  
   332  	r, err := sg.convertToRevision(ctx, v)
   333  	if err != nil {
   334  		return err
   335  	}
   336  
   337  	err = sg.suprvsr.do(ctx, sg.src.upstreamURL(), ctExportTree, func(ctx context.Context) error {
   338  		return sg.src.exportRevisionTo(ctx, r, to)
   339  	})
   340  
   341  	// It's possible (in git) that we may have tried this against a version that
   342  	// doesn't exist in the repository cache, even though we know it exists in
   343  	// the upstream. If it looks like that might be the case, update the local
   344  	// and retry.
   345  	// TODO(sdboyer) It'd be better if we could check the error to see if this
   346  	// actually was the cause of the problem.
   347  	if err != nil && sg.srcState&sourceHasLatestLocally == 0 {
   348  		if err = sg.require(ctx, sourceHasLatestLocally); err == nil {
   349  			err = sg.suprvsr.do(ctx, sg.src.upstreamURL(), ctExportTree, func(ctx context.Context) error {
   350  				return sg.src.exportRevisionTo(ctx, r, to)
   351  			})
   352  		}
   353  	}
   354  
   355  	return err
   356  }
   357  
   358  func (sg *sourceGateway) exportPrunedVersionTo(ctx context.Context, lp LockedProject, prune PruneOptions, to string) error {
   359  	sg.mu.Lock()
   360  	defer sg.mu.Unlock()
   361  
   362  	err := sg.require(ctx, sourceExistsLocally)
   363  	if err != nil {
   364  		return err
   365  	}
   366  
   367  	r, err := sg.convertToRevision(ctx, lp.Version())
   368  	if err != nil {
   369  		return err
   370  	}
   371  
   372  	if fastprune, ok := sg.src.(sourceFastPrune); ok {
   373  		return sg.suprvsr.do(ctx, sg.src.upstreamURL(), ctExportTree, func(ctx context.Context) error {
   374  			return fastprune.exportPrunedRevisionTo(ctx, r, lp.Packages(), prune, to)
   375  		})
   376  	}
   377  
   378  	if err = sg.suprvsr.do(ctx, sg.src.upstreamURL(), ctExportTree, func(ctx context.Context) error {
   379  		return sg.src.exportRevisionTo(ctx, r, to)
   380  	}); err != nil {
   381  		return err
   382  	}
   383  
   384  	return PruneProject(to, lp, prune)
   385  }
   386  
   387  func (sg *sourceGateway) getManifestAndLock(ctx context.Context, pr ProjectRoot, v Version, an ProjectAnalyzer) (Manifest, Lock, error) {
   388  	sg.mu.Lock()
   389  	defer sg.mu.Unlock()
   390  
   391  	r, err := sg.convertToRevision(ctx, v)
   392  	if err != nil {
   393  		return nil, nil, err
   394  	}
   395  
   396  	m, l, has := sg.cache.getManifestAndLock(r, an.Info())
   397  	if has {
   398  		return m, l, nil
   399  	}
   400  
   401  	err = sg.require(ctx, sourceExistsLocally)
   402  	if err != nil {
   403  		return nil, nil, err
   404  	}
   405  
   406  	label := fmt.Sprintf("%s:%s", sg.src.upstreamURL(), an.Info())
   407  	err = sg.suprvsr.do(ctx, label, ctGetManifestAndLock, func(ctx context.Context) error {
   408  		m, l, err = sg.src.getManifestAndLock(ctx, pr, r, an)
   409  		return err
   410  	})
   411  
   412  	// It's possible (in git) that we may have tried this against a version that
   413  	// doesn't exist in the repository cache, even though we know it exists in
   414  	// the upstream. If it looks like that might be the case, update the local
   415  	// and retry.
   416  	// TODO(sdboyer) It'd be better if we could check the error to see if this
   417  	// actually was the cause of the problem.
   418  	if err != nil && sg.srcState&sourceHasLatestLocally == 0 {
   419  		// TODO(sdboyer) we should warn/log/something in adaptive recovery
   420  		// situations like this
   421  		err = sg.require(ctx, sourceHasLatestLocally)
   422  		if err != nil {
   423  			return nil, nil, err
   424  		}
   425  
   426  		err = sg.suprvsr.do(ctx, label, ctGetManifestAndLock, func(ctx context.Context) error {
   427  			m, l, err = sg.src.getManifestAndLock(ctx, pr, r, an)
   428  			return err
   429  		})
   430  	}
   431  
   432  	if err != nil {
   433  		return nil, nil, err
   434  	}
   435  
   436  	sg.cache.setManifestAndLock(r, an.Info(), m, l)
   437  	return m, l, nil
   438  }
   439  
   440  func (sg *sourceGateway) listPackages(ctx context.Context, pr ProjectRoot, v Version) (pkgtree.PackageTree, error) {
   441  	sg.mu.Lock()
   442  	defer sg.mu.Unlock()
   443  
   444  	r, err := sg.convertToRevision(ctx, v)
   445  	if err != nil {
   446  		return pkgtree.PackageTree{}, err
   447  	}
   448  
   449  	ptree, has := sg.cache.getPackageTree(r, pr)
   450  	if has {
   451  		return ptree, nil
   452  	}
   453  
   454  	err = sg.require(ctx, sourceExistsLocally)
   455  	if err != nil {
   456  		return pkgtree.PackageTree{}, err
   457  	}
   458  
   459  	label := fmt.Sprintf("%s:%s", pr, sg.src.upstreamURL())
   460  	err = sg.suprvsr.do(ctx, label, ctListPackages, func(ctx context.Context) error {
   461  		ptree, err = sg.src.listPackages(ctx, pr, r)
   462  		return err
   463  	})
   464  
   465  	// It's possible (in git) that we may have tried this against a version that
   466  	// doesn't exist in the repository cache, even though we know it exists in
   467  	// the upstream. If it looks like that might be the case, update the local
   468  	// and retry.
   469  	// TODO(sdboyer) It'd be better if we could check the error to see if this
   470  	// actually was the cause of the problem.
   471  	if err != nil && sg.srcState&sourceHasLatestLocally == 0 {
   472  		// TODO(sdboyer) we should warn/log/something in adaptive recovery
   473  		// situations like this
   474  		err = sg.require(ctx, sourceHasLatestLocally)
   475  		if err != nil {
   476  			return pkgtree.PackageTree{}, err
   477  		}
   478  
   479  		err = sg.suprvsr.do(ctx, label, ctListPackages, func(ctx context.Context) error {
   480  			ptree, err = sg.src.listPackages(ctx, pr, r)
   481  			return err
   482  		})
   483  	}
   484  
   485  	if err != nil {
   486  		return pkgtree.PackageTree{}, err
   487  	}
   488  
   489  	sg.cache.setPackageTree(r, ptree)
   490  	return ptree, nil
   491  }
   492  
   493  // caller must hold sg.mu.
   494  func (sg *sourceGateway) convertToRevision(ctx context.Context, v Version) (Revision, error) {
   495  	// When looking up by Version, there are four states that may have
   496  	// differing opinions about version->revision mappings:
   497  	//
   498  	//   1. The upstream source/repo (canonical)
   499  	//   2. The local source/repo
   500  	//   3. The local cache
   501  	//   4. The input (params to this method)
   502  	//
   503  	// If the input differs from any of the above, it's likely because some lock
   504  	// got written somewhere with a version/rev pair that has since changed or
   505  	// been removed. But correct operation dictates that such a mis-mapping be
   506  	// respected; if the mis-mapping is to be corrected, it has to be done
   507  	// intentionally by the caller, not automatically here.
   508  	r, has := sg.cache.toRevision(v)
   509  	if has {
   510  		return r, nil
   511  	}
   512  
   513  	if sg.srcState&sourceHasLatestVersionList != 0 {
   514  		// We have the latest version list already and didn't get a match, so
   515  		// this is definitely a failure case.
   516  		return "", fmt.Errorf("version %q does not exist in source", v)
   517  	}
   518  
   519  	// The version list is out of date; it's possible this version might
   520  	// show up after loading it.
   521  	err := sg.require(ctx, sourceHasLatestVersionList)
   522  	if err != nil {
   523  		return "", err
   524  	}
   525  
   526  	r, has = sg.cache.toRevision(v)
   527  	if !has {
   528  		return "", fmt.Errorf("version %q does not exist in source", v)
   529  	}
   530  
   531  	return r, nil
   532  }
   533  
   534  func (sg *sourceGateway) listVersions(ctx context.Context) ([]PairedVersion, error) {
   535  	sg.mu.Lock()
   536  	defer sg.mu.Unlock()
   537  
   538  	if pvs, ok := sg.cache.getAllVersions(); ok {
   539  		return pvs, nil
   540  	}
   541  
   542  	err := sg.require(ctx, sourceHasLatestVersionList)
   543  	if err != nil {
   544  		return nil, err
   545  	}
   546  	if pvs, ok := sg.cache.getAllVersions(); ok {
   547  		return pvs, nil
   548  	}
   549  	return nil, nil
   550  }
   551  
   552  func (sg *sourceGateway) revisionPresentIn(ctx context.Context, r Revision) (bool, error) {
   553  	sg.mu.Lock()
   554  	defer sg.mu.Unlock()
   555  
   556  	err := sg.require(ctx, sourceExistsLocally)
   557  	if err != nil {
   558  		return false, err
   559  	}
   560  
   561  	if _, exists := sg.cache.getVersionsFor(r); exists {
   562  		return true, nil
   563  	}
   564  
   565  	present, err := sg.src.revisionPresentIn(r)
   566  	if err == nil && present {
   567  		sg.cache.markRevisionExists(r)
   568  	}
   569  	return present, err
   570  }
   571  
   572  func (sg *sourceGateway) disambiguateRevision(ctx context.Context, r Revision) (Revision, error) {
   573  	sg.mu.Lock()
   574  	defer sg.mu.Unlock()
   575  
   576  	err := sg.require(ctx, sourceExistsLocally)
   577  	if err != nil {
   578  		return "", err
   579  	}
   580  
   581  	return sg.src.disambiguateRevision(ctx, r)
   582  }
   583  
   584  // sourceExistsUpstream verifies that the source exists upstream and that the
   585  // upstreamURL has not changed and returns any additional sourceState, or an error.
   586  func (sg *sourceGateway) sourceExistsUpstream(ctx context.Context) (sourceState, error) {
   587  	if sg.src.existsCallsListVersions() {
   588  		return sg.loadLatestVersionList(ctx)
   589  	}
   590  	err := sg.suprvsr.do(ctx, sg.src.sourceType(), ctSourcePing, func(ctx context.Context) error {
   591  		if !sg.src.existsUpstream(ctx) {
   592  			return errors.Errorf("source does not exist upstream: %s: %s", sg.src.sourceType(), sg.src.upstreamURL())
   593  		}
   594  		return nil
   595  	})
   596  	return 0, err
   597  }
   598  
   599  // initLocal initializes the source locally and returns the resulting sourceState.
   600  func (sg *sourceGateway) initLocal(ctx context.Context) (sourceState, error) {
   601  	if err := sg.suprvsr.do(ctx, sg.src.sourceType(), ctSourceInit, func(ctx context.Context) error {
   602  		err := sg.src.initLocal(ctx)
   603  		return errors.Wrapf(err, "failed to fetch source for %s", sg.src.upstreamURL())
   604  	}); err != nil {
   605  		return 0, err
   606  	}
   607  	return sourceExistsUpstream | sourceExistsLocally | sourceHasLatestLocally, nil
   608  }
   609  
   610  // loadLatestVersionList loads the latest version list, possibly ensuring the source
   611  // exists locally first, and returns the resulting sourceState.
   612  func (sg *sourceGateway) loadLatestVersionList(ctx context.Context) (sourceState, error) {
   613  	var addlState sourceState
   614  	if sg.src.listVersionsRequiresLocal() && !sg.src.existsLocally(ctx) {
   615  		as, err := sg.initLocal(ctx)
   616  		if err != nil {
   617  			return 0, err
   618  		}
   619  		addlState |= as
   620  	}
   621  	var pvl []PairedVersion
   622  	if err := sg.suprvsr.do(ctx, sg.src.sourceType(), ctListVersions, func(ctx context.Context) error {
   623  		var err error
   624  		pvl, err = sg.src.listVersions(ctx)
   625  		return errors.Wrapf(err, "failed to list versions for %s", sg.src.upstreamURL())
   626  	}); err != nil {
   627  		return addlState, err
   628  	}
   629  	sg.cache.setVersionMap(pvl)
   630  	return addlState | sourceHasLatestVersionList, nil
   631  }
   632  
   633  // require ensures the sourceGateway has the wanted sourceState, fetching more
   634  // data if necessary. Returns an error if the state could not be reached.
   635  // caller must hold sg.mu
   636  func (sg *sourceGateway) require(ctx context.Context, wanted sourceState) (err error) {
   637  	todo := (^sg.srcState) & wanted
   638  	var flag sourceState = 1
   639  
   640  	for todo != 0 {
   641  		if todo&flag != 0 {
   642  			// Set up addlState so that individual ops can easily attach
   643  			// more states that were incidentally satisfied by the op.
   644  			var addlState sourceState
   645  
   646  			switch flag {
   647  			case sourceExistsUpstream:
   648  				addlState, err = sg.sourceExistsUpstream(ctx)
   649  			case sourceExistsLocally:
   650  				if !sg.src.existsLocally(ctx) {
   651  					addlState, err = sg.initLocal(ctx)
   652  				}
   653  			case sourceHasLatestVersionList:
   654  				if _, ok := sg.cache.getAllVersions(); !ok {
   655  					addlState, err = sg.loadLatestVersionList(ctx)
   656  				}
   657  			case sourceHasLatestLocally:
   658  				err = sg.suprvsr.do(ctx, sg.src.sourceType(), ctSourceFetch, func(ctx context.Context) error {
   659  					return sg.src.updateLocal(ctx)
   660  				})
   661  				addlState = sourceExistsUpstream | sourceExistsLocally
   662  			}
   663  
   664  			if err != nil {
   665  				return
   666  			}
   667  
   668  			checked := flag | addlState
   669  			sg.srcState |= checked
   670  			todo &= ^checked
   671  		}
   672  
   673  		flag <<= 1
   674  	}
   675  
   676  	return nil
   677  }
   678  
   679  // source is an abstraction around the different underlying types (git, bzr, hg,
   680  // svn, maybe raw on-disk code, and maybe eventually a registry) that can
   681  // provide versioned project source trees.
   682  type source interface {
   683  	existsLocally(context.Context) bool
   684  	existsUpstream(context.Context) bool
   685  	upstreamURL() string
   686  	initLocal(context.Context) error
   687  	updateLocal(context.Context) error
   688  	// maybeClean is a no-op when the underlying source does not support cleaning.
   689  	maybeClean(context.Context) error
   690  	listVersions(context.Context) ([]PairedVersion, error)
   691  	getManifestAndLock(context.Context, ProjectRoot, Revision, ProjectAnalyzer) (Manifest, Lock, error)
   692  	listPackages(context.Context, ProjectRoot, Revision) (pkgtree.PackageTree, error)
   693  	revisionPresentIn(Revision) (bool, error)
   694  	disambiguateRevision(context.Context, Revision) (Revision, error)
   695  	exportRevisionTo(context.Context, Revision, string) error
   696  	sourceType() string
   697  	// existsCallsListVersions returns true if calling existsUpstream actually lists
   698  	// versions underneath, meaning listVersions might as well be used instead.
   699  	existsCallsListVersions() bool
   700  	// listVersionsRequiresLocal returns true if calling listVersions first
   701  	// requires the source to exist locally.
   702  	listVersionsRequiresLocal() bool
   703  }
   704  
   705  type sourceFastPrune interface {
   706  	source
   707  	exportPrunedRevisionTo(context.Context, Revision, []string, PruneOptions, string) error
   708  }