github.com/jhump/golang-x-tools@v0.0.0-20220218190644-4958d6d39439/internal/lsp/cache/workspace.go (about)

     1  // Copyright 2020 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package cache
     6  
     7  import (
     8  	"context"
     9  	"fmt"
    10  	"os"
    11  	"path/filepath"
    12  	"sort"
    13  	"strings"
    14  	"sync"
    15  
    16  	"golang.org/x/mod/modfile"
    17  	"github.com/jhump/golang-x-tools/internal/event"
    18  	"github.com/jhump/golang-x-tools/internal/lsp/source"
    19  	"github.com/jhump/golang-x-tools/internal/span"
    20  	"github.com/jhump/golang-x-tools/internal/xcontext"
    21  	errors "golang.org/x/xerrors"
    22  )
    23  
    24  // workspaceSource reports how the set of active modules has been derived.
    25  type workspaceSource int
    26  
    27  const (
    28  	legacyWorkspace     = iota // non-module or single module mode
    29  	goplsModWorkspace          // modules provided by a gopls.mod file
    30  	goWorkWorkspace            // modules provided by a go.work file
    31  	fileSystemWorkspace        // modules scanned from the filesystem
    32  )
    33  
    34  func (s workspaceSource) String() string {
    35  	switch s {
    36  	case legacyWorkspace:
    37  		return "legacy"
    38  	case goplsModWorkspace:
    39  		return "gopls.mod"
    40  	case goWorkWorkspace:
    41  		return "go.work"
    42  	case fileSystemWorkspace:
    43  		return "file system"
    44  	default:
    45  		return "!(unknown module source)"
    46  	}
    47  }
    48  
    49  // workspace tracks go.mod files in the workspace, along with the
    50  // gopls.mod file, to provide support for multi-module workspaces.
    51  //
    52  // Specifically, it provides:
    53  //  - the set of modules contained within in the workspace root considered to
    54  //    be 'active'
    55  //  - the workspace modfile, to be used for the go command `-modfile` flag
    56  //  - the set of workspace directories
    57  //
    58  // This type is immutable (or rather, idempotent), so that it may be shared
    59  // across multiple snapshots.
    60  type workspace struct {
    61  	root         span.URI
    62  	excludePath  func(string) bool
    63  	moduleSource workspaceSource
    64  
    65  	// activeModFiles holds the active go.mod files.
    66  	activeModFiles map[span.URI]struct{}
    67  
    68  	// knownModFiles holds the set of all go.mod files in the workspace.
    69  	// In all modes except for legacy, this is equivalent to modFiles.
    70  	knownModFiles map[span.URI]struct{}
    71  
    72  	// The workspace module is lazily re-built once after being invalidated.
    73  	// buildMu+built guards this reconstruction.
    74  	//
    75  	// file and wsDirs may be non-nil even if built == false, if they were copied
    76  	// from the previous workspace module version. In this case, they will be
    77  	// preserved if building fails.
    78  	buildMu  sync.Mutex
    79  	built    bool
    80  	buildErr error
    81  	mod      *modfile.File
    82  	sum      []byte
    83  	wsDirs   map[span.URI]struct{}
    84  }
    85  
    86  // newWorkspace creates a new workspace at the given root directory,
    87  // determining its module source based on the presence of a gopls.mod or
    88  // go.work file, and the go111moduleOff and useWsModule settings.
    89  //
    90  // If useWsModule is set, the workspace may use a synthetic mod file replacing
    91  // all modules in the root.
    92  //
    93  // If there is no active workspace file (a gopls.mod or go.work), newWorkspace
    94  // scans the filesystem to find modules.
    95  func newWorkspace(ctx context.Context, root span.URI, fs source.FileSource, excludePath func(string) bool, go111moduleOff bool, useWsModule bool) (*workspace, error) {
    96  	ws := &workspace{
    97  		root:        root,
    98  		excludePath: excludePath,
    99  	}
   100  
   101  	// The user may have a gopls.mod or go.work file that defines their
   102  	// workspace.
   103  	if err := loadExplicitWorkspaceFile(ctx, ws, fs); err == nil {
   104  		if ws.mod == nil {
   105  			panic("BUG: explicit workspace file was not parsed")
   106  		}
   107  		return ws, nil
   108  	}
   109  
   110  	// Otherwise, in all other modes, search for all of the go.mod files in the
   111  	// workspace.
   112  	knownModFiles, err := findModules(root, excludePath, 0)
   113  	if err != nil {
   114  		return nil, err
   115  	}
   116  	ws.knownModFiles = knownModFiles
   117  
   118  	switch {
   119  	case go111moduleOff:
   120  		ws.moduleSource = legacyWorkspace
   121  	case useWsModule:
   122  		ws.activeModFiles = knownModFiles
   123  		ws.moduleSource = fileSystemWorkspace
   124  	default:
   125  		ws.moduleSource = legacyWorkspace
   126  		activeModFiles, err := getLegacyModules(ctx, root, fs)
   127  		if err != nil {
   128  			return nil, err
   129  		}
   130  		ws.activeModFiles = activeModFiles
   131  	}
   132  	return ws, nil
   133  }
   134  
   135  // loadExplicitWorkspaceFile loads workspace information from go.work or
   136  // gopls.mod files, setting the active modules, mod file, and module source
   137  // accordingly.
   138  func loadExplicitWorkspaceFile(ctx context.Context, ws *workspace, fs source.FileSource) error {
   139  	for _, src := range []workspaceSource{goWorkWorkspace, goplsModWorkspace} {
   140  		fh, err := fs.GetFile(ctx, uriForSource(ws.root, src))
   141  		if err != nil {
   142  			return err
   143  		}
   144  		contents, err := fh.Read()
   145  		if err != nil {
   146  			continue
   147  		}
   148  		var file *modfile.File
   149  		var activeModFiles map[span.URI]struct{}
   150  		switch src {
   151  		case goWorkWorkspace:
   152  			file, activeModFiles, err = parseGoWork(ctx, ws.root, fh.URI(), contents, fs)
   153  		case goplsModWorkspace:
   154  			file, activeModFiles, err = parseGoplsMod(ws.root, fh.URI(), contents)
   155  		}
   156  		if err != nil {
   157  			return err
   158  		}
   159  		ws.mod = file
   160  		ws.activeModFiles = activeModFiles
   161  		ws.moduleSource = src
   162  		return nil
   163  	}
   164  	return noHardcodedWorkspace
   165  }
   166  
   167  var noHardcodedWorkspace = errors.New("no hardcoded workspace")
   168  
   169  func (w *workspace) getKnownModFiles() map[span.URI]struct{} {
   170  	return w.knownModFiles
   171  }
   172  
   173  func (w *workspace) getActiveModFiles() map[span.URI]struct{} {
   174  	return w.activeModFiles
   175  }
   176  
   177  // modFile gets the workspace modfile associated with this workspace,
   178  // computing it if it doesn't exist.
   179  //
   180  // A fileSource must be passed in to solve a chicken-egg problem: it is not
   181  // correct to pass in the snapshot file source to newWorkspace when
   182  // invalidating, because at the time these are called the snapshot is locked.
   183  // So we must pass it in later on when actually using the modFile.
   184  func (w *workspace) modFile(ctx context.Context, fs source.FileSource) (*modfile.File, error) {
   185  	w.build(ctx, fs)
   186  	return w.mod, w.buildErr
   187  }
   188  
   189  func (w *workspace) sumFile(ctx context.Context, fs source.FileSource) ([]byte, error) {
   190  	w.build(ctx, fs)
   191  	return w.sum, w.buildErr
   192  }
   193  
   194  func (w *workspace) build(ctx context.Context, fs source.FileSource) {
   195  	w.buildMu.Lock()
   196  	defer w.buildMu.Unlock()
   197  
   198  	if w.built {
   199  		return
   200  	}
   201  	// Building should never be cancelled. Since the workspace module is shared
   202  	// across multiple snapshots, doing so would put us in a bad state, and it
   203  	// would not be obvious to the user how to recover.
   204  	ctx = xcontext.Detach(ctx)
   205  
   206  	// If our module source is not gopls.mod, try to build the workspace module
   207  	// from modules. Fall back on the pre-existing mod file if parsing fails.
   208  	if w.moduleSource != goplsModWorkspace {
   209  		file, err := buildWorkspaceModFile(ctx, w.activeModFiles, fs)
   210  		switch {
   211  		case err == nil:
   212  			w.mod = file
   213  		case w.mod != nil:
   214  			// Parsing failed, but we have a previous file version.
   215  			event.Error(ctx, "building workspace mod file", err)
   216  		default:
   217  			// No file to fall back on.
   218  			w.buildErr = err
   219  		}
   220  	}
   221  	if w.mod != nil {
   222  		w.wsDirs = map[span.URI]struct{}{
   223  			w.root: {},
   224  		}
   225  		for _, r := range w.mod.Replace {
   226  			// We may be replacing a module with a different version, not a path
   227  			// on disk.
   228  			if r.New.Version != "" {
   229  				continue
   230  			}
   231  			w.wsDirs[span.URIFromPath(r.New.Path)] = struct{}{}
   232  		}
   233  	}
   234  	// Ensure that there is always at least the root dir.
   235  	if len(w.wsDirs) == 0 {
   236  		w.wsDirs = map[span.URI]struct{}{
   237  			w.root: {},
   238  		}
   239  	}
   240  	sum, err := buildWorkspaceSumFile(ctx, w.activeModFiles, fs)
   241  	if err == nil {
   242  		w.sum = sum
   243  	} else {
   244  		event.Error(ctx, "building workspace sum file", err)
   245  	}
   246  	w.built = true
   247  }
   248  
   249  // dirs returns the workspace directories for the loaded modules.
   250  func (w *workspace) dirs(ctx context.Context, fs source.FileSource) []span.URI {
   251  	w.build(ctx, fs)
   252  	var dirs []span.URI
   253  	for d := range w.wsDirs {
   254  		dirs = append(dirs, d)
   255  	}
   256  	sort.Slice(dirs, func(i, j int) bool {
   257  		return source.CompareURI(dirs[i], dirs[j]) < 0
   258  	})
   259  	return dirs
   260  }
   261  
   262  // invalidate returns a (possibly) new workspace after invalidating the changed
   263  // files. If w is still valid in the presence of changedURIs, it returns itself
   264  // unmodified.
   265  //
   266  // The returned changed and reload flags control the level of invalidation.
   267  // Some workspace changes may affect workspace contents without requiring a
   268  // reload of metadata (for example, unsaved changes to a go.mod or go.sum
   269  // file).
   270  func (w *workspace) invalidate(ctx context.Context, changes map[span.URI]*fileChange, fs source.FileSource) (_ *workspace, changed, reload bool) {
   271  	// Prevent races to w.modFile or w.wsDirs below, if wmhas not yet been built.
   272  	w.buildMu.Lock()
   273  	defer w.buildMu.Unlock()
   274  
   275  	// Clone the workspace. This may be discarded if nothing changed.
   276  	result := &workspace{
   277  		root:           w.root,
   278  		moduleSource:   w.moduleSource,
   279  		knownModFiles:  make(map[span.URI]struct{}),
   280  		activeModFiles: make(map[span.URI]struct{}),
   281  		mod:            w.mod,
   282  		sum:            w.sum,
   283  		wsDirs:         w.wsDirs,
   284  		excludePath:    w.excludePath,
   285  	}
   286  	for k, v := range w.knownModFiles {
   287  		result.knownModFiles[k] = v
   288  	}
   289  	for k, v := range w.activeModFiles {
   290  		result.activeModFiles[k] = v
   291  	}
   292  
   293  	// First handle changes to the go.work or gopls.mod file. This must be
   294  	// considered before any changes to go.mod or go.sum files, as these files
   295  	// determine which modules we care about. If go.work/gopls.mod has changed
   296  	// we need to either re-read it if it exists or walk the filesystem if it
   297  	// has been deleted. go.work should override the gopls.mod if both exist.
   298  	changed, reload = handleWorkspaceFileChanges(ctx, result, changes, fs)
   299  	// Next, handle go.mod changes that could affect our workspace. If we're
   300  	// reading our tracked modules from the gopls.mod, there's nothing to do
   301  	// here.
   302  	if result.moduleSource != goplsModWorkspace && result.moduleSource != goWorkWorkspace {
   303  		for uri, change := range changes {
   304  			// Otherwise, we only care about go.mod files in the workspace directory.
   305  			if change.isUnchanged || !isGoMod(uri) || !source.InDir(result.root.Filename(), uri.Filename()) {
   306  				continue
   307  			}
   308  			changed = true
   309  			active := result.moduleSource != legacyWorkspace || source.CompareURI(modURI(w.root), uri) == 0
   310  			reload = reload || (active && change.fileHandle.Saved())
   311  			if change.exists {
   312  				result.knownModFiles[uri] = struct{}{}
   313  				if active {
   314  					result.activeModFiles[uri] = struct{}{}
   315  				}
   316  			} else {
   317  				delete(result.knownModFiles, uri)
   318  				delete(result.activeModFiles, uri)
   319  			}
   320  		}
   321  	}
   322  
   323  	// Finally, process go.sum changes for any modules that are now active.
   324  	for uri, change := range changes {
   325  		if !isGoSum(uri) {
   326  			continue
   327  		}
   328  		// TODO(rFindley) factor out this URI mangling.
   329  		dir := filepath.Dir(uri.Filename())
   330  		modURI := span.URIFromPath(filepath.Join(dir, "go.mod"))
   331  		if _, active := result.activeModFiles[modURI]; !active {
   332  			continue
   333  		}
   334  		// Only changes to active go.sum files actually cause the workspace to
   335  		// change.
   336  		changed = true
   337  		reload = reload || change.fileHandle.Saved()
   338  	}
   339  
   340  	if !changed {
   341  		return w, false, false
   342  	}
   343  
   344  	return result, changed, reload
   345  }
   346  
   347  // handleWorkspaceFileChanges handles changes related to a go.work or gopls.mod
   348  // file, updating ws accordingly. ws.root must be set.
   349  func handleWorkspaceFileChanges(ctx context.Context, ws *workspace, changes map[span.URI]*fileChange, fs source.FileSource) (changed, reload bool) {
   350  	// If go.work/gopls.mod has changed we need to either re-read it if it
   351  	// exists or walk the filesystem if it has been deleted.
   352  	// go.work should override the gopls.mod if both exist.
   353  	for _, src := range []workspaceSource{goWorkWorkspace, goplsModWorkspace} {
   354  		uri := uriForSource(ws.root, src)
   355  		// File opens/closes are just no-ops.
   356  		change, ok := changes[uri]
   357  		if !ok {
   358  			continue
   359  		}
   360  		if change.isUnchanged {
   361  			break
   362  		}
   363  		if change.exists {
   364  			// Only invalidate if the file if it actually parses.
   365  			// Otherwise, stick with the current file.
   366  			var parsedFile *modfile.File
   367  			var parsedModules map[span.URI]struct{}
   368  			var err error
   369  			switch src {
   370  			case goWorkWorkspace:
   371  				parsedFile, parsedModules, err = parseGoWork(ctx, ws.root, uri, change.content, fs)
   372  			case goplsModWorkspace:
   373  				parsedFile, parsedModules, err = parseGoplsMod(ws.root, uri, change.content)
   374  			}
   375  			if err != nil {
   376  				// An unparseable file should not invalidate the workspace:
   377  				// nothing good could come from changing the workspace in
   378  				// this case.
   379  				event.Error(ctx, fmt.Sprintf("parsing %s", filepath.Base(uri.Filename())), err)
   380  			} else {
   381  				// only update the modfile if it parsed.
   382  				changed = true
   383  				reload = change.fileHandle.Saved()
   384  				ws.mod = parsedFile
   385  				ws.moduleSource = src
   386  				ws.knownModFiles = parsedModules
   387  				ws.activeModFiles = make(map[span.URI]struct{})
   388  				for k, v := range parsedModules {
   389  					ws.activeModFiles[k] = v
   390  				}
   391  			}
   392  			break // We've found an explicit workspace file, so can stop looking.
   393  		} else {
   394  			// go.work/gopls.mod is deleted. search for modules again.
   395  			changed = true
   396  			reload = true
   397  			ws.moduleSource = fileSystemWorkspace
   398  			// The parsed file is no longer valid.
   399  			ws.mod = nil
   400  			knownModFiles, err := findModules(ws.root, ws.excludePath, 0)
   401  			if err != nil {
   402  				ws.knownModFiles = nil
   403  				ws.activeModFiles = nil
   404  				event.Error(ctx, "finding file system modules", err)
   405  			} else {
   406  				ws.knownModFiles = knownModFiles
   407  				ws.activeModFiles = make(map[span.URI]struct{})
   408  				for k, v := range ws.knownModFiles {
   409  					ws.activeModFiles[k] = v
   410  				}
   411  			}
   412  		}
   413  	}
   414  	return changed, reload
   415  }
   416  
   417  // goplsModURI returns the URI for the gopls.mod file contained in root.
   418  func uriForSource(root span.URI, src workspaceSource) span.URI {
   419  	var basename string
   420  	switch src {
   421  	case goplsModWorkspace:
   422  		basename = "gopls.mod"
   423  	case goWorkWorkspace:
   424  		basename = "go.work"
   425  	default:
   426  		return ""
   427  	}
   428  	return span.URIFromPath(filepath.Join(root.Filename(), basename))
   429  }
   430  
   431  // modURI returns the URI for the go.mod file contained in root.
   432  func modURI(root span.URI) span.URI {
   433  	return span.URIFromPath(filepath.Join(root.Filename(), "go.mod"))
   434  }
   435  
   436  // isGoMod reports if uri is a go.mod file.
   437  func isGoMod(uri span.URI) bool {
   438  	return filepath.Base(uri.Filename()) == "go.mod"
   439  }
   440  
   441  func isGoSum(uri span.URI) bool {
   442  	return filepath.Base(uri.Filename()) == "go.sum" || filepath.Base(uri.Filename()) == "go.work.sum"
   443  }
   444  
   445  // fileExists reports if the file uri exists within source.
   446  func fileExists(ctx context.Context, uri span.URI, source source.FileSource) (bool, error) {
   447  	fh, err := source.GetFile(ctx, uri)
   448  	if err != nil {
   449  		return false, err
   450  	}
   451  	return fileHandleExists(fh)
   452  }
   453  
   454  // fileHandleExists reports if the file underlying fh actually exits.
   455  func fileHandleExists(fh source.FileHandle) (bool, error) {
   456  	_, err := fh.Read()
   457  	if err == nil {
   458  		return true, nil
   459  	}
   460  	if os.IsNotExist(err) {
   461  		return false, nil
   462  	}
   463  	return false, err
   464  }
   465  
   466  // TODO(rFindley): replace this (and similar) with a uripath package analogous
   467  // to filepath.
   468  func dirURI(uri span.URI) span.URI {
   469  	return span.URIFromPath(filepath.Dir(uri.Filename()))
   470  }
   471  
   472  // getLegacyModules returns a module set containing at most the root module.
   473  func getLegacyModules(ctx context.Context, root span.URI, fs source.FileSource) (map[span.URI]struct{}, error) {
   474  	uri := span.URIFromPath(filepath.Join(root.Filename(), "go.mod"))
   475  	modules := make(map[span.URI]struct{})
   476  	exists, err := fileExists(ctx, uri, fs)
   477  	if err != nil {
   478  		return nil, err
   479  	}
   480  	if exists {
   481  		modules[uri] = struct{}{}
   482  	}
   483  	return modules, nil
   484  }
   485  
   486  func parseGoWork(ctx context.Context, root, uri span.URI, contents []byte, fs source.FileSource) (*modfile.File, map[span.URI]struct{}, error) {
   487  	workFile, err := modfile.ParseWork(uri.Filename(), contents, nil)
   488  	if err != nil {
   489  		return nil, nil, errors.Errorf("parsing go.work: %w", err)
   490  	}
   491  	modFiles := make(map[span.URI]struct{})
   492  	for _, dir := range workFile.Use {
   493  		// The resulting modfile must use absolute paths, so that it can be
   494  		// written to a temp directory.
   495  		dir.Path = absolutePath(root, dir.Path)
   496  		modURI := span.URIFromPath(filepath.Join(dir.Path, "go.mod"))
   497  		modFiles[modURI] = struct{}{}
   498  	}
   499  	modFile, err := buildWorkspaceModFile(ctx, modFiles, fs)
   500  	if err != nil {
   501  		return nil, nil, err
   502  	}
   503  	if workFile.Go.Version != "" {
   504  		if err := modFile.AddGoStmt(workFile.Go.Version); err != nil {
   505  			return nil, nil, err
   506  		}
   507  	}
   508  
   509  	return modFile, modFiles, nil
   510  }
   511  
   512  func parseGoplsMod(root, uri span.URI, contents []byte) (*modfile.File, map[span.URI]struct{}, error) {
   513  	modFile, err := modfile.Parse(uri.Filename(), contents, nil)
   514  	if err != nil {
   515  		return nil, nil, errors.Errorf("parsing gopls.mod: %w", err)
   516  	}
   517  	modFiles := make(map[span.URI]struct{})
   518  	for _, replace := range modFile.Replace {
   519  		if replace.New.Version != "" {
   520  			return nil, nil, errors.Errorf("gopls.mod: replaced module %q@%q must not have version", replace.New.Path, replace.New.Version)
   521  		}
   522  		// The resulting modfile must use absolute paths, so that it can be
   523  		// written to a temp directory.
   524  		replace.New.Path = absolutePath(root, replace.New.Path)
   525  		modURI := span.URIFromPath(filepath.Join(replace.New.Path, "go.mod"))
   526  		modFiles[modURI] = struct{}{}
   527  	}
   528  	return modFile, modFiles, nil
   529  }
   530  
   531  func absolutePath(root span.URI, path string) string {
   532  	dirFP := filepath.FromSlash(path)
   533  	if !filepath.IsAbs(dirFP) {
   534  		dirFP = filepath.Join(root.Filename(), dirFP)
   535  	}
   536  	return dirFP
   537  }
   538  
   539  // errExhausted is returned by findModules if the file scan limit is reached.
   540  var errExhausted = errors.New("exhausted")
   541  
   542  // Limit go.mod search to 1 million files. As a point of reference,
   543  // Kubernetes has 22K files (as of 2020-11-24).
   544  const fileLimit = 1000000
   545  
   546  // findModules recursively walks the root directory looking for go.mod files,
   547  // returning the set of modules it discovers. If modLimit is non-zero,
   548  // searching stops once modLimit modules have been found.
   549  //
   550  // TODO(rfindley): consider overlays.
   551  func findModules(root span.URI, excludePath func(string) bool, modLimit int) (map[span.URI]struct{}, error) {
   552  	// Walk the view's folder to find all modules in the view.
   553  	modFiles := make(map[span.URI]struct{})
   554  	searched := 0
   555  	errDone := errors.New("done")
   556  	err := filepath.Walk(root.Filename(), func(path string, info os.FileInfo, err error) error {
   557  		if err != nil {
   558  			// Probably a permission error. Keep looking.
   559  			return filepath.SkipDir
   560  		}
   561  		// For any path that is not the workspace folder, check if the path
   562  		// would be ignored by the go command. Vendor directories also do not
   563  		// contain workspace modules.
   564  		if info.IsDir() && path != root.Filename() {
   565  			suffix := strings.TrimPrefix(path, root.Filename())
   566  			switch {
   567  			case checkIgnored(suffix),
   568  				strings.Contains(filepath.ToSlash(suffix), "/vendor/"),
   569  				excludePath(suffix):
   570  				return filepath.SkipDir
   571  			}
   572  		}
   573  		// We're only interested in go.mod files.
   574  		uri := span.URIFromPath(path)
   575  		if isGoMod(uri) {
   576  			modFiles[uri] = struct{}{}
   577  		}
   578  		if modLimit > 0 && len(modFiles) >= modLimit {
   579  			return errDone
   580  		}
   581  		searched++
   582  		if fileLimit > 0 && searched >= fileLimit {
   583  			return errExhausted
   584  		}
   585  		return nil
   586  	})
   587  	if err == errDone {
   588  		return modFiles, nil
   589  	}
   590  	return modFiles, err
   591  }