github.com/jhump/golang-x-tools@v0.0.0-20220218190644-4958d6d39439/internal/lsp/cache/workspace.go (about) 1 // Copyright 2020 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package cache 6 7 import ( 8 "context" 9 "fmt" 10 "os" 11 "path/filepath" 12 "sort" 13 "strings" 14 "sync" 15 16 "golang.org/x/mod/modfile" 17 "github.com/jhump/golang-x-tools/internal/event" 18 "github.com/jhump/golang-x-tools/internal/lsp/source" 19 "github.com/jhump/golang-x-tools/internal/span" 20 "github.com/jhump/golang-x-tools/internal/xcontext" 21 errors "golang.org/x/xerrors" 22 ) 23 24 // workspaceSource reports how the set of active modules has been derived. 25 type workspaceSource int 26 27 const ( 28 legacyWorkspace = iota // non-module or single module mode 29 goplsModWorkspace // modules provided by a gopls.mod file 30 goWorkWorkspace // modules provided by a go.work file 31 fileSystemWorkspace // modules scanned from the filesystem 32 ) 33 34 func (s workspaceSource) String() string { 35 switch s { 36 case legacyWorkspace: 37 return "legacy" 38 case goplsModWorkspace: 39 return "gopls.mod" 40 case goWorkWorkspace: 41 return "go.work" 42 case fileSystemWorkspace: 43 return "file system" 44 default: 45 return "!(unknown module source)" 46 } 47 } 48 49 // workspace tracks go.mod files in the workspace, along with the 50 // gopls.mod file, to provide support for multi-module workspaces. 51 // 52 // Specifically, it provides: 53 // - the set of modules contained within in the workspace root considered to 54 // be 'active' 55 // - the workspace modfile, to be used for the go command `-modfile` flag 56 // - the set of workspace directories 57 // 58 // This type is immutable (or rather, idempotent), so that it may be shared 59 // across multiple snapshots. 60 type workspace struct { 61 root span.URI 62 excludePath func(string) bool 63 moduleSource workspaceSource 64 65 // activeModFiles holds the active go.mod files. 66 activeModFiles map[span.URI]struct{} 67 68 // knownModFiles holds the set of all go.mod files in the workspace. 69 // In all modes except for legacy, this is equivalent to modFiles. 70 knownModFiles map[span.URI]struct{} 71 72 // The workspace module is lazily re-built once after being invalidated. 73 // buildMu+built guards this reconstruction. 74 // 75 // file and wsDirs may be non-nil even if built == false, if they were copied 76 // from the previous workspace module version. In this case, they will be 77 // preserved if building fails. 78 buildMu sync.Mutex 79 built bool 80 buildErr error 81 mod *modfile.File 82 sum []byte 83 wsDirs map[span.URI]struct{} 84 } 85 86 // newWorkspace creates a new workspace at the given root directory, 87 // determining its module source based on the presence of a gopls.mod or 88 // go.work file, and the go111moduleOff and useWsModule settings. 89 // 90 // If useWsModule is set, the workspace may use a synthetic mod file replacing 91 // all modules in the root. 92 // 93 // If there is no active workspace file (a gopls.mod or go.work), newWorkspace 94 // scans the filesystem to find modules. 95 func newWorkspace(ctx context.Context, root span.URI, fs source.FileSource, excludePath func(string) bool, go111moduleOff bool, useWsModule bool) (*workspace, error) { 96 ws := &workspace{ 97 root: root, 98 excludePath: excludePath, 99 } 100 101 // The user may have a gopls.mod or go.work file that defines their 102 // workspace. 103 if err := loadExplicitWorkspaceFile(ctx, ws, fs); err == nil { 104 if ws.mod == nil { 105 panic("BUG: explicit workspace file was not parsed") 106 } 107 return ws, nil 108 } 109 110 // Otherwise, in all other modes, search for all of the go.mod files in the 111 // workspace. 112 knownModFiles, err := findModules(root, excludePath, 0) 113 if err != nil { 114 return nil, err 115 } 116 ws.knownModFiles = knownModFiles 117 118 switch { 119 case go111moduleOff: 120 ws.moduleSource = legacyWorkspace 121 case useWsModule: 122 ws.activeModFiles = knownModFiles 123 ws.moduleSource = fileSystemWorkspace 124 default: 125 ws.moduleSource = legacyWorkspace 126 activeModFiles, err := getLegacyModules(ctx, root, fs) 127 if err != nil { 128 return nil, err 129 } 130 ws.activeModFiles = activeModFiles 131 } 132 return ws, nil 133 } 134 135 // loadExplicitWorkspaceFile loads workspace information from go.work or 136 // gopls.mod files, setting the active modules, mod file, and module source 137 // accordingly. 138 func loadExplicitWorkspaceFile(ctx context.Context, ws *workspace, fs source.FileSource) error { 139 for _, src := range []workspaceSource{goWorkWorkspace, goplsModWorkspace} { 140 fh, err := fs.GetFile(ctx, uriForSource(ws.root, src)) 141 if err != nil { 142 return err 143 } 144 contents, err := fh.Read() 145 if err != nil { 146 continue 147 } 148 var file *modfile.File 149 var activeModFiles map[span.URI]struct{} 150 switch src { 151 case goWorkWorkspace: 152 file, activeModFiles, err = parseGoWork(ctx, ws.root, fh.URI(), contents, fs) 153 case goplsModWorkspace: 154 file, activeModFiles, err = parseGoplsMod(ws.root, fh.URI(), contents) 155 } 156 if err != nil { 157 return err 158 } 159 ws.mod = file 160 ws.activeModFiles = activeModFiles 161 ws.moduleSource = src 162 return nil 163 } 164 return noHardcodedWorkspace 165 } 166 167 var noHardcodedWorkspace = errors.New("no hardcoded workspace") 168 169 func (w *workspace) getKnownModFiles() map[span.URI]struct{} { 170 return w.knownModFiles 171 } 172 173 func (w *workspace) getActiveModFiles() map[span.URI]struct{} { 174 return w.activeModFiles 175 } 176 177 // modFile gets the workspace modfile associated with this workspace, 178 // computing it if it doesn't exist. 179 // 180 // A fileSource must be passed in to solve a chicken-egg problem: it is not 181 // correct to pass in the snapshot file source to newWorkspace when 182 // invalidating, because at the time these are called the snapshot is locked. 183 // So we must pass it in later on when actually using the modFile. 184 func (w *workspace) modFile(ctx context.Context, fs source.FileSource) (*modfile.File, error) { 185 w.build(ctx, fs) 186 return w.mod, w.buildErr 187 } 188 189 func (w *workspace) sumFile(ctx context.Context, fs source.FileSource) ([]byte, error) { 190 w.build(ctx, fs) 191 return w.sum, w.buildErr 192 } 193 194 func (w *workspace) build(ctx context.Context, fs source.FileSource) { 195 w.buildMu.Lock() 196 defer w.buildMu.Unlock() 197 198 if w.built { 199 return 200 } 201 // Building should never be cancelled. Since the workspace module is shared 202 // across multiple snapshots, doing so would put us in a bad state, and it 203 // would not be obvious to the user how to recover. 204 ctx = xcontext.Detach(ctx) 205 206 // If our module source is not gopls.mod, try to build the workspace module 207 // from modules. Fall back on the pre-existing mod file if parsing fails. 208 if w.moduleSource != goplsModWorkspace { 209 file, err := buildWorkspaceModFile(ctx, w.activeModFiles, fs) 210 switch { 211 case err == nil: 212 w.mod = file 213 case w.mod != nil: 214 // Parsing failed, but we have a previous file version. 215 event.Error(ctx, "building workspace mod file", err) 216 default: 217 // No file to fall back on. 218 w.buildErr = err 219 } 220 } 221 if w.mod != nil { 222 w.wsDirs = map[span.URI]struct{}{ 223 w.root: {}, 224 } 225 for _, r := range w.mod.Replace { 226 // We may be replacing a module with a different version, not a path 227 // on disk. 228 if r.New.Version != "" { 229 continue 230 } 231 w.wsDirs[span.URIFromPath(r.New.Path)] = struct{}{} 232 } 233 } 234 // Ensure that there is always at least the root dir. 235 if len(w.wsDirs) == 0 { 236 w.wsDirs = map[span.URI]struct{}{ 237 w.root: {}, 238 } 239 } 240 sum, err := buildWorkspaceSumFile(ctx, w.activeModFiles, fs) 241 if err == nil { 242 w.sum = sum 243 } else { 244 event.Error(ctx, "building workspace sum file", err) 245 } 246 w.built = true 247 } 248 249 // dirs returns the workspace directories for the loaded modules. 250 func (w *workspace) dirs(ctx context.Context, fs source.FileSource) []span.URI { 251 w.build(ctx, fs) 252 var dirs []span.URI 253 for d := range w.wsDirs { 254 dirs = append(dirs, d) 255 } 256 sort.Slice(dirs, func(i, j int) bool { 257 return source.CompareURI(dirs[i], dirs[j]) < 0 258 }) 259 return dirs 260 } 261 262 // invalidate returns a (possibly) new workspace after invalidating the changed 263 // files. If w is still valid in the presence of changedURIs, it returns itself 264 // unmodified. 265 // 266 // The returned changed and reload flags control the level of invalidation. 267 // Some workspace changes may affect workspace contents without requiring a 268 // reload of metadata (for example, unsaved changes to a go.mod or go.sum 269 // file). 270 func (w *workspace) invalidate(ctx context.Context, changes map[span.URI]*fileChange, fs source.FileSource) (_ *workspace, changed, reload bool) { 271 // Prevent races to w.modFile or w.wsDirs below, if wmhas not yet been built. 272 w.buildMu.Lock() 273 defer w.buildMu.Unlock() 274 275 // Clone the workspace. This may be discarded if nothing changed. 276 result := &workspace{ 277 root: w.root, 278 moduleSource: w.moduleSource, 279 knownModFiles: make(map[span.URI]struct{}), 280 activeModFiles: make(map[span.URI]struct{}), 281 mod: w.mod, 282 sum: w.sum, 283 wsDirs: w.wsDirs, 284 excludePath: w.excludePath, 285 } 286 for k, v := range w.knownModFiles { 287 result.knownModFiles[k] = v 288 } 289 for k, v := range w.activeModFiles { 290 result.activeModFiles[k] = v 291 } 292 293 // First handle changes to the go.work or gopls.mod file. This must be 294 // considered before any changes to go.mod or go.sum files, as these files 295 // determine which modules we care about. If go.work/gopls.mod has changed 296 // we need to either re-read it if it exists or walk the filesystem if it 297 // has been deleted. go.work should override the gopls.mod if both exist. 298 changed, reload = handleWorkspaceFileChanges(ctx, result, changes, fs) 299 // Next, handle go.mod changes that could affect our workspace. If we're 300 // reading our tracked modules from the gopls.mod, there's nothing to do 301 // here. 302 if result.moduleSource != goplsModWorkspace && result.moduleSource != goWorkWorkspace { 303 for uri, change := range changes { 304 // Otherwise, we only care about go.mod files in the workspace directory. 305 if change.isUnchanged || !isGoMod(uri) || !source.InDir(result.root.Filename(), uri.Filename()) { 306 continue 307 } 308 changed = true 309 active := result.moduleSource != legacyWorkspace || source.CompareURI(modURI(w.root), uri) == 0 310 reload = reload || (active && change.fileHandle.Saved()) 311 if change.exists { 312 result.knownModFiles[uri] = struct{}{} 313 if active { 314 result.activeModFiles[uri] = struct{}{} 315 } 316 } else { 317 delete(result.knownModFiles, uri) 318 delete(result.activeModFiles, uri) 319 } 320 } 321 } 322 323 // Finally, process go.sum changes for any modules that are now active. 324 for uri, change := range changes { 325 if !isGoSum(uri) { 326 continue 327 } 328 // TODO(rFindley) factor out this URI mangling. 329 dir := filepath.Dir(uri.Filename()) 330 modURI := span.URIFromPath(filepath.Join(dir, "go.mod")) 331 if _, active := result.activeModFiles[modURI]; !active { 332 continue 333 } 334 // Only changes to active go.sum files actually cause the workspace to 335 // change. 336 changed = true 337 reload = reload || change.fileHandle.Saved() 338 } 339 340 if !changed { 341 return w, false, false 342 } 343 344 return result, changed, reload 345 } 346 347 // handleWorkspaceFileChanges handles changes related to a go.work or gopls.mod 348 // file, updating ws accordingly. ws.root must be set. 349 func handleWorkspaceFileChanges(ctx context.Context, ws *workspace, changes map[span.URI]*fileChange, fs source.FileSource) (changed, reload bool) { 350 // If go.work/gopls.mod has changed we need to either re-read it if it 351 // exists or walk the filesystem if it has been deleted. 352 // go.work should override the gopls.mod if both exist. 353 for _, src := range []workspaceSource{goWorkWorkspace, goplsModWorkspace} { 354 uri := uriForSource(ws.root, src) 355 // File opens/closes are just no-ops. 356 change, ok := changes[uri] 357 if !ok { 358 continue 359 } 360 if change.isUnchanged { 361 break 362 } 363 if change.exists { 364 // Only invalidate if the file if it actually parses. 365 // Otherwise, stick with the current file. 366 var parsedFile *modfile.File 367 var parsedModules map[span.URI]struct{} 368 var err error 369 switch src { 370 case goWorkWorkspace: 371 parsedFile, parsedModules, err = parseGoWork(ctx, ws.root, uri, change.content, fs) 372 case goplsModWorkspace: 373 parsedFile, parsedModules, err = parseGoplsMod(ws.root, uri, change.content) 374 } 375 if err != nil { 376 // An unparseable file should not invalidate the workspace: 377 // nothing good could come from changing the workspace in 378 // this case. 379 event.Error(ctx, fmt.Sprintf("parsing %s", filepath.Base(uri.Filename())), err) 380 } else { 381 // only update the modfile if it parsed. 382 changed = true 383 reload = change.fileHandle.Saved() 384 ws.mod = parsedFile 385 ws.moduleSource = src 386 ws.knownModFiles = parsedModules 387 ws.activeModFiles = make(map[span.URI]struct{}) 388 for k, v := range parsedModules { 389 ws.activeModFiles[k] = v 390 } 391 } 392 break // We've found an explicit workspace file, so can stop looking. 393 } else { 394 // go.work/gopls.mod is deleted. search for modules again. 395 changed = true 396 reload = true 397 ws.moduleSource = fileSystemWorkspace 398 // The parsed file is no longer valid. 399 ws.mod = nil 400 knownModFiles, err := findModules(ws.root, ws.excludePath, 0) 401 if err != nil { 402 ws.knownModFiles = nil 403 ws.activeModFiles = nil 404 event.Error(ctx, "finding file system modules", err) 405 } else { 406 ws.knownModFiles = knownModFiles 407 ws.activeModFiles = make(map[span.URI]struct{}) 408 for k, v := range ws.knownModFiles { 409 ws.activeModFiles[k] = v 410 } 411 } 412 } 413 } 414 return changed, reload 415 } 416 417 // goplsModURI returns the URI for the gopls.mod file contained in root. 418 func uriForSource(root span.URI, src workspaceSource) span.URI { 419 var basename string 420 switch src { 421 case goplsModWorkspace: 422 basename = "gopls.mod" 423 case goWorkWorkspace: 424 basename = "go.work" 425 default: 426 return "" 427 } 428 return span.URIFromPath(filepath.Join(root.Filename(), basename)) 429 } 430 431 // modURI returns the URI for the go.mod file contained in root. 432 func modURI(root span.URI) span.URI { 433 return span.URIFromPath(filepath.Join(root.Filename(), "go.mod")) 434 } 435 436 // isGoMod reports if uri is a go.mod file. 437 func isGoMod(uri span.URI) bool { 438 return filepath.Base(uri.Filename()) == "go.mod" 439 } 440 441 func isGoSum(uri span.URI) bool { 442 return filepath.Base(uri.Filename()) == "go.sum" || filepath.Base(uri.Filename()) == "go.work.sum" 443 } 444 445 // fileExists reports if the file uri exists within source. 446 func fileExists(ctx context.Context, uri span.URI, source source.FileSource) (bool, error) { 447 fh, err := source.GetFile(ctx, uri) 448 if err != nil { 449 return false, err 450 } 451 return fileHandleExists(fh) 452 } 453 454 // fileHandleExists reports if the file underlying fh actually exits. 455 func fileHandleExists(fh source.FileHandle) (bool, error) { 456 _, err := fh.Read() 457 if err == nil { 458 return true, nil 459 } 460 if os.IsNotExist(err) { 461 return false, nil 462 } 463 return false, err 464 } 465 466 // TODO(rFindley): replace this (and similar) with a uripath package analogous 467 // to filepath. 468 func dirURI(uri span.URI) span.URI { 469 return span.URIFromPath(filepath.Dir(uri.Filename())) 470 } 471 472 // getLegacyModules returns a module set containing at most the root module. 473 func getLegacyModules(ctx context.Context, root span.URI, fs source.FileSource) (map[span.URI]struct{}, error) { 474 uri := span.URIFromPath(filepath.Join(root.Filename(), "go.mod")) 475 modules := make(map[span.URI]struct{}) 476 exists, err := fileExists(ctx, uri, fs) 477 if err != nil { 478 return nil, err 479 } 480 if exists { 481 modules[uri] = struct{}{} 482 } 483 return modules, nil 484 } 485 486 func parseGoWork(ctx context.Context, root, uri span.URI, contents []byte, fs source.FileSource) (*modfile.File, map[span.URI]struct{}, error) { 487 workFile, err := modfile.ParseWork(uri.Filename(), contents, nil) 488 if err != nil { 489 return nil, nil, errors.Errorf("parsing go.work: %w", err) 490 } 491 modFiles := make(map[span.URI]struct{}) 492 for _, dir := range workFile.Use { 493 // The resulting modfile must use absolute paths, so that it can be 494 // written to a temp directory. 495 dir.Path = absolutePath(root, dir.Path) 496 modURI := span.URIFromPath(filepath.Join(dir.Path, "go.mod")) 497 modFiles[modURI] = struct{}{} 498 } 499 modFile, err := buildWorkspaceModFile(ctx, modFiles, fs) 500 if err != nil { 501 return nil, nil, err 502 } 503 if workFile.Go.Version != "" { 504 if err := modFile.AddGoStmt(workFile.Go.Version); err != nil { 505 return nil, nil, err 506 } 507 } 508 509 return modFile, modFiles, nil 510 } 511 512 func parseGoplsMod(root, uri span.URI, contents []byte) (*modfile.File, map[span.URI]struct{}, error) { 513 modFile, err := modfile.Parse(uri.Filename(), contents, nil) 514 if err != nil { 515 return nil, nil, errors.Errorf("parsing gopls.mod: %w", err) 516 } 517 modFiles := make(map[span.URI]struct{}) 518 for _, replace := range modFile.Replace { 519 if replace.New.Version != "" { 520 return nil, nil, errors.Errorf("gopls.mod: replaced module %q@%q must not have version", replace.New.Path, replace.New.Version) 521 } 522 // The resulting modfile must use absolute paths, so that it can be 523 // written to a temp directory. 524 replace.New.Path = absolutePath(root, replace.New.Path) 525 modURI := span.URIFromPath(filepath.Join(replace.New.Path, "go.mod")) 526 modFiles[modURI] = struct{}{} 527 } 528 return modFile, modFiles, nil 529 } 530 531 func absolutePath(root span.URI, path string) string { 532 dirFP := filepath.FromSlash(path) 533 if !filepath.IsAbs(dirFP) { 534 dirFP = filepath.Join(root.Filename(), dirFP) 535 } 536 return dirFP 537 } 538 539 // errExhausted is returned by findModules if the file scan limit is reached. 540 var errExhausted = errors.New("exhausted") 541 542 // Limit go.mod search to 1 million files. As a point of reference, 543 // Kubernetes has 22K files (as of 2020-11-24). 544 const fileLimit = 1000000 545 546 // findModules recursively walks the root directory looking for go.mod files, 547 // returning the set of modules it discovers. If modLimit is non-zero, 548 // searching stops once modLimit modules have been found. 549 // 550 // TODO(rfindley): consider overlays. 551 func findModules(root span.URI, excludePath func(string) bool, modLimit int) (map[span.URI]struct{}, error) { 552 // Walk the view's folder to find all modules in the view. 553 modFiles := make(map[span.URI]struct{}) 554 searched := 0 555 errDone := errors.New("done") 556 err := filepath.Walk(root.Filename(), func(path string, info os.FileInfo, err error) error { 557 if err != nil { 558 // Probably a permission error. Keep looking. 559 return filepath.SkipDir 560 } 561 // For any path that is not the workspace folder, check if the path 562 // would be ignored by the go command. Vendor directories also do not 563 // contain workspace modules. 564 if info.IsDir() && path != root.Filename() { 565 suffix := strings.TrimPrefix(path, root.Filename()) 566 switch { 567 case checkIgnored(suffix), 568 strings.Contains(filepath.ToSlash(suffix), "/vendor/"), 569 excludePath(suffix): 570 return filepath.SkipDir 571 } 572 } 573 // We're only interested in go.mod files. 574 uri := span.URIFromPath(path) 575 if isGoMod(uri) { 576 modFiles[uri] = struct{}{} 577 } 578 if modLimit > 0 && len(modFiles) >= modLimit { 579 return errDone 580 } 581 searched++ 582 if fileLimit > 0 && searched >= fileLimit { 583 return errExhausted 584 } 585 return nil 586 }) 587 if err == errDone { 588 return modFiles, nil 589 } 590 return modFiles, err 591 }