github.com/april1989/origin-go-tools@v0.0.32/internal/lsp/cache/snapshot.go (about) 1 // Copyright 2019 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package cache 6 7 import ( 8 "bytes" 9 "context" 10 "fmt" 11 "go/ast" 12 "go/parser" 13 "go/token" 14 "go/types" 15 "io" 16 "os" 17 "path/filepath" 18 "sort" 19 "strconv" 20 "strings" 21 "sync" 22 23 "github.com/april1989/origin-go-tools/go/analysis" 24 "github.com/april1989/origin-go-tools/go/packages" 25 "github.com/april1989/origin-go-tools/internal/event" 26 "github.com/april1989/origin-go-tools/internal/gocommand" 27 "github.com/april1989/origin-go-tools/internal/lsp/debug/tag" 28 "github.com/april1989/origin-go-tools/internal/lsp/source" 29 "github.com/april1989/origin-go-tools/internal/memoize" 30 "github.com/april1989/origin-go-tools/internal/packagesinternal" 31 "github.com/april1989/origin-go-tools/internal/span" 32 "github.com/april1989/origin-go-tools/internal/typesinternal" 33 errors "golang.org/x/xerrors" 34 ) 35 36 type snapshot struct { 37 memoize.Arg // allow as a memoize.Function arg 38 39 id uint64 40 view *View 41 42 // the cache generation that contains the data for this snapshot. 43 generation *memoize.Generation 44 45 // builtin pins the AST and package for builtin.go in memory. 46 builtin *builtinPackageHandle 47 48 // mu guards all of the maps in the snapshot. 49 mu sync.Mutex 50 51 // ids maps file URIs to package IDs. 52 // It may be invalidated on calls to go/packages. 53 ids map[span.URI][]packageID 54 55 // metadata maps file IDs to their associated metadata. 56 // It may invalidated on calls to go/packages. 57 metadata map[packageID]*metadata 58 59 // importedBy maps package IDs to the list of packages that import them. 60 importedBy map[packageID][]packageID 61 62 // files maps file URIs to their corresponding FileHandles. 63 // It may invalidated when a file's content changes. 64 files map[span.URI]source.VersionedFileHandle 65 66 // goFiles maps a parseKey to its parseGoHandle. 67 goFiles map[parseKey]*parseGoHandle 68 69 // packages maps a packageKey to a set of packageHandles to which that file belongs. 70 // It may be invalidated when a file's content changes. 71 packages map[packageKey]*packageHandle 72 73 // actions maps an actionkey to its actionHandle. 74 actions map[actionKey]*actionHandle 75 76 // workspacePackages contains the workspace's packages, which are loaded 77 // when the view is created. 78 workspacePackages map[packageID]packagePath 79 80 // workspaceDirectories are the directories containing workspace packages. 81 // They are the view's root, as well as any replace targets. 82 workspaceDirectories map[span.URI]struct{} 83 84 // unloadableFiles keeps track of files that we've failed to load. 85 unloadableFiles map[span.URI]struct{} 86 87 // parseModHandles keeps track of any ParseModHandles for the snapshot. 88 // The handles need not refer to only the view's go.mod file. 89 parseModHandles map[span.URI]*parseModHandle 90 91 // Preserve go.mod-related handles to avoid garbage-collecting the results 92 // of various calls to the go command. The handles need not refer to only 93 // the view's go.mod file. 94 modTidyHandles map[span.URI]*modTidyHandle 95 modUpgradeHandles map[span.URI]*modUpgradeHandle 96 modWhyHandles map[span.URI]*modWhyHandle 97 } 98 99 type packageKey struct { 100 mode source.ParseMode 101 id packageID 102 } 103 104 type actionKey struct { 105 pkg packageKey 106 analyzer *analysis.Analyzer 107 } 108 109 func (s *snapshot) ID() uint64 { 110 return s.id 111 } 112 113 func (s *snapshot) View() source.View { 114 return s.view 115 } 116 117 func (s *snapshot) FileSet() *token.FileSet { 118 return s.view.session.cache.fset 119 } 120 121 // config returns a *packages.Config with the working directory set to the 122 // view's root. 123 func (s *snapshot) config(ctx context.Context) *packages.Config { 124 return s.configWithDir(ctx, s.view.root.Filename()) 125 } 126 127 // configWithDir returns the configuration used for the snapshot's interaction 128 // with the go/packages API. It uses the given working directory. 129 // TODO(rstambler): go/packages requires that we do not provide overlays for 130 // multiple modules in on config, so buildOverlay needs to filter overlays by 131 // module. 132 func (s *snapshot) configWithDir(ctx context.Context, dir string) *packages.Config { 133 s.view.optionsMu.Lock() 134 env, buildFlags := s.view.envLocked() 135 verboseOutput := s.view.options.VerboseOutput 136 s.view.optionsMu.Unlock() 137 138 cfg := &packages.Config{ 139 Context: ctx, 140 Dir: dir, 141 Env: append([]string{}, env...), 142 BuildFlags: append([]string{}, buildFlags...), 143 Mode: packages.NeedName | 144 packages.NeedFiles | 145 packages.NeedCompiledGoFiles | 146 packages.NeedImports | 147 packages.NeedDeps | 148 packages.NeedTypesSizes | 149 packages.NeedModule, 150 Fset: s.view.session.cache.fset, 151 Overlay: s.buildOverlay(), 152 ParseFile: func(*token.FileSet, string, []byte) (*ast.File, error) { 153 panic("go/packages must not be used to parse files") 154 }, 155 Logf: func(format string, args ...interface{}) { 156 if verboseOutput { 157 event.Log(ctx, fmt.Sprintf(format, args...)) 158 } 159 }, 160 Tests: true, 161 } 162 // We want to type check cgo code if go/types supports it. 163 if typesinternal.SetUsesCgo(&types.Config{}) { 164 cfg.Mode |= packages.LoadMode(packagesinternal.TypecheckCgo) 165 } 166 packagesinternal.SetGoCmdRunner(cfg, s.view.session.gocmdRunner) 167 168 return cfg 169 } 170 171 func (s *snapshot) RunGoCommandDirect(ctx context.Context, verb string, args []string) error { 172 _, runner, inv, cleanup, err := s.goCommandInvocation(ctx, false, verb, args) 173 if err != nil { 174 return err 175 } 176 defer cleanup() 177 178 _, err = runner.Run(ctx, *inv) 179 return err 180 } 181 182 func (s *snapshot) RunGoCommand(ctx context.Context, verb string, args []string) (*bytes.Buffer, error) { 183 _, runner, inv, cleanup, err := s.goCommandInvocation(ctx, true, verb, args) 184 if err != nil { 185 return nil, err 186 } 187 defer cleanup() 188 189 return runner.Run(ctx, *inv) 190 } 191 192 func (s *snapshot) RunGoCommandPiped(ctx context.Context, verb string, args []string, stdout, stderr io.Writer) error { 193 _, runner, inv, cleanup, err := s.goCommandInvocation(ctx, true, verb, args) 194 if err != nil { 195 return err 196 } 197 defer cleanup() 198 return runner.RunPiped(ctx, *inv, stdout, stderr) 199 } 200 201 // Assumes that modURI is only provided when the -modfile flag is enabled. 202 func (s *snapshot) goCommandInvocation(ctx context.Context, allowTempModfile bool, verb string, args []string) (tmpURI span.URI, runner *gocommand.Runner, inv *gocommand.Invocation, cleanup func(), err error) { 203 cleanup = func() {} // fallback 204 cfg := s.config(ctx) 205 if allowTempModfile && s.view.tmpMod { 206 modFH, err := s.GetFile(ctx, s.view.modURI) 207 if err != nil { 208 return "", nil, nil, cleanup, err 209 } 210 // Use the go.sum if it happens to be available. 211 sumFH, _ := s.sumFH(ctx, modFH) 212 213 tmpURI, cleanup, err = tempModFile(modFH, sumFH) 214 if err != nil { 215 return "", nil, nil, cleanup, err 216 } 217 cfg.BuildFlags = append(cfg.BuildFlags, fmt.Sprintf("-modfile=%s", tmpURI.Filename())) 218 } 219 runner = packagesinternal.GetGoCmdRunner(cfg) 220 return tmpURI, runner, &gocommand.Invocation{ 221 Verb: verb, 222 Args: args, 223 Env: cfg.Env, 224 BuildFlags: cfg.BuildFlags, 225 WorkingDir: cfg.Dir, 226 }, cleanup, nil 227 } 228 229 func (s *snapshot) buildOverlay() map[string][]byte { 230 s.mu.Lock() 231 defer s.mu.Unlock() 232 233 overlays := make(map[string][]byte) 234 for uri, fh := range s.files { 235 overlay, ok := fh.(*overlay) 236 if !ok { 237 continue 238 } 239 if overlay.saved { 240 continue 241 } 242 // TODO(rstambler): Make sure not to send overlays outside of the current view. 243 overlays[uri.Filename()] = overlay.text 244 } 245 return overlays 246 } 247 248 func hashUnsavedOverlays(files map[span.URI]source.VersionedFileHandle) string { 249 var unsaved []string 250 for uri, fh := range files { 251 if overlay, ok := fh.(*overlay); ok && !overlay.saved { 252 unsaved = append(unsaved, uri.Filename()) 253 } 254 } 255 sort.Strings(unsaved) 256 return hashContents([]byte(strings.Join(unsaved, ""))) 257 } 258 259 func (s *snapshot) PackagesForFile(ctx context.Context, uri span.URI, mode source.TypecheckMode) ([]source.Package, error) { 260 ctx = event.Label(ctx, tag.URI.Of(uri)) 261 262 // Check if we should reload metadata for the file. We don't invalidate IDs 263 // (though we should), so the IDs will be a better source of truth than the 264 // metadata. If there are no IDs for the file, then we should also reload. 265 ids := s.getIDsForURI(uri) 266 reload := len(ids) == 0 267 for _, id := range ids { 268 // Reload package metadata if any of the metadata has missing 269 // dependencies, in case something has changed since the last time we 270 // reloaded it. 271 if m := s.getMetadata(id); m == nil { 272 reload = true 273 break 274 } 275 // TODO(golang/go#36918): Previously, we would reload any package with 276 // missing dependencies. This is expensive and results in too many 277 // calls to packages.Load. Determine what we should do instead. 278 } 279 if reload { 280 if err := s.load(ctx, fileURI(uri)); err != nil { 281 return nil, err 282 } 283 } 284 // Get the list of IDs from the snapshot again, in case it has changed. 285 var pkgs []source.Package 286 for _, id := range s.getIDsForURI(uri) { 287 var parseModes []source.ParseMode 288 switch mode { 289 case source.TypecheckAll: 290 if s.workspaceParseMode(id) == source.ParseFull { 291 parseModes = []source.ParseMode{source.ParseFull} 292 } else { 293 parseModes = []source.ParseMode{source.ParseExported, source.ParseFull} 294 } 295 case source.TypecheckFull: 296 parseModes = []source.ParseMode{source.ParseFull} 297 case source.TypecheckWorkspace: 298 parseModes = []source.ParseMode{s.workspaceParseMode(id)} 299 } 300 301 for _, parseMode := range parseModes { 302 pkg, err := s.checkedPackage(ctx, id, parseMode) 303 if err != nil { 304 return nil, err 305 } 306 pkgs = append(pkgs, pkg) 307 } 308 } 309 return pkgs, nil 310 } 311 312 func (s *snapshot) checkedPackage(ctx context.Context, id packageID, mode source.ParseMode) (*pkg, error) { 313 ph, err := s.buildPackageHandle(ctx, id, mode) 314 if err != nil { 315 return nil, err 316 } 317 return ph.check(ctx, s) 318 } 319 320 func (s *snapshot) GetReverseDependencies(ctx context.Context, id string) ([]source.Package, error) { 321 if err := s.awaitLoaded(ctx); err != nil { 322 return nil, err 323 } 324 ids := make(map[packageID]struct{}) 325 s.transitiveReverseDependencies(packageID(id), ids) 326 327 // Make sure to delete the original package ID from the map. 328 delete(ids, packageID(id)) 329 330 var pkgs []source.Package 331 for id := range ids { 332 pkg, err := s.checkedPackage(ctx, id, s.workspaceParseMode(id)) 333 if err != nil { 334 return nil, err 335 } 336 pkgs = append(pkgs, pkg) 337 } 338 return pkgs, nil 339 } 340 341 // transitiveReverseDependencies populates the uris map with file URIs 342 // belonging to the provided package and its transitive reverse dependencies. 343 func (s *snapshot) transitiveReverseDependencies(id packageID, ids map[packageID]struct{}) { 344 if _, ok := ids[id]; ok { 345 return 346 } 347 if s.getMetadata(id) == nil { 348 return 349 } 350 ids[id] = struct{}{} 351 importedBy := s.getImportedBy(id) 352 for _, parentID := range importedBy { 353 s.transitiveReverseDependencies(parentID, ids) 354 } 355 } 356 357 func (s *snapshot) getGoFile(key parseKey) *parseGoHandle { 358 s.mu.Lock() 359 defer s.mu.Unlock() 360 return s.goFiles[key] 361 } 362 363 func (s *snapshot) addGoFile(key parseKey, pgh *parseGoHandle) *parseGoHandle { 364 s.mu.Lock() 365 defer s.mu.Unlock() 366 if existing, ok := s.goFiles[key]; ok { 367 return existing 368 } 369 s.goFiles[key] = pgh 370 return pgh 371 } 372 373 func (s *snapshot) getParseModHandle(uri span.URI) *parseModHandle { 374 s.mu.Lock() 375 defer s.mu.Unlock() 376 return s.parseModHandles[uri] 377 } 378 379 func (s *snapshot) getModWhyHandle(uri span.URI) *modWhyHandle { 380 s.mu.Lock() 381 defer s.mu.Unlock() 382 return s.modWhyHandles[uri] 383 } 384 385 func (s *snapshot) getModUpgradeHandle(uri span.URI) *modUpgradeHandle { 386 s.mu.Lock() 387 defer s.mu.Unlock() 388 return s.modUpgradeHandles[uri] 389 } 390 391 func (s *snapshot) getModTidyHandle(uri span.URI) *modTidyHandle { 392 s.mu.Lock() 393 defer s.mu.Unlock() 394 return s.modTidyHandles[uri] 395 } 396 397 func (s *snapshot) getImportedBy(id packageID) []packageID { 398 s.mu.Lock() 399 defer s.mu.Unlock() 400 return s.getImportedByLocked(id) 401 } 402 403 func (s *snapshot) getImportedByLocked(id packageID) []packageID { 404 // If we haven't rebuilt the import graph since creating the snapshot. 405 if len(s.importedBy) == 0 { 406 s.rebuildImportGraph() 407 } 408 return s.importedBy[id] 409 } 410 411 func (s *snapshot) clearAndRebuildImportGraph() { 412 s.mu.Lock() 413 defer s.mu.Unlock() 414 415 // Completely invalidate the original map. 416 s.importedBy = make(map[packageID][]packageID) 417 s.rebuildImportGraph() 418 } 419 420 func (s *snapshot) rebuildImportGraph() { 421 for id, m := range s.metadata { 422 for _, importID := range m.deps { 423 s.importedBy[importID] = append(s.importedBy[importID], id) 424 } 425 } 426 } 427 428 func (s *snapshot) addPackageHandle(ph *packageHandle) *packageHandle { 429 s.mu.Lock() 430 defer s.mu.Unlock() 431 432 // If the package handle has already been cached, 433 // return the cached handle instead of overriding it. 434 if ph, ok := s.packages[ph.packageKey()]; ok { 435 return ph 436 } 437 s.packages[ph.packageKey()] = ph 438 return ph 439 } 440 441 func (s *snapshot) workspacePackageIDs() (ids []packageID) { 442 s.mu.Lock() 443 defer s.mu.Unlock() 444 445 for id := range s.workspacePackages { 446 ids = append(ids, id) 447 } 448 return ids 449 } 450 451 func (s *snapshot) WorkspaceDirectories(ctx context.Context) []span.URI { 452 s.mu.Lock() 453 defer s.mu.Unlock() 454 455 var dirs []span.URI 456 for d := range s.workspaceDirectories { 457 dirs = append(dirs, d) 458 } 459 return dirs 460 } 461 462 func (s *snapshot) WorkspacePackages(ctx context.Context) ([]source.Package, error) { 463 if err := s.awaitLoaded(ctx); err != nil { 464 return nil, err 465 } 466 var pkgs []source.Package 467 for _, pkgID := range s.workspacePackageIDs() { 468 pkg, err := s.checkedPackage(ctx, pkgID, s.workspaceParseMode(pkgID)) 469 if err != nil { 470 return nil, err 471 } 472 pkgs = append(pkgs, pkg) 473 } 474 return pkgs, nil 475 } 476 477 func (s *snapshot) KnownPackages(ctx context.Context) ([]source.Package, error) { 478 if err := s.awaitLoaded(ctx); err != nil { 479 return nil, err 480 } 481 482 // The WorkspaceSymbols implementation relies on this function returning 483 // workspace packages first. 484 ids := s.workspacePackageIDs() 485 s.mu.Lock() 486 for id := range s.metadata { 487 if _, ok := s.workspacePackages[id]; ok { 488 continue 489 } 490 ids = append(ids, id) 491 } 492 s.mu.Unlock() 493 494 var pkgs []source.Package 495 for _, id := range ids { 496 pkg, err := s.checkedPackage(ctx, id, s.workspaceParseMode(id)) 497 if err != nil { 498 return nil, err 499 } 500 pkgs = append(pkgs, pkg) 501 } 502 return pkgs, nil 503 } 504 505 func (s *snapshot) CachedImportPaths(ctx context.Context) (map[string]source.Package, error) { 506 // Don't reload workspace package metadata. 507 // This function is meant to only return currently cached information. 508 s.view.AwaitInitialized(ctx) 509 510 s.mu.Lock() 511 defer s.mu.Unlock() 512 513 results := map[string]source.Package{} 514 for _, ph := range s.packages { 515 cachedPkg, err := ph.cached(s.generation) 516 if err != nil { 517 continue 518 } 519 for importPath, newPkg := range cachedPkg.imports { 520 if oldPkg, ok := results[string(importPath)]; ok { 521 // Using the same trick as NarrowestPackage, prefer non-variants. 522 if len(newPkg.compiledGoFiles) < len(oldPkg.(*pkg).compiledGoFiles) { 523 results[string(importPath)] = newPkg 524 } 525 } else { 526 results[string(importPath)] = newPkg 527 } 528 } 529 } 530 return results, nil 531 } 532 533 func (s *snapshot) getPackage(id packageID, mode source.ParseMode) *packageHandle { 534 s.mu.Lock() 535 defer s.mu.Unlock() 536 537 key := packageKey{ 538 id: id, 539 mode: mode, 540 } 541 return s.packages[key] 542 } 543 544 func (s *snapshot) getActionHandle(id packageID, m source.ParseMode, a *analysis.Analyzer) *actionHandle { 545 s.mu.Lock() 546 defer s.mu.Unlock() 547 548 key := actionKey{ 549 pkg: packageKey{ 550 id: id, 551 mode: m, 552 }, 553 analyzer: a, 554 } 555 return s.actions[key] 556 } 557 558 func (s *snapshot) addActionHandle(ah *actionHandle) *actionHandle { 559 s.mu.Lock() 560 defer s.mu.Unlock() 561 562 key := actionKey{ 563 analyzer: ah.analyzer, 564 pkg: packageKey{ 565 id: ah.pkg.m.id, 566 mode: ah.pkg.mode, 567 }, 568 } 569 if ah, ok := s.actions[key]; ok { 570 return ah 571 } 572 s.actions[key] = ah 573 return ah 574 } 575 576 func (s *snapshot) getIDsForURI(uri span.URI) []packageID { 577 s.mu.Lock() 578 defer s.mu.Unlock() 579 580 return s.ids[uri] 581 } 582 583 func (s *snapshot) getMetadataForURILocked(uri span.URI) (metadata []*metadata) { 584 // TODO(matloob): uri can be a file or directory. Should we update the mappings 585 // to map directories to their contained packages? 586 587 for _, id := range s.ids[uri] { 588 if m, ok := s.metadata[id]; ok { 589 metadata = append(metadata, m) 590 } 591 } 592 return metadata 593 } 594 595 func (s *snapshot) getMetadata(id packageID) *metadata { 596 s.mu.Lock() 597 defer s.mu.Unlock() 598 599 return s.metadata[id] 600 } 601 602 func (s *snapshot) addID(uri span.URI, id packageID) { 603 s.mu.Lock() 604 defer s.mu.Unlock() 605 606 for i, existingID := range s.ids[uri] { 607 // TODO: We should make sure not to set duplicate IDs, 608 // and instead panic here. This can be done by making sure not to 609 // reset metadata information for packages we've already seen. 610 if existingID == id { 611 return 612 } 613 // If we are setting a real ID, when the package had only previously 614 // had a command-line-arguments ID, we should just replace it. 615 if existingID == "command-line-arguments" { 616 s.ids[uri][i] = id 617 // Delete command-line-arguments if it was a workspace package. 618 delete(s.workspacePackages, existingID) 619 return 620 } 621 } 622 s.ids[uri] = append(s.ids[uri], id) 623 } 624 625 func (s *snapshot) isWorkspacePackage(id packageID) (packagePath, bool) { 626 s.mu.Lock() 627 defer s.mu.Unlock() 628 629 scope, ok := s.workspacePackages[id] 630 return scope, ok 631 } 632 633 func (s *snapshot) FindFile(uri span.URI) source.VersionedFileHandle { 634 f, err := s.view.getFile(uri) 635 if err != nil { 636 return nil 637 } 638 639 s.mu.Lock() 640 defer s.mu.Unlock() 641 642 return s.files[f.URI()] 643 } 644 645 // GetFile returns a File for the given URI. It will always succeed because it 646 // adds the file to the managed set if needed. 647 func (s *snapshot) GetFile(ctx context.Context, uri span.URI) (source.VersionedFileHandle, error) { 648 f, err := s.view.getFile(uri) 649 if err != nil { 650 return nil, err 651 } 652 653 s.mu.Lock() 654 defer s.mu.Unlock() 655 656 if fh, ok := s.files[f.URI()]; ok { 657 return fh, nil 658 } 659 660 fh, err := s.view.session.cache.getFile(ctx, uri) 661 if err != nil { 662 return nil, err 663 } 664 closed := &closedFile{fh} 665 s.files[f.URI()] = closed 666 return closed, nil 667 } 668 669 func (s *snapshot) IsOpen(uri span.URI) bool { 670 s.mu.Lock() 671 defer s.mu.Unlock() 672 673 _, open := s.files[uri].(*overlay) 674 return open 675 } 676 677 func (s *snapshot) IsSaved(uri span.URI) bool { 678 s.mu.Lock() 679 defer s.mu.Unlock() 680 681 ovl, open := s.files[uri].(*overlay) 682 return !open || ovl.saved 683 } 684 685 func (s *snapshot) awaitLoaded(ctx context.Context) error { 686 // Do not return results until the snapshot's view has been initialized. 687 s.view.AwaitInitialized(ctx) 688 689 if err := s.reloadWorkspace(ctx); err != nil { 690 return err 691 } 692 if err := s.reloadOrphanedFiles(ctx); err != nil { 693 return err 694 } 695 // If we still have absolutely no metadata, check if the view failed to 696 // initialize and return any errors. 697 // TODO(rstambler): Should we clear the error after we return it? 698 s.mu.Lock() 699 defer s.mu.Unlock() 700 if len(s.metadata) == 0 { 701 return s.view.initializedErr 702 } 703 return nil 704 } 705 706 // reloadWorkspace reloads the metadata for all invalidated workspace packages. 707 func (s *snapshot) reloadWorkspace(ctx context.Context) error { 708 // If the view's build configuration is invalid, we cannot reload by package path. 709 // Just reload the directory instead. 710 if !s.view.hasValidBuildConfiguration { 711 return s.load(ctx, viewLoadScope("LOAD_INVALID_VIEW")) 712 } 713 714 // See which of the workspace packages are missing metadata. 715 s.mu.Lock() 716 pkgPathSet := map[packagePath]struct{}{} 717 for id, pkgPath := range s.workspacePackages { 718 // Don't try to reload "command-line-arguments" directly. 719 if pkgPath == "command-line-arguments" { 720 continue 721 } 722 if s.metadata[id] == nil { 723 pkgPathSet[pkgPath] = struct{}{} 724 } 725 } 726 s.mu.Unlock() 727 728 if len(pkgPathSet) == 0 { 729 return nil 730 } 731 var pkgPaths []interface{} 732 for pkgPath := range pkgPathSet { 733 pkgPaths = append(pkgPaths, pkgPath) 734 } 735 return s.load(ctx, pkgPaths...) 736 } 737 738 func (s *snapshot) reloadOrphanedFiles(ctx context.Context) error { 739 // When we load ./... or a package path directly, we may not get packages 740 // that exist only in overlays. As a workaround, we search all of the files 741 // available in the snapshot and reload their metadata individually using a 742 // file= query if the metadata is unavailable. 743 scopes := s.orphanedFileScopes() 744 if len(scopes) == 0 { 745 return nil 746 } 747 748 err := s.load(ctx, scopes...) 749 750 // If we failed to load some files, i.e. they have no metadata, 751 // mark the failures so we don't bother retrying until the file's 752 // content changes. 753 // 754 // TODO(rstambler): This may be an overestimate if the load stopped 755 // early for an unrelated errors. Add a fallback? 756 // 757 // Check for context cancellation so that we don't incorrectly mark files 758 // as unloadable, but don't return before setting all workspace packages. 759 if ctx.Err() == nil && err != nil { 760 event.Error(ctx, "reloadOrphanedFiles: failed to load", err, tag.Query.Of(scopes)) 761 s.mu.Lock() 762 for _, scope := range scopes { 763 uri := span.URI(scope.(fileURI)) 764 if s.getMetadataForURILocked(uri) == nil { 765 s.unloadableFiles[uri] = struct{}{} 766 } 767 } 768 s.mu.Unlock() 769 } 770 return nil 771 } 772 773 func (s *snapshot) orphanedFileScopes() []interface{} { 774 s.mu.Lock() 775 defer s.mu.Unlock() 776 777 scopeSet := make(map[span.URI]struct{}) 778 for uri, fh := range s.files { 779 // Don't try to reload metadata for go.mod files. 780 if fh.Kind() != source.Go { 781 continue 782 } 783 // If the URI doesn't belong to this view, then it's not in a workspace 784 // package and should not be reloaded directly. 785 if !contains(s.view.session.viewsOf(uri), s.view) { 786 continue 787 } 788 // Don't reload metadata for files we've already deemed unloadable. 789 if _, ok := s.unloadableFiles[uri]; ok { 790 continue 791 } 792 if s.getMetadataForURILocked(uri) == nil { 793 scopeSet[uri] = struct{}{} 794 } 795 } 796 var scopes []interface{} 797 for uri := range scopeSet { 798 scopes = append(scopes, fileURI(uri)) 799 } 800 return scopes 801 } 802 803 func contains(views []*View, view *View) bool { 804 for _, v := range views { 805 if v == view { 806 return true 807 } 808 } 809 return false 810 } 811 812 func generationName(v *View, snapshotID uint64) string { 813 return fmt.Sprintf("v%v/%v", v.id, snapshotID) 814 } 815 816 func (s *snapshot) clone(ctx context.Context, withoutURIs map[span.URI]source.VersionedFileHandle, forceReloadMetadata bool) *snapshot { 817 s.mu.Lock() 818 defer s.mu.Unlock() 819 820 newGen := s.view.session.cache.store.Generation(generationName(s.view, s.id+1)) 821 result := &snapshot{ 822 id: s.id + 1, 823 generation: newGen, 824 view: s.view, 825 builtin: s.builtin, 826 ids: make(map[span.URI][]packageID), 827 importedBy: make(map[packageID][]packageID), 828 metadata: make(map[packageID]*metadata), 829 packages: make(map[packageKey]*packageHandle), 830 actions: make(map[actionKey]*actionHandle), 831 files: make(map[span.URI]source.VersionedFileHandle), 832 goFiles: make(map[parseKey]*parseGoHandle), 833 workspaceDirectories: make(map[span.URI]struct{}), 834 workspacePackages: make(map[packageID]packagePath), 835 unloadableFiles: make(map[span.URI]struct{}), 836 parseModHandles: make(map[span.URI]*parseModHandle), 837 modTidyHandles: make(map[span.URI]*modTidyHandle), 838 modUpgradeHandles: make(map[span.URI]*modUpgradeHandle), 839 modWhyHandles: make(map[span.URI]*modWhyHandle), 840 } 841 842 if s.builtin != nil { 843 newGen.Inherit(s.builtin.handle) 844 } 845 846 // Copy all of the FileHandles. 847 for k, v := range s.files { 848 result.files[k] = v 849 } 850 // Copy the set of unloadable files. 851 for k, v := range s.unloadableFiles { 852 result.unloadableFiles[k] = v 853 } 854 // Copy all of the modHandles. 855 for k, v := range s.parseModHandles { 856 newGen.Inherit(v.handle) 857 result.parseModHandles[k] = v 858 } 859 // Copy all of the workspace directories. They may be reset later. 860 for k, v := range s.workspaceDirectories { 861 result.workspaceDirectories[k] = v 862 } 863 864 for k, v := range s.goFiles { 865 if _, ok := withoutURIs[k.file.URI]; ok { 866 continue 867 } 868 newGen.Inherit(v.handle) 869 newGen.Inherit(v.astCacheHandle) 870 result.goFiles[k] = v 871 } 872 873 // Copy all of the go.mod-related handles. They may be invalidated later, 874 // so we inherit them at the end of the function. 875 for k, v := range s.modTidyHandles { 876 if _, ok := withoutURIs[k]; ok { 877 continue 878 } 879 result.modTidyHandles[k] = v 880 } 881 for k, v := range s.modUpgradeHandles { 882 if _, ok := withoutURIs[k]; ok { 883 continue 884 } 885 result.modUpgradeHandles[k] = v 886 } 887 for k, v := range s.modWhyHandles { 888 if _, ok := withoutURIs[k]; ok { 889 continue 890 } 891 result.modWhyHandles[k] = v 892 } 893 894 // transitiveIDs keeps track of transitive reverse dependencies. 895 // If an ID is present in the map, invalidate its types. 896 // If an ID's value is true, invalidate its metadata too. 897 transitiveIDs := make(map[packageID]bool) 898 for withoutURI, currentFH := range withoutURIs { 899 directIDs := map[packageID]struct{}{} 900 901 // Collect all of the package IDs that correspond to the given file. 902 // TODO: if the file has moved into a new package, we should invalidate that too. 903 for _, id := range s.ids[withoutURI] { 904 directIDs[id] = struct{}{} 905 } 906 // The original FileHandle for this URI is cached on the snapshot. 907 originalFH := s.files[withoutURI] 908 909 // Check if the file's package name or imports have changed, 910 // and if so, invalidate this file's packages' metadata. 911 invalidateMetadata := forceReloadMetadata || s.shouldInvalidateMetadata(ctx, originalFH, currentFH) 912 913 // Invalidate the previous modTidyHandle if any of the files have been 914 // saved or if any of the metadata has been invalidated. 915 if invalidateMetadata || fileWasSaved(originalFH, currentFH) { 916 // TODO(rstambler): Only delete mod handles for which the 917 // withoutURI is relevant. 918 for k := range s.modTidyHandles { 919 delete(result.modTidyHandles, k) 920 } 921 for k := range s.modUpgradeHandles { 922 delete(result.modUpgradeHandles, k) 923 } 924 for k := range s.modWhyHandles { 925 delete(result.modWhyHandles, k) 926 } 927 } 928 if currentFH.Kind() == source.Mod { 929 // If the view's go.mod file's contents have changed, invalidate the 930 // metadata for every known package in the snapshot. 931 if invalidateMetadata { 932 for k := range s.packages { 933 directIDs[k.id] = struct{}{} 934 } 935 } 936 937 delete(result.parseModHandles, withoutURI) 938 939 if currentFH.URI() == s.view.modURI { 940 // The go.mod's replace directives may have changed. We may 941 // need to update our set of workspace directories. Use the new 942 // snapshot, as it can be locked without causing issues. 943 result.workspaceDirectories = result.findWorkspaceDirectories(ctx, currentFH) 944 } 945 } 946 947 // If this is a file we don't yet know about, 948 // then we do not yet know what packages it should belong to. 949 // Make a rough estimate of what metadata to invalidate by finding the package IDs 950 // of all of the files in the same directory as this one. 951 // TODO(rstambler): Speed this up by mapping directories to filenames. 952 if len(directIDs) == 0 { 953 if dirStat, err := os.Stat(filepath.Dir(withoutURI.Filename())); err == nil { 954 for uri := range s.files { 955 if fdirStat, err := os.Stat(filepath.Dir(uri.Filename())); err == nil { 956 if os.SameFile(dirStat, fdirStat) { 957 for _, id := range s.ids[uri] { 958 directIDs[id] = struct{}{} 959 } 960 } 961 } 962 } 963 } 964 } 965 966 // Invalidate reverse dependencies too. 967 // TODO(heschi): figure out the locking model and use transitiveReverseDeps? 968 var addRevDeps func(packageID) 969 addRevDeps = func(id packageID) { 970 current, seen := transitiveIDs[id] 971 newInvalidateMetadata := current || invalidateMetadata 972 973 // If we've already seen this ID, and the value of invalidate 974 // metadata has not changed, we can return early. 975 if seen && current == newInvalidateMetadata { 976 return 977 } 978 transitiveIDs[id] = newInvalidateMetadata 979 for _, rid := range s.getImportedByLocked(id) { 980 addRevDeps(rid) 981 } 982 } 983 for id := range directIDs { 984 addRevDeps(id) 985 } 986 987 // Handle the invalidated file; it may have new contents or not exist. 988 if _, err := currentFH.Read(); os.IsNotExist(err) { 989 delete(result.files, withoutURI) 990 } else { 991 result.files[withoutURI] = currentFH 992 } 993 // Make sure to remove the changed file from the unloadable set. 994 delete(result.unloadableFiles, withoutURI) 995 } 996 // Copy the package type information. 997 for k, v := range s.packages { 998 if _, ok := transitiveIDs[k.id]; ok { 999 continue 1000 } 1001 newGen.Inherit(v.handle) 1002 result.packages[k] = v 1003 } 1004 // Copy the package analysis information. 1005 for k, v := range s.actions { 1006 if _, ok := transitiveIDs[k.pkg.id]; ok { 1007 continue 1008 } 1009 newGen.Inherit(v.handle) 1010 result.actions[k] = v 1011 } 1012 // Copy the package metadata. We only need to invalidate packages directly 1013 // containing the affected file, and only if it changed in a relevant way. 1014 for k, v := range s.metadata { 1015 if invalidateMetadata, ok := transitiveIDs[k]; invalidateMetadata && ok { 1016 continue 1017 } 1018 result.metadata[k] = v 1019 } 1020 // Copy the URI to package ID mappings, skipping only those URIs whose 1021 // metadata will be reloaded in future calls to load. 1022 copyIDs: 1023 for k, ids := range s.ids { 1024 for _, id := range ids { 1025 if invalidateMetadata, ok := transitiveIDs[id]; invalidateMetadata && ok { 1026 continue copyIDs 1027 } 1028 } 1029 result.ids[k] = ids 1030 } 1031 // Copy the set of initally loaded packages. 1032 for id, pkgPath := range s.workspacePackages { 1033 if id == "command-line-arguments" { 1034 if invalidateMetadata, ok := transitiveIDs[id]; invalidateMetadata && ok { 1035 continue 1036 } 1037 } 1038 1039 // If all the files we know about in a package have been deleted, 1040 // the package is gone and we should no longer try to load it. 1041 if m := s.metadata[id]; m != nil { 1042 hasFiles := false 1043 for _, uri := range s.metadata[id].goFiles { 1044 if _, ok := result.files[uri]; ok { 1045 hasFiles = true 1046 break 1047 } 1048 } 1049 if !hasFiles { 1050 continue 1051 } 1052 } 1053 1054 result.workspacePackages[id] = pkgPath 1055 } 1056 1057 // Inherit all of the go.mod-related handles. 1058 for _, v := range s.modTidyHandles { 1059 newGen.Inherit(v.handle) 1060 } 1061 for _, v := range s.modUpgradeHandles { 1062 newGen.Inherit(v.handle) 1063 } 1064 for _, v := range s.modWhyHandles { 1065 newGen.Inherit(v.handle) 1066 } 1067 1068 // Don't bother copying the importedBy graph, 1069 // as it changes each time we update metadata. 1070 return result 1071 } 1072 1073 // fileWasSaved reports whether the FileHandle passed in has been saved. It 1074 // accomplishes this by checking to see if the original and current FileHandles 1075 // are both overlays, and if the current FileHandle is saved while the original 1076 // FileHandle was not saved. 1077 func fileWasSaved(originalFH, currentFH source.FileHandle) bool { 1078 c, ok := currentFH.(*overlay) 1079 if !ok || c == nil { 1080 return true 1081 } 1082 o, ok := originalFH.(*overlay) 1083 if !ok || o == nil { 1084 return c.saved 1085 } 1086 return !o.saved && c.saved 1087 } 1088 1089 // shouldInvalidateMetadata reparses a file's package and import declarations to 1090 // determine if the file requires a metadata reload. 1091 func (s *snapshot) shouldInvalidateMetadata(ctx context.Context, originalFH, currentFH source.FileHandle) bool { 1092 if originalFH == nil { 1093 return currentFH.Kind() == source.Go 1094 } 1095 // If the file hasn't changed, there's no need to reload. 1096 if originalFH.FileIdentity() == currentFH.FileIdentity() { 1097 return false 1098 } 1099 // If a go.mod file's contents have changed, always invalidate metadata. 1100 if kind := originalFH.Kind(); kind == source.Mod { 1101 return originalFH.URI() == s.view.modURI 1102 } 1103 // Get the original and current parsed files in order to check package name and imports. 1104 // Use the direct parsing API to avoid modifying the snapshot we're cloning. 1105 parse := func(fh source.FileHandle) (*ast.File, error) { 1106 data, err := fh.Read() 1107 if err != nil { 1108 return nil, err 1109 } 1110 fset := token.NewFileSet() 1111 return parser.ParseFile(fset, fh.URI().Filename(), data, parser.ImportsOnly) 1112 } 1113 original, originalErr := parse(originalFH) 1114 current, currentErr := parse(currentFH) 1115 if originalErr != nil || currentErr != nil { 1116 return (originalErr == nil) != (currentErr == nil) 1117 } 1118 // Check if the package's metadata has changed. The cases handled are: 1119 // 1. A package's name has changed 1120 // 2. A file's imports have changed 1121 if original.Name.Name != current.Name.Name { 1122 return true 1123 } 1124 importSet := make(map[string]struct{}) 1125 for _, importSpec := range original.Imports { 1126 importSet[importSpec.Path.Value] = struct{}{} 1127 } 1128 // If any of the current imports were not in the original imports. 1129 for _, importSpec := range current.Imports { 1130 if _, ok := importSet[importSpec.Path.Value]; ok { 1131 continue 1132 } 1133 // If the import path is obviously not valid, we can skip reloading 1134 // metadata. For now, valid means properly quoted and without a 1135 // terminal slash. 1136 path, err := strconv.Unquote(importSpec.Path.Value) 1137 if err != nil { 1138 continue 1139 } 1140 if path == "" { 1141 continue 1142 } 1143 if path[len(path)-1] == '/' { 1144 continue 1145 } 1146 return true 1147 } 1148 return false 1149 } 1150 1151 // findWorkspaceDirectoriesLocked returns all of the directories that are 1152 // considered to be part of the view's workspace. For GOPATH workspaces, this 1153 // is just the view's root. For modules-based workspaces, this is the module 1154 // root and any replace targets. It also returns the parseModHandle for the 1155 // view's go.mod file if it has one. 1156 // 1157 // It assumes that the file handle is the view's go.mod file, if it has one. 1158 // The caller need not be holding the snapshot's mutex, but it might be. 1159 func (s *snapshot) findWorkspaceDirectories(ctx context.Context, modFH source.FileHandle) map[span.URI]struct{} { 1160 m := map[span.URI]struct{}{ 1161 s.view.root: {}, 1162 } 1163 // If the view does not have a go.mod file, only the root directory 1164 // is known. In GOPATH mode, we should really watch the entire GOPATH, 1165 // but that's too expensive. 1166 modURI := s.view.modURI 1167 if modURI == "" { 1168 return m 1169 } 1170 if modFH == nil { 1171 return m 1172 } 1173 // Ignore parse errors. An invalid go.mod is not fatal. 1174 mod, err := s.ParseMod(ctx, modFH) 1175 if err != nil { 1176 return m 1177 } 1178 for _, r := range mod.File.Replace { 1179 // We may be replacing a module with a different version, not a path 1180 // on disk. 1181 if r.New.Version != "" { 1182 continue 1183 } 1184 uri := span.URIFromPath(r.New.Path) 1185 m[uri] = struct{}{} 1186 } 1187 return m 1188 } 1189 1190 func (s *snapshot) BuiltinPackage(ctx context.Context) (*source.BuiltinPackage, error) { 1191 s.view.AwaitInitialized(ctx) 1192 1193 if s.builtin == nil { 1194 return nil, errors.Errorf("no builtin package for view %s", s.view.name) 1195 } 1196 d, err := s.builtin.handle.Get(ctx, s.generation, s) 1197 if err != nil { 1198 return nil, err 1199 } 1200 data := d.(*builtinPackageData) 1201 return data.parsed, data.err 1202 } 1203 1204 func (s *snapshot) buildBuiltinPackage(ctx context.Context, goFiles []string) error { 1205 if len(goFiles) != 1 { 1206 return errors.Errorf("only expected 1 file, got %v", len(goFiles)) 1207 } 1208 uri := span.URIFromPath(goFiles[0]) 1209 1210 // Get the FileHandle through the cache to avoid adding it to the snapshot 1211 // and to get the file content from disk. 1212 fh, err := s.view.session.cache.getFile(ctx, uri) 1213 if err != nil { 1214 return err 1215 } 1216 h := s.generation.Bind(fh.FileIdentity(), func(ctx context.Context, arg memoize.Arg) interface{} { 1217 snapshot := arg.(*snapshot) 1218 1219 pgh := snapshot.parseGoHandle(ctx, fh, source.ParseFull) 1220 pgf, _, err := snapshot.parseGo(ctx, pgh) 1221 if err != nil { 1222 return &builtinPackageData{err: err} 1223 } 1224 pkg, err := ast.NewPackage(snapshot.view.session.cache.fset, map[string]*ast.File{ 1225 pgf.URI.Filename(): pgf.File, 1226 }, nil, nil) 1227 if err != nil { 1228 return &builtinPackageData{err: err} 1229 } 1230 return &builtinPackageData{ 1231 parsed: &source.BuiltinPackage{ 1232 ParsedFile: pgf, 1233 Package: pkg, 1234 }, 1235 } 1236 }) 1237 s.builtin = &builtinPackageHandle{handle: h} 1238 return nil 1239 }