golang.org/x/tools@v0.21.0/internal/imports/mod.go (about) 1 // Copyright 2019 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package imports 6 7 import ( 8 "bytes" 9 "context" 10 "encoding/json" 11 "fmt" 12 "os" 13 "path" 14 "path/filepath" 15 "regexp" 16 "sort" 17 "strconv" 18 "strings" 19 20 "golang.org/x/mod/module" 21 "golang.org/x/tools/internal/event" 22 "golang.org/x/tools/internal/gocommand" 23 "golang.org/x/tools/internal/gopathwalk" 24 "golang.org/x/tools/internal/stdlib" 25 ) 26 27 // Notes(rfindley): ModuleResolver appears to be heavily optimized for scanning 28 // as fast as possible, which is desirable for a call to goimports from the 29 // command line, but it doesn't work as well for gopls, where it suffers from 30 // slow startup (golang/go#44863) and intermittent hanging (golang/go#59216), 31 // both caused by populating the cache, albeit in slightly different ways. 32 // 33 // A high level list of TODOs: 34 // - Optimize the scan itself, as there is some redundancy statting and 35 // reading go.mod files. 36 // - Invert the relationship between ProcessEnv and Resolver (see the 37 // docstring of ProcessEnv). 38 // - Make it easier to use an external resolver implementation. 39 // 40 // Smaller TODOs are annotated in the code below. 41 42 // ModuleResolver implements the Resolver interface for a workspace using 43 // modules. 44 // 45 // A goal of the ModuleResolver is to invoke the Go command as little as 46 // possible. To this end, it runs the Go command only for listing module 47 // information (i.e. `go list -m -e -json ...`). Package scanning, the process 48 // of loading package information for the modules, is implemented internally 49 // via the scan method. 50 // 51 // It has two types of state: the state derived from the go command, which 52 // is populated by init, and the state derived from scans, which is populated 53 // via scan. A root is considered scanned if it has been walked to discover 54 // directories. However, if the scan did not require additional information 55 // from the directory (such as package name or exports), the directory 56 // information itself may be partially populated. It will be lazily filled in 57 // as needed by scans, using the scanCallback. 58 type ModuleResolver struct { 59 env *ProcessEnv 60 61 // Module state, populated during construction 62 dummyVendorMod *gocommand.ModuleJSON // if vendoring is enabled, a pseudo-module to represent the /vendor directory 63 moduleCacheDir string // GOMODCACHE, inferred from GOPATH if unset 64 roots []gopathwalk.Root // roots to scan, in approximate order of importance 65 mains []*gocommand.ModuleJSON // main modules 66 mainByDir map[string]*gocommand.ModuleJSON // module information by dir, to join with roots 67 modsByModPath []*gocommand.ModuleJSON // all modules, ordered by # of path components in their module path 68 modsByDir []*gocommand.ModuleJSON // ...or by the number of path components in their Dir. 69 70 // Scanning state, populated by scan 71 72 // scanSema prevents concurrent scans, and guards scannedRoots and the cache 73 // fields below (though the caches themselves are concurrency safe). 74 // Receive to acquire, send to release. 75 scanSema chan struct{} 76 scannedRoots map[gopathwalk.Root]bool // if true, root has been walked 77 78 // Caches of directory info, populated by scans and scan callbacks 79 // 80 // moduleCacheCache stores cached information about roots in the module 81 // cache, which are immutable and therefore do not need to be invalidated. 82 // 83 // otherCache stores information about all other roots (even GOROOT), which 84 // may change. 85 moduleCacheCache *DirInfoCache 86 otherCache *DirInfoCache 87 } 88 89 // newModuleResolver returns a new module-aware goimports resolver. 90 // 91 // Note: use caution when modifying this constructor: changes must also be 92 // reflected in ModuleResolver.ClearForNewScan. 93 func newModuleResolver(e *ProcessEnv, moduleCacheCache *DirInfoCache) (*ModuleResolver, error) { 94 r := &ModuleResolver{ 95 env: e, 96 scanSema: make(chan struct{}, 1), 97 } 98 r.scanSema <- struct{}{} // release 99 100 goenv, err := r.env.goEnv() 101 if err != nil { 102 return nil, err 103 } 104 105 // TODO(rfindley): can we refactor to share logic with r.env.invokeGo? 106 inv := gocommand.Invocation{ 107 BuildFlags: r.env.BuildFlags, 108 ModFlag: r.env.ModFlag, 109 Env: r.env.env(), 110 Logf: r.env.Logf, 111 WorkingDir: r.env.WorkingDir, 112 } 113 114 vendorEnabled := false 115 var mainModVendor *gocommand.ModuleJSON // for module vendoring 116 var mainModsVendor []*gocommand.ModuleJSON // for workspace vendoring 117 118 goWork := r.env.Env["GOWORK"] 119 if len(goWork) == 0 { 120 // TODO(rfindley): VendorEnabled runs the go command to get GOFLAGS, but 121 // they should be available from the ProcessEnv. Can we avoid the redundant 122 // invocation? 123 vendorEnabled, mainModVendor, err = gocommand.VendorEnabled(context.TODO(), inv, r.env.GocmdRunner) 124 if err != nil { 125 return nil, err 126 } 127 } else { 128 vendorEnabled, mainModsVendor, err = gocommand.WorkspaceVendorEnabled(context.Background(), inv, r.env.GocmdRunner) 129 if err != nil { 130 return nil, err 131 } 132 } 133 134 if vendorEnabled { 135 if mainModVendor != nil { 136 // Module vendor mode is on, so all the non-Main modules are irrelevant, 137 // and we need to search /vendor for everything. 138 r.mains = []*gocommand.ModuleJSON{mainModVendor} 139 r.dummyVendorMod = &gocommand.ModuleJSON{ 140 Path: "", 141 Dir: filepath.Join(mainModVendor.Dir, "vendor"), 142 } 143 r.modsByModPath = []*gocommand.ModuleJSON{mainModVendor, r.dummyVendorMod} 144 r.modsByDir = []*gocommand.ModuleJSON{mainModVendor, r.dummyVendorMod} 145 } else { 146 // Workspace vendor mode is on, so all the non-Main modules are irrelevant, 147 // and we need to search /vendor for everything. 148 r.mains = mainModsVendor 149 r.dummyVendorMod = &gocommand.ModuleJSON{ 150 Path: "", 151 Dir: filepath.Join(filepath.Dir(goWork), "vendor"), 152 } 153 r.modsByModPath = append(append([]*gocommand.ModuleJSON{}, mainModsVendor...), r.dummyVendorMod) 154 r.modsByDir = append(append([]*gocommand.ModuleJSON{}, mainModsVendor...), r.dummyVendorMod) 155 } 156 } else { 157 // Vendor mode is off, so run go list -m ... to find everything. 158 err := r.initAllMods() 159 // We expect an error when running outside of a module with 160 // GO111MODULE=on. Other errors are fatal. 161 if err != nil { 162 if errMsg := err.Error(); !strings.Contains(errMsg, "working directory is not part of a module") && !strings.Contains(errMsg, "go.mod file not found") { 163 return nil, err 164 } 165 } 166 } 167 168 r.moduleCacheDir = gomodcacheForEnv(goenv) 169 if r.moduleCacheDir == "" { 170 return nil, fmt.Errorf("cannot resolve GOMODCACHE") 171 } 172 173 sort.Slice(r.modsByModPath, func(i, j int) bool { 174 count := func(x int) int { 175 return strings.Count(r.modsByModPath[x].Path, "/") 176 } 177 return count(j) < count(i) // descending order 178 }) 179 sort.Slice(r.modsByDir, func(i, j int) bool { 180 count := func(x int) int { 181 return strings.Count(r.modsByDir[x].Dir, string(filepath.Separator)) 182 } 183 return count(j) < count(i) // descending order 184 }) 185 186 r.roots = []gopathwalk.Root{} 187 if goenv["GOROOT"] != "" { // "" happens in tests 188 r.roots = append(r.roots, gopathwalk.Root{Path: filepath.Join(goenv["GOROOT"], "/src"), Type: gopathwalk.RootGOROOT}) 189 } 190 r.mainByDir = make(map[string]*gocommand.ModuleJSON) 191 for _, main := range r.mains { 192 r.roots = append(r.roots, gopathwalk.Root{Path: main.Dir, Type: gopathwalk.RootCurrentModule}) 193 r.mainByDir[main.Dir] = main 194 } 195 if vendorEnabled { 196 r.roots = append(r.roots, gopathwalk.Root{Path: r.dummyVendorMod.Dir, Type: gopathwalk.RootOther}) 197 } else { 198 addDep := func(mod *gocommand.ModuleJSON) { 199 if mod.Replace == nil { 200 // This is redundant with the cache, but we'll skip it cheaply enough 201 // when we encounter it in the module cache scan. 202 // 203 // Including it at a lower index in r.roots than the module cache dir 204 // helps prioritize matches from within existing dependencies. 205 r.roots = append(r.roots, gopathwalk.Root{Path: mod.Dir, Type: gopathwalk.RootModuleCache}) 206 } else { 207 r.roots = append(r.roots, gopathwalk.Root{Path: mod.Dir, Type: gopathwalk.RootOther}) 208 } 209 } 210 // Walk dependent modules before scanning the full mod cache, direct deps first. 211 for _, mod := range r.modsByModPath { 212 if !mod.Indirect && !mod.Main { 213 addDep(mod) 214 } 215 } 216 for _, mod := range r.modsByModPath { 217 if mod.Indirect && !mod.Main { 218 addDep(mod) 219 } 220 } 221 // If provided, share the moduleCacheCache. 222 // 223 // TODO(rfindley): The module cache is immutable. However, the loaded 224 // exports do depend on GOOS and GOARCH. Fortunately, the 225 // ProcessEnv.buildContext does not adjust these from build.DefaultContext 226 // (even though it should). So for now, this is OK to share, but we need to 227 // add logic for handling GOOS/GOARCH. 228 r.moduleCacheCache = moduleCacheCache 229 r.roots = append(r.roots, gopathwalk.Root{Path: r.moduleCacheDir, Type: gopathwalk.RootModuleCache}) 230 } 231 232 r.scannedRoots = map[gopathwalk.Root]bool{} 233 if r.moduleCacheCache == nil { 234 r.moduleCacheCache = NewDirInfoCache() 235 } 236 r.otherCache = NewDirInfoCache() 237 return r, nil 238 } 239 240 // gomodcacheForEnv returns the GOMODCACHE value to use based on the given env 241 // map, which must have GOMODCACHE and GOPATH populated. 242 // 243 // TODO(rfindley): this is defensive refactoring. 244 // 1. Is this even relevant anymore? Can't we just read GOMODCACHE. 245 // 2. Use this to separate module cache scanning from other scanning. 246 func gomodcacheForEnv(goenv map[string]string) string { 247 if gmc := goenv["GOMODCACHE"]; gmc != "" { 248 return gmc 249 } 250 gopaths := filepath.SplitList(goenv["GOPATH"]) 251 if len(gopaths) == 0 { 252 return "" 253 } 254 return filepath.Join(gopaths[0], "/pkg/mod") 255 } 256 257 func (r *ModuleResolver) initAllMods() error { 258 stdout, err := r.env.invokeGo(context.TODO(), "list", "-m", "-e", "-json", "...") 259 if err != nil { 260 return err 261 } 262 for dec := json.NewDecoder(stdout); dec.More(); { 263 mod := &gocommand.ModuleJSON{} 264 if err := dec.Decode(mod); err != nil { 265 return err 266 } 267 if mod.Dir == "" { 268 if r.env.Logf != nil { 269 r.env.Logf("module %v has not been downloaded and will be ignored", mod.Path) 270 } 271 // Can't do anything with a module that's not downloaded. 272 continue 273 } 274 // golang/go#36193: the go command doesn't always clean paths. 275 mod.Dir = filepath.Clean(mod.Dir) 276 r.modsByModPath = append(r.modsByModPath, mod) 277 r.modsByDir = append(r.modsByDir, mod) 278 if mod.Main { 279 r.mains = append(r.mains, mod) 280 } 281 } 282 return nil 283 } 284 285 // ClearForNewScan invalidates the last scan. 286 // 287 // It preserves the set of roots, but forgets about the set of directories. 288 // Though it forgets the set of module cache directories, it remembers their 289 // contents, since they are assumed to be immutable. 290 func (r *ModuleResolver) ClearForNewScan() Resolver { 291 <-r.scanSema // acquire r, to guard scannedRoots 292 r2 := &ModuleResolver{ 293 env: r.env, 294 dummyVendorMod: r.dummyVendorMod, 295 moduleCacheDir: r.moduleCacheDir, 296 roots: r.roots, 297 mains: r.mains, 298 mainByDir: r.mainByDir, 299 modsByModPath: r.modsByModPath, 300 301 scanSema: make(chan struct{}, 1), 302 scannedRoots: make(map[gopathwalk.Root]bool), 303 otherCache: NewDirInfoCache(), 304 moduleCacheCache: r.moduleCacheCache, 305 } 306 r2.scanSema <- struct{}{} // r2 must start released 307 // Invalidate root scans. We don't need to invalidate module cache roots, 308 // because they are immutable. 309 // (We don't support a use case where GOMODCACHE is cleaned in the middle of 310 // e.g. a gopls session: the user must restart gopls to get accurate 311 // imports.) 312 // 313 // Scanning for new directories in GOMODCACHE should be handled elsewhere, 314 // via a call to ScanModuleCache. 315 for _, root := range r.roots { 316 if root.Type == gopathwalk.RootModuleCache && r.scannedRoots[root] { 317 r2.scannedRoots[root] = true 318 } 319 } 320 r.scanSema <- struct{}{} // release r 321 return r2 322 } 323 324 // ClearModuleInfo invalidates resolver state that depends on go.mod file 325 // contents (essentially, the output of go list -m -json ...). 326 // 327 // Notably, it does not forget directory contents, which are reset 328 // asynchronously via ClearForNewScan. 329 // 330 // If the ProcessEnv is a GOPATH environment, ClearModuleInfo is a no op. 331 // 332 // TODO(rfindley): move this to a new env.go, consolidating ProcessEnv methods. 333 func (e *ProcessEnv) ClearModuleInfo() { 334 if r, ok := e.resolver.(*ModuleResolver); ok { 335 resolver, err := newModuleResolver(e, e.ModCache) 336 if err != nil { 337 e.resolver = nil 338 e.resolverErr = err 339 return 340 } 341 342 <-r.scanSema // acquire (guards caches) 343 resolver.moduleCacheCache = r.moduleCacheCache 344 resolver.otherCache = r.otherCache 345 r.scanSema <- struct{}{} // release 346 347 e.UpdateResolver(resolver) 348 } 349 } 350 351 // UpdateResolver sets the resolver for the ProcessEnv to use in imports 352 // operations. Only for use with the result of [Resolver.ClearForNewScan]. 353 // 354 // TODO(rfindley): this awkward API is a result of the (arguably) inverted 355 // relationship between configuration and state described in the doc comment 356 // for [ProcessEnv]. 357 func (e *ProcessEnv) UpdateResolver(r Resolver) { 358 e.resolver = r 359 e.resolverErr = nil 360 } 361 362 // findPackage returns the module and directory from within the main modules 363 // and their dependencies that contains the package at the given import path, 364 // or returns nil, "" if no module is in scope. 365 func (r *ModuleResolver) findPackage(importPath string) (*gocommand.ModuleJSON, string) { 366 // This can't find packages in the stdlib, but that's harmless for all 367 // the existing code paths. 368 for _, m := range r.modsByModPath { 369 if !strings.HasPrefix(importPath, m.Path) { 370 continue 371 } 372 pathInModule := importPath[len(m.Path):] 373 pkgDir := filepath.Join(m.Dir, pathInModule) 374 if r.dirIsNestedModule(pkgDir, m) { 375 continue 376 } 377 378 if info, ok := r.cacheLoad(pkgDir); ok { 379 if loaded, err := info.reachedStatus(nameLoaded); loaded { 380 if err != nil { 381 continue // No package in this dir. 382 } 383 return m, pkgDir 384 } 385 if scanned, err := info.reachedStatus(directoryScanned); scanned && err != nil { 386 continue // Dir is unreadable, etc. 387 } 388 // This is slightly wrong: a directory doesn't have to have an 389 // importable package to count as a package for package-to-module 390 // resolution. package main or _test files should count but 391 // don't. 392 // TODO(heschi): fix this. 393 if _, err := r.cachePackageName(info); err == nil { 394 return m, pkgDir 395 } 396 } 397 398 // Not cached. Read the filesystem. 399 pkgFiles, err := os.ReadDir(pkgDir) 400 if err != nil { 401 continue 402 } 403 // A module only contains a package if it has buildable go 404 // files in that directory. If not, it could be provided by an 405 // outer module. See #29736. 406 for _, fi := range pkgFiles { 407 if ok, _ := r.env.matchFile(pkgDir, fi.Name()); ok { 408 return m, pkgDir 409 } 410 } 411 } 412 return nil, "" 413 } 414 415 func (r *ModuleResolver) cacheLoad(dir string) (directoryPackageInfo, bool) { 416 if info, ok := r.moduleCacheCache.Load(dir); ok { 417 return info, ok 418 } 419 return r.otherCache.Load(dir) 420 } 421 422 func (r *ModuleResolver) cacheStore(info directoryPackageInfo) { 423 if info.rootType == gopathwalk.RootModuleCache { 424 r.moduleCacheCache.Store(info.dir, info) 425 } else { 426 r.otherCache.Store(info.dir, info) 427 } 428 } 429 430 // cachePackageName caches the package name for a dir already in the cache. 431 func (r *ModuleResolver) cachePackageName(info directoryPackageInfo) (string, error) { 432 if info.rootType == gopathwalk.RootModuleCache { 433 return r.moduleCacheCache.CachePackageName(info) 434 } 435 return r.otherCache.CachePackageName(info) 436 } 437 438 func (r *ModuleResolver) cacheExports(ctx context.Context, env *ProcessEnv, info directoryPackageInfo) (string, []stdlib.Symbol, error) { 439 if info.rootType == gopathwalk.RootModuleCache { 440 return r.moduleCacheCache.CacheExports(ctx, env, info) 441 } 442 return r.otherCache.CacheExports(ctx, env, info) 443 } 444 445 // findModuleByDir returns the module that contains dir, or nil if no such 446 // module is in scope. 447 func (r *ModuleResolver) findModuleByDir(dir string) *gocommand.ModuleJSON { 448 // This is quite tricky and may not be correct. dir could be: 449 // - a package in the main module. 450 // - a replace target underneath the main module's directory. 451 // - a nested module in the above. 452 // - a replace target somewhere totally random. 453 // - a nested module in the above. 454 // - in the mod cache. 455 // - in /vendor/ in -mod=vendor mode. 456 // - nested module? Dunno. 457 // Rumor has it that replace targets cannot contain other replace targets. 458 // 459 // Note that it is critical here that modsByDir is sorted to have deeper dirs 460 // first. This ensures that findModuleByDir finds the innermost module. 461 // See also golang/go#56291. 462 for _, m := range r.modsByDir { 463 if !strings.HasPrefix(dir, m.Dir) { 464 continue 465 } 466 467 if r.dirIsNestedModule(dir, m) { 468 continue 469 } 470 471 return m 472 } 473 return nil 474 } 475 476 // dirIsNestedModule reports if dir is contained in a nested module underneath 477 // mod, not actually in mod. 478 func (r *ModuleResolver) dirIsNestedModule(dir string, mod *gocommand.ModuleJSON) bool { 479 if !strings.HasPrefix(dir, mod.Dir) { 480 return false 481 } 482 if r.dirInModuleCache(dir) { 483 // Nested modules in the module cache are pruned, 484 // so it cannot be a nested module. 485 return false 486 } 487 if mod != nil && mod == r.dummyVendorMod { 488 // The /vendor pseudomodule is flattened and doesn't actually count. 489 return false 490 } 491 modDir, _ := r.modInfo(dir) 492 if modDir == "" { 493 return false 494 } 495 return modDir != mod.Dir 496 } 497 498 func readModName(modFile string) string { 499 modBytes, err := os.ReadFile(modFile) 500 if err != nil { 501 return "" 502 } 503 return modulePath(modBytes) 504 } 505 506 func (r *ModuleResolver) modInfo(dir string) (modDir, modName string) { 507 if r.dirInModuleCache(dir) { 508 if matches := modCacheRegexp.FindStringSubmatch(dir); len(matches) == 3 { 509 index := strings.Index(dir, matches[1]+"@"+matches[2]) 510 modDir := filepath.Join(dir[:index], matches[1]+"@"+matches[2]) 511 return modDir, readModName(filepath.Join(modDir, "go.mod")) 512 } 513 } 514 for { 515 if info, ok := r.cacheLoad(dir); ok { 516 return info.moduleDir, info.moduleName 517 } 518 f := filepath.Join(dir, "go.mod") 519 info, err := os.Stat(f) 520 if err == nil && !info.IsDir() { 521 return dir, readModName(f) 522 } 523 524 d := filepath.Dir(dir) 525 if len(d) >= len(dir) { 526 return "", "" // reached top of file system, no go.mod 527 } 528 dir = d 529 } 530 } 531 532 func (r *ModuleResolver) dirInModuleCache(dir string) bool { 533 if r.moduleCacheDir == "" { 534 return false 535 } 536 return strings.HasPrefix(dir, r.moduleCacheDir) 537 } 538 539 func (r *ModuleResolver) loadPackageNames(importPaths []string, srcDir string) (map[string]string, error) { 540 names := map[string]string{} 541 for _, path := range importPaths { 542 // TODO(rfindley): shouldn't this use the dirInfoCache? 543 _, packageDir := r.findPackage(path) 544 if packageDir == "" { 545 continue 546 } 547 name, err := packageDirToName(packageDir) 548 if err != nil { 549 continue 550 } 551 names[path] = name 552 } 553 return names, nil 554 } 555 556 func (r *ModuleResolver) scan(ctx context.Context, callback *scanCallback) error { 557 ctx, done := event.Start(ctx, "imports.ModuleResolver.scan") 558 defer done() 559 560 processDir := func(info directoryPackageInfo) { 561 // Skip this directory if we were not able to get the package information successfully. 562 if scanned, err := info.reachedStatus(directoryScanned); !scanned || err != nil { 563 return 564 } 565 pkg, err := r.canonicalize(info) 566 if err != nil { 567 return 568 } 569 if !callback.dirFound(pkg) { 570 return 571 } 572 573 pkg.packageName, err = r.cachePackageName(info) 574 if err != nil { 575 return 576 } 577 if !callback.packageNameLoaded(pkg) { 578 return 579 } 580 581 _, exports, err := r.loadExports(ctx, pkg, false) 582 if err != nil { 583 return 584 } 585 callback.exportsLoaded(pkg, exports) 586 } 587 588 // Start processing everything in the cache, and listen for the new stuff 589 // we discover in the walk below. 590 stop1 := r.moduleCacheCache.ScanAndListen(ctx, processDir) 591 defer stop1() 592 stop2 := r.otherCache.ScanAndListen(ctx, processDir) 593 defer stop2() 594 595 // We assume cached directories are fully cached, including all their 596 // children, and have not changed. We can skip them. 597 skip := func(root gopathwalk.Root, dir string) bool { 598 if r.env.SkipPathInScan != nil && root.Type == gopathwalk.RootCurrentModule { 599 if root.Path == dir { 600 return false 601 } 602 603 if r.env.SkipPathInScan(filepath.Clean(dir)) { 604 return true 605 } 606 } 607 608 info, ok := r.cacheLoad(dir) 609 if !ok { 610 return false 611 } 612 // This directory can be skipped as long as we have already scanned it. 613 // Packages with errors will continue to have errors, so there is no need 614 // to rescan them. 615 packageScanned, _ := info.reachedStatus(directoryScanned) 616 return packageScanned 617 } 618 619 add := func(root gopathwalk.Root, dir string) { 620 r.cacheStore(r.scanDirForPackage(root, dir)) 621 } 622 623 // r.roots and the callback are not necessarily safe to use in the 624 // goroutine below. Process them eagerly. 625 roots := filterRoots(r.roots, callback.rootFound) 626 // We can't cancel walks, because we need them to finish to have a usable 627 // cache. Instead, run them in a separate goroutine and detach. 628 scanDone := make(chan struct{}) 629 go func() { 630 select { 631 case <-ctx.Done(): 632 return 633 case <-r.scanSema: // acquire 634 } 635 defer func() { r.scanSema <- struct{}{} }() // release 636 // We have the lock on r.scannedRoots, and no other scans can run. 637 for _, root := range roots { 638 if ctx.Err() != nil { 639 return 640 } 641 642 if r.scannedRoots[root] { 643 continue 644 } 645 gopathwalk.WalkSkip([]gopathwalk.Root{root}, add, skip, gopathwalk.Options{Logf: r.env.Logf, ModulesEnabled: true}) 646 r.scannedRoots[root] = true 647 } 648 close(scanDone) 649 }() 650 select { 651 case <-ctx.Done(): 652 case <-scanDone: 653 } 654 return nil 655 } 656 657 func (r *ModuleResolver) scoreImportPath(ctx context.Context, path string) float64 { 658 if stdlib.HasPackage(path) { 659 return MaxRelevance 660 } 661 mod, _ := r.findPackage(path) 662 return modRelevance(mod) 663 } 664 665 func modRelevance(mod *gocommand.ModuleJSON) float64 { 666 var relevance float64 667 switch { 668 case mod == nil: // out of scope 669 return MaxRelevance - 4 670 case mod.Indirect: 671 relevance = MaxRelevance - 3 672 case !mod.Main: 673 relevance = MaxRelevance - 2 674 default: 675 relevance = MaxRelevance - 1 // main module ties with stdlib 676 } 677 678 _, versionString, ok := module.SplitPathVersion(mod.Path) 679 if ok { 680 index := strings.Index(versionString, "v") 681 if index == -1 { 682 return relevance 683 } 684 if versionNumber, err := strconv.ParseFloat(versionString[index+1:], 64); err == nil { 685 relevance += versionNumber / 1000 686 } 687 } 688 689 return relevance 690 } 691 692 // canonicalize gets the result of canonicalizing the packages using the results 693 // of initializing the resolver from 'go list -m'. 694 func (r *ModuleResolver) canonicalize(info directoryPackageInfo) (*pkg, error) { 695 // Packages in GOROOT are already canonical, regardless of the std/cmd modules. 696 if info.rootType == gopathwalk.RootGOROOT { 697 return &pkg{ 698 importPathShort: info.nonCanonicalImportPath, 699 dir: info.dir, 700 packageName: path.Base(info.nonCanonicalImportPath), 701 relevance: MaxRelevance, 702 }, nil 703 } 704 705 importPath := info.nonCanonicalImportPath 706 mod := r.findModuleByDir(info.dir) 707 // Check if the directory is underneath a module that's in scope. 708 if mod != nil { 709 // It is. If dir is the target of a replace directive, 710 // our guessed import path is wrong. Use the real one. 711 if mod.Dir == info.dir { 712 importPath = mod.Path 713 } else { 714 dirInMod := info.dir[len(mod.Dir)+len("/"):] 715 importPath = path.Join(mod.Path, filepath.ToSlash(dirInMod)) 716 } 717 } else if !strings.HasPrefix(importPath, info.moduleName) { 718 // The module's name doesn't match the package's import path. It 719 // probably needs a replace directive we don't have. 720 return nil, fmt.Errorf("package in %q is not valid without a replace statement", info.dir) 721 } 722 723 res := &pkg{ 724 importPathShort: importPath, 725 dir: info.dir, 726 relevance: modRelevance(mod), 727 } 728 // We may have discovered a package that has a different version 729 // in scope already. Canonicalize to that one if possible. 730 if _, canonicalDir := r.findPackage(importPath); canonicalDir != "" { 731 res.dir = canonicalDir 732 } 733 return res, nil 734 } 735 736 func (r *ModuleResolver) loadExports(ctx context.Context, pkg *pkg, includeTest bool) (string, []stdlib.Symbol, error) { 737 if info, ok := r.cacheLoad(pkg.dir); ok && !includeTest { 738 return r.cacheExports(ctx, r.env, info) 739 } 740 return loadExportsFromFiles(ctx, r.env, pkg.dir, includeTest) 741 } 742 743 func (r *ModuleResolver) scanDirForPackage(root gopathwalk.Root, dir string) directoryPackageInfo { 744 subdir := "" 745 if dir != root.Path { 746 subdir = dir[len(root.Path)+len("/"):] 747 } 748 importPath := filepath.ToSlash(subdir) 749 if strings.HasPrefix(importPath, "vendor/") { 750 // Only enter vendor directories if they're explicitly requested as a root. 751 return directoryPackageInfo{ 752 status: directoryScanned, 753 err: fmt.Errorf("unwanted vendor directory"), 754 } 755 } 756 switch root.Type { 757 case gopathwalk.RootCurrentModule: 758 importPath = path.Join(r.mainByDir[root.Path].Path, filepath.ToSlash(subdir)) 759 case gopathwalk.RootModuleCache: 760 matches := modCacheRegexp.FindStringSubmatch(subdir) 761 if len(matches) == 0 { 762 return directoryPackageInfo{ 763 status: directoryScanned, 764 err: fmt.Errorf("invalid module cache path: %v", subdir), 765 } 766 } 767 modPath, err := module.UnescapePath(filepath.ToSlash(matches[1])) 768 if err != nil { 769 if r.env.Logf != nil { 770 r.env.Logf("decoding module cache path %q: %v", subdir, err) 771 } 772 return directoryPackageInfo{ 773 status: directoryScanned, 774 err: fmt.Errorf("decoding module cache path %q: %v", subdir, err), 775 } 776 } 777 importPath = path.Join(modPath, filepath.ToSlash(matches[3])) 778 } 779 780 modDir, modName := r.modInfo(dir) 781 result := directoryPackageInfo{ 782 status: directoryScanned, 783 dir: dir, 784 rootType: root.Type, 785 nonCanonicalImportPath: importPath, 786 moduleDir: modDir, 787 moduleName: modName, 788 } 789 if root.Type == gopathwalk.RootGOROOT { 790 // stdlib packages are always in scope, despite the confusing go.mod 791 return result 792 } 793 return result 794 } 795 796 // modCacheRegexp splits a path in a module cache into module, module version, and package. 797 var modCacheRegexp = regexp.MustCompile(`(.*)@([^/\\]*)(.*)`) 798 799 var ( 800 slashSlash = []byte("//") 801 moduleStr = []byte("module") 802 ) 803 804 // modulePath returns the module path from the gomod file text. 805 // If it cannot find a module path, it returns an empty string. 806 // It is tolerant of unrelated problems in the go.mod file. 807 // 808 // Copied from cmd/go/internal/modfile. 809 func modulePath(mod []byte) string { 810 for len(mod) > 0 { 811 line := mod 812 mod = nil 813 if i := bytes.IndexByte(line, '\n'); i >= 0 { 814 line, mod = line[:i], line[i+1:] 815 } 816 if i := bytes.Index(line, slashSlash); i >= 0 { 817 line = line[:i] 818 } 819 line = bytes.TrimSpace(line) 820 if !bytes.HasPrefix(line, moduleStr) { 821 continue 822 } 823 line = line[len(moduleStr):] 824 n := len(line) 825 line = bytes.TrimSpace(line) 826 if len(line) == n || len(line) == 0 { 827 continue 828 } 829 830 if line[0] == '"' || line[0] == '`' { 831 p, err := strconv.Unquote(string(line)) 832 if err != nil { 833 return "" // malformed quoted string or multiline module path 834 } 835 return p 836 } 837 838 return string(line) 839 } 840 return "" // missing module path 841 }