golang.org/x/tools@v0.21.0/go/packages/golist.go (about) 1 // Copyright 2018 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package packages 6 7 import ( 8 "bytes" 9 "context" 10 "encoding/json" 11 "fmt" 12 "log" 13 "os" 14 "os/exec" 15 "path" 16 "path/filepath" 17 "reflect" 18 "sort" 19 "strconv" 20 "strings" 21 "sync" 22 "unicode" 23 24 "golang.org/x/tools/go/internal/packagesdriver" 25 "golang.org/x/tools/internal/gocommand" 26 "golang.org/x/tools/internal/packagesinternal" 27 ) 28 29 // debug controls verbose logging. 30 var debug, _ = strconv.ParseBool(os.Getenv("GOPACKAGESDEBUG")) 31 32 // A goTooOldError reports that the go command 33 // found by exec.LookPath is too old to use the new go list behavior. 34 type goTooOldError struct { 35 error 36 } 37 38 // responseDeduper wraps a DriverResponse, deduplicating its contents. 39 type responseDeduper struct { 40 seenRoots map[string]bool 41 seenPackages map[string]*Package 42 dr *DriverResponse 43 } 44 45 func newDeduper() *responseDeduper { 46 return &responseDeduper{ 47 dr: &DriverResponse{}, 48 seenRoots: map[string]bool{}, 49 seenPackages: map[string]*Package{}, 50 } 51 } 52 53 // addAll fills in r with a DriverResponse. 54 func (r *responseDeduper) addAll(dr *DriverResponse) { 55 for _, pkg := range dr.Packages { 56 r.addPackage(pkg) 57 } 58 for _, root := range dr.Roots { 59 r.addRoot(root) 60 } 61 r.dr.GoVersion = dr.GoVersion 62 } 63 64 func (r *responseDeduper) addPackage(p *Package) { 65 if r.seenPackages[p.ID] != nil { 66 return 67 } 68 r.seenPackages[p.ID] = p 69 r.dr.Packages = append(r.dr.Packages, p) 70 } 71 72 func (r *responseDeduper) addRoot(id string) { 73 if r.seenRoots[id] { 74 return 75 } 76 r.seenRoots[id] = true 77 r.dr.Roots = append(r.dr.Roots, id) 78 } 79 80 type golistState struct { 81 cfg *Config 82 ctx context.Context 83 84 envOnce sync.Once 85 goEnvError error 86 goEnv map[string]string 87 88 rootsOnce sync.Once 89 rootDirsError error 90 rootDirs map[string]string 91 92 goVersionOnce sync.Once 93 goVersionError error 94 goVersion int // The X in Go 1.X. 95 96 // vendorDirs caches the (non)existence of vendor directories. 97 vendorDirs map[string]bool 98 } 99 100 // getEnv returns Go environment variables. Only specific variables are 101 // populated -- computing all of them is slow. 102 func (state *golistState) getEnv() (map[string]string, error) { 103 state.envOnce.Do(func() { 104 var b *bytes.Buffer 105 b, state.goEnvError = state.invokeGo("env", "-json", "GOMOD", "GOPATH") 106 if state.goEnvError != nil { 107 return 108 } 109 110 state.goEnv = make(map[string]string) 111 decoder := json.NewDecoder(b) 112 if state.goEnvError = decoder.Decode(&state.goEnv); state.goEnvError != nil { 113 return 114 } 115 }) 116 return state.goEnv, state.goEnvError 117 } 118 119 // mustGetEnv is a convenience function that can be used if getEnv has already succeeded. 120 func (state *golistState) mustGetEnv() map[string]string { 121 env, err := state.getEnv() 122 if err != nil { 123 panic(fmt.Sprintf("mustGetEnv: %v", err)) 124 } 125 return env 126 } 127 128 // goListDriver uses the go list command to interpret the patterns and produce 129 // the build system package structure. 130 // See driver for more details. 131 func goListDriver(cfg *Config, patterns ...string) (_ *DriverResponse, err error) { 132 // Make sure that any asynchronous go commands are killed when we return. 133 parentCtx := cfg.Context 134 if parentCtx == nil { 135 parentCtx = context.Background() 136 } 137 ctx, cancel := context.WithCancel(parentCtx) 138 defer cancel() 139 140 response := newDeduper() 141 142 state := &golistState{ 143 cfg: cfg, 144 ctx: ctx, 145 vendorDirs: map[string]bool{}, 146 } 147 148 // Fill in response.Sizes asynchronously if necessary. 149 if cfg.Mode&NeedTypesSizes != 0 || cfg.Mode&NeedTypes != 0 { 150 errCh := make(chan error) 151 go func() { 152 compiler, arch, err := packagesdriver.GetSizesForArgsGolist(ctx, state.cfgInvocation(), cfg.gocmdRunner) 153 response.dr.Compiler = compiler 154 response.dr.Arch = arch 155 errCh <- err 156 }() 157 defer func() { 158 if sizesErr := <-errCh; sizesErr != nil { 159 err = sizesErr 160 } 161 }() 162 } 163 164 // Determine files requested in contains patterns 165 var containFiles []string 166 restPatterns := make([]string, 0, len(patterns)) 167 // Extract file= and other [querytype]= patterns. Report an error if querytype 168 // doesn't exist. 169 extractQueries: 170 for _, pattern := range patterns { 171 eqidx := strings.Index(pattern, "=") 172 if eqidx < 0 { 173 restPatterns = append(restPatterns, pattern) 174 } else { 175 query, value := pattern[:eqidx], pattern[eqidx+len("="):] 176 switch query { 177 case "file": 178 containFiles = append(containFiles, value) 179 case "pattern": 180 restPatterns = append(restPatterns, value) 181 case "": // not a reserved query 182 restPatterns = append(restPatterns, pattern) 183 default: 184 for _, rune := range query { 185 if rune < 'a' || rune > 'z' { // not a reserved query 186 restPatterns = append(restPatterns, pattern) 187 continue extractQueries 188 } 189 } 190 // Reject all other patterns containing "=" 191 return nil, fmt.Errorf("invalid query type %q in query pattern %q", query, pattern) 192 } 193 } 194 } 195 196 // See if we have any patterns to pass through to go list. Zero initial 197 // patterns also requires a go list call, since it's the equivalent of 198 // ".". 199 if len(restPatterns) > 0 || len(patterns) == 0 { 200 dr, err := state.createDriverResponse(restPatterns...) 201 if err != nil { 202 return nil, err 203 } 204 response.addAll(dr) 205 } 206 207 if len(containFiles) != 0 { 208 if err := state.runContainsQueries(response, containFiles); err != nil { 209 return nil, err 210 } 211 } 212 213 // (We may yet return an error due to defer.) 214 return response.dr, nil 215 } 216 217 func (state *golistState) runContainsQueries(response *responseDeduper, queries []string) error { 218 for _, query := range queries { 219 // TODO(matloob): Do only one query per directory. 220 fdir := filepath.Dir(query) 221 // Pass absolute path of directory to go list so that it knows to treat it as a directory, 222 // not a package path. 223 pattern, err := filepath.Abs(fdir) 224 if err != nil { 225 return fmt.Errorf("could not determine absolute path of file= query path %q: %v", query, err) 226 } 227 dirResponse, err := state.createDriverResponse(pattern) 228 229 // If there was an error loading the package, or no packages are returned, 230 // or the package is returned with errors, try to load the file as an 231 // ad-hoc package. 232 // Usually the error will appear in a returned package, but may not if we're 233 // in module mode and the ad-hoc is located outside a module. 234 if err != nil || len(dirResponse.Packages) == 0 || len(dirResponse.Packages) == 1 && len(dirResponse.Packages[0].GoFiles) == 0 && 235 len(dirResponse.Packages[0].Errors) == 1 { 236 var queryErr error 237 if dirResponse, queryErr = state.adhocPackage(pattern, query); queryErr != nil { 238 return err // return the original error 239 } 240 } 241 isRoot := make(map[string]bool, len(dirResponse.Roots)) 242 for _, root := range dirResponse.Roots { 243 isRoot[root] = true 244 } 245 for _, pkg := range dirResponse.Packages { 246 // Add any new packages to the main set 247 // We don't bother to filter packages that will be dropped by the changes of roots, 248 // that will happen anyway during graph construction outside this function. 249 // Over-reporting packages is not a problem. 250 response.addPackage(pkg) 251 // if the package was not a root one, it cannot have the file 252 if !isRoot[pkg.ID] { 253 continue 254 } 255 for _, pkgFile := range pkg.GoFiles { 256 if filepath.Base(query) == filepath.Base(pkgFile) { 257 response.addRoot(pkg.ID) 258 break 259 } 260 } 261 } 262 } 263 return nil 264 } 265 266 // adhocPackage attempts to load or construct an ad-hoc package for a given 267 // query, if the original call to the driver produced inadequate results. 268 func (state *golistState) adhocPackage(pattern, query string) (*DriverResponse, error) { 269 response, err := state.createDriverResponse(query) 270 if err != nil { 271 return nil, err 272 } 273 // If we get nothing back from `go list`, 274 // try to make this file into its own ad-hoc package. 275 // TODO(rstambler): Should this check against the original response? 276 if len(response.Packages) == 0 { 277 response.Packages = append(response.Packages, &Package{ 278 ID: "command-line-arguments", 279 PkgPath: query, 280 GoFiles: []string{query}, 281 CompiledGoFiles: []string{query}, 282 Imports: make(map[string]*Package), 283 }) 284 response.Roots = append(response.Roots, "command-line-arguments") 285 } 286 // Handle special cases. 287 if len(response.Packages) == 1 { 288 // golang/go#33482: If this is a file= query for ad-hoc packages where 289 // the file only exists on an overlay, and exists outside of a module, 290 // add the file to the package and remove the errors. 291 if response.Packages[0].ID == "command-line-arguments" || 292 filepath.ToSlash(response.Packages[0].PkgPath) == filepath.ToSlash(query) { 293 if len(response.Packages[0].GoFiles) == 0 { 294 filename := filepath.Join(pattern, filepath.Base(query)) // avoid recomputing abspath 295 // TODO(matloob): check if the file is outside of a root dir? 296 for path := range state.cfg.Overlay { 297 if path == filename { 298 response.Packages[0].Errors = nil 299 response.Packages[0].GoFiles = []string{path} 300 response.Packages[0].CompiledGoFiles = []string{path} 301 } 302 } 303 } 304 } 305 } 306 return response, nil 307 } 308 309 // Fields must match go list; 310 // see $GOROOT/src/cmd/go/internal/load/pkg.go. 311 type jsonPackage struct { 312 ImportPath string 313 Dir string 314 Name string 315 Export string 316 GoFiles []string 317 CompiledGoFiles []string 318 IgnoredGoFiles []string 319 IgnoredOtherFiles []string 320 EmbedPatterns []string 321 EmbedFiles []string 322 CFiles []string 323 CgoFiles []string 324 CXXFiles []string 325 MFiles []string 326 HFiles []string 327 FFiles []string 328 SFiles []string 329 SwigFiles []string 330 SwigCXXFiles []string 331 SysoFiles []string 332 Imports []string 333 ImportMap map[string]string 334 Deps []string 335 Module *Module 336 TestGoFiles []string 337 TestImports []string 338 XTestGoFiles []string 339 XTestImports []string 340 ForTest string // q in a "p [q.test]" package, else "" 341 DepOnly bool 342 343 Error *packagesinternal.PackageError 344 DepsErrors []*packagesinternal.PackageError 345 } 346 347 type jsonPackageError struct { 348 ImportStack []string 349 Pos string 350 Err string 351 } 352 353 func otherFiles(p *jsonPackage) [][]string { 354 return [][]string{p.CFiles, p.CXXFiles, p.MFiles, p.HFiles, p.FFiles, p.SFiles, p.SwigFiles, p.SwigCXXFiles, p.SysoFiles} 355 } 356 357 // createDriverResponse uses the "go list" command to expand the pattern 358 // words and return a response for the specified packages. 359 func (state *golistState) createDriverResponse(words ...string) (*DriverResponse, error) { 360 // go list uses the following identifiers in ImportPath and Imports: 361 // 362 // "p" -- importable package or main (command) 363 // "q.test" -- q's test executable 364 // "p [q.test]" -- variant of p as built for q's test executable 365 // "q_test [q.test]" -- q's external test package 366 // 367 // The packages p that are built differently for a test q.test 368 // are q itself, plus any helpers used by the external test q_test, 369 // typically including "testing" and all its dependencies. 370 371 // Run "go list" for complete 372 // information on the specified packages. 373 goVersion, err := state.getGoVersion() 374 if err != nil { 375 return nil, err 376 } 377 buf, err := state.invokeGo("list", golistargs(state.cfg, words, goVersion)...) 378 if err != nil { 379 return nil, err 380 } 381 382 seen := make(map[string]*jsonPackage) 383 pkgs := make(map[string]*Package) 384 additionalErrors := make(map[string][]Error) 385 // Decode the JSON and convert it to Package form. 386 response := &DriverResponse{ 387 GoVersion: goVersion, 388 } 389 for dec := json.NewDecoder(buf); dec.More(); { 390 p := new(jsonPackage) 391 if err := dec.Decode(p); err != nil { 392 return nil, fmt.Errorf("JSON decoding failed: %v", err) 393 } 394 395 if p.ImportPath == "" { 396 // The documentation for go list says that “[e]rroneous packages will have 397 // a non-empty ImportPath”. If for some reason it comes back empty, we 398 // prefer to error out rather than silently discarding data or handing 399 // back a package without any way to refer to it. 400 if p.Error != nil { 401 return nil, Error{ 402 Pos: p.Error.Pos, 403 Msg: p.Error.Err, 404 } 405 } 406 return nil, fmt.Errorf("package missing import path: %+v", p) 407 } 408 409 // Work around https://golang.org/issue/33157: 410 // go list -e, when given an absolute path, will find the package contained at 411 // that directory. But when no package exists there, it will return a fake package 412 // with an error and the ImportPath set to the absolute path provided to go list. 413 // Try to convert that absolute path to what its package path would be if it's 414 // contained in a known module or GOPATH entry. This will allow the package to be 415 // properly "reclaimed" when overlays are processed. 416 if filepath.IsAbs(p.ImportPath) && p.Error != nil { 417 pkgPath, ok, err := state.getPkgPath(p.ImportPath) 418 if err != nil { 419 return nil, err 420 } 421 if ok { 422 p.ImportPath = pkgPath 423 } 424 } 425 426 if old, found := seen[p.ImportPath]; found { 427 // If one version of the package has an error, and the other doesn't, assume 428 // that this is a case where go list is reporting a fake dependency variant 429 // of the imported package: When a package tries to invalidly import another 430 // package, go list emits a variant of the imported package (with the same 431 // import path, but with an error on it, and the package will have a 432 // DepError set on it). An example of when this can happen is for imports of 433 // main packages: main packages can not be imported, but they may be 434 // separately matched and listed by another pattern. 435 // See golang.org/issue/36188 for more details. 436 437 // The plan is that eventually, hopefully in Go 1.15, the error will be 438 // reported on the importing package rather than the duplicate "fake" 439 // version of the imported package. Once all supported versions of Go 440 // have the new behavior this logic can be deleted. 441 // TODO(matloob): delete the workaround logic once all supported versions of 442 // Go return the errors on the proper package. 443 444 // There should be exactly one version of a package that doesn't have an 445 // error. 446 if old.Error == nil && p.Error == nil { 447 if !reflect.DeepEqual(p, old) { 448 return nil, fmt.Errorf("internal error: go list gives conflicting information for package %v", p.ImportPath) 449 } 450 continue 451 } 452 453 // Determine if this package's error needs to be bubbled up. 454 // This is a hack, and we expect for go list to eventually set the error 455 // on the package. 456 if old.Error != nil { 457 var errkind string 458 if strings.Contains(old.Error.Err, "not an importable package") { 459 errkind = "not an importable package" 460 } else if strings.Contains(old.Error.Err, "use of internal package") && strings.Contains(old.Error.Err, "not allowed") { 461 errkind = "use of internal package not allowed" 462 } 463 if errkind != "" { 464 if len(old.Error.ImportStack) < 1 { 465 return nil, fmt.Errorf(`internal error: go list gave a %q error with empty import stack`, errkind) 466 } 467 importingPkg := old.Error.ImportStack[len(old.Error.ImportStack)-1] 468 if importingPkg == old.ImportPath { 469 // Using an older version of Go which put this package itself on top of import 470 // stack, instead of the importer. Look for importer in second from top 471 // position. 472 if len(old.Error.ImportStack) < 2 { 473 return nil, fmt.Errorf(`internal error: go list gave a %q error with an import stack without importing package`, errkind) 474 } 475 importingPkg = old.Error.ImportStack[len(old.Error.ImportStack)-2] 476 } 477 additionalErrors[importingPkg] = append(additionalErrors[importingPkg], Error{ 478 Pos: old.Error.Pos, 479 Msg: old.Error.Err, 480 Kind: ListError, 481 }) 482 } 483 } 484 485 // Make sure that if there's a version of the package without an error, 486 // that's the one reported to the user. 487 if old.Error == nil { 488 continue 489 } 490 491 // This package will replace the old one at the end of the loop. 492 } 493 seen[p.ImportPath] = p 494 495 pkg := &Package{ 496 Name: p.Name, 497 ID: p.ImportPath, 498 GoFiles: absJoin(p.Dir, p.GoFiles, p.CgoFiles), 499 CompiledGoFiles: absJoin(p.Dir, p.CompiledGoFiles), 500 OtherFiles: absJoin(p.Dir, otherFiles(p)...), 501 EmbedFiles: absJoin(p.Dir, p.EmbedFiles), 502 EmbedPatterns: absJoin(p.Dir, p.EmbedPatterns), 503 IgnoredFiles: absJoin(p.Dir, p.IgnoredGoFiles, p.IgnoredOtherFiles), 504 forTest: p.ForTest, 505 depsErrors: p.DepsErrors, 506 Module: p.Module, 507 } 508 509 if (state.cfg.Mode&typecheckCgo) != 0 && len(p.CgoFiles) != 0 { 510 if len(p.CompiledGoFiles) > len(p.GoFiles) { 511 // We need the cgo definitions, which are in the first 512 // CompiledGoFile after the non-cgo ones. This is a hack but there 513 // isn't currently a better way to find it. We also need the pure 514 // Go files and unprocessed cgo files, all of which are already 515 // in pkg.GoFiles. 516 cgoTypes := p.CompiledGoFiles[len(p.GoFiles)] 517 pkg.CompiledGoFiles = append([]string{cgoTypes}, pkg.GoFiles...) 518 } else { 519 // golang/go#38990: go list silently fails to do cgo processing 520 pkg.CompiledGoFiles = nil 521 pkg.Errors = append(pkg.Errors, Error{ 522 Msg: "go list failed to return CompiledGoFiles. This may indicate failure to perform cgo processing; try building at the command line. See https://golang.org/issue/38990.", 523 Kind: ListError, 524 }) 525 } 526 } 527 528 // Work around https://golang.org/issue/28749: 529 // cmd/go puts assembly, C, and C++ files in CompiledGoFiles. 530 // Remove files from CompiledGoFiles that are non-go files 531 // (or are not files that look like they are from the cache). 532 if len(pkg.CompiledGoFiles) > 0 { 533 out := pkg.CompiledGoFiles[:0] 534 for _, f := range pkg.CompiledGoFiles { 535 if ext := filepath.Ext(f); ext != ".go" && ext != "" { // ext == "" means the file is from the cache, so probably cgo-processed file 536 continue 537 } 538 out = append(out, f) 539 } 540 pkg.CompiledGoFiles = out 541 } 542 543 // Extract the PkgPath from the package's ID. 544 if i := strings.IndexByte(pkg.ID, ' '); i >= 0 { 545 pkg.PkgPath = pkg.ID[:i] 546 } else { 547 pkg.PkgPath = pkg.ID 548 } 549 550 if pkg.PkgPath == "unsafe" { 551 pkg.CompiledGoFiles = nil // ignore fake unsafe.go file (#59929) 552 } else if len(pkg.CompiledGoFiles) == 0 { 553 // Work around for pre-go.1.11 versions of go list. 554 // TODO(matloob): they should be handled by the fallback. 555 // Can we delete this? 556 pkg.CompiledGoFiles = pkg.GoFiles 557 } 558 559 // Assume go list emits only absolute paths for Dir. 560 if p.Dir != "" && !filepath.IsAbs(p.Dir) { 561 log.Fatalf("internal error: go list returned non-absolute Package.Dir: %s", p.Dir) 562 } 563 564 if p.Export != "" && !filepath.IsAbs(p.Export) { 565 pkg.ExportFile = filepath.Join(p.Dir, p.Export) 566 } else { 567 pkg.ExportFile = p.Export 568 } 569 570 // imports 571 // 572 // Imports contains the IDs of all imported packages. 573 // ImportsMap records (path, ID) only where they differ. 574 ids := make(map[string]bool) 575 for _, id := range p.Imports { 576 ids[id] = true 577 } 578 pkg.Imports = make(map[string]*Package) 579 for path, id := range p.ImportMap { 580 pkg.Imports[path] = &Package{ID: id} // non-identity import 581 delete(ids, id) 582 } 583 for id := range ids { 584 if id == "C" { 585 continue 586 } 587 588 pkg.Imports[id] = &Package{ID: id} // identity import 589 } 590 if !p.DepOnly { 591 response.Roots = append(response.Roots, pkg.ID) 592 } 593 594 // Temporary work-around for golang/go#39986. Parse filenames out of 595 // error messages. This happens if there are unrecoverable syntax 596 // errors in the source, so we can't match on a specific error message. 597 // 598 // TODO(rfindley): remove this heuristic, in favor of considering 599 // InvalidGoFiles from the list driver. 600 if err := p.Error; err != nil && state.shouldAddFilenameFromError(p) { 601 addFilenameFromPos := func(pos string) bool { 602 split := strings.Split(pos, ":") 603 if len(split) < 1 { 604 return false 605 } 606 filename := strings.TrimSpace(split[0]) 607 if filename == "" { 608 return false 609 } 610 if !filepath.IsAbs(filename) { 611 filename = filepath.Join(state.cfg.Dir, filename) 612 } 613 info, _ := os.Stat(filename) 614 if info == nil { 615 return false 616 } 617 pkg.CompiledGoFiles = append(pkg.CompiledGoFiles, filename) 618 pkg.GoFiles = append(pkg.GoFiles, filename) 619 return true 620 } 621 found := addFilenameFromPos(err.Pos) 622 // In some cases, go list only reports the error position in the 623 // error text, not the error position. One such case is when the 624 // file's package name is a keyword (see golang.org/issue/39763). 625 if !found { 626 addFilenameFromPos(err.Err) 627 } 628 } 629 630 if p.Error != nil { 631 msg := strings.TrimSpace(p.Error.Err) // Trim to work around golang.org/issue/32363. 632 // Address golang.org/issue/35964 by appending import stack to error message. 633 if msg == "import cycle not allowed" && len(p.Error.ImportStack) != 0 { 634 msg += fmt.Sprintf(": import stack: %v", p.Error.ImportStack) 635 } 636 pkg.Errors = append(pkg.Errors, Error{ 637 Pos: p.Error.Pos, 638 Msg: msg, 639 Kind: ListError, 640 }) 641 } 642 643 pkgs[pkg.ID] = pkg 644 } 645 646 for id, errs := range additionalErrors { 647 if p, ok := pkgs[id]; ok { 648 p.Errors = append(p.Errors, errs...) 649 } 650 } 651 for _, pkg := range pkgs { 652 response.Packages = append(response.Packages, pkg) 653 } 654 sort.Slice(response.Packages, func(i, j int) bool { return response.Packages[i].ID < response.Packages[j].ID }) 655 656 return response, nil 657 } 658 659 func (state *golistState) shouldAddFilenameFromError(p *jsonPackage) bool { 660 if len(p.GoFiles) > 0 || len(p.CompiledGoFiles) > 0 { 661 return false 662 } 663 664 goV, err := state.getGoVersion() 665 if err != nil { 666 return false 667 } 668 669 // On Go 1.14 and earlier, only add filenames from errors if the import stack is empty. 670 // The import stack behaves differently for these versions than newer Go versions. 671 if goV < 15 { 672 return len(p.Error.ImportStack) == 0 673 } 674 675 // On Go 1.15 and later, only parse filenames out of error if there's no import stack, 676 // or the current package is at the top of the import stack. This is not guaranteed 677 // to work perfectly, but should avoid some cases where files in errors don't belong to this 678 // package. 679 return len(p.Error.ImportStack) == 0 || p.Error.ImportStack[len(p.Error.ImportStack)-1] == p.ImportPath 680 } 681 682 // getGoVersion returns the effective minor version of the go command. 683 func (state *golistState) getGoVersion() (int, error) { 684 state.goVersionOnce.Do(func() { 685 state.goVersion, state.goVersionError = gocommand.GoVersion(state.ctx, state.cfgInvocation(), state.cfg.gocmdRunner) 686 }) 687 return state.goVersion, state.goVersionError 688 } 689 690 // getPkgPath finds the package path of a directory if it's relative to a root 691 // directory. 692 func (state *golistState) getPkgPath(dir string) (string, bool, error) { 693 absDir, err := filepath.Abs(dir) 694 if err != nil { 695 return "", false, err 696 } 697 roots, err := state.determineRootDirs() 698 if err != nil { 699 return "", false, err 700 } 701 702 for rdir, rpath := range roots { 703 // Make sure that the directory is in the module, 704 // to avoid creating a path relative to another module. 705 if !strings.HasPrefix(absDir, rdir) { 706 continue 707 } 708 // TODO(matloob): This doesn't properly handle symlinks. 709 r, err := filepath.Rel(rdir, dir) 710 if err != nil { 711 continue 712 } 713 if rpath != "" { 714 // We choose only one root even though the directory even it can belong in multiple modules 715 // or GOPATH entries. This is okay because we only need to work with absolute dirs when a 716 // file is missing from disk, for instance when gopls calls go/packages in an overlay. 717 // Once the file is saved, gopls, or the next invocation of the tool will get the correct 718 // result straight from golist. 719 // TODO(matloob): Implement module tiebreaking? 720 return path.Join(rpath, filepath.ToSlash(r)), true, nil 721 } 722 return filepath.ToSlash(r), true, nil 723 } 724 return "", false, nil 725 } 726 727 // absJoin absolutizes and flattens the lists of files. 728 func absJoin(dir string, fileses ...[]string) (res []string) { 729 for _, files := range fileses { 730 for _, file := range files { 731 if !filepath.IsAbs(file) { 732 file = filepath.Join(dir, file) 733 } 734 res = append(res, file) 735 } 736 } 737 return res 738 } 739 740 func jsonFlag(cfg *Config, goVersion int) string { 741 if goVersion < 19 { 742 return "-json" 743 } 744 var fields []string 745 added := make(map[string]bool) 746 addFields := func(fs ...string) { 747 for _, f := range fs { 748 if !added[f] { 749 added[f] = true 750 fields = append(fields, f) 751 } 752 } 753 } 754 addFields("Name", "ImportPath", "Error") // These fields are always needed 755 if cfg.Mode&NeedFiles != 0 || cfg.Mode&NeedTypes != 0 { 756 addFields("Dir", "GoFiles", "IgnoredGoFiles", "IgnoredOtherFiles", "CFiles", 757 "CgoFiles", "CXXFiles", "MFiles", "HFiles", "FFiles", "SFiles", 758 "SwigFiles", "SwigCXXFiles", "SysoFiles") 759 if cfg.Tests { 760 addFields("TestGoFiles", "XTestGoFiles") 761 } 762 } 763 if cfg.Mode&NeedTypes != 0 { 764 // CompiledGoFiles seems to be required for the test case TestCgoNoSyntax, 765 // even when -compiled isn't passed in. 766 // TODO(#52435): Should we make the test ask for -compiled, or automatically 767 // request CompiledGoFiles in certain circumstances? 768 addFields("Dir", "CompiledGoFiles") 769 } 770 if cfg.Mode&NeedCompiledGoFiles != 0 { 771 addFields("Dir", "CompiledGoFiles", "Export") 772 } 773 if cfg.Mode&NeedImports != 0 { 774 // When imports are requested, DepOnly is used to distinguish between packages 775 // explicitly requested and transitive imports of those packages. 776 addFields("DepOnly", "Imports", "ImportMap") 777 if cfg.Tests { 778 addFields("TestImports", "XTestImports") 779 } 780 } 781 if cfg.Mode&NeedDeps != 0 { 782 addFields("DepOnly") 783 } 784 if usesExportData(cfg) { 785 // Request Dir in the unlikely case Export is not absolute. 786 addFields("Dir", "Export") 787 } 788 if cfg.Mode&needInternalForTest != 0 { 789 addFields("ForTest") 790 } 791 if cfg.Mode&needInternalDepsErrors != 0 { 792 addFields("DepsErrors") 793 } 794 if cfg.Mode&NeedModule != 0 { 795 addFields("Module") 796 } 797 if cfg.Mode&NeedEmbedFiles != 0 { 798 addFields("EmbedFiles") 799 } 800 if cfg.Mode&NeedEmbedPatterns != 0 { 801 addFields("EmbedPatterns") 802 } 803 return "-json=" + strings.Join(fields, ",") 804 } 805 806 func golistargs(cfg *Config, words []string, goVersion int) []string { 807 const findFlags = NeedImports | NeedTypes | NeedSyntax | NeedTypesInfo 808 fullargs := []string{ 809 "-e", jsonFlag(cfg, goVersion), 810 fmt.Sprintf("-compiled=%t", cfg.Mode&(NeedCompiledGoFiles|NeedSyntax|NeedTypes|NeedTypesInfo|NeedTypesSizes) != 0), 811 fmt.Sprintf("-test=%t", cfg.Tests), 812 fmt.Sprintf("-export=%t", usesExportData(cfg)), 813 fmt.Sprintf("-deps=%t", cfg.Mode&NeedImports != 0), 814 // go list doesn't let you pass -test and -find together, 815 // probably because you'd just get the TestMain. 816 fmt.Sprintf("-find=%t", !cfg.Tests && cfg.Mode&findFlags == 0 && !usesExportData(cfg)), 817 } 818 819 // golang/go#60456: with go1.21 and later, go list serves pgo variants, which 820 // can be costly to compute and may result in redundant processing for the 821 // caller. Disable these variants. If someone wants to add e.g. a NeedPGO 822 // mode flag, that should be a separate proposal. 823 if goVersion >= 21 { 824 fullargs = append(fullargs, "-pgo=off") 825 } 826 827 fullargs = append(fullargs, cfg.BuildFlags...) 828 fullargs = append(fullargs, "--") 829 fullargs = append(fullargs, words...) 830 return fullargs 831 } 832 833 // cfgInvocation returns an Invocation that reflects cfg's settings. 834 func (state *golistState) cfgInvocation() gocommand.Invocation { 835 cfg := state.cfg 836 return gocommand.Invocation{ 837 BuildFlags: cfg.BuildFlags, 838 ModFile: cfg.modFile, 839 ModFlag: cfg.modFlag, 840 CleanEnv: cfg.Env != nil, 841 Env: cfg.Env, 842 Logf: cfg.Logf, 843 WorkingDir: cfg.Dir, 844 } 845 } 846 847 // invokeGo returns the stdout of a go command invocation. 848 func (state *golistState) invokeGo(verb string, args ...string) (*bytes.Buffer, error) { 849 cfg := state.cfg 850 851 inv := state.cfgInvocation() 852 853 // For Go versions 1.16 and above, `go list` accepts overlays directly via 854 // the -overlay flag. Set it, if it's available. 855 // 856 // The check for "list" is not necessarily required, but we should avoid 857 // getting the go version if possible. 858 if verb == "list" { 859 goVersion, err := state.getGoVersion() 860 if err != nil { 861 return nil, err 862 } 863 if goVersion >= 16 { 864 filename, cleanup, err := state.writeOverlays() 865 if err != nil { 866 return nil, err 867 } 868 defer cleanup() 869 inv.Overlay = filename 870 } 871 } 872 inv.Verb = verb 873 inv.Args = args 874 gocmdRunner := cfg.gocmdRunner 875 if gocmdRunner == nil { 876 gocmdRunner = &gocommand.Runner{} 877 } 878 stdout, stderr, friendlyErr, err := gocmdRunner.RunRaw(cfg.Context, inv) 879 if err != nil { 880 // Check for 'go' executable not being found. 881 if ee, ok := err.(*exec.Error); ok && ee.Err == exec.ErrNotFound { 882 return nil, fmt.Errorf("'go list' driver requires 'go', but %s", exec.ErrNotFound) 883 } 884 885 exitErr, ok := err.(*exec.ExitError) 886 if !ok { 887 // Catastrophic error: 888 // - context cancellation 889 return nil, fmt.Errorf("couldn't run 'go': %w", err) 890 } 891 892 // Old go version? 893 if strings.Contains(stderr.String(), "flag provided but not defined") { 894 return nil, goTooOldError{fmt.Errorf("unsupported version of go: %s: %s", exitErr, stderr)} 895 } 896 897 // Related to #24854 898 if len(stderr.String()) > 0 && strings.Contains(stderr.String(), "unexpected directory layout") { 899 return nil, friendlyErr 900 } 901 902 // Is there an error running the C compiler in cgo? This will be reported in the "Error" field 903 // and should be suppressed by go list -e. 904 // 905 // This condition is not perfect yet because the error message can include other error messages than runtime/cgo. 906 isPkgPathRune := func(r rune) bool { 907 // From https://golang.org/ref/spec#Import_declarations: 908 // Implementation restriction: A compiler may restrict ImportPaths to non-empty strings 909 // using only characters belonging to Unicode's L, M, N, P, and S general categories 910 // (the Graphic characters without spaces) and may also exclude the 911 // characters !"#$%&'()*,:;<=>?[\]^`{|} and the Unicode replacement character U+FFFD. 912 return unicode.IsOneOf([]*unicode.RangeTable{unicode.L, unicode.M, unicode.N, unicode.P, unicode.S}, r) && 913 !strings.ContainsRune("!\"#$%&'()*,:;<=>?[\\]^`{|}\uFFFD", r) 914 } 915 // golang/go#36770: Handle case where cmd/go prints module download messages before the error. 916 msg := stderr.String() 917 for strings.HasPrefix(msg, "go: downloading") { 918 msg = msg[strings.IndexRune(msg, '\n')+1:] 919 } 920 if len(stderr.String()) > 0 && strings.HasPrefix(stderr.String(), "# ") { 921 msg := msg[len("# "):] 922 if strings.HasPrefix(strings.TrimLeftFunc(msg, isPkgPathRune), "\n") { 923 return stdout, nil 924 } 925 // Treat pkg-config errors as a special case (golang.org/issue/36770). 926 if strings.HasPrefix(msg, "pkg-config") { 927 return stdout, nil 928 } 929 } 930 931 // This error only appears in stderr. See golang.org/cl/166398 for a fix in go list to show 932 // the error in the Err section of stdout in case -e option is provided. 933 // This fix is provided for backwards compatibility. 934 if len(stderr.String()) > 0 && strings.Contains(stderr.String(), "named files must be .go files") { 935 output := fmt.Sprintf(`{"ImportPath": "command-line-arguments","Incomplete": true,"Error": {"Pos": "","Err": %q}}`, 936 strings.Trim(stderr.String(), "\n")) 937 return bytes.NewBufferString(output), nil 938 } 939 940 // Similar to the previous error, but currently lacks a fix in Go. 941 if len(stderr.String()) > 0 && strings.Contains(stderr.String(), "named files must all be in one directory") { 942 output := fmt.Sprintf(`{"ImportPath": "command-line-arguments","Incomplete": true,"Error": {"Pos": "","Err": %q}}`, 943 strings.Trim(stderr.String(), "\n")) 944 return bytes.NewBufferString(output), nil 945 } 946 947 // Backwards compatibility for Go 1.11 because 1.12 and 1.13 put the directory in the ImportPath. 948 // If the package doesn't exist, put the absolute path of the directory into the error message, 949 // as Go 1.13 list does. 950 const noSuchDirectory = "no such directory" 951 if len(stderr.String()) > 0 && strings.Contains(stderr.String(), noSuchDirectory) { 952 errstr := stderr.String() 953 abspath := strings.TrimSpace(errstr[strings.Index(errstr, noSuchDirectory)+len(noSuchDirectory):]) 954 output := fmt.Sprintf(`{"ImportPath": %q,"Incomplete": true,"Error": {"Pos": "","Err": %q}}`, 955 abspath, strings.Trim(stderr.String(), "\n")) 956 return bytes.NewBufferString(output), nil 957 } 958 959 // Workaround for #29280: go list -e has incorrect behavior when an ad-hoc package doesn't exist. 960 // Note that the error message we look for in this case is different that the one looked for above. 961 if len(stderr.String()) > 0 && strings.Contains(stderr.String(), "no such file or directory") { 962 output := fmt.Sprintf(`{"ImportPath": "command-line-arguments","Incomplete": true,"Error": {"Pos": "","Err": %q}}`, 963 strings.Trim(stderr.String(), "\n")) 964 return bytes.NewBufferString(output), nil 965 } 966 967 // Workaround for #34273. go list -e with GO111MODULE=on has incorrect behavior when listing a 968 // directory outside any module. 969 if len(stderr.String()) > 0 && strings.Contains(stderr.String(), "outside available modules") { 970 output := fmt.Sprintf(`{"ImportPath": %q,"Incomplete": true,"Error": {"Pos": "","Err": %q}}`, 971 // TODO(matloob): command-line-arguments isn't correct here. 972 "command-line-arguments", strings.Trim(stderr.String(), "\n")) 973 return bytes.NewBufferString(output), nil 974 } 975 976 // Another variation of the previous error 977 if len(stderr.String()) > 0 && strings.Contains(stderr.String(), "outside module root") { 978 output := fmt.Sprintf(`{"ImportPath": %q,"Incomplete": true,"Error": {"Pos": "","Err": %q}}`, 979 // TODO(matloob): command-line-arguments isn't correct here. 980 "command-line-arguments", strings.Trim(stderr.String(), "\n")) 981 return bytes.NewBufferString(output), nil 982 } 983 984 // Workaround for an instance of golang.org/issue/26755: go list -e will return a non-zero exit 985 // status if there's a dependency on a package that doesn't exist. But it should return 986 // a zero exit status and set an error on that package. 987 if len(stderr.String()) > 0 && strings.Contains(stderr.String(), "no Go files in") { 988 // Don't clobber stdout if `go list` actually returned something. 989 if len(stdout.String()) > 0 { 990 return stdout, nil 991 } 992 // try to extract package name from string 993 stderrStr := stderr.String() 994 var importPath string 995 colon := strings.Index(stderrStr, ":") 996 if colon > 0 && strings.HasPrefix(stderrStr, "go build ") { 997 importPath = stderrStr[len("go build "):colon] 998 } 999 output := fmt.Sprintf(`{"ImportPath": %q,"Incomplete": true,"Error": {"Pos": "","Err": %q}}`, 1000 importPath, strings.Trim(stderrStr, "\n")) 1001 return bytes.NewBufferString(output), nil 1002 } 1003 1004 // Export mode entails a build. 1005 // If that build fails, errors appear on stderr 1006 // (despite the -e flag) and the Export field is blank. 1007 // Do not fail in that case. 1008 // The same is true if an ad-hoc package given to go list doesn't exist. 1009 // TODO(matloob): Remove these once we can depend on go list to exit with a zero status with -e even when 1010 // packages don't exist or a build fails. 1011 if !usesExportData(cfg) && !containsGoFile(args) { 1012 return nil, friendlyErr 1013 } 1014 } 1015 return stdout, nil 1016 } 1017 1018 // OverlayJSON is the format overlay files are expected to be in. 1019 // The Replace map maps from overlaid paths to replacement paths: 1020 // the Go command will forward all reads trying to open 1021 // each overlaid path to its replacement path, or consider the overlaid 1022 // path not to exist if the replacement path is empty. 1023 // 1024 // From golang/go#39958. 1025 type OverlayJSON struct { 1026 Replace map[string]string `json:"replace,omitempty"` 1027 } 1028 1029 // writeOverlays writes out files for go list's -overlay flag, as described 1030 // above. 1031 func (state *golistState) writeOverlays() (filename string, cleanup func(), err error) { 1032 // Do nothing if there are no overlays in the config. 1033 if len(state.cfg.Overlay) == 0 { 1034 return "", func() {}, nil 1035 } 1036 dir, err := os.MkdirTemp("", "gopackages-*") 1037 if err != nil { 1038 return "", nil, err 1039 } 1040 // The caller must clean up this directory, unless this function returns an 1041 // error. 1042 cleanup = func() { 1043 os.RemoveAll(dir) 1044 } 1045 defer func() { 1046 if err != nil { 1047 cleanup() 1048 } 1049 }() 1050 overlays := map[string]string{} 1051 for k, v := range state.cfg.Overlay { 1052 // Create a unique filename for the overlaid files, to avoid 1053 // creating nested directories. 1054 noSeparator := strings.Join(strings.Split(filepath.ToSlash(k), "/"), "") 1055 f, err := os.CreateTemp(dir, fmt.Sprintf("*-%s", noSeparator)) 1056 if err != nil { 1057 return "", func() {}, err 1058 } 1059 if _, err := f.Write(v); err != nil { 1060 return "", func() {}, err 1061 } 1062 if err := f.Close(); err != nil { 1063 return "", func() {}, err 1064 } 1065 overlays[k] = f.Name() 1066 } 1067 b, err := json.Marshal(OverlayJSON{Replace: overlays}) 1068 if err != nil { 1069 return "", func() {}, err 1070 } 1071 // Write out the overlay file that contains the filepath mappings. 1072 filename = filepath.Join(dir, "overlay.json") 1073 if err := os.WriteFile(filename, b, 0665); err != nil { 1074 return "", func() {}, err 1075 } 1076 return filename, cleanup, nil 1077 } 1078 1079 func containsGoFile(s []string) bool { 1080 for _, f := range s { 1081 if strings.HasSuffix(f, ".go") { 1082 return true 1083 } 1084 } 1085 return false 1086 } 1087 1088 func cmdDebugStr(cmd *exec.Cmd) string { 1089 env := make(map[string]string) 1090 for _, kv := range cmd.Env { 1091 split := strings.SplitN(kv, "=", 2) 1092 k, v := split[0], split[1] 1093 env[k] = v 1094 } 1095 1096 var args []string 1097 for _, arg := range cmd.Args { 1098 quoted := strconv.Quote(arg) 1099 if quoted[1:len(quoted)-1] != arg || strings.Contains(arg, " ") { 1100 args = append(args, quoted) 1101 } else { 1102 args = append(args, arg) 1103 } 1104 } 1105 return fmt.Sprintf("GOROOT=%v GOPATH=%v GO111MODULE=%v GOPROXY=%v PWD=%v %v", env["GOROOT"], env["GOPATH"], env["GO111MODULE"], env["GOPROXY"], env["PWD"], strings.Join(args, " ")) 1106 }