github.com/mkideal/godep@v0.0.0-20160710170555-3c0ccb9a2415/list.go (about) 1 package main 2 3 import ( 4 "errors" 5 "fmt" 6 "go/build" 7 "go/parser" 8 "go/token" 9 "log" 10 "os" 11 "path/filepath" 12 "regexp" 13 "strconv" 14 "strings" 15 "unicode" 16 17 pathpkg "path" 18 ) 19 20 var ( 21 gorootSrc = filepath.Join(build.Default.GOROOT, "src") 22 ignoreTags = []string{"appengine", "ignore"} //TODO: appengine is a special case for now: https://github.com/tools/godep/issues/353 23 versionMatch = regexp.MustCompile(`\Ago\d+\.\d+\z`) 24 versionNegativeMatch = regexp.MustCompile(`\A\!go\d+\.\d+\z`) 25 ) 26 27 type errorMissingDep struct { 28 i, dir string // import, dir 29 } 30 31 func (e errorMissingDep) Error() string { 32 return "Unable to find dependent package " + e.i + " in context of " + e.dir 33 } 34 35 // packageContext is used to track an import and which package imported it. 36 type packageContext struct { 37 pkg *build.Package // package that imports the import 38 imp string // import 39 } 40 41 // depScanner tracks the processed and to be processed packageContexts 42 type depScanner struct { 43 processed []packageContext 44 todo []packageContext 45 } 46 47 // Next package and import to process 48 func (ds *depScanner) Next() (*build.Package, string) { 49 c := ds.todo[0] 50 ds.processed = append(ds.processed, c) 51 ds.todo = ds.todo[1:] 52 return c.pkg, c.imp 53 } 54 55 // Continue looping? 56 func (ds *depScanner) Continue() bool { 57 if len(ds.todo) > 0 { 58 return true 59 } 60 return false 61 } 62 63 // Add a package and imports to the depScanner. Skips already processed/pending package/import combos 64 func (ds *depScanner) Add(pkg *build.Package, imports ...string) { 65 NextImport: 66 for _, i := range imports { 67 if i == "C" { 68 i = "runtime/cgo" 69 } 70 for _, epc := range ds.processed { 71 if pkg.Dir == epc.pkg.Dir && i == epc.imp { 72 debugln("ctxts epc.pkg.Dir == pkg.Dir && i == epc.imp, skipping", epc.pkg.Dir, i) 73 continue NextImport 74 } 75 } 76 for _, epc := range ds.todo { 77 if pkg.Dir == epc.pkg.Dir && i == epc.imp { 78 debugln("ctxts epc.pkg.Dir == pkg.Dir && i == epc.imp, skipping", epc.pkg.Dir, i) 79 continue NextImport 80 } 81 } 82 pc := packageContext{pkg, i} 83 debugln("Adding pc:", pc.pkg.Dir, pc.imp) 84 ds.todo = append(ds.todo, pc) 85 } 86 } 87 88 var ( 89 pkgCache = make(map[string]*build.Package) // dir => *build.Package 90 ) 91 92 // returns the package in dir either from a cache or by importing it and then caching it 93 func fullPackageInDir(dir string) (*build.Package, error) { 94 var err error 95 pkg, ok := pkgCache[dir] 96 if !ok { 97 pkg, err = build.ImportDir(dir, build.FindOnly) 98 if pkg.Goroot { 99 pkg, err = build.ImportDir(pkg.Dir, 0) 100 } else { 101 fillPackage(pkg) 102 } 103 if err == nil { 104 pkgCache[dir] = pkg 105 } 106 } 107 return pkg, err 108 } 109 110 // listPackage specified by path 111 func listPackage(path string) (*Package, error) { 112 debugln("listPackage", path) 113 var lp *build.Package 114 dir, err := findDirForPath(path, nil) 115 if err != nil { 116 return nil, err 117 } 118 lp, err = fullPackageInDir(dir) 119 p := &Package{ 120 Dir: lp.Dir, 121 Root: lp.Root, 122 ImportPath: lp.ImportPath, 123 XTestImports: lp.XTestImports, 124 TestImports: lp.TestImports, 125 GoFiles: lp.GoFiles, 126 CgoFiles: lp.CgoFiles, 127 TestGoFiles: lp.TestGoFiles, 128 XTestGoFiles: lp.XTestGoFiles, 129 IgnoredGoFiles: lp.IgnoredGoFiles, 130 } 131 p.Standard = lp.Goroot && lp.ImportPath != "" && !strings.Contains(lp.ImportPath, ".") 132 if err != nil || p.Standard { 133 return p, err 134 } 135 debugln("Looking For Package:", path, "in", dir) 136 ppln(lp) 137 138 ds := depScanner{} 139 ds.Add(lp, lp.Imports...) 140 for ds.Continue() { 141 ip, i := ds.Next() 142 143 debugf("Processing import %s for %s\n", i, ip.Dir) 144 pdir, err := findDirForPath(i, ip) 145 if err != nil { 146 return nil, err 147 } 148 dp, err := fullPackageInDir(pdir) 149 if err != nil { // This really should happen in this context though 150 ppln(err) 151 return nil, errorMissingDep{i: i, dir: ip.Dir} 152 } 153 ppln(dp) 154 if !dp.Goroot { 155 // Don't bother adding packages in GOROOT to the dependency scanner, they don't import things from outside of it. 156 ds.Add(dp, dp.Imports...) 157 } 158 debugln("lp:") 159 ppln(lp) 160 debugln("ip:") 161 ppln(ip) 162 if lp == ip { 163 debugln("lp == ip") 164 p.Imports = append(p.Imports, dp.ImportPath) 165 } 166 p.Deps = append(p.Deps, dp.ImportPath) 167 p.Dependencies = addDependency(p.Dependencies, dp) 168 } 169 p.Imports = uniq(p.Imports) 170 p.Deps = uniq(p.Deps) 171 debugln("Done Looking For Package:", path, "in", dir) 172 ppln(p) 173 return p, nil 174 } 175 176 func addDependency(deps []build.Package, d *build.Package) []build.Package { 177 for i := range deps { 178 if deps[i].Dir == d.Dir { 179 return deps 180 } 181 } 182 return append(deps, *d) 183 } 184 185 // finds the directory for the given import path in the context of the provided build.Package (if provided) 186 func findDirForPath(path string, ip *build.Package) (string, error) { 187 debugln("findDirForPath", path, ip) 188 var search []string 189 190 if build.IsLocalImport(path) { 191 dir := path 192 if !filepath.IsAbs(dir) { 193 if abs, err := filepath.Abs(dir); err == nil { 194 // interpret relative to current directory 195 dir = abs 196 } 197 } 198 return dir, nil 199 } 200 201 // We need to check to see if the import exists in vendor/ folders up the hierarchy of the importing package 202 if VendorExperiment && ip != nil { 203 debugln("resolving vendor posibilities:", ip.Dir, ip.Root) 204 cr := cleanPath(ip.Root) 205 206 for base := cleanPath(ip.Dir); !pathEqual(base, cr); base = cleanPath(filepath.Dir(base)) { 207 s := filepath.Join(base, "vendor", path) 208 debugln("Adding search dir:", s) 209 search = append(search, s) 210 } 211 } 212 213 for _, base := range build.Default.SrcDirs() { 214 search = append(search, filepath.Join(base, path)) 215 } 216 217 for _, dir := range search { 218 debugln("searching", dir) 219 fi, err := stat(dir) 220 if err == nil && fi.IsDir() { 221 return dir, nil 222 } 223 } 224 225 return "", errPackageNotFound{path} 226 } 227 228 type statEntry struct { 229 fi os.FileInfo 230 err error 231 } 232 233 var ( 234 statCache = make(map[string]statEntry) 235 ) 236 237 func clearStatCache() { 238 statCache = make(map[string]statEntry) 239 } 240 241 func stat(p string) (os.FileInfo, error) { 242 if e, ok := statCache[p]; ok { 243 return e.fi, e.err 244 } 245 fi, err := os.Stat(p) 246 statCache[p] = statEntry{fi, err} 247 return fi, err 248 } 249 250 // fillPackage full of info. Assumes p.Dir is set at a minimum 251 func fillPackage(p *build.Package) error { 252 if p.Goroot { 253 return nil 254 } 255 256 if p.SrcRoot == "" { 257 for _, base := range build.Default.SrcDirs() { 258 if strings.HasPrefix(p.Dir, base) { 259 p.SrcRoot = base 260 } 261 } 262 } 263 264 if p.SrcRoot == "" { 265 return errors.New("Unable to find SrcRoot for package " + p.ImportPath) 266 } 267 268 if p.Root == "" { 269 p.Root = filepath.Dir(p.SrcRoot) 270 } 271 272 var buildMatch = "+build " 273 var buildFieldSplit = func(r rune) bool { 274 return unicode.IsSpace(r) || r == ',' 275 } 276 277 debugln("Filling package:", p.ImportPath, "from", p.Dir) 278 gofiles, err := filepath.Glob(filepath.Join(p.Dir, "*.go")) 279 if err != nil { 280 debugln("Error globbing", err) 281 return err 282 } 283 284 var testImports []string 285 var imports []string 286 NextFile: 287 for _, file := range gofiles { 288 debugln(file) 289 pf, err := parser.ParseFile(token.NewFileSet(), file, nil, parser.ImportsOnly|parser.ParseComments) 290 if err != nil { 291 return err 292 } 293 testFile := strings.HasSuffix(file, "_test.go") 294 fname := filepath.Base(file) 295 for _, c := range pf.Comments { 296 ct := c.Text() 297 if i := strings.Index(ct, buildMatch); i != -1 { 298 for _, t := range strings.FieldsFunc(ct[i+len(buildMatch):], buildFieldSplit) { 299 for _, tag := range ignoreTags { 300 if t == tag { 301 p.IgnoredGoFiles = append(p.IgnoredGoFiles, fname) 302 continue NextFile 303 } 304 } 305 306 if versionMatch.MatchString(t) && !isSameOrNewer(t, majorGoVersion) { 307 debugln("Adding", fname, "to ignored list because of version tag", t) 308 p.IgnoredGoFiles = append(p.IgnoredGoFiles, fname) 309 continue NextFile 310 } 311 if versionNegativeMatch.MatchString(t) && isSameOrNewer(t[1:], majorGoVersion) { 312 debugln("Adding", fname, "to ignored list because of version tag", t) 313 p.IgnoredGoFiles = append(p.IgnoredGoFiles, fname) 314 continue NextFile 315 } 316 } 317 } 318 } 319 if testFile { 320 p.TestGoFiles = append(p.TestGoFiles, fname) 321 } else { 322 p.GoFiles = append(p.GoFiles, fname) 323 } 324 for _, is := range pf.Imports { 325 name, err := strconv.Unquote(is.Path.Value) 326 if err != nil { 327 return err // can't happen? 328 } 329 if testFile { 330 testImports = append(testImports, name) 331 } else { 332 imports = append(imports, name) 333 } 334 } 335 } 336 imports = uniq(imports) 337 testImports = uniq(testImports) 338 p.Imports = imports 339 p.TestImports = testImports 340 return nil 341 } 342 343 // All of the following functions were vendored from go proper. Locations are noted in comments, but may change in future Go versions. 344 345 // importPaths returns the import paths to use for the given command line. 346 // $GOROOT/src/cmd/main.go:366 347 func importPaths(args []string) []string { 348 args = importPathsNoDotExpansion(args) 349 var out []string 350 for _, a := range args { 351 if strings.Contains(a, "...") { 352 if build.IsLocalImport(a) { 353 out = append(out, allPackagesInFS(a)...) 354 } else { 355 out = append(out, allPackages(a)...) 356 } 357 continue 358 } 359 out = append(out, a) 360 } 361 return out 362 } 363 364 // importPathsNoDotExpansion returns the import paths to use for the given 365 // command line, but it does no ... expansion. 366 // $GOROOT/src/cmd/main.go:332 367 func importPathsNoDotExpansion(args []string) []string { 368 if len(args) == 0 { 369 return []string{"."} 370 } 371 var out []string 372 for _, a := range args { 373 // Arguments are supposed to be import paths, but 374 // as a courtesy to Windows developers, rewrite \ to / 375 // in command-line arguments. Handles .\... and so on. 376 if filepath.Separator == '\\' { 377 a = strings.Replace(a, `\`, `/`, -1) 378 } 379 380 // Put argument in canonical form, but preserve leading ./. 381 if strings.HasPrefix(a, "./") { 382 a = "./" + pathpkg.Clean(a) 383 if a == "./." { 384 a = "." 385 } 386 } else { 387 a = pathpkg.Clean(a) 388 } 389 if a == "all" || a == "std" || a == "cmd" { 390 out = append(out, allPackages(a)...) 391 continue 392 } 393 out = append(out, a) 394 } 395 return out 396 } 397 398 // allPackagesInFS is like allPackages but is passed a pattern 399 // beginning ./ or ../, meaning it should scan the tree rooted 400 // at the given directory. There are ... in the pattern too. 401 // $GOROOT/src/cmd/main.go:620 402 func allPackagesInFS(pattern string) []string { 403 pkgs := matchPackagesInFS(pattern) 404 if len(pkgs) == 0 { 405 fmt.Fprintf(os.Stderr, "warning: %q matched no packages\n", pattern) 406 } 407 return pkgs 408 } 409 410 // allPackages returns all the packages that can be found 411 // under the $GOPATH directories and $GOROOT matching pattern. 412 // The pattern is either "all" (all packages), "std" (standard packages), 413 // "cmd" (standard commands), or a path including "...". 414 // $GOROOT/src/cmd/main.go:542 415 func allPackages(pattern string) []string { 416 pkgs := matchPackages(pattern) 417 if len(pkgs) == 0 { 418 fmt.Fprintf(os.Stderr, "warning: %q matched no packages\n", pattern) 419 } 420 return pkgs 421 } 422 423 // $GOROOT/src/cmd/main.go:554 424 func matchPackages(pattern string) []string { 425 match := func(string) bool { return true } 426 treeCanMatch := func(string) bool { return true } 427 if pattern != "all" && pattern != "std" && pattern != "cmd" { 428 match = matchPattern(pattern) 429 treeCanMatch = treeCanMatchPattern(pattern) 430 } 431 432 have := map[string]bool{ 433 "builtin": true, // ignore pseudo-package that exists only for documentation 434 } 435 if !build.Default.CgoEnabled { 436 have["runtime/cgo"] = true // ignore during walk 437 } 438 var pkgs []string 439 440 for _, src := range build.Default.SrcDirs() { 441 if (pattern == "std" || pattern == "cmd") && src != gorootSrc { 442 continue 443 } 444 src = filepath.Clean(src) + string(filepath.Separator) 445 root := src 446 if pattern == "cmd" { 447 root += "cmd" + string(filepath.Separator) 448 } 449 filepath.Walk(root, func(path string, fi os.FileInfo, err error) error { 450 if err != nil || !fi.IsDir() || path == src { 451 return nil 452 } 453 454 // Avoid .foo, _foo, and testdata directory trees. 455 _, elem := filepath.Split(path) 456 if strings.HasPrefix(elem, ".") || strings.HasPrefix(elem, "_") || elem == "testdata" { 457 return filepath.SkipDir 458 } 459 460 name := filepath.ToSlash(path[len(src):]) 461 if pattern == "std" && (strings.Contains(name, ".") || name == "cmd") { 462 // The name "std" is only the standard library. 463 // If the name has a dot, assume it's a domain name for go get, 464 // and if the name is cmd, it's the root of the command tree. 465 return filepath.SkipDir 466 } 467 if !treeCanMatch(name) { 468 return filepath.SkipDir 469 } 470 if have[name] { 471 return nil 472 } 473 have[name] = true 474 if !match(name) { 475 return nil 476 } 477 _, err = build.ImportDir(path, 0) 478 if err != nil { 479 if _, noGo := err.(*build.NoGoError); noGo { 480 return nil 481 } 482 } 483 pkgs = append(pkgs, name) 484 return nil 485 }) 486 } 487 return pkgs 488 } 489 490 // treeCanMatchPattern(pattern)(name) reports whether 491 // name or children of name can possibly match pattern. 492 // Pattern is the same limited glob accepted by matchPattern. 493 // $GOROOT/src/cmd/main.go:527 494 func treeCanMatchPattern(pattern string) func(name string) bool { 495 wildCard := false 496 if i := strings.Index(pattern, "..."); i >= 0 { 497 wildCard = true 498 pattern = pattern[:i] 499 } 500 return func(name string) bool { 501 return len(name) <= len(pattern) && hasPathPrefix(pattern, name) || 502 wildCard && strings.HasPrefix(name, pattern) 503 } 504 } 505 506 // hasPathPrefix reports whether the path s begins with the 507 // elements in prefix. 508 // $GOROOT/src/cmd/main.go:489 509 func hasPathPrefix(s, prefix string) bool { 510 switch { 511 default: 512 return false 513 case len(s) == len(prefix): 514 return s == prefix 515 case len(s) > len(prefix): 516 if prefix != "" && prefix[len(prefix)-1] == '/' { 517 return strings.HasPrefix(s, prefix) 518 } 519 return s[len(prefix)] == '/' && s[:len(prefix)] == prefix 520 } 521 } 522 523 // $GOROOT/src/cmd/go/main.go:631 524 func matchPackagesInFS(pattern string) []string { 525 // Find directory to begin the scan. 526 // Could be smarter but this one optimization 527 // is enough for now, since ... is usually at the 528 // end of a path. 529 i := strings.Index(pattern, "...") 530 dir, _ := pathpkg.Split(pattern[:i]) 531 532 // pattern begins with ./ or ../. 533 // path.Clean will discard the ./ but not the ../. 534 // We need to preserve the ./ for pattern matching 535 // and in the returned import paths. 536 prefix := "" 537 if strings.HasPrefix(pattern, "./") { 538 prefix = "./" 539 } 540 match := matchPattern(pattern) 541 542 var pkgs []string 543 filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error { 544 if err != nil || !fi.IsDir() { 545 return nil 546 } 547 if path == dir { 548 // filepath.Walk starts at dir and recurses. For the recursive case, 549 // the path is the result of filepath.Join, which calls filepath.Clean. 550 // The initial case is not Cleaned, though, so we do this explicitly. 551 // 552 // This converts a path like "./io/" to "io". Without this step, running 553 // "cd $GOROOT/src; go list ./io/..." would incorrectly skip the io 554 // package, because prepending the prefix "./" to the unclean path would 555 // result in "././io", and match("././io") returns false. 556 path = filepath.Clean(path) 557 } 558 559 // Avoid .foo, _foo, and testdata directory trees, but do not avoid "." or "..". 560 _, elem := filepath.Split(path) 561 dot := strings.HasPrefix(elem, ".") && elem != "." && elem != ".." 562 if dot || strings.HasPrefix(elem, "_") || elem == "testdata" { 563 return filepath.SkipDir 564 } 565 566 name := prefix + filepath.ToSlash(path) 567 if !match(name) { 568 return nil 569 } 570 if _, err = build.ImportDir(path, 0); err != nil { 571 if _, noGo := err.(*build.NoGoError); !noGo { 572 log.Print(err) 573 } 574 return nil 575 } 576 pkgs = append(pkgs, name) 577 return nil 578 }) 579 return pkgs 580 }