github.com/andrewlunde/glide@v0.20.1/dependency/resolver.go (about) 1 package dependency 2 3 import ( 4 "container/list" 5 "errors" 6 "runtime" 7 "sort" 8 9 //"go/build" 10 "os" 11 "path/filepath" 12 "strings" 13 14 "github.com/andrewlunde/glide/cfg" 15 "github.com/andrewlunde/glide/msg" 16 gpath "github.com/andrewlunde/glide/path" 17 "github.com/andrewlunde/glide/util" 18 ) 19 20 // MissingPackageHandler handles the case where a package is missing during scanning. 21 // 22 // It returns true if the package can be passed to the resolver, false otherwise. 23 // False may be returned even if error is nil. 24 type MissingPackageHandler interface { 25 // NotFound is called when the Resolver fails to find a package with the given name. 26 // 27 // NotFound returns true when the resolver should attempt to re-resole the 28 // dependency (e.g. when NotFound has gone and fetched the missing package). 29 // 30 // When NotFound returns false, the Resolver does not try to do any additional 31 // work on the missing package. 32 // 33 // NotFound only returns errors when it fails to perform its internal goals. 34 // When it returns false with no error, this indicates that the handler did 35 // its job, but the resolver should not do any additional work on the 36 // package. 37 NotFound(pkg string, addTest bool) (bool, error) 38 39 // OnGopath is called when the Resolver finds a dependency, but it's only on GOPATH. 40 // 41 // OnGopath provides an opportunity to copy, move, warn, or ignore cases like this. 42 // 43 // OnGopath returns true when the resolver should attempt to re-resolve the 44 // dependency (e.g. when the dependency is copied to a new location). 45 // 46 // When OnGopath returns false, the Resolver does not try to do any additional 47 // work on the package. 48 // 49 // An error indicates that OnGopath cannot complete its intended operation. 50 // Not all false results are errors. 51 OnGopath(pkg string, addTest bool) (bool, error) 52 53 // InVendor is called when the Resolver finds a dependency in the vendor/ directory. 54 // 55 // This can be used update a project found in the vendor/ folder. 56 InVendor(pkg string, addTest bool) error 57 58 // PkgPath is called to find the location locally to scan. This gives the 59 // handler to do things such as use a cached location. 60 PkgPath(pkg string) string 61 } 62 63 // DefaultMissingPackageHandler is the default handler for missing packages. 64 // 65 // When asked to handle a missing package, it will report the miss as a warning, 66 // and then store the package in the Missing slice for later access. 67 type DefaultMissingPackageHandler struct { 68 Missing []string 69 Gopath []string 70 Prefix string 71 } 72 73 // NotFound prints a warning and then stores the package name in Missing. 74 // 75 // It never returns an error, and it always returns false. 76 func (d *DefaultMissingPackageHandler) NotFound(pkg string, addTest bool) (bool, error) { 77 msg.Warn("Package %s is not installed", pkg) 78 d.Missing = append(d.Missing, pkg) 79 return false, nil 80 } 81 82 // OnGopath is run when a package is missing from vendor/ but found in the GOPATH 83 func (d *DefaultMissingPackageHandler) OnGopath(pkg string, addTest bool) (bool, error) { 84 msg.Warn("Package %s is only on GOPATH.", pkg) 85 d.Gopath = append(d.Gopath, pkg) 86 return false, nil 87 } 88 89 // InVendor is run when a package is found in the vendor/ folder 90 func (d *DefaultMissingPackageHandler) InVendor(pkg string, addTest bool) error { 91 msg.Info("Package %s found in vendor/ folder", pkg) 92 return nil 93 } 94 95 // PkgPath returns the path to the package 96 func (d *DefaultMissingPackageHandler) PkgPath(pkg string) string { 97 if d.Prefix != "" { 98 return filepath.Join(d.Prefix, pkg) 99 } 100 return pkg 101 } 102 103 // VersionHandler sets the version for a package when found while scanning. 104 // 105 // When a package if found it needs to be on the correct version before 106 // scanning its contents to be sure to pick up the right elements for that 107 // version. 108 type VersionHandler interface { 109 110 // Process provides an opportunity to process the codebase for version setting. 111 Process(pkg string) error 112 113 // SetVersion sets the version for a package. An error is returned if there 114 // was a problem setting the version. 115 SetVersion(pkg string, testDep bool) error 116 } 117 118 // DefaultVersionHandler is the default handler for setting the version. 119 // 120 // The default handler leaves the current version and skips setting a version. 121 // For a handler that alters the version see the handler included in the repo 122 // package as part of the installer. 123 type DefaultVersionHandler struct{} 124 125 // Process a package to aide in version setting. 126 func (d *DefaultVersionHandler) Process(pkg string) error { 127 return nil 128 } 129 130 // SetVersion here sends a message when a package is found noting that it 131 // did not set the version. 132 func (d *DefaultVersionHandler) SetVersion(pkg string, testDep bool) error { 133 msg.Warn("Version not set for package %s", pkg) 134 return nil 135 } 136 137 // Resolver resolves a dependency tree. 138 // 139 // It operates in two modes: 140 // - local resolution (ResolveLocal) determines the dependencies of the local project. 141 // - vendor resolving (Resolve, ResolveAll) determines the dependencies of vendored 142 // projects. 143 // 144 // Local resolution is for guessing initial dependencies. Vendor resolution is 145 // for determining vendored dependencies. 146 type Resolver struct { 147 Handler MissingPackageHandler 148 VersionHandler VersionHandler 149 VendorDir string 150 BuildContext *util.BuildCtxt 151 Config *cfg.Config 152 153 // ResolveAllFiles toggles deep scanning. 154 // If this is true, resolve by scanning all files, not by walking the 155 // import tree. 156 ResolveAllFiles bool 157 158 // ResolveTest sets if test dependencies should be resolved. 159 ResolveTest bool 160 161 // Items already in the queue. 162 alreadyQ map[string]bool 163 164 // Attempts to scan that had unrecoverable error. 165 hadError map[string]bool 166 167 basedir string 168 seen map[string]bool 169 170 // findCache caches hits from Find. This reduces the number of filesystem 171 // touches that have to be done for dependency resolution. 172 findCache map[string]*PkgInfo 173 } 174 175 // NewResolver returns a new Resolver initialized with the DefaultMissingPackageHandler. 176 // 177 // This will return an error if the given path does not meet the basic criteria 178 // for a Go source project. For example, basedir must have a vendor subdirectory. 179 // 180 // The BuildContext uses the "go/build".Default to resolve dependencies. 181 func NewResolver(basedir string) (*Resolver, error) { 182 183 var err error 184 basedir, err = filepath.Abs(basedir) 185 if err != nil { 186 return nil, err 187 } 188 189 basedir, err = checkForBasedirSymlink(basedir) 190 191 if err != nil { 192 return nil, err 193 } 194 195 vdir := filepath.Join(basedir, "vendor") 196 197 buildContext, err := util.GetBuildContext() 198 if err != nil { 199 return nil, err 200 } 201 202 r := &Resolver{ 203 Handler: &DefaultMissingPackageHandler{Missing: []string{}, Gopath: []string{}}, 204 VersionHandler: &DefaultVersionHandler{}, 205 basedir: basedir, 206 VendorDir: vdir, 207 BuildContext: buildContext, 208 seen: map[string]bool{}, 209 alreadyQ: map[string]bool{}, 210 hadError: map[string]bool{}, 211 findCache: map[string]*PkgInfo{}, 212 213 // The config instance here should really be replaced with a real one. 214 Config: &cfg.Config{}, 215 } 216 217 // TODO: Make sure the build context is correctly set up. Especially in 218 // regards to GOROOT, which is not always set. 219 220 return r, nil 221 } 222 223 // Resolve takes a package name and returns all of the imported package names. 224 // 225 // If a package is not found, this calls the Fetcher. If the Fetcher returns 226 // true, it will re-try traversing that package for dependencies. Otherwise it 227 // will add that package to the deps array and continue on without trying it. 228 // And if the Fetcher returns an error, this will stop resolution and return 229 // the error. 230 // 231 // If basepath is set to $GOPATH, this will start from that package's root there. 232 // If basepath is set to a project's vendor path, the scanning will begin from 233 // there. 234 func (r *Resolver) Resolve(pkg, basepath string) ([]string, error) { 235 target := filepath.Join(basepath, filepath.FromSlash(pkg)) 236 //msg.Debug("Scanning %s", target) 237 l := list.New() 238 l.PushBack(target) 239 240 // In this mode, walk the entire tree. 241 if r.ResolveAllFiles { 242 return r.resolveList(l, false, false) 243 } 244 return r.resolveImports(l, false, false) 245 } 246 247 // dirHasPrefix tests whether the directory dir begins with prefix. 248 func dirHasPrefix(dir, prefix string) bool { 249 if runtime.GOOS != "windows" { 250 return strings.HasPrefix(dir, prefix) 251 } 252 return len(dir) >= len(prefix) && strings.EqualFold(dir[:len(prefix)], prefix) 253 } 254 255 // ResolveLocal resolves dependencies for the current project. 256 // 257 // This begins with the project, builds up a list of external dependencies. 258 // 259 // If the deep flag is set to true, this will then resolve all of the dependencies 260 // of the dependencies it has found. If not, it will return just the packages that 261 // the base project relies upon. 262 func (r *Resolver) ResolveLocal(deep bool) ([]string, []string, error) { 263 // We build a list of local source to walk, then send this list 264 // to resolveList. 265 msg.Debug("Resolving local dependencies") 266 l := list.New() 267 tl := list.New() 268 alreadySeen := map[string]bool{} 269 talreadySeen := map[string]bool{} 270 err := filepath.Walk(r.basedir, func(path string, fi os.FileInfo, err error) error { 271 if err != nil && err != filepath.SkipDir { 272 return err 273 } 274 pt := strings.TrimPrefix(path, r.basedir+string(os.PathSeparator)) 275 pt = strings.TrimSuffix(pt, string(os.PathSeparator)) 276 if r.Config.HasExclude(pt) { 277 msg.Debug("Excluding %s", pt) 278 return filepath.SkipDir 279 } 280 if !fi.IsDir() { 281 return nil 282 } 283 if !srcDir(fi) { 284 return filepath.SkipDir 285 } 286 287 // Scan for dependencies, and anything that's not part of the local 288 // package gets added to the scan list. 289 var imps []string 290 var testImps []string 291 p, err := r.BuildContext.ImportDir(path, 0) 292 if err != nil { 293 if strings.HasPrefix(err.Error(), "no buildable Go source") { 294 return nil 295 } else if strings.HasPrefix(err.Error(), "found packages ") { 296 // If we got here it's because a package and multiple packages 297 // declared. This is often because of an example with a package 298 // or main but +build ignore as a build tag. In that case we 299 // try to brute force the packages with a slower scan. 300 imps, testImps, err = IterativeScan(path) 301 if err != nil { 302 return err 303 } 304 } else { 305 return err 306 } 307 } else { 308 imps = p.Imports 309 testImps = dedupeStrings(p.TestImports, p.XTestImports) 310 } 311 312 // We are only looking for dependencies in vendor. No root, cgo, etc. 313 for _, imp := range imps { 314 if r.Config.HasIgnore(imp) { 315 continue 316 } 317 if alreadySeen[imp] { 318 continue 319 } 320 alreadySeen[imp] = true 321 info := r.FindPkg(imp) 322 switch info.Loc { 323 case LocUnknown, LocVendor: 324 l.PushBack(filepath.Join(r.VendorDir, filepath.FromSlash(imp))) // Do we need a path on this? 325 case LocGopath: 326 if !dirHasPrefix(info.Path, r.basedir) { 327 // FIXME: This is a package outside of the project we're 328 // scanning. It should really be on vendor. But we don't 329 // want it to reference GOPATH. We want it to be detected 330 // and moved. 331 l.PushBack(filepath.Join(r.VendorDir, filepath.FromSlash(imp))) 332 } 333 case LocRelative: 334 if strings.HasPrefix(imp, "./"+gpath.VendorDir) { 335 msg.Warn("Go package resolving will resolve %s without the ./%s/ prefix", imp, gpath.VendorDir) 336 } 337 } 338 } 339 340 if r.ResolveTest { 341 for _, imp := range testImps { 342 if talreadySeen[imp] { 343 continue 344 } 345 talreadySeen[imp] = true 346 info := r.FindPkg(imp) 347 switch info.Loc { 348 case LocUnknown, LocVendor: 349 tl.PushBack(filepath.Join(r.VendorDir, filepath.FromSlash(imp))) // Do we need a path on this? 350 case LocGopath: 351 if !dirHasPrefix(info.Path, r.basedir) { 352 // FIXME: This is a package outside of the project we're 353 // scanning. It should really be on vendor. But we don't 354 // want it to reference GOPATH. We want it to be detected 355 // and moved. 356 tl.PushBack(filepath.Join(r.VendorDir, filepath.FromSlash(imp))) 357 } 358 case LocRelative: 359 if strings.HasPrefix(imp, "./"+gpath.VendorDir) { 360 msg.Warn("Go package resolving will resolve %s without the ./%s/ prefix", imp, gpath.VendorDir) 361 } 362 } 363 } 364 } 365 366 return nil 367 }) 368 369 if err != nil { 370 msg.Err("Failed to build an initial list of packages to scan: %s", err) 371 return []string{}, []string{}, err 372 } 373 374 if deep { 375 if r.ResolveAllFiles { 376 re, err := r.resolveList(l, false, false) 377 if err != nil { 378 return []string{}, []string{}, err 379 } 380 tre, err := r.resolveList(l, false, true) 381 return re, tre, err 382 } 383 re, err := r.resolveImports(l, false, false) 384 if err != nil { 385 return []string{}, []string{}, err 386 } 387 tre, err := r.resolveImports(tl, true, true) 388 return re, tre, err 389 } 390 391 // If we're not doing a deep scan, we just convert the list into an 392 // array and return. 393 res := make([]string, 0, l.Len()) 394 for e := l.Front(); e != nil; e = e.Next() { 395 res = append(res, e.Value.(string)) 396 } 397 tres := make([]string, 0, l.Len()) 398 if r.ResolveTest { 399 for e := tl.Front(); e != nil; e = e.Next() { 400 tres = append(tres, e.Value.(string)) 401 } 402 } 403 404 return res, tres, nil 405 } 406 407 // ResolveAll takes a list of packages and returns an inclusive list of all 408 // vendored dependencies. 409 // 410 // While this will scan all of the source code it can find, it will only return 411 // packages that were either explicitly passed in as deps, or were explicitly 412 // imported by the code. 413 // 414 // Packages that are either CGO or on GOROOT are ignored. Packages that are 415 // on GOPATH, but not vendored currently generate a warning. 416 // 417 // If one of the passed in packages does not exist in the vendor directory, 418 // an error is returned. 419 func (r *Resolver) ResolveAll(deps []*cfg.Dependency, addTest bool) ([]string, error) { 420 421 queue := sliceToQueue(deps, r.VendorDir) 422 423 if r.ResolveAllFiles { 424 return r.resolveList(queue, false, addTest) 425 } 426 return r.resolveImports(queue, false, addTest) 427 } 428 429 // Stripv strips the vendor/ prefix from vendored packages. 430 func (r *Resolver) Stripv(str string) string { 431 return strings.TrimPrefix(str, r.VendorDir+string(os.PathSeparator)) 432 } 433 434 // vpath adds an absolute vendor path. 435 func (r *Resolver) vpath(str string) string { 436 return filepath.Join(r.basedir, "vendor", str) 437 } 438 439 // resolveImports takes a list of existing packages and resolves their imports. 440 // 441 // It returns a list of all of the packages that it can determine are required 442 // for the given code to function. 443 // 444 // The expectation is that each item in the queue is an absolute path to a 445 // vendored package. This attempts to read that package, and then find 446 // its referenced packages. Those packages are then added to the list 447 // to be scanned next. 448 // 449 // The resolver's handler is used in the cases where a package cannot be 450 // located. 451 // 452 // testDeps specifies if the test dependencies should be resolved and addTest 453 // specifies if the dependencies should be added to the Config.DevImports. This 454 // is important because we may resolve normal dependencies of test deps and add 455 // them to the DevImports list. 456 func (r *Resolver) resolveImports(queue *list.List, testDeps, addTest bool) ([]string, error) { 457 msg.Debug("Resolving import path") 458 459 // When test deps passed in but not resolving return empty. 460 if (testDeps || addTest) && !r.ResolveTest { 461 return []string{}, nil 462 } 463 464 alreadySeen := make(map[string]bool, queue.Len()) 465 466 for e := queue.Front(); e != nil; e = e.Next() { 467 vdep := e.Value.(string) 468 dep := r.Stripv(vdep) 469 // Check if marked in the Q and then explicitly mark it. We want to know 470 // if it had previously been marked and ensure it for the future. 471 472 if alreadySeen[dep] { 473 continue 474 } 475 alreadySeen[dep] = true 476 477 _, foundQ := r.alreadyQ[dep] 478 r.alreadyQ[dep] = true 479 480 // If we've already encountered an error processing this dependency 481 // skip it. 482 _, foundErr := r.hadError[dep] 483 if foundErr { 484 continue 485 } 486 487 // Skip ignored packages 488 if r.Config.HasIgnore(dep) { 489 msg.Debug("Ignoring: %s", dep) 490 continue 491 } 492 r.VersionHandler.Process(dep) 493 // Here, we want to import the package and see what imports it has. 494 msg.Debug("Trying to open %s (%s)", dep, r.Handler.PkgPath(dep)) 495 var imps []string 496 pkg, err := r.BuildContext.ImportDir(r.Handler.PkgPath(dep), 0) 497 if err != nil && strings.HasPrefix(err.Error(), "found packages ") { 498 // If we got here it's because a package and multiple packages 499 // declared. This is often because of an example with a package 500 // or main but +build ignore as a build tag. In that case we 501 // try to brute force the packages with a slower scan. 502 msg.Debug("Using Iterative Scanning for %s", dep) 503 if testDeps { 504 _, imps, err = IterativeScan(r.Handler.PkgPath(dep)) 505 } else { 506 imps, _, err = IterativeScan(r.Handler.PkgPath(dep)) 507 } 508 509 if err != nil { 510 msg.Err("Iterative scanning error %s: %s", dep, err) 511 continue 512 } 513 } else if err != nil { 514 errStr := err.Error() 515 msg.Debug("ImportDir error on %s: %s", r.Handler.PkgPath(dep), err) 516 if strings.HasPrefix(errStr, "no buildable Go source") { 517 msg.Debug("No subpackages declared. Skipping %s.", dep) 518 continue 519 } else if osDirNotFound(err, r.Handler.PkgPath(dep)) && !foundErr && !foundQ { 520 // If the location doesn't exist, there hasn't already been an 521 // error, it's not already been in the Q then try to fetch it. 522 // When there's an error or it's already in the Q (it should be 523 // fetched if it's marked in r.alreadyQ) we skip to make sure 524 // not to get stuck in a recursion. 525 526 // If the location doesn't exist try to fetch it. 527 if ok, err2 := r.Handler.NotFound(dep, addTest); ok { 528 r.alreadyQ[dep] = true 529 alreadySeen[dep] = false 530 531 // By adding to the queue it will get reprocessed now that 532 // it exists. 533 queue.PushBack(r.vpath(dep)) 534 r.VersionHandler.SetVersion(dep, addTest) 535 } else if err2 != nil { 536 r.hadError[dep] = true 537 msg.Err("Error looking for %s: %s", dep, err2) 538 } else { 539 r.hadError[dep] = true 540 // TODO (mpb): Should we toss this into a Handler to 541 // see if this is on GOPATH and copy it? 542 msg.Info("Not found in vendor/: %s (1)", dep) 543 } 544 } else if strings.Contains(errStr, "no such file or directory") { 545 r.hadError[dep] = true 546 msg.Err("Error scanning %s: %s", dep, err) 547 msg.Err("This error means the referenced package was not found.") 548 msg.Err("Missing file or directory errors usually occur when multiple packages") 549 msg.Err("share a common dependency and the first reference encountered by the scanner") 550 msg.Err("sets the version to one that does not contain a subpackage needed required") 551 msg.Err("by another package that uses the shared dependency. Try setting a") 552 msg.Err("version in your glide.yaml that works for all packages that share this") 553 msg.Err("dependency.") 554 } else { 555 r.hadError[dep] = true 556 msg.Err("Error scanning %s: %s", dep, err) 557 } 558 continue 559 } else { 560 if testDeps { 561 imps = dedupeStrings(pkg.TestImports, pkg.XTestImports) 562 } else { 563 imps = pkg.Imports 564 } 565 566 } 567 568 // Range over all of the identified imports and see which ones we 569 // can locate. 570 for _, imp := range imps { 571 if r.Config.HasIgnore(imp) { 572 msg.Debug("Ignoring: %s", imp) 573 continue 574 } 575 pi := r.FindPkg(imp) 576 if pi.Loc != LocCgo && pi.Loc != LocGoroot && pi.Loc != LocAppengine { 577 msg.Debug("Package %s imports %s", dep, imp) 578 } 579 switch pi.Loc { 580 case LocVendor: 581 msg.Debug("In vendor: %s", imp) 582 if _, ok := r.alreadyQ[imp]; !ok { 583 msg.Debug("Marking %s to be scanned.", imp) 584 r.alreadyQ[imp] = true 585 queue.PushBack(r.vpath(imp)) 586 if err := r.Handler.InVendor(imp, addTest); err == nil { 587 r.VersionHandler.SetVersion(imp, addTest) 588 } else { 589 msg.Warn("Error updating %s: %s", imp, err) 590 } 591 } 592 case LocUnknown: 593 msg.Debug("Missing %s. Trying to resolve.", imp) 594 if ok, err := r.Handler.NotFound(imp, addTest); ok { 595 r.alreadyQ[imp] = true 596 queue.PushBack(r.vpath(imp)) 597 r.VersionHandler.SetVersion(imp, addTest) 598 } else if err != nil { 599 r.hadError[imp] = true 600 msg.Err("Error looking for %s: %s", imp, err) 601 } else { 602 r.hadError[imp] = true 603 msg.Err("Not found: %s (2)", imp) 604 } 605 case LocGopath: 606 msg.Debug("Found on GOPATH, not vendor: %s", imp) 607 if _, ok := r.alreadyQ[imp]; !ok { 608 // Only scan it if it gets moved into vendor/ 609 if ok, _ := r.Handler.OnGopath(imp, addTest); ok { 610 r.alreadyQ[imp] = true 611 queue.PushBack(r.vpath(imp)) 612 r.VersionHandler.SetVersion(imp, addTest) 613 } 614 } 615 } 616 } 617 618 } 619 620 if len(r.hadError) > 0 { 621 // Errors occurred so we return. 622 return []string{}, errors.New("Error resolving imports") 623 } 624 625 // FIXME: From here to the end is a straight copy of the resolveList() func. 626 res := make([]string, 0, queue.Len()) 627 628 // In addition to generating a list 629 for e := queue.Front(); e != nil; e = e.Next() { 630 t := r.Stripv(e.Value.(string)) 631 root, sp := util.NormalizeName(t) 632 633 if root == r.Config.Name { 634 continue 635 } 636 637 // Skip ignored packages 638 if r.Config.HasIgnore(e.Value.(string)) { 639 msg.Debug("Ignoring: %s", e.Value.(string)) 640 continue 641 } 642 643 // TODO(mattfarina): Need to eventually support devImport 644 existing := r.Config.Imports.Get(root) 645 if existing == nil && addTest { 646 existing = r.Config.DevImports.Get(root) 647 } 648 if existing != nil { 649 if sp != "" && !existing.HasSubpackage(sp) { 650 existing.Subpackages = append(existing.Subpackages, sp) 651 } 652 } else { 653 newDep := &cfg.Dependency{ 654 Name: root, 655 } 656 if sp != "" { 657 newDep.Subpackages = []string{sp} 658 } 659 660 if addTest { 661 r.Config.DevImports = append(r.Config.DevImports, newDep) 662 } else { 663 r.Config.Imports = append(r.Config.Imports, newDep) 664 } 665 } 666 res = append(res, t) 667 } 668 669 return res, nil 670 } 671 672 // resolveList takes a list and resolves it. 673 // 674 // This walks the entire file tree for the given dependencies, not just the 675 // parts that are imported directly. Using this will discover dependencies 676 // regardless of OS, and arch. 677 func (r *Resolver) resolveList(queue *list.List, testDeps, addTest bool) ([]string, error) { 678 // When test deps passed in but not resolving return empty. 679 if testDeps && !r.ResolveTest { 680 return []string{}, nil 681 } 682 683 var failedDep string 684 var failedDepPath string 685 var pkgPath string 686 for e := queue.Front(); e != nil; e = e.Next() { 687 dep := e.Value.(string) 688 t := strings.TrimPrefix(dep, r.VendorDir+string(os.PathSeparator)) 689 if r.Config.HasIgnore(t) { 690 msg.Debug("Ignoring: %s", t) 691 continue 692 } 693 r.VersionHandler.Process(t) 694 //msg.Warn("#### %s ####", dep) 695 //msg.Info("Seen Count: %d", len(r.seen)) 696 // Catch the outtermost dependency. 697 pkgPath = r.Handler.PkgPath(t) 698 failedDep = t 699 failedDepPath = pkgPath 700 err := filepath.Walk(pkgPath, func(path string, fi os.FileInfo, err error) error { 701 if err != nil && err != filepath.SkipDir { 702 return err 703 } 704 705 // Skip files. 706 if !fi.IsDir() { 707 return nil 708 } 709 // Skip dirs that are not source. 710 if !srcDir(fi) { 711 //msg.Debug("Skip resource %s", fi.Name()) 712 return filepath.SkipDir 713 } 714 715 // Anything that comes through here has already been through 716 // the queue. 717 r.alreadyQ[path] = true 718 e := r.queueUnseen(path, queue, testDeps, addTest) 719 if e != nil { 720 failedDepPath = path 721 //msg.Err("Failed to fetch dependency %s: %s", path, err) 722 } 723 return e 724 }) 725 if err != nil && err != filepath.SkipDir { 726 msg.Err("Dependency %s (%s) failed to resolve: %s.", failedDep, failedDepPath, err) 727 return []string{}, err 728 } 729 } 730 731 res := make([]string, 0, queue.Len()) 732 733 // In addition to generating a list 734 for e := queue.Front(); e != nil; e = e.Next() { 735 t := strings.TrimPrefix(e.Value.(string), r.VendorDir+string(os.PathSeparator)) 736 root, sp := util.NormalizeName(t) 737 738 if root == r.Config.Name { 739 continue 740 } 741 742 existing := r.Config.Imports.Get(root) 743 if existing == nil && addTest { 744 existing = r.Config.DevImports.Get(root) 745 } 746 747 if existing != nil { 748 if sp != "" && !existing.HasSubpackage(sp) { 749 existing.Subpackages = append(existing.Subpackages, sp) 750 } 751 } else { 752 newDep := &cfg.Dependency{ 753 Name: root, 754 } 755 if sp != "" { 756 newDep.Subpackages = []string{sp} 757 } 758 759 if addTest { 760 r.Config.DevImports = append(r.Config.DevImports, newDep) 761 } else { 762 r.Config.Imports = append(r.Config.Imports, newDep) 763 } 764 } 765 res = append(res, e.Value.(string)) 766 } 767 768 return res, nil 769 } 770 771 // queueUnseenImports scans a package's imports and adds any new ones to the 772 // processing queue. 773 func (r *Resolver) queueUnseen(pkg string, queue *list.List, testDeps, addTest bool) error { 774 // A pkg is marked "seen" as soon as we have inspected it the first time. 775 // Seen means that we have added all of its imports to the list. 776 777 // Already queued indicates that we've either already put it into the queue 778 // or intentionally not put it in the queue for fatal reasons (e.g. no 779 // buildable source). 780 781 deps, err := r.imports(pkg, testDeps, addTest) 782 if err != nil && !strings.HasPrefix(err.Error(), "no buildable Go source") { 783 msg.Err("Could not find %s: %s", pkg, err) 784 return err 785 // NOTE: If we uncomment this, we get lots of "no buildable Go source" errors, 786 // which don't ever seem to be helpful. They don't actually indicate an error 787 // condition, and it's perfectly okay to run into that condition. 788 //} else if err != nil { 789 // msg.Warn(err.Error()) 790 } 791 792 for _, d := range deps { 793 if _, ok := r.alreadyQ[d]; !ok { 794 r.alreadyQ[d] = true 795 queue.PushBack(d) 796 } 797 } 798 return nil 799 } 800 801 // imports gets all of the imports for a given package. 802 // 803 // If the package is in GOROOT, this will return an empty list (but not 804 // an error). 805 // If it cannot resolve the pkg, it will return an error. 806 func (r *Resolver) imports(pkg string, testDeps, addTest bool) ([]string, error) { 807 808 if r.Config.HasIgnore(pkg) { 809 msg.Debug("Ignoring %s", pkg) 810 return []string{}, nil 811 } 812 813 // If this pkg is marked seen, we don't scan it again. 814 if _, ok := r.seen[pkg]; ok { 815 msg.Debug("Already saw %s", pkg) 816 return []string{}, nil 817 } 818 819 // FIXME: On error this should try to NotFound to the dependency, and then import 820 // it again. 821 var imps []string 822 p, err := r.BuildContext.ImportDir(pkg, 0) 823 if err != nil && strings.HasPrefix(err.Error(), "found packages ") { 824 // If we got here it's because a package and multiple packages 825 // declared. This is often because of an example with a package 826 // or main but +build ignore as a build tag. In that case we 827 // try to brute force the packages with a slower scan. 828 if testDeps { 829 _, imps, err = IterativeScan(r.Handler.PkgPath(pkg)) 830 } else { 831 imps, _, err = IterativeScan(r.Handler.PkgPath(pkg)) 832 } 833 834 if err != nil { 835 return []string{}, err 836 } 837 } else if err != nil { 838 return []string{}, err 839 } else { 840 if testDeps { 841 imps = dedupeStrings(p.TestImports, p.XTestImports) 842 } else { 843 imps = p.Imports 844 } 845 } 846 847 // It is okay to scan a package more than once. In some cases, this is 848 // desirable because the package can change between scans (e.g. as a result 849 // of a failed scan resolving the situation). 850 msg.Debug("=> Scanning %s (%s)", p.ImportPath, pkg) 851 r.seen[pkg] = true 852 853 // Optimization: If it's in GOROOT, it has no imports worth scanning. 854 if p.Goroot { 855 return []string{}, nil 856 } 857 858 // We are only looking for dependencies in vendor. No root, cgo, etc. 859 buf := []string{} 860 for _, imp := range imps { 861 if r.Config.HasIgnore(imp) { 862 msg.Debug("Ignoring %s", imp) 863 continue 864 } 865 info := r.FindPkg(imp) 866 switch info.Loc { 867 case LocUnknown: 868 // Do we resolve here? 869 found, err := r.Handler.NotFound(imp, addTest) 870 if err != nil { 871 msg.Err("Failed to fetch %s: %s", imp, err) 872 } 873 if found { 874 buf = append(buf, filepath.Join(r.VendorDir, filepath.FromSlash(imp))) 875 r.VersionHandler.SetVersion(imp, addTest) 876 continue 877 } 878 r.seen[info.Path] = true 879 case LocVendor: 880 //msg.Debug("Vendored: %s", imp) 881 buf = append(buf, info.Path) 882 if err := r.Handler.InVendor(imp, addTest); err == nil { 883 r.VersionHandler.SetVersion(imp, addTest) 884 } else { 885 msg.Warn("Error updating %s: %s", imp, err) 886 } 887 case LocGopath: 888 found, err := r.Handler.OnGopath(imp, addTest) 889 if err != nil { 890 msg.Err("Failed to fetch %s: %s", imp, err) 891 } 892 // If the Handler marks this as found, we drop it into the buffer 893 // for subsequent processing. Otherwise, we assume that we're 894 // in a less-than-perfect, but functional, situation. 895 if found { 896 buf = append(buf, filepath.Join(r.VendorDir, filepath.FromSlash(imp))) 897 r.VersionHandler.SetVersion(imp, addTest) 898 continue 899 } 900 msg.Warn("Package %s is on GOPATH, but not vendored. Ignoring.", imp) 901 r.seen[info.Path] = true 902 default: 903 // Local packages are an odd case. CGO cannot be scanned. 904 msg.Debug("===> Skipping %s", imp) 905 } 906 } 907 908 return buf, nil 909 } 910 911 // sliceToQueue is a special-purpose function for unwrapping a slice of 912 // dependencies into a queue of fully qualified paths. 913 func sliceToQueue(deps []*cfg.Dependency, basepath string) *list.List { 914 l := list.New() 915 for _, e := range deps { 916 if len(e.Subpackages) > 0 { 917 for _, v := range e.Subpackages { 918 ip := e.Name 919 if v != "." && v != "" { 920 ip = ip + "/" + v 921 } 922 msg.Debug("Adding local Import %s to queue", ip) 923 l.PushBack(filepath.Join(basepath, filepath.FromSlash(ip))) 924 } 925 } else { 926 msg.Debug("Adding local Import %s to queue", e.Name) 927 l.PushBack(filepath.Join(basepath, filepath.FromSlash(e.Name))) 928 } 929 930 } 931 return l 932 } 933 934 // PkgLoc describes the location of the package. 935 type PkgLoc uint8 936 937 const ( 938 // LocUnknown indicates the package location is unknown (probably not present) 939 LocUnknown PkgLoc = iota 940 // LocLocal inidcates that the package is in a local dir, not GOPATH or GOROOT. 941 LocLocal 942 // LocVendor indicates that the package is in a vendor/ dir 943 LocVendor 944 // LocGopath inidcates that the package is in GOPATH 945 LocGopath 946 // LocGoroot indicates that the package is in GOROOT 947 LocGoroot 948 // LocCgo indicates that the package is a a CGO package 949 LocCgo 950 // LocAppengine indicates the package is part of the appengine SDK. It's a 951 // special build mode. https://blog.golang.org/the-app-engine-sdk-and-workspaces-gopath 952 // Why does a Google product get a special case build mode with a local 953 // package? 954 LocAppengine 955 // LocRelative indicates the package is a relative directory 956 LocRelative 957 ) 958 959 // PkgInfo represents metadata about a package found by the resolver. 960 type PkgInfo struct { 961 Name, Path string 962 Vendored bool 963 Loc PkgLoc 964 } 965 966 // PackagesAddedToStdlib is the list of packages added to the go standard lib 967 // at various points. 968 var PackagesAddedToStdlib = map[string]struct{}{ 969 // context and net/http/httptrace are packages being added to 970 // the Go 1.7 standard library. Some packages, such as golang.org/x/net 971 // are importing it with build flags in files for go1.7. 972 "context": struct{}{}, 973 "net/http/httptrace": struct{}{}, 974 975 // math.bits are packages being added to the Go 1.9 standard library. 976 // Some packages, such as github.com/RoaringBitmap/roaring are importing 977 // it with build flags in files for go1.9. 978 "math/bits": struct{}{}, 979 980 // crypto/ed25519 is a package being added to the Go 1.13 standard library. 981 // It is importing itself with build flags in files for go1.13. 982 "crypto/ed25519": struct{}{}, 983 } 984 985 // FindPkg takes a package name and attempts to find it on the filesystem 986 // 987 // The resulting PkgInfo will indicate where it was found. 988 func (r *Resolver) FindPkg(name string) *PkgInfo { 989 // We cachae results for FindPkg to reduce the number of filesystem ops 990 // that we have to do. This is a little risky because certain directories, 991 // like GOPATH, can be modified while we're running an operation, and 992 // render the cache inaccurate. 993 // 994 // Unfound items (LocUnknown) are never cached because we assume that as 995 // part of the response, the Resolver may fetch that dependency. 996 if i, ok := r.findCache[name]; ok { 997 //msg.Info("Cache hit on %s", name) 998 return i 999 } 1000 1001 // 502 individual packages scanned. 1002 // No cache: 1003 // glide -y etcd.yaml list 0.27s user 0.19s system 85% cpu 0.534 total 1004 // With cache: 1005 // glide -y etcd.yaml list 0.22s user 0.15s system 85% cpu 0.438 total 1006 1007 var p string 1008 info := &PkgInfo{ 1009 Name: name, 1010 } 1011 1012 if strings.HasPrefix(name, "./") || strings.HasPrefix(name, "../") { 1013 info.Loc = LocRelative 1014 r.findCache[name] = info 1015 return info 1016 } 1017 1018 // Check _only_ if this dep is in the current vendor directory. 1019 p = filepath.Join(r.VendorDir, filepath.FromSlash(name)) 1020 if pkgExists(p) { 1021 info.Path = p 1022 info.Loc = LocVendor 1023 info.Vendored = true 1024 r.findCache[name] = info 1025 return info 1026 } 1027 1028 // TODO: Do we need this if we always flatten? 1029 // Recurse backward to scan other vendor/ directories 1030 //for wd := cwd; wd != "/"; wd = filepath.Dir(wd) { 1031 //p = filepath.Join(wd, "vendor", filepath.FromSlash(name)) 1032 //if fi, err = os.Stat(p); err == nil && (fi.IsDir() || isLink(fi)) { 1033 //info.Path = p 1034 //info.PType = ptypeVendor 1035 //info.Vendored = true 1036 //return info 1037 //} 1038 //} 1039 1040 // Check $GOPATH 1041 for _, rr := range filepath.SplitList(r.BuildContext.GOPATH) { 1042 p = filepath.Join(rr, "src", filepath.FromSlash(name)) 1043 if pkgExists(p) { 1044 info.Path = p 1045 info.Loc = LocGopath 1046 r.findCache[name] = info 1047 return info 1048 } 1049 } 1050 1051 // Check $GOROOT 1052 for _, rr := range filepath.SplitList(r.BuildContext.GOROOT) { 1053 p = filepath.Join(rr, "src", filepath.FromSlash(name)) 1054 if pkgExists(p) { 1055 info.Path = p 1056 info.Loc = LocGoroot 1057 r.findCache[name] = info 1058 return info 1059 } 1060 } 1061 1062 // If this is "C", we're dealing with cgo 1063 if name == "C" { 1064 info.Loc = LocCgo 1065 r.findCache[name] = info 1066 } else if name == "appengine" || name == "appengine_internal" || 1067 strings.HasPrefix(name, "appengine/") || 1068 strings.HasPrefix(name, "appengine_internal/") { 1069 // Appengine is a special case when it comes to Go builds. It is a local 1070 // looking package only available within appengine. It's a special case 1071 // where Google products are playing with each other. 1072 // https://blog.golang.org/the-app-engine-sdk-and-workspaces-gopath 1073 info.Loc = LocAppengine 1074 r.findCache[name] = info 1075 } else if _, ok := PackagesAddedToStdlib[name]; ok { 1076 // Various packages are being added to the Go standard library, and being imported 1077 // with build flags. Need to detect this and handle it. 1078 info.Loc = LocGoroot 1079 r.findCache[name] = info 1080 } 1081 1082 return info 1083 } 1084 1085 func pkgExists(path string) bool { 1086 fi, err := os.Stat(path) 1087 return err == nil && (fi.IsDir() || isLink(fi)) 1088 } 1089 1090 // isLink returns true if the given FileInfo is a symbolic link. 1091 func isLink(fi os.FileInfo) bool { 1092 return fi.Mode()&os.ModeSymlink == os.ModeSymlink 1093 } 1094 1095 // IsSrcDir returns true if this is a directory that could have source code, 1096 // false otherwise. 1097 // 1098 // Directories with _ or . prefixes are skipped, as are testdata and vendor. 1099 func IsSrcDir(fi os.FileInfo) bool { 1100 return srcDir(fi) 1101 } 1102 1103 func srcDir(fi os.FileInfo) bool { 1104 if !fi.IsDir() { 1105 return false 1106 } 1107 1108 // Ignore _foo and .foo 1109 if strings.HasPrefix(fi.Name(), "_") || strings.HasPrefix(fi.Name(), ".") { 1110 return false 1111 } 1112 1113 // Ignore testdata. For now, ignore vendor. 1114 if fi.Name() == "testdata" || fi.Name() == "vendor" { 1115 return false 1116 } 1117 1118 return true 1119 } 1120 1121 // checkForBasedirSymlink checks to see if the given basedir is actually a 1122 // symlink. In the case that it is a symlink, the symlink is read and returned. 1123 // If the basedir is not a symlink, the provided basedir argument is simply 1124 // returned back to the caller. 1125 func checkForBasedirSymlink(basedir string) (string, error) { 1126 fi, err := os.Lstat(basedir) 1127 if err != nil { 1128 return "", err 1129 } 1130 1131 if fi.Mode()&os.ModeSymlink != 0 { 1132 return os.Readlink(basedir) 1133 } 1134 1135 return basedir, nil 1136 } 1137 1138 // helper func to merge, dedupe, and sort strings 1139 func dedupeStrings(s1, s2 []string) (r []string) { 1140 dedupe := make(map[string]bool) 1141 1142 if len(s1) > 0 && len(s2) > 0 { 1143 for _, i := range s1 { 1144 dedupe[i] = true 1145 } 1146 for _, i := range s2 { 1147 dedupe[i] = true 1148 } 1149 1150 for i := range dedupe { 1151 r = append(r, i) 1152 } 1153 // And then re-sort them 1154 sort.Strings(r) 1155 } else if len(s1) > 0 { 1156 r = s1 1157 } else if len(s2) > 0 { 1158 r = s2 1159 } 1160 1161 return 1162 } 1163 1164 // In Go 1.9 go/build.ImportDir changed so that a missing dir 1165 // no longer responses with os.IsNotExist. Instead the error changed 1166 // one in the form of fmt.Errorf("cannot find package %q in:\n\t%s", path, p.Dir) 1167 // which is similar to other go/build.ImportDir errors. This function 1168 // attempts to detect when ImportDir thinks something is not found 1169 func osDirNotFound(err error, p string) bool { 1170 1171 if os.IsNotExist(err) { 1172 return true 1173 } 1174 1175 // Since there are multiple errors that start like this we need to make 1176 // sure the directory is not present 1177 if strings.HasPrefix(err.Error(), "cannot find package ") { 1178 _, nferr := os.Stat(p) 1179 if os.IsNotExist(nferr) { 1180 return true 1181 } 1182 } 1183 1184 return false 1185 }