github.com/szyn/glide@v0.12.2/dependency/resolver.go (about) 1 package dependency 2 3 import ( 4 "container/list" 5 "errors" 6 "runtime" 7 "sort" 8 //"go/build" 9 "os" 10 "path/filepath" 11 "strings" 12 13 "github.com/Masterminds/glide/cfg" 14 "github.com/Masterminds/glide/msg" 15 gpath "github.com/Masterminds/glide/path" 16 "github.com/Masterminds/glide/util" 17 ) 18 19 // MissingPackageHandler handles the case where a package is missing during scanning. 20 // 21 // It returns true if the package can be passed to the resolver, false otherwise. 22 // False may be returned even if error is nil. 23 type MissingPackageHandler interface { 24 // NotFound is called when the Resolver fails to find a package with the given name. 25 // 26 // NotFound returns true when the resolver should attempt to re-resole the 27 // dependency (e.g. when NotFound has gone and fetched the missing package). 28 // 29 // When NotFound returns false, the Resolver does not try to do any additional 30 // work on the missing package. 31 // 32 // NotFound only returns errors when it fails to perform its internal goals. 33 // When it returns false with no error, this indicates that the handler did 34 // its job, but the resolver should not do any additional work on the 35 // package. 36 NotFound(pkg string, addTest bool) (bool, error) 37 38 // OnGopath is called when the Resolver finds a dependency, but it's only on GOPATH. 39 // 40 // OnGopath provides an opportunity to copy, move, warn, or ignore cases like this. 41 // 42 // OnGopath returns true when the resolver should attempt to re-resolve the 43 // dependency (e.g. when the dependency is copied to a new location). 44 // 45 // When OnGopath returns false, the Resolver does not try to do any additional 46 // work on the package. 47 // 48 // An error indicates that OnGopath cannot complete its intended operation. 49 // Not all false results are errors. 50 OnGopath(pkg string, addTest bool) (bool, error) 51 52 // InVendor is called when the Resolver finds a dependency in the vendor/ directory. 53 // 54 // This can be used update a project found in the vendor/ folder. 55 InVendor(pkg string, addTest bool) error 56 57 // PkgPath is called to find the location locally to scan. This gives the 58 // handler to do things such as use a cached location. 59 PkgPath(pkg string) string 60 } 61 62 // DefaultMissingPackageHandler is the default handler for missing packages. 63 // 64 // When asked to handle a missing package, it will report the miss as a warning, 65 // and then store the package in the Missing slice for later access. 66 type DefaultMissingPackageHandler struct { 67 Missing []string 68 Gopath []string 69 Prefix string 70 } 71 72 // NotFound prints a warning and then stores the package name in Missing. 73 // 74 // It never returns an error, and it always returns false. 75 func (d *DefaultMissingPackageHandler) NotFound(pkg string, addTest bool) (bool, error) { 76 msg.Warn("Package %s is not installed", pkg) 77 d.Missing = append(d.Missing, pkg) 78 return false, nil 79 } 80 81 // OnGopath is run when a package is missing from vendor/ but found in the GOPATH 82 func (d *DefaultMissingPackageHandler) OnGopath(pkg string, addTest bool) (bool, error) { 83 msg.Warn("Package %s is only on GOPATH.", pkg) 84 d.Gopath = append(d.Gopath, pkg) 85 return false, nil 86 } 87 88 // InVendor is run when a package is found in the vendor/ folder 89 func (d *DefaultMissingPackageHandler) InVendor(pkg string, addTest bool) error { 90 msg.Info("Package %s found in vendor/ folder", pkg) 91 return nil 92 } 93 94 // PkgPath returns the path to the package 95 func (d *DefaultMissingPackageHandler) PkgPath(pkg string) string { 96 if d.Prefix != "" { 97 return filepath.Join(d.Prefix, pkg) 98 } 99 return pkg 100 } 101 102 // VersionHandler sets the version for a package when found while scanning. 103 // 104 // When a package if found it needs to be on the correct version before 105 // scanning its contents to be sure to pick up the right elements for that 106 // version. 107 type VersionHandler interface { 108 109 // Process provides an opportunity to process the codebase for version setting. 110 Process(pkg string) error 111 112 // SetVersion sets the version for a package. An error is returned if there 113 // was a problem setting the version. 114 SetVersion(pkg string, testDep bool) error 115 } 116 117 // DefaultVersionHandler is the default handler for setting the version. 118 // 119 // The default handler leaves the current version and skips setting a version. 120 // For a handler that alters the version see the handler included in the repo 121 // package as part of the installer. 122 type DefaultVersionHandler struct{} 123 124 // Process a package to aide in version setting. 125 func (d *DefaultVersionHandler) Process(pkg string) error { 126 return nil 127 } 128 129 // SetVersion here sends a message when a package is found noting that it 130 // did not set the version. 131 func (d *DefaultVersionHandler) SetVersion(pkg string, testDep bool) error { 132 msg.Warn("Version not set for package %s", pkg) 133 return nil 134 } 135 136 // Resolver resolves a dependency tree. 137 // 138 // It operates in two modes: 139 // - local resolution (ResolveLocal) determines the dependencies of the local project. 140 // - vendor resolving (Resolve, ResolveAll) determines the dependencies of vendored 141 // projects. 142 // 143 // Local resolution is for guessing initial dependencies. Vendor resolution is 144 // for determining vendored dependencies. 145 type Resolver struct { 146 Handler MissingPackageHandler 147 VersionHandler VersionHandler 148 VendorDir string 149 BuildContext *util.BuildCtxt 150 Config *cfg.Config 151 152 // ResolveAllFiles toggles deep scanning. 153 // If this is true, resolve by scanning all files, not by walking the 154 // import tree. 155 ResolveAllFiles bool 156 157 // ResolveTest sets if test dependencies should be resolved. 158 ResolveTest bool 159 160 // Items already in the queue. 161 alreadyQ map[string]bool 162 163 // Attempts to scan that had unrecoverable error. 164 hadError map[string]bool 165 166 basedir string 167 seen map[string]bool 168 169 // findCache caches hits from Find. This reduces the number of filesystem 170 // touches that have to be done for dependency resolution. 171 findCache map[string]*PkgInfo 172 } 173 174 // NewResolver returns a new Resolver initialized with the DefaultMissingPackageHandler. 175 // 176 // This will return an error if the given path does not meet the basic criteria 177 // for a Go source project. For example, basedir must have a vendor subdirectory. 178 // 179 // The BuildContext uses the "go/build".Default to resolve dependencies. 180 func NewResolver(basedir string) (*Resolver, error) { 181 182 var err error 183 basedir, err = filepath.Abs(basedir) 184 if err != nil { 185 return nil, err 186 } 187 188 basedir, err = checkForBasedirSymlink(basedir) 189 190 if err != nil { 191 return nil, err 192 } 193 194 vdir := filepath.Join(basedir, "vendor") 195 196 buildContext, err := util.GetBuildContext() 197 if err != nil { 198 return nil, err 199 } 200 201 r := &Resolver{ 202 Handler: &DefaultMissingPackageHandler{Missing: []string{}, Gopath: []string{}}, 203 VersionHandler: &DefaultVersionHandler{}, 204 basedir: basedir, 205 VendorDir: vdir, 206 BuildContext: buildContext, 207 seen: map[string]bool{}, 208 alreadyQ: map[string]bool{}, 209 hadError: map[string]bool{}, 210 findCache: map[string]*PkgInfo{}, 211 212 // The config instance here should really be replaced with a real one. 213 Config: &cfg.Config{}, 214 } 215 216 // TODO: Make sure the build context is correctly set up. Especially in 217 // regards to GOROOT, which is not always set. 218 219 return r, nil 220 } 221 222 // Resolve takes a package name and returns all of the imported package names. 223 // 224 // If a package is not found, this calls the Fetcher. If the Fetcher returns 225 // true, it will re-try traversing that package for dependencies. Otherwise it 226 // will add that package to the deps array and continue on without trying it. 227 // And if the Fetcher returns an error, this will stop resolution and return 228 // the error. 229 // 230 // If basepath is set to $GOPATH, this will start from that package's root there. 231 // If basepath is set to a project's vendor path, the scanning will begin from 232 // there. 233 func (r *Resolver) Resolve(pkg, basepath string) ([]string, error) { 234 target := filepath.Join(basepath, filepath.FromSlash(pkg)) 235 //msg.Debug("Scanning %s", target) 236 l := list.New() 237 l.PushBack(target) 238 239 // In this mode, walk the entire tree. 240 if r.ResolveAllFiles { 241 return r.resolveList(l, false, false) 242 } 243 return r.resolveImports(l, false, false) 244 } 245 246 // dirHasPrefix tests whether the directory dir begins with prefix. 247 func dirHasPrefix(dir, prefix string) bool { 248 if runtime.GOOS != "windows" { 249 return strings.HasPrefix(dir, prefix) 250 } 251 return len(dir) >= len(prefix) && strings.EqualFold(dir[:len(prefix)], prefix) 252 } 253 254 // ResolveLocal resolves dependencies for the current project. 255 // 256 // This begins with the project, builds up a list of external dependencies. 257 // 258 // If the deep flag is set to true, this will then resolve all of the dependencies 259 // of the dependencies it has found. If not, it will return just the packages that 260 // the base project relies upon. 261 func (r *Resolver) ResolveLocal(deep bool) ([]string, []string, error) { 262 // We build a list of local source to walk, then send this list 263 // to resolveList. 264 msg.Debug("Resolving local dependencies") 265 l := list.New() 266 tl := list.New() 267 alreadySeen := map[string]bool{} 268 talreadySeen := map[string]bool{} 269 err := filepath.Walk(r.basedir, func(path string, fi os.FileInfo, err error) error { 270 if err != nil && err != filepath.SkipDir { 271 return err 272 } 273 pt := strings.TrimPrefix(path, r.basedir+string(os.PathSeparator)) 274 pt = strings.TrimSuffix(pt, string(os.PathSeparator)) 275 if r.Config.HasExclude(pt) { 276 msg.Debug("Excluding %s", pt) 277 return filepath.SkipDir 278 } 279 if !fi.IsDir() { 280 return nil 281 } 282 if !srcDir(fi) { 283 return filepath.SkipDir 284 } 285 286 // Scan for dependencies, and anything that's not part of the local 287 // package gets added to the scan list. 288 var imps []string 289 var testImps []string 290 p, err := r.BuildContext.ImportDir(path, 0) 291 if err != nil { 292 if strings.HasPrefix(err.Error(), "no buildable Go source") { 293 return nil 294 } else if strings.HasPrefix(err.Error(), "found packages ") { 295 // If we got here it's because a package and multiple packages 296 // declared. This is often because of an example with a package 297 // or main but +build ignore as a build tag. In that case we 298 // try to brute force the packages with a slower scan. 299 imps, testImps, err = IterativeScan(path) 300 if err != nil { 301 return err 302 } 303 } else { 304 return err 305 } 306 } else { 307 imps = p.Imports 308 testImps = dedupeStrings(p.TestImports, p.XTestImports) 309 } 310 311 // We are only looking for dependencies in vendor. No root, cgo, etc. 312 for _, imp := range imps { 313 if r.Config.HasIgnore(imp) { 314 continue 315 } 316 if alreadySeen[imp] { 317 continue 318 } 319 alreadySeen[imp] = true 320 info := r.FindPkg(imp) 321 switch info.Loc { 322 case LocUnknown, LocVendor: 323 l.PushBack(filepath.Join(r.VendorDir, filepath.FromSlash(imp))) // Do we need a path on this? 324 case LocGopath: 325 if !dirHasPrefix(info.Path, r.basedir) { 326 // FIXME: This is a package outside of the project we're 327 // scanning. It should really be on vendor. But we don't 328 // want it to reference GOPATH. We want it to be detected 329 // and moved. 330 l.PushBack(filepath.Join(r.VendorDir, filepath.FromSlash(imp))) 331 } 332 case LocRelative: 333 if strings.HasPrefix(imp, "./"+gpath.VendorDir) { 334 msg.Warn("Go package resolving will resolve %s without the ./%s/ prefix", imp, gpath.VendorDir) 335 } 336 } 337 } 338 339 if r.ResolveTest { 340 for _, imp := range testImps { 341 if talreadySeen[imp] { 342 continue 343 } 344 talreadySeen[imp] = true 345 info := r.FindPkg(imp) 346 switch info.Loc { 347 case LocUnknown, LocVendor: 348 tl.PushBack(filepath.Join(r.VendorDir, filepath.FromSlash(imp))) // Do we need a path on this? 349 case LocGopath: 350 if !dirHasPrefix(info.Path, r.basedir) { 351 // FIXME: This is a package outside of the project we're 352 // scanning. It should really be on vendor. But we don't 353 // want it to reference GOPATH. We want it to be detected 354 // and moved. 355 tl.PushBack(filepath.Join(r.VendorDir, filepath.FromSlash(imp))) 356 } 357 case LocRelative: 358 if strings.HasPrefix(imp, "./"+gpath.VendorDir) { 359 msg.Warn("Go package resolving will resolve %s without the ./%s/ prefix", imp, gpath.VendorDir) 360 } 361 } 362 } 363 } 364 365 return nil 366 }) 367 368 if err != nil { 369 msg.Err("Failed to build an initial list of packages to scan: %s", err) 370 return []string{}, []string{}, err 371 } 372 373 if deep { 374 if r.ResolveAllFiles { 375 re, err := r.resolveList(l, false, false) 376 if err != nil { 377 return []string{}, []string{}, err 378 } 379 tre, err := r.resolveList(l, false, true) 380 return re, tre, err 381 } 382 re, err := r.resolveImports(l, false, false) 383 if err != nil { 384 return []string{}, []string{}, err 385 } 386 tre, err := r.resolveImports(tl, true, true) 387 return re, tre, err 388 } 389 390 // If we're not doing a deep scan, we just convert the list into an 391 // array and return. 392 res := make([]string, 0, l.Len()) 393 for e := l.Front(); e != nil; e = e.Next() { 394 res = append(res, e.Value.(string)) 395 } 396 tres := make([]string, 0, l.Len()) 397 if r.ResolveTest { 398 for e := tl.Front(); e != nil; e = e.Next() { 399 tres = append(tres, e.Value.(string)) 400 } 401 } 402 403 return res, tres, nil 404 } 405 406 // ResolveAll takes a list of packages and returns an inclusive list of all 407 // vendored dependencies. 408 // 409 // While this will scan all of the source code it can find, it will only return 410 // packages that were either explicitly passed in as deps, or were explicitly 411 // imported by the code. 412 // 413 // Packages that are either CGO or on GOROOT are ignored. Packages that are 414 // on GOPATH, but not vendored currently generate a warning. 415 // 416 // If one of the passed in packages does not exist in the vendor directory, 417 // an error is returned. 418 func (r *Resolver) ResolveAll(deps []*cfg.Dependency, addTest bool) ([]string, error) { 419 420 queue := sliceToQueue(deps, r.VendorDir) 421 422 if r.ResolveAllFiles { 423 return r.resolveList(queue, false, addTest) 424 } 425 return r.resolveImports(queue, false, addTest) 426 } 427 428 // Stripv strips the vendor/ prefix from vendored packages. 429 func (r *Resolver) Stripv(str string) string { 430 return strings.TrimPrefix(str, r.VendorDir+string(os.PathSeparator)) 431 } 432 433 // vpath adds an absolute vendor path. 434 func (r *Resolver) vpath(str string) string { 435 return filepath.Join(r.basedir, "vendor", str) 436 } 437 438 // resolveImports takes a list of existing packages and resolves their imports. 439 // 440 // It returns a list of all of the packages that it can determine are required 441 // for the given code to function. 442 // 443 // The expectation is that each item in the queue is an absolute path to a 444 // vendored package. This attempts to read that package, and then find 445 // its referenced packages. Those packages are then added to the list 446 // to be scanned next. 447 // 448 // The resolver's handler is used in the cases where a package cannot be 449 // located. 450 // 451 // testDeps specifies if the test dependencies should be resolved and addTest 452 // specifies if the dependencies should be added to the Config.DevImports. This 453 // is important because we may resolve normal dependencies of test deps and add 454 // them to the DevImports list. 455 func (r *Resolver) resolveImports(queue *list.List, testDeps, addTest bool) ([]string, error) { 456 msg.Debug("Resolving import path") 457 458 // When test deps passed in but not resolving return empty. 459 if (testDeps || addTest) && !r.ResolveTest { 460 return []string{}, nil 461 } 462 463 for e := queue.Front(); e != nil; e = e.Next() { 464 vdep := e.Value.(string) 465 dep := r.Stripv(vdep) 466 // Check if marked in the Q and then explicitly mark it. We want to know 467 // if it had previously been marked and ensure it for the future. 468 469 _, foundQ := r.alreadyQ[dep] 470 r.alreadyQ[dep] = true 471 472 // If we've already encountered an error processing this dependency 473 // skip it. 474 _, foundErr := r.hadError[dep] 475 if foundErr { 476 continue 477 } 478 479 // Skip ignored packages 480 if r.Config.HasIgnore(dep) { 481 msg.Debug("Ignoring: %s", dep) 482 continue 483 } 484 r.VersionHandler.Process(dep) 485 // Here, we want to import the package and see what imports it has. 486 msg.Debug("Trying to open %s (%s)", dep, r.Handler.PkgPath(dep)) 487 var imps []string 488 pkg, err := r.BuildContext.ImportDir(r.Handler.PkgPath(dep), 0) 489 if err != nil && strings.HasPrefix(err.Error(), "found packages ") { 490 // If we got here it's because a package and multiple packages 491 // declared. This is often because of an example with a package 492 // or main but +build ignore as a build tag. In that case we 493 // try to brute force the packages with a slower scan. 494 msg.Debug("Using Iterative Scanning for %s", dep) 495 if testDeps { 496 _, imps, err = IterativeScan(r.Handler.PkgPath(dep)) 497 } else { 498 imps, _, err = IterativeScan(r.Handler.PkgPath(dep)) 499 } 500 501 if err != nil { 502 msg.Err("Iterative scanning error %s: %s", dep, err) 503 continue 504 } 505 } else if err != nil { 506 errStr := err.Error() 507 msg.Debug("ImportDir error on %s: %s", r.Handler.PkgPath(dep), err) 508 if strings.HasPrefix(errStr, "no buildable Go source") { 509 msg.Debug("No subpackages declared. Skipping %s.", dep) 510 continue 511 } else if os.IsNotExist(err) && !foundErr && !foundQ { 512 // If the location doesn't exist, there hasn't already been an 513 // error, it's not already been in the Q then try to fetch it. 514 // When there's an error or it's already in the Q (it should be 515 // fetched if it's marked in r.alreadyQ) we skip to make sure 516 // not to get stuck in a recursion. 517 518 // If the location doesn't exist try to fetch it. 519 if ok, err2 := r.Handler.NotFound(dep, addTest); ok { 520 r.alreadyQ[dep] = true 521 522 // By adding to the queue it will get reprocessed now that 523 // it exists. 524 queue.PushBack(r.vpath(dep)) 525 r.VersionHandler.SetVersion(dep, addTest) 526 } else if err2 != nil { 527 r.hadError[dep] = true 528 msg.Err("Error looking for %s: %s", dep, err2) 529 } else { 530 r.hadError[dep] = true 531 // TODO (mpb): Should we toss this into a Handler to 532 // see if this is on GOPATH and copy it? 533 msg.Info("Not found in vendor/: %s (1)", dep) 534 } 535 } else if strings.Contains(errStr, "no such file or directory") { 536 r.hadError[dep] = true 537 msg.Err("Error scanning %s: %s", dep, err) 538 msg.Err("This error means the referenced package was not found.") 539 msg.Err("Missing file or directory errors usually occur when multiple packages") 540 msg.Err("share a common dependency and the first reference encountered by the scanner") 541 msg.Err("sets the version to one that does not contain a subpackage needed required") 542 msg.Err("by another package that uses the shared dependency. Try setting a") 543 msg.Err("version in your glide.yaml that works for all packages that share this") 544 msg.Err("dependency.") 545 } else { 546 r.hadError[dep] = true 547 msg.Err("Error scanning %s: %s", dep, err) 548 } 549 continue 550 } else { 551 if testDeps { 552 imps = dedupeStrings(pkg.TestImports, pkg.XTestImports) 553 } else { 554 imps = pkg.Imports 555 } 556 557 } 558 559 // Range over all of the identified imports and see which ones we 560 // can locate. 561 for _, imp := range imps { 562 if r.Config.HasIgnore(imp) { 563 msg.Debug("Ignoring: %s", imp) 564 continue 565 } 566 pi := r.FindPkg(imp) 567 if pi.Loc != LocCgo && pi.Loc != LocGoroot && pi.Loc != LocAppengine { 568 msg.Debug("Package %s imports %s", dep, imp) 569 } 570 switch pi.Loc { 571 case LocVendor: 572 msg.Debug("In vendor: %s", imp) 573 if _, ok := r.alreadyQ[imp]; !ok { 574 msg.Debug("Marking %s to be scanned.", imp) 575 r.alreadyQ[dep] = true 576 queue.PushBack(r.vpath(imp)) 577 if err := r.Handler.InVendor(imp, addTest); err == nil { 578 r.VersionHandler.SetVersion(imp, addTest) 579 } else { 580 msg.Warn("Error updating %s: %s", imp, err) 581 } 582 } 583 case LocUnknown: 584 msg.Debug("Missing %s. Trying to resolve.", imp) 585 if ok, err := r.Handler.NotFound(imp, addTest); ok { 586 r.alreadyQ[dep] = true 587 queue.PushBack(r.vpath(imp)) 588 r.VersionHandler.SetVersion(imp, addTest) 589 } else if err != nil { 590 r.hadError[dep] = true 591 msg.Err("Error looking for %s: %s", imp, err) 592 } else { 593 r.hadError[dep] = true 594 msg.Err("Not found: %s (2)", imp) 595 } 596 case LocGopath: 597 msg.Debug("Found on GOPATH, not vendor: %s", imp) 598 if _, ok := r.alreadyQ[imp]; !ok { 599 // Only scan it if it gets moved into vendor/ 600 if ok, _ := r.Handler.OnGopath(imp, addTest); ok { 601 r.alreadyQ[dep] = true 602 queue.PushBack(r.vpath(imp)) 603 r.VersionHandler.SetVersion(imp, addTest) 604 } 605 } 606 } 607 } 608 609 } 610 611 if len(r.hadError) > 0 { 612 // Errors occurred so we return. 613 return []string{}, errors.New("Error resolving imports") 614 } 615 616 // FIXME: From here to the end is a straight copy of the resolveList() func. 617 res := make([]string, 0, queue.Len()) 618 619 // In addition to generating a list 620 for e := queue.Front(); e != nil; e = e.Next() { 621 t := r.Stripv(e.Value.(string)) 622 root, sp := util.NormalizeName(t) 623 624 if root == r.Config.Name { 625 continue 626 } 627 628 // Skip ignored packages 629 if r.Config.HasIgnore(e.Value.(string)) { 630 msg.Debug("Ignoring: %s", e.Value.(string)) 631 continue 632 } 633 634 // TODO(mattfarina): Need to eventually support devImport 635 existing := r.Config.Imports.Get(root) 636 if existing == nil && addTest { 637 existing = r.Config.DevImports.Get(root) 638 } 639 if existing != nil { 640 if sp != "" && !existing.HasSubpackage(sp) { 641 existing.Subpackages = append(existing.Subpackages, sp) 642 } 643 } else { 644 newDep := &cfg.Dependency{ 645 Name: root, 646 } 647 if sp != "" { 648 newDep.Subpackages = []string{sp} 649 } 650 651 if addTest { 652 r.Config.DevImports = append(r.Config.DevImports, newDep) 653 } else { 654 r.Config.Imports = append(r.Config.Imports, newDep) 655 } 656 } 657 res = append(res, t) 658 } 659 660 return res, nil 661 } 662 663 // resolveList takes a list and resolves it. 664 // 665 // This walks the entire file tree for the given dependencies, not just the 666 // parts that are imported directly. Using this will discover dependencies 667 // regardless of OS, and arch. 668 func (r *Resolver) resolveList(queue *list.List, testDeps, addTest bool) ([]string, error) { 669 // When test deps passed in but not resolving return empty. 670 if testDeps && !r.ResolveTest { 671 return []string{}, nil 672 } 673 674 var failedDep string 675 var failedDepPath string 676 var pkgPath string 677 for e := queue.Front(); e != nil; e = e.Next() { 678 dep := e.Value.(string) 679 t := strings.TrimPrefix(dep, r.VendorDir+string(os.PathSeparator)) 680 if r.Config.HasIgnore(t) { 681 msg.Debug("Ignoring: %s", t) 682 continue 683 } 684 r.VersionHandler.Process(t) 685 //msg.Warn("#### %s ####", dep) 686 //msg.Info("Seen Count: %d", len(r.seen)) 687 // Catch the outtermost dependency. 688 pkgPath = r.Handler.PkgPath(t) 689 failedDep = t 690 failedDepPath = pkgPath 691 err := filepath.Walk(pkgPath, func(path string, fi os.FileInfo, err error) error { 692 if err != nil && err != filepath.SkipDir { 693 return err 694 } 695 696 // Skip files. 697 if !fi.IsDir() { 698 return nil 699 } 700 // Skip dirs that are not source. 701 if !srcDir(fi) { 702 //msg.Debug("Skip resource %s", fi.Name()) 703 return filepath.SkipDir 704 } 705 706 // Anything that comes through here has already been through 707 // the queue. 708 r.alreadyQ[path] = true 709 e := r.queueUnseen(path, queue, testDeps, addTest) 710 if e != nil { 711 failedDepPath = path 712 //msg.Err("Failed to fetch dependency %s: %s", path, err) 713 } 714 return e 715 }) 716 if err != nil && err != filepath.SkipDir { 717 msg.Err("Dependency %s (%s) failed to resolve: %s.", failedDep, failedDepPath, err) 718 return []string{}, err 719 } 720 } 721 722 res := make([]string, 0, queue.Len()) 723 724 // In addition to generating a list 725 for e := queue.Front(); e != nil; e = e.Next() { 726 t := strings.TrimPrefix(e.Value.(string), r.VendorDir+string(os.PathSeparator)) 727 root, sp := util.NormalizeName(t) 728 729 if root == r.Config.Name { 730 continue 731 } 732 733 existing := r.Config.Imports.Get(root) 734 if existing == nil && addTest { 735 existing = r.Config.DevImports.Get(root) 736 } 737 738 if existing != nil { 739 if sp != "" && !existing.HasSubpackage(sp) { 740 existing.Subpackages = append(existing.Subpackages, sp) 741 } 742 } else { 743 newDep := &cfg.Dependency{ 744 Name: root, 745 } 746 if sp != "" { 747 newDep.Subpackages = []string{sp} 748 } 749 750 if addTest { 751 r.Config.DevImports = append(r.Config.DevImports, newDep) 752 } else { 753 r.Config.Imports = append(r.Config.Imports, newDep) 754 } 755 } 756 res = append(res, e.Value.(string)) 757 } 758 759 return res, nil 760 } 761 762 // queueUnseenImports scans a package's imports and adds any new ones to the 763 // processing queue. 764 func (r *Resolver) queueUnseen(pkg string, queue *list.List, testDeps, addTest bool) error { 765 // A pkg is marked "seen" as soon as we have inspected it the first time. 766 // Seen means that we have added all of its imports to the list. 767 768 // Already queued indicates that we've either already put it into the queue 769 // or intentionally not put it in the queue for fatal reasons (e.g. no 770 // buildable source). 771 772 deps, err := r.imports(pkg, testDeps, addTest) 773 if err != nil && !strings.HasPrefix(err.Error(), "no buildable Go source") { 774 msg.Err("Could not find %s: %s", pkg, err) 775 return err 776 // NOTE: If we uncomment this, we get lots of "no buildable Go source" errors, 777 // which don't ever seem to be helpful. They don't actually indicate an error 778 // condition, and it's perfectly okay to run into that condition. 779 //} else if err != nil { 780 // msg.Warn(err.Error()) 781 } 782 783 for _, d := range deps { 784 if _, ok := r.alreadyQ[d]; !ok { 785 r.alreadyQ[d] = true 786 queue.PushBack(d) 787 } 788 } 789 return nil 790 } 791 792 // imports gets all of the imports for a given package. 793 // 794 // If the package is in GOROOT, this will return an empty list (but not 795 // an error). 796 // If it cannot resolve the pkg, it will return an error. 797 func (r *Resolver) imports(pkg string, testDeps, addTest bool) ([]string, error) { 798 799 if r.Config.HasIgnore(pkg) { 800 msg.Debug("Ignoring %s", pkg) 801 return []string{}, nil 802 } 803 804 // If this pkg is marked seen, we don't scan it again. 805 if _, ok := r.seen[pkg]; ok { 806 msg.Debug("Already saw %s", pkg) 807 return []string{}, nil 808 } 809 810 // FIXME: On error this should try to NotFound to the dependency, and then import 811 // it again. 812 var imps []string 813 p, err := r.BuildContext.ImportDir(pkg, 0) 814 if err != nil && strings.HasPrefix(err.Error(), "found packages ") { 815 // If we got here it's because a package and multiple packages 816 // declared. This is often because of an example with a package 817 // or main but +build ignore as a build tag. In that case we 818 // try to brute force the packages with a slower scan. 819 if testDeps { 820 _, imps, err = IterativeScan(r.Handler.PkgPath(pkg)) 821 } else { 822 imps, _, err = IterativeScan(r.Handler.PkgPath(pkg)) 823 } 824 825 if err != nil { 826 return []string{}, err 827 } 828 } else if err != nil { 829 return []string{}, err 830 } else { 831 if testDeps { 832 imps = dedupeStrings(p.TestImports, p.XTestImports) 833 } else { 834 imps = p.Imports 835 } 836 } 837 838 // It is okay to scan a package more than once. In some cases, this is 839 // desirable because the package can change between scans (e.g. as a result 840 // of a failed scan resolving the situation). 841 msg.Debug("=> Scanning %s (%s)", p.ImportPath, pkg) 842 r.seen[pkg] = true 843 844 // Optimization: If it's in GOROOT, it has no imports worth scanning. 845 if p.Goroot { 846 return []string{}, nil 847 } 848 849 // We are only looking for dependencies in vendor. No root, cgo, etc. 850 buf := []string{} 851 for _, imp := range imps { 852 if r.Config.HasIgnore(imp) { 853 msg.Debug("Ignoring %s", imp) 854 continue 855 } 856 info := r.FindPkg(imp) 857 switch info.Loc { 858 case LocUnknown: 859 // Do we resolve here? 860 found, err := r.Handler.NotFound(imp, addTest) 861 if err != nil { 862 msg.Err("Failed to fetch %s: %s", imp, err) 863 } 864 if found { 865 buf = append(buf, filepath.Join(r.VendorDir, filepath.FromSlash(imp))) 866 r.VersionHandler.SetVersion(imp, addTest) 867 continue 868 } 869 r.seen[info.Path] = true 870 case LocVendor: 871 //msg.Debug("Vendored: %s", imp) 872 buf = append(buf, info.Path) 873 if err := r.Handler.InVendor(imp, addTest); err == nil { 874 r.VersionHandler.SetVersion(imp, addTest) 875 } else { 876 msg.Warn("Error updating %s: %s", imp, err) 877 } 878 case LocGopath: 879 found, err := r.Handler.OnGopath(imp, addTest) 880 if err != nil { 881 msg.Err("Failed to fetch %s: %s", imp, err) 882 } 883 // If the Handler marks this as found, we drop it into the buffer 884 // for subsequent processing. Otherwise, we assume that we're 885 // in a less-than-perfect, but functional, situation. 886 if found { 887 buf = append(buf, filepath.Join(r.VendorDir, filepath.FromSlash(imp))) 888 r.VersionHandler.SetVersion(imp, addTest) 889 continue 890 } 891 msg.Warn("Package %s is on GOPATH, but not vendored. Ignoring.", imp) 892 r.seen[info.Path] = true 893 default: 894 // Local packages are an odd case. CGO cannot be scanned. 895 msg.Debug("===> Skipping %s", imp) 896 } 897 } 898 899 return buf, nil 900 } 901 902 // sliceToQueue is a special-purpose function for unwrapping a slice of 903 // dependencies into a queue of fully qualified paths. 904 func sliceToQueue(deps []*cfg.Dependency, basepath string) *list.List { 905 l := list.New() 906 for _, e := range deps { 907 if len(e.Subpackages) > 0 { 908 for _, v := range e.Subpackages { 909 ip := e.Name 910 if v != "." && v != "" { 911 ip = ip + "/" + v 912 } 913 msg.Debug("Adding local Import %s to queue", ip) 914 l.PushBack(filepath.Join(basepath, filepath.FromSlash(ip))) 915 } 916 } else { 917 msg.Debug("Adding local Import %s to queue", e.Name) 918 l.PushBack(filepath.Join(basepath, filepath.FromSlash(e.Name))) 919 } 920 921 } 922 return l 923 } 924 925 // PkgLoc describes the location of the package. 926 type PkgLoc uint8 927 928 const ( 929 // LocUnknown indicates the package location is unknown (probably not present) 930 LocUnknown PkgLoc = iota 931 // LocLocal inidcates that the package is in a local dir, not GOPATH or GOROOT. 932 LocLocal 933 // LocVendor indicates that the package is in a vendor/ dir 934 LocVendor 935 // LocGopath inidcates that the package is in GOPATH 936 LocGopath 937 // LocGoroot indicates that the package is in GOROOT 938 LocGoroot 939 // LocCgo indicates that the package is a a CGO package 940 LocCgo 941 // LocAppengine indicates the package is part of the appengine SDK. It's a 942 // special build mode. https://blog.golang.org/the-app-engine-sdk-and-workspaces-gopath 943 // Why does a Google product get a special case build mode with a local 944 // package? 945 LocAppengine 946 // LocRelative indicates the package is a relative directory 947 LocRelative 948 ) 949 950 // PkgInfo represents metadata about a package found by the resolver. 951 type PkgInfo struct { 952 Name, Path string 953 Vendored bool 954 Loc PkgLoc 955 } 956 957 // FindPkg takes a package name and attempts to find it on the filesystem 958 // 959 // The resulting PkgInfo will indicate where it was found. 960 func (r *Resolver) FindPkg(name string) *PkgInfo { 961 // We cachae results for FindPkg to reduce the number of filesystem ops 962 // that we have to do. This is a little risky because certain directories, 963 // like GOPATH, can be modified while we're running an operation, and 964 // render the cache inaccurate. 965 // 966 // Unfound items (LocUnknown) are never cached because we assume that as 967 // part of the response, the Resolver may fetch that dependency. 968 if i, ok := r.findCache[name]; ok { 969 //msg.Info("Cache hit on %s", name) 970 return i 971 } 972 973 // 502 individual packages scanned. 974 // No cache: 975 // glide -y etcd.yaml list 0.27s user 0.19s system 85% cpu 0.534 total 976 // With cache: 977 // glide -y etcd.yaml list 0.22s user 0.15s system 85% cpu 0.438 total 978 979 var p string 980 info := &PkgInfo{ 981 Name: name, 982 } 983 984 if strings.HasPrefix(name, "./") || strings.HasPrefix(name, "../") { 985 info.Loc = LocRelative 986 r.findCache[name] = info 987 return info 988 } 989 990 // Check _only_ if this dep is in the current vendor directory. 991 p = filepath.Join(r.VendorDir, filepath.FromSlash(name)) 992 if pkgExists(p) { 993 info.Path = p 994 info.Loc = LocVendor 995 info.Vendored = true 996 r.findCache[name] = info 997 return info 998 } 999 1000 // TODO: Do we need this if we always flatten? 1001 // Recurse backward to scan other vendor/ directories 1002 //for wd := cwd; wd != "/"; wd = filepath.Dir(wd) { 1003 //p = filepath.Join(wd, "vendor", filepath.FromSlash(name)) 1004 //if fi, err = os.Stat(p); err == nil && (fi.IsDir() || isLink(fi)) { 1005 //info.Path = p 1006 //info.PType = ptypeVendor 1007 //info.Vendored = true 1008 //return info 1009 //} 1010 //} 1011 1012 // Check $GOPATH 1013 for _, rr := range filepath.SplitList(r.BuildContext.GOPATH) { 1014 p = filepath.Join(rr, "src", filepath.FromSlash(name)) 1015 if pkgExists(p) { 1016 info.Path = p 1017 info.Loc = LocGopath 1018 r.findCache[name] = info 1019 return info 1020 } 1021 } 1022 1023 // Check $GOROOT 1024 for _, rr := range filepath.SplitList(r.BuildContext.GOROOT) { 1025 p = filepath.Join(rr, "src", filepath.FromSlash(name)) 1026 if pkgExists(p) { 1027 info.Path = p 1028 info.Loc = LocGoroot 1029 r.findCache[name] = info 1030 return info 1031 } 1032 } 1033 1034 // If this is "C", we're dealing with cgo 1035 if name == "C" { 1036 info.Loc = LocCgo 1037 r.findCache[name] = info 1038 } else if name == "appengine" || name == "appengine_internal" || 1039 strings.HasPrefix(name, "appengine/") || 1040 strings.HasPrefix(name, "appengine_internal/") { 1041 // Appengine is a special case when it comes to Go builds. It is a local 1042 // looking package only available within appengine. It's a special case 1043 // where Google products are playing with each other. 1044 // https://blog.golang.org/the-app-engine-sdk-and-workspaces-gopath 1045 info.Loc = LocAppengine 1046 r.findCache[name] = info 1047 } else if name == "context" || name == "net/http/httptrace" { 1048 // context and net/http/httptrace are packages being added to 1049 // the Go 1.7 standard library. Some packages, such as golang.org/x/net 1050 // are importing it with build flags in files for go1.7. Need to detect 1051 // this and handle it. 1052 info.Loc = LocGoroot 1053 r.findCache[name] = info 1054 } 1055 1056 return info 1057 } 1058 1059 func pkgExists(path string) bool { 1060 fi, err := os.Stat(path) 1061 return err == nil && (fi.IsDir() || isLink(fi)) 1062 } 1063 1064 // isLink returns true if the given FileInfo is a symbolic link. 1065 func isLink(fi os.FileInfo) bool { 1066 return fi.Mode()&os.ModeSymlink == os.ModeSymlink 1067 } 1068 1069 // IsSrcDir returns true if this is a directory that could have source code, 1070 // false otherwise. 1071 // 1072 // Directories with _ or . prefixes are skipped, as are testdata and vendor. 1073 func IsSrcDir(fi os.FileInfo) bool { 1074 return srcDir(fi) 1075 } 1076 1077 func srcDir(fi os.FileInfo) bool { 1078 if !fi.IsDir() { 1079 return false 1080 } 1081 1082 // Ignore _foo and .foo 1083 if strings.HasPrefix(fi.Name(), "_") || strings.HasPrefix(fi.Name(), ".") { 1084 return false 1085 } 1086 1087 // Ignore testdata. For now, ignore vendor. 1088 if fi.Name() == "testdata" || fi.Name() == "vendor" { 1089 return false 1090 } 1091 1092 return true 1093 } 1094 1095 // checkForBasedirSymlink checks to see if the given basedir is actually a 1096 // symlink. In the case that it is a symlink, the symlink is read and returned. 1097 // If the basedir is not a symlink, the provided basedir argument is simply 1098 // returned back to the caller. 1099 func checkForBasedirSymlink(basedir string) (string, error) { 1100 fi, err := os.Lstat(basedir) 1101 if err != nil { 1102 return "", err 1103 } 1104 1105 if fi.Mode()&os.ModeSymlink != 0 { 1106 return os.Readlink(basedir) 1107 } 1108 1109 return basedir, nil 1110 } 1111 1112 // helper func to merge, dedupe, and sort strings 1113 func dedupeStrings(s1, s2 []string) (r []string) { 1114 dedupe := make(map[string]bool) 1115 1116 if len(s1) > 0 && len(s2) > 0 { 1117 for _, i := range s1 { 1118 dedupe[i] = true 1119 } 1120 for _, i := range s2 { 1121 dedupe[i] = true 1122 } 1123 1124 for i := range dedupe { 1125 r = append(r, i) 1126 } 1127 // And then re-sort them 1128 sort.Strings(r) 1129 } else if len(s1) > 0 { 1130 r = s1 1131 } else if len(s2) > 0 { 1132 r = s2 1133 } 1134 1135 return 1136 }