github.com/alash3al/go@v0.0.0-20150827002835-d497eeb00540/src/cmd/go/pkg.go (about) 1 // Copyright 2011 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package main 6 7 import ( 8 "bytes" 9 "crypto/sha1" 10 "errors" 11 "fmt" 12 "go/build" 13 "go/scanner" 14 "go/token" 15 "io" 16 "io/ioutil" 17 "os" 18 pathpkg "path" 19 "path/filepath" 20 "runtime" 21 "sort" 22 "strconv" 23 "strings" 24 "unicode" 25 ) 26 27 // A Package describes a single package found in a directory. 28 type Package struct { 29 // Note: These fields are part of the go command's public API. 30 // See list.go. It is okay to add fields, but not to change or 31 // remove existing ones. Keep in sync with list.go 32 Dir string `json:",omitempty"` // directory containing package sources 33 ImportPath string `json:",omitempty"` // import path of package in dir 34 ImportComment string `json:",omitempty"` // path in import comment on package statement 35 Name string `json:",omitempty"` // package name 36 Doc string `json:",omitempty"` // package documentation string 37 Target string `json:",omitempty"` // install path 38 Shlib string `json:",omitempty"` // the shared library that contains this package (only set when -linkshared) 39 Goroot bool `json:",omitempty"` // is this package found in the Go root? 40 Standard bool `json:",omitempty"` // is this package part of the standard Go library? 41 Stale bool `json:",omitempty"` // would 'go install' do anything for this package? 42 Root string `json:",omitempty"` // Go root or Go path dir containing this package 43 ConflictDir string `json:",omitempty"` // Dir is hidden by this other directory 44 45 // Source files 46 GoFiles []string `json:",omitempty"` // .go source files (excluding CgoFiles, TestGoFiles, XTestGoFiles) 47 CgoFiles []string `json:",omitempty"` // .go sources files that import "C" 48 IgnoredGoFiles []string `json:",omitempty"` // .go sources ignored due to build constraints 49 CFiles []string `json:",omitempty"` // .c source files 50 CXXFiles []string `json:",omitempty"` // .cc, .cpp and .cxx source files 51 MFiles []string `json:",omitempty"` // .m source files 52 HFiles []string `json:",omitempty"` // .h, .hh, .hpp and .hxx source files 53 SFiles []string `json:",omitempty"` // .s source files 54 SwigFiles []string `json:",omitempty"` // .swig files 55 SwigCXXFiles []string `json:",omitempty"` // .swigcxx files 56 SysoFiles []string `json:",omitempty"` // .syso system object files added to package 57 58 // Cgo directives 59 CgoCFLAGS []string `json:",omitempty"` // cgo: flags for C compiler 60 CgoCPPFLAGS []string `json:",omitempty"` // cgo: flags for C preprocessor 61 CgoCXXFLAGS []string `json:",omitempty"` // cgo: flags for C++ compiler 62 CgoLDFLAGS []string `json:",omitempty"` // cgo: flags for linker 63 CgoPkgConfig []string `json:",omitempty"` // cgo: pkg-config names 64 65 // Dependency information 66 Imports []string `json:",omitempty"` // import paths used by this package 67 Deps []string `json:",omitempty"` // all (recursively) imported dependencies 68 69 // Error information 70 Incomplete bool `json:",omitempty"` // was there an error loading this package or dependencies? 71 Error *PackageError `json:",omitempty"` // error loading this package (not dependencies) 72 DepsErrors []*PackageError `json:",omitempty"` // errors loading dependencies 73 74 // Test information 75 TestGoFiles []string `json:",omitempty"` // _test.go files in package 76 TestImports []string `json:",omitempty"` // imports from TestGoFiles 77 XTestGoFiles []string `json:",omitempty"` // _test.go files outside package 78 XTestImports []string `json:",omitempty"` // imports from XTestGoFiles 79 80 // Unexported fields are not part of the public API. 81 build *build.Package 82 pkgdir string // overrides build.PkgDir 83 imports []*Package 84 deps []*Package 85 gofiles []string // GoFiles+CgoFiles+TestGoFiles+XTestGoFiles files, absolute paths 86 sfiles []string 87 allgofiles []string // gofiles + IgnoredGoFiles, absolute paths 88 target string // installed file for this package (may be executable) 89 fake bool // synthesized package 90 external bool // synthesized external test package 91 forceBuild bool // this package must be rebuilt 92 forceLibrary bool // this package is a library (even if named "main") 93 cmdline bool // defined by files listed on command line 94 local bool // imported via local path (./ or ../) 95 localPrefix string // interpret ./ and ../ imports relative to this prefix 96 exeName string // desired name for temporary executable 97 coverMode string // preprocess Go source files with the coverage tool in this mode 98 coverVars map[string]*CoverVar // variables created by coverage analysis 99 omitDWARF bool // tell linker not to write DWARF information 100 buildID string // expected build ID for generated package 101 gobinSubdir bool // install target would be subdir of GOBIN 102 } 103 104 // vendored returns the vendor-resolved version of imports, 105 // which should be p.TestImports or p.XTestImports, NOT p.Imports. 106 // The imports in p.TestImports and p.XTestImports are not recursively 107 // loaded during the initial load of p, so they list the imports found in 108 // the source file, but most processing should be over the vendor-resolved 109 // import paths. We do this resolution lazily both to avoid file system work 110 // and because the eventual real load of the test imports (during 'go test') 111 // can produce better error messages if it starts with the original paths. 112 // The initial load of p loads all the non-test imports and rewrites 113 // the vendored paths, so nothing should ever call p.vendored(p.Imports). 114 func (p *Package) vendored(imports []string) []string { 115 if len(imports) > 0 && len(p.Imports) > 0 && &imports[0] == &p.Imports[0] { 116 panic("internal error: p.vendored(p.Imports) called") 117 } 118 seen := make(map[string]bool) 119 var all []string 120 for _, path := range imports { 121 path, _ = vendoredImportPath(p, path) 122 if !seen[path] { 123 seen[path] = true 124 all = append(all, path) 125 } 126 } 127 sort.Strings(all) 128 return all 129 } 130 131 // CoverVar holds the name of the generated coverage variables targeting the named file. 132 type CoverVar struct { 133 File string // local file name 134 Var string // name of count struct 135 } 136 137 func (p *Package) copyBuild(pp *build.Package) { 138 p.build = pp 139 140 if pp.PkgTargetRoot != "" && buildPkgdir != "" { 141 old := pp.PkgTargetRoot 142 pp.PkgRoot = buildPkgdir 143 pp.PkgTargetRoot = buildPkgdir 144 pp.PkgObj = filepath.Join(buildPkgdir, strings.TrimPrefix(pp.PkgObj, old)) 145 } 146 147 p.Dir = pp.Dir 148 p.ImportPath = pp.ImportPath 149 p.ImportComment = pp.ImportComment 150 p.Name = pp.Name 151 p.Doc = pp.Doc 152 p.Root = pp.Root 153 p.ConflictDir = pp.ConflictDir 154 // TODO? Target 155 p.Goroot = pp.Goroot 156 p.Standard = p.Goroot && p.ImportPath != "" && !strings.Contains(p.ImportPath, ".") 157 p.GoFiles = pp.GoFiles 158 p.CgoFiles = pp.CgoFiles 159 p.IgnoredGoFiles = pp.IgnoredGoFiles 160 p.CFiles = pp.CFiles 161 p.CXXFiles = pp.CXXFiles 162 p.MFiles = pp.MFiles 163 p.HFiles = pp.HFiles 164 p.SFiles = pp.SFiles 165 p.SwigFiles = pp.SwigFiles 166 p.SwigCXXFiles = pp.SwigCXXFiles 167 p.SysoFiles = pp.SysoFiles 168 p.CgoCFLAGS = pp.CgoCFLAGS 169 p.CgoCPPFLAGS = pp.CgoCPPFLAGS 170 p.CgoCXXFLAGS = pp.CgoCXXFLAGS 171 p.CgoLDFLAGS = pp.CgoLDFLAGS 172 p.CgoPkgConfig = pp.CgoPkgConfig 173 p.Imports = pp.Imports 174 p.TestGoFiles = pp.TestGoFiles 175 p.TestImports = pp.TestImports 176 p.XTestGoFiles = pp.XTestGoFiles 177 p.XTestImports = pp.XTestImports 178 } 179 180 // A PackageError describes an error loading information about a package. 181 type PackageError struct { 182 ImportStack []string // shortest path from package named on command line to this one 183 Pos string // position of error 184 Err string // the error itself 185 isImportCycle bool // the error is an import cycle 186 hard bool // whether the error is soft or hard; soft errors are ignored in some places 187 } 188 189 func (p *PackageError) Error() string { 190 // Import cycles deserve special treatment. 191 if p.isImportCycle { 192 return fmt.Sprintf("%s\npackage %s\n", p.Err, strings.Join(p.ImportStack, "\n\timports ")) 193 } 194 if p.Pos != "" { 195 // Omit import stack. The full path to the file where the error 196 // is the most important thing. 197 return p.Pos + ": " + p.Err 198 } 199 if len(p.ImportStack) == 0 { 200 return p.Err 201 } 202 return "package " + strings.Join(p.ImportStack, "\n\timports ") + ": " + p.Err 203 } 204 205 // An importStack is a stack of import paths. 206 type importStack []string 207 208 func (s *importStack) push(p string) { 209 *s = append(*s, p) 210 } 211 212 func (s *importStack) pop() { 213 *s = (*s)[0 : len(*s)-1] 214 } 215 216 func (s *importStack) copy() []string { 217 return append([]string{}, *s...) 218 } 219 220 // shorterThan reports whether sp is shorter than t. 221 // We use this to record the shortest import sequence 222 // that leads to a particular package. 223 func (sp *importStack) shorterThan(t []string) bool { 224 s := *sp 225 if len(s) != len(t) { 226 return len(s) < len(t) 227 } 228 // If they are the same length, settle ties using string ordering. 229 for i := range s { 230 if s[i] != t[i] { 231 return s[i] < t[i] 232 } 233 } 234 return false // they are equal 235 } 236 237 // packageCache is a lookup cache for loadPackage, 238 // so that if we look up a package multiple times 239 // we return the same pointer each time. 240 var packageCache = map[string]*Package{} 241 242 // reloadPackage is like loadPackage but makes sure 243 // not to use the package cache. 244 func reloadPackage(arg string, stk *importStack) *Package { 245 p := packageCache[arg] 246 if p != nil { 247 delete(packageCache, p.Dir) 248 delete(packageCache, p.ImportPath) 249 } 250 return loadPackage(arg, stk) 251 } 252 253 // The Go 1.5 vendoring experiment is enabled by setting GO15VENDOREXPERIMENT=1. 254 // The variable is obnoxiously long so that years from now when people find it in 255 // their profiles and wonder what it does, there is some chance that a web search 256 // might answer the question. 257 var go15VendorExperiment = os.Getenv("GO15VENDOREXPERIMENT") == "1" 258 259 // dirToImportPath returns the pseudo-import path we use for a package 260 // outside the Go path. It begins with _/ and then contains the full path 261 // to the directory. If the package lives in c:\home\gopher\my\pkg then 262 // the pseudo-import path is _/c_/home/gopher/my/pkg. 263 // Using a pseudo-import path like this makes the ./ imports no longer 264 // a special case, so that all the code to deal with ordinary imports works 265 // automatically. 266 func dirToImportPath(dir string) string { 267 return pathpkg.Join("_", strings.Map(makeImportValid, filepath.ToSlash(dir))) 268 } 269 270 func makeImportValid(r rune) rune { 271 // Should match Go spec, compilers, and ../../go/parser/parser.go:/isValidImport. 272 const illegalChars = `!"#$%&'()*,:;<=>?[\]^{|}` + "`\uFFFD" 273 if !unicode.IsGraphic(r) || unicode.IsSpace(r) || strings.ContainsRune(illegalChars, r) { 274 return '_' 275 } 276 return r 277 } 278 279 // Mode flags for loadImport and download (in get.go). 280 const ( 281 // useVendor means that loadImport should do vendor expansion 282 // (provided the vendoring experiment is enabled). 283 // That is, useVendor means that the import path came from 284 // a source file and has not been vendor-expanded yet. 285 // Every import path should be loaded initially with useVendor, 286 // and then the expanded version (with the /vendor/ in it) gets 287 // recorded as the canonical import path. At that point, future loads 288 // of that package must not pass useVendor, because 289 // disallowVendor will reject direct use of paths containing /vendor/. 290 useVendor = 1 << iota 291 292 // getTestDeps is for download (part of "go get") and indicates 293 // that test dependencies should be fetched too. 294 getTestDeps 295 ) 296 297 // loadImport scans the directory named by path, which must be an import path, 298 // but possibly a local import path (an absolute file system path or one beginning 299 // with ./ or ../). A local relative path is interpreted relative to srcDir. 300 // It returns a *Package describing the package found in that directory. 301 func loadImport(path, srcDir string, parent *Package, stk *importStack, importPos []token.Position, mode int) *Package { 302 stk.push(path) 303 defer stk.pop() 304 305 // Determine canonical identifier for this package. 306 // For a local import the identifier is the pseudo-import path 307 // we create from the full directory to the package. 308 // Otherwise it is the usual import path. 309 // For vendored imports, it is the expanded form. 310 importPath := path 311 origPath := path 312 isLocal := build.IsLocalImport(path) 313 var vendorSearch []string 314 if isLocal { 315 importPath = dirToImportPath(filepath.Join(srcDir, path)) 316 } else if mode&useVendor != 0 { 317 path, vendorSearch = vendoredImportPath(parent, path) 318 importPath = path 319 } 320 321 if p := packageCache[importPath]; p != nil { 322 if perr := disallowInternal(srcDir, p, stk); perr != p { 323 return perr 324 } 325 if mode&useVendor != 0 { 326 if perr := disallowVendor(srcDir, origPath, p, stk); perr != p { 327 return perr 328 } 329 } 330 return reusePackage(p, stk) 331 } 332 333 p := new(Package) 334 p.local = isLocal 335 p.ImportPath = importPath 336 packageCache[importPath] = p 337 338 // Load package. 339 // Import always returns bp != nil, even if an error occurs, 340 // in order to return partial information. 341 // 342 // TODO: After Go 1, decide when to pass build.AllowBinary here. 343 // See issue 3268 for mistakes to avoid. 344 bp, err := buildContext.Import(path, srcDir, build.ImportComment) 345 346 // If we got an error from go/build about package not found, 347 // it contains the directories from $GOROOT and $GOPATH that 348 // were searched. Add to that message the vendor directories 349 // that were searched. 350 if err != nil && len(vendorSearch) > 0 { 351 // NOTE(rsc): The direct text manipulation here is fairly awful, 352 // but it avoids defining new go/build API (an exported error type) 353 // late in the Go 1.5 release cycle. If this turns out to be a more general 354 // problem we could define a real error type when the decision can be 355 // considered more carefully. 356 text := err.Error() 357 if strings.Contains(text, "cannot find package \"") && strings.Contains(text, "\" in any of:\n\t") { 358 old := strings.SplitAfter(text, "\n") 359 lines := []string{old[0]} 360 for _, dir := range vendorSearch { 361 lines = append(lines, "\t"+dir+" (vendor tree)\n") 362 } 363 lines = append(lines, old[1:]...) 364 err = errors.New(strings.Join(lines, "")) 365 } 366 } 367 bp.ImportPath = importPath 368 if gobin != "" { 369 bp.BinDir = gobin 370 } 371 if err == nil && !isLocal && bp.ImportComment != "" && bp.ImportComment != path && (!go15VendorExperiment || !strings.Contains(path, "/vendor/")) { 372 err = fmt.Errorf("code in directory %s expects import %q", bp.Dir, bp.ImportComment) 373 } 374 p.load(stk, bp, err) 375 if p.Error != nil && len(importPos) > 0 { 376 pos := importPos[0] 377 pos.Filename = shortPath(pos.Filename) 378 p.Error.Pos = pos.String() 379 } 380 381 if perr := disallowInternal(srcDir, p, stk); perr != p { 382 return perr 383 } 384 if mode&useVendor != 0 { 385 if perr := disallowVendor(srcDir, origPath, p, stk); perr != p { 386 return perr 387 } 388 } 389 390 return p 391 } 392 393 var isDirCache = map[string]bool{} 394 395 func isDir(path string) bool { 396 result, ok := isDirCache[path] 397 if ok { 398 return result 399 } 400 401 fi, err := os.Stat(path) 402 result = err == nil && fi.IsDir() 403 isDirCache[path] = result 404 return result 405 } 406 407 // vendoredImportPath returns the expansion of path when it appears in parent. 408 // If parent is x/y/z, then path might expand to x/y/z/vendor/path, x/y/vendor/path, 409 // x/vendor/path, vendor/path, or else stay path if none of those exist. 410 // vendoredImportPath returns the expanded path or, if no expansion is found, the original. 411 // If no expansion is found, vendoredImportPath also returns a list of vendor directories 412 // it searched along the way, to help prepare a useful error message should path turn 413 // out not to exist. 414 func vendoredImportPath(parent *Package, path string) (found string, searched []string) { 415 if parent == nil || parent.Root == "" || !go15VendorExperiment { 416 return path, nil 417 } 418 dir := filepath.Clean(parent.Dir) 419 root := filepath.Join(parent.Root, "src") 420 if !hasFilePathPrefix(dir, root) || len(dir) <= len(root) || dir[len(root)] != filepath.Separator { 421 fatalf("invalid vendoredImportPath: dir=%q root=%q separator=%q", dir, root, string(filepath.Separator)) 422 } 423 vpath := "vendor/" + path 424 for i := len(dir); i >= len(root); i-- { 425 if i < len(dir) && dir[i] != filepath.Separator { 426 continue 427 } 428 // Note: checking for the vendor directory before checking 429 // for the vendor/path directory helps us hit the 430 // isDir cache more often. It also helps us prepare a more useful 431 // list of places we looked, to report when an import is not found. 432 if !isDir(filepath.Join(dir[:i], "vendor")) { 433 continue 434 } 435 targ := filepath.Join(dir[:i], vpath) 436 if isDir(targ) { 437 // We started with parent's dir c:\gopath\src\foo\bar\baz\quux\xyzzy. 438 // We know the import path for parent's dir. 439 // We chopped off some number of path elements and 440 // added vendor\path to produce c:\gopath\src\foo\bar\baz\vendor\path. 441 // Now we want to know the import path for that directory. 442 // Construct it by chopping the same number of path elements 443 // (actually the same number of bytes) from parent's import path 444 // and then append /vendor/path. 445 chopped := len(dir) - i 446 if chopped == len(parent.ImportPath)+1 { 447 // We walked up from c:\gopath\src\foo\bar 448 // and found c:\gopath\src\vendor\path. 449 // We chopped \foo\bar (length 8) but the import path is "foo/bar" (length 7). 450 // Use "vendor/path" without any prefix. 451 return vpath, nil 452 } 453 return parent.ImportPath[:len(parent.ImportPath)-chopped] + "/" + vpath, nil 454 } 455 // Note the existence of a vendor directory in case path is not found anywhere. 456 searched = append(searched, targ) 457 } 458 return path, searched 459 } 460 461 // reusePackage reuses package p to satisfy the import at the top 462 // of the import stack stk. If this use causes an import loop, 463 // reusePackage updates p's error information to record the loop. 464 func reusePackage(p *Package, stk *importStack) *Package { 465 // We use p.imports==nil to detect a package that 466 // is in the midst of its own loadPackage call 467 // (all the recursion below happens before p.imports gets set). 468 if p.imports == nil { 469 if p.Error == nil { 470 p.Error = &PackageError{ 471 ImportStack: stk.copy(), 472 Err: "import cycle not allowed", 473 isImportCycle: true, 474 } 475 } 476 p.Incomplete = true 477 } 478 // Don't rewrite the import stack in the error if we have an import cycle. 479 // If we do, we'll lose the path that describes the cycle. 480 if p.Error != nil && !p.Error.isImportCycle && stk.shorterThan(p.Error.ImportStack) { 481 p.Error.ImportStack = stk.copy() 482 } 483 return p 484 } 485 486 // disallowInternal checks that srcDir is allowed to import p. 487 // If the import is allowed, disallowInternal returns the original package p. 488 // If not, it returns a new package containing just an appropriate error. 489 func disallowInternal(srcDir string, p *Package, stk *importStack) *Package { 490 // golang.org/s/go14internal: 491 // An import of a path containing the element “internal” 492 // is disallowed if the importing code is outside the tree 493 // rooted at the parent of the “internal” directory. 494 495 // There was an error loading the package; stop here. 496 if p.Error != nil { 497 return p 498 } 499 500 // The stack includes p.ImportPath. 501 // If that's the only thing on the stack, we started 502 // with a name given on the command line, not an 503 // import. Anything listed on the command line is fine. 504 if len(*stk) == 1 { 505 return p 506 } 507 508 // Check for "internal" element: four cases depending on begin of string and/or end of string. 509 i, ok := findInternal(p.ImportPath) 510 if !ok { 511 return p 512 } 513 514 // Internal is present. 515 // Map import path back to directory corresponding to parent of internal. 516 if i > 0 { 517 i-- // rewind over slash in ".../internal" 518 } 519 parent := p.Dir[:i+len(p.Dir)-len(p.ImportPath)] 520 if hasPathPrefix(filepath.ToSlash(srcDir), filepath.ToSlash(parent)) { 521 return p 522 } 523 524 // Internal is present, and srcDir is outside parent's tree. Not allowed. 525 perr := *p 526 perr.Error = &PackageError{ 527 ImportStack: stk.copy(), 528 Err: "use of internal package not allowed", 529 } 530 perr.Incomplete = true 531 return &perr 532 } 533 534 // findInternal looks for the final "internal" path element in the given import path. 535 // If there isn't one, findInternal returns ok=false. 536 // Otherwise, findInternal returns ok=true and the index of the "internal". 537 func findInternal(path string) (index int, ok bool) { 538 // Four cases, depending on internal at start/end of string or not. 539 // The order matters: we must return the index of the final element, 540 // because the final one produces the most restrictive requirement 541 // on the importer. 542 switch { 543 case strings.HasSuffix(path, "/internal"): 544 return len(path) - len("internal"), true 545 case strings.Contains(path, "/internal/"): 546 return strings.LastIndex(path, "/internal/") + 1, true 547 case path == "internal", strings.HasPrefix(path, "internal/"): 548 return 0, true 549 } 550 return 0, false 551 } 552 553 // disallowVendor checks that srcDir is allowed to import p as path. 554 // If the import is allowed, disallowVendor returns the original package p. 555 // If not, it returns a new package containing just an appropriate error. 556 func disallowVendor(srcDir, path string, p *Package, stk *importStack) *Package { 557 if !go15VendorExperiment { 558 return p 559 } 560 561 // The stack includes p.ImportPath. 562 // If that's the only thing on the stack, we started 563 // with a name given on the command line, not an 564 // import. Anything listed on the command line is fine. 565 if len(*stk) == 1 { 566 return p 567 } 568 569 if perr := disallowVendorVisibility(srcDir, p, stk); perr != p { 570 return perr 571 } 572 573 // Paths like x/vendor/y must be imported as y, never as x/vendor/y. 574 if i, ok := findVendor(path); ok { 575 perr := *p 576 perr.Error = &PackageError{ 577 ImportStack: stk.copy(), 578 Err: "must be imported as " + path[i+len("vendor/"):], 579 } 580 perr.Incomplete = true 581 return &perr 582 } 583 584 return p 585 } 586 587 // disallowVendorVisibility checks that srcDir is allowed to import p. 588 // The rules are the same as for /internal/ except that a path ending in /vendor 589 // is not subject to the rules, only subdirectories of vendor. 590 // This allows people to have packages and commands named vendor, 591 // for maximal compatibility with existing source trees. 592 func disallowVendorVisibility(srcDir string, p *Package, stk *importStack) *Package { 593 // The stack includes p.ImportPath. 594 // If that's the only thing on the stack, we started 595 // with a name given on the command line, not an 596 // import. Anything listed on the command line is fine. 597 if len(*stk) == 1 { 598 return p 599 } 600 601 // Check for "vendor" element. 602 i, ok := findVendor(p.ImportPath) 603 if !ok { 604 return p 605 } 606 607 // Vendor is present. 608 // Map import path back to directory corresponding to parent of vendor. 609 if i > 0 { 610 i-- // rewind over slash in ".../vendor" 611 } 612 truncateTo := i + len(p.Dir) - len(p.ImportPath) 613 if truncateTo < 0 || len(p.Dir) < truncateTo { 614 return p 615 } 616 parent := p.Dir[:truncateTo] 617 if hasPathPrefix(filepath.ToSlash(srcDir), filepath.ToSlash(parent)) { 618 return p 619 } 620 621 // Vendor is present, and srcDir is outside parent's tree. Not allowed. 622 perr := *p 623 perr.Error = &PackageError{ 624 ImportStack: stk.copy(), 625 Err: "use of vendored package not allowed", 626 } 627 perr.Incomplete = true 628 return &perr 629 } 630 631 // findVendor looks for the last non-terminating "vendor" path element in the given import path. 632 // If there isn't one, findVendor returns ok=false. 633 // Otherwise, findInternal returns ok=true and the index of the "vendor". 634 // 635 // Note that terminating "vendor" elements don't count: "x/vendor" is its own package, 636 // not the vendored copy of an import "" (the empty import path). 637 // This will allow people to have packages or commands named vendor. 638 // This may help reduce breakage, or it may just be confusing. We'll see. 639 func findVendor(path string) (index int, ok bool) { 640 // Two cases, depending on internal at start of string or not. 641 // The order matters: we must return the index of the final element, 642 // because the final one is where the effective import path starts. 643 switch { 644 case strings.Contains(path, "/vendor/"): 645 return strings.LastIndex(path, "/vendor/") + 1, true 646 case strings.HasPrefix(path, "vendor/"): 647 return 0, true 648 } 649 return 0, false 650 } 651 652 type targetDir int 653 654 const ( 655 toRoot targetDir = iota // to bin dir inside package root (default) 656 toTool // GOROOT/pkg/tool 657 toBin // GOROOT/bin 658 stalePath // the old import path; fail to build 659 ) 660 661 // goTools is a map of Go program import path to install target directory. 662 var goTools = map[string]targetDir{ 663 "cmd/addr2line": toTool, 664 "cmd/api": toTool, 665 "cmd/asm": toTool, 666 "cmd/compile": toTool, 667 "cmd/cgo": toTool, 668 "cmd/cover": toTool, 669 "cmd/dist": toTool, 670 "cmd/doc": toTool, 671 "cmd/fix": toTool, 672 "cmd/link": toTool, 673 "cmd/newlink": toTool, 674 "cmd/nm": toTool, 675 "cmd/objdump": toTool, 676 "cmd/pack": toTool, 677 "cmd/pprof": toTool, 678 "cmd/trace": toTool, 679 "cmd/vet": toTool, 680 "cmd/yacc": toTool, 681 "golang.org/x/tools/cmd/godoc": toBin, 682 "code.google.com/p/go.tools/cmd/cover": stalePath, 683 "code.google.com/p/go.tools/cmd/godoc": stalePath, 684 "code.google.com/p/go.tools/cmd/vet": stalePath, 685 } 686 687 // expandScanner expands a scanner.List error into all the errors in the list. 688 // The default Error method only shows the first error. 689 func expandScanner(err error) error { 690 // Look for parser errors. 691 if err, ok := err.(scanner.ErrorList); ok { 692 // Prepare error with \n before each message. 693 // When printed in something like context: %v 694 // this will put the leading file positions each on 695 // its own line. It will also show all the errors 696 // instead of just the first, as err.Error does. 697 var buf bytes.Buffer 698 for _, e := range err { 699 e.Pos.Filename = shortPath(e.Pos.Filename) 700 buf.WriteString("\n") 701 buf.WriteString(e.Error()) 702 } 703 return errors.New(buf.String()) 704 } 705 return err 706 } 707 708 var raceExclude = map[string]bool{ 709 "runtime/race": true, 710 "runtime/cgo": true, 711 "cmd/cgo": true, 712 "syscall": true, 713 "errors": true, 714 } 715 716 var cgoExclude = map[string]bool{ 717 "runtime/cgo": true, 718 } 719 720 var cgoSyscallExclude = map[string]bool{ 721 "runtime/cgo": true, 722 "runtime/race": true, 723 } 724 725 // load populates p using information from bp, err, which should 726 // be the result of calling build.Context.Import. 727 func (p *Package) load(stk *importStack, bp *build.Package, err error) *Package { 728 p.copyBuild(bp) 729 730 // The localPrefix is the path we interpret ./ imports relative to. 731 // Synthesized main packages sometimes override this. 732 p.localPrefix = dirToImportPath(p.Dir) 733 734 if err != nil { 735 p.Incomplete = true 736 err = expandScanner(err) 737 p.Error = &PackageError{ 738 ImportStack: stk.copy(), 739 Err: err.Error(), 740 } 741 return p 742 } 743 744 useBindir := p.Name == "main" 745 if !p.Standard { 746 switch buildBuildmode { 747 case "c-archive", "c-shared": 748 useBindir = false 749 } 750 } 751 752 if useBindir { 753 // Report an error when the old code.google.com/p/go.tools paths are used. 754 if goTools[p.ImportPath] == stalePath { 755 newPath := strings.Replace(p.ImportPath, "code.google.com/p/go.", "golang.org/x/", 1) 756 e := fmt.Sprintf("the %v command has moved; use %v instead.", p.ImportPath, newPath) 757 p.Error = &PackageError{Err: e} 758 return p 759 } 760 _, elem := filepath.Split(p.Dir) 761 full := buildContext.GOOS + "_" + buildContext.GOARCH + "/" + elem 762 if buildContext.GOOS != toolGOOS || buildContext.GOARCH != toolGOARCH { 763 // Install cross-compiled binaries to subdirectories of bin. 764 elem = full 765 } 766 if p.build.BinDir != gobin && goTools[p.ImportPath] == toBin { 767 // Override BinDir. 768 // This is from a subrepo but installs to $GOROOT/bin 769 // by default anyway (like godoc). 770 p.target = filepath.Join(gorootBin, elem) 771 } else if p.build.BinDir != "" { 772 // Install to GOBIN or bin of GOPATH entry. 773 p.target = filepath.Join(p.build.BinDir, elem) 774 if !p.Goroot && strings.Contains(elem, "/") && gobin != "" { 775 // Do not create $GOBIN/goos_goarch/elem. 776 p.target = "" 777 p.gobinSubdir = true 778 } 779 } 780 if goTools[p.ImportPath] == toTool { 781 // This is for 'go tool'. 782 // Override all the usual logic and force it into the tool directory. 783 p.target = filepath.Join(gorootPkg, "tool", full) 784 } 785 if p.target != "" && buildContext.GOOS == "windows" { 786 p.target += ".exe" 787 } 788 } else if p.local { 789 // Local import turned into absolute path. 790 // No permanent install target. 791 p.target = "" 792 } else { 793 p.target = p.build.PkgObj 794 if buildLinkshared { 795 shlibnamefile := p.target[:len(p.target)-2] + ".shlibname" 796 shlib, err := ioutil.ReadFile(shlibnamefile) 797 if err == nil { 798 libname := strings.TrimSpace(string(shlib)) 799 if buildContext.Compiler == "gccgo" { 800 p.Shlib = filepath.Join(p.build.PkgTargetRoot, "shlibs", libname) 801 } else { 802 p.Shlib = filepath.Join(p.build.PkgTargetRoot, libname) 803 804 } 805 } else if !os.IsNotExist(err) { 806 fatalf("unexpected error reading %s: %v", shlibnamefile, err) 807 } 808 } 809 } 810 811 importPaths := p.Imports 812 // Packages that use cgo import runtime/cgo implicitly. 813 // Packages that use cgo also import syscall implicitly, 814 // to wrap errno. 815 // Exclude certain packages to avoid circular dependencies. 816 if len(p.CgoFiles) > 0 && (!p.Standard || !cgoExclude[p.ImportPath]) { 817 importPaths = append(importPaths, "runtime/cgo") 818 } 819 if len(p.CgoFiles) > 0 && (!p.Standard || !cgoSyscallExclude[p.ImportPath]) { 820 importPaths = append(importPaths, "syscall") 821 } 822 823 // Currently build mode c-shared, or -linkshared, forces 824 // external linking mode, and external linking mode forces an 825 // import of runtime/cgo. 826 if p.Name == "main" && !p.Goroot && (buildBuildmode == "c-shared" || buildLinkshared) { 827 importPaths = append(importPaths, "runtime/cgo") 828 } 829 830 // Everything depends on runtime, except runtime and unsafe. 831 if !p.Standard || (p.ImportPath != "runtime" && p.ImportPath != "unsafe") { 832 importPaths = append(importPaths, "runtime") 833 // When race detection enabled everything depends on runtime/race. 834 // Exclude certain packages to avoid circular dependencies. 835 if buildRace && (!p.Standard || !raceExclude[p.ImportPath]) { 836 importPaths = append(importPaths, "runtime/race") 837 } 838 // On ARM with GOARM=5, everything depends on math for the link. 839 if p.Name == "main" && goarch == "arm" { 840 importPaths = append(importPaths, "math") 841 } 842 } 843 844 // Build list of full paths to all Go files in the package, 845 // for use by commands like go fmt. 846 p.gofiles = stringList(p.GoFiles, p.CgoFiles, p.TestGoFiles, p.XTestGoFiles) 847 for i := range p.gofiles { 848 p.gofiles[i] = filepath.Join(p.Dir, p.gofiles[i]) 849 } 850 sort.Strings(p.gofiles) 851 852 p.sfiles = stringList(p.SFiles) 853 for i := range p.sfiles { 854 p.sfiles[i] = filepath.Join(p.Dir, p.sfiles[i]) 855 } 856 sort.Strings(p.sfiles) 857 858 p.allgofiles = stringList(p.IgnoredGoFiles) 859 for i := range p.allgofiles { 860 p.allgofiles[i] = filepath.Join(p.Dir, p.allgofiles[i]) 861 } 862 p.allgofiles = append(p.allgofiles, p.gofiles...) 863 sort.Strings(p.allgofiles) 864 865 // Check for case-insensitive collision of input files. 866 // To avoid problems on case-insensitive files, we reject any package 867 // where two different input files have equal names under a case-insensitive 868 // comparison. 869 f1, f2 := foldDup(stringList( 870 p.GoFiles, 871 p.CgoFiles, 872 p.IgnoredGoFiles, 873 p.CFiles, 874 p.CXXFiles, 875 p.MFiles, 876 p.HFiles, 877 p.SFiles, 878 p.SysoFiles, 879 p.SwigFiles, 880 p.SwigCXXFiles, 881 p.TestGoFiles, 882 p.XTestGoFiles, 883 )) 884 if f1 != "" { 885 p.Error = &PackageError{ 886 ImportStack: stk.copy(), 887 Err: fmt.Sprintf("case-insensitive file name collision: %q and %q", f1, f2), 888 } 889 return p 890 } 891 892 // Build list of imported packages and full dependency list. 893 imports := make([]*Package, 0, len(p.Imports)) 894 deps := make(map[string]*Package) 895 for i, path := range importPaths { 896 if path == "C" { 897 continue 898 } 899 p1 := loadImport(path, p.Dir, p, stk, p.build.ImportPos[path], useVendor) 900 if p1.Name == "main" { 901 p.Error = &PackageError{ 902 ImportStack: stk.copy(), 903 Err: fmt.Sprintf("import %q is a program, not an importable package", path), 904 } 905 pos := p.build.ImportPos[path] 906 if len(pos) > 0 { 907 p.Error.Pos = pos[0].String() 908 } 909 } 910 if p1.local { 911 if !p.local && p.Error == nil { 912 p.Error = &PackageError{ 913 ImportStack: stk.copy(), 914 Err: fmt.Sprintf("local import %q in non-local package", path), 915 } 916 pos := p.build.ImportPos[path] 917 if len(pos) > 0 { 918 p.Error.Pos = pos[0].String() 919 } 920 } 921 } 922 path = p1.ImportPath 923 importPaths[i] = path 924 if i < len(p.Imports) { 925 p.Imports[i] = path 926 } 927 deps[path] = p1 928 imports = append(imports, p1) 929 for _, dep := range p1.deps { 930 // The same import path could produce an error or not, 931 // depending on what tries to import it. 932 // Prefer to record entries with errors, so we can report them. 933 if deps[dep.ImportPath] == nil || dep.Error != nil { 934 deps[dep.ImportPath] = dep 935 } 936 } 937 if p1.Incomplete { 938 p.Incomplete = true 939 } 940 } 941 p.imports = imports 942 943 p.Deps = make([]string, 0, len(deps)) 944 for dep := range deps { 945 p.Deps = append(p.Deps, dep) 946 } 947 sort.Strings(p.Deps) 948 for _, dep := range p.Deps { 949 p1 := deps[dep] 950 if p1 == nil { 951 panic("impossible: missing entry in package cache for " + dep + " imported by " + p.ImportPath) 952 } 953 p.deps = append(p.deps, p1) 954 if p1.Error != nil { 955 p.DepsErrors = append(p.DepsErrors, p1.Error) 956 } 957 } 958 959 // unsafe is a fake package. 960 if p.Standard && (p.ImportPath == "unsafe" || buildContext.Compiler == "gccgo") { 961 p.target = "" 962 } 963 p.Target = p.target 964 965 // The gc toolchain only permits C source files with cgo. 966 if len(p.CFiles) > 0 && !p.usesCgo() && !p.usesSwig() && buildContext.Compiler == "gc" { 967 p.Error = &PackageError{ 968 ImportStack: stk.copy(), 969 Err: fmt.Sprintf("C source files not allowed when not using cgo or SWIG: %s", strings.Join(p.CFiles, " ")), 970 } 971 return p 972 } 973 974 // In the absence of errors lower in the dependency tree, 975 // check for case-insensitive collisions of import paths. 976 if len(p.DepsErrors) == 0 { 977 dep1, dep2 := foldDup(p.Deps) 978 if dep1 != "" { 979 p.Error = &PackageError{ 980 ImportStack: stk.copy(), 981 Err: fmt.Sprintf("case-insensitive import collision: %q and %q", dep1, dep2), 982 } 983 return p 984 } 985 } 986 987 computeBuildID(p) 988 return p 989 } 990 991 // usesSwig reports whether the package needs to run SWIG. 992 func (p *Package) usesSwig() bool { 993 return len(p.SwigFiles) > 0 || len(p.SwigCXXFiles) > 0 994 } 995 996 // usesCgo reports whether the package needs to run cgo 997 func (p *Package) usesCgo() bool { 998 return len(p.CgoFiles) > 0 999 } 1000 1001 // packageList returns the list of packages in the dag rooted at roots 1002 // as visited in a depth-first post-order traversal. 1003 func packageList(roots []*Package) []*Package { 1004 seen := map[*Package]bool{} 1005 all := []*Package{} 1006 var walk func(*Package) 1007 walk = func(p *Package) { 1008 if seen[p] { 1009 return 1010 } 1011 seen[p] = true 1012 for _, p1 := range p.imports { 1013 walk(p1) 1014 } 1015 all = append(all, p) 1016 } 1017 for _, root := range roots { 1018 walk(root) 1019 } 1020 return all 1021 } 1022 1023 // computeStale computes the Stale flag in the package dag that starts 1024 // at the named pkgs (command-line arguments). 1025 func computeStale(pkgs ...*Package) { 1026 for _, p := range packageList(pkgs) { 1027 p.Stale = isStale(p) 1028 } 1029 } 1030 1031 // The runtime version string takes one of two forms: 1032 // "go1.X[.Y]" for Go releases, and "devel +hash" at tip. 1033 // Determine whether we are in a released copy by 1034 // inspecting the version. 1035 var isGoRelease = strings.HasPrefix(runtime.Version(), "go1") 1036 1037 // isStale and computeBuildID 1038 // 1039 // Theory of Operation 1040 // 1041 // There is an installed copy of the package (or binary). 1042 // Can we reuse the installed copy, or do we need to build a new one? 1043 // 1044 // We can use the installed copy if it matches what we'd get 1045 // by building a new one. The hard part is predicting that without 1046 // actually running a build. 1047 // 1048 // To start, we must know the set of inputs to the build process that can 1049 // affect the generated output. At a minimum, that includes the source 1050 // files for the package and also any compiled packages imported by those 1051 // source files. The *Package has these, and we use them. One might also 1052 // argue for including in the input set: the build tags, whether the race 1053 // detector is in use, the target operating system and architecture, the 1054 // compiler and linker binaries being used, the additional flags being 1055 // passed to those, the cgo binary being used, the additional flags cgo 1056 // passes to the host C compiler, the host C compiler being used, the set 1057 // of host C include files and installed C libraries, and so on. 1058 // We include some but not all of this information. 1059 // 1060 // Once we have decided on a set of inputs, we must next decide how to 1061 // tell whether the content of that set has changed since the last build 1062 // of p. If there have been no changes, then we assume a new build would 1063 // produce the same result and reuse the installed package or binary. 1064 // But if there have been changes, then we assume a new build might not 1065 // produce the same result, so we rebuild. 1066 // 1067 // There are two common ways to decide whether the content of the set has 1068 // changed: modification times and content hashes. We use a mixture of both. 1069 // 1070 // The use of modification times (mtimes) was pioneered by make: 1071 // assuming that a file's mtime is an accurate record of when that file was last written, 1072 // and assuming that the modification time of an installed package or 1073 // binary is the time that it was built, if the mtimes of the inputs 1074 // predate the mtime of the installed object, then the build of that 1075 // object saw those versions of the files, and therefore a rebuild using 1076 // those same versions would produce the same object. In contrast, if any 1077 // mtime of an input is newer than the mtime of the installed object, a 1078 // change has occurred since the build, and the build should be redone. 1079 // 1080 // Modification times are attractive because the logic is easy to 1081 // understand and the file system maintains the mtimes automatically 1082 // (less work for us). Unfortunately, there are a variety of ways in 1083 // which the mtime approach fails to detect a change and reuses a stale 1084 // object file incorrectly. (Making the opposite mistake, rebuilding 1085 // unnecessarily, is only a performance problem and not a correctness 1086 // problem, so we ignore that one.) 1087 // 1088 // As a warmup, one problem is that to be perfectly precise, we need to 1089 // compare the input mtimes against the time at the beginning of the 1090 // build, but the object file time is the time at the end of the build. 1091 // If an input file changes after being read but before the object is 1092 // written, the next build will see an object newer than the input and 1093 // will incorrectly decide that the object is up to date. We make no 1094 // attempt to detect or solve this problem. 1095 // 1096 // Another problem is that due to file system imprecision, an input and 1097 // output that are actually ordered in time have the same mtime. 1098 // This typically happens on file systems with 1-second (or, worse, 1099 // 2-second) mtime granularity and with automated scripts that write an 1100 // input and then immediately run a build, or vice versa. If an input and 1101 // an output have the same mtime, the conservative behavior is to treat 1102 // the output as out-of-date and rebuild. This can cause one or more 1103 // spurious rebuilds, but only for 1 second, until the object finally has 1104 // an mtime later than the input. 1105 // 1106 // Another problem is that binary distributions often set the mtime on 1107 // all files to the same time. If the distribution includes both inputs 1108 // and cached build outputs, the conservative solution to the previous 1109 // problem will cause unnecessary rebuilds. Worse, in such a binary 1110 // distribution, those rebuilds might not even have permission to update 1111 // the cached build output. To avoid these write errors, if an input and 1112 // output have the same mtime, we assume the output is up-to-date. 1113 // This is the opposite of what the previous problem would have us do, 1114 // but binary distributions are more common than instances of the 1115 // previous problem. 1116 // 1117 // A variant of the last problem is that some binary distributions do not 1118 // set the mtime on all files to the same time. Instead they let the file 1119 // system record mtimes as the distribution is unpacked. If the outputs 1120 // are unpacked before the inputs, they'll be older and a build will try 1121 // to rebuild them. That rebuild might hit the same write errors as in 1122 // the last scenario. We don't make any attempt to solve this, and we 1123 // haven't had many reports of it. Perhaps the only time this happens is 1124 // when people manually unpack the distribution, and most of the time 1125 // that's done as the same user who will be using it, so an initial 1126 // rebuild on first use succeeds quietly. 1127 // 1128 // More generally, people and programs change mtimes on files. The last 1129 // few problems were specific examples of this, but it's a general problem. 1130 // For example, instead of a binary distribution, copying a home 1131 // directory from one directory or machine to another might copy files 1132 // but not preserve mtimes. If the inputs are new than the outputs on the 1133 // first machine but copied first, they end up older than the outputs on 1134 // the second machine. 1135 // 1136 // Because many other build systems have the same sensitivity to mtimes, 1137 // most programs manipulating source code take pains not to break the 1138 // mtime assumptions. For example, Git does not set the mtime of files 1139 // during a checkout operation, even when checking out an old version of 1140 // the code. This decision was made specifically to work well with 1141 // mtime-based build systems. 1142 // 1143 // The killer problem, though, for mtime-based build systems is that the 1144 // build only has access to the mtimes of the inputs that still exist. 1145 // If it is possible to remove an input without changing any other inputs, 1146 // a later build will think the object is up-to-date when it is not. 1147 // This happens for Go because a package is made up of all source 1148 // files in a directory. If a source file is removed, there is no newer 1149 // mtime available recording that fact. The mtime on the directory could 1150 // be used, but it also changes when unrelated files are added to or 1151 // removed from the directory, so including the directory mtime would 1152 // cause unnecessary rebuilds, possibly many. It would also exacerbate 1153 // the problems mentioned earlier, since even programs that are careful 1154 // to maintain mtimes on files rarely maintain mtimes on directories. 1155 // 1156 // A variant of the last problem is when the inputs change for other 1157 // reasons. For example, Go 1.4 and Go 1.5 both install $GOPATH/src/mypkg 1158 // into the same target, $GOPATH/pkg/$GOOS_$GOARCH/mypkg.a. 1159 // If Go 1.4 has built mypkg into mypkg.a, a build using Go 1.5 must 1160 // rebuild mypkg.a, but from mtimes alone mypkg.a looks up-to-date. 1161 // If Go 1.5 has just been installed, perhaps the compiler will have a 1162 // newer mtime; since the compiler is considered an input, that would 1163 // trigger a rebuild. But only once, and only the last Go 1.4 build of 1164 // mypkg.a happened before Go 1.5 was installed. If a user has the two 1165 // versions installed in different locations and flips back and forth, 1166 // mtimes alone cannot tell what to do. Changing the toolchain is 1167 // changing the set of inputs, without affecting any mtimes. 1168 // 1169 // To detect the set of inputs changing, we turn away from mtimes and to 1170 // an explicit data comparison. Specifically, we build a list of the 1171 // inputs to the build, compute its SHA1 hash, and record that as the 1172 // ``build ID'' in the generated object. At the next build, we can 1173 // recompute the buid ID and compare it to the one in the generated 1174 // object. If they differ, the list of inputs has changed, so the object 1175 // is out of date and must be rebuilt. 1176 // 1177 // Because this build ID is computed before the build begins, the 1178 // comparison does not have the race that mtime comparison does. 1179 // 1180 // Making the build sensitive to changes in other state is 1181 // straightforward: include the state in the build ID hash, and if it 1182 // changes, so does the build ID, triggering a rebuild. 1183 // 1184 // To detect changes in toolchain, we include the toolchain version in 1185 // the build ID hash for package runtime, and then we include the build 1186 // IDs of all imported packages in the build ID for p. 1187 // 1188 // It is natural to think about including build tags in the build ID, but 1189 // the naive approach of just dumping the tags into the hash would cause 1190 // spurious rebuilds. For example, 'go install' and 'go install -tags neverusedtag' 1191 // produce the same binaries (assuming neverusedtag is never used). 1192 // A more precise approach would be to include only tags that have an 1193 // effect on the build. But the effect of a tag on the build is to 1194 // include or exclude a file from the compilation, and that file list is 1195 // already in the build ID hash. So the build ID is already tag-sensitive 1196 // in a perfectly precise way. So we do NOT explicitly add build tags to 1197 // the build ID hash. 1198 // 1199 // We do not include as part of the build ID the operating system, 1200 // architecture, or whether the race detector is enabled, even though all 1201 // three have an effect on the output, because that information is used 1202 // to decide the install location. Binaries for linux and binaries for 1203 // darwin are written to different directory trees; including that 1204 // information in the build ID is unnecessary (although it would be 1205 // harmless). 1206 // 1207 // TODO(rsc): Investigate the cost of putting source file content into 1208 // the build ID hash as a replacement for the use of mtimes. Using the 1209 // file content would avoid all the mtime problems, but it does require 1210 // reading all the source files, something we avoid today (we read the 1211 // beginning to find the build tags and the imports, but we stop as soon 1212 // as we see the import block is over). If the package is stale, the compiler 1213 // is going to read the files anyway. But if the package is up-to-date, the 1214 // read is overhead. 1215 // 1216 // TODO(rsc): Investigate the complexity of making the build more 1217 // precise about when individual results are needed. To be fully precise, 1218 // there are two results of a compilation: the entire .a file used by the link 1219 // and the subpiece used by later compilations (__.PKGDEF only). 1220 // If a rebuild is needed but produces the previous __.PKGDEF, then 1221 // no more recompilation due to the rebuilt package is needed, only 1222 // relinking. To date, there is nothing in the Go command to express this. 1223 // 1224 // Special Cases 1225 // 1226 // When the go command makes the wrong build decision and does not 1227 // rebuild something it should, users fall back to adding the -a flag. 1228 // Any common use of the -a flag should be considered prima facie evidence 1229 // that isStale is returning an incorrect false result in some important case. 1230 // Bugs reported in the behavior of -a itself should prompt the question 1231 // ``Why is -a being used at all? What bug does that indicate?'' 1232 // 1233 // There is a long history of changes to isStale to try to make -a into a 1234 // suitable workaround for bugs in the mtime-based decisions. 1235 // It is worth recording that history to inform (and, as much as possible, deter) future changes. 1236 // 1237 // (1) Before the build IDs were introduced, building with alternate tags 1238 // would happily reuse installed objects built without those tags. 1239 // For example, "go build -tags netgo myprog.go" would use the installed 1240 // copy of package net, even if that copy had been built without netgo. 1241 // (The netgo tag controls whether package net uses cgo or pure Go for 1242 // functionality such as name resolution.) 1243 // Using the installed non-netgo package defeats the purpose. 1244 // 1245 // Users worked around this with "go build -tags netgo -a myprog.go". 1246 // 1247 // Build IDs have made that workaround unnecessary: 1248 // "go build -tags netgo myprog.go" 1249 // cannot use a non-netgo copy of package net. 1250 // 1251 // (2) Before the build IDs were introduced, building with different toolchains, 1252 // especially changing between toolchains, tried to reuse objects stored in 1253 // $GOPATH/pkg, resulting in link-time errors about object file mismatches. 1254 // 1255 // Users worked around this with "go install -a ./...". 1256 // 1257 // Build IDs have made that workaround unnecessary: 1258 // "go install ./..." will rebuild any objects it finds that were built against 1259 // a different toolchain. 1260 // 1261 // (3) The common use of "go install -a ./..." led to reports of problems 1262 // when the -a forced the rebuild of the standard library, which for some 1263 // users was not writable. Because we didn't understand that the real 1264 // problem was the bug -a was working around, we changed -a not to 1265 // apply to the standard library. 1266 // 1267 // (4) The common use of "go build -tags netgo -a myprog.go" broke 1268 // when we changed -a not to apply to the standard library, because 1269 // if go build doesn't rebuild package net, it uses the non-netgo version. 1270 // 1271 // Users worked around this with "go build -tags netgo -installsuffix barf myprog.go". 1272 // The -installsuffix here is making the go command look for packages 1273 // in pkg/$GOOS_$GOARCH_barf instead of pkg/$GOOS_$GOARCH. 1274 // Since the former presumably doesn't exist, go build decides to rebuild 1275 // everything, including the standard library. Since go build doesn't 1276 // install anything it builds, nothing is ever written to pkg/$GOOS_$GOARCH_barf, 1277 // so repeated invocations continue to work. 1278 // 1279 // If the use of -a wasn't a red flag, the use of -installsuffix to point to 1280 // a non-existent directory in a command that installs nothing should 1281 // have been. 1282 // 1283 // (5) Now that (1) and (2) no longer need -a, we have removed the kludge 1284 // introduced in (3): once again, -a means ``rebuild everything,'' not 1285 // ``rebuild everything except the standard library.'' Only Go 1.4 had 1286 // the restricted meaning. 1287 // 1288 // In addition to these cases trying to trigger rebuilds, there are 1289 // special cases trying NOT to trigger rebuilds. The main one is that for 1290 // a variety of reasons (see above), the install process for a Go release 1291 // cannot be relied upon to set the mtimes such that the go command will 1292 // think the standard library is up to date. So the mtime evidence is 1293 // ignored for the standard library if we find ourselves in a release 1294 // version of Go. Build ID-based staleness checks still apply to the 1295 // standard library, even in release versions. This makes 1296 // 'go build -tags netgo' work, among other things. 1297 1298 // isStale reports whether package p needs to be rebuilt. 1299 func isStale(p *Package) bool { 1300 if p.Standard && (p.ImportPath == "unsafe" || buildContext.Compiler == "gccgo") { 1301 // fake, builtin package 1302 return false 1303 } 1304 if p.Error != nil { 1305 return true 1306 } 1307 1308 // A package without Go sources means we only found 1309 // the installed .a file. Since we don't know how to rebuild 1310 // it, it can't be stale, even if -a is set. This enables binary-only 1311 // distributions of Go packages, although such binaries are 1312 // only useful with the specific version of the toolchain that 1313 // created them. 1314 if len(p.gofiles) == 0 && !p.usesSwig() { 1315 return false 1316 } 1317 1318 // If the -a flag is given, rebuild everything. 1319 if buildA { 1320 return true 1321 } 1322 1323 // If there's no install target or it's already marked stale, we have to rebuild. 1324 if p.target == "" || p.Stale { 1325 return true 1326 } 1327 1328 // Package is stale if completely unbuilt. 1329 fi, err := os.Stat(p.target) 1330 if err != nil { 1331 return true 1332 } 1333 1334 // Package is stale if the expected build ID differs from the 1335 // recorded build ID. This catches changes like a source file 1336 // being removed from a package directory. See issue 3895. 1337 // It also catches changes in build tags that affect the set of 1338 // files being compiled. See issue 9369. 1339 // It also catches changes in toolchain, like when flipping between 1340 // two versions of Go compiling a single GOPATH. 1341 // See issue 8290 and issue 10702. 1342 targetBuildID, err := readBuildID(p) 1343 if err == nil && targetBuildID != p.buildID { 1344 return true 1345 } 1346 1347 // Package is stale if a dependency is. 1348 for _, p1 := range p.deps { 1349 if p1.Stale { 1350 return true 1351 } 1352 } 1353 1354 // The checks above are content-based staleness. 1355 // We assume they are always accurate. 1356 // 1357 // The checks below are mtime-based staleness. 1358 // We hope they are accurate, but we know that they fail in the case of 1359 // prebuilt Go installations that don't preserve the build mtimes 1360 // (for example, if the pkg/ mtimes are before the src/ mtimes). 1361 // See the large comment above isStale for details. 1362 1363 // If we are running a release copy of Go and didn't find a content-based 1364 // reason to rebuild the standard packages, do not rebuild them. 1365 // They may not be writable anyway, but they are certainly not changing. 1366 // This makes 'go build' skip the standard packages when 1367 // using an official release, even when the mtimes have been changed. 1368 // See issue 3036, issue 3149, issue 4106, issue 8290. 1369 // (If a change to a release tree must be made by hand, the way to force the 1370 // install is to run make.bash, which will remove the old package archives 1371 // before rebuilding.) 1372 if p.Standard && isGoRelease { 1373 return false 1374 } 1375 1376 // Time-based staleness. 1377 1378 built := fi.ModTime() 1379 1380 olderThan := func(file string) bool { 1381 fi, err := os.Stat(file) 1382 return err != nil || fi.ModTime().After(built) 1383 } 1384 1385 // Package is stale if a dependency is, or if a dependency is newer. 1386 for _, p1 := range p.deps { 1387 if p1.target != "" && olderThan(p1.target) { 1388 return true 1389 } 1390 } 1391 1392 // As a courtesy to developers installing new versions of the compiler 1393 // frequently, define that packages are stale if they are 1394 // older than the compiler, and commands if they are older than 1395 // the linker. This heuristic will not work if the binaries are 1396 // back-dated, as some binary distributions may do, but it does handle 1397 // a very common case. 1398 // See issue 3036. 1399 // Exclude $GOROOT, under the assumption that people working on 1400 // the compiler may want to control when everything gets rebuilt, 1401 // and people updating the Go repository will run make.bash or all.bash 1402 // and get a full rebuild anyway. 1403 // Excluding $GOROOT used to also fix issue 4106, but that's now 1404 // taken care of above (at least when the installed Go is a released version). 1405 if p.Root != goroot { 1406 if olderThan(buildToolchain.compiler()) { 1407 return true 1408 } 1409 if p.build.IsCommand() && olderThan(buildToolchain.linker()) { 1410 return true 1411 } 1412 } 1413 1414 // Note: Until Go 1.5, we had an additional shortcut here. 1415 // We built a list of the workspace roots ($GOROOT, each $GOPATH) 1416 // containing targets directly named on the command line, 1417 // and if p were not in any of those, it would be treated as up-to-date 1418 // as long as it is built. The goal was to avoid rebuilding a system-installed 1419 // $GOROOT, unless something from $GOROOT were explicitly named 1420 // on the command line (like go install math). 1421 // That's now handled by the isGoRelease clause above. 1422 // The other effect of the shortcut was to isolate different entries in 1423 // $GOPATH from each other. This had the unfortunate effect that 1424 // if you had (say), GOPATH listing two entries, one for commands 1425 // and one for libraries, and you did a 'git pull' in the library one 1426 // and then tried 'go install commands/...', it would build the new libraries 1427 // during the first build (because they wouldn't have been installed at all) 1428 // but then subsequent builds would not rebuild the libraries, even if the 1429 // mtimes indicate they are stale, because the different GOPATH entries 1430 // were treated differently. This behavior was confusing when using 1431 // non-trivial GOPATHs, which were particularly common with some 1432 // code management conventions, like the original godep. 1433 // Since the $GOROOT case (the original motivation) is handled separately, 1434 // we no longer put a barrier between the different $GOPATH entries. 1435 // 1436 // One implication of this is that if there is a system directory for 1437 // non-standard Go packages that is included in $GOPATH, the mtimes 1438 // on those compiled packages must be no earlier than the mtimes 1439 // on the source files. Since most distributions use the same mtime 1440 // for all files in a tree, they will be unaffected. People using plain 1441 // tar x to extract system-installed packages will need to adjust mtimes, 1442 // but it's better to force them to get the mtimes right than to ignore 1443 // the mtimes and thereby do the wrong thing in common use cases. 1444 // 1445 // So there is no GOPATH vs GOPATH shortcut here anymore. 1446 // 1447 // If something needs to come back here, we could try writing a dummy 1448 // file with a random name to the $GOPATH/pkg directory (and removing it) 1449 // to test for write access, and then skip GOPATH roots we don't have write 1450 // access to. But hopefully we can just use the mtimes always. 1451 1452 srcs := stringList(p.GoFiles, p.CFiles, p.CXXFiles, p.MFiles, p.HFiles, p.SFiles, p.CgoFiles, p.SysoFiles, p.SwigFiles, p.SwigCXXFiles) 1453 for _, src := range srcs { 1454 if olderThan(filepath.Join(p.Dir, src)) { 1455 return true 1456 } 1457 } 1458 1459 return false 1460 } 1461 1462 // computeBuildID computes the build ID for p, leaving it in p.buildID. 1463 // Build ID is a hash of the information we want to detect changes in. 1464 // See the long comment in isStale for details. 1465 func computeBuildID(p *Package) { 1466 h := sha1.New() 1467 1468 // Include the list of files compiled as part of the package. 1469 // This lets us detect removed files. See issue 3895. 1470 inputFiles := stringList( 1471 p.GoFiles, 1472 p.CgoFiles, 1473 p.CFiles, 1474 p.CXXFiles, 1475 p.MFiles, 1476 p.HFiles, 1477 p.SFiles, 1478 p.SysoFiles, 1479 p.SwigFiles, 1480 p.SwigCXXFiles, 1481 ) 1482 for _, file := range inputFiles { 1483 fmt.Fprintf(h, "file %s\n", file) 1484 } 1485 1486 // Include the content of runtime/zversion.go in the hash 1487 // for package runtime. This will give package runtime a 1488 // different build ID in each Go release. 1489 if p.Standard && p.ImportPath == "runtime" { 1490 data, _ := ioutil.ReadFile(filepath.Join(p.Dir, "zversion.go")) 1491 fmt.Fprintf(h, "zversion %q\n", string(data)) 1492 } 1493 1494 // Include the build IDs of any dependencies in the hash. 1495 // This, combined with the runtime/zversion content, 1496 // will cause packages to have different build IDs when 1497 // compiled with different Go releases. 1498 // This helps the go command know to recompile when 1499 // people use the same GOPATH but switch between 1500 // different Go releases. See issue 10702. 1501 // This is also a better fix for issue 8290. 1502 for _, p1 := range p.deps { 1503 fmt.Fprintf(h, "dep %s %s\n", p1.ImportPath, p1.buildID) 1504 } 1505 1506 p.buildID = fmt.Sprintf("%x", h.Sum(nil)) 1507 } 1508 1509 var cwd, _ = os.Getwd() 1510 1511 var cmdCache = map[string]*Package{} 1512 1513 // loadPackage is like loadImport but is used for command-line arguments, 1514 // not for paths found in import statements. In addition to ordinary import paths, 1515 // loadPackage accepts pseudo-paths beginning with cmd/ to denote commands 1516 // in the Go command directory, as well as paths to those directories. 1517 func loadPackage(arg string, stk *importStack) *Package { 1518 if build.IsLocalImport(arg) { 1519 dir := arg 1520 if !filepath.IsAbs(dir) { 1521 if abs, err := filepath.Abs(dir); err == nil { 1522 // interpret relative to current directory 1523 dir = abs 1524 } 1525 } 1526 if sub, ok := hasSubdir(gorootSrc, dir); ok && strings.HasPrefix(sub, "cmd/") && !strings.Contains(sub[4:], "/") { 1527 arg = sub 1528 } 1529 } 1530 if strings.HasPrefix(arg, "cmd/") && !strings.Contains(arg[4:], "/") { 1531 if p := cmdCache[arg]; p != nil { 1532 return p 1533 } 1534 stk.push(arg) 1535 defer stk.pop() 1536 1537 bp, err := buildContext.ImportDir(filepath.Join(gorootSrc, arg), 0) 1538 bp.ImportPath = arg 1539 bp.Goroot = true 1540 bp.BinDir = gorootBin 1541 if gobin != "" { 1542 bp.BinDir = gobin 1543 } 1544 bp.Root = goroot 1545 bp.SrcRoot = gorootSrc 1546 p := new(Package) 1547 cmdCache[arg] = p 1548 p.load(stk, bp, err) 1549 if p.Error == nil && p.Name != "main" { 1550 p.Error = &PackageError{ 1551 ImportStack: stk.copy(), 1552 Err: fmt.Sprintf("expected package main but found package %s in %s", p.Name, p.Dir), 1553 } 1554 } 1555 return p 1556 } 1557 1558 // Wasn't a command; must be a package. 1559 // If it is a local import path but names a standard package, 1560 // we treat it as if the user specified the standard package. 1561 // This lets you run go test ./ioutil in package io and be 1562 // referring to io/ioutil rather than a hypothetical import of 1563 // "./ioutil". 1564 if build.IsLocalImport(arg) { 1565 bp, _ := buildContext.ImportDir(filepath.Join(cwd, arg), build.FindOnly) 1566 if bp.ImportPath != "" && bp.ImportPath != "." { 1567 arg = bp.ImportPath 1568 } 1569 } 1570 1571 return loadImport(arg, cwd, nil, stk, nil, 0) 1572 } 1573 1574 // packages returns the packages named by the 1575 // command line arguments 'args'. If a named package 1576 // cannot be loaded at all (for example, if the directory does not exist), 1577 // then packages prints an error and does not include that 1578 // package in the results. However, if errors occur trying 1579 // to load dependencies of a named package, the named 1580 // package is still returned, with p.Incomplete = true 1581 // and details in p.DepsErrors. 1582 func packages(args []string) []*Package { 1583 var pkgs []*Package 1584 for _, pkg := range packagesAndErrors(args) { 1585 if pkg.Error != nil { 1586 errorf("can't load package: %s", pkg.Error) 1587 continue 1588 } 1589 pkgs = append(pkgs, pkg) 1590 } 1591 return pkgs 1592 } 1593 1594 // packagesAndErrors is like 'packages' but returns a 1595 // *Package for every argument, even the ones that 1596 // cannot be loaded at all. 1597 // The packages that fail to load will have p.Error != nil. 1598 func packagesAndErrors(args []string) []*Package { 1599 if len(args) > 0 && strings.HasSuffix(args[0], ".go") { 1600 return []*Package{goFilesPackage(args)} 1601 } 1602 1603 args = importPaths(args) 1604 var pkgs []*Package 1605 var stk importStack 1606 var set = make(map[string]bool) 1607 1608 for _, arg := range args { 1609 if !set[arg] { 1610 pkgs = append(pkgs, loadPackage(arg, &stk)) 1611 set[arg] = true 1612 } 1613 } 1614 computeStale(pkgs...) 1615 1616 return pkgs 1617 } 1618 1619 // packagesForBuild is like 'packages' but fails if any of 1620 // the packages or their dependencies have errors 1621 // (cannot be built). 1622 func packagesForBuild(args []string) []*Package { 1623 pkgs := packagesAndErrors(args) 1624 printed := map[*PackageError]bool{} 1625 for _, pkg := range pkgs { 1626 if pkg.Error != nil { 1627 errorf("can't load package: %s", pkg.Error) 1628 } 1629 for _, err := range pkg.DepsErrors { 1630 // Since these are errors in dependencies, 1631 // the same error might show up multiple times, 1632 // once in each package that depends on it. 1633 // Only print each once. 1634 if !printed[err] { 1635 printed[err] = true 1636 errorf("%s", err) 1637 } 1638 } 1639 } 1640 exitIfErrors() 1641 1642 // Check for duplicate loads of the same package. 1643 // That should be impossible, but if it does happen then 1644 // we end up trying to build the same package twice, 1645 // usually in parallel overwriting the same files, 1646 // which doesn't work very well. 1647 seen := map[string]bool{} 1648 reported := map[string]bool{} 1649 for _, pkg := range packageList(pkgs) { 1650 if seen[pkg.ImportPath] && !reported[pkg.ImportPath] { 1651 reported[pkg.ImportPath] = true 1652 errorf("internal error: duplicate loads of %s", pkg.ImportPath) 1653 } 1654 seen[pkg.ImportPath] = true 1655 } 1656 exitIfErrors() 1657 1658 return pkgs 1659 } 1660 1661 // hasSubdir reports whether dir is a subdirectory of 1662 // (possibly multiple levels below) root. 1663 // If so, it sets rel to the path fragment that must be 1664 // appended to root to reach dir. 1665 func hasSubdir(root, dir string) (rel string, ok bool) { 1666 if p, err := filepath.EvalSymlinks(root); err == nil { 1667 root = p 1668 } 1669 if p, err := filepath.EvalSymlinks(dir); err == nil { 1670 dir = p 1671 } 1672 const sep = string(filepath.Separator) 1673 root = filepath.Clean(root) 1674 if !strings.HasSuffix(root, sep) { 1675 root += sep 1676 } 1677 dir = filepath.Clean(dir) 1678 if !strings.HasPrefix(dir, root) { 1679 return "", false 1680 } 1681 return filepath.ToSlash(dir[len(root):]), true 1682 } 1683 1684 var ( 1685 errBuildIDToolchain = fmt.Errorf("build ID only supported in gc toolchain") 1686 errBuildIDMalformed = fmt.Errorf("malformed object file") 1687 errBuildIDUnknown = fmt.Errorf("lost build ID") 1688 ) 1689 1690 var ( 1691 bangArch = []byte("!<arch>") 1692 pkgdef = []byte("__.PKGDEF") 1693 goobject = []byte("go object ") 1694 buildid = []byte("build id ") 1695 ) 1696 1697 // readBuildID reads the build ID from an archive or binary. 1698 // It only supports the gc toolchain. 1699 // Other toolchain maintainers should adjust this function. 1700 func readBuildID(p *Package) (id string, err error) { 1701 if buildToolchain != (gcToolchain{}) { 1702 return "", errBuildIDToolchain 1703 } 1704 1705 // For commands, read build ID directly from binary. 1706 if p.Name == "main" { 1707 return ReadBuildIDFromBinary(p.Target) 1708 } 1709 1710 // Otherwise, we expect to have an archive (.a) file, 1711 // and we can read the build ID from the Go export data. 1712 if !strings.HasSuffix(p.Target, ".a") { 1713 return "", &os.PathError{Op: "parse", Path: p.Target, Err: errBuildIDUnknown} 1714 } 1715 1716 // Read just enough of the target to fetch the build ID. 1717 // The archive is expected to look like: 1718 // 1719 // !<arch> 1720 // __.PKGDEF 0 0 0 644 7955 ` 1721 // go object darwin amd64 devel X:none 1722 // build id "b41e5c45250e25c9fd5e9f9a1de7857ea0d41224" 1723 // 1724 // The variable-sized strings are GOOS, GOARCH, and the experiment list (X:none). 1725 // Reading the first 1024 bytes should be plenty. 1726 f, err := os.Open(p.Target) 1727 if err != nil { 1728 return "", err 1729 } 1730 data := make([]byte, 1024) 1731 n, err := io.ReadFull(f, data) 1732 f.Close() 1733 1734 if err != nil && n == 0 { 1735 return "", err 1736 } 1737 1738 bad := func() (string, error) { 1739 return "", &os.PathError{Op: "parse", Path: p.Target, Err: errBuildIDMalformed} 1740 } 1741 1742 // Archive header. 1743 for i := 0; ; i++ { // returns during i==3 1744 j := bytes.IndexByte(data, '\n') 1745 if j < 0 { 1746 return bad() 1747 } 1748 line := data[:j] 1749 data = data[j+1:] 1750 switch i { 1751 case 0: 1752 if !bytes.Equal(line, bangArch) { 1753 return bad() 1754 } 1755 case 1: 1756 if !bytes.HasPrefix(line, pkgdef) { 1757 return bad() 1758 } 1759 case 2: 1760 if !bytes.HasPrefix(line, goobject) { 1761 return bad() 1762 } 1763 case 3: 1764 if !bytes.HasPrefix(line, buildid) { 1765 // Found the object header, just doesn't have a build id line. 1766 // Treat as successful, with empty build id. 1767 return "", nil 1768 } 1769 id, err := strconv.Unquote(string(line[len(buildid):])) 1770 if err != nil { 1771 return bad() 1772 } 1773 return id, nil 1774 } 1775 } 1776 } 1777 1778 var ( 1779 goBuildPrefix = []byte("\xff Go build ID: \"") 1780 goBuildEnd = []byte("\"\n \xff") 1781 1782 elfPrefix = []byte("\x7fELF") 1783 ) 1784 1785 // ReadBuildIDFromBinary reads the build ID from a binary. 1786 // 1787 // ELF binaries store the build ID in a proper PT_NOTE section. 1788 // 1789 // Other binary formats are not so flexible. For those, the linker 1790 // stores the build ID as non-instruction bytes at the very beginning 1791 // of the text segment, which should appear near the beginning 1792 // of the file. This is clumsy but fairly portable. Custom locations 1793 // can be added for other binary types as needed, like we did for ELF. 1794 func ReadBuildIDFromBinary(filename string) (id string, err error) { 1795 if filename == "" { 1796 return "", &os.PathError{Op: "parse", Path: filename, Err: errBuildIDUnknown} 1797 } 1798 1799 // Read the first 16 kB of the binary file. 1800 // That should be enough to find the build ID. 1801 // In ELF files, the build ID is in the leading headers, 1802 // which are typically less than 4 kB, not to mention 16 kB. 1803 // On other systems, we're trying to read enough that 1804 // we get the beginning of the text segment in the read. 1805 // The offset where the text segment begins in a hello 1806 // world compiled for each different object format today: 1807 // 1808 // Plan 9: 0x20 1809 // Windows: 0x600 1810 // Mach-O: 0x2000 1811 // 1812 f, err := os.Open(filename) 1813 if err != nil { 1814 return "", err 1815 } 1816 defer f.Close() 1817 1818 data := make([]byte, 16*1024) 1819 _, err = io.ReadFull(f, data) 1820 if err == io.ErrUnexpectedEOF { 1821 err = nil 1822 } 1823 if err != nil { 1824 return "", err 1825 } 1826 1827 if bytes.HasPrefix(data, elfPrefix) { 1828 return readELFGoBuildID(filename, f, data) 1829 } 1830 1831 i := bytes.Index(data, goBuildPrefix) 1832 if i < 0 { 1833 // Missing. Treat as successful but build ID empty. 1834 return "", nil 1835 } 1836 1837 j := bytes.Index(data[i+len(goBuildPrefix):], goBuildEnd) 1838 if j < 0 { 1839 return "", &os.PathError{Op: "parse", Path: filename, Err: errBuildIDMalformed} 1840 } 1841 1842 quoted := data[i+len(goBuildPrefix)-1 : i+len(goBuildPrefix)+j+1] 1843 id, err = strconv.Unquote(string(quoted)) 1844 if err != nil { 1845 return "", &os.PathError{Op: "parse", Path: filename, Err: errBuildIDMalformed} 1846 } 1847 1848 return id, nil 1849 }