github.com/peggyl/go@v0.0.0-20151008231540-ae315999c2d5/src/cmd/go/pkg.go (about) 1 // Copyright 2011 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package main 6 7 import ( 8 "bytes" 9 "crypto/sha1" 10 "errors" 11 "fmt" 12 "go/build" 13 "go/scanner" 14 "go/token" 15 "io" 16 "io/ioutil" 17 "os" 18 pathpkg "path" 19 "path/filepath" 20 "runtime" 21 "sort" 22 "strconv" 23 "strings" 24 "unicode" 25 ) 26 27 // A Package describes a single package found in a directory. 28 type Package struct { 29 // Note: These fields are part of the go command's public API. 30 // See list.go. It is okay to add fields, but not to change or 31 // remove existing ones. Keep in sync with list.go 32 Dir string `json:",omitempty"` // directory containing package sources 33 ImportPath string `json:",omitempty"` // import path of package in dir 34 ImportComment string `json:",omitempty"` // path in import comment on package statement 35 Name string `json:",omitempty"` // package name 36 Doc string `json:",omitempty"` // package documentation string 37 Target string `json:",omitempty"` // install path 38 Shlib string `json:",omitempty"` // the shared library that contains this package (only set when -linkshared) 39 Goroot bool `json:",omitempty"` // is this package found in the Go root? 40 Standard bool `json:",omitempty"` // is this package part of the standard Go library? 41 Stale bool `json:",omitempty"` // would 'go install' do anything for this package? 42 Root string `json:",omitempty"` // Go root or Go path dir containing this package 43 ConflictDir string `json:",omitempty"` // Dir is hidden by this other directory 44 45 // Source files 46 GoFiles []string `json:",omitempty"` // .go source files (excluding CgoFiles, TestGoFiles, XTestGoFiles) 47 CgoFiles []string `json:",omitempty"` // .go sources files that import "C" 48 IgnoredGoFiles []string `json:",omitempty"` // .go sources ignored due to build constraints 49 CFiles []string `json:",omitempty"` // .c source files 50 CXXFiles []string `json:",omitempty"` // .cc, .cpp and .cxx source files 51 MFiles []string `json:",omitempty"` // .m source files 52 HFiles []string `json:",omitempty"` // .h, .hh, .hpp and .hxx source files 53 SFiles []string `json:",omitempty"` // .s source files 54 SwigFiles []string `json:",omitempty"` // .swig files 55 SwigCXXFiles []string `json:",omitempty"` // .swigcxx files 56 SysoFiles []string `json:",omitempty"` // .syso system object files added to package 57 58 // Cgo directives 59 CgoCFLAGS []string `json:",omitempty"` // cgo: flags for C compiler 60 CgoCPPFLAGS []string `json:",omitempty"` // cgo: flags for C preprocessor 61 CgoCXXFLAGS []string `json:",omitempty"` // cgo: flags for C++ compiler 62 CgoLDFLAGS []string `json:",omitempty"` // cgo: flags for linker 63 CgoPkgConfig []string `json:",omitempty"` // cgo: pkg-config names 64 65 // Dependency information 66 Imports []string `json:",omitempty"` // import paths used by this package 67 Deps []string `json:",omitempty"` // all (recursively) imported dependencies 68 69 // Error information 70 Incomplete bool `json:",omitempty"` // was there an error loading this package or dependencies? 71 Error *PackageError `json:",omitempty"` // error loading this package (not dependencies) 72 DepsErrors []*PackageError `json:",omitempty"` // errors loading dependencies 73 74 // Test information 75 TestGoFiles []string `json:",omitempty"` // _test.go files in package 76 TestImports []string `json:",omitempty"` // imports from TestGoFiles 77 XTestGoFiles []string `json:",omitempty"` // _test.go files outside package 78 XTestImports []string `json:",omitempty"` // imports from XTestGoFiles 79 80 // Unexported fields are not part of the public API. 81 build *build.Package 82 pkgdir string // overrides build.PkgDir 83 imports []*Package 84 deps []*Package 85 gofiles []string // GoFiles+CgoFiles+TestGoFiles+XTestGoFiles files, absolute paths 86 sfiles []string 87 allgofiles []string // gofiles + IgnoredGoFiles, absolute paths 88 target string // installed file for this package (may be executable) 89 fake bool // synthesized package 90 external bool // synthesized external test package 91 forceBuild bool // this package must be rebuilt 92 forceLibrary bool // this package is a library (even if named "main") 93 cmdline bool // defined by files listed on command line 94 local bool // imported via local path (./ or ../) 95 localPrefix string // interpret ./ and ../ imports relative to this prefix 96 exeName string // desired name for temporary executable 97 coverMode string // preprocess Go source files with the coverage tool in this mode 98 coverVars map[string]*CoverVar // variables created by coverage analysis 99 omitDWARF bool // tell linker not to write DWARF information 100 buildID string // expected build ID for generated package 101 gobinSubdir bool // install target would be subdir of GOBIN 102 } 103 104 // vendored returns the vendor-resolved version of imports, 105 // which should be p.TestImports or p.XTestImports, NOT p.Imports. 106 // The imports in p.TestImports and p.XTestImports are not recursively 107 // loaded during the initial load of p, so they list the imports found in 108 // the source file, but most processing should be over the vendor-resolved 109 // import paths. We do this resolution lazily both to avoid file system work 110 // and because the eventual real load of the test imports (during 'go test') 111 // can produce better error messages if it starts with the original paths. 112 // The initial load of p loads all the non-test imports and rewrites 113 // the vendored paths, so nothing should ever call p.vendored(p.Imports). 114 func (p *Package) vendored(imports []string) []string { 115 if len(imports) > 0 && len(p.Imports) > 0 && &imports[0] == &p.Imports[0] { 116 panic("internal error: p.vendored(p.Imports) called") 117 } 118 seen := make(map[string]bool) 119 var all []string 120 for _, path := range imports { 121 path, _ = vendoredImportPath(p, path) 122 if !seen[path] { 123 seen[path] = true 124 all = append(all, path) 125 } 126 } 127 sort.Strings(all) 128 return all 129 } 130 131 // CoverVar holds the name of the generated coverage variables targeting the named file. 132 type CoverVar struct { 133 File string // local file name 134 Var string // name of count struct 135 } 136 137 func (p *Package) copyBuild(pp *build.Package) { 138 p.build = pp 139 140 if pp.PkgTargetRoot != "" && buildPkgdir != "" { 141 old := pp.PkgTargetRoot 142 pp.PkgRoot = buildPkgdir 143 pp.PkgTargetRoot = buildPkgdir 144 pp.PkgObj = filepath.Join(buildPkgdir, strings.TrimPrefix(pp.PkgObj, old)) 145 } 146 147 p.Dir = pp.Dir 148 p.ImportPath = pp.ImportPath 149 p.ImportComment = pp.ImportComment 150 p.Name = pp.Name 151 p.Doc = pp.Doc 152 p.Root = pp.Root 153 p.ConflictDir = pp.ConflictDir 154 // TODO? Target 155 p.Goroot = pp.Goroot 156 p.Standard = p.Goroot && p.ImportPath != "" && !strings.Contains(p.ImportPath, ".") 157 p.GoFiles = pp.GoFiles 158 p.CgoFiles = pp.CgoFiles 159 p.IgnoredGoFiles = pp.IgnoredGoFiles 160 p.CFiles = pp.CFiles 161 p.CXXFiles = pp.CXXFiles 162 p.MFiles = pp.MFiles 163 p.HFiles = pp.HFiles 164 p.SFiles = pp.SFiles 165 p.SwigFiles = pp.SwigFiles 166 p.SwigCXXFiles = pp.SwigCXXFiles 167 p.SysoFiles = pp.SysoFiles 168 p.CgoCFLAGS = pp.CgoCFLAGS 169 p.CgoCPPFLAGS = pp.CgoCPPFLAGS 170 p.CgoCXXFLAGS = pp.CgoCXXFLAGS 171 p.CgoLDFLAGS = pp.CgoLDFLAGS 172 p.CgoPkgConfig = pp.CgoPkgConfig 173 p.Imports = pp.Imports 174 p.TestGoFiles = pp.TestGoFiles 175 p.TestImports = pp.TestImports 176 p.XTestGoFiles = pp.XTestGoFiles 177 p.XTestImports = pp.XTestImports 178 } 179 180 // A PackageError describes an error loading information about a package. 181 type PackageError struct { 182 ImportStack []string // shortest path from package named on command line to this one 183 Pos string // position of error 184 Err string // the error itself 185 isImportCycle bool // the error is an import cycle 186 hard bool // whether the error is soft or hard; soft errors are ignored in some places 187 } 188 189 func (p *PackageError) Error() string { 190 // Import cycles deserve special treatment. 191 if p.isImportCycle { 192 return fmt.Sprintf("%s\npackage %s\n", p.Err, strings.Join(p.ImportStack, "\n\timports ")) 193 } 194 if p.Pos != "" { 195 // Omit import stack. The full path to the file where the error 196 // is the most important thing. 197 return p.Pos + ": " + p.Err 198 } 199 if len(p.ImportStack) == 0 { 200 return p.Err 201 } 202 return "package " + strings.Join(p.ImportStack, "\n\timports ") + ": " + p.Err 203 } 204 205 // An importStack is a stack of import paths. 206 type importStack []string 207 208 func (s *importStack) push(p string) { 209 *s = append(*s, p) 210 } 211 212 func (s *importStack) pop() { 213 *s = (*s)[0 : len(*s)-1] 214 } 215 216 func (s *importStack) copy() []string { 217 return append([]string{}, *s...) 218 } 219 220 // shorterThan reports whether sp is shorter than t. 221 // We use this to record the shortest import sequence 222 // that leads to a particular package. 223 func (sp *importStack) shorterThan(t []string) bool { 224 s := *sp 225 if len(s) != len(t) { 226 return len(s) < len(t) 227 } 228 // If they are the same length, settle ties using string ordering. 229 for i := range s { 230 if s[i] != t[i] { 231 return s[i] < t[i] 232 } 233 } 234 return false // they are equal 235 } 236 237 // packageCache is a lookup cache for loadPackage, 238 // so that if we look up a package multiple times 239 // we return the same pointer each time. 240 var packageCache = map[string]*Package{} 241 242 // reloadPackage is like loadPackage but makes sure 243 // not to use the package cache. 244 func reloadPackage(arg string, stk *importStack) *Package { 245 p := packageCache[arg] 246 if p != nil { 247 delete(packageCache, p.Dir) 248 delete(packageCache, p.ImportPath) 249 } 250 return loadPackage(arg, stk) 251 } 252 253 // The Go 1.5 vendoring experiment was enabled by setting GO15VENDOREXPERIMENT=1. 254 // In Go 1.6 this is on by default and is disabled by setting GO15VENDOREXPERIMENT=0. 255 // In Go 1.7 the variable will stop having any effect. 256 // The variable is obnoxiously long so that years from now when people find it in 257 // their profiles and wonder what it does, there is some chance that a web search 258 // might answer the question. 259 var go15VendorExperiment = os.Getenv("GO15VENDOREXPERIMENT") != "0" 260 261 // dirToImportPath returns the pseudo-import path we use for a package 262 // outside the Go path. It begins with _/ and then contains the full path 263 // to the directory. If the package lives in c:\home\gopher\my\pkg then 264 // the pseudo-import path is _/c_/home/gopher/my/pkg. 265 // Using a pseudo-import path like this makes the ./ imports no longer 266 // a special case, so that all the code to deal with ordinary imports works 267 // automatically. 268 func dirToImportPath(dir string) string { 269 return pathpkg.Join("_", strings.Map(makeImportValid, filepath.ToSlash(dir))) 270 } 271 272 func makeImportValid(r rune) rune { 273 // Should match Go spec, compilers, and ../../go/parser/parser.go:/isValidImport. 274 const illegalChars = `!"#$%&'()*,:;<=>?[\]^{|}` + "`\uFFFD" 275 if !unicode.IsGraphic(r) || unicode.IsSpace(r) || strings.ContainsRune(illegalChars, r) { 276 return '_' 277 } 278 return r 279 } 280 281 // Mode flags for loadImport and download (in get.go). 282 const ( 283 // useVendor means that loadImport should do vendor expansion 284 // (provided the vendoring experiment is enabled). 285 // That is, useVendor means that the import path came from 286 // a source file and has not been vendor-expanded yet. 287 // Every import path should be loaded initially with useVendor, 288 // and then the expanded version (with the /vendor/ in it) gets 289 // recorded as the canonical import path. At that point, future loads 290 // of that package must not pass useVendor, because 291 // disallowVendor will reject direct use of paths containing /vendor/. 292 useVendor = 1 << iota 293 294 // getTestDeps is for download (part of "go get") and indicates 295 // that test dependencies should be fetched too. 296 getTestDeps 297 ) 298 299 // loadImport scans the directory named by path, which must be an import path, 300 // but possibly a local import path (an absolute file system path or one beginning 301 // with ./ or ../). A local relative path is interpreted relative to srcDir. 302 // It returns a *Package describing the package found in that directory. 303 func loadImport(path, srcDir string, parent *Package, stk *importStack, importPos []token.Position, mode int) *Package { 304 stk.push(path) 305 defer stk.pop() 306 307 // Determine canonical identifier for this package. 308 // For a local import the identifier is the pseudo-import path 309 // we create from the full directory to the package. 310 // Otherwise it is the usual import path. 311 // For vendored imports, it is the expanded form. 312 importPath := path 313 origPath := path 314 isLocal := build.IsLocalImport(path) 315 var vendorSearch []string 316 if isLocal { 317 importPath = dirToImportPath(filepath.Join(srcDir, path)) 318 } else if mode&useVendor != 0 { 319 path, vendorSearch = vendoredImportPath(parent, path) 320 importPath = path 321 } 322 323 if p := packageCache[importPath]; p != nil { 324 if perr := disallowInternal(srcDir, p, stk); perr != p { 325 return perr 326 } 327 if mode&useVendor != 0 { 328 if perr := disallowVendor(srcDir, origPath, p, stk); perr != p { 329 return perr 330 } 331 } 332 return reusePackage(p, stk) 333 } 334 335 p := new(Package) 336 p.local = isLocal 337 p.ImportPath = importPath 338 packageCache[importPath] = p 339 340 // Load package. 341 // Import always returns bp != nil, even if an error occurs, 342 // in order to return partial information. 343 // 344 // TODO: After Go 1, decide when to pass build.AllowBinary here. 345 // See issue 3268 for mistakes to avoid. 346 bp, err := buildContext.Import(path, srcDir, build.ImportComment) 347 348 // If we got an error from go/build about package not found, 349 // it contains the directories from $GOROOT and $GOPATH that 350 // were searched. Add to that message the vendor directories 351 // that were searched. 352 if err != nil && len(vendorSearch) > 0 { 353 // NOTE(rsc): The direct text manipulation here is fairly awful, 354 // but it avoids defining new go/build API (an exported error type) 355 // late in the Go 1.5 release cycle. If this turns out to be a more general 356 // problem we could define a real error type when the decision can be 357 // considered more carefully. 358 text := err.Error() 359 if strings.Contains(text, "cannot find package \"") && strings.Contains(text, "\" in any of:\n\t") { 360 old := strings.SplitAfter(text, "\n") 361 lines := []string{old[0]} 362 for _, dir := range vendorSearch { 363 lines = append(lines, "\t"+dir+" (vendor tree)\n") 364 } 365 lines = append(lines, old[1:]...) 366 err = errors.New(strings.Join(lines, "")) 367 } 368 } 369 bp.ImportPath = importPath 370 if gobin != "" { 371 bp.BinDir = gobin 372 } 373 if err == nil && !isLocal && bp.ImportComment != "" && bp.ImportComment != path && 374 (!go15VendorExperiment || (!strings.Contains(path, "/vendor/") && !strings.HasPrefix(path, "vendor/"))) { 375 err = fmt.Errorf("code in directory %s expects import %q", bp.Dir, bp.ImportComment) 376 } 377 p.load(stk, bp, err) 378 if p.Error != nil && len(importPos) > 0 { 379 pos := importPos[0] 380 pos.Filename = shortPath(pos.Filename) 381 p.Error.Pos = pos.String() 382 } 383 384 if perr := disallowInternal(srcDir, p, stk); perr != p { 385 return perr 386 } 387 if mode&useVendor != 0 { 388 if perr := disallowVendor(srcDir, origPath, p, stk); perr != p { 389 return perr 390 } 391 } 392 393 return p 394 } 395 396 var isDirCache = map[string]bool{} 397 398 func isDir(path string) bool { 399 result, ok := isDirCache[path] 400 if ok { 401 return result 402 } 403 404 fi, err := os.Stat(path) 405 result = err == nil && fi.IsDir() 406 isDirCache[path] = result 407 return result 408 } 409 410 // vendoredImportPath returns the expansion of path when it appears in parent. 411 // If parent is x/y/z, then path might expand to x/y/z/vendor/path, x/y/vendor/path, 412 // x/vendor/path, vendor/path, or else stay path if none of those exist. 413 // vendoredImportPath returns the expanded path or, if no expansion is found, the original. 414 // If no expansion is found, vendoredImportPath also returns a list of vendor directories 415 // it searched along the way, to help prepare a useful error message should path turn 416 // out not to exist. 417 func vendoredImportPath(parent *Package, path string) (found string, searched []string) { 418 if parent == nil || parent.Root == "" || !go15VendorExperiment { 419 return path, nil 420 } 421 dir := filepath.Clean(parent.Dir) 422 root := filepath.Join(parent.Root, "src") 423 if !hasFilePathPrefix(dir, root) || len(dir) <= len(root) || dir[len(root)] != filepath.Separator { 424 fatalf("invalid vendoredImportPath: dir=%q root=%q separator=%q", dir, root, string(filepath.Separator)) 425 } 426 vpath := "vendor/" + path 427 for i := len(dir); i >= len(root); i-- { 428 if i < len(dir) && dir[i] != filepath.Separator { 429 continue 430 } 431 // Note: checking for the vendor directory before checking 432 // for the vendor/path directory helps us hit the 433 // isDir cache more often. It also helps us prepare a more useful 434 // list of places we looked, to report when an import is not found. 435 if !isDir(filepath.Join(dir[:i], "vendor")) { 436 continue 437 } 438 targ := filepath.Join(dir[:i], vpath) 439 if isDir(targ) { 440 // We started with parent's dir c:\gopath\src\foo\bar\baz\quux\xyzzy. 441 // We know the import path for parent's dir. 442 // We chopped off some number of path elements and 443 // added vendor\path to produce c:\gopath\src\foo\bar\baz\vendor\path. 444 // Now we want to know the import path for that directory. 445 // Construct it by chopping the same number of path elements 446 // (actually the same number of bytes) from parent's import path 447 // and then append /vendor/path. 448 chopped := len(dir) - i 449 if chopped == len(parent.ImportPath)+1 { 450 // We walked up from c:\gopath\src\foo\bar 451 // and found c:\gopath\src\vendor\path. 452 // We chopped \foo\bar (length 8) but the import path is "foo/bar" (length 7). 453 // Use "vendor/path" without any prefix. 454 return vpath, nil 455 } 456 return parent.ImportPath[:len(parent.ImportPath)-chopped] + "/" + vpath, nil 457 } 458 // Note the existence of a vendor directory in case path is not found anywhere. 459 searched = append(searched, targ) 460 } 461 return path, searched 462 } 463 464 // reusePackage reuses package p to satisfy the import at the top 465 // of the import stack stk. If this use causes an import loop, 466 // reusePackage updates p's error information to record the loop. 467 func reusePackage(p *Package, stk *importStack) *Package { 468 // We use p.imports==nil to detect a package that 469 // is in the midst of its own loadPackage call 470 // (all the recursion below happens before p.imports gets set). 471 if p.imports == nil { 472 if p.Error == nil { 473 p.Error = &PackageError{ 474 ImportStack: stk.copy(), 475 Err: "import cycle not allowed", 476 isImportCycle: true, 477 } 478 } 479 p.Incomplete = true 480 } 481 // Don't rewrite the import stack in the error if we have an import cycle. 482 // If we do, we'll lose the path that describes the cycle. 483 if p.Error != nil && !p.Error.isImportCycle && stk.shorterThan(p.Error.ImportStack) { 484 p.Error.ImportStack = stk.copy() 485 } 486 return p 487 } 488 489 // disallowInternal checks that srcDir is allowed to import p. 490 // If the import is allowed, disallowInternal returns the original package p. 491 // If not, it returns a new package containing just an appropriate error. 492 func disallowInternal(srcDir string, p *Package, stk *importStack) *Package { 493 // golang.org/s/go14internal: 494 // An import of a path containing the element “internal” 495 // is disallowed if the importing code is outside the tree 496 // rooted at the parent of the “internal” directory. 497 498 // There was an error loading the package; stop here. 499 if p.Error != nil { 500 return p 501 } 502 503 // The stack includes p.ImportPath. 504 // If that's the only thing on the stack, we started 505 // with a name given on the command line, not an 506 // import. Anything listed on the command line is fine. 507 if len(*stk) == 1 { 508 return p 509 } 510 511 // Check for "internal" element: four cases depending on begin of string and/or end of string. 512 i, ok := findInternal(p.ImportPath) 513 if !ok { 514 return p 515 } 516 517 // Internal is present. 518 // Map import path back to directory corresponding to parent of internal. 519 if i > 0 { 520 i-- // rewind over slash in ".../internal" 521 } 522 parent := p.Dir[:i+len(p.Dir)-len(p.ImportPath)] 523 if hasPathPrefix(filepath.ToSlash(srcDir), filepath.ToSlash(parent)) { 524 return p 525 } 526 527 // Internal is present, and srcDir is outside parent's tree. Not allowed. 528 perr := *p 529 perr.Error = &PackageError{ 530 ImportStack: stk.copy(), 531 Err: "use of internal package not allowed", 532 } 533 perr.Incomplete = true 534 return &perr 535 } 536 537 // findInternal looks for the final "internal" path element in the given import path. 538 // If there isn't one, findInternal returns ok=false. 539 // Otherwise, findInternal returns ok=true and the index of the "internal". 540 func findInternal(path string) (index int, ok bool) { 541 // Four cases, depending on internal at start/end of string or not. 542 // The order matters: we must return the index of the final element, 543 // because the final one produces the most restrictive requirement 544 // on the importer. 545 switch { 546 case strings.HasSuffix(path, "/internal"): 547 return len(path) - len("internal"), true 548 case strings.Contains(path, "/internal/"): 549 return strings.LastIndex(path, "/internal/") + 1, true 550 case path == "internal", strings.HasPrefix(path, "internal/"): 551 return 0, true 552 } 553 return 0, false 554 } 555 556 // disallowVendor checks that srcDir is allowed to import p as path. 557 // If the import is allowed, disallowVendor returns the original package p. 558 // If not, it returns a new package containing just an appropriate error. 559 func disallowVendor(srcDir, path string, p *Package, stk *importStack) *Package { 560 if !go15VendorExperiment { 561 return p 562 } 563 564 // The stack includes p.ImportPath. 565 // If that's the only thing on the stack, we started 566 // with a name given on the command line, not an 567 // import. Anything listed on the command line is fine. 568 if len(*stk) == 1 { 569 return p 570 } 571 572 if perr := disallowVendorVisibility(srcDir, p, stk); perr != p { 573 return perr 574 } 575 576 // Paths like x/vendor/y must be imported as y, never as x/vendor/y. 577 if i, ok := findVendor(path); ok { 578 perr := *p 579 perr.Error = &PackageError{ 580 ImportStack: stk.copy(), 581 Err: "must be imported as " + path[i+len("vendor/"):], 582 } 583 perr.Incomplete = true 584 return &perr 585 } 586 587 return p 588 } 589 590 // disallowVendorVisibility checks that srcDir is allowed to import p. 591 // The rules are the same as for /internal/ except that a path ending in /vendor 592 // is not subject to the rules, only subdirectories of vendor. 593 // This allows people to have packages and commands named vendor, 594 // for maximal compatibility with existing source trees. 595 func disallowVendorVisibility(srcDir string, p *Package, stk *importStack) *Package { 596 // The stack includes p.ImportPath. 597 // If that's the only thing on the stack, we started 598 // with a name given on the command line, not an 599 // import. Anything listed on the command line is fine. 600 if len(*stk) == 1 { 601 return p 602 } 603 604 // Check for "vendor" element. 605 i, ok := findVendor(p.ImportPath) 606 if !ok { 607 return p 608 } 609 610 // Vendor is present. 611 // Map import path back to directory corresponding to parent of vendor. 612 if i > 0 { 613 i-- // rewind over slash in ".../vendor" 614 } 615 truncateTo := i + len(p.Dir) - len(p.ImportPath) 616 if truncateTo < 0 || len(p.Dir) < truncateTo { 617 return p 618 } 619 parent := p.Dir[:truncateTo] 620 if hasPathPrefix(filepath.ToSlash(srcDir), filepath.ToSlash(parent)) { 621 return p 622 } 623 624 // Vendor is present, and srcDir is outside parent's tree. Not allowed. 625 perr := *p 626 perr.Error = &PackageError{ 627 ImportStack: stk.copy(), 628 Err: "use of vendored package not allowed", 629 } 630 perr.Incomplete = true 631 return &perr 632 } 633 634 // findVendor looks for the last non-terminating "vendor" path element in the given import path. 635 // If there isn't one, findVendor returns ok=false. 636 // Otherwise, findInternal returns ok=true and the index of the "vendor". 637 // 638 // Note that terminating "vendor" elements don't count: "x/vendor" is its own package, 639 // not the vendored copy of an import "" (the empty import path). 640 // This will allow people to have packages or commands named vendor. 641 // This may help reduce breakage, or it may just be confusing. We'll see. 642 func findVendor(path string) (index int, ok bool) { 643 // Two cases, depending on internal at start of string or not. 644 // The order matters: we must return the index of the final element, 645 // because the final one is where the effective import path starts. 646 switch { 647 case strings.Contains(path, "/vendor/"): 648 return strings.LastIndex(path, "/vendor/") + 1, true 649 case strings.HasPrefix(path, "vendor/"): 650 return 0, true 651 } 652 return 0, false 653 } 654 655 type targetDir int 656 657 const ( 658 toRoot targetDir = iota // to bin dir inside package root (default) 659 toTool // GOROOT/pkg/tool 660 toBin // GOROOT/bin 661 stalePath // the old import path; fail to build 662 ) 663 664 // goTools is a map of Go program import path to install target directory. 665 var goTools = map[string]targetDir{ 666 "cmd/addr2line": toTool, 667 "cmd/api": toTool, 668 "cmd/asm": toTool, 669 "cmd/compile": toTool, 670 "cmd/cgo": toTool, 671 "cmd/cover": toTool, 672 "cmd/dist": toTool, 673 "cmd/doc": toTool, 674 "cmd/fix": toTool, 675 "cmd/link": toTool, 676 "cmd/newlink": toTool, 677 "cmd/nm": toTool, 678 "cmd/objdump": toTool, 679 "cmd/pack": toTool, 680 "cmd/pprof": toTool, 681 "cmd/trace": toTool, 682 "cmd/vet": toTool, 683 "cmd/yacc": toTool, 684 "golang.org/x/tools/cmd/godoc": toBin, 685 "code.google.com/p/go.tools/cmd/cover": stalePath, 686 "code.google.com/p/go.tools/cmd/godoc": stalePath, 687 "code.google.com/p/go.tools/cmd/vet": stalePath, 688 } 689 690 // expandScanner expands a scanner.List error into all the errors in the list. 691 // The default Error method only shows the first error. 692 func expandScanner(err error) error { 693 // Look for parser errors. 694 if err, ok := err.(scanner.ErrorList); ok { 695 // Prepare error with \n before each message. 696 // When printed in something like context: %v 697 // this will put the leading file positions each on 698 // its own line. It will also show all the errors 699 // instead of just the first, as err.Error does. 700 var buf bytes.Buffer 701 for _, e := range err { 702 e.Pos.Filename = shortPath(e.Pos.Filename) 703 buf.WriteString("\n") 704 buf.WriteString(e.Error()) 705 } 706 return errors.New(buf.String()) 707 } 708 return err 709 } 710 711 var raceExclude = map[string]bool{ 712 "runtime/race": true, 713 "runtime/cgo": true, 714 "cmd/cgo": true, 715 "syscall": true, 716 "errors": true, 717 } 718 719 var cgoExclude = map[string]bool{ 720 "runtime/cgo": true, 721 } 722 723 var cgoSyscallExclude = map[string]bool{ 724 "runtime/cgo": true, 725 "runtime/race": true, 726 } 727 728 // load populates p using information from bp, err, which should 729 // be the result of calling build.Context.Import. 730 func (p *Package) load(stk *importStack, bp *build.Package, err error) *Package { 731 p.copyBuild(bp) 732 733 // The localPrefix is the path we interpret ./ imports relative to. 734 // Synthesized main packages sometimes override this. 735 p.localPrefix = dirToImportPath(p.Dir) 736 737 if err != nil { 738 p.Incomplete = true 739 err = expandScanner(err) 740 p.Error = &PackageError{ 741 ImportStack: stk.copy(), 742 Err: err.Error(), 743 } 744 return p 745 } 746 747 useBindir := p.Name == "main" 748 if !p.Standard { 749 switch buildBuildmode { 750 case "c-archive", "c-shared": 751 useBindir = false 752 } 753 } 754 755 if useBindir { 756 // Report an error when the old code.google.com/p/go.tools paths are used. 757 if goTools[p.ImportPath] == stalePath { 758 newPath := strings.Replace(p.ImportPath, "code.google.com/p/go.", "golang.org/x/", 1) 759 e := fmt.Sprintf("the %v command has moved; use %v instead.", p.ImportPath, newPath) 760 p.Error = &PackageError{Err: e} 761 return p 762 } 763 _, elem := filepath.Split(p.Dir) 764 full := buildContext.GOOS + "_" + buildContext.GOARCH + "/" + elem 765 if buildContext.GOOS != toolGOOS || buildContext.GOARCH != toolGOARCH { 766 // Install cross-compiled binaries to subdirectories of bin. 767 elem = full 768 } 769 if p.build.BinDir != gobin && goTools[p.ImportPath] == toBin { 770 // Override BinDir. 771 // This is from a subrepo but installs to $GOROOT/bin 772 // by default anyway (like godoc). 773 p.target = filepath.Join(gorootBin, elem) 774 } else if p.build.BinDir != "" { 775 // Install to GOBIN or bin of GOPATH entry. 776 p.target = filepath.Join(p.build.BinDir, elem) 777 if !p.Goroot && strings.Contains(elem, "/") && gobin != "" { 778 // Do not create $GOBIN/goos_goarch/elem. 779 p.target = "" 780 p.gobinSubdir = true 781 } 782 } 783 if goTools[p.ImportPath] == toTool { 784 // This is for 'go tool'. 785 // Override all the usual logic and force it into the tool directory. 786 p.target = filepath.Join(gorootPkg, "tool", full) 787 } 788 if p.target != "" && buildContext.GOOS == "windows" { 789 p.target += ".exe" 790 } 791 } else if p.local { 792 // Local import turned into absolute path. 793 // No permanent install target. 794 p.target = "" 795 } else { 796 p.target = p.build.PkgObj 797 if buildLinkshared { 798 shlibnamefile := p.target[:len(p.target)-2] + ".shlibname" 799 shlib, err := ioutil.ReadFile(shlibnamefile) 800 if err == nil { 801 libname := strings.TrimSpace(string(shlib)) 802 if buildContext.Compiler == "gccgo" { 803 p.Shlib = filepath.Join(p.build.PkgTargetRoot, "shlibs", libname) 804 } else { 805 p.Shlib = filepath.Join(p.build.PkgTargetRoot, libname) 806 807 } 808 } else if !os.IsNotExist(err) { 809 fatalf("unexpected error reading %s: %v", shlibnamefile, err) 810 } 811 } 812 } 813 814 importPaths := p.Imports 815 // Packages that use cgo import runtime/cgo implicitly. 816 // Packages that use cgo also import syscall implicitly, 817 // to wrap errno. 818 // Exclude certain packages to avoid circular dependencies. 819 if len(p.CgoFiles) > 0 && (!p.Standard || !cgoExclude[p.ImportPath]) { 820 importPaths = append(importPaths, "runtime/cgo") 821 } 822 if len(p.CgoFiles) > 0 && (!p.Standard || !cgoSyscallExclude[p.ImportPath]) { 823 importPaths = append(importPaths, "syscall") 824 } 825 826 // Currently build mode c-shared, or -linkshared, forces 827 // external linking mode, and external linking mode forces an 828 // import of runtime/cgo. 829 if p.Name == "main" && !p.Goroot && (buildBuildmode == "c-shared" || buildLinkshared) { 830 importPaths = append(importPaths, "runtime/cgo") 831 } 832 833 // Everything depends on runtime, except runtime and unsafe. 834 if !p.Standard || (p.ImportPath != "runtime" && p.ImportPath != "unsafe") { 835 importPaths = append(importPaths, "runtime") 836 // When race detection enabled everything depends on runtime/race. 837 // Exclude certain packages to avoid circular dependencies. 838 if buildRace && (!p.Standard || !raceExclude[p.ImportPath]) { 839 importPaths = append(importPaths, "runtime/race") 840 } 841 // On ARM with GOARM=5, everything depends on math for the link. 842 if p.Name == "main" && goarch == "arm" { 843 importPaths = append(importPaths, "math") 844 } 845 } 846 847 // Build list of full paths to all Go files in the package, 848 // for use by commands like go fmt. 849 p.gofiles = stringList(p.GoFiles, p.CgoFiles, p.TestGoFiles, p.XTestGoFiles) 850 for i := range p.gofiles { 851 p.gofiles[i] = filepath.Join(p.Dir, p.gofiles[i]) 852 } 853 sort.Strings(p.gofiles) 854 855 p.sfiles = stringList(p.SFiles) 856 for i := range p.sfiles { 857 p.sfiles[i] = filepath.Join(p.Dir, p.sfiles[i]) 858 } 859 sort.Strings(p.sfiles) 860 861 p.allgofiles = stringList(p.IgnoredGoFiles) 862 for i := range p.allgofiles { 863 p.allgofiles[i] = filepath.Join(p.Dir, p.allgofiles[i]) 864 } 865 p.allgofiles = append(p.allgofiles, p.gofiles...) 866 sort.Strings(p.allgofiles) 867 868 // Check for case-insensitive collision of input files. 869 // To avoid problems on case-insensitive files, we reject any package 870 // where two different input files have equal names under a case-insensitive 871 // comparison. 872 f1, f2 := foldDup(stringList( 873 p.GoFiles, 874 p.CgoFiles, 875 p.IgnoredGoFiles, 876 p.CFiles, 877 p.CXXFiles, 878 p.MFiles, 879 p.HFiles, 880 p.SFiles, 881 p.SysoFiles, 882 p.SwigFiles, 883 p.SwigCXXFiles, 884 p.TestGoFiles, 885 p.XTestGoFiles, 886 )) 887 if f1 != "" { 888 p.Error = &PackageError{ 889 ImportStack: stk.copy(), 890 Err: fmt.Sprintf("case-insensitive file name collision: %q and %q", f1, f2), 891 } 892 return p 893 } 894 895 // Build list of imported packages and full dependency list. 896 imports := make([]*Package, 0, len(p.Imports)) 897 deps := make(map[string]*Package) 898 for i, path := range importPaths { 899 if path == "C" { 900 continue 901 } 902 p1 := loadImport(path, p.Dir, p, stk, p.build.ImportPos[path], useVendor) 903 if p1.Name == "main" { 904 p.Error = &PackageError{ 905 ImportStack: stk.copy(), 906 Err: fmt.Sprintf("import %q is a program, not an importable package", path), 907 } 908 pos := p.build.ImportPos[path] 909 if len(pos) > 0 { 910 p.Error.Pos = pos[0].String() 911 } 912 } 913 if p1.local { 914 if !p.local && p.Error == nil { 915 p.Error = &PackageError{ 916 ImportStack: stk.copy(), 917 Err: fmt.Sprintf("local import %q in non-local package", path), 918 } 919 pos := p.build.ImportPos[path] 920 if len(pos) > 0 { 921 p.Error.Pos = pos[0].String() 922 } 923 } 924 } 925 path = p1.ImportPath 926 importPaths[i] = path 927 if i < len(p.Imports) { 928 p.Imports[i] = path 929 } 930 deps[path] = p1 931 imports = append(imports, p1) 932 for _, dep := range p1.deps { 933 // The same import path could produce an error or not, 934 // depending on what tries to import it. 935 // Prefer to record entries with errors, so we can report them. 936 if deps[dep.ImportPath] == nil || dep.Error != nil { 937 deps[dep.ImportPath] = dep 938 } 939 } 940 if p1.Incomplete { 941 p.Incomplete = true 942 } 943 } 944 p.imports = imports 945 946 p.Deps = make([]string, 0, len(deps)) 947 for dep := range deps { 948 p.Deps = append(p.Deps, dep) 949 } 950 sort.Strings(p.Deps) 951 for _, dep := range p.Deps { 952 p1 := deps[dep] 953 if p1 == nil { 954 panic("impossible: missing entry in package cache for " + dep + " imported by " + p.ImportPath) 955 } 956 p.deps = append(p.deps, p1) 957 if p1.Error != nil { 958 p.DepsErrors = append(p.DepsErrors, p1.Error) 959 } 960 } 961 962 // unsafe is a fake package. 963 if p.Standard && (p.ImportPath == "unsafe" || buildContext.Compiler == "gccgo") { 964 p.target = "" 965 } 966 p.Target = p.target 967 968 // The gc toolchain only permits C source files with cgo. 969 if len(p.CFiles) > 0 && !p.usesCgo() && !p.usesSwig() && buildContext.Compiler == "gc" { 970 p.Error = &PackageError{ 971 ImportStack: stk.copy(), 972 Err: fmt.Sprintf("C source files not allowed when not using cgo or SWIG: %s", strings.Join(p.CFiles, " ")), 973 } 974 return p 975 } 976 977 // In the absence of errors lower in the dependency tree, 978 // check for case-insensitive collisions of import paths. 979 if len(p.DepsErrors) == 0 { 980 dep1, dep2 := foldDup(p.Deps) 981 if dep1 != "" { 982 p.Error = &PackageError{ 983 ImportStack: stk.copy(), 984 Err: fmt.Sprintf("case-insensitive import collision: %q and %q", dep1, dep2), 985 } 986 return p 987 } 988 } 989 990 computeBuildID(p) 991 return p 992 } 993 994 // usesSwig reports whether the package needs to run SWIG. 995 func (p *Package) usesSwig() bool { 996 return len(p.SwigFiles) > 0 || len(p.SwigCXXFiles) > 0 997 } 998 999 // usesCgo reports whether the package needs to run cgo 1000 func (p *Package) usesCgo() bool { 1001 return len(p.CgoFiles) > 0 1002 } 1003 1004 // packageList returns the list of packages in the dag rooted at roots 1005 // as visited in a depth-first post-order traversal. 1006 func packageList(roots []*Package) []*Package { 1007 seen := map[*Package]bool{} 1008 all := []*Package{} 1009 var walk func(*Package) 1010 walk = func(p *Package) { 1011 if seen[p] { 1012 return 1013 } 1014 seen[p] = true 1015 for _, p1 := range p.imports { 1016 walk(p1) 1017 } 1018 all = append(all, p) 1019 } 1020 for _, root := range roots { 1021 walk(root) 1022 } 1023 return all 1024 } 1025 1026 // computeStale computes the Stale flag in the package dag that starts 1027 // at the named pkgs (command-line arguments). 1028 func computeStale(pkgs ...*Package) { 1029 for _, p := range packageList(pkgs) { 1030 p.Stale = isStale(p) 1031 } 1032 } 1033 1034 // The runtime version string takes one of two forms: 1035 // "go1.X[.Y]" for Go releases, and "devel +hash" at tip. 1036 // Determine whether we are in a released copy by 1037 // inspecting the version. 1038 var isGoRelease = strings.HasPrefix(runtime.Version(), "go1") 1039 1040 // isStale and computeBuildID 1041 // 1042 // Theory of Operation 1043 // 1044 // There is an installed copy of the package (or binary). 1045 // Can we reuse the installed copy, or do we need to build a new one? 1046 // 1047 // We can use the installed copy if it matches what we'd get 1048 // by building a new one. The hard part is predicting that without 1049 // actually running a build. 1050 // 1051 // To start, we must know the set of inputs to the build process that can 1052 // affect the generated output. At a minimum, that includes the source 1053 // files for the package and also any compiled packages imported by those 1054 // source files. The *Package has these, and we use them. One might also 1055 // argue for including in the input set: the build tags, whether the race 1056 // detector is in use, the target operating system and architecture, the 1057 // compiler and linker binaries being used, the additional flags being 1058 // passed to those, the cgo binary being used, the additional flags cgo 1059 // passes to the host C compiler, the host C compiler being used, the set 1060 // of host C include files and installed C libraries, and so on. 1061 // We include some but not all of this information. 1062 // 1063 // Once we have decided on a set of inputs, we must next decide how to 1064 // tell whether the content of that set has changed since the last build 1065 // of p. If there have been no changes, then we assume a new build would 1066 // produce the same result and reuse the installed package or binary. 1067 // But if there have been changes, then we assume a new build might not 1068 // produce the same result, so we rebuild. 1069 // 1070 // There are two common ways to decide whether the content of the set has 1071 // changed: modification times and content hashes. We use a mixture of both. 1072 // 1073 // The use of modification times (mtimes) was pioneered by make: 1074 // assuming that a file's mtime is an accurate record of when that file was last written, 1075 // and assuming that the modification time of an installed package or 1076 // binary is the time that it was built, if the mtimes of the inputs 1077 // predate the mtime of the installed object, then the build of that 1078 // object saw those versions of the files, and therefore a rebuild using 1079 // those same versions would produce the same object. In contrast, if any 1080 // mtime of an input is newer than the mtime of the installed object, a 1081 // change has occurred since the build, and the build should be redone. 1082 // 1083 // Modification times are attractive because the logic is easy to 1084 // understand and the file system maintains the mtimes automatically 1085 // (less work for us). Unfortunately, there are a variety of ways in 1086 // which the mtime approach fails to detect a change and reuses a stale 1087 // object file incorrectly. (Making the opposite mistake, rebuilding 1088 // unnecessarily, is only a performance problem and not a correctness 1089 // problem, so we ignore that one.) 1090 // 1091 // As a warmup, one problem is that to be perfectly precise, we need to 1092 // compare the input mtimes against the time at the beginning of the 1093 // build, but the object file time is the time at the end of the build. 1094 // If an input file changes after being read but before the object is 1095 // written, the next build will see an object newer than the input and 1096 // will incorrectly decide that the object is up to date. We make no 1097 // attempt to detect or solve this problem. 1098 // 1099 // Another problem is that due to file system imprecision, an input and 1100 // output that are actually ordered in time have the same mtime. 1101 // This typically happens on file systems with 1-second (or, worse, 1102 // 2-second) mtime granularity and with automated scripts that write an 1103 // input and then immediately run a build, or vice versa. If an input and 1104 // an output have the same mtime, the conservative behavior is to treat 1105 // the output as out-of-date and rebuild. This can cause one or more 1106 // spurious rebuilds, but only for 1 second, until the object finally has 1107 // an mtime later than the input. 1108 // 1109 // Another problem is that binary distributions often set the mtime on 1110 // all files to the same time. If the distribution includes both inputs 1111 // and cached build outputs, the conservative solution to the previous 1112 // problem will cause unnecessary rebuilds. Worse, in such a binary 1113 // distribution, those rebuilds might not even have permission to update 1114 // the cached build output. To avoid these write errors, if an input and 1115 // output have the same mtime, we assume the output is up-to-date. 1116 // This is the opposite of what the previous problem would have us do, 1117 // but binary distributions are more common than instances of the 1118 // previous problem. 1119 // 1120 // A variant of the last problem is that some binary distributions do not 1121 // set the mtime on all files to the same time. Instead they let the file 1122 // system record mtimes as the distribution is unpacked. If the outputs 1123 // are unpacked before the inputs, they'll be older and a build will try 1124 // to rebuild them. That rebuild might hit the same write errors as in 1125 // the last scenario. We don't make any attempt to solve this, and we 1126 // haven't had many reports of it. Perhaps the only time this happens is 1127 // when people manually unpack the distribution, and most of the time 1128 // that's done as the same user who will be using it, so an initial 1129 // rebuild on first use succeeds quietly. 1130 // 1131 // More generally, people and programs change mtimes on files. The last 1132 // few problems were specific examples of this, but it's a general problem. 1133 // For example, instead of a binary distribution, copying a home 1134 // directory from one directory or machine to another might copy files 1135 // but not preserve mtimes. If the inputs are new than the outputs on the 1136 // first machine but copied first, they end up older than the outputs on 1137 // the second machine. 1138 // 1139 // Because many other build systems have the same sensitivity to mtimes, 1140 // most programs manipulating source code take pains not to break the 1141 // mtime assumptions. For example, Git does not set the mtime of files 1142 // during a checkout operation, even when checking out an old version of 1143 // the code. This decision was made specifically to work well with 1144 // mtime-based build systems. 1145 // 1146 // The killer problem, though, for mtime-based build systems is that the 1147 // build only has access to the mtimes of the inputs that still exist. 1148 // If it is possible to remove an input without changing any other inputs, 1149 // a later build will think the object is up-to-date when it is not. 1150 // This happens for Go because a package is made up of all source 1151 // files in a directory. If a source file is removed, there is no newer 1152 // mtime available recording that fact. The mtime on the directory could 1153 // be used, but it also changes when unrelated files are added to or 1154 // removed from the directory, so including the directory mtime would 1155 // cause unnecessary rebuilds, possibly many. It would also exacerbate 1156 // the problems mentioned earlier, since even programs that are careful 1157 // to maintain mtimes on files rarely maintain mtimes on directories. 1158 // 1159 // A variant of the last problem is when the inputs change for other 1160 // reasons. For example, Go 1.4 and Go 1.5 both install $GOPATH/src/mypkg 1161 // into the same target, $GOPATH/pkg/$GOOS_$GOARCH/mypkg.a. 1162 // If Go 1.4 has built mypkg into mypkg.a, a build using Go 1.5 must 1163 // rebuild mypkg.a, but from mtimes alone mypkg.a looks up-to-date. 1164 // If Go 1.5 has just been installed, perhaps the compiler will have a 1165 // newer mtime; since the compiler is considered an input, that would 1166 // trigger a rebuild. But only once, and only the last Go 1.4 build of 1167 // mypkg.a happened before Go 1.5 was installed. If a user has the two 1168 // versions installed in different locations and flips back and forth, 1169 // mtimes alone cannot tell what to do. Changing the toolchain is 1170 // changing the set of inputs, without affecting any mtimes. 1171 // 1172 // To detect the set of inputs changing, we turn away from mtimes and to 1173 // an explicit data comparison. Specifically, we build a list of the 1174 // inputs to the build, compute its SHA1 hash, and record that as the 1175 // ``build ID'' in the generated object. At the next build, we can 1176 // recompute the buid ID and compare it to the one in the generated 1177 // object. If they differ, the list of inputs has changed, so the object 1178 // is out of date and must be rebuilt. 1179 // 1180 // Because this build ID is computed before the build begins, the 1181 // comparison does not have the race that mtime comparison does. 1182 // 1183 // Making the build sensitive to changes in other state is 1184 // straightforward: include the state in the build ID hash, and if it 1185 // changes, so does the build ID, triggering a rebuild. 1186 // 1187 // To detect changes in toolchain, we include the toolchain version in 1188 // the build ID hash for package runtime, and then we include the build 1189 // IDs of all imported packages in the build ID for p. 1190 // 1191 // It is natural to think about including build tags in the build ID, but 1192 // the naive approach of just dumping the tags into the hash would cause 1193 // spurious rebuilds. For example, 'go install' and 'go install -tags neverusedtag' 1194 // produce the same binaries (assuming neverusedtag is never used). 1195 // A more precise approach would be to include only tags that have an 1196 // effect on the build. But the effect of a tag on the build is to 1197 // include or exclude a file from the compilation, and that file list is 1198 // already in the build ID hash. So the build ID is already tag-sensitive 1199 // in a perfectly precise way. So we do NOT explicitly add build tags to 1200 // the build ID hash. 1201 // 1202 // We do not include as part of the build ID the operating system, 1203 // architecture, or whether the race detector is enabled, even though all 1204 // three have an effect on the output, because that information is used 1205 // to decide the install location. Binaries for linux and binaries for 1206 // darwin are written to different directory trees; including that 1207 // information in the build ID is unnecessary (although it would be 1208 // harmless). 1209 // 1210 // TODO(rsc): Investigate the cost of putting source file content into 1211 // the build ID hash as a replacement for the use of mtimes. Using the 1212 // file content would avoid all the mtime problems, but it does require 1213 // reading all the source files, something we avoid today (we read the 1214 // beginning to find the build tags and the imports, but we stop as soon 1215 // as we see the import block is over). If the package is stale, the compiler 1216 // is going to read the files anyway. But if the package is up-to-date, the 1217 // read is overhead. 1218 // 1219 // TODO(rsc): Investigate the complexity of making the build more 1220 // precise about when individual results are needed. To be fully precise, 1221 // there are two results of a compilation: the entire .a file used by the link 1222 // and the subpiece used by later compilations (__.PKGDEF only). 1223 // If a rebuild is needed but produces the previous __.PKGDEF, then 1224 // no more recompilation due to the rebuilt package is needed, only 1225 // relinking. To date, there is nothing in the Go command to express this. 1226 // 1227 // Special Cases 1228 // 1229 // When the go command makes the wrong build decision and does not 1230 // rebuild something it should, users fall back to adding the -a flag. 1231 // Any common use of the -a flag should be considered prima facie evidence 1232 // that isStale is returning an incorrect false result in some important case. 1233 // Bugs reported in the behavior of -a itself should prompt the question 1234 // ``Why is -a being used at all? What bug does that indicate?'' 1235 // 1236 // There is a long history of changes to isStale to try to make -a into a 1237 // suitable workaround for bugs in the mtime-based decisions. 1238 // It is worth recording that history to inform (and, as much as possible, deter) future changes. 1239 // 1240 // (1) Before the build IDs were introduced, building with alternate tags 1241 // would happily reuse installed objects built without those tags. 1242 // For example, "go build -tags netgo myprog.go" would use the installed 1243 // copy of package net, even if that copy had been built without netgo. 1244 // (The netgo tag controls whether package net uses cgo or pure Go for 1245 // functionality such as name resolution.) 1246 // Using the installed non-netgo package defeats the purpose. 1247 // 1248 // Users worked around this with "go build -tags netgo -a myprog.go". 1249 // 1250 // Build IDs have made that workaround unnecessary: 1251 // "go build -tags netgo myprog.go" 1252 // cannot use a non-netgo copy of package net. 1253 // 1254 // (2) Before the build IDs were introduced, building with different toolchains, 1255 // especially changing between toolchains, tried to reuse objects stored in 1256 // $GOPATH/pkg, resulting in link-time errors about object file mismatches. 1257 // 1258 // Users worked around this with "go install -a ./...". 1259 // 1260 // Build IDs have made that workaround unnecessary: 1261 // "go install ./..." will rebuild any objects it finds that were built against 1262 // a different toolchain. 1263 // 1264 // (3) The common use of "go install -a ./..." led to reports of problems 1265 // when the -a forced the rebuild of the standard library, which for some 1266 // users was not writable. Because we didn't understand that the real 1267 // problem was the bug -a was working around, we changed -a not to 1268 // apply to the standard library. 1269 // 1270 // (4) The common use of "go build -tags netgo -a myprog.go" broke 1271 // when we changed -a not to apply to the standard library, because 1272 // if go build doesn't rebuild package net, it uses the non-netgo version. 1273 // 1274 // Users worked around this with "go build -tags netgo -installsuffix barf myprog.go". 1275 // The -installsuffix here is making the go command look for packages 1276 // in pkg/$GOOS_$GOARCH_barf instead of pkg/$GOOS_$GOARCH. 1277 // Since the former presumably doesn't exist, go build decides to rebuild 1278 // everything, including the standard library. Since go build doesn't 1279 // install anything it builds, nothing is ever written to pkg/$GOOS_$GOARCH_barf, 1280 // so repeated invocations continue to work. 1281 // 1282 // If the use of -a wasn't a red flag, the use of -installsuffix to point to 1283 // a non-existent directory in a command that installs nothing should 1284 // have been. 1285 // 1286 // (5) Now that (1) and (2) no longer need -a, we have removed the kludge 1287 // introduced in (3): once again, -a means ``rebuild everything,'' not 1288 // ``rebuild everything except the standard library.'' Only Go 1.4 had 1289 // the restricted meaning. 1290 // 1291 // In addition to these cases trying to trigger rebuilds, there are 1292 // special cases trying NOT to trigger rebuilds. The main one is that for 1293 // a variety of reasons (see above), the install process for a Go release 1294 // cannot be relied upon to set the mtimes such that the go command will 1295 // think the standard library is up to date. So the mtime evidence is 1296 // ignored for the standard library if we find ourselves in a release 1297 // version of Go. Build ID-based staleness checks still apply to the 1298 // standard library, even in release versions. This makes 1299 // 'go build -tags netgo' work, among other things. 1300 1301 // isStale reports whether package p needs to be rebuilt. 1302 func isStale(p *Package) bool { 1303 if p.Standard && (p.ImportPath == "unsafe" || buildContext.Compiler == "gccgo") { 1304 // fake, builtin package 1305 return false 1306 } 1307 if p.Error != nil { 1308 return true 1309 } 1310 1311 // A package without Go sources means we only found 1312 // the installed .a file. Since we don't know how to rebuild 1313 // it, it can't be stale, even if -a is set. This enables binary-only 1314 // distributions of Go packages, although such binaries are 1315 // only useful with the specific version of the toolchain that 1316 // created them. 1317 if len(p.gofiles) == 0 && !p.usesSwig() { 1318 return false 1319 } 1320 1321 // If the -a flag is given, rebuild everything. 1322 if buildA { 1323 return true 1324 } 1325 1326 // If there's no install target or it's already marked stale, we have to rebuild. 1327 if p.target == "" || p.Stale { 1328 return true 1329 } 1330 1331 // Package is stale if completely unbuilt. 1332 fi, err := os.Stat(p.target) 1333 if err != nil { 1334 return true 1335 } 1336 1337 // Package is stale if the expected build ID differs from the 1338 // recorded build ID. This catches changes like a source file 1339 // being removed from a package directory. See issue 3895. 1340 // It also catches changes in build tags that affect the set of 1341 // files being compiled. See issue 9369. 1342 // It also catches changes in toolchain, like when flipping between 1343 // two versions of Go compiling a single GOPATH. 1344 // See issue 8290 and issue 10702. 1345 targetBuildID, err := readBuildID(p) 1346 if err == nil && targetBuildID != p.buildID { 1347 return true 1348 } 1349 1350 // Package is stale if a dependency is. 1351 for _, p1 := range p.deps { 1352 if p1.Stale { 1353 return true 1354 } 1355 } 1356 1357 // The checks above are content-based staleness. 1358 // We assume they are always accurate. 1359 // 1360 // The checks below are mtime-based staleness. 1361 // We hope they are accurate, but we know that they fail in the case of 1362 // prebuilt Go installations that don't preserve the build mtimes 1363 // (for example, if the pkg/ mtimes are before the src/ mtimes). 1364 // See the large comment above isStale for details. 1365 1366 // If we are running a release copy of Go and didn't find a content-based 1367 // reason to rebuild the standard packages, do not rebuild them. 1368 // They may not be writable anyway, but they are certainly not changing. 1369 // This makes 'go build' skip the standard packages when 1370 // using an official release, even when the mtimes have been changed. 1371 // See issue 3036, issue 3149, issue 4106, issue 8290. 1372 // (If a change to a release tree must be made by hand, the way to force the 1373 // install is to run make.bash, which will remove the old package archives 1374 // before rebuilding.) 1375 if p.Standard && isGoRelease { 1376 return false 1377 } 1378 1379 // Time-based staleness. 1380 1381 built := fi.ModTime() 1382 1383 olderThan := func(file string) bool { 1384 fi, err := os.Stat(file) 1385 return err != nil || fi.ModTime().After(built) 1386 } 1387 1388 // Package is stale if a dependency is, or if a dependency is newer. 1389 for _, p1 := range p.deps { 1390 if p1.target != "" && olderThan(p1.target) { 1391 return true 1392 } 1393 } 1394 1395 // As a courtesy to developers installing new versions of the compiler 1396 // frequently, define that packages are stale if they are 1397 // older than the compiler, and commands if they are older than 1398 // the linker. This heuristic will not work if the binaries are 1399 // back-dated, as some binary distributions may do, but it does handle 1400 // a very common case. 1401 // See issue 3036. 1402 // Exclude $GOROOT, under the assumption that people working on 1403 // the compiler may want to control when everything gets rebuilt, 1404 // and people updating the Go repository will run make.bash or all.bash 1405 // and get a full rebuild anyway. 1406 // Excluding $GOROOT used to also fix issue 4106, but that's now 1407 // taken care of above (at least when the installed Go is a released version). 1408 if p.Root != goroot { 1409 if olderThan(buildToolchain.compiler()) { 1410 return true 1411 } 1412 if p.build.IsCommand() && olderThan(buildToolchain.linker()) { 1413 return true 1414 } 1415 } 1416 1417 // Note: Until Go 1.5, we had an additional shortcut here. 1418 // We built a list of the workspace roots ($GOROOT, each $GOPATH) 1419 // containing targets directly named on the command line, 1420 // and if p were not in any of those, it would be treated as up-to-date 1421 // as long as it is built. The goal was to avoid rebuilding a system-installed 1422 // $GOROOT, unless something from $GOROOT were explicitly named 1423 // on the command line (like go install math). 1424 // That's now handled by the isGoRelease clause above. 1425 // The other effect of the shortcut was to isolate different entries in 1426 // $GOPATH from each other. This had the unfortunate effect that 1427 // if you had (say), GOPATH listing two entries, one for commands 1428 // and one for libraries, and you did a 'git pull' in the library one 1429 // and then tried 'go install commands/...', it would build the new libraries 1430 // during the first build (because they wouldn't have been installed at all) 1431 // but then subsequent builds would not rebuild the libraries, even if the 1432 // mtimes indicate they are stale, because the different GOPATH entries 1433 // were treated differently. This behavior was confusing when using 1434 // non-trivial GOPATHs, which were particularly common with some 1435 // code management conventions, like the original godep. 1436 // Since the $GOROOT case (the original motivation) is handled separately, 1437 // we no longer put a barrier between the different $GOPATH entries. 1438 // 1439 // One implication of this is that if there is a system directory for 1440 // non-standard Go packages that is included in $GOPATH, the mtimes 1441 // on those compiled packages must be no earlier than the mtimes 1442 // on the source files. Since most distributions use the same mtime 1443 // for all files in a tree, they will be unaffected. People using plain 1444 // tar x to extract system-installed packages will need to adjust mtimes, 1445 // but it's better to force them to get the mtimes right than to ignore 1446 // the mtimes and thereby do the wrong thing in common use cases. 1447 // 1448 // So there is no GOPATH vs GOPATH shortcut here anymore. 1449 // 1450 // If something needs to come back here, we could try writing a dummy 1451 // file with a random name to the $GOPATH/pkg directory (and removing it) 1452 // to test for write access, and then skip GOPATH roots we don't have write 1453 // access to. But hopefully we can just use the mtimes always. 1454 1455 srcs := stringList(p.GoFiles, p.CFiles, p.CXXFiles, p.MFiles, p.HFiles, p.SFiles, p.CgoFiles, p.SysoFiles, p.SwigFiles, p.SwigCXXFiles) 1456 for _, src := range srcs { 1457 if olderThan(filepath.Join(p.Dir, src)) { 1458 return true 1459 } 1460 } 1461 1462 return false 1463 } 1464 1465 // computeBuildID computes the build ID for p, leaving it in p.buildID. 1466 // Build ID is a hash of the information we want to detect changes in. 1467 // See the long comment in isStale for details. 1468 func computeBuildID(p *Package) { 1469 h := sha1.New() 1470 1471 // Include the list of files compiled as part of the package. 1472 // This lets us detect removed files. See issue 3895. 1473 inputFiles := stringList( 1474 p.GoFiles, 1475 p.CgoFiles, 1476 p.CFiles, 1477 p.CXXFiles, 1478 p.MFiles, 1479 p.HFiles, 1480 p.SFiles, 1481 p.SysoFiles, 1482 p.SwigFiles, 1483 p.SwigCXXFiles, 1484 ) 1485 for _, file := range inputFiles { 1486 fmt.Fprintf(h, "file %s\n", file) 1487 } 1488 1489 // Include the content of runtime/zversion.go in the hash 1490 // for package runtime. This will give package runtime a 1491 // different build ID in each Go release. 1492 if p.Standard && p.ImportPath == "runtime" { 1493 data, _ := ioutil.ReadFile(filepath.Join(p.Dir, "zversion.go")) 1494 fmt.Fprintf(h, "zversion %q\n", string(data)) 1495 } 1496 1497 // Include the build IDs of any dependencies in the hash. 1498 // This, combined with the runtime/zversion content, 1499 // will cause packages to have different build IDs when 1500 // compiled with different Go releases. 1501 // This helps the go command know to recompile when 1502 // people use the same GOPATH but switch between 1503 // different Go releases. See issue 10702. 1504 // This is also a better fix for issue 8290. 1505 for _, p1 := range p.deps { 1506 fmt.Fprintf(h, "dep %s %s\n", p1.ImportPath, p1.buildID) 1507 } 1508 1509 p.buildID = fmt.Sprintf("%x", h.Sum(nil)) 1510 } 1511 1512 var cwd, _ = os.Getwd() 1513 1514 var cmdCache = map[string]*Package{} 1515 1516 // loadPackage is like loadImport but is used for command-line arguments, 1517 // not for paths found in import statements. In addition to ordinary import paths, 1518 // loadPackage accepts pseudo-paths beginning with cmd/ to denote commands 1519 // in the Go command directory, as well as paths to those directories. 1520 func loadPackage(arg string, stk *importStack) *Package { 1521 if build.IsLocalImport(arg) { 1522 dir := arg 1523 if !filepath.IsAbs(dir) { 1524 if abs, err := filepath.Abs(dir); err == nil { 1525 // interpret relative to current directory 1526 dir = abs 1527 } 1528 } 1529 if sub, ok := hasSubdir(gorootSrc, dir); ok && strings.HasPrefix(sub, "cmd/") && !strings.Contains(sub[4:], "/") { 1530 arg = sub 1531 } 1532 } 1533 if strings.HasPrefix(arg, "cmd/") && !strings.Contains(arg[4:], "/") { 1534 if p := cmdCache[arg]; p != nil { 1535 return p 1536 } 1537 stk.push(arg) 1538 defer stk.pop() 1539 1540 bp, err := buildContext.ImportDir(filepath.Join(gorootSrc, arg), 0) 1541 bp.ImportPath = arg 1542 bp.Goroot = true 1543 bp.BinDir = gorootBin 1544 if gobin != "" { 1545 bp.BinDir = gobin 1546 } 1547 bp.Root = goroot 1548 bp.SrcRoot = gorootSrc 1549 p := new(Package) 1550 cmdCache[arg] = p 1551 p.load(stk, bp, err) 1552 if p.Error == nil && p.Name != "main" { 1553 p.Error = &PackageError{ 1554 ImportStack: stk.copy(), 1555 Err: fmt.Sprintf("expected package main but found package %s in %s", p.Name, p.Dir), 1556 } 1557 } 1558 return p 1559 } 1560 1561 // Wasn't a command; must be a package. 1562 // If it is a local import path but names a standard package, 1563 // we treat it as if the user specified the standard package. 1564 // This lets you run go test ./ioutil in package io and be 1565 // referring to io/ioutil rather than a hypothetical import of 1566 // "./ioutil". 1567 if build.IsLocalImport(arg) { 1568 bp, _ := buildContext.ImportDir(filepath.Join(cwd, arg), build.FindOnly) 1569 if bp.ImportPath != "" && bp.ImportPath != "." { 1570 arg = bp.ImportPath 1571 } 1572 } 1573 1574 return loadImport(arg, cwd, nil, stk, nil, 0) 1575 } 1576 1577 // packages returns the packages named by the 1578 // command line arguments 'args'. If a named package 1579 // cannot be loaded at all (for example, if the directory does not exist), 1580 // then packages prints an error and does not include that 1581 // package in the results. However, if errors occur trying 1582 // to load dependencies of a named package, the named 1583 // package is still returned, with p.Incomplete = true 1584 // and details in p.DepsErrors. 1585 func packages(args []string) []*Package { 1586 var pkgs []*Package 1587 for _, pkg := range packagesAndErrors(args) { 1588 if pkg.Error != nil { 1589 errorf("can't load package: %s", pkg.Error) 1590 continue 1591 } 1592 pkgs = append(pkgs, pkg) 1593 } 1594 return pkgs 1595 } 1596 1597 // packagesAndErrors is like 'packages' but returns a 1598 // *Package for every argument, even the ones that 1599 // cannot be loaded at all. 1600 // The packages that fail to load will have p.Error != nil. 1601 func packagesAndErrors(args []string) []*Package { 1602 if len(args) > 0 && strings.HasSuffix(args[0], ".go") { 1603 return []*Package{goFilesPackage(args)} 1604 } 1605 1606 args = importPaths(args) 1607 var pkgs []*Package 1608 var stk importStack 1609 var set = make(map[string]bool) 1610 1611 for _, arg := range args { 1612 if !set[arg] { 1613 pkgs = append(pkgs, loadPackage(arg, &stk)) 1614 set[arg] = true 1615 } 1616 } 1617 computeStale(pkgs...) 1618 1619 return pkgs 1620 } 1621 1622 // packagesForBuild is like 'packages' but fails if any of 1623 // the packages or their dependencies have errors 1624 // (cannot be built). 1625 func packagesForBuild(args []string) []*Package { 1626 pkgs := packagesAndErrors(args) 1627 printed := map[*PackageError]bool{} 1628 for _, pkg := range pkgs { 1629 if pkg.Error != nil { 1630 errorf("can't load package: %s", pkg.Error) 1631 } 1632 for _, err := range pkg.DepsErrors { 1633 // Since these are errors in dependencies, 1634 // the same error might show up multiple times, 1635 // once in each package that depends on it. 1636 // Only print each once. 1637 if !printed[err] { 1638 printed[err] = true 1639 errorf("%s", err) 1640 } 1641 } 1642 } 1643 exitIfErrors() 1644 1645 // Check for duplicate loads of the same package. 1646 // That should be impossible, but if it does happen then 1647 // we end up trying to build the same package twice, 1648 // usually in parallel overwriting the same files, 1649 // which doesn't work very well. 1650 seen := map[string]bool{} 1651 reported := map[string]bool{} 1652 for _, pkg := range packageList(pkgs) { 1653 if seen[pkg.ImportPath] && !reported[pkg.ImportPath] { 1654 reported[pkg.ImportPath] = true 1655 errorf("internal error: duplicate loads of %s", pkg.ImportPath) 1656 } 1657 seen[pkg.ImportPath] = true 1658 } 1659 exitIfErrors() 1660 1661 return pkgs 1662 } 1663 1664 // hasSubdir reports whether dir is a subdirectory of 1665 // (possibly multiple levels below) root. 1666 // If so, it sets rel to the path fragment that must be 1667 // appended to root to reach dir. 1668 func hasSubdir(root, dir string) (rel string, ok bool) { 1669 if p, err := filepath.EvalSymlinks(root); err == nil { 1670 root = p 1671 } 1672 if p, err := filepath.EvalSymlinks(dir); err == nil { 1673 dir = p 1674 } 1675 const sep = string(filepath.Separator) 1676 root = filepath.Clean(root) 1677 if !strings.HasSuffix(root, sep) { 1678 root += sep 1679 } 1680 dir = filepath.Clean(dir) 1681 if !strings.HasPrefix(dir, root) { 1682 return "", false 1683 } 1684 return filepath.ToSlash(dir[len(root):]), true 1685 } 1686 1687 var ( 1688 errBuildIDToolchain = fmt.Errorf("build ID only supported in gc toolchain") 1689 errBuildIDMalformed = fmt.Errorf("malformed object file") 1690 errBuildIDUnknown = fmt.Errorf("lost build ID") 1691 ) 1692 1693 var ( 1694 bangArch = []byte("!<arch>") 1695 pkgdef = []byte("__.PKGDEF") 1696 goobject = []byte("go object ") 1697 buildid = []byte("build id ") 1698 ) 1699 1700 // readBuildID reads the build ID from an archive or binary. 1701 // It only supports the gc toolchain. 1702 // Other toolchain maintainers should adjust this function. 1703 func readBuildID(p *Package) (id string, err error) { 1704 if buildToolchain != (gcToolchain{}) { 1705 return "", errBuildIDToolchain 1706 } 1707 1708 // For commands, read build ID directly from binary. 1709 if p.Name == "main" { 1710 return ReadBuildIDFromBinary(p.Target) 1711 } 1712 1713 // Otherwise, we expect to have an archive (.a) file, 1714 // and we can read the build ID from the Go export data. 1715 if !strings.HasSuffix(p.Target, ".a") { 1716 return "", &os.PathError{Op: "parse", Path: p.Target, Err: errBuildIDUnknown} 1717 } 1718 1719 // Read just enough of the target to fetch the build ID. 1720 // The archive is expected to look like: 1721 // 1722 // !<arch> 1723 // __.PKGDEF 0 0 0 644 7955 ` 1724 // go object darwin amd64 devel X:none 1725 // build id "b41e5c45250e25c9fd5e9f9a1de7857ea0d41224" 1726 // 1727 // The variable-sized strings are GOOS, GOARCH, and the experiment list (X:none). 1728 // Reading the first 1024 bytes should be plenty. 1729 f, err := os.Open(p.Target) 1730 if err != nil { 1731 return "", err 1732 } 1733 data := make([]byte, 1024) 1734 n, err := io.ReadFull(f, data) 1735 f.Close() 1736 1737 if err != nil && n == 0 { 1738 return "", err 1739 } 1740 1741 bad := func() (string, error) { 1742 return "", &os.PathError{Op: "parse", Path: p.Target, Err: errBuildIDMalformed} 1743 } 1744 1745 // Archive header. 1746 for i := 0; ; i++ { // returns during i==3 1747 j := bytes.IndexByte(data, '\n') 1748 if j < 0 { 1749 return bad() 1750 } 1751 line := data[:j] 1752 data = data[j+1:] 1753 switch i { 1754 case 0: 1755 if !bytes.Equal(line, bangArch) { 1756 return bad() 1757 } 1758 case 1: 1759 if !bytes.HasPrefix(line, pkgdef) { 1760 return bad() 1761 } 1762 case 2: 1763 if !bytes.HasPrefix(line, goobject) { 1764 return bad() 1765 } 1766 case 3: 1767 if !bytes.HasPrefix(line, buildid) { 1768 // Found the object header, just doesn't have a build id line. 1769 // Treat as successful, with empty build id. 1770 return "", nil 1771 } 1772 id, err := strconv.Unquote(string(line[len(buildid):])) 1773 if err != nil { 1774 return bad() 1775 } 1776 return id, nil 1777 } 1778 } 1779 } 1780 1781 var ( 1782 goBuildPrefix = []byte("\xff Go build ID: \"") 1783 goBuildEnd = []byte("\"\n \xff") 1784 1785 elfPrefix = []byte("\x7fELF") 1786 ) 1787 1788 // ReadBuildIDFromBinary reads the build ID from a binary. 1789 // 1790 // ELF binaries store the build ID in a proper PT_NOTE section. 1791 // 1792 // Other binary formats are not so flexible. For those, the linker 1793 // stores the build ID as non-instruction bytes at the very beginning 1794 // of the text segment, which should appear near the beginning 1795 // of the file. This is clumsy but fairly portable. Custom locations 1796 // can be added for other binary types as needed, like we did for ELF. 1797 func ReadBuildIDFromBinary(filename string) (id string, err error) { 1798 if filename == "" { 1799 return "", &os.PathError{Op: "parse", Path: filename, Err: errBuildIDUnknown} 1800 } 1801 1802 // Read the first 16 kB of the binary file. 1803 // That should be enough to find the build ID. 1804 // In ELF files, the build ID is in the leading headers, 1805 // which are typically less than 4 kB, not to mention 16 kB. 1806 // On other systems, we're trying to read enough that 1807 // we get the beginning of the text segment in the read. 1808 // The offset where the text segment begins in a hello 1809 // world compiled for each different object format today: 1810 // 1811 // Plan 9: 0x20 1812 // Windows: 0x600 1813 // Mach-O: 0x2000 1814 // 1815 f, err := os.Open(filename) 1816 if err != nil { 1817 return "", err 1818 } 1819 defer f.Close() 1820 1821 data := make([]byte, 16*1024) 1822 _, err = io.ReadFull(f, data) 1823 if err == io.ErrUnexpectedEOF { 1824 err = nil 1825 } 1826 if err != nil { 1827 return "", err 1828 } 1829 1830 if bytes.HasPrefix(data, elfPrefix) { 1831 return readELFGoBuildID(filename, f, data) 1832 } 1833 1834 i := bytes.Index(data, goBuildPrefix) 1835 if i < 0 { 1836 // Missing. Treat as successful but build ID empty. 1837 return "", nil 1838 } 1839 1840 j := bytes.Index(data[i+len(goBuildPrefix):], goBuildEnd) 1841 if j < 0 { 1842 return "", &os.PathError{Op: "parse", Path: filename, Err: errBuildIDMalformed} 1843 } 1844 1845 quoted := data[i+len(goBuildPrefix)-1 : i+len(goBuildPrefix)+j+1] 1846 id, err = strconv.Unquote(string(quoted)) 1847 if err != nil { 1848 return "", &os.PathError{Op: "parse", Path: filename, Err: errBuildIDMalformed} 1849 } 1850 1851 return id, nil 1852 }