github.com/dannin/go@v0.0.0-20161031215817-d35dfd405eaa/src/cmd/go/pkg.go (about) 1 // Copyright 2011 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package main 6 7 import ( 8 "bytes" 9 "crypto/sha1" 10 "errors" 11 "fmt" 12 "go/build" 13 "go/scanner" 14 "go/token" 15 "io" 16 "io/ioutil" 17 "os" 18 pathpkg "path" 19 "path/filepath" 20 "runtime" 21 "sort" 22 "strconv" 23 "strings" 24 "unicode" 25 ) 26 27 var ignoreImports bool // control whether we ignore imports in packages 28 29 // A Package describes a single package found in a directory. 30 type Package struct { 31 // Note: These fields are part of the go command's public API. 32 // See list.go. It is okay to add fields, but not to change or 33 // remove existing ones. Keep in sync with list.go 34 Dir string `json:",omitempty"` // directory containing package sources 35 ImportPath string `json:",omitempty"` // import path of package in dir 36 ImportComment string `json:",omitempty"` // path in import comment on package statement 37 Name string `json:",omitempty"` // package name 38 Doc string `json:",omitempty"` // package documentation string 39 Target string `json:",omitempty"` // install path 40 Shlib string `json:",omitempty"` // the shared library that contains this package (only set when -linkshared) 41 Goroot bool `json:",omitempty"` // is this package found in the Go root? 42 Standard bool `json:",omitempty"` // is this package part of the standard Go library? 43 Stale bool `json:",omitempty"` // would 'go install' do anything for this package? 44 StaleReason string `json:",omitempty"` // why is Stale true? 45 Root string `json:",omitempty"` // Go root or Go path dir containing this package 46 ConflictDir string `json:",omitempty"` // Dir is hidden by this other directory 47 BinaryOnly bool `json:",omitempty"` // package cannot be recompiled 48 49 // Source files 50 GoFiles []string `json:",omitempty"` // .go source files (excluding CgoFiles, TestGoFiles, XTestGoFiles) 51 CgoFiles []string `json:",omitempty"` // .go sources files that import "C" 52 IgnoredGoFiles []string `json:",omitempty"` // .go sources ignored due to build constraints 53 CFiles []string `json:",omitempty"` // .c source files 54 CXXFiles []string `json:",omitempty"` // .cc, .cpp and .cxx source files 55 MFiles []string `json:",omitempty"` // .m source files 56 HFiles []string `json:",omitempty"` // .h, .hh, .hpp and .hxx source files 57 FFiles []string `json:",omitempty"` // .f, .F, .for and .f90 Fortran source files 58 SFiles []string `json:",omitempty"` // .s source files 59 SwigFiles []string `json:",omitempty"` // .swig files 60 SwigCXXFiles []string `json:",omitempty"` // .swigcxx files 61 SysoFiles []string `json:",omitempty"` // .syso system object files added to package 62 63 // Cgo directives 64 CgoCFLAGS []string `json:",omitempty"` // cgo: flags for C compiler 65 CgoCPPFLAGS []string `json:",omitempty"` // cgo: flags for C preprocessor 66 CgoCXXFLAGS []string `json:",omitempty"` // cgo: flags for C++ compiler 67 CgoFFLAGS []string `json:",omitempty"` // cgo: flags for Fortran compiler 68 CgoLDFLAGS []string `json:",omitempty"` // cgo: flags for linker 69 CgoPkgConfig []string `json:",omitempty"` // cgo: pkg-config names 70 71 // Dependency information 72 Imports []string `json:",omitempty"` // import paths used by this package 73 Deps []string `json:",omitempty"` // all (recursively) imported dependencies 74 75 // Error information 76 Incomplete bool `json:",omitempty"` // was there an error loading this package or dependencies? 77 Error *PackageError `json:",omitempty"` // error loading this package (not dependencies) 78 DepsErrors []*PackageError `json:",omitempty"` // errors loading dependencies 79 80 // Test information 81 TestGoFiles []string `json:",omitempty"` // _test.go files in package 82 TestImports []string `json:",omitempty"` // imports from TestGoFiles 83 XTestGoFiles []string `json:",omitempty"` // _test.go files outside package 84 XTestImports []string `json:",omitempty"` // imports from XTestGoFiles 85 86 // Unexported fields are not part of the public API. 87 build *build.Package 88 pkgdir string // overrides build.PkgDir 89 imports []*Package 90 deps []*Package 91 gofiles []string // GoFiles+CgoFiles+TestGoFiles+XTestGoFiles files, absolute paths 92 sfiles []string 93 allgofiles []string // gofiles + IgnoredGoFiles, absolute paths 94 target string // installed file for this package (may be executable) 95 fake bool // synthesized package 96 external bool // synthesized external test package 97 forceLibrary bool // this package is a library (even if named "main") 98 cmdline bool // defined by files listed on command line 99 local bool // imported via local path (./ or ../) 100 localPrefix string // interpret ./ and ../ imports relative to this prefix 101 exeName string // desired name for temporary executable 102 coverMode string // preprocess Go source files with the coverage tool in this mode 103 coverVars map[string]*CoverVar // variables created by coverage analysis 104 omitDWARF bool // tell linker not to write DWARF information 105 buildID string // expected build ID for generated package 106 gobinSubdir bool // install target would be subdir of GOBIN 107 } 108 109 // vendored returns the vendor-resolved version of imports, 110 // which should be p.TestImports or p.XTestImports, NOT p.Imports. 111 // The imports in p.TestImports and p.XTestImports are not recursively 112 // loaded during the initial load of p, so they list the imports found in 113 // the source file, but most processing should be over the vendor-resolved 114 // import paths. We do this resolution lazily both to avoid file system work 115 // and because the eventual real load of the test imports (during 'go test') 116 // can produce better error messages if it starts with the original paths. 117 // The initial load of p loads all the non-test imports and rewrites 118 // the vendored paths, so nothing should ever call p.vendored(p.Imports). 119 func (p *Package) vendored(imports []string) []string { 120 if len(imports) > 0 && len(p.Imports) > 0 && &imports[0] == &p.Imports[0] { 121 panic("internal error: p.vendored(p.Imports) called") 122 } 123 seen := make(map[string]bool) 124 var all []string 125 for _, path := range imports { 126 path = vendoredImportPath(p, path) 127 if !seen[path] { 128 seen[path] = true 129 all = append(all, path) 130 } 131 } 132 sort.Strings(all) 133 return all 134 } 135 136 // CoverVar holds the name of the generated coverage variables targeting the named file. 137 type CoverVar struct { 138 File string // local file name 139 Var string // name of count struct 140 } 141 142 func (p *Package) copyBuild(pp *build.Package) { 143 p.build = pp 144 145 if pp.PkgTargetRoot != "" && buildPkgdir != "" { 146 old := pp.PkgTargetRoot 147 pp.PkgRoot = buildPkgdir 148 pp.PkgTargetRoot = buildPkgdir 149 pp.PkgObj = filepath.Join(buildPkgdir, strings.TrimPrefix(pp.PkgObj, old)) 150 } 151 152 p.Dir = pp.Dir 153 p.ImportPath = pp.ImportPath 154 p.ImportComment = pp.ImportComment 155 p.Name = pp.Name 156 p.Doc = pp.Doc 157 p.Root = pp.Root 158 p.ConflictDir = pp.ConflictDir 159 p.BinaryOnly = pp.BinaryOnly 160 161 // TODO? Target 162 p.Goroot = pp.Goroot 163 p.Standard = p.Goroot && p.ImportPath != "" && isStandardImportPath(p.ImportPath) 164 p.GoFiles = pp.GoFiles 165 p.CgoFiles = pp.CgoFiles 166 p.IgnoredGoFiles = pp.IgnoredGoFiles 167 p.CFiles = pp.CFiles 168 p.CXXFiles = pp.CXXFiles 169 p.MFiles = pp.MFiles 170 p.HFiles = pp.HFiles 171 p.FFiles = pp.FFiles 172 p.SFiles = pp.SFiles 173 p.SwigFiles = pp.SwigFiles 174 p.SwigCXXFiles = pp.SwigCXXFiles 175 p.SysoFiles = pp.SysoFiles 176 p.CgoCFLAGS = pp.CgoCFLAGS 177 p.CgoCPPFLAGS = pp.CgoCPPFLAGS 178 p.CgoCXXFLAGS = pp.CgoCXXFLAGS 179 p.CgoLDFLAGS = pp.CgoLDFLAGS 180 p.CgoPkgConfig = pp.CgoPkgConfig 181 p.Imports = pp.Imports 182 p.TestGoFiles = pp.TestGoFiles 183 p.TestImports = pp.TestImports 184 p.XTestGoFiles = pp.XTestGoFiles 185 p.XTestImports = pp.XTestImports 186 if ignoreImports { 187 p.Imports = nil 188 p.TestImports = nil 189 p.XTestImports = nil 190 } 191 } 192 193 // isStandardImportPath reports whether $GOROOT/src/path should be considered 194 // part of the standard distribution. For historical reasons we allow people to add 195 // their own code to $GOROOT instead of using $GOPATH, but we assume that 196 // code will start with a domain name (dot in the first element). 197 func isStandardImportPath(path string) bool { 198 i := strings.Index(path, "/") 199 if i < 0 { 200 i = len(path) 201 } 202 elem := path[:i] 203 return !strings.Contains(elem, ".") 204 } 205 206 // A PackageError describes an error loading information about a package. 207 type PackageError struct { 208 ImportStack []string // shortest path from package named on command line to this one 209 Pos string // position of error 210 Err string // the error itself 211 isImportCycle bool // the error is an import cycle 212 hard bool // whether the error is soft or hard; soft errors are ignored in some places 213 } 214 215 func (p *PackageError) Error() string { 216 // Import cycles deserve special treatment. 217 if p.isImportCycle { 218 return fmt.Sprintf("%s\npackage %s\n", p.Err, strings.Join(p.ImportStack, "\n\timports ")) 219 } 220 if p.Pos != "" { 221 // Omit import stack. The full path to the file where the error 222 // is the most important thing. 223 return p.Pos + ": " + p.Err 224 } 225 if len(p.ImportStack) == 0 { 226 return p.Err 227 } 228 return "package " + strings.Join(p.ImportStack, "\n\timports ") + ": " + p.Err 229 } 230 231 // An importStack is a stack of import paths. 232 type importStack []string 233 234 func (s *importStack) push(p string) { 235 *s = append(*s, p) 236 } 237 238 func (s *importStack) pop() { 239 *s = (*s)[0 : len(*s)-1] 240 } 241 242 func (s *importStack) copy() []string { 243 return append([]string{}, *s...) 244 } 245 246 // shorterThan reports whether sp is shorter than t. 247 // We use this to record the shortest import sequence 248 // that leads to a particular package. 249 func (sp *importStack) shorterThan(t []string) bool { 250 s := *sp 251 if len(s) != len(t) { 252 return len(s) < len(t) 253 } 254 // If they are the same length, settle ties using string ordering. 255 for i := range s { 256 if s[i] != t[i] { 257 return s[i] < t[i] 258 } 259 } 260 return false // they are equal 261 } 262 263 // packageCache is a lookup cache for loadPackage, 264 // so that if we look up a package multiple times 265 // we return the same pointer each time. 266 var packageCache = map[string]*Package{} 267 268 // reloadPackage is like loadPackage but makes sure 269 // not to use the package cache. 270 func reloadPackage(arg string, stk *importStack) *Package { 271 p := packageCache[arg] 272 if p != nil { 273 delete(packageCache, p.Dir) 274 delete(packageCache, p.ImportPath) 275 } 276 return loadPackage(arg, stk) 277 } 278 279 // dirToImportPath returns the pseudo-import path we use for a package 280 // outside the Go path. It begins with _/ and then contains the full path 281 // to the directory. If the package lives in c:\home\gopher\my\pkg then 282 // the pseudo-import path is _/c_/home/gopher/my/pkg. 283 // Using a pseudo-import path like this makes the ./ imports no longer 284 // a special case, so that all the code to deal with ordinary imports works 285 // automatically. 286 func dirToImportPath(dir string) string { 287 return pathpkg.Join("_", strings.Map(makeImportValid, filepath.ToSlash(dir))) 288 } 289 290 func makeImportValid(r rune) rune { 291 // Should match Go spec, compilers, and ../../go/parser/parser.go:/isValidImport. 292 const illegalChars = `!"#$%&'()*,:;<=>?[\]^{|}` + "`\uFFFD" 293 if !unicode.IsGraphic(r) || unicode.IsSpace(r) || strings.ContainsRune(illegalChars, r) { 294 return '_' 295 } 296 return r 297 } 298 299 // Mode flags for loadImport and download (in get.go). 300 const ( 301 // useVendor means that loadImport should do vendor expansion 302 // (provided the vendoring experiment is enabled). 303 // That is, useVendor means that the import path came from 304 // a source file and has not been vendor-expanded yet. 305 // Every import path should be loaded initially with useVendor, 306 // and then the expanded version (with the /vendor/ in it) gets 307 // recorded as the canonical import path. At that point, future loads 308 // of that package must not pass useVendor, because 309 // disallowVendor will reject direct use of paths containing /vendor/. 310 useVendor = 1 << iota 311 312 // getTestDeps is for download (part of "go get") and indicates 313 // that test dependencies should be fetched too. 314 getTestDeps 315 ) 316 317 // loadImport scans the directory named by path, which must be an import path, 318 // but possibly a local import path (an absolute file system path or one beginning 319 // with ./ or ../). A local relative path is interpreted relative to srcDir. 320 // It returns a *Package describing the package found in that directory. 321 func loadImport(path, srcDir string, parent *Package, stk *importStack, importPos []token.Position, mode int) *Package { 322 stk.push(path) 323 defer stk.pop() 324 325 // Determine canonical identifier for this package. 326 // For a local import the identifier is the pseudo-import path 327 // we create from the full directory to the package. 328 // Otherwise it is the usual import path. 329 // For vendored imports, it is the expanded form. 330 importPath := path 331 origPath := path 332 isLocal := build.IsLocalImport(path) 333 if isLocal { 334 importPath = dirToImportPath(filepath.Join(srcDir, path)) 335 } else if mode&useVendor != 0 { 336 // We do our own vendor resolution, because we want to 337 // find out the key to use in packageCache without the 338 // overhead of repeated calls to buildContext.Import. 339 // The code is also needed in a few other places anyway. 340 path = vendoredImportPath(parent, path) 341 importPath = path 342 } 343 344 p := packageCache[importPath] 345 if p != nil { 346 p = reusePackage(p, stk) 347 } else { 348 p = new(Package) 349 p.local = isLocal 350 p.ImportPath = importPath 351 packageCache[importPath] = p 352 353 // Load package. 354 // Import always returns bp != nil, even if an error occurs, 355 // in order to return partial information. 356 // 357 // TODO: After Go 1, decide when to pass build.AllowBinary here. 358 // See issue 3268 for mistakes to avoid. 359 buildMode := build.ImportComment 360 if mode&useVendor == 0 || path != origPath { 361 // Not vendoring, or we already found the vendored path. 362 buildMode |= build.IgnoreVendor 363 } 364 bp, err := buildContext.Import(path, srcDir, buildMode) 365 bp.ImportPath = importPath 366 if gobin != "" { 367 bp.BinDir = gobin 368 } 369 if err == nil && !isLocal && bp.ImportComment != "" && bp.ImportComment != path && 370 !strings.Contains(path, "/vendor/") && !strings.HasPrefix(path, "vendor/") { 371 err = fmt.Errorf("code in directory %s expects import %q", bp.Dir, bp.ImportComment) 372 } 373 p.load(stk, bp, err) 374 if p.Error != nil && p.Error.Pos == "" && len(importPos) > 0 { 375 pos := importPos[0] 376 pos.Filename = shortPath(pos.Filename) 377 p.Error.Pos = pos.String() 378 } 379 380 if origPath != cleanImport(origPath) { 381 p.Error = &PackageError{ 382 ImportStack: stk.copy(), 383 Err: fmt.Sprintf("non-canonical import path: %q should be %q", origPath, pathpkg.Clean(origPath)), 384 } 385 p.Incomplete = true 386 } 387 } 388 389 // Checked on every import because the rules depend on the code doing the importing. 390 if perr := disallowInternal(srcDir, p, stk); perr != p { 391 return perr 392 } 393 if mode&useVendor != 0 { 394 if perr := disallowVendor(srcDir, origPath, p, stk); perr != p { 395 return perr 396 } 397 } 398 399 if p.Name == "main" && parent != nil && parent.Dir != p.Dir { 400 perr := *p 401 perr.Error = &PackageError{ 402 ImportStack: stk.copy(), 403 Err: fmt.Sprintf("import %q is a program, not an importable package", path), 404 } 405 if len(importPos) > 0 { 406 pos := importPos[0] 407 pos.Filename = shortPath(pos.Filename) 408 perr.Error.Pos = pos.String() 409 } 410 return &perr 411 } 412 413 if p.local && parent != nil && !parent.local { 414 perr := *p 415 perr.Error = &PackageError{ 416 ImportStack: stk.copy(), 417 Err: fmt.Sprintf("local import %q in non-local package", path), 418 } 419 if len(importPos) > 0 { 420 pos := importPos[0] 421 pos.Filename = shortPath(pos.Filename) 422 perr.Error.Pos = pos.String() 423 } 424 return &perr 425 } 426 427 return p 428 } 429 430 func cleanImport(path string) string { 431 orig := path 432 path = pathpkg.Clean(path) 433 if strings.HasPrefix(orig, "./") && path != ".." && path != "." && !strings.HasPrefix(path, "../") { 434 path = "./" + path 435 } 436 return path 437 } 438 439 var isDirCache = map[string]bool{} 440 441 func isDir(path string) bool { 442 result, ok := isDirCache[path] 443 if ok { 444 return result 445 } 446 447 fi, err := os.Stat(path) 448 result = err == nil && fi.IsDir() 449 isDirCache[path] = result 450 return result 451 } 452 453 // vendoredImportPath returns the expansion of path when it appears in parent. 454 // If parent is x/y/z, then path might expand to x/y/z/vendor/path, x/y/vendor/path, 455 // x/vendor/path, vendor/path, or else stay path if none of those exist. 456 // vendoredImportPath returns the expanded path or, if no expansion is found, the original. 457 func vendoredImportPath(parent *Package, path string) (found string) { 458 if parent == nil || parent.Root == "" { 459 return path 460 } 461 462 dir := filepath.Clean(parent.Dir) 463 root := filepath.Join(parent.Root, "src") 464 if !hasFilePathPrefix(dir, root) || parent.ImportPath != "command-line-arguments" && filepath.Join(root, parent.ImportPath) != dir { 465 // Look for symlinks before reporting error. 466 dir = expandPath(dir) 467 root = expandPath(root) 468 } 469 470 if !hasFilePathPrefix(dir, root) || len(dir) <= len(root) || dir[len(root)] != filepath.Separator || parent.ImportPath != "command-line-arguments" && !parent.local && filepath.Join(root, parent.ImportPath) != dir { 471 fatalf("unexpected directory layout:\n"+ 472 " import path: %s\n"+ 473 " root: %s\n"+ 474 " dir: %s\n"+ 475 " expand root: %s\n"+ 476 " expand dir: %s\n"+ 477 " separator: %s", 478 parent.ImportPath, 479 filepath.Join(parent.Root, "src"), 480 filepath.Clean(parent.Dir), 481 root, 482 dir, 483 string(filepath.Separator)) 484 } 485 486 vpath := "vendor/" + path 487 for i := len(dir); i >= len(root); i-- { 488 if i < len(dir) && dir[i] != filepath.Separator { 489 continue 490 } 491 // Note: checking for the vendor directory before checking 492 // for the vendor/path directory helps us hit the 493 // isDir cache more often. It also helps us prepare a more useful 494 // list of places we looked, to report when an import is not found. 495 if !isDir(filepath.Join(dir[:i], "vendor")) { 496 continue 497 } 498 targ := filepath.Join(dir[:i], vpath) 499 if isDir(targ) && hasGoFiles(targ) { 500 importPath := parent.ImportPath 501 if importPath == "command-line-arguments" { 502 // If parent.ImportPath is 'command-line-arguments'. 503 // set to relative directory to root (also chopped root directory) 504 importPath = dir[len(root)+1:] 505 } 506 // We started with parent's dir c:\gopath\src\foo\bar\baz\quux\xyzzy. 507 // We know the import path for parent's dir. 508 // We chopped off some number of path elements and 509 // added vendor\path to produce c:\gopath\src\foo\bar\baz\vendor\path. 510 // Now we want to know the import path for that directory. 511 // Construct it by chopping the same number of path elements 512 // (actually the same number of bytes) from parent's import path 513 // and then append /vendor/path. 514 chopped := len(dir) - i 515 if chopped == len(importPath)+1 { 516 // We walked up from c:\gopath\src\foo\bar 517 // and found c:\gopath\src\vendor\path. 518 // We chopped \foo\bar (length 8) but the import path is "foo/bar" (length 7). 519 // Use "vendor/path" without any prefix. 520 return vpath 521 } 522 return importPath[:len(importPath)-chopped] + "/" + vpath 523 } 524 } 525 return path 526 } 527 528 // hasGoFiles reports whether dir contains any files with names ending in .go. 529 // For a vendor check we must exclude directories that contain no .go files. 530 // Otherwise it is not possible to vendor just a/b/c and still import the 531 // non-vendored a/b. See golang.org/issue/13832. 532 func hasGoFiles(dir string) bool { 533 fis, _ := ioutil.ReadDir(dir) 534 for _, fi := range fis { 535 if !fi.IsDir() && strings.HasSuffix(fi.Name(), ".go") { 536 return true 537 } 538 } 539 return false 540 } 541 542 // reusePackage reuses package p to satisfy the import at the top 543 // of the import stack stk. If this use causes an import loop, 544 // reusePackage updates p's error information to record the loop. 545 func reusePackage(p *Package, stk *importStack) *Package { 546 // We use p.imports==nil to detect a package that 547 // is in the midst of its own loadPackage call 548 // (all the recursion below happens before p.imports gets set). 549 if p.imports == nil { 550 if p.Error == nil { 551 p.Error = &PackageError{ 552 ImportStack: stk.copy(), 553 Err: "import cycle not allowed", 554 isImportCycle: true, 555 } 556 } 557 p.Incomplete = true 558 } 559 // Don't rewrite the import stack in the error if we have an import cycle. 560 // If we do, we'll lose the path that describes the cycle. 561 if p.Error != nil && !p.Error.isImportCycle && stk.shorterThan(p.Error.ImportStack) { 562 p.Error.ImportStack = stk.copy() 563 } 564 return p 565 } 566 567 // disallowInternal checks that srcDir is allowed to import p. 568 // If the import is allowed, disallowInternal returns the original package p. 569 // If not, it returns a new package containing just an appropriate error. 570 func disallowInternal(srcDir string, p *Package, stk *importStack) *Package { 571 // golang.org/s/go14internal: 572 // An import of a path containing the element “internal” 573 // is disallowed if the importing code is outside the tree 574 // rooted at the parent of the “internal” directory. 575 576 // There was an error loading the package; stop here. 577 if p.Error != nil { 578 return p 579 } 580 581 // The stack includes p.ImportPath. 582 // If that's the only thing on the stack, we started 583 // with a name given on the command line, not an 584 // import. Anything listed on the command line is fine. 585 if len(*stk) == 1 { 586 return p 587 } 588 589 // Check for "internal" element: three cases depending on begin of string and/or end of string. 590 i, ok := findInternal(p.ImportPath) 591 if !ok { 592 return p 593 } 594 595 // Internal is present. 596 // Map import path back to directory corresponding to parent of internal. 597 if i > 0 { 598 i-- // rewind over slash in ".../internal" 599 } 600 parent := p.Dir[:i+len(p.Dir)-len(p.ImportPath)] 601 if hasFilePathPrefix(filepath.Clean(srcDir), filepath.Clean(parent)) { 602 return p 603 } 604 605 // Look for symlinks before reporting error. 606 srcDir = expandPath(srcDir) 607 parent = expandPath(parent) 608 if hasFilePathPrefix(filepath.Clean(srcDir), filepath.Clean(parent)) { 609 return p 610 } 611 612 // Internal is present, and srcDir is outside parent's tree. Not allowed. 613 perr := *p 614 perr.Error = &PackageError{ 615 ImportStack: stk.copy(), 616 Err: "use of internal package not allowed", 617 } 618 perr.Incomplete = true 619 return &perr 620 } 621 622 // findInternal looks for the final "internal" path element in the given import path. 623 // If there isn't one, findInternal returns ok=false. 624 // Otherwise, findInternal returns ok=true and the index of the "internal". 625 func findInternal(path string) (index int, ok bool) { 626 // Three cases, depending on internal at start/end of string or not. 627 // The order matters: we must return the index of the final element, 628 // because the final one produces the most restrictive requirement 629 // on the importer. 630 switch { 631 case strings.HasSuffix(path, "/internal"): 632 return len(path) - len("internal"), true 633 case strings.Contains(path, "/internal/"): 634 return strings.LastIndex(path, "/internal/") + 1, true 635 case path == "internal", strings.HasPrefix(path, "internal/"): 636 return 0, true 637 } 638 return 0, false 639 } 640 641 // disallowVendor checks that srcDir is allowed to import p as path. 642 // If the import is allowed, disallowVendor returns the original package p. 643 // If not, it returns a new package containing just an appropriate error. 644 func disallowVendor(srcDir, path string, p *Package, stk *importStack) *Package { 645 // The stack includes p.ImportPath. 646 // If that's the only thing on the stack, we started 647 // with a name given on the command line, not an 648 // import. Anything listed on the command line is fine. 649 if len(*stk) == 1 { 650 return p 651 } 652 653 if perr := disallowVendorVisibility(srcDir, p, stk); perr != p { 654 return perr 655 } 656 657 // Paths like x/vendor/y must be imported as y, never as x/vendor/y. 658 if i, ok := findVendor(path); ok { 659 perr := *p 660 perr.Error = &PackageError{ 661 ImportStack: stk.copy(), 662 Err: "must be imported as " + path[i+len("vendor/"):], 663 } 664 perr.Incomplete = true 665 return &perr 666 } 667 668 return p 669 } 670 671 // disallowVendorVisibility checks that srcDir is allowed to import p. 672 // The rules are the same as for /internal/ except that a path ending in /vendor 673 // is not subject to the rules, only subdirectories of vendor. 674 // This allows people to have packages and commands named vendor, 675 // for maximal compatibility with existing source trees. 676 func disallowVendorVisibility(srcDir string, p *Package, stk *importStack) *Package { 677 // The stack includes p.ImportPath. 678 // If that's the only thing on the stack, we started 679 // with a name given on the command line, not an 680 // import. Anything listed on the command line is fine. 681 if len(*stk) == 1 { 682 return p 683 } 684 685 // Check for "vendor" element. 686 i, ok := findVendor(p.ImportPath) 687 if !ok { 688 return p 689 } 690 691 // Vendor is present. 692 // Map import path back to directory corresponding to parent of vendor. 693 if i > 0 { 694 i-- // rewind over slash in ".../vendor" 695 } 696 truncateTo := i + len(p.Dir) - len(p.ImportPath) 697 if truncateTo < 0 || len(p.Dir) < truncateTo { 698 return p 699 } 700 parent := p.Dir[:truncateTo] 701 if hasFilePathPrefix(filepath.Clean(srcDir), filepath.Clean(parent)) { 702 return p 703 } 704 705 // Look for symlinks before reporting error. 706 srcDir = expandPath(srcDir) 707 parent = expandPath(parent) 708 if hasFilePathPrefix(filepath.Clean(srcDir), filepath.Clean(parent)) { 709 return p 710 } 711 712 // Vendor is present, and srcDir is outside parent's tree. Not allowed. 713 perr := *p 714 perr.Error = &PackageError{ 715 ImportStack: stk.copy(), 716 Err: "use of vendored package not allowed", 717 } 718 perr.Incomplete = true 719 return &perr 720 } 721 722 // findVendor looks for the last non-terminating "vendor" path element in the given import path. 723 // If there isn't one, findVendor returns ok=false. 724 // Otherwise, findVendor returns ok=true and the index of the "vendor". 725 // 726 // Note that terminating "vendor" elements don't count: "x/vendor" is its own package, 727 // not the vendored copy of an import "" (the empty import path). 728 // This will allow people to have packages or commands named vendor. 729 // This may help reduce breakage, or it may just be confusing. We'll see. 730 func findVendor(path string) (index int, ok bool) { 731 // Two cases, depending on internal at start of string or not. 732 // The order matters: we must return the index of the final element, 733 // because the final one is where the effective import path starts. 734 switch { 735 case strings.Contains(path, "/vendor/"): 736 return strings.LastIndex(path, "/vendor/") + 1, true 737 case strings.HasPrefix(path, "vendor/"): 738 return 0, true 739 } 740 return 0, false 741 } 742 743 type targetDir int 744 745 const ( 746 toRoot targetDir = iota // to bin dir inside package root (default) 747 toTool // GOROOT/pkg/tool 748 stalePath // the old import path; fail to build 749 ) 750 751 // goTools is a map of Go program import path to install target directory. 752 var goTools = map[string]targetDir{ 753 "cmd/addr2line": toTool, 754 "cmd/api": toTool, 755 "cmd/asm": toTool, 756 "cmd/compile": toTool, 757 "cmd/cgo": toTool, 758 "cmd/cover": toTool, 759 "cmd/dist": toTool, 760 "cmd/doc": toTool, 761 "cmd/fix": toTool, 762 "cmd/link": toTool, 763 "cmd/newlink": toTool, 764 "cmd/nm": toTool, 765 "cmd/objdump": toTool, 766 "cmd/pack": toTool, 767 "cmd/pprof": toTool, 768 "cmd/trace": toTool, 769 "cmd/vet": toTool, 770 "code.google.com/p/go.tools/cmd/cover": stalePath, 771 "code.google.com/p/go.tools/cmd/godoc": stalePath, 772 "code.google.com/p/go.tools/cmd/vet": stalePath, 773 } 774 775 // expandScanner expands a scanner.List error into all the errors in the list. 776 // The default Error method only shows the first error. 777 func expandScanner(err error) error { 778 // Look for parser errors. 779 if err, ok := err.(scanner.ErrorList); ok { 780 // Prepare error with \n before each message. 781 // When printed in something like context: %v 782 // this will put the leading file positions each on 783 // its own line. It will also show all the errors 784 // instead of just the first, as err.Error does. 785 var buf bytes.Buffer 786 for _, e := range err { 787 e.Pos.Filename = shortPath(e.Pos.Filename) 788 buf.WriteString("\n") 789 buf.WriteString(e.Error()) 790 } 791 return errors.New(buf.String()) 792 } 793 return err 794 } 795 796 var raceExclude = map[string]bool{ 797 "runtime/race": true, 798 "runtime/msan": true, 799 "runtime/cgo": true, 800 "cmd/cgo": true, 801 "syscall": true, 802 "errors": true, 803 } 804 805 var cgoExclude = map[string]bool{ 806 "runtime/cgo": true, 807 } 808 809 var cgoSyscallExclude = map[string]bool{ 810 "runtime/cgo": true, 811 "runtime/race": true, 812 "runtime/msan": true, 813 } 814 815 // load populates p using information from bp, err, which should 816 // be the result of calling build.Context.Import. 817 func (p *Package) load(stk *importStack, bp *build.Package, err error) *Package { 818 p.copyBuild(bp) 819 820 // The localPrefix is the path we interpret ./ imports relative to. 821 // Synthesized main packages sometimes override this. 822 p.localPrefix = dirToImportPath(p.Dir) 823 824 if err != nil { 825 p.Incomplete = true 826 err = expandScanner(err) 827 p.Error = &PackageError{ 828 ImportStack: stk.copy(), 829 Err: err.Error(), 830 } 831 return p 832 } 833 834 useBindir := p.Name == "main" 835 if !p.Standard { 836 switch buildBuildmode { 837 case "c-archive", "c-shared", "plugin": 838 useBindir = false 839 } 840 } 841 842 if useBindir { 843 // Report an error when the old code.google.com/p/go.tools paths are used. 844 if goTools[p.ImportPath] == stalePath { 845 newPath := strings.Replace(p.ImportPath, "code.google.com/p/go.", "golang.org/x/", 1) 846 e := fmt.Sprintf("the %v command has moved; use %v instead.", p.ImportPath, newPath) 847 p.Error = &PackageError{Err: e} 848 return p 849 } 850 _, elem := filepath.Split(p.Dir) 851 full := buildContext.GOOS + "_" + buildContext.GOARCH + "/" + elem 852 if buildContext.GOOS != toolGOOS || buildContext.GOARCH != toolGOARCH { 853 // Install cross-compiled binaries to subdirectories of bin. 854 elem = full 855 } 856 if p.build.BinDir != "" { 857 // Install to GOBIN or bin of GOPATH entry. 858 p.target = filepath.Join(p.build.BinDir, elem) 859 if !p.Goroot && strings.Contains(elem, "/") && gobin != "" { 860 // Do not create $GOBIN/goos_goarch/elem. 861 p.target = "" 862 p.gobinSubdir = true 863 } 864 } 865 if goTools[p.ImportPath] == toTool { 866 // This is for 'go tool'. 867 // Override all the usual logic and force it into the tool directory. 868 p.target = filepath.Join(gorootPkg, "tool", full) 869 } 870 if p.target != "" && buildContext.GOOS == "windows" { 871 p.target += ".exe" 872 } 873 } else if p.local { 874 // Local import turned into absolute path. 875 // No permanent install target. 876 p.target = "" 877 } else { 878 p.target = p.build.PkgObj 879 if buildLinkshared { 880 shlibnamefile := p.target[:len(p.target)-2] + ".shlibname" 881 shlib, err := ioutil.ReadFile(shlibnamefile) 882 if err == nil { 883 libname := strings.TrimSpace(string(shlib)) 884 if buildContext.Compiler == "gccgo" { 885 p.Shlib = filepath.Join(p.build.PkgTargetRoot, "shlibs", libname) 886 } else { 887 p.Shlib = filepath.Join(p.build.PkgTargetRoot, libname) 888 889 } 890 } else if !os.IsNotExist(err) { 891 fatalf("unexpected error reading %s: %v", shlibnamefile, err) 892 } 893 } 894 } 895 896 importPaths := p.Imports 897 // Packages that use cgo import runtime/cgo implicitly. 898 // Packages that use cgo also import syscall implicitly, 899 // to wrap errno. 900 // Exclude certain packages to avoid circular dependencies. 901 if len(p.CgoFiles) > 0 && (!p.Standard || !cgoExclude[p.ImportPath]) { 902 importPaths = append(importPaths, "runtime/cgo") 903 } 904 if len(p.CgoFiles) > 0 && (!p.Standard || !cgoSyscallExclude[p.ImportPath]) { 905 importPaths = append(importPaths, "syscall") 906 } 907 908 // Currently build modes c-shared, pie, plugin, and -linkshared force 909 // external linking mode, and external linking mode forces an 910 // import of runtime/cgo. 911 pieCgo := buildBuildmode == "pie" && (buildContext.GOOS != "linux" || buildContext.GOARCH != "amd64") 912 if p.Name == "main" && !p.Goroot && (buildBuildmode == "c-shared" || buildBuildmode == "plugin" || pieCgo || buildLinkshared) { 913 importPaths = append(importPaths, "runtime/cgo") 914 } 915 916 // Everything depends on runtime, except runtime, its internal 917 // subpackages, and unsafe. 918 if !p.Standard || (p.ImportPath != "runtime" && !strings.HasPrefix(p.ImportPath, "runtime/internal/") && p.ImportPath != "unsafe") { 919 importPaths = append(importPaths, "runtime") 920 // When race detection enabled everything depends on runtime/race. 921 // Exclude certain packages to avoid circular dependencies. 922 if buildRace && (!p.Standard || !raceExclude[p.ImportPath]) { 923 importPaths = append(importPaths, "runtime/race") 924 } 925 // MSan uses runtime/msan. 926 if buildMSan && (!p.Standard || !raceExclude[p.ImportPath]) { 927 importPaths = append(importPaths, "runtime/msan") 928 } 929 // On ARM with GOARM=5, everything depends on math for the link. 930 if p.Name == "main" && goarch == "arm" { 931 importPaths = append(importPaths, "math") 932 } 933 } 934 935 // Runtime and its internal packages depend on runtime/internal/sys, 936 // so that they pick up the generated zversion.go file. 937 // This can be an issue particularly for runtime/internal/atomic; 938 // see issue 13655. 939 if p.Standard && (p.ImportPath == "runtime" || strings.HasPrefix(p.ImportPath, "runtime/internal/")) && p.ImportPath != "runtime/internal/sys" { 940 importPaths = append(importPaths, "runtime/internal/sys") 941 } 942 943 // Build list of full paths to all Go files in the package, 944 // for use by commands like go fmt. 945 p.gofiles = stringList(p.GoFiles, p.CgoFiles, p.TestGoFiles, p.XTestGoFiles) 946 for i := range p.gofiles { 947 p.gofiles[i] = filepath.Join(p.Dir, p.gofiles[i]) 948 } 949 sort.Strings(p.gofiles) 950 951 p.sfiles = stringList(p.SFiles) 952 for i := range p.sfiles { 953 p.sfiles[i] = filepath.Join(p.Dir, p.sfiles[i]) 954 } 955 sort.Strings(p.sfiles) 956 957 p.allgofiles = stringList(p.IgnoredGoFiles) 958 for i := range p.allgofiles { 959 p.allgofiles[i] = filepath.Join(p.Dir, p.allgofiles[i]) 960 } 961 p.allgofiles = append(p.allgofiles, p.gofiles...) 962 sort.Strings(p.allgofiles) 963 964 // Check for case-insensitive collision of input files. 965 // To avoid problems on case-insensitive files, we reject any package 966 // where two different input files have equal names under a case-insensitive 967 // comparison. 968 f1, f2 := foldDup(stringList( 969 p.GoFiles, 970 p.CgoFiles, 971 p.IgnoredGoFiles, 972 p.CFiles, 973 p.CXXFiles, 974 p.MFiles, 975 p.HFiles, 976 p.FFiles, 977 p.SFiles, 978 p.SysoFiles, 979 p.SwigFiles, 980 p.SwigCXXFiles, 981 p.TestGoFiles, 982 p.XTestGoFiles, 983 )) 984 if f1 != "" { 985 p.Error = &PackageError{ 986 ImportStack: stk.copy(), 987 Err: fmt.Sprintf("case-insensitive file name collision: %q and %q", f1, f2), 988 } 989 return p 990 } 991 992 // Build list of imported packages and full dependency list. 993 imports := make([]*Package, 0, len(p.Imports)) 994 deps := make(map[string]*Package) 995 save := func(path string, p1 *Package) { 996 // The same import path could produce an error or not, 997 // depending on what tries to import it. 998 // Prefer to record entries with errors, so we can report them. 999 p0 := deps[path] 1000 if p0 == nil || p1.Error != nil && (p0.Error == nil || len(p0.Error.ImportStack) > len(p1.Error.ImportStack)) { 1001 deps[path] = p1 1002 } 1003 } 1004 1005 for i, path := range importPaths { 1006 if path == "C" { 1007 continue 1008 } 1009 p1 := loadImport(path, p.Dir, p, stk, p.build.ImportPos[path], useVendor) 1010 if p.Standard && p.Error == nil && !p1.Standard && p1.Error == nil { 1011 p.Error = &PackageError{ 1012 ImportStack: stk.copy(), 1013 Err: fmt.Sprintf("non-standard import %q in standard package %q", path, p.ImportPath), 1014 } 1015 pos := p.build.ImportPos[path] 1016 if len(pos) > 0 { 1017 p.Error.Pos = pos[0].String() 1018 } 1019 } 1020 1021 path = p1.ImportPath 1022 importPaths[i] = path 1023 if i < len(p.Imports) { 1024 p.Imports[i] = path 1025 } 1026 1027 save(path, p1) 1028 imports = append(imports, p1) 1029 for _, dep := range p1.deps { 1030 save(dep.ImportPath, dep) 1031 } 1032 if p1.Incomplete { 1033 p.Incomplete = true 1034 } 1035 } 1036 p.imports = imports 1037 1038 p.Deps = make([]string, 0, len(deps)) 1039 for dep := range deps { 1040 p.Deps = append(p.Deps, dep) 1041 } 1042 sort.Strings(p.Deps) 1043 for _, dep := range p.Deps { 1044 p1 := deps[dep] 1045 if p1 == nil { 1046 panic("impossible: missing entry in package cache for " + dep + " imported by " + p.ImportPath) 1047 } 1048 p.deps = append(p.deps, p1) 1049 if p1.Error != nil { 1050 p.DepsErrors = append(p.DepsErrors, p1.Error) 1051 } 1052 } 1053 1054 // unsafe is a fake package. 1055 if p.Standard && (p.ImportPath == "unsafe" || buildContext.Compiler == "gccgo") { 1056 p.target = "" 1057 } 1058 p.Target = p.target 1059 1060 // If cgo is not enabled, ignore cgo supporting sources 1061 // just as we ignore go files containing import "C". 1062 if !buildContext.CgoEnabled { 1063 p.CFiles = nil 1064 p.CXXFiles = nil 1065 p.MFiles = nil 1066 p.SwigFiles = nil 1067 p.SwigCXXFiles = nil 1068 // Note that SFiles are okay (they go to the Go assembler) 1069 // and HFiles are okay (they might be used by the SFiles). 1070 // Also Sysofiles are okay (they might not contain object 1071 // code; see issue #16050). 1072 } 1073 1074 // The gc toolchain only permits C source files with cgo. 1075 if len(p.CFiles) > 0 && !p.usesCgo() && !p.usesSwig() && buildContext.Compiler == "gc" { 1076 p.Error = &PackageError{ 1077 ImportStack: stk.copy(), 1078 Err: fmt.Sprintf("C source files not allowed when not using cgo or SWIG: %s", strings.Join(p.CFiles, " ")), 1079 } 1080 return p 1081 } 1082 1083 // In the absence of errors lower in the dependency tree, 1084 // check for case-insensitive collisions of import paths. 1085 if len(p.DepsErrors) == 0 { 1086 dep1, dep2 := foldDup(p.Deps) 1087 if dep1 != "" { 1088 p.Error = &PackageError{ 1089 ImportStack: stk.copy(), 1090 Err: fmt.Sprintf("case-insensitive import collision: %q and %q", dep1, dep2), 1091 } 1092 return p 1093 } 1094 } 1095 1096 if p.BinaryOnly { 1097 // For binary-only package, use build ID from supplied package binary. 1098 buildID, err := readBuildID(p) 1099 if err == nil { 1100 p.buildID = buildID 1101 } 1102 } else { 1103 computeBuildID(p) 1104 } 1105 return p 1106 } 1107 1108 // usesSwig reports whether the package needs to run SWIG. 1109 func (p *Package) usesSwig() bool { 1110 return len(p.SwigFiles) > 0 || len(p.SwigCXXFiles) > 0 1111 } 1112 1113 // usesCgo reports whether the package needs to run cgo 1114 func (p *Package) usesCgo() bool { 1115 return len(p.CgoFiles) > 0 1116 } 1117 1118 // packageList returns the list of packages in the dag rooted at roots 1119 // as visited in a depth-first post-order traversal. 1120 func packageList(roots []*Package) []*Package { 1121 seen := map[*Package]bool{} 1122 all := []*Package{} 1123 var walk func(*Package) 1124 walk = func(p *Package) { 1125 if seen[p] { 1126 return 1127 } 1128 seen[p] = true 1129 for _, p1 := range p.imports { 1130 walk(p1) 1131 } 1132 all = append(all, p) 1133 } 1134 for _, root := range roots { 1135 walk(root) 1136 } 1137 return all 1138 } 1139 1140 // computeStale computes the Stale flag in the package dag that starts 1141 // at the named pkgs (command-line arguments). 1142 func computeStale(pkgs ...*Package) { 1143 for _, p := range packageList(pkgs) { 1144 p.Stale, p.StaleReason = isStale(p) 1145 } 1146 } 1147 1148 // The runtime version string takes one of two forms: 1149 // "go1.X[.Y]" for Go releases, and "devel +hash" at tip. 1150 // Determine whether we are in a released copy by 1151 // inspecting the version. 1152 var isGoRelease = strings.HasPrefix(runtime.Version(), "go1") 1153 1154 // isStale and computeBuildID 1155 // 1156 // Theory of Operation 1157 // 1158 // There is an installed copy of the package (or binary). 1159 // Can we reuse the installed copy, or do we need to build a new one? 1160 // 1161 // We can use the installed copy if it matches what we'd get 1162 // by building a new one. The hard part is predicting that without 1163 // actually running a build. 1164 // 1165 // To start, we must know the set of inputs to the build process that can 1166 // affect the generated output. At a minimum, that includes the source 1167 // files for the package and also any compiled packages imported by those 1168 // source files. The *Package has these, and we use them. One might also 1169 // argue for including in the input set: the build tags, whether the race 1170 // detector is in use, the target operating system and architecture, the 1171 // compiler and linker binaries being used, the additional flags being 1172 // passed to those, the cgo binary being used, the additional flags cgo 1173 // passes to the host C compiler, the host C compiler being used, the set 1174 // of host C include files and installed C libraries, and so on. 1175 // We include some but not all of this information. 1176 // 1177 // Once we have decided on a set of inputs, we must next decide how to 1178 // tell whether the content of that set has changed since the last build 1179 // of p. If there have been no changes, then we assume a new build would 1180 // produce the same result and reuse the installed package or binary. 1181 // But if there have been changes, then we assume a new build might not 1182 // produce the same result, so we rebuild. 1183 // 1184 // There are two common ways to decide whether the content of the set has 1185 // changed: modification times and content hashes. We use a mixture of both. 1186 // 1187 // The use of modification times (mtimes) was pioneered by make: 1188 // assuming that a file's mtime is an accurate record of when that file was last written, 1189 // and assuming that the modification time of an installed package or 1190 // binary is the time that it was built, if the mtimes of the inputs 1191 // predate the mtime of the installed object, then the build of that 1192 // object saw those versions of the files, and therefore a rebuild using 1193 // those same versions would produce the same object. In contrast, if any 1194 // mtime of an input is newer than the mtime of the installed object, a 1195 // change has occurred since the build, and the build should be redone. 1196 // 1197 // Modification times are attractive because the logic is easy to 1198 // understand and the file system maintains the mtimes automatically 1199 // (less work for us). Unfortunately, there are a variety of ways in 1200 // which the mtime approach fails to detect a change and reuses a stale 1201 // object file incorrectly. (Making the opposite mistake, rebuilding 1202 // unnecessarily, is only a performance problem and not a correctness 1203 // problem, so we ignore that one.) 1204 // 1205 // As a warmup, one problem is that to be perfectly precise, we need to 1206 // compare the input mtimes against the time at the beginning of the 1207 // build, but the object file time is the time at the end of the build. 1208 // If an input file changes after being read but before the object is 1209 // written, the next build will see an object newer than the input and 1210 // will incorrectly decide that the object is up to date. We make no 1211 // attempt to detect or solve this problem. 1212 // 1213 // Another problem is that due to file system imprecision, an input and 1214 // output that are actually ordered in time have the same mtime. 1215 // This typically happens on file systems with 1-second (or, worse, 1216 // 2-second) mtime granularity and with automated scripts that write an 1217 // input and then immediately run a build, or vice versa. If an input and 1218 // an output have the same mtime, the conservative behavior is to treat 1219 // the output as out-of-date and rebuild. This can cause one or more 1220 // spurious rebuilds, but only for 1 second, until the object finally has 1221 // an mtime later than the input. 1222 // 1223 // Another problem is that binary distributions often set the mtime on 1224 // all files to the same time. If the distribution includes both inputs 1225 // and cached build outputs, the conservative solution to the previous 1226 // problem will cause unnecessary rebuilds. Worse, in such a binary 1227 // distribution, those rebuilds might not even have permission to update 1228 // the cached build output. To avoid these write errors, if an input and 1229 // output have the same mtime, we assume the output is up-to-date. 1230 // This is the opposite of what the previous problem would have us do, 1231 // but binary distributions are more common than instances of the 1232 // previous problem. 1233 // 1234 // A variant of the last problem is that some binary distributions do not 1235 // set the mtime on all files to the same time. Instead they let the file 1236 // system record mtimes as the distribution is unpacked. If the outputs 1237 // are unpacked before the inputs, they'll be older and a build will try 1238 // to rebuild them. That rebuild might hit the same write errors as in 1239 // the last scenario. We don't make any attempt to solve this, and we 1240 // haven't had many reports of it. Perhaps the only time this happens is 1241 // when people manually unpack the distribution, and most of the time 1242 // that's done as the same user who will be using it, so an initial 1243 // rebuild on first use succeeds quietly. 1244 // 1245 // More generally, people and programs change mtimes on files. The last 1246 // few problems were specific examples of this, but it's a general problem. 1247 // For example, instead of a binary distribution, copying a home 1248 // directory from one directory or machine to another might copy files 1249 // but not preserve mtimes. If the inputs are new than the outputs on the 1250 // first machine but copied first, they end up older than the outputs on 1251 // the second machine. 1252 // 1253 // Because many other build systems have the same sensitivity to mtimes, 1254 // most programs manipulating source code take pains not to break the 1255 // mtime assumptions. For example, Git does not set the mtime of files 1256 // during a checkout operation, even when checking out an old version of 1257 // the code. This decision was made specifically to work well with 1258 // mtime-based build systems. 1259 // 1260 // The killer problem, though, for mtime-based build systems is that the 1261 // build only has access to the mtimes of the inputs that still exist. 1262 // If it is possible to remove an input without changing any other inputs, 1263 // a later build will think the object is up-to-date when it is not. 1264 // This happens for Go because a package is made up of all source 1265 // files in a directory. If a source file is removed, there is no newer 1266 // mtime available recording that fact. The mtime on the directory could 1267 // be used, but it also changes when unrelated files are added to or 1268 // removed from the directory, so including the directory mtime would 1269 // cause unnecessary rebuilds, possibly many. It would also exacerbate 1270 // the problems mentioned earlier, since even programs that are careful 1271 // to maintain mtimes on files rarely maintain mtimes on directories. 1272 // 1273 // A variant of the last problem is when the inputs change for other 1274 // reasons. For example, Go 1.4 and Go 1.5 both install $GOPATH/src/mypkg 1275 // into the same target, $GOPATH/pkg/$GOOS_$GOARCH/mypkg.a. 1276 // If Go 1.4 has built mypkg into mypkg.a, a build using Go 1.5 must 1277 // rebuild mypkg.a, but from mtimes alone mypkg.a looks up-to-date. 1278 // If Go 1.5 has just been installed, perhaps the compiler will have a 1279 // newer mtime; since the compiler is considered an input, that would 1280 // trigger a rebuild. But only once, and only the last Go 1.4 build of 1281 // mypkg.a happened before Go 1.5 was installed. If a user has the two 1282 // versions installed in different locations and flips back and forth, 1283 // mtimes alone cannot tell what to do. Changing the toolchain is 1284 // changing the set of inputs, without affecting any mtimes. 1285 // 1286 // To detect the set of inputs changing, we turn away from mtimes and to 1287 // an explicit data comparison. Specifically, we build a list of the 1288 // inputs to the build, compute its SHA1 hash, and record that as the 1289 // ``build ID'' in the generated object. At the next build, we can 1290 // recompute the build ID and compare it to the one in the generated 1291 // object. If they differ, the list of inputs has changed, so the object 1292 // is out of date and must be rebuilt. 1293 // 1294 // Because this build ID is computed before the build begins, the 1295 // comparison does not have the race that mtime comparison does. 1296 // 1297 // Making the build sensitive to changes in other state is 1298 // straightforward: include the state in the build ID hash, and if it 1299 // changes, so does the build ID, triggering a rebuild. 1300 // 1301 // To detect changes in toolchain, we include the toolchain version in 1302 // the build ID hash for package runtime, and then we include the build 1303 // IDs of all imported packages in the build ID for p. 1304 // 1305 // It is natural to think about including build tags in the build ID, but 1306 // the naive approach of just dumping the tags into the hash would cause 1307 // spurious rebuilds. For example, 'go install' and 'go install -tags neverusedtag' 1308 // produce the same binaries (assuming neverusedtag is never used). 1309 // A more precise approach would be to include only tags that have an 1310 // effect on the build. But the effect of a tag on the build is to 1311 // include or exclude a file from the compilation, and that file list is 1312 // already in the build ID hash. So the build ID is already tag-sensitive 1313 // in a perfectly precise way. So we do NOT explicitly add build tags to 1314 // the build ID hash. 1315 // 1316 // We do not include as part of the build ID the operating system, 1317 // architecture, or whether the race detector is enabled, even though all 1318 // three have an effect on the output, because that information is used 1319 // to decide the install location. Binaries for linux and binaries for 1320 // darwin are written to different directory trees; including that 1321 // information in the build ID is unnecessary (although it would be 1322 // harmless). 1323 // 1324 // TODO(rsc): Investigate the cost of putting source file content into 1325 // the build ID hash as a replacement for the use of mtimes. Using the 1326 // file content would avoid all the mtime problems, but it does require 1327 // reading all the source files, something we avoid today (we read the 1328 // beginning to find the build tags and the imports, but we stop as soon 1329 // as we see the import block is over). If the package is stale, the compiler 1330 // is going to read the files anyway. But if the package is up-to-date, the 1331 // read is overhead. 1332 // 1333 // TODO(rsc): Investigate the complexity of making the build more 1334 // precise about when individual results are needed. To be fully precise, 1335 // there are two results of a compilation: the entire .a file used by the link 1336 // and the subpiece used by later compilations (__.PKGDEF only). 1337 // If a rebuild is needed but produces the previous __.PKGDEF, then 1338 // no more recompilation due to the rebuilt package is needed, only 1339 // relinking. To date, there is nothing in the Go command to express this. 1340 // 1341 // Special Cases 1342 // 1343 // When the go command makes the wrong build decision and does not 1344 // rebuild something it should, users fall back to adding the -a flag. 1345 // Any common use of the -a flag should be considered prima facie evidence 1346 // that isStale is returning an incorrect false result in some important case. 1347 // Bugs reported in the behavior of -a itself should prompt the question 1348 // ``Why is -a being used at all? What bug does that indicate?'' 1349 // 1350 // There is a long history of changes to isStale to try to make -a into a 1351 // suitable workaround for bugs in the mtime-based decisions. 1352 // It is worth recording that history to inform (and, as much as possible, deter) future changes. 1353 // 1354 // (1) Before the build IDs were introduced, building with alternate tags 1355 // would happily reuse installed objects built without those tags. 1356 // For example, "go build -tags netgo myprog.go" would use the installed 1357 // copy of package net, even if that copy had been built without netgo. 1358 // (The netgo tag controls whether package net uses cgo or pure Go for 1359 // functionality such as name resolution.) 1360 // Using the installed non-netgo package defeats the purpose. 1361 // 1362 // Users worked around this with "go build -tags netgo -a myprog.go". 1363 // 1364 // Build IDs have made that workaround unnecessary: 1365 // "go build -tags netgo myprog.go" 1366 // cannot use a non-netgo copy of package net. 1367 // 1368 // (2) Before the build IDs were introduced, building with different toolchains, 1369 // especially changing between toolchains, tried to reuse objects stored in 1370 // $GOPATH/pkg, resulting in link-time errors about object file mismatches. 1371 // 1372 // Users worked around this with "go install -a ./...". 1373 // 1374 // Build IDs have made that workaround unnecessary: 1375 // "go install ./..." will rebuild any objects it finds that were built against 1376 // a different toolchain. 1377 // 1378 // (3) The common use of "go install -a ./..." led to reports of problems 1379 // when the -a forced the rebuild of the standard library, which for some 1380 // users was not writable. Because we didn't understand that the real 1381 // problem was the bug -a was working around, we changed -a not to 1382 // apply to the standard library. 1383 // 1384 // (4) The common use of "go build -tags netgo -a myprog.go" broke 1385 // when we changed -a not to apply to the standard library, because 1386 // if go build doesn't rebuild package net, it uses the non-netgo version. 1387 // 1388 // Users worked around this with "go build -tags netgo -installsuffix barf myprog.go". 1389 // The -installsuffix here is making the go command look for packages 1390 // in pkg/$GOOS_$GOARCH_barf instead of pkg/$GOOS_$GOARCH. 1391 // Since the former presumably doesn't exist, go build decides to rebuild 1392 // everything, including the standard library. Since go build doesn't 1393 // install anything it builds, nothing is ever written to pkg/$GOOS_$GOARCH_barf, 1394 // so repeated invocations continue to work. 1395 // 1396 // If the use of -a wasn't a red flag, the use of -installsuffix to point to 1397 // a non-existent directory in a command that installs nothing should 1398 // have been. 1399 // 1400 // (5) Now that (1) and (2) no longer need -a, we have removed the kludge 1401 // introduced in (3): once again, -a means ``rebuild everything,'' not 1402 // ``rebuild everything except the standard library.'' Only Go 1.4 had 1403 // the restricted meaning. 1404 // 1405 // In addition to these cases trying to trigger rebuilds, there are 1406 // special cases trying NOT to trigger rebuilds. The main one is that for 1407 // a variety of reasons (see above), the install process for a Go release 1408 // cannot be relied upon to set the mtimes such that the go command will 1409 // think the standard library is up to date. So the mtime evidence is 1410 // ignored for the standard library if we find ourselves in a release 1411 // version of Go. Build ID-based staleness checks still apply to the 1412 // standard library, even in release versions. This makes 1413 // 'go build -tags netgo' work, among other things. 1414 1415 // isStale reports whether package p needs to be rebuilt, 1416 // along with the reason why. 1417 func isStale(p *Package) (bool, string) { 1418 if p.Standard && (p.ImportPath == "unsafe" || buildContext.Compiler == "gccgo") { 1419 // fake, builtin package 1420 return false, "builtin package" 1421 } 1422 if p.Error != nil { 1423 return true, "errors loading package" 1424 } 1425 if p.Stale { 1426 return true, p.StaleReason 1427 } 1428 1429 // If this is a package with no source code, it cannot be rebuilt. 1430 // If the binary is missing, we mark the package stale so that 1431 // if a rebuild is needed, that rebuild attempt will produce a useful error. 1432 // (Some commands, such as 'go list', do not attempt to rebuild.) 1433 if p.BinaryOnly { 1434 if p.target == "" { 1435 // Fail if a build is attempted. 1436 return true, "no source code for package, but no install target" 1437 } 1438 if _, err := os.Stat(p.target); err != nil { 1439 // Fail if a build is attempted. 1440 return true, "no source code for package, but cannot access install target: " + err.Error() 1441 } 1442 return false, "no source code for package" 1443 } 1444 1445 // If the -a flag is given, rebuild everything. 1446 if buildA { 1447 return true, "build -a flag in use" 1448 } 1449 1450 // If there's no install target, we have to rebuild. 1451 if p.target == "" { 1452 return true, "no install target" 1453 } 1454 1455 // Package is stale if completely unbuilt. 1456 fi, err := os.Stat(p.target) 1457 if err != nil { 1458 return true, "cannot stat install target" 1459 } 1460 1461 // Package is stale if the expected build ID differs from the 1462 // recorded build ID. This catches changes like a source file 1463 // being removed from a package directory. See issue 3895. 1464 // It also catches changes in build tags that affect the set of 1465 // files being compiled. See issue 9369. 1466 // It also catches changes in toolchain, like when flipping between 1467 // two versions of Go compiling a single GOPATH. 1468 // See issue 8290 and issue 10702. 1469 targetBuildID, err := readBuildID(p) 1470 if err == nil && targetBuildID != p.buildID { 1471 return true, "build ID mismatch" 1472 } 1473 1474 // Package is stale if a dependency is. 1475 for _, p1 := range p.deps { 1476 if p1.Stale { 1477 return true, "stale dependency" 1478 } 1479 } 1480 1481 // The checks above are content-based staleness. 1482 // We assume they are always accurate. 1483 // 1484 // The checks below are mtime-based staleness. 1485 // We hope they are accurate, but we know that they fail in the case of 1486 // prebuilt Go installations that don't preserve the build mtimes 1487 // (for example, if the pkg/ mtimes are before the src/ mtimes). 1488 // See the large comment above isStale for details. 1489 1490 // If we are running a release copy of Go and didn't find a content-based 1491 // reason to rebuild the standard packages, do not rebuild them. 1492 // They may not be writable anyway, but they are certainly not changing. 1493 // This makes 'go build' skip the standard packages when 1494 // using an official release, even when the mtimes have been changed. 1495 // See issue 3036, issue 3149, issue 4106, issue 8290. 1496 // (If a change to a release tree must be made by hand, the way to force the 1497 // install is to run make.bash, which will remove the old package archives 1498 // before rebuilding.) 1499 if p.Standard && isGoRelease { 1500 return false, "standard package in Go release distribution" 1501 } 1502 1503 // Time-based staleness. 1504 1505 built := fi.ModTime() 1506 1507 olderThan := func(file string) bool { 1508 fi, err := os.Stat(file) 1509 return err != nil || fi.ModTime().After(built) 1510 } 1511 1512 // Package is stale if a dependency is, or if a dependency is newer. 1513 for _, p1 := range p.deps { 1514 if p1.target != "" && olderThan(p1.target) { 1515 return true, "newer dependency" 1516 } 1517 } 1518 1519 // As a courtesy to developers installing new versions of the compiler 1520 // frequently, define that packages are stale if they are 1521 // older than the compiler, and commands if they are older than 1522 // the linker. This heuristic will not work if the binaries are 1523 // back-dated, as some binary distributions may do, but it does handle 1524 // a very common case. 1525 // See issue 3036. 1526 // Exclude $GOROOT, under the assumption that people working on 1527 // the compiler may want to control when everything gets rebuilt, 1528 // and people updating the Go repository will run make.bash or all.bash 1529 // and get a full rebuild anyway. 1530 // Excluding $GOROOT used to also fix issue 4106, but that's now 1531 // taken care of above (at least when the installed Go is a released version). 1532 if p.Root != goroot { 1533 if olderThan(buildToolchain.compiler()) { 1534 return true, "newer compiler" 1535 } 1536 if p.build.IsCommand() && olderThan(buildToolchain.linker()) { 1537 return true, "newer linker" 1538 } 1539 } 1540 1541 // Note: Until Go 1.5, we had an additional shortcut here. 1542 // We built a list of the workspace roots ($GOROOT, each $GOPATH) 1543 // containing targets directly named on the command line, 1544 // and if p were not in any of those, it would be treated as up-to-date 1545 // as long as it is built. The goal was to avoid rebuilding a system-installed 1546 // $GOROOT, unless something from $GOROOT were explicitly named 1547 // on the command line (like go install math). 1548 // That's now handled by the isGoRelease clause above. 1549 // The other effect of the shortcut was to isolate different entries in 1550 // $GOPATH from each other. This had the unfortunate effect that 1551 // if you had (say), GOPATH listing two entries, one for commands 1552 // and one for libraries, and you did a 'git pull' in the library one 1553 // and then tried 'go install commands/...', it would build the new libraries 1554 // during the first build (because they wouldn't have been installed at all) 1555 // but then subsequent builds would not rebuild the libraries, even if the 1556 // mtimes indicate they are stale, because the different GOPATH entries 1557 // were treated differently. This behavior was confusing when using 1558 // non-trivial GOPATHs, which were particularly common with some 1559 // code management conventions, like the original godep. 1560 // Since the $GOROOT case (the original motivation) is handled separately, 1561 // we no longer put a barrier between the different $GOPATH entries. 1562 // 1563 // One implication of this is that if there is a system directory for 1564 // non-standard Go packages that is included in $GOPATH, the mtimes 1565 // on those compiled packages must be no earlier than the mtimes 1566 // on the source files. Since most distributions use the same mtime 1567 // for all files in a tree, they will be unaffected. People using plain 1568 // tar x to extract system-installed packages will need to adjust mtimes, 1569 // but it's better to force them to get the mtimes right than to ignore 1570 // the mtimes and thereby do the wrong thing in common use cases. 1571 // 1572 // So there is no GOPATH vs GOPATH shortcut here anymore. 1573 // 1574 // If something needs to come back here, we could try writing a dummy 1575 // file with a random name to the $GOPATH/pkg directory (and removing it) 1576 // to test for write access, and then skip GOPATH roots we don't have write 1577 // access to. But hopefully we can just use the mtimes always. 1578 1579 srcs := stringList(p.GoFiles, p.CFiles, p.CXXFiles, p.MFiles, p.HFiles, p.FFiles, p.SFiles, p.CgoFiles, p.SysoFiles, p.SwigFiles, p.SwigCXXFiles) 1580 for _, src := range srcs { 1581 if olderThan(filepath.Join(p.Dir, src)) { 1582 return true, "newer source file" 1583 } 1584 } 1585 1586 return false, "" 1587 } 1588 1589 // computeBuildID computes the build ID for p, leaving it in p.buildID. 1590 // Build ID is a hash of the information we want to detect changes in. 1591 // See the long comment in isStale for details. 1592 func computeBuildID(p *Package) { 1593 h := sha1.New() 1594 1595 // Include the list of files compiled as part of the package. 1596 // This lets us detect removed files. See issue 3895. 1597 inputFiles := stringList( 1598 p.GoFiles, 1599 p.CgoFiles, 1600 p.CFiles, 1601 p.CXXFiles, 1602 p.MFiles, 1603 p.HFiles, 1604 p.SFiles, 1605 p.SysoFiles, 1606 p.SwigFiles, 1607 p.SwigCXXFiles, 1608 ) 1609 for _, file := range inputFiles { 1610 fmt.Fprintf(h, "file %s\n", file) 1611 } 1612 1613 // Include the content of runtime/internal/sys/zversion.go in the hash 1614 // for package runtime. This will give package runtime a 1615 // different build ID in each Go release. 1616 if p.Standard && p.ImportPath == "runtime/internal/sys" { 1617 data, err := ioutil.ReadFile(filepath.Join(p.Dir, "zversion.go")) 1618 if err != nil { 1619 fatalf("go: %s", err) 1620 } 1621 fmt.Fprintf(h, "zversion %q\n", string(data)) 1622 } 1623 1624 // Include the build IDs of any dependencies in the hash. 1625 // This, combined with the runtime/zversion content, 1626 // will cause packages to have different build IDs when 1627 // compiled with different Go releases. 1628 // This helps the go command know to recompile when 1629 // people use the same GOPATH but switch between 1630 // different Go releases. See issue 10702. 1631 // This is also a better fix for issue 8290. 1632 for _, p1 := range p.deps { 1633 fmt.Fprintf(h, "dep %s %s\n", p1.ImportPath, p1.buildID) 1634 } 1635 1636 p.buildID = fmt.Sprintf("%x", h.Sum(nil)) 1637 } 1638 1639 var cwd, _ = os.Getwd() 1640 1641 var cmdCache = map[string]*Package{} 1642 1643 // loadPackage is like loadImport but is used for command-line arguments, 1644 // not for paths found in import statements. In addition to ordinary import paths, 1645 // loadPackage accepts pseudo-paths beginning with cmd/ to denote commands 1646 // in the Go command directory, as well as paths to those directories. 1647 func loadPackage(arg string, stk *importStack) *Package { 1648 if build.IsLocalImport(arg) { 1649 dir := arg 1650 if !filepath.IsAbs(dir) { 1651 if abs, err := filepath.Abs(dir); err == nil { 1652 // interpret relative to current directory 1653 dir = abs 1654 } 1655 } 1656 if sub, ok := hasSubdir(gorootSrc, dir); ok && strings.HasPrefix(sub, "cmd/") && !strings.Contains(sub[4:], "/") { 1657 arg = sub 1658 } 1659 } 1660 if strings.HasPrefix(arg, "cmd/") && !strings.Contains(arg[4:], "/") { 1661 if p := cmdCache[arg]; p != nil { 1662 return p 1663 } 1664 stk.push(arg) 1665 defer stk.pop() 1666 1667 bp, err := buildContext.ImportDir(filepath.Join(gorootSrc, arg), 0) 1668 bp.ImportPath = arg 1669 bp.Goroot = true 1670 bp.BinDir = gorootBin 1671 if gobin != "" { 1672 bp.BinDir = gobin 1673 } 1674 bp.Root = goroot 1675 bp.SrcRoot = gorootSrc 1676 p := new(Package) 1677 cmdCache[arg] = p 1678 p.load(stk, bp, err) 1679 if p.Error == nil && p.Name != "main" { 1680 p.Error = &PackageError{ 1681 ImportStack: stk.copy(), 1682 Err: fmt.Sprintf("expected package main but found package %s in %s", p.Name, p.Dir), 1683 } 1684 } 1685 return p 1686 } 1687 1688 // Wasn't a command; must be a package. 1689 // If it is a local import path but names a standard package, 1690 // we treat it as if the user specified the standard package. 1691 // This lets you run go test ./ioutil in package io and be 1692 // referring to io/ioutil rather than a hypothetical import of 1693 // "./ioutil". 1694 if build.IsLocalImport(arg) { 1695 bp, _ := buildContext.ImportDir(filepath.Join(cwd, arg), build.FindOnly) 1696 if bp.ImportPath != "" && bp.ImportPath != "." { 1697 arg = bp.ImportPath 1698 } 1699 } 1700 1701 return loadImport(arg, cwd, nil, stk, nil, 0) 1702 } 1703 1704 // packages returns the packages named by the 1705 // command line arguments 'args'. If a named package 1706 // cannot be loaded at all (for example, if the directory does not exist), 1707 // then packages prints an error and does not include that 1708 // package in the results. However, if errors occur trying 1709 // to load dependencies of a named package, the named 1710 // package is still returned, with p.Incomplete = true 1711 // and details in p.DepsErrors. 1712 func packages(args []string) []*Package { 1713 var pkgs []*Package 1714 for _, pkg := range packagesAndErrors(args) { 1715 if pkg.Error != nil { 1716 errorf("can't load package: %s", pkg.Error) 1717 continue 1718 } 1719 pkgs = append(pkgs, pkg) 1720 } 1721 return pkgs 1722 } 1723 1724 // packagesAndErrors is like 'packages' but returns a 1725 // *Package for every argument, even the ones that 1726 // cannot be loaded at all. 1727 // The packages that fail to load will have p.Error != nil. 1728 func packagesAndErrors(args []string) []*Package { 1729 if len(args) > 0 && strings.HasSuffix(args[0], ".go") { 1730 return []*Package{goFilesPackage(args)} 1731 } 1732 1733 args = importPaths(args) 1734 var ( 1735 pkgs []*Package 1736 stk importStack 1737 seenArg = make(map[string]bool) 1738 seenPkg = make(map[*Package]bool) 1739 ) 1740 1741 for _, arg := range args { 1742 if seenArg[arg] { 1743 continue 1744 } 1745 seenArg[arg] = true 1746 pkg := loadPackage(arg, &stk) 1747 if seenPkg[pkg] { 1748 continue 1749 } 1750 seenPkg[pkg] = true 1751 pkgs = append(pkgs, pkg) 1752 } 1753 computeStale(pkgs...) 1754 1755 return pkgs 1756 } 1757 1758 // packagesForBuild is like 'packages' but fails if any of 1759 // the packages or their dependencies have errors 1760 // (cannot be built). 1761 func packagesForBuild(args []string) []*Package { 1762 pkgs := packagesAndErrors(args) 1763 printed := map[*PackageError]bool{} 1764 for _, pkg := range pkgs { 1765 if pkg.Error != nil { 1766 errorf("can't load package: %s", pkg.Error) 1767 } 1768 for _, err := range pkg.DepsErrors { 1769 // Since these are errors in dependencies, 1770 // the same error might show up multiple times, 1771 // once in each package that depends on it. 1772 // Only print each once. 1773 if !printed[err] { 1774 printed[err] = true 1775 errorf("%s", err) 1776 } 1777 } 1778 } 1779 exitIfErrors() 1780 1781 // Check for duplicate loads of the same package. 1782 // That should be impossible, but if it does happen then 1783 // we end up trying to build the same package twice, 1784 // usually in parallel overwriting the same files, 1785 // which doesn't work very well. 1786 seen := map[string]bool{} 1787 reported := map[string]bool{} 1788 for _, pkg := range packageList(pkgs) { 1789 if seen[pkg.ImportPath] && !reported[pkg.ImportPath] { 1790 reported[pkg.ImportPath] = true 1791 errorf("internal error: duplicate loads of %s", pkg.ImportPath) 1792 } 1793 seen[pkg.ImportPath] = true 1794 } 1795 exitIfErrors() 1796 1797 return pkgs 1798 } 1799 1800 // hasSubdir reports whether dir is a subdirectory of 1801 // (possibly multiple levels below) root. 1802 // If so, it sets rel to the path fragment that must be 1803 // appended to root to reach dir. 1804 func hasSubdir(root, dir string) (rel string, ok bool) { 1805 if p, err := filepath.EvalSymlinks(root); err == nil { 1806 root = p 1807 } 1808 if p, err := filepath.EvalSymlinks(dir); err == nil { 1809 dir = p 1810 } 1811 const sep = string(filepath.Separator) 1812 root = filepath.Clean(root) 1813 if !strings.HasSuffix(root, sep) { 1814 root += sep 1815 } 1816 dir = filepath.Clean(dir) 1817 if !strings.HasPrefix(dir, root) { 1818 return "", false 1819 } 1820 return filepath.ToSlash(dir[len(root):]), true 1821 } 1822 1823 var ( 1824 errBuildIDToolchain = fmt.Errorf("build ID only supported in gc toolchain") 1825 errBuildIDMalformed = fmt.Errorf("malformed object file") 1826 errBuildIDUnknown = fmt.Errorf("lost build ID") 1827 ) 1828 1829 var ( 1830 bangArch = []byte("!<arch>") 1831 pkgdef = []byte("__.PKGDEF") 1832 goobject = []byte("go object ") 1833 buildid = []byte("build id ") 1834 ) 1835 1836 // readBuildID reads the build ID from an archive or binary. 1837 // It only supports the gc toolchain. 1838 // Other toolchain maintainers should adjust this function. 1839 func readBuildID(p *Package) (id string, err error) { 1840 if buildToolchain != (gcToolchain{}) { 1841 return "", errBuildIDToolchain 1842 } 1843 1844 // For commands, read build ID directly from binary. 1845 if p.Name == "main" { 1846 return ReadBuildIDFromBinary(p.Target) 1847 } 1848 1849 // Otherwise, we expect to have an archive (.a) file, 1850 // and we can read the build ID from the Go export data. 1851 if !strings.HasSuffix(p.Target, ".a") { 1852 return "", &os.PathError{Op: "parse", Path: p.Target, Err: errBuildIDUnknown} 1853 } 1854 1855 // Read just enough of the target to fetch the build ID. 1856 // The archive is expected to look like: 1857 // 1858 // !<arch> 1859 // __.PKGDEF 0 0 0 644 7955 ` 1860 // go object darwin amd64 devel X:none 1861 // build id "b41e5c45250e25c9fd5e9f9a1de7857ea0d41224" 1862 // 1863 // The variable-sized strings are GOOS, GOARCH, and the experiment list (X:none). 1864 // Reading the first 1024 bytes should be plenty. 1865 f, err := os.Open(p.Target) 1866 if err != nil { 1867 return "", err 1868 } 1869 data := make([]byte, 1024) 1870 n, err := io.ReadFull(f, data) 1871 f.Close() 1872 1873 if err != nil && n == 0 { 1874 return "", err 1875 } 1876 1877 bad := func() (string, error) { 1878 return "", &os.PathError{Op: "parse", Path: p.Target, Err: errBuildIDMalformed} 1879 } 1880 1881 // Archive header. 1882 for i := 0; ; i++ { // returns during i==3 1883 j := bytes.IndexByte(data, '\n') 1884 if j < 0 { 1885 return bad() 1886 } 1887 line := data[:j] 1888 data = data[j+1:] 1889 switch i { 1890 case 0: 1891 if !bytes.Equal(line, bangArch) { 1892 return bad() 1893 } 1894 case 1: 1895 if !bytes.HasPrefix(line, pkgdef) { 1896 return bad() 1897 } 1898 case 2: 1899 if !bytes.HasPrefix(line, goobject) { 1900 return bad() 1901 } 1902 case 3: 1903 if !bytes.HasPrefix(line, buildid) { 1904 // Found the object header, just doesn't have a build id line. 1905 // Treat as successful, with empty build id. 1906 return "", nil 1907 } 1908 id, err := strconv.Unquote(string(line[len(buildid):])) 1909 if err != nil { 1910 return bad() 1911 } 1912 return id, nil 1913 } 1914 } 1915 } 1916 1917 var ( 1918 goBuildPrefix = []byte("\xff Go build ID: \"") 1919 goBuildEnd = []byte("\"\n \xff") 1920 1921 elfPrefix = []byte("\x7fELF") 1922 1923 machoPrefixes = [][]byte{ 1924 {0xfe, 0xed, 0xfa, 0xce}, 1925 {0xfe, 0xed, 0xfa, 0xcf}, 1926 {0xce, 0xfa, 0xed, 0xfe}, 1927 {0xcf, 0xfa, 0xed, 0xfe}, 1928 } 1929 ) 1930 1931 var BuildIDReadSize = 32 * 1024 // changed for testing 1932 1933 // ReadBuildIDFromBinary reads the build ID from a binary. 1934 // 1935 // ELF binaries store the build ID in a proper PT_NOTE section. 1936 // 1937 // Other binary formats are not so flexible. For those, the linker 1938 // stores the build ID as non-instruction bytes at the very beginning 1939 // of the text segment, which should appear near the beginning 1940 // of the file. This is clumsy but fairly portable. Custom locations 1941 // can be added for other binary types as needed, like we did for ELF. 1942 func ReadBuildIDFromBinary(filename string) (id string, err error) { 1943 if filename == "" { 1944 return "", &os.PathError{Op: "parse", Path: filename, Err: errBuildIDUnknown} 1945 } 1946 1947 // Read the first 32 kB of the binary file. 1948 // That should be enough to find the build ID. 1949 // In ELF files, the build ID is in the leading headers, 1950 // which are typically less than 4 kB, not to mention 32 kB. 1951 // In Mach-O files, there's no limit, so we have to parse the file. 1952 // On other systems, we're trying to read enough that 1953 // we get the beginning of the text segment in the read. 1954 // The offset where the text segment begins in a hello 1955 // world compiled for each different object format today: 1956 // 1957 // Plan 9: 0x20 1958 // Windows: 0x600 1959 // 1960 f, err := os.Open(filename) 1961 if err != nil { 1962 return "", err 1963 } 1964 defer f.Close() 1965 1966 data := make([]byte, BuildIDReadSize) 1967 _, err = io.ReadFull(f, data) 1968 if err == io.ErrUnexpectedEOF { 1969 err = nil 1970 } 1971 if err != nil { 1972 return "", err 1973 } 1974 1975 if bytes.HasPrefix(data, elfPrefix) { 1976 return readELFGoBuildID(filename, f, data) 1977 } 1978 for _, m := range machoPrefixes { 1979 if bytes.HasPrefix(data, m) { 1980 return readMachoGoBuildID(filename, f, data) 1981 } 1982 } 1983 1984 return readRawGoBuildID(filename, data) 1985 } 1986 1987 // readRawGoBuildID finds the raw build ID stored in text segment data. 1988 func readRawGoBuildID(filename string, data []byte) (id string, err error) { 1989 i := bytes.Index(data, goBuildPrefix) 1990 if i < 0 { 1991 // Missing. Treat as successful but build ID empty. 1992 return "", nil 1993 } 1994 1995 j := bytes.Index(data[i+len(goBuildPrefix):], goBuildEnd) 1996 if j < 0 { 1997 return "", &os.PathError{Op: "parse", Path: filename, Err: errBuildIDMalformed} 1998 } 1999 2000 quoted := data[i+len(goBuildPrefix)-1 : i+len(goBuildPrefix)+j+1] 2001 id, err = strconv.Unquote(string(quoted)) 2002 if err != nil { 2003 return "", &os.PathError{Op: "parse", Path: filename, Err: errBuildIDMalformed} 2004 } 2005 2006 return id, nil 2007 }