github.com/zxy12/go_duplicate_112_new@v0.0.0-20200807091221-747231827200/src/cmd/go/internal/work/gc.go (about) 1 // Copyright 2011 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package work 6 7 import ( 8 "bufio" 9 "bytes" 10 "fmt" 11 "io" 12 "io/ioutil" 13 "log" 14 "os" 15 "path/filepath" 16 "runtime" 17 "strings" 18 19 "cmd/go/internal/base" 20 "cmd/go/internal/cfg" 21 "cmd/go/internal/load" 22 "cmd/go/internal/str" 23 "cmd/internal/objabi" 24 "crypto/sha1" 25 ) 26 27 // The Go toolchain. 28 29 type gcToolchain struct{} 30 31 func (gcToolchain) compiler() string { 32 return base.Tool("compile") 33 } 34 35 func (gcToolchain) linker() string { 36 return base.Tool("link") 37 } 38 39 func (gcToolchain) gc(b *Builder, a *Action, archive string, importcfg []byte, symabis string, asmhdr bool, gofiles []string) (ofile string, output []byte, err error) { 40 p := a.Package 41 objdir := a.Objdir 42 if archive != "" { 43 ofile = archive 44 } else { 45 out := "_go_.o" 46 ofile = objdir + out 47 } 48 49 pkgpath := p.ImportPath 50 if cfg.BuildBuildmode == "plugin" { 51 pkgpath = pluginPath(a) 52 } else if p.Name == "main" && !p.Internal.ForceLibrary { 53 pkgpath = "main" 54 } 55 gcargs := []string{"-p", pkgpath} 56 if p.Module != nil && p.Module.GoVersion != "" && allowedVersion(p.Module.GoVersion) { 57 gcargs = append(gcargs, "-lang=go"+p.Module.GoVersion) 58 } 59 if p.Standard { 60 gcargs = append(gcargs, "-std") 61 } 62 compilingRuntime := p.Standard && (p.ImportPath == "runtime" || strings.HasPrefix(p.ImportPath, "runtime/internal")) 63 // The runtime package imports a couple of general internal packages. 64 if p.Standard && (p.ImportPath == "internal/cpu" || p.ImportPath == "internal/bytealg") { 65 compilingRuntime = true 66 } 67 if compilingRuntime { 68 // runtime compiles with a special gc flag to check for 69 // memory allocations that are invalid in the runtime package, 70 // and to implement some special compiler pragmas. 71 gcargs = append(gcargs, "-+") 72 } 73 74 // If we're giving the compiler the entire package (no C etc files), tell it that, 75 // so that it can give good error messages about forward declarations. 76 // Exceptions: a few standard packages have forward declarations for 77 // pieces supplied behind-the-scenes by package runtime. 78 extFiles := len(p.CgoFiles) + len(p.CFiles) + len(p.CXXFiles) + len(p.MFiles) + len(p.FFiles) + len(p.SFiles) + len(p.SysoFiles) + len(p.SwigFiles) + len(p.SwigCXXFiles) 79 if p.Standard { 80 switch p.ImportPath { 81 case "bytes", "internal/poll", "net", "os", "runtime/pprof", "runtime/trace", "sync", "syscall", "time": 82 extFiles++ 83 } 84 } 85 if extFiles == 0 { 86 gcargs = append(gcargs, "-complete") 87 } 88 if cfg.BuildContext.InstallSuffix != "" { 89 gcargs = append(gcargs, "-installsuffix", cfg.BuildContext.InstallSuffix) 90 } 91 if a.buildID != "" { 92 gcargs = append(gcargs, "-buildid", a.buildID) 93 } 94 platform := cfg.Goos + "/" + cfg.Goarch 95 if p.Internal.OmitDebug || platform == "nacl/amd64p32" || cfg.Goos == "plan9" || cfg.Goarch == "wasm" { 96 gcargs = append(gcargs, "-dwarf=false") 97 } 98 if strings.HasPrefix(runtimeVersion, "go1") && !strings.Contains(os.Args[0], "go_bootstrap") { 99 gcargs = append(gcargs, "-goversion", runtimeVersion) 100 } 101 if symabis != "" { 102 gcargs = append(gcargs, "-symabis", symabis) 103 } 104 105 gcflags := str.StringList(forcedGcflags, p.Internal.Gcflags) 106 if compilingRuntime { 107 // Remove -N, if present. 108 // It is not possible to build the runtime with no optimizations, 109 // because the compiler cannot eliminate enough write barriers. 110 for i := 0; i < len(gcflags); i++ { 111 if gcflags[i] == "-N" { 112 copy(gcflags[i:], gcflags[i+1:]) 113 gcflags = gcflags[:len(gcflags)-1] 114 i-- 115 } 116 } 117 } 118 119 args := []interface{}{cfg.BuildToolexec, base.Tool("compile"), "-o", ofile, "-trimpath", trimDir(a.Objdir), gcflags, gcargs, "-D", p.Internal.LocalPrefix} 120 if importcfg != nil { 121 if err := b.writeFile(objdir+"importcfg", importcfg); err != nil { 122 return "", nil, err 123 } 124 args = append(args, "-importcfg", objdir+"importcfg") 125 } 126 if ofile == archive { 127 args = append(args, "-pack") 128 } 129 if asmhdr { 130 args = append(args, "-asmhdr", objdir+"go_asm.h") 131 } 132 133 // Add -c=N to use concurrent backend compilation, if possible. 134 if c := gcBackendConcurrency(gcflags); c > 1 { 135 args = append(args, fmt.Sprintf("-c=%d", c)) 136 } 137 138 for _, f := range gofiles { 139 args = append(args, mkAbs(p.Dir, f)) 140 } 141 142 output, err = b.runOut(p.Dir, nil, args...) 143 return ofile, output, err 144 } 145 146 // gcBackendConcurrency returns the backend compiler concurrency level for a package compilation. 147 func gcBackendConcurrency(gcflags []string) int { 148 // First, check whether we can use -c at all for this compilation. 149 canDashC := concurrentGCBackendCompilationEnabledByDefault 150 151 switch e := os.Getenv("GO19CONCURRENTCOMPILATION"); e { 152 case "0": 153 canDashC = false 154 case "1": 155 canDashC = true 156 case "": 157 // Not set. Use default. 158 default: 159 log.Fatalf("GO19CONCURRENTCOMPILATION must be 0, 1, or unset, got %q", e) 160 } 161 162 CheckFlags: 163 for _, flag := range gcflags { 164 // Concurrent compilation is presumed incompatible with any gcflags, 165 // except for a small whitelist of commonly used flags. 166 // If the user knows better, they can manually add their own -c to the gcflags. 167 switch flag { 168 case "-N", "-l", "-S", "-B", "-C", "-I": 169 // OK 170 default: 171 canDashC = false 172 break CheckFlags 173 } 174 } 175 176 // TODO: Test and delete these conditions. 177 if objabi.Fieldtrack_enabled != 0 || objabi.Preemptibleloops_enabled != 0 { 178 canDashC = false 179 } 180 181 if !canDashC { 182 return 1 183 } 184 185 // Decide how many concurrent backend compilations to allow. 186 // 187 // If we allow too many, in theory we might end up with p concurrent processes, 188 // each with c concurrent backend compiles, all fighting over the same resources. 189 // However, in practice, that seems not to happen too much. 190 // Most build graphs are surprisingly serial, so p==1 for much of the build. 191 // Furthermore, concurrent backend compilation is only enabled for a part 192 // of the overall compiler execution, so c==1 for much of the build. 193 // So don't worry too much about that interaction for now. 194 // 195 // However, in practice, setting c above 4 tends not to help very much. 196 // See the analysis in CL 41192. 197 // 198 // TODO(josharian): attempt to detect whether this particular compilation 199 // is likely to be a bottleneck, e.g. when: 200 // - it has no successor packages to compile (usually package main) 201 // - all paths through the build graph pass through it 202 // - critical path scheduling says it is high priority 203 // and in such a case, set c to runtime.NumCPU. 204 // We do this now when p==1. 205 if cfg.BuildP == 1 { 206 // No process parallelism. Max out c. 207 return runtime.NumCPU() 208 } 209 // Some process parallelism. Set c to min(4, numcpu). 210 c := 4 211 if ncpu := runtime.NumCPU(); ncpu < c { 212 c = ncpu 213 } 214 return c 215 } 216 217 func trimDir(dir string) string { 218 if len(dir) > 1 && dir[len(dir)-1] == filepath.Separator { 219 dir = dir[:len(dir)-1] 220 } 221 return dir 222 } 223 224 func asmArgs(a *Action, p *load.Package) []interface{} { 225 // Add -I pkg/GOOS_GOARCH so #include "textflag.h" works in .s files. 226 inc := filepath.Join(cfg.GOROOT, "pkg", "include") 227 args := []interface{}{cfg.BuildToolexec, base.Tool("asm"), "-trimpath", trimDir(a.Objdir), "-I", a.Objdir, "-I", inc, "-D", "GOOS_" + cfg.Goos, "-D", "GOARCH_" + cfg.Goarch, forcedAsmflags, p.Internal.Asmflags} 228 if p.ImportPath == "runtime" && cfg.Goarch == "386" { 229 for _, arg := range forcedAsmflags { 230 if arg == "-dynlink" { 231 args = append(args, "-D=GOBUILDMODE_shared=1") 232 } 233 } 234 } 235 236 if cfg.Goarch == "mips" || cfg.Goarch == "mipsle" { 237 // Define GOMIPS_value from cfg.GOMIPS. 238 args = append(args, "-D", "GOMIPS_"+cfg.GOMIPS) 239 } 240 241 if cfg.Goarch == "mips64" || cfg.Goarch == "mips64le" { 242 // Define GOMIPS64_value from cfg.GOMIPS64. 243 args = append(args, "-D", "GOMIPS64_"+cfg.GOMIPS64) 244 } 245 246 return args 247 } 248 249 func (gcToolchain) asm(b *Builder, a *Action, sfiles []string) ([]string, error) { 250 p := a.Package 251 args := asmArgs(a, p) 252 253 var ofiles []string 254 for _, sfile := range sfiles { 255 ofile := a.Objdir + sfile[:len(sfile)-len(".s")] + ".o" 256 ofiles = append(ofiles, ofile) 257 args1 := append(args, "-o", ofile, mkAbs(p.Dir, sfile)) 258 if err := b.run(a, p.Dir, p.ImportPath, nil, args1...); err != nil { 259 return nil, err 260 } 261 } 262 return ofiles, nil 263 } 264 265 func (gcToolchain) symabis(b *Builder, a *Action, sfiles []string) (string, error) { 266 mkSymabis := func(p *load.Package, sfiles []string, path string) error { 267 args := asmArgs(a, p) 268 args = append(args, "-gensymabis", "-o", path) 269 for _, sfile := range sfiles { 270 if p.ImportPath == "runtime/cgo" && strings.HasPrefix(sfile, "gcc_") { 271 continue 272 } 273 args = append(args, mkAbs(p.Dir, sfile)) 274 } 275 276 // Supply an empty go_asm.h as if the compiler had been run. 277 // -gensymabis parsing is lax enough that we don't need the 278 // actual definitions that would appear in go_asm.h. 279 if err := b.writeFile(a.Objdir+"go_asm.h", nil); err != nil { 280 return err 281 } 282 283 return b.run(a, p.Dir, p.ImportPath, nil, args...) 284 } 285 286 var symabis string // Only set if we actually create the file 287 p := a.Package 288 if len(sfiles) != 0 { 289 symabis = a.Objdir + "symabis" 290 if err := mkSymabis(p, sfiles, symabis); err != nil { 291 return "", err 292 } 293 } 294 295 // Gather known cross-package references from assembly code. 296 var otherPkgs []string 297 if p.ImportPath == "runtime" { 298 // Assembly in the following packages references 299 // symbols in runtime. 300 otherPkgs = []string{"syscall", "internal/syscall/unix", "runtime/cgo"} 301 } else if p.ImportPath == "runtime/internal/atomic" { 302 // sync/atomic is an assembly wrapper around 303 // runtime/internal/atomic. 304 otherPkgs = []string{"sync/atomic"} 305 } 306 for _, p2name := range otherPkgs { 307 p2 := load.LoadPackage(p2name, &load.ImportStack{}) 308 if len(p2.SFiles) == 0 { 309 continue 310 } 311 312 symabis2 := a.Objdir + "symabis2" 313 if err := mkSymabis(p2, p2.SFiles, symabis2); err != nil { 314 return "", err 315 } 316 317 // Filter out just the symbol refs and append them to 318 // the symabis file. 319 if cfg.BuildN { 320 // -x will print the lines from symabis2 that are actually appended 321 // to symabis. With -n, we don't know what those lines will be. 322 b.Showcmd("", `grep '^ref' <%s | grep -v '^ref\s*""\.' >>%s`, symabis2, a.Objdir+"symabis") 323 continue 324 } 325 abis2, err := ioutil.ReadFile(symabis2) 326 if err != nil { 327 return "", err 328 } 329 var refs bytes.Buffer 330 for _, line := range strings.Split(string(abis2), "\n") { 331 fs := strings.Fields(line) 332 if len(fs) >= 2 && fs[0] == "ref" && !strings.HasPrefix(fs[1], `"".`) { 333 fmt.Fprintf(&refs, "%s\n", line) 334 } 335 } 336 if refs.Len() != 0 { 337 symabis = a.Objdir + "symabis" 338 if err := b.appendFile(symabis, refs.Bytes()); err != nil { 339 return "", err 340 } 341 } 342 } 343 344 return symabis, nil 345 } 346 347 // toolVerify checks that the command line args writes the same output file 348 // if run using newTool instead. 349 // Unused now but kept around for future use. 350 func toolVerify(a *Action, b *Builder, p *load.Package, newTool string, ofile string, args []interface{}) error { 351 newArgs := make([]interface{}, len(args)) 352 copy(newArgs, args) 353 newArgs[1] = base.Tool(newTool) 354 newArgs[3] = ofile + ".new" // x.6 becomes x.6.new 355 if err := b.run(a, p.Dir, p.ImportPath, nil, newArgs...); err != nil { 356 return err 357 } 358 data1, err := ioutil.ReadFile(ofile) 359 if err != nil { 360 return err 361 } 362 data2, err := ioutil.ReadFile(ofile + ".new") 363 if err != nil { 364 return err 365 } 366 if !bytes.Equal(data1, data2) { 367 return fmt.Errorf("%s and %s produced different output files:\n%s\n%s", filepath.Base(args[1].(string)), newTool, strings.Join(str.StringList(args...), " "), strings.Join(str.StringList(newArgs...), " ")) 368 } 369 os.Remove(ofile + ".new") 370 return nil 371 } 372 373 func (gcToolchain) pack(b *Builder, a *Action, afile string, ofiles []string) error { 374 var absOfiles []string 375 for _, f := range ofiles { 376 absOfiles = append(absOfiles, mkAbs(a.Objdir, f)) 377 } 378 absAfile := mkAbs(a.Objdir, afile) 379 380 // The archive file should have been created by the compiler. 381 // Since it used to not work that way, verify. 382 if !cfg.BuildN { 383 if _, err := os.Stat(absAfile); err != nil { 384 base.Fatalf("os.Stat of archive file failed: %v", err) 385 } 386 } 387 388 p := a.Package 389 if cfg.BuildN || cfg.BuildX { 390 cmdline := str.StringList(base.Tool("pack"), "r", absAfile, absOfiles) 391 b.Showcmd(p.Dir, "%s # internal", joinUnambiguously(cmdline)) 392 } 393 if cfg.BuildN { 394 return nil 395 } 396 if err := packInternal(absAfile, absOfiles); err != nil { 397 b.showOutput(a, p.Dir, p.Desc(), err.Error()+"\n") 398 return errPrintedOutput 399 } 400 return nil 401 } 402 403 func packInternal(afile string, ofiles []string) error { 404 dst, err := os.OpenFile(afile, os.O_WRONLY|os.O_APPEND, 0) 405 if err != nil { 406 return err 407 } 408 defer dst.Close() // only for error returns or panics 409 w := bufio.NewWriter(dst) 410 411 for _, ofile := range ofiles { 412 src, err := os.Open(ofile) 413 if err != nil { 414 return err 415 } 416 fi, err := src.Stat() 417 if err != nil { 418 src.Close() 419 return err 420 } 421 // Note: Not using %-16.16s format because we care 422 // about bytes, not runes. 423 name := fi.Name() 424 if len(name) > 16 { 425 name = name[:16] 426 } else { 427 name += strings.Repeat(" ", 16-len(name)) 428 } 429 size := fi.Size() 430 fmt.Fprintf(w, "%s%-12d%-6d%-6d%-8o%-10d`\n", 431 name, 0, 0, 0, 0644, size) 432 n, err := io.Copy(w, src) 433 src.Close() 434 if err == nil && n < size { 435 err = io.ErrUnexpectedEOF 436 } else if err == nil && n > size { 437 err = fmt.Errorf("file larger than size reported by stat") 438 } 439 if err != nil { 440 return fmt.Errorf("copying %s to %s: %v", ofile, afile, err) 441 } 442 if size&1 != 0 { 443 w.WriteByte(0) 444 } 445 } 446 447 if err := w.Flush(); err != nil { 448 return err 449 } 450 return dst.Close() 451 } 452 453 // setextld sets the appropriate linker flags for the specified compiler. 454 func setextld(ldflags []string, compiler []string) []string { 455 for _, f := range ldflags { 456 if f == "-extld" || strings.HasPrefix(f, "-extld=") { 457 // don't override -extld if supplied 458 return ldflags 459 } 460 } 461 ldflags = append(ldflags, "-extld="+compiler[0]) 462 if len(compiler) > 1 { 463 extldflags := false 464 add := strings.Join(compiler[1:], " ") 465 for i, f := range ldflags { 466 if f == "-extldflags" && i+1 < len(ldflags) { 467 ldflags[i+1] = add + " " + ldflags[i+1] 468 extldflags = true 469 break 470 } else if strings.HasPrefix(f, "-extldflags=") { 471 ldflags[i] = "-extldflags=" + add + " " + ldflags[i][len("-extldflags="):] 472 extldflags = true 473 break 474 } 475 } 476 if !extldflags { 477 ldflags = append(ldflags, "-extldflags="+add) 478 } 479 } 480 return ldflags 481 } 482 483 // pluginPath computes the package path for a plugin main package. 484 // 485 // This is typically the import path of the main package p, unless the 486 // plugin is being built directly from source files. In that case we 487 // combine the package build ID with the contents of the main package 488 // source files. This allows us to identify two different plugins 489 // built from two source files with the same name. 490 func pluginPath(a *Action) string { 491 p := a.Package 492 if p.ImportPath != "command-line-arguments" { 493 return p.ImportPath 494 } 495 h := sha1.New() 496 fmt.Fprintf(h, "build ID: %s\n", a.buildID) 497 for _, file := range str.StringList(p.GoFiles, p.CgoFiles, p.SFiles) { 498 data, err := ioutil.ReadFile(filepath.Join(p.Dir, file)) 499 if err != nil { 500 base.Fatalf("go: %s", err) 501 } 502 h.Write(data) 503 } 504 return fmt.Sprintf("plugin/unnamed-%x", h.Sum(nil)) 505 } 506 507 func (gcToolchain) ld(b *Builder, root *Action, out, importcfg, mainpkg string) error { 508 cxx := len(root.Package.CXXFiles) > 0 || len(root.Package.SwigCXXFiles) > 0 509 for _, a := range root.Deps { 510 if a.Package != nil && (len(a.Package.CXXFiles) > 0 || len(a.Package.SwigCXXFiles) > 0) { 511 cxx = true 512 } 513 } 514 var ldflags []string 515 if cfg.BuildContext.InstallSuffix != "" { 516 ldflags = append(ldflags, "-installsuffix", cfg.BuildContext.InstallSuffix) 517 } 518 if root.Package.Internal.OmitDebug { 519 ldflags = append(ldflags, "-s", "-w") 520 } 521 if cfg.BuildBuildmode == "plugin" { 522 ldflags = append(ldflags, "-pluginpath", pluginPath(root)) 523 } 524 525 // Store BuildID inside toolchain binaries as a unique identifier of the 526 // tool being run, for use by content-based staleness determination. 527 if root.Package.Goroot && strings.HasPrefix(root.Package.ImportPath, "cmd/") { 528 ldflags = append(ldflags, "-X=cmd/internal/objabi.buildID="+root.buildID) 529 } 530 531 // If the user has not specified the -extld option, then specify the 532 // appropriate linker. In case of C++ code, use the compiler named 533 // by the CXX environment variable or defaultCXX if CXX is not set. 534 // Else, use the CC environment variable and defaultCC as fallback. 535 var compiler []string 536 if cxx { 537 compiler = envList("CXX", cfg.DefaultCXX(cfg.Goos, cfg.Goarch)) 538 } else { 539 compiler = envList("CC", cfg.DefaultCC(cfg.Goos, cfg.Goarch)) 540 } 541 ldflags = append(ldflags, "-buildmode="+ldBuildmode) 542 if root.buildID != "" { 543 ldflags = append(ldflags, "-buildid="+root.buildID) 544 } 545 ldflags = append(ldflags, forcedLdflags...) 546 ldflags = append(ldflags, root.Package.Internal.Ldflags...) 547 ldflags = setextld(ldflags, compiler) 548 549 // On OS X when using external linking to build a shared library, 550 // the argument passed here to -o ends up recorded in the final 551 // shared library in the LC_ID_DYLIB load command. 552 // To avoid putting the temporary output directory name there 553 // (and making the resulting shared library useless), 554 // run the link in the output directory so that -o can name 555 // just the final path element. 556 // On Windows, DLL file name is recorded in PE file 557 // export section, so do like on OS X. 558 dir := "." 559 if (cfg.Goos == "darwin" || cfg.Goos == "windows") && cfg.BuildBuildmode == "c-shared" { 560 dir, out = filepath.Split(out) 561 } 562 563 return b.run(root, dir, root.Package.ImportPath, nil, cfg.BuildToolexec, base.Tool("link"), "-o", out, "-importcfg", importcfg, ldflags, mainpkg) 564 } 565 566 func (gcToolchain) ldShared(b *Builder, root *Action, toplevelactions []*Action, out, importcfg string, allactions []*Action) error { 567 ldflags := []string{"-installsuffix", cfg.BuildContext.InstallSuffix} 568 ldflags = append(ldflags, "-buildmode=shared") 569 ldflags = append(ldflags, forcedLdflags...) 570 ldflags = append(ldflags, root.Package.Internal.Ldflags...) 571 cxx := false 572 for _, a := range allactions { 573 if a.Package != nil && (len(a.Package.CXXFiles) > 0 || len(a.Package.SwigCXXFiles) > 0) { 574 cxx = true 575 } 576 } 577 // If the user has not specified the -extld option, then specify the 578 // appropriate linker. In case of C++ code, use the compiler named 579 // by the CXX environment variable or defaultCXX if CXX is not set. 580 // Else, use the CC environment variable and defaultCC as fallback. 581 var compiler []string 582 if cxx { 583 compiler = envList("CXX", cfg.DefaultCXX(cfg.Goos, cfg.Goarch)) 584 } else { 585 compiler = envList("CC", cfg.DefaultCC(cfg.Goos, cfg.Goarch)) 586 } 587 ldflags = setextld(ldflags, compiler) 588 for _, d := range toplevelactions { 589 if !strings.HasSuffix(d.Target, ".a") { // omit unsafe etc and actions for other shared libraries 590 continue 591 } 592 ldflags = append(ldflags, d.Package.ImportPath+"="+d.Target) 593 } 594 return b.run(root, ".", out, nil, cfg.BuildToolexec, base.Tool("link"), "-o", out, "-importcfg", importcfg, ldflags) 595 } 596 597 func (gcToolchain) cc(b *Builder, a *Action, ofile, cfile string) error { 598 return fmt.Errorf("%s: C source files not supported without cgo", mkAbs(a.Package.Dir, cfile)) 599 }