github.com/sirkon/goproxy@v1.4.8/internal/work/gc.go (about) 1 // Copyright 2011 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package work 6 7 import ( 8 "bufio" 9 "bytes" 10 "fmt" 11 "io" 12 "io/ioutil" 13 "log" 14 "os" 15 "path/filepath" 16 "runtime" 17 "strings" 18 19 "github.com/sirkon/goproxy/internal/base" 20 "github.com/sirkon/goproxy/internal/cfg" 21 "github.com/sirkon/goproxy/internal/load" 22 "github.com/sirkon/goproxy/internal/str" 23 "github.com/sirkon/goproxy/internal/objabi" 24 "crypto/sha1" 25 ) 26 27 // The Go toolchain. 28 29 type gcToolchain struct{} 30 31 func (gcToolchain) compiler() string { 32 return base.Tool("compile") 33 } 34 35 func (gcToolchain) linker() string { 36 return base.Tool("link") 37 } 38 39 func (gcToolchain) gc(b *Builder, a *Action, archive string, importcfg []byte, asmhdr bool, gofiles []string) (ofile string, output []byte, err error) { 40 p := a.Package 41 objdir := a.Objdir 42 if archive != "" { 43 ofile = archive 44 } else { 45 out := "_go_.o" 46 ofile = objdir + out 47 } 48 49 pkgpath := p.ImportPath 50 if cfg.BuildBuildmode == "plugin" { 51 pkgpath = pluginPath(a) 52 } else if p.Name == "main" && !p.Internal.ForceLibrary { 53 pkgpath = "main" 54 } 55 gcargs := []string{"-p", pkgpath} 56 if p.Standard { 57 gcargs = append(gcargs, "-std") 58 } 59 compilingRuntime := p.Standard && (p.ImportPath == "runtime" || strings.HasPrefix(p.ImportPath, "runtime/internal")) 60 // The runtime package imports a couple of general internal packages. 61 if p.Standard && (p.ImportPath == "internal/cpu" || p.ImportPath == "internal/bytealg") { 62 compilingRuntime = true 63 } 64 if compilingRuntime { 65 // runtime compiles with a special gc flag to check for 66 // memory allocations that are invalid in the runtime package, 67 // and to implement some special compiler pragmas. 68 gcargs = append(gcargs, "-+") 69 } 70 71 // If we're giving the compiler the entire package (no C etc files), tell it that, 72 // so that it can give good error messages about forward declarations. 73 // Exceptions: a few standard packages have forward declarations for 74 // pieces supplied behind-the-scenes by package runtime. 75 extFiles := len(p.CgoFiles) + len(p.CFiles) + len(p.CXXFiles) + len(p.MFiles) + len(p.FFiles) + len(p.SFiles) + len(p.SysoFiles) + len(p.SwigFiles) + len(p.SwigCXXFiles) 76 if p.Standard { 77 switch p.ImportPath { 78 case "bytes", "internal/poll", "net", "os", "runtime/pprof", "runtime/trace", "sync", "syscall", "time": 79 extFiles++ 80 } 81 } 82 if extFiles == 0 { 83 gcargs = append(gcargs, "-complete") 84 } 85 if cfg.BuildContext.InstallSuffix != "" { 86 gcargs = append(gcargs, "-installsuffix", cfg.BuildContext.InstallSuffix) 87 } 88 if a.buildID != "" { 89 gcargs = append(gcargs, "-buildid", a.buildID) 90 } 91 platform := cfg.Goos + "/" + cfg.Goarch 92 if p.Internal.OmitDebug || platform == "nacl/amd64p32" || cfg.Goos == "plan9" || cfg.Goarch == "wasm" { 93 gcargs = append(gcargs, "-dwarf=false") 94 } 95 if strings.HasPrefix(runtimeVersion, "go1") && !strings.Contains(os.Args[0], "go_bootstrap") { 96 gcargs = append(gcargs, "-goversion", runtimeVersion) 97 } 98 99 gcflags := str.StringList(forcedGcflags, p.Internal.Gcflags) 100 if compilingRuntime { 101 // Remove -N, if present. 102 // It is not possible to build the runtime with no optimizations, 103 // because the compiler cannot eliminate enough write barriers. 104 for i := 0; i < len(gcflags); i++ { 105 if gcflags[i] == "-N" { 106 copy(gcflags[i:], gcflags[i+1:]) 107 gcflags = gcflags[:len(gcflags)-1] 108 i-- 109 } 110 } 111 } 112 113 args := []interface{}{cfg.BuildToolexec, base.Tool("compile"), "-o", ofile, "-trimpath", trimDir(a.Objdir), gcflags, gcargs, "-D", p.Internal.LocalPrefix} 114 if importcfg != nil { 115 if err := b.writeFile(objdir+"importcfg", importcfg); err != nil { 116 return "", nil, err 117 } 118 args = append(args, "-importcfg", objdir+"importcfg") 119 } 120 if ofile == archive { 121 args = append(args, "-pack") 122 } 123 if asmhdr { 124 args = append(args, "-asmhdr", objdir+"go_asm.h") 125 } 126 127 // Add -c=N to use concurrent backend compilation, if possible. 128 if c := gcBackendConcurrency(gcflags); c > 1 { 129 args = append(args, fmt.Sprintf("-c=%d", c)) 130 } 131 132 for _, f := range gofiles { 133 args = append(args, mkAbs(p.Dir, f)) 134 } 135 136 output, err = b.runOut(p.Dir, nil, args...) 137 return ofile, output, err 138 } 139 140 // gcBackendConcurrency returns the backend compiler concurrency level for a package compilation. 141 func gcBackendConcurrency(gcflags []string) int { 142 // First, check whether we can use -c at all for this compilation. 143 canDashC := concurrentGCBackendCompilationEnabledByDefault 144 145 switch e := os.Getenv("GO19CONCURRENTCOMPILATION"); e { 146 case "0": 147 canDashC = false 148 case "1": 149 canDashC = true 150 case "": 151 // Not set. Use default. 152 default: 153 log.Fatalf("GO19CONCURRENTCOMPILATION must be 0, 1, or unset, got %q", e) 154 } 155 156 CheckFlags: 157 for _, flag := range gcflags { 158 // Concurrent compilation is presumed incompatible with any gcflags, 159 // except for a small whitelist of commonly used flags. 160 // If the user knows better, they can manually add their own -c to the gcflags. 161 switch flag { 162 case "-N", "-l", "-S", "-B", "-C", "-I": 163 // OK 164 default: 165 canDashC = false 166 break CheckFlags 167 } 168 } 169 170 // TODO: Test and delete these conditions. 171 if objabi.Fieldtrack_enabled != 0 || objabi.Preemptibleloops_enabled != 0 || objabi.Clobberdead_enabled != 0 { 172 canDashC = false 173 } 174 175 if !canDashC { 176 return 1 177 } 178 179 // Decide how many concurrent backend compilations to allow. 180 // 181 // If we allow too many, in theory we might end up with p concurrent processes, 182 // each with c concurrent backend compiles, all fighting over the same resources. 183 // However, in practice, that seems not to happen too much. 184 // Most build graphs are surprisingly serial, so p==1 for much of the build. 185 // Furthermore, concurrent backend compilation is only enabled for a part 186 // of the overall compiler execution, so c==1 for much of the build. 187 // So don't worry too much about that interaction for now. 188 // 189 // However, in practice, setting c above 4 tends not to help very much. 190 // See the analysis in CL 41192. 191 // 192 // TODO(josharian): attempt to detect whether this particular compilation 193 // is likely to be a bottleneck, e.g. when: 194 // - it has no successor packages to compile (usually package main) 195 // - all paths through the build graph pass through it 196 // - critical path scheduling says it is high priority 197 // and in such a case, set c to runtime.NumCPU. 198 // We do this now when p==1. 199 if cfg.BuildP == 1 { 200 // No process parallelism. Max out c. 201 return runtime.NumCPU() 202 } 203 // Some process parallelism. Set c to min(4, numcpu). 204 c := 4 205 if ncpu := runtime.NumCPU(); ncpu < c { 206 c = ncpu 207 } 208 return c 209 } 210 211 func trimDir(dir string) string { 212 if len(dir) > 1 && dir[len(dir)-1] == filepath.Separator { 213 dir = dir[:len(dir)-1] 214 } 215 return dir 216 } 217 218 func (gcToolchain) asm(b *Builder, a *Action, sfiles []string) ([]string, error) { 219 p := a.Package 220 // Add -I pkg/GOOS_GOARCH so #include "textflag.h" works in .s files. 221 inc := filepath.Join(cfg.GOROOT, "pkg", "include") 222 args := []interface{}{cfg.BuildToolexec, base.Tool("asm"), "-trimpath", trimDir(a.Objdir), "-I", a.Objdir, "-I", inc, "-D", "GOOS_" + cfg.Goos, "-D", "GOARCH_" + cfg.Goarch, forcedAsmflags, p.Internal.Asmflags} 223 if p.ImportPath == "runtime" && cfg.Goarch == "386" { 224 for _, arg := range forcedAsmflags { 225 if arg == "-dynlink" { 226 args = append(args, "-D=GOBUILDMODE_shared=1") 227 } 228 } 229 } 230 231 if cfg.Goarch == "mips" || cfg.Goarch == "mipsle" { 232 // Define GOMIPS_value from cfg.GOMIPS. 233 args = append(args, "-D", "GOMIPS_"+cfg.GOMIPS) 234 } 235 236 if cfg.Goarch == "mips64" || cfg.Goarch == "mips64le" { 237 // Define GOMIPS64_value from cfg.GOMIPS64. 238 args = append(args, "-D", "GOMIPS64_"+cfg.GOMIPS64) 239 } 240 241 var ofiles []string 242 for _, sfile := range sfiles { 243 ofile := a.Objdir + sfile[:len(sfile)-len(".s")] + ".o" 244 ofiles = append(ofiles, ofile) 245 args1 := append(args, "-o", ofile, mkAbs(p.Dir, sfile)) 246 if err := b.run(a, p.Dir, p.ImportPath, nil, args1...); err != nil { 247 return nil, err 248 } 249 } 250 return ofiles, nil 251 } 252 253 // toolVerify checks that the command line args writes the same output file 254 // if run using newTool instead. 255 // Unused now but kept around for future use. 256 func toolVerify(a *Action, b *Builder, p *load.Package, newTool string, ofile string, args []interface{}) error { 257 newArgs := make([]interface{}, len(args)) 258 copy(newArgs, args) 259 newArgs[1] = base.Tool(newTool) 260 newArgs[3] = ofile + ".new" // x.6 becomes x.6.new 261 if err := b.run(a, p.Dir, p.ImportPath, nil, newArgs...); err != nil { 262 return err 263 } 264 data1, err := ioutil.ReadFile(ofile) 265 if err != nil { 266 return err 267 } 268 data2, err := ioutil.ReadFile(ofile + ".new") 269 if err != nil { 270 return err 271 } 272 if !bytes.Equal(data1, data2) { 273 return fmt.Errorf("%s and %s produced different output files:\n%s\n%s", filepath.Base(args[1].(string)), newTool, strings.Join(str.StringList(args...), " "), strings.Join(str.StringList(newArgs...), " ")) 274 } 275 os.Remove(ofile + ".new") 276 return nil 277 } 278 279 func (gcToolchain) pack(b *Builder, a *Action, afile string, ofiles []string) error { 280 var absOfiles []string 281 for _, f := range ofiles { 282 absOfiles = append(absOfiles, mkAbs(a.Objdir, f)) 283 } 284 absAfile := mkAbs(a.Objdir, afile) 285 286 // The archive file should have been created by the compiler. 287 // Since it used to not work that way, verify. 288 if !cfg.BuildN { 289 if _, err := os.Stat(absAfile); err != nil { 290 base.Fatalf("os.Stat of archive file failed: %v", err) 291 } 292 } 293 294 p := a.Package 295 if cfg.BuildN || cfg.BuildX { 296 cmdline := str.StringList(base.Tool("pack"), "r", absAfile, absOfiles) 297 b.Showcmd(p.Dir, "%s # internal", joinUnambiguously(cmdline)) 298 } 299 if cfg.BuildN { 300 return nil 301 } 302 if err := packInternal(absAfile, absOfiles); err != nil { 303 b.showOutput(a, p.Dir, p.Desc(), err.Error()+"\n") 304 return errPrintedOutput 305 } 306 return nil 307 } 308 309 func packInternal(afile string, ofiles []string) error { 310 dst, err := os.OpenFile(afile, os.O_WRONLY|os.O_APPEND, 0) 311 if err != nil { 312 return err 313 } 314 defer dst.Close() // only for error returns or panics 315 w := bufio.NewWriter(dst) 316 317 for _, ofile := range ofiles { 318 src, err := os.Open(ofile) 319 if err != nil { 320 return err 321 } 322 fi, err := src.Stat() 323 if err != nil { 324 src.Close() 325 return err 326 } 327 // Note: Not using %-16.16s format because we care 328 // about bytes, not runes. 329 name := fi.Name() 330 if len(name) > 16 { 331 name = name[:16] 332 } else { 333 name += strings.Repeat(" ", 16-len(name)) 334 } 335 size := fi.Size() 336 fmt.Fprintf(w, "%s%-12d%-6d%-6d%-8o%-10d`\n", 337 name, 0, 0, 0, 0644, size) 338 n, err := io.Copy(w, src) 339 src.Close() 340 if err == nil && n < size { 341 err = io.ErrUnexpectedEOF 342 } else if err == nil && n > size { 343 err = fmt.Errorf("file larger than size reported by stat") 344 } 345 if err != nil { 346 return fmt.Errorf("copying %s to %s: %v", ofile, afile, err) 347 } 348 if size&1 != 0 { 349 w.WriteByte(0) 350 } 351 } 352 353 if err := w.Flush(); err != nil { 354 return err 355 } 356 return dst.Close() 357 } 358 359 // setextld sets the appropriate linker flags for the specified compiler. 360 func setextld(ldflags []string, compiler []string) []string { 361 for _, f := range ldflags { 362 if f == "-extld" || strings.HasPrefix(f, "-extld=") { 363 // don't override -extld if supplied 364 return ldflags 365 } 366 } 367 ldflags = append(ldflags, "-extld="+compiler[0]) 368 if len(compiler) > 1 { 369 extldflags := false 370 add := strings.Join(compiler[1:], " ") 371 for i, f := range ldflags { 372 if f == "-extldflags" && i+1 < len(ldflags) { 373 ldflags[i+1] = add + " " + ldflags[i+1] 374 extldflags = true 375 break 376 } else if strings.HasPrefix(f, "-extldflags=") { 377 ldflags[i] = "-extldflags=" + add + " " + ldflags[i][len("-extldflags="):] 378 extldflags = true 379 break 380 } 381 } 382 if !extldflags { 383 ldflags = append(ldflags, "-extldflags="+add) 384 } 385 } 386 return ldflags 387 } 388 389 // pluginPath computes the package path for a plugin main package. 390 // 391 // This is typically the import path of the main package p, unless the 392 // plugin is being built directly from source files. In that case we 393 // combine the package build ID with the contents of the main package 394 // source files. This allows us to identify two different plugins 395 // built from two source files with the same name. 396 func pluginPath(a *Action) string { 397 p := a.Package 398 if p.ImportPath != "command-line-arguments" { 399 return p.ImportPath 400 } 401 h := sha1.New() 402 fmt.Fprintf(h, "build ID: %s\n", a.buildID) 403 for _, file := range str.StringList(p.GoFiles, p.CgoFiles, p.SFiles) { 404 data, err := ioutil.ReadFile(filepath.Join(p.Dir, file)) 405 if err != nil { 406 base.Fatalf("go: %s", err) 407 } 408 h.Write(data) 409 } 410 return fmt.Sprintf("plugin/unnamed-%x", h.Sum(nil)) 411 } 412 413 func (gcToolchain) ld(b *Builder, root *Action, out, importcfg, mainpkg string) error { 414 cxx := len(root.Package.CXXFiles) > 0 || len(root.Package.SwigCXXFiles) > 0 415 for _, a := range root.Deps { 416 if a.Package != nil && (len(a.Package.CXXFiles) > 0 || len(a.Package.SwigCXXFiles) > 0) { 417 cxx = true 418 } 419 } 420 var ldflags []string 421 if cfg.BuildContext.InstallSuffix != "" { 422 ldflags = append(ldflags, "-installsuffix", cfg.BuildContext.InstallSuffix) 423 } 424 if root.Package.Internal.OmitDebug { 425 ldflags = append(ldflags, "-s", "-w") 426 } 427 if cfg.BuildBuildmode == "plugin" { 428 ldflags = append(ldflags, "-pluginpath", pluginPath(root)) 429 } 430 431 // Store BuildID inside toolchain binaries as a unique identifier of the 432 // tool being run, for use by content-based staleness determination. 433 if root.Package.Goroot && strings.HasPrefix(root.Package.ImportPath, "cmd/") { 434 ldflags = append(ldflags, "-X=github.com/sirkon/goproxy/internal/objabi.buildID="+root.buildID) 435 } 436 437 // If the user has not specified the -extld option, then specify the 438 // appropriate linker. In case of C++ code, use the compiler named 439 // by the CXX environment variable or defaultCXX if CXX is not set. 440 // Else, use the CC environment variable and defaultCC as fallback. 441 var compiler []string 442 if cxx { 443 compiler = envList("CXX", cfg.DefaultCXX(cfg.Goos, cfg.Goarch)) 444 } else { 445 compiler = envList("CC", cfg.DefaultCC(cfg.Goos, cfg.Goarch)) 446 } 447 ldflags = append(ldflags, "-buildmode="+ldBuildmode) 448 if root.buildID != "" { 449 ldflags = append(ldflags, "-buildid="+root.buildID) 450 } 451 ldflags = append(ldflags, forcedLdflags...) 452 ldflags = append(ldflags, root.Package.Internal.Ldflags...) 453 ldflags = setextld(ldflags, compiler) 454 455 // On OS X when using external linking to build a shared library, 456 // the argument passed here to -o ends up recorded in the final 457 // shared library in the LC_ID_DYLIB load command. 458 // To avoid putting the temporary output directory name there 459 // (and making the resulting shared library useless), 460 // run the link in the output directory so that -o can name 461 // just the final path element. 462 // On Windows, DLL file name is recorded in PE file 463 // export section, so do like on OS X. 464 dir := "." 465 if (cfg.Goos == "darwin" || cfg.Goos == "windows") && cfg.BuildBuildmode == "c-shared" { 466 dir, out = filepath.Split(out) 467 } 468 469 return b.run(root, dir, root.Package.ImportPath, nil, cfg.BuildToolexec, base.Tool("link"), "-o", out, "-importcfg", importcfg, ldflags, mainpkg) 470 } 471 472 func (gcToolchain) ldShared(b *Builder, root *Action, toplevelactions []*Action, out, importcfg string, allactions []*Action) error { 473 ldflags := []string{"-installsuffix", cfg.BuildContext.InstallSuffix} 474 ldflags = append(ldflags, "-buildmode=shared") 475 ldflags = append(ldflags, forcedLdflags...) 476 ldflags = append(ldflags, root.Package.Internal.Ldflags...) 477 cxx := false 478 for _, a := range allactions { 479 if a.Package != nil && (len(a.Package.CXXFiles) > 0 || len(a.Package.SwigCXXFiles) > 0) { 480 cxx = true 481 } 482 } 483 // If the user has not specified the -extld option, then specify the 484 // appropriate linker. In case of C++ code, use the compiler named 485 // by the CXX environment variable or defaultCXX if CXX is not set. 486 // Else, use the CC environment variable and defaultCC as fallback. 487 var compiler []string 488 if cxx { 489 compiler = envList("CXX", cfg.DefaultCXX(cfg.Goos, cfg.Goarch)) 490 } else { 491 compiler = envList("CC", cfg.DefaultCC(cfg.Goos, cfg.Goarch)) 492 } 493 ldflags = setextld(ldflags, compiler) 494 for _, d := range toplevelactions { 495 if !strings.HasSuffix(d.Target, ".a") { // omit unsafe etc and actions for other shared libraries 496 continue 497 } 498 ldflags = append(ldflags, d.Package.ImportPath+"="+d.Target) 499 } 500 return b.run(root, ".", out, nil, cfg.BuildToolexec, base.Tool("link"), "-o", out, "-importcfg", importcfg, ldflags) 501 } 502 503 func (gcToolchain) cc(b *Builder, a *Action, ofile, cfile string) error { 504 return fmt.Errorf("%s: C source files not supported without cgo", mkAbs(a.Package.Dir, cfile)) 505 }