github.com/go-asm/go@v1.21.1-0.20240213172139-40c5ead50c48/cmd/dist/build.go (about) 1 // Copyright 2012 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package dist 6 7 import ( 8 "bytes" 9 "encoding/json" 10 "flag" 11 "fmt" 12 "io" 13 "io/fs" 14 "log" 15 "os" 16 "os/exec" 17 "path/filepath" 18 "regexp" 19 "sort" 20 "strings" 21 "sync" 22 "time" 23 ) 24 25 // Initialization for any invocation. 26 27 // The usual variables. 28 var ( 29 goarch string 30 gorootBin string 31 gorootBinGo string 32 gohostarch string 33 gohostos string 34 goos string 35 goarm string 36 go386 string 37 goamd64 string 38 gomips string 39 gomips64 string 40 goppc64 string 41 goroot string 42 goroot_final string 43 goextlinkenabled string 44 gogcflags string // For running built compiler 45 goldflags string 46 goexperiment string 47 workdir string 48 tooldir string 49 oldgoos string 50 oldgoarch string 51 oldgocache string 52 exe string 53 defaultcc map[string]string 54 defaultcxx map[string]string 55 defaultpkgconfig string 56 defaultldso string 57 58 rebuildall bool 59 noOpt bool 60 isRelease bool 61 62 vflag int // verbosity 63 ) 64 65 // The known architectures. 66 var okgoarch = []string{ 67 "386", 68 "amd64", 69 "arm", 70 "arm64", 71 "loong64", 72 "mips", 73 "mipsle", 74 "mips64", 75 "mips64le", 76 "ppc64", 77 "ppc64le", 78 "riscv64", 79 "s390x", 80 "sparc64", 81 "wasm", 82 } 83 84 // The known operating systems. 85 var okgoos = []string{ 86 "darwin", 87 "dragonfly", 88 "illumos", 89 "ios", 90 "js", 91 "wasip1", 92 "linux", 93 "android", 94 "solaris", 95 "freebsd", 96 "nacl", // keep; 97 "netbsd", 98 "openbsd", 99 "plan9", 100 "windows", 101 "aix", 102 } 103 104 // find reports the first index of p in l[0:n], or else -1. 105 func find(p string, l []string) int { 106 for i, s := range l { 107 if p == s { 108 return i 109 } 110 } 111 return -1 112 } 113 114 // xinit handles initialization of the various global state, like goroot and goarch. 115 func xinit() { 116 b := os.Getenv("GOROOT") 117 if b == "" { 118 fatalf("$GOROOT must be set") 119 } 120 goroot = filepath.Clean(b) 121 gorootBin = pathf("%s/bin", goroot) 122 123 // Don't run just 'go' because the build infrastructure 124 // runs cmd/dist inside go/bin often, and on Windows 125 // it will be found in the current directory and refuse to exec. 126 // All exec calls rewrite "go" into gorootBinGo. 127 gorootBinGo = pathf("%s/bin/go", goroot) 128 129 b = os.Getenv("GOROOT_FINAL") 130 if b == "" { 131 b = goroot 132 } 133 goroot_final = b 134 135 b = os.Getenv("GOOS") 136 if b == "" { 137 b = gohostos 138 } 139 goos = b 140 if find(goos, okgoos) < 0 { 141 fatalf("unknown $GOOS %s", goos) 142 } 143 144 b = os.Getenv("GOARM") 145 if b == "" { 146 b = xgetgoarm() 147 } 148 goarm = b 149 150 b = os.Getenv("GO386") 151 if b == "" { 152 b = "sse2" 153 } 154 go386 = b 155 156 b = os.Getenv("GOAMD64") 157 if b == "" { 158 b = "v1" 159 } 160 goamd64 = b 161 162 b = os.Getenv("GOMIPS") 163 if b == "" { 164 b = "hardfloat" 165 } 166 gomips = b 167 168 b = os.Getenv("GOMIPS64") 169 if b == "" { 170 b = "hardfloat" 171 } 172 gomips64 = b 173 174 b = os.Getenv("GOPPC64") 175 if b == "" { 176 b = "power8" 177 } 178 goppc64 = b 179 180 if p := pathf("%s/src/all.bash", goroot); !isfile(p) { 181 fatalf("$GOROOT is not set correctly or not exported\n"+ 182 "\tGOROOT=%s\n"+ 183 "\t%s does not exist", goroot, p) 184 } 185 186 b = os.Getenv("GOHOSTARCH") 187 if b != "" { 188 gohostarch = b 189 } 190 if find(gohostarch, okgoarch) < 0 { 191 fatalf("unknown $GOHOSTARCH %s", gohostarch) 192 } 193 194 b = os.Getenv("GOARCH") 195 if b == "" { 196 b = gohostarch 197 } 198 goarch = b 199 if find(goarch, okgoarch) < 0 { 200 fatalf("unknown $GOARCH %s", goarch) 201 } 202 203 b = os.Getenv("GO_EXTLINK_ENABLED") 204 if b != "" { 205 if b != "0" && b != "1" { 206 fatalf("unknown $GO_EXTLINK_ENABLED %s", b) 207 } 208 goextlinkenabled = b 209 } 210 211 goexperiment = os.Getenv("GOEXPERIMENT") 212 // TODO(mdempsky): Validate known experiments? 213 214 gogcflags = os.Getenv("BOOT_GO_GCFLAGS") 215 goldflags = os.Getenv("BOOT_GO_LDFLAGS") 216 217 defaultcc = compilerEnv("CC", "") 218 defaultcxx = compilerEnv("CXX", "") 219 220 b = os.Getenv("PKG_CONFIG") 221 if b == "" { 222 b = "pkg-config" 223 } 224 defaultpkgconfig = b 225 226 defaultldso = os.Getenv("GO_LDSO") 227 228 // For tools being invoked but also for os.ExpandEnv. 229 os.Setenv("GO386", go386) 230 os.Setenv("GOAMD64", goamd64) 231 os.Setenv("GOARCH", goarch) 232 os.Setenv("GOARM", goarm) 233 os.Setenv("GOHOSTARCH", gohostarch) 234 os.Setenv("GOHOSTOS", gohostos) 235 os.Setenv("GOOS", goos) 236 os.Setenv("GOMIPS", gomips) 237 os.Setenv("GOMIPS64", gomips64) 238 os.Setenv("GOPPC64", goppc64) 239 os.Setenv("GOROOT", goroot) 240 os.Setenv("GOROOT_FINAL", goroot_final) 241 242 // Set GOBIN to GOROOT/bin. The meaning of GOBIN has drifted over time 243 // (see https://go.dev/issue/3269, https://go.dev/cl/183058, 244 // https://go.dev/issue/31576). Since we want binaries installed by 'dist' to 245 // always go to GOROOT/bin anyway. 246 os.Setenv("GOBIN", gorootBin) 247 248 // Make the environment more predictable. 249 os.Setenv("LANG", "C") 250 os.Setenv("LANGUAGE", "en_US.UTF8") 251 os.Unsetenv("GO111MODULE") 252 os.Setenv("GOENV", "off") 253 os.Unsetenv("GOFLAGS") 254 os.Setenv("GOWORK", "off") 255 256 workdir = xworkdir() 257 if err := os.WriteFile(pathf("%s/go.mod", workdir), []byte("module bootstrap"), 0666); err != nil { 258 fatalf("cannot write stub go.mod: %s", err) 259 } 260 xatexit(rmworkdir) 261 262 tooldir = pathf("%s/pkg/tool/%s_%s", goroot, gohostos, gohostarch) 263 264 goversion := findgoversion() 265 isRelease = strings.HasPrefix(goversion, "release.") || strings.HasPrefix(goversion, "go") 266 } 267 268 // compilerEnv returns a map from "goos/goarch" to the 269 // compiler setting to use for that platform. 270 // The entry for key "" covers any goos/goarch not explicitly set in the map. 271 // For example, compilerEnv("CC", "gcc") returns the C compiler settings 272 // read from $CC, defaulting to gcc. 273 // 274 // The result is a map because additional environment variables 275 // can be set to change the compiler based on goos/goarch settings. 276 // The following applies to all envNames but CC is assumed to simplify 277 // the presentation. 278 // 279 // If no environment variables are set, we use def for all goos/goarch. 280 // $CC, if set, applies to all goos/goarch but is overridden by the following. 281 // $CC_FOR_TARGET, if set, applies to all goos/goarch except gohostos/gohostarch, 282 // but is overridden by the following. 283 // If gohostos=goos and gohostarch=goarch, then $CC_FOR_TARGET applies even for gohostos/gohostarch. 284 // $CC_FOR_goos_goarch, if set, applies only to goos/goarch. 285 func compilerEnv(envName, def string) map[string]string { 286 m := map[string]string{"": def} 287 288 if env := os.Getenv(envName); env != "" { 289 m[""] = env 290 } 291 if env := os.Getenv(envName + "_FOR_TARGET"); env != "" { 292 if gohostos != goos || gohostarch != goarch { 293 m[gohostos+"/"+gohostarch] = m[""] 294 } 295 m[""] = env 296 } 297 298 for _, goos := range okgoos { 299 for _, goarch := range okgoarch { 300 if env := os.Getenv(envName + "_FOR_" + goos + "_" + goarch); env != "" { 301 m[goos+"/"+goarch] = env 302 } 303 } 304 } 305 306 return m 307 } 308 309 // clangos lists the operating systems where we prefer clang to gcc. 310 var clangos = []string{ 311 "darwin", "ios", // macOS 10.9 and later require clang 312 "freebsd", // FreeBSD 10 and later do not ship gcc 313 "openbsd", // OpenBSD ships with GCC 4.2, which is now quite old. 314 } 315 316 // compilerEnvLookup returns the compiler settings for goos/goarch in map m. 317 // kind is "CC" or "CXX". 318 func compilerEnvLookup(kind string, m map[string]string, goos, goarch string) string { 319 if !needCC() { 320 return "" 321 } 322 if cc := m[goos+"/"+goarch]; cc != "" { 323 return cc 324 } 325 if cc := m[""]; cc != "" { 326 return cc 327 } 328 for _, os := range clangos { 329 if goos == os { 330 if kind == "CXX" { 331 return "clang++" 332 } 333 return "clang" 334 } 335 } 336 if kind == "CXX" { 337 return "g++" 338 } 339 return "gcc" 340 } 341 342 // rmworkdir deletes the work directory. 343 func rmworkdir() { 344 if vflag > 1 { 345 errprintf("rm -rf %s\n", workdir) 346 } 347 xremoveall(workdir) 348 } 349 350 // Remove trailing spaces. 351 func chomp(s string) string { 352 return strings.TrimRight(s, " \t\r\n") 353 } 354 355 // findgoversion determines the Go version to use in the version string. 356 // It also parses any other metadata found in the version file. 357 func findgoversion() string { 358 // The $GOROOT/VERSION file takes priority, for distributions 359 // without the source repo. 360 path := pathf("%s/VERSION", goroot) 361 if isfile(path) { 362 b := chomp(readfile(path)) 363 364 // Starting in Go 1.21 the VERSION file starts with the 365 // version on a line by itself but then can contain other 366 // metadata about the release, one item per line. 367 if i := strings.Index(b, "\n"); i >= 0 { 368 rest := b[i+1:] 369 b = chomp(b[:i]) 370 for _, line := range strings.Split(rest, "\n") { 371 f := strings.Fields(line) 372 if len(f) == 0 { 373 continue 374 } 375 switch f[0] { 376 default: 377 fatalf("VERSION: unexpected line: %s", line) 378 case "time": 379 if len(f) != 2 { 380 fatalf("VERSION: unexpected time line: %s", line) 381 } 382 _, err := time.Parse(time.RFC3339, f[1]) 383 if err != nil { 384 fatalf("VERSION: bad time: %s", err) 385 } 386 } 387 } 388 } 389 390 // Commands such as "dist version > VERSION" will cause 391 // the shell to create an empty VERSION file and set dist's 392 // stdout to its fd. dist in turn looks at VERSION and uses 393 // its content if available, which is empty at this point. 394 // Only use the VERSION file if it is non-empty. 395 if b != "" { 396 return b 397 } 398 } 399 400 // The $GOROOT/VERSION.cache file is a cache to avoid invoking 401 // git every time we run this command. Unlike VERSION, it gets 402 // deleted by the clean command. 403 path = pathf("%s/VERSION.cache", goroot) 404 if isfile(path) { 405 return chomp(readfile(path)) 406 } 407 408 // Show a nicer error message if this isn't a Git repo. 409 if !isGitRepo() { 410 fatalf("FAILED: not a Git repo; must put a VERSION file in $GOROOT") 411 } 412 413 // Otherwise, use Git. 414 // 415 // Include 1.x base version, hash, and date in the version. 416 // 417 // Note that we lightly parse github.com/go-asm/go/goversion/goversion.go to 418 // obtain the base version. We can't just import the package, 419 // because cmd/dist is built with a bootstrap GOROOT which could 420 // be an entirely different version of Go. We assume 421 // that the file contains "const Version = <Integer>". 422 goversionSource := readfile(pathf("%s/src/github.com/go-asm/go/goversion/goversion.go", goroot)) 423 m := regexp.MustCompile(`(?m)^const Version = (\d+)`).FindStringSubmatch(goversionSource) 424 if m == nil { 425 fatalf("github.com/go-asm/go/goversion/goversion.go does not contain 'const Version = ...'") 426 } 427 version := fmt.Sprintf("devel go1.%s-", m[1]) 428 version += chomp(run(goroot, CheckExit, "git", "log", "-n", "1", "--format=format:%h %cd", "HEAD")) 429 430 // Cache version. 431 writefile(version, path, 0) 432 433 return version 434 } 435 436 // isGitRepo reports whether the working directory is inside a Git repository. 437 func isGitRepo() bool { 438 // NB: simply checking the exit code of `git rev-parse --git-dir` would 439 // suffice here, but that requires deviating from the infrastructure 440 // provided by `run`. 441 gitDir := chomp(run(goroot, 0, "git", "rev-parse", "--git-dir")) 442 if !filepath.IsAbs(gitDir) { 443 gitDir = filepath.Join(goroot, gitDir) 444 } 445 return isdir(gitDir) 446 } 447 448 /* 449 * Initial tree setup. 450 */ 451 452 // The old tools that no longer live in $GOBIN or $GOROOT/bin. 453 var oldtool = []string{ 454 "5a", "5c", "5g", "5l", 455 "6a", "6c", "6g", "6l", 456 "8a", "8c", "8g", "8l", 457 "9a", "9c", "9g", "9l", 458 "6cov", 459 "6nm", 460 "6prof", 461 "cgo", 462 "ebnflint", 463 "goapi", 464 "gofix", 465 "goinstall", 466 "gomake", 467 "gopack", 468 "gopprof", 469 "gotest", 470 "gotype", 471 "govet", 472 "goyacc", 473 "quietgcc", 474 } 475 476 // Unreleased directories (relative to $GOROOT) that should 477 // not be in release branches. 478 var unreleased = []string{ 479 "src/cmd/newlink", 480 "src/cmd/objwriter", 481 "src/debug/goobj", 482 "src/old", 483 } 484 485 // setup sets up the tree for the initial build. 486 func setup() { 487 // Create bin directory. 488 if p := pathf("%s/bin", goroot); !isdir(p) { 489 xmkdir(p) 490 } 491 492 // Create package directory. 493 if p := pathf("%s/pkg", goroot); !isdir(p) { 494 xmkdir(p) 495 } 496 497 goosGoarch := pathf("%s/pkg/%s_%s", goroot, gohostos, gohostarch) 498 if rebuildall { 499 xremoveall(goosGoarch) 500 } 501 xmkdirall(goosGoarch) 502 xatexit(func() { 503 if files := xreaddir(goosGoarch); len(files) == 0 { 504 xremove(goosGoarch) 505 } 506 }) 507 508 if goos != gohostos || goarch != gohostarch { 509 p := pathf("%s/pkg/%s_%s", goroot, goos, goarch) 510 if rebuildall { 511 xremoveall(p) 512 } 513 xmkdirall(p) 514 } 515 516 // Create object directory. 517 // We used to use it for C objects. 518 // Now we use it for the build cache, to separate dist's cache 519 // from any other cache the user might have, and for the location 520 // to build the bootstrap versions of the standard library. 521 obj := pathf("%s/pkg/obj", goroot) 522 if !isdir(obj) { 523 xmkdir(obj) 524 } 525 xatexit(func() { xremove(obj) }) 526 527 // Create build cache directory. 528 objGobuild := pathf("%s/pkg/obj/go-build", goroot) 529 if rebuildall { 530 xremoveall(objGobuild) 531 } 532 xmkdirall(objGobuild) 533 xatexit(func() { xremoveall(objGobuild) }) 534 535 // Create directory for bootstrap versions of standard library .a files. 536 objGoBootstrap := pathf("%s/pkg/obj/go-bootstrap", goroot) 537 if rebuildall { 538 xremoveall(objGoBootstrap) 539 } 540 xmkdirall(objGoBootstrap) 541 xatexit(func() { xremoveall(objGoBootstrap) }) 542 543 // Create tool directory. 544 // We keep it in pkg/, just like the object directory above. 545 if rebuildall { 546 xremoveall(tooldir) 547 } 548 xmkdirall(tooldir) 549 550 // Remove tool binaries from before the tool/gohostos_gohostarch 551 xremoveall(pathf("%s/bin/tool", goroot)) 552 553 // Remove old pre-tool binaries. 554 for _, old := range oldtool { 555 xremove(pathf("%s/bin/%s", goroot, old)) 556 } 557 558 // Special release-specific setup. 559 if isRelease { 560 // Make sure release-excluded things are excluded. 561 for _, dir := range unreleased { 562 if p := pathf("%s/%s", goroot, dir); isdir(p) { 563 fatalf("%s should not exist in release build", p) 564 } 565 } 566 } 567 } 568 569 /* 570 * Tool building 571 */ 572 573 // mustLinkExternal is a copy of github.com/go-asm/go/platform.MustLinkExternal, 574 // duplicated here to avoid version skew in the MustLinkExternal function 575 // during bootstrapping. 576 func mustLinkExternal(goos, goarch string, cgoEnabled bool) bool { 577 if cgoEnabled { 578 switch goarch { 579 case "loong64", "mips", "mipsle", "mips64", "mips64le": 580 // Internally linking cgo is incomplete on some architectures. 581 // https://golang.org/issue/14449 582 return true 583 case "arm64": 584 if goos == "windows" { 585 // windows/arm64 internal linking is not implemented. 586 return true 587 } 588 case "ppc64": 589 // Big Endian PPC64 cgo internal linking is not implemented for aix or linux. 590 if goos == "aix" || goos == "linux" { 591 return true 592 } 593 } 594 595 switch goos { 596 case "android": 597 return true 598 case "dragonfly": 599 // It seems that on Dragonfly thread local storage is 600 // set up by the dynamic linker, so internal cgo linking 601 // doesn't work. Test case is "go test runtime/cgo". 602 return true 603 } 604 } 605 606 switch goos { 607 case "android": 608 if goarch != "arm64" { 609 return true 610 } 611 case "ios": 612 if goarch == "arm64" { 613 return true 614 } 615 } 616 return false 617 } 618 619 // depsuffix records the allowed suffixes for source files. 620 var depsuffix = []string{ 621 ".s", 622 ".go", 623 } 624 625 // gentab records how to generate some trivial files. 626 // Files listed here should also be listed in ../distpack/pack.go's srcArch.Remove list. 627 var gentab = []struct { 628 pkg string // Relative to $GOROOT/src 629 file string 630 gen func(dir, file string) 631 }{ 632 {"go/build", "zcgo.go", mkzcgo}, 633 {"github.com/go-asm/go/cmd/go/cfg", "zdefaultcc.go", mkzdefaultcc}, 634 {"runtime/github.com/go-asm/go/sys", "zversion.go", mkzversion}, 635 {"time/tzdata", "zzipdata.go", mktzdata}, 636 } 637 638 // installed maps from a dir name (as given to install) to a chan 639 // closed when the dir's package is installed. 640 var installed = make(map[string]chan struct{}) 641 var installedMu sync.Mutex 642 643 func install(dir string) { 644 <-startInstall(dir) 645 } 646 647 func startInstall(dir string) chan struct{} { 648 installedMu.Lock() 649 ch := installed[dir] 650 if ch == nil { 651 ch = make(chan struct{}) 652 installed[dir] = ch 653 go runInstall(dir, ch) 654 } 655 installedMu.Unlock() 656 return ch 657 } 658 659 // runInstall installs the library, package, or binary associated with pkg, 660 // which is relative to $GOROOT/src. 661 func runInstall(pkg string, ch chan struct{}) { 662 if pkg == "net" || pkg == "os/user" || pkg == "crypto/x509" { 663 fatalf("go_bootstrap cannot depend on cgo package %s", pkg) 664 } 665 666 defer close(ch) 667 668 if pkg == "unsafe" { 669 return 670 } 671 672 if vflag > 0 { 673 if goos != gohostos || goarch != gohostarch { 674 errprintf("%s (%s/%s)\n", pkg, goos, goarch) 675 } else { 676 errprintf("%s\n", pkg) 677 } 678 } 679 680 workdir := pathf("%s/%s", workdir, pkg) 681 xmkdirall(workdir) 682 683 var clean []string 684 defer func() { 685 for _, name := range clean { 686 xremove(name) 687 } 688 }() 689 690 // dir = full path to pkg. 691 dir := pathf("%s/src/%s", goroot, pkg) 692 name := filepath.Base(dir) 693 694 // ispkg predicts whether the package should be linked as a binary, based 695 // on the name. There should be no "main" packages in vendor, since 696 // 'go mod vendor' will only copy imported packages there. 697 ispkg := !strings.HasPrefix(pkg, "github.com/go-asm/go/cmd/") || strings.Contains(pkg, "/") || strings.Contains(pkg, "/vendor/") 698 699 // Start final link command line. 700 // Note: code below knows that link.p[targ] is the target. 701 var ( 702 link []string 703 targ int 704 ispackcmd bool 705 ) 706 if ispkg { 707 // Go library (package). 708 ispackcmd = true 709 link = []string{"pack", packagefile(pkg)} 710 targ = len(link) - 1 711 xmkdirall(filepath.Dir(link[targ])) 712 } else { 713 // Go command. 714 elem := name 715 if elem == "go" { 716 elem = "go_bootstrap" 717 } 718 link = []string{pathf("%s/link", tooldir)} 719 if goos == "android" { 720 link = append(link, "-buildmode=pie") 721 } 722 if goldflags != "" { 723 link = append(link, goldflags) 724 } 725 link = append(link, "-extld="+compilerEnvLookup("CC", defaultcc, goos, goarch)) 726 link = append(link, "-L="+pathf("%s/pkg/obj/go-bootstrap/%s_%s", goroot, goos, goarch)) 727 link = append(link, "-o", pathf("%s/%s%s", tooldir, elem, exe)) 728 targ = len(link) - 1 729 } 730 ttarg := mtime(link[targ]) 731 732 // Gather files that are sources for this target. 733 // Everything in that directory, and any target-specific 734 // additions. 735 files := xreaddir(dir) 736 737 // Remove files beginning with . or _, 738 // which are likely to be editor temporary files. 739 // This is the same heuristic build.ScanDir uses. 740 // There do exist real C files beginning with _, 741 // so limit that check to just Go files. 742 files = filter(files, func(p string) bool { 743 return !strings.HasPrefix(p, ".") && (!strings.HasPrefix(p, "_") || !strings.HasSuffix(p, ".go")) 744 }) 745 746 // Add generated files for this package. 747 for _, gt := range gentab { 748 if gt.pkg == pkg { 749 files = append(files, gt.file) 750 } 751 } 752 files = uniq(files) 753 754 // Convert to absolute paths. 755 for i, p := range files { 756 if !filepath.IsAbs(p) { 757 files[i] = pathf("%s/%s", dir, p) 758 } 759 } 760 761 // Is the target up-to-date? 762 var gofiles, sfiles []string 763 stale := rebuildall 764 files = filter(files, func(p string) bool { 765 for _, suf := range depsuffix { 766 if strings.HasSuffix(p, suf) { 767 goto ok 768 } 769 } 770 return false 771 ok: 772 t := mtime(p) 773 if !t.IsZero() && !strings.HasSuffix(p, ".a") && !shouldbuild(p, pkg) { 774 return false 775 } 776 if strings.HasSuffix(p, ".go") { 777 gofiles = append(gofiles, p) 778 } else if strings.HasSuffix(p, ".s") { 779 sfiles = append(sfiles, p) 780 } 781 if t.After(ttarg) { 782 stale = true 783 } 784 return true 785 }) 786 787 // If there are no files to compile, we're done. 788 if len(files) == 0 { 789 return 790 } 791 792 if !stale { 793 return 794 } 795 796 // For package runtime, copy some files into the work space. 797 if pkg == "runtime" { 798 xmkdirall(pathf("%s/pkg/include", goroot)) 799 // For use by assembly and C files. 800 copyfile(pathf("%s/pkg/include/textflag.h", goroot), 801 pathf("%s/src/runtime/textflag.h", goroot), 0) 802 copyfile(pathf("%s/pkg/include/funcdata.h", goroot), 803 pathf("%s/src/runtime/funcdata.h", goroot), 0) 804 copyfile(pathf("%s/pkg/include/asm_ppc64x.h", goroot), 805 pathf("%s/src/runtime/asm_ppc64x.h", goroot), 0) 806 copyfile(pathf("%s/pkg/include/asm_amd64.h", goroot), 807 pathf("%s/src/runtime/asm_amd64.h", goroot), 0) 808 } 809 810 // Generate any missing files; regenerate existing ones. 811 for _, gt := range gentab { 812 if gt.pkg != pkg { 813 continue 814 } 815 p := pathf("%s/%s", dir, gt.file) 816 if vflag > 1 { 817 errprintf("generate %s\n", p) 818 } 819 gt.gen(dir, p) 820 // Do not add generated file to clean list. 821 // In runtime, we want to be able to 822 // build the package with the go tool, 823 // and it assumes these generated files already 824 // exist (it does not know how to build them). 825 // The 'clean' command can remove 826 // the generated files. 827 } 828 829 // Resolve imported packages to actual package paths. 830 // Make sure they're installed. 831 importMap := make(map[string]string) 832 for _, p := range gofiles { 833 for _, imp := range readimports(p) { 834 if imp == "C" { 835 fatalf("%s imports C", p) 836 } 837 importMap[imp] = resolveVendor(imp, dir) 838 } 839 } 840 sortedImports := make([]string, 0, len(importMap)) 841 for imp := range importMap { 842 sortedImports = append(sortedImports, imp) 843 } 844 sort.Strings(sortedImports) 845 846 for _, dep := range importMap { 847 if dep == "C" { 848 fatalf("%s imports C", pkg) 849 } 850 startInstall(dep) 851 } 852 for _, dep := range importMap { 853 install(dep) 854 } 855 856 if goos != gohostos || goarch != gohostarch { 857 // We've generated the right files; the go command can do the build. 858 if vflag > 1 { 859 errprintf("skip build for cross-compile %s\n", pkg) 860 } 861 return 862 } 863 864 asmArgs := []string{ 865 pathf("%s/asm", tooldir), 866 "-I", workdir, 867 "-I", pathf("%s/pkg/include", goroot), 868 "-D", "GOOS_" + goos, 869 "-D", "GOARCH_" + goarch, 870 "-D", "GOOS_GOARCH_" + goos + "_" + goarch, 871 "-p", pkg, 872 } 873 if goarch == "mips" || goarch == "mipsle" { 874 // Define GOMIPS_value from gomips. 875 asmArgs = append(asmArgs, "-D", "GOMIPS_"+gomips) 876 } 877 if goarch == "mips64" || goarch == "mips64le" { 878 // Define GOMIPS64_value from gomips64. 879 asmArgs = append(asmArgs, "-D", "GOMIPS64_"+gomips64) 880 } 881 if goarch == "ppc64" || goarch == "ppc64le" { 882 // We treat each powerpc version as a superset of functionality. 883 switch goppc64 { 884 case "power10": 885 asmArgs = append(asmArgs, "-D", "GOPPC64_power10") 886 fallthrough 887 case "power9": 888 asmArgs = append(asmArgs, "-D", "GOPPC64_power9") 889 fallthrough 890 default: // This should always be power8. 891 asmArgs = append(asmArgs, "-D", "GOPPC64_power8") 892 } 893 } 894 goasmh := pathf("%s/go_asm.h", workdir) 895 896 // Collect symabis from assembly code. 897 var symabis string 898 if len(sfiles) > 0 { 899 symabis = pathf("%s/symabis", workdir) 900 var wg sync.WaitGroup 901 asmabis := append(asmArgs[:len(asmArgs):len(asmArgs)], "-gensymabis", "-o", symabis) 902 asmabis = append(asmabis, sfiles...) 903 if err := os.WriteFile(goasmh, nil, 0666); err != nil { 904 fatalf("cannot write empty go_asm.h: %s", err) 905 } 906 bgrun(&wg, dir, asmabis...) 907 bgwait(&wg) 908 } 909 910 // Build an importcfg file for the compiler. 911 buf := &bytes.Buffer{} 912 for _, imp := range sortedImports { 913 if imp == "unsafe" { 914 continue 915 } 916 dep := importMap[imp] 917 if imp != dep { 918 fmt.Fprintf(buf, "importmap %s=%s\n", imp, dep) 919 } 920 fmt.Fprintf(buf, "packagefile %s=%s\n", dep, packagefile(dep)) 921 } 922 importcfg := pathf("%s/importcfg", workdir) 923 if err := os.WriteFile(importcfg, buf.Bytes(), 0666); err != nil { 924 fatalf("cannot write importcfg file: %v", err) 925 } 926 927 var archive string 928 // The next loop will compile individual non-Go files. 929 // Hand the Go files to the compiler en masse. 930 // For packages containing assembly, this writes go_asm.h, which 931 // the assembly files will need. 932 pkgName := pkg 933 if strings.HasPrefix(pkg, "cmd/") && strings.Count(pkg, "/") == 1 { 934 pkgName = "main" 935 } 936 b := pathf("%s/_go_.a", workdir) 937 clean = append(clean, b) 938 if !ispackcmd { 939 link = append(link, b) 940 } else { 941 archive = b 942 } 943 944 // Compile Go code. 945 compile := []string{pathf("%s/compile", tooldir), "-std", "-pack", "-o", b, "-p", pkgName, "-importcfg", importcfg} 946 if gogcflags != "" { 947 compile = append(compile, strings.Fields(gogcflags)...) 948 } 949 if len(sfiles) > 0 { 950 compile = append(compile, "-asmhdr", goasmh) 951 } 952 if symabis != "" { 953 compile = append(compile, "-symabis", symabis) 954 } 955 if goos == "android" { 956 compile = append(compile, "-shared") 957 } 958 959 compile = append(compile, gofiles...) 960 var wg sync.WaitGroup 961 // We use bgrun and immediately wait for it instead of calling run() synchronously. 962 // This executes all jobs through the bgwork channel and allows the process 963 // to exit cleanly in case an error occurs. 964 bgrun(&wg, dir, compile...) 965 bgwait(&wg) 966 967 // Compile the files. 968 for _, p := range sfiles { 969 // Assembly file for a Go package. 970 compile := asmArgs[:len(asmArgs):len(asmArgs)] 971 972 doclean := true 973 b := pathf("%s/%s", workdir, filepath.Base(p)) 974 975 // Change the last character of the output file (which was c or s). 976 b = b[:len(b)-1] + "o" 977 compile = append(compile, "-o", b, p) 978 bgrun(&wg, dir, compile...) 979 980 link = append(link, b) 981 if doclean { 982 clean = append(clean, b) 983 } 984 } 985 bgwait(&wg) 986 987 if ispackcmd { 988 xremove(link[targ]) 989 dopack(link[targ], archive, link[targ+1:]) 990 return 991 } 992 993 // Remove target before writing it. 994 xremove(link[targ]) 995 bgrun(&wg, "", link...) 996 bgwait(&wg) 997 } 998 999 // packagefile returns the path to a compiled .a file for the given package 1000 // path. Paths may need to be resolved with resolveVendor first. 1001 func packagefile(pkg string) string { 1002 return pathf("%s/pkg/obj/go-bootstrap/%s_%s/%s.a", goroot, goos, goarch, pkg) 1003 } 1004 1005 // unixOS is the set of GOOS values matched by the "unix" build tag. 1006 // This is the same list as in go/build/syslist.go and 1007 // github.com/go-asm/go/cmd/go/imports/build.go. 1008 var unixOS = map[string]bool{ 1009 "aix": true, 1010 "android": true, 1011 "darwin": true, 1012 "dragonfly": true, 1013 "freebsd": true, 1014 "hurd": true, 1015 "illumos": true, 1016 "ios": true, 1017 "linux": true, 1018 "netbsd": true, 1019 "openbsd": true, 1020 "solaris": true, 1021 } 1022 1023 // matchtag reports whether the tag matches this build. 1024 func matchtag(tag string) bool { 1025 switch tag { 1026 case "gc", "cmd_go_bootstrap", "go1.1": 1027 return true 1028 case "linux": 1029 return goos == "linux" || goos == "android" 1030 case "solaris": 1031 return goos == "solaris" || goos == "illumos" 1032 case "darwin": 1033 return goos == "darwin" || goos == "ios" 1034 case goos, goarch: 1035 return true 1036 case "unix": 1037 return unixOS[goos] 1038 default: 1039 return false 1040 } 1041 } 1042 1043 // shouldbuild reports whether we should build this file. 1044 // It applies the same rules that are used with context tags 1045 // in package go/build, except it's less picky about the order 1046 // of GOOS and GOARCH. 1047 // We also allow the special tag cmd_go_bootstrap. 1048 // See ../go/bootstrap.go and package go/build. 1049 func shouldbuild(file, pkg string) bool { 1050 // Check file name for GOOS or GOARCH. 1051 name := filepath.Base(file) 1052 excluded := func(list []string, ok string) bool { 1053 for _, x := range list { 1054 if x == ok || (ok == "android" && x == "linux") || (ok == "illumos" && x == "solaris") || (ok == "ios" && x == "darwin") { 1055 continue 1056 } 1057 i := strings.Index(name, x) 1058 if i <= 0 || name[i-1] != '_' { 1059 continue 1060 } 1061 i += len(x) 1062 if i == len(name) || name[i] == '.' || name[i] == '_' { 1063 return true 1064 } 1065 } 1066 return false 1067 } 1068 if excluded(okgoos, goos) || excluded(okgoarch, goarch) { 1069 return false 1070 } 1071 1072 // Omit test files. 1073 if strings.Contains(name, "_test") { 1074 return false 1075 } 1076 1077 // Check file contents for //go:build lines. 1078 for _, p := range strings.Split(readfile(file), "\n") { 1079 p = strings.TrimSpace(p) 1080 if p == "" { 1081 continue 1082 } 1083 code := p 1084 i := strings.Index(code, "//") 1085 if i > 0 { 1086 code = strings.TrimSpace(code[:i]) 1087 } 1088 if code == "package documentation" { 1089 return false 1090 } 1091 if code == "package dist" && pkg != "cmd/go" && pkg != "cmd/cgo" { 1092 return false 1093 } 1094 if !strings.HasPrefix(p, "//") { 1095 break 1096 } 1097 if strings.HasPrefix(p, "//go:build ") { 1098 matched, err := matchexpr(p[len("//go:build "):]) 1099 if err != nil { 1100 errprintf("%s: %v", file, err) 1101 } 1102 return matched 1103 } 1104 } 1105 1106 return true 1107 } 1108 1109 // copyfile copies the file src to dst, via memory (so only good for small files). 1110 func copyfile(dst, src string, flag int) { 1111 if vflag > 1 { 1112 errprintf("cp %s %s\n", src, dst) 1113 } 1114 writefile(readfile(src), dst, flag) 1115 } 1116 1117 // dopack copies the package src to dst, 1118 // appending the files listed in extra. 1119 // The archive format is the traditional Unix ar format. 1120 func dopack(dst, src string, extra []string) { 1121 bdst := bytes.NewBufferString(readfile(src)) 1122 for _, file := range extra { 1123 b := readfile(file) 1124 // find last path element for archive member name 1125 i := strings.LastIndex(file, "/") + 1 1126 j := strings.LastIndex(file, `\`) + 1 1127 if i < j { 1128 i = j 1129 } 1130 fmt.Fprintf(bdst, "%-16.16s%-12d%-6d%-6d%-8o%-10d`\n", file[i:], 0, 0, 0, 0644, len(b)) 1131 bdst.WriteString(b) 1132 if len(b)&1 != 0 { 1133 bdst.WriteByte(0) 1134 } 1135 } 1136 writefile(bdst.String(), dst, 0) 1137 } 1138 1139 func clean() { 1140 generated := []byte(generatedHeader) 1141 1142 // Remove generated source files. 1143 filepath.WalkDir(pathf("%s/src", goroot), func(path string, d fs.DirEntry, err error) error { 1144 switch { 1145 case err != nil: 1146 // ignore 1147 case d.IsDir() && (d.Name() == "vendor" || d.Name() == "testdata"): 1148 return filepath.SkipDir 1149 case d.IsDir() && d.Name() != "dist": 1150 // Remove generated binary named for directory, but not dist out from under us. 1151 exe := filepath.Join(path, d.Name()) 1152 if info, err := os.Stat(exe); err == nil && !info.IsDir() { 1153 xremove(exe) 1154 } 1155 xremove(exe + ".exe") 1156 case !d.IsDir() && strings.HasPrefix(d.Name(), "z"): 1157 // Remove generated file, identified by marker string. 1158 head := make([]byte, 512) 1159 if f, err := os.Open(path); err == nil { 1160 io.ReadFull(f, head) 1161 f.Close() 1162 } 1163 if bytes.HasPrefix(head, generated) { 1164 xremove(path) 1165 } 1166 } 1167 return nil 1168 }) 1169 1170 if rebuildall { 1171 // Remove object tree. 1172 xremoveall(pathf("%s/pkg/obj/%s_%s", goroot, gohostos, gohostarch)) 1173 1174 // Remove installed packages and tools. 1175 xremoveall(pathf("%s/pkg/%s_%s", goroot, gohostos, gohostarch)) 1176 xremoveall(pathf("%s/pkg/%s_%s", goroot, goos, goarch)) 1177 xremoveall(pathf("%s/pkg/%s_%s_race", goroot, gohostos, gohostarch)) 1178 xremoveall(pathf("%s/pkg/%s_%s_race", goroot, goos, goarch)) 1179 xremoveall(tooldir) 1180 1181 // Remove cached version info. 1182 xremove(pathf("%s/VERSION.cache", goroot)) 1183 1184 // Remove distribution packages. 1185 xremoveall(pathf("%s/pkg/distpack", goroot)) 1186 } 1187 } 1188 1189 /* 1190 * command implementations 1191 */ 1192 1193 // The env command prints the default environment. 1194 func cmdenv() { 1195 path := flag.Bool("p", false, "emit updated PATH") 1196 plan9 := flag.Bool("9", gohostos == "plan9", "emit plan 9 syntax") 1197 windows := flag.Bool("w", gohostos == "windows", "emit windows syntax") 1198 xflagparse(0) 1199 1200 format := "%s=\"%s\";\n" // Include ; to separate variables when 'dist env' output is used with eval. 1201 switch { 1202 case *plan9: 1203 format = "%s='%s'\n" 1204 case *windows: 1205 format = "set %s=%s\r\n" 1206 } 1207 1208 xprintf(format, "GO111MODULE", "") 1209 xprintf(format, "GOARCH", goarch) 1210 xprintf(format, "GOBIN", gorootBin) 1211 xprintf(format, "GODEBUG", os.Getenv("GODEBUG")) 1212 xprintf(format, "GOENV", "off") 1213 xprintf(format, "GOFLAGS", "") 1214 xprintf(format, "GOHOSTARCH", gohostarch) 1215 xprintf(format, "GOHOSTOS", gohostos) 1216 xprintf(format, "GOOS", goos) 1217 xprintf(format, "GOPROXY", os.Getenv("GOPROXY")) 1218 xprintf(format, "GOROOT", goroot) 1219 xprintf(format, "GOTMPDIR", os.Getenv("GOTMPDIR")) 1220 xprintf(format, "GOTOOLDIR", tooldir) 1221 if goarch == "arm" { 1222 xprintf(format, "GOARM", goarm) 1223 } 1224 if goarch == "386" { 1225 xprintf(format, "GO386", go386) 1226 } 1227 if goarch == "amd64" { 1228 xprintf(format, "GOAMD64", goamd64) 1229 } 1230 if goarch == "mips" || goarch == "mipsle" { 1231 xprintf(format, "GOMIPS", gomips) 1232 } 1233 if goarch == "mips64" || goarch == "mips64le" { 1234 xprintf(format, "GOMIPS64", gomips64) 1235 } 1236 if goarch == "ppc64" || goarch == "ppc64le" { 1237 xprintf(format, "GOPPC64", goppc64) 1238 } 1239 xprintf(format, "GOWORK", "off") 1240 1241 if *path { 1242 sep := ":" 1243 if gohostos == "windows" { 1244 sep = ";" 1245 } 1246 xprintf(format, "PATH", fmt.Sprintf("%s%s%s", gorootBin, sep, os.Getenv("PATH"))) 1247 1248 // Also include $DIST_UNMODIFIED_PATH with the original $PATH 1249 // for the internal needs of "dist banner", along with export 1250 // so that it reaches the dist process. See its comment below. 1251 var exportFormat string 1252 if !*windows && !*plan9 { 1253 exportFormat = "export " + format 1254 } else { 1255 exportFormat = format 1256 } 1257 xprintf(exportFormat, "DIST_UNMODIFIED_PATH", os.Getenv("PATH")) 1258 } 1259 } 1260 1261 var ( 1262 timeLogEnabled = os.Getenv("GOBUILDTIMELOGFILE") != "" 1263 timeLogMu sync.Mutex 1264 timeLogFile *os.File 1265 timeLogStart time.Time 1266 ) 1267 1268 func timelog(op, name string) { 1269 if !timeLogEnabled { 1270 return 1271 } 1272 timeLogMu.Lock() 1273 defer timeLogMu.Unlock() 1274 if timeLogFile == nil { 1275 f, err := os.OpenFile(os.Getenv("GOBUILDTIMELOGFILE"), os.O_RDWR|os.O_APPEND, 0666) 1276 if err != nil { 1277 log.Fatal(err) 1278 } 1279 buf := make([]byte, 100) 1280 n, _ := f.Read(buf) 1281 s := string(buf[:n]) 1282 if i := strings.Index(s, "\n"); i >= 0 { 1283 s = s[:i] 1284 } 1285 i := strings.Index(s, " start") 1286 if i < 0 { 1287 log.Fatalf("time log %s does not begin with start line", os.Getenv("GOBUILDTIMELOGFILE")) 1288 } 1289 t, err := time.Parse(time.UnixDate, s[:i]) 1290 if err != nil { 1291 log.Fatalf("cannot parse time log line %q: %v", s, err) 1292 } 1293 timeLogStart = t 1294 timeLogFile = f 1295 } 1296 t := time.Now() 1297 fmt.Fprintf(timeLogFile, "%s %+.1fs %s %s\n", t.Format(time.UnixDate), t.Sub(timeLogStart).Seconds(), op, name) 1298 } 1299 1300 // toolenv returns the environment to use when building commands in cmd. 1301 // 1302 // This is a function instead of a variable because the exact toolenv depends 1303 // on the GOOS and GOARCH, and (at least for now) those are modified in place 1304 // to switch between the host and target configurations when cross-compiling. 1305 func toolenv() []string { 1306 var env []string 1307 if !mustLinkExternal(goos, goarch, false) { 1308 // Unless the platform requires external linking, 1309 // we disable cgo to get static binaries for cmd/go and cmd/pprof, 1310 // so that they work on systems without the same dynamic libraries 1311 // as the original build system. 1312 env = append(env, "CGO_ENABLED=0") 1313 } 1314 if isRelease || os.Getenv("GO_BUILDER_NAME") != "" { 1315 // Add -trimpath for reproducible builds of releases. 1316 // Include builders so that -trimpath is well-tested ahead of releases. 1317 // Do not include local development, so that people working in the 1318 // main branch for day-to-day work on the Go toolchain itself can 1319 // still have full paths for stack traces for compiler crashes and the like. 1320 env = append(env, "GOFLAGS=-trimpath -ldflags=-w -gcflags=cmd/...=-dwarf=false") 1321 } 1322 return env 1323 } 1324 1325 var toolchain = []string{"cmd/asm", "cmd/cgo", "cmd/compile", "cmd/link"} 1326 1327 // The bootstrap command runs a build from scratch, 1328 // stopping at having installed the go_bootstrap command. 1329 // 1330 // WARNING: This command runs after cmd/dist is built with the Go bootstrap toolchain. 1331 // It rebuilds and installs cmd/dist with the new toolchain, so other 1332 // commands (like "go tool dist test" in run.bash) can rely on bug fixes 1333 // made since the Go bootstrap version, but this function cannot. 1334 func cmdbootstrap() { 1335 timelog("start", "dist bootstrap") 1336 defer timelog("end", "dist bootstrap") 1337 1338 var debug, distpack, force, noBanner, noClean bool 1339 flag.BoolVar(&rebuildall, "a", rebuildall, "rebuild all") 1340 flag.BoolVar(&debug, "d", debug, "enable debugging of bootstrap process") 1341 flag.BoolVar(&distpack, "distpack", distpack, "write distribution files to pkg/distpack") 1342 flag.BoolVar(&force, "force", force, "build even if the port is marked as broken") 1343 flag.BoolVar(&noBanner, "no-banner", noBanner, "do not print banner") 1344 flag.BoolVar(&noClean, "no-clean", noClean, "print deprecation warning") 1345 1346 xflagparse(0) 1347 1348 if noClean { 1349 xprintf("warning: --no-clean is deprecated and has no effect; use 'go install std cmd' instead\n") 1350 } 1351 1352 // Don't build broken ports by default. 1353 if broken[goos+"/"+goarch] && !force { 1354 fatalf("build stopped because the port %s/%s is marked as broken\n\n"+ 1355 "Use the -force flag to build anyway.\n", goos, goarch) 1356 } 1357 1358 // Set GOPATH to an internal directory. We shouldn't actually 1359 // need to store files here, since the toolchain won't 1360 // depend on modules outside of vendor directories, but if 1361 // GOPATH points somewhere else (e.g., to GOROOT), the 1362 // go tool may complain. 1363 os.Setenv("GOPATH", pathf("%s/pkg/obj/gopath", goroot)) 1364 1365 // Use a build cache separate from the default user one. 1366 // Also one that will be wiped out during startup, so that 1367 // make.bash really does start from a clean slate. 1368 oldgocache = os.Getenv("GOCACHE") 1369 os.Setenv("GOCACHE", pathf("%s/pkg/obj/go-build", goroot)) 1370 1371 // Disable GOEXPERIMENT when building toolchain1 and 1372 // go_bootstrap. We don't need any experiments for the 1373 // bootstrap toolchain, and this lets us avoid duplicating the 1374 // GOEXPERIMENT-related build logic from cmd/go here. If the 1375 // bootstrap toolchain is < Go 1.17, it will ignore this 1376 // anyway since GOEXPERIMENT is baked in; otherwise it will 1377 // pick it up from the environment we set here. Once we're 1378 // using toolchain1 with dist as the build system, we need to 1379 // override this to keep the experiments assumed by the 1380 // toolchain and by dist consistent. Once go_bootstrap takes 1381 // over the build process, we'll set this back to the original 1382 // GOEXPERIMENT. 1383 os.Setenv("GOEXPERIMENT", "none") 1384 1385 if debug { 1386 // cmd/buildid is used in debug mode. 1387 toolchain = append(toolchain, "cmd/buildid") 1388 } 1389 1390 if isdir(pathf("%s/src/pkg", goroot)) { 1391 fatalf("\n\n"+ 1392 "The Go package sources have moved to $GOROOT/src.\n"+ 1393 "*** %s still exists. ***\n"+ 1394 "It probably contains stale files that may confuse the build.\n"+ 1395 "Please (check what's there and) remove it and try again.\n"+ 1396 "See https://golang.org/s/go14nopkg\n", 1397 pathf("%s/src/pkg", goroot)) 1398 } 1399 1400 if rebuildall { 1401 clean() 1402 } 1403 1404 setup() 1405 1406 timelog("build", "toolchain1") 1407 checkCC() 1408 bootstrapBuildTools() 1409 1410 // Remember old content of $GOROOT/bin for comparison below. 1411 oldBinFiles, err := filepath.Glob(pathf("%s/bin/*", goroot)) 1412 if err != nil { 1413 fatalf("glob: %v", err) 1414 } 1415 1416 // For the main bootstrap, building for host os/arch. 1417 oldgoos = goos 1418 oldgoarch = goarch 1419 goos = gohostos 1420 goarch = gohostarch 1421 os.Setenv("GOHOSTARCH", gohostarch) 1422 os.Setenv("GOHOSTOS", gohostos) 1423 os.Setenv("GOARCH", goarch) 1424 os.Setenv("GOOS", goos) 1425 1426 timelog("build", "go_bootstrap") 1427 xprintf("Building Go bootstrap cmd/go (go_bootstrap) using Go toolchain1.\n") 1428 install("runtime") // dependency not visible in sources; also sets up textflag.h 1429 install("time/tzdata") // no dependency in sources; creates generated file 1430 install("cmd/go") 1431 if vflag > 0 { 1432 xprintf("\n") 1433 } 1434 1435 gogcflags = os.Getenv("GO_GCFLAGS") // we were using $BOOT_GO_GCFLAGS until now 1436 setNoOpt() 1437 goldflags = os.Getenv("GO_LDFLAGS") // we were using $BOOT_GO_LDFLAGS until now 1438 goBootstrap := pathf("%s/go_bootstrap", tooldir) 1439 if debug { 1440 run("", ShowOutput|CheckExit, pathf("%s/compile", tooldir), "-V=full") 1441 copyfile(pathf("%s/compile1", tooldir), pathf("%s/compile", tooldir), writeExec) 1442 } 1443 1444 // To recap, so far we have built the new toolchain 1445 // (cmd/asm, cmd/cgo, cmd/compile, cmd/link) 1446 // using the Go bootstrap toolchain and go command. 1447 // Then we built the new go command (as go_bootstrap) 1448 // using the new toolchain and our own build logic (above). 1449 // 1450 // toolchain1 = mk(new toolchain, go1.17 toolchain, go1.17 cmd/go) 1451 // go_bootstrap = mk(new cmd/go, toolchain1, cmd/dist) 1452 // 1453 // The toolchain1 we built earlier is built from the new sources, 1454 // but because it was built using cmd/go it has no build IDs. 1455 // The eventually installed toolchain needs build IDs, so we need 1456 // to do another round: 1457 // 1458 // toolchain2 = mk(new toolchain, toolchain1, go_bootstrap) 1459 // 1460 timelog("build", "toolchain2") 1461 if vflag > 0 { 1462 xprintf("\n") 1463 } 1464 xprintf("Building Go toolchain2 using go_bootstrap and Go toolchain1.\n") 1465 os.Setenv("CC", compilerEnvLookup("CC", defaultcc, goos, goarch)) 1466 // Now that cmd/go is in charge of the build process, enable GOEXPERIMENT. 1467 os.Setenv("GOEXPERIMENT", goexperiment) 1468 // No need to enable PGO for toolchain2. 1469 goInstall(toolenv(), goBootstrap, append([]string{"-pgo=off"}, toolchain...)...) 1470 if debug { 1471 run("", ShowOutput|CheckExit, pathf("%s/compile", tooldir), "-V=full") 1472 copyfile(pathf("%s/compile2", tooldir), pathf("%s/compile", tooldir), writeExec) 1473 } 1474 1475 // Toolchain2 should be semantically equivalent to toolchain1, 1476 // but it was built using the newly built compiler instead of the Go bootstrap compiler, 1477 // so it should at the least run faster. Also, toolchain1 had no build IDs 1478 // in the binaries, while toolchain2 does. In non-release builds, the 1479 // toolchain's build IDs feed into constructing the build IDs of built targets, 1480 // so in non-release builds, everything now looks out-of-date due to 1481 // toolchain2 having build IDs - that is, due to the go command seeing 1482 // that there are new compilers. In release builds, the toolchain's reported 1483 // version is used in place of the build ID, and the go command does not 1484 // see that change from toolchain1 to toolchain2, so in release builds, 1485 // nothing looks out of date. 1486 // To keep the behavior the same in both non-release and release builds, 1487 // we force-install everything here. 1488 // 1489 // toolchain3 = mk(new toolchain, toolchain2, go_bootstrap) 1490 // 1491 timelog("build", "toolchain3") 1492 if vflag > 0 { 1493 xprintf("\n") 1494 } 1495 xprintf("Building Go toolchain3 using go_bootstrap and Go toolchain2.\n") 1496 goInstall(toolenv(), goBootstrap, append([]string{"-a"}, toolchain...)...) 1497 if debug { 1498 run("", ShowOutput|CheckExit, pathf("%s/compile", tooldir), "-V=full") 1499 copyfile(pathf("%s/compile3", tooldir), pathf("%s/compile", tooldir), writeExec) 1500 } 1501 1502 // Now that toolchain3 has been built from scratch, its compiler and linker 1503 // should have accurate build IDs suitable for caching. 1504 // Now prime the build cache with the rest of the standard library for 1505 // testing, and so that the user can run 'go install std cmd' to quickly 1506 // iterate on local changes without waiting for a full rebuild. 1507 if _, err := os.Stat(pathf("%s/VERSION", goroot)); err == nil { 1508 // If we have a VERSION file, then we use the Go version 1509 // instead of build IDs as a cache key, and there is no guarantee 1510 // that code hasn't changed since the last time we ran a build 1511 // with this exact VERSION file (especially if someone is working 1512 // on a release branch). We must not fall back to the shared build cache 1513 // in this case. Leave $GOCACHE alone. 1514 } else { 1515 os.Setenv("GOCACHE", oldgocache) 1516 } 1517 1518 if goos == oldgoos && goarch == oldgoarch { 1519 // Common case - not setting up for cross-compilation. 1520 timelog("build", "toolchain") 1521 if vflag > 0 { 1522 xprintf("\n") 1523 } 1524 xprintf("Building packages and commands for %s/%s.\n", goos, goarch) 1525 } else { 1526 // GOOS/GOARCH does not match GOHOSTOS/GOHOSTARCH. 1527 // Finish GOHOSTOS/GOHOSTARCH installation and then 1528 // run GOOS/GOARCH installation. 1529 timelog("build", "host toolchain") 1530 if vflag > 0 { 1531 xprintf("\n") 1532 } 1533 xprintf("Building commands for host, %s/%s.\n", goos, goarch) 1534 goInstall(toolenv(), goBootstrap, "cmd") 1535 checkNotStale(toolenv(), goBootstrap, "cmd") 1536 checkNotStale(toolenv(), gorootBinGo, "cmd") 1537 1538 timelog("build", "target toolchain") 1539 if vflag > 0 { 1540 xprintf("\n") 1541 } 1542 goos = oldgoos 1543 goarch = oldgoarch 1544 os.Setenv("GOOS", goos) 1545 os.Setenv("GOARCH", goarch) 1546 os.Setenv("CC", compilerEnvLookup("CC", defaultcc, goos, goarch)) 1547 xprintf("Building packages and commands for target, %s/%s.\n", goos, goarch) 1548 } 1549 goInstall(nil, goBootstrap, "std") 1550 goInstall(toolenv(), goBootstrap, "cmd") 1551 checkNotStale(toolenv(), goBootstrap, toolchain...) 1552 checkNotStale(nil, goBootstrap, "std") 1553 checkNotStale(toolenv(), goBootstrap, "cmd") 1554 checkNotStale(nil, gorootBinGo, "std") 1555 checkNotStale(toolenv(), gorootBinGo, "cmd") 1556 if debug { 1557 run("", ShowOutput|CheckExit, pathf("%s/compile", tooldir), "-V=full") 1558 checkNotStale(toolenv(), goBootstrap, toolchain...) 1559 copyfile(pathf("%s/compile4", tooldir), pathf("%s/compile", tooldir), writeExec) 1560 } 1561 1562 // Check that there are no new files in $GOROOT/bin other than 1563 // go and gofmt and $GOOS_$GOARCH (target bin when cross-compiling). 1564 binFiles, err := filepath.Glob(pathf("%s/bin/*", goroot)) 1565 if err != nil { 1566 fatalf("glob: %v", err) 1567 } 1568 1569 ok := map[string]bool{} 1570 for _, f := range oldBinFiles { 1571 ok[f] = true 1572 } 1573 for _, f := range binFiles { 1574 if gohostos == "darwin" && filepath.Base(f) == ".DS_Store" { 1575 continue // unfortunate but not unexpected 1576 } 1577 elem := strings.TrimSuffix(filepath.Base(f), ".exe") 1578 if !ok[f] && elem != "go" && elem != "gofmt" && elem != goos+"_"+goarch { 1579 fatalf("unexpected new file in $GOROOT/bin: %s", elem) 1580 } 1581 } 1582 1583 // Remove go_bootstrap now that we're done. 1584 xremove(pathf("%s/go_bootstrap"+exe, tooldir)) 1585 1586 if goos == "android" { 1587 // Make sure the exec wrapper will sync a fresh $GOROOT to the device. 1588 xremove(pathf("%s/go_android_exec-adb-sync-status", os.TempDir())) 1589 } 1590 1591 if wrapperPath := wrapperPathFor(goos, goarch); wrapperPath != "" { 1592 oldcc := os.Getenv("CC") 1593 os.Setenv("GOOS", gohostos) 1594 os.Setenv("GOARCH", gohostarch) 1595 os.Setenv("CC", compilerEnvLookup("CC", defaultcc, gohostos, gohostarch)) 1596 goCmd(nil, gorootBinGo, "build", "-o", pathf("%s/go_%s_%s_exec%s", gorootBin, goos, goarch, exe), wrapperPath) 1597 // Restore environment. 1598 // TODO(elias.naur): support environment variables in goCmd? 1599 os.Setenv("GOOS", goos) 1600 os.Setenv("GOARCH", goarch) 1601 os.Setenv("CC", oldcc) 1602 } 1603 1604 if distpack { 1605 xprintf("Packaging archives for %s/%s.\n", goos, goarch) 1606 run("", ShowOutput|CheckExit, pathf("%s/distpack", tooldir)) 1607 } 1608 1609 // Print trailing banner unless instructed otherwise. 1610 if !noBanner { 1611 banner() 1612 } 1613 } 1614 1615 func wrapperPathFor(goos, goarch string) string { 1616 switch { 1617 case goos == "android": 1618 if gohostos != "android" { 1619 return pathf("%s/misc/go_android_exec/main.go", goroot) 1620 } 1621 case goos == "ios": 1622 if gohostos != "ios" { 1623 return pathf("%s/misc/ios/go_ios_exec.go", goroot) 1624 } 1625 } 1626 return "" 1627 } 1628 1629 func goInstall(env []string, goBinary string, args ...string) { 1630 goCmd(env, goBinary, "install", args...) 1631 } 1632 1633 func appendCompilerFlags(args []string) []string { 1634 if gogcflags != "" { 1635 args = append(args, "-gcflags=all="+gogcflags) 1636 } 1637 if goldflags != "" { 1638 args = append(args, "-ldflags=all="+goldflags) 1639 } 1640 return args 1641 } 1642 1643 func goCmd(env []string, goBinary string, cmd string, args ...string) { 1644 goCmd := []string{goBinary, cmd} 1645 if noOpt { 1646 goCmd = append(goCmd, "-tags=noopt") 1647 } 1648 goCmd = appendCompilerFlags(goCmd) 1649 if vflag > 0 { 1650 goCmd = append(goCmd, "-v") 1651 } 1652 1653 // Force only one process at a time on vx32 emulation. 1654 if gohostos == "plan9" && os.Getenv("sysname") == "vx32" { 1655 goCmd = append(goCmd, "-p=1") 1656 } 1657 1658 runEnv(workdir, ShowOutput|CheckExit, env, append(goCmd, args...)...) 1659 } 1660 1661 func checkNotStale(env []string, goBinary string, targets ...string) { 1662 goCmd := []string{goBinary, "list"} 1663 if noOpt { 1664 goCmd = append(goCmd, "-tags=noopt") 1665 } 1666 goCmd = appendCompilerFlags(goCmd) 1667 goCmd = append(goCmd, "-f={{if .Stale}}\tSTALE {{.ImportPath}}: {{.StaleReason}}{{end}}") 1668 1669 out := runEnv(workdir, CheckExit, env, append(goCmd, targets...)...) 1670 if strings.Contains(out, "\tSTALE ") { 1671 os.Setenv("GODEBUG", "gocachehash=1") 1672 for _, target := range []string{"runtime/github.com/go-asm/go/sys", "cmd/dist", "cmd/link"} { 1673 if strings.Contains(out, "STALE "+target) { 1674 run(workdir, ShowOutput|CheckExit, goBinary, "list", "-f={{.ImportPath}} {{.Stale}}", target) 1675 break 1676 } 1677 } 1678 fatalf("unexpected stale targets reported by %s list -gcflags=\"%s\" -ldflags=\"%s\" for %v (consider rerunning with GOMAXPROCS=1 GODEBUG=gocachehash=1):\n%s", goBinary, gogcflags, goldflags, targets, out) 1679 } 1680 } 1681 1682 // Cannot use go/build directly because cmd/dist for a new release 1683 // builds against an old release's go/build, which may be out of sync. 1684 // To reduce duplication, we generate the list for go/build from this. 1685 // 1686 // We list all supported platforms in this list, so that this is the 1687 // single point of truth for supported platforms. This list is used 1688 // by 'go tool dist list'. 1689 var cgoEnabled = map[string]bool{ 1690 "aix/ppc64": true, 1691 "darwin/amd64": true, 1692 "darwin/arm64": true, 1693 "dragonfly/amd64": true, 1694 "freebsd/386": true, 1695 "freebsd/amd64": true, 1696 "freebsd/arm": true, 1697 "freebsd/arm64": true, 1698 "freebsd/riscv64": true, 1699 "illumos/amd64": true, 1700 "linux/386": true, 1701 "linux/amd64": true, 1702 "linux/arm": true, 1703 "linux/arm64": true, 1704 "linux/loong64": true, 1705 "linux/ppc64": false, 1706 "linux/ppc64le": true, 1707 "linux/mips": true, 1708 "linux/mipsle": true, 1709 "linux/mips64": true, 1710 "linux/mips64le": true, 1711 "linux/riscv64": true, 1712 "linux/s390x": true, 1713 "linux/sparc64": true, 1714 "android/386": true, 1715 "android/amd64": true, 1716 "android/arm": true, 1717 "android/arm64": true, 1718 "ios/arm64": true, 1719 "ios/amd64": true, 1720 "js/wasm": false, 1721 "wasip1/wasm": false, 1722 "netbsd/386": true, 1723 "netbsd/amd64": true, 1724 "netbsd/arm": true, 1725 "netbsd/arm64": true, 1726 "openbsd/386": true, 1727 "openbsd/amd64": true, 1728 "openbsd/arm": true, 1729 "openbsd/arm64": true, 1730 "openbsd/mips64": true, 1731 "openbsd/ppc64": false, 1732 "openbsd/riscv64": false, 1733 "plan9/386": false, 1734 "plan9/amd64": false, 1735 "plan9/arm": false, 1736 "solaris/amd64": true, 1737 "windows/386": true, 1738 "windows/amd64": true, 1739 "windows/arm": false, 1740 "windows/arm64": true, 1741 } 1742 1743 // List of platforms that are marked as broken ports. 1744 // These require -force flag to build, and also 1745 // get filtered out of cgoEnabled for 'dist list'. 1746 // See go.dev/issue/56679. 1747 var broken = map[string]bool{ 1748 "linux/sparc64": true, // An incomplete port. See CL 132155. 1749 "openbsd/mips64": true, // Broken: go.dev/issue/58110. 1750 "openbsd/riscv64": true, // An incomplete port: go.dev/issue/55999. 1751 } 1752 1753 // List of platforms which are first class ports. See go.dev/issue/38874. 1754 var firstClass = map[string]bool{ 1755 "darwin/amd64": true, 1756 "darwin/arm64": true, 1757 "linux/386": true, 1758 "linux/amd64": true, 1759 "linux/arm": true, 1760 "linux/arm64": true, 1761 "windows/386": true, 1762 "windows/amd64": true, 1763 } 1764 1765 // We only need CC if cgo is forced on, or if the platform requires external linking. 1766 // Otherwise the go command will automatically disable it. 1767 func needCC() bool { 1768 return os.Getenv("CGO_ENABLED") == "1" || mustLinkExternal(gohostos, gohostarch, false) 1769 } 1770 1771 func checkCC() { 1772 if !needCC() { 1773 return 1774 } 1775 cc1 := defaultcc[""] 1776 if cc1 == "" { 1777 cc1 = "gcc" 1778 for _, os := range clangos { 1779 if gohostos == os { 1780 cc1 = "clang" 1781 break 1782 } 1783 } 1784 } 1785 cc, err := quotedSplit(cc1) 1786 if err != nil { 1787 fatalf("split CC: %v", err) 1788 } 1789 var ccHelp = append(cc, "--help") 1790 1791 if output, err := exec.Command(ccHelp[0], ccHelp[1:]...).CombinedOutput(); err != nil { 1792 outputHdr := "" 1793 if len(output) > 0 { 1794 outputHdr = "\nCommand output:\n\n" 1795 } 1796 fatalf("cannot invoke C compiler %q: %v\n\n"+ 1797 "Go needs a system C compiler for use with cgo.\n"+ 1798 "To set a C compiler, set CC=the-compiler.\n"+ 1799 "To disable cgo, set CGO_ENABLED=0.\n%s%s", cc, err, outputHdr, output) 1800 } 1801 } 1802 1803 func defaulttarg() string { 1804 // xgetwd might return a path with symlinks fully resolved, and if 1805 // there happens to be symlinks in goroot, then the hasprefix test 1806 // will never succeed. Instead, we use xrealwd to get a canonical 1807 // goroot/src before the comparison to avoid this problem. 1808 pwd := xgetwd() 1809 src := pathf("%s/src/", goroot) 1810 real_src := xrealwd(src) 1811 if !strings.HasPrefix(pwd, real_src) { 1812 fatalf("current directory %s is not under %s", pwd, real_src) 1813 } 1814 pwd = pwd[len(real_src):] 1815 // guard against xrealwd returning the directory without the trailing / 1816 pwd = strings.TrimPrefix(pwd, "/") 1817 1818 return pwd 1819 } 1820 1821 // Install installs the list of packages named on the command line. 1822 func cmdinstall() { 1823 xflagparse(-1) 1824 1825 if flag.NArg() == 0 { 1826 install(defaulttarg()) 1827 } 1828 1829 for _, arg := range flag.Args() { 1830 install(arg) 1831 } 1832 } 1833 1834 // Clean deletes temporary objects. 1835 func cmdclean() { 1836 xflagparse(0) 1837 clean() 1838 } 1839 1840 // Banner prints the 'now you've installed Go' banner. 1841 func cmdbanner() { 1842 xflagparse(0) 1843 banner() 1844 } 1845 1846 func banner() { 1847 if vflag > 0 { 1848 xprintf("\n") 1849 } 1850 xprintf("---\n") 1851 xprintf("Installed Go for %s/%s in %s\n", goos, goarch, goroot) 1852 xprintf("Installed commands in %s\n", gorootBin) 1853 1854 if !xsamefile(goroot_final, goroot) { 1855 // If the files are to be moved, don't check that gobin 1856 // is on PATH; assume they know what they are doing. 1857 } else if gohostos == "plan9" { 1858 // Check that GOROOT/bin is bound before /bin. 1859 pid := strings.Replace(readfile("#c/pid"), " ", "", -1) 1860 ns := fmt.Sprintf("/proc/%s/ns", pid) 1861 if !strings.Contains(readfile(ns), fmt.Sprintf("bind -b %s /bin", gorootBin)) { 1862 xprintf("*** You need to bind %s before /bin.\n", gorootBin) 1863 } 1864 } else { 1865 // Check that GOROOT/bin appears in $PATH. 1866 pathsep := ":" 1867 if gohostos == "windows" { 1868 pathsep = ";" 1869 } 1870 path := os.Getenv("PATH") 1871 if p, ok := os.LookupEnv("DIST_UNMODIFIED_PATH"); ok { 1872 // Scripts that modify $PATH and then run dist should also provide 1873 // dist with an unmodified copy of $PATH via $DIST_UNMODIFIED_PATH. 1874 // Use it here when determining if the user still needs to update 1875 // their $PATH. See go.dev/issue/42563. 1876 path = p 1877 } 1878 if !strings.Contains(pathsep+path+pathsep, pathsep+gorootBin+pathsep) { 1879 xprintf("*** You need to add %s to your PATH.\n", gorootBin) 1880 } 1881 } 1882 1883 if !xsamefile(goroot_final, goroot) { 1884 xprintf("\n"+ 1885 "The binaries expect %s to be copied or moved to %s\n", 1886 goroot, goroot_final) 1887 } 1888 } 1889 1890 // Version prints the Go version. 1891 func cmdversion() { 1892 xflagparse(0) 1893 xprintf("%s\n", findgoversion()) 1894 } 1895 1896 // cmdlist lists all supported platforms. 1897 func cmdlist() { 1898 jsonFlag := flag.Bool("json", false, "produce JSON output") 1899 brokenFlag := flag.Bool("broken", false, "include broken ports") 1900 xflagparse(0) 1901 1902 var plats []string 1903 for p := range cgoEnabled { 1904 if broken[p] && !*brokenFlag { 1905 continue 1906 } 1907 plats = append(plats, p) 1908 } 1909 sort.Strings(plats) 1910 1911 if !*jsonFlag { 1912 for _, p := range plats { 1913 xprintf("%s\n", p) 1914 } 1915 return 1916 } 1917 1918 type jsonResult struct { 1919 GOOS string 1920 GOARCH string 1921 CgoSupported bool 1922 FirstClass bool 1923 Broken bool `json:",omitempty"` 1924 } 1925 var results []jsonResult 1926 for _, p := range plats { 1927 fields := strings.Split(p, "/") 1928 results = append(results, jsonResult{ 1929 GOOS: fields[0], 1930 GOARCH: fields[1], 1931 CgoSupported: cgoEnabled[p], 1932 FirstClass: firstClass[p], 1933 Broken: broken[p], 1934 }) 1935 } 1936 out, err := json.MarshalIndent(results, "", "\t") 1937 if err != nil { 1938 fatalf("json marshal error: %v", err) 1939 } 1940 if _, err := os.Stdout.Write(out); err != nil { 1941 fatalf("write failed: %v", err) 1942 } 1943 } 1944 1945 func setNoOpt() { 1946 for _, gcflag := range strings.Split(gogcflags, " ") { 1947 if gcflag == "-N" || gcflag == "-l" { 1948 noOpt = true 1949 break 1950 } 1951 } 1952 }