golang.org/x/build@v0.0.0-20240506185731-218518f32b70/cmd/updatestd/updatestd.go

// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// updatestd is an experimental program that has been used to update
// the standard library modules as part of golang.org/issue/36905 in
// CL 255860 and CL 266898. It's expected to be modified to meet the
// ongoing needs of that recurring maintenance work.
package main

import (
	"bytes"
	"context"
	"debug/buildinfo"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"go/ast"
	"go/parser"
	"go/token"
	"io"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"

	"golang.org/x/build/gerrit"
	"golang.org/x/build/internal/envutil"
)

var goCmd string // the go command

func main() {
	log.SetFlags(0)

	flag.Usage = func() {
		fmt.Fprintln(os.Stderr, "Usage: updatestd -goroot=<goroot> -branch=<branch>")
		flag.PrintDefaults()
	}
	goroot := flag.String("goroot", "", "path to a working copy of https://go.googlesource.com/go (required)")
	branch := flag.String("branch", "", "branch to target, such as master or release-branch.go1.Y (required)")
	flag.Parse()
	if flag.NArg() != 0 || *goroot == "" || *branch == "" {
		flag.Usage()
		os.Exit(2)
	}

	// Determine the Go version from the GOROOT source tree.
	goVersion, err := gorootVersion(*goroot)
	if err != nil {
		log.Fatalln(err)
	}

	goCmd = filepath.Join(*goroot, "bin", "go")

	// Confirm that bundle is in PATH.
	// It's needed for a go generate step later.
	bundlePath, err := exec.LookPath("bundle")
	if err != nil {
		log.Fatalln("can't find bundle in PATH; did you run 'go install golang.org/x/tools/cmd/bundle@latest' and add it to PATH?")
	}
	if bi, err := buildinfo.ReadFile(bundlePath); err != nil || bi.Path != "golang.org/x/tools/cmd/bundle" {
		// Not the bundle command we want.
		log.Fatalln("unexpected bundle command in PATH; did you run 'go install golang.org/x/tools/cmd/bundle@latest' and add it to PATH?")
	}

	// Fetch latest hashes of Go projects from Gerrit,
	// using the specified branch name.
	//
	// We get a fairly consistent snapshot of all golang.org/x module versions
	// at a given point in time. This ensures selection of latest available
	// pseudo-versions is done without being subject to module mirror caching,
	// and that selected pseudo-versions can be re-used across multiple modules.
	//
	// TODO: Consider a future enhancement of fetching build status for all
	// commits that are selected and reporting if any of them have a failure.
	//
	cl := gerrit.NewClient("https://go-review.googlesource.com", gerrit.NoAuth)
	projs, err := cl.ListProjects(context.Background())
	if err != nil {
		log.Fatalln("failed to get a list of Gerrit projects:", err)
	}
	hashes := map[string]string{}
	for _, p := range projs {
		b, err := cl.GetBranch(context.Background(), p.Name, *branch)
		if errors.Is(err, gerrit.ErrResourceNotExist) {
			continue
		} else if err != nil {
			log.Fatalf("failed to get the %q branch of Gerrit project %q: %v\n", *branch, p.Name, err)
		}
		hashes[p.Name] = b.Revision
	}

	w := Work{
		Branch:        *branch,
		GoVersion:     fmt.Sprintf("1.%d", goVersion),
		ProjectHashes: hashes,
	}

	// Print environment information.
	r := runner{filepath.Join(*goroot, "src")}
	r.run(goCmd, "version")
	r.run(goCmd, "env", "GOROOT")
	r.run(goCmd, "version", "-m", bundlePath)
	log.Println()

	// Walk the standard library source tree (GOROOT/src),
	// skipping directories that the Go command ignores (see go help packages)
	// and update modules that are found.
	err = filepath.Walk(filepath.Join(*goroot, "src"), func(path string, fi os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if fi.IsDir() && (strings.HasPrefix(fi.Name(), ".") || strings.HasPrefix(fi.Name(), "_") || fi.Name() == "testdata" || fi.Name() == "vendor") {
			return filepath.SkipDir
		}
		goModFile := fi.Name() == "go.mod" && !fi.IsDir()
		if goModFile {
			moduleDir := filepath.Dir(path)
			err := w.UpdateModule(moduleDir)
			if err != nil {
				return fmt.Errorf("failed to update module in %s: %v", moduleDir, err)
			}
			return filepath.SkipDir // Skip the remaining files in this directory.
		}
		return nil
	})
	if err != nil {
		log.Fatalln(err)
	}

	// Re-bundle packages in the standard library.
	//
	// TODO: Maybe do GOBIN=$(mktemp -d) go install golang.org/x/tools/cmd/bundle@version or so,
	// and add it to PATH to eliminate variance in bundle tool version. Can be considered later.
	//
	log.Println("updating bundles in", r.dir)
	r.run(goCmd, "generate", "-run=bundle", "std", "cmd")
}

type Work struct {
	Branch        string            // Target branch name.
	GoVersion     string            // Major Go version, like "1.x".
	ProjectHashes map[string]string // Gerrit project name → commit hash.
}

// UpdateModule updates the standard library module found in dir:
//
//  1. Set the expected Go version in go.mod file to w.GoVersion.
//  2. For modules in the build list with "golang.org/x/" prefix,
//     update to pseudo-version corresponding to w.ProjectHashes.
//  3. Run go mod tidy.
//  4. Run go mod vendor.
//
// The logic in this method needs to serve the dependency update
// policy for the purpose of golang.org/issue/36905, although it
// does not directly define said policy.
func (w Work) UpdateModule(dir string) error {
	// Determine the build list.
	main, deps := buildList(dir)

	// Determine module versions to get.
	goGet := []string{goCmd, "get", "-d"}
	for _, m := range deps {
		if !strings.HasPrefix(m.Path, "golang.org/x/") {
			log.Printf("skipping %s (out of scope, it's not a golang.org/x dependency)\n", m.Path)
			continue
		}
		gerritProj := m.Path[len("golang.org/x/"):]
		hash, ok := w.ProjectHashes[gerritProj]
		if !ok {
			if m.Indirect {
				log.Printf("skipping %s because branch %s doesn't exist and it's indirect\n", m.Path, w.Branch)
				continue
			}
			return fmt.Errorf("no hash for Gerrit project %q", gerritProj)
		}
		goGet = append(goGet, m.Path+"@"+hash)
	}

	// Run all the commands.
	log.Println("updating module", main.Path, "in", dir)
	r := runner{dir}
	gowork := strings.TrimSpace(string(r.runOut(goCmd, "env", "GOWORK")))
	if gowork != "" && gowork != "off" {
		log.Printf("warning: GOWORK=%q, things may go wrong?", gowork)
	}
	r.run(goCmd, "mod", "edit", "-go="+w.GoVersion)
	r.run(goGet...)
	r.run(goCmd, "mod", "tidy")
	r.run(goCmd, "mod", "vendor")
	log.Println()
	return nil
}

// buildList determines the build list in the directory dir
// by invoking the go command. It uses -mod=readonly mode.
// It returns the main module and other modules separately
// for convenience to the UpdateModule caller.
//
// See https://go.dev/ref/mod#go-list-m and https://go.dev/ref/mod#glos-build-list.
func buildList(dir string) (main module, deps []module) {
	out := runner{dir}.runOut(goCmd, "list", "-mod=readonly", "-m", "-json", "all")
	for dec := json.NewDecoder(bytes.NewReader(out)); ; {
		var m module
		err := dec.Decode(&m)
		if err == io.EOF {
			break
		} else if err != nil {
			log.Fatalf("internal error: unexpected problem decoding JSON returned by go list -json: %v", err)
		}
		if m.Main {
			main = m
			continue
		}
		deps = append(deps, m)
	}
	return main, deps
}

type module struct {
	Path     string // Module path.
	Main     bool   // Is this the main module?
	Indirect bool   // Is this module only an indirect dependency of main module?
}

// gorootVersion reads the GOROOT/src/internal/goversion/goversion.go
// file and reports the Version declaration value found therein.
func gorootVersion(goroot string) (int, error) {
	// Parse the goversion.go file, extract the declaration from the AST.
	//
	// This is a pragmatic approach that relies on the trajectory of the
	// internal/goversion package being predictable and unlikely to change.
	// If that stops being true, this small helper is easy to re-write.
	//
	fset := token.NewFileSet()
	f, err := parser.ParseFile(fset, filepath.Join(goroot, "src", "internal", "goversion", "goversion.go"), nil, 0)
	if os.IsNotExist(err) {
		return 0, fmt.Errorf("did not find goversion.go file (%v); wrong goroot or did internal/goversion package change?", err)
	} else if err != nil {
		return 0, err
	}
	for _, d := range f.Decls {
		g, ok := d.(*ast.GenDecl)
		if !ok {
			continue
		}
		for _, s := range g.Specs {
			v, ok := s.(*ast.ValueSpec)
			if !ok || len(v.Names) != 1 || v.Names[0].String() != "Version" || len(v.Values) != 1 {
				continue
			}
			l, ok := v.Values[0].(*ast.BasicLit)
			if !ok || l.Kind != token.INT {
				continue
			}
			return strconv.Atoi(l.Value)
		}
	}
	return 0, fmt.Errorf("did not find Version declaration in %s; wrong goroot or did internal/goversion package change?", fset.File(f.Pos()).Name())
}

type runner struct{ dir string }

// run runs the command and requires that it succeeds.
// It logs the command's combined output.
func (r runner) run(args ...string) {
	log.Printf("> %s\n", strings.Join(args, " "))
	cmd := exec.Command(args[0], args[1:]...)
	envutil.SetDir(cmd, r.dir)
	out, err := cmd.CombinedOutput()
	if err != nil {
		log.Fatalf("command failed: %s\n%s", err, out)
	}
	if len(out) != 0 {
		log.Print(string(out))
	}
}

// runOut runs the command, requires that it succeeds,
// and returns the command's standard output.
func (r runner) runOut(args ...string) []byte {
	cmd := exec.Command(args[0], args[1:]...)
	envutil.SetDir(cmd, r.dir)
	out, err := cmd.Output()
	if err != nil {
		log.Printf("> %s\n", strings.Join(args, " "))
		if ee := (*exec.ExitError)(nil); errors.As(err, &ee) {
			out = append(out, ee.Stderr...)
		}
		log.Fatalf("command failed: %s\n%s", err, out)
	}
	return out
}