go.fuchsia.dev/infra@v0.0.0-20240507153436-9b593402251b/cmd/cl-util/run_postsubmit_tryjobs.go

// Copyright 2022 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package main

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"log"
	"net/url"
	"os"
	"slices"
	"sort"
	"strings"

	"github.com/golang/protobuf/proto"
	"github.com/maruel/subcommands"
	"go.chromium.org/luci/auth"
	buildbucketpb "go.chromium.org/luci/buildbucket/proto"
	gerritpb "go.chromium.org/luci/common/proto/gerrit"
	cvpb "go.chromium.org/luci/cv/api/config/v2"
	"go.fuchsia.dev/infra/buildbucket"
	"go.fuchsia.dev/infra/flagutil"
	"go.fuchsia.dev/infra/gerrit"
	"go.fuchsia.dev/infra/gitiles"
	"google.golang.org/genproto/protobuf/field_mask"
	"google.golang.org/protobuf/encoding/protojson"
	"google.golang.org/protobuf/types/known/structpb"
)

const rptLongDesc = `
run-postsubmit-tryjobs run all available presubmit tryjobs for a CL that are
required by postsubmit. Due to a number of reasons
outlined in go/what-belongs-in-presubmit we can't run all tryjobs on all
changes by default, so this tool is for allowing automated tooling or humans
to specify which changes to run all builders for.
`

const buildbucketHost = "cr-buildbucket.appspot.com"

func cmdRunPostsubmitTryjobs(authOpts auth.Options) *subcommands.Command {
	return &subcommands.Command{
		UsageLine: "run-postsubmit-tryjobs <CL URL>",
		ShortDesc: "Trigger available tryjobs that are required for postsubmit for a CL.",
		LongDesc:  rptLongDesc,
		CommandRun: func() subcommands.CommandRun {
			c := &rptCmd{}
			c.Init(authOpts)
			return c
		},
	}
}

type rptCmd struct {
	commonFlags
	gerritChangeID        int64
	gerritPatchset        int64
	jsonOutputFile        string
	CommitQueueCfgPath    string
	LUCIConfigHost        string
	LUCIConfigPath        string
	LUCIConfigProject     string
	CrBuildBucketCfgPaths flagutil.RepeatedStringValue
	dryRun                bool
	force                 bool
	allPresubmit          bool
	verbose               bool
}

type BuildersToTrigger struct {
	Builders []string `json:"builders"`
}

type LUCIConfigurationFiles struct {
	commitQueueConfig    *cvpb.Config
	crBuildBucketConfigs map[string]*buildbucketpb.BuildbucketCfg
}

func (c *rptCmd) Init(defaultAuthOpts auth.Options) {
	c.commonFlags.Init(defaultAuthOpts)
	c.Flags.Int64Var(
		&c.gerritChangeID,
		"gerrit-change-id",
		0,
		"The ChangeID of the GerritChange to trigger builders against.",
	)
	c.Flags.StringVar(
		&c.LUCIConfigHost,
		"luci-config-host",
		"turquoise-internal.googlesource.com",
		"Gerrit host for the project containing the LUCI configuration files.",
	)
	c.Flags.StringVar(
		&c.LUCIConfigProject,
		"luci-config-project",
		"integration",
		"Project name containing the LUCI configuration files.",
	)
	c.Flags.StringVar(
		&c.CommitQueueCfgPath,
		"commit-queue-cfg-path",
		"infra/config/generated/turquoise/luci/commit-queue.cfg",
		"Path to the LUCI commit-queue.cfg configuration file.",
	)
	c.Flags.Var(
		&c.CrBuildBucketCfgPaths,
		"cr-buildbucket-cfg-project-path",
		"One or more comma-separated strings containing a project name and path to a LUCI cr-buildbucket.cfg configuration file. e.g. 'project,path/to/cr-buildbucket.cfg'",
	)
	c.Flags.BoolVar(
		&c.allPresubmit,
		"trigger-all-presubmit",
		false,
		"Whether to include presubmit builders not tagged for run-postsubmit-tryjobs. Projects outside Fuchsia that have not tagged their builders should enable this flag.",
	)
	c.Flags.BoolVar(
		&c.force,
		"f",
		false,
		"Whether to skip command line confirmation when triggering builders.",
	)
	c.Flags.BoolVar(
		&c.verbose,
		"v",
		false,
		"Whether to print all builders to be run.",
	)
	c.Flags.StringVar(
		&c.jsonOutputFile,
		"json-output",
		"",
		"Filepath to write json output to. Use '-' for stdout.",
	)
	c.Flags.BoolVar(
		&c.dryRun,
		"dry-run",
		false,
		"Whether to actually trigger the builders or just print out which builders would have been triggered.",
	)
}

func (c *rptCmd) Parse(a subcommands.Application, args []string) error {
	if len(args) != 0 && !strings.HasPrefix(args[0], "-") {
		// The first positional argument is the changelink, parse it into constituent parts.
		changeUrl, err := url.Parse(args[0])
		if err != nil {
			return err
		}

		gerritHost, changeID, err := gerrit.ResolveChangeUrl(changeUrl)
		if err != nil {
			return err
		}
		c.gerritHost = gerritHost
		c.gerritChangeID = changeID
	}

	// Set placeholder value for gerritProject if it isn't set at this point.
	// commonFlags.Parse includes validation for gerritProject to not be null,
	// but we set it in getChangeDetails.
	if c.gerritProject == "" {
		c.gerritProject = "placeholder-project-name"
	}

	// Custom vars can't have defaults set so populate here in case it's unset.
	if len(c.CrBuildBucketCfgPaths) == 0 {
		c.CrBuildBucketCfgPaths = flagutil.RepeatedStringValue{
			"turquoise,infra/config/generated/turquoise/luci/cr-buildbucket.cfg",
			"fuchsia,infra/config/generated/fuchsia/luci/cr-buildbucket.cfg",
		}
	}

	if err := c.commonFlags.Parse(); err != nil {
		return err
	}

	return nil
}

func (c *rptCmd) Run(a subcommands.Application, args []string, _ subcommands.Env) int {
	if err := c.Parse(a, args); err != nil {
		fmt.Fprintf(a.GetErr(), "%s: %s\n", a.GetName(), err)
		return 1
	}

	if err := c.main(); err != nil {
		fmt.Fprintf(a.GetErr(), "%s: %s\n", a.GetName(), err)
		return 1
	}
	return 0
}

func (c *rptCmd) main() error {
	if !c.verbose {
		log.SetOutput(io.Discard)
	}

	ctx := context.Background()

	buildClient, err := buildbucket.NewBuildsClient(
		ctx,
		buildbucketHost,
		c.commonFlags.parsedAuthOpts,
	)
	if err != nil {
		return err
	}

	authClient, err := auth.NewAuthenticator(ctx, auth.OptionalLogin, c.commonFlags.parsedAuthOpts).Client()
	if err != nil {
		return fmt.Errorf("failed to get authenticated http client: %w", err)
	}

	gitilesClient, err := gitiles.NewClient(c.LUCIConfigHost, c.LUCIConfigProject, authClient)
	if err != nil {
		return err
	}

	lucicfg, err := c.getLUCIConfigs(ctx, *gitilesClient)
	if err != nil {
		return err
	}

	gerritClient, err := gerrit.NewClient(c.gerritHost, c.gerritProject, authClient)
	if err != nil {
		return err
	}

	if err = c.getChangeDetails(ctx, *gerritClient); err != nil {
		return err
	}

	availablePresubmitBuilders := c.getAvailablePresubmitBuilders(lucicfg, c.gerritHost, c.gerritProject)

	triggeredBuilds, err := c.getTriggeredBuildsForChange(ctx, buildClient)
	if err != nil {
		return err
	}

	triggeredBuilds = c.filterFailedBuilders(triggeredBuilds)
	missingBuilders := c.getMissingBuilders(availablePresubmitBuilders, triggeredBuilds)
	if err := c.writeJSONOutput(missingBuilders); err != nil {
		return err
	}
	return c.triggerMissingBuilders(ctx, buildClient, missingBuilders)
}

// getLUCIConfigs fetches the LUCI configuration files contained at
// the project specified in c.LUCIConfigProject and c.LUCIConfigPath
func (c *rptCmd) getLUCIConfigs(ctx context.Context, gitilesClient gitiles.Client) (LUCIConfigurationFiles, error) {
	log.Println("Downloading LUCI configuration files...")
	// Set non-nil values for the configs.
	commitQueueConfig := &cvpb.Config{}
	crBuildBucketConfigs := map[string]*buildbucketpb.BuildbucketCfg{}

	commitQueueFileContents, err := gitilesClient.DownloadFile(
		ctx,
		c.CommitQueueCfgPath,
		"refs/heads/main",
	)
	if err != nil {
		return LUCIConfigurationFiles{}, err
	}

	if err := proto.UnmarshalText(commitQueueFileContents, commitQueueConfig); err != nil {
		return LUCIConfigurationFiles{}, err
	}

	for _, projectPath := range c.CrBuildBucketCfgPaths {
		projectPathSplit := strings.Split(projectPath, ",")
		if len(projectPathSplit) != 2 {
			return LUCIConfigurationFiles{}, errors.New(
				fmt.Sprintf(
					"Invalid entry to cr-buildbucket-cfg-project-path, expected 'project,path/to/cr-buildbucket.cfg' got %s",
					projectPath,
				),
			)
		}
		project := projectPathSplit[0]
		path := projectPathSplit[1]

		cfg := &buildbucketpb.BuildbucketCfg{}
		CrBuildBucketFileContents, err := gitilesClient.DownloadFile(
			ctx,
			path,
			"refs/heads/main",
		)
		if err != nil {
			return LUCIConfigurationFiles{}, err
		}

		if err = proto.UnmarshalText(CrBuildBucketFileContents, cfg); err != nil {
			return LUCIConfigurationFiles{}, err
		}

		crBuildBucketConfigs[project] = cfg
	}

	return LUCIConfigurationFiles{
		commitQueueConfig:    commitQueueConfig,
		crBuildBucketConfigs: crBuildBucketConfigs,
	}, nil
}

// getChangeDetails queries Gerrit for the target change to determine
// the ChangeRef, ChangeRepo, Project and Patchset (if not specified)
func (c *rptCmd) getChangeDetails(ctx context.Context, gerritClient gerrit.Client) error {
	change, err := gerritClient.GetChange(
		ctx,
		c.gerritChangeID,
		gerritpb.QueryOption_ALL_REVISIONS,
	)
	if err != nil {
		return err
	}

	c.gerritPatchset = int64(change.Revisions[change.CurrentRevision].Number)
	c.gerritProject = change.Project

	return nil
}

// getAvailablePresubmitBuilders parses through available builders in
// presubmit and if an equivalent builder is required in postsubmit, adds it to
// the list of builders we check to trigger.
func (c *rptCmd) getAvailablePresubmitBuilders(lucicfg LUCIConfigurationFiles, gerritHost string, gerritProject string) []*buildbucketpb.BuilderID {
	// Throw buckets into a map for cross referencing.
	flaggedBuildersByBucketAndProjectMap := map[string]map[string]map[string]bool{}
	for project, cfg := range lucicfg.crBuildBucketConfigs {
		if _, ok := flaggedBuildersByBucketAndProjectMap[project]; !ok {
			flaggedBuildersByBucketAndProjectMap[project] = map[string]map[string]bool{}
		}
		for _, bucket := range cfg.GetBuckets() {
			flaggedBuildersByBucketAndProjectMap[project][bucket.GetName()] = getBuildersFlaggedForRpt(bucket)
		}
	}

	// Find CQ for the target change.
	configGroups := lucicfg.commitQueueConfig.GetConfigGroups()
	var availablePresubmitBuilders []*buildbucketpb.BuilderID

	for _, cfg := range configGroups {
		projectMatch := cfg.GetGerrit()[0].GetProjects()[0].GetName() == gerritProject
		cfgUrl, _ := url.Parse(cfg.GetGerrit()[0].GetUrl())
		hostMatch := cfgUrl.Host == gerritHost
		if projectMatch && hostMatch {
			for _, builder := range cfg.GetVerifiers().GetTryjob().GetBuilders() {
				builderSplit := strings.Split(builder.GetName(), "/")
				builderID := &buildbucketpb.BuilderID{
					Project: builderSplit[0],
					Bucket:  builderSplit[1],
					Builder: builderSplit[2],
				}

				if c.allPresubmit || flaggedBuildersByBucketAndProjectMap[builderID.Project][builderID.Bucket][builderID.Builder] {
					availablePresubmitBuilders = append(availablePresubmitBuilders, builderID)
				}
			}
		}
	}

	// Sort for logging output clarity.
	sort.SliceStable(availablePresubmitBuilders, func(i, j int) bool {
		return availablePresubmitBuilders[i].Builder < availablePresubmitBuilders[j].Builder
	})

	return availablePresubmitBuilders
}

// getTriggeredBuildersForChange calls the buildbucket SearchBuilds RPC
// and reports back all builders that have already triggered against the change.
func (c *rptCmd) getTriggeredBuildsForChange(ctx context.Context, buildClient buildbucketpb.BuildsClient) ([]*buildbucketpb.Build, error) {
	// Pull all builds triggered against the CL.
	resp, err := buildClient.SearchBuilds(
		ctx,
		&buildbucketpb.SearchBuildsRequest{
			Predicate: &buildbucketpb.BuildPredicate{
				GerritChanges: []*buildbucketpb.GerritChange{
					{
						Host:     c.gerritHost,
						Project:  c.gerritProject,
						Change:   c.gerritChangeID,
						Patchset: c.gerritPatchset,
					},
				},
			},
			Fields: &field_mask.FieldMask{Paths: []string{
				"builds.*.builder",
				"builds.*.status",
			}},
			PageSize: 1000,
		},
	)
	if err != nil {
		return []*buildbucketpb.Build{}, err
	}

	return resp.GetBuilds(), nil
}

// filterFailedBuilders filters failed builders out of the triggered builders
// so that they will be re-triggered by run-postsubmit-tryjobs.
func (c *rptCmd) filterFailedBuilders(triggeredBuilds []*buildbucketpb.Build) []*buildbucketpb.Build {
	var filteredBuilds []*buildbucketpb.Build
	for _, build := range triggeredBuilds {
		if !slices.Contains(
			[]buildbucketpb.Status{
				buildbucketpb.Status_FAILURE,
				buildbucketpb.Status_INFRA_FAILURE,
			},
			build.GetStatus(),
		) {
			filteredBuilds = append(filteredBuilds, build)
		}
	}
	return filteredBuilds
}

// getMissingBuilders finds the diff between availablePresubmitBuilders and the
// builds already triggered by the change.
func (c *rptCmd) getMissingBuilders(availablePresubmitBuilders []*buildbucketpb.BuilderID, triggeredPresubmitBuilds []*buildbucketpb.Build) []*buildbucketpb.BuilderID {
	var missingBuilders []*buildbucketpb.BuilderID
	var triggeredBuilders []string
	for _, build := range triggeredPresubmitBuilds {
		triggeredBuilders = append(triggeredBuilders, build.GetBuilder().GetBuilder())
	}

	// Find which builders haven't been triggered yet.
	for _, builder := range availablePresubmitBuilders {
		if !slices.Contains(triggeredBuilders, builder.GetBuilder()) && !builderSliceContains(missingBuilders, builder) {
			missingBuilders = append(missingBuilders, builder)
		}
	}

	return missingBuilders
}

// triggerMissingBuilders triggers the diff between all builders available
// and the builders that have already been triggered for the change.
func (c *rptCmd) triggerMissingBuilders(ctx context.Context, buildClient buildbucketpb.BuildsClient, missingBuilders []*buildbucketpb.BuilderID) error {
	// If we have no builders to trigger, return.
	if len(missingBuilders) == 0 {
		log.Println("No builders found to trigger.")
		return nil
	}

	// Print out list of builders to be triggered for logging.
	for _, builder := range missingBuilders {
		log.Printf("Builder: %s\n", builder.GetBuilder())
	}

	// Require command line confirmation.  Automated tools should pass the -f flag.
	if !(c.force || c.dryRun) {
		// The cost has been calculated roughly on average as of 2022 to be $0.57 per builder triggered
		// GCE Cost: $0.53/h
		// Orchestrator builder: 0.45h
		// Subbuild builder: 0.45h
		// Testing Subtasks: 0.17h
		fmt.Printf("You are about to trigger %d tryjobs with an estimated cost of $%.2f USD. Additionally, this will likely consume inelastic resources. Do you wish to proceed? (y/n)\n", len(missingBuilders), 0.57*float64(len(missingBuilders)))
		confirm := ""
		fmt.Scanln(&confirm)
		if !slices.Contains([]string{"y", "Y", "yes", "Yes", "YES"}, confirm) {
			fmt.Println("Process aborted.")
			return nil
		} else {
			fmt.Println("You can skip this confirmation next time by passing the -f flag.")
		}
	}

	// Assemble requests into a batch.
	var requestBatch []*buildbucketpb.BatchRequest_Request
	for _, builder := range missingBuilders {
		req := &buildbucketpb.BatchRequest_Request{
			Request: &buildbucketpb.BatchRequest_Request_ScheduleBuild{
				ScheduleBuild: &buildbucketpb.ScheduleBuildRequest{
					Builder: builder,
					GerritChanges: []*buildbucketpb.GerritChange{
						{
							Host:     c.gerritHost,
							Project:  c.gerritProject,
							Change:   c.gerritChangeID,
							Patchset: c.gerritPatchset,
						},
					},
					// Slightly below default priority.
					Priority: 31,
				},
			},
		}
		requestBatch = append(requestBatch, req)
	}

	if !c.dryRun {
		resp, err := buildClient.Batch(
			ctx,
			&buildbucketpb.BatchRequest{
				Requests: requestBatch,
			},
		)
		if err != nil {
			return err
		}

		jsonResp, err := json.MarshalIndent(resp, "", "  ")
		if err != nil {
			return err
		}

		log.Printf(
			"BatchRequest response: \n%s",
			jsonResp,
		)
	}

	return nil
}

func (c *rptCmd) writeJSONOutput(builders []*buildbucketpb.BuilderID) error {
	if c.jsonOutputFile == "" {
		return nil
	}
	// Assemble builder data into easy to consume json output.
	jsonOutput := &BuildersToTrigger{Builders: []string{}}
	for _, b := range builders {
		jsonOutput.Builders = append(jsonOutput.Builders, b.Builder)
	}

	// Marshal output to json.
	var rawJSON []byte
	var err error
	var f io.WriteCloser = os.Stdout
	if c.jsonOutputFile != "-" {
		if f, err = os.Create(c.jsonOutputFile); err != nil {
			return err
		}
		defer f.Close()
	}

	if c.jsonOutputFile == "-" {
		rawJSON, err = json.MarshalIndent(jsonOutput, "", "    ")
		rawJSON = append(rawJSON, '\n')
	} else {
		rawJSON, err = json.Marshal(jsonOutput)
		rawJSON = append(rawJSON, '\n')
	}
	if err != nil {
		return err
	}

	_, err = f.Write(rawJSON)
	return err
}

// slices.Contains doesn't really work with slices of pointers, so this helper
// performs a low effort check for inclusion.
func builderSliceContains(slice []*buildbucketpb.BuilderID, builder *buildbucketpb.BuilderID) bool {
	for _, b := range slice {
		if b.GetBuilder() == builder.GetBuilder() {
			return true
		}
	}
	return false
}

// getBuildersFlaggedForRpt finds builders with the `run_postsubmit_tryjobs_include` property.
func getBuildersFlaggedForRpt(bucket *buildbucketpb.Bucket) map[string]bool {
	flaggedBuilders := map[string]bool{}
	for _, b := range bucket.GetSwarming().GetBuilders() {
		properties := &structpb.Struct{}
		protojson.Unmarshal([]byte(b.GetProperties()), properties)
		if builderTags, ok := properties.GetFields()["$fuchsia/builder_tags"]; ok {
			if runPostsubmitTryjobsInclude, ok := builderTags.GetStructValue().GetFields()["run_postsubmit_tryjobs_include"]; ok {
				flaggedBuilders[b.GetName()] = runPostsubmitTryjobsInclude.GetBoolValue()
			} else {
				flaggedBuilders[b.GetName()] = false
			}
		} else {
			flaggedBuilders[b.GetName()] = false
		}
	}

	return flaggedBuilders
}