go.fuchsia.dev/infra@v0.0.0-20240507153436-9b593402251b/cmd/recipe_wrapper/recipes/recipes.go

// Copyright 2021 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package recipes

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/bazelbuild/remote-apis-sdks/go/pkg/digest"
	"github.com/bazelbuild/remote-apis-sdks/go/pkg/filemetadata"
	buildbucketpb "go.chromium.org/luci/buildbucket/proto"
	"go.chromium.org/luci/client/casclient"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/hardcoded/chromeinfra"
	"go.chromium.org/luci/luciexe/exe"
	apipb "go.chromium.org/luci/swarming/proto/api"
	"google.golang.org/protobuf/encoding/protojson"
	"google.golang.org/protobuf/proto"
	"google.golang.org/protobuf/types/known/structpb"

	infracheckout "go.fuchsia.dev/infra/checkout"
	rbc "go.fuchsia.dev/infra/cmd/recipe_wrapper/checkout"
	"go.fuchsia.dev/infra/cmd/recipe_wrapper/git"
	"go.fuchsia.dev/infra/cmd/recipe_wrapper/manifest"
	"go.fuchsia.dev/infra/cmd/recipe_wrapper/props"
)

const (
	// `led edit-recipe-bundle -property-only` sets this property. If set, its
	// value will be a jsonpb representation of a CASReference pointing to the
	// uploaded recipe bundle in CAS.
	ledCASRecipeBundleProp = "led_cas_recipe_bundle"

	// Recipe manifest filename relative to root of an integration checkout.
	recipesManifest = "infra/recipes"

	// Recipes.git remote.
	recipesRemote = "https://fuchsia.googlesource.com/infra/recipes"

	// Name of the build input property used by the recipe engine and various
	// other tools to determine the recipe that a build uses.
	recipeProperty = "recipe"
)

// Checkout represents a directory containing recipe code capable of being run
// as a luciexe.
type Checkout interface {
	// LuciexeCommand returns a command that will run the recipes as a LUCI
	// executable.
	LuciexeCommand() ([]string, error)
	// Repo returns the Git repo that these recipes were downloaded from.
	// Returns an empty string if unknown.
	Repo() string
	// SHA1 returns the Git SHA1 that these recipes were found at.
	// Returns an empty string if unknown.
	SHA1() string
}

// bundle represents a recipe bundle directory downloaded from CAS and
// originally produced using `recipes.py bundle`.
type bundle struct {
	dir string
}

func (r *bundle) LuciexeCommand() ([]string, error) {
	return []string{filepath.Join(r.dir, "luciexe")}, nil
}
func (*bundle) Repo() string { return "" }
func (*bundle) SHA1() string { return "" }

// Repo represents a recipe repository downloaded from Git.
type Repo struct {
	dir, repo, sha1 string
}

func (r *Repo) LuciexeCommand() ([]string, error) {
	return pyCommand(r.dir, "luciexe")
}

func (r *Repo) Repo() string { return r.repo }
func (r *Repo) SHA1() string { return r.sha1 }

// SetUp processes the build input to determine which version of recipes
// to use (possibly from CAS rather than from Git if run by a led job) and
// downloads that version of the recipes. It returns a `RecipesExe` that can be
// used to run the recipes (whether they were downloaded from CAS or from Git)
// as a luciexe, along with the deserialized build.proto.
func SetUp(ctx context.Context, recipesDir string) (Checkout, *buildbucketpb.Build, error) {
	data, err := io.ReadAll(os.Stdin)
	if err != nil {
		return nil, nil, fmt.Errorf("failed to read build.proto from stdin: %w", err)
	}
	build := &buildbucketpb.Build{}
	if err := proto.Unmarshal(data, build); err != nil {
		return nil, nil, fmt.Errorf("failed to unmarshal build.proto from stdin: %w", err)
	}
	if err := git.EnsureGitilesCommit(ctx, build); err != nil {
		return nil, nil, fmt.Errorf("failed to ensure that build has a gitiles commit: %w", err)
	}
	integrationDir, err := git.CheckoutIntegration(ctx, build)
	defer os.RemoveAll(integrationDir)
	if err != nil {
		return nil, nil, err
	}
	if err := resolveBuildProperties(ctx, integrationDir, build); err != nil {
		return nil, nil, fmt.Errorf("failed to resolve the build properties: %w", err)
	}
	b, err := json.MarshalIndent(build.Input.Properties, "", "  ")
	if err != nil {
		return nil, nil, fmt.Errorf("failed to marshal build input properties: %w", err)
	}
	// TODO(fxbug.dev/83858): Stop setting this env var once once we no longer
	// need to support running recipes from release branches where python3
	// wasn't the default.
	if err := os.Setenv("RECIPES_USE_PY3", "true"); err != nil {
		return nil, nil, err
	}
	// TODO(fxbug.dev/72956): Emit the build properties to a separate output log
	// once it's possible for recipe_wrapper to emit build.proto to the same
	// Logdog stream as the recipe engine.
	logging.Infof(ctx, "Resolved input properties: %s", string(b))
	exe, err := downloadRecipes(ctx, build, integrationDir, recipesDir)
	if err != nil {
		return nil, nil, err
	}
	return exe, build, nil
}

// resolveBuildProperties resolves the build input properties to use by reading
// the builder's properties from integration.git and merging them with the input
// properties.
func resolveBuildProperties(ctx context.Context, integrationDir string, build *buildbucketpb.Build) error {
	versionedProperties, err := versionedProperties(ctx, integrationDir, build.Builder)
	if err != nil {
		return fmt.Errorf("failed to read builder properties from integration repo: %w", err)
	}
	// Make a copy of the original input properties, which we'll use to override
	// the versioned properties.
	originalProperties := build.Input.Properties.AsMap()

	// The versioned "recipe" property should take precedence over the "recipe"
	// property from the build input. This makes it possible to:
	// 1) Test a configuration change that swaps a builder's recipe in presubmit.
	// 2) Swap a builder's recipe without breaking release branches.
	//
	// The downside is that any out-of-band tooling that relies on this property
	// will no longer behave correctly, but this isn't the end of the world
	// because recipe swaps should be very rare and only affect a small number
	// of release branch builds.
	if versionedRecipe, ok := versionedProperties.AsMap()[recipeProperty]; ok {
		originalProperties[recipeProperty] = versionedRecipe
	}

	// Replace the input properties with the resolved versioned properties.
	build.Input.Properties = versionedProperties
	// Merge the original input properties (primarily request properties) into
	// the versioned properties, letting the original input properties take
	// precedence.
	return exe.WriteProperties(build.Input.Properties, originalProperties)
}

// versionedProperties loads the builder's properties from a JSON file in
// the integration checkout.
func versionedProperties(ctx context.Context, integrationDir string, builder *buildbucketpb.BuilderID) (*structpb.Struct, error) {
	absPath := filepath.Join(integrationDir, propertiesFileForBuilder(builder))
	logging.Infof(ctx, "Using versioned properties from %s", absPath)
	contents, err := os.ReadFile(absPath)
	if err != nil {
		return nil, fmt.Errorf("failed to locate properties file for builder %s: %w", builder, err)
	}
	properties := &structpb.Struct{}
	if err := protojson.Unmarshal(contents, properties); err != nil {
		return nil, err
	}
	return properties, nil
}

func propertiesFileForBuilder(builder *buildbucketpb.BuilderID) string {
	return filepath.Join(
		"infra",
		"config",
		"generated",
		builder.Project,
		"properties",
		// Builders launched by led with -real-build use shadow buckets in the form of
		// `original_bucket.shadow`, but use the same properties file as the
		// corresponding builder in the original bucket it shadows.
		strings.ReplaceAll(builder.Bucket, ".shadow", ""),
		builder.Builder+".json",
	)
}

// downloadRecipes downloads recipes from Git or CAS depending on the build
// input.
func downloadRecipes(ctx context.Context, build *buildbucketpb.Build, integrationDir, recipesDir string) (Checkout, error) {
	casRef := &apipb.CASReference{}
	if err := props.Proto(build, ledCASRecipeBundleProp, casRef); err != nil {
		return nil, err
	}
	if casRef.CasInstance != "" {
		// We're running as a led job that was launched using `led
		// edit-recipe-bundle -property-only`. So download the referenced recipe
		// bundle from CAS and execute that, rather than checking out recipes
		// from Git.
		logging.Infof(ctx, "Downloading recipe bundle from CAS: %+v", casRef)
		if err := downloadBundle(ctx, casRef, recipesDir); err != nil {
			return nil, err
		}
		return &bundle{dir: recipesDir}, nil
	}

	input, err := checkout(ctx, build, recipesRemote, integrationDir, recipesDir)
	if err != nil {
		return nil, err
	}
	if err := fetchDeps(ctx, recipesDir); err != nil {
		return nil, err
	}
	return &Repo{dir: recipesDir, repo: recipesRemote, sha1: input.GitilesCommit.Id}, nil
}

// checkout resolves the recipe version to check out, then checks out the
// recipes repo at the specified recipesDir.
// If the build input has a recipe change, then checkout directly using the
// build input.
// Otherwise, checkout at the revision specified by Build.Input.Properties.
// The input resolved and used is returned.
func checkout(ctx context.Context, build *buildbucketpb.Build, remote string, integrationDir, recipesDir string) (*buildbucketpb.Build_Input, error) {
	recipesURL, err := url.Parse(remote)
	if err != nil {
		return nil, fmt.Errorf("could not parse URL %s", remote)
	}
	var input *buildbucketpb.Build_Input
	hasRecipeChange, err := rbc.HasRepoChange(build.Input, recipesRemote)
	if err != nil {
		return nil, fmt.Errorf("could not determine whether build input has recipe change: %w", err)
	}
	if hasRecipeChange {
		input = build.Input
	} else {
		// Use the recipe version from the checkout integrationDir.
		manifestXML, err := os.Open(filepath.Join(integrationDir, recipesManifest))
		if err != nil {
			return nil, fmt.Errorf("failed to read recipes manifest file: %s: %w", filepath.Join(integrationDir, recipesManifest), err)
		}
		defer manifestXML.Close()
		proj, err := manifest.ResolveRecipesProject(manifestXML, recipesRemote)
		if err != nil {
			return nil, err
		}
		input = &buildbucketpb.Build_Input{
			GitilesCommit: &buildbucketpb.GitilesCommit{
				Host:    recipesURL.Host,
				Project: strings.TrimLeft(recipesURL.Path, "/"),
				Id:      proj.Revision,
			},
		}
	}
	tctx, cancel := context.WithTimeout(ctx, rbc.DefaultCheckoutTimeout)
	defer cancel()
	if err := infracheckout.Checkout(tctx, input, *recipesURL, "", recipesDir); err != nil {
		return nil, fmt.Errorf("failed to checkout recipes repo: %w", err)
	}
	return input, nil
}

// fetchDeps runs `recipes.py fetch`, which checks out all recipe git
// repos that the current recipes repo repends on. `recipes.py luciexe` would
// handle fetching deps even if we didn't run `recipes.py fetch` separately
// beforehand, but running `recipes.py fetch` separately makes it easier to
// distinguish between fetch errors (which are generally caused by transient Git
// server issues) and true build errors within the executed recipe.
func fetchDeps(ctx context.Context, recipesDir string) error {
	args, err := pyCommand(recipesDir, "fetch")
	if err != nil {
		return err
	}
	cmd := exec.CommandContext(ctx, args[0], args[1:]...)
	// We want to make logs visible, but stdout is reserved for luciexe
	// communication so we must pipe all output to stderr.
	cmd.Stdout = os.Stderr
	cmd.Stderr = os.Stderr
	if err := cmd.Run(); err != nil {
		return fmt.Errorf("failed to fetch recipe deps: %w", err)
	}
	return nil
}

// pyCommand constructs inputs to an exec.Cmd for running the `recipes.py`
// script found in `recipesDir` with the specified arguments.
func pyCommand(recipesDir string, recipesPyArgs ...string) ([]string, error) {
	var cmd []string
	if runtime.GOOS == "windows" {
		// Windows doesn't support shebangs, so we can't run recipes.py
		// directly. Instead we need to pass it as a script to the Python
		// executable.
		//
		// TODO(olivernweman): For simplicity, we should really do this
		// regardless of the OS once all release branches use a version of
		// recipes.py that supports Python 3. It just so happens that no Windows
		// builders run on release branches, which is why we can get away with
		// doing this for Windows.
		python, err := exec.LookPath("python3")
		if err != nil {
			return nil, fmt.Errorf("could not find python3 on PATH: %w", err)
		}
		cmd = append(cmd, python)
	}
	cmd = append(cmd, filepath.Join(recipesDir, "recipes.py"))
	cmd = append(cmd, recipesPyArgs...)
	return cmd, nil
}

// downloadBundle downloads the CAS archive identified by casRef to
// recipesDir.
func downloadBundle(ctx context.Context, casRef *apipb.CASReference, recipesDir string) error {
	client, err := casclient.NewLegacy(ctx, casclient.AddrProd, casRef.CasInstance, chromeinfra.DefaultAuthOptions(), true)
	if err != nil {
		return err
	}
	defer client.Close()

	d := digest.Digest{
		Hash: casRef.Digest.Hash,
		Size: casRef.Digest.SizeBytes,
	}
	if _, _, err := client.DownloadDirectory(ctx, d, recipesDir, filemetadata.NewNoopCache()); err != nil {
		return fmt.Errorf("failed to download recipe bundle from CAS: %w", err)
	}
	return nil
}