github.com/tiagovtristao/plz@v13.4.0+incompatible/src/parse/parse_step.go

// Package parse implements parsing of the BUILD files via an embedded Python interpreter.
//
// The actual work here is done by an embedded PyPy instance. Various rules are built in to
// the binary itself using go-bindata to embed the .py files; these are always available to
// all programs which is rather nice, but it does mean that must be run before 'go run' etc
// will work as expected.
package parse

import (
	"fmt"
	"path"
	"strings"

	"gopkg.in/op/go-logging.v1"

	"github.com/thought-machine/please/src/cli"
	"github.com/thought-machine/please/src/core"
	"github.com/thought-machine/please/src/fs"
	"github.com/thought-machine/please/src/worker"
)

var log = logging.MustGetLogger("parse")

// Parse parses the package corresponding to a single build label. The label can be :all to add all targets in a package.
// It is not an error if the package has already been parsed.
//
// By default, after the package is parsed, any targets that are now needed for the build and ready
// to be built are queued, and any new packages are queued for parsing. When a specific label is requested
// this is straightforward, but when parsing for pseudo-targets like :all and ..., various flags affect it:
// 'include' and 'exclude' refer to the labels of targets to be added. If 'include' is non-empty then only
// targets with at least one matching label are added. Any targets with a label in 'exclude' are not added.
// 'forSubinclude' is set when the parse is required for a subinclude target so should proceed
// even when we're not otherwise building targets.
func Parse(tid int, state *core.BuildState, label, dependor core.BuildLabel, include, exclude []string, forSubinclude bool) {
	if err := parse(tid, state, label, dependor, include, exclude, forSubinclude); err != nil {
		state.LogBuildError(tid, label, core.ParseFailed, err, "Failed to parse package")
	}
}

func parse(tid int, state *core.BuildState, label, dependor core.BuildLabel, include, exclude []string, forSubinclude bool) error {
	// See if something else has parsed this package first.
	pkg := state.WaitForPackage(label)
	if pkg != nil {
		// Does exist, all we need to do is toggle on this target
		return activateTarget(state, pkg, label, dependor, forSubinclude, include, exclude)
	}
	// If we get here then it falls to us to parse this package.
	state.LogBuildResult(tid, label, core.PackageParsing, "Parsing...")

	subrepo, err := checkSubrepo(tid, state, label, dependor)
	if err != nil {
		return err
	} else if subrepo != nil && subrepo.Target != nil {
		// We have got the definition of the subrepo but it depends on something, make sure that has been built.
		state.WaitForBuiltTarget(subrepo.Target.Label, label)
	}
	pkg, err = parsePackage(state, label, dependor, subrepo)
	if err != nil {
		return err
	}
	state.LogBuildResult(tid, label, core.PackageParsed, "Parsed package")
	return activateTarget(state, pkg, label, dependor, forSubinclude, include, exclude)
}

// checkSubrepo checks whether this guy exists within a subrepo. If so we will need to make sure that's available first.
func checkSubrepo(tid int, state *core.BuildState, label, dependor core.BuildLabel) (*core.Subrepo, error) {
	if label.Subrepo == "" {
		return nil, nil
	} else if subrepo := state.Graph.Subrepo(label.Subrepo); subrepo != nil {
		return subrepo, nil
	}
	// We don't have the definition of it at all. Need to parse that first.
	sl := label.SubrepoLabel()
	if err := parseSubrepoPackage(tid, state, sl.PackageName, "", label); err != nil {
		return nil, err
	} else if err := parseSubrepoPackage(tid, state, sl.PackageName, dependor.Subrepo, label); err != nil {
		return nil, err
	}
	if subrepo := state.Graph.Subrepo(label.Subrepo); subrepo != nil {
		return subrepo, nil
	} else if subrepo := checkArchSubrepo(state, label.Subrepo); subrepo != nil {
		return subrepo, nil
	}
	return nil, fmt.Errorf("Subrepo %s is not defined", label.Subrepo)
}

// parseSubrepoPackage parses a package to make sure subrepos are available.
func parseSubrepoPackage(tid int, state *core.BuildState, pkg, subrepo string, dependor core.BuildLabel) error {
	if state.Graph.Package(pkg, subrepo) == nil {
		// Don't have it already, must parse.
		label := core.BuildLabel{Subrepo: subrepo, PackageName: pkg, Name: "all"}
		return parse(tid, state, label, dependor, nil, nil, true)
	}
	return nil
}

// checkArchSubrepo checks if a target refers to a cross-compiling subrepo.
// Those don't have to be explicitly defined - maybe we should insist on that, but it's nicer not to have to.
func checkArchSubrepo(state *core.BuildState, name string) *core.Subrepo {
	var arch cli.Arch
	if err := arch.UnmarshalFlag(name); err == nil {
		return state.Graph.MaybeAddSubrepo(core.SubrepoForArch(state, arch))
	}
	return nil
}

// activateTarget marks a target as active (ie. to be built) and adds its dependencies as pending parses.
func activateTarget(state *core.BuildState, pkg *core.Package, label, dependor core.BuildLabel, forSubinclude bool, include, exclude []string) error {
	if !label.IsAllTargets() && state.Graph.Target(label) == nil {
		if label.Subrepo == "" && label.PackageName == "" && label.Name == dependor.Subrepo {
			if subrepo := checkArchSubrepo(state, label.Name); subrepo != nil {
				return nil
			}
		}
		msg := fmt.Sprintf("Parsed build file %s but it doesn't contain target %s", pkg.Filename, label.Name)
		if dependor != core.OriginalTarget {
			msg += fmt.Sprintf(" (depended on by %s)", dependor)
		}
		return fmt.Errorf(msg + suggestTargets(pkg, label, dependor))
	}
	if state.ParsePackageOnly && !forSubinclude {
		return nil // Some kinds of query don't need a full recursive parse.
	} else if label.IsAllTargets() {
		for _, target := range pkg.AllTargets() {
			// Don't activate targets that were added in a post-build function; that causes a race condition
			// between the post-build functions running and other things trying to activate them too early.
			if state.ShouldInclude(target) && !target.AddedPostBuild {
				// Must always do this for coverage because we need to calculate sources of
				// non-test targets later on.
				if !state.NeedTests || target.IsTest || state.NeedCoverage {
					addDep(state, target.Label, dependor, false, dependor.IsAllTargets())
				}
			}
		}
	} else {
		for _, l := range state.Graph.DependentTargets(dependor, label) {
			// We use :all to indicate a dependency needed for parse.
			addDep(state, l, dependor, false, forSubinclude || dependor.IsAllTargets())
		}
	}
	return nil
}

// parsePackage performs the initial parse of a package.
func parsePackage(state *core.BuildState, label, dependor core.BuildLabel, subrepo *core.Subrepo) (*core.Package, error) {
	packageName := label.PackageName
	pkg := core.NewPackage(packageName)
	pkg.Subrepo = subrepo
	if subrepo != nil {
		pkg.SubrepoName = subrepo.Name
	}
	filename, dir := buildFileName(state, label.PackageName, subrepo)
	if filename == "" {
		if success, err := providePackage(state, pkg); err != nil {
			return nil, err
		} else if !success && packageName == "" && dependor.Subrepo == "pleasings" && subrepo == nil && state.Config.Parse.BuiltinPleasings {
			// Deliberate fallthrough, for the case where someone depended on the default
			// @pleasings subrepo, and there is no BUILD file at their root.
		} else if !success {
			exists := core.PathExists(dir)
			// Handle quite a few cases to provide more obvious error messages.
			if dependor != core.OriginalTarget && exists {
				return nil, fmt.Errorf("%s depends on %s, but there's no BUILD file in %s/", dependor, label, dir)
			} else if dependor != core.OriginalTarget {
				return nil, fmt.Errorf("%s depends on %s, but the directory %s doesn't exist", dependor, label, dir)
			} else if exists {
				return nil, fmt.Errorf("Can't build %s; there's no BUILD file in %s/", label, dir)
			}
			return nil, fmt.Errorf("Can't build %s; the directory %s doesn't exist", label, dir)
		}
	} else {
		pkg.Filename = filename
		if err := state.Parser.ParseFile(state, pkg, pkg.Filename); err != nil {
			return nil, err
		}
	}
	// If the config setting is on, we "magically" register a default repo called @pleasings.
	if packageName == "" && subrepo == nil && state.Config.Parse.BuiltinPleasings && pkg.Target("pleasings") == nil {
		if _, err := state.Parser.(*aspParser).asp.ParseReader(pkg, strings.NewReader(pleasings)); err != nil {
			log.Fatalf("Failed to load pleasings: %s", err) // This shouldn't happen, of course.
		}
	}
	addPackage(state, pkg)
	return pkg, nil
}

// addPackage adds the given package to the graph, with appropriate dependencies and whatnot.
func addPackage(state *core.BuildState, pkg *core.Package) {
	allTargets := pkg.AllTargets()
	for _, target := range allTargets {
		state.Graph.AddTarget(target)
		if target.IsFilegroup {
			// At least register these guys as outputs.
			// It's difficult to handle non-file sources because we don't know if they're
			// parsed yet - recall filegroups are a special case for this since they don't
			// explicitly declare their outputs but can re-output other rules' outputs.
			for _, src := range target.AllLocalSources() {
				pkg.MustRegisterOutput(src, target)
			}
		} else {
			for _, out := range target.DeclaredOutputs() {
				pkg.MustRegisterOutput(out, target)
			}
			for _, out := range target.TestOutputs {
				if !fs.IsGlob(out) {
					pkg.MustRegisterOutput(out, target)
				}
			}
		}
	}
	// Do this in a separate loop so we get intra-package dependencies right now.
	for _, target := range allTargets {
		for _, dep := range target.DeclaredDependencies() {
			state.Graph.AddDependency(target.Label, dep)
		}
	}
	// Verify some details of the output files in the background. Don't need to wait for this
	// since it only issues warnings sometimes.
	go pkg.VerifyOutputs()
	state.Graph.AddPackage(pkg) // Calling this means nobody else will add entries to pendingTargets for this package.
}

// buildFileName returns the name of the BUILD file for a package, or the empty string if one
// doesn't exist. It also returns the directory that it looked in.
func buildFileName(state *core.BuildState, pkgName string, subrepo *core.Subrepo) (string, string) {
	config := state.Config
	if subrepo != nil {
		pkgName = subrepo.Dir(pkgName)
		config = subrepo.State.Config
	}
	// Bazel defines targets in its "external" package from its WORKSPACE file.
	// We will fake this by treating that as an actual package file...
	// TODO(peterebden): They may be moving away from their "external" nomenclature?
	if state.Config.Bazel.Compatibility && pkgName == "external" || pkgName == "workspace" {
		return "WORKSPACE", ""
	}
	for _, buildFileName := range config.Parse.BuildFileName {
		if filename := path.Join(core.RepoRoot, pkgName, buildFileName); fs.FileExists(filename) {
			return filename, pkgName
		}
	}
	return "", pkgName
}

// Adds a single target to the build queue.
func addDep(state *core.BuildState, label, dependor core.BuildLabel, rescan, forceBuild bool) {
	// Stop at any package that's not loaded yet
	if state.Graph.PackageByLabel(label) == nil {
		if forceBuild {
			log.Debug("Adding forced pending parse of %s", label)
		}
		state.AddPendingParse(label, dependor, forceBuild)
		return
	}
	target := state.Graph.Target(label)
	if target == nil {
		log.Fatalf("Target %s (referenced by %s) doesn't exist\n", label, dependor)
	}
	if target.State() >= core.Active && !rescan && !forceBuild {
		return // Target is already tagged to be built and likely on the queue.
	}
	// Only do this bit if we actually need to build the target
	if !target.SyncUpdateState(core.Inactive, core.Semiactive) && !rescan && !forceBuild {
		return
	}
	if state.NeedBuild || forceBuild {
		if target.SyncUpdateState(core.Semiactive, core.Active) {
			state.AddActiveTarget()
			if target.IsTest && state.NeedTests {
				state.AddActiveTarget() // Tests count twice if we're gonna run them.
			}
		}
	}
	// If this target has no deps, add it to the queue now, otherwise handle its deps.
	// Only add if we need to build targets (not if we're just parsing) but we might need it to parse...
	if target.State() == core.Active && state.Graph.AllDepsBuilt(target) {
		if target.SyncUpdateState(core.Active, core.Pending) {
			state.AddPendingBuild(label, dependor.IsAllTargets())
		}
		if !rescan {
			return
		}
	}
	for _, dep := range target.DeclaredDependencies() {
		// Check the require/provide stuff; we may need to add a different target.
		if len(target.Requires) > 0 {
			if depTarget := state.Graph.Target(dep); depTarget != nil && len(depTarget.Provides) > 0 {
				for _, provided := range depTarget.ProvideFor(target) {
					addDep(state, provided, label, false, forceBuild)
				}
				continue
			}
		}
		addDep(state, dep, label, false, forceBuild)
	}
}

func rescanDeps(state *core.BuildState, changed map[*core.BuildTarget]struct{}) {
	// Run over all the changed targets in this package and ensure that any newly added dependencies enter the build queue.
	for target := range changed {
		if !state.Graph.AllDependenciesResolved(target) {
			for _, dep := range target.DeclaredDependencies() {
				state.Graph.AddDependency(target.Label, dep)
			}
		}
		if s := target.State(); s < core.Built && s > core.Inactive {
			addDep(state, target.Label, core.OriginalTarget, true, false)
		}
	}
}

// This is the builtin subrepo for pleasings.
// TODO(peterebden): Should really provide a github_archive builtin that knows how to construct
//                   the URL and strip_prefix etc.
const pleasings = `
http_archive(
    name = "pleasings",
    strip_prefix = "pleasings-master",
    urls = ["https://github.com/thought-machine/pleasings/archive/master.zip"],
)
`

// providePackage looks through all the configured BUILD file providers to see if any of them
// can handle the given package. It returns true if any of them did.
// N.B. More than one is allowed to handle a single directory.
func providePackage(state *core.BuildState, pkg *core.Package) (bool, error) {
	if len(state.Config.Provider) == 0 {
		return false, nil
	}
	success := false
	label := pkg.Label()
	for name, p := range state.Config.Provider {
		if !shouldProvide(p.Path, label) {
			continue
		}
		t := state.WaitForBuiltTarget(p.Target, label)
		outs := t.Outputs()
		if !t.IsBinary && len(outs) != 1 {
			log.Error("Cannot use %s as build provider %s, it must be a binary with exactly 1 output.", p.Target, name)
			continue
		}
		dir := pkg.SourceRoot()
		resp, err := worker.ProvideParse(state, path.Join(t.OutDir(), outs[0]), dir)
		if err != nil {
			return false, fmt.Errorf("Failed to start build provider %s: %s", name, err)
		} else if resp != "" {
			log.Debug("Received BUILD file from %s provider for %s: %s", name, dir, resp)
			if err := state.Parser.ParseReader(state, pkg, strings.NewReader(resp)); err != nil {
				return false, err
			}
			success = true
		}
	}
	return success, nil
}

// shouldProvide returns true if a provider's set of configured paths overlaps a package.
func shouldProvide(paths []core.BuildLabel, label core.BuildLabel) bool {
	for _, p := range paths {
		if p.Includes(label) {
			return true
		}
	}
	return false
}