github.com/sercand/please@v13.4.0+incompatible/src/core/state.go (about)

     1  package core
     2  
     3  import (
     4  	"fmt"
     5  	"io"
     6  	"sort"
     7  	"sync"
     8  	"sync/atomic"
     9  	"time"
    10  
    11  	"github.com/Workiva/go-datastructures/queue"
    12  
    13  	"github.com/thought-machine/please/src/cli"
    14  	"github.com/thought-machine/please/src/fs"
    15  )
    16  
    17  // startTime is as close as we can conveniently get to process start time.
    18  var startTime = time.Now()
    19  
    20  // A TaskType identifies the kind of task returned from NextTask()
    21  type TaskType int
    22  
    23  // The values here are fiddled to make Compare work easily.
    24  // Essentially we prioritise on the higher bits only and use the lower ones to make
    25  // the values unique.
    26  // Subinclude tasks order first, but we're happy for all build / parse / test tasks
    27  // to be treated equivalently.
    28  const (
    29  	Kill            TaskType = 0x0000 | 0
    30  	SubincludeBuild          = 0x1000 | 1
    31  	SubincludeParse          = 0x2000 | 2
    32  	Build                    = 0x4000 | 3
    33  	Parse                    = 0x4000 | 4
    34  	Test                     = 0x4000 | 5
    35  	Stop                     = 0x8000 | 6
    36  	priorityMask             = ^0x00FF
    37  )
    38  
    39  type pendingTask struct {
    40  	Label    BuildLabel // Label of target to parse
    41  	Dependor BuildLabel // The target that depended on it (only for parse tasks)
    42  	Type     TaskType
    43  }
    44  
    45  func (t pendingTask) Compare(that queue.Item) int {
    46  	return int((t.Type & priorityMask) - (that.(pendingTask).Type & priorityMask))
    47  }
    48  
    49  // A Parser is the interface to reading and interacting with BUILD files.
    50  type Parser interface {
    51  	// ParseFile parses a single BUILD file into the given package.
    52  	ParseFile(state *BuildState, pkg *Package, filename string) error
    53  	// ParseReader parses a single BUILD file into the given package.
    54  	ParseReader(state *BuildState, pkg *Package, reader io.ReadSeeker) error
    55  	// RunPreBuildFunction runs a pre-build function for a target.
    56  	RunPreBuildFunction(threadID int, state *BuildState, target *BuildTarget) error
    57  	// RunPostBuildFunction runs a post-build function for a target.
    58  	RunPostBuildFunction(threadID int, state *BuildState, target *BuildTarget, output string) error
    59  }
    60  
    61  // A BuildState tracks the current state of the build & related data.
    62  // As well as tracking the build graph and config, it also tracks the set of current
    63  // tasks and maintains a queue of them, along with various related counters which are
    64  // used to determine when we're finished.
    65  // Tasks are internally tracked by priority, which is determined by their type.
    66  type BuildState struct {
    67  	Graph *BuildGraph
    68  	// Stream of pending tasks
    69  	pendingTasks *queue.PriorityQueue
    70  	// Stream of results from the build
    71  	results chan *BuildResult
    72  	// Stream of results pushed to remote clients.
    73  	remoteResults chan *BuildResult
    74  	// Last results for each thread. These are used to catch up remote clients quickly.
    75  	lastResults []*BuildResult
    76  	// Timestamp that the build is considered to start at.
    77  	StartTime time.Time
    78  	// Various system statistics. Mostly used during remote communication.
    79  	Stats *SystemStats
    80  	// Configuration options
    81  	Config *Configuration
    82  	// Parser implementation. Other things can call this to perform various external parse tasks.
    83  	Parser Parser
    84  	// Worker pool for the parser
    85  	ParsePool Pool
    86  	// Hashes of variouts bits of the configuration, used for incrementality.
    87  	Hashes struct {
    88  		// Hash of the general config, not including specialised bits.
    89  		Config []byte
    90  		// Hash of the config relating to containerisation for tests.
    91  		Containerisation []byte
    92  	}
    93  	// Tracks file hashes during the build.
    94  	PathHasher *fs.PathHasher
    95  	// Level of verbosity during the build
    96  	Verbosity int
    97  	// Cache to store / retrieve old build results.
    98  	Cache Cache
    99  	// Targets that we were originally requested to build
   100  	OriginalTargets []BuildLabel
   101  	// Arguments to tests.
   102  	TestArgs []string
   103  	// Labels of targets that we will include / exclude
   104  	Include, Exclude []string
   105  	// Actual targets to exclude from discovery
   106  	ExcludeTargets []BuildLabel
   107  	// True if we require rule hashes to be correctly verified (usually the case).
   108  	VerifyHashes bool
   109  	// Aggregated coverage for this run
   110  	Coverage TestCoverage
   111  	// True if the build has been successful so far (i.e. nothing has failed yet).
   112  	Success bool
   113  	// True if tests should calculate coverage metrics
   114  	NeedCoverage bool
   115  	// True if we intend to build targets. False if we're just parsing
   116  	// (although some may be built if they're needed for parse).
   117  	NeedBuild bool
   118  	// True if we're running tests. False if we're only building or parsing.
   119  	NeedTests bool
   120  	// True if we will run targets at the end of the build.
   121  	NeedRun bool
   122  	// True if we want to calculate target hashes (ie. 'plz hash').
   123  	NeedHashesOnly bool
   124  	// True if we only want to prepare build directories (ie. 'plz build --prepare')
   125  	PrepareOnly bool
   126  	// True if we're going to run a shell after builds are prepared.
   127  	PrepareShell bool
   128  	// True if we only need to parse the initial package (i.e. don't search downwards
   129  	// through deps) - for example when doing `plz query print`.
   130  	ParsePackageOnly bool
   131  	// True if this build is triggered by watching for changes
   132  	Watch bool
   133  	// Number of times to run each test target. 1 == once each, plus flakes if necessary.
   134  	NumTestRuns int
   135  	// True to clean working directories after successful builds.
   136  	CleanWorkdirs bool
   137  	// True if we're forcing a rebuild of the original targets.
   138  	ForceRebuild bool
   139  	// True to always show test output, even on success.
   140  	ShowTestOutput bool
   141  	// True to print all output of all tasks to stderr.
   142  	ShowAllOutput bool
   143  	// True to attach a debugger on test failure.
   144  	DebugTests bool
   145  	// True once we have killed the workers, so we only do it once.
   146  	workersKilled bool
   147  	// Number of running workers
   148  	numWorkers int
   149  	// Experimental directories
   150  	experimentalLabels []BuildLabel
   151  	// Various items for tracking progress.
   152  	progress *stateProgress
   153  }
   154  
   155  // A stateProgress records various points of progress for a State.
   156  // This is split out from above so we can share it between multiple instances.
   157  type stateProgress struct {
   158  	// Used to count the number of currently active/pending targets
   159  	numActive  int64
   160  	numPending int64
   161  	numRunning int64
   162  	numDone    int64
   163  	mutex      sync.Mutex
   164  	// Used to track subinclude() calls that block until targets are built.
   165  	pendingTargets     map[BuildLabel]chan struct{}
   166  	pendingTargetMutex sync.Mutex
   167  	// Used to track general package parsing requests.
   168  	pendingPackages     map[packageKey]chan struct{}
   169  	pendingPackageMutex sync.Mutex
   170  	// The set of known states
   171  	allStates []*BuildState
   172  }
   173  
   174  // SystemStats stores information about the system.
   175  type SystemStats struct {
   176  	Memory struct {
   177  		Total, Used uint64
   178  		UsedPercent float64
   179  	}
   180  	// This is somewhat abbreviated to the "interesting" values and is aggregated
   181  	// across all CPUs for convenience of display.
   182  	// We're a bit casual about the exact definition of Used (it's the sum of some
   183  	// fields that seem relevant) and IOWait is not fully reliable.
   184  	CPU struct {
   185  		Used, IOWait float64
   186  		Count        int
   187  	}
   188  	// Number of currently running worker processes.
   189  	// N.B. These are background workers as started by $(worker) commands, not the internal worker
   190  	//      threads tracked in the state object.
   191  	NumWorkerProcesses int
   192  }
   193  
   194  // AddActiveTarget increments the counter for a newly active build target.
   195  func (state *BuildState) AddActiveTarget() {
   196  	atomic.AddInt64(&state.progress.numActive, 1)
   197  }
   198  
   199  // AddPendingParse adds a task for a pending parse of a build label.
   200  func (state *BuildState) AddPendingParse(label, dependor BuildLabel, forSubinclude bool) {
   201  	atomic.AddInt64(&state.progress.numActive, 1)
   202  	atomic.AddInt64(&state.progress.numPending, 1)
   203  	if forSubinclude {
   204  		state.pendingTasks.Put(pendingTask{Label: label, Dependor: dependor, Type: SubincludeParse})
   205  	} else {
   206  		state.pendingTasks.Put(pendingTask{Label: label, Dependor: dependor, Type: Parse})
   207  	}
   208  }
   209  
   210  // AddPendingBuild adds a task for a pending build of a target.
   211  func (state *BuildState) AddPendingBuild(label BuildLabel, forSubinclude bool) {
   212  	if forSubinclude {
   213  		state.addPending(label, SubincludeBuild)
   214  	} else {
   215  		state.addPending(label, Build)
   216  	}
   217  }
   218  
   219  // AddPendingTest adds a task for a pending test of a target.
   220  func (state *BuildState) AddPendingTest(label BuildLabel) {
   221  	if state.NeedTests {
   222  		state.addPending(label, Test)
   223  	}
   224  }
   225  
   226  // NextTask receives the next task that should be processed according to the priority queues.
   227  func (state *BuildState) NextTask() (BuildLabel, BuildLabel, TaskType) {
   228  	t, err := state.pendingTasks.Get(1)
   229  	if err != nil {
   230  		log.Fatalf("error receiving next task: %s", err)
   231  	}
   232  	task := t[0].(pendingTask)
   233  	if task.Type == Build || task.Type == SubincludeBuild || task.Type == Test {
   234  		atomic.AddInt64(&state.progress.numRunning, 1)
   235  	}
   236  	return task.Label, task.Dependor, task.Type
   237  }
   238  
   239  func (state *BuildState) addPending(label BuildLabel, t TaskType) {
   240  	atomic.AddInt64(&state.progress.numPending, 1)
   241  	state.pendingTasks.Put(pendingTask{Label: label, Type: t})
   242  }
   243  
   244  // TaskDone indicates that a single task is finished. Should be called after one is finished with
   245  // a task returned from NextTask(), or from a call to ExtraTask().
   246  func (state *BuildState) TaskDone(wasBuildOrTest bool) {
   247  	atomic.AddInt64(&state.progress.numDone, 1)
   248  	if wasBuildOrTest {
   249  		atomic.AddInt64(&state.progress.numRunning, -1)
   250  	}
   251  	if atomic.AddInt64(&state.progress.numPending, -1) <= 0 {
   252  		state.Stop(state.numWorkers)
   253  		state.killall(Stop)
   254  	}
   255  }
   256  
   257  // Stop adds n stop tasks to the list of pending tasks, which stops n workers after all their other tasks are done.
   258  func (state *BuildState) Stop(n int) {
   259  	for i := 0; i < n; i++ {
   260  		state.pendingTasks.Put(pendingTask{Type: Stop})
   261  	}
   262  }
   263  
   264  // KillAll kills all the workers.
   265  func (state *BuildState) KillAll() {
   266  	state.killall(Kill)
   267  }
   268  
   269  func (state *BuildState) killall(signal TaskType) {
   270  	if !state.workersKilled {
   271  		state.workersKilled = true
   272  		for i := 0; i < state.numWorkers; i++ {
   273  			state.pendingTasks.Put(pendingTask{Type: signal})
   274  		}
   275  		if state.results != nil {
   276  			close(state.results)
   277  		}
   278  		if state.remoteResults != nil {
   279  			close(state.remoteResults)
   280  		}
   281  	}
   282  }
   283  
   284  // DelayedKillAll waits until no workers are running
   285  func (state *BuildState) DelayedKillAll() {
   286  	for state.anyRunningTasks() {
   287  	}
   288  	if state.progress.numPending > 0 {
   289  		log.Error("All workers seem deadlocked, stopping.")
   290  		state.KillAll()
   291  	}
   292  }
   293  
   294  // anyRunningTasks checks over a little while whether there are any tasks still running and
   295  // returns true if so.
   296  func (state *BuildState) anyRunningTasks() bool {
   297  	for i := 0; i < 10; i++ {
   298  		if state.progress.numRunning > 0 {
   299  			return true
   300  		}
   301  		time.Sleep(10 * time.Millisecond) // Give it a little time to see if anything wakes.
   302  	}
   303  	return state.progress.numRunning > 0
   304  }
   305  
   306  // IsOriginalTarget returns true if a target is an original target, ie. one specified on the command line.
   307  func (state *BuildState) IsOriginalTarget(label BuildLabel) bool {
   308  	return state.isOriginalTarget(label, false)
   309  }
   310  
   311  // isOriginalTarget implementsIsOriginalTarget, optionally allowing disabling matching :all labels.
   312  func (state *BuildState) isOriginalTarget(label BuildLabel, exact bool) bool {
   313  	for _, original := range state.OriginalTargets {
   314  		if original == label || (!exact && original.IsAllTargets() && original.PackageName == label.PackageName) {
   315  			return true
   316  		}
   317  	}
   318  	return false
   319  }
   320  
   321  // SetIncludeAndExclude sets the include / exclude labels.
   322  // Handles build labels on Exclude so should be preferred over setting them directly.
   323  func (state *BuildState) SetIncludeAndExclude(include, exclude []string) {
   324  	state.Include = include
   325  	for _, e := range exclude {
   326  		if LooksLikeABuildLabel(e) {
   327  			if label, err := parseMaybeRelativeBuildLabel(e, ""); err != nil {
   328  				log.Fatalf("%s", err)
   329  			} else {
   330  				state.ExcludeTargets = append(state.ExcludeTargets, label)
   331  			}
   332  		} else {
   333  			state.Exclude = append(state.Exclude, e)
   334  		}
   335  	}
   336  }
   337  
   338  // ShouldInclude returns true if the given target is included by the include/exclude flags.
   339  func (state *BuildState) ShouldInclude(target *BuildTarget) bool {
   340  	for _, e := range state.ExcludeTargets {
   341  		if e.Includes(target.Label) {
   342  			return false
   343  		}
   344  	}
   345  	return target.ShouldInclude(state.Include, state.Exclude)
   346  }
   347  
   348  // AddOriginalTarget adds one of the original targets and enqueues it for parsing / building.
   349  func (state *BuildState) AddOriginalTarget(label BuildLabel, addToList bool) {
   350  	// Check it's not excluded first.
   351  	for _, e := range state.ExcludeTargets {
   352  		if e.Includes(label) {
   353  			return
   354  		}
   355  	}
   356  	if addToList {
   357  		// The sets of original targets are duplicated between states for all architectures,
   358  		// we must add it to all of them to ensure everything sees the same set.
   359  		for _, s := range state.progress.allStates {
   360  			s.OriginalTargets = append(s.OriginalTargets, label)
   361  		}
   362  	}
   363  	state.AddPendingParse(label, OriginalTarget, false)
   364  }
   365  
   366  // LogBuildResult logs the result of a target either building or parsing.
   367  func (state *BuildState) LogBuildResult(tid int, label BuildLabel, status BuildResultStatus, description string) {
   368  	if status == PackageParsed {
   369  		// We may have parse tasks waiting for this package to exist, check for them.
   370  		state.progress.pendingPackageMutex.Lock()
   371  		if ch, present := state.progress.pendingPackages[packageKey{Name: label.PackageName, Subrepo: label.Subrepo}]; present {
   372  			close(ch) // This signals to anyone waiting that it's done.
   373  		}
   374  		state.progress.pendingPackageMutex.Unlock()
   375  		return // We don't notify anything else on these.
   376  	}
   377  	state.LogResult(&BuildResult{
   378  		ThreadID:    tid,
   379  		Time:        time.Now(),
   380  		Label:       label,
   381  		Status:      status,
   382  		Err:         nil,
   383  		Description: description,
   384  	})
   385  	if status == TargetBuilt || status == TargetCached {
   386  		// We may have parse tasks waiting for this guy to build, check for them.
   387  		state.progress.pendingTargetMutex.Lock()
   388  		if ch, present := state.progress.pendingTargets[label]; present {
   389  			close(ch) // This signals to anyone waiting that it's done.
   390  		}
   391  		state.progress.pendingTargetMutex.Unlock()
   392  	}
   393  }
   394  
   395  // LogTestResult logs the result of a target once its tests have completed.
   396  func (state *BuildState) LogTestResult(tid int, label BuildLabel, status BuildResultStatus, results *TestSuite, coverage *TestCoverage, err error, format string, args ...interface{}) {
   397  	state.LogResult(&BuildResult{
   398  		ThreadID:    tid,
   399  		Time:        time.Now(),
   400  		Label:       label,
   401  		Status:      status,
   402  		Err:         err,
   403  		Description: fmt.Sprintf(format, args...),
   404  		Tests:       *results,
   405  	})
   406  	state.progress.mutex.Lock()
   407  	defer state.progress.mutex.Unlock()
   408  	state.Coverage.Aggregate(coverage)
   409  }
   410  
   411  // LogBuildError logs a failure for a target to parse, build or test.
   412  func (state *BuildState) LogBuildError(tid int, label BuildLabel, status BuildResultStatus, err error, format string, args ...interface{}) {
   413  	state.LogResult(&BuildResult{
   414  		ThreadID:    tid,
   415  		Time:        time.Now(),
   416  		Label:       label,
   417  		Status:      status,
   418  		Err:         err,
   419  		Description: fmt.Sprintf(format, args...),
   420  	})
   421  }
   422  
   423  // LogResult logs a build result directly to the state's queue.
   424  func (state *BuildState) LogResult(result *BuildResult) {
   425  	defer func() {
   426  		if r := recover(); r != nil {
   427  			// This is basically always "send on closed channel" which can happen because this
   428  			// channel gets closed while there might still be some other workers doing stuff.
   429  			// At that point we don't care much because the build has already failed.
   430  			log.Notice("%s", r)
   431  		}
   432  	}()
   433  	if state.results != nil {
   434  		state.results <- result
   435  	}
   436  	if state.remoteResults != nil {
   437  		state.remoteResults <- result
   438  		state.lastResults[result.ThreadID] = result
   439  	}
   440  	if result.Status.IsFailure() {
   441  		state.Success = false
   442  	}
   443  }
   444  
   445  // Results returns a channel on which the caller can listen for results.
   446  func (state *BuildState) Results() <-chan *BuildResult {
   447  	if state.results == nil {
   448  		state.results = make(chan *BuildResult, 100*state.numWorkers)
   449  	}
   450  	return state.results
   451  }
   452  
   453  // RemoteResults returns a channel for distributing remote results too, as well as
   454  // the last set of results per thread.
   455  func (state *BuildState) RemoteResults() (<-chan *BuildResult, []*BuildResult) {
   456  	if state.remoteResults == nil {
   457  		state.remoteResults = make(chan *BuildResult, 100*state.numWorkers)
   458  	}
   459  	return state.remoteResults, state.lastResults
   460  }
   461  
   462  // NumActive returns the number of currently active tasks (i.e. those that are
   463  // scheduled to be built at some point, or have been built already).
   464  func (state *BuildState) NumActive() int {
   465  	return int(atomic.LoadInt64(&state.progress.numActive))
   466  }
   467  
   468  // NumDone returns the number of tasks that have been completed so far.
   469  func (state *BuildState) NumDone() int {
   470  	return int(atomic.LoadInt64(&state.progress.numDone))
   471  }
   472  
   473  // SetTaskNumbers allows a caller to set the number of active and done tasks.
   474  // This may drastically confuse matters if used incorrectly.
   475  func (state *BuildState) SetTaskNumbers(active, done int64) {
   476  	atomic.StoreInt64(&state.progress.numActive, active)
   477  	atomic.StoreInt64(&state.progress.numDone, done)
   478  }
   479  
   480  // ExpandOriginalTargets expands any pseudo-targets (ie. :all, ... has already been resolved to a bunch :all targets)
   481  // from the set of original targets.
   482  // Deprecated: Callers should use ExpandOriginalLabels instead.
   483  func (state *BuildState) ExpandOriginalTargets() BuildLabels {
   484  	return state.ExpandOriginalLabels()
   485  }
   486  
   487  // ExpandOriginalLabels expands any pseudo-labels (ie. :all, ... has already been resolved to a bunch :all targets)
   488  // from the set of original labels.
   489  func (state *BuildState) ExpandOriginalLabels() BuildLabels {
   490  	ret := BuildLabels{}
   491  	for _, label := range state.OriginalTargets {
   492  		if label.IsAllTargets() || label.IsAllSubpackages() {
   493  			ret = append(ret, state.expandOriginalPseudoTarget(label)...)
   494  		} else {
   495  			ret = append(ret, label)
   496  		}
   497  	}
   498  	return ret
   499  }
   500  
   501  // expandOriginalPseudoTarget expands one original pseudo-target (i.e. :all or /...) and sorts it
   502  func (state *BuildState) expandOriginalPseudoTarget(label BuildLabel) BuildLabels {
   503  	ret := BuildLabels{}
   504  	addPackage := func(pkg *Package) {
   505  		for _, target := range pkg.AllTargets() {
   506  			if target.ShouldInclude(state.Include, state.Exclude) && (!state.NeedTests || target.IsTest) {
   507  				ret = append(ret, target.Label)
   508  			}
   509  		}
   510  	}
   511  	if label.IsAllTargets() {
   512  		if pkg := state.Graph.PackageByLabel(label); pkg != nil {
   513  			addPackage(pkg)
   514  		} else {
   515  			log.Warning("Package %s does not exist in graph", label.PackageName)
   516  		}
   517  	} else {
   518  		for name, pkg := range state.Graph.PackageMap() {
   519  			if label.Includes(BuildLabel{PackageName: name}) {
   520  				addPackage(pkg)
   521  			}
   522  		}
   523  	}
   524  	sort.Sort(ret)
   525  	return ret
   526  }
   527  
   528  // ExpandVisibleOriginalTargets expands any pseudo-targets (ie. :all, ... has already been resolved to a bunch :all targets)
   529  // from the set of original targets. Hidden targets are not included.
   530  func (state *BuildState) ExpandVisibleOriginalTargets() BuildLabels {
   531  	ret := BuildLabels{}
   532  	for _, target := range state.ExpandOriginalTargets() {
   533  		if !target.HasParent() || state.isOriginalTarget(target, true) {
   534  			ret = append(ret, target)
   535  		}
   536  	}
   537  	return ret
   538  }
   539  
   540  // WaitForPackage either returns the given package which is already parsed and available,
   541  // or returns nil if nothing's parsed it already, in which case everything else calling this
   542  // will wait for the caller to parse it themselves.
   543  func (state *BuildState) WaitForPackage(label BuildLabel) *Package {
   544  	if p := state.Graph.PackageByLabel(label); p != nil {
   545  		return p
   546  	}
   547  	key := packageKey{Name: label.PackageName, Subrepo: label.Subrepo}
   548  	state.progress.pendingPackageMutex.Lock()
   549  	if ch, present := state.progress.pendingPackages[key]; present {
   550  		state.progress.pendingPackageMutex.Unlock()
   551  		state.ParsePool.AddWorker()
   552  		<-ch
   553  		state.ParsePool.StopWorker()
   554  		return state.Graph.PackageByLabel(label)
   555  	}
   556  	// Nothing's registered this so we do it ourselves.
   557  	state.progress.pendingPackages[key] = make(chan struct{})
   558  	state.progress.pendingPackageMutex.Unlock()
   559  	return state.Graph.PackageByLabel(label) // Important to check again; it's possible to race against this whole lot.
   560  }
   561  
   562  // WaitForBuiltTarget blocks until the given label is available as a build target and has been successfully built.
   563  func (state *BuildState) WaitForBuiltTarget(l, dependor BuildLabel) *BuildTarget {
   564  	if t := state.Graph.Target(l); t != nil {
   565  		if state := t.State(); state >= Built && state != Failed {
   566  			return t
   567  		}
   568  	}
   569  	dependor.Name = "all" // Every target in this package depends on this one.
   570  	// okay, we need to register and wait for this guy.
   571  	state.progress.pendingTargetMutex.Lock()
   572  	if ch, present := state.progress.pendingTargets[l]; present {
   573  		// Something's already registered for this, get on the train
   574  		state.progress.pendingTargetMutex.Unlock()
   575  		log.Debug("Pausing parse of %s to wait for %s", dependor, l)
   576  		state.ParsePool.AddWorker()
   577  		<-ch
   578  		state.ParsePool.StopWorker()
   579  		log.Debug("Resuming parse of %s now %s is ready", dependor, l)
   580  		return state.Graph.Target(l)
   581  	}
   582  	// Nothing's registered this, set it up.
   583  	state.progress.pendingTargets[l] = make(chan struct{})
   584  	state.progress.pendingTargetMutex.Unlock()
   585  	state.AddPendingParse(l, dependor, true)
   586  	// Do this all over; the re-checking that happens here is actually fairly important to resolve
   587  	// a potential race condition if the target was built between us checking earlier and registering
   588  	// the channel just now.
   589  	return state.WaitForBuiltTarget(l, dependor)
   590  }
   591  
   592  // ForTarget returns the state associated with a given target.
   593  // This differs if the target is in a subrepo for a different architecture.
   594  func (state *BuildState) ForTarget(target *BuildTarget) *BuildState {
   595  	if target.Subrepo != nil && target.Subrepo.State != nil {
   596  		return target.Subrepo.State
   597  	}
   598  	return state
   599  }
   600  
   601  // ForArch creates a copy of this BuildState for a different architecture.
   602  func (state *BuildState) ForArch(arch cli.Arch) *BuildState {
   603  	// Check if we've got this one already.
   604  	// N.B. This implicitly handles the case of the host architecture
   605  	if s := state.findArch(arch); s != nil {
   606  		return s
   607  	}
   608  	// Copy with the architecture-specific config file.
   609  	// This is slightly wrong in that other things (e.g. user-specified command line overrides) should
   610  	// in fact take priority over this, but that's a lot more fiddly to get right.
   611  	s := state.ForConfig(".plzconfig_" + arch.String())
   612  	s.Config.Build.Arch = arch
   613  	return s
   614  }
   615  
   616  // findArch returns an existing state for the given architecture, if one exists.
   617  func (state *BuildState) findArch(arch cli.Arch) *BuildState {
   618  	state.progress.mutex.Lock()
   619  	defer state.progress.mutex.Unlock()
   620  	for _, s := range state.progress.allStates {
   621  		if s.Config.Build.Arch == arch {
   622  			return s
   623  		}
   624  	}
   625  	return nil
   626  }
   627  
   628  // ForConfig creates a copy of this BuildState based on the given config files.
   629  func (state *BuildState) ForConfig(config ...string) *BuildState {
   630  	state.progress.mutex.Lock()
   631  	defer state.progress.mutex.Unlock()
   632  	// Duplicate & alter configuration
   633  	c := &Configuration{}
   634  	*c = *state.Config
   635  	c.buildEnvStored = &storedBuildEnv{}
   636  	for _, filename := range config {
   637  		if err := readConfigFile(c, filename); err != nil {
   638  			log.Fatalf("Failed to read config file %s: %s", filename, err)
   639  		}
   640  	}
   641  	s := &BuildState{}
   642  	*s = *state
   643  	s.Config = c
   644  	state.progress.allStates = append(state.progress.allStates, s)
   645  	return s
   646  }
   647  
   648  // NewBuildState constructs and returns a new BuildState.
   649  // Everyone should use this rather than attempting to construct it themselves;
   650  // callers can't initialise all the required private fields.
   651  func NewBuildState(numThreads int, cache Cache, verbosity int, config *Configuration) *BuildState {
   652  	state := &BuildState{
   653  		Graph:        NewGraph(),
   654  		pendingTasks: queue.NewPriorityQueue(10000, true), // big hint, why not
   655  		lastResults:  make([]*BuildResult, numThreads),
   656  		PathHasher:   fs.NewPathHasher(RepoRoot),
   657  		StartTime:    startTime,
   658  		Config:       config,
   659  		Verbosity:    verbosity,
   660  		Cache:        cache,
   661  		ParsePool:    NewPool(numThreads),
   662  		VerifyHashes: true,
   663  		NeedBuild:    true,
   664  		Success:      true,
   665  		Coverage:     TestCoverage{Files: map[string][]LineCoverage{}},
   666  		numWorkers:   numThreads,
   667  		Stats:        &SystemStats{},
   668  		progress: &stateProgress{
   669  			numActive:       1, // One for the initial target adding on the main thread.
   670  			numRunning:      1, // Similarly.
   671  			numPending:      1,
   672  			pendingTargets:  map[BuildLabel]chan struct{}{},
   673  			pendingPackages: map[packageKey]chan struct{}{},
   674  		},
   675  	}
   676  	state.progress.allStates = []*BuildState{state}
   677  	state.Hashes.Config = config.Hash()
   678  	state.Hashes.Containerisation = config.ContainerisationHash()
   679  	config.Please.NumThreads = numThreads
   680  	for _, exp := range config.Parse.ExperimentalDir {
   681  		state.experimentalLabels = append(state.experimentalLabels, BuildLabel{PackageName: exp, Name: "..."})
   682  	}
   683  	return state
   684  }
   685  
   686  // NewDefaultBuildState creates a BuildState for the default configuration.
   687  // This is useful for tests etc that don't need to customise anything about it.
   688  func NewDefaultBuildState() *BuildState {
   689  	return NewBuildState(1, nil, 4, DefaultConfiguration())
   690  }
   691  
   692  // A BuildResult represents a single event in the build process, i.e. a target starting or finishing
   693  // building, or reaching some milestone within those steps.
   694  type BuildResult struct {
   695  	// Thread id (or goroutine id, really) that generated this result.
   696  	ThreadID int
   697  	// Timestamp of this event
   698  	Time time.Time
   699  	// Target which has just changed
   700  	Label BuildLabel
   701  	// Its current status
   702  	Status BuildResultStatus
   703  	// Error, only populated for failure statuses
   704  	Err error
   705  	// Description of what's going on right now.
   706  	Description string
   707  	// Test results
   708  	Tests TestSuite
   709  }
   710  
   711  // A BuildResultStatus represents the status of a target when we log a build result.
   712  type BuildResultStatus int
   713  
   714  // The collection of expected build result statuses.
   715  const (
   716  	PackageParsing BuildResultStatus = iota
   717  	PackageParsed
   718  	ParseFailed
   719  	TargetBuilding
   720  	TargetBuildStopped
   721  	TargetBuilt
   722  	TargetCached
   723  	TargetBuildFailed
   724  	TargetTesting
   725  	TargetTestStopped
   726  	TargetTested
   727  	TargetTestFailed
   728  )
   729  
   730  // Category returns the broad area that this event represents in the tasks we perform for a target.
   731  func (s BuildResultStatus) Category() string {
   732  	switch s {
   733  	case PackageParsing, PackageParsed, ParseFailed:
   734  		return "Parse"
   735  	case TargetBuilding, TargetBuildStopped, TargetBuilt, TargetBuildFailed:
   736  		return "Build"
   737  	case TargetTesting, TargetTestStopped, TargetTested, TargetTestFailed:
   738  		return "Test"
   739  	default:
   740  		return "Other"
   741  	}
   742  }
   743  
   744  // IsFailure returns true if this status represents a failure.
   745  func (s BuildResultStatus) IsFailure() bool {
   746  	return s == ParseFailed || s == TargetBuildFailed || s == TargetTestFailed
   747  }
   748  
   749  // IsActive returns true if this status represents a target that is not yet finished.
   750  func (s BuildResultStatus) IsActive() bool {
   751  	return s == PackageParsing || s == TargetBuilding || s == TargetTesting
   752  }