github.com/maruel/nin@v0.0.0-20220112143044-f35891e3ce7e/build.go (about) 1 // Copyright 2011 Google Inc. All Rights Reserved. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package nin 16 17 import ( 18 "errors" 19 "fmt" 20 "os" 21 "time" 22 ) 23 24 type edgeResult bool 25 26 const ( 27 edgeFailed edgeResult = false 28 edgeSucceeded edgeResult = true 29 ) 30 31 // Want enumerates possible steps we want for an edge. 32 type Want int32 33 34 const ( 35 // WantNothing means we do not want to build the edge, but we might want to 36 // build one of its dependents. 37 WantNothing Want = iota 38 // WantToStart means we want to build the edge, but have not yet scheduled 39 // it. 40 WantToStart 41 // WantToFinish means we want to build the edge, have scheduled it, and are 42 // waiting for it to complete. 43 WantToFinish 44 ) 45 46 // commandRunner is an interface that wraps running the build 47 // subcommands. This allows tests to abstract out running commands. 48 // RealCommandRunner is an implementation that actually runs commands. 49 type commandRunner interface { 50 CanRunMore() bool 51 StartCommand(edge *Edge) bool 52 53 /// Wait for a command to complete, or return false if interrupted. 54 WaitForCommand(result *Result) bool 55 56 GetActiveEdges() []*Edge 57 Abort() 58 } 59 60 // Result is the result of waiting for a command. 61 type Result struct { 62 Edge *Edge 63 ExitCode ExitStatus 64 Output string 65 } 66 67 // TODO(maruel): The build per se shouldn't have verbosity as a flag. It should 68 // be composed. 69 70 // BuildConfig are the options (e.g. verbosity, parallelism) passed to a build. 71 type BuildConfig struct { 72 Verbosity Verbosity 73 DryRun bool 74 Parallelism int 75 FailuresAllowed int 76 // The maximum load average we must not exceed. A negative or zero value 77 // means that we do not have any limit. 78 MaxLoadAvg float64 79 } 80 81 // NewBuildConfig returns the default build configuration. 82 func NewBuildConfig() BuildConfig { 83 return BuildConfig{ 84 Verbosity: Normal, 85 Parallelism: 1, 86 FailuresAllowed: 1, 87 } 88 } 89 90 // Verbosity controls the verbosity of the status updates. 91 type Verbosity int32 92 93 const ( 94 // Quiet means no output -- used when testing. 95 Quiet Verbosity = iota 96 // NoStatusUpdate means just regular output but suppress status update. 97 NoStatusUpdate 98 // Normal provides regular output and status update. 99 Normal 100 // Verbose prints out commands executed. 101 Verbose 102 ) 103 104 // A CommandRunner that doesn't actually run the commands. 105 type dryRunCommandRunner struct { 106 finished []*Edge 107 } 108 109 // Overridden from CommandRunner: 110 func (d *dryRunCommandRunner) CanRunMore() bool { 111 return true 112 } 113 114 func (d *dryRunCommandRunner) StartCommand(edge *Edge) bool { 115 // In C++ it's a queue. In Go it's a bit less efficient but it shouldn't be 116 // performance critical. 117 // TODO(maruel): Move items when cap() is significantly larger than len(). 118 d.finished = append([]*Edge{edge}, d.finished...) 119 return true 120 } 121 122 func (d *dryRunCommandRunner) WaitForCommand(result *Result) bool { 123 if len(d.finished) == 0 { 124 return false 125 } 126 127 result.ExitCode = ExitSuccess 128 result.Edge = d.finished[len(d.finished)-1] 129 d.finished = d.finished[:len(d.finished)-1] 130 return true 131 } 132 133 func (d *dryRunCommandRunner) GetActiveEdges() []*Edge { 134 return nil 135 } 136 137 func (d *dryRunCommandRunner) Abort() { 138 } 139 140 type realCommandRunner struct { 141 config *BuildConfig 142 subprocs *subprocessSet 143 subprocToEdge map[*subprocess]*Edge 144 } 145 146 func newRealCommandRunner(config *BuildConfig) *realCommandRunner { 147 return &realCommandRunner{ 148 config: config, 149 subprocs: newSubprocessSet(), 150 subprocToEdge: map[*subprocess]*Edge{}, 151 } 152 } 153 154 func (r *realCommandRunner) GetActiveEdges() []*Edge { 155 var edges []*Edge 156 for _, e := range r.subprocToEdge { 157 edges = append(edges, e) 158 } 159 return edges 160 } 161 162 func (r *realCommandRunner) Abort() { 163 r.subprocs.Clear() 164 } 165 166 func (r *realCommandRunner) CanRunMore() bool { 167 subprocNumber := r.subprocs.Running() + r.subprocs.Finished() 168 more := subprocNumber < r.config.Parallelism 169 load := r.subprocs.Running() == 0 || r.config.MaxLoadAvg <= 0. || getLoadAverage() < r.config.MaxLoadAvg 170 return more && load 171 } 172 173 func (r *realCommandRunner) StartCommand(edge *Edge) bool { 174 command := edge.EvaluateCommand(false) 175 subproc := r.subprocs.Add(command, edge.Pool == ConsolePool) 176 if subproc == nil { 177 return false 178 } 179 r.subprocToEdge[subproc] = edge 180 return true 181 } 182 183 func (r *realCommandRunner) WaitForCommand(result *Result) bool { 184 var subproc *subprocess 185 for { 186 subproc = r.subprocs.NextFinished() 187 if subproc != nil { 188 break 189 } 190 if r.subprocs.DoWork() { 191 return false 192 } 193 } 194 195 result.ExitCode = subproc.Finish() 196 result.Output = subproc.GetOutput() 197 198 e := r.subprocToEdge[subproc] 199 result.Edge = e 200 delete(r.subprocToEdge, subproc) 201 return true 202 } 203 204 // 205 206 // plan stores the state of a build plan: what we intend to build, 207 // which steps we're ready to execute. 208 type plan struct { 209 // Keep track of which edges we want to build in this plan. If this map does 210 // not contain an entry for an edge, we do not want to build the entry or its 211 // dependents. If it does contain an entry, the enumeration indicates what 212 // we want for the edge. 213 want map[*Edge]Want 214 215 ready *EdgeSet 216 217 builder *Builder 218 219 // Total number of edges that have commands (not phony). 220 commandEdges int 221 222 // Total remaining number of wanted edges. 223 wantedEdges int 224 } 225 226 // Returns true if there's more work to be done. 227 func (p *plan) moreToDo() bool { 228 return p.wantedEdges > 0 && p.commandEdges > 0 229 } 230 231 func newPlan(builder *Builder) plan { 232 return plan{ 233 want: map[*Edge]Want{}, 234 ready: NewEdgeSet(), 235 builder: builder, 236 } 237 } 238 239 // Reset state. Clears want and ready sets. 240 // 241 // Only used in unit tests. 242 func (p *plan) Reset() { 243 p.commandEdges = 0 244 p.wantedEdges = 0 245 p.want = map[*Edge]Want{} 246 p.ready = NewEdgeSet() 247 } 248 249 // Add a target to our plan (including all its dependencies). 250 // Returns false if we don't need to build this target; may 251 // fill in |err| with an error message if there's a problem. 252 func (p *plan) addTarget(target *Node) (bool, error) { 253 return p.addSubTarget(target, nil, nil) 254 } 255 256 func (p *plan) addSubTarget(node *Node, dependent *Node, dyndepWalk map[*Edge]struct{}) (bool, error) { 257 edge := node.InEdge 258 if edge == nil { // Leaf node. 259 if node.Dirty { 260 referenced := "" 261 if dependent != nil { 262 // TODO(maruel): Use %q for real quoting. 263 referenced = fmt.Sprintf(", needed by '%s',", dependent.Path) 264 } 265 // TODO(maruel): Use %q for real quoting. 266 return false, fmt.Errorf("'%s'%s missing and no known rule to make it", node.Path, referenced) 267 } 268 return false, nil 269 } 270 271 if edge.OutputsReady { 272 // Don't need to do anything. 273 return false, nil 274 } 275 276 // If an entry in want does not already exist for edge, create an entry which 277 // maps to WantNothing, indicating that we do not want to build this entry itself. 278 want, ok := p.want[edge] 279 if !ok { 280 p.want[edge] = WantNothing 281 } else if len(dyndepWalk) != 0 && want == WantToFinish { 282 // Don't need to do anything with already-scheduled edge. 283 return false, nil 284 } 285 286 // If we do need to build edge and we haven't already marked it as wanted, 287 // mark it now. 288 if node.Dirty && want == WantNothing { 289 want = WantToStart 290 p.want[edge] = want 291 p.edgeWanted(edge) 292 if len(dyndepWalk) == 0 && edge.allInputsReady() { 293 p.ScheduleWork(edge, want) 294 } 295 } 296 297 if len(dyndepWalk) != 0 { 298 dyndepWalk[edge] = struct{}{} 299 } 300 301 if ok { 302 // We've already processed the inputs. 303 return true, nil 304 } 305 306 for _, i := range edge.Inputs { 307 if _, err := p.addSubTarget(i, node, dyndepWalk); err != nil { 308 return false, err 309 } 310 } 311 return true, nil 312 } 313 314 func (p *plan) edgeWanted(edge *Edge) { 315 p.wantedEdges++ 316 if edge.Rule != PhonyRule { 317 p.commandEdges++ 318 } 319 } 320 321 // Pop a ready edge off the queue of edges to build. 322 // Returns NULL if there's no work to do. 323 func (p *plan) findWork() *Edge { 324 return p.ready.Pop() 325 } 326 327 // Submits a ready edge as a candidate for execution. 328 // The edge may be delayed from running, for example if it's a member of a 329 // currently-full pool. 330 func (p *plan) ScheduleWork(edge *Edge, want Want) { 331 if want == WantToFinish { 332 // This edge has already been scheduled. We can get here again if an edge 333 // and one of its dependencies share an order-only input, or if a node 334 // duplicates an out edge (see https://github.com/ninja-build/ninja/pull/519). 335 // Avoid scheduling the work again. 336 return 337 } 338 if want != WantToStart { 339 panic("M-A") 340 } 341 p.want[edge] = WantToFinish 342 343 pool := edge.Pool 344 if pool.shouldDelayEdge() { 345 pool.delayEdge(edge) 346 pool.retrieveReadyEdges(p.ready) 347 } else { 348 pool.edgeScheduled(edge) 349 p.ready.Add(edge) 350 } 351 } 352 353 // edgeFinished marks an edge as done building (whether it succeeded or 354 // failed). 355 // 356 // If any of the edge's outputs are dyndep bindings of their dependents, this 357 // loads dynamic dependencies from the nodes' paths. 358 func (p *plan) edgeFinished(edge *Edge, result edgeResult) error { 359 directlyWanted := p.want[edge] != WantNothing 360 361 // See if this job frees up any delayed jobs. 362 if directlyWanted { 363 edge.Pool.edgeFinished(edge) 364 } 365 edge.Pool.retrieveReadyEdges(p.ready) 366 367 // The rest of this function only applies to successful commands. 368 if result != edgeSucceeded { 369 return nil 370 } 371 372 if directlyWanted { 373 p.wantedEdges-- 374 } 375 delete(p.want, edge) 376 edge.OutputsReady = true 377 378 // Check off any nodes we were waiting for with this edge. 379 for _, o := range edge.Outputs { 380 if err := p.nodeFinished(o); err != nil { 381 return err 382 } 383 } 384 return nil 385 } 386 387 // nodeFinished updates plan with knowledge that the given node is up to date. 388 // 389 // If the node is a dyndep binding on any of its dependents, this 390 // loads dynamic dependencies from the node's path. 391 // 392 // Returns 'false' if loading dyndep info fails and 'true' otherwise. 393 func (p *plan) nodeFinished(node *Node) error { 394 // If this node provides dyndep info, load it now. 395 if node.DyndepPending { 396 if p.builder == nil { 397 return errors.New("dyndep requires Plan to have a Builder") 398 } 399 // Load the now-clean dyndep file. This will also update the 400 // build plan and schedule any new work that is ready. 401 return p.builder.loadDyndeps(node) 402 } 403 404 // See if we we want any edges from this node. 405 for _, oe := range node.OutEdges { 406 want, ok := p.want[oe] 407 if !ok { 408 continue 409 } 410 411 // See if the edge is now ready. 412 if err := p.edgeMaybeReady(oe, want); err != nil { 413 return err 414 } 415 } 416 return nil 417 } 418 419 func (p *plan) edgeMaybeReady(edge *Edge, want Want) error { 420 if edge.allInputsReady() { 421 if want != WantNothing { 422 p.ScheduleWork(edge, want) 423 } else { 424 // We do not need to build this edge, but we might need to build one of 425 // its dependents. 426 if err := p.edgeFinished(edge, edgeSucceeded); err != nil { 427 return err 428 } 429 } 430 } 431 return nil 432 } 433 434 // Clean the given node during the build. 435 // Return false on error. 436 func (p *plan) cleanNode(scan *DependencyScan, node *Node) error { 437 node.Dirty = false 438 439 for _, oe := range node.OutEdges { 440 // Don't process edges that we don't actually want. 441 want, ok := p.want[oe] 442 if !ok || want == WantNothing { 443 continue 444 } 445 446 // Don't attempt to clean an edge if it failed to load deps. 447 if oe.DepsMissing { 448 continue 449 } 450 451 // If all non-order-only inputs for this edge are now clean, 452 // we might have changed the dirty state of the outputs. 453 end := len(oe.Inputs) - int(oe.OrderOnlyDeps) 454 found := false 455 for i := 0; i < end; i++ { 456 if oe.Inputs[i].Dirty { 457 found = true 458 break 459 } 460 } 461 if !found { 462 // Recompute mostRecentInput. 463 var mostRecentInput *Node 464 if end > 0 { 465 mostRecentInput = oe.Inputs[0] 466 for i := 1; i != end; i++ { 467 if oe.Inputs[i].MTime > mostRecentInput.MTime { 468 mostRecentInput = oe.Inputs[i] 469 } 470 } 471 } 472 473 // TODO(maruel): This code doesn't have unit test coverage when 474 // mostRecentInput is nil. 475 476 // Now, this edge is dirty if any of the outputs are dirty. 477 // If the edge isn't dirty, clean the outputs and mark the edge as not 478 // wanted. 479 // The C++ code conditions on its return value but always returns true. 480 outputsDirty := scan.recomputeOutputsDirty(oe, mostRecentInput) 481 if !outputsDirty { 482 for _, o := range oe.Outputs { 483 if err := p.cleanNode(scan, o); err != nil { 484 return err 485 } 486 } 487 488 p.want[oe] = WantNothing 489 p.wantedEdges-- 490 if oe.Rule != PhonyRule { 491 p.commandEdges-- 492 } 493 } 494 } 495 } 496 return nil 497 } 498 499 // dyndepsLoaded updates the build plan to account for modifications made to 500 // the graph by information loaded from a dyndep file. 501 func (p *plan) dyndepsLoaded(scan *DependencyScan, node *Node, ddf DyndepFile) error { 502 // Recompute the dirty state of all our direct and indirect dependents now 503 // that our dyndep information has been loaded. 504 if do, err := p.refreshDyndepDependents(scan, node); !do || err != nil { 505 return err 506 } 507 508 // We loaded dyndep information for those outEdges of the dyndep node that 509 // specify the node in a dyndep binding, but they may not be in the plan. 510 // Starting with those already in the plan, walk newly-reachable portion 511 // of the graph through the dyndep-discovered dependencies. 512 513 // Find edges in the the build plan for which we have new dyndep info. 514 var dyndepRoots []*Edge 515 for edge := range ddf { 516 // If the edge outputs are ready we do not need to consider it here. 517 if edge.OutputsReady { 518 continue 519 } 520 521 // If the edge has not been encountered before then nothing already in the 522 // plan depends on it so we do not need to consider the edge yet either. 523 if _, ok := p.want[edge]; !ok { 524 continue 525 } 526 527 // This edge is already in the plan so queue it for the walk. 528 dyndepRoots = append(dyndepRoots, edge) 529 } 530 531 // Walk dyndep-discovered portion of the graph to add it to the build plan. 532 dyndepWalk := map[*Edge]struct{}{} 533 for _, oe := range dyndepRoots { 534 for _, i := range ddf[oe].implicitInputs { 535 if _, err := p.addSubTarget(i, oe.Outputs[0], dyndepWalk); err != nil { 536 return err 537 } 538 } 539 } 540 541 // Add out edges from this node that are in the plan (just as 542 // NodeFinished would have without taking the dyndep code path). 543 for _, oe := range node.OutEdges { 544 if _, ok := p.want[oe]; !ok { 545 continue 546 } 547 dyndepWalk[oe] = struct{}{} 548 } 549 550 // See if any encountered edges are now ready. 551 for wi := range dyndepWalk { 552 want, ok := p.want[wi] 553 if !ok { 554 continue 555 } 556 if err := p.edgeMaybeReady(wi, want); err != nil { 557 return err 558 } 559 } 560 return nil 561 } 562 563 func (p *plan) refreshDyndepDependents(scan *DependencyScan, node *Node) (bool, error) { 564 // Collect the transitive closure of dependents and mark their edges 565 // as not yet visited by RecomputeDirty. 566 dependents := map[*Node]struct{}{} 567 p.unmarkDependents(node, dependents) 568 569 // Update the dirty state of all dependents and check if their edges 570 // have become wanted. 571 for n := range dependents { 572 // Check if this dependent node is now dirty. Also checks for new cycles. 573 validationNodes, err := scan.RecomputeDirty(n) 574 if err != nil { 575 return false, err 576 } 577 578 // Add any validation nodes found during RecomputeDirty as new top level 579 // targets. 580 for _, v := range validationNodes { 581 if inEdge := v.InEdge; inEdge != nil { 582 if !inEdge.OutputsReady { 583 if do, err := p.addTarget(v); !do || err != nil { 584 return false, err 585 } 586 } 587 } 588 } 589 if !n.Dirty { 590 continue 591 } 592 593 // This edge was encountered before. However, we may not have wanted to 594 // build it if the outputs were not known to be dirty. With dyndep 595 // information an output is now known to be dirty, so we want the edge. 596 edge := n.InEdge 597 if edge == nil || edge.OutputsReady { 598 panic("M-A") 599 } 600 wantE, ok := p.want[edge] 601 if !ok { 602 panic("M-A") 603 } 604 if wantE == WantNothing { 605 p.want[edge] = WantToStart 606 p.edgeWanted(edge) 607 } 608 } 609 return true, nil 610 } 611 612 func (p *plan) unmarkDependents(node *Node, dependents map[*Node]struct{}) { 613 for _, edge := range node.OutEdges { 614 _, ok := p.want[edge] 615 if !ok { 616 continue 617 } 618 619 if edge.Mark != VisitNone { 620 edge.Mark = VisitNone 621 for _, o := range edge.Outputs { 622 _, ok := dependents[o] 623 if ok { 624 p.unmarkDependents(o, dependents) 625 } else { 626 dependents[o] = struct{}{} 627 } 628 } 629 } 630 } 631 } 632 633 // Dumps the current state of the plan. 634 func (p *plan) Dump() { 635 fmt.Printf("pending: %d\n", len(p.want)) 636 for e, w := range p.want { 637 if w != WantNothing { 638 fmt.Printf("want ") 639 } 640 e.Dump("") 641 } 642 // TODO(maruel): Uses inner knowledge 643 fmt.Printf("ready:\n") 644 p.ready.recreate() 645 for i := range p.ready.sorted { 646 fmt.Printf("\t") 647 p.ready.sorted[len(p.ready.sorted)-1-i].Dump("") 648 } 649 } 650 651 // 652 653 // Builder wraps the build process: starting commands, updating status. 654 type Builder struct { 655 state *State 656 config *BuildConfig 657 plan plan 658 commandRunner commandRunner 659 status Status 660 661 // Map of running edge to time the edge started running. 662 runningEdges map[*Edge]int32 663 664 // Time the build started. 665 startTimeMillis int64 666 667 di DiskInterface 668 scan DependencyScan 669 } 670 671 // NewBuilder returns an initialized Builder. 672 func NewBuilder(state *State, config *BuildConfig, buildLog *BuildLog, depsLog *DepsLog, di DiskInterface, status Status, startTimeMillis int64) *Builder { 673 b := &Builder{ 674 state: state, 675 config: config, 676 status: status, 677 runningEdges: map[*Edge]int32{}, 678 startTimeMillis: startTimeMillis, 679 di: di, 680 } 681 b.plan = newPlan(b) 682 b.scan = NewDependencyScan(state, buildLog, depsLog, di) 683 return b 684 } 685 686 // cleanup cleans up after interrupted commands by deleting output files. 687 func (b *Builder) cleanup() { 688 if b.commandRunner != nil { 689 activeEdges := b.commandRunner.GetActiveEdges() 690 b.commandRunner.Abort() 691 692 for _, e := range activeEdges { 693 depfile := e.GetUnescapedDepfile() 694 for _, o := range e.Outputs { 695 // Only delete this output if it was actually modified. This is 696 // important for things like the generator where we don't want to 697 // delete the manifest file if we can avoid it. But if the rule 698 // uses a depfile, always delete. (Consider the case where we 699 // need to rebuild an output because of a modified header file 700 // mentioned in a depfile, and the command touches its depfile 701 // but is interrupted before it touches its output file.) 702 newMtime, err := b.di.Stat(o.Path) 703 if newMtime == -1 { // Log and ignore Stat() errors. 704 b.status.Error("%s", err) 705 } 706 if depfile != "" || o.MTime != newMtime { 707 if err := b.di.RemoveFile(o.Path); err != nil { 708 b.status.Error("%s", err) 709 } 710 } 711 } 712 if len(depfile) != 0 { 713 if err := b.di.RemoveFile(depfile); err != nil { 714 b.status.Error("%s", err) 715 } 716 } 717 } 718 } 719 } 720 721 // addTargetName adds a target to the build, scanning dependencies. 722 // 723 // Returns false on error. 724 func (b *Builder) addTargetName(name string) (*Node, error) { 725 node := b.state.Paths[name] 726 if node == nil { 727 // TODO(maruel): Use %q for real quoting. 728 return nil, fmt.Errorf("unknown target: '%s'", name) 729 } 730 if _, err := b.AddTarget(node); err != nil { 731 return nil, err 732 } 733 return node, nil 734 } 735 736 // AddTarget adds a target to the build, scanning dependencies. 737 // 738 // Returns true if the target is dirty. Returns false and no error if the 739 // target is up to date. 740 func (b *Builder) AddTarget(target *Node) (bool, error) { 741 validationNodes, err := b.scan.RecomputeDirty(target) 742 if err != nil { 743 return false, err 744 } 745 746 inEdge := target.InEdge 747 if inEdge == nil || !inEdge.OutputsReady { 748 if do, err := b.plan.addTarget(target); !do { 749 return false, err 750 } 751 } 752 753 // Also add any validation nodes found during RecomputeDirty as top level 754 // targets. 755 for _, n := range validationNodes { 756 if validationInEdge := n.InEdge; validationInEdge != nil { 757 if !validationInEdge.OutputsReady { 758 if do, err := b.plan.addTarget(n); !do { 759 return false, err 760 } 761 } 762 } 763 } 764 return true, nil 765 } 766 767 // AlreadyUpToDate returns true if the build targets are already up to date. 768 func (b *Builder) AlreadyUpToDate() bool { 769 return !b.plan.moreToDo() 770 } 771 772 // Build runs the build. 773 // 774 // It is an error to call this function when AlreadyUpToDate() is true. 775 func (b *Builder) Build() error { 776 if b.AlreadyUpToDate() { 777 return errors.New("already up to date") 778 } 779 780 b.status.PlanHasTotalEdges(b.plan.commandEdges) 781 pendingCommands := 0 782 failuresAllowed := b.config.FailuresAllowed 783 784 // Set up the command runner if we haven't done so already. 785 if b.commandRunner == nil { 786 if b.config.DryRun { 787 b.commandRunner = &dryRunCommandRunner{} 788 } else { 789 b.commandRunner = newRealCommandRunner(b.config) 790 } 791 } 792 793 // We are about to start the build process. 794 b.status.BuildStarted() 795 796 // This main loop runs the entire build process. 797 // It is structured like this: 798 // First, we attempt to start as many commands as allowed by the 799 // command runner. 800 // Second, we attempt to wait for / reap the next finished command. 801 for b.plan.moreToDo() { 802 // See if we can start any more commands. 803 if failuresAllowed != 0 && b.commandRunner.CanRunMore() { 804 if edge := b.plan.findWork(); edge != nil { 805 if edge.GetBinding("generator") != "" { 806 if err := b.scan.buildLog.Close(); err != nil { 807 panic("M-A") 808 // New. 809 //b.cleanup() 810 //return err 811 } 812 } 813 814 if err := b.startEdge(edge); err != nil { 815 b.cleanup() 816 b.status.BuildFinished() 817 return err 818 } 819 820 if edge.Rule == PhonyRule { 821 if err := b.plan.edgeFinished(edge, edgeSucceeded); err != nil { 822 b.cleanup() 823 b.status.BuildFinished() 824 return err 825 } 826 } else { 827 pendingCommands++ 828 } 829 830 // We made some progress; go back to the main loop. 831 continue 832 } 833 } 834 835 // See if we can reap any finished commands. 836 if pendingCommands != 0 { 837 var result Result 838 if !b.commandRunner.WaitForCommand(&result) || result.ExitCode == ExitInterrupted { 839 b.cleanup() 840 b.status.BuildFinished() 841 // TODO(maruel): This will use context. 842 return errors.New("interrupted by user") 843 } 844 845 pendingCommands-- 846 if err := b.finishCommand(&result); err != nil { 847 b.cleanup() 848 b.status.BuildFinished() 849 return err 850 } 851 852 if result.ExitCode != ExitSuccess { 853 if failuresAllowed != 0 { 854 failuresAllowed-- 855 } 856 } 857 858 // We made some progress; start the main loop over. 859 continue 860 } 861 862 // If we get here, we cannot make any more progress. 863 b.status.BuildFinished() 864 if failuresAllowed == 0 { 865 if b.config.FailuresAllowed > 1 { 866 return errors.New("subcommands failed") 867 } 868 return errors.New("subcommand failed") 869 } else if failuresAllowed < b.config.FailuresAllowed { 870 return errors.New("cannot make progress due to previous errors") 871 } 872 return errors.New("stuck [this is a bug]") 873 } 874 b.status.BuildFinished() 875 return nil 876 } 877 878 func (b *Builder) startEdge(edge *Edge) error { 879 defer metricRecord("StartEdge")() 880 if edge.Rule == PhonyRule { 881 return nil 882 } 883 startTimeMillis := int32(time.Now().UnixMilli() - b.startTimeMillis) 884 b.runningEdges[edge] = startTimeMillis 885 886 b.status.BuildEdgeStarted(edge, startTimeMillis) 887 888 // Create directories necessary for outputs. 889 // XXX: this will block; do we care? 890 for _, o := range edge.Outputs { 891 if err := MakeDirs(b.di, o.Path); err != nil { 892 return err 893 } 894 } 895 896 // Create response file, if needed 897 // XXX: this may also block; do we care? 898 rspfile := edge.GetUnescapedRspfile() 899 if len(rspfile) != 0 { 900 content := edge.GetBinding("rspfile_content") 901 if err := b.di.WriteFile(rspfile, content); err != nil { 902 return err 903 } 904 } 905 906 // start command computing and run it 907 if !b.commandRunner.StartCommand(edge) { 908 // TODO(maruel): Use %q for real quoting. 909 return fmt.Errorf("command '%s' failed", edge.EvaluateCommand(len(rspfile) != 0)) 910 } 911 return nil 912 } 913 914 // finishCommand updates status ninja logs following a command termination. 915 // 916 // Return an error if the build can not proceed further due to a fatal error. 917 func (b *Builder) finishCommand(result *Result) error { 918 defer metricRecord("FinishCommand")() 919 edge := result.Edge 920 921 // First try to extract dependencies from the result, if any. 922 // This must happen first as it filters the command output (we want 923 // to filter /showIncludes output, even on compile failure) and 924 // extraction itself can fail, which makes the command fail from a 925 // build perspective. 926 var depsNodes []*Node 927 depsType := edge.GetBinding("deps") 928 depsPrefix := edge.GetBinding("msvc_deps_prefix") 929 if depsType != "" { 930 var err error 931 depsNodes, err = b.extractDeps(result, depsType, depsPrefix) 932 if err != nil && result.ExitCode == ExitSuccess { 933 if result.Output != "" { 934 result.Output += "\n" 935 } 936 result.Output += err.Error() 937 result.ExitCode = ExitFailure 938 } 939 } 940 941 var startTimeMillis, endTimeMillis int32 942 startTimeMillis = b.runningEdges[edge] 943 endTimeMillis = int32(time.Now().UnixMilli() - b.startTimeMillis) 944 delete(b.runningEdges, edge) 945 946 b.status.BuildEdgeFinished(edge, endTimeMillis, result.ExitCode == ExitSuccess, result.Output) 947 948 // The rest of this function only applies to successful commands. 949 if result.ExitCode != ExitSuccess { 950 return b.plan.edgeFinished(edge, edgeFailed) 951 } 952 // Restat the edge outputs 953 outputMtime := TimeStamp(0) 954 restat := edge.GetBinding("restat") != "" 955 if !b.config.DryRun { 956 nodeCleaned := false 957 958 for _, o := range edge.Outputs { 959 newMtime, err := b.di.Stat(o.Path) 960 if newMtime == -1 { 961 return err 962 } 963 if newMtime > outputMtime { 964 outputMtime = newMtime 965 } 966 if o.MTime == newMtime && restat { 967 // The rule command did not change the output. Propagate the clean 968 // state through the build graph. 969 // Note that this also applies to nonexistent outputs (mtime == 0). 970 if err := b.plan.cleanNode(&b.scan, o); err != nil { 971 return err 972 } 973 nodeCleaned = true 974 } 975 } 976 977 if nodeCleaned { 978 restatMtime := TimeStamp(0) 979 // If any output was cleaned, find the most recent mtime of any 980 // (existing) non-order-only input or the depfile. 981 for _, i := range edge.Inputs[:len(edge.Inputs)-int(edge.OrderOnlyDeps)] { 982 inputMtime, err := b.di.Stat(i.Path) 983 if inputMtime == -1 { 984 return err 985 } 986 if inputMtime > restatMtime { 987 restatMtime = inputMtime 988 } 989 } 990 991 depfile := edge.GetUnescapedDepfile() 992 if restatMtime != 0 && depsType == "" && depfile != "" { 993 depfileMtime, err := b.di.Stat(depfile) 994 if depfileMtime == -1 { 995 return err 996 } 997 if depfileMtime > restatMtime { 998 restatMtime = depfileMtime 999 } 1000 } 1001 1002 // The total number of edges in the plan may have changed as a result 1003 // of a restat. 1004 b.status.PlanHasTotalEdges(b.plan.commandEdges) 1005 1006 outputMtime = restatMtime 1007 } 1008 } 1009 1010 if err := b.plan.edgeFinished(edge, edgeSucceeded); err != nil { 1011 return err 1012 } 1013 1014 // Delete any left over response file. 1015 rspfile := edge.GetUnescapedRspfile() 1016 if rspfile != "" && !Debug.KeepRsp { 1017 // Ignore the error for now. 1018 _ = b.di.RemoveFile(rspfile) 1019 } 1020 1021 if b.scan.buildLog != nil { 1022 if err := b.scan.buildLog.RecordCommand(edge, startTimeMillis, endTimeMillis, outputMtime); err != nil { 1023 return fmt.Errorf("error writing to build log: %w", err) 1024 } 1025 } 1026 1027 if depsType != "" && !b.config.DryRun { 1028 if len(edge.Outputs) == 0 { 1029 return errors.New("should have been rejected by parser") 1030 } 1031 for _, o := range edge.Outputs { 1032 depsMtime, err := b.di.Stat(o.Path) 1033 if depsMtime == -1 { 1034 return err 1035 } 1036 if err := b.scan.depsLog().recordDeps(o, depsMtime, depsNodes); err != nil { 1037 return fmt.Errorf("error writing to deps log: %w", err) 1038 } 1039 } 1040 } 1041 1042 return nil 1043 } 1044 1045 func (b *Builder) extractDeps(result *Result, depsType string, depsPrefix string) ([]*Node, error) { 1046 switch depsType { 1047 case "msvc": 1048 parser := NewCLParser() 1049 output := "" 1050 if err := parser.Parse(result.Output, depsPrefix, &output); err != nil { 1051 return nil, err 1052 } 1053 result.Output = output 1054 depsNodes := make([]*Node, 0, len(parser.includes)) 1055 for i := range parser.includes { 1056 // ~0 is assuming that with MSVC-parsed headers, it's ok to always make 1057 // all backslashes (as some of the slashes will certainly be backslashes 1058 // anyway). This could be fixed if necessary with some additional 1059 // complexity in IncludesNormalize.relativize. 1060 depsNodes = append(depsNodes, b.state.GetNode(i, 0xFFFFFFFF)) 1061 } 1062 return depsNodes, nil 1063 case "gcc": 1064 depfile := result.Edge.GetUnescapedDepfile() 1065 if len(depfile) == 0 { 1066 return nil, errors.New("edge with deps=gcc but no depfile makes no sense") 1067 } 1068 1069 // Read depfile content. Treat a missing depfile as empty. 1070 content, err := b.di.ReadFile(depfile) 1071 if err != nil && !os.IsNotExist(err) { 1072 return nil, err 1073 } 1074 if len(content) == 0 { 1075 return nil, nil 1076 } 1077 1078 deps := DepfileParser{} 1079 if err := deps.Parse(content); err != nil { 1080 return nil, err 1081 } 1082 1083 // XXX check depfile matches expected output. 1084 depsNodes := make([]*Node, len(deps.ins)) 1085 for i, s := range deps.ins { 1086 depsNodes[i] = b.state.GetNode(CanonicalizePathBits(s)) 1087 } 1088 1089 if !Debug.KeepDepfile { 1090 if err := b.di.RemoveFile(depfile); err != nil { 1091 return depsNodes, err 1092 } 1093 } 1094 return depsNodes, nil 1095 default: 1096 return nil, fmt.Errorf("unknown deps type '%s'", depsType) 1097 } 1098 } 1099 1100 // Load the dyndep information provided by the given node. 1101 func (b *Builder) loadDyndeps(node *Node) error { 1102 b.status.BuildLoadDyndeps() 1103 1104 // Load the dyndep information provided by this node. 1105 ddf := DyndepFile{} 1106 if err := b.scan.LoadDyndeps(node, ddf); err != nil { 1107 return err 1108 } 1109 1110 // Update the build plan to account for dyndep modifications to the graph. 1111 if err := b.plan.dyndepsLoaded(&b.scan, node, ddf); err != nil { 1112 return err 1113 } 1114 1115 // New command edges may have been added to the plan. 1116 b.status.PlanHasTotalEdges(b.plan.commandEdges) 1117 return nil 1118 }