github.com/maruel/nin@v0.0.0-20220112143044-f35891e3ce7e/graph.go (about) 1 // Copyright 2011 Google Inc. All Rights Reserved. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package nin 16 17 import ( 18 "bytes" 19 "errors" 20 "fmt" 21 "os" 22 "runtime" 23 "sort" 24 ) 25 26 // ExistenceStatus represents the knowledge of the file's existence. 27 type ExistenceStatus int32 28 29 const ( 30 // ExistenceStatusUnknown means the file hasn't been examined. 31 ExistenceStatusUnknown ExistenceStatus = iota 32 // ExistenceStatusMissing means the file doesn't exist. MTime will be the 33 // latest mtime of its dependencies. 34 ExistenceStatusMissing 35 // ExistenceStatusExists means the path is an actual file. MTime will be the 36 // file's mtime. 37 ExistenceStatusExists 38 ) 39 40 // Node represents information about a node in the dependency graph: the file, 41 // whether it's dirty, mtime, etc. 42 type Node struct { 43 // Immutable. 44 45 // Path is the path of the file that this node represents. 46 Path string 47 48 // Set bits starting from lowest for backslashes that were normalized to 49 // forward slashes by CanonicalizePathBits. See |PathDecanonicalized|. 50 SlashBits uint64 51 52 // Mutable. 53 54 // The Edge that produces this Node, or NULL when there is no 55 // known edge to produce it. 56 InEdge *Edge 57 58 // All Edges that use this Node as an input. 59 OutEdges []*Edge 60 61 // All Edges that use this Node as a validation. 62 ValidationOutEdges []*Edge 63 64 // Possible values of MTime: 65 // -1: file hasn't been examined 66 // 0: we looked, and file doesn't exist 67 // >0: actual file's mtime, or the latest mtime of its dependencies if it doesn't exist 68 MTime TimeStamp 69 70 // A dense integer id for the node, assigned and used by DepsLog. 71 ID int32 72 73 Exists ExistenceStatus 74 75 // Dirty is true when the underlying file is out-of-date. 76 // But note that Edge.OutputsReady is also used in judging which 77 // edges to build. 78 Dirty bool 79 80 // Store whether dyndep information is expected from this node but 81 // has not yet been loaded. 82 DyndepPending bool 83 } 84 85 func (n *Node) statIfNecessary(di DiskInterface) error { 86 if n.Exists != ExistenceStatusUnknown { 87 return nil 88 } 89 return n.Stat(di) 90 } 91 92 // PathDecanonicalized return |Path| but use SlashBits to convert back to 93 // original slash styles. 94 func (n *Node) PathDecanonicalized() string { 95 return PathDecanonicalized(n.Path, n.SlashBits) 96 } 97 98 // Stat stat's the file. 99 func (n *Node) Stat(di DiskInterface) error { 100 defer metricRecord("node stat")() 101 mtime, err := di.Stat(n.Path) 102 n.MTime = mtime 103 if mtime == -1 { 104 return err 105 } 106 if n.MTime != 0 { 107 n.Exists = ExistenceStatusExists 108 } else { 109 n.Exists = ExistenceStatusMissing 110 } 111 return nil 112 } 113 114 // If the file doesn't exist, set the MTime from its dependencies 115 func (n *Node) updatePhonyMtime(mtime TimeStamp) { 116 if n.Exists != ExistenceStatusExists { 117 if mtime > n.MTime { 118 n.MTime = mtime 119 } 120 } 121 } 122 123 // Dump prints out Node's details to stdout. 124 func (n *Node) Dump(prefix string) { 125 s := "" 126 if n.Exists != ExistenceStatusExists { 127 s = " (:missing)" 128 } 129 t := " clean" 130 if n.Dirty { 131 t = " dirty" 132 } 133 fmt.Printf("%s <%s 0x%p> mtime: %x%s, (:%s), ", prefix, n.Path, n, n.MTime, s, t) 134 if n.InEdge != nil { 135 n.InEdge.Dump("in-edge: ") 136 } else { 137 fmt.Printf("no in-edge\n") 138 } 139 fmt.Printf(" out edges:\n") 140 for _, e := range n.OutEdges { 141 if e == nil { 142 break 143 } 144 e.Dump(" +- ") 145 } 146 if len(n.ValidationOutEdges) != 0 { 147 fmt.Printf(" validation out edges:\n") 148 for _, e := range n.ValidationOutEdges { 149 e.Dump(" +- ") 150 } 151 } 152 } 153 154 // 155 156 // VisitMark is a market to determine if an edge is visited. 157 type VisitMark int32 158 159 // Valid VisitMark values. 160 const ( 161 VisitNone VisitMark = iota 162 VisitInStack 163 VisitDone 164 ) 165 166 // Edge is an edge in the dependency graph; links between Nodes using Rules. 167 type Edge struct { 168 Inputs []*Node 169 Outputs []*Node 170 Validations []*Node 171 Rule *Rule 172 Pool *Pool 173 Dyndep *Node 174 Env *BindingEnv 175 Mark VisitMark 176 ID int32 177 178 // There are three types of inputs. 179 // 1) explicit deps, which show up as $in on the command line; 180 // 2) implicit deps, which the target depends on implicitly (e.g. C headers), 181 // and changes in them cause the target to rebuild; 182 // 3) order-only deps, which are needed before the target builds but which 183 // don't cause the target to rebuild. 184 // These are stored in Inputs in that order, and we keep counts of 185 // #2 and #3 when we need to access the various subsets. 186 ImplicitDeps int32 187 OrderOnlyDeps int32 188 189 // There are two types of outputs. 190 // 1) explicit outs, which show up as $out on the command line; 191 // 2) implicit outs, which the target generates but are not part of $out. 192 // These are stored in Outputs in that order, and we keep a count of 193 // #2 to use when we need to access the various subsets. 194 ImplicitOuts int32 195 196 OutputsReady bool 197 DepsLoaded bool 198 DepsMissing bool 199 GeneratedByDepLoader bool 200 } 201 202 // If this ever gets changed, update DelayedEdgesSet to take this into account. 203 func (e *Edge) weight() int { 204 return 1 205 } 206 207 // IsImplicit returns if the inputs at the specified index is implicit and not 208 // for ordering only. 209 func (e *Edge) IsImplicit(index int) bool { 210 return index >= len(e.Inputs)-int(e.OrderOnlyDeps)-int(e.ImplicitDeps) && !e.IsOrderOnly(index) 211 } 212 213 // IsOrderOnly returns if the input at the specified index is only used for 214 // ordering. 215 func (e *Edge) IsOrderOnly(index int) bool { 216 return index >= len(e.Inputs)-int(e.OrderOnlyDeps) 217 } 218 219 // isImplicitOut is only used in unit tests. 220 func (e *Edge) isImplicitOut(index int) bool { 221 return index >= len(e.Outputs)-int(e.ImplicitOuts) 222 } 223 224 // EvaluateCommand expands all variables in a command and return it as a string. 225 // 226 // If inclRspFile is enabled, the string will also contain the 227 // full contents of a response file (if applicable) 228 func (e *Edge) EvaluateCommand(inclRspFile bool) string { 229 command := e.GetBinding("command") 230 if inclRspFile { 231 rspfileContent := e.GetBinding("rspfile_content") 232 if rspfileContent != "" { 233 command += ";rspfile=" + rspfileContent 234 } 235 } 236 return command 237 } 238 239 // GetBinding returns the shell-escaped value of |key|. 240 func (e *Edge) GetBinding(key string) string { 241 env := edgeEnv{ 242 edge: e, 243 escapeInOut: shellEscape, 244 } 245 return env.LookupVariable(key) 246 } 247 248 // GetUnescapedDepfile returns like GetBinding("depfile"), but without shell 249 // escaping. 250 func (e *Edge) GetUnescapedDepfile() string { 251 env := edgeEnv{ 252 edge: e, 253 escapeInOut: doNotEscape, 254 } 255 return env.LookupVariable("depfile") 256 } 257 258 // GetUnescapedDyndep returns like GetBinding("dyndep"), but without shell 259 // escaping. 260 func (e *Edge) GetUnescapedDyndep() string { 261 env := edgeEnv{ 262 edge: e, 263 escapeInOut: doNotEscape, 264 } 265 return env.LookupVariable("dyndep") 266 } 267 268 // GetUnescapedRspfile returns like GetBinding("rspfile"), but without shell 269 // escaping. 270 func (e *Edge) GetUnescapedRspfile() string { 271 env := edgeEnv{ 272 edge: e, 273 escapeInOut: doNotEscape, 274 } 275 return env.LookupVariable("rspfile") 276 } 277 278 // Dump prints the Edge details to stdout. 279 func (e *Edge) Dump(prefix string) { 280 fmt.Printf("%s[ ", prefix) 281 for _, i := range e.Inputs { 282 if i != nil { 283 fmt.Printf("%s ", i.Path) 284 } 285 } 286 fmt.Printf("--%s-> ", e.Rule.Name) 287 for _, i := range e.Outputs { 288 fmt.Printf("%s ", i.Path) 289 } 290 if len(e.Validations) != 0 { 291 fmt.Printf(" validations ") 292 for _, i := range e.Validations { 293 fmt.Printf("%s ", i.Path) 294 } 295 } 296 if e.Pool != nil { 297 if e.Pool.Name != "" { 298 fmt.Printf("(in pool '%s')", e.Pool.Name) 299 } 300 } else { 301 fmt.Printf("(null pool?)") 302 } 303 fmt.Printf("] 0x%p\n", e) 304 } 305 306 func (e *Edge) maybePhonycycleDiagnostic() bool { 307 // CMake 2.8.12.x and 3.0.x produced self-referencing phony rules 308 // of the form "build a: phony ... a ...". Restrict our 309 // "phonycycle" diagnostic option to the form it used. 310 return e.Rule == PhonyRule && len(e.Outputs) == 1 && e.ImplicitOuts == 0 && e.ImplicitDeps == 0 311 } 312 313 // Return true if all inputs' in-edges are ready. 314 func (e *Edge) allInputsReady() bool { 315 for _, i := range e.Inputs { 316 if i.InEdge != nil && !i.InEdge.OutputsReady { 317 return false 318 } 319 } 320 return true 321 } 322 323 // 324 325 // EdgeSet acts as a sorted set of *Edge, so map[*Edge]struct{} but with sorted 326 // pop. 327 type EdgeSet struct { 328 edges map[*Edge]struct{} 329 dirty bool 330 sorted []*Edge 331 } 332 333 // NewEdgeSet returns an initialized EdgeSet. 334 func NewEdgeSet() *EdgeSet { 335 return &EdgeSet{ 336 edges: make(map[*Edge]struct{}), 337 } 338 } 339 340 // IsEmpty return true if the set is empty. 341 func (e *EdgeSet) IsEmpty() bool { 342 return len(e.edges) == 0 343 } 344 345 // Add the edge to the set. 346 func (e *EdgeSet) Add(ed *Edge) { 347 e.edges[ed] = struct{}{} 348 e.dirty = true 349 } 350 351 // Pop returns the lowest ID. 352 func (e *EdgeSet) Pop() *Edge { 353 e.recreate() 354 if len(e.sorted) == 0 { 355 return nil 356 } 357 // Do not set dirty. 358 ed := e.sorted[len(e.sorted)-1] 359 e.sorted = e.sorted[:len(e.sorted)-1] 360 delete(e.edges, ed) 361 return ed 362 } 363 364 func (e *EdgeSet) recreate() { 365 if !e.dirty { 366 return 367 } 368 e.dirty = false 369 if len(e.edges) == 0 { 370 if len(e.sorted) != 0 { 371 e.sorted = e.sorted[:0] 372 } 373 return 374 } 375 // Resize e.sorted to be the same size as e.edges 376 le := len(e.edges) 377 if cap(e.sorted) < le { 378 e.sorted = make([]*Edge, le) 379 } else { 380 delta := le - len(e.sorted) 381 if delta < 0 { 382 // TODO(maruel): Not sure how to tell the Go compiler to do it as a 383 // single operation. 384 for i := 0; i < delta; i++ { 385 e.sorted = append(e.sorted, nil) 386 } 387 } else if delta > 0 { 388 e.sorted = e.sorted[:le] 389 } 390 } 391 i := 0 392 for k := range e.edges { 393 e.sorted[i] = k 394 i++ 395 } 396 // Sort in reverse order, so that Pop() removes the last (smallest) item. 397 sort.Slice(e.sorted, func(i, j int) bool { 398 return e.sorted[i].ID > e.sorted[j].ID 399 }) 400 } 401 402 // 403 404 type escapeKind bool 405 406 const ( 407 shellEscape escapeKind = false 408 doNotEscape escapeKind = true 409 ) 410 411 // An Env for an Edge, providing $in and $out. 412 type edgeEnv struct { 413 lookups []string 414 edge *Edge 415 escapeInOut escapeKind 416 recursive bool 417 } 418 419 func (e *edgeEnv) LookupVariable(v string) string { 420 edge := e.edge 421 switch v { 422 case "in": 423 explicitDepsCount := len(edge.Inputs) - int(edge.ImplicitDeps) - int(edge.OrderOnlyDeps) 424 return makePathList(edge.Inputs[:explicitDepsCount], ' ', e.escapeInOut) 425 case "in_newline": 426 explicitDepsCount := len(edge.Inputs) - int(edge.ImplicitDeps) - int(edge.OrderOnlyDeps) 427 return makePathList(edge.Inputs[:explicitDepsCount], '\n', e.escapeInOut) 428 case "out": 429 explicitOutsCount := len(edge.Outputs) - int(edge.ImplicitOuts) 430 return makePathList(edge.Outputs[:explicitOutsCount], ' ', e.escapeInOut) 431 default: 432 // TODO(maruel): Remove here and move to a post parsing evaluation in a 433 // separate goroutine. 434 for i := 0; i < len(e.lookups); i++ { 435 if e.lookups[i] == v { 436 cycle := "" 437 for ; i < len(e.lookups); i++ { 438 cycle += e.lookups[i] + " -> " 439 } 440 cycle += v 441 fatalf("cycle in rule variables: " + cycle) 442 } 443 } 444 445 // See notes on BindingEnv.lookupWithFallback. 446 eval := edge.Rule.Bindings[v] 447 if e.recursive { 448 if eval != nil { 449 e.lookups = append(e.lookups, v) 450 } 451 } else { 452 // In practice, variables defined on rules never use another rule variable. 453 e.recursive = true 454 } 455 return edge.Env.lookupWithFallback(v, eval, e) 456 } 457 } 458 459 // Given a span of Nodes, construct a list of paths suitable for a command 460 // line. 461 func makePathList(span []*Node, sep byte, escapeInOut escapeKind) string { 462 var z [64]string 463 var s []string 464 if l := len(span); l <= cap(z) { 465 s = z[:l] 466 } else { 467 s = make([]string, l) 468 } 469 total := 0 470 first := false 471 for i, x := range span { 472 path := x.PathDecanonicalized() 473 if escapeInOut == shellEscape { 474 if runtime.GOOS == "windows" { 475 path = getWin32EscapedString(path) 476 } else { 477 path = getShellEscapedString(path) 478 } 479 } 480 l := len(path) 481 if !first { 482 if l != 0 { 483 first = true 484 } 485 } else { 486 // For the separator. 487 total++ 488 } 489 s[i] = path 490 total += l 491 } 492 493 out := make([]byte, total) 494 offset := 0 495 for _, x := range s { 496 if offset != 0 { 497 out[offset] = sep 498 offset++ 499 } 500 copy(out[offset:], x) 501 offset += len(x) 502 } 503 return unsafeString(out) 504 } 505 506 // PathDecanonicalized does the reverse process of CanonicalizePath(). 507 // 508 // Only does anything on Windows. 509 func PathDecanonicalized(path string, slashBits uint64) string { 510 if runtime.GOOS != "windows" { 511 return path 512 } 513 result := []byte(path) 514 mask := uint64(1) 515 516 for c := 0; ; c++ { 517 d := bytes.IndexByte(result[c:], '/') 518 if d == -1 { 519 break 520 } 521 c += d 522 if slashBits&mask != 0 { 523 result[c] = '\\' 524 } 525 mask <<= 1 526 } 527 return unsafeString(result) 528 } 529 530 // 531 532 // DependencyScan manages the process of scanning the files in a graph 533 // and updating the dirty/outputsReady state of all the nodes and edges. 534 type DependencyScan struct { 535 buildLog *BuildLog 536 di DiskInterface 537 depLoader implicitDepLoader 538 dyndepLoader DyndepLoader 539 } 540 541 // NewDependencyScan returns an initialized DependencyScan. 542 func NewDependencyScan(state *State, buildLog *BuildLog, depsLog *DepsLog, di DiskInterface) DependencyScan { 543 return DependencyScan{ 544 buildLog: buildLog, 545 di: di, 546 depLoader: newImplicitDepLoader(state, depsLog, di), 547 dyndepLoader: NewDyndepLoader(state, di), 548 } 549 } 550 551 func (d *DependencyScan) depsLog() *DepsLog { 552 return d.depLoader.depsLog 553 } 554 555 // RecomputeDirty updates the |dirty| state of the given Node by transitively 556 // inspecting their input edges. 557 // 558 // Examine inputs, outputs, and command lines to judge whether an edge 559 // needs to be re-run, and update OutputsReady and each outputs' Dirty 560 // state accordingly. 561 // 562 // Appends any validation nodes found to the nodes parameter. 563 func (d *DependencyScan) RecomputeDirty(initialNode *Node) ([]*Node, error) { 564 var stack, validationNodes, newValidationNodes []*Node 565 // The C++ code uses a dequeue. 566 nodes := []*Node{initialNode} 567 568 // recomputeNodeDirty might return new validation nodes that need to be 569 // checked for dirty state, keep a queue of nodes to visit. 570 for len(nodes) != 0 { 571 node := nodes[0] 572 nodes = nodes[1:] 573 // Reuse slices to reduce overall memory allocations. 574 stack = stack[:0] 575 newValidationNodes = newValidationNodes[:0] 576 var err error 577 stack, newValidationNodes, err = d.recomputeNodeDirty(node, stack, newValidationNodes) 578 if err != nil { 579 return nil, err 580 } 581 nodes = append(nodes, newValidationNodes...) 582 validationNodes = append(validationNodes, newValidationNodes...) 583 } 584 return validationNodes, nil 585 } 586 587 // recomputeNodeDirty updates Node.Dirty. 588 // 589 // It is recursive. 590 func (d *DependencyScan) recomputeNodeDirty(node *Node, stack, validationNodes []*Node) ([]*Node, []*Node, error) { 591 edge := node.InEdge 592 if edge == nil { 593 // If we already visited this leaf node then we are done. 594 if node.Exists != ExistenceStatusUnknown { 595 return stack, validationNodes, nil 596 } 597 // This node has no in-edge; it is dirty if it is missing. 598 if err := node.statIfNecessary(d.di); err != nil { 599 return stack, validationNodes, err 600 } 601 if node.Exists != ExistenceStatusExists { 602 explain("%s has no in-edge and is missing", node.Path) 603 } 604 node.Dirty = node.Exists != ExistenceStatusExists 605 return stack, validationNodes, nil 606 } 607 608 // If we already finished this edge then we are done. 609 if edge.Mark == VisitDone { 610 return stack, validationNodes, nil 611 } 612 613 // If we encountered this edge earlier in the call stack we have a cycle. 614 if err := d.verifyDAG(node, stack); err != nil { 615 return stack, validationNodes, err 616 } 617 618 // Mark the edge temporarily while in the call stack. 619 edge.Mark = VisitInStack 620 stack = append(stack, node) 621 622 dirty := false 623 edge.OutputsReady = true 624 edge.DepsMissing = false 625 626 if !edge.DepsLoaded { 627 // This is our first encounter with this edge. 628 // If there is a pending dyndep file, visit it now: 629 // * If the dyndep file is ready then load it now to get any 630 // additional inputs and outputs for this and other edges. 631 // Once the dyndep file is loaded it will no longer be pending 632 // if any other edges encounter it, but they will already have 633 // been updated. 634 // * If the dyndep file is not ready then since is known to be an 635 // input to this edge, the edge will not be considered ready below. 636 // Later during the build the dyndep file will become ready and be 637 // loaded to update this edge before it can possibly be scheduled. 638 if edge.Dyndep != nil && edge.Dyndep.DyndepPending { 639 var err error 640 stack, validationNodes, err = d.recomputeNodeDirty(edge.Dyndep, stack, validationNodes) 641 if err != nil { 642 return stack, validationNodes, err 643 } 644 645 if edge.Dyndep.InEdge == nil || edge.Dyndep.InEdge.OutputsReady { 646 // The dyndep file is ready, so load it now. 647 if err := d.LoadDyndeps(edge.Dyndep, DyndepFile{}); err != nil { 648 return stack, validationNodes, err 649 } 650 } 651 } 652 } 653 654 // Load output mtimes so we can compare them to the most recent input below. 655 for _, o := range edge.Outputs { 656 if err := o.statIfNecessary(d.di); err != nil { 657 return stack, validationNodes, err 658 } 659 } 660 661 if !edge.DepsLoaded { 662 // This is our first encounter with this edge. Load discovered deps. 663 edge.DepsLoaded = true 664 if found, err := d.depLoader.loadDeps(edge); err != nil { 665 return stack, validationNodes, err 666 } else if !found { 667 // Failed to load dependency info: rebuild to regenerate it. 668 // loadDeps() did Explain() already, no need to do it here. 669 dirty = true 670 edge.DepsMissing = true 671 } 672 } 673 674 // Store any validation nodes from the edge for adding to the initial 675 // nodes. Don't recurse into them, that would trigger the dependency 676 // cycle detector if the validation node depends on this node. 677 // RecomputeDirty will add the validation nodes to the initial nodes 678 // and recurse into them. 679 validationNodes = append(validationNodes, edge.Validations...) 680 681 // Visit all inputs; we're dirty if any of the inputs are dirty. 682 var mostRecentInput *Node 683 for j, i := range edge.Inputs { 684 // Visit this input. 685 var err error 686 stack, validationNodes, err = d.recomputeNodeDirty(i, stack, validationNodes) 687 if err != nil { 688 return stack, validationNodes, err 689 } 690 691 // If an input is not ready, neither are our outputs. 692 if inEdge := i.InEdge; inEdge != nil { 693 if !inEdge.OutputsReady { 694 edge.OutputsReady = false 695 } 696 } 697 698 if !edge.IsOrderOnly(j) { 699 // If a regular input is dirty (or missing), we're dirty. 700 // Otherwise consider mtime. 701 if i.Dirty { 702 explain("%s is dirty", i.Path) 703 dirty = true 704 } else { 705 if mostRecentInput == nil || i.MTime > mostRecentInput.MTime { 706 mostRecentInput = i 707 } 708 } 709 } 710 } 711 712 // We may also be dirty due to output state: missing outputs, out of 713 // date outputs, etc. Visit all outputs and determine whether they're dirty. 714 if !dirty { 715 // The C++ code conditions on this but I think there's a bug in there. 716 dirty = d.recomputeOutputsDirty(edge, mostRecentInput) 717 } 718 719 // Finally, visit each output and update their dirty state if necessary. 720 for _, o := range edge.Outputs { 721 if dirty { 722 o.Dirty = true 723 } 724 } 725 726 // If an edge is dirty, its outputs are normally not ready. (It's 727 // possible to be clean but still not be ready in the presence of 728 // order-only inputs.) 729 // But phony edges with no inputs have nothing to do, so are always 730 // ready. 731 if dirty && !(edge.Rule == PhonyRule && len(edge.Inputs) == 0) { 732 edge.OutputsReady = false 733 } 734 735 // Mark the edge as finished during this walk now that it will no longer 736 // be in the call stack. 737 edge.Mark = VisitDone 738 // assert(stack[len(stack)-1] == node) 739 return stack[:len(stack)-1], validationNodes, nil 740 } 741 742 // verifyDAG checks that the node is a directed acyclic graph. 743 // 744 // Mutates stack in-place in case of error. 745 func (d *DependencyScan) verifyDAG(node *Node, stack []*Node) error { 746 edge := node.InEdge 747 748 // If we have no temporary mark on the edge then we do not yet have a cycle. 749 if edge.Mark != VisitInStack { 750 return nil 751 } 752 753 // We have this edge earlier in the call stack. Find it. 754 start := -1 755 for i := range stack { 756 if stack[i].InEdge == edge { 757 start = i 758 break 759 } 760 } 761 762 // Make the cycle clear by reporting its start as the node at its end 763 // instead of some other output of the starting edge. For example, 764 // running 'ninja b' on 765 // build a b: cat c 766 // build c: cat a 767 // should report a -> c -> a instead of b -> c -> a. 768 stack[start] = node 769 770 // Construct the error message rejecting the cycle. 771 err := "dependency cycle: " 772 for i := start; i != len(stack); i++ { 773 err += stack[i].Path 774 err += " -> " 775 } 776 err += stack[start].Path 777 778 if (start+1) == len(stack) && edge.maybePhonycycleDiagnostic() { 779 // The manifest parser would have filtered out the self-referencing 780 // input if it were not configured to allow the error. 781 err += " [-w phonycycle=err]" 782 } 783 return errors.New(err) 784 } 785 786 // recomputeOutputsDirty recomputes whether any output of the edge is dirty. 787 // 788 // Returns true if dirty. 789 func (d *DependencyScan) recomputeOutputsDirty(edge *Edge, mostRecentInput *Node) bool { 790 command := edge.EvaluateCommand(true) // inclRspFile= 791 for _, o := range edge.Outputs { 792 if d.recomputeOutputDirty(edge, mostRecentInput, command, o) { 793 return true 794 } 795 } 796 return false 797 } 798 799 // recomputeOutputDirty recomputes whether a given single output should be 800 // marked dirty. 801 // 802 // Returns true if so. 803 func (d *DependencyScan) recomputeOutputDirty(edge *Edge, mostRecentInput *Node, command string, output *Node) bool { 804 if edge.Rule == PhonyRule { 805 // Phony edges don't write any output. Outputs are only dirty if 806 // there are no inputs and we're missing the output. 807 if len(edge.Inputs) == 0 && output.Exists != ExistenceStatusExists { 808 explain("output %s of phony edge with no inputs doesn't exist", output.Path) 809 return true 810 } 811 812 // Update the mtime with the newest input. Dependents can thus call mtime() 813 // on the fake node and get the latest mtime of the dependencies 814 if mostRecentInput != nil { 815 output.updatePhonyMtime(mostRecentInput.MTime) 816 } 817 818 // Phony edges are clean, nothing to do. 819 return false 820 } 821 822 var entry *LogEntry 823 824 // Dirty if we're missing the output. 825 if output.Exists != ExistenceStatusExists { 826 explain("output %s doesn't exist", output.Path) 827 return true 828 } 829 830 // Dirty if the output is older than the input. 831 if mostRecentInput != nil && output.MTime < mostRecentInput.MTime { 832 outputMtime := output.MTime 833 834 // If this is a restat rule, we may have cleaned the output with a restat 835 // rule in a previous run and stored the most recent input mtime in the 836 // build log. Use that mtime instead, so that the file will only be 837 // considered dirty if an input was modified since the previous run. 838 usedRestat := false 839 if edge.GetBinding("restat") != "" && d.buildLog != nil { 840 if entry = d.buildLog.Entries[output.Path]; entry != nil { 841 outputMtime = entry.mtime 842 usedRestat = true 843 } 844 } 845 846 if outputMtime < mostRecentInput.MTime { 847 s := "" 848 if usedRestat { 849 s = "restat of " 850 } 851 explain("%soutput %s older than most recent input %s (%x vs %x)", s, output.Path, mostRecentInput.Path, outputMtime, mostRecentInput.MTime) 852 return true 853 } 854 } 855 856 if d.buildLog != nil { 857 generator := edge.GetBinding("generator") != "" 858 if entry == nil { 859 entry = d.buildLog.Entries[output.Path] 860 } 861 if entry != nil { 862 if !generator && HashCommand(command) != entry.commandHash { 863 // May also be dirty due to the command changing since the last build. 864 // But if this is a generator rule, the command changing does not make us 865 // dirty. 866 explain("command line changed for %s", output.Path) 867 return true 868 } 869 if mostRecentInput != nil && entry.mtime < mostRecentInput.MTime { 870 // May also be dirty due to the mtime in the log being older than the 871 // mtime of the most recent input. This can occur even when the mtime 872 // on disk is newer if a previous run wrote to the output file but 873 // exited with an error or was interrupted. 874 explain("recorded mtime of %s older than most recent input %s (%x vs %x)", output.Path, mostRecentInput.Path, entry.mtime, mostRecentInput.MTime) 875 return true 876 } 877 } 878 if entry == nil && !generator { 879 explain("command line not found in log for %s", output.Path) 880 return true 881 } 882 } 883 return false 884 } 885 886 // LoadDyndeps loads a dyndep file from the given node's path and update the 887 // build graph with the new information. 888 // 889 // The 'DyndepFile' object stores the information loaded from the dyndep file. 890 func (d *DependencyScan) LoadDyndeps(node *Node, ddf DyndepFile) error { 891 return d.dyndepLoader.LoadDyndeps(node, ddf) 892 } 893 894 // 895 896 // implicitDepLoader loads implicit dependencies, as referenced via the 897 // "depfile" attribute in build files. 898 type implicitDepLoader struct { 899 state *State 900 di DiskInterface 901 depsLog *DepsLog 902 } 903 904 func newImplicitDepLoader(state *State, depsLog *DepsLog, di DiskInterface) implicitDepLoader { 905 return implicitDepLoader{ 906 state: state, 907 di: di, 908 depsLog: depsLog, 909 } 910 } 911 912 // loadDeps loads implicit dependencies for edge. 913 // 914 // Returns false if info is just missing or out of date. 915 func (i *implicitDepLoader) loadDeps(edge *Edge) (bool, error) { 916 depsType := edge.GetBinding("deps") 917 if len(depsType) != 0 { 918 return i.loadDepsFromLog(edge), nil 919 } 920 921 depfile := edge.GetUnescapedDepfile() 922 if len(depfile) != 0 { 923 return i.loadDepFile(edge, depfile) 924 } 925 926 // No deps to load. 927 return true, nil 928 } 929 930 // loadDepFile loads implicit dependencies for edge from a depfile attribute. 931 // 932 // Returns false if info is just missing or on error. 933 func (i *implicitDepLoader) loadDepFile(edge *Edge, path string) (bool, error) { 934 defer metricRecord("depfile load")() 935 // Read depfile content. Treat a missing depfile as empty. 936 content, err := i.di.ReadFile(path) 937 if err != nil && !os.IsNotExist(err) { 938 // TODO(maruel): Use %q for real quoting. 939 return false, fmt.Errorf("loading '%s': %w", path, err) 940 } 941 // On a missing depfile: return false and empty error. 942 if len(content) == 0 { 943 // TODO(maruel): Use %q for real quoting. 944 explain("depfile '%s' is missing", path) 945 return false, nil 946 } 947 948 depfile := DepfileParser{} 949 if err := depfile.Parse(content); err != nil { 950 return false, fmt.Errorf("%s: %w", path, err) 951 } 952 953 if len(depfile.outs) == 0 { 954 return false, errors.New(path + ": no outputs declared") 955 } 956 957 // Check that this depfile matches the edge's output, if not return false to 958 // mark the edge as dirty. 959 firstOutput := edge.Outputs[0] 960 if primaryOut := CanonicalizePath(depfile.outs[0]); firstOutput.Path != primaryOut { 961 explain("expected depfile '%s' to mention '%s', got '%s'", path, firstOutput.Path, primaryOut) 962 return false, nil 963 } 964 965 // Ensure that all mentioned outputs are outputs of the edge. 966 for _, o := range depfile.outs { 967 found := false 968 for _, n := range edge.Outputs { 969 if n.Path == o { 970 found = true 971 break 972 } 973 } 974 if !found { 975 // TODO(maruel): Use %q for real quoting. 976 return false, fmt.Errorf("%s: depfile mentions '%s' as an output, but no such output was declared", path, o) 977 } 978 } 979 return i.processDepfileDeps(edge, depfile.ins), nil 980 } 981 982 // processDepfileDeps processes loaded implicit dependencies for edge and 983 // update the graph. 984 // 985 // Returns false with info is just missing. 986 func (i *implicitDepLoader) processDepfileDeps(edge *Edge, depfileIns []string) bool { 987 // Preallocate space in edge.Inputs to be filled in below. 988 implicitDep := i.preallocateSpace(edge, len(depfileIns)) 989 990 // Add all its in-edges. 991 for _, j := range depfileIns { 992 node := i.state.GetNode(CanonicalizePathBits(j)) 993 edge.Inputs[implicitDep] = node 994 node.OutEdges = append(node.OutEdges, edge) 995 i.createPhonyInEdge(node) 996 implicitDep++ 997 } 998 return true 999 } 1000 1001 // loadDepsFromLog loads implicit dependencies for edge from the DepsLog. 1002 // 1003 // Returns false if info is missing. 1004 func (i *implicitDepLoader) loadDepsFromLog(edge *Edge) bool { 1005 // NOTE: deps are only supported for single-target edges. 1006 output := edge.Outputs[0] 1007 var deps *Deps 1008 if i.depsLog != nil { 1009 deps = i.depsLog.GetDeps(output) 1010 } 1011 if deps == nil { 1012 explain("deps for '%s' are missing", output.Path) 1013 return false 1014 } 1015 1016 // Deps are invalid if the output is newer than the deps. 1017 if output.MTime > deps.MTime { 1018 explain("stored deps info out of date for '%s' (%x vs %x)", output.Path, deps.MTime, output.MTime) 1019 return false 1020 } 1021 1022 implicitDep := i.preallocateSpace(edge, len(deps.Nodes)) 1023 for _, node := range deps.Nodes { 1024 edge.Inputs[implicitDep] = node 1025 node.OutEdges = append(node.OutEdges, edge) 1026 i.createPhonyInEdge(node) 1027 implicitDep++ 1028 } 1029 return true 1030 } 1031 1032 // preallocateSpace preallocates count spaces in the input array on edge, 1033 // returning the index at the first new space. 1034 func (i *implicitDepLoader) preallocateSpace(edge *Edge, count int) int { 1035 offset := len(edge.Inputs) - int(edge.OrderOnlyDeps) 1036 old := edge.Inputs 1037 edge.Inputs = make([]*Node, len(old)+count) 1038 copy(edge.Inputs, old[:offset]) 1039 copy(edge.Inputs[offset+count:], old[offset:]) 1040 edge.ImplicitDeps += int32(count) 1041 return len(edge.Inputs) - int(edge.OrderOnlyDeps) - count 1042 } 1043 1044 // createPhonyInEdge creates an edge that generates this input if we don't have 1045 // one already. 1046 // 1047 // This makes us not abort if the input is missing, but instead will rebuild in 1048 // that circumstance. 1049 func (i *implicitDepLoader) createPhonyInEdge(node *Node) { 1050 if node.InEdge != nil { 1051 return 1052 } 1053 1054 phonyEdge := i.state.addEdge(PhonyRule) 1055 phonyEdge.GeneratedByDepLoader = true 1056 node.InEdge = phonyEdge 1057 phonyEdge.Outputs = append(phonyEdge.Outputs, node) 1058 1059 // RecomputeDirty might not be called for phonyEdge if a previous call 1060 // to RecomputeDirty had caused the file to be stat'ed. Because previous 1061 // invocations of RecomputeDirty would have seen this node without an 1062 // input edge (and therefore ready), we have to set OutputsReady to true 1063 // to avoid a potential stuck build. If we do call RecomputeDirty for 1064 // this node, it will simply set OutputsReady to the correct value. 1065 phonyEdge.OutputsReady = true 1066 }