github.com/maruel/nin@v0.0.0-20220112143044-f35891e3ce7e/manifest_parser_concurrent.go (about) 1 // Copyright 2011 Google Inc. All Rights Reserved. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package nin 16 17 import ( 18 "fmt" 19 "strconv" 20 ) 21 22 // manifestParserConcurrent parses .ninja files. 23 type manifestParserConcurrent struct { 24 manifestParserRoutine 25 manifestParserState 26 } 27 28 // manifestParserRoutine is the state of the parsing goroutine. 29 type manifestParserRoutine struct { 30 // Mutable. 31 lexer lexer 32 manifestParserContext 33 } 34 35 type manifestParserContext struct { 36 env *BindingEnv 37 doneParsing barrier 38 subninjas chan error 39 subninjasEnqueued int32 40 } 41 42 type barrier struct { 43 want chan struct{} 44 } 45 46 func (b *barrier) unblock() { 47 // TODO(maruel): Memory leak? 48 go func() { 49 for range b.want { 50 } 51 }() 52 } 53 54 func (b *barrier) wait() { 55 b.want <- struct{}{} 56 } 57 58 type actionBatch [10]interface{} 59 60 // manifestParserState is the state of the processing goroutine. 61 type manifestParserState struct { 62 // Mutable. 63 state *State 64 65 // Immutable. 66 options ParseManifestOpts 67 fr FileReader 68 // These need to be saved since this goroutine doesn't have access to lexer 69 // to reconstruct errors. 70 filename string 71 input []byte 72 } 73 74 // parse parses a file, given its contents as a string. 75 // 76 // Primary Processing Subninja Nth 77 // Where goroutine goroutine 78 // ParseManifest() process() processSubninjaReal() 79 // was called │ │ 80 // │ │ │ 81 // ▼ ▼ ▼ 82 // ─────────────────────────────────────────────────────────────── 83 // │ 84 // Initiate 85 // │ 86 // ───────────────────────►│ 87 // │ │ 88 // ┌────────────┐ │ 89 // │parses and │ │ 90 // │send actions│ actions│ 91 // └──────────────────────►│ 92 // ├─────────────────────┐ 93 // │EvalString.Evaluate()│ 94 // │update m.state │ 95 // └─────────────────────┘ 96 // │ │ 97 // ├────────────┐ │ 98 // │parses and │ │ 99 // │send actions│ actions│ 100 // └──────────────────────►│ 101 // ├─────────────────────┐ 102 // │EvalString.Evaluate()│ 103 // │update m.state │ 104 // └─────────────────────┘ 105 // │ │ 106 // ├──────────────┐ │ 107 // │parseInclude()│ actions│ 108 // └──────────────────────►│ 109 // ├─────────────────────┐ 110 // │processInclude() │ 111 // │EvalString.Evaluate()│ 112 // │run new parser here │ 113 // └─────────────────────┘ 114 // │ │ 115 // ├───────────────┐ │ 116 // │parseSubninja()│actions│ 117 // └──────────────────────►│ 118 // ├─────────────────────┐ 119 // │processSubninja() │ 120 // │EvalString.Evaluate()│ 121 // │Start goroutine to │ 122 // │read file │ 123 // └────────────────────────►│ 124 // │ 125 // │ ┌─────┴───────┐ 126 // ├───────┐ │read subninja│ 127 // │Done │ └─────────────┘ 128 // │parsing│ doneParsing │ 129 // └────────────────────────────────────────────────►│ 130 // ┌────┴───────┐ 131 // │ │parses and │ 132 // │actions │send actions│ 133 // │◄────────────────────────────────┘ 134 // │ 135 // ├─────────────────────┐ 136 // │EvalString.Evaluate()│ 137 // │update m.state │ 138 // └─────────────────────┘ 139 // │ │ 140 // │ │ 141 // │processResult │ 142 // │◄─────────────────────── 143 // │ 144 // ▼ 145 // Done 146 // 147 // "parses and send actions" is one of: parsePool(), parseEdge(), parseRule(), 148 // parseDefault() or parseIdent(). 149 // 150 // Warning: a subninja can have includes and subninjas itself. They need to 151 // block the subninja parsing goroutine and until main action routine unblocks 152 // the barrier. 153 func (m *manifestParserConcurrent) parseMain(filename string, input []byte) error { 154 defer metricRecord(".ninja parse")() 155 156 // We want some amount of buffering to help with the parsing getting ahead of 157 // the processing. 158 actions := make(chan actionBatch, 128) 159 processResult := make(chan error) 160 161 // For error(). 162 m.manifestParserState.filename = filename 163 m.manifestParserState.input = input 164 165 // Processing goroutine 166 go func() { 167 // This goroutine doesn't have access to m.lexer. It will enqueue 168 // goroutines to read subninjas in parallel, hence the wg to ensure we wait 169 // for them when we terminate early. 170 processResult <- m.manifestParserState.process(actions) 171 }() 172 173 // First, block on the parser to be done. 174 err := m.parse(filename, input, actions) 175 close(actions) 176 177 // Between actions results and parsing results, prioritize parsing results. 178 if err2 := <-processResult; err == nil { 179 err = err2 180 } 181 return err 182 } 183 184 // parse runs the lexer parsing loop. 185 // 186 // It parses but does not process. It enqueues actions into actions that the 187 // main goroutine shall execute. 188 func (m *manifestParserRoutine) parse(filename string, input []byte, actions chan<- actionBatch) error { 189 // The parsing done by the lexer can be done in a separate thread. What is 190 // important is that m.state is never touched concurrently. 191 m.subninjas = make(chan error) 192 193 // Parse the main manifest (build.ninja). 194 if err := m.lexer.Start(filename, input); err != nil { 195 return err 196 } 197 // subninja files are read as soon as the statement is parsed but they are 198 // only processed once the current file is done. This enables lower latency 199 // overall. 200 var err error 201 var array actionBatch 202 index := 0 203 loop: 204 for err == nil { 205 if index == len(array) { 206 actions <- array 207 index = 0 208 } 209 switch token := m.lexer.ReadToken(); token { 210 case POOL: 211 array[index], err = m.parsePool() 212 index++ 213 case BUILD: 214 array[index], err = m.parseEdge() 215 index++ 216 case RULE: 217 array[index], err = m.parseRule() 218 index++ 219 case DEFAULT: 220 array[index], err = m.parseDefault() 221 index++ 222 case IDENT: 223 array[index], err = m.parseIdent() 224 index++ 225 case INCLUDE: 226 array[index], err = m.parseInclude() 227 index++ 228 case SUBNINJA: 229 array[index], err = m.parseSubninja() 230 index++ 231 case ERROR: 232 err = m.lexer.Error(m.lexer.DescribeLastError()) 233 case TEOF: 234 // Don't forget the last item too. 235 break loop 236 case NEWLINE: 237 default: 238 err = m.lexer.Error("unexpected " + token.String()) 239 } 240 } 241 242 // Send any remaining stragglers. 243 if err == nil && index != 0 { 244 for i := index; i < len(array); i++ { 245 array[i] = nil 246 } 247 actions <- array 248 } 249 // Send an event of ourselves to notify actions that it is ready to process 250 // subninjas. 251 array[0] = m 252 for i := 1; i < len(array); i++ { 253 array[i] = nil 254 } 255 actions <- array 256 257 m.doneParsing.wait() 258 259 // Once all the actions created by m.parse() are processed, m.env is 260 // completely immutable and can be accessed concurrently. 261 // 262 // We can safely process the subninja files since m.env becomes static. 263 // 264 // This is done here because parse() doesn't have access to m.subninjas. 265 for i := int32(0); i < m.subninjasEnqueued; i++ { 266 if err2 := <-m.subninjas; err == nil { 267 err = err2 268 } 269 } 270 return err 271 } 272 273 func (m *manifestParserState) process(actions chan actionBatch) error { 274 var err error 275 for s := range actions { 276 for _, a := range s { 277 if err != nil { 278 // Ignore following actions if we got an error but we still need to 279 // continue emptying the channel. 280 switch d := a.(type) { 281 case *manifestParserRoutine: 282 // Unblock all the subninjas for this specific routine. It is important 283 // because recursive subninjas have to wait for their parent subninja to 284 // be processed. 285 d.doneParsing.unblock() 286 default: 287 } 288 continue 289 } 290 switch d := a.(type) { 291 case dataPool: 292 err = m.processPool(d) 293 case dataEdge: 294 err = m.processEdge(d) 295 case dataRule: 296 err = m.processRule(d) 297 case dataDefault: 298 err = m.processDefault(d) 299 case dataIdent: 300 err = m.processIdent(d) 301 case dataInclude: 302 // Loads the included file immediately. 303 // An include can be in a subninja, so the right BindingsEnv must be 304 // loaded. 305 err = m.processInclude(d) 306 case dataSubninja: 307 // Enqueues the file to read into m.subninjas. 308 // A subninja can be from another subninja, so the right BindingsEnv must 309 // be loaded. 310 err = m.processSubninja(d, actions) 311 case *manifestParserRoutine: 312 // Unblock all the subninjas for this specific routine. It is important 313 // because recursive subninjas have to wait for their parent subninja to 314 // be processed. 315 d.doneParsing.unblock() 316 } 317 } 318 } 319 return err 320 } 321 322 // parsePool parses a "pool" statement. 323 func (m *manifestParserRoutine) parsePool() (dataPool, error) { 324 d := dataPool{env: m.env} 325 d.name = m.lexer.readIdent() 326 if d.name == "" { 327 return d, m.lexer.Error("expected pool name") 328 } 329 if err := m.expectToken(NEWLINE); err != nil { 330 return d, err 331 } 332 d.ls = m.lexer.lexerState 333 for m.lexer.PeekToken(INDENT) { 334 key := "" 335 var err error 336 key, d.eval, err = m.parseLet() 337 if err != nil { 338 return d, err 339 } 340 if key != "depth" { 341 // TODO(maruel): Use %q for real quoting. 342 return d, m.lexer.Error(fmt.Sprintf("unexpected variable '%s'", key)) 343 } 344 d.dls = m.lexer.lexerState 345 } 346 if len(d.eval.Parsed) == 0 { 347 return d, m.lexer.Error("expected 'depth =' line") 348 } 349 return d, nil 350 } 351 352 // processPool updates m.state with a parsed pool statement. 353 func (m *manifestParserState) processPool(d dataPool) error { 354 if m.state.Pools[d.name] != nil { 355 // TODO(maruel): Use %q for real quoting. 356 return m.error(fmt.Sprintf("duplicate pool '%s'", d.name), d.ls) 357 } 358 // TODO(maruel): Do we want to use ParseInt() here? Aka support hex. 359 depth, err := strconv.Atoi(d.eval.Evaluate(d.env)) 360 if depth < 0 || err != nil { 361 return m.error("invalid pool depth", d.dls) 362 } 363 m.state.Pools[d.name] = NewPool(d.name, depth) 364 return nil 365 } 366 367 // parseRule parses a "rule" statement. 368 func (m *manifestParserRoutine) parseRule() (dataRule, error) { 369 d := dataRule{env: m.env} 370 name := m.lexer.readIdent() 371 if name == "" { 372 return d, m.lexer.Error("expected rule name") 373 } 374 if err := m.expectToken(NEWLINE); err != nil { 375 return d, err 376 } 377 d.ls = m.lexer 378 d.rule = NewRule(name) 379 for m.lexer.PeekToken(INDENT) { 380 key, value, err := m.parseLet() 381 if err != nil { 382 return d, err 383 } 384 385 if !IsReservedBinding(key) { 386 // Die on other keyvals for now; revisit if we want to add a 387 // scope here. 388 // TODO(maruel): Use %q for real quoting. 389 return d, m.lexer.Error(fmt.Sprintf("unexpected variable '%s'", key)) 390 } 391 d.rule.Bindings[key] = &value 392 } 393 394 b1, ok1 := d.rule.Bindings["rspfile"] 395 b2, ok2 := d.rule.Bindings["rspfile_content"] 396 if ok1 != ok2 || (ok1 && (len(b1.Parsed) == 0) != (len(b2.Parsed) == 0)) { 397 return d, m.lexer.Error("rspfile and rspfile_content need to be both specified") 398 } 399 400 b, ok := d.rule.Bindings["command"] 401 if !ok || len(b.Parsed) == 0 { 402 return d, m.lexer.Error("expected 'command =' line") 403 } 404 return d, nil 405 } 406 407 // processRule updates m.state with a parsed rule statement. 408 func (m *manifestParserState) processRule(d dataRule) error { 409 if d.env.Rules[d.rule.Name] != nil { 410 // TODO(maruel): Use %q for real quoting. 411 return d.ls.Error(fmt.Sprintf("duplicate rule '%s'", d.rule.Name)) 412 } 413 d.env.Rules[d.rule.Name] = d.rule 414 return nil 415 } 416 417 // parseDefault parses a "default" statement. 418 func (m *manifestParserRoutine) parseDefault() (dataDefault, error) { 419 d := dataDefault{env: m.env} 420 eval, err := m.lexer.readEvalString(true) 421 if err != nil { 422 return d, err 423 } 424 if len(eval.Parsed) == 0 { 425 return d, m.lexer.Error("expected target name") 426 } 427 428 d.evals = []*parsedEval{{eval, m.lexer}} 429 for { 430 if eval, err = m.lexer.readEvalString(true); err != nil { 431 return d, err 432 } 433 if len(eval.Parsed) == 0 { 434 break 435 } 436 d.evals = append(d.evals, &parsedEval{eval, m.lexer}) 437 } 438 return d, m.expectToken(NEWLINE) 439 } 440 441 // processDefault updates m.state with a parsed default statement. 442 func (m *manifestParserState) processDefault(d dataDefault) error { 443 for i := 0; i < len(d.evals); i++ { 444 path := d.evals[i].eval.Evaluate(d.env) 445 if len(path) == 0 { 446 return d.evals[i].ls.Error("empty path") 447 } 448 if err := m.state.addDefault(CanonicalizePath(path)); err != nil { 449 return d.evals[i].ls.Error(err.Error()) 450 } 451 } 452 return nil 453 } 454 455 // parseIdent parses a generic statement as a fallback. 456 func (m *manifestParserRoutine) parseIdent() (dataIdent, error) { 457 d := dataIdent{env: m.env} 458 m.lexer.UnreadToken() 459 var err error 460 d.name, d.eval, err = m.parseLet() 461 return d, err 462 } 463 464 // processIdent updates m.state with a parsed ident statement. 465 func (m *manifestParserState) processIdent(d dataIdent) error { 466 value := d.eval.Evaluate(d.env) 467 // Check ninjaRequiredVersion immediately so we can exit 468 // before encountering any syntactic surprises. 469 if d.name == "ninja_required_version" { 470 if err := checkNinjaVersion(value); err != nil { 471 return err 472 } 473 } 474 d.env.Bindings[d.name] = value 475 return nil 476 } 477 478 // parseEdge parses a "build" statement that results into an edge, which 479 // defines inputs and outputs. 480 func (m *manifestParserRoutine) parseEdge() (dataEdge, error) { 481 d := dataEdge{env: m.env} 482 for { 483 ev, err := m.lexer.readEvalString(true) 484 if err != nil { 485 return d, err 486 } 487 if len(ev.Parsed) == 0 { 488 break 489 } 490 d.outs = append(d.outs, ev) 491 } 492 493 // Add all implicit outs, counting how many as we go. 494 if m.lexer.PeekToken(PIPE) { 495 for { 496 ev, err := m.lexer.readEvalString(true) 497 if err != nil { 498 return d, err 499 } 500 if len(ev.Parsed) == 0 { 501 break 502 } 503 d.outs = append(d.outs, ev) 504 d.implicitOuts++ 505 } 506 } 507 508 if len(d.outs) == 0 { 509 return d, m.lexer.Error("expected path") 510 } 511 512 if err := m.expectToken(COLON); err != nil { 513 return d, err 514 } 515 516 d.ruleName = m.lexer.readIdent() 517 if d.ruleName == "" { 518 return d, m.lexer.Error("expected build command name") 519 } 520 521 // Save the lexer for unknown rule check later. 522 d.lsRule = m.lexer 523 524 for { 525 // XXX should we require one path here? 526 ev, err := m.lexer.readEvalString(true) 527 if err != nil { 528 return d, err 529 } 530 if len(ev.Parsed) == 0 { 531 break 532 } 533 d.ins = append(d.ins, ev) 534 } 535 536 // Add all implicit deps, counting how many as we go. 537 if m.lexer.PeekToken(PIPE) { 538 for { 539 ev, err := m.lexer.readEvalString(true) 540 if err != nil { 541 return d, err 542 } 543 if len(ev.Parsed) == 0 { 544 break 545 } 546 d.ins = append(d.ins, ev) 547 d.implicit++ 548 } 549 } 550 551 // Add all order-only deps, counting how many as we go. 552 if m.lexer.PeekToken(PIPE2) { 553 for { 554 ev, err := m.lexer.readEvalString(true) 555 if err != nil { 556 return d, err 557 } 558 if len(ev.Parsed) == 0 { 559 break 560 } 561 d.ins = append(d.ins, ev) 562 d.orderOnly++ 563 } 564 } 565 566 // Add all validations, counting how many as we go. 567 if m.lexer.PeekToken(PIPEAT) { 568 for { 569 ev, err := m.lexer.readEvalString(true) 570 if err != nil { 571 return d, err 572 } 573 if len(ev.Parsed) == 0 { 574 break 575 } 576 d.validations = append(d.validations, ev) 577 } 578 } 579 580 if err := m.expectToken(NEWLINE); err != nil { 581 return d, err 582 } 583 584 // Bindings on edges are rare, so allocate per-edge envs only when needed. 585 d.hadIndentToken = m.lexer.PeekToken(INDENT) 586 // Accumulate the bindings for now, will process them later. 587 for h := d.hadIndentToken; h; h = m.lexer.PeekToken(INDENT) { 588 key, val, err := m.parseLet() 589 if err != nil { 590 return d, err 591 } 592 d.bindings = append(d.bindings, &keyEval{key, val}) 593 } 594 d.lsEnd = m.lexer.lexerState 595 return d, nil 596 } 597 598 // processEdge updates m.state with a parsed edge statement. 599 func (m *manifestParserState) processEdge(d dataEdge) error { 600 rule := d.env.LookupRule(d.ruleName) 601 if rule == nil { 602 // TODO(maruel): Use %q for real quoting. 603 return d.lsRule.Error(fmt.Sprintf("unknown build rule '%s'", d.ruleName)) 604 } 605 env := d.env 606 if d.hadIndentToken { 607 env = NewBindingEnv(d.env) 608 } 609 for _, i := range d.bindings { 610 env.Bindings[i.key] = i.eval.Evaluate(d.env) 611 } 612 613 edge := m.state.addEdge(rule) 614 edge.Env = env 615 616 if poolName := edge.GetBinding("pool"); poolName != "" { 617 pool := m.state.Pools[poolName] 618 if pool == nil { 619 // TODO(maruel): Use %q for real quoting. 620 return d.lsEnd.error(fmt.Sprintf("unknown pool name '%s'", poolName), d.lsRule.filename, d.lsRule.input) 621 } 622 edge.Pool = pool 623 } 624 625 edge.Outputs = make([]*Node, 0, len(d.outs)) 626 for i, o := range d.outs { 627 path := o.Evaluate(env) 628 if len(path) == 0 { 629 return d.lsEnd.error("empty path", d.lsRule.filename, d.lsRule.input) 630 } 631 path, slashBits := CanonicalizePathBits(path) 632 if !m.state.addOut(edge, path, slashBits) { 633 if m.options.ErrOnDupeEdge { 634 return d.lsEnd.error("multiple rules generate "+path, d.lsRule.filename, d.lsRule.input) 635 } 636 if !m.options.Quiet { 637 warningf("multiple rules generate %s. builds involving this target will not be correct; continuing anyway", path) 638 } 639 if len(d.outs)-i <= d.implicitOuts { 640 d.implicitOuts-- 641 } 642 } 643 } 644 if len(edge.Outputs) == 0 { 645 // All outputs of the edge are already created by other edges. Don't add 646 // this edge. Do this check before input nodes are connected to the edge. 647 m.state.Edges = m.state.Edges[:len(m.state.Edges)-1] 648 return nil 649 } 650 edge.ImplicitOuts = int32(d.implicitOuts) 651 652 edge.Inputs = make([]*Node, 0, len(d.ins)) 653 for _, i := range d.ins { 654 path := i.Evaluate(env) 655 if len(path) == 0 { 656 return d.lsEnd.error("empty path", d.lsRule.filename, d.lsRule.input) 657 } 658 path, slashBits := CanonicalizePathBits(path) 659 m.state.addIn(edge, path, slashBits) 660 } 661 edge.ImplicitDeps = int32(d.implicit) 662 edge.OrderOnlyDeps = int32(d.orderOnly) 663 664 edge.Validations = make([]*Node, 0, len(d.validations)) 665 for _, v := range d.validations { 666 path := v.Evaluate(env) 667 if path == "" { 668 return d.lsEnd.error("empty path", d.lsRule.filename, d.lsRule.input) 669 } 670 path, slashBits := CanonicalizePathBits(path) 671 m.state.addValidation(edge, path, slashBits) 672 } 673 674 if !m.options.ErrOnPhonyCycle && edge.maybePhonycycleDiagnostic() { 675 // CMake 2.8.12.x and 3.0.x incorrectly write phony build statements 676 // that reference themselves. Ninja used to tolerate these in the 677 // build graph but that has since been fixed. Filter them out to 678 // support users of those old CMake versions. 679 out := edge.Outputs[0] 680 for i, n := range edge.Inputs { 681 if n == out { 682 copy(edge.Inputs[i:], edge.Inputs[i+1:]) 683 edge.Inputs = edge.Inputs[:len(edge.Inputs)-1] 684 if !m.options.Quiet { 685 warningf("phony target '%s' names itself as an input; ignoring [-w phonycycle=warn]", out.Path) 686 } 687 break 688 } 689 } 690 } 691 692 // Lookup, validate, and save any dyndep binding. It will be used later 693 // to load generated dependency information dynamically, but it must 694 // be one of our manifest-specified inputs. 695 dyndep := edge.GetUnescapedDyndep() 696 if len(dyndep) != 0 { 697 n := m.state.GetNode(CanonicalizePathBits(dyndep)) 698 n.DyndepPending = true 699 edge.Dyndep = n 700 found := false 701 for _, x := range edge.Inputs { 702 if x == n { 703 found = true 704 break 705 } 706 } 707 if !found { 708 // TODO(maruel): Use %q for real quoting. 709 return d.lsEnd.error(fmt.Sprintf("dyndep '%s' is not an input", dyndep), d.lsRule.filename, d.lsRule.input) 710 } 711 } 712 return nil 713 } 714 715 // parseInclude parses a "include" line. 716 func (m *manifestParserRoutine) parseInclude() (dataInclude, error) { 717 d := dataInclude{env: m.env} 718 var err error 719 if d.eval, err = m.lexer.readEvalString(true); err != nil { 720 return d, err 721 } 722 d.ls = m.lexer 723 return d, m.expectToken(NEWLINE) 724 } 725 726 // processInclude updates m.state by parsing an included ninja file. 727 // 728 // This is a stop-the-world event. 729 func (m *manifestParserState) processInclude(d dataInclude) error { 730 path := d.eval.Evaluate(d.env) 731 input, err := m.fr.ReadFile(path) 732 if err != nil { 733 // Wrap it. 734 // TODO(maruel): Use %q for real quoting. 735 return d.ls.Error(fmt.Sprintf("loading '%s': %s", path, err)) 736 } 737 738 // Synchronously parse the inner file. This is because the following lines 739 // may require declarations from this file. 740 // 741 // Manually construct the object instead of using ParseManifest(), because 742 // d.env may not equal to m.state.Bindings. This happens when the include 743 // statement is inside a subninja. 744 subparser := manifestParserConcurrent{ 745 manifestParserRoutine: manifestParserRoutine{ 746 manifestParserContext: manifestParserContext{ 747 env: d.env, 748 doneParsing: barrier{ 749 want: make(chan struct{}), 750 }, 751 }, 752 }, 753 manifestParserState: manifestParserState{ 754 fr: m.fr, 755 options: m.options, 756 state: m.state, 757 }, 758 } 759 // Recursively parse the input into the current state. This works because we 760 // completely hang the primary process() goroutine. 761 // Do not wrap error inside the included ninja. 762 return subparser.parseMain(path, input) 763 } 764 765 // parseSubninja parses a "subninja" statement. 766 func (m *manifestParserRoutine) parseSubninja() (dataSubninja, error) { 767 d := dataSubninja{context: &m.manifestParserContext} 768 var err error 769 if d.eval, err = m.lexer.readEvalString(true); err != nil { 770 return d, err 771 } 772 d.ls = m.lexer 773 return d, m.expectToken(NEWLINE) 774 } 775 776 // processSubninja is an action in the main thread to evaluate the subninja to 777 // parse and start up a separate goroutine to read it. 778 func (m *manifestParserState) processSubninja(d dataSubninja, actions chan<- actionBatch) error { 779 // We can finally resolve what file path it is. Start the read to process 780 // later. 781 filename := d.eval.Evaluate(d.context.env) 782 // Start the goroutine to read it asynchronously. It will send an action back. 783 // TODO(maruel): Use a workerpool, something around runtime.NumCPU() ? 784 d.context.subninjasEnqueued++ 785 go m.processSubninjaReal(filename, d, actions) 786 return nil 787 } 788 789 // processSubninjaReal is the goroutine that reads the subninja file in parallel 790 // to the main build.ninja to reduce overall latency, then parse the subninja 791 // once the parent's parser is done processing its actions. 792 // 793 // Contrary to the include, here we run a separate concurrent parsing loop. The 794 // state modification is still in the main loop. 795 func (m *manifestParserState) processSubninjaReal(filename string, d dataSubninja, actions chan<- actionBatch) { 796 input, err := m.fr.ReadFile(filename) 797 if err != nil { 798 // Wrap it. 799 // TODO(maruel): Use %q for real quoting. 800 err = d.ls.Error(fmt.Sprintf("loading '%s': %s", filename, err.Error())) 801 } 802 803 // We are NOT allowed to write to actions, because we are in a completely new 804 // and separate goroutine. 805 d.context.doneParsing.wait() 806 807 // At this point, the parent's m.env is completely immutable and can be 808 // accessed concurrently. We can now safely process the subninja 809 // concurrently! 810 811 if err == nil { 812 subparser := manifestParserConcurrent{ 813 manifestParserRoutine: manifestParserRoutine{ 814 manifestParserContext: manifestParserContext{ 815 // Reset the binding fresh with a temporary one that will not affect the 816 // root one. 817 env: NewBindingEnv(d.context.env), 818 doneParsing: barrier{ 819 want: make(chan struct{}), 820 }, 821 }, 822 }, 823 manifestParserState: manifestParserState{ 824 fr: m.fr, 825 options: m.options, 826 state: m.state, 827 }, 828 } 829 // We must not use subparser.parseMain() here, since we want it to send the 830 // actions to the main thread. So use parse() directly. This is fine 831 // because we do not want to handle grand-children subninjas here. 832 err = subparser.parse(filename, input, actions) 833 } 834 d.context.subninjas <- err 835 } 836 837 func (m *manifestParserRoutine) parseLet() (string, EvalString, error) { 838 eval := EvalString{} 839 key := m.lexer.readIdent() 840 if key == "" { 841 return key, eval, m.lexer.Error("expected variable name") 842 } 843 var err error 844 if err = m.expectToken(EQUALS); err == nil { 845 eval, err = m.lexer.readEvalString(false) 846 } 847 return key, eval, err 848 } 849 850 // expectToken produces an error string if the next token is not expected. 851 // 852 // The error says "expected foo, got bar". 853 func (m *manifestParserRoutine) expectToken(expected Token) error { 854 if token := m.lexer.ReadToken(); token != expected { 855 msg := "expected " + expected.String() + ", got " + token.String() + expected.errorHint() 856 return m.lexer.Error(msg) 857 } 858 return nil 859 } 860 861 func (m *manifestParserState) error(msg string, ls lexerState) error { 862 return ls.error(msg, m.filename, m.input) 863 } 864 865 type dataPool struct { 866 env *BindingEnv 867 name string 868 eval EvalString 869 ls, dls lexerState 870 } 871 872 type dataEdge struct { 873 env *BindingEnv 874 ruleName string 875 bindings []*keyEval 876 lsRule lexer 877 lsEnd lexerState // Kind of a hack. 878 ins, outs, validations []EvalString 879 implicit, orderOnly int 880 implicitOuts int 881 hadIndentToken bool 882 } 883 884 type dataRule struct { 885 env *BindingEnv 886 rule *Rule 887 ls lexer 888 } 889 890 type dataDefault struct { 891 env *BindingEnv 892 evals []*parsedEval 893 } 894 895 type dataIdent struct { 896 env *BindingEnv 897 name string 898 eval EvalString 899 } 900 901 type dataInclude struct { 902 env *BindingEnv 903 eval EvalString 904 ls lexer 905 } 906 907 type dataSubninja struct { 908 eval EvalString 909 ls lexer 910 context *manifestParserContext 911 } 912 913 type parsedEval struct { 914 eval EvalString 915 ls lexer 916 } 917 918 type keyEval struct { 919 key string 920 eval EvalString 921 }