github.com/maruel/nin@v0.0.0-20220112143044-f35891e3ce7e/manifest_parser_serial.go (about) 1 // Copyright 2011 Google Inc. All Rights Reserved. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package nin 16 17 import ( 18 "fmt" 19 "strconv" 20 ) 21 22 // manifestParserSerial parses .ninja files. 23 type manifestParserSerial struct { 24 // Immutable 25 fr FileReader 26 options ParseManifestOpts 27 28 // Mutable. 29 lexer lexer 30 state *State 31 env *BindingEnv 32 subninjas chan subninja 33 subninjasEnqueued int32 34 } 35 36 // parse parses a file, given its contents as a string. 37 func (m *manifestParserSerial) parse(filename string, input []byte) error { 38 defer metricRecord(".ninja parse")() 39 40 m.subninjas = make(chan subninja) 41 42 if err := m.lexer.Start(filename, input); err != nil { 43 return err 44 } 45 46 // subninja files are read as soon as the statement is parsed but they are 47 // only processed once the current file is done. This enables lower latency 48 // overall. 49 var err error 50 loop: 51 for err == nil { 52 switch token := m.lexer.ReadToken(); token { 53 case POOL: 54 err = m.parsePool() 55 case BUILD: 56 err = m.parseEdge() 57 case RULE: 58 err = m.parseRule() 59 case DEFAULT: 60 err = m.parseDefault() 61 case IDENT: 62 err = m.parseIdent() 63 case INCLUDE: 64 err = m.parseInclude() 65 case SUBNINJA: 66 err = m.parseSubninja() 67 case ERROR: 68 err = m.lexer.Error(m.lexer.DescribeLastError()) 69 case TEOF: 70 break loop 71 case NEWLINE: 72 default: 73 err = m.lexer.Error("unexpected " + token.String()) 74 } 75 } 76 77 // At this point, m.env is completely immutable and can be accessed 78 // concurrently. 79 80 // Did the loop complete because of an error? 81 if err != nil { 82 // Do not forget to unblock the goroutines. 83 for i := int32(0); i < m.subninjasEnqueued; i++ { 84 <-m.subninjas 85 } 86 return err 87 } 88 89 // Finish the processing by parsing the subninja files. 90 return m.processSubninjaQueue() 91 } 92 93 // parsePool parses a "pool" statement. 94 func (m *manifestParserSerial) parsePool() error { 95 name := m.lexer.readIdent() 96 if name == "" { 97 return m.lexer.Error("expected pool name") 98 } 99 100 if err := m.expectToken(NEWLINE); err != nil { 101 return err 102 } 103 104 if m.state.Pools[name] != nil { 105 // TODO(maruel): Use %q for real quoting. 106 return m.lexer.Error(fmt.Sprintf("duplicate pool '%s'", name)) 107 } 108 109 depth := -1 110 111 for m.lexer.PeekToken(INDENT) { 112 key, value, err := m.parseLet() 113 if err != nil { 114 return err 115 } 116 if key != "depth" { 117 // TODO(maruel): Use %q for real quoting. 118 return m.lexer.Error(fmt.Sprintf("unexpected variable '%s'", key)) 119 } 120 // TODO(maruel): Do we want to use ParseInt() here? Aka support hex. 121 if depth, err = strconv.Atoi(value.Evaluate(m.env)); depth < 0 || err != nil { 122 return m.lexer.Error("invalid pool depth") 123 } 124 } 125 126 if depth < 0 { 127 return m.lexer.Error("expected 'depth =' line") 128 } 129 130 m.state.Pools[name] = NewPool(name, depth) 131 return nil 132 } 133 134 // parseRule parses a "rule" statement. 135 func (m *manifestParserSerial) parseRule() error { 136 name := m.lexer.readIdent() 137 if name == "" { 138 return m.lexer.Error("expected rule name") 139 } 140 141 if err := m.expectToken(NEWLINE); err != nil { 142 return err 143 } 144 145 if m.env.Rules[name] != nil { 146 // TODO(maruel): Use %q for real quoting. 147 return m.lexer.Error(fmt.Sprintf("duplicate rule '%s'", name)) 148 } 149 150 rule := NewRule(name) 151 for m.lexer.PeekToken(INDENT) { 152 key, value, err := m.parseLet() 153 if err != nil { 154 return err 155 } 156 157 if !IsReservedBinding(key) { 158 // Die on other keyvals for now; revisit if we want to add a 159 // scope here. 160 // TODO(maruel): Use %q for real quoting. 161 return m.lexer.Error(fmt.Sprintf("unexpected variable '%s'", key)) 162 } 163 rule.Bindings[key] = &value 164 } 165 166 b1, ok1 := rule.Bindings["rspfile"] 167 b2, ok2 := rule.Bindings["rspfile_content"] 168 if ok1 != ok2 || (ok1 && (len(b1.Parsed) == 0) != (len(b2.Parsed) == 0)) { 169 return m.lexer.Error("rspfile and rspfile_content need to be both specified") 170 } 171 172 b, ok := rule.Bindings["command"] 173 if !ok || len(b.Parsed) == 0 { 174 return m.lexer.Error("expected 'command =' line") 175 } 176 m.env.Rules[rule.Name] = rule 177 return nil 178 } 179 180 // parseDefault parses a "default" statement. 181 func (m *manifestParserSerial) parseDefault() error { 182 eval, err := m.lexer.readEvalString(true) 183 if err != nil { 184 return err 185 } 186 if len(eval.Parsed) == 0 { 187 return m.lexer.Error("expected target name") 188 } 189 190 for { 191 path := eval.Evaluate(m.env) 192 if len(path) == 0 { 193 return m.lexer.Error("empty path") 194 195 } 196 if err = m.state.addDefault(CanonicalizePath(path)); err != nil { 197 return m.lexer.Error(err.Error()) 198 } 199 200 eval, err = m.lexer.readEvalString(true) 201 if err != nil { 202 return err 203 } 204 if len(eval.Parsed) == 0 { 205 break 206 } 207 } 208 209 return m.expectToken(NEWLINE) 210 } 211 212 // parseIdent parses a generic statement as a fallback. 213 func (m *manifestParserSerial) parseIdent() error { 214 m.lexer.UnreadToken() 215 name, letValue, err := m.parseLet() 216 if err != nil { 217 return err 218 } 219 value := letValue.Evaluate(m.env) 220 // Check ninjaRequiredVersion immediately so we can exit 221 // before encountering any syntactic surprises. 222 if name == "ninja_required_version" { 223 if err := checkNinjaVersion(value); err != nil { 224 return err 225 } 226 } 227 m.env.Bindings[name] = value 228 return nil 229 } 230 231 // parseEdge parses a "build" statement that results into an edge, which 232 // defines inputs and outputs. 233 func (m *manifestParserSerial) parseEdge() error { 234 var outs []EvalString 235 for { 236 ev, err := m.lexer.readEvalString(true) 237 if err != nil { 238 return err 239 } 240 if len(ev.Parsed) == 0 { 241 break 242 } 243 outs = append(outs, ev) 244 } 245 246 // Add all implicit outs, counting how many as we go. 247 implicitOuts := 0 248 if m.lexer.PeekToken(PIPE) { 249 for { 250 ev, err := m.lexer.readEvalString(true) 251 if err != nil { 252 return err 253 } 254 if len(ev.Parsed) == 0 { 255 break 256 } 257 outs = append(outs, ev) 258 implicitOuts++ 259 } 260 } 261 262 if len(outs) == 0 { 263 return m.lexer.Error("expected path") 264 } 265 266 if err := m.expectToken(COLON); err != nil { 267 return err 268 } 269 270 ruleName := m.lexer.readIdent() 271 if ruleName == "" { 272 return m.lexer.Error("expected build command name") 273 } 274 275 rule := m.env.LookupRule(ruleName) 276 if rule == nil { 277 // TODO(maruel): Use %q for real quoting. 278 return m.lexer.Error(fmt.Sprintf("unknown build rule '%s'", ruleName)) 279 } 280 281 var ins []EvalString 282 for { 283 // XXX should we require one path here? 284 ev, err := m.lexer.readEvalString(true) 285 if err != nil { 286 return err 287 } 288 if len(ev.Parsed) == 0 { 289 break 290 } 291 ins = append(ins, ev) 292 } 293 294 // Add all implicit deps, counting how many as we go. 295 implicit := 0 296 if m.lexer.PeekToken(PIPE) { 297 for { 298 ev, err := m.lexer.readEvalString(true) 299 if err != nil { 300 return err 301 } 302 if len(ev.Parsed) == 0 { 303 break 304 } 305 ins = append(ins, ev) 306 implicit++ 307 } 308 } 309 310 // Add all order-only deps, counting how many as we go. 311 orderOnly := 0 312 if m.lexer.PeekToken(PIPE2) { 313 for { 314 ev, err := m.lexer.readEvalString(true) 315 if err != nil { 316 return err 317 } 318 if len(ev.Parsed) == 0 { 319 break 320 } 321 ins = append(ins, ev) 322 orderOnly++ 323 } 324 } 325 326 // Add all validations, counting how many as we go. 327 var validations []EvalString 328 if m.lexer.PeekToken(PIPEAT) { 329 for { 330 ev, err := m.lexer.readEvalString(true) 331 if err != nil { 332 return err 333 } 334 if len(ev.Parsed) == 0 { 335 break 336 } 337 validations = append(validations, ev) 338 } 339 } 340 341 if err := m.expectToken(NEWLINE); err != nil { 342 return err 343 } 344 345 // Bindings on edges are rare, so allocate per-edge envs only when needed. 346 hasIndentToken := m.lexer.PeekToken(INDENT) 347 env := m.env 348 if hasIndentToken { 349 env = NewBindingEnv(m.env) 350 } 351 for hasIndentToken { 352 key, val, err := m.parseLet() 353 if err != nil { 354 return err 355 } 356 357 env.Bindings[key] = val.Evaluate(m.env) 358 hasIndentToken = m.lexer.PeekToken(INDENT) 359 } 360 361 edge := m.state.addEdge(rule) 362 edge.Env = env 363 364 poolName := edge.GetBinding("pool") 365 if poolName != "" { 366 pool := m.state.Pools[poolName] 367 if pool == nil { 368 // TODO(maruel): Use %q for real quoting. 369 return m.lexer.Error(fmt.Sprintf("unknown pool name '%s'", poolName)) 370 } 371 edge.Pool = pool 372 } 373 374 edge.Outputs = make([]*Node, 0, len(outs)) 375 for i := range outs { 376 path := outs[i].Evaluate(env) 377 if len(path) == 0 { 378 return m.lexer.Error("empty path") 379 } 380 path, slashBits := CanonicalizePathBits(path) 381 if !m.state.addOut(edge, path, slashBits) { 382 if m.options.ErrOnDupeEdge { 383 return m.lexer.Error("multiple rules generate " + path) 384 } 385 if !m.options.Quiet { 386 warningf("multiple rules generate %s. builds involving this target will not be correct; continuing anyway", path) 387 } 388 if len(outs)-i <= implicitOuts { 389 implicitOuts-- 390 } 391 } 392 } 393 if len(edge.Outputs) == 0 { 394 // All outputs of the edge are already created by other edges. Don't add 395 // this edge. Do this check before input nodes are connected to the edge. 396 m.state.Edges = m.state.Edges[:len(m.state.Edges)-1] 397 return nil 398 } 399 edge.ImplicitOuts = int32(implicitOuts) 400 401 edge.Inputs = make([]*Node, 0, len(ins)) 402 for _, i := range ins { 403 path := i.Evaluate(env) 404 if len(path) == 0 { 405 return m.lexer.Error("empty path") 406 } 407 path, slashBits := CanonicalizePathBits(path) 408 m.state.addIn(edge, path, slashBits) 409 } 410 edge.ImplicitDeps = int32(implicit) 411 edge.OrderOnlyDeps = int32(orderOnly) 412 413 edge.Validations = make([]*Node, 0, len(validations)) 414 for _, v := range validations { 415 path := v.Evaluate(env) 416 if path == "" { 417 return m.lexer.Error("empty path") 418 } 419 path, slashBits := CanonicalizePathBits(path) 420 m.state.addValidation(edge, path, slashBits) 421 } 422 423 if !m.options.ErrOnPhonyCycle && edge.maybePhonycycleDiagnostic() { 424 // CMake 2.8.12.x and 3.0.x incorrectly write phony build statements 425 // that reference themselves. Ninja used to tolerate these in the 426 // build graph but that has since been fixed. Filter them out to 427 // support users of those old CMake versions. 428 out := edge.Outputs[0] 429 for i, n := range edge.Inputs { 430 if n == out { 431 copy(edge.Inputs[i:], edge.Inputs[i+1:]) 432 edge.Inputs = edge.Inputs[:len(edge.Inputs)-1] 433 if !m.options.Quiet { 434 warningf("phony target '%s' names itself as an input; ignoring [-w phonycycle=warn]", out.Path) 435 } 436 break 437 } 438 } 439 } 440 441 // Lookup, validate, and save any dyndep binding. It will be used later 442 // to load generated dependency information dynamically, but it must 443 // be one of our manifest-specified inputs. 444 dyndep := edge.GetUnescapedDyndep() 445 if len(dyndep) != 0 { 446 n := m.state.GetNode(CanonicalizePathBits(dyndep)) 447 n.DyndepPending = true 448 edge.Dyndep = n 449 found := false 450 for _, x := range edge.Inputs { 451 if x == n { 452 found = true 453 break 454 } 455 } 456 if !found { 457 // TODO(maruel): Use %q for real quoting. 458 return m.lexer.Error(fmt.Sprintf("dyndep '%s' is not an input", dyndep)) 459 } 460 } 461 return nil 462 } 463 464 // parseInclude parses a "include" line. 465 func (m *manifestParserSerial) parseInclude() error { 466 eval, err := m.lexer.readEvalString(true) 467 if err != nil { 468 return err 469 } 470 ls := m.lexer.lexerState 471 if err = m.expectToken(NEWLINE); err != nil { 472 return err 473 } 474 475 // Process state. 476 path := eval.Evaluate(m.env) 477 input, err := m.fr.ReadFile(path) 478 if err != nil { 479 // Wrap it. 480 // TODO(maruel): Use %q for real quoting. 481 return m.error(fmt.Sprintf("loading '%s': %s", path, err), ls) 482 } 483 484 // Manually construct the object instead of using ParseManifest(), because 485 // m.env may not equal to m.state.Bindings. This happens when the include 486 // statement is inside a subninja. 487 subparser := manifestParserSerial{ 488 fr: m.fr, 489 options: m.options, 490 state: m.state, 491 env: m.env, 492 } 493 // Recursively parse the input into the current state. 494 if err = subparser.parse(path, input); err != nil { 495 // Do not wrap error inside the included ninja. 496 return err 497 } 498 return nil 499 } 500 501 // parseSubninja parses a "subninja" statement. 502 // 503 // If options.Concurrency != ParseManifestSerial, it starts a goroutine that 504 // reads the file and send the content to the channel, but not process it. 505 // 506 // Otherwise, it processes it serially. 507 func (m *manifestParserSerial) parseSubninja() error { 508 eval, err := m.lexer.readEvalString(true) 509 if err != nil { 510 return err 511 } 512 filename := eval.Evaluate(m.env) 513 ls := m.lexer.lexerState 514 if err = m.expectToken(NEWLINE); err != nil { 515 return err 516 } 517 518 if m.options.Concurrency != ParseManifestSerial { 519 // Start the goroutine to read it asynchronously. It will be processed 520 // after the main manifest. 521 go readSubninjaAsync(m.fr, filename, m.subninjas, ls, m.env) 522 m.subninjasEnqueued++ 523 return nil 524 } 525 526 // Process the subninja right away. This is the most compatible way. 527 input, err := m.fr.ReadFile(filename) 528 if err != nil { 529 // Wrap it. 530 return m.error(fmt.Sprintf("loading '%s': %s", filename, err.Error()), ls) 531 } 532 return m.processOneSubninja(filename, input, m.env) 533 } 534 535 // processSubninjaQueue empties the queue of subninja files to process. 536 func (m *manifestParserSerial) processSubninjaQueue() error { 537 // Out of order flow. This is the faster but may be incompatible? 538 var err error 539 for i := int32(0); i < m.subninjasEnqueued; i++ { 540 s := <-m.subninjas 541 if err != nil { 542 continue 543 } 544 if s.err != nil { 545 // Wrap it. 546 // TODO(maruel): Use %q for real quoting. 547 err = m.error(fmt.Sprintf("loading '%s': %s", s.filename, s.err.Error()), s.ls) 548 continue 549 } 550 err = m.processOneSubninja(s.filename, s.input, s.env) 551 } 552 return err 553 } 554 555 func (m *manifestParserSerial) processOneSubninja(filename string, input []byte, env *BindingEnv) error { 556 subparser := manifestParserSerial{ 557 fr: m.fr, 558 options: m.options, 559 state: m.state, 560 // Reset the binding fresh with a temporary one that will not affect the 561 // root one. 562 env: NewBindingEnv(env), 563 } 564 // Do not wrap error inside the subninja. 565 return subparser.parse(filename, input) 566 } 567 568 func (m *manifestParserSerial) parseLet() (string, EvalString, error) { 569 eval := EvalString{} 570 key := m.lexer.readIdent() 571 if key == "" { 572 return key, eval, m.lexer.Error("expected variable name") 573 } 574 var err error 575 if err = m.expectToken(EQUALS); err == nil { 576 eval, err = m.lexer.readEvalString(false) 577 } 578 return key, eval, err 579 } 580 581 // expectToken produces an error string if the next token is not expected. 582 // 583 // The error says "expected foo, got bar". 584 func (m *manifestParserSerial) expectToken(expected Token) error { 585 if token := m.lexer.ReadToken(); token != expected { 586 msg := "expected " + expected.String() + ", got " + token.String() + expected.errorHint() 587 return m.lexer.Error(msg) 588 } 589 return nil 590 } 591 592 func (m *manifestParserSerial) error(msg string, ls lexerState) error { 593 return ls.error(msg, m.lexer.filename, m.lexer.input) 594 }