github.com/cilki/sh@v2.6.4+incompatible/syntax/parser.go (about)

     1  // Copyright (c) 2016, Daniel Martí <mvdan@mvdan.cc>
     2  // See LICENSE for licensing information
     3  
     4  package syntax
     5  
     6  import (
     7  	"bytes"
     8  	"fmt"
     9  	"io"
    10  	"strconv"
    11  	"strings"
    12  	"unicode/utf8"
    13  )
    14  
    15  // KeepComments makes the parser parse comments and attach them to
    16  // nodes, as opposed to discarding them.
    17  func KeepComments(p *Parser) { p.keepComments = true }
    18  
    19  type LangVariant int
    20  
    21  const (
    22  	LangBash LangVariant = iota
    23  	LangPOSIX
    24  	LangMirBSDKorn
    25  )
    26  
    27  // Variant changes the shell language variant that the parser will
    28  // accept.
    29  func Variant(l LangVariant) func(*Parser) {
    30  	return func(p *Parser) { p.lang = l }
    31  }
    32  
    33  func (l LangVariant) String() string {
    34  	switch l {
    35  	case LangBash:
    36  		return "bash"
    37  	case LangPOSIX:
    38  		return "posix"
    39  	case LangMirBSDKorn:
    40  		return "mksh"
    41  	}
    42  	return "unknown shell language variant"
    43  }
    44  
    45  // StopAt configures the lexer to stop at an arbitrary word, treating it
    46  // as if it were the end of the input. It can contain any characters
    47  // except whitespace, and cannot be over four bytes in size.
    48  //
    49  // This can be useful to embed shell code within another language, as
    50  // one can use a special word to mark the delimiters between the two.
    51  //
    52  // As a word, it will only apply when following whitespace or a
    53  // separating token. For example, StopAt("$$") will act on the inputs
    54  // "foo $$" and "foo;$$", but not on "foo '$$'".
    55  //
    56  // The match is done by prefix, so the example above will also act on
    57  // "foo $$bar".
    58  func StopAt(word string) func(*Parser) {
    59  	if len(word) > 4 {
    60  		panic("stop word can't be over four bytes in size")
    61  	}
    62  	if strings.ContainsAny(word, " \t\n\r") {
    63  		panic("stop word can't contain whitespace characters")
    64  	}
    65  	return func(p *Parser) { p.stopAt = []byte(word) }
    66  }
    67  
    68  // NewParser allocates a new Parser and applies any number of options.
    69  func NewParser(options ...func(*Parser)) *Parser {
    70  	p := &Parser{helperBuf: new(bytes.Buffer)}
    71  	for _, opt := range options {
    72  		opt(p)
    73  	}
    74  	return p
    75  }
    76  
    77  // Parse reads and parses a shell program with an optional name. It
    78  // returns the parsed program if no issues were encountered. Otherwise,
    79  // an error is returned. Reads from r are buffered.
    80  //
    81  // Parse can be called more than once, but not concurrently. That is, a
    82  // Parser can be reused once it is done working.
    83  func (p *Parser) Parse(r io.Reader, name string) (*File, error) {
    84  	p.reset()
    85  	p.f = &File{Name: name}
    86  	p.src = r
    87  	p.rune()
    88  	p.next()
    89  	p.f.StmtList = p.stmtList()
    90  	if p.err == nil {
    91  		// EOF immediately after heredoc word so no newline to
    92  		// trigger it
    93  		p.doHeredocs()
    94  	}
    95  	return p.f, p.err
    96  }
    97  
    98  // Stmts reads and parses statements one at a time, calling a function
    99  // each time one is parsed. If the function returns false, parsing is
   100  // stopped and the function is not called again.
   101  func (p *Parser) Stmts(r io.Reader, fn func(*Stmt) bool) error {
   102  	p.reset()
   103  	p.f = &File{}
   104  	p.src = r
   105  	p.rune()
   106  	p.next()
   107  	p.stmts(fn)
   108  	if p.err == nil {
   109  		// EOF immediately after heredoc word so no newline to
   110  		// trigger it
   111  		p.doHeredocs()
   112  	}
   113  	return p.err
   114  }
   115  
   116  type wrappedReader struct {
   117  	*Parser
   118  	io.Reader
   119  
   120  	lastLine    uint16
   121  	accumulated []*Stmt
   122  	fn          func([]*Stmt) bool
   123  }
   124  
   125  func (w *wrappedReader) Read(p []byte) (n int, err error) {
   126  	// If we lexed a newline for the first time, we just finished a line, so
   127  	// we may need to give a callback for the edge cases below not covered
   128  	// by Parser.Stmts.
   129  	if w.r == '\n' && w.npos.line > w.lastLine {
   130  		if w.Incomplete() {
   131  			// Incomplete statement; call back to print "> ".
   132  			if !w.fn(w.accumulated) {
   133  				return 0, io.EOF
   134  			}
   135  		} else if len(w.accumulated) == 0 {
   136  			// Nothing was parsed; call back to print another "$ ".
   137  			if !w.fn(nil) {
   138  				return 0, io.EOF
   139  			}
   140  		}
   141  		w.lastLine = w.npos.line
   142  	}
   143  	return w.Reader.Read(p)
   144  }
   145  
   146  // Interactive implements what is necessary to parse statements in an
   147  // interactive shell. The parser will call the given function under two
   148  // circumstances outlined below.
   149  //
   150  // If a line containing any number of statements is parsed, the function will be
   151  // called with said statements.
   152  //
   153  // If a line ending in an incomplete statement is parsed, the function will be
   154  // called with any fully parsed statents, and Parser.Incomplete will return
   155  // true.
   156  //
   157  // One can imagine a simple interactive shell implementation as follows:
   158  //
   159  //         fmt.Fprintf(os.Stdout, "$ ")
   160  //         parser.Interactive(os.Stdin, func(stmts []*syntax.Stmt) bool {
   161  //                 if parser.Incomplete() {
   162  //                         fmt.Fprintf(os.Stdout, "> ")
   163  //                         return true
   164  //                 }
   165  //                 run(stmts)
   166  //                 fmt.Fprintf(os.Stdout, "$ ")
   167  //                 return true
   168  //         }
   169  //
   170  // If the callback function returns false, parsing is stopped and the function
   171  // is not called again.
   172  func (p *Parser) Interactive(r io.Reader, fn func([]*Stmt) bool) error {
   173  	w := wrappedReader{Parser: p, Reader: r, fn: fn}
   174  	return p.Stmts(&w, func(stmt *Stmt) bool {
   175  		w.accumulated = append(w.accumulated, stmt)
   176  		// We finished parsing a statement and we're at a newline token,
   177  		// so we finished fully parsing a number of statements. Call
   178  		// back to run the statements and print "$ ".
   179  		if p.tok == _Newl {
   180  			if !fn(w.accumulated) {
   181  				return false
   182  			}
   183  			w.accumulated = w.accumulated[:0]
   184  			// The callback above would already print "$ ", so we
   185  			// don't want the subsequent wrappedReader.Read to cause
   186  			// another "$ " print thinking that nothing was parsed.
   187  			w.lastLine = w.npos.line + 1
   188  		}
   189  		return true
   190  	})
   191  }
   192  
   193  // Words reads and parses words one at a time, calling a function each time one
   194  // is parsed. If the function returns false, parsing is stopped and the function
   195  // is not called again.
   196  //
   197  // Newlines are skipped, meaning that multi-line input will work fine. If the
   198  // parser encounters a token that isn't a word, such as a semicolon, an error
   199  // will be returned.
   200  //
   201  // Note that the lexer doesn't currently tokenize spaces, so it may need to read
   202  // a non-space byte such as a newline or a letter before finishing the parsing
   203  // of a word. This will be fixed in the future.
   204  func (p *Parser) Words(r io.Reader, fn func(*Word) bool) error {
   205  	p.reset()
   206  	p.f = &File{}
   207  	p.src = r
   208  	p.rune()
   209  	p.next()
   210  	for {
   211  		p.got(_Newl)
   212  		w := p.getWord()
   213  		if w == nil {
   214  			if p.tok != _EOF {
   215  				p.curErr("%s is not a valid word", p.tok)
   216  			}
   217  			return p.err
   218  		}
   219  		if !fn(w) {
   220  			return nil
   221  		}
   222  	}
   223  }
   224  
   225  // Document parses a single here-document word. That is, it parses the input as
   226  // if they were lines following a <<EOF redirection.
   227  //
   228  // In practice, this is the same as parsing the input as if it were within
   229  // double quotes, but without having to escape all double quote characters.
   230  // Similarly, the here-document word parsed here cannot be ended by any
   231  // delimiter other than reaching the end of the input.
   232  func (p *Parser) Document(r io.Reader) (*Word, error) {
   233  	p.reset()
   234  	p.f = &File{}
   235  	p.src = r
   236  	p.rune()
   237  	p.quote = hdocBody
   238  	p.hdocStop = []byte("MVDAN_CC_SH_SYNTAX_EOF")
   239  	p.parsingDoc = true
   240  	p.next()
   241  	w := p.getWord()
   242  	return w, p.err
   243  }
   244  
   245  // Parser holds the internal state of the parsing mechanism of a
   246  // program.
   247  type Parser struct {
   248  	src io.Reader
   249  	bs  []byte // current chunk of read bytes
   250  	bsp int    // pos within chunk for the rune after r
   251  	r   rune   // next rune
   252  	w   uint16 // width of r
   253  
   254  	f *File
   255  
   256  	spaced bool // whether tok has whitespace on its left
   257  
   258  	err     error // lexer/parser error
   259  	readErr error // got a read error, but bytes left
   260  
   261  	tok token  // current token
   262  	val string // current value (valid if tok is _Lit*)
   263  
   264  	offs int
   265  	pos  Pos // position of tok
   266  	npos Pos // next position (of r)
   267  
   268  	quote   quoteState // current lexer state
   269  	eqlOffs int        // position of '=' in val (a literal)
   270  
   271  	keepComments bool
   272  	lang         LangVariant
   273  
   274  	stopAt []byte
   275  
   276  	forbidNested bool
   277  
   278  	// list of pending heredoc bodies
   279  	buriedHdocs int
   280  	heredocs    []*Redirect
   281  	hdocStop    []byte
   282  	parsingDoc  bool
   283  
   284  	// openStmts is how many entire statements we're currently parsing. A
   285  	// non-zero number means that we require certain tokens or words before
   286  	// reaching EOF.
   287  	openStmts int
   288  	// openBquotes is how many levels of backquotes are open at the moment.
   289  	openBquotes int
   290  
   291  	// lastBquoteEsc is how many times the last backquote token was escaped
   292  	lastBquoteEsc int
   293  	// buriedBquotes is like openBquotes, but saved for when the parser
   294  	// comes out of single quotes
   295  	buriedBquotes int
   296  
   297  	rxOpenParens int
   298  	rxFirstPart  bool
   299  
   300  	accComs []Comment
   301  	curComs *[]Comment
   302  
   303  	helperBuf *bytes.Buffer
   304  
   305  	litBatch    []Lit
   306  	wordBatch   []Word
   307  	wpsBatch    []WordPart
   308  	stmtBatch   []Stmt
   309  	stListBatch []*Stmt
   310  	callBatch   []callAlloc
   311  
   312  	readBuf [bufSize]byte
   313  	litBuf  [bufSize]byte
   314  	litBs   []byte
   315  }
   316  
   317  func (p *Parser) Incomplete() bool {
   318  	// If we're in a quote state other than noState, we're parsing a node
   319  	// such as a double-quoted string.
   320  	// If there are any open statements, we need to finish them.
   321  	// If we're constructing a literal, we need to finish it.
   322  	return p.quote != noState || p.openStmts > 0 || p.litBs != nil
   323  }
   324  
   325  const bufSize = 1 << 10
   326  
   327  func (p *Parser) reset() {
   328  	p.tok, p.val = illegalTok, ""
   329  	p.eqlOffs = 0
   330  	p.bs, p.bsp = nil, 0
   331  	p.offs = 0
   332  	p.npos = Pos{line: 1, col: 1}
   333  	p.r, p.w = 0, 0
   334  	p.err, p.readErr = nil, nil
   335  	p.quote, p.forbidNested = noState, false
   336  	p.openStmts = 0
   337  	p.heredocs, p.buriedHdocs = p.heredocs[:0], 0
   338  	p.parsingDoc = false
   339  	p.openBquotes, p.buriedBquotes = 0, 0
   340  	p.accComs, p.curComs = nil, &p.accComs
   341  }
   342  
   343  func (p *Parser) getPos() Pos {
   344  	p.npos.offs = uint32(p.offs + p.bsp - int(p.w))
   345  	return p.npos
   346  }
   347  
   348  func (p *Parser) lit(pos Pos, val string) *Lit {
   349  	if len(p.litBatch) == 0 {
   350  		p.litBatch = make([]Lit, 128)
   351  	}
   352  	l := &p.litBatch[0]
   353  	p.litBatch = p.litBatch[1:]
   354  	l.ValuePos = pos
   355  	l.ValueEnd = p.getPos()
   356  	l.Value = val
   357  	return l
   358  }
   359  
   360  func (p *Parser) word(parts []WordPart) *Word {
   361  	if len(p.wordBatch) == 0 {
   362  		p.wordBatch = make([]Word, 64)
   363  	}
   364  	w := &p.wordBatch[0]
   365  	p.wordBatch = p.wordBatch[1:]
   366  	w.Parts = parts
   367  	return w
   368  }
   369  
   370  func (p *Parser) wps(wp WordPart) []WordPart {
   371  	if len(p.wpsBatch) == 0 {
   372  		p.wpsBatch = make([]WordPart, 64)
   373  	}
   374  	wps := p.wpsBatch[:1:1]
   375  	p.wpsBatch = p.wpsBatch[1:]
   376  	wps[0] = wp
   377  	return wps
   378  }
   379  
   380  func (p *Parser) stmt(pos Pos) *Stmt {
   381  	if len(p.stmtBatch) == 0 {
   382  		p.stmtBatch = make([]Stmt, 64)
   383  	}
   384  	s := &p.stmtBatch[0]
   385  	p.stmtBatch = p.stmtBatch[1:]
   386  	s.Position = pos
   387  	return s
   388  }
   389  
   390  func (p *Parser) stList() []*Stmt {
   391  	if len(p.stListBatch) == 0 {
   392  		p.stListBatch = make([]*Stmt, 256)
   393  	}
   394  	stmts := p.stListBatch[:0:4]
   395  	p.stListBatch = p.stListBatch[4:]
   396  	return stmts
   397  }
   398  
   399  type callAlloc struct {
   400  	ce CallExpr
   401  	ws [4]*Word
   402  }
   403  
   404  func (p *Parser) call(w *Word) *CallExpr {
   405  	if len(p.callBatch) == 0 {
   406  		p.callBatch = make([]callAlloc, 32)
   407  	}
   408  	alloc := &p.callBatch[0]
   409  	p.callBatch = p.callBatch[1:]
   410  	ce := &alloc.ce
   411  	ce.Args = alloc.ws[:1]
   412  	ce.Args[0] = w
   413  	return ce
   414  }
   415  
   416  //go:generate stringer -type=quoteState
   417  
   418  type quoteState uint32
   419  
   420  const (
   421  	noState quoteState = 1 << iota
   422  	subCmd
   423  	subCmdBckquo
   424  	dblQuotes
   425  	hdocWord
   426  	hdocBody
   427  	hdocBodyTabs
   428  	arithmExpr
   429  	arithmExprLet
   430  	arithmExprCmd
   431  	arithmExprBrack
   432  	testRegexp
   433  	switchCase
   434  	paramExpName
   435  	paramExpSlice
   436  	paramExpRepl
   437  	paramExpExp
   438  	arrayElems
   439  
   440  	allKeepSpaces = paramExpRepl | dblQuotes | hdocBody |
   441  		hdocBodyTabs | paramExpExp
   442  	allRegTokens = noState | subCmd | subCmdBckquo | hdocWord |
   443  		switchCase | arrayElems
   444  	allArithmExpr = arithmExpr | arithmExprLet | arithmExprCmd |
   445  		arithmExprBrack | paramExpSlice
   446  	allParamReg = paramExpName | paramExpSlice
   447  	allParamExp = allParamReg | paramExpRepl | paramExpExp | arithmExprBrack
   448  )
   449  
   450  type saveState struct {
   451  	quote       quoteState
   452  	buriedHdocs int
   453  }
   454  
   455  func (p *Parser) preNested(quote quoteState) (s saveState) {
   456  	s.quote, s.buriedHdocs = p.quote, p.buriedHdocs
   457  	p.buriedHdocs, p.quote = len(p.heredocs), quote
   458  	return
   459  }
   460  
   461  func (p *Parser) postNested(s saveState) {
   462  	p.quote, p.buriedHdocs = s.quote, s.buriedHdocs
   463  }
   464  
   465  func (p *Parser) unquotedWordBytes(w *Word) ([]byte, bool) {
   466  	p.helperBuf.Reset()
   467  	didUnquote := false
   468  	for _, wp := range w.Parts {
   469  		if p.unquotedWordPart(p.helperBuf, wp, false) {
   470  			didUnquote = true
   471  		}
   472  	}
   473  	return p.helperBuf.Bytes(), didUnquote
   474  }
   475  
   476  func (p *Parser) unquotedWordPart(buf *bytes.Buffer, wp WordPart, quotes bool) (quoted bool) {
   477  	switch x := wp.(type) {
   478  	case *Lit:
   479  		for i := 0; i < len(x.Value); i++ {
   480  			if b := x.Value[i]; b == '\\' && !quotes {
   481  				if i++; i < len(x.Value) {
   482  					buf.WriteByte(x.Value[i])
   483  				}
   484  				quoted = true
   485  			} else {
   486  				buf.WriteByte(b)
   487  			}
   488  		}
   489  	case *SglQuoted:
   490  		buf.WriteString(x.Value)
   491  		quoted = true
   492  	case *DblQuoted:
   493  		for _, wp2 := range x.Parts {
   494  			p.unquotedWordPart(buf, wp2, true)
   495  		}
   496  		quoted = true
   497  	}
   498  	return
   499  }
   500  
   501  func (p *Parser) doHeredocs() {
   502  	p.rune() // consume '\n', since we know p.tok == _Newl
   503  	old := p.quote
   504  	hdocs := p.heredocs[p.buriedHdocs:]
   505  	p.heredocs = p.heredocs[:p.buriedHdocs]
   506  	for i, r := range hdocs {
   507  		if p.err != nil {
   508  			break
   509  		}
   510  		p.quote = hdocBody
   511  		if r.Op == DashHdoc {
   512  			p.quote = hdocBodyTabs
   513  		}
   514  		var quoted bool
   515  		p.hdocStop, quoted = p.unquotedWordBytes(r.Word)
   516  		if i > 0 && p.r == '\n' {
   517  			p.rune()
   518  		}
   519  		if quoted {
   520  			r.Hdoc = p.quotedHdocWord()
   521  		} else {
   522  			p.next()
   523  			r.Hdoc = p.getWord()
   524  		}
   525  		if p.hdocStop != nil {
   526  			p.posErr(r.Pos(), "unclosed here-document '%s'",
   527  				string(p.hdocStop))
   528  		}
   529  	}
   530  	p.quote = old
   531  }
   532  
   533  func (p *Parser) got(tok token) bool {
   534  	if p.tok == tok {
   535  		p.next()
   536  		return true
   537  	}
   538  	return false
   539  }
   540  
   541  func (p *Parser) gotRsrv(val string) (Pos, bool) {
   542  	pos := p.pos
   543  	if p.tok == _LitWord && p.val == val {
   544  		p.next()
   545  		return pos, true
   546  	}
   547  	return pos, false
   548  }
   549  
   550  func readableStr(s string) string {
   551  	// don't quote tokens like & or }
   552  	if s != "" && s[0] >= 'a' && s[0] <= 'z' {
   553  		return strconv.Quote(s)
   554  	}
   555  	return s
   556  }
   557  
   558  func (p *Parser) followErr(pos Pos, left, right string) {
   559  	leftStr := readableStr(left)
   560  	p.posErr(pos, "%s must be followed by %s", leftStr, right)
   561  }
   562  
   563  func (p *Parser) followErrExp(pos Pos, left string) {
   564  	p.followErr(pos, left, "an expression")
   565  }
   566  
   567  func (p *Parser) follow(lpos Pos, left string, tok token) {
   568  	if !p.got(tok) {
   569  		p.followErr(lpos, left, tok.String())
   570  	}
   571  }
   572  
   573  func (p *Parser) followRsrv(lpos Pos, left, val string) Pos {
   574  	pos, ok := p.gotRsrv(val)
   575  	if !ok {
   576  		p.followErr(lpos, left, fmt.Sprintf("%q", val))
   577  	}
   578  	return pos
   579  }
   580  
   581  func (p *Parser) followStmts(left string, lpos Pos, stops ...string) StmtList {
   582  	if p.got(semicolon) {
   583  		return StmtList{}
   584  	}
   585  	newLine := p.got(_Newl)
   586  	sl := p.stmtList(stops...)
   587  	if len(sl.Stmts) < 1 && !newLine {
   588  		p.followErr(lpos, left, "a statement list")
   589  	}
   590  	return sl
   591  }
   592  
   593  func (p *Parser) followWordTok(tok token, pos Pos) *Word {
   594  	w := p.getWord()
   595  	if w == nil {
   596  		p.followErr(pos, tok.String(), "a word")
   597  	}
   598  	return w
   599  }
   600  
   601  func (p *Parser) followWord(s string, pos Pos) *Word {
   602  	w := p.getWord()
   603  	if w == nil {
   604  		p.followErr(pos, s, "a word")
   605  	}
   606  	return w
   607  }
   608  
   609  func (p *Parser) stmtEnd(n Node, start, end string) Pos {
   610  	pos, ok := p.gotRsrv(end)
   611  	if !ok {
   612  		p.posErr(n.Pos(), "%s statement must end with %q", start, end)
   613  	}
   614  	return pos
   615  }
   616  
   617  func (p *Parser) quoteErr(lpos Pos, quote token) {
   618  	p.posErr(lpos, "reached %s without closing quote %s",
   619  		p.tok.String(), quote)
   620  }
   621  
   622  func (p *Parser) matchingErr(lpos Pos, left, right interface{}) {
   623  	p.posErr(lpos, "reached %s without matching %s with %s",
   624  		p.tok.String(), left, right)
   625  }
   626  
   627  func (p *Parser) matched(lpos Pos, left, right token) Pos {
   628  	pos := p.pos
   629  	if !p.got(right) {
   630  		p.matchingErr(lpos, left, right)
   631  	}
   632  	return pos
   633  }
   634  
   635  func (p *Parser) errPass(err error) {
   636  	if p.err == nil {
   637  		p.err = err
   638  		p.bsp = len(p.bs) + 1
   639  		p.r = utf8.RuneSelf
   640  		p.w = 1
   641  		p.tok = _EOF
   642  	}
   643  }
   644  
   645  // ParseError represents an error found when parsing a source file, from which
   646  // the parser cannot recover.
   647  type ParseError struct {
   648  	Filename string
   649  	Pos
   650  	Text string
   651  }
   652  
   653  func (e ParseError) Error() string {
   654  	if e.Filename == "" {
   655  		return fmt.Sprintf("%s: %s", e.Pos.String(), e.Text)
   656  	}
   657  	return fmt.Sprintf("%s:%s: %s", e.Filename, e.Pos.String(), e.Text)
   658  }
   659  
   660  // LangError is returned when the parser encounters code that is only valid in
   661  // other shell language variants. The error includes what feature is not present
   662  // in the current language variant, and what languages support it.
   663  type LangError struct {
   664  	Filename string
   665  	Pos
   666  	Feature string
   667  	Langs   []LangVariant
   668  }
   669  
   670  func (e LangError) Error() string {
   671  	var buf bytes.Buffer
   672  	if e.Filename != "" {
   673  		buf.WriteString(e.Filename + ":")
   674  	}
   675  	buf.WriteString(e.Pos.String() + ": ")
   676  	buf.WriteString(e.Feature)
   677  	if strings.HasSuffix(e.Feature, "s") {
   678  		buf.WriteString(" are a ")
   679  	} else {
   680  		buf.WriteString(" is a ")
   681  	}
   682  	for i, lang := range e.Langs {
   683  		if i > 0 {
   684  			buf.WriteString("/")
   685  		}
   686  		buf.WriteString(lang.String())
   687  	}
   688  	buf.WriteString(" feature")
   689  	return buf.String()
   690  }
   691  
   692  func (p *Parser) posErr(pos Pos, format string, a ...interface{}) {
   693  	p.errPass(ParseError{
   694  		Filename: p.f.Name,
   695  		Pos:      pos,
   696  		Text:     fmt.Sprintf(format, a...),
   697  	})
   698  }
   699  
   700  func (p *Parser) curErr(format string, a ...interface{}) {
   701  	p.posErr(p.pos, format, a...)
   702  }
   703  
   704  func (p *Parser) langErr(pos Pos, feature string, langs ...LangVariant) {
   705  	p.errPass(LangError{
   706  		Filename: p.f.Name,
   707  		Pos:      pos,
   708  		Feature:  feature,
   709  		Langs:    langs,
   710  	})
   711  }
   712  
   713  func (p *Parser) stmts(fn func(*Stmt) bool, stops ...string) {
   714  	gotEnd := true
   715  loop:
   716  	for p.tok != _EOF {
   717  		newLine := p.got(_Newl)
   718  		switch p.tok {
   719  		case _LitWord:
   720  			for _, stop := range stops {
   721  				if p.val == stop {
   722  					break loop
   723  				}
   724  			}
   725  		case rightParen:
   726  			if p.quote == subCmd {
   727  				break loop
   728  			}
   729  		case bckQuote:
   730  			if p.backquoteEnd() {
   731  				break loop
   732  			}
   733  		case dblSemicolon, semiAnd, dblSemiAnd, semiOr:
   734  			if p.quote == switchCase {
   735  				break loop
   736  			}
   737  			p.curErr("%s can only be used in a case clause", p.tok)
   738  		}
   739  		if !newLine && !gotEnd {
   740  			p.curErr("statements must be separated by &, ; or a newline")
   741  		}
   742  		if p.tok == _EOF {
   743  			break
   744  		}
   745  		p.openStmts++
   746  		s := p.getStmt(true, false, false)
   747  		p.openStmts--
   748  		if s == nil {
   749  			p.invalidStmtStart()
   750  			break
   751  		}
   752  		gotEnd = s.Semicolon.IsValid()
   753  		if !fn(s) {
   754  			break
   755  		}
   756  	}
   757  }
   758  
   759  func (p *Parser) stmtList(stops ...string) (sl StmtList) {
   760  	fn := func(s *Stmt) bool {
   761  		if sl.Stmts == nil {
   762  			sl.Stmts = p.stList()
   763  		}
   764  		sl.Stmts = append(sl.Stmts, s)
   765  		return true
   766  	}
   767  	p.stmts(fn, stops...)
   768  	split := len(p.accComs)
   769  	if p.tok == _LitWord && (p.val == "elif" || p.val == "else" || p.val == "fi") {
   770  		// Split the comments, so that any aligned with an opening token
   771  		// get attached to it. For example:
   772  		//
   773  		//     if foo; then
   774  		//         # inside the body
   775  		//     # document the else
   776  		//     else
   777  		//     fi
   778  		// TODO(mvdan): look into deduplicating this with similar logic
   779  		// in caseItems.
   780  		for i := len(p.accComs) - 1; i >= 0; i-- {
   781  			c := p.accComs[i]
   782  			if c.Pos().Col() != p.pos.Col() {
   783  				break
   784  			}
   785  			split = i
   786  		}
   787  	}
   788  	sl.Last = p.accComs[:split]
   789  	p.accComs = p.accComs[split:]
   790  	return
   791  }
   792  
   793  func (p *Parser) invalidStmtStart() {
   794  	switch p.tok {
   795  	case semicolon, and, or, andAnd, orOr:
   796  		p.curErr("%s can only immediately follow a statement", p.tok)
   797  	case rightParen:
   798  		p.curErr("%s can only be used to close a subshell", p.tok)
   799  	default:
   800  		p.curErr("%s is not a valid start for a statement", p.tok)
   801  	}
   802  }
   803  
   804  func (p *Parser) getWord() *Word {
   805  	if parts := p.wordParts(); len(parts) > 0 {
   806  		return p.word(parts)
   807  	}
   808  	return nil
   809  }
   810  
   811  func (p *Parser) getLit() *Lit {
   812  	switch p.tok {
   813  	case _Lit, _LitWord, _LitRedir:
   814  		l := p.lit(p.pos, p.val)
   815  		p.next()
   816  		return l
   817  	}
   818  	return nil
   819  }
   820  
   821  func (p *Parser) wordParts() (wps []WordPart) {
   822  	for {
   823  		n := p.wordPart()
   824  		if n == nil {
   825  			return
   826  		}
   827  		if wps == nil {
   828  			wps = p.wps(n)
   829  		} else {
   830  			wps = append(wps, n)
   831  		}
   832  		if p.spaced {
   833  			return
   834  		}
   835  	}
   836  }
   837  
   838  func (p *Parser) ensureNoNested() {
   839  	if p.forbidNested {
   840  		p.curErr("expansions not allowed in heredoc words")
   841  	}
   842  }
   843  
   844  func (p *Parser) wordPart() WordPart {
   845  	switch p.tok {
   846  	case _Lit, _LitWord:
   847  		l := p.lit(p.pos, p.val)
   848  		p.next()
   849  		return l
   850  	case dollBrace:
   851  		p.ensureNoNested()
   852  		switch p.r {
   853  		case '|':
   854  			if p.lang != LangMirBSDKorn {
   855  				p.curErr(`"${|stmts;}" is a mksh feature`)
   856  			}
   857  			fallthrough
   858  		case ' ', '\t', '\n':
   859  			if p.lang != LangMirBSDKorn {
   860  				p.curErr(`"${ stmts;}" is a mksh feature`)
   861  			}
   862  			cs := &CmdSubst{
   863  				Left:     p.pos,
   864  				TempFile: p.r != '|',
   865  				ReplyVar: p.r == '|',
   866  			}
   867  			old := p.preNested(subCmd)
   868  			p.rune() // don't tokenize '|'
   869  			p.next()
   870  			cs.StmtList = p.stmtList("}")
   871  			p.postNested(old)
   872  			pos, ok := p.gotRsrv("}")
   873  			if !ok {
   874  				p.matchingErr(cs.Left, "${", "}")
   875  			}
   876  			cs.Right = pos
   877  			return cs
   878  		default:
   879  			return p.paramExp()
   880  		}
   881  	case dollDblParen, dollBrack:
   882  		p.ensureNoNested()
   883  		left := p.tok
   884  		ar := &ArithmExp{Left: p.pos, Bracket: left == dollBrack}
   885  		var old saveState
   886  		if ar.Bracket {
   887  			old = p.preNested(arithmExprBrack)
   888  		} else {
   889  			old = p.preNested(arithmExpr)
   890  		}
   891  		p.next()
   892  		if p.got(hash) {
   893  			if p.lang != LangMirBSDKorn {
   894  				p.langErr(ar.Pos(), "unsigned expressions", LangMirBSDKorn)
   895  			}
   896  			ar.Unsigned = true
   897  		}
   898  		ar.X = p.followArithm(left, ar.Left)
   899  		if ar.Bracket {
   900  			if p.tok != rightBrack {
   901  				p.matchingErr(ar.Left, dollBrack, rightBrack)
   902  			}
   903  			p.postNested(old)
   904  			ar.Right = p.pos
   905  			p.next()
   906  		} else {
   907  			ar.Right = p.arithmEnd(dollDblParen, ar.Left, old)
   908  		}
   909  		return ar
   910  	case dollParen:
   911  		p.ensureNoNested()
   912  		cs := &CmdSubst{Left: p.pos}
   913  		old := p.preNested(subCmd)
   914  		p.next()
   915  		cs.StmtList = p.stmtList()
   916  		p.postNested(old)
   917  		cs.Right = p.matched(cs.Left, leftParen, rightParen)
   918  		return cs
   919  	case dollar:
   920  		r := p.r
   921  		switch {
   922  		case singleRuneParam(r):
   923  			p.tok, p.val = _LitWord, string(r)
   924  			p.rune()
   925  		case 'a' <= r && r <= 'z', 'A' <= r && r <= 'Z',
   926  			'0' <= r && r <= '9', r == '_', r == '\\':
   927  			p.advanceNameCont(r)
   928  		default:
   929  			l := p.lit(p.pos, "$")
   930  			p.next()
   931  			return l
   932  		}
   933  		p.ensureNoNested()
   934  		pe := &ParamExp{Dollar: p.pos, Short: true}
   935  		p.pos = posAddCol(p.pos, 1)
   936  		pe.Param = p.getLit()
   937  		if pe.Param != nil && pe.Param.Value == "" {
   938  			l := p.lit(pe.Dollar, "$")
   939  			if p.val == "" {
   940  				// e.g. "$\\\n" followed by a closing double
   941  				// quote, so we need the next token.
   942  				p.next()
   943  			} else {
   944  				// e.g. "$\\\"" within double quotes, so we must
   945  				// keep the rest of the literal characters.
   946  				l.ValueEnd = posAddCol(l.ValuePos, 1)
   947  			}
   948  			return l
   949  		}
   950  		return pe
   951  	case cmdIn, cmdOut:
   952  		p.ensureNoNested()
   953  		ps := &ProcSubst{Op: ProcOperator(p.tok), OpPos: p.pos}
   954  		old := p.preNested(subCmd)
   955  		p.next()
   956  		ps.StmtList = p.stmtList()
   957  		p.postNested(old)
   958  		ps.Rparen = p.matched(ps.OpPos, token(ps.Op), rightParen)
   959  		return ps
   960  	case sglQuote, dollSglQuote:
   961  		sq := &SglQuoted{Left: p.pos, Dollar: p.tok == dollSglQuote}
   962  		r := p.r
   963  		for p.newLit(r); ; r = p.rune() {
   964  			switch r {
   965  			case '\\':
   966  				if sq.Dollar {
   967  					p.rune()
   968  				}
   969  			case '\'':
   970  				sq.Right = p.getPos()
   971  				sq.Value = p.endLit()
   972  
   973  				// restore openBquotes
   974  				p.openBquotes = p.buriedBquotes
   975  				p.buriedBquotes = 0
   976  
   977  				p.rune()
   978  				p.next()
   979  				return sq
   980  			case utf8.RuneSelf:
   981  				p.posErr(sq.Pos(), "reached EOF without closing quote %s", sglQuote)
   982  				return nil
   983  			}
   984  		}
   985  	case dblQuote, dollDblQuote:
   986  		if p.quote == dblQuotes {
   987  			// p.tok == dblQuote, as "foo$" puts $ in the lit
   988  			return nil
   989  		}
   990  		return p.dblQuoted()
   991  	case bckQuote:
   992  		if p.backquoteEnd() {
   993  			return nil
   994  		}
   995  		p.ensureNoNested()
   996  		cs := &CmdSubst{Left: p.pos}
   997  		old := p.preNested(subCmdBckquo)
   998  		p.openBquotes++
   999  
  1000  		// The lexer didn't call p.rune for us, so that it could have
  1001  		// the right p.openBquotes to properly handle backslashes.
  1002  		p.rune()
  1003  
  1004  		p.next()
  1005  		cs.StmtList = p.stmtList()
  1006  		if p.tok == bckQuote && p.lastBquoteEsc < p.openBquotes-1 {
  1007  			// e.g. found ` before the nested backquote \` was closed.
  1008  			p.tok = _EOF
  1009  			p.quoteErr(cs.Pos(), bckQuote)
  1010  		}
  1011  		p.postNested(old)
  1012  		p.openBquotes--
  1013  		cs.Right = p.pos
  1014  
  1015  		// Like above, the lexer didn't call p.rune for us.
  1016  		p.rune()
  1017  		if !p.got(bckQuote) {
  1018  			p.quoteErr(cs.Pos(), bckQuote)
  1019  		}
  1020  		return cs
  1021  	case globQuest, globStar, globPlus, globAt, globExcl:
  1022  		if p.lang == LangPOSIX {
  1023  			p.langErr(p.pos, "extended globs", LangBash, LangMirBSDKorn)
  1024  		}
  1025  		eg := &ExtGlob{Op: GlobOperator(p.tok), OpPos: p.pos}
  1026  		lparens := 1
  1027  		r := p.r
  1028  	globLoop:
  1029  		for p.newLit(r); ; r = p.rune() {
  1030  			switch r {
  1031  			case utf8.RuneSelf:
  1032  				break globLoop
  1033  			case '(':
  1034  				lparens++
  1035  			case ')':
  1036  				if lparens--; lparens == 0 {
  1037  					break globLoop
  1038  				}
  1039  			}
  1040  		}
  1041  		eg.Pattern = p.lit(posAddCol(eg.OpPos, 2), p.endLit())
  1042  		p.rune()
  1043  		p.next()
  1044  		if lparens != 0 {
  1045  			p.matchingErr(eg.OpPos, eg.Op, rightParen)
  1046  		}
  1047  		return eg
  1048  	default:
  1049  		return nil
  1050  	}
  1051  }
  1052  
  1053  func (p *Parser) dblQuoted() *DblQuoted {
  1054  	q := &DblQuoted{Position: p.pos, Dollar: p.tok == dollDblQuote}
  1055  	old := p.quote
  1056  	p.quote = dblQuotes
  1057  	p.next()
  1058  	q.Parts = p.wordParts()
  1059  	p.quote = old
  1060  	if !p.got(dblQuote) {
  1061  		p.quoteErr(q.Pos(), dblQuote)
  1062  	}
  1063  	return q
  1064  }
  1065  
  1066  func arithmOpLevel(op BinAritOperator) int {
  1067  	switch op {
  1068  	case Comma:
  1069  		return 0
  1070  	case AddAssgn, SubAssgn, MulAssgn, QuoAssgn, RemAssgn, AndAssgn,
  1071  		OrAssgn, XorAssgn, ShlAssgn, ShrAssgn:
  1072  		return 1
  1073  	case Assgn:
  1074  		return 2
  1075  	case Quest, Colon:
  1076  		return 3
  1077  	case AndArit, OrArit:
  1078  		return 4
  1079  	case And, Or, Xor:
  1080  		return 5
  1081  	case Eql, Neq:
  1082  		return 6
  1083  	case Lss, Gtr, Leq, Geq:
  1084  		return 7
  1085  	case Shl, Shr:
  1086  		return 8
  1087  	case Add, Sub:
  1088  		return 9
  1089  	case Mul, Quo, Rem:
  1090  		return 10
  1091  	case Pow:
  1092  		return 11
  1093  	}
  1094  	return -1
  1095  }
  1096  
  1097  func (p *Parser) followArithm(ftok token, fpos Pos) ArithmExpr {
  1098  	x := p.arithmExpr(0, false, false)
  1099  	if x == nil {
  1100  		p.followErrExp(fpos, ftok.String())
  1101  	}
  1102  	return x
  1103  }
  1104  
  1105  func (p *Parser) arithmExpr(level int, compact, tern bool) ArithmExpr {
  1106  	if p.tok == _EOF || p.peekArithmEnd() {
  1107  		return nil
  1108  	}
  1109  	var left ArithmExpr
  1110  	if level > 11 {
  1111  		left = p.arithmExprBase(compact)
  1112  	} else {
  1113  		left = p.arithmExpr(level+1, compact, false)
  1114  	}
  1115  	if compact && p.spaced {
  1116  		return left
  1117  	}
  1118  	p.got(_Newl)
  1119  	newLevel := arithmOpLevel(BinAritOperator(p.tok))
  1120  	if !tern && p.tok == colon && p.quote == paramExpSlice {
  1121  		newLevel = -1
  1122  	}
  1123  	if newLevel < 0 {
  1124  		switch p.tok {
  1125  		case _Lit, _LitWord:
  1126  			p.curErr("not a valid arithmetic operator: %s", p.val)
  1127  			return nil
  1128  		case leftBrack:
  1129  			p.curErr("[ must follow a name")
  1130  			return nil
  1131  		case rightParen, _EOF:
  1132  		default:
  1133  			if p.quote == arithmExpr {
  1134  				p.curErr("not a valid arithmetic operator: %v", p.tok)
  1135  				return nil
  1136  			}
  1137  		}
  1138  	}
  1139  	if newLevel < level {
  1140  		return left
  1141  	}
  1142  	if left == nil {
  1143  		p.curErr("%s must follow an expression", p.tok.String())
  1144  		return nil
  1145  	}
  1146  	b := &BinaryArithm{
  1147  		OpPos: p.pos,
  1148  		Op:    BinAritOperator(p.tok),
  1149  		X:     left,
  1150  	}
  1151  	switch b.Op {
  1152  	case Colon:
  1153  		if !tern {
  1154  			p.posErr(b.Pos(), "ternary operator missing ? before :")
  1155  		}
  1156  	case AddAssgn, SubAssgn, MulAssgn, QuoAssgn, RemAssgn, AndAssgn,
  1157  		OrAssgn, XorAssgn, ShlAssgn, ShrAssgn, Assgn:
  1158  		if !isArithName(b.X) {
  1159  			p.posErr(b.OpPos, "%s must follow a name", b.Op.String())
  1160  		}
  1161  	}
  1162  	if p.next(); compact && p.spaced {
  1163  		p.followErrExp(b.OpPos, b.Op.String())
  1164  	}
  1165  	b.Y = p.arithmExpr(newLevel, compact, b.Op == Quest)
  1166  	if b.Y == nil {
  1167  		p.followErrExp(b.OpPos, b.Op.String())
  1168  	}
  1169  	if b.Op == Quest {
  1170  		if b2, ok := b.Y.(*BinaryArithm); !ok || b2.Op != Colon {
  1171  			p.posErr(b.Pos(), "ternary operator missing : after ?")
  1172  		}
  1173  	}
  1174  	return b
  1175  }
  1176  
  1177  func isArithName(left ArithmExpr) bool {
  1178  	w, ok := left.(*Word)
  1179  	if !ok || len(w.Parts) != 1 {
  1180  		return false
  1181  	}
  1182  	switch x := w.Parts[0].(type) {
  1183  	case *Lit:
  1184  		return ValidName(x.Value)
  1185  	case *ParamExp:
  1186  		return x.nakedIndex()
  1187  	default:
  1188  		return false
  1189  	}
  1190  }
  1191  
  1192  func (p *Parser) arithmExprBase(compact bool) ArithmExpr {
  1193  	p.got(_Newl)
  1194  	var x ArithmExpr
  1195  	switch p.tok {
  1196  	case exclMark:
  1197  		ue := &UnaryArithm{OpPos: p.pos, Op: UnAritOperator(p.tok)}
  1198  		p.next()
  1199  		if ue.X = p.arithmExprBase(compact); ue.X == nil {
  1200  			p.followErrExp(ue.OpPos, ue.Op.String())
  1201  		}
  1202  		return ue
  1203  	case addAdd, subSub:
  1204  		ue := &UnaryArithm{OpPos: p.pos, Op: UnAritOperator(p.tok)}
  1205  		p.next()
  1206  		if p.tok != _LitWord {
  1207  			p.followErr(ue.OpPos, token(ue.Op).String(), "a literal")
  1208  		}
  1209  		ue.X = p.arithmExprBase(compact)
  1210  		return ue
  1211  	case leftParen:
  1212  		pe := &ParenArithm{Lparen: p.pos}
  1213  		p.next()
  1214  		pe.X = p.followArithm(leftParen, pe.Lparen)
  1215  		pe.Rparen = p.matched(pe.Lparen, leftParen, rightParen)
  1216  		x = pe
  1217  	case plus, minus:
  1218  		ue := &UnaryArithm{OpPos: p.pos, Op: UnAritOperator(p.tok)}
  1219  		if p.next(); compact && p.spaced {
  1220  			p.followErrExp(ue.OpPos, ue.Op.String())
  1221  		}
  1222  		ue.X = p.arithmExprBase(compact)
  1223  		if ue.X == nil {
  1224  			p.followErrExp(ue.OpPos, ue.Op.String())
  1225  		}
  1226  		x = ue
  1227  	case _LitWord:
  1228  		l := p.getLit()
  1229  		if p.tok != leftBrack {
  1230  			x = p.word(p.wps(l))
  1231  			break
  1232  		}
  1233  		pe := &ParamExp{Dollar: l.ValuePos, Short: true, Param: l}
  1234  		pe.Index = p.eitherIndex()
  1235  		x = p.word(p.wps(pe))
  1236  	case bckQuote:
  1237  		if p.quote == arithmExprLet && p.openBquotes > 0 {
  1238  			return nil
  1239  		}
  1240  		fallthrough
  1241  	default:
  1242  		if w := p.getWord(); w != nil {
  1243  			// we want real nil, not (*Word)(nil) as that
  1244  			// sets the type to non-nil and then x != nil
  1245  			x = w
  1246  		}
  1247  	}
  1248  	if compact && p.spaced {
  1249  		return x
  1250  	}
  1251  	if p.tok == addAdd || p.tok == subSub {
  1252  		if !isArithName(x) {
  1253  			p.curErr("%s must follow a name", p.tok.String())
  1254  		}
  1255  		u := &UnaryArithm{
  1256  			Post:  true,
  1257  			OpPos: p.pos,
  1258  			Op:    UnAritOperator(p.tok),
  1259  			X:     x,
  1260  		}
  1261  		p.next()
  1262  		return u
  1263  	}
  1264  	return x
  1265  }
  1266  
  1267  func singleRuneParam(r rune) bool {
  1268  	switch r {
  1269  	case '@', '*', '#', '$', '?', '!', '-',
  1270  		'0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
  1271  		return true
  1272  	}
  1273  	return false
  1274  }
  1275  
  1276  func (p *Parser) paramExp() *ParamExp {
  1277  	pe := &ParamExp{Dollar: p.pos}
  1278  	old := p.quote
  1279  	p.quote = paramExpName
  1280  	if p.r == '#' {
  1281  		p.tok = hash
  1282  		p.pos = p.getPos()
  1283  		p.rune()
  1284  	} else {
  1285  		p.next()
  1286  	}
  1287  	switch p.tok {
  1288  	case hash:
  1289  		if paramNameOp(p.r) {
  1290  			pe.Length = true
  1291  			p.next()
  1292  		}
  1293  	case perc:
  1294  		if p.lang != LangMirBSDKorn {
  1295  			p.posErr(pe.Pos(), `"${%%foo}" is a mksh feature`)
  1296  		}
  1297  		if paramNameOp(p.r) {
  1298  			pe.Width = true
  1299  			p.next()
  1300  		}
  1301  	case exclMark:
  1302  		if paramNameOp(p.r) {
  1303  			if p.lang == LangPOSIX {
  1304  				p.langErr(p.pos, "${!foo}", LangBash, LangMirBSDKorn)
  1305  			}
  1306  			pe.Excl = true
  1307  			p.next()
  1308  		}
  1309  	}
  1310  	op := p.tok
  1311  	switch p.tok {
  1312  	case _Lit, _LitWord:
  1313  		if !numberLiteral(p.val) && !ValidName(p.val) {
  1314  			p.curErr("invalid parameter name")
  1315  		}
  1316  		pe.Param = p.lit(p.pos, p.val)
  1317  		p.next()
  1318  	case quest, minus:
  1319  		if pe.Length && p.r != '}' {
  1320  			// actually ${#-default}, not ${#-}; error below
  1321  			pe.Length = false
  1322  			pe.Param = p.lit(p.pos, "#")
  1323  			break
  1324  		}
  1325  		fallthrough
  1326  	case at, star, hash, exclMark, dollar:
  1327  		pe.Param = p.lit(p.pos, p.tok.String())
  1328  		p.next()
  1329  	default:
  1330  		p.curErr("parameter expansion requires a literal")
  1331  	}
  1332  	switch p.tok {
  1333  	case _Lit, _LitWord:
  1334  		p.curErr("%s cannot be followed by a word", op)
  1335  	case rightBrace:
  1336  		pe.Rbrace = p.pos
  1337  		p.quote = old
  1338  		p.next()
  1339  		return pe
  1340  	case leftBrack:
  1341  		if p.lang == LangPOSIX {
  1342  			p.langErr(p.pos, "arrays", LangBash, LangMirBSDKorn)
  1343  		}
  1344  		if !ValidName(pe.Param.Value) {
  1345  			p.curErr("cannot index a special parameter name")
  1346  		}
  1347  		pe.Index = p.eitherIndex()
  1348  	}
  1349  	if p.tok == rightBrace {
  1350  		pe.Rbrace = p.pos
  1351  		p.quote = old
  1352  		p.next()
  1353  		return pe
  1354  	}
  1355  	if p.tok != _EOF && (pe.Length || pe.Width) {
  1356  		p.curErr("cannot combine multiple parameter expansion operators")
  1357  	}
  1358  	switch p.tok {
  1359  	case slash, dblSlash:
  1360  		// pattern search and replace
  1361  		if p.lang == LangPOSIX {
  1362  			p.langErr(p.pos, "search and replace", LangBash, LangMirBSDKorn)
  1363  		}
  1364  		pe.Repl = &Replace{All: p.tok == dblSlash}
  1365  		p.quote = paramExpRepl
  1366  		p.next()
  1367  		pe.Repl.Orig = p.getWord()
  1368  		p.quote = paramExpExp
  1369  		if p.got(slash) {
  1370  			pe.Repl.With = p.getWord()
  1371  		}
  1372  	case colon:
  1373  		// slicing
  1374  		if p.lang == LangPOSIX {
  1375  			p.langErr(p.pos, "slicing", LangBash, LangMirBSDKorn)
  1376  		}
  1377  		pe.Slice = &Slice{}
  1378  		colonPos := p.pos
  1379  		p.quote = paramExpSlice
  1380  		if p.next(); p.tok != colon {
  1381  			pe.Slice.Offset = p.followArithm(colon, colonPos)
  1382  		}
  1383  		colonPos = p.pos
  1384  		if p.got(colon) {
  1385  			pe.Slice.Length = p.followArithm(colon, colonPos)
  1386  		}
  1387  	case caret, dblCaret, comma, dblComma:
  1388  		// upper/lower case
  1389  		if p.lang != LangBash {
  1390  			p.langErr(p.pos, "this expansion operator", LangBash)
  1391  		}
  1392  		pe.Exp = p.paramExpExp()
  1393  	case at, star:
  1394  		switch {
  1395  		case p.tok == at && p.lang == LangPOSIX:
  1396  			p.langErr(p.pos, "this expansion operator", LangBash, LangMirBSDKorn)
  1397  		case p.tok == star && !pe.Excl:
  1398  			p.curErr("not a valid parameter expansion operator: %v", p.tok)
  1399  		case pe.Excl:
  1400  			pe.Names = ParNamesOperator(p.tok)
  1401  			p.next()
  1402  		default:
  1403  			pe.Exp = p.paramExpExp()
  1404  		}
  1405  	case plus, colPlus, minus, colMinus, quest, colQuest, assgn, colAssgn,
  1406  		perc, dblPerc, hash, dblHash:
  1407  		pe.Exp = p.paramExpExp()
  1408  	case _EOF:
  1409  	default:
  1410  		p.curErr("not a valid parameter expansion operator: %v", p.tok)
  1411  	}
  1412  	p.quote = old
  1413  	pe.Rbrace = p.pos
  1414  	p.matched(pe.Dollar, dollBrace, rightBrace)
  1415  	return pe
  1416  }
  1417  
  1418  func (p *Parser) paramExpExp() *Expansion {
  1419  	op := ParExpOperator(p.tok)
  1420  	p.quote = paramExpExp
  1421  	p.next()
  1422  	if op == OtherParamOps {
  1423  		switch p.tok {
  1424  		case _Lit, _LitWord:
  1425  		default:
  1426  			p.curErr("@ expansion operator requires a literal")
  1427  		}
  1428  		switch p.val {
  1429  		case "Q", "E", "P", "A", "a":
  1430  		default:
  1431  			p.curErr("invalid @ expansion operator")
  1432  		}
  1433  	}
  1434  	return &Expansion{Op: op, Word: p.getWord()}
  1435  }
  1436  
  1437  func (p *Parser) eitherIndex() ArithmExpr {
  1438  	old := p.quote
  1439  	lpos := p.pos
  1440  	p.quote = arithmExprBrack
  1441  	p.next()
  1442  	if p.tok == star || p.tok == at {
  1443  		p.tok, p.val = _LitWord, p.tok.String()
  1444  	}
  1445  	expr := p.followArithm(leftBrack, lpos)
  1446  	p.quote = old
  1447  	p.matched(lpos, leftBrack, rightBrack)
  1448  	return expr
  1449  }
  1450  
  1451  func (p *Parser) peekArithmEnd() bool {
  1452  	return p.tok == rightParen && p.r == ')'
  1453  }
  1454  
  1455  func (p *Parser) arithmEnd(ltok token, lpos Pos, old saveState) Pos {
  1456  	if !p.peekArithmEnd() {
  1457  		p.matchingErr(lpos, ltok, dblRightParen)
  1458  	}
  1459  	p.rune()
  1460  	p.postNested(old)
  1461  	pos := p.pos
  1462  	p.next()
  1463  	return pos
  1464  }
  1465  
  1466  func stopToken(tok token) bool {
  1467  	switch tok {
  1468  	case _EOF, _Newl, semicolon, and, or, andAnd, orOr, orAnd, dblSemicolon,
  1469  		semiAnd, dblSemiAnd, semiOr, rightParen:
  1470  		return true
  1471  	}
  1472  	return false
  1473  }
  1474  
  1475  func (p *Parser) backquoteEnd() bool {
  1476  	return p.quote == subCmdBckquo && p.lastBquoteEsc < p.openBquotes
  1477  }
  1478  
  1479  // ValidName returns whether val is a valid name as per the POSIX spec.
  1480  func ValidName(val string) bool {
  1481  	if val == "" {
  1482  		return false
  1483  	}
  1484  	for i, r := range val {
  1485  		switch {
  1486  		case 'a' <= r && r <= 'z':
  1487  		case 'A' <= r && r <= 'Z':
  1488  		case r == '_':
  1489  		case i > 0 && '0' <= r && r <= '9':
  1490  		default:
  1491  			return false
  1492  		}
  1493  	}
  1494  	return true
  1495  }
  1496  
  1497  func numberLiteral(val string) bool {
  1498  	for _, r := range val {
  1499  		if '0' > r || r > '9' {
  1500  			return false
  1501  		}
  1502  	}
  1503  	return true
  1504  }
  1505  
  1506  func (p *Parser) hasValidIdent() bool {
  1507  	if p.tok != _Lit && p.tok != _LitWord {
  1508  		return false
  1509  	}
  1510  	if end := p.eqlOffs; end > 0 {
  1511  		if p.val[end-1] == '+' && p.lang != LangPOSIX {
  1512  			end--
  1513  		}
  1514  		if ValidName(p.val[:end]) {
  1515  			return true
  1516  		}
  1517  	}
  1518  	return p.r == '['
  1519  }
  1520  
  1521  func (p *Parser) getAssign(needEqual bool) *Assign {
  1522  	as := &Assign{}
  1523  	if p.eqlOffs > 0 { // foo=bar
  1524  		nameEnd := p.eqlOffs
  1525  		if p.lang != LangPOSIX && p.val[p.eqlOffs-1] == '+' {
  1526  			// a+=b
  1527  			as.Append = true
  1528  			nameEnd--
  1529  		}
  1530  		as.Name = p.lit(p.pos, p.val[:nameEnd])
  1531  		// since we're not using the entire p.val
  1532  		as.Name.ValueEnd = posAddCol(as.Name.ValuePos, nameEnd)
  1533  		left := p.lit(posAddCol(p.pos, 1), p.val[p.eqlOffs+1:])
  1534  		if left.Value != "" {
  1535  			left.ValuePos = posAddCol(left.ValuePos, p.eqlOffs)
  1536  			as.Value = p.word(p.wps(left))
  1537  		}
  1538  		p.next()
  1539  	} else { // foo[x]=bar
  1540  		as.Name = p.lit(p.pos, p.val)
  1541  		// hasValidIdent already checks p.r is '['
  1542  		p.rune()
  1543  		p.pos = posAddCol(p.pos, 1)
  1544  		as.Index = p.eitherIndex()
  1545  		if !needEqual && (p.spaced || stopToken(p.tok)) {
  1546  			as.Naked = true
  1547  			return as
  1548  		}
  1549  		if len(p.val) > 0 && p.val[0] == '+' {
  1550  			as.Append = true
  1551  			p.val = p.val[1:]
  1552  			p.pos = posAddCol(p.pos, 1)
  1553  		}
  1554  		if len(p.val) < 1 || p.val[0] != '=' {
  1555  			if as.Append {
  1556  				p.followErr(as.Pos(), "a[b]+", "=")
  1557  			} else {
  1558  				p.followErr(as.Pos(), "a[b]", "=")
  1559  			}
  1560  			return nil
  1561  		}
  1562  		p.pos = posAddCol(p.pos, 1)
  1563  		p.val = p.val[1:]
  1564  		if p.val == "" {
  1565  			p.next()
  1566  		}
  1567  	}
  1568  	if p.spaced || stopToken(p.tok) {
  1569  		return as
  1570  	}
  1571  	if as.Value == nil && p.tok == leftParen {
  1572  		if p.lang == LangPOSIX {
  1573  			p.langErr(p.pos, "arrays", LangBash, LangMirBSDKorn)
  1574  		}
  1575  		if as.Index != nil {
  1576  			p.curErr("arrays cannot be nested")
  1577  		}
  1578  		as.Array = &ArrayExpr{Lparen: p.pos}
  1579  		newQuote := p.quote
  1580  		if p.lang == LangBash {
  1581  			newQuote = arrayElems
  1582  		}
  1583  		old := p.preNested(newQuote)
  1584  		p.next()
  1585  		p.got(_Newl)
  1586  		for p.tok != _EOF && p.tok != rightParen {
  1587  			ae := &ArrayElem{}
  1588  			ae.Comments, p.accComs = p.accComs, nil
  1589  			if p.tok == leftBrack {
  1590  				left := p.pos
  1591  				ae.Index = p.eitherIndex()
  1592  				p.follow(left, `"[x]"`, assgn)
  1593  			}
  1594  			if ae.Value = p.getWord(); ae.Value == nil {
  1595  				switch p.tok {
  1596  				case leftParen:
  1597  					p.curErr("arrays cannot be nested")
  1598  					return nil
  1599  				case _Newl, rightParen, leftBrack:
  1600  					// TODO: support [index]=[
  1601  				default:
  1602  					p.curErr("array element values must be words")
  1603  					break
  1604  				}
  1605  			}
  1606  			if len(p.accComs) > 0 {
  1607  				c := p.accComs[0]
  1608  				if c.Pos().Line() == ae.End().Line() {
  1609  					ae.Comments = append(ae.Comments, c)
  1610  					p.accComs = p.accComs[1:]
  1611  				}
  1612  			}
  1613  			as.Array.Elems = append(as.Array.Elems, ae)
  1614  			p.got(_Newl)
  1615  		}
  1616  		as.Array.Last, p.accComs = p.accComs, nil
  1617  		p.postNested(old)
  1618  		as.Array.Rparen = p.matched(as.Array.Lparen, leftParen, rightParen)
  1619  	} else if w := p.getWord(); w != nil {
  1620  		if as.Value == nil {
  1621  			as.Value = w
  1622  		} else {
  1623  			as.Value.Parts = append(as.Value.Parts, w.Parts...)
  1624  		}
  1625  	}
  1626  	return as
  1627  }
  1628  
  1629  func (p *Parser) peekRedir() bool {
  1630  	switch p.tok {
  1631  	case rdrOut, appOut, rdrIn, dplIn, dplOut, clbOut, rdrInOut,
  1632  		hdoc, dashHdoc, wordHdoc, rdrAll, appAll, _LitRedir:
  1633  		return true
  1634  	}
  1635  	return false
  1636  }
  1637  
  1638  func (p *Parser) doRedirect(s *Stmt) {
  1639  	var r *Redirect
  1640  	if s.Redirs == nil {
  1641  		var alloc struct {
  1642  			redirs [4]*Redirect
  1643  			redir  Redirect
  1644  		}
  1645  		s.Redirs = alloc.redirs[:0]
  1646  		r = &alloc.redir
  1647  		s.Redirs = append(s.Redirs, r)
  1648  	} else {
  1649  		r = &Redirect{}
  1650  		s.Redirs = append(s.Redirs, r)
  1651  	}
  1652  	r.N = p.getLit()
  1653  	if p.lang != LangBash && r.N != nil && r.N.Value[0] == '{' {
  1654  		p.langErr(r.N.Pos(), "{varname} redirects", LangBash)
  1655  	}
  1656  	r.Op, r.OpPos = RedirOperator(p.tok), p.pos
  1657  	p.next()
  1658  	switch r.Op {
  1659  	case Hdoc, DashHdoc:
  1660  		old := p.quote
  1661  		p.quote, p.forbidNested = hdocWord, true
  1662  		p.heredocs = append(p.heredocs, r)
  1663  		r.Word = p.followWordTok(token(r.Op), r.OpPos)
  1664  		p.quote, p.forbidNested = old, false
  1665  		if p.tok == _Newl {
  1666  			if len(p.accComs) > 0 {
  1667  				c := p.accComs[0]
  1668  				if c.Pos().Line() == s.End().Line() {
  1669  					s.Comments = append(s.Comments, c)
  1670  					p.accComs = p.accComs[1:]
  1671  				}
  1672  			}
  1673  			p.doHeredocs()
  1674  		}
  1675  	default:
  1676  		r.Word = p.followWordTok(token(r.Op), r.OpPos)
  1677  	}
  1678  }
  1679  
  1680  func (p *Parser) getStmt(readEnd, binCmd, fnBody bool) *Stmt {
  1681  	pos, ok := p.gotRsrv("!")
  1682  	s := p.stmt(pos)
  1683  	if ok {
  1684  		s.Negated = true
  1685  		if stopToken(p.tok) {
  1686  			p.posErr(s.Pos(), `"!" cannot form a statement alone`)
  1687  		}
  1688  		if _, ok := p.gotRsrv("!"); ok {
  1689  			p.posErr(s.Pos(), `cannot negate a command multiple times`)
  1690  		}
  1691  	}
  1692  	if s = p.gotStmtPipe(s); s == nil || p.err != nil {
  1693  		return nil
  1694  	}
  1695  	// instead of using recursion, iterate manually
  1696  	for p.tok == andAnd || p.tok == orOr {
  1697  		// left associativity: in a list of BinaryCmds, the
  1698  		// right recursion should only read a single element
  1699  		if binCmd {
  1700  			return s
  1701  		}
  1702  		b := &BinaryCmd{
  1703  			OpPos: p.pos,
  1704  			Op:    BinCmdOperator(p.tok),
  1705  			X:     s,
  1706  		}
  1707  		p.next()
  1708  		p.got(_Newl)
  1709  		b.Y = p.getStmt(false, true, false)
  1710  		if b.Y == nil || p.err != nil {
  1711  			p.followErr(b.OpPos, b.Op.String(), "a statement")
  1712  			return nil
  1713  		}
  1714  		s = p.stmt(s.Position)
  1715  		s.Cmd = b
  1716  		s.Comments, b.X.Comments = b.X.Comments, nil
  1717  	}
  1718  	if readEnd {
  1719  		switch p.tok {
  1720  		case semicolon:
  1721  			s.Semicolon = p.pos
  1722  			p.next()
  1723  		case and:
  1724  			s.Semicolon = p.pos
  1725  			p.next()
  1726  			s.Background = true
  1727  		case orAnd:
  1728  			s.Semicolon = p.pos
  1729  			p.next()
  1730  			s.Coprocess = true
  1731  		}
  1732  	}
  1733  	if len(p.accComs) > 0 && !binCmd && !fnBody {
  1734  		c := p.accComs[0]
  1735  		if c.Pos().Line() == s.End().Line() {
  1736  			s.Comments = append(s.Comments, c)
  1737  			p.accComs = p.accComs[1:]
  1738  		}
  1739  	}
  1740  	return s
  1741  }
  1742  
  1743  func (p *Parser) gotStmtPipe(s *Stmt) *Stmt {
  1744  	s.Comments, p.accComs = p.accComs, nil
  1745  	switch p.tok {
  1746  	case _LitWord:
  1747  		switch p.val {
  1748  		case "{":
  1749  			p.block(s)
  1750  		case "if":
  1751  			p.ifClause(s)
  1752  		case "while", "until":
  1753  			p.whileClause(s, p.val == "until")
  1754  		case "for":
  1755  			p.forClause(s)
  1756  		case "case":
  1757  			p.caseClause(s)
  1758  		case "}":
  1759  			p.curErr(`%q can only be used to close a block`, p.val)
  1760  		case "then":
  1761  			p.curErr(`%q can only be used in an if`, p.val)
  1762  		case "elif":
  1763  			p.curErr(`%q can only be used in an if`, p.val)
  1764  		case "fi":
  1765  			p.curErr(`%q can only be used to end an if`, p.val)
  1766  		case "do":
  1767  			p.curErr(`%q can only be used in a loop`, p.val)
  1768  		case "done":
  1769  			p.curErr(`%q can only be used to end a loop`, p.val)
  1770  		case "esac":
  1771  			p.curErr(`%q can only be used to end a case`, p.val)
  1772  		case "!":
  1773  			if !s.Negated {
  1774  				p.curErr(`"!" can only be used in full statements`)
  1775  				break
  1776  			}
  1777  		case "[[":
  1778  			if p.lang != LangPOSIX {
  1779  				p.testClause(s)
  1780  			}
  1781  		case "]]":
  1782  			if p.lang != LangPOSIX {
  1783  				p.curErr(`%q can only be used to close a test`,
  1784  					p.val)
  1785  			}
  1786  		case "let":
  1787  			if p.lang != LangPOSIX {
  1788  				p.letClause(s)
  1789  			}
  1790  		case "function":
  1791  			if p.lang != LangPOSIX {
  1792  				p.bashFuncDecl(s)
  1793  			}
  1794  		case "declare":
  1795  			if p.lang == LangBash {
  1796  				p.declClause(s)
  1797  			}
  1798  		case "local", "export", "readonly", "typeset", "nameref":
  1799  			if p.lang != LangPOSIX {
  1800  				p.declClause(s)
  1801  			}
  1802  		case "time":
  1803  			if p.lang != LangPOSIX {
  1804  				p.timeClause(s)
  1805  			}
  1806  		case "coproc":
  1807  			if p.lang == LangBash {
  1808  				p.coprocClause(s)
  1809  			}
  1810  		case "select":
  1811  			if p.lang != LangPOSIX {
  1812  				p.selectClause(s)
  1813  			}
  1814  		}
  1815  		if s.Cmd != nil {
  1816  			break
  1817  		}
  1818  		if p.hasValidIdent() {
  1819  			p.callExpr(s, nil, true)
  1820  			break
  1821  		}
  1822  		name := p.lit(p.pos, p.val)
  1823  		if p.next(); p.got(leftParen) {
  1824  			p.follow(name.ValuePos, "foo(", rightParen)
  1825  			if p.lang == LangPOSIX && !ValidName(name.Value) {
  1826  				p.posErr(name.Pos(), "invalid func name")
  1827  			}
  1828  			p.funcDecl(s, name, name.ValuePos)
  1829  		} else {
  1830  			p.callExpr(s, p.word(p.wps(name)), false)
  1831  		}
  1832  	case rdrOut, appOut, rdrIn, dplIn, dplOut, clbOut, rdrInOut,
  1833  		hdoc, dashHdoc, wordHdoc, rdrAll, appAll, _LitRedir:
  1834  		p.doRedirect(s)
  1835  		p.callExpr(s, nil, false)
  1836  	case bckQuote:
  1837  		if p.backquoteEnd() {
  1838  			return nil
  1839  		}
  1840  		fallthrough
  1841  	case _Lit, dollBrace, dollDblParen, dollParen, dollar, cmdIn, cmdOut,
  1842  		sglQuote, dollSglQuote, dblQuote, dollDblQuote, dollBrack,
  1843  		globQuest, globStar, globPlus, globAt, globExcl:
  1844  		if p.hasValidIdent() {
  1845  			p.callExpr(s, nil, true)
  1846  			break
  1847  		}
  1848  		w := p.word(p.wordParts())
  1849  		if p.got(leftParen) && p.err == nil {
  1850  			p.posErr(w.Pos(), "invalid func name")
  1851  		}
  1852  		p.callExpr(s, w, false)
  1853  	case leftParen:
  1854  		p.subshell(s)
  1855  	case dblLeftParen:
  1856  		p.arithmExpCmd(s)
  1857  	default:
  1858  		if len(s.Redirs) == 0 {
  1859  			return nil
  1860  		}
  1861  	}
  1862  	for p.peekRedir() {
  1863  		p.doRedirect(s)
  1864  	}
  1865  	switch p.tok {
  1866  	case orAnd:
  1867  		if p.lang == LangMirBSDKorn {
  1868  			break
  1869  		}
  1870  		fallthrough
  1871  	case or:
  1872  		b := &BinaryCmd{OpPos: p.pos, Op: BinCmdOperator(p.tok), X: s}
  1873  		p.next()
  1874  		p.got(_Newl)
  1875  		if b.Y = p.gotStmtPipe(p.stmt(p.pos)); b.Y == nil || p.err != nil {
  1876  			p.followErr(b.OpPos, b.Op.String(), "a statement")
  1877  			break
  1878  		}
  1879  		s = p.stmt(s.Position)
  1880  		s.Cmd = b
  1881  		s.Comments, b.X.Comments = b.X.Comments, nil
  1882  		// in "! x | y", the bang applies to the entire pipeline
  1883  		s.Negated = b.X.Negated
  1884  		b.X.Negated = false
  1885  	}
  1886  	return s
  1887  }
  1888  
  1889  func (p *Parser) subshell(s *Stmt) {
  1890  	sub := &Subshell{Lparen: p.pos}
  1891  	old := p.preNested(subCmd)
  1892  	p.next()
  1893  	sub.StmtList = p.stmtList()
  1894  	p.postNested(old)
  1895  	sub.Rparen = p.matched(sub.Lparen, leftParen, rightParen)
  1896  	s.Cmd = sub
  1897  }
  1898  
  1899  func (p *Parser) arithmExpCmd(s *Stmt) {
  1900  	ar := &ArithmCmd{Left: p.pos}
  1901  	old := p.preNested(arithmExprCmd)
  1902  	p.next()
  1903  	if p.got(hash) {
  1904  		if p.lang != LangMirBSDKorn {
  1905  			p.langErr(ar.Pos(), "unsigned expressions", LangMirBSDKorn)
  1906  		}
  1907  		ar.Unsigned = true
  1908  	}
  1909  	ar.X = p.followArithm(dblLeftParen, ar.Left)
  1910  	ar.Right = p.arithmEnd(dblLeftParen, ar.Left, old)
  1911  	s.Cmd = ar
  1912  }
  1913  
  1914  func (p *Parser) block(s *Stmt) {
  1915  	b := &Block{Lbrace: p.pos}
  1916  	p.next()
  1917  	b.StmtList = p.stmtList("}")
  1918  	pos, ok := p.gotRsrv("}")
  1919  	b.Rbrace = pos
  1920  	if !ok {
  1921  		p.matchingErr(b.Lbrace, "{", "}")
  1922  	}
  1923  	s.Cmd = b
  1924  }
  1925  
  1926  func (p *Parser) ifClause(s *Stmt) {
  1927  	rif := &IfClause{IfPos: p.pos}
  1928  	p.next()
  1929  	rif.Cond = p.followStmts("if", rif.IfPos, "then")
  1930  	rif.ThenPos = p.followRsrv(rif.IfPos, "if <cond>", "then")
  1931  	rif.Then = p.followStmts("then", rif.ThenPos, "fi", "elif", "else")
  1932  	curIf := rif
  1933  	for p.tok == _LitWord && p.val == "elif" {
  1934  		elf := &IfClause{IfPos: p.pos, Elif: true}
  1935  		s := p.stmt(elf.IfPos)
  1936  		s.Cmd = elf
  1937  		s.Comments = p.accComs
  1938  		p.accComs = nil
  1939  		p.next()
  1940  		elf.Cond = p.followStmts("elif", elf.IfPos, "then")
  1941  		elf.ThenPos = p.followRsrv(elf.IfPos, "elif <cond>", "then")
  1942  		elf.Then = p.followStmts("then", elf.ThenPos, "fi", "elif", "else")
  1943  		curIf.ElsePos = elf.IfPos
  1944  		curIf.Else.Stmts = []*Stmt{s}
  1945  		curIf = elf
  1946  	}
  1947  	if elsePos, ok := p.gotRsrv("else"); ok {
  1948  		curIf.ElseComments = p.accComs
  1949  		p.accComs = nil
  1950  		curIf.ElsePos = elsePos
  1951  		curIf.Else = p.followStmts("else", curIf.ElsePos, "fi")
  1952  	}
  1953  	curIf.FiComments = p.accComs
  1954  	p.accComs = nil
  1955  	rif.FiPos = p.stmtEnd(rif, "if", "fi")
  1956  	curIf.FiPos = rif.FiPos
  1957  	s.Cmd = rif
  1958  }
  1959  
  1960  func (p *Parser) whileClause(s *Stmt, until bool) {
  1961  	wc := &WhileClause{WhilePos: p.pos, Until: until}
  1962  	rsrv := "while"
  1963  	rsrvCond := "while <cond>"
  1964  	if wc.Until {
  1965  		rsrv = "until"
  1966  		rsrvCond = "until <cond>"
  1967  	}
  1968  	p.next()
  1969  	wc.Cond = p.followStmts(rsrv, wc.WhilePos, "do")
  1970  	wc.DoPos = p.followRsrv(wc.WhilePos, rsrvCond, "do")
  1971  	wc.Do = p.followStmts("do", wc.DoPos, "done")
  1972  	wc.DonePos = p.stmtEnd(wc, rsrv, "done")
  1973  	s.Cmd = wc
  1974  }
  1975  
  1976  func (p *Parser) forClause(s *Stmt) {
  1977  	fc := &ForClause{ForPos: p.pos}
  1978  	p.next()
  1979  	fc.Loop = p.loop(fc.ForPos)
  1980  	fc.DoPos = p.followRsrv(fc.ForPos, "for foo [in words]", "do")
  1981  
  1982  	s.Comments = append(s.Comments, p.accComs...)
  1983  	p.accComs = nil
  1984  	fc.Do = p.followStmts("do", fc.DoPos, "done")
  1985  	fc.DonePos = p.stmtEnd(fc, "for", "done")
  1986  	s.Cmd = fc
  1987  }
  1988  
  1989  func (p *Parser) loop(fpos Pos) Loop {
  1990  	if p.lang != LangBash {
  1991  		switch p.tok {
  1992  		case leftParen, dblLeftParen:
  1993  			p.langErr(p.pos, "c-style fors", LangBash)
  1994  		}
  1995  	}
  1996  	if p.tok == dblLeftParen {
  1997  		cl := &CStyleLoop{Lparen: p.pos}
  1998  		old := p.preNested(arithmExprCmd)
  1999  		p.next()
  2000  		cl.Init = p.arithmExpr(0, false, false)
  2001  		if !p.got(dblSemicolon) {
  2002  			p.follow(p.pos, "expr", semicolon)
  2003  			cl.Cond = p.arithmExpr(0, false, false)
  2004  			p.follow(p.pos, "expr", semicolon)
  2005  		}
  2006  		cl.Post = p.arithmExpr(0, false, false)
  2007  		cl.Rparen = p.arithmEnd(dblLeftParen, cl.Lparen, old)
  2008  		p.got(semicolon)
  2009  		p.got(_Newl)
  2010  		return cl
  2011  	}
  2012  	return p.wordIter("for", fpos)
  2013  }
  2014  
  2015  func (p *Parser) wordIter(ftok string, fpos Pos) *WordIter {
  2016  	wi := &WordIter{}
  2017  	if wi.Name = p.getLit(); wi.Name == nil {
  2018  		p.followErr(fpos, ftok, "a literal")
  2019  	}
  2020  	if p.got(semicolon) {
  2021  		p.got(_Newl)
  2022  		return wi
  2023  	}
  2024  	p.got(_Newl)
  2025  	if pos, ok := p.gotRsrv("in"); ok {
  2026  		wi.InPos = pos
  2027  		for !stopToken(p.tok) {
  2028  			if w := p.getWord(); w == nil {
  2029  				p.curErr("word list can only contain words")
  2030  			} else {
  2031  				wi.Items = append(wi.Items, w)
  2032  			}
  2033  		}
  2034  		p.got(semicolon)
  2035  		p.got(_Newl)
  2036  	} else if p.tok == _LitWord && p.val == "do" {
  2037  	} else {
  2038  		p.followErr(fpos, ftok+" foo", `"in", "do", ;, or a newline`)
  2039  	}
  2040  	return wi
  2041  }
  2042  
  2043  func (p *Parser) selectClause(s *Stmt) {
  2044  	fc := &ForClause{ForPos: p.pos, Select: true}
  2045  	p.next()
  2046  	fc.Loop = p.wordIter("select", fc.ForPos)
  2047  	fc.DoPos = p.followRsrv(fc.ForPos, "select foo [in words]", "do")
  2048  	fc.Do = p.followStmts("do", fc.DoPos, "done")
  2049  	fc.DonePos = p.stmtEnd(fc, "select", "done")
  2050  	s.Cmd = fc
  2051  }
  2052  
  2053  func (p *Parser) caseClause(s *Stmt) {
  2054  	cc := &CaseClause{Case: p.pos}
  2055  	p.next()
  2056  	cc.Word = p.followWord("case", cc.Case)
  2057  	end := "esac"
  2058  	p.got(_Newl)
  2059  	if _, ok := p.gotRsrv("{"); ok {
  2060  		if p.lang != LangMirBSDKorn {
  2061  			p.posErr(cc.Pos(), `"case i {" is a mksh feature`)
  2062  		}
  2063  		end = "}"
  2064  	} else {
  2065  		p.followRsrv(cc.Case, "case x", "in")
  2066  	}
  2067  	cc.Items = p.caseItems(end)
  2068  	cc.Last, p.accComs = p.accComs, nil
  2069  	cc.Esac = p.stmtEnd(cc, "case", end)
  2070  	s.Cmd = cc
  2071  }
  2072  
  2073  func (p *Parser) caseItems(stop string) (items []*CaseItem) {
  2074  	p.got(_Newl)
  2075  	for p.tok != _EOF && !(p.tok == _LitWord && p.val == stop) {
  2076  		ci := &CaseItem{}
  2077  		ci.Comments, p.accComs = p.accComs, nil
  2078  		p.got(leftParen)
  2079  		for p.tok != _EOF {
  2080  			if w := p.getWord(); w == nil {
  2081  				p.curErr("case patterns must consist of words")
  2082  			} else {
  2083  				ci.Patterns = append(ci.Patterns, w)
  2084  			}
  2085  			if p.tok == rightParen {
  2086  				break
  2087  			}
  2088  			if !p.got(or) {
  2089  				p.curErr("case patterns must be separated with |")
  2090  			}
  2091  		}
  2092  		old := p.preNested(switchCase)
  2093  		p.next()
  2094  		ci.StmtList = p.stmtList(stop)
  2095  		p.postNested(old)
  2096  		switch p.tok {
  2097  		case dblSemicolon, semiAnd, dblSemiAnd, semiOr:
  2098  		default:
  2099  			ci.Op = Break
  2100  			items = append(items, ci)
  2101  			return
  2102  		}
  2103  		ci.Last = append(ci.Last, p.accComs...)
  2104  		p.accComs = nil
  2105  		ci.OpPos = p.pos
  2106  		ci.Op = CaseOperator(p.tok)
  2107  		p.next()
  2108  		p.got(_Newl)
  2109  		split := len(p.accComs)
  2110  		if p.tok == _LitWord && p.val != stop {
  2111  			for i := len(p.accComs) - 1; i >= 0; i-- {
  2112  				c := p.accComs[i]
  2113  				if c.Pos().Col() != p.pos.Col() {
  2114  					break
  2115  				}
  2116  				split = i
  2117  			}
  2118  		}
  2119  		ci.Comments = append(ci.Comments, p.accComs[:split]...)
  2120  		p.accComs = p.accComs[split:]
  2121  		items = append(items, ci)
  2122  	}
  2123  	return
  2124  }
  2125  
  2126  func (p *Parser) testClause(s *Stmt) {
  2127  	tc := &TestClause{Left: p.pos}
  2128  	p.next()
  2129  	if _, ok := p.gotRsrv("]]"); ok || p.tok == _EOF {
  2130  		p.posErr(tc.Left, "test clause requires at least one expression")
  2131  	}
  2132  	tc.X = p.testExpr(dblLeftBrack, tc.Left, false)
  2133  	tc.Right = p.pos
  2134  	if _, ok := p.gotRsrv("]]"); !ok {
  2135  		p.matchingErr(tc.Left, "[[", "]]")
  2136  	}
  2137  	s.Cmd = tc
  2138  }
  2139  
  2140  func (p *Parser) testExpr(ftok token, fpos Pos, pastAndOr bool) TestExpr {
  2141  	p.got(_Newl)
  2142  	var left TestExpr
  2143  	if pastAndOr {
  2144  		left = p.testExprBase(ftok, fpos)
  2145  	} else {
  2146  		left = p.testExpr(ftok, fpos, true)
  2147  	}
  2148  	if left == nil {
  2149  		return left
  2150  	}
  2151  	p.got(_Newl)
  2152  	switch p.tok {
  2153  	case andAnd, orOr:
  2154  	case _LitWord:
  2155  		if p.val == "]]" {
  2156  			return left
  2157  		}
  2158  	case rdrIn, rdrOut:
  2159  	case _EOF, rightParen:
  2160  		return left
  2161  	case _Lit:
  2162  		p.curErr("test operator words must consist of a single literal")
  2163  	default:
  2164  		p.curErr("not a valid test operator: %v", p.tok)
  2165  	}
  2166  	if p.tok == _LitWord {
  2167  		if p.tok = token(testBinaryOp(p.val)); p.tok == illegalTok {
  2168  			p.curErr("not a valid test operator: %s", p.val)
  2169  		}
  2170  	}
  2171  	b := &BinaryTest{
  2172  		OpPos: p.pos,
  2173  		Op:    BinTestOperator(p.tok),
  2174  		X:     left,
  2175  	}
  2176  	// Save the previous quoteState, since we change it in TsReMatch.
  2177  	oldQuote := p.quote
  2178  
  2179  	switch b.Op {
  2180  	case AndTest, OrTest:
  2181  		p.next()
  2182  		if b.Y = p.testExpr(token(b.Op), b.OpPos, false); b.Y == nil {
  2183  			p.followErrExp(b.OpPos, b.Op.String())
  2184  		}
  2185  	case TsReMatch:
  2186  		if p.lang != LangBash {
  2187  			p.langErr(p.pos, "regex tests", LangBash)
  2188  		}
  2189  		p.rxOpenParens = 0
  2190  		p.rxFirstPart = true
  2191  		// TODO(mvdan): Using nested states within a regex will break in
  2192  		// all sorts of ways. The better fix is likely to use a stop
  2193  		// token, like we do with heredocs.
  2194  		p.quote = testRegexp
  2195  		fallthrough
  2196  	default:
  2197  		if _, ok := b.X.(*Word); !ok {
  2198  			p.posErr(b.OpPos, "expected %s, %s or %s after complex expr",
  2199  				AndTest, OrTest, "]]")
  2200  		}
  2201  		p.next()
  2202  		b.Y = p.followWordTok(token(b.Op), b.OpPos)
  2203  	}
  2204  	p.quote = oldQuote
  2205  	return b
  2206  }
  2207  
  2208  func (p *Parser) testExprBase(ftok token, fpos Pos) TestExpr {
  2209  	switch p.tok {
  2210  	case _EOF, rightParen:
  2211  		return nil
  2212  	case _LitWord:
  2213  		op := token(testUnaryOp(p.val))
  2214  		switch op {
  2215  		case illegalTok:
  2216  		case tsRefVar, tsModif: // not available in mksh
  2217  			if p.lang == LangBash {
  2218  				p.tok = op
  2219  			}
  2220  		default:
  2221  			p.tok = op
  2222  		}
  2223  	}
  2224  	switch p.tok {
  2225  	case exclMark:
  2226  		u := &UnaryTest{OpPos: p.pos, Op: TsNot}
  2227  		p.next()
  2228  		if u.X = p.testExpr(token(u.Op), u.OpPos, false); u.X == nil {
  2229  			p.followErrExp(u.OpPos, u.Op.String())
  2230  		}
  2231  		return u
  2232  	case tsExists, tsRegFile, tsDirect, tsCharSp, tsBlckSp, tsNmPipe,
  2233  		tsSocket, tsSmbLink, tsSticky, tsGIDSet, tsUIDSet, tsGrpOwn,
  2234  		tsUsrOwn, tsModif, tsRead, tsWrite, tsExec, tsNoEmpty,
  2235  		tsFdTerm, tsEmpStr, tsNempStr, tsOptSet, tsVarSet, tsRefVar:
  2236  		u := &UnaryTest{OpPos: p.pos, Op: UnTestOperator(p.tok)}
  2237  		p.next()
  2238  		u.X = p.followWordTok(token(u.Op), u.OpPos)
  2239  		return u
  2240  	case leftParen:
  2241  		pe := &ParenTest{Lparen: p.pos}
  2242  		p.next()
  2243  		if pe.X = p.testExpr(leftParen, pe.Lparen, false); pe.X == nil {
  2244  			p.followErrExp(pe.Lparen, "(")
  2245  		}
  2246  		pe.Rparen = p.matched(pe.Lparen, leftParen, rightParen)
  2247  		return pe
  2248  	default:
  2249  		return p.followWordTok(ftok, fpos)
  2250  	}
  2251  }
  2252  
  2253  func (p *Parser) declClause(s *Stmt) {
  2254  	ds := &DeclClause{Variant: p.lit(p.pos, p.val)}
  2255  	p.next()
  2256  	for (p.tok == _LitWord || p.tok == _Lit) &&
  2257  		(p.val[0] == '-' || p.val[0] == '+') {
  2258  		ds.Opts = append(ds.Opts, p.getWord())
  2259  	}
  2260  	for !stopToken(p.tok) && !p.peekRedir() {
  2261  		if p.hasValidIdent() {
  2262  			ds.Assigns = append(ds.Assigns, p.getAssign(false))
  2263  		} else if p.eqlOffs > 0 {
  2264  			p.curErr("invalid var name")
  2265  		} else if p.tok == _LitWord {
  2266  			ds.Assigns = append(ds.Assigns, &Assign{
  2267  				Naked: true,
  2268  				Name:  p.getLit(),
  2269  			})
  2270  		} else if w := p.getWord(); w != nil {
  2271  			ds.Assigns = append(ds.Assigns, &Assign{
  2272  				Naked: true,
  2273  				Value: w,
  2274  			})
  2275  		} else {
  2276  			p.followErr(p.pos, ds.Variant.Value, "names or assignments")
  2277  		}
  2278  	}
  2279  	s.Cmd = ds
  2280  }
  2281  
  2282  func isBashCompoundCommand(tok token, val string) bool {
  2283  	switch tok {
  2284  	case leftParen, dblLeftParen:
  2285  		return true
  2286  	case _LitWord:
  2287  		switch val {
  2288  		case "{", "if", "while", "until", "for", "case", "[[",
  2289  			"coproc", "let", "function", "declare", "local",
  2290  			"export", "readonly", "typeset", "nameref":
  2291  			return true
  2292  		}
  2293  	}
  2294  	return false
  2295  }
  2296  
  2297  func (p *Parser) timeClause(s *Stmt) {
  2298  	tc := &TimeClause{Time: p.pos}
  2299  	p.next()
  2300  	if _, ok := p.gotRsrv("-p"); ok {
  2301  		tc.PosixFormat = true
  2302  	}
  2303  	tc.Stmt = p.gotStmtPipe(p.stmt(p.pos))
  2304  	s.Cmd = tc
  2305  }
  2306  
  2307  func (p *Parser) coprocClause(s *Stmt) {
  2308  	cc := &CoprocClause{Coproc: p.pos}
  2309  	if p.next(); isBashCompoundCommand(p.tok, p.val) {
  2310  		// has no name
  2311  		cc.Stmt = p.gotStmtPipe(p.stmt(p.pos))
  2312  		s.Cmd = cc
  2313  		return
  2314  	}
  2315  	cc.Name = p.getLit()
  2316  	cc.Stmt = p.gotStmtPipe(p.stmt(p.pos))
  2317  	if cc.Stmt == nil {
  2318  		if cc.Name == nil {
  2319  			p.posErr(cc.Coproc, "coproc clause requires a command")
  2320  			return
  2321  		}
  2322  		// name was in fact the stmt
  2323  		cc.Stmt = p.stmt(cc.Name.ValuePos)
  2324  		cc.Stmt.Cmd = p.call(p.word(p.wps(cc.Name)))
  2325  		cc.Name = nil
  2326  	} else if cc.Name != nil {
  2327  		if call, ok := cc.Stmt.Cmd.(*CallExpr); ok {
  2328  			// name was in fact the start of a call
  2329  			call.Args = append([]*Word{p.word(p.wps(cc.Name))},
  2330  				call.Args...)
  2331  			cc.Name = nil
  2332  		}
  2333  	}
  2334  	s.Cmd = cc
  2335  }
  2336  
  2337  func (p *Parser) letClause(s *Stmt) {
  2338  	lc := &LetClause{Let: p.pos}
  2339  	old := p.preNested(arithmExprLet)
  2340  	p.next()
  2341  	for !stopToken(p.tok) && !p.peekRedir() {
  2342  		x := p.arithmExpr(0, true, false)
  2343  		if x == nil {
  2344  			break
  2345  		}
  2346  		lc.Exprs = append(lc.Exprs, x)
  2347  	}
  2348  	if len(lc.Exprs) == 0 {
  2349  		p.followErrExp(lc.Let, "let")
  2350  	}
  2351  	p.postNested(old)
  2352  	s.Cmd = lc
  2353  }
  2354  
  2355  func (p *Parser) bashFuncDecl(s *Stmt) {
  2356  	fpos := p.pos
  2357  	if p.next(); p.tok != _LitWord {
  2358  		if w := p.followWord("function", fpos); p.err == nil {
  2359  			p.posErr(w.Pos(), "invalid func name")
  2360  		}
  2361  	}
  2362  	name := p.lit(p.pos, p.val)
  2363  	if p.next(); p.got(leftParen) {
  2364  		p.follow(name.ValuePos, "foo(", rightParen)
  2365  	}
  2366  	p.funcDecl(s, name, fpos)
  2367  }
  2368  
  2369  func (p *Parser) callExpr(s *Stmt, w *Word, assign bool) {
  2370  	ce := p.call(w)
  2371  	if w == nil {
  2372  		ce.Args = ce.Args[:0]
  2373  	}
  2374  	if assign {
  2375  		ce.Assigns = append(ce.Assigns, p.getAssign(true))
  2376  	}
  2377  loop:
  2378  	for {
  2379  		switch p.tok {
  2380  		case _EOF, _Newl, semicolon, and, or, andAnd, orOr, orAnd,
  2381  			dblSemicolon, semiAnd, dblSemiAnd, semiOr:
  2382  			break loop
  2383  		case _LitWord:
  2384  			if len(ce.Args) == 0 && p.hasValidIdent() {
  2385  				ce.Assigns = append(ce.Assigns, p.getAssign(true))
  2386  				break
  2387  			}
  2388  			ce.Args = append(ce.Args, p.word(
  2389  				p.wps(p.lit(p.pos, p.val)),
  2390  			))
  2391  			p.next()
  2392  		case _Lit:
  2393  			if len(ce.Args) == 0 && p.hasValidIdent() {
  2394  				ce.Assigns = append(ce.Assigns, p.getAssign(true))
  2395  				break
  2396  			}
  2397  			ce.Args = append(ce.Args, p.word(p.wordParts()))
  2398  		case bckQuote:
  2399  			if p.backquoteEnd() {
  2400  				break loop
  2401  			}
  2402  			fallthrough
  2403  		case dollBrace, dollDblParen, dollParen, dollar, cmdIn, cmdOut,
  2404  			sglQuote, dollSglQuote, dblQuote, dollDblQuote, dollBrack,
  2405  			globQuest, globStar, globPlus, globAt, globExcl:
  2406  			ce.Args = append(ce.Args, p.word(p.wordParts()))
  2407  		case rdrOut, appOut, rdrIn, dplIn, dplOut, clbOut, rdrInOut,
  2408  			hdoc, dashHdoc, wordHdoc, rdrAll, appAll, _LitRedir:
  2409  			p.doRedirect(s)
  2410  		case dblLeftParen:
  2411  			p.curErr("%s can only be used to open an arithmetic cmd", p.tok)
  2412  		case rightParen:
  2413  			if p.quote == subCmd {
  2414  				break loop
  2415  			}
  2416  			fallthrough
  2417  		default:
  2418  			p.curErr("a command can only contain words and redirects")
  2419  		}
  2420  	}
  2421  	if len(ce.Assigns) == 0 && len(ce.Args) == 0 {
  2422  		return
  2423  	}
  2424  	if len(ce.Args) == 0 {
  2425  		ce.Args = nil
  2426  	} else {
  2427  		for _, asgn := range ce.Assigns {
  2428  			if asgn.Index != nil || asgn.Array != nil {
  2429  				p.posErr(asgn.Pos(), "inline variables cannot be arrays")
  2430  			}
  2431  		}
  2432  	}
  2433  	s.Cmd = ce
  2434  }
  2435  
  2436  func (p *Parser) funcDecl(s *Stmt, name *Lit, pos Pos) {
  2437  	fd := &FuncDecl{
  2438  		Position: pos,
  2439  		RsrvWord: pos != name.ValuePos,
  2440  		Name:     name,
  2441  	}
  2442  	p.got(_Newl)
  2443  	if fd.Body = p.getStmt(false, false, true); fd.Body == nil {
  2444  		p.followErr(fd.Pos(), "foo()", "a statement")
  2445  	}
  2446  	s.Cmd = fd
  2447  }